From e241b33f2653737854357b96dd80d9ca85f237e4 Mon Sep 17 00:00:00 2001 From: upstream-mirror Date: Tue, 15 Sep 2026 20:40:15 +0500 Subject: [PATCH] snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted) --- .gemini/config.yaml | 13 + .gemini/styleguide.md | 16 + .github/ISSUE_TEMPLATE/bug.md | 27 + .github/ISSUE_TEMPLATE/feature.md | 16 + .github/ISSUE_TEMPLATE/question.md | 8 + .github/codecov.yml | 25 + .github/pull_request_template.md | 4 + .github/workflows/codeql-analysis.yml | 37 + .github/workflows/coverage.yml | 26 + .github/workflows/deps.yml | 60 + .github/workflows/govulncheck.yaml | 27 + .github/workflows/lock.yml | 22 + .github/workflows/pr-validation.yml | 55 + .github/workflows/release.yml | 63 + .github/workflows/stale.yml | 33 + .github/workflows/testing.yml | 135 + AUTHORS | 1 + CODE-OF-CONDUCT.md | 3 + CONTRIBUTING.md | 182 + Documentation/anti-patterns.md | 183 + Documentation/benchmark.md | 75 + Documentation/compression.md | 80 + Documentation/concurrency.md | 39 + Documentation/encoding.md | 146 + Documentation/grpc-auth-support.md | 83 + Documentation/grpc-metadata.md | 244 ++ Documentation/keepalive.md | 57 + Documentation/log_levels.md | 49 + Documentation/proxy.md | 25 + Documentation/rpc-errors.md | 68 + Documentation/server-reflection-tutorial.md | 171 + Documentation/versioning.md | 34 + GOVERNANCE.md | 1 + LICENSE | 202 + MAINTAINERS.md | 36 + Makefile | 49 + NOTICE.txt | 13 + README.md | 108 + SECURITY.md | 3 + admin/admin.go | 58 + admin/admin_test.go | 34 + admin/test/admin_test.go | 38 + admin/test/utils.go | 104 + attributes/attributes.go | 174 + attributes/attributes_test.go | 189 + authz/audit/audit_logger.go | 127 + authz/audit/audit_logging_test.go | 370 ++ authz/audit/stdout/stdout_logger.go | 110 + authz/audit/stdout/stdout_logger_test.go | 140 + authz/grpc_authz_end2end_test.go | 708 ++++ authz/grpc_authz_server_interceptors.go | 211 + authz/grpc_authz_server_interceptors_test.go | 174 + authz/rbac_translator.go | 398 ++ authz/rbac_translator_test.go | 1052 +++++ backoff.go | 61 + backoff/backoff.go | 52 + balancer/autosharding/autosharding.go | 112 + balancer/autosharding/autosharding_test.go | 144 + balancer/autosharding/slice_map.go | 120 + balancer/autosharding/slice_map_test.go | 216 ++ balancer/balancer.go | 394 ++ balancer/base/balancer.go | 260 ++ balancer/base/balancer_test.go | 122 + balancer/base/base.go | 71 + balancer/conn_state_evaluator.go | 74 + balancer/conn_state_evaluator_test.go | 245 ++ balancer/endpointsharding/endpointsharding.go | 385 ++ .../endpointsharding_ext_test.go | 353 ++ .../endpointsharding/endpointsharding_test.go | 83 + .../grpclb/grpc_lb_v1/load_balancer.pb.go | 763 ++++ .../grpc_lb_v1/load_balancer_grpc.pb.go | 134 + balancer/grpclb/grpclb.go | 535 +++ balancer/grpclb/grpclb_config.go | 67 + balancer/grpclb/grpclb_config_test.go | 123 + balancer/grpclb/grpclb_picker.go | 193 + balancer/grpclb/grpclb_remote_balancer.go | 459 +++ balancer/grpclb/grpclb_test.go | 1649 ++++++++ balancer/grpclb/grpclb_util.go | 173 + balancer/grpclb/grpclb_util_test.go | 267 ++ balancer/grpclb/state/state.go | 51 + balancer/lazy/lazy.go | 157 + balancer/lazy/lazy_ext_test.go | 466 +++ balancer/leastrequest/leastrequest.go | 250 ++ balancer/leastrequest/leastrequest_test.go | 770 ++++ balancer/pickfirst/internal/internal.go | 37 + balancer/pickfirst/pickfirst.go | 921 +++++ balancer/pickfirst/pickfirst_ext_test.go | 2957 ++++++++++++++ balancer/pickfirst/pickfirst_test.go | 376 ++ balancer/randomsubsetting/randomsubsetting.go | 201 + .../randomsubsetting/randomsubsetting_test.go | 281 ++ balancer/ringhash/config.go | 77 + balancer/ringhash/config_test.go | 173 + balancer/ringhash/logging.go | 34 + balancer/ringhash/picker.go | 124 + balancer/ringhash/picker_test.go | 301 ++ balancer/ringhash/ring.go | 189 + balancer/ringhash/ring_test.go | 112 + balancer/ringhash/ringhash.go | 410 ++ balancer/ringhash/ringhash_e2e_test.go | 2944 ++++++++++++++ balancer/ringhash/ringhash_test.go | 739 ++++ balancer/rls/balancer.go | 733 ++++ balancer/rls/balancer_test.go | 1536 ++++++++ balancer/rls/cache.go | 378 ++ balancer/rls/cache_test.go | 243 ++ balancer/rls/child_policy.go | 109 + balancer/rls/config.go | 330 ++ balancer/rls/config_test.go | 515 +++ balancer/rls/control_channel.go | 219 ++ balancer/rls/control_channel_test.go | 465 +++ balancer/rls/helpers_test.go | 299 ++ balancer/rls/internal/adaptive/adaptive.go | 131 + .../rls/internal/adaptive/adaptive_test.go | 231 ++ balancer/rls/internal/adaptive/lookback.go | 84 + .../rls/internal/adaptive/lookback_test.go | 85 + balancer/rls/internal/keys/builder.go | 267 ++ balancer/rls/internal/keys/builder_test.go | 650 ++++ balancer/rls/internal/test/e2e/e2e.go | 20 + .../rls/internal/test/e2e/rls_child_policy.go | 131 + .../rls/internal/test/e2e/rls_lb_config.go | 103 + balancer/rls/metrics_test.go | 375 ++ balancer/rls/picker.go | 401 ++ balancer/rls/picker_test.go | 1049 +++++ balancer/roundrobin/roundrobin.go | 72 + balancer/subconn.go | 120 + balancer/weightedroundrobin/balancer.go | 674 ++++ balancer/weightedroundrobin/balancer_test.go | 869 +++++ balancer/weightedroundrobin/config.go | 59 + .../weightedroundrobin/internal/internal.go | 44 + balancer/weightedroundrobin/logging.go | 34 + balancer/weightedroundrobin/metrics_test.go | 173 + balancer/weightedroundrobin/scheduler.go | 146 + balancer/weightedtarget/logging.go | 34 + .../weightedaggregator/aggregator.go | 318 ++ balancer/weightedtarget/weightedtarget.go | 196 + .../weightedtarget/weightedtarget_config.go | 49 + .../weightedtarget_config_test.go | 104 + .../weightedtarget/weightedtarget_test.go | 1519 ++++++++ balancer_wrapper.go | 517 +++ balancer_wrapper_test.go | 82 + benchmark/benchmain/main.go | 1022 +++++ benchmark/benchmark.go | 349 ++ benchmark/benchresult/main.go | 152 + benchmark/client/main.go | 217 ++ benchmark/flags/flags.go | 178 + benchmark/flags/flags_test.go | 153 + benchmark/latency/latency.go | 334 ++ benchmark/latency/latency_test.go | 363 ++ benchmark/primitives/code_string_test.go | 125 + benchmark/primitives/context_test.go | 152 + benchmark/primitives/primitives_test.go | 466 +++ .../primitives/safe_config_selector_test.go | 114 + benchmark/primitives/syncmap_test.go | 165 + benchmark/run_bench.sh | 188 + benchmark/server/main.go | 97 + benchmark/stats/curve.go | 179 + benchmark/stats/histogram.go | 221 ++ benchmark/stats/stats.go | 496 +++ benchmark/worker/benchmark_client.go | 428 +++ benchmark/worker/benchmark_server.go | 187 + benchmark/worker/main.go | 239 ++ binarylog/binarylog_end2end_test.go | 1053 +++++ binarylog/grpc_binarylog_v1/binarylog.pb.go | 1006 +++++ binarylog/sink.go | 68 + call.go | 97 + channelz/channelz.go | 36 + channelz/grpc_channelz_v1/channelz.pb.go | 3418 +++++++++++++++++ channelz/grpc_channelz_v1/channelz_grpc.pb.go | 390 ++ channelz/internal/protoconv/channel.go | 136 + channelz/internal/protoconv/server.go | 73 + channelz/internal/protoconv/socket.go | 136 + channelz/internal/protoconv/sockopt_linux.go | 125 + .../internal/protoconv/sockopt_nonlinux.go | 31 + channelz/internal/protoconv/subchannel.go | 65 + channelz/internal/protoconv/util.go | 28 + channelz/service/service.go | 102 + channelz/service/service_sktopt_test.go | 98 + channelz/service/service_test.go | 803 ++++ clientconn.go | 1981 ++++++++++ clientconn_authority_test.go | 135 + clientconn_disconnect_reason_noplan9.go | 48 + clientconn_disconnect_reason_plan9.go | 39 + clientconn_parsed_target_test.go | 332 ++ clientconn_test.go | 1079 ++++++ cmd/protoc-gen-go-grpc/README.md | 28 + cmd/protoc-gen-go-grpc/go.mod | 15 + cmd/protoc-gen-go-grpc/go.sum | 38 + cmd/protoc-gen-go-grpc/grpc.go | 513 +++ cmd/protoc-gen-go-grpc/main.go | 74 + .../protoc-gen-go-grpc_test.sh | 50 + cmd/protoc-gen-go-grpc/unimpl_test.go | 47 + codec.go | 105 + codec_test.go | 32 + codes/code_string.go | 111 + codes/codes.go | 250 ++ codes/codes_test.go | 103 + connectivity/connectivity.go | 94 + credentials/alts/alts.go | 322 ++ credentials/alts/alts_test.go | 505 +++ .../alts/internal/authinfo/authinfo.go | 95 + .../alts/internal/authinfo/authinfo_test.go | 156 + credentials/alts/internal/common.go | 67 + credentials/alts/internal/conn/aeadrekey.go | 131 + .../alts/internal/conn/aeadrekey_test.go | 263 ++ credentials/alts/internal/conn/aes128gcm.go | 105 + .../alts/internal/conn/aes128gcm_test.go | 223 ++ .../alts/internal/conn/aes128gcmrekey.go | 116 + .../alts/internal/conn/aes128gcmrekey_test.go | 112 + credentials/alts/internal/conn/common.go | 80 + credentials/alts/internal/conn/counter.go | 62 + .../alts/internal/conn/counter_test.go | 141 + credentials/alts/internal/conn/record.go | 380 ++ credentials/alts/internal/conn/record_test.go | 563 +++ credentials/alts/internal/conn/utils.go | 63 + .../alts/internal/handshaker/handshaker.go | 381 ++ .../internal/handshaker/handshaker_test.go | 437 +++ .../internal/handshaker/service/service.go | 91 + .../handshaker/service/service_test.go | 80 + .../internal/proto/grpc_gcp/altscontext.pb.go | 211 + .../internal/proto/grpc_gcp/handshaker.pb.go | 1245 ++++++ .../proto/grpc_gcp/handshaker_grpc.pb.go | 144 + .../grpc_gcp/transport_security_common.pb.go | 366 ++ .../alts/internal/testutil/testutil.go | 323 ++ credentials/alts/utils.go | 70 + credentials/alts/utils_test.go | 167 + credentials/credentials.go | 337 ++ credentials/credentials_ext_test.go | 538 +++ credentials/credentials_test.go | 322 ++ ...cp_service_account_identity_credentials.go | 278 ++ ...rvice_account_identity_credentials_test.go | 545 +++ credentials/google/google.go | 178 + credentials/google/google_test.go | 268 ++ credentials/google/internal/internal.go | 35 + credentials/google/xds.go | 128 + credentials/google/xds_test.go | 60 + credentials/insecure/insecure.go | 104 + credentials/jwt/doc.go | 50 + credentials/jwt/file_reader.go | 118 + credentials/jwt/file_reader_test.go | 172 + credentials/jwt/token_file_call_creds.go | 180 + .../jwt/token_file_call_creds_ext_test.go | 265 ++ credentials/jwt/token_file_call_creds_test.go | 555 +++ credentials/local/local.go | 121 + credentials/local/local_test.go | 225 ++ credentials/oauth/oauth.go | 244 ++ credentials/oauth/oauth_test.go | 60 + credentials/sts/sts.go | 407 ++ credentials/sts/sts_test.go | 803 ++++ credentials/tls.go | 322 ++ credentials/tls/certprovider/distributor.go | 114 + .../tls/certprovider/distributor_test.go | 110 + .../tls/certprovider/pemfile/builder.go | 103 + .../tls/certprovider/pemfile/builder_test.go | 162 + .../tls/certprovider/pemfile/watcher.go | 297 ++ .../tls/certprovider/pemfile/watcher_test.go | 629 +++ credentials/tls/certprovider/provider.go | 115 + credentials/tls/certprovider/store.go | 196 + credentials/tls/certprovider/store_test.go | 400 ++ credentials/tls_ext_test.go | 608 +++ credentials/xds/xds.go | 236 ++ credentials/xds/xds_client_test.go | 676 ++++ credentials/xds/xds_server_test.go | 543 +++ default_dial_option_server_option_test.go | 178 + dial_test.go | 348 ++ dialoptions.go | 812 ++++ doc.go | 26 + encoding/compressor_test.go | 281 ++ encoding/encoding.go | 150 + encoding/encoding_test.go | 400 ++ encoding/encoding_v2.go | 81 + encoding/gzip/gzip.go | 124 + encoding/internal/internal.go | 28 + encoding/proto/proto.go | 112 + encoding/proto/proto_benchmark_test.go | 99 + encoding/proto/proto_test.go | 139 + examples/README.md | 10 + examples/data/data.go | 44 + examples/data/rbac/policy.json | 30 + examples/data/x509/README.md | 6 + examples/data/x509/ca_cert.pem | 34 + examples/data/x509/ca_key.pem | 52 + examples/data/x509/client_ca_cert.pem | 34 + examples/data/x509/client_ca_key.pem | 52 + examples/data/x509/client_cert.pem | 32 + examples/data/x509/client_key.pem | 51 + examples/data/x509/create.sh | 69 + examples/data/x509/openssl.cnf | 28 + examples/data/x509/server_cert.pem | 32 + examples/data/x509/server_key.pem | 51 + examples/examples_test.sh | 248 ++ examples/features/advancedtls/README.md | 28 + examples/features/advancedtls/client/main.go | 306 ++ .../features/advancedtls/creds/ca_cert.pem | 35 + .../features/advancedtls/creds/ca_key.pem | 52 + .../advancedtls/creds/client_cert.pem | 127 + .../advancedtls/creds/client_cert_revoked.pem | 127 + .../features/advancedtls/creds/client_key.pem | 52 + .../advancedtls/creds/crl/client_revoked.crl | 20 + .../advancedtls/creds/localhost-openssl.cnf | 24 + .../features/advancedtls/creds/openssl-ca.cnf | 94 + .../advancedtls/creds/server_cert.pem | 127 + .../advancedtls/creds/server_cert_revoked.pem | 127 + .../features/advancedtls/creds/server_key.pem | 52 + .../advancedtls/creds/server_revoked.crl | 19 + examples/features/advancedtls/generate.sh | 91 + .../advancedtls/localhost-openssl.cnf | 24 + examples/features/advancedtls/openssl-ca.cnf | 94 + examples/features/advancedtls/server/main.go | 177 + examples/features/authentication/README.md | 62 + .../features/authentication/client/main.go | 86 + .../features/authentication/server/main.go | 112 + examples/features/authz/README.md | 75 + examples/features/authz/client/main.go | 141 + examples/features/authz/server/main.go | 238 ++ examples/features/authz/token/token.go | 55 + examples/features/cancellation/README.md | 13 + examples/features/cancellation/client/main.go | 96 + examples/features/cancellation/server/main.go | 67 + examples/features/compression/README.md | 14 + examples/features/compression/client/main.go | 62 + examples/features/compression/server/main.go | 61 + examples/features/csm_observability/README.md | 38 + .../csm_observability/client/Dockerfile | 35 + .../features/csm_observability/client/main.go | 88 + .../csm_observability/server/Dockerfile | 34 + .../features/csm_observability/server/main.go | 91 + .../features/customloadbalancer/README.md | 52 + .../customroundrobin/customroundrobin.go | 148 + .../customloadbalancer/client/main.go | 155 + .../customloadbalancer/server/main.go | 67 + examples/features/deadline/client/main.go | 97 + examples/features/deadline/server/main.go | 124 + examples/features/debugging/README.md | 29 + examples/features/debugging/client/main.go | 93 + examples/features/debugging/server/main.go | 92 + examples/features/dualstack/README.md | 29 + examples/features/dualstack/client/main.go | 189 + examples/features/dualstack/server/main.go | 84 + .../features/encryption/ALTS/client/main.go | 63 + .../features/encryption/ALTS/server/main.go | 64 + examples/features/encryption/README.md | 119 + .../features/encryption/TLS/client/main.go | 67 + .../features/encryption/TLS/server/main.go | 68 + .../features/encryption/mTLS/client/main.go | 81 + .../features/encryption/mTLS/server/main.go | 82 + examples/features/error_details/README.md | 25 + .../features/error_details/client/main.go | 69 + .../features/error_details/server/main.go | 85 + examples/features/error_handling/README.md | 22 + .../features/error_handling/client/main.go | 70 + .../features/error_handling/server/main.go | 66 + examples/features/flow_control/README.md | 55 + examples/features/flow_control/client/main.go | 103 + examples/features/flow_control/server/main.go | 111 + examples/features/gracefulstop/README.md | 45 + examples/features/gracefulstop/client/main.go | 80 + examples/features/gracefulstop/server/main.go | 117 + examples/features/health/README.md | 74 + examples/features/health/client/main.go | 88 + examples/features/health/server/main.go | 90 + examples/features/interceptor/README.md | 130 + examples/features/interceptor/client/main.go | 167 + examples/features/interceptor/server/main.go | 166 + examples/features/keepalive/README.md | 15 + examples/features/keepalive/client/main.go | 64 + examples/features/keepalive/server/main.go | 76 + examples/features/load_balancing/README.md | 83 + .../features/load_balancing/client/main.go | 127 + .../features/load_balancing/server/main.go | 71 + examples/features/metadata/README.md | 18 + examples/features/metadata/client/main.go | 308 ++ examples/features/metadata/server/main.go | 208 + .../features/metadata_interceptor/README.md | 70 + .../metadata_interceptor/client/main.go | 114 + .../metadata_interceptor/server/main.go | 164 + examples/features/multiplex/README.md | 12 + examples/features/multiplex/client/main.go | 79 + examples/features/multiplex/server/main.go | 75 + examples/features/name_resolving/README.md | 41 + .../features/name_resolving/client/main.go | 137 + .../features/name_resolving/server/main.go | 56 + examples/features/observability/README.md | 3 + .../observability/client/clientConfig.json | 17 + .../features/observability/client/main.go | 71 + .../features/observability/server/main.go | 90 + .../observability/server/serverConfig.json | 17 + examples/features/opentelemetry/README.md | 39 + .../features/opentelemetry/client/main.go | 100 + .../features/opentelemetry/server/main.go | 93 + examples/features/orca/README.md | 48 + examples/features/orca/client/main.go | 159 + examples/features/orca/server/main.go | 97 + examples/features/proto/echo/echo.pb.go | 201 + examples/features/proto/echo/echo.proto | 45 + examples/features/proto/echo/echo_grpc.pb.go | 255 ++ examples/features/reflection/README.md | 21 + examples/features/reflection/server/main.go | 79 + examples/features/retry/README.md | 64 + examples/features/retry/client/main.go | 77 + examples/features/retry/server/main.go | 93 + examples/features/stats_monitoring/README.md | 58 + .../features/stats_monitoring/client/main.go | 60 + .../features/stats_monitoring/server/main.go | 62 + .../stats_monitoring/statshandler/handler.go | 93 + examples/features/unix_abstract/README.md | 29 + .../features/unix_abstract/client/main.go | 75 + .../features/unix_abstract/server/main.go | 65 + examples/features/wait_for_ready/README.md | 11 + examples/features/wait_for_ready/main.go | 117 + examples/features/xds/README.md | 35 + examples/features/xds/client/main.go | 73 + examples/features/xds/server/main.go | 107 + examples/go.mod | 92 + examples/gotutorial.md | 423 ++ examples/helloworld/README.md | 29 + examples/helloworld/greeter_client/main.go | 60 + examples/helloworld/greeter_server/main.go | 60 + .../helloworld/helloworld/helloworld.pb.go | 192 + .../helloworld/helloworld/helloworld.proto | 38 + .../helloworld/helloworld_grpc.pb.go | 141 + examples/route_guide/README.md | 35 + examples/route_guide/client/client.go | 194 + .../route_guide/routeguide/route_guide.pb.go | 434 +++ .../route_guide/routeguide/route_guide.proto | 112 + .../routeguide/route_guide_grpc.pb.go | 284 ++ examples/route_guide/server/server.go | 845 ++++ .../route_guide/testdata/route_guide_db.json | 601 +++ experimental/balancer/hostname/hostname.go | 47 + .../balancer/hostname/hostname_test.go | 75 + experimental/balancer/weight/weight.go | 60 + experimental/balancer/weight/weight_test.go | 83 + experimental/credentials/credentials_test.go | 252 ++ experimental/credentials/internal/spiffe.go | 75 + .../credentials/internal/spiffe_test.go | 233 ++ .../credentials/internal/syscallconn.go | 58 + .../credentials/internal/syscallconn_test.go | 56 + experimental/credentials/tls.go | 249 ++ experimental/credentials/tls_ext_test.go | 604 +++ experimental/experimental.go | 85 + experimental/opentelemetry/trace_options.go | 36 + experimental/shared_buffer_pool_test.go | 211 + experimental/stats/metricregistry.go | 342 ++ experimental/stats/metricregistry_test.go | 318 ++ experimental/stats/metrics.go | 148 + experimental/stats/telemetry/labels.go | 42 + gcp/observability/config.go | 273 ++ gcp/observability/exporting.go | 108 + gcp/observability/go.mod | 67 + gcp/observability/logging.go | 505 +++ gcp/observability/logging_test.go | 1347 +++++++ gcp/observability/observability.go | 92 + gcp/observability/observability_test.go | 1088 ++++++ gcp/observability/opencensus.go | 179 + go.mod | 54 + go.sum | 99 + grpc_test.go | 33 + grpclog/component.go | 115 + grpclog/glogger/glogger.go | 104 + grpclog/grpclog.go | 186 + grpclog/internal/grpclog.go | 26 + grpclog/internal/logger.go | 87 + grpclog/internal/loggerv2.go | 267 ++ grpclog/internal/loggerv2_test.go | 350 ++ grpclog/logger.go | 34 + grpclog/loggerv2.go | 97 + health/client.go | 117 + health/client_test.go | 63 + health/grpc_health_v1/health.pb.go | 350 ++ health/grpc_health_v1/health_grpc.pb.go | 290 ++ health/logging.go | 23 + health/producer.go | 106 + health/server.go | 187 + health/server_internal_test.go | 153 + health/server_test.go | 43 + interceptor.go | 108 + internal/admin/admin.go | 60 + internal/backoff/backoff.go | 109 + internal/balancer/gracefulswitch/config.go | 84 + .../balancer/gracefulswitch/gracefulswitch.go | 421 ++ .../gracefulswitch/gracefulswitch_test.go | 1089 ++++++ internal/balancer/nop/nop.go | 65 + internal/balancer/stub/stub.go | 116 + internal/balancergroup/balancergroup.go | 607 +++ internal/balancergroup/balancergroup_test.go | 930 +++++ .../balancergroup/balancerstateaggregator.go | 37 + internal/balancerload/load.go | 46 + internal/binarylog/binarylog.go | 192 + internal/binarylog/binarylog_test.go | 156 + internal/binarylog/binarylog_testutil.go | 42 + internal/binarylog/env_config.go | 208 + internal/binarylog/env_config_test.go | 478 +++ internal/binarylog/method_logger.go | 446 +++ internal/binarylog/method_logger_test.go | 562 +++ internal/binarylog/regexp_test.go | 179 + internal/binarylog/sink.go | 170 + internal/buffer/unbounded.go | 115 + internal/buffer/unbounded_test.go | 148 + internal/cache/timeoutCache.go | 151 + internal/cache/timeoutCache_test.go | 254 ++ internal/channelz/channel.go | 270 ++ internal/channelz/channelmap.go | 395 ++ internal/channelz/funcs.go | 230 ++ internal/channelz/logging.go | 75 + internal/channelz/server.go | 121 + internal/channelz/socket.go | 137 + internal/channelz/subchannel.go | 153 + internal/channelz/syscall_linux.go | 65 + internal/channelz/syscall_nonlinux.go | 47 + internal/channelz/syscall_test.go | 95 + internal/channelz/trace.go | 213 + internal/credentials/credentials.go | 35 + internal/credentials/spiffe.go | 75 + internal/credentials/spiffe/spiffe.go | 107 + internal/credentials/spiffe/spiffe_test.go | 246 ++ internal/credentials/spiffe_test.go | 233 ++ internal/credentials/syscallconn.go | 58 + internal/credentials/syscallconn_test.go | 56 + internal/credentials/util.go | 52 + internal/credentials/util_test.go | 60 + internal/credentials/xds/handshake_info.go | 469 +++ .../credentials/xds/handshake_info_test.go | 668 ++++ internal/envconfig/envconfig.go | 244 ++ internal/envconfig/envconfig_test.go | 218 ++ internal/envconfig/observability.go | 42 + internal/envconfig/xds.go | 106 + internal/experimental.go | 35 + internal/googlecloud/googlecloud.go | 72 + internal/googlecloud/googlecloud_test.go | 46 + internal/googlecloud/manufacturer.go | 26 + internal/googlecloud/manufacturer_linux.go | 27 + internal/googlecloud/manufacturer_windows.go | 50 + internal/grpclog/prefix_logger.go | 79 + internal/grpcsync/callback_serializer.go | 124 + internal/grpcsync/callback_serializer_test.go | 247 ++ internal/grpcsync/event.go | 58 + internal/grpcsync/event_test.go | 81 + internal/grpcsync/pubsub.go | 121 + internal/grpcsync/pubsub_test.go | 196 + internal/grpcsync/refcounted.go | 109 + internal/grpcsync/refcounted_test.go | 155 + internal/grpctest/example_test.go | 61 + internal/grpctest/grpctest.go | 135 + internal/grpctest/grpctest_test.go | 46 + internal/grpctest/tlogger.go | 300 ++ internal/grpctest/tlogger_test.go | 79 + internal/grpcutil/compressor.go | 42 + internal/grpcutil/compressor_test.go | 32 + internal/grpcutil/encode_duration.go | 62 + internal/grpcutil/encode_duration_test.go | 51 + internal/grpcutil/grpcutil.go | 20 + internal/grpcutil/metadata.go | 40 + internal/grpcutil/method.go | 88 + internal/grpcutil/method_test.go | 69 + internal/hierarchy/hierarchy.go | 108 + internal/hierarchy/hierarchy_ext_test.go | 174 + internal/idle/idle.go | 289 ++ internal/idle/idle_e2e_test.go | 589 +++ internal/idle/idle_test.go | 378 ++ internal/internal.go | 304 ++ internal/leakcheck/leakcheck.go | 489 +++ internal/leakcheck/leakcheck_enabled.go | 25 + internal/leakcheck/leakcheck_test.go | 173 + internal/mem/buffer_pool.go | 363 ++ internal/mem/buffer_pool_ext_test.go | 118 + internal/mem/buffer_pool_test.go | 130 + internal/metadata/metadata.go | 144 + internal/metadata/metadata_test.go | 117 + internal/optional/optional.go | 47 + internal/optional/optional_test.go | 130 + internal/pretty/pretty.go | 73 + internal/profiling/buffer/buffer.go | 272 ++ internal/profiling/buffer/buffer_test.go | 186 + internal/profiling/goid_modified.go | 82 + internal/profiling/goid_regular.go | 30 + internal/profiling/profiling.go | 223 ++ internal/profiling/profiling_test.go | 154 + internal/proto/grpc_lookup_v1/rls.pb.go | 328 ++ .../proto/grpc_lookup_v1/rls_config.pb.go | 774 ++++ internal/proto/grpc_lookup_v1/rls_grpc.pb.go | 137 + internal/proxyattributes/proxyattributes.go | 54 + .../proxyattributes/proxyattributes_test.go | 116 + internal/resolver/config_selector.go | 100 + internal/resolver/config_selector_test.go | 156 + .../delegatingresolver/delegatingresolver.go | 477 +++ .../delegatingresolver_ext_test.go | 1025 +++++ .../delegatingresolver_test.go | 107 + internal/resolver/dns/dns_resolver.go | 472 +++ internal/resolver/dns/dns_resolver_test.go | 1511 ++++++++ .../resolver/dns/fake_net_resolver_test.go | 126 + internal/resolver/dns/internal/internal.go | 77 + internal/resolver/passthrough/passthrough.go | 64 + internal/resolver/unix/unix.go | 78 + internal/ringhash/ringhash.go | 58 + internal/serviceconfig/duration.go | 130 + internal/serviceconfig/duration_test.go | 86 + internal/serviceconfig/serviceconfig.go | 180 + internal/serviceconfig/serviceconfig_test.go | 182 + internal/stats/labels.go | 74 + internal/stats/labels_test.go | 216 ++ internal/stats/metrics_recorder_list.go | 175 + internal/stats/metrics_recorder_list_test.go | 336 ++ internal/stats/stats.go | 70 + internal/status/status.go | 246 ++ internal/status/status_test.go | 40 + internal/stubserver/stubserver.go | 302 ++ internal/syscall/syscall_linux.go | 112 + internal/syscall/syscall_nonlinux.go | 77 + internal/tcp_keepalive_others.go | 29 + internal/tcp_keepalive_unix.go | 54 + internal/tcp_keepalive_windows.go | 54 + internal/testutils/balancer.go | 413 ++ internal/testutils/blocking_context_dialer.go | 139 + .../testutils/blocking_context_dialer_test.go | 214 ++ internal/testutils/channel.go | 118 + internal/testutils/envconfig.go | 33 + internal/testutils/fakegrpclb/server.go | 250 ++ internal/testutils/http_client.go | 63 + internal/testutils/local_listener.go | 26 + internal/testutils/marshal_any.go | 37 + internal/testutils/parse_port.go | 39 + internal/testutils/parse_url.go | 34 + internal/testutils/pickfirst/pickfirst.go | 82 + internal/testutils/pipe_listener.go | 96 + internal/testutils/pipe_listener_test.go | 172 + internal/testutils/proxyserver/proxyserver.go | 134 + internal/testutils/resolver.go | 70 + internal/testutils/restartable_listener.go | 102 + internal/testutils/rls/fake_rls_server.go | 134 + internal/testutils/roundrobin/roundrobin.go | 286 ++ internal/testutils/state.go | 85 + .../testutils/stats/test_metrics_recorder.go | 323 ++ internal/testutils/status_equal.go | 38 + internal/testutils/status_equal_test.go | 66 + internal/testutils/stubstatshandler.go | 65 + internal/testutils/tls_creds.go | 156 + internal/testutils/wrappers.go | 74 + internal/testutils/wrr.go | 73 + internal/testutils/xds/e2e/bootstrap.go | 196 + internal/testutils/xds/e2e/clientresources.go | 846 ++++ internal/testutils/xds/e2e/logging.go | 40 + internal/testutils/xds/e2e/server.go | 256 ++ internal/testutils/xds/e2e/setup/setup.go | 85 + internal/testutils/xds/fakeserver/server.go | 254 ++ internal/testutils/xds_bootstrap.go | 52 + internal/transport/bdp_estimator.go | 141 + internal/transport/client_stream.go | 189 + internal/transport/controlbuf.go | 1051 +++++ internal/transport/controlbuf_test.go | 85 + internal/transport/defaults.go | 56 + internal/transport/flowcontrol.go | 211 + internal/transport/handler_server.go | 506 +++ internal/transport/handler_server_test.go | 614 +++ internal/transport/http2_client.go | 1921 +++++++++ .../http2_client_header_bench_test.go | 96 + internal/transport/http2_server.go | 1532 ++++++++ internal/transport/http_util.go | 643 ++++ internal/transport/http_util_test.go | 637 +++ internal/transport/internal/internal.go | 25 + internal/transport/keepalive_test.go | 860 +++++ internal/transport/logging.go | 40 + internal/transport/networktype/networktype.go | 46 + internal/transport/proxy.go | 116 + internal/transport/proxy_ext_test.go | 441 +++ internal/transport/proxy_test.go | 105 + .../transport/readyreader/raw_conn_linux.go | 39 + .../readyreader/raw_conn_nonlinux.go | 35 + .../transport/readyreader/ready_reader.go | 253 ++ .../readyreader/ready_reader_ext_test.go | 414 ++ internal/transport/server_metrics_ext_test.go | 185 + internal/transport/server_stream.go | 189 + internal/transport/transport.go | 878 +++++ internal/wrr/edf.go | 92 + internal/wrr/edf_test.go | 35 + internal/wrr/random.go | 84 + internal/wrr/wrr.go | 32 + internal/wrr/wrr_test.go | 193 + internal/xds/balancer/balancer.go | 30 + .../cdsbalancer/aggregate_cluster_test.go | 932 +++++ .../xds/balancer/cdsbalancer/cdsbalancer.go | 472 +++ .../balancer/cdsbalancer/cdsbalancer_test.go | 1135 ++++++ .../xds/balancer/cdsbalancer/configbuilder.go | 375 ++ .../cdsbalancer/configbuilder_childname.go | 111 + .../configbuilder_childname_test.go | 311 ++ .../cdsbalancer/configbuilder_test.go | 1445 +++++++ .../e2e_test/aggregate_cluster_test.go | 1467 +++++++ .../cdsbalancer/e2e_test/balancer_test.go | 420 ++ .../cdsbalancer/e2e_test/dns_impl_test.go | 136 + .../cdsbalancer/e2e_test/eds_impl_test.go | 1105 ++++++ internal/xds/balancer/cdsbalancer/logging.go | 34 + .../xds/balancer/clusterimpl/balancer_test.go | 356 ++ .../xds/balancer/clusterimpl/clusterimpl.go | 644 ++++ internal/xds/balancer/clusterimpl/config.go | 48 + .../xds/balancer/clusterimpl/config_test.go | 132 + .../balancer/clusterimpl/internal/internal.go | 25 + internal/xds/balancer/clusterimpl/logging.go | 34 + internal/xds/balancer/clusterimpl/picker.go | 239 ++ .../clusterimpl/tests/balancer_test.go | 1650 ++++++++ .../tests/clusterimpl_security_test.go | 783 ++++ .../clustermanager/balancerstateaggregator.go | 196 + .../balancer/clustermanager/clustermanager.go | 205 + .../clustermanager/clustermanager_test.go | 775 ++++ .../xds/balancer/clustermanager/config.go | 46 + .../balancer/clustermanager/config_test.go | 144 + .../e2e_test/clustermanager_test.go | 342 ++ .../xds/balancer/clustermanager/picker.go | 65 + .../balancer/loadstore/load_store_wrapper.go | 128 + .../xds/balancer/outlierdetection/balancer.go | 927 +++++ .../outlierdetection/balancer_ext_test.go | 518 +++ .../outlierdetection/balancer_test.go | 1533 ++++++++ .../balancer/outlierdetection/callcounter.go | 66 + .../outlierdetection/callcounter_test.go | 92 + .../xds/balancer/outlierdetection/config.go | 240 ++ .../balancer/outlierdetection/config_test.go | 72 + .../e2e_test/outlierdetection_test.go | 425 ++ .../xds/balancer/outlierdetection/logging.go | 34 + .../outlierdetection/subconn_wrapper.go | 175 + internal/xds/balancer/priority/balancer.go | 292 ++ .../xds/balancer/priority/balancer_child.go | 186 + .../balancer/priority/balancer_priority.go | 223 ++ .../xds/balancer/priority/balancer_test.go | 2366 ++++++++++++ internal/xds/balancer/priority/config.go | 67 + internal/xds/balancer/priority/config_test.go | 108 + .../balancer/priority/ignore_resolve_now.go | 50 + .../priority/ignore_resolve_now_test.go | 66 + internal/xds/balancer/priority/logging.go | 34 + internal/xds/balancer/wrrlocality/balancer.go | 201 + .../xds/balancer/wrrlocality/balancer_test.go | 250 ++ internal/xds/balancer/wrrlocality/logging.go | 34 + internal/xds/bootstrap/bootstrap.go | 1099 ++++++ internal/xds/bootstrap/bootstrap_test.go | 1947 ++++++++++ internal/xds/bootstrap/jwtcreds/call_creds.go | 54 + .../xds/bootstrap/jwtcreds/call_creds_test.go | 168 + internal/xds/bootstrap/logging.go | 28 + internal/xds/bootstrap/template.go | 47 + internal/xds/bootstrap/template_test.go | 97 + internal/xds/bootstrap/tlscreds/bundle.go | 194 + .../xds/bootstrap/tlscreds/bundle_ext_test.go | 520 +++ .../xds/bootstrap/tlscreds/bundle_test.go | 162 + internal/xds/clients/config.go | 140 + .../clients/grpctransport/examples_test.go | 39 + .../clients/grpctransport/grpc_transport.go | 237 ++ .../grpctransport/grpc_transport_ext_test.go | 316 ++ .../grpctransport/grpc_transport_test.go | 425 ++ .../xds/clients/internal/backoff/backoff.go | 124 + .../xds/clients/internal/buffer/unbounded.go | 115 + .../clients/internal/buffer/unbounded_test.go | 148 + internal/xds/clients/internal/internal.go | 71 + .../xds/clients/internal/internal_test.go | 213 + .../xds/clients/internal/pretty/pretty.go | 73 + .../internal/syncutil/callback_serializer.go | 98 + .../syncutil/callback_serializer_test.go | 206 + .../xds/clients/internal/syncutil/event.go | 61 + .../clients/internal/syncutil/event_test.go | 81 + .../xds/clients/internal/testutils/channel.go | 87 + .../internal/testutils/e2e/clientresources.go | 75 + .../clients/internal/testutils/e2e/logging.go | 40 + .../clients/internal/testutils/e2e/server.go | 257 ++ .../internal/testutils/fakeserver/server.go | 254 ++ .../faketransport/xds_fake_transport.go | 252 ++ .../clients/internal/testutils/marshal_any.go | 37 + .../testutils/restartable_listener.go | 106 + .../clients/internal/testutils/wrappers.go | 74 + .../clients/lrsclient/internal/internal.go | 26 + internal/xds/clients/lrsclient/load_store.go | 438 +++ .../xds/clients/lrsclient/load_store_test.go | 525 +++ .../xds/clients/lrsclient/loadreport_test.go | 730 ++++ internal/xds/clients/lrsclient/logging.go | 36 + internal/xds/clients/lrsclient/lrs_stream.go | 334 ++ internal/xds/clients/lrsclient/lrsclient.go | 175 + internal/xds/clients/lrsclient/lrsconfig.go | 35 + internal/xds/clients/transport_builder.go | 53 + internal/xds/clients/xdsclient/ads_stream.go | 733 ++++ internal/xds/clients/xdsclient/authority.go | 996 +++++ internal/xds/clients/xdsclient/channel.go | 329 ++ .../xds/clients/xdsclient/channel_test.go | 821 ++++ .../clients/xdsclient/clientimpl_watchers.go | 109 + .../xds/clients/xdsclient/helpers_test.go | 233 ++ .../clients/xdsclient/internal/internal.go | 32 + .../internal/xdsresource/ads_stream.go | 46 + .../xdsclient/internal/xdsresource/errors.go | 78 + .../xdsclient/internal/xdsresource/name.go | 127 + .../xdsclient/internal/xdsresource/type.go | 101 + .../xdsclient/internal/xdsresource/version.go | 31 + internal/xds/clients/xdsclient/logging.go | 36 + .../xds/clients/xdsclient/metrics/metrics.go | 64 + .../xds/clients/xdsclient/resource_type.go | 123 + .../xds/clients/xdsclient/resource_watcher.go | 48 + .../test/ads_stream_ack_nack_test.go | 671 ++++ .../xdsclient/test/ads_stream_backoff_test.go | 444 +++ .../test/ads_stream_flow_control_test.go | 625 +++ .../xdsclient/test/ads_stream_restart_test.go | 202 + .../xdsclient/test/ads_stream_watch_test.go | 222 ++ .../clients/xdsclient/test/authority_test.go | 334 ++ .../test/custom_resource_watch_test.go | 254 ++ .../xds/clients/xdsclient/test/dump_test.go | 557 +++ .../clients/xdsclient/test/helpers_test.go | 390 ++ .../xdsclient/test/lds_watchers_test.go | 1402 +++++++ .../clients/xdsclient/test/metrics_test.go | 911 +++++ .../xdsclient/test/misc_watchers_test.go | 529 +++ internal/xds/clients/xdsclient/xdsclient.go | 516 +++ .../xds/clients/xdsclient/xdsclient_test.go | 128 + internal/xds/clients/xdsclient/xdsconfig.go | 112 + .../xds/clusterspecifier/cluster_specifier.go | 72 + internal/xds/clusterspecifier/rls/rls.go | 95 + internal/xds/clusterspecifier/rls/rls_test.go | 162 + internal/xds/credentials/access_token.go | 71 + internal/xds/credentials/access_token_test.go | 126 + internal/xds/credentials/credentials.go | 217 ++ internal/xds/credentials/tls.go | 174 + internal/xds/credentials/tls_test.go | 287 ++ internal/xds/credentials/xds_test.go | 76 + internal/xds/grpcservice/grpcservice.go | 306 ++ internal/xds/grpcservice/grpcservice_test.go | 462 +++ internal/xds/httpfilter/ext_authz/config.go | 79 + .../xds/httpfilter/ext_authz/ext_authz.go | 184 + .../httpfilter/ext_authz/ext_authz_test.go | 439 +++ internal/xds/httpfilter/extconfig.go | 326 ++ internal/xds/httpfilter/extconfig_test.go | 712 ++++ internal/xds/httpfilter/extproc/config.go | 184 + .../xds/httpfilter/extproc/config_test.go | 980 +++++ internal/xds/httpfilter/extproc/ext_proc.go | 2240 +++++++++++ .../httpfilter/extproc/internal/internal.go | 61 + internal/xds/httpfilter/extproc/metrics.go | 54 + internal/xds/httpfilter/fault/fault.go | 311 ++ internal/xds/httpfilter/fault/fault_test.go | 661 ++++ .../httpfilter/gcp_authn/gcp_authn_filter.go | 333 ++ .../gcp_authn/gcp_authn_filter_e2e_test.go | 952 +++++ .../gcp_authn/gcp_authn_filter_test.go | 295 ++ internal/xds/httpfilter/httpfilter.go | 204 + internal/xds/httpfilter/rbac/rbac.go | 279 ++ internal/xds/httpfilter/rbac/rbac_test.go | 185 + internal/xds/httpfilter/router/router.go | 122 + internal/xds/matcher/matcher_header.go | 273 ++ internal/xds/matcher/matcher_header_test.go | 549 +++ internal/xds/matcher/string_matcher.go | 229 ++ internal/xds/matcher/string_matcher_test.go | 349 ++ internal/xds/rbac/converter.go | 101 + internal/xds/rbac/converter_test.go | 170 + internal/xds/rbac/matchers.go | 505 +++ internal/xds/rbac/rbac_engine.go | 319 ++ internal/xds/rbac/rbac_engine_test.go | 2521 ++++++++++++ .../resolver/cluster_specifier_plugin_test.go | 479 +++ internal/xds/resolver/helpers_test.go | 288 ++ internal/xds/resolver/internal/internal.go | 30 + internal/xds/resolver/logging.go | 34 + internal/xds/resolver/serviceconfig.go | 381 ++ internal/xds/resolver/serviceconfig_test.go | 187 + internal/xds/resolver/watch_service_test.go | 187 + .../xds/resolver/xds_http_filters_test.go | 1462 +++++++ internal/xds/resolver/xds_resolver.go | 731 ++++ internal/xds/resolver/xds_resolver_test.go | 1511 ++++++++ internal/xds/server/conn_wrapper.go | 203 + internal/xds/server/filter_chain_manager.go | 607 +++ .../xds/server/filter_chain_manager_test.go | 796 ++++ internal/xds/server/listener_wrapper.go | 521 +++ internal/xds/server/listener_wrapper_test.go | 148 + internal/xds/server/rds_handler.go | 188 + internal/xds/server/rds_handler_test.go | 272 ++ internal/xds/server/routing.go | 192 + internal/xds/server/routing_test.go | 155 + internal/xds/server/server_options.go | 69 + internal/xds/test/e2e/README.md | 19 + internal/xds/test/e2e/controlplane.go | 45 + internal/xds/test/e2e/e2e.go | 173 + internal/xds/test/e2e/e2e_test.go | 258 ++ internal/xds/test/e2e/e2e_utils.go | 179 + internal/xds/test/e2e/run.sh | 6 + internal/xds/testutils/balancer_test.go | 147 + internal/xds/testutils/fakeclient/client.go | 147 + internal/xds/testutils/resource_watcher.go | 83 + internal/xds/testutils/testutils.go | 51 + internal/xds/xds.go | 123 + internal/xds/xds_test.go | 115 + internal/xds/xdsclient/attributes.go | 36 + internal/xds/xdsclient/client.go | 62 + .../xds/xdsclient/client_refcounted_test.go | 277 ++ internal/xds/xdsclient/client_test.go | 40 + internal/xds/xdsclient/clientimpl.go | 328 ++ .../xds/xdsclient/clientimpl_loadreport.go | 49 + internal/xds/xdsclient/clientimpl_test.go | 363 ++ internal/xds/xdsclient/internal/internal.go | 33 + internal/xds/xdsclient/logging.go | 36 + internal/xds/xdsclient/metrics_test.go | 361 ++ internal/xds/xdsclient/pool.go | 320 ++ internal/xds/xdsclient/pool/pool_ext_test.go | 250 ++ internal/xds/xdsclient/requests_counter.go | 83 + .../xds/xdsclient/requests_counter_test.go | 162 + internal/xds/xdsclient/resource_types.go | 54 + .../tests/ads_stream_ack_nack_test.go | 512 +++ .../tests/ads_stream_restart_test.go | 196 + .../xds/xdsclient/tests/authority_test.go | 375 ++ .../xds/xdsclient/tests/cds_watchers_test.go | 1141 ++++++ .../tests/client_custom_dialopts_test.go | 165 + internal/xds/xdsclient/tests/dump_test.go | 575 +++ .../xds/xdsclient/tests/eds_watchers_test.go | 1045 +++++ .../xdsclient/tests/fallback/fallback_test.go | 931 +++++ .../tests/federation_watchers_test.go | 324 ++ internal/xds/xdsclient/tests/helpers_test.go | 72 + .../xds/xdsclient/tests/lds_watchers_test.go | 1397 +++++++ .../xds/xdsclient/tests/loadreport_test.go | 566 +++ .../xds/xdsclient/tests/rds_watchers_test.go | 996 +++++ .../xdsclient/tests/resource_update_test.go | 1259 ++++++ internal/xds/xdsclient/xdsclient_test.go | 33 + .../xdslbregistry/converter/converter.go | 252 ++ .../xdsclient/xdslbregistry/xdslbregistry.go | 82 + .../xdslbregistry/xdslbregistry_test.go | 459 +++ .../xdsresource/cluster_resource_type.go | 140 + .../xdsresource/endpoints_resource_type.go | 126 + internal/xds/xdsclient/xdsresource/errors.go | 79 + .../xds/xdsclient/xdsresource/filter_chain.go | 180 + .../xdsresource/filter_chain_test.go | 2568 +++++++++++++ .../xdsresource/listener_resource_type.go | 185 + internal/xds/xdsclient/xdsresource/logging.go | 28 + internal/xds/xdsclient/xdsresource/matcher.go | 231 ++ .../xds/xdsclient/xdsresource/matcher_path.go | 100 + .../xdsresource/matcher_path_test.go | 93 + .../xds/xdsclient/xdsresource/matcher_test.go | 244 ++ .../xds/xdsclient/xdsresource/metadata.go | 164 + .../xdsclient/xdsresource/metadata_test.go | 259 ++ internal/xds/xdsclient/xdsresource/name.go | 127 + .../xds/xdsclient/xdsresource/name_test.go | 103 + .../xdsclient/xdsresource/resource_type.go | 139 + .../xdsresource/route_config_resource_type.go | 132 + .../xdsclient/xdsresource/test_utils_test.go | 124 + .../xdsresource/tests/unmarshal_cds_test.go | 575 +++ internal/xds/xdsclient/xdsresource/type.go | 135 + .../xds/xdsclient/xdsresource/type_cds.go | 228 ++ .../xds/xdsclient/xdsresource/type_eds.go | 78 + .../xds/xdsclient/xdsresource/type_lds.go | 92 + .../xds/xdsclient/xdsresource/type_rds.go | 184 + .../xdsclient/xdsresource/unmarshal_cds.go | 805 ++++ .../xdsresource/unmarshal_cds_test.go | 2600 +++++++++++++ .../xdsclient/xdsresource/unmarshal_eds.go | 294 ++ .../xdsresource/unmarshal_eds_test.go | 1721 +++++++++ .../xdsclient/xdsresource/unmarshal_lds.go | 618 +++ .../xdsresource/unmarshal_lds_test.go | 2066 ++++++++++ .../xdsclient/xdsresource/unmarshal_rds.go | 464 +++ .../xdsresource/unmarshal_rds_test.go | 1910 +++++++++ .../xdsclient/xdsresource/version/version.go | 44 + .../xds/xdsclient/xdsresource/xdsconfig.go | 133 + .../xds/xdsdepmgr/xds_dependency_manager.go | 1027 +++++ .../xdsdepmgr/xds_dependency_manager_test.go | 1760 +++++++++ interop/alts/client/client.go | 69 + interop/alts/server/server.go | 88 + interop/client/client.go | 408 ++ interop/fake_grpclb/fake_grpclb.go | 83 + interop/grpc_testing/benchmark_service.pb.go | 97 + .../grpc_testing/benchmark_service_grpc.pb.go | 297 ++ interop/grpc_testing/control.pb.go | 2069 ++++++++++ interop/grpc_testing/core/stats.pb.go | 343 ++ interop/grpc_testing/empty.pb.go | 135 + interop/grpc_testing/messages.pb.go | 2369 ++++++++++++ interop/grpc_testing/payloads.pb.go | 352 ++ .../report_qps_scenario_service.pb.go | 85 + .../report_qps_scenario_service_grpc.pb.go | 141 + interop/grpc_testing/stats.pb.go | 525 +++ interop/grpc_testing/test.pb.go | 166 + interop/grpc_testing/test_grpc.pb.go | 1316 +++++++ interop/grpc_testing/worker_service.pb.go | 100 + .../grpc_testing/worker_service_grpc.pb.go | 269 ++ interop/grpclb_fallback/client_linux.go | 192 + interop/http2/negative_http2_client.go | 165 + interop/interop_test.sh | 140 + interop/observability/Dockerfile | 53 + interop/observability/build_docker.sh | 26 + interop/observability/client/client.go | 84 + interop/observability/go.mod | 73 + interop/observability/run.sh | 29 + interop/observability/server/server.go | 60 + interop/orcalb.go | 256 ++ interop/orcalb_test.go | 363 ++ interop/server/server.go | 97 + interop/soak_tests.go | 203 + interop/stress/client/main.go | 364 ++ interop/stress/grpc_testing/metrics.pb.go | 309 ++ interop/stress/grpc_testing/metrics.proto | 51 + .../stress/grpc_testing/metrics_grpc.pb.go | 190 + interop/stress/metrics_client/main.go | 86 + interop/test_utils.go | 990 +++++ interop/xds/client/Dockerfile | 37 + interop/xds/client/client.go | 554 +++ interop/xds/custom_lb.go | 140 + interop/xds/custom_lb_test.go | 135 + interop/xds/go.mod | 56 + interop/xds/go.sum | 119 + interop/xds/server/Dockerfile | 37 + interop/xds/server/server.go | 342 ++ interop/xds_federation/client.go | 152 + keepalive/keepalive.go | 99 + mem/buffer_pool.go | 92 + mem/buffer_pool_test.go | 148 + mem/buffer_slice.go | 345 ++ mem/buffer_slice_test.go | 676 ++++ mem/buffers.go | 317 ++ mem/buffers_test.go | 493 +++ metadata/metadata.go | 426 ++ metadata/metadata_test.go | 677 ++++ orca/call_metrics.go | 196 + orca/call_metrics_test.go | 287 ++ orca/internal/internal.go | 71 + orca/orca.go | 57 + orca/orca_test.go | 102 + orca/producer.go | 225 ++ orca/producer_test.go | 572 +++ orca/server_metrics.go | 352 ++ orca/server_metrics_test.go | 175 + orca/service.go | 161 + orca/service_test.go | 187 + peer/peer.go | 83 + peer/peer_test.go | 101 + picker_wrapper.go | 221 ++ picker_wrapper_test.go | 184 + preloader.go | 82 + producer_ext_test.go | 116 + profiling/cmd/catapult.go | 394 ++ profiling/cmd/flags.go | 76 + profiling/cmd/local.go | 68 + profiling/cmd/main.go | 53 + profiling/cmd/remote.go | 98 + profiling/profiling.go | 40 + profiling/proto/service.pb.go | 455 +++ profiling/proto/service.proto | 98 + profiling/proto/service_grpc.pb.go | 183 + profiling/service/service.go | 150 + reflection/README.md | 18 + reflection/adapt.go | 57 + .../grpc_reflection_v1/reflection.pb.go | 777 ++++ .../grpc_reflection_v1/reflection_grpc.pb.go | 138 + .../grpc_reflection_v1alpha/reflection.pb.go | 847 ++++ .../reflection_grpc.pb.go | 135 + reflection/grpc_testing/proto2.pb.go | 138 + reflection/grpc_testing/proto2.proto | 24 + reflection/grpc_testing/proto2_ext.pb.go | 186 + reflection/grpc_testing/proto2_ext.proto | 32 + reflection/grpc_testing/proto2_ext2.pb.go | 171 + reflection/grpc_testing/proto2_ext2.proto | 30 + reflection/grpc_testing/test.pb.go | 256 ++ reflection/grpc_testing/test.proto | 37 + reflection/grpc_testing/test_grpc.pb.go | 168 + reflection/internal/internal.go | 436 +++ reflection/serverreflection.go | 160 + reflection/test/serverreflection_test.go | 856 +++++ resolver/dns/dns_resolver.go | 60 + resolver/manual/manual.go | 130 + resolver/manual/manual_test.go | 63 + resolver/map.go | 281 ++ resolver/map_test.go | 365 ++ resolver/passthrough/passthrough.go | 26 + resolver/resolver.go | 359 ++ resolver/resolver_test.go | 105 + resolver/ringhash/attr.go | 60 + resolver_balancer_ext_test.go | 306 ++ resolver_test.go | 197 + resolver_wrapper.go | 222 ++ rpc_util.go | 1205 ++++++ rpc_util_test.go | 676 ++++ scripts/common.sh | 24 + scripts/gen-deps.sh | 23 + scripts/install-protoc.sh | 94 + scripts/regenerate.sh | 131 + scripts/revive.toml | 60 + scripts/vet-proto.sh | 46 + scripts/vet.sh | 217 ++ security/advancedtls/advancedtls.go | 690 ++++ .../advancedtls_integration_test.go | 942 +++++ security/advancedtls/advancedtls_test.go | 1324 +++++++ .../advancedtls/certificate_utils_test.go | 81 + security/advancedtls/crl.go | 436 +++ security/advancedtls/crl_provider.go | 239 ++ security/advancedtls/crl_provider_test.go | 356 ++ security/advancedtls/crl_test.go | 588 +++ .../credential_reloading_from_files/README.md | 25 + .../client/main.go | 111 + .../server/main.go | 112 + .../advancedtls/examples/examples_test.sh | 114 + security/advancedtls/examples/go.mod | 26 + security/advancedtls/examples/go.sum | 30 + security/advancedtls/go.mod | 24 + security/advancedtls/go.sum | 30 + .../internal/testutils/testutils.go | 129 + security/advancedtls/sni.go | 51 + security/advancedtls/testdata/README.md | 48 + .../testdata/another_client_cert_1.pem | 122 + .../testdata/another_client_key_1.pem | 51 + .../advancedtls/testdata/client_cert_1.pem | 122 + .../advancedtls/testdata/client_cert_2.pem | 122 + .../advancedtls/testdata/client_key_1.pem | 51 + .../advancedtls/testdata/client_key_2.pem | 51 + .../testdata/client_trust_cert_1.pem | 32 + .../testdata/client_trust_cert_2.pem | 33 + .../testdata/client_trust_key_1.pem | 52 + .../testdata/client_trust_key_2.pem | 52 + security/advancedtls/testdata/crl/0b35a562.r0 | 1 + security/advancedtls/testdata/crl/0b35a562.r1 | 1 + security/advancedtls/testdata/crl/1.crl | 10 + security/advancedtls/testdata/crl/1ab871c8.r0 | 1 + security/advancedtls/testdata/crl/2.crl | 10 + security/advancedtls/testdata/crl/2f11f022.r0 | 7 + security/advancedtls/testdata/crl/3.crl | 11 + security/advancedtls/testdata/crl/4.crl | 10 + security/advancedtls/testdata/crl/5.crl | 10 + security/advancedtls/testdata/crl/6.crl | 11 + security/advancedtls/testdata/crl/71eac5a2.r0 | 1 + security/advancedtls/testdata/crl/7a1799af.r0 | 1 + security/advancedtls/testdata/crl/8828a7e6.r0 | 1 + security/advancedtls/testdata/crl/README.md | 92 + security/advancedtls/testdata/crl/deee447d.r0 | 1 + .../advancedtls/testdata/crl/provider_ca.cnf | 14 + .../testdata/crl/provider_client_cert.key | 52 + .../testdata/crl/provider_client_cert.pem | 32 + .../crl/provider_client_trust_cert.pem | 32 + .../crl/provider_client_trust_key.pem | 52 + .../testdata/crl/provider_create.sh | 116 + .../advancedtls/testdata/crl/provider_crl.cnf | 14 + .../testdata/crl/provider_crl_empty.pem | 20 + .../crl/provider_crl_server_revoked.pem | 20 + .../testdata/crl/provider_extensions.conf | 5 + .../provider_malicious_client_trust_cert.pem | 31 + .../provider_malicious_client_trust_key.pem | 52 + .../crl/provider_malicious_crl_empty.pem | 20 + .../testdata/crl/provider_server_cert.key | 52 + .../testdata/crl/provider_server_cert.pem | 32 + .../crl/provider_server_trust_cert.pem | 32 + .../crl/provider_server_trust_key.pem | 52 + .../advancedtls/testdata/crl/revokedInt.pem | 58 + .../advancedtls/testdata/crl/revokedLeaf.pem | 59 + .../advancedtls/testdata/crl/unrevoked.pem | 58 + .../testdata/localhost-openssl.cnf | 23 + security/advancedtls/testdata/openssl-ca.cnf | 87 + .../advancedtls/testdata/server_cert_1.pem | 122 + .../advancedtls/testdata/server_cert_1.txt | 90 + .../advancedtls/testdata/server_cert_2.pem | 122 + .../advancedtls/testdata/server_cert_2.txt | 90 + .../advancedtls/testdata/server_cert_3.pem | 23 + .../advancedtls/testdata/server_cert_3.txt | 60 + .../testdata/server_cert_localhost_1.pem | 124 + .../advancedtls/testdata/server_key_1.pem | 51 + .../advancedtls/testdata/server_key_2.pem | 51 + .../advancedtls/testdata/server_key_3.pem | 28 + .../testdata/server_key_localhost_1.pem | 51 + .../testdata/server_trust_cert_1.pem | 32 + .../testdata/server_trust_cert_2.pem | 33 + .../testdata/server_trust_key_1.pem | 52 + .../testdata/server_trust_key_2.pem | 52 + security/advancedtls/testdata/testdata.go | 43 + server.go | 1991 ++++++++++ server_ext_test.go | 332 ++ server_test.go | 235 ++ service_config.go | 360 ++ service_config_test.go | 609 +++ serviceconfig/serviceconfig.go | 44 + stats/handlers.go | 72 + stats/metrics.go | 81 + stats/opencensus/client_metrics.go | 150 + stats/opencensus/e2e_test.go | 1619 ++++++++ stats/opencensus/go.mod | 20 + stats/opencensus/opencensus.go | 251 ++ stats/opencensus/server_metrics.go | 135 + stats/opencensus/stats.go | 226 ++ stats/opencensus/trace.go | 130 + stats/opentelemetry/client_metrics.go | 287 ++ stats/opentelemetry/client_tracing.go | 155 + stats/opentelemetry/csm/observability.go | 84 + stats/opentelemetry/csm/observability_test.go | 649 ++++ stats/opentelemetry/csm/pluginoption.go | 279 ++ stats/opentelemetry/csm/pluginoption_test.go | 490 +++ stats/opentelemetry/e2e_test.go | 2772 +++++++++++++ stats/opentelemetry/example_test.go | 151 + .../grpc_trace_bin_propagator.go | 119 + .../grpc_trace_bin_propagator_test.go | 219 ++ stats/opentelemetry/internal/pluginoption.go | 41 + .../internal/testutils/testutils.go | 853 ++++ .../opentelemetry/internal/tracing/carrier.go | 131 + .../internal/tracing/carrier_test.go | 190 + stats/opentelemetry/metricsregistry_test.go | 341 ++ stats/opentelemetry/opentelemetry.go | 626 +++ stats/opentelemetry/server_metrics.go | 254 ++ stats/opentelemetry/server_tracing.go | 84 + stats/opentelemetry/trace.go | 74 + stats/stats.go | 321 ++ stats/stats_test.go | 1540 ++++++++ status/status.go | 162 + status/status_ext_test.go | 317 ++ status/status_test.go | 482 +++ stream.go | 2021 ++++++++++ stream_interfaces.go | 238 ++ stream_test.go | 243 ++ tap/tap.go | 62 + test/authority_test.go | 237 ++ test/balancer_switching_test.go | 530 +++ test/balancer_test.go | 1429 +++++++ test/bufconn/bufconn.go | 317 ++ test/bufconn/bufconn_test.go | 328 ++ test/channelz_linux_test.go | 95 + test/channelz_test.go | 2165 +++++++++++ test/clientconn_state_transition_test.go | 832 ++++ test/clientconn_test.go | 197 + test/clienttester.go | 109 + test/codec_perf/perf.pb.go | 142 + test/codec_perf/perf.proto | 27 + test/compressor_test.go | 694 ++++ test/config_selector_test.go | 216 ++ test/context_canceled_test.go | 164 + test/control_plane_status_test.go | 234 ++ test/creds_test.go | 626 +++ test/goaway_test.go | 823 ++++ test/gracefulstop_test.go | 364 ++ test/healthcheck_test.go | 1281 ++++++ test/http_header_end2end_test.go | 259 ++ test/insecure_creds_test.go | 198 + test/interceptor_test.go | 281 ++ test/invoke_test.go | 152 + test/kokoro/README.md | 1 + test/kokoro/psm-csm.cfg | 17 + test/kokoro/psm-dualstack.cfg | 17 + test/kokoro/psm-interop-build-go.sh | 34 + test/kokoro/psm-interop-test-go.sh | 40 + test/kokoro/psm-light.cfg | 17 + test/kokoro/psm-regional_td.cfg | 17 + test/kokoro/psm-security.cfg | 17 + test/kokoro/psm-spiffe.cfg | 17 + test/kokoro/xds.cfg | 11 + test/kokoro/xds.sh | 45 + test/kokoro/xds_k8s_lb.cfg | 17 + test/kokoro/xds_url_map.cfg | 17 + test/kokoro/xds_v3.cfg | 11 + test/kokoro/xds_v3.sh | 3 + test/local_creds_test.go | 218 ++ test/logging.go | 23 + test/malformed_method_test.go | 159 + test/metadata_test.go | 155 + test/parse_config.go | 38 + test/race_test.go | 25 + test/rawConnWrapper.go | 247 ++ test/resolver_update_test.go | 263 ++ test/retry_test.go | 1069 ++++++ test/roundrobin_test.go | 318 ++ test/server_test.go | 587 +++ test/servertester.go | 320 ++ test/stats_test.go | 142 + test/stream_cleanup_test.go | 139 + test/subconn_test.go | 126 + test/timeouts.go | 29 + test/tools/go.mod | 29 + test/tools/go.sum | 44 + test/tools/tools.go | 34 + test/tools/tools_vet.go | 21 + test/transport_test.go | 355 ++ test/xds/xds_client_ack_nack_test.go | 202 + test/xds/xds_client_affinity_test.go | 128 + .../xds_client_certificate_providers_test.go | 508 +++ test/xds/xds_client_custom_lb_test.go | 269 ++ test/xds/xds_client_federation_test.go | 343 ++ test/xds/xds_client_filter_e2e_test.go | 239 ++ test/xds/xds_client_http_connect_test.go | 642 ++++ ...ds_client_ignore_resource_deletion_test.go | 502 +++ test/xds/xds_client_integration_test.go | 244 ++ test/xds/xds_client_outlier_detection_test.go | 323 ++ test/xds/xds_client_priority_locality_test.go | 211 + test/xds/xds_client_retry_test.go | 192 + test/xds/xds_client_sni_test.go | 851 ++++ test/xds/xds_resolver_e2e_test.go | 754 ++++ .../xds_rls_clusterspecifier_plugin_test.go | 168 + test/xds/xds_security_config_nack_test.go | 377 ++ test/xds/xds_server_filter_e2e_test.go | 402 ++ test/xds/xds_server_filter_override_test.go | 393 ++ .../xds_server_filter_state_retention_test.go | 699 ++++ test/xds/xds_server_integration_test.go | 548 +++ test/xds/xds_server_interceptor_leak_test.go | 274 ++ test/xds/xds_server_rbac_test.go | 1201 ++++++ test/xds/xds_telemetry_labels_test.go | 281 ++ testdata/README.md | 3 + testdata/ca.pem | 20 + .../grpc_testing_not_regenerated/README.md | 12 + .../grpc_testing_not_regenerated/dynamic.go | 35 + .../dynamic.proto | 33 + .../grpc_testing_not_regenerated/simple.proto | 27 + .../simple_message_v1.go | 83 + .../grpc_testing_not_regenerated/testv3.go | 476 +++ .../grpc_testing_not_regenerated/testv3.proto | 58 + testdata/server1.key | 28 + testdata/server1.pem | 22 + testdata/spiffe/README.md | 76 + testdata/spiffe/client_spiffe.pem | 25 + testdata/spiffe/server1_spiffe.pem | 26 + testdata/spiffe/spiffe-openssl.cnf | 28 + testdata/spiffe/spiffe_cert.pem | 33 + testdata/spiffe/spiffe_multi_uri_san_cert.pem | 25 + testdata/spiffe/spiffe_test.json | 13 + testdata/spiffe/spiffebundle.json | 31 + testdata/spiffe/spiffebundle2.json | 18 + .../spiffe/spiffebundle_corrupted_cert.json | 31 + testdata/spiffe/spiffebundle_empty_keys.json | 9 + .../spiffe/spiffebundle_empty_string_key.json | 18 + .../spiffebundle_invalid_trustdomain.json | 18 + testdata/spiffe/spiffebundle_malformed.json | 4 + .../spiffebundle_match_client_spiffe.json | 31 + testdata/spiffe/spiffebundle_wrong_kid.json | 31 + testdata/spiffe/spiffebundle_wrong_kty.json | 31 + .../spiffebundle_wrong_multi_certs.json | 19 + testdata/spiffe/spiffebundle_wrong_root.json | 31 + .../spiffe/spiffebundle_wrong_seq_type.json | 31 + testdata/spiffe/spiffebundle_wrong_use.json | 18 + testdata/spiffe_end2end/README.md | 38 + testdata/spiffe_end2end/ca.key | 52 + testdata/spiffe_end2end/ca.pem | 32 + testdata/spiffe_end2end/client.key | 28 + testdata/spiffe_end2end/client_spiffe.pem | 28 + .../spiffe_end2end/client_spiffebundle.json | 16 + testdata/spiffe_end2end/generate.sh | 30 + testdata/spiffe_end2end/intermediate.cnf | 38 + testdata/spiffe_end2end/intermediate_ca.key | 28 + testdata/spiffe_end2end/intermediate_ca.pem | 27 + testdata/spiffe_end2end/intermediate_gen.sh | 55 + .../leaf_and_intermediate_chain.pem | 50 + .../leaf_signed_by_intermediate.key | 28 + .../leaf_signed_by_intermediate.pem | 23 + testdata/spiffe_end2end/server.key | 28 + testdata/spiffe_end2end/server_spiffe.pem | 29 + .../spiffe_end2end/server_spiffebundle.json | 16 + testdata/spiffe_end2end/spiffe-openssl.cnf | 28 + testdata/testdata.go | 42 + testdata/x509/README.md | 6 + testdata/x509/client1_cert.pem | 32 + testdata/x509/client1_key.pem | 51 + testdata/x509/client2_cert.pem | 32 + testdata/x509/client2_key.pem | 51 + testdata/x509/client_ca_cert.pem | 35 + testdata/x509/client_ca_key.pem | 52 + testdata/x509/client_with_spiffe_cert.pem | 33 + testdata/x509/client_with_spiffe_key.pem | 52 + testdata/x509/client_with_spiffe_openssl.cnf | 17 + testdata/x509/create.sh | 151 + testdata/x509/multiple_uri_cert.pem | 34 + testdata/x509/multiple_uri_key.pem | 52 + testdata/x509/openssl.cnf | 28 + testdata/x509/server1_cert.pem | 32 + testdata/x509/server1_key.pem | 51 + testdata/x509/server2_cert.pem | 32 + testdata/x509/server2_key.pem | 51 + testdata/x509/server_ca_cert.pem | 35 + testdata/x509/server_ca_key.pem | 52 + testdata/x509/spiffe_cert.pem | 33 + testdata/x509/spiffe_key.pem | 52 + trace.go | 143 + trace_notrace.go | 52 + trace_test.go | 50 + trace_withtrace.go | 39 + version.go | 22 + xds/bootstrap/bootstrap.go | 102 + xds/bootstrap/bootstrap_test.go | 119 + xds/bootstrap/credentials.go | 85 + xds/csds/csds.go | 108 + xds/csds/csds_e2e_test.go | 699 ++++ xds/googledirectpath/googlec2p.go | 250 ++ xds/googledirectpath/googlec2p_test.go | 768 ++++ xds/googledirectpath/utils.go | 101 + xds/server.go | 235 ++ xds/server_ext_test.go | 532 +++ xds/server_options.go | 102 + xds/server_resource_ext_test.go | 654 ++++ xds/server_security_ext_test.go | 475 +++ xds/server_serving_mode_ext_test.go | 606 +++ xds/server_test.go | 814 ++++ xds/test/eds_resource_missing_test.go | 188 + xds/xds.go | 100 + 1365 files changed, 339570 insertions(+) create mode 100644 .gemini/config.yaml create mode 100644 .gemini/styleguide.md create mode 100644 .github/ISSUE_TEMPLATE/bug.md create mode 100644 .github/ISSUE_TEMPLATE/feature.md create mode 100644 .github/ISSUE_TEMPLATE/question.md create mode 100644 .github/codecov.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/coverage.yml create mode 100644 .github/workflows/deps.yml create mode 100644 .github/workflows/govulncheck.yaml create mode 100644 .github/workflows/lock.yml create mode 100644 .github/workflows/pr-validation.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/stale.yml create mode 100644 .github/workflows/testing.yml create mode 100644 AUTHORS create mode 100644 CODE-OF-CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 Documentation/anti-patterns.md create mode 100644 Documentation/benchmark.md create mode 100644 Documentation/compression.md create mode 100644 Documentation/concurrency.md create mode 100644 Documentation/encoding.md create mode 100644 Documentation/grpc-auth-support.md create mode 100644 Documentation/grpc-metadata.md create mode 100644 Documentation/keepalive.md create mode 100644 Documentation/log_levels.md create mode 100644 Documentation/proxy.md create mode 100644 Documentation/rpc-errors.md create mode 100644 Documentation/server-reflection-tutorial.md create mode 100644 Documentation/versioning.md create mode 100644 GOVERNANCE.md create mode 100644 LICENSE create mode 100644 MAINTAINERS.md create mode 100644 Makefile create mode 100644 NOTICE.txt create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 admin/admin.go create mode 100644 admin/admin_test.go create mode 100644 admin/test/admin_test.go create mode 100644 admin/test/utils.go create mode 100644 attributes/attributes.go create mode 100644 attributes/attributes_test.go create mode 100644 authz/audit/audit_logger.go create mode 100644 authz/audit/audit_logging_test.go create mode 100644 authz/audit/stdout/stdout_logger.go create mode 100644 authz/audit/stdout/stdout_logger_test.go create mode 100644 authz/grpc_authz_end2end_test.go create mode 100644 authz/grpc_authz_server_interceptors.go create mode 100644 authz/grpc_authz_server_interceptors_test.go create mode 100644 authz/rbac_translator.go create mode 100644 authz/rbac_translator_test.go create mode 100644 backoff.go create mode 100644 backoff/backoff.go create mode 100644 balancer/autosharding/autosharding.go create mode 100644 balancer/autosharding/autosharding_test.go create mode 100644 balancer/autosharding/slice_map.go create mode 100644 balancer/autosharding/slice_map_test.go create mode 100644 balancer/balancer.go create mode 100644 balancer/base/balancer.go create mode 100644 balancer/base/balancer_test.go create mode 100644 balancer/base/base.go create mode 100644 balancer/conn_state_evaluator.go create mode 100644 balancer/conn_state_evaluator_test.go create mode 100644 balancer/endpointsharding/endpointsharding.go create mode 100644 balancer/endpointsharding/endpointsharding_ext_test.go create mode 100644 balancer/endpointsharding/endpointsharding_test.go create mode 100644 balancer/grpclb/grpc_lb_v1/load_balancer.pb.go create mode 100644 balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go create mode 100644 balancer/grpclb/grpclb.go create mode 100644 balancer/grpclb/grpclb_config.go create mode 100644 balancer/grpclb/grpclb_config_test.go create mode 100644 balancer/grpclb/grpclb_picker.go create mode 100644 balancer/grpclb/grpclb_remote_balancer.go create mode 100644 balancer/grpclb/grpclb_test.go create mode 100644 balancer/grpclb/grpclb_util.go create mode 100644 balancer/grpclb/grpclb_util_test.go create mode 100644 balancer/grpclb/state/state.go create mode 100644 balancer/lazy/lazy.go create mode 100644 balancer/lazy/lazy_ext_test.go create mode 100644 balancer/leastrequest/leastrequest.go create mode 100644 balancer/leastrequest/leastrequest_test.go create mode 100644 balancer/pickfirst/internal/internal.go create mode 100644 balancer/pickfirst/pickfirst.go create mode 100644 balancer/pickfirst/pickfirst_ext_test.go create mode 100644 balancer/pickfirst/pickfirst_test.go create mode 100644 balancer/randomsubsetting/randomsubsetting.go create mode 100644 balancer/randomsubsetting/randomsubsetting_test.go create mode 100644 balancer/ringhash/config.go create mode 100644 balancer/ringhash/config_test.go create mode 100644 balancer/ringhash/logging.go create mode 100644 balancer/ringhash/picker.go create mode 100644 balancer/ringhash/picker_test.go create mode 100644 balancer/ringhash/ring.go create mode 100644 balancer/ringhash/ring_test.go create mode 100644 balancer/ringhash/ringhash.go create mode 100644 balancer/ringhash/ringhash_e2e_test.go create mode 100644 balancer/ringhash/ringhash_test.go create mode 100644 balancer/rls/balancer.go create mode 100644 balancer/rls/balancer_test.go create mode 100644 balancer/rls/cache.go create mode 100644 balancer/rls/cache_test.go create mode 100644 balancer/rls/child_policy.go create mode 100644 balancer/rls/config.go create mode 100644 balancer/rls/config_test.go create mode 100644 balancer/rls/control_channel.go create mode 100644 balancer/rls/control_channel_test.go create mode 100644 balancer/rls/helpers_test.go create mode 100644 balancer/rls/internal/adaptive/adaptive.go create mode 100644 balancer/rls/internal/adaptive/adaptive_test.go create mode 100644 balancer/rls/internal/adaptive/lookback.go create mode 100644 balancer/rls/internal/adaptive/lookback_test.go create mode 100644 balancer/rls/internal/keys/builder.go create mode 100644 balancer/rls/internal/keys/builder_test.go create mode 100644 balancer/rls/internal/test/e2e/e2e.go create mode 100644 balancer/rls/internal/test/e2e/rls_child_policy.go create mode 100644 balancer/rls/internal/test/e2e/rls_lb_config.go create mode 100644 balancer/rls/metrics_test.go create mode 100644 balancer/rls/picker.go create mode 100644 balancer/rls/picker_test.go create mode 100644 balancer/roundrobin/roundrobin.go create mode 100644 balancer/subconn.go create mode 100644 balancer/weightedroundrobin/balancer.go create mode 100644 balancer/weightedroundrobin/balancer_test.go create mode 100644 balancer/weightedroundrobin/config.go create mode 100644 balancer/weightedroundrobin/internal/internal.go create mode 100644 balancer/weightedroundrobin/logging.go create mode 100644 balancer/weightedroundrobin/metrics_test.go create mode 100644 balancer/weightedroundrobin/scheduler.go create mode 100644 balancer/weightedtarget/logging.go create mode 100644 balancer/weightedtarget/weightedaggregator/aggregator.go create mode 100644 balancer/weightedtarget/weightedtarget.go create mode 100644 balancer/weightedtarget/weightedtarget_config.go create mode 100644 balancer/weightedtarget/weightedtarget_config_test.go create mode 100644 balancer/weightedtarget/weightedtarget_test.go create mode 100644 balancer_wrapper.go create mode 100644 balancer_wrapper_test.go create mode 100644 benchmark/benchmain/main.go create mode 100644 benchmark/benchmark.go create mode 100644 benchmark/benchresult/main.go create mode 100644 benchmark/client/main.go create mode 100644 benchmark/flags/flags.go create mode 100644 benchmark/flags/flags_test.go create mode 100644 benchmark/latency/latency.go create mode 100644 benchmark/latency/latency_test.go create mode 100644 benchmark/primitives/code_string_test.go create mode 100644 benchmark/primitives/context_test.go create mode 100644 benchmark/primitives/primitives_test.go create mode 100644 benchmark/primitives/safe_config_selector_test.go create mode 100644 benchmark/primitives/syncmap_test.go create mode 100755 benchmark/run_bench.sh create mode 100644 benchmark/server/main.go create mode 100644 benchmark/stats/curve.go create mode 100644 benchmark/stats/histogram.go create mode 100644 benchmark/stats/stats.go create mode 100644 benchmark/worker/benchmark_client.go create mode 100644 benchmark/worker/benchmark_server.go create mode 100644 benchmark/worker/main.go create mode 100644 binarylog/binarylog_end2end_test.go create mode 100644 binarylog/grpc_binarylog_v1/binarylog.pb.go create mode 100644 binarylog/sink.go create mode 100644 call.go create mode 100644 channelz/channelz.go create mode 100644 channelz/grpc_channelz_v1/channelz.pb.go create mode 100644 channelz/grpc_channelz_v1/channelz_grpc.pb.go create mode 100644 channelz/internal/protoconv/channel.go create mode 100644 channelz/internal/protoconv/server.go create mode 100644 channelz/internal/protoconv/socket.go create mode 100644 channelz/internal/protoconv/sockopt_linux.go create mode 100644 channelz/internal/protoconv/sockopt_nonlinux.go create mode 100644 channelz/internal/protoconv/subchannel.go create mode 100644 channelz/internal/protoconv/util.go create mode 100644 channelz/service/service.go create mode 100644 channelz/service/service_sktopt_test.go create mode 100644 channelz/service/service_test.go create mode 100644 clientconn.go create mode 100644 clientconn_authority_test.go create mode 100644 clientconn_disconnect_reason_noplan9.go create mode 100644 clientconn_disconnect_reason_plan9.go create mode 100644 clientconn_parsed_target_test.go create mode 100644 clientconn_test.go create mode 100644 cmd/protoc-gen-go-grpc/README.md create mode 100644 cmd/protoc-gen-go-grpc/go.mod create mode 100644 cmd/protoc-gen-go-grpc/go.sum create mode 100644 cmd/protoc-gen-go-grpc/grpc.go create mode 100644 cmd/protoc-gen-go-grpc/main.go create mode 100755 cmd/protoc-gen-go-grpc/protoc-gen-go-grpc_test.sh create mode 100644 cmd/protoc-gen-go-grpc/unimpl_test.go create mode 100644 codec.go create mode 100644 codec_test.go create mode 100644 codes/code_string.go create mode 100644 codes/codes.go create mode 100644 codes/codes_test.go create mode 100644 connectivity/connectivity.go create mode 100644 credentials/alts/alts.go create mode 100644 credentials/alts/alts_test.go create mode 100644 credentials/alts/internal/authinfo/authinfo.go create mode 100644 credentials/alts/internal/authinfo/authinfo_test.go create mode 100644 credentials/alts/internal/common.go create mode 100644 credentials/alts/internal/conn/aeadrekey.go create mode 100644 credentials/alts/internal/conn/aeadrekey_test.go create mode 100644 credentials/alts/internal/conn/aes128gcm.go create mode 100644 credentials/alts/internal/conn/aes128gcm_test.go create mode 100644 credentials/alts/internal/conn/aes128gcmrekey.go create mode 100644 credentials/alts/internal/conn/aes128gcmrekey_test.go create mode 100644 credentials/alts/internal/conn/common.go create mode 100644 credentials/alts/internal/conn/counter.go create mode 100644 credentials/alts/internal/conn/counter_test.go create mode 100644 credentials/alts/internal/conn/record.go create mode 100644 credentials/alts/internal/conn/record_test.go create mode 100644 credentials/alts/internal/conn/utils.go create mode 100644 credentials/alts/internal/handshaker/handshaker.go create mode 100644 credentials/alts/internal/handshaker/handshaker_test.go create mode 100644 credentials/alts/internal/handshaker/service/service.go create mode 100644 credentials/alts/internal/handshaker/service/service_test.go create mode 100644 credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go create mode 100644 credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go create mode 100644 credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go create mode 100644 credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go create mode 100644 credentials/alts/internal/testutil/testutil.go create mode 100644 credentials/alts/utils.go create mode 100644 credentials/alts/utils_test.go create mode 100644 credentials/credentials.go create mode 100644 credentials/credentials_ext_test.go create mode 100644 credentials/credentials_test.go create mode 100644 credentials/google/gcp_service_account_identity_credentials.go create mode 100644 credentials/google/gcp_service_account_identity_credentials_test.go create mode 100644 credentials/google/google.go create mode 100644 credentials/google/google_test.go create mode 100644 credentials/google/internal/internal.go create mode 100644 credentials/google/xds.go create mode 100644 credentials/google/xds_test.go create mode 100644 credentials/insecure/insecure.go create mode 100644 credentials/jwt/doc.go create mode 100644 credentials/jwt/file_reader.go create mode 100644 credentials/jwt/file_reader_test.go create mode 100644 credentials/jwt/token_file_call_creds.go create mode 100644 credentials/jwt/token_file_call_creds_ext_test.go create mode 100644 credentials/jwt/token_file_call_creds_test.go create mode 100644 credentials/local/local.go create mode 100644 credentials/local/local_test.go create mode 100644 credentials/oauth/oauth.go create mode 100644 credentials/oauth/oauth_test.go create mode 100644 credentials/sts/sts.go create mode 100644 credentials/sts/sts_test.go create mode 100644 credentials/tls.go create mode 100644 credentials/tls/certprovider/distributor.go create mode 100644 credentials/tls/certprovider/distributor_test.go create mode 100644 credentials/tls/certprovider/pemfile/builder.go create mode 100644 credentials/tls/certprovider/pemfile/builder_test.go create mode 100644 credentials/tls/certprovider/pemfile/watcher.go create mode 100644 credentials/tls/certprovider/pemfile/watcher_test.go create mode 100644 credentials/tls/certprovider/provider.go create mode 100644 credentials/tls/certprovider/store.go create mode 100644 credentials/tls/certprovider/store_test.go create mode 100644 credentials/tls_ext_test.go create mode 100644 credentials/xds/xds.go create mode 100644 credentials/xds/xds_client_test.go create mode 100644 credentials/xds/xds_server_test.go create mode 100644 default_dial_option_server_option_test.go create mode 100644 dial_test.go create mode 100644 dialoptions.go create mode 100644 doc.go create mode 100644 encoding/compressor_test.go create mode 100644 encoding/encoding.go create mode 100644 encoding/encoding_test.go create mode 100644 encoding/encoding_v2.go create mode 100644 encoding/gzip/gzip.go create mode 100644 encoding/internal/internal.go create mode 100644 encoding/proto/proto.go create mode 100644 encoding/proto/proto_benchmark_test.go create mode 100644 encoding/proto/proto_test.go create mode 100644 examples/README.md create mode 100644 examples/data/data.go create mode 100644 examples/data/rbac/policy.json create mode 100644 examples/data/x509/README.md create mode 100644 examples/data/x509/ca_cert.pem create mode 100644 examples/data/x509/ca_key.pem create mode 100644 examples/data/x509/client_ca_cert.pem create mode 100644 examples/data/x509/client_ca_key.pem create mode 100644 examples/data/x509/client_cert.pem create mode 100644 examples/data/x509/client_key.pem create mode 100755 examples/data/x509/create.sh create mode 100644 examples/data/x509/openssl.cnf create mode 100644 examples/data/x509/server_cert.pem create mode 100644 examples/data/x509/server_key.pem create mode 100755 examples/examples_test.sh create mode 100644 examples/features/advancedtls/README.md create mode 100644 examples/features/advancedtls/client/main.go create mode 100644 examples/features/advancedtls/creds/ca_cert.pem create mode 100644 examples/features/advancedtls/creds/ca_key.pem create mode 100644 examples/features/advancedtls/creds/client_cert.pem create mode 100644 examples/features/advancedtls/creds/client_cert_revoked.pem create mode 100644 examples/features/advancedtls/creds/client_key.pem create mode 100644 examples/features/advancedtls/creds/crl/client_revoked.crl create mode 100644 examples/features/advancedtls/creds/localhost-openssl.cnf create mode 100644 examples/features/advancedtls/creds/openssl-ca.cnf create mode 100644 examples/features/advancedtls/creds/server_cert.pem create mode 100644 examples/features/advancedtls/creds/server_cert_revoked.pem create mode 100644 examples/features/advancedtls/creds/server_key.pem create mode 100644 examples/features/advancedtls/creds/server_revoked.crl create mode 100755 examples/features/advancedtls/generate.sh create mode 100644 examples/features/advancedtls/localhost-openssl.cnf create mode 100644 examples/features/advancedtls/openssl-ca.cnf create mode 100644 examples/features/advancedtls/server/main.go create mode 100644 examples/features/authentication/README.md create mode 100644 examples/features/authentication/client/main.go create mode 100644 examples/features/authentication/server/main.go create mode 100644 examples/features/authz/README.md create mode 100644 examples/features/authz/client/main.go create mode 100644 examples/features/authz/server/main.go create mode 100644 examples/features/authz/token/token.go create mode 100644 examples/features/cancellation/README.md create mode 100644 examples/features/cancellation/client/main.go create mode 100644 examples/features/cancellation/server/main.go create mode 100644 examples/features/compression/README.md create mode 100644 examples/features/compression/client/main.go create mode 100644 examples/features/compression/server/main.go create mode 100644 examples/features/csm_observability/README.md create mode 100644 examples/features/csm_observability/client/Dockerfile create mode 100644 examples/features/csm_observability/client/main.go create mode 100644 examples/features/csm_observability/server/Dockerfile create mode 100644 examples/features/csm_observability/server/main.go create mode 100644 examples/features/customloadbalancer/README.md create mode 100644 examples/features/customloadbalancer/client/customroundrobin/customroundrobin.go create mode 100644 examples/features/customloadbalancer/client/main.go create mode 100644 examples/features/customloadbalancer/server/main.go create mode 100644 examples/features/deadline/client/main.go create mode 100644 examples/features/deadline/server/main.go create mode 100644 examples/features/debugging/README.md create mode 100644 examples/features/debugging/client/main.go create mode 100644 examples/features/debugging/server/main.go create mode 100644 examples/features/dualstack/README.md create mode 100644 examples/features/dualstack/client/main.go create mode 100644 examples/features/dualstack/server/main.go create mode 100644 examples/features/encryption/ALTS/client/main.go create mode 100644 examples/features/encryption/ALTS/server/main.go create mode 100644 examples/features/encryption/README.md create mode 100644 examples/features/encryption/TLS/client/main.go create mode 100644 examples/features/encryption/TLS/server/main.go create mode 100644 examples/features/encryption/mTLS/client/main.go create mode 100644 examples/features/encryption/mTLS/server/main.go create mode 100644 examples/features/error_details/README.md create mode 100644 examples/features/error_details/client/main.go create mode 100644 examples/features/error_details/server/main.go create mode 100644 examples/features/error_handling/README.md create mode 100644 examples/features/error_handling/client/main.go create mode 100644 examples/features/error_handling/server/main.go create mode 100644 examples/features/flow_control/README.md create mode 100644 examples/features/flow_control/client/main.go create mode 100644 examples/features/flow_control/server/main.go create mode 100644 examples/features/gracefulstop/README.md create mode 100644 examples/features/gracefulstop/client/main.go create mode 100644 examples/features/gracefulstop/server/main.go create mode 100644 examples/features/health/README.md create mode 100644 examples/features/health/client/main.go create mode 100644 examples/features/health/server/main.go create mode 100644 examples/features/interceptor/README.md create mode 100644 examples/features/interceptor/client/main.go create mode 100644 examples/features/interceptor/server/main.go create mode 100644 examples/features/keepalive/README.md create mode 100644 examples/features/keepalive/client/main.go create mode 100644 examples/features/keepalive/server/main.go create mode 100644 examples/features/load_balancing/README.md create mode 100644 examples/features/load_balancing/client/main.go create mode 100644 examples/features/load_balancing/server/main.go create mode 100644 examples/features/metadata/README.md create mode 100644 examples/features/metadata/client/main.go create mode 100644 examples/features/metadata/server/main.go create mode 100644 examples/features/metadata_interceptor/README.md create mode 100644 examples/features/metadata_interceptor/client/main.go create mode 100644 examples/features/metadata_interceptor/server/main.go create mode 100644 examples/features/multiplex/README.md create mode 100644 examples/features/multiplex/client/main.go create mode 100644 examples/features/multiplex/server/main.go create mode 100644 examples/features/name_resolving/README.md create mode 100644 examples/features/name_resolving/client/main.go create mode 100644 examples/features/name_resolving/server/main.go create mode 100644 examples/features/observability/README.md create mode 100644 examples/features/observability/client/clientConfig.json create mode 100644 examples/features/observability/client/main.go create mode 100644 examples/features/observability/server/main.go create mode 100644 examples/features/observability/server/serverConfig.json create mode 100644 examples/features/opentelemetry/README.md create mode 100644 examples/features/opentelemetry/client/main.go create mode 100644 examples/features/opentelemetry/server/main.go create mode 100644 examples/features/orca/README.md create mode 100644 examples/features/orca/client/main.go create mode 100644 examples/features/orca/server/main.go create mode 100644 examples/features/proto/echo/echo.pb.go create mode 100644 examples/features/proto/echo/echo.proto create mode 100644 examples/features/proto/echo/echo_grpc.pb.go create mode 100644 examples/features/reflection/README.md create mode 100644 examples/features/reflection/server/main.go create mode 100644 examples/features/retry/README.md create mode 100644 examples/features/retry/client/main.go create mode 100644 examples/features/retry/server/main.go create mode 100644 examples/features/stats_monitoring/README.md create mode 100644 examples/features/stats_monitoring/client/main.go create mode 100644 examples/features/stats_monitoring/server/main.go create mode 100644 examples/features/stats_monitoring/statshandler/handler.go create mode 100644 examples/features/unix_abstract/README.md create mode 100644 examples/features/unix_abstract/client/main.go create mode 100644 examples/features/unix_abstract/server/main.go create mode 100644 examples/features/wait_for_ready/README.md create mode 100644 examples/features/wait_for_ready/main.go create mode 100644 examples/features/xds/README.md create mode 100644 examples/features/xds/client/main.go create mode 100644 examples/features/xds/server/main.go create mode 100644 examples/go.mod create mode 100644 examples/gotutorial.md create mode 100644 examples/helloworld/README.md create mode 100644 examples/helloworld/greeter_client/main.go create mode 100644 examples/helloworld/greeter_server/main.go create mode 100644 examples/helloworld/helloworld/helloworld.pb.go create mode 100644 examples/helloworld/helloworld/helloworld.proto create mode 100644 examples/helloworld/helloworld/helloworld_grpc.pb.go create mode 100644 examples/route_guide/README.md create mode 100644 examples/route_guide/client/client.go create mode 100644 examples/route_guide/routeguide/route_guide.pb.go create mode 100644 examples/route_guide/routeguide/route_guide.proto create mode 100644 examples/route_guide/routeguide/route_guide_grpc.pb.go create mode 100644 examples/route_guide/server/server.go create mode 100644 examples/route_guide/testdata/route_guide_db.json create mode 100644 experimental/balancer/hostname/hostname.go create mode 100644 experimental/balancer/hostname/hostname_test.go create mode 100644 experimental/balancer/weight/weight.go create mode 100644 experimental/balancer/weight/weight_test.go create mode 100644 experimental/credentials/credentials_test.go create mode 100644 experimental/credentials/internal/spiffe.go create mode 100644 experimental/credentials/internal/spiffe_test.go create mode 100644 experimental/credentials/internal/syscallconn.go create mode 100644 experimental/credentials/internal/syscallconn_test.go create mode 100644 experimental/credentials/tls.go create mode 100644 experimental/credentials/tls_ext_test.go create mode 100644 experimental/experimental.go create mode 100644 experimental/opentelemetry/trace_options.go create mode 100644 experimental/shared_buffer_pool_test.go create mode 100644 experimental/stats/metricregistry.go create mode 100644 experimental/stats/metricregistry_test.go create mode 100644 experimental/stats/metrics.go create mode 100644 experimental/stats/telemetry/labels.go create mode 100644 gcp/observability/config.go create mode 100644 gcp/observability/exporting.go create mode 100644 gcp/observability/go.mod create mode 100644 gcp/observability/logging.go create mode 100644 gcp/observability/logging_test.go create mode 100644 gcp/observability/observability.go create mode 100644 gcp/observability/observability_test.go create mode 100644 gcp/observability/opencensus.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 grpc_test.go create mode 100644 grpclog/component.go create mode 100644 grpclog/glogger/glogger.go create mode 100644 grpclog/grpclog.go create mode 100644 grpclog/internal/grpclog.go create mode 100644 grpclog/internal/logger.go create mode 100644 grpclog/internal/loggerv2.go create mode 100644 grpclog/internal/loggerv2_test.go create mode 100644 grpclog/logger.go create mode 100644 grpclog/loggerv2.go create mode 100644 health/client.go create mode 100644 health/client_test.go create mode 100644 health/grpc_health_v1/health.pb.go create mode 100644 health/grpc_health_v1/health_grpc.pb.go create mode 100644 health/logging.go create mode 100644 health/producer.go create mode 100644 health/server.go create mode 100644 health/server_internal_test.go create mode 100644 health/server_test.go create mode 100644 interceptor.go create mode 100644 internal/admin/admin.go create mode 100644 internal/backoff/backoff.go create mode 100644 internal/balancer/gracefulswitch/config.go create mode 100644 internal/balancer/gracefulswitch/gracefulswitch.go create mode 100644 internal/balancer/gracefulswitch/gracefulswitch_test.go create mode 100644 internal/balancer/nop/nop.go create mode 100644 internal/balancer/stub/stub.go create mode 100644 internal/balancergroup/balancergroup.go create mode 100644 internal/balancergroup/balancergroup_test.go create mode 100644 internal/balancergroup/balancerstateaggregator.go create mode 100644 internal/balancerload/load.go create mode 100644 internal/binarylog/binarylog.go create mode 100644 internal/binarylog/binarylog_test.go create mode 100644 internal/binarylog/binarylog_testutil.go create mode 100644 internal/binarylog/env_config.go create mode 100644 internal/binarylog/env_config_test.go create mode 100644 internal/binarylog/method_logger.go create mode 100644 internal/binarylog/method_logger_test.go create mode 100644 internal/binarylog/regexp_test.go create mode 100644 internal/binarylog/sink.go create mode 100644 internal/buffer/unbounded.go create mode 100644 internal/buffer/unbounded_test.go create mode 100644 internal/cache/timeoutCache.go create mode 100644 internal/cache/timeoutCache_test.go create mode 100644 internal/channelz/channel.go create mode 100644 internal/channelz/channelmap.go create mode 100644 internal/channelz/funcs.go create mode 100644 internal/channelz/logging.go create mode 100644 internal/channelz/server.go create mode 100644 internal/channelz/socket.go create mode 100644 internal/channelz/subchannel.go create mode 100644 internal/channelz/syscall_linux.go create mode 100644 internal/channelz/syscall_nonlinux.go create mode 100644 internal/channelz/syscall_test.go create mode 100644 internal/channelz/trace.go create mode 100644 internal/credentials/credentials.go create mode 100644 internal/credentials/spiffe.go create mode 100644 internal/credentials/spiffe/spiffe.go create mode 100644 internal/credentials/spiffe/spiffe_test.go create mode 100644 internal/credentials/spiffe_test.go create mode 100644 internal/credentials/syscallconn.go create mode 100644 internal/credentials/syscallconn_test.go create mode 100644 internal/credentials/util.go create mode 100644 internal/credentials/util_test.go create mode 100644 internal/credentials/xds/handshake_info.go create mode 100644 internal/credentials/xds/handshake_info_test.go create mode 100644 internal/envconfig/envconfig.go create mode 100644 internal/envconfig/envconfig_test.go create mode 100644 internal/envconfig/observability.go create mode 100644 internal/envconfig/xds.go create mode 100644 internal/experimental.go create mode 100644 internal/googlecloud/googlecloud.go create mode 100644 internal/googlecloud/googlecloud_test.go create mode 100644 internal/googlecloud/manufacturer.go create mode 100644 internal/googlecloud/manufacturer_linux.go create mode 100644 internal/googlecloud/manufacturer_windows.go create mode 100644 internal/grpclog/prefix_logger.go create mode 100644 internal/grpcsync/callback_serializer.go create mode 100644 internal/grpcsync/callback_serializer_test.go create mode 100644 internal/grpcsync/event.go create mode 100644 internal/grpcsync/event_test.go create mode 100644 internal/grpcsync/pubsub.go create mode 100644 internal/grpcsync/pubsub_test.go create mode 100644 internal/grpcsync/refcounted.go create mode 100644 internal/grpcsync/refcounted_test.go create mode 100644 internal/grpctest/example_test.go create mode 100644 internal/grpctest/grpctest.go create mode 100644 internal/grpctest/grpctest_test.go create mode 100644 internal/grpctest/tlogger.go create mode 100644 internal/grpctest/tlogger_test.go create mode 100644 internal/grpcutil/compressor.go create mode 100644 internal/grpcutil/compressor_test.go create mode 100644 internal/grpcutil/encode_duration.go create mode 100644 internal/grpcutil/encode_duration_test.go create mode 100644 internal/grpcutil/grpcutil.go create mode 100644 internal/grpcutil/metadata.go create mode 100644 internal/grpcutil/method.go create mode 100644 internal/grpcutil/method_test.go create mode 100644 internal/hierarchy/hierarchy.go create mode 100644 internal/hierarchy/hierarchy_ext_test.go create mode 100644 internal/idle/idle.go create mode 100644 internal/idle/idle_e2e_test.go create mode 100644 internal/idle/idle_test.go create mode 100644 internal/internal.go create mode 100644 internal/leakcheck/leakcheck.go create mode 100644 internal/leakcheck/leakcheck_enabled.go create mode 100644 internal/leakcheck/leakcheck_test.go create mode 100644 internal/mem/buffer_pool.go create mode 100644 internal/mem/buffer_pool_ext_test.go create mode 100644 internal/mem/buffer_pool_test.go create mode 100644 internal/metadata/metadata.go create mode 100644 internal/metadata/metadata_test.go create mode 100644 internal/optional/optional.go create mode 100644 internal/optional/optional_test.go create mode 100644 internal/pretty/pretty.go create mode 100644 internal/profiling/buffer/buffer.go create mode 100644 internal/profiling/buffer/buffer_test.go create mode 100644 internal/profiling/goid_modified.go create mode 100644 internal/profiling/goid_regular.go create mode 100644 internal/profiling/profiling.go create mode 100644 internal/profiling/profiling_test.go create mode 100644 internal/proto/grpc_lookup_v1/rls.pb.go create mode 100644 internal/proto/grpc_lookup_v1/rls_config.pb.go create mode 100644 internal/proto/grpc_lookup_v1/rls_grpc.pb.go create mode 100644 internal/proxyattributes/proxyattributes.go create mode 100644 internal/proxyattributes/proxyattributes_test.go create mode 100644 internal/resolver/config_selector.go create mode 100644 internal/resolver/config_selector_test.go create mode 100644 internal/resolver/delegatingresolver/delegatingresolver.go create mode 100644 internal/resolver/delegatingresolver/delegatingresolver_ext_test.go create mode 100644 internal/resolver/delegatingresolver/delegatingresolver_test.go create mode 100644 internal/resolver/dns/dns_resolver.go create mode 100644 internal/resolver/dns/dns_resolver_test.go create mode 100644 internal/resolver/dns/fake_net_resolver_test.go create mode 100644 internal/resolver/dns/internal/internal.go create mode 100644 internal/resolver/passthrough/passthrough.go create mode 100644 internal/resolver/unix/unix.go create mode 100644 internal/ringhash/ringhash.go create mode 100644 internal/serviceconfig/duration.go create mode 100644 internal/serviceconfig/duration_test.go create mode 100644 internal/serviceconfig/serviceconfig.go create mode 100644 internal/serviceconfig/serviceconfig_test.go create mode 100644 internal/stats/labels.go create mode 100644 internal/stats/labels_test.go create mode 100644 internal/stats/metrics_recorder_list.go create mode 100644 internal/stats/metrics_recorder_list_test.go create mode 100644 internal/stats/stats.go create mode 100644 internal/status/status.go create mode 100644 internal/status/status_test.go create mode 100644 internal/stubserver/stubserver.go create mode 100644 internal/syscall/syscall_linux.go create mode 100644 internal/syscall/syscall_nonlinux.go create mode 100644 internal/tcp_keepalive_others.go create mode 100644 internal/tcp_keepalive_unix.go create mode 100644 internal/tcp_keepalive_windows.go create mode 100644 internal/testutils/balancer.go create mode 100644 internal/testutils/blocking_context_dialer.go create mode 100644 internal/testutils/blocking_context_dialer_test.go create mode 100644 internal/testutils/channel.go create mode 100644 internal/testutils/envconfig.go create mode 100644 internal/testutils/fakegrpclb/server.go create mode 100644 internal/testutils/http_client.go create mode 100644 internal/testutils/local_listener.go create mode 100644 internal/testutils/marshal_any.go create mode 100644 internal/testutils/parse_port.go create mode 100644 internal/testutils/parse_url.go create mode 100644 internal/testutils/pickfirst/pickfirst.go create mode 100644 internal/testutils/pipe_listener.go create mode 100644 internal/testutils/pipe_listener_test.go create mode 100644 internal/testutils/proxyserver/proxyserver.go create mode 100644 internal/testutils/resolver.go create mode 100644 internal/testutils/restartable_listener.go create mode 100644 internal/testutils/rls/fake_rls_server.go create mode 100644 internal/testutils/roundrobin/roundrobin.go create mode 100644 internal/testutils/state.go create mode 100644 internal/testutils/stats/test_metrics_recorder.go create mode 100644 internal/testutils/status_equal.go create mode 100644 internal/testutils/status_equal_test.go create mode 100644 internal/testutils/stubstatshandler.go create mode 100644 internal/testutils/tls_creds.go create mode 100644 internal/testutils/wrappers.go create mode 100644 internal/testutils/wrr.go create mode 100644 internal/testutils/xds/e2e/bootstrap.go create mode 100644 internal/testutils/xds/e2e/clientresources.go create mode 100644 internal/testutils/xds/e2e/logging.go create mode 100644 internal/testutils/xds/e2e/server.go create mode 100644 internal/testutils/xds/e2e/setup/setup.go create mode 100644 internal/testutils/xds/fakeserver/server.go create mode 100644 internal/testutils/xds_bootstrap.go create mode 100644 internal/transport/bdp_estimator.go create mode 100644 internal/transport/client_stream.go create mode 100644 internal/transport/controlbuf.go create mode 100644 internal/transport/controlbuf_test.go create mode 100644 internal/transport/defaults.go create mode 100644 internal/transport/flowcontrol.go create mode 100644 internal/transport/handler_server.go create mode 100644 internal/transport/handler_server_test.go create mode 100644 internal/transport/http2_client.go create mode 100644 internal/transport/http2_client_header_bench_test.go create mode 100644 internal/transport/http2_server.go create mode 100644 internal/transport/http_util.go create mode 100644 internal/transport/http_util_test.go create mode 100644 internal/transport/internal/internal.go create mode 100644 internal/transport/keepalive_test.go create mode 100644 internal/transport/logging.go create mode 100644 internal/transport/networktype/networktype.go create mode 100644 internal/transport/proxy.go create mode 100644 internal/transport/proxy_ext_test.go create mode 100644 internal/transport/proxy_test.go create mode 100644 internal/transport/readyreader/raw_conn_linux.go create mode 100644 internal/transport/readyreader/raw_conn_nonlinux.go create mode 100644 internal/transport/readyreader/ready_reader.go create mode 100644 internal/transport/readyreader/ready_reader_ext_test.go create mode 100644 internal/transport/server_metrics_ext_test.go create mode 100644 internal/transport/server_stream.go create mode 100644 internal/transport/transport.go create mode 100644 internal/wrr/edf.go create mode 100644 internal/wrr/edf_test.go create mode 100644 internal/wrr/random.go create mode 100644 internal/wrr/wrr.go create mode 100644 internal/wrr/wrr_test.go create mode 100644 internal/xds/balancer/balancer.go create mode 100644 internal/xds/balancer/cdsbalancer/aggregate_cluster_test.go create mode 100644 internal/xds/balancer/cdsbalancer/cdsbalancer.go create mode 100644 internal/xds/balancer/cdsbalancer/cdsbalancer_test.go create mode 100644 internal/xds/balancer/cdsbalancer/configbuilder.go create mode 100644 internal/xds/balancer/cdsbalancer/configbuilder_childname.go create mode 100644 internal/xds/balancer/cdsbalancer/configbuilder_childname_test.go create mode 100644 internal/xds/balancer/cdsbalancer/configbuilder_test.go create mode 100644 internal/xds/balancer/cdsbalancer/e2e_test/aggregate_cluster_test.go create mode 100644 internal/xds/balancer/cdsbalancer/e2e_test/balancer_test.go create mode 100644 internal/xds/balancer/cdsbalancer/e2e_test/dns_impl_test.go create mode 100644 internal/xds/balancer/cdsbalancer/e2e_test/eds_impl_test.go create mode 100644 internal/xds/balancer/cdsbalancer/logging.go create mode 100644 internal/xds/balancer/clusterimpl/balancer_test.go create mode 100644 internal/xds/balancer/clusterimpl/clusterimpl.go create mode 100644 internal/xds/balancer/clusterimpl/config.go create mode 100644 internal/xds/balancer/clusterimpl/config_test.go create mode 100644 internal/xds/balancer/clusterimpl/internal/internal.go create mode 100644 internal/xds/balancer/clusterimpl/logging.go create mode 100644 internal/xds/balancer/clusterimpl/picker.go create mode 100644 internal/xds/balancer/clusterimpl/tests/balancer_test.go create mode 100644 internal/xds/balancer/clusterimpl/tests/clusterimpl_security_test.go create mode 100644 internal/xds/balancer/clustermanager/balancerstateaggregator.go create mode 100644 internal/xds/balancer/clustermanager/clustermanager.go create mode 100644 internal/xds/balancer/clustermanager/clustermanager_test.go create mode 100644 internal/xds/balancer/clustermanager/config.go create mode 100644 internal/xds/balancer/clustermanager/config_test.go create mode 100644 internal/xds/balancer/clustermanager/e2e_test/clustermanager_test.go create mode 100644 internal/xds/balancer/clustermanager/picker.go create mode 100644 internal/xds/balancer/loadstore/load_store_wrapper.go create mode 100644 internal/xds/balancer/outlierdetection/balancer.go create mode 100644 internal/xds/balancer/outlierdetection/balancer_ext_test.go create mode 100644 internal/xds/balancer/outlierdetection/balancer_test.go create mode 100644 internal/xds/balancer/outlierdetection/callcounter.go create mode 100644 internal/xds/balancer/outlierdetection/callcounter_test.go create mode 100644 internal/xds/balancer/outlierdetection/config.go create mode 100644 internal/xds/balancer/outlierdetection/config_test.go create mode 100644 internal/xds/balancer/outlierdetection/e2e_test/outlierdetection_test.go create mode 100644 internal/xds/balancer/outlierdetection/logging.go create mode 100644 internal/xds/balancer/outlierdetection/subconn_wrapper.go create mode 100644 internal/xds/balancer/priority/balancer.go create mode 100644 internal/xds/balancer/priority/balancer_child.go create mode 100644 internal/xds/balancer/priority/balancer_priority.go create mode 100644 internal/xds/balancer/priority/balancer_test.go create mode 100644 internal/xds/balancer/priority/config.go create mode 100644 internal/xds/balancer/priority/config_test.go create mode 100644 internal/xds/balancer/priority/ignore_resolve_now.go create mode 100644 internal/xds/balancer/priority/ignore_resolve_now_test.go create mode 100644 internal/xds/balancer/priority/logging.go create mode 100644 internal/xds/balancer/wrrlocality/balancer.go create mode 100644 internal/xds/balancer/wrrlocality/balancer_test.go create mode 100644 internal/xds/balancer/wrrlocality/logging.go create mode 100644 internal/xds/bootstrap/bootstrap.go create mode 100644 internal/xds/bootstrap/bootstrap_test.go create mode 100644 internal/xds/bootstrap/jwtcreds/call_creds.go create mode 100644 internal/xds/bootstrap/jwtcreds/call_creds_test.go create mode 100644 internal/xds/bootstrap/logging.go create mode 100644 internal/xds/bootstrap/template.go create mode 100644 internal/xds/bootstrap/template_test.go create mode 100644 internal/xds/bootstrap/tlscreds/bundle.go create mode 100644 internal/xds/bootstrap/tlscreds/bundle_ext_test.go create mode 100644 internal/xds/bootstrap/tlscreds/bundle_test.go create mode 100644 internal/xds/clients/config.go create mode 100644 internal/xds/clients/grpctransport/examples_test.go create mode 100644 internal/xds/clients/grpctransport/grpc_transport.go create mode 100644 internal/xds/clients/grpctransport/grpc_transport_ext_test.go create mode 100644 internal/xds/clients/grpctransport/grpc_transport_test.go create mode 100644 internal/xds/clients/internal/backoff/backoff.go create mode 100644 internal/xds/clients/internal/buffer/unbounded.go create mode 100644 internal/xds/clients/internal/buffer/unbounded_test.go create mode 100644 internal/xds/clients/internal/internal.go create mode 100644 internal/xds/clients/internal/internal_test.go create mode 100644 internal/xds/clients/internal/pretty/pretty.go create mode 100644 internal/xds/clients/internal/syncutil/callback_serializer.go create mode 100644 internal/xds/clients/internal/syncutil/callback_serializer_test.go create mode 100644 internal/xds/clients/internal/syncutil/event.go create mode 100644 internal/xds/clients/internal/syncutil/event_test.go create mode 100644 internal/xds/clients/internal/testutils/channel.go create mode 100644 internal/xds/clients/internal/testutils/e2e/clientresources.go create mode 100644 internal/xds/clients/internal/testutils/e2e/logging.go create mode 100644 internal/xds/clients/internal/testutils/e2e/server.go create mode 100644 internal/xds/clients/internal/testutils/fakeserver/server.go create mode 100644 internal/xds/clients/internal/testutils/faketransport/xds_fake_transport.go create mode 100644 internal/xds/clients/internal/testutils/marshal_any.go create mode 100644 internal/xds/clients/internal/testutils/restartable_listener.go create mode 100644 internal/xds/clients/internal/testutils/wrappers.go create mode 100644 internal/xds/clients/lrsclient/internal/internal.go create mode 100644 internal/xds/clients/lrsclient/load_store.go create mode 100644 internal/xds/clients/lrsclient/load_store_test.go create mode 100644 internal/xds/clients/lrsclient/loadreport_test.go create mode 100644 internal/xds/clients/lrsclient/logging.go create mode 100644 internal/xds/clients/lrsclient/lrs_stream.go create mode 100644 internal/xds/clients/lrsclient/lrsclient.go create mode 100644 internal/xds/clients/lrsclient/lrsconfig.go create mode 100644 internal/xds/clients/transport_builder.go create mode 100644 internal/xds/clients/xdsclient/ads_stream.go create mode 100644 internal/xds/clients/xdsclient/authority.go create mode 100644 internal/xds/clients/xdsclient/channel.go create mode 100644 internal/xds/clients/xdsclient/channel_test.go create mode 100644 internal/xds/clients/xdsclient/clientimpl_watchers.go create mode 100644 internal/xds/clients/xdsclient/helpers_test.go create mode 100644 internal/xds/clients/xdsclient/internal/internal.go create mode 100644 internal/xds/clients/xdsclient/internal/xdsresource/ads_stream.go create mode 100644 internal/xds/clients/xdsclient/internal/xdsresource/errors.go create mode 100644 internal/xds/clients/xdsclient/internal/xdsresource/name.go create mode 100644 internal/xds/clients/xdsclient/internal/xdsresource/type.go create mode 100644 internal/xds/clients/xdsclient/internal/xdsresource/version.go create mode 100644 internal/xds/clients/xdsclient/logging.go create mode 100644 internal/xds/clients/xdsclient/metrics/metrics.go create mode 100644 internal/xds/clients/xdsclient/resource_type.go create mode 100644 internal/xds/clients/xdsclient/resource_watcher.go create mode 100644 internal/xds/clients/xdsclient/test/ads_stream_ack_nack_test.go create mode 100644 internal/xds/clients/xdsclient/test/ads_stream_backoff_test.go create mode 100644 internal/xds/clients/xdsclient/test/ads_stream_flow_control_test.go create mode 100644 internal/xds/clients/xdsclient/test/ads_stream_restart_test.go create mode 100644 internal/xds/clients/xdsclient/test/ads_stream_watch_test.go create mode 100644 internal/xds/clients/xdsclient/test/authority_test.go create mode 100644 internal/xds/clients/xdsclient/test/custom_resource_watch_test.go create mode 100644 internal/xds/clients/xdsclient/test/dump_test.go create mode 100644 internal/xds/clients/xdsclient/test/helpers_test.go create mode 100644 internal/xds/clients/xdsclient/test/lds_watchers_test.go create mode 100644 internal/xds/clients/xdsclient/test/metrics_test.go create mode 100644 internal/xds/clients/xdsclient/test/misc_watchers_test.go create mode 100644 internal/xds/clients/xdsclient/xdsclient.go create mode 100644 internal/xds/clients/xdsclient/xdsclient_test.go create mode 100644 internal/xds/clients/xdsclient/xdsconfig.go create mode 100644 internal/xds/clusterspecifier/cluster_specifier.go create mode 100644 internal/xds/clusterspecifier/rls/rls.go create mode 100644 internal/xds/clusterspecifier/rls/rls_test.go create mode 100644 internal/xds/credentials/access_token.go create mode 100644 internal/xds/credentials/access_token_test.go create mode 100644 internal/xds/credentials/credentials.go create mode 100644 internal/xds/credentials/tls.go create mode 100644 internal/xds/credentials/tls_test.go create mode 100644 internal/xds/credentials/xds_test.go create mode 100644 internal/xds/grpcservice/grpcservice.go create mode 100644 internal/xds/grpcservice/grpcservice_test.go create mode 100644 internal/xds/httpfilter/ext_authz/config.go create mode 100644 internal/xds/httpfilter/ext_authz/ext_authz.go create mode 100644 internal/xds/httpfilter/ext_authz/ext_authz_test.go create mode 100644 internal/xds/httpfilter/extconfig.go create mode 100644 internal/xds/httpfilter/extconfig_test.go create mode 100644 internal/xds/httpfilter/extproc/config.go create mode 100644 internal/xds/httpfilter/extproc/config_test.go create mode 100644 internal/xds/httpfilter/extproc/ext_proc.go create mode 100644 internal/xds/httpfilter/extproc/internal/internal.go create mode 100644 internal/xds/httpfilter/extproc/metrics.go create mode 100644 internal/xds/httpfilter/fault/fault.go create mode 100644 internal/xds/httpfilter/fault/fault_test.go create mode 100644 internal/xds/httpfilter/gcp_authn/gcp_authn_filter.go create mode 100644 internal/xds/httpfilter/gcp_authn/gcp_authn_filter_e2e_test.go create mode 100644 internal/xds/httpfilter/gcp_authn/gcp_authn_filter_test.go create mode 100644 internal/xds/httpfilter/httpfilter.go create mode 100644 internal/xds/httpfilter/rbac/rbac.go create mode 100644 internal/xds/httpfilter/rbac/rbac_test.go create mode 100644 internal/xds/httpfilter/router/router.go create mode 100644 internal/xds/matcher/matcher_header.go create mode 100644 internal/xds/matcher/matcher_header_test.go create mode 100644 internal/xds/matcher/string_matcher.go create mode 100644 internal/xds/matcher/string_matcher_test.go create mode 100644 internal/xds/rbac/converter.go create mode 100644 internal/xds/rbac/converter_test.go create mode 100644 internal/xds/rbac/matchers.go create mode 100644 internal/xds/rbac/rbac_engine.go create mode 100644 internal/xds/rbac/rbac_engine_test.go create mode 100644 internal/xds/resolver/cluster_specifier_plugin_test.go create mode 100644 internal/xds/resolver/helpers_test.go create mode 100644 internal/xds/resolver/internal/internal.go create mode 100644 internal/xds/resolver/logging.go create mode 100644 internal/xds/resolver/serviceconfig.go create mode 100644 internal/xds/resolver/serviceconfig_test.go create mode 100644 internal/xds/resolver/watch_service_test.go create mode 100644 internal/xds/resolver/xds_http_filters_test.go create mode 100644 internal/xds/resolver/xds_resolver.go create mode 100644 internal/xds/resolver/xds_resolver_test.go create mode 100644 internal/xds/server/conn_wrapper.go create mode 100644 internal/xds/server/filter_chain_manager.go create mode 100644 internal/xds/server/filter_chain_manager_test.go create mode 100644 internal/xds/server/listener_wrapper.go create mode 100644 internal/xds/server/listener_wrapper_test.go create mode 100644 internal/xds/server/rds_handler.go create mode 100644 internal/xds/server/rds_handler_test.go create mode 100644 internal/xds/server/routing.go create mode 100644 internal/xds/server/routing_test.go create mode 100644 internal/xds/server/server_options.go create mode 100644 internal/xds/test/e2e/README.md create mode 100644 internal/xds/test/e2e/controlplane.go create mode 100644 internal/xds/test/e2e/e2e.go create mode 100644 internal/xds/test/e2e/e2e_test.go create mode 100644 internal/xds/test/e2e/e2e_utils.go create mode 100755 internal/xds/test/e2e/run.sh create mode 100644 internal/xds/testutils/balancer_test.go create mode 100644 internal/xds/testutils/fakeclient/client.go create mode 100644 internal/xds/testutils/resource_watcher.go create mode 100644 internal/xds/testutils/testutils.go create mode 100644 internal/xds/xds.go create mode 100644 internal/xds/xds_test.go create mode 100644 internal/xds/xdsclient/attributes.go create mode 100644 internal/xds/xdsclient/client.go create mode 100644 internal/xds/xdsclient/client_refcounted_test.go create mode 100644 internal/xds/xdsclient/client_test.go create mode 100644 internal/xds/xdsclient/clientimpl.go create mode 100644 internal/xds/xdsclient/clientimpl_loadreport.go create mode 100644 internal/xds/xdsclient/clientimpl_test.go create mode 100644 internal/xds/xdsclient/internal/internal.go create mode 100644 internal/xds/xdsclient/logging.go create mode 100644 internal/xds/xdsclient/metrics_test.go create mode 100644 internal/xds/xdsclient/pool.go create mode 100644 internal/xds/xdsclient/pool/pool_ext_test.go create mode 100644 internal/xds/xdsclient/requests_counter.go create mode 100644 internal/xds/xdsclient/requests_counter_test.go create mode 100644 internal/xds/xdsclient/resource_types.go create mode 100644 internal/xds/xdsclient/tests/ads_stream_ack_nack_test.go create mode 100644 internal/xds/xdsclient/tests/ads_stream_restart_test.go create mode 100644 internal/xds/xdsclient/tests/authority_test.go create mode 100644 internal/xds/xdsclient/tests/cds_watchers_test.go create mode 100644 internal/xds/xdsclient/tests/client_custom_dialopts_test.go create mode 100644 internal/xds/xdsclient/tests/dump_test.go create mode 100644 internal/xds/xdsclient/tests/eds_watchers_test.go create mode 100644 internal/xds/xdsclient/tests/fallback/fallback_test.go create mode 100644 internal/xds/xdsclient/tests/federation_watchers_test.go create mode 100644 internal/xds/xdsclient/tests/helpers_test.go create mode 100644 internal/xds/xdsclient/tests/lds_watchers_test.go create mode 100644 internal/xds/xdsclient/tests/loadreport_test.go create mode 100644 internal/xds/xdsclient/tests/rds_watchers_test.go create mode 100644 internal/xds/xdsclient/tests/resource_update_test.go create mode 100644 internal/xds/xdsclient/xdsclient_test.go create mode 100644 internal/xds/xdsclient/xdslbregistry/converter/converter.go create mode 100644 internal/xds/xdsclient/xdslbregistry/xdslbregistry.go create mode 100644 internal/xds/xdsclient/xdslbregistry/xdslbregistry_test.go create mode 100644 internal/xds/xdsclient/xdsresource/cluster_resource_type.go create mode 100644 internal/xds/xdsclient/xdsresource/endpoints_resource_type.go create mode 100644 internal/xds/xdsclient/xdsresource/errors.go create mode 100644 internal/xds/xdsclient/xdsresource/filter_chain.go create mode 100644 internal/xds/xdsclient/xdsresource/filter_chain_test.go create mode 100644 internal/xds/xdsclient/xdsresource/listener_resource_type.go create mode 100644 internal/xds/xdsclient/xdsresource/logging.go create mode 100644 internal/xds/xdsclient/xdsresource/matcher.go create mode 100644 internal/xds/xdsclient/xdsresource/matcher_path.go create mode 100644 internal/xds/xdsclient/xdsresource/matcher_path_test.go create mode 100644 internal/xds/xdsclient/xdsresource/matcher_test.go create mode 100644 internal/xds/xdsclient/xdsresource/metadata.go create mode 100644 internal/xds/xdsclient/xdsresource/metadata_test.go create mode 100644 internal/xds/xdsclient/xdsresource/name.go create mode 100644 internal/xds/xdsclient/xdsresource/name_test.go create mode 100644 internal/xds/xdsclient/xdsresource/resource_type.go create mode 100644 internal/xds/xdsclient/xdsresource/route_config_resource_type.go create mode 100644 internal/xds/xdsclient/xdsresource/test_utils_test.go create mode 100644 internal/xds/xdsclient/xdsresource/tests/unmarshal_cds_test.go create mode 100644 internal/xds/xdsclient/xdsresource/type.go create mode 100644 internal/xds/xdsclient/xdsresource/type_cds.go create mode 100644 internal/xds/xdsclient/xdsresource/type_eds.go create mode 100644 internal/xds/xdsclient/xdsresource/type_lds.go create mode 100644 internal/xds/xdsclient/xdsresource/type_rds.go create mode 100644 internal/xds/xdsclient/xdsresource/unmarshal_cds.go create mode 100644 internal/xds/xdsclient/xdsresource/unmarshal_cds_test.go create mode 100644 internal/xds/xdsclient/xdsresource/unmarshal_eds.go create mode 100644 internal/xds/xdsclient/xdsresource/unmarshal_eds_test.go create mode 100644 internal/xds/xdsclient/xdsresource/unmarshal_lds.go create mode 100644 internal/xds/xdsclient/xdsresource/unmarshal_lds_test.go create mode 100644 internal/xds/xdsclient/xdsresource/unmarshal_rds.go create mode 100644 internal/xds/xdsclient/xdsresource/unmarshal_rds_test.go create mode 100644 internal/xds/xdsclient/xdsresource/version/version.go create mode 100644 internal/xds/xdsclient/xdsresource/xdsconfig.go create mode 100644 internal/xds/xdsdepmgr/xds_dependency_manager.go create mode 100644 internal/xds/xdsdepmgr/xds_dependency_manager_test.go create mode 100644 interop/alts/client/client.go create mode 100644 interop/alts/server/server.go create mode 100644 interop/client/client.go create mode 100644 interop/fake_grpclb/fake_grpclb.go create mode 100644 interop/grpc_testing/benchmark_service.pb.go create mode 100644 interop/grpc_testing/benchmark_service_grpc.pb.go create mode 100644 interop/grpc_testing/control.pb.go create mode 100644 interop/grpc_testing/core/stats.pb.go create mode 100644 interop/grpc_testing/empty.pb.go create mode 100644 interop/grpc_testing/messages.pb.go create mode 100644 interop/grpc_testing/payloads.pb.go create mode 100644 interop/grpc_testing/report_qps_scenario_service.pb.go create mode 100644 interop/grpc_testing/report_qps_scenario_service_grpc.pb.go create mode 100644 interop/grpc_testing/stats.pb.go create mode 100644 interop/grpc_testing/test.pb.go create mode 100644 interop/grpc_testing/test_grpc.pb.go create mode 100644 interop/grpc_testing/worker_service.pb.go create mode 100644 interop/grpc_testing/worker_service_grpc.pb.go create mode 100644 interop/grpclb_fallback/client_linux.go create mode 100644 interop/http2/negative_http2_client.go create mode 100755 interop/interop_test.sh create mode 100644 interop/observability/Dockerfile create mode 100755 interop/observability/build_docker.sh create mode 100644 interop/observability/client/client.go create mode 100644 interop/observability/go.mod create mode 100755 interop/observability/run.sh create mode 100644 interop/observability/server/server.go create mode 100644 interop/orcalb.go create mode 100644 interop/orcalb_test.go create mode 100644 interop/server/server.go create mode 100644 interop/soak_tests.go create mode 100644 interop/stress/client/main.go create mode 100644 interop/stress/grpc_testing/metrics.pb.go create mode 100644 interop/stress/grpc_testing/metrics.proto create mode 100644 interop/stress/grpc_testing/metrics_grpc.pb.go create mode 100644 interop/stress/metrics_client/main.go create mode 100644 interop/test_utils.go create mode 100644 interop/xds/client/Dockerfile create mode 100644 interop/xds/client/client.go create mode 100644 interop/xds/custom_lb.go create mode 100644 interop/xds/custom_lb_test.go create mode 100644 interop/xds/go.mod create mode 100644 interop/xds/go.sum create mode 100644 interop/xds/server/Dockerfile create mode 100644 interop/xds/server/server.go create mode 100644 interop/xds_federation/client.go create mode 100644 keepalive/keepalive.go create mode 100644 mem/buffer_pool.go create mode 100644 mem/buffer_pool_test.go create mode 100644 mem/buffer_slice.go create mode 100644 mem/buffer_slice_test.go create mode 100644 mem/buffers.go create mode 100644 mem/buffers_test.go create mode 100644 metadata/metadata.go create mode 100644 metadata/metadata_test.go create mode 100644 orca/call_metrics.go create mode 100644 orca/call_metrics_test.go create mode 100644 orca/internal/internal.go create mode 100644 orca/orca.go create mode 100644 orca/orca_test.go create mode 100644 orca/producer.go create mode 100644 orca/producer_test.go create mode 100644 orca/server_metrics.go create mode 100644 orca/server_metrics_test.go create mode 100644 orca/service.go create mode 100644 orca/service_test.go create mode 100644 peer/peer.go create mode 100644 peer/peer_test.go create mode 100644 picker_wrapper.go create mode 100644 picker_wrapper_test.go create mode 100644 preloader.go create mode 100644 producer_ext_test.go create mode 100644 profiling/cmd/catapult.go create mode 100644 profiling/cmd/flags.go create mode 100644 profiling/cmd/local.go create mode 100644 profiling/cmd/main.go create mode 100644 profiling/cmd/remote.go create mode 100644 profiling/profiling.go create mode 100644 profiling/proto/service.pb.go create mode 100644 profiling/proto/service.proto create mode 100644 profiling/proto/service_grpc.pb.go create mode 100644 profiling/service/service.go create mode 100644 reflection/README.md create mode 100644 reflection/adapt.go create mode 100644 reflection/grpc_reflection_v1/reflection.pb.go create mode 100644 reflection/grpc_reflection_v1/reflection_grpc.pb.go create mode 100644 reflection/grpc_reflection_v1alpha/reflection.pb.go create mode 100644 reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go create mode 100644 reflection/grpc_testing/proto2.pb.go create mode 100644 reflection/grpc_testing/proto2.proto create mode 100644 reflection/grpc_testing/proto2_ext.pb.go create mode 100644 reflection/grpc_testing/proto2_ext.proto create mode 100644 reflection/grpc_testing/proto2_ext2.pb.go create mode 100644 reflection/grpc_testing/proto2_ext2.proto create mode 100644 reflection/grpc_testing/test.pb.go create mode 100644 reflection/grpc_testing/test.proto create mode 100644 reflection/grpc_testing/test_grpc.pb.go create mode 100644 reflection/internal/internal.go create mode 100644 reflection/serverreflection.go create mode 100644 reflection/test/serverreflection_test.go create mode 100644 resolver/dns/dns_resolver.go create mode 100644 resolver/manual/manual.go create mode 100644 resolver/manual/manual_test.go create mode 100644 resolver/map.go create mode 100644 resolver/map_test.go create mode 100644 resolver/passthrough/passthrough.go create mode 100644 resolver/resolver.go create mode 100644 resolver/resolver_test.go create mode 100644 resolver/ringhash/attr.go create mode 100644 resolver_balancer_ext_test.go create mode 100644 resolver_test.go create mode 100644 resolver_wrapper.go create mode 100644 rpc_util.go create mode 100644 rpc_util_test.go create mode 100755 scripts/common.sh create mode 100755 scripts/gen-deps.sh create mode 100755 scripts/install-protoc.sh create mode 100755 scripts/regenerate.sh create mode 100644 scripts/revive.toml create mode 100755 scripts/vet-proto.sh create mode 100755 scripts/vet.sh create mode 100644 security/advancedtls/advancedtls.go create mode 100644 security/advancedtls/advancedtls_integration_test.go create mode 100644 security/advancedtls/advancedtls_test.go create mode 100644 security/advancedtls/certificate_utils_test.go create mode 100644 security/advancedtls/crl.go create mode 100644 security/advancedtls/crl_provider.go create mode 100644 security/advancedtls/crl_provider_test.go create mode 100644 security/advancedtls/crl_test.go create mode 100644 security/advancedtls/examples/credential_reloading_from_files/README.md create mode 100644 security/advancedtls/examples/credential_reloading_from_files/client/main.go create mode 100644 security/advancedtls/examples/credential_reloading_from_files/server/main.go create mode 100755 security/advancedtls/examples/examples_test.sh create mode 100644 security/advancedtls/examples/go.mod create mode 100644 security/advancedtls/examples/go.sum create mode 100644 security/advancedtls/go.mod create mode 100644 security/advancedtls/go.sum create mode 100644 security/advancedtls/internal/testutils/testutils.go create mode 100644 security/advancedtls/sni.go create mode 100644 security/advancedtls/testdata/README.md create mode 100644 security/advancedtls/testdata/another_client_cert_1.pem create mode 100644 security/advancedtls/testdata/another_client_key_1.pem create mode 100644 security/advancedtls/testdata/client_cert_1.pem create mode 100644 security/advancedtls/testdata/client_cert_2.pem create mode 100644 security/advancedtls/testdata/client_key_1.pem create mode 100644 security/advancedtls/testdata/client_key_2.pem create mode 100644 security/advancedtls/testdata/client_trust_cert_1.pem create mode 100644 security/advancedtls/testdata/client_trust_cert_2.pem create mode 100644 security/advancedtls/testdata/client_trust_key_1.pem create mode 100644 security/advancedtls/testdata/client_trust_key_2.pem create mode 120000 security/advancedtls/testdata/crl/0b35a562.r0 create mode 120000 security/advancedtls/testdata/crl/0b35a562.r1 create mode 100644 security/advancedtls/testdata/crl/1.crl create mode 120000 security/advancedtls/testdata/crl/1ab871c8.r0 create mode 100644 security/advancedtls/testdata/crl/2.crl create mode 100644 security/advancedtls/testdata/crl/2f11f022.r0 create mode 100644 security/advancedtls/testdata/crl/3.crl create mode 100644 security/advancedtls/testdata/crl/4.crl create mode 100644 security/advancedtls/testdata/crl/5.crl create mode 100644 security/advancedtls/testdata/crl/6.crl create mode 120000 security/advancedtls/testdata/crl/71eac5a2.r0 create mode 120000 security/advancedtls/testdata/crl/7a1799af.r0 create mode 120000 security/advancedtls/testdata/crl/8828a7e6.r0 create mode 100644 security/advancedtls/testdata/crl/README.md create mode 120000 security/advancedtls/testdata/crl/deee447d.r0 create mode 100644 security/advancedtls/testdata/crl/provider_ca.cnf create mode 100644 security/advancedtls/testdata/crl/provider_client_cert.key create mode 100644 security/advancedtls/testdata/crl/provider_client_cert.pem create mode 100644 security/advancedtls/testdata/crl/provider_client_trust_cert.pem create mode 100644 security/advancedtls/testdata/crl/provider_client_trust_key.pem create mode 100755 security/advancedtls/testdata/crl/provider_create.sh create mode 100644 security/advancedtls/testdata/crl/provider_crl.cnf create mode 100644 security/advancedtls/testdata/crl/provider_crl_empty.pem create mode 100644 security/advancedtls/testdata/crl/provider_crl_server_revoked.pem create mode 100644 security/advancedtls/testdata/crl/provider_extensions.conf create mode 100644 security/advancedtls/testdata/crl/provider_malicious_client_trust_cert.pem create mode 100644 security/advancedtls/testdata/crl/provider_malicious_client_trust_key.pem create mode 100644 security/advancedtls/testdata/crl/provider_malicious_crl_empty.pem create mode 100644 security/advancedtls/testdata/crl/provider_server_cert.key create mode 100644 security/advancedtls/testdata/crl/provider_server_cert.pem create mode 100644 security/advancedtls/testdata/crl/provider_server_trust_cert.pem create mode 100644 security/advancedtls/testdata/crl/provider_server_trust_key.pem create mode 100644 security/advancedtls/testdata/crl/revokedInt.pem create mode 100644 security/advancedtls/testdata/crl/revokedLeaf.pem create mode 100644 security/advancedtls/testdata/crl/unrevoked.pem create mode 100644 security/advancedtls/testdata/localhost-openssl.cnf create mode 100644 security/advancedtls/testdata/openssl-ca.cnf create mode 100644 security/advancedtls/testdata/server_cert_1.pem create mode 100644 security/advancedtls/testdata/server_cert_1.txt create mode 100644 security/advancedtls/testdata/server_cert_2.pem create mode 100644 security/advancedtls/testdata/server_cert_2.txt create mode 100644 security/advancedtls/testdata/server_cert_3.pem create mode 100644 security/advancedtls/testdata/server_cert_3.txt create mode 100644 security/advancedtls/testdata/server_cert_localhost_1.pem create mode 100644 security/advancedtls/testdata/server_key_1.pem create mode 100644 security/advancedtls/testdata/server_key_2.pem create mode 100644 security/advancedtls/testdata/server_key_3.pem create mode 100644 security/advancedtls/testdata/server_key_localhost_1.pem create mode 100644 security/advancedtls/testdata/server_trust_cert_1.pem create mode 100644 security/advancedtls/testdata/server_trust_cert_2.pem create mode 100644 security/advancedtls/testdata/server_trust_key_1.pem create mode 100644 security/advancedtls/testdata/server_trust_key_2.pem create mode 100644 security/advancedtls/testdata/testdata.go create mode 100644 server.go create mode 100644 server_ext_test.go create mode 100644 server_test.go create mode 100644 service_config.go create mode 100644 service_config_test.go create mode 100644 serviceconfig/serviceconfig.go create mode 100644 stats/handlers.go create mode 100644 stats/metrics.go create mode 100644 stats/opencensus/client_metrics.go create mode 100644 stats/opencensus/e2e_test.go create mode 100644 stats/opencensus/go.mod create mode 100644 stats/opencensus/opencensus.go create mode 100644 stats/opencensus/server_metrics.go create mode 100644 stats/opencensus/stats.go create mode 100644 stats/opencensus/trace.go create mode 100644 stats/opentelemetry/client_metrics.go create mode 100644 stats/opentelemetry/client_tracing.go create mode 100644 stats/opentelemetry/csm/observability.go create mode 100644 stats/opentelemetry/csm/observability_test.go create mode 100644 stats/opentelemetry/csm/pluginoption.go create mode 100644 stats/opentelemetry/csm/pluginoption_test.go create mode 100644 stats/opentelemetry/e2e_test.go create mode 100644 stats/opentelemetry/example_test.go create mode 100644 stats/opentelemetry/grpc_trace_bin_propagator.go create mode 100644 stats/opentelemetry/grpc_trace_bin_propagator_test.go create mode 100644 stats/opentelemetry/internal/pluginoption.go create mode 100644 stats/opentelemetry/internal/testutils/testutils.go create mode 100644 stats/opentelemetry/internal/tracing/carrier.go create mode 100644 stats/opentelemetry/internal/tracing/carrier_test.go create mode 100644 stats/opentelemetry/metricsregistry_test.go create mode 100644 stats/opentelemetry/opentelemetry.go create mode 100644 stats/opentelemetry/server_metrics.go create mode 100644 stats/opentelemetry/server_tracing.go create mode 100644 stats/opentelemetry/trace.go create mode 100644 stats/stats.go create mode 100644 stats/stats_test.go create mode 100644 status/status.go create mode 100644 status/status_ext_test.go create mode 100644 status/status_test.go create mode 100644 stream.go create mode 100644 stream_interfaces.go create mode 100644 stream_test.go create mode 100644 tap/tap.go create mode 100644 test/authority_test.go create mode 100644 test/balancer_switching_test.go create mode 100644 test/balancer_test.go create mode 100644 test/bufconn/bufconn.go create mode 100644 test/bufconn/bufconn_test.go create mode 100644 test/channelz_linux_test.go create mode 100644 test/channelz_test.go create mode 100644 test/clientconn_state_transition_test.go create mode 100644 test/clientconn_test.go create mode 100644 test/clienttester.go create mode 100644 test/codec_perf/perf.pb.go create mode 100644 test/codec_perf/perf.proto create mode 100644 test/compressor_test.go create mode 100644 test/config_selector_test.go create mode 100644 test/context_canceled_test.go create mode 100644 test/control_plane_status_test.go create mode 100644 test/creds_test.go create mode 100644 test/goaway_test.go create mode 100644 test/gracefulstop_test.go create mode 100644 test/healthcheck_test.go create mode 100644 test/http_header_end2end_test.go create mode 100644 test/insecure_creds_test.go create mode 100644 test/interceptor_test.go create mode 100644 test/invoke_test.go create mode 100644 test/kokoro/README.md create mode 100644 test/kokoro/psm-csm.cfg create mode 100644 test/kokoro/psm-dualstack.cfg create mode 100755 test/kokoro/psm-interop-build-go.sh create mode 100755 test/kokoro/psm-interop-test-go.sh create mode 100644 test/kokoro/psm-light.cfg create mode 100644 test/kokoro/psm-regional_td.cfg create mode 100644 test/kokoro/psm-security.cfg create mode 100644 test/kokoro/psm-spiffe.cfg create mode 100644 test/kokoro/xds.cfg create mode 100755 test/kokoro/xds.sh create mode 100644 test/kokoro/xds_k8s_lb.cfg create mode 100644 test/kokoro/xds_url_map.cfg create mode 100644 test/kokoro/xds_v3.cfg create mode 100755 test/kokoro/xds_v3.sh create mode 100644 test/local_creds_test.go create mode 100644 test/logging.go create mode 100644 test/malformed_method_test.go create mode 100644 test/metadata_test.go create mode 100644 test/parse_config.go create mode 100644 test/race_test.go create mode 100644 test/rawConnWrapper.go create mode 100644 test/resolver_update_test.go create mode 100644 test/retry_test.go create mode 100644 test/roundrobin_test.go create mode 100644 test/server_test.go create mode 100644 test/servertester.go create mode 100644 test/stats_test.go create mode 100644 test/stream_cleanup_test.go create mode 100644 test/subconn_test.go create mode 100644 test/timeouts.go create mode 100644 test/tools/go.mod create mode 100644 test/tools/go.sum create mode 100644 test/tools/tools.go create mode 100644 test/tools/tools_vet.go create mode 100644 test/transport_test.go create mode 100644 test/xds/xds_client_ack_nack_test.go create mode 100644 test/xds/xds_client_affinity_test.go create mode 100644 test/xds/xds_client_certificate_providers_test.go create mode 100644 test/xds/xds_client_custom_lb_test.go create mode 100644 test/xds/xds_client_federation_test.go create mode 100644 test/xds/xds_client_filter_e2e_test.go create mode 100644 test/xds/xds_client_http_connect_test.go create mode 100644 test/xds/xds_client_ignore_resource_deletion_test.go create mode 100644 test/xds/xds_client_integration_test.go create mode 100644 test/xds/xds_client_outlier_detection_test.go create mode 100644 test/xds/xds_client_priority_locality_test.go create mode 100644 test/xds/xds_client_retry_test.go create mode 100644 test/xds/xds_client_sni_test.go create mode 100644 test/xds/xds_resolver_e2e_test.go create mode 100644 test/xds/xds_rls_clusterspecifier_plugin_test.go create mode 100644 test/xds/xds_security_config_nack_test.go create mode 100644 test/xds/xds_server_filter_e2e_test.go create mode 100644 test/xds/xds_server_filter_override_test.go create mode 100644 test/xds/xds_server_filter_state_retention_test.go create mode 100644 test/xds/xds_server_integration_test.go create mode 100644 test/xds/xds_server_interceptor_leak_test.go create mode 100644 test/xds/xds_server_rbac_test.go create mode 100644 test/xds/xds_telemetry_labels_test.go create mode 100644 testdata/README.md create mode 100644 testdata/ca.pem create mode 100644 testdata/grpc_testing_not_regenerated/README.md create mode 100644 testdata/grpc_testing_not_regenerated/dynamic.go create mode 100644 testdata/grpc_testing_not_regenerated/dynamic.proto create mode 100644 testdata/grpc_testing_not_regenerated/simple.proto create mode 100644 testdata/grpc_testing_not_regenerated/simple_message_v1.go create mode 100644 testdata/grpc_testing_not_regenerated/testv3.go create mode 100644 testdata/grpc_testing_not_regenerated/testv3.proto create mode 100644 testdata/server1.key create mode 100644 testdata/server1.pem create mode 100644 testdata/spiffe/README.md create mode 100644 testdata/spiffe/client_spiffe.pem create mode 100644 testdata/spiffe/server1_spiffe.pem create mode 100644 testdata/spiffe/spiffe-openssl.cnf create mode 100644 testdata/spiffe/spiffe_cert.pem create mode 100644 testdata/spiffe/spiffe_multi_uri_san_cert.pem create mode 100644 testdata/spiffe/spiffe_test.json create mode 100644 testdata/spiffe/spiffebundle.json create mode 100644 testdata/spiffe/spiffebundle2.json create mode 100644 testdata/spiffe/spiffebundle_corrupted_cert.json create mode 100644 testdata/spiffe/spiffebundle_empty_keys.json create mode 100644 testdata/spiffe/spiffebundle_empty_string_key.json create mode 100644 testdata/spiffe/spiffebundle_invalid_trustdomain.json create mode 100644 testdata/spiffe/spiffebundle_malformed.json create mode 100644 testdata/spiffe/spiffebundle_match_client_spiffe.json create mode 100644 testdata/spiffe/spiffebundle_wrong_kid.json create mode 100644 testdata/spiffe/spiffebundle_wrong_kty.json create mode 100644 testdata/spiffe/spiffebundle_wrong_multi_certs.json create mode 100644 testdata/spiffe/spiffebundle_wrong_root.json create mode 100644 testdata/spiffe/spiffebundle_wrong_seq_type.json create mode 100644 testdata/spiffe/spiffebundle_wrong_use.json create mode 100644 testdata/spiffe_end2end/README.md create mode 100644 testdata/spiffe_end2end/ca.key create mode 100644 testdata/spiffe_end2end/ca.pem create mode 100644 testdata/spiffe_end2end/client.key create mode 100644 testdata/spiffe_end2end/client_spiffe.pem create mode 100644 testdata/spiffe_end2end/client_spiffebundle.json create mode 100755 testdata/spiffe_end2end/generate.sh create mode 100644 testdata/spiffe_end2end/intermediate.cnf create mode 100644 testdata/spiffe_end2end/intermediate_ca.key create mode 100644 testdata/spiffe_end2end/intermediate_ca.pem create mode 100755 testdata/spiffe_end2end/intermediate_gen.sh create mode 100644 testdata/spiffe_end2end/leaf_and_intermediate_chain.pem create mode 100644 testdata/spiffe_end2end/leaf_signed_by_intermediate.key create mode 100644 testdata/spiffe_end2end/leaf_signed_by_intermediate.pem create mode 100644 testdata/spiffe_end2end/server.key create mode 100644 testdata/spiffe_end2end/server_spiffe.pem create mode 100644 testdata/spiffe_end2end/server_spiffebundle.json create mode 100644 testdata/spiffe_end2end/spiffe-openssl.cnf create mode 100644 testdata/testdata.go create mode 100644 testdata/x509/README.md create mode 100644 testdata/x509/client1_cert.pem create mode 100644 testdata/x509/client1_key.pem create mode 100644 testdata/x509/client2_cert.pem create mode 100644 testdata/x509/client2_key.pem create mode 100644 testdata/x509/client_ca_cert.pem create mode 100644 testdata/x509/client_ca_key.pem create mode 100644 testdata/x509/client_with_spiffe_cert.pem create mode 100644 testdata/x509/client_with_spiffe_key.pem create mode 100644 testdata/x509/client_with_spiffe_openssl.cnf create mode 100755 testdata/x509/create.sh create mode 100644 testdata/x509/multiple_uri_cert.pem create mode 100644 testdata/x509/multiple_uri_key.pem create mode 100644 testdata/x509/openssl.cnf create mode 100644 testdata/x509/server1_cert.pem create mode 100644 testdata/x509/server1_key.pem create mode 100644 testdata/x509/server2_cert.pem create mode 100644 testdata/x509/server2_key.pem create mode 100644 testdata/x509/server_ca_cert.pem create mode 100644 testdata/x509/server_ca_key.pem create mode 100644 testdata/x509/spiffe_cert.pem create mode 100644 testdata/x509/spiffe_key.pem create mode 100644 trace.go create mode 100644 trace_notrace.go create mode 100644 trace_test.go create mode 100644 trace_withtrace.go create mode 100644 version.go create mode 100644 xds/bootstrap/bootstrap.go create mode 100644 xds/bootstrap/bootstrap_test.go create mode 100644 xds/bootstrap/credentials.go create mode 100644 xds/csds/csds.go create mode 100644 xds/csds/csds_e2e_test.go create mode 100644 xds/googledirectpath/googlec2p.go create mode 100644 xds/googledirectpath/googlec2p_test.go create mode 100644 xds/googledirectpath/utils.go create mode 100644 xds/server.go create mode 100644 xds/server_ext_test.go create mode 100644 xds/server_options.go create mode 100644 xds/server_resource_ext_test.go create mode 100644 xds/server_security_ext_test.go create mode 100644 xds/server_serving_mode_ext_test.go create mode 100644 xds/server_test.go create mode 100644 xds/test/eds_resource_missing_test.go create mode 100644 xds/xds.go diff --git a/.gemini/config.yaml b/.gemini/config.yaml new file mode 100644 index 0000000..71adf79 --- /dev/null +++ b/.gemini/config.yaml @@ -0,0 +1,13 @@ +have_fun: false +memory_config: + disabled: false +code_review: + disable: false + comment_severity_threshold: MEDIUM + max_review_comments: -1 + pull_request_opened: + help: false + summary: false + code_review: false + include_drafts: false +ignore_patterns: [] diff --git a/.gemini/styleguide.md b/.gemini/styleguide.md new file mode 100644 index 0000000..f87ded7 --- /dev/null +++ b/.gemini/styleguide.md @@ -0,0 +1,16 @@ +# Code Review Guidelines + +* Primary Style: Strictly adhere to the Google Go Style Guide + (https://google.github.io/styleguide/go/). Evaluate all pull requests against + these conventions. + +* gRPC and Protobuf Imports: When importing generated gRPC service and Protobuf + message code from the same package, you **MUST** use separate named imports to + distinguish them. Alias the message code import with a `pb` suffix and the + service code import with a `grpc` suffix. + ```go + import ( + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + ) + ``` diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md new file mode 100644 index 0000000..5384f2d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -0,0 +1,27 @@ +--- +name: Bug Report +about: Report a non-security bug. For suspected security vulnerabilities or crashes, please use "Report a Security Vulnerability", below. +labels: 'Type: Bug' + +--- + +NOTE: if you are reporting is a potential security vulnerability or a crash, +please follow our CVE process at +https://github.com/grpc/proposal/blob/master/P4-grpc-cve-process.md instead of +filing an issue here. + +Please see the FAQ in our main README.md, then answer the questions below +before submitting your issue. + +### What version of gRPC are you using? + +### What version of Go are you using (`go version`)? + +### What operating system (Linux, Windows, …) and version? + +### What did you do? +If possible, provide a recipe for reproducing the error. + +### What did you expect to see? + +### What did you see instead? diff --git a/.github/ISSUE_TEMPLATE/feature.md b/.github/ISSUE_TEMPLATE/feature.md new file mode 100644 index 0000000..2b28691 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.md @@ -0,0 +1,16 @@ +--- +name: Feature Request +about: Suggest an idea for gRPC-Go +labels: 'Type: Feature' + +--- + +Please see the FAQ in our main README.md before submitting your issue. + +### Use case(s) - what problem will this feature solve? + +### Proposed Solution + +### Alternatives Considered + +### Additional Context diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000..1d1752a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,8 @@ +--- +name: Question +about: Ask a question about gRPC-Go +labels: 'Type: Question' + +--- + +Please see the FAQ in our main README.md before submitting your issue. diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000..ea9e773 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,25 @@ +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true +ignore: + # All 'pb.go's. + - "**/*.pb.go" + # Tests and test related files. + - "**/test" + - "**/testdata" + - "**/testutils" + - "benchmark" + - "interop" + # Other submodules. + - "cmd" + - "examples" + - "gcp" + - "security" + - "stats/opencensus" +comment: + layout: "header, diff, files" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..deb08ca --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,4 @@ +Thank you for your PR. Please read and follow +https://github.com/grpc/grpc-go/blob/master/CONTRIBUTING.md, especially the +"Guidelines for Pull Requests" section, and then delete this text before +entering your PR description. diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..237137a --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,37 @@ +name: "CodeQL" + +on: + push: + branches: [ master ] + schedule: + - cron: '24 20 * * 3' + +permissions: + contents: read + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + timeout-minutes: 30 + + permissions: + security-events: write + pull-requests: read + actions: read + + strategy: + fail-fast: false + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: go + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..970a48f --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,26 @@ +name: codecov +on: [push, pull_request] + +permissions: + contents: read + +jobs: + upload: + runs-on: ubuntu-latest + steps: + - name: Install checkout + uses: actions/checkout@v4 + + - name: Install checkout + uses: actions/setup-go@v5 + with: + go-version: "stable" + + - name: Run coverage + run: go test -coverprofile=coverage.out -coverpkg=./... ./... + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml new file mode 100644 index 0000000..a0457ec --- /dev/null +++ b/.github/workflows/deps.yml @@ -0,0 +1,60 @@ +name: Dependency Changes + +# Trigger on PRs. +on: + pull_request: + +permissions: + contents: read + +jobs: + # Compare dependencies before and after this PR. + dependencies: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: true + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: stable + cache-dependency-path: "**/*go.sum" + + # Run the commands to generate dependencies before and after and compare. + - name: Compare dependencies + run: | + set -eu + TEMP_DIR="$(mktemp -d)" + # GITHUB_BASE_REF is set when the job is triggered by a PR. + TARGET_REF="${GITHUB_BASE_REF:-master}" + + mkdir "${TEMP_DIR}/after" + scripts/gen-deps.sh "${TEMP_DIR}/after" + + git checkout "origin/${TARGET_REF}" + mkdir "${TEMP_DIR}/before" + scripts/gen-deps.sh "${TEMP_DIR}/before" + + echo -e " \nComparing dependencies..." + cd "${TEMP_DIR}" + # Run grep in a sub-shell since bash does not support ! in the middle of a pipe. + if diff -u0 -r "before" "after" | bash -c '! grep -v "@@"'; then + echo "No changes detected." + exit 0 + fi + + # Print packages in `after` but not `before`. + for x in $(ls -1 after | grep -vF "$(ls -1 before)"); do + echo -e " \nDependencies of new package $x:" + cat "after/$x" + done + + echo -e " \nChanges detected; exiting with error." + exit 1 diff --git a/.github/workflows/govulncheck.yaml b/.github/workflows/govulncheck.yaml new file mode 100644 index 0000000..6a4ea48 --- /dev/null +++ b/.github/workflows/govulncheck.yaml @@ -0,0 +1,27 @@ +name: govulncheck +on: + push: + pull_request: + schedule: # daily at 10:22 UTC + - cron: '22 10 * * *' + workflow_dispatch: +permissions: + contents: read +jobs: + govulncheck: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + work-dir: ['.', + './gcp/observability', + './cmd/protoc-gen-go-grpc', + './security/advancedtls', + './stats/opencensus'] + steps: + - name: Run govulncheck + uses: golang/govulncheck-action@v1 + with: + check-latest: true + work-dir: ${{ matrix.work-dir }} diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml new file mode 100644 index 0000000..901db85 --- /dev/null +++ b/.github/workflows/lock.yml @@ -0,0 +1,22 @@ +name: 'Lock Threads' + +on: + workflow_dispatch: + schedule: + - cron: '22 1 * * *' + +permissions: + contents: read + +jobs: + lock: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: dessant/lock-threads@v5 + with: + github-token: ${{ github.token }} + issue-inactive-days: 180 + pr-inactive-days: 180 diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml new file mode 100644 index 0000000..2b32391 --- /dev/null +++ b/.github/workflows/pr-validation.yml @@ -0,0 +1,55 @@ +name: PR Validation +on: + pull_request: + types: [opened, edited, synchronize, labeled, unlabeled, milestoned, demilestoned] + +permissions: + contents: read + +jobs: + validate: + name: Validate PR + runs-on: ubuntu-latest + steps: + - name: Validate Description + uses: actions/github-script@v6 + with: + script: | + const body = context.payload.pull_request.body; + const requiredRegex = new RegExp('^RELEASE NOTES:\\s*([Nn][Oo][Nn][Ee]|[Nn]/[Aa]|\\n(\\*|-)\\s*.+)$', 'm'); + if (!requiredRegex.test(body)) { + core.setFailed(` + The PR description must include a RELEASE NOTES section. + It should be in one of the following formats: + - "RELEASE NOTES: none" (case-insensitive) + - "RELEASE NOTES: N/A" (case-insensitive) + - A bulleted list under "RELEASE NOTES:", for example: + RELEASE NOTES: + * my_package: Fix bug causing crash... + `); + } + + - name: Validate Label + uses: actions/github-script@v6 + with: + script: | + const labels = context.payload.pull_request.labels.map(label => label.name); + const requiredRegex = new RegExp('^Type:'); + const hasRequiredLabel = labels.some(label => requiredRegex.test(label)); + if (!hasRequiredLabel) { + core.setFailed("This PR must have a label starting with 'Type:'."); + } + + - name: Validate Milestone + uses: actions/github-script@v6 + with: + script: | + const milestone = context.payload.pull_request.milestone; + if (!milestone) { + core.setFailed("This PR must be associated with a milestone."); + } else { + const requiredRegex = new RegExp('Release$'); + if (!requiredRegex.test(milestone.title)) { + core.setFailed("The milestone for this PR must end with 'Release'."); + } + } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..1138fe2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,63 @@ +name: Release + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + release: + permissions: + contents: write # to upload release asset (actions/upload-release-asset) + + name: Release cmd/protoc-gen-go-grpc + runs-on: ubuntu-latest + if: startsWith(github.event.release.tag_name, 'cmd/protoc-gen-go-grpc/') + strategy: + matrix: + goos: [linux, darwin, windows] + goarch: [386, amd64, arm64] + exclude: + - goos: darwin + goarch: 386 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + + - name: Download dependencies + run: | + cd cmd/protoc-gen-go-grpc + go mod download + + - name: Prepare build directory + run: | + mkdir -p build/ + cp README.md build/ + cp LICENSE build/ + + - name: Build + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + run: | + cd cmd/protoc-gen-go-grpc + go build -trimpath -o $GITHUB_WORKSPACE/build + + - name: Create package + id: package + run: | + PACKAGE_NAME=protoc-gen-go-grpc.${GITHUB_REF#refs/tags/cmd/protoc-gen-go-grpc/}.${{ matrix.goos }}.${{ matrix.goarch }}.tar.gz + tar -czvf $PACKAGE_NAME -C build . + echo "name=${PACKAGE_NAME}" >> $GITHUB_OUTPUT + + - name: Upload asset + run: | + gh release upload ${{ github.event.release.tag_name }} ./${{ steps.package.outputs.name }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..f9bae4c --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,33 @@ +name: Stale bot + +on: + workflow_dispatch: + schedule: + - cron: "44 */2 * * *" + +permissions: + contents: read + +jobs: + stale: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v8 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + days-before-stale: 6 + days-before-close: 7 + only-labels: 'Status: Requires Reporter Clarification' + stale-issue-label: 'stale' + stale-pr-label: 'stale' + operations-per-run: 999 + stale-issue-message: > + This issue is labeled as requiring an update from the reporter, and no update has been received + after 6 days. If no update is provided in the next 7 days, this issue will be automatically closed. + stale-pr-message: > + This PR is labeled as requiring an update from the reporter, and no update has been received + after 6 days. If no update is provided in the next 7 days, this issue will be automatically closed. diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 0000000..f429f46 --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,135 @@ +name: Testing + +# Trigger on pushes, PRs (excluding documentation changes), and nightly. +on: + push: + pull_request: + schedule: + - cron: 0 0 * * * # daily at 00:00 + +permissions: + contents: read + +# Always force the use of Go modules +env: + GO111MODULE: on + +jobs: + # Check generated protos match their source repos (optional for PRs). + vet-proto: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + # Setup the environment. + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.26' + cache-dependency-path: "**/go.sum" + + # Run the vet-proto checks. + - name: vet-proto + run: ./scripts/vet-proto.sh -install && ./scripts/vet-proto.sh + + # Run the main gRPC-Go tests. + tests: + name: ${{ matrix.display_name }} + + # Use the matrix variable to set the runner, with 'ubuntu-latest' as the + # default. + runs-on: ${{ matrix.runner || 'ubuntu-latest' }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - type: vet + goversion: '1.25' + display_name: 'static checks (latest-1)' + + - type: extras + goversion: '1.26' + display_name: 'extras (latest)' + + - type: tests + goversion: '1.26' + display_name: 'tests (latest)' + + - type: tests + goversion: '1.26' + testflags: -race + display_name: 'tests (-race, latest)' + + - type: tests + goversion: '1.26' + goarch: 386 + display_name: 'tests (i386, latest)' + + - type: tests + goversion: '1.26' + goarch: arm64 + runner: ubuntu-24.04-arm + display_name: 'tests (arm, latest)' + + - type: tests + goversion: '1.25' + display_name: 'tests (latest-1)' + + steps: + # Setup the environment. + - name: Setup GOARCH + if: matrix.goarch != '' + run: echo "GOARCH=${{ matrix.goarch }}" >> $GITHUB_ENV + + - name: Setup GRPC environment + if: matrix.grpcenv != '' + run: echo "${{ matrix.grpcenv }}" >> $GITHUB_ENV + + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.goversion }} + cache-dependency-path: "**/*go.sum" + + # Only run vet for 'vet' runs. + - name: Run vet.sh + if: matrix.type == 'vet' + run: ./scripts/vet.sh -install && ./scripts/vet.sh + + # Main tests run for everything except when testing "extras" + # (where we run a reduced set of tests). + - name: Run tests + if: matrix.type == 'tests' + run: | + go version + go test ${{ matrix.testflags }} -cpu 1,4 -timeout 7m ./... + cd "${GITHUB_WORKSPACE}" + for MOD_FILE in $(find . -name 'go.mod' | grep -Ev '^\./go\.mod'); do + pushd "$(dirname ${MOD_FILE})" + go test ${{ matrix.testflags }} -cpu 1,4 -timeout 2m ./... + popd + done + + # Non-core gRPC tests (examples, interop, etc) + - name: Run extras tests + if: matrix.type == 'extras' + run: | + export TERM=${TERM:-xterm} + go version + echo -e "\n-- Running Examples --" + examples/examples_test.sh + echo -e "\n-- Running AdvancedTLS Examples --" + security/advancedtls/examples/examples_test.sh + echo -e "\n-- Running Interop Test --" + interop/interop_test.sh + echo -e "\n-- Running xDS E2E Test --" + internal/xds/test/e2e/run.sh + echo -e "\n-- Running protoc-gen-go-grpc test --" + ./scripts/vet-proto.sh -install + cmd/protoc-gen-go-grpc/protoc-gen-go-grpc_test.sh diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..e491a9e --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Google Inc. diff --git a/CODE-OF-CONDUCT.md b/CODE-OF-CONDUCT.md new file mode 100644 index 0000000..9d4213e --- /dev/null +++ b/CODE-OF-CONDUCT.md @@ -0,0 +1,3 @@ +## Community Code of Conduct + +gRPC follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..bd80b1f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,182 @@ +# How to contribute + +We welcome your patches and contributions to gRPC! Please read the gRPC +organization's [governance +rules](https://github.com/grpc/grpc-community/blob/master/governance.md) before +proceeding. + +If you are new to GitHub, please start by reading [Pull Request howto](https://help.github.com/articles/about-pull-requests/) + +## Legal requirements + +In order to protect both you and ourselves, you will need to sign the +[Contributor License +Agreement](https://identity.linuxfoundation.org/projects/cncf). When you create +your first PR, a link will be added as a comment that contains the steps needed +to complete this process. + +## Getting Started + +A great way to start is by searching through our open issues. [Unassigned issues +labeled as "help +wanted"](https://github.com/grpc/grpc-go/issues?q=sort%3Aupdated-desc%20is%3Aissue%20is%3Aopen%20label%3A%22Status%3A%20Help%20Wanted%22%20no%3Aassignee) +are especially nice for first-time contributors, as they should be well-defined +problems that already have agreed-upon solutions. + +## Generative AI Policy + +AI tools have the ability to produce more code than is possible for the gRPC +team to read, understand, review, and accept into the repository. For this +reason, we request that all contributions adhere to the following rules: + +1. **No AI-Generated Interactions:** All communication in the repo must be + authored by a human. _Exception: AIs may be used for directed writing + assistance or translation._ Absolutely no automated agents are allowed to + directly publish to GitHub. + +2. **Author Ownership and Accountability:** Code contributions are expected to + be fully owned and understood by the human contributor. If the code was + produced by generative AI, the author is expected to have reviewed and + understood it in its entirety before submitting it for review. This includes + all content: production code, tests, examples, tools, etc. + +In addition to the above requirements, any AI-assisted contributions must also +comply with the [Linux Foundation Generative AI +Policy](https://www.linuxfoundation.org/legal/generative-ai). This includes +confirming that all contributions are legally allowed to be contributed to the +gRPC project under the applicable license terms. + +## Code Style + +We follow [Google's published Go style +guide](https://google.github.io/styleguide/go/). Note that there are three +primary documents that make up this style guide; please follow them as closely +as possible. If a reviewer recommends something that contradicts those +guidelines, there may be valid reasons to do so, but it should be rare. + +## Guidelines for Pull Requests + +Please read the following carefully to ensure your contributions can be merged +smoothly and quickly. + +### PR Contents + +- Create **small PRs** that are narrowly focused on **addressing a single + concern**. We often receive PRs that attempt to fix several things at the same + time, and if one part of the PR has a problem, that will hold up the entire + PR. + +- If your change does not address an **open issue** with an **agreed + resolution**, consider opening an issue and discussing it first. If you are + suggesting a behavioral or API change, consider starting with a [gRFC + proposal](https://github.com/grpc/proposal). Many new features that are not + bug fixes will require cross-language agreement. + +- If you want to fix **formatting or style**, consider whether your changes are + an obvious improvement or might be considered a personal preference. If a + style change is based on preference, it likely will not be accepted. If it + corrects widely agreed-upon anti-patterns, then please do create a PR and + explain the benefits of the change. + +- For correcting **misspellings**, please be aware that we use some terms that + are sometimes flagged by spell checkers. As an example, "if an only if" is + often written as "iff". Please do not make spelling correction changes unless + you are certain they are misspellings. + +- **All tests need to be passing** before your change can be merged. We + recommend you run tests locally before creating your PR to catch breakages + early on: + + - `./scripts/vet.sh` to catch vet errors. + - `go test -cpu 1,4 -timeout 7m ./...` to run the tests. + - `go test -race -cpu 1,4 -timeout 7m ./...` to run tests in race mode. + + Note that we have a multi-module repo, so `go test` commands may need to be + run from the root of each module in order to cause all tests to run. + + *Alternatively*, you may find it easier to push your changes to your fork on + GitHub, which will trigger a GitHub Actions run that you can use to verify + everything is passing. + +- Note that there are two GitHub actions checks that need not be green: + + 1. We test the freshness of the generated proto code we maintain via the + `vet-proto` check. If the source proto files are updated, but our repo is + not updated, an optional checker will fail. This will be fixed by our team + in a separate PR and will not prevent the merge of your PR. + + 2. We run a checker that will fail if there is any change in dependencies of + an exported package via the `dependencies` check. If new dependencies are + added that are not appropriate, we may not accept your PR (see below). + +- If you are adding a **new file**, make sure it has the **copyright message** + template at the top as a comment. You can copy the message from an existing + file and update the year. + +- The grpc package should only depend on standard Go packages and a small number + of exceptions. **If your contribution introduces new dependencies**, you will + need a discussion with gRPC-Go maintainers. + +### PR Descriptions + +- **PR titles** should start with the name of the component being addressed, or + the type of change. Examples: transport, client, server, round_robin, xds, + cleanup, deps. + +- Read and follow the **guidelines for PR titles and descriptions** here: + https://google.github.io/eng-practices/review/developer/cl-descriptions.html + + *particularly* the sections "First Line" and "Body is Informative". + + Note: your PR description will be used as the git commit message in a + squash-and-merge if your PR is approved. We may make changes to this as + necessary. + +- **Does this PR relate to an open issue?** On the first line, please use the + tag `Fixes #` to ensure the issue is closed when the PR is merged. Or + use `Updates #` if the PR is related to an open issue, but does not fix + it. Consider filing an issue if one does not already exist. + +- PR descriptions *must* conclude with **release notes** as follows: + + ``` + RELEASE NOTES: + * : + ``` + + This need not match the PR title. + + The summary must: + + * be something that gRPC users will understand. + + * clearly explain the feature being added, the issue being fixed, or the + behavior being changed, etc. If fixing a bug, be clear about how the bug + can be triggered by an end-user. + + * begin with a capital letter and use complete sentences. + + * be as short as possible to describe the change being made. + + If a PR is *not* end-user visible -- e.g. a cleanup, testing change, or + GitHub-related, use `RELEASE NOTES: n/a`. + +### PR Process + +- Please **self-review** your code changes before sending your PR. This will + prevent simple, obvious errors from causing delays. + +- Maintain a **clean commit history** and use **meaningful commit messages**. + PRs with messy commit histories are difficult to review and won't be merged. + Before sending your PR, ensure your changes are based on top of the latest + `upstream/master` commits, and avoid rebasing in the middle of a code review. + You should **never use `git push -f`** unless absolutely necessary during a + review, as it can interfere with GitHub's tracking of comments. + +- Unless your PR is trivial, you should **expect reviewer comments** that you + will need to address before merging. We'll label the PR as `Status: Requires + Reporter Clarification` if we expect you to respond to these comments in a + timely manner. If the PR remains inactive for 6 days, it will be marked as + `stale`, and we will automatically close it after 7 days if we don't hear back + from you. Please feel free to ping issues or bugs if you do not get a response + within a week. diff --git a/Documentation/anti-patterns.md b/Documentation/anti-patterns.md new file mode 100644 index 0000000..8faaa37 --- /dev/null +++ b/Documentation/anti-patterns.md @@ -0,0 +1,183 @@ +## Anti-Patterns of Client creation + +### How to properly create a `ClientConn`: `grpc.NewClient` + +[`grpc.NewClient`](https://pkg.go.dev/google.golang.org/grpc#NewClient) is the +function in the gRPC library that creates a virtual connection from a client +application to a gRPC server. It takes a target URI (which represents the name +of a logical backend service and resolves to one or more physical addresses) and +a list of options, and returns a +[`ClientConn`](https://pkg.go.dev/google.golang.org/grpc#ClientConn) object that +represents the virtual connection to the server. The `ClientConn` contains one +or more actual connections to real servers and attempts to maintain these +connections by automatically reconnecting to them when they break. `NewClient` +was introduced in gRPC-Go v1.63. + +### The wrong way: `grpc.Dial` + +[`grpc.Dial`](https://pkg.go.dev/google.golang.org/grpc#Dial) is a deprecated +function that also creates the same virtual connection pool as `grpc.NewClient`. +However, unlike `grpc.NewClient`, it immediately starts connecting and supports +a few additional `DialOption`s that control this initial connection attempt. +These are: `WithBlock`, `WithTimeout`, `WithReturnConnectionError`, and +`FailOnNonTempDialError`. + +That `grpc.Dial` creates connections immediately is not a problem in and of +itself, but this behavior differs from how gRPC works in all other languages, +and it can be convenient to have a constructor that does not perform I/O. It +can also be confusing to users, as most people expect a function called `Dial` +to create _a_ connection which may need to be recreated if it is lost. + +`grpc.Dial` uses "passthrough" as the default name resolver for backward +compatibility while `grpc.NewClient` uses "dns" as its default name resolver. +This subtle difference is important to legacy systems that also specified a +custom dialer and expected it to receive the target string directly. + +For these reasons, using `grpc.Dial` is discouraged. Even though it is marked +as deprecated, we will continue to support it until a v2 is released (and no +plans for a v2 exist at the time this was written). + +### Especially bad: using deprecated `DialOptions` + +`FailOnNonTempDialError`, `WithBlock`, and `WithReturnConnectionError` are three +`DialOption`s that are only supported by `Dial` because they only affect the +behavior of `Dial` itself. `WithBlock` causes `Dial` to wait until the +`ClientConn` reports its `State` as `connectivity.Connected`. The other two deal +with returning connection errors before the timeout (`WithTimeout` or on the +context when using `DialContext`). + +The reason these options can be a problem is that connections with a +`ClientConn` are dynamic -- they may come and go over time. If your client +successfully connects, the server could go down 1 second later, and your RPCs +will fail. "Knowing you are connected" does not tell you much in this regard. + +Additionally, _all_ RPCs created on an "idle" or a "connecting" `ClientConn` +will wait until their deadline or until a connection is established before +failing. This means that you don't need to check that a `ClientConn` is "ready" +before starting your RPCs. By default, RPCs will fail if the `ClientConn` +enters the "transient failure" state, but setting `WaitForReady(true)` on a +call will cause it to queue even in the "transient failure" state, and it will +only ever fail due to a deadline, a server response, or a connection loss after +the RPC was sent to a server. + +Some users of `Dial` use it as a way to validate the configuration of their +system. If you wish to maintain this behavior but migrate to `NewClient`, you +can call `GetState`, then `Connect` if the state is `Idle` and +`WaitForStateChange` until the channel is connected. However, if this fails, +it does not mean that your configuration was bad - it could also mean the +service is not reachable by the client due to connectivity reasons. + +## Best practices for error handling in gRPC + +Instead of relying on failures at dial time, we strongly encourage developers to +rely on errors from RPCs. When a client makes an RPC, it can receive an error +response from the server. These errors can provide valuable information about +what went wrong, including information about network issues, server-side errors, +and incorrect usage of the gRPC API. + +By handling errors from RPCs correctly, developers can write more reliable and +robust gRPC applications. Here are some best practices for error handling in +gRPC: + +- Always check for error responses from RPCs and handle them appropriately. +- Use the `status` field of the error response to determine the type of error + that occurred. +- When retrying failed RPCs, consider using the built-in retry mechanism + provided by gRPC-Go, if available, instead of manually implementing retries. + Refer to the [gRPC-Go retry example + documentation](https://github.com/grpc/grpc-go/blob/master/examples/features/retry/README.md) + for more information. Note that this is not a substitute for client-side + retries as errors that occur after an RPC starts on a server cannot be + retried through gRPC's built-in mechanism. +- If making an outgoing RPC from a server handler, be sure to translate the + status code before returning the error from your method handler. For example, + if the error is an `INVALID_ARGUMENT` status code, that probably means + your service has a bug (otherwise it shouldn't have triggered this error), in + which case `INTERNAL` is more appropriate to return back to your users. + +### Example: Handling errors from an RPC + +The following code snippet demonstrates how to handle errors from an RPC in +gRPC: + +```go +ctx, cancel := context.WithTimeout(context.Background(), time.Second) +defer cancel() + +res, err := client.MyRPC(ctx, &MyRequest{}) +if err != nil { + // Handle the error appropriately, + // log it & return an error to the caller, etc. + log.Printf("Error calling MyRPC: %v", err) + return nil, err +} + +// Use the response as appropriate +log.Printf("MyRPC response: %v", res) +``` + +To determine the type of error that occurred, you can use the status field of +the error response: + +```go +resp, err := client.MakeRPC(context.TODO(), request) +if err != nil { + if status, ok := status.FromError(err); ok { + // Handle the error based on its status code + if status.Code() == codes.NotFound { + log.Println("Requested resource not found") + } else { + log.Printf("RPC error: %v", status.Message()) + } + } else { + // Handle non-RPC errors + log.Printf("Non-RPC error: %v", err) + } + return +} + +// Use the response as needed +log.Printf("Response received: %v", resp) +``` + +### Example: Using a backoff strategy + +When retrying failed RPCs, use a backoff strategy to avoid overwhelming the +server or exacerbating network issues: + +```go +var res *MyResponse +var err error + +retryableStatusCodes := map[codes.Code]bool{ + codes.Unavailable: true, // etc +} + +// Retry the RPC a maximum number of times. +for i := 0; i < maxRetries; i++ { + // Make the RPC. + res, err = client.MyRPC(context.TODO(), &MyRequest{}) + + // Check if the RPC was successful. + if !retryableStatusCodes[status.Code(err)] { + // The RPC was successful or errored in a non-retryable way; + // do not retry. + break + } + + // The RPC is retryable; wait for a backoff period before retrying. + backoff := time.Duration(i+1) * time.Second + log.Printf("Error calling MyRPC: %v; retrying in %v", err, backoff) + time.Sleep(backoff) +} + +// Check if the RPC was successful after all retries. +if err != nil { + // All retries failed, so handle the error appropriately + log.Printf("Error calling MyRPC: %v", err) + return nil, err +} + +// Use the response as appropriate. +log.Printf("MyRPC response: %v", res) +``` diff --git a/Documentation/benchmark.md b/Documentation/benchmark.md new file mode 100644 index 0000000..f8097a2 --- /dev/null +++ b/Documentation/benchmark.md @@ -0,0 +1,75 @@ +# Benchmark + +gRPC-Go comes with a set of benchmarking utilities to measure performance. +These utilities can be found in the `benchmark` directory within the project's +root directory. + +The main utility, aptly named `benchmain`, supports a host of configurable +parameters to simulate various environments and workloads. For example, if your +server's workload is primarily streaming RPCs with large messages with +compression turned on, invoking `benchmain` in the following way may closely +simulate your application: + +```bash +$ go run google.golang.org/grpc/benchmark/benchmain/main.go \ + -workloads=streaming \ + -reqSizeBytes=1024 \ + -respSizeBytes=1024 \ + -compression=gzip +``` + +Pass the `-h` flag to the `benchmain` utility to see other flags and workloads +that are supported. + +## Varying Payload Sizes (Weighted Random Distribution) + +The `benchmain` utility supports two flags, `-reqPayloadCurveFiles` and +`-respPayloadCurveFiles`, that can be used to specify histograms representing +a weighted random distribution of request and response payload sizes, +respectively. This is useful to simulate workloads with arbitrary payload +sizes. + +The options take a comma-separated list of file paths as value. Each file must +be a valid CSV file with three columns in each row. Each row represents a range +of payload sizes (first two columns) and the weight associated with that range +(third column). For example, consider the below file: + +```csv +1,32,12.5 +128,256,12.5 +1024,2048,25.0 +``` + +Assume that `benchmain` is invoked like so: + +```bash +$ go run google.golang.org/grpc/benchmark/benchmain/main.go \ + -workloads=unary \ + -reqPayloadCurveFiles=/path/to/csv \ + -respPayloadCurveFiles=/path/to/csv +``` + +This tells the `benchmain` utility to generate unary RPC requests with a 25% +probability of payload sizes in the ranges 1-32 bytes, 25% probability in the +128-256 bytes range, and 50% probability in the 1024-2048 bytes range. RPC +requests outside these ranges will not be generated. + +You may specify multiple CSV files delimited by a comma. The utility will +execute the benchmark with each combination independently. That is, the +following command will execute four benchmarks: + +```bash +$ go run google.golang.org/grpc/benchmark/benchmain/main.go \ + -workloads=unary \ + -reqPayloadCurveFiles=/path/to/csv1,/path/to/csv2 \ + -respPayloadCurveFiles=/path/to/csv3,/path/to/csv4 +``` + +You may also combine `PayloadCurveFiles` with `SizeBytes` options. For example: + +``` +$ go run google.golang.org/grpc/benchmark/benchmain/main.go \ + -workloads=unary \ + -reqPayloadCurveFiles=/path/to/csv \ + -respSizeBytes=1 +``` diff --git a/Documentation/compression.md b/Documentation/compression.md new file mode 100644 index 0000000..204f880 --- /dev/null +++ b/Documentation/compression.md @@ -0,0 +1,80 @@ +# Compression + +The preferred method for configuring message compression on both clients and +servers is to use +[`encoding.RegisterCompressor`](https://godoc.org/google.golang.org/grpc/encoding#RegisterCompressor) +to register an implementation of a compression algorithm. See +`grpc/encoding/gzip/gzip.go` for an example of how to implement one. + +Once a compressor has been registered on the client-side, RPCs may be sent using +it via the +[`UseCompressor`](https://godoc.org/google.golang.org/grpc#UseCompressor) +`CallOption`. Remember that `CallOption`s may be turned into defaults for all +calls from a `ClientConn` by using the +[`WithDefaultCallOptions`](https://godoc.org/google.golang.org/grpc#WithDefaultCallOptions) +`DialOption`. If `UseCompressor` is used and the corresponding compressor has +not been installed, an `Internal` error will be returned to the application +before the RPC is sent. + +Server-side, registered compressors will be used automatically to decode request +messages and encode the responses. Servers currently always respond using the +same compression method specified by the client. If the corresponding +compressor has not been registered, an `Unimplemented` status will be returned +to the client. + +## Deprecated API + +There is a deprecated API for setting compression as well. It is not +recommended for use. However, if you were previously using it, the following +section may be helpful in understanding how it works in combination with the new +API. + +### Client-Side + +There are two legacy functions and one new function to configure compression: + +```go +func WithCompressor(grpc.Compressor) DialOption {} +func WithDecompressor(grpc.Decompressor) DialOption {} +func UseCompressor(name) CallOption {} +``` + +For outgoing requests, the following rules are applied in order: +1. If `UseCompressor` is used, messages will be compressed using the compressor + named. + * If the compressor named is not registered, an Internal error is returned + back to the client before sending the RPC. + * If UseCompressor("identity"), no compressor will be used, but "identity" + will be sent in the header to the server. +1. If `WithCompressor` is used, messages will be compressed using that + compressor implementation. +1. Otherwise, outbound messages will be uncompressed. + +For incoming responses, the following rules are applied in order: +1. If `WithDecompressor` is used and it matches the message's encoding, it will + be used. +1. If a registered compressor matches the response's encoding, it will be used. +1. Otherwise, the stream will be closed and an `Unimplemented` status error will + be returned to the application. + +### Server-Side + +There are two legacy functions to configure compression: +```go +func RPCCompressor(grpc.Compressor) ServerOption {} +func RPCDecompressor(grpc.Decompressor) ServerOption {} +``` + +For incoming requests, the following rules are applied in order: +1. If `RPCDecompressor` is used and that decompressor matches the request's + encoding: it will be used. +1. If a registered compressor matches the request's encoding, it will be used. +1. Otherwise, an `Unimplemented` status will be returned to the client. + +For outgoing responses, the following rules are applied in order: +1. If `RPCCompressor` is used, that compressor will be used to compress all + response messages. +1. If compression was used for the incoming request and a registered compressor + supports it, that same compression method will be used for the outgoing + response. +1. Otherwise, no compression will be used for the outgoing response. diff --git a/Documentation/concurrency.md b/Documentation/concurrency.md new file mode 100644 index 0000000..5f6c631 --- /dev/null +++ b/Documentation/concurrency.md @@ -0,0 +1,39 @@ +# Concurrency + +In general, gRPC-go provides a concurrency-friendly API. What follows are some +guidelines. + +## Clients + +A [ClientConn][client-conn] can safely be accessed concurrently. Using +[helloworld][helloworld] as an example, one could share the `ClientConn` across +multiple goroutines to create multiple `GreeterClient` types. In this case, +RPCs would be sent in parallel. `GreeterClient`, generated from the proto +definitions and wrapping `ClientConn`, is also concurrency safe, and may be +directly shared in the same way. Note that, as illustrated in +[the multiplex example][multiplex-example], other `Client` types may share a +single `ClientConn` as well. + +## Streams + +When using streams, one must take care to avoid calling either `SendMsg` or +`RecvMsg` multiple times against the same [Stream][stream] from different +goroutines. In other words, it's safe to have a goroutine calling `SendMsg` and +another goroutine calling `RecvMsg` on the same stream at the same time. But it +is not safe to call `SendMsg` on the same stream in different goroutines, or to +call `RecvMsg` on the same stream in different goroutines. + +## Servers + +Each RPC handler attached to a registered server will be invoked in its own +goroutine. For example, [SayHello][say-hello] will be invoked in its own +goroutine. The same is true for service handlers for streaming RPCs, as seen +in the route guide example [here][route-guide-stream]. Similar to clients, +multiple services can be registered to the same server. + +[helloworld]: https://github.com/grpc/grpc-go/blob/master/examples/helloworld/greeter_client/main.go#L43 +[client-conn]: https://godoc.org/google.golang.org/grpc#ClientConn +[stream]: https://godoc.org/google.golang.org/grpc#Stream +[say-hello]: https://github.com/grpc/grpc-go/blob/master/examples/helloworld/greeter_server/main.go#L41 +[route-guide-stream]: https://github.com/grpc/grpc-go/blob/master/examples/route_guide/server/server.go#L126 +[multiplex-example]: https://github.com/grpc/grpc-go/tree/master/examples/features/multiplex diff --git a/Documentation/encoding.md b/Documentation/encoding.md new file mode 100644 index 0000000..00e19cc --- /dev/null +++ b/Documentation/encoding.md @@ -0,0 +1,146 @@ +# Encoding + +The gRPC API for sending and receiving is based upon *messages*. However, +messages cannot be transmitted directly over a network; they must first be +converted into *bytes*. This document describes how gRPC-Go converts messages +into bytes and vice-versa for the purposes of network transmission. + +## Codecs (Serialization and Deserialization) + +A `Codec` contains code to serialize a message into a byte slice (`Marshal`) and +deserialize a byte slice back into a message (`Unmarshal`). `Codec`s are +registered by name into a global registry maintained in the `encoding` package. + +### Implementing a `Codec` + +A typical `Codec` will be implemented in its own package with an `init` function +that registers itself, and is imported anonymously. For example: + +```go +package proto + +import "google.golang.org/grpc/encoding" + +func init() { + encoding.RegisterCodec(protoCodec{}) +} + +// ... implementation of protoCodec ... +``` + +For an example, gRPC's implementation of the `proto` codec can be found in +[`encoding/proto`](https://godoc.org/google.golang.org/grpc/encoding/proto). + +### Using a `Codec` + +By default, gRPC registers and uses the "proto" codec, so it is not necessary to +do this in your own code to send and receive proto messages. To use another +`Codec` from a client or server: + +```go +package myclient + +import _ "path/to/another/codec" +``` + +`Codec`s, by definition, must be symmetric, so the same desired `Codec` should +be registered in both client and server binaries. + +On the client-side, to specify a `Codec` to use for message transmission, the +`CallOption` `CallContentSubtype` should be used as follows: + +```go + response, err := myclient.MyCall(ctx, request, grpc.CallContentSubtype("mycodec")) +``` + +As a reminder, all `CallOption`s may be converted into `DialOption`s that become +the default for all RPCs sent through a client using `grpc.WithDefaultCallOptions`: + +```go + myclient := grpc.NewClient(target, grpc.WithDefaultCallOptions(grpc.CallContentSubtype("mycodec"))) +``` + +When specified in either of these ways, messages will be encoded using this +codec and sent along with headers indicating the codec (`content-type` set to +`application/grpc+`). + +On the server-side, using a `Codec` is as simple as registering it into the +global registry (i.e. `import`ing it). If a message is encoded with the content +sub-type supported by a registered `Codec`, it will be used automatically for +decoding the request and encoding the response. Otherwise, for +backward-compatibility reasons, gRPC will attempt to use the "proto" codec. In +an upcoming change (tracked in [this +issue](https://github.com/grpc/grpc-go/issues/1824)), such requests will be +rejected with status code `Unimplemented` instead. + +## Compressors (Compression and Decompression) + +Sometimes, the resulting serialization of a message is not space-efficient, and +it may be beneficial to compress this byte stream before transmitting it over +the network. To facilitate this operation, gRPC supports a mechanism for +performing compression and decompression. + +A `Compressor` contains code to compress and decompress by wrapping `io.Writer`s +and `io.Reader`s, respectively. (The form of `Compress` and `Decompress` were +chosen to most closely match Go's standard package +[implementations](https://golang.org/pkg/compress/) of compressors). Like +`Codec`s, `Compressor`s are registered by name into a global registry maintained +in the `encoding` package. + +### Implementing a `Compressor` + +A typical `Compressor` will be implemented in its own package with an `init` +function that registers itself, and is imported anonymously. For example: + +```go +package gzip + +import "google.golang.org/grpc/encoding" + +func init() { + encoding.RegisterCompressor(compressor{}) +} + +// ... implementation of compressor ... +``` + +An implementation of a `gzip` compressor can be found in +[`encoding/gzip`](https://godoc.org/google.golang.org/grpc/encoding/gzip). + +### Using a `Compressor` + +By default, gRPC does not register or use any compressors. To use a +`Compressor` from a client or server: + +```go +package myclient + +import _ "google.golang.org/grpc/encoding/gzip" +``` + +`Compressor`s, by definition, must be symmetric, so the same desired +`Compressor` should be registered in both client and server binaries. + +On the client-side, to specify a `Compressor` to use for message transmission, +the `CallOption` `UseCompressor` should be used as follows: + +```go + response, err := myclient.MyCall(ctx, request, grpc.UseCompressor("gzip")) +``` + +As a reminder, all `CallOption`s may be converted into `DialOption`s that become +the default for all RPCs sent through a client using `grpc.WithDefaultCallOptions`: + +```go + myclient := grpc.NewClient(target, grpc.WithDefaultCallOptions(grpc.UseCompressor("gzip"))) +``` + +When specified in either of these ways, messages will be compressed using this +compressor and sent along with headers indicating the compressor +(`content-coding` set to ``). + +On the server-side, using a `Compressor` is as simple as registering it into the +global registry (i.e. `import`ing it). If a message is compressed with the +content coding supported by a registered `Compressor`, it will be used +automatically for decompressing the request and compressing the response. +Otherwise, the request will be rejected with status code `Unimplemented`. diff --git a/Documentation/grpc-auth-support.md b/Documentation/grpc-auth-support.md new file mode 100644 index 0000000..2b8865c --- /dev/null +++ b/Documentation/grpc-auth-support.md @@ -0,0 +1,83 @@ +# Authentication + +As outlined in the +[gRPC authentication guide](https://grpc.io/docs/guides/auth.html), there are a +number of different mechanisms for asserting identity between a client and +server. We'll present some code-samples here demonstrating how to provide TLS +support encryption and identity assertions as well as passing OAuth2 tokens to +services that support it. + +## Enabling TLS on a gRPC client + +```go +conn, err := grpc.NewClient(serverAddr, grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, ""))) +``` + +## Enabling TLS on a gRPC server + +```go +creds, err := credentials.NewServerTLSFromFile(certFile, keyFile) +if err != nil { + log.Fatalf("Failed to generate credentials %v", err) +} +lis, err := net.Listen("tcp", ":0") +server := grpc.NewServer(grpc.Creds(creds)) +... +server.Serve(lis) +``` + +## OAuth2 + +For an example of how to configure client and server to use OAuth2 tokens, see +[here](https://github.com/grpc/grpc-go/tree/master/examples/features/authentication). + +### Validating a token on the server + +Clients may use +[metadata.MD](https://godoc.org/google.golang.org/grpc/metadata#MD) to store +tokens and other authentication-related data. To gain access to the +`metadata.MD` object, a server may use +[metadata.FromIncomingContext](https://godoc.org/google.golang.org/grpc/metadata#FromIncomingContext). +With a reference to `metadata.MD` on the server, one needs to simply look up the +`authorization` key. Note, all keys stored within `metadata.MD` are normalized +to lowercase. See [here](https://godoc.org/google.golang.org/grpc/metadata#New). + +It is possible to configure token validation for all RPCs using an interceptor. +A server may configure either a +[grpc.UnaryInterceptor](https://godoc.org/google.golang.org/grpc#UnaryInterceptor) +or a +[grpc.StreamInterceptor](https://godoc.org/google.golang.org/grpc#StreamInterceptor). + +### Adding a token to all outgoing client RPCs + +To send an OAuth2 token with each RPC, a client may configure the +`grpc.DialOption` +[grpc.WithPerRPCCredentials](https://godoc.org/google.golang.org/grpc#WithPerRPCCredentials). +Alternatively, a client may also use the `grpc.CallOption` +[grpc.PerRPCCredentials](https://godoc.org/google.golang.org/grpc#PerRPCCredentials) +on each invocation of an RPC. + +To create a `credentials.PerRPCCredentials`, use +[oauth.TokenSource](https://godoc.org/google.golang.org/grpc/credentials/oauth#TokenSource). +Note, the OAuth2 implementation of `grpc.PerRPCCredentials` requires a client to +use +[grpc.WithTransportCredentials](https://godoc.org/google.golang.org/grpc#WithTransportCredentials) +to prevent any insecure transmission of tokens. + +## Authenticating with Google + +### Google Compute Engine (GCE) + +```go +conn, err := grpc.NewClient(serverAddr, grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")), grpc.WithPerRPCCredentials(oauth.NewComputeEngine())) +``` + +### JWT + +```go +jwtCreds, err := oauth.NewServiceAccountFromFile(*serviceAccountKeyFile, *oauthScope) +if err != nil { + log.Fatalf("Failed to create JWT credentials: %v", err) +} +conn, err := grpc.NewClient(serverAddr, grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")), grpc.WithPerRPCCredentials(jwtCreds)) +``` diff --git a/Documentation/grpc-metadata.md b/Documentation/grpc-metadata.md new file mode 100644 index 0000000..488fc7a --- /dev/null +++ b/Documentation/grpc-metadata.md @@ -0,0 +1,244 @@ +# Metadata + +gRPC supports sending metadata between client and server. +This doc shows how to send and receive metadata in gRPC-go. + +## Background + +Four kinds of service method: + +- [Unary RPC](https://grpc.io/docs/guides/concepts.html#unary-rpc) +- [Server streaming RPC](https://grpc.io/docs/guides/concepts.html#server-streaming-rpc) +- [Client streaming RPC](https://grpc.io/docs/guides/concepts.html#client-streaming-rpc) +- [Bidirectional streaming RPC](https://grpc.io/docs/guides/concepts.html#bidirectional-streaming-rpc) + +And concept of [metadata]. + +## Constructing metadata + +A metadata can be created using package [metadata]. +The type MD is actually a map from string to a list of strings: + +```go +type MD map[string][]string +``` + +Metadata can be read like a normal map. +Note that the value type of this map is `[]string`, +so that users can attach multiple values using a single key. + +### Creating a new metadata + +A metadata can be created from a `map[string]string` using function `New`: + +```go +md := metadata.New(map[string]string{"key1": "val1", "key2": "val2"}) +``` + +Another way is to use `Pairs`. +Values with the same key will be merged into a list: + +```go +md := metadata.Pairs( + "key1", "val1", + "key1", "val1-2", // "key1" will have map value []string{"val1", "val1-2"} + "key2", "val2", +) +``` + +__Note:__ all the keys will be automatically converted to lowercase, +so "key1" and "kEy1" will be the same key and their values will be merged into the same list. +This happens for both `New` and `Pairs`. + +### Storing binary data in metadata + +In metadata, keys are always strings. But values can be strings or binary data. +To store binary data value in metadata, simply add "-bin" suffix to the key. +The values with "-bin" suffixed keys will be encoded when creating the metadata: + +```go +md := metadata.Pairs( + "key", "string value", + "key-bin", string([]byte{96, 102}), // this binary data will be encoded (base64) before sending + // and will be decoded after being transferred. +) +``` + +## Sending and receiving metadata - client side + +Client side metadata sending and receiving examples are available +[here](../examples/features/metadata/client/main.go). + +### Sending metadata + +There are two ways to send metadata to the server. The recommended way is to append kv pairs to the context using +`AppendToOutgoingContext`. This can be used with or without existing metadata on the context. When there is no prior +metadata, metadata is added; when metadata already exists on the context, kv pairs are merged in. + +```go +// create a new context with some metadata +ctx := metadata.AppendToOutgoingContext(ctx, "k1", "v1", "k1", "v2", "k2", "v3") + +// later, add some more metadata to the context (e.g. in an interceptor) +ctx := metadata.AppendToOutgoingContext(ctx, "k3", "v4") + +// make unary RPC +response, err := client.SomeRPC(ctx, someRequest) + +// or make streaming RPC +stream, err := client.SomeStreamingRPC(ctx) +``` + +Alternatively, metadata may be attached to the context using `NewOutgoingContext`. However, this +replaces any existing metadata in the context, so care must be taken to preserve the existing +metadata if desired. This is slower than using `AppendToOutgoingContext`. An example of this +is below: + +```go +// create a new context with some metadata +md := metadata.Pairs("k1", "v1", "k1", "v2", "k2", "v3") +ctx := metadata.NewOutgoingContext(context.Background(), md) + +// later, add some more metadata to the context (e.g. in an interceptor) +send, _ := metadata.FromOutgoingContext(ctx) +newMD := metadata.Pairs("k3", "v3") +ctx = metadata.NewOutgoingContext(ctx, metadata.Join(send, newMD)) + +// make unary RPC +response, err := client.SomeRPC(ctx, someRequest) + +// or make streaming RPC +stream, err := client.SomeStreamingRPC(ctx) +``` + +### Receiving metadata + +Metadata that a client can receive includes header and trailer. + +#### Unary call + +Header and trailer sent along with a unary call can be retrieved using function +[Header] and [Trailer] in [CallOption]: + +```go +var header, trailer metadata.MD // variable to store header and trailer +r, err := client.SomeRPC( + ctx, + someRequest, + grpc.Header(&header), // will retrieve header + grpc.Trailer(&trailer), // will retrieve trailer +) + +// do something with header and trailer +``` + +#### Streaming call + +For streaming calls including: + +- Server streaming RPC +- Client streaming RPC +- Bidirectional streaming RPC + +Header and trailer can be retrieved from the returned stream using function +`Header` and `Trailer` in interface [ClientStream]: + +```go +stream, err := client.SomeStreamingRPC(ctx) + +// retrieve header +header, err := stream.Header() + +// retrieve trailer +trailer := stream.Trailer() + +``` + +## Sending and receiving metadata - server side + +Server side metadata sending and receiving examples are available +[here](../examples/features/metadata/server/main.go). + +### Receiving metadata + +To read metadata sent by the client, the server needs to retrieve it from RPC +context using [FromIncomingContext]. +If it is a unary call, the RPC handler's context can be used. +For streaming calls, the server needs to get context from the stream. + +#### Unary call + +```go +func (s *server) SomeRPC(ctx context.Context, in *pb.someRequest) (*pb.someResponse, error) { + md, ok := metadata.FromIncomingContext(ctx) + // do something with metadata +} +``` + +#### Streaming call + +```go +func (s *server) SomeStreamingRPC(stream pb.Service_SomeStreamingRPCServer) error { + md, ok := metadata.FromIncomingContext(stream.Context()) // get context from stream + // do something with metadata +} +``` + +### Sending metadata + +#### Unary call + +To send header and trailer to client in unary call, the server can call +[SetHeader] and [SetTrailer] functions in module [grpc]. +These two functions take a context as the first parameter. +It should be the RPC handler's context or one derived from it: + +```go +func (s *server) SomeRPC(ctx context.Context, in *pb.someRequest) (*pb.someResponse, error) { + // create and set header + header := metadata.Pairs("header-key", "val") + grpc.SetHeader(ctx, header) + // create and set trailer + trailer := metadata.Pairs("trailer-key", "val") + grpc.SetTrailer(ctx, trailer) +} +``` + +#### Streaming call + +For streaming calls, header and trailer can be sent using function +[SetHeader] and [SetTrailer] in interface [ServerStream]: + +```go +func (s *server) SomeStreamingRPC(stream pb.Service_SomeStreamingRPCServer) error { + // create and set header + header := metadata.Pairs("header-key", "val") + stream.SetHeader(header) + // create and set trailer + trailer := metadata.Pairs("trailer-key", "val") + stream.SetTrailer(trailer) +} +``` + +**Important** + +Do not use +[FromOutgoingContext] on the server to write metadata to be sent to the client. +[FromOutgoingContext] is for client-side use only. + +## Updating metadata from a server interceptor + +An example for updating metadata from a server interceptor is +available [here](../examples/features/metadata_interceptor/server/main.go). + +[FromIncomingContext]: +[SetHeader]: +[SetTrailer]: https://godoc.org/google.golang.org/grpc#SetTrailer +[FromOutgoingContext]: https://pkg.go.dev/google.golang.org/grpc/metadata#FromOutgoingContext +[ServerStream]: https://godoc.org/google.golang.org/grpc#ServerStream +[grpc]: https://godoc.org/google.golang.org/grpc +[ClientStream]: https://godoc.org/google.golang.org/grpc#ClientStream +[Header]: https://godoc.org/google.golang.org/grpc#Header +[Trailer]: https://godoc.org/google.golang.org/grpc#Trailer +[CallOption]: https://godoc.org/google.golang.org/grpc#CallOption +[metadata]: https://godoc.org/google.golang.org/grpc/metadata diff --git a/Documentation/keepalive.md b/Documentation/keepalive.md new file mode 100644 index 0000000..6bd9536 --- /dev/null +++ b/Documentation/keepalive.md @@ -0,0 +1,57 @@ +# Keepalive + +gRPC sends http2 pings on the transport to detect if the connection is down. If +the ping is not acknowledged by the other side within a certain period, the +connection will be closed. Note that pings are only necessary when there's no +activity on the connection. + +For how to configure keepalive, see +https://godoc.org/google.golang.org/grpc/keepalive for the options. + +## Why do I need this? + +Keepalive can be useful to detect TCP level connection failures. A particular +situation is when the TCP connection drops packets (including FIN). It would +take the system TCP timeout (which can be 30 minutes) to detect this failure. +Keepalive would allow gRPC to detect this failure much sooner. + +Another usage is (as the name suggests) to keep the connection alive. For +example in cases where the L4 proxies are configured to kill "idle" connections. +Sending pings would make the connections not "idle". + +## What should I set? + +It should be sufficient for most users to set [client +parameters](https://godoc.org/google.golang.org/grpc/keepalive) as a [dial +option](https://godoc.org/google.golang.org/grpc#WithKeepaliveParams). + +## What will happen? + +(The behavior described here is specific for gRPC-go, it might be slightly +different in other languages.) + +When there's no activity on a connection (note that an ongoing stream results in +__no activity__ when there's no message being sent), after `Time`, a ping will +be sent by the client and the server will send a ping ack when it gets the ping. +Client will wait for `Timeout`, and check if there's any activity on the +connection during this period (a ping ack is an activity). + +## What about server side? + +Server has similar `Time` and `Timeout` settings as client. Server can also +configure connection max-age. See [server +parameters](https://godoc.org/google.golang.org/grpc/keepalive#ServerParameters) +for details. + +### Enforcement policy + +[Enforcement +policy](https://godoc.org/google.golang.org/grpc/keepalive#EnforcementPolicy) is +a special setting on server side to protect server from malicious or misbehaving +clients. + +Server sends GOAWAY with ENHANCE_YOUR_CALM and close the connection when bad +behaviors are detected: + - Client sends too frequent pings + - Client sends pings when there's no stream and this is disallowed by server + config diff --git a/Documentation/log_levels.md b/Documentation/log_levels.md new file mode 100644 index 0000000..6839f6d --- /dev/null +++ b/Documentation/log_levels.md @@ -0,0 +1,49 @@ +# Log Levels + +This document describes the different log levels supported by the grpc-go +library, and under what conditions they should be used. + +### Info + +Info messages are for informational purposes and may aid in the debugging of +applications or the gRPC library. + +Examples: +- The name resolver received an update. +- The balancer updated its picker. +- Significant gRPC state is changing. + +At verbosity of 0 (the default), any single info message should not be output +more than once every 5 minutes under normal operation. + +### Warning + +Warning messages indicate problems that are non-fatal for the application, but +could lead to unexpected behavior or subsequent errors. + +Examples: +- Resolver could not resolve target name. +- Error received while connecting to a server. +- Lost or corrupt connection with remote endpoint. + +### Error + +Error messages represent errors in the usage of gRPC that cannot be returned to +the application as errors, or internal gRPC-Go errors that are recoverable. + +Internal errors are detected during gRPC tests and will result in test failures. + +Examples: +- Invalid arguments passed to a function that cannot return an error. +- An internal error that cannot be returned or would be inappropriate to return + to the user. + +### Fatal + +Fatal errors are severe internal errors that are unrecoverable. These lead +directly to panics, and are avoided as much as possible. + +Example: +- Internal invariant was violated. +- User attempted an action that cannot return an error gracefully, but would + lead to an invalid state if performed. diff --git a/Documentation/proxy.md b/Documentation/proxy.md new file mode 100644 index 0000000..0424b94 --- /dev/null +++ b/Documentation/proxy.md @@ -0,0 +1,25 @@ +# Proxy + +HTTP CONNECT proxies are supported by default in gRPC. The proxy address can be +specified by the environment variables `HTTPS_PROXY` and `NO_PROXY`. (Note that +these environment variables are case insensitive.) + +**NOTE**: Using CONNECT proxies via https is not supported. gRPC performs a +plaintext CONNECT handshake to establish a tunnel and does not support the +additional encryption required to secure the initial connection to the proxy +itself. + +When using TLS, the gRPC traffic is encrypted end-to-end between the client and +the destination server. Even when using a CONNECT proxy without https, the +security is not compromised, as the proxy only sees the destination address and +cannot intercept the encrypted gRPC data. + +## Custom proxy + +Currently, proxy support is implemented in the default dialer. It does one more +handshake (a CONNECT handshake in the case of HTTP CONNECT proxy) on the +connection before giving it to gRPC. + +If the default proxy doesn't work for you, replace the default dialer with your +custom proxy dialer. This can be done using +[`WithContextDialer`](https://pkg.go.dev/google.golang.org/grpc#WithContextDialer). diff --git a/Documentation/rpc-errors.md b/Documentation/rpc-errors.md new file mode 100644 index 0000000..c277d26 --- /dev/null +++ b/Documentation/rpc-errors.md @@ -0,0 +1,68 @@ +# RPC Errors + +All service method handlers should return `nil` or errors from the +`status.Status` type. Clients have direct access to the errors. + +Upon encountering an error, a gRPC server method handler should create a +`status.Status`. In typical usage, one would use [status.New][new-status] +passing in an appropriate [codes.Code][code] as well as a description of the +error to produce a `status.Status`. Calling [status.Err][status-err] converts +the `status.Status` type into an `error`. As a convenience method, there is also +[status.Error][status-error] which obviates the conversion step. Compare: + +``` +st := status.New(codes.NotFound, "some description") +err := st.Err() + +// vs. + +err := status.Error(codes.NotFound, "some description") +``` + +## Adding additional details to errors + +In some cases, it may be necessary to add details for a particular error on the +server side. The [status.WithDetails][with-details] method exists for this +purpose. Clients may then read those details by first converting the plain +`error` type back to a [status.Status][status] and then using +[status.Details][details]. + +## Example + +The [example][example] demonstrates the API discussed above and shows how to add +information about rate limits to the error message using `status.Status`. + +To run the example, first start the server: + +``` +$ go run examples/rpc_errors/server/main.go +``` + +In a separate session, run the client: + +``` +$ go run examples/rpc_errors/client/main.go +``` + +On the first run of the client, all is well: + +``` +2018/03/12 19:39:33 Greeting: Hello world +``` + +Upon running the client a second time, the client exceeds the rate limit and +receives an error with details: + +``` +2018/03/19 16:42:01 Quota failure: violations: +exit status 1 +``` + +[status]: https://godoc.org/google.golang.org/grpc/status#Status +[new-status]: https://godoc.org/google.golang.org/grpc/status#New +[code]: https://godoc.org/google.golang.org/grpc/codes#Code +[with-details]: https://godoc.org/google.golang.org/grpc/internal/status#Status.WithDetails +[details]: https://godoc.org/google.golang.org/grpc/internal/status#Status.Details +[status-err]: https://godoc.org/google.golang.org/grpc/internal/status#Status.Err +[status-error]: https://godoc.org/google.golang.org/grpc/status#Error +[example]: https://github.com/grpc/grpc-go/tree/master/examples/features/error_details diff --git a/Documentation/server-reflection-tutorial.md b/Documentation/server-reflection-tutorial.md new file mode 100644 index 0000000..6bffaf0 --- /dev/null +++ b/Documentation/server-reflection-tutorial.md @@ -0,0 +1,171 @@ +# gRPC Server Reflection Tutorial + +gRPC Server Reflection provides information about publicly-accessible gRPC +services on a server, and assists clients at runtime to construct RPC requests +and responses without precompiled service information. It is used by +[gRPCurl](https://github.com/fullstorydev/grpcurl), which can be used to +introspect server protos and send/receive test RPCs. + +## Enable Server Reflection + +gRPC-go Server Reflection is implemented in package +[reflection](https://github.com/grpc/grpc-go/tree/master/reflection). To enable +server reflection, you need to import this package and register reflection +service on your gRPC server. + +For example, to enable server reflection in `example/helloworld`, we need to +make the following changes: + +```diff +--- a/examples/helloworld/greeter_server/main.go ++++ b/examples/helloworld/greeter_server/main.go +@@ -40,6 +40,7 @@ import ( + "google.golang.org/grpc" + pb "google.golang.org/grpc/examples/helloworld/helloworld" ++ "google.golang.org/grpc/reflection" + ) + + const ( +@@ -61,6 +62,8 @@ func main() { + } + s := grpc.NewServer() + pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHello}) ++ // Register reflection service on gRPC server. ++ reflection.Register(s) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +``` + +An example server with reflection registered can be found at +`examples/features/reflection/server`. + +## gRPCurl + +After enabling Server Reflection in a server application, you can use gRPCurl +to check its services. gRPCurl is built with Go and has packages available. +Instructions on how to install and use gRPCurl can be found at +[gRPCurl Installation](https://github.com/fullstorydev/grpcurl#installation). + +## Use gRPCurl to check services + +First, start the helloworld server in grpc-go directory: + +```sh +$ cd /examples +$ go run features/reflection/server/main.go +``` + +output: +```sh +server listening at [::]:50051 +``` + +After installing gRPCurl, open a new terminal and run the commands from the new +terminal. + +**NOTE:** gRPCurl expects a TLS-encrypted connection by default. For all of +the commands below, use the `-plaintext` flag to use an unencrypted connection. + +### List services and methods + +The `list` command lists services exposed at a given port: + +- List all the services exposed at a given port + + ```sh + $ grpcurl -plaintext localhost:50051 list + ``` + + output: + ```sh + grpc.examples.echo.Echo + grpc.reflection.v1alpha.ServerReflection + helloworld.Greeter + ``` + +- List all the methods of a service + + The `list` command lists methods given the full service name (in the format of + \.\). + + ```sh + $ grpcurl -plaintext localhost:50051 list helloworld.Greeter + ``` + + output: + ```sh + helloworld.Greeter.SayHello + ``` + +### Describe services and methods + +- Describe all services + + The `describe` command inspects a service given its full name (in the format + of \.\). + + ```sh + $ grpcurl -plaintext localhost:50051 describe helloworld.Greeter + ``` + + output: + ```sh + helloworld.Greeter is a service: + service Greeter { + rpc SayHello ( .helloworld.HelloRequest ) returns ( .helloworld.HelloReply ); + } + ``` + +- Describe all methods of a service + + The `describe` command inspects a method given its full name (in the format of + \.\.\). + + ```sh + $ grpcurl -plaintext localhost:50051 describe helloworld.Greeter.SayHello + ``` + + output: + ```sh + helloworld.Greeter.SayHello is a method: + rpc SayHello ( .helloworld.HelloRequest ) returns ( .helloworld.HelloReply ); + ``` + +### Inspect message types + +We can use the `describe` command to inspect request/response types given the +full name of the type (in the format of \.\). + +- Get information about the request type + + ```sh + $ grpcurl -plaintext localhost:50051 describe helloworld.HelloRequest + ``` + + output: + ```sh + helloworld.HelloRequest is a message: + message HelloRequest { + string name = 1; + } + ``` + +### Call a remote method + +We can send RPCs to a server and get responses using the full method name (in +the format of \.\.\). The `-d ` flag +represents the request data and the `-format text` flag indicates that the +request data is in text format. + +- Call a unary method + + ```sh + $ grpcurl -plaintext -format text -d 'name: "gRPCurl"' \ + localhost:50051 helloworld.Greeter.SayHello + ``` + + output: + ```sh + message: "Hello gRPCurl" + ``` diff --git a/Documentation/versioning.md b/Documentation/versioning.md new file mode 100644 index 0000000..03e868c --- /dev/null +++ b/Documentation/versioning.md @@ -0,0 +1,34 @@ +# Versioning and Releases + +Note: This document references terminology defined at http://semver.org. + +## Release Frequency + +Regular MINOR releases of gRPC-Go are performed every six weeks. Patch releases +to the previous two MINOR releases may be performed on demand or if serious +security problems are discovered. + +## Versioning Policy + +The gRPC-Go versioning policy follows the Semantic Versioning 2.0.0 +specification, with the following exceptions: + +- A MINOR version will not _necessarily_ add new functionality. + +- MINOR releases will not break backward compatibility, except in the following +circumstances: + + - An API was marked as EXPERIMENTAL upon its introduction. + - An API was marked as DEPRECATED in the initial MAJOR release. + - An API is inherently flawed and cannot provide correct or secure behavior. + + In these cases, APIs MAY be changed or removed without a MAJOR release. +Otherwise, backward compatibility will be preserved by MINOR releases. + + For an API marked as DEPRECATED, an alternative will be available (if +appropriate) for at least three months prior to its removal. + +## Release History + +Please see our release history on GitHub: +https://github.com/grpc/grpc-go/releases diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 0000000..d6ff267 --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1 @@ +This repository is governed by the gRPC organization's [governance rules](https://github.com/grpc/grpc-community/blob/master/governance.md). diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 0000000..df35bb9 --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1,36 @@ +This page lists all active maintainers of this repository. If you were a +maintainer and would like to add your name to the Emeritus list, please send us a +PR. + +See [GOVERNANCE.md](https://github.com/grpc/grpc-community/blob/master/governance.md) +for governance guidelines and how to become a maintainer. +See [CONTRIBUTING.md](https://github.com/grpc/grpc-community/blob/master/CONTRIBUTING.md) +for general contribution guidelines. + +## Maintainers (in alphabetical order) + +- [arjan-bal](https://github.com/arjan-bal), Google LLC +- [arvindbr8](https://github.com/arvindbr8), Google LLC +- [atollena](https://github.com/atollena), Datadog, Inc. +- [dfawley](https://github.com/dfawley), Google LLC +- [easwars](https://github.com/easwars), Google LLC +- [gtcooke94](https://github.com/gtcooke94), Google LLC + +## Emeritus Maintainers (in alphabetical order) +- [adelez](https://github.com/adelez) +- [aranjans](https://github.com/aranjans) +- [canguler](https://github.com/canguler) +- [cesarghali](https://github.com/cesarghali) +- [erm-g](https://github.com/erm-g) +- [iamqizhao](https://github.com/iamqizhao) +- [jeanbza](https://github.com/jeanbza) +- [jtattermusch](https://github.com/jtattermusch) +- [lyuxuan](https://github.com/lyuxuan) +- [makmukhi](https://github.com/makmukhi) +- [matt-kwong](https://github.com/matt-kwong) +- [menghanl](https://github.com/menghanl) +- [nicolasnoble](https://github.com/nicolasnoble) +- [purnesh42h](https://github.com/purnesh42h) +- [srini100](https://github.com/srini100) +- [yongni](https://github.com/yongni) +- [zasweq](https://github.com/zasweq) diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..be38384 --- /dev/null +++ b/Makefile @@ -0,0 +1,49 @@ +all: vet test testrace + +build: + go build google.golang.org/grpc/... + +clean: + go clean -i google.golang.org/grpc/... + +deps: + GO111MODULE=on go get -d -v google.golang.org/grpc/... + +proto: + @ if ! which protoc > /dev/null; then \ + echo "error: protoc not installed" >&2; \ + exit 1; \ + fi + go generate google.golang.org/grpc/... + +test: + go test -cpu 1,4 -timeout 7m google.golang.org/grpc/... + +testsubmodule: + cd security/advancedtls && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/advancedtls/... + cd security/authorization && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/authorization/... + +testrace: + go test -race -cpu 1,4 -timeout 7m google.golang.org/grpc/... + +testdeps: + GO111MODULE=on go get -d -v -t google.golang.org/grpc/... + +vet: vetdeps + ./scripts/vet.sh + +vetdeps: + ./scripts/vet.sh -install + +.PHONY: \ + all \ + build \ + clean \ + deps \ + proto \ + test \ + testsubmodule \ + testrace \ + testdeps \ + vet \ + vetdeps diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 0000000..5301977 --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,13 @@ +Copyright 2014 gRPC authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f9a88d5 --- /dev/null +++ b/README.md @@ -0,0 +1,108 @@ +# gRPC-Go + +[![GoDoc](https://pkg.go.dev/badge/google.golang.org/grpc)][API] +[![GoReportCard](https://goreportcard.com/badge/grpc/grpc-go)](https://goreportcard.com/report/github.com/grpc/grpc-go) +[![codecov](https://codecov.io/gh/grpc/grpc-go/graph/badge.svg)](https://codecov.io/gh/grpc/grpc-go) + +The [Go][] implementation of [gRPC][]: A high performance, open source, general +RPC framework that puts mobile and HTTP/2 first. For more information see the +[Go gRPC docs][], or jump directly into the [quick start][]. + +## Prerequisites + +- **[Go][]**: any one of the **two latest major** [releases][go-releases]. + +## Installation + +Simply add the following import to your code, and then `go [build|run|test]` +will automatically fetch the necessary dependencies: + + +```go +import "google.golang.org/grpc" +``` + +> **Note:** If you are trying to access `grpc-go` from **China**, see the +> [FAQ](#FAQ) below. + +## Learn more + +- [Go gRPC docs][], which include a [quick start][] and [API + reference][API] among other resources +- [Low-level technical docs](Documentation) from this repository +- [Performance benchmark][] +- [Examples](examples) +- [Contribution guidelines](CONTRIBUTING.md) + +## FAQ + +### I/O Timeout Errors + +The `golang.org` domain may be blocked from some countries. `go get` usually +produces an error like the following when this happens: + +```console +$ go get -u google.golang.org/grpc +package google.golang.org/grpc: unrecognized import path "google.golang.org/grpc" (https fetch: Get https://google.golang.org/grpc?go-get=1: dial tcp 216.239.37.1:443: i/o timeout) +``` + +To build Go code, there are several options: + +- Set up a VPN and access google.golang.org through that. + +- With Go module support: it is possible to use the `replace` feature of `go + mod` to create aliases for golang.org packages. In your project's directory: + + ```sh + go mod edit -replace=google.golang.org/grpc=github.com/grpc/grpc-go@latest + go mod tidy + go mod vendor + go build -mod=vendor + ``` + + Again, this will need to be done for all transitive dependencies hosted on + golang.org as well. For details, refer to [golang/go issue + #28652](https://github.com/golang/go/issues/28652). + +### Compiling error, undefined: grpc.SupportPackageIsVersion + +Please update to the latest version of gRPC-Go using +`go get google.golang.org/grpc`. + +### How to turn on logging + +The default logger is controlled by environment variables. Turn everything on +like this: + +```console +$ export GRPC_GO_LOG_VERBOSITY_LEVEL=99 +$ export GRPC_GO_LOG_SEVERITY_LEVEL=info +``` + +### The RPC failed with error `"code = Unavailable desc = transport is closing"` + +This error means the connection the RPC is using was closed, and there are many +possible reasons, including: + 1. mis-configured transport credentials, connection failed on handshaking + 1. bytes disrupted, possibly by a proxy in between + 1. server shutdown + 1. Keepalive parameters caused connection shutdown, for example if you have + configured your server to terminate connections regularly to [trigger DNS + lookups](https://github.com/grpc/grpc-go/issues/3170#issuecomment-552517779). + If this is the case, you may want to increase your + [MaxConnectionAgeGrace](https://pkg.go.dev/google.golang.org/grpc/keepalive?tab=doc#ServerParameters), + to allow longer RPC calls to finish. + +It can be tricky to debug this because the error happens on the client side but +the root cause of the connection being closed is on the server side. Turn on +logging on __both client and server__, and see if there are any transport +errors. + +[API]: https://pkg.go.dev/google.golang.org/grpc +[Go]: https://golang.org +[Go module]: https://github.com/golang/go/wiki/Modules +[gRPC]: https://grpc.io +[Go gRPC docs]: https://grpc.io/docs/languages/go +[Performance benchmark]: https://performance-dot-grpc-testing.appspot.com/explore?dashboard=5180705743044608 +[quick start]: https://grpc.io/docs/languages/go/quickstart +[go-releases]: https://golang.org/doc/devel/release.html diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..abab279 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +For information on gRPC Security Policy and reporting potential security issues, please see [gRPC CVE Process](https://github.com/grpc/proposal/blob/master/P4-grpc-cve-process.md). diff --git a/admin/admin.go b/admin/admin.go new file mode 100644 index 0000000..41ae156 --- /dev/null +++ b/admin/admin.go @@ -0,0 +1,58 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package admin provides a convenient method for registering a collection of +// administration services to a gRPC server. The services registered are: +// +// - Channelz: https://github.com/grpc/proposal/blob/master/A14-channelz.md +// +// - CSDS: https://github.com/grpc/proposal/blob/master/A40-csds-support.md +// +// # Experimental +// +// Notice: All APIs in this package are experimental and may be removed in a +// later release. +package admin + +import ( + "google.golang.org/grpc" + channelzservice "google.golang.org/grpc/channelz/service" + internaladmin "google.golang.org/grpc/internal/admin" +) + +func init() { + // Add a list of default services to admin here. Optional services, like + // CSDS, will be added by other packages. + internaladmin.AddService(func(registrar grpc.ServiceRegistrar) (func(), error) { + channelzservice.RegisterChannelzServiceToServer(registrar) + return nil, nil + }) +} + +// Register registers the set of admin services to the given server. +// +// The returned cleanup function should be called to clean up the resources +// allocated for the service handlers after the server is stopped. +// +// Note that if `s` is not a *grpc.Server or a *xds.GRPCServer, CSDS will not be +// registered because CSDS generated code is old and doesn't support interface +// `grpc.ServiceRegistrar`. +// https://github.com/envoyproxy/go-control-plane/issues/403 +func Register(s grpc.ServiceRegistrar) (cleanup func(), _ error) { + return internaladmin.Register(s) +} diff --git a/admin/admin_test.go b/admin/admin_test.go new file mode 100644 index 0000000..0ee4aad --- /dev/null +++ b/admin/admin_test.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package admin_test + +import ( + "testing" + + "google.golang.org/grpc/admin/test" + "google.golang.org/grpc/codes" +) + +func TestRegisterNoCSDS(t *testing.T) { + test.RunRegisterTests(t, test.ExpectedStatusCodes{ + ChannelzCode: codes.OK, + // CSDS is not registered because xDS isn't imported. + CSDSCode: codes.Unimplemented, + }) +} diff --git a/admin/test/admin_test.go b/admin/test/admin_test.go new file mode 100644 index 0000000..f0f784b --- /dev/null +++ b/admin/test/admin_test.go @@ -0,0 +1,38 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// This file has the same content as admin_test.go, difference is that this is +// in another package, and it imports "xds", so we can test that csds is +// registered when xds is imported. + +package test_test + +import ( + "testing" + + "google.golang.org/grpc/admin/test" + "google.golang.org/grpc/codes" + _ "google.golang.org/grpc/xds" +) + +func TestRegisterWithCSDS(t *testing.T) { + test.RunRegisterTests(t, test.ExpectedStatusCodes{ + ChannelzCode: codes.OK, + CSDSCode: codes.OK, + }) +} diff --git a/admin/test/utils.go b/admin/test/utils.go new file mode 100644 index 0000000..2f86c15 --- /dev/null +++ b/admin/test/utils.go @@ -0,0 +1,104 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package test contains test only functions for package admin. It's used by +// admin/admin_test.go and admin/test/admin_test.go. +package test + +import ( + "context" + "net" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/admin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + v3statusgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + channelzgrpc "google.golang.org/grpc/channelz/grpc_channelz_v1" + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" +) + +const ( + defaultTestTimeout = 10 * time.Second +) + +// ExpectedStatusCodes contains the expected status code for each RPC (can be +// OK). +type ExpectedStatusCodes struct { + ChannelzCode codes.Code + CSDSCode codes.Code +} + +// RunRegisterTests makes a client, runs the RPCs, and compares the status +// codes. +func RunRegisterTests(t *testing.T, ec ExpectedStatusCodes) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("cannot create listener: %v", err) + } + + server := grpc.NewServer() + defer server.Stop() + cleanup, err := admin.Register(server) + if err != nil { + t.Fatalf("failed to register admin: %v", err) + } + defer cleanup() + go func() { + server.Serve(lis) + }() + + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err) + } + + t.Run("channelz", func(t *testing.T) { + if err := RunChannelz(conn); status.Code(err) != ec.ChannelzCode { + t.Fatalf("%s RPC failed with error %v, want code %v", "channelz", err, ec.ChannelzCode) + } + }) + t.Run("csds", func(t *testing.T) { + if err := RunCSDS(conn); status.Code(err) != ec.CSDSCode { + t.Fatalf("%s RPC failed with error %v, want code %v", "CSDS", err, ec.CSDSCode) + } + }) +} + +// RunChannelz makes a channelz RPC. +func RunChannelz(conn *grpc.ClientConn) error { + c := channelzgrpc.NewChannelzClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := c.GetTopChannels(ctx, &channelzpb.GetTopChannelsRequest{}, grpc.WaitForReady(true)) + return err +} + +// RunCSDS makes a CSDS RPC. +func RunCSDS(conn *grpc.ClientConn) error { + c := v3statusgrpc.NewClientStatusDiscoveryServiceClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := c.FetchClientStatus(ctx, &v3statuspb.ClientStatusRequest{}, grpc.WaitForReady(true)) + return err +} diff --git a/attributes/attributes.go b/attributes/attributes.go new file mode 100644 index 0000000..4c60518 --- /dev/null +++ b/attributes/attributes.go @@ -0,0 +1,174 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package attributes defines a generic key/value store used in various gRPC +// components. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +package attributes + +import ( + "fmt" + "iter" + "maps" + "strings" +) + +// Attributes is an immutable struct for storing and retrieving generic +// key/value pairs. Keys must be hashable, and users should define their own +// types for keys. Values should not be modified after they are added to an +// Attributes or if they were received from one. If values implement 'Equal(o +// any) bool', it will be called by (*Attributes).Equal to determine whether +// two values with the same key should be considered equal. +type Attributes struct { + parent *Attributes + key, value any +} + +// New returns a new Attributes containing the key/value pair. +func New(key, value any) *Attributes { + return &Attributes{ + key: key, + value: value, + } +} + +// WithValue returns a new Attributes containing the previous keys and values +// and the new key/value pair. If the same key appears multiple times, the +// last value overwrites all previous values for that key. value should not be +// modified later. +// +// Note that Attributes do not support deletion. Avoid using untyped nil values. +// Since the Value method returns an untyped nil when a key is absent, it is +// impossible to distinguish between a missing key and a key explicitly set to +// an untyped nil. If you need to represent a value being unset, consider +// storing a specific sentinel type or a wrapper struct with a boolean field +// indicating presence. +func (a *Attributes) WithValue(key, value any) *Attributes { + return &Attributes{ + parent: a, + key: key, + value: value, + } +} + +// Value returns the value associated with these attributes for key, or nil if +// no value is associated with key. The returned value should not be modified. +func (a *Attributes) Value(key any) any { + for cur := a; cur != nil; cur = cur.parent { + if cur.key == key { + return cur.value + } + } + return nil +} + +// Equal returns whether a and o are equivalent. If 'Equal(o any) bool' is +// implemented for a value in the attributes, it is called to determine if the +// value matches the one stored in the other attributes. If Equal is not +// implemented, standard equality is used to determine if the two values are +// equal. Note that some types (e.g. maps) aren't comparable by default, so +// they must be wrapped in a struct, or in an alias type, with Equal defined. +func (a *Attributes) Equal(o *Attributes) bool { + if a == nil && o == nil { + return true + } + if a == nil || o == nil { + return false + } + if a == o { + return true + } + m := maps.Collect(o.all()) + lenA := 0 + + for k, v := range a.all() { + lenA++ + ov, ok := m[k] + if !ok { + // o missing element of a + return false + } + if eq, ok := v.(interface{ Equal(o any) bool }); ok { + if !eq.Equal(ov) { + return false + } + } else if v != ov { + // Fallback to a standard equality check if Value is unimplemented. + return false + } + } + return lenA == len(m) +} + +// String prints the attribute map. If any key or values throughout the map +// implement fmt.Stringer, it calls that method and appends. +func (a *Attributes) String() string { + var sb strings.Builder + sb.WriteString("{") + first := true + for k, v := range a.all() { + if !first { + sb.WriteString(", ") + } + fmt.Fprintf(&sb, "%q: %q ", str(k), str(v)) + first = false + } + sb.WriteString("}") + return sb.String() +} + +func str(x any) (s string) { + if v, ok := x.(fmt.Stringer); ok { + return fmt.Sprint(v) + } else if v, ok := x.(string); ok { + return v + } + return fmt.Sprintf("<%p>", x) +} + +// MarshalJSON helps implement the json.Marshaler interface, thereby rendering +// the Attributes correctly when printing (via pretty.JSON) structs containing +// Attributes as fields. +// +// Is it impossible to unmarshal attributes from a JSON representation and this +// method is meant only for debugging purposes. +func (a *Attributes) MarshalJSON() ([]byte, error) { + return []byte(a.String()), nil +} + +// all returns an iterator that yields all key-value pairs in the Attributes +// chain. If a key appears multiple times, only the most recently added value +// is yielded. +func (a *Attributes) all() iter.Seq2[any, any] { + return func(yield func(any, any) bool) { + seen := map[any]bool{} + for cur := a; cur != nil; cur = cur.parent { + if seen[cur.key] { + continue + } + if !yield(cur.key, cur.value) { + return + } + seen[cur.key] = true + } + } +} diff --git a/attributes/attributes_test.go b/attributes/attributes_test.go new file mode 100644 index 0000000..7d0e336 --- /dev/null +++ b/attributes/attributes_test.go @@ -0,0 +1,189 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package attributes_test + +import ( + "fmt" + "testing" + + "google.golang.org/grpc/attributes" +) + +type stringVal struct { + s string +} + +func (s stringVal) Equal(o any) bool { + os, ok := o.(stringVal) + return ok && s.s == os.s +} + +type stringerVal struct { + s string +} + +func (s stringerVal) String() string { + return s.s +} + +func ExampleAttributes() { + type keyOne struct{} + type keyTwo struct{} + a := attributes.New(keyOne{}, 1).WithValue(keyTwo{}, stringVal{s: "two"}) + fmt.Println("Key one:", a.Value(keyOne{})) + fmt.Println("Key two:", a.Value(keyTwo{})) + // Output: + // Key one: 1 + // Key two: {two} +} + +func ExampleAttributes_WithValue() { + type keyOne struct{} + type keyTwo struct{} + a := attributes.New(keyOne{}, 1) + a = a.WithValue(keyTwo{}, stringVal{s: "two"}) + fmt.Println("Key one:", a.Value(keyOne{})) + fmt.Println("Key two:", a.Value(keyTwo{})) + // Output: + // Key one: 1 + // Key two: {two} +} + +func ExampleAttributes_String() { + type key struct{} + var typedNil *stringerVal + a1 := attributes.New(key{}, typedNil) // typed nil implements [fmt.Stringer] + a2 := attributes.New(key{}, (*stringerVal)(nil)) // typed nil implements [fmt.Stringer] + a3 := attributes.New(key{}, (*stringVal)(nil)) // typed nil not implements [fmt.Stringer] + a4 := attributes.New(key{}, nil) // untyped nil + a5 := attributes.New(key{}, 1) + a6 := attributes.New(key{}, stringerVal{s: "two"}) + a7 := attributes.New(key{}, stringVal{s: "two"}) + a8 := attributes.New(1, true) + fmt.Println("a1:", a1.String()) + fmt.Println("a2:", a2.String()) + fmt.Println("a3:", a3.String()) + fmt.Println("a4:", a4.String()) + fmt.Println("a5:", a5.String()) + fmt.Println("a6:", a6.String()) + fmt.Println("a7:", a7.String()) + fmt.Println("a8:", a8.String()) + // Output: + // a1: {"<%!p(attributes_test.key={})>": "" } + // a2: {"<%!p(attributes_test.key={})>": "" } + // a3: {"<%!p(attributes_test.key={})>": "<0x0>" } + // a4: {"<%!p(attributes_test.key={})>": "<%!p()>" } + // a5: {"<%!p(attributes_test.key={})>": "<%!p(int=1)>" } + // a6: {"<%!p(attributes_test.key={})>": "two" } + // a7: {"<%!p(attributes_test.key={})>": "<%!p(attributes_test.stringVal={two})>" } + // a8: {"<%!p(int=1)>": "<%!p(bool=true)>" } +} + +// Test that two attributes with different content are not Equal. +func TestEqual(t *testing.T) { + type keyOne struct{} + type keyTwo struct{} + tests := []struct { + name string + a *attributes.Attributes + b *attributes.Attributes + want bool + }{ + { + name: "different_first_value", + a: attributes.New(keyOne{}, 1).WithValue(keyTwo{}, stringVal{s: "two"}), + b: attributes.New(keyOne{}, 2).WithValue(keyTwo{}, stringVal{s: "two"}), + want: false, + }, + { + name: "different_second_value", + a: attributes.New(keyOne{}, 1).WithValue(keyTwo{}, stringVal{s: "one"}), + b: attributes.New(keyOne{}, 1).WithValue(keyTwo{}, stringVal{s: "two"}), + want: false, + }, + { + name: "same", + a: attributes.New(keyOne{}, 1).WithValue(keyTwo{}, stringVal{s: "two"}), + b: attributes.New(keyOne{}, 1).WithValue(keyTwo{}, stringVal{s: "two"}), + want: true, + }, + { + name: "subset", + a: attributes.New(keyOne{}, 1), + b: attributes.New(keyOne{}, 1).WithValue(keyTwo{}, stringVal{s: "two"}), + want: false, + }, + { + name: "superset", + a: attributes.New(keyOne{}, 1).WithValue(keyTwo{}, stringVal{s: "two"}), + b: attributes.New(keyTwo{}, stringVal{s: "two"}), + want: false, + }, + { + name: "a_nil", + a: nil, + b: attributes.New(keyOne{}, 1), + want: false, + }, + { + name: "b_nil", + a: attributes.New(keyOne{}, 1), + b: nil, + want: false, + }, + { + name: "both_nil", + a: nil, + b: nil, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.a.Equal(tt.b); got != tt.want { + t.Errorf("%+v.Equal(%+v) = %v; want %v", tt.a, tt.b, got, tt.want) + } + // The Equal function should be symmetric, i.e. a.Equals(b) == + // b.Equals(a). + if got := tt.b.Equal(tt.a); got != tt.want { + t.Errorf("%+v.Equal(%+v) = %v; want %v", tt.b, tt.a, got, tt.want) + } + }) + } +} + +func BenchmarkWithValue(b *testing.B) { + keys := make([]any, 10) + for i := range 10 { + keys[i] = i + } + b.ReportAllocs() + + for b.Loop() { + // 50 endpoints + for range 50 { + a := attributes.New(keys[0], keys[0]) + // 10 attributes each. + for j := 1; j < 10; j++ { + a = a.WithValue(keys[j], keys[j]) + } + } + } +} diff --git a/authz/audit/audit_logger.go b/authz/audit/audit_logger.go new file mode 100644 index 0000000..7ea7941 --- /dev/null +++ b/authz/audit/audit_logger.go @@ -0,0 +1,127 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package audit contains interfaces for audit logging during authorization. +package audit + +import ( + "encoding/json" + "sync" +) + +// loggerBuilderRegistry holds a map of audit logger builders and a mutex +// to facilitate thread-safe reading/writing operations. +type loggerBuilderRegistry struct { + mu sync.Mutex + builders map[string]LoggerBuilder +} + +var ( + registry = loggerBuilderRegistry{ + builders: make(map[string]LoggerBuilder), + } +) + +// RegisterLoggerBuilder registers the builder in a global map +// using b.Name() as the key. +// +// This should only be called during initialization time (i.e. in an init() +// function). If multiple builders are registered with the same name, +// the one registered last will take effect. +func RegisterLoggerBuilder(b LoggerBuilder) { + registry.mu.Lock() + defer registry.mu.Unlock() + registry.builders[b.Name()] = b +} + +// GetLoggerBuilder returns a builder with the given name. +// It returns nil if the builder is not found in the registry. +func GetLoggerBuilder(name string) LoggerBuilder { + registry.mu.Lock() + defer registry.mu.Unlock() + return registry.builders[name] +} + +// Event contains information passed to the audit logger as part of an +// audit logging event. +type Event struct { + // FullMethodName is the full method name of the audited RPC, in the format + // of "/pkg.Service/Method". For example, "/helloworld.Greeter/SayHello". + FullMethodName string + // Principal is the identity of the caller. Currently it will only be + // available in certificate-based TLS authentication. + Principal string + // PolicyName is the authorization policy name or the xDS RBAC filter name. + PolicyName string + // MatchedRule is the matched rule or policy name in the xDS RBAC filter. + // It will be empty if there is no match. + MatchedRule string + // Authorized indicates whether the audited RPC is authorized or not. + Authorized bool +} + +// LoggerConfig represents an opaque data structure holding an audit +// logger configuration. Concrete types representing configuration of specific +// audit loggers must embed this interface to implement it. +type LoggerConfig interface { + loggerConfig() +} + +// Logger is the interface to be implemented by audit loggers. +// +// An audit logger is a logger instance that can be configured via the +// authorization policy API or xDS HTTP RBAC filters. When the authorization +// decision meets the condition for audit, all the configured audit loggers' +// Log() method will be invoked to log that event. +// +// Please refer to +// https://github.com/grpc/proposal/blob/master/A59-audit-logging.md for more +// details about audit logging. +type Logger interface { + // Log performs audit logging for the provided audit event. + // + // This method is invoked in the RPC path and therefore implementations + // must not block. + Log(*Event) +} + +// LoggerBuilder is the interface to be implemented by audit logger +// builders that are used at runtime to configure and instantiate audit loggers. +// +// Users who want to implement their own audit logging logic should +// implement this interface, along with the Logger interface, and register +// it by calling RegisterLoggerBuilder() at init time. +// +// Please refer to +// https://github.com/grpc/proposal/blob/master/A59-audit-logging.md for more +// details about audit logging. +type LoggerBuilder interface { + // ParseLoggerConfig parses the given JSON bytes into a structured + // logger config this builder can use to build an audit logger. + ParseLoggerConfig(config json.RawMessage) (LoggerConfig, error) + // Build builds an audit logger with the given logger config. + // This will only be called with valid configs returned from + // ParseLoggerConfig() and any runtime issues such as failing to + // create a file should be handled by the logger implementation instead of + // failing the logger instantiation. So implementers need to make sure it + // can return a logger without error at this stage. + Build(LoggerConfig) Logger + // Name returns the name of logger built by this builder. + // This is used to register and pick the builder. + Name() string +} diff --git a/authz/audit/audit_logging_test.go b/authz/audit/audit_logging_test.go new file mode 100644 index 0000000..e24f4ae --- /dev/null +++ b/authz/audit/audit_logging_test.go @@ -0,0 +1,370 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package audit_test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "io" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/authz" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" + + _ "google.golang.org/grpc/authz/audit/stdout" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type statAuditLogger struct { + authzDecisionStat map[bool]int // Map to hold the counts of authorization decisions + lastEvent *audit.Event // Field to store last received event +} + +func (s *statAuditLogger) Log(event *audit.Event) { + s.authzDecisionStat[event.Authorized]++ + *s.lastEvent = *event +} + +type loggerBuilder struct { + authzDecisionStat map[bool]int + lastEvent *audit.Event +} + +func (loggerBuilder) Name() string { + return "stat_logger" +} + +func (lb *loggerBuilder) Build(audit.LoggerConfig) audit.Logger { + return &statAuditLogger{ + authzDecisionStat: lb.authzDecisionStat, + lastEvent: lb.lastEvent, + } +} + +func (*loggerBuilder) ParseLoggerConfig(json.RawMessage) (audit.LoggerConfig, error) { + return nil, nil +} + +// TestAuditLogger examines audit logging invocations using four different +// authorization policies. It covers scenarios including a disabled audit, +// auditing both 'allow' and 'deny' outcomes, and separately auditing 'allow' +// and 'deny' outcomes. Additionally, it checks if SPIFFE ID from a certificate +// is propagated correctly. +func (s) TestAuditLogger(t *testing.T) { + // Each test data entry contains an authz policy for a grpc server, + // how many 'allow' and 'deny' outcomes we expect (each test case makes 2 + // unary calls and one client-streaming call), and a structure to check if + // the audit.Event fields are properly populated. Additionally, we specify + // directly which authz outcome we expect from each type of call. + tests := []struct { + name string + authzPolicy string + wantAuthzOutcomes map[bool]int + eventContent *audit.Event + wantUnaryCallCode codes.Code + wantStreamingCallCode codes.Code + }{ + { + name: "No audit", + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_UnaryCall", + "request": { + "paths": [ + "/grpc.testing.TestService/UnaryCall" + ] + } + } + ], + "audit_logging_options": { + "audit_condition": "NONE", + "audit_loggers": [ + { + "name": "stat_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantAuthzOutcomes: map[bool]int{true: 0, false: 0}, + wantUnaryCallCode: codes.OK, + wantStreamingCallCode: codes.PermissionDenied, + }, + { + name: "Allow All Deny Streaming - Audit All", + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_all", + "request": { + "paths": [ + "*" + ] + } + } + ], + "deny_rules": [ + { + "name": "deny_all", + "request": { + "paths": [ + "/grpc.testing.TestService/StreamingInputCall" + ] + } + } + ], + "audit_logging_options": { + "audit_condition": "ON_DENY_AND_ALLOW", + "audit_loggers": [ + { + "name": "stat_logger", + "config": {}, + "is_optional": false + }, + { + "name": "stdout_logger", + "is_optional": false + } + ] + } + }`, + wantAuthzOutcomes: map[bool]int{true: 2, false: 1}, + eventContent: &audit.Event{ + FullMethodName: "/grpc.testing.TestService/StreamingInputCall", + Principal: "spiffe://foo.bar.com/client/workload/1", + PolicyName: "authz", + MatchedRule: "authz_deny_all", + Authorized: false, + }, + wantUnaryCallCode: codes.OK, + wantStreamingCallCode: codes.PermissionDenied, + }, + { + name: "Allow Unary - Audit Allow", + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_UnaryCall", + "request": { + "paths": [ + "/grpc.testing.TestService/UnaryCall" + ] + } + } + ], + "audit_logging_options": { + "audit_condition": "ON_ALLOW", + "audit_loggers": [ + { + "name": "stat_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantAuthzOutcomes: map[bool]int{true: 2, false: 0}, + wantUnaryCallCode: codes.OK, + wantStreamingCallCode: codes.PermissionDenied, + }, + { + name: "Allow Typo - Audit Deny", + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_UnaryCall", + "request": { + "paths": [ + "/grpc.testing.TestService/UnaryCall_Z" + ] + } + } + ], + "audit_logging_options": { + "audit_condition": "ON_DENY", + "audit_loggers": [ + { + "name": "stat_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantAuthzOutcomes: map[bool]int{true: 0, false: 3}, + wantUnaryCallCode: codes.PermissionDenied, + wantStreamingCallCode: codes.PermissionDenied, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Construct the credentials for the tests and the stub server + serverCreds := loadServerCreds(t) + clientCreds := loadClientCreds(t) + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != io.EOF { + return err + } + return nil + }, + } + // Setup test statAuditLogger, gRPC test server with authzPolicy, unary + // and stream interceptors. + lb := &loggerBuilder{ + authzDecisionStat: map[bool]int{true: 0, false: 0}, + lastEvent: &audit.Event{}, + } + audit.RegisterLoggerBuilder(lb) + i, _ := authz.NewStatic(test.authzPolicy) + + s := grpc.NewServer(grpc.Creds(serverCreds), grpc.ChainUnaryInterceptor(i.UnaryInterceptor), grpc.ChainStreamInterceptor(i.StreamInterceptor)) + defer s.Stop() + ss.S = s + stubserver.StartTestService(t, ss) + + // Setup gRPC test client with certificates containing a SPIFFE Id. + cc, err := grpc.NewClient(ss.Address, grpc.WithTransportCredentials(clientCreds)) + if err != nil { + t.Fatalf("grpc.NewClient(%v) failed: %v", ss.Address, err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != test.wantUnaryCallCode { + t.Errorf("Unexpected UnaryCall fail: got %v want %v", err, test.wantUnaryCallCode) + } + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != test.wantUnaryCallCode { + t.Errorf("Unexpected UnaryCall fail: got %v want %v", err, test.wantUnaryCallCode) + } + stream, err := client.StreamingInputCall(ctx) + if err != nil { + t.Fatalf("StreamingInputCall failed: %v", err) + } + req := &testpb.StreamingInputCallRequest{ + Payload: &testpb.Payload{ + Body: []byte("hi"), + }, + } + if err := stream.Send(req); err != nil && err != io.EOF { + t.Fatalf("stream.Send failed: %v", err) + } + if _, err := stream.CloseAndRecv(); status.Code(err) != test.wantStreamingCallCode { + t.Errorf("Unexpected stream.CloseAndRecv fail: got %v want %v", err, test.wantStreamingCallCode) + } + + // Compare expected number of allows/denies with content of the internal + // map of statAuditLogger. + if diff := cmp.Diff(lb.authzDecisionStat, test.wantAuthzOutcomes); diff != "" { + t.Errorf("Authorization decisions do not match\ndiff (-got +want):\n%s", diff) + } + // Compare last event received by statAuditLogger with expected event. + if test.eventContent != nil { + if diff := cmp.Diff(lb.lastEvent, test.eventContent); diff != "" { + t.Errorf("Unexpected message\ndiff (-got +want):\n%s", diff) + } + } + }) + } +} + +// loadServerCreds constructs TLS containing server certs and CA +func loadServerCreds(t *testing.T) credentials.TransportCredentials { + t.Helper() + cert := loadKeys(t, "x509/server1_cert.pem", "x509/server1_key.pem") + certPool := loadCACerts(t, "x509/client_ca_cert.pem") + return credentials.NewTLS(&tls.Config{ + ClientAuth: tls.RequireAndVerifyClientCert, + Certificates: []tls.Certificate{cert}, + ClientCAs: certPool, + }) +} + +// loadClientCreds constructs TLS containing client certs and CA +func loadClientCreds(t *testing.T) credentials.TransportCredentials { + t.Helper() + cert := loadKeys(t, "x509/client_with_spiffe_cert.pem", "x509/client_with_spiffe_key.pem") + roots := loadCACerts(t, "x509/server_ca_cert.pem") + return credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: roots, + ServerName: "x.test.example.com", + }) + +} + +// loadKeys loads X509 key pair from the provided file paths. +// It is used for loading both client and server certificates for the test +func loadKeys(t *testing.T, certPath, key string) tls.Certificate { + t.Helper() + cert, err := tls.LoadX509KeyPair(testdata.Path(certPath), testdata.Path(key)) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(%q, %q) failed: %v", certPath, key, err) + } + return cert +} + +// loadCACerts loads CA certificates and constructs x509.CertPool +// It is used for loading both client and server CAs for the test +func loadCACerts(t *testing.T, certPath string) *x509.CertPool { + t.Helper() + ca, err := os.ReadFile(testdata.Path(certPath)) + if err != nil { + t.Fatalf("os.ReadFile(%q) failed: %v", certPath, err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(ca) { + t.Fatal("Failed to append certificates") + } + return roots +} diff --git a/authz/audit/stdout/stdout_logger.go b/authz/audit/stdout/stdout_logger.go new file mode 100644 index 0000000..f9bfc2d --- /dev/null +++ b/authz/audit/stdout/stdout_logger.go @@ -0,0 +1,110 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package stdout defines an stdout audit logger. +package stdout + +import ( + "encoding/json" + "log" + "os" + "time" + + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/grpclog" +) + +var grpcLogger = grpclog.Component("authz-audit") + +// Name is the string to identify this logger type in the registry +const Name = "stdout_logger" + +func init() { + audit.RegisterLoggerBuilder(&loggerBuilder{ + goLogger: log.New(os.Stdout, "", 0), + }) +} + +type event struct { + FullMethodName string `json:"rpc_method"` + Principal string `json:"principal"` + PolicyName string `json:"policy_name"` + MatchedRule string `json:"matched_rule"` + Authorized bool `json:"authorized"` + Timestamp string `json:"timestamp"` // Time when the audit event is logged via Log method +} + +// logger implements the audit.logger interface by logging to standard output. +type logger struct { + goLogger *log.Logger +} + +// Log marshals the audit.Event to json and prints it to standard output. +func (l *logger) Log(event *audit.Event) { + jsonContainer := map[string]any{ + "grpc_audit_log": convertEvent(event), + } + jsonBytes, err := json.Marshal(jsonContainer) + if err != nil { + grpcLogger.Errorf("failed to marshal AuditEvent data to JSON: %v", err) + return + } + l.goLogger.Println(string(jsonBytes)) +} + +// loggerConfig represents the configuration for the stdout logger. +// It is currently empty and implements the audit.Logger interface by embedding it. +type loggerConfig struct { + audit.LoggerConfig +} + +type loggerBuilder struct { + goLogger *log.Logger +} + +func (loggerBuilder) Name() string { + return Name +} + +// Build returns a new instance of the stdout logger. +// Passed in configuration is ignored as the stdout logger does not +// expect any configuration to be provided. +func (lb *loggerBuilder) Build(audit.LoggerConfig) audit.Logger { + return &logger{ + goLogger: lb.goLogger, + } +} + +// ParseLoggerConfig is a no-op since the stdout logger does not accept any configuration. +func (*loggerBuilder) ParseLoggerConfig(config json.RawMessage) (audit.LoggerConfig, error) { + if len(config) != 0 && string(config) != "{}" { + grpcLogger.Warningf("Stdout logger doesn't support custom configs. Ignoring:\n%s", string(config)) + } + return &loggerConfig{}, nil +} + +func convertEvent(auditEvent *audit.Event) *event { + return &event{ + FullMethodName: auditEvent.FullMethodName, + Principal: auditEvent.Principal, + PolicyName: auditEvent.PolicyName, + MatchedRule: auditEvent.MatchedRule, + Authorized: auditEvent.Authorized, + Timestamp: time.Now().Format(time.RFC3339Nano), + } +} diff --git a/authz/audit/stdout/stdout_logger_test.go b/authz/audit/stdout/stdout_logger_test.go new file mode 100644 index 0000000..02f9860 --- /dev/null +++ b/authz/audit/stdout/stdout_logger_test.go @@ -0,0 +1,140 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package stdout + +import ( + "bytes" + "encoding/json" + "log" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestStdoutLogger_Log(t *testing.T) { + tests := map[string]struct { + event *audit.Event + wantMessage string + wantErr string + }{ + "few fields": { + event: &audit.Event{PolicyName: "test policy", Principal: "test principal"}, + wantMessage: `{"fullMethodName":"","principal":"test principal","policyName":"test policy","matchedRule":"","authorized":false`, + }, + "all fields": { + event: &audit.Event{ + FullMethodName: "/helloworld.Greeter/SayHello", + Principal: "spiffe://example.org/ns/default/sa/default/backend", + PolicyName: "example-policy", + MatchedRule: "dev-access", + Authorized: true, + }, + wantMessage: `{"fullMethodName":"/helloworld.Greeter/SayHello",` + + `"principal":"spiffe://example.org/ns/default/sa/default/backend","policyName":"example-policy",` + + `"matchedRule":"dev-access","authorized":true`, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + before := time.Now().Unix() + var buf bytes.Buffer + builder := &loggerBuilder{goLogger: log.New(&buf, "", 0)} + auditLogger := builder.Build(nil) + + auditLogger.Log(test.event) + + var container map[string]any + if err := json.Unmarshal(buf.Bytes(), &container); err != nil { + t.Fatalf("Failed to unmarshal audit log event: %v", err) + } + innerEvent := extractEvent(container["grpc_audit_log"].(map[string]any)) + if innerEvent.Timestamp == "" { + t.Fatalf("Resulted event has no timestamp: %v", innerEvent) + } + after := time.Now().Unix() + innerEventUnixTime, err := time.Parse(time.RFC3339Nano, innerEvent.Timestamp) + if err != nil { + t.Fatalf("Failed to convert event timestamp into Unix time format: %v", err) + } + if before > innerEventUnixTime.Unix() || after < innerEventUnixTime.Unix() { + t.Errorf("The audit event timestamp is outside of the test interval: test start %v, event timestamp %v, test end %v", before, innerEventUnixTime.Unix(), after) + } + if diff := cmp.Diff(trimEvent(innerEvent), test.event); diff != "" { + t.Fatalf("Unexpected message\ndiff (-got +want):\n%s", diff) + } + }) + } +} + +func (s) TestStdoutLoggerBuilder_NilConfig(t *testing.T) { + builder := &loggerBuilder{ + goLogger: log.New(os.Stdout, "", log.LstdFlags), + } + config, err := builder.ParseLoggerConfig(nil) + if err != nil { + t.Fatalf("Failed to parse stdout logger configuration: %v", err) + } + if l := builder.Build(config); l == nil { + t.Fatal("Failed to build stdout audit logger") + } +} + +func (s) TestStdoutLoggerBuilder_Registration(t *testing.T) { + if audit.GetLoggerBuilder("stdout_logger") == nil { + t.Fatal("stdout logger is not registered") + } +} + +// extractEvent extracts an stdout.event from a map +// unmarshalled from a logged json message. +func extractEvent(container map[string]any) event { + return event{ + FullMethodName: container["rpc_method"].(string), + Principal: container["principal"].(string), + PolicyName: container["policy_name"].(string), + MatchedRule: container["matched_rule"].(string), + Authorized: container["authorized"].(bool), + Timestamp: container["timestamp"].(string), + } +} + +// trimEvent converts a logged stdout.event into an audit.Event +// by removing Timestamp field. It is used for comparing events during testing. +func trimEvent(testEvent event) *audit.Event { + return &audit.Event{ + FullMethodName: testEvent.FullMethodName, + Principal: testEvent.Principal, + PolicyName: testEvent.PolicyName, + MatchedRule: testEvent.MatchedRule, + Authorized: testEvent.Authorized, + } +} diff --git a/authz/grpc_authz_end2end_test.go b/authz/grpc_authz_end2end_test.go new file mode 100644 index 0000000..7791d1d --- /dev/null +++ b/authz/grpc_authz_end2end_test.go @@ -0,0 +1,708 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package authz_test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "io" + "os" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/authz" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +var authzTests = map[string]struct { + authzPolicy string + md metadata.MD + wantStatus *status.Status +}{ + "DeniesRPCMatchInDenyNoMatchInAllow": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_StreamingOutputCall", + "request": { + "paths": + [ + "/grpc.testing.TestService/StreamingOutputCall" + ] + } + } + ], + "deny_rules": + [ + { + "name": "deny_TestServiceCalls", + "request": { + "paths": + [ + "/grpc.testing.TestService/*" + ], + "headers": + [ + { + "key": "key-abc", + "values": + [ + "val-abc", + "val-def" + ] + } + ] + } + } + ] + }`, + md: metadata.Pairs("key-abc", "val-abc"), + wantStatus: status.New(codes.PermissionDenied, "authz: unauthorized RPC request rejected"), + }, + "DeniesRPCMatchInDenyAndAllow": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_all", + "request": { + "paths": + [ + "*" + ] + } + } + ], + "deny_rules": + [ + { + "name": "deny_all", + "request": { + "paths": + [ + "*" + ] + } + } + ] + }`, + wantStatus: status.New(codes.PermissionDenied, "authz: unauthorized RPC request rejected"), + }, + "AllowsRPCNoMatchInDenyMatchInAllow": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_all" + } + ], + "deny_rules": + [ + { + "name": "deny_TestServiceCalls", + "request": { + "paths": + [ + "/grpc.testing.TestService/UnaryCall", + "/grpc.testing.TestService/StreamingInputCall" + ], + "headers": + [ + { + "key": "key-abc", + "values": + [ + "val-abc", + "val-def" + ] + } + ] + } + } + ] + }`, + md: metadata.Pairs("key-xyz", "val-xyz"), + wantStatus: status.New(codes.OK, ""), + }, + "DeniesRPCNoMatchInDenyAndAllow": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_some_user", + "source": { + "principals": + [ + "some_user" + ] + } + } + ], + "deny_rules": + [ + { + "name": "deny_StreamingOutputCall", + "request": { + "paths": + [ + "/grpc.testing.TestService/StreamingOutputCall" + ] + } + } + ] + }`, + wantStatus: status.New(codes.PermissionDenied, "authz: unauthorized RPC request rejected"), + }, + "AllowsRPCEmptyDenyMatchInAllow": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_UnaryCall", + "request": + { + "paths": + [ + "/grpc.testing.TestService/UnaryCall" + ] + } + }, + { + "name": "allow_StreamingInputCall", + "request": + { + "paths": + [ + "/grpc.testing.TestService/StreamingInputCall" + ] + } + } + ] + }`, + wantStatus: status.New(codes.OK, ""), + }, + "DeniesRPCEmptyDenyNoMatchInAllow": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_StreamingOutputCall", + "request": + { + "paths": + [ + "/grpc.testing.TestService/StreamingOutputCall" + ] + } + } + ] + }`, + wantStatus: status.New(codes.PermissionDenied, "authz: unauthorized RPC request rejected"), + }, + "DeniesRPCRequestWithPrincipalsFieldOnUnauthenticatedConnection": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_authenticated", + "source": { + "principals": ["*", ""] + } + } + ] + }`, + wantStatus: status.New(codes.PermissionDenied, "authz: unauthorized RPC request rejected"), + }, + "DeniesRPCRequestNoMatchInAllowFailsPresenceMatch": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_TestServiceCalls", + "request": { + "paths": + [ + "/grpc.testing.TestService/*" + ], + "headers": + [ + { + "key": "key-abc", + "values": + [ + "*" + ] + } + ] + } + } + ] + }`, + md: metadata.Pairs("key-abc", ""), + wantStatus: status.New(codes.PermissionDenied, "authz: unauthorized RPC request rejected"), + }, +} + +func (s) TestStaticPolicyEnd2End(t *testing.T) { + for name, test := range authzTests { + t.Run(name, func(t *testing.T) { + // Start a gRPC server with gRPC authz unary and stream server interceptors. + i, _ := authz.NewStatic(test.authzPolicy) + + stub := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + StreamingInputCallF: func(stream testgrpc.TestService_StreamingInputCallServer) error { + for { + _, err := stream.Recv() + if err == io.EOF { + return stream.SendAndClose(&testpb.StreamingInputCallResponse{}) + } + if err != nil { + return err + } + } + }, + S: grpc.NewServer(grpc.ChainUnaryInterceptor(i.UnaryInterceptor), grpc.ChainStreamInterceptor(i.StreamInterceptor)), + } + stubserver.StartTestService(t, stub) + defer stub.Stop() + + // Establish a connection to the server. + cc, err := grpc.NewClient(stub.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%v) failed: %v", stub.Address, err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, test.md) + + // Verifying authorization decision for Unary RPC. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != test.wantStatus.Code() || got.Message() != test.wantStatus.Message() { + t.Fatalf("[UnaryCall] error want:{%v} got:{%v}", test.wantStatus.Err(), got.Err()) + } + + // Verifying authorization decision for Streaming RPC. + stream, err := client.StreamingInputCall(ctx) + if err != nil { + t.Fatalf("failed StreamingInputCall err: %v", err) + } + req := &testpb.StreamingInputCallRequest{ + Payload: &testpb.Payload{ + Body: []byte("hi"), + }, + } + if err := stream.Send(req); err != nil && err != io.EOF { + t.Fatalf("failed stream.Send err: %v", err) + } + _, err = stream.CloseAndRecv() + if got := status.Convert(err); got.Code() != test.wantStatus.Code() || got.Message() != test.wantStatus.Message() { + t.Fatalf("[StreamingCall] error want:{%v} got:{%v}", test.wantStatus.Err(), got.Err()) + } + }) + } +} + +func (s) TestAllowsRPCRequestWithPrincipalsFieldOnTLSAuthenticatedConnection(t *testing.T) { + authzPolicy := `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_authenticated", + "source": { + "principals": ["*", ""] + } + } + ] + }` + // Start a gRPC server with gRPC authz unary server interceptor. + i, _ := authz.NewStatic(authzPolicy) + creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("failed to generate credentials: %v", err) + } + + stub := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + S: grpc.NewServer(grpc.Creds(creds), grpc.ChainUnaryInterceptor(i.UnaryInterceptor)), + } + stubserver.StartTestService(t, stub) + defer stub.S.Stop() + + // Establish a connection to the server. + creds, err = credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") + if err != nil { + t.Fatalf("failed to load credentials: %v", err) + } + cc, err := grpc.NewClient(stub.Address, grpc.WithTransportCredentials(creds)) + if err != nil { + t.Fatalf("grpc.NewClient(%v) failed: %v", stub.Address, err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Verifying authorization decision. + if _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("client.UnaryCall(_, _) = %v; want nil", err) + } +} + +func (s) TestAllowsRPCRequestWithPrincipalsFieldOnMTLSAuthenticatedConnection(t *testing.T) { + authzPolicy := `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_authenticated", + "source": { + "principals": ["*", ""] + } + } + ] + }` + // Start a gRPC server with gRPC authz unary server interceptor. + i, _ := authz.NewStatic(authzPolicy) + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(x509/server1_cert.pem, x509/server1_key.pem) failed: %v", err) + } + ca, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem")) + if err != nil { + t.Fatalf("os.ReadFile(x509/client_ca_cert.pem) failed: %v", err) + } + certPool := x509.NewCertPool() + if !certPool.AppendCertsFromPEM(ca) { + t.Fatal("failed to append certificates") + } + creds := credentials.NewTLS(&tls.Config{ + ClientAuth: tls.RequireAndVerifyClientCert, + Certificates: []tls.Certificate{cert}, + ClientCAs: certPool, + }) + stub := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + S: grpc.NewServer(grpc.Creds(creds), grpc.ChainUnaryInterceptor(i.UnaryInterceptor)), + } + stubserver.StartTestService(t, stub) + defer stub.Stop() + + // Establish a connection to the server. + cert, err = tls.LoadX509KeyPair(testdata.Path("x509/client1_cert.pem"), testdata.Path("x509/client1_key.pem")) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(x509/client1_cert.pem, x509/client1_key.pem) failed: %v", err) + } + ca, err = os.ReadFile(testdata.Path("x509/server_ca_cert.pem")) + if err != nil { + t.Fatalf("os.ReadFile(x509/server_ca_cert.pem) failed: %v", err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(ca) { + t.Fatal("failed to append certificates") + } + creds = credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: roots, + ServerName: "x.test.example.com", + }) + cc, err := grpc.NewClient(stub.Address, grpc.WithTransportCredentials(creds)) + if err != nil { + t.Fatalf("grpc.NewClient(%v) failed: %v", stub.Address, err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Verifying authorization decision. + if _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("client.UnaryCall(_, _) = %v; want nil", err) + } +} + +func (s) TestFileWatcherEnd2End(t *testing.T) { + for name, test := range authzTests { + t.Run(name, func(t *testing.T) { + file := createTmpPolicyFile(t, name, []byte(test.authzPolicy)) + i, _ := authz.NewFileWatcher(file, 1*time.Second) + defer i.Close() + + stub := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + StreamingInputCallF: func(stream testgrpc.TestService_StreamingInputCallServer) error { + for { + _, err := stream.Recv() + if err == io.EOF { + return stream.SendAndClose(&testpb.StreamingInputCallResponse{}) + } + if err != nil { + return err + } + } + }, + // Start a gRPC server with gRPC authz unary and stream server interceptors. + S: grpc.NewServer(grpc.ChainUnaryInterceptor(i.UnaryInterceptor), grpc.ChainStreamInterceptor(i.StreamInterceptor)), + } + stubserver.StartTestService(t, stub) + defer stub.Stop() + + // Establish a connection to the server. + cc, err := grpc.NewClient(stub.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%v) failed: %v", stub.Address, err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, test.md) + + // Verifying authorization decision for Unary RPC. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != test.wantStatus.Code() || got.Message() != test.wantStatus.Message() { + t.Fatalf("[UnaryCall] error want:{%v} got:{%v}", test.wantStatus.Err(), got.Err()) + } + + // Verifying authorization decision for Streaming RPC. + stream, err := client.StreamingInputCall(ctx) + if err != nil { + t.Fatalf("failed StreamingInputCall : %v", err) + } + req := &testpb.StreamingInputCallRequest{ + Payload: &testpb.Payload{ + Body: []byte("hi"), + }, + } + if err := stream.Send(req); err != nil && err != io.EOF { + t.Fatalf("failed stream.Send : %v", err) + } + _, err = stream.CloseAndRecv() + if got := status.Convert(err); got.Code() != test.wantStatus.Code() || got.Message() != test.wantStatus.Message() { + t.Fatalf("[StreamingCall] error want:{%v} got:{%v}", test.wantStatus.Err(), got.Err()) + } + }) + } +} + +func retryUntil(ctx context.Context, tsc testgrpc.TestServiceClient, want *status.Status) (lastErr error) { + for ctx.Err() == nil { + _, lastErr = tsc.UnaryCall(ctx, &testpb.SimpleRequest{}) + if s := status.Convert(lastErr); s.Code() == want.Code() && s.Message() == want.Message() { + return nil + } + time.Sleep(20 * time.Millisecond) + } + return lastErr +} + +func (s) TestFileWatcher_ValidPolicyRefresh(t *testing.T) { + valid1 := authzTests["DeniesRPCMatchInDenyAndAllow"] + file := createTmpPolicyFile(t, "valid_policy_refresh", []byte(valid1.authzPolicy)) + i, _ := authz.NewFileWatcher(file, 100*time.Millisecond) + defer i.Close() + + stub := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + // Start a gRPC server with gRPC authz unary server interceptor. + S: grpc.NewServer(grpc.ChainUnaryInterceptor(i.UnaryInterceptor)), + } + stubserver.StartTestService(t, stub) + defer stub.Stop() + + // Establish a connection to the server. + cc, err := grpc.NewClient(stub.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%v) failed: %v", stub.Address, err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Verifying authorization decision. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != valid1.wantStatus.Code() || got.Message() != valid1.wantStatus.Message() { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got.Err(), valid1.wantStatus.Err()) + } + + // Rewrite the file with a different valid authorization policy. + valid2 := authzTests["AllowsRPCEmptyDenyMatchInAllow"] + if err := os.WriteFile(file, []byte(valid2.authzPolicy), os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(%q) failed: %v", file, err) + } + + // Verifying authorization decision. + if got := retryUntil(ctx, client, valid2.wantStatus); got != nil { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got, valid2.wantStatus.Err()) + } +} + +func (s) TestFileWatcher_InvalidPolicySkipReload(t *testing.T) { + valid := authzTests["DeniesRPCMatchInDenyAndAllow"] + file := createTmpPolicyFile(t, "invalid_policy_skip_reload", []byte(valid.authzPolicy)) + i, _ := authz.NewFileWatcher(file, 20*time.Millisecond) + defer i.Close() + + stub := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + // Start a gRPC server with gRPC authz unary server interceptors. + S: grpc.NewServer(grpc.ChainUnaryInterceptor(i.UnaryInterceptor)), + } + stubserver.StartTestService(t, stub) + defer stub.Stop() + + // Establish a connection to the server. + cc, err := grpc.NewClient(stub.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%v) failed: %v", stub.Address, err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Verifying authorization decision. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != valid.wantStatus.Code() || got.Message() != valid.wantStatus.Message() { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got.Err(), valid.wantStatus.Err()) + } + + // Skips the invalid policy update, and continues to use the valid policy. + if err := os.WriteFile(file, []byte("{}"), os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(%q) failed: %v", file, err) + } + + // Wait 40 ms for background go routine to read updated files. + time.Sleep(40 * time.Millisecond) + + // Verifying authorization decision. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != valid.wantStatus.Code() || got.Message() != valid.wantStatus.Message() { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got.Err(), valid.wantStatus.Err()) + } +} + +func (s) TestFileWatcher_RecoversFromReloadFailure(t *testing.T) { + valid1 := authzTests["DeniesRPCMatchInDenyAndAllow"] + file := createTmpPolicyFile(t, "recovers_from_reload_failure", []byte(valid1.authzPolicy)) + i, _ := authz.NewFileWatcher(file, 100*time.Millisecond) + defer i.Close() + + stub := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + S: grpc.NewServer(grpc.ChainUnaryInterceptor(i.UnaryInterceptor)), + } + stubserver.StartTestService(t, stub) + defer stub.Stop() + + // Establish a connection to the server. + cc, err := grpc.NewClient(stub.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%v) failed: %v", stub.Address, err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Verifying authorization decision. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != valid1.wantStatus.Code() || got.Message() != valid1.wantStatus.Message() { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got.Err(), valid1.wantStatus.Err()) + } + + // Skips the invalid policy update, and continues to use the valid policy. + if err := os.WriteFile(file, []byte("{}"), os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(%q) failed: %v", file, err) + } + + // Wait 120 ms for background go routine to read updated files. + time.Sleep(120 * time.Millisecond) + + // Verifying authorization decision. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != valid1.wantStatus.Code() || got.Message() != valid1.wantStatus.Message() { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got.Err(), valid1.wantStatus.Err()) + } + + // Rewrite the file with a different valid authorization policy. + valid2 := authzTests["AllowsRPCEmptyDenyMatchInAllow"] + if err := os.WriteFile(file, []byte(valid2.authzPolicy), os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(%q) failed: %v", file, err) + } + + // Verifying authorization decision. + if got := retryUntil(ctx, client, valid2.wantStatus); got != nil { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got, valid2.wantStatus.Err()) + } +} diff --git a/authz/grpc_authz_server_interceptors.go b/authz/grpc_authz_server_interceptors.go new file mode 100644 index 0000000..8ab2068 --- /dev/null +++ b/authz/grpc_authz_server_interceptors.go @@ -0,0 +1,211 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package authz + +import ( + "bytes" + "context" + "fmt" + "os" + "sync/atomic" + "time" + "unsafe" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/xds/rbac" + "google.golang.org/grpc/status" +) + +var logger = grpclog.Component("authz") + +// StaticInterceptor contains engines used to make authorization decisions. It +// either contains two engines deny engine followed by an allow engine or only +// one allow engine. +type StaticInterceptor struct { + engines rbac.ChainEngine +} + +// NewStatic returns a new StaticInterceptor from a static authorization policy +// JSON string. +func NewStatic(authzPolicy string) (*StaticInterceptor, error) { + rbacs, policyName, err := translatePolicy(authzPolicy) + if err != nil { + return nil, err + } + chainEngine, err := rbac.NewChainEngine(rbacs, policyName) + if err != nil { + return nil, err + } + return &StaticInterceptor{*chainEngine}, nil +} + +// UnaryInterceptor intercepts incoming Unary RPC requests. +// Only authorized requests are allowed to pass. Otherwise, an unauthorized +// error is returned to the client. +func (i *StaticInterceptor) UnaryInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + err := i.engines.IsAuthorized(ctx) + if err != nil { + if status.Code(err) == codes.PermissionDenied { + if logger.V(2) { + logger.Infof("unauthorized RPC request rejected: %v", err) + } + return nil, status.Errorf(codes.PermissionDenied, "authz: unauthorized RPC request rejected") + } + return nil, err + } + return handler(ctx, req) +} + +// StreamInterceptor intercepts incoming Stream RPC requests. +// Only authorized requests are allowed to pass. Otherwise, an unauthorized +// error is returned to the client. +func (i *StaticInterceptor) StreamInterceptor(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + err := i.engines.IsAuthorized(ss.Context()) + if err != nil { + if status.Code(err) == codes.PermissionDenied { + if logger.V(2) { + logger.Infof("unauthorized RPC request rejected: %v", err) + } + return status.Errorf(codes.PermissionDenied, "authz: unauthorized RPC request rejected") + } + return err + } + return handler(srv, ss) +} + +// FileWatcherInterceptor contains details used to make authorization decisions +// by watching a file path that contains authorization policy in JSON format. +type FileWatcherInterceptor struct { + options FileWatcherOptions + internalInterceptor unsafe.Pointer // *StaticInterceptor + policyContents []byte + cancel context.CancelFunc +} + +// FileWatcherOptions contains configuration options for the +// FileWatcherInterceptor. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +type FileWatcherOptions struct { + // PolicyFile contains a JSON string of the authorization policy. + PolicyFile string + // RefreshDuration is the delay between policy refreshes. + RefreshDuration time.Duration + // OnPolicyUpdate is a callback to be invoked when a policy is + // loaded/updated. The loaded policy string is passed as an argument. + // + // The callback is executed synchronously, so should complete quickly or + // risk blocking future updates. + OnPolicyUpdate func(string) +} + +// NewFileWatcher returns a new FileWatcherInterceptor from a policy file +// that contains JSON string of authorization policy and a refresh duration to +// specify the amount of time between policy refreshes. +func NewFileWatcher(file string, duration time.Duration) (*FileWatcherInterceptor, error) { + return NewFileWatcherWithOptions(FileWatcherOptions{PolicyFile: file, RefreshDuration: duration, OnPolicyUpdate: nil}) +} + +// NewFileWatcherWithOptions returns a new FileWatcherInterceptor from a set of +// options. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func NewFileWatcherWithOptions(options FileWatcherOptions) (*FileWatcherInterceptor, error) { + if options.PolicyFile == "" { + return nil, fmt.Errorf("authz: authorization policy file path is empty") + } + if options.RefreshDuration <= time.Duration(0) { + return nil, fmt.Errorf("authz: requires refresh interval(%v) greater than 0s", options.RefreshDuration) + } + i := &FileWatcherInterceptor{options: options} + if err := i.updateInternalInterceptor(); err != nil { + return nil, err + } + ctx, cancel := context.WithCancel(context.Background()) + i.cancel = cancel + // Create a background go routine for policy refresh. + go i.run(ctx) + return i, nil +} + +func (i *FileWatcherInterceptor) run(ctx context.Context) { + ticker := time.NewTicker(i.options.RefreshDuration) + for { + if err := i.updateInternalInterceptor(); err != nil { + logger.Warningf("authorization policy reload status err: %v", err) + } + select { + case <-ctx.Done(): + ticker.Stop() + return + case <-ticker.C: + } + } +} + +// updateInternalInterceptor checks if the policy file that is watching has changed, +// and if so, updates the internalInterceptor with the policy. Unlike the +// constructor, if there is an error in reading the file or parsing the policy, the +// previous internalInterceptors will not be replaced. +func (i *FileWatcherInterceptor) updateInternalInterceptor() error { + policyContents, err := os.ReadFile(i.options.PolicyFile) + if err != nil { + return fmt.Errorf("policyFile(%s) read failed: %v", i.options.PolicyFile, err) + } + if bytes.Equal(i.policyContents, policyContents) { + return nil + } + i.policyContents = policyContents + policyContentsString := string(policyContents) + interceptor, err := NewStatic(policyContentsString) + if err != nil { + return err + } + atomic.StorePointer(&i.internalInterceptor, unsafe.Pointer(interceptor)) + logger.Infof("authorization policy reload status: successfully loaded new policy %v", policyContentsString) + if i.options.OnPolicyUpdate != nil { + i.options.OnPolicyUpdate(policyContentsString) + } + return nil +} + +// Close cleans up resources allocated by the interceptor. +func (i *FileWatcherInterceptor) Close() { + i.cancel() +} + +// UnaryInterceptor intercepts incoming Unary RPC requests. +// Only authorized requests are allowed to pass. Otherwise, an unauthorized +// error is returned to the client. +func (i *FileWatcherInterceptor) UnaryInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + return ((*StaticInterceptor)(atomic.LoadPointer(&i.internalInterceptor))).UnaryInterceptor(ctx, req, info, handler) +} + +// StreamInterceptor intercepts incoming Stream RPC requests. +// Only authorized requests are allowed to pass. Otherwise, an unauthorized +// error is returned to the client. +func (i *FileWatcherInterceptor) StreamInterceptor(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + return ((*StaticInterceptor)(atomic.LoadPointer(&i.internalInterceptor))).StreamInterceptor(srv, ss, info, handler) +} diff --git a/authz/grpc_authz_server_interceptors_test.go b/authz/grpc_authz_server_interceptors_test.go new file mode 100644 index 0000000..ee5a215 --- /dev/null +++ b/authz/grpc_authz_server_interceptors_test.go @@ -0,0 +1,174 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package authz_test + +import ( + "context" + "fmt" + "os" + "path" + "testing" + "time" + + "google.golang.org/grpc/authz" +) + +const defaultTestTimeout = 10 * time.Second + +func createTmpPolicyFile(t *testing.T, dirSuffix string, policy []byte) string { + t.Helper() + + // Create a temp directory. Passing an empty string for the first argument + // uses the system temp directory. + dir, err := os.MkdirTemp("", dirSuffix) + if err != nil { + t.Fatalf("os.MkdirTemp() failed: %v", err) + } + t.Logf("Using tmpdir: %s", dir) + // Write policy into file. + filename := path.Join(dir, "policy.json") + if err := os.WriteFile(filename, policy, os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(%q) failed: %v", filename, err) + } + t.Logf("Wrote policy %s to file at %s", string(policy), filename) + return filename +} + +func (s) TestNewStatic(t *testing.T) { + tests := map[string]struct { + authzPolicy string + wantErr error + }{ + "InvalidPolicyFailsToCreateInterceptor": { + authzPolicy: `{}`, + wantErr: fmt.Errorf(`"name" is not present`), + }, + "ValidPolicyCreatesInterceptor": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_all" + } + ] + }`, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + if _, err := authz.NewStatic(test.authzPolicy); fmt.Sprint(err) != fmt.Sprint(test.wantErr) { + t.Fatalf("NewStatic(%v) returned err: %v, want err: %v", test.authzPolicy, err, test.wantErr) + } + }) + } +} + +func (s) TestNewFileWatcher(t *testing.T) { + tests := map[string]struct { + authzPolicy string + refreshDuration time.Duration + wantErr error + }{ + "InvalidRefreshDurationFailsToCreateInterceptor": { + refreshDuration: time.Duration(0), + wantErr: fmt.Errorf("authz: requires refresh interval(0s) greater than 0s"), + }, + "InvalidPolicyFailsToCreateInterceptor": { + authzPolicy: `{}`, + refreshDuration: time.Duration(1), + wantErr: fmt.Errorf(`"name" is not present`), + }, + "ValidPolicyCreatesInterceptor": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_all" + } + ] + }`, + refreshDuration: time.Duration(1), + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + file := createTmpPolicyFile(t, name, []byte(test.authzPolicy)) + i, err := authz.NewFileWatcher(file, test.refreshDuration) + if fmt.Sprint(err) != fmt.Sprint(test.wantErr) { + t.Fatalf("NewFileWatcher(%v) returned err: %v, want err: %v", test.authzPolicy, err, test.wantErr) + } + if i != nil { + i.Close() + } + }) + } +} + +func (s) TestOnPolicyUpdate(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + updates := make(chan string, 1) + onPolicyUpdate := func(s string) { + updates <- s + } + + content := `{"name": "foo1", "allow_rules":[{"name":"bar"}]}` + file := createTmpPolicyFile(t, "onpolicyupdate", []byte(content)) + opts := authz.FileWatcherOptions{PolicyFile: file, RefreshDuration: 50 * time.Millisecond, OnPolicyUpdate: onPolicyUpdate} + i, err := authz.NewFileWatcherWithOptions(opts) + if err != nil { + t.Fatalf("NewFileWatcherWithOptions(%v) returned err: %v", opts, err) + } + defer i.Close() + + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for policy update") + case update := <-updates: + if update != content { + t.Fatalf("Unexpected contents on first load of policy file: got=%v, want=%v", update, content) + } + } + + // Tweak the file, expect an update. + content = `{"name": "foo2", "allow_rules":[{"name":"bar"}]}` + if err := os.WriteFile(file, []byte(content), os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(%q) failed: %v", file, err) + } + + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for policy update") + case update := <-updates: + if update != content { + t.Fatalf("Unexpected contents after policy file changed: got=%v, want=%v", update, content) + } + } + + sCtx, sCancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer sCancel() + select { + case <-updates: + t.Fatal("OnPolicyUpdate callback invoked more times than expected") + case <-sCtx.Done(): + } +} diff --git a/authz/rbac_translator.go b/authz/rbac_translator.go new file mode 100644 index 0000000..730ec9d --- /dev/null +++ b/authz/rbac_translator.go @@ -0,0 +1,398 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package authz exposes methods to manage authorization within gRPC. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed +// in a later release. +package authz + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" +) + +// This is used when converting a custom config from raw JSON to a TypedStruct +// The TypeURL of the TypeStruct will be "grpc.authz.audit_logging/" +const typeURLPrefix = "grpc.authz.audit_logging/" + +type header struct { + Key string + Values []string +} + +type peer struct { + Principals []string +} + +type request struct { + Paths []string + Headers []header +} + +type rule struct { + Name string + Source peer + Request request +} + +type auditLogger struct { + Name string `json:"name"` + Config *structpb.Struct `json:"config"` + IsOptional bool `json:"is_optional"` +} + +type auditLoggingOptions struct { + AuditCondition string `json:"audit_condition"` + AuditLoggers []*auditLogger `json:"audit_loggers"` +} + +// Represents the SDK authorization policy provided by user. +type authorizationPolicy struct { + Name string + DenyRules []rule `json:"deny_rules"` + AllowRules []rule `json:"allow_rules"` + AuditLoggingOptions auditLoggingOptions `json:"audit_logging_options"` +} + +func principalOr(principals []*v3rbacpb.Principal) *v3rbacpb.Principal { + return &v3rbacpb.Principal{ + Identifier: &v3rbacpb.Principal_OrIds{ + OrIds: &v3rbacpb.Principal_Set{ + Ids: principals, + }, + }, + } +} + +func permissionOr(permission []*v3rbacpb.Permission) *v3rbacpb.Permission { + return &v3rbacpb.Permission{ + Rule: &v3rbacpb.Permission_OrRules{ + OrRules: &v3rbacpb.Permission_Set{ + Rules: permission, + }, + }, + } +} + +func permissionAnd(permission []*v3rbacpb.Permission) *v3rbacpb.Permission { + return &v3rbacpb.Permission{ + Rule: &v3rbacpb.Permission_AndRules{ + AndRules: &v3rbacpb.Permission_Set{ + Rules: permission, + }, + }, + } +} + +func getStringMatcher(value string) *v3matcherpb.StringMatcher { + switch { + case value == "*": + return &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{ + SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + } + case strings.HasSuffix(value, "*"): + prefix := strings.TrimSuffix(value, "*") + return &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: prefix}, + } + case strings.HasPrefix(value, "*"): + suffix := strings.TrimPrefix(value, "*") + return &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: suffix}, + } + default: + return &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: value}, + } + } +} + +func getHeaderMatcher(key, value string) *v3routepb.HeaderMatcher { + switch { + case value == "*": + return &v3routepb.HeaderMatcher{ + Name: key, + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{ + SafeRegexMatch: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + } + case strings.HasSuffix(value, "*"): + prefix := strings.TrimSuffix(value, "*") + return &v3routepb.HeaderMatcher{ + Name: key, + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: prefix}, + } + case strings.HasPrefix(value, "*"): + suffix := strings.TrimPrefix(value, "*") + return &v3routepb.HeaderMatcher{ + Name: key, + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SuffixMatch{SuffixMatch: suffix}, + } + default: + return &v3routepb.HeaderMatcher{ + Name: key, + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: value}, + } + } +} + +func parsePrincipalNames(principalNames []string) []*v3rbacpb.Principal { + ps := make([]*v3rbacpb.Principal, 0, len(principalNames)) + for _, principalName := range principalNames { + newPrincipalName := &v3rbacpb.Principal{ + Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{ + PrincipalName: getStringMatcher(principalName), + }, + }} + ps = append(ps, newPrincipalName) + } + return ps +} + +func parsePeer(source peer) *v3rbacpb.Principal { + if len(source.Principals) == 0 { + return &v3rbacpb.Principal{ + Identifier: &v3rbacpb.Principal_Any{ + Any: true, + }, + } + } + return principalOr(parsePrincipalNames(source.Principals)) +} + +func parsePaths(paths []string) []*v3rbacpb.Permission { + ps := make([]*v3rbacpb.Permission, 0, len(paths)) + for _, path := range paths { + newPath := &v3rbacpb.Permission{ + Rule: &v3rbacpb.Permission_UrlPath{ + UrlPath: &v3matcherpb.PathMatcher{ + Rule: &v3matcherpb.PathMatcher_Path{Path: getStringMatcher(path)}}}} + ps = append(ps, newPath) + } + return ps +} + +func parseHeaderValues(key string, values []string) []*v3rbacpb.Permission { + vs := make([]*v3rbacpb.Permission, 0, len(values)) + for _, value := range values { + newHeader := &v3rbacpb.Permission{ + Rule: &v3rbacpb.Permission_Header{ + Header: getHeaderMatcher(key, value)}} + vs = append(vs, newHeader) + } + return vs +} + +var unsupportedHeaders = map[string]bool{ + "host": true, + "connection": true, + "keep-alive": true, + "proxy-authenticate": true, + "proxy-authorization": true, + "te": true, + "trailer": true, + "transfer-encoding": true, + "upgrade": true, +} + +func unsupportedHeader(key string) bool { + return key[0] == ':' || strings.HasPrefix(key, "grpc-") || unsupportedHeaders[key] +} + +func parseHeaders(headers []header) ([]*v3rbacpb.Permission, error) { + hs := make([]*v3rbacpb.Permission, 0, len(headers)) + for i, header := range headers { + if header.Key == "" { + return nil, fmt.Errorf(`"headers" %d: "key" is not present`, i) + } + header.Key = strings.ToLower(header.Key) + if unsupportedHeader(header.Key) { + return nil, fmt.Errorf(`"headers" %d: unsupported "key" %s`, i, header.Key) + } + if len(header.Values) == 0 { + return nil, fmt.Errorf(`"headers" %d: "values" is not present`, i) + } + values := parseHeaderValues(header.Key, header.Values) + hs = append(hs, permissionOr(values)) + } + return hs, nil +} + +func parseRequest(request request) (*v3rbacpb.Permission, error) { + var and []*v3rbacpb.Permission + if len(request.Paths) > 0 { + and = append(and, permissionOr(parsePaths(request.Paths))) + } + if len(request.Headers) > 0 { + headers, err := parseHeaders(request.Headers) + if err != nil { + return nil, err + } + and = append(and, permissionAnd(headers)) + } + if len(and) > 0 { + return permissionAnd(and), nil + } + return &v3rbacpb.Permission{ + Rule: &v3rbacpb.Permission_Any{ + Any: true, + }, + }, nil +} + +func parseRules(rules []rule, prefixName string) (map[string]*v3rbacpb.Policy, error) { + policies := make(map[string]*v3rbacpb.Policy) + for i, rule := range rules { + if rule.Name == "" { + return policies, fmt.Errorf(`%d: "name" is not present`, i) + } + permission, err := parseRequest(rule.Request) + if err != nil { + return nil, fmt.Errorf("%d: %v", i, err) + } + policyName := prefixName + "_" + rule.Name + policies[policyName] = &v3rbacpb.Policy{ + Principals: []*v3rbacpb.Principal{parsePeer(rule.Source)}, + Permissions: []*v3rbacpb.Permission{permission}, + } + } + return policies, nil +} + +// Parse auditLoggingOptions to the associated RBAC protos. The single +// auditLoggingOptions results in two different parsed protos, one for the allow +// policy and one for the deny policy +func (options *auditLoggingOptions) toProtos() (allow *v3rbacpb.RBAC_AuditLoggingOptions, deny *v3rbacpb.RBAC_AuditLoggingOptions, err error) { + allow = &v3rbacpb.RBAC_AuditLoggingOptions{} + deny = &v3rbacpb.RBAC_AuditLoggingOptions{} + + if options.AuditCondition != "" { + rbacCondition, ok := v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition_value[options.AuditCondition] + if !ok { + return nil, nil, fmt.Errorf("failed to parse AuditCondition %v. Allowed values {NONE, ON_DENY, ON_ALLOW, ON_DENY_AND_ALLOW}", options.AuditCondition) + } + allow.AuditCondition = v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition(rbacCondition) + deny.AuditCondition = toDenyCondition(v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition(rbacCondition)) + } + + for i, config := range options.AuditLoggers { + if config.Name == "" { + return nil, nil, fmt.Errorf("missing required field: name in audit_logging_options.audit_loggers[%v]", i) + } + if config.Config == nil { + config.Config = &structpb.Struct{} + } + typedStruct := &v1xdsudpatypepb.TypedStruct{ + TypeUrl: typeURLPrefix + config.Name, + Value: config.Config, + } + customConfig, err := anypb.New(typedStruct) + if err != nil { + return nil, nil, fmt.Errorf("error parsing custom audit logger config: %v", err) + } + + logger := &v3corepb.TypedExtensionConfig{Name: config.Name, TypedConfig: customConfig} + rbacConfig := v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + IsOptional: config.IsOptional, + AuditLogger: logger, + } + allow.LoggerConfigs = append(allow.LoggerConfigs, &rbacConfig) + deny.LoggerConfigs = append(deny.LoggerConfigs, &rbacConfig) + } + + return allow, deny, nil +} + +// Maps the AuditCondition coming from AuditLoggingOptions to the proper +// condition for the deny policy RBAC proto +func toDenyCondition(condition v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition) v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition { + // Mapping the overall policy AuditCondition to what it must be for the Deny and Allow RBAC + // See gRPC A59 for details - https://github.com/grpc/proposal/pull/346/files + // |Authorization Policy |DENY RBAC |ALLOW RBAC | + // |----------------------|-------------------|---------------------| + // |NONE |NONE |NONE | + // |ON_DENY |ON_DENY |ON_DENY | + // |ON_ALLOW |NONE |ON_ALLOW | + // |ON_DENY_AND_ALLOW |ON_DENY |ON_DENY_AND_ALLOW | + switch condition { + case v3rbacpb.RBAC_AuditLoggingOptions_NONE: + return v3rbacpb.RBAC_AuditLoggingOptions_NONE + case v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY: + return v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY + case v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW: + return v3rbacpb.RBAC_AuditLoggingOptions_NONE + case v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY_AND_ALLOW: + return v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY + default: + return v3rbacpb.RBAC_AuditLoggingOptions_NONE + } +} + +// translatePolicy translates SDK authorization policy in JSON format to two +// Envoy RBAC polices (deny followed by allow policy) or only one Envoy RBAC +// allow policy. Also returns the overall policy name. If the input policy +// cannot be parsed or is invalid, an error will be returned. +func translatePolicy(policyStr string) ([]*v3rbacpb.RBAC, string, error) { + policy := &authorizationPolicy{} + d := json.NewDecoder(bytes.NewReader([]byte(policyStr))) + d.DisallowUnknownFields() + if err := d.Decode(policy); err != nil { + return nil, "", fmt.Errorf("failed to unmarshal policy: %v", err) + } + if policy.Name == "" { + return nil, "", fmt.Errorf(`"name" is not present`) + } + if len(policy.AllowRules) == 0 { + return nil, "", fmt.Errorf(`"allow_rules" is not present`) + } + allowLogger, denyLogger, err := policy.AuditLoggingOptions.toProtos() + if err != nil { + return nil, "", err + } + rbacs := make([]*v3rbacpb.RBAC, 0, 2) + if len(policy.DenyRules) > 0 { + denyPolicies, err := parseRules(policy.DenyRules, policy.Name) + if err != nil { + return nil, "", fmt.Errorf(`"deny_rules" %v`, err) + } + denyRBAC := &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_DENY, + Policies: denyPolicies, + AuditLoggingOptions: denyLogger, + } + rbacs = append(rbacs, denyRBAC) + } + allowPolicies, err := parseRules(policy.AllowRules, policy.Name) + if err != nil { + return nil, "", fmt.Errorf(`"allow_rules" %v`, err) + } + allowRBAC := &v3rbacpb.RBAC{Action: v3rbacpb.RBAC_ALLOW, Policies: allowPolicies, AuditLoggingOptions: allowLogger} + return append(rbacs, allowRBAC), policy.Name, nil +} diff --git a/authz/rbac_translator_test.go b/authz/rbac_translator_test.go new file mode 100644 index 0000000..0de0d50 --- /dev/null +++ b/authz/rbac_translator_test.go @@ -0,0 +1,1052 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package authz + +import ( + "strings" + "testing" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" +) + +func TestTranslatePolicy(t *testing.T) { + tests := map[string]struct { + authzPolicy string + wantErr string + wantPolicies []*v3rbacpb.RBAC + wantPolicyName string + }{ + "valid policy": { + authzPolicy: `{ + "name": "authz", + "deny_rules": [ + { + "name": "deny_policy_1", + "source": { + "principals":[ + "spiffe://foo.abc", + "spiffe://bar*", + "*baz", + "spiffe://abc.*.com" + ] + } + }], + "allow_rules": [ + { + "name": "allow_policy_1", + "source": { + "principals":["*"] + }, + "request": { + "paths": ["path-foo*"] + } + }, + { + "name": "allow_policy_2", + "request": { + "paths": [ + "path-bar", + "*baz" + ], + "headers": [ + { + "key": "key-1", + "values": ["foo", "*bar"] + }, + { + "key": "key-2", + "values": ["baz*"] + } + ] + } + }] + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "authz_deny_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://foo.abc"}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "spiffe://bar"}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "baz"}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://abc.*.com"}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{}, + }, + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_AndRules{AndRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_OrRules{OrRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{ + UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "path-foo"}, + }}}, + }}, + }, + }}}, + }, + }}}, + }, + }, + "authz_allow_policy_2": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_AndRules{AndRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_OrRules{OrRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{ + UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "path-bar"}, + }}}, + }}, + {Rule: &v3rbacpb.Permission_UrlPath{ + UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "baz"}, + }}}, + }}, + }, + }}}, + {Rule: &v3rbacpb.Permission_AndRules{AndRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_OrRules{OrRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{ + Header: &v3routepb.HeaderMatcher{ + Name: "key-1", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "foo"}, + }, + }}, + {Rule: &v3rbacpb.Permission_Header{ + Header: &v3routepb.HeaderMatcher{ + Name: "key-1", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SuffixMatch{SuffixMatch: "bar"}, + }, + }}, + }, + }}}, + {Rule: &v3rbacpb.Permission_OrRules{OrRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{ + Header: &v3routepb.HeaderMatcher{ + Name: "key-2", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: "baz"}, + }, + }}, + }, + }}}, + }, + }}}, + }, + }}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{}, + }, + }, + wantPolicyName: "authz", + }, + "allow authenticated": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }] + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{}, + }, + }, + }, + "audit_logging_ALLOW empty config": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "deny_rules": [ + { + "name": "deny_policy_1", + "source": { + "principals":[ + "spiffe://foo.abc" + ] + } + }], + "audit_logging_options": { + "audit_condition": "ON_ALLOW", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "authz_deny_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://foo.abc"}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "audit_logging_DENY_AND_ALLOW": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "deny_rules": [ + { + "name": "deny_policy_1", + "source": { + "principals":[ + "spiffe://foo.abc" + ] + } + }], + "audit_logging_options": { + "audit_condition": "ON_DENY_AND_ALLOW", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "authz_deny_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://foo.abc"}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY_AND_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "audit_logging_NONE": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "deny_rules": [ + { + "name": "deny_policy_1", + "source": { + "principals":[ + "spiffe://foo.abc" + ] + } + }], + "audit_logging_options": { + "audit_condition": "NONE", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "authz_deny_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://foo.abc"}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "audit_logging_custom_config simple": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "deny_rules": [ + { + "name": "deny_policy_1", + "source": { + "principals":[ + "spiffe://foo.abc" + ] + } + }], + "audit_logging_options": { + "audit_condition": "NONE", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {"abc":123, "xyz":"123"}, + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "authz_deny_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://foo.abc"}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{"abc": 123, "xyz": "123"}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{"abc": 123, "xyz": "123"}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "audit_logging_custom_config nested": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "audit_logging_options": { + "audit_condition": "NONE", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {"abc":123, "xyz":{"abc":123}}, + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{"abc": 123, "xyz": map[string]any{"abc": 123}}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "missing audit logger config": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "audit_logging_options": { + "audit_condition": "NONE" + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{}, + }, + }, + }, + }, + "missing audit condition": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "audit_logging_options": { + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "missing custom config audit logger": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "deny_rules": [ + { + "name": "deny_policy_1", + "source": { + "principals":[ + "spiffe://foo.abc" + ] + } + }], + "audit_logging_options": { + "audit_condition": "ON_DENY", + "audit_loggers": [ + { + "name": "stdout_logger", + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "authz_deny_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://foo.abc"}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "unknown field": { + authzPolicy: `{"random": 123}`, + wantErr: "failed to unmarshal policy", + }, + "missing name field": { + authzPolicy: `{}`, + wantErr: `"name" is not present`, + }, + "invalid field type": { + authzPolicy: `{"name": 123}`, + wantErr: "failed to unmarshal policy", + }, + "missing allow rules field": { + authzPolicy: `{"name": "authz-foo"}`, + wantErr: `"allow_rules" is not present`, + }, + "missing rule name field": { + authzPolicy: `{ + "name": "authz-foo", + "allow_rules": [{}] + }`, + wantErr: `"allow_rules" 0: "name" is not present`, + }, + "missing header key": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [{ + "name": "allow_policy_1", + "request": {"headers":[{"key":"key-a", "values": ["value-a"]}, {}]} + }] + }`, + wantErr: `"allow_rules" 0: "headers" 1: "key" is not present`, + }, + "missing header values": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [{ + "name": "allow_policy_1", + "request": {"headers":[{"key":"key-a"}]} + }] + }`, + wantErr: `"allow_rules" 0: "headers" 0: "values" is not present`, + }, + "unsupported header": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [{ + "name": "allow_policy_1", + "request": {"headers":[{"key":":method", "values":["GET"]}]} + }] + }`, + wantErr: `"allow_rules" 0: "headers" 0: unsupported "key" :method`, + }, + "bad audit condition": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "audit_logging_options": { + "audit_condition": "ABC", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantErr: `failed to parse AuditCondition ABC`, + }, + "bad audit logger config": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "audit_logging_options": { + "audit_condition": "NONE", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": "abc", + "is_optional": false + } + ] + } + }`, + wantErr: `failed to unmarshal policy`, + }, + "missing audit logger name": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "audit_logging_options": { + "audit_condition": "NONE", + "audit_loggers": [ + { + "name": "", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantErr: `missing required field: name`, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + gotPolicies, gotPolicyName, gotErr := translatePolicy(test.authzPolicy) + if gotErr != nil && !strings.HasPrefix(gotErr.Error(), test.wantErr) { + t.Fatalf("unexpected error\nwant:%v\ngot:%v", test.wantErr, gotErr) + } + if diff := cmp.Diff(gotPolicies, test.wantPolicies, protocmp.Transform()); diff != "" { + t.Fatalf("unexpected policy\ndiff (-want +got):\n%s", diff) + } + if test.wantPolicyName != "" && gotPolicyName != test.wantPolicyName { + t.Fatalf("unexpected policy name\nwant:%v\ngot:%v", test.wantPolicyName, gotPolicyName) + } + }) + } +} + +func anyPbHelper(t *testing.T, in map[string]any, name string) *anypb.Any { + t.Helper() + pb, err := structpb.NewStruct(in) + typedStruct := &v1xdsudpatypepb.TypedStruct{ + TypeUrl: typeURLPrefix + name, + Value: pb, + } + if err != nil { + t.Fatal(err) + } + customConfig, err := anypb.New(typedStruct) + if err != nil { + t.Fatal(err) + } + return customConfig +} diff --git a/backoff.go b/backoff.go new file mode 100644 index 0000000..29475e3 --- /dev/null +++ b/backoff.go @@ -0,0 +1,61 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// See internal/backoff package for the backoff implementation. This file is +// kept for the exported types and API backward compatibility. + +package grpc + +import ( + "time" + + "google.golang.org/grpc/backoff" +) + +// DefaultBackoffConfig uses values specified for backoff in +// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. +// +// Deprecated: use ConnectParams instead. Will be supported throughout 1.x. +var DefaultBackoffConfig = BackoffConfig{ + MaxDelay: 120 * time.Second, +} + +// BackoffConfig defines the parameters for the default gRPC backoff strategy. +// +// Deprecated: use ConnectParams instead. Will be supported throughout 1.x. +type BackoffConfig struct { + // MaxDelay is the upper bound of backoff delay. + MaxDelay time.Duration +} + +// ConnectParams defines the parameters for connecting and retrying. Users are +// encouraged to use this instead of the BackoffConfig type defined above. See +// here for more details: +// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type ConnectParams struct { + // Backoff specifies the configuration options for connection backoff. + Backoff backoff.Config + // MinConnectTimeout is the minimum amount of time we are willing to give a + // connection to complete. + MinConnectTimeout time.Duration +} diff --git a/backoff/backoff.go b/backoff/backoff.go new file mode 100644 index 0000000..d7b40b7 --- /dev/null +++ b/backoff/backoff.go @@ -0,0 +1,52 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package backoff provides configuration options for backoff. +// +// More details can be found at: +// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. +// +// All APIs in this package are experimental. +package backoff + +import "time" + +// Config defines the configuration options for backoff. +type Config struct { + // BaseDelay is the amount of time to backoff after the first failure. + BaseDelay time.Duration + // Multiplier is the factor with which to multiply backoffs after a + // failed retry. Should ideally be greater than 1. + Multiplier float64 + // Jitter is the factor with which backoffs are randomized. + Jitter float64 + // MaxDelay is the upper bound of backoff delay. + MaxDelay time.Duration +} + +// DefaultConfig is a backoff configuration with the default values specified +// at https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. +// +// This should be useful for callers who want to configure backoff with +// non-default values only for a subset of the options. +var DefaultConfig = Config{ + BaseDelay: 1.0 * time.Second, + Multiplier: 1.6, + Jitter: 0.2, + MaxDelay: 120 * time.Second, +} diff --git a/balancer/autosharding/autosharding.go b/balancer/autosharding/autosharding.go new file mode 100644 index 0000000..a7ff611 --- /dev/null +++ b/balancer/autosharding/autosharding.go @@ -0,0 +1,112 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package autosharding implements the autosharding load balancing policy. +package autosharding + +import ( + "encoding/json" + "fmt" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/endpointsharding" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Name is the name of the autosharding balancer. +const Name = "autosharding_experimental" + +func init() { + balancer.Register(bb{}) +} + +// lbConfig is the balancer config for the autosharding balancer. +type lbConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + ChannelFactoryKey string `json:"channelFactoryKey,omitempty"` + AutoShardingTarget string `json:"autoshardingTarget,omitempty"` + KeyHeaderName string `json:"keyHeaderName,omitempty"` + EnableFallback bool `json:"enableFallback,omitempty"` + InitialAssignmentTimeout iserviceconfig.Duration `json:"initialAssignmentTimeout,omitempty"` +} + +type bb struct{} + +func (bb) Name() string { + return Name +} + +func (bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + lbConfig := &lbConfig{InitialAssignmentTimeout: iserviceconfig.Duration(60 * time.Second)} + if err := json.Unmarshal(s, lbConfig); err != nil { + return nil, fmt.Errorf("autosharding: unable to unmarshal LBConfig: %v", err) + } + if lbConfig.ChannelFactoryKey == "" { + return nil, fmt.Errorf("autosharding: channelFactoryKey field is required") + } + if lbConfig.AutoShardingTarget == "" { + return nil, fmt.Errorf("autosharding: autoshardingTarget field is required") + } + if lbConfig.KeyHeaderName == "" { + return nil, fmt.Errorf("autosharding: keyHeaderName field is required") + } + return lbConfig, nil +} + +func (bb) Build(balancer.ClientConn, balancer.BuildOptions) balancer.Balancer { + return &autoshardingBalancer{} +} + +// slice represents a key range and its assigned endpoints. +// +//lint:ignore U1000 Struct fields planned for future implementation +type slice struct { + startKey []byte // Inclusive start key of the key-range + endKey []byte // Exclusive, nil for sentinel/infinity + endpoints []int // Indices into assignment.endpointNames +} + +// assignment represents a complete snapshot of sharding assignments, and is +// expected to cover the entire key range. +type assignment struct { + slices []slice // Sorted by startKey + endpointNames []string // Complete list of endpoint names + generation int64 +} + +// endpointState represents the state associated with an endpoint in the LB policy. +// +//lint:ignore U1000 Struct fields planned for future implementation +type endpointState struct { + index int // Index of the endpoint within the NR update + endpoint resolver.Endpoint // The actual endpoint returned by the NR + childState endpointsharding.ChildState // State as reported by the child policy +} + +// endpointMap maps from endpoint hostname to endpoint state. +type endpointMap struct { + m map[string]*endpointState +} + +type autoshardingBalancer struct { + balancer.Balancer +} diff --git a/balancer/autosharding/autosharding_test.go b/balancer/autosharding/autosharding_test.go new file mode 100644 index 0000000..2a4c1e4 --- /dev/null +++ b/balancer/autosharding/autosharding_test.go @@ -0,0 +1,144 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package autosharding + +import ( + "encoding/json" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestParseConfig_Success(t *testing.T) { + parser := bb{} + tests := []struct { + name string + input string + wantCfg serviceconfig.LoadBalancingConfig + }{ + { + name: "all-fields", + input: `{ + "channelFactoryKey": "factory-key", + "autoshardingTarget": "target", + "keyHeaderName": "header", + "enableFallback": true, + "initialAssignmentTimeout": "30s" + }`, + wantCfg: &lbConfig{ + ChannelFactoryKey: "factory-key", + AutoShardingTarget: "target", + KeyHeaderName: "header", + EnableFallback: true, + InitialAssignmentTimeout: iserviceconfig.Duration(30 * time.Second), + }, + }, + { + name: "default-timeout", + input: `{ + "channelFactoryKey": "factory-key", + "autoshardingTarget": "target", + "keyHeaderName": "header" + }`, + wantCfg: &lbConfig{ + ChannelFactoryKey: "factory-key", + AutoShardingTarget: "target", + KeyHeaderName: "header", + InitialAssignmentTimeout: iserviceconfig.Duration(60 * time.Second), + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotCfg, err := parser.ParseConfig(json.RawMessage(test.input)) + if err != nil { + t.Fatalf("ParseConfig() error = %v, want nil", err) + } + if diff := cmp.Diff(test.wantCfg, gotCfg); diff != "" { + t.Errorf("ParseConfig() config diff (-want +got):\n%s", diff) + } + }) + } +} + +func (s) TestParseConfig_Failure(t *testing.T) { + parser := bb{} + tests := []struct { + name string + input string + }{ + { + name: "invalid-json", + input: "{{invalidjson{{", + }, + { + name: "invalid-duration", + input: `{ + "channelFactoryKey": "factory-key", + "autoshardingTarget": "target", + "keyHeaderName": "header", + "initialAssignmentTimeout": "invalid" + }`, + }, + { + name: "missing-channel-factory-key", + input: `{ + "autoshardingTarget": "target", + "keyHeaderName": "header" + }`, + }, + { + name: "missing-autosharding-target", + input: `{ + "channelFactoryKey": "factory-key", + "keyHeaderName": "header" + }`, + }, + { + name: "missing-key-header-name", + input: `{ + "channelFactoryKey": "factory-key", + "autoshardingTarget": "target" + }`, + }, + { + name: "empty-config", + input: `{}`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := parser.ParseConfig(json.RawMessage(test.input)); err == nil { + t.Fatalf("ParseConfig() succeeded, want error") + } + }) + } +} diff --git a/balancer/autosharding/slice_map.go b/balancer/autosharding/slice_map.go new file mode 100644 index 0000000..ac8adab --- /dev/null +++ b/balancer/autosharding/slice_map.go @@ -0,0 +1,120 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package autosharding + +import ( + "bytes" + "slices" +) + +// sliceMapEntry represents an entry for a key-range in the sliceMap. +type sliceMapEntry struct { + startKey []byte // Inclusive start key of the key-range + endpoints []int // Indices into list[PickerEndpoint] in the Picker +} + +// sliceMap is an immutable data structure used by the Picker to map request +// keys to their assigned key-range. Because it is immutable, it allows the +// Picker to access it without any explicit synchronization with the LB policy. +// +// The sliceMap is meant to be used by the Picker in conjunction with a list of +// pickerEndpoints such that the list can be swapped out, as long as the number +// and order of endpoints don't change. +type sliceMap struct { + slices []sliceMapEntry // Sorted by startKey + fallbackPool []int // Indices into list[PickerEndpoint] for all endpoints + generation int64 // Snapshot generation number +} + +// lookup returns the index of the slice covering the given key. +// +// Because assignments are pre-validated to have no gaps and cover the full key +// range, and since slices is sorted by startKey, lookup boils down to a binary +// search, looking for the insertion point. +// +// If the key matches the startKey of a slice, that slice index is returned. +// Otherwise, the index of the slice immediately preceding the insertion point +// is returned (i.e., the slice covering the range [startKey, nextStartKey)). +// +// Returns -1 when the sliceMap is empty, which is the case before the first +// assignment is received. +func (sm *sliceMap) lookup(key []byte) int { + if len(sm.slices) == 0 { + return -1 + } + + idx, found := slices.BinarySearchFunc(sm.slices, key, func(e sliceMapEntry, k []byte) int { + return bytes.Compare(e.startKey, k) + }) + + if found { + return idx + } + + // Key falls in range [slices[idx - 1].startKey, slices[idx].startKey). + return idx - 1 +} + +// buildSliceMap is used to generate a new sliceMap from the EndpointMap and +// Assignment when either of them change. +func buildSliceMap(endpointMap *endpointMap, assignment *assignment) *sliceMap { + sm := &sliceMap{} + + // Populate fallbackPool with values [0, 1, 2, ... N-1] where N is the + // number of endpoints returned by the name resolver. The only reason to + // have this slice is to allow the Picker to share code that picks a random + // endpoint from a slice of indices, either from the fallbackPool or from a + // sliceMapEntry. + sm.fallbackPool = make([]int, len(endpointMap.m)) + for i := range sm.fallbackPool { + sm.fallbackPool[i] = i + } + + // If no assignment has been received yet (startup case), return early with + // empty slices. + if assignment == nil { + return sm + } + + sm.generation = assignment.generation + + // Build sliceMapEntry for each Slice in the assignment. + sm.slices = make([]sliceMapEntry, 0, len(assignment.slices)) + for _, s := range assignment.slices { + entry := sliceMapEntry{ + startKey: s.startKey, + endpoints: make([]int, 0, len(s.endpoints)), + } + + // Populate the endpoints for this slice by looking up the endpoint + // names in the EndpointMap. If an endpoint name is not found in the + // EndpointMap, it is skipped. This ensures that the sliceMap only + // contains valid endpoints that are currently available. + for _, idx := range s.endpoints { + hostname := assignment.endpointNames[idx] + if es, ok := endpointMap.m[hostname]; ok { + entry.endpoints = append(entry.endpoints, es.index) + } + } + + sm.slices = append(sm.slices, entry) + } + + return sm +} diff --git a/balancer/autosharding/slice_map_test.go b/balancer/autosharding/slice_map_test.go new file mode 100644 index 0000000..2c7b8b3 --- /dev/null +++ b/balancer/autosharding/slice_map_test.go @@ -0,0 +1,216 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package autosharding + +import ( + "bytes" + "fmt" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func (s) TestSliceMap_Lookup(t *testing.T) { + sm := &sliceMap{ + slices: []sliceMapEntry{ + {startKey: []byte("a"), endpoints: []int{1}}, + {startKey: []byte("d"), endpoints: []int{2}}, + {startKey: []byte("f"), endpoints: []int{3}}, + }, + } + + tests := []struct { + name string + key []byte + want int + }{ + { + name: "exact-match-first", + key: []byte("a"), + want: 0, + }, + { + name: "exact-match-middle", + key: []byte("d"), + want: 1, + }, + { + name: "exact-match-last", + key: []byte("f"), + want: 2, + }, + { + name: "between-first-and-middle", + key: []byte("c"), + want: 0, + }, + { + name: "between-middle-and-last", + key: []byte("e"), + want: 1, + }, + { + name: "after-last", + key: []byte("g"), + want: 2, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sm.lookup(tc.key); got != tc.want { + t.Errorf("lookup(%q) = %d, want %d", tc.key, got, tc.want) + } + }) + } +} + +func (s) TestSliceMap_Lookup_Empty(t *testing.T) { + sm := &sliceMap{} + if got := sm.lookup([]byte("a")); got != -1 { + t.Errorf("lookup() on empty map = %d, want -1", got) + } +} + +func (s) TestBuildSliceMap(t *testing.T) { + // Setup endpointMap with unsorted order in map to verify deterministic + // sorting of fallbackPool. + epMap := &endpointMap{ + m: map[string]*endpointState{ + "hostC": {index: 2}, + "hostA": {index: 0}, + "hostB": {index: 1}, + }, + } + + tests := []struct { + name string + assignment *assignment + want *sliceMap + }{ + { + name: "nil-assignment-startup", + assignment: nil, + want: &sliceMap{fallbackPool: []int{0, 1, 2}}, + }, + { + name: "valid-assignment", + assignment: &assignment{ + endpointNames: []string{"hostA", "hostB", "hostC"}, + slices: []slice{ + {startKey: []byte("a"), endpoints: []int{0, 1}}, // hostA, hostB -> indices 0, 1 + {startKey: []byte("m"), endpoints: []int{1, 2}}, // hostB, hostC -> indices 1, 2 + }, + generation: 42, + }, + want: &sliceMap{ + slices: []sliceMapEntry{ + {startKey: []byte("a"), endpoints: []int{0, 1}}, + {startKey: []byte("m"), endpoints: []int{1, 2}}, + }, + fallbackPool: []int{0, 1, 2}, + generation: 42, + }, + }, + { + name: "assignment-with-unknown-host", + assignment: &assignment{ + endpointNames: []string{"hostA", "hostUnknown", "hostC"}, + slices: []slice{ + {startKey: []byte("a"), endpoints: []int{0, 1}}, // hostUnknown is skipped + }, + generation: 43, + }, + want: &sliceMap{ + slices: []sliceMapEntry{ + {startKey: []byte("a"), endpoints: []int{0}}, // Only hostA (index 0) + }, + fallbackPool: []int{0, 1, 2}, + generation: 43, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := buildSliceMap(epMap, tc.assignment) + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("buildSliceMap() diff (-want +got):\n%s", diff) + } + }) + } +} + +func (e sliceMapEntry) Equal(b sliceMapEntry) bool { + return bytes.Equal(e.startKey, b.startKey) && slices.Equal(e.endpoints, b.endpoints) +} + +func (sm *sliceMap) Equal(b *sliceMap) bool { + if sm == nil || b == nil { + return sm == b + } + fallbackPoolEqual := slices.Equal(sm.fallbackPool, b.fallbackPool) + slicesEqual := slices.EqualFunc(sm.slices, b.slices, func(x, y sliceMapEntry) bool { return x.Equal(y) }) + return sm.generation == b.generation && fallbackPoolEqual && slicesEqual +} + +func BenchmarkSliceMap_Lookup(b *testing.B) { + for _, numSlices := range []int{1, 10, 100, 1000, 10000} { + for _, keySize := range []int{16, 32, 64, 128, 256, 512} { + b.Run(fmt.Sprintf("slices_%d_keySize_%d", numSlices, keySize), func(b *testing.B) { + sm := &sliceMap{ + slices: make([]sliceMapEntry, numSlices), + } + for i := 0; i < numSlices; i++ { + // Generate lexicographically sorted keys to serve as slice + // boundaries. We multiply by 1000 to create ranges (gaps) between + // successive slices (e.g., Slice 0 covers [0, 1000), Slice 1 covers + // [1000, 2000)). This allows us to test lookups that fall + // *mid-slice*, rather than just exact matches. + // + // We use zero-padding on the left via "%0*d" (where * is the keySize + // width). This fills the key to its full length (e.g., 512 bytes) + // with leading zeros. Scanning long identical prefixes forces + // bytes.Compare to traverse the full length, simulating a + // conservative, worst-case latency scenario. + key := fmt.Appendf(nil, "%0*d", keySize, i*1000) + sm.slices[i] = sliceMapEntry{startKey: key, endpoints: []int{i}} + } + + // Pre-generate a pool of lookup keys. This helps avoid + // measuring string formatting overhead inside the timer loop. + lookupKeys := make([][]byte, 10000) + for i := 0; i < 10000; i++ { + // Distribute the 10,000 lookup keys proportionally across the entire + // synthetic keyspace [0, numSlices * 1000). This ensures a good mix + // of boundary hits and interior hits. + val := i * (numSlices * 1000) / 10000 + lookupKeys[i] = fmt.Appendf(nil, "%0*d", keySize, val) + } + + var i int + for b.Loop() { + sm.lookup(lookupKeys[i%10000]) + i++ + } + }) + } + } +} diff --git a/balancer/balancer.go b/balancer/balancer.go new file mode 100644 index 0000000..7e3dbaa --- /dev/null +++ b/balancer/balancer.go @@ -0,0 +1,394 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package balancer defines APIs for load balancing in gRPC. +// All APIs in this package are experimental. +package balancer + +import ( + "context" + "encoding/json" + "errors" + "net" + "strings" + + "google.golang.org/grpc/channelz" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +var ( + // m is a map from name to balancer builder. + m = make(map[string]Builder) + + logger = grpclog.Component("balancer") +) + +// Register registers the balancer builder to the balancer map. b.Name +// will be used as the name registered with this builder. If the Builder +// implements ConfigParser, ParseConfig will be called when new service +// configs are received by the resolver, and the result will be provided to the +// Balancer in UpdateClientConnState. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple Balancers are +// registered with the same name, the one registered last will take effect. +func Register(b Builder) { + name := b.Name() + if !envconfig.CaseSensitiveBalancerRegistries { + name = strings.ToLower(name) + if name != b.Name() { + logger.Warningf("Balancer registered with name %q. grpc-go has switched to case sensitive balancer registries. GRPC_GO_EXPERIMENTAL_CASE_SENSITIVE_BALANCER_REGISTRIES env variable will be removed in release v1.82.0", b.Name()) + } + } + m[name] = b +} + +// unregisterForTesting deletes the balancer with the given name from the +// balancer map. +// +// This function is not thread-safe. +func unregisterForTesting(name string) { + delete(m, name) +} + +func init() { + internal.BalancerUnregister = unregisterForTesting +} + +// Get returns the resolver builder registered with the given name. +// Note that the compare is done in a case-sensitive fashion. +// If no builder is register with the name, nil will be returned. +func Get(name string) Builder { + if !envconfig.CaseSensitiveBalancerRegistries { + lowerName := strings.ToLower(name) + if lowerName != name { + logger.Warningf("Balancer retrieved for name %q. grpc-go has switched to case sensitive balancer registries. GRPC_GO_EXPERIMENTAL_CASE_SENSITIVE_BALANCER_REGISTRIES env variable will be removed in release v1.82.0", name) + } + name = lowerName + } + if b, ok := m[name]; ok { + return b + } + return nil +} + +// NewSubConnOptions contains options to create new SubConn. +type NewSubConnOptions struct { + // CredsBundle is the credentials bundle that will be used in the created + // SubConn. If it's nil, the original creds from grpc DialOptions will be + // used. + // + // Deprecated: Use the Attributes field in resolver.Address to pass + // arbitrary data to the credential handshaker. + CredsBundle credentials.Bundle + // HealthCheckEnabled indicates whether health check service should be + // enabled on this SubConn + HealthCheckEnabled bool + // StateListener is called when the state of the subconn changes. If nil, + // Balancer.UpdateSubConnState will be called instead. Will never be + // invoked until after Connect() is called on the SubConn created with + // these options. + StateListener func(SubConnState) +} + +// State contains the balancer's state relevant to the gRPC ClientConn. +type State struct { + // State contains the connectivity state of the balancer, which is used to + // determine the state of the ClientConn. + ConnectivityState connectivity.State + // Picker is used to choose connections (SubConns) for RPCs. + Picker Picker +} + +// ClientConn represents a gRPC ClientConn. +// +// This interface is to be implemented by gRPC. Users should not need a +// brand new implementation of this interface. For the situations like +// testing, the new implementation should embed this interface. This allows +// gRPC to add new methods to this interface. +// +// NOTICE: This interface is intended to be implemented by gRPC, or intercepted +// by custom load balancing polices. Users should not need their own complete +// implementation of this interface -- they should always delegate to a +// ClientConn passed to Builder.Build() by embedding it in their +// implementations. An embedded ClientConn must never be nil, or runtime panics +// will occur. +type ClientConn interface { + // NewSubConn is called by balancer to create a new SubConn. + // It doesn't block and wait for the connections to be established. + // Behaviors of the SubConn can be controlled by options. + // + // Deprecated: please be aware that in a future version, SubConns will only + // support one address per SubConn. + NewSubConn([]resolver.Address, NewSubConnOptions) (SubConn, error) + // RemoveSubConn removes the SubConn from ClientConn. + // The SubConn will be shutdown. + // + // Deprecated: use SubConn.Shutdown instead. + RemoveSubConn(SubConn) + // UpdateAddresses updates the addresses used in the passed in SubConn. + // gRPC checks if the currently connected address is still in the new list. + // If so, the connection will be kept. Else, the connection will be + // gracefully closed, and a new connection will be created. + // + // This may trigger a state transition for the SubConn. + // + // Deprecated: this method will be removed. Create new SubConns for new + // addresses instead. + UpdateAddresses(SubConn, []resolver.Address) + + // UpdateState notifies gRPC that the balancer's internal state has + // changed. + // + // gRPC will update the connectivity state of the ClientConn, and will call + // Pick on the new Picker to pick new SubConns. + UpdateState(State) + + // ResolveNow is called by balancer to notify gRPC to do a name resolving. + ResolveNow(resolver.ResolveNowOptions) + + // Target returns the dial target for this ClientConn. + // + // Deprecated: Use the Target field in the BuildOptions instead. + Target() string + + // MetricsRecorder provides the metrics recorder that balancers can use to + // record metrics. Balancer implementations which do not register metrics on + // metrics registry and record on them can ignore this method. The returned + // MetricsRecorder is guaranteed to never be nil. + MetricsRecorder() estats.MetricsRecorder + + // EnforceClientConnEmbedding is included to force implementers to embed + // another implementation of this interface, allowing gRPC to add methods + // without breaking users. + internal.EnforceClientConnEmbedding +} + +// BuildOptions contains additional information for Build. +type BuildOptions struct { + // DialCreds is the transport credentials to use when communicating with a + // remote load balancer server. Balancer implementations which do not + // communicate with a remote load balancer server can ignore this field. + DialCreds credentials.TransportCredentials + // CredsBundle is the credentials bundle to use when communicating with a + // remote load balancer server. Balancer implementations which do not + // communicate with a remote load balancer server can ignore this field. + CredsBundle credentials.Bundle + // Dialer is the custom dialer to use when communicating with a remote load + // balancer server. Balancer implementations which do not communicate with a + // remote load balancer server can ignore this field. + Dialer func(context.Context, string) (net.Conn, error) + // Authority is the server name to use as part of the authentication + // handshake when communicating with a remote load balancer server. Balancer + // implementations which do not communicate with a remote load balancer + // server can ignore this field. + Authority string + // ChannelzParent is the parent ClientConn's channelz channel. + ChannelzParent channelz.Identifier + // CustomUserAgent is the custom user agent set on the parent ClientConn. + // The balancer should set the same custom user agent if it creates a + // ClientConn. + CustomUserAgent string + // Target contains the parsed address info of the dial target. It is the + // same resolver.Target as passed to the resolver. See the documentation for + // the resolver.Target type for details about what it contains. + Target resolver.Target +} + +// Builder creates a balancer. +type Builder interface { + // Build creates a new balancer with the ClientConn. + Build(cc ClientConn, opts BuildOptions) Balancer + // Name returns the name of balancers built by this builder. + // It will be used to pick balancers (for example in service config). + Name() string +} + +// ConfigParser parses load balancer configs. +type ConfigParser interface { + // ParseConfig parses the JSON load balancer config provided into an + // internal form or returns an error if the config is invalid. For future + // compatibility reasons, unknown fields in the config should be ignored. + ParseConfig(LoadBalancingConfigJSON json.RawMessage) (serviceconfig.LoadBalancingConfig, error) +} + +// PickInfo contains additional information for the Pick operation. +type PickInfo struct { + // FullMethodName is the method name that NewClientStream() is called + // with. The canonical format is /service/Method. + FullMethodName string + // Ctx is the RPC's context, and may contain relevant RPC-level information + // like the outgoing header metadata. + Ctx context.Context +} + +// DoneInfo contains additional information for done. +type DoneInfo struct { + // Err is the rpc error the RPC finished with. It could be nil. + Err error + // Trailer contains the metadata from the RPC's trailer, if present. + Trailer metadata.MD + // BytesSent indicates if any bytes have been sent to the server. + BytesSent bool + // BytesReceived indicates if any byte has been received from the server. + BytesReceived bool + // ServerLoad is the load received from server. It's usually sent as part of + // trailing metadata. + // + // The only supported type now is *orca_v3.LoadReport. + ServerLoad any +} + +var ( + // ErrNoSubConnAvailable indicates no SubConn is available for pick(). + // gRPC will block the RPC until a new picker is available via UpdateState(). + ErrNoSubConnAvailable = errors.New("no SubConn is available") + // ErrTransientFailure indicates all SubConns are in TransientFailure. + // WaitForReady RPCs will block, non-WaitForReady RPCs will fail. + // + // Deprecated: return an appropriate error based on the last resolution or + // connection attempt instead. The behavior is the same for any non-gRPC + // status error. + ErrTransientFailure = errors.New("all SubConns are in TransientFailure") +) + +// PickResult contains information related to a connection chosen for an RPC. +type PickResult struct { + // SubConn is the connection to use for this pick, if its state is Ready. + // If the state is not Ready, gRPC will block the RPC until a new Picker is + // provided by the balancer (using ClientConn.UpdateState). The SubConn + // must be one returned by ClientConn.NewSubConn. + SubConn SubConn + + // Done is called when the RPC is completed. If the SubConn is not ready, + // this will be called with a nil parameter. If the SubConn is not a valid + // type, Done may not be called. May be nil if the balancer does not wish + // to be notified when the RPC completes. + Done func(DoneInfo) + + // Metadata provides a way for LB policies to inject arbitrary per-call + // metadata. Any metadata returned here will be merged with existing + // metadata added by the client application. + // + // LB policies with child policies are responsible for propagating metadata + // injected by their children to the ClientConn, as part of Pick(). + Metadata metadata.MD +} + +// TransientFailureError returns e. It exists for backward compatibility and +// will be deleted soon. +// +// Deprecated: no longer necessary, picker errors are treated this way by +// default. +func TransientFailureError(e error) error { return e } + +// Picker is used by gRPC to pick a SubConn to send an RPC. +// Balancer is expected to generate a new picker from its snapshot every time its +// internal state has changed. +// +// The pickers used by gRPC can be updated by ClientConn.UpdateState(). +type Picker interface { + // Pick returns the connection to use for this RPC and related information. + // + // Pick should not block. If the balancer needs to do I/O or any blocking + // or time-consuming work to service this call, it should return + // ErrNoSubConnAvailable, and the Pick call will be repeated by gRPC when + // the Picker is updated (using ClientConn.UpdateState). + // + // If an error is returned: + // + // - If the error is ErrNoSubConnAvailable, gRPC will block until a new + // Picker is provided by the balancer (using ClientConn.UpdateState). + // + // - If the error is a status error (implemented by the grpc/status + // package), gRPC will terminate the RPC with the code and message + // provided. + // + // - For all other errors, wait for ready RPCs will wait, but non-wait for + // ready RPCs will be terminated with this error's Error() string and + // status code Unavailable. + Pick(info PickInfo) (PickResult, error) +} + +// Balancer takes input from gRPC, manages SubConns, and collects and aggregates +// the connectivity states. +// +// It also generates and updates the Picker used by gRPC to pick SubConns for RPCs. +// +// UpdateClientConnState, ResolverError, UpdateSubConnState, and Close are +// guaranteed to be called synchronously from the same goroutine. There's no +// guarantee on picker.Pick, it may be called anytime. +type Balancer interface { + // UpdateClientConnState is called by gRPC when the state of the ClientConn + // changes. If the error returned is ErrBadResolverState, the ClientConn + // will begin calling ResolveNow on the active name resolver with + // exponential backoff until a subsequent call to UpdateClientConnState + // returns a nil error. Any other errors are currently ignored. + UpdateClientConnState(ClientConnState) error + // ResolverError is called by gRPC when the name resolver reports an error. + ResolverError(error) + // UpdateSubConnState is called by gRPC when the state of a SubConn + // changes. + // + // Deprecated: Use NewSubConnOptions.StateListener when creating the + // SubConn instead. + UpdateSubConnState(SubConn, SubConnState) + // Close closes the balancer. The balancer is not currently required to + // call SubConn.Shutdown for its existing SubConns; however, this will be + // required in a future release, so it is recommended. + Close() + // ExitIdle instructs the LB policy to reconnect to backends / exit the + // IDLE state, if appropriate and possible. Note that SubConns that enter + // the IDLE state will not reconnect until SubConn.Connect is called. + ExitIdle() +} + +// ExitIdler is an optional interface for balancers to implement. If +// implemented, ExitIdle will be called when ClientConn.Connect is called, if +// the ClientConn is idle. If unimplemented, ClientConn.Connect will cause +// all SubConns to connect. +// +// Deprecated: All balancers must implement this interface. This interface will +// be removed in a future release. +type ExitIdler interface { + // ExitIdle instructs the LB policy to reconnect to backends / exit the + // IDLE state, if appropriate and possible. Note that SubConns that enter + // the IDLE state will not reconnect until SubConn.Connect is called. + ExitIdle() +} + +// ClientConnState describes the state of a ClientConn relevant to the +// balancer. +type ClientConnState struct { + ResolverState resolver.State + // The parsed load balancing configuration returned by the builder's + // ParseConfig method, if implemented. + BalancerConfig serviceconfig.LoadBalancingConfig +} + +// ErrBadResolverState may be returned by UpdateClientConnState to indicate a +// problem with the provided name resolver data. +var ErrBadResolverState = errors.New("bad resolver state") diff --git a/balancer/base/balancer.go b/balancer/base/balancer.go new file mode 100644 index 0000000..4399ba0 --- /dev/null +++ b/balancer/base/balancer.go @@ -0,0 +1,260 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package base + +import ( + "errors" + "fmt" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" +) + +var logger = grpclog.Component("balancer") + +type baseBuilder struct { + name string + pickerBuilder PickerBuilder + config Config +} + +func (bb *baseBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { + bal := &baseBalancer{ + cc: cc, + pickerBuilder: bb.pickerBuilder, + + subConns: resolver.NewAddressMapV2[balancer.SubConn](), + scStates: make(map[balancer.SubConn]connectivity.State), + csEvltr: &balancer.ConnectivityStateEvaluator{}, + config: bb.config, + state: connectivity.Connecting, + } + // Initialize picker to a picker that always returns + // ErrNoSubConnAvailable, because when state of a SubConn changes, we + // may call UpdateState with this picker. + bal.picker = NewErrPicker(balancer.ErrNoSubConnAvailable) + return bal +} + +func (bb *baseBuilder) Name() string { + return bb.name +} + +type baseBalancer struct { + cc balancer.ClientConn + pickerBuilder PickerBuilder + + csEvltr *balancer.ConnectivityStateEvaluator + state connectivity.State + + subConns *resolver.AddressMapV2[balancer.SubConn] + scStates map[balancer.SubConn]connectivity.State + picker balancer.Picker + config Config + + resolverErr error // the last error reported by the resolver; cleared on successful resolution + connErr error // the last connection error; cleared upon leaving TransientFailure +} + +func (b *baseBalancer) ResolverError(err error) { + b.resolverErr = err + if b.subConns.Len() == 0 { + b.state = connectivity.TransientFailure + } + + if b.state != connectivity.TransientFailure { + // The picker will not change since the balancer does not currently + // report an error. + return + } + b.regeneratePicker() + b.cc.UpdateState(balancer.State{ + ConnectivityState: b.state, + Picker: b.picker, + }) +} + +func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + // TODO: handle s.ResolverState.ServiceConfig? + if logger.V(2) { + logger.Info("base.baseBalancer: got new ClientConn state: ", s) + } + // Successful resolution; clear resolver error and ensure we return nil. + b.resolverErr = nil + // addrsSet is the set converted from addrs, it's used for quick lookup of an address. + addrsSet := resolver.NewAddressMapV2[any]() + for _, a := range s.ResolverState.Addresses { + addrsSet.Set(a, nil) + if _, ok := b.subConns.Get(a); !ok { + // a is a new address (not existing in b.subConns). + var sc balancer.SubConn + opts := balancer.NewSubConnOptions{ + HealthCheckEnabled: b.config.HealthCheck, + StateListener: func(scs balancer.SubConnState) { b.updateSubConnState(sc, scs) }, + } + sc, err := b.cc.NewSubConn([]resolver.Address{a}, opts) + if err != nil { + logger.Warningf("base.baseBalancer: failed to create new SubConn: %v", err) + continue + } + b.subConns.Set(a, sc) + b.scStates[sc] = connectivity.Idle + b.csEvltr.RecordTransition(connectivity.Shutdown, connectivity.Idle) + sc.Connect() + } + } + for a, sc := range b.subConns.All() { + // a was removed by resolver. + if _, ok := addrsSet.Get(a); !ok { + sc.Shutdown() + b.subConns.Delete(a) + // Keep the state of this sc in b.scStates until sc's state becomes Shutdown. + // The entry will be deleted in updateSubConnState. + } + } + // If resolver state contains no addresses, return an error so ClientConn + // will trigger re-resolve. Also records this as a resolver error, so when + // the overall state turns transient failure, the error message will have + // the zero address information. + if len(s.ResolverState.Addresses) == 0 { + b.ResolverError(errors.New("produced zero addresses")) + return balancer.ErrBadResolverState + } + + b.regeneratePicker() + b.cc.UpdateState(balancer.State{ConnectivityState: b.state, Picker: b.picker}) + return nil +} + +// mergeErrors builds an error from the last connection error and the last +// resolver error. Must only be called if b.state is TransientFailure. +func (b *baseBalancer) mergeErrors() error { + // connErr must always be non-nil unless there are no SubConns, in which + // case resolverErr must be non-nil. + if b.connErr == nil { + return fmt.Errorf("last resolver error: %v", b.resolverErr) + } + if b.resolverErr == nil { + return fmt.Errorf("last connection error: %v", b.connErr) + } + return fmt.Errorf("last connection error: %v; last resolver error: %v", b.connErr, b.resolverErr) +} + +// regeneratePicker takes a snapshot of the balancer, and generates a picker +// from it. The picker is +// - errPicker if the balancer is in TransientFailure, +// - built by the pickerBuilder with all READY SubConns otherwise. +func (b *baseBalancer) regeneratePicker() { + if b.state == connectivity.TransientFailure { + b.picker = NewErrPicker(b.mergeErrors()) + return + } + readySCs := make(map[balancer.SubConn]SubConnInfo) + + // Filter out all ready SCs from full subConn map. + for addr, sc := range b.subConns.All() { + if st, ok := b.scStates[sc]; ok && st == connectivity.Ready { + readySCs[sc] = SubConnInfo{Address: addr} + } + } + b.picker = b.pickerBuilder.Build(PickerBuildInfo{ReadySCs: readySCs}) +} + +// UpdateSubConnState is a nop because a StateListener is always set in NewSubConn. +func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + logger.Errorf("base.baseBalancer: UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *baseBalancer) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + s := state.ConnectivityState + if logger.V(2) { + logger.Infof("base.baseBalancer: handle SubConn state change: %p, %v", sc, s) + } + oldS, ok := b.scStates[sc] + if !ok { + if logger.V(2) { + logger.Infof("base.baseBalancer: got state changes for an unknown SubConn: %p, %v", sc, s) + } + return + } + if oldS == connectivity.TransientFailure && + (s == connectivity.Connecting || s == connectivity.Idle) { + // Once a subconn enters TRANSIENT_FAILURE, ignore subsequent IDLE or + // CONNECTING transitions to prevent the aggregated state from being + // always CONNECTING when many backends exist but are all down. + if s == connectivity.Idle { + sc.Connect() + } + return + } + b.scStates[sc] = s + switch s { + case connectivity.Idle: + sc.Connect() + case connectivity.Shutdown: + // When an address was removed by resolver, b called Shutdown but kept + // the sc's state in scStates. Remove state for this sc here. + delete(b.scStates, sc) + case connectivity.TransientFailure: + // Save error to be reported via picker. + b.connErr = state.ConnectionError + } + + b.state = b.csEvltr.RecordTransition(oldS, s) + + // Regenerate picker when one of the following happens: + // - this sc entered or left ready + // - the aggregated state of balancer is TransientFailure + // (may need to update error message) + if (s == connectivity.Ready) != (oldS == connectivity.Ready) || + b.state == connectivity.TransientFailure { + b.regeneratePicker() + } + b.cc.UpdateState(balancer.State{ConnectivityState: b.state, Picker: b.picker}) +} + +// Close is a nop because base balancer doesn't have internal state to clean up, +// and it doesn't need to call Shutdown for the SubConns. +func (b *baseBalancer) Close() { +} + +// ExitIdle is a nop because the base balancer attempts to stay connected to +// all SubConns at all times. +func (b *baseBalancer) ExitIdle() { +} + +// NewErrPicker returns a Picker that always returns err on Pick(). +func NewErrPicker(err error) balancer.Picker { + return &errPicker{err: err} +} + +// NewErrPickerV2 is temporarily defined for backward compatibility reasons. +// +// Deprecated: use NewErrPicker instead. +var NewErrPickerV2 = NewErrPicker + +type errPicker struct { + err error // Pick() always returns this err. +} + +func (p *errPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + return balancer.PickResult{}, p.err +} diff --git a/balancer/base/balancer_test.go b/balancer/base/balancer_test.go new file mode 100644 index 0000000..7486cf1 --- /dev/null +++ b/balancer/base/balancer_test.go @@ -0,0 +1,122 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package base + +import ( + "context" + "testing" + "time" + + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" +) + +type testClientConn struct { + balancer.ClientConn + newSubConn func([]resolver.Address, balancer.NewSubConnOptions) (balancer.SubConn, error) +} + +func (c *testClientConn) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + return c.newSubConn(addrs, opts) +} + +func (c *testClientConn) UpdateState(balancer.State) {} + +type testSubConn struct { + balancer.SubConn + updateState func(balancer.SubConnState) +} + +func (sc *testSubConn) UpdateAddresses([]resolver.Address) {} + +func (sc *testSubConn) Connect() {} + +func (sc *testSubConn) Shutdown() {} + +func (sc *testSubConn) GetOrBuildProducer(balancer.ProducerBuilder) (balancer.Producer, func()) { + return nil, nil +} + +// RegisterHealthListener is a no-op. +func (*testSubConn) RegisterHealthListener(func(balancer.SubConnState)) {} + +// testPickBuilder creates balancer.Picker for test. +type testPickBuilder struct { + validate func(info PickerBuildInfo) +} + +func (p *testPickBuilder) Build(info PickerBuildInfo) balancer.Picker { + p.validate(info) + return nil +} + +func TestBaseBalancerReserveAttributes(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + validated := make(chan struct{}, 1) + v := func(info PickerBuildInfo) { + defer func() { validated <- struct{}{} }() + for _, sc := range info.ReadySCs { + if sc.Address.Addr == "1.1.1.1" { + if sc.Address.Attributes == nil { + t.Errorf("in picker.validate, got address %+v with nil attributes, want not nil", sc.Address) + } + foo, ok := sc.Address.Attributes.Value("foo").(string) + if !ok || foo != "2233niang" { + t.Errorf("in picker.validate, got address[1.1.1.1] with invalid attributes value %v, want 2233niang", sc.Address.Attributes.Value("foo")) + } + } else if sc.Address.Addr == "2.2.2.2" { + if sc.Address.Attributes != nil { + t.Error("in b.subConns, got address[2.2.2.2] with not nil attributes, want nil") + } + } + } + } + pickBuilder := &testPickBuilder{validate: v} + b := (&baseBuilder{pickerBuilder: pickBuilder}).Build(&testClientConn{ + newSubConn: func(_ []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + return &testSubConn{updateState: opts.StateListener}, nil + }, + }, balancer.BuildOptions{}).(*baseBalancer) + + b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + {Addr: "1.1.1.1", Attributes: attributes.New("foo", "2233niang")}, + {Addr: "2.2.2.2", Attributes: nil}, + }, + }, + }) + select { + case <-validated: + case <-ctx.Done(): + t.Fatalf("timed out waiting for UpdateClientConnState to call picker.Build") + } + + for sc := range b.scStates { + sc.(*testSubConn).updateState(balancer.SubConnState{ConnectivityState: connectivity.Ready, ConnectionError: nil}) + select { + case <-validated: + case <-ctx.Done(): + t.Fatalf("timed out waiting for UpdateClientConnState to call picker.Build") + } + } +} diff --git a/balancer/base/base.go b/balancer/base/base.go new file mode 100644 index 0000000..e31d76e --- /dev/null +++ b/balancer/base/base.go @@ -0,0 +1,71 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package base defines a balancer base that can be used to build balancers with +// different picking algorithms. +// +// The base balancer creates a new SubConn for each resolved address. The +// provided picker will only be notified about READY SubConns. +// +// This package is the base of round_robin balancer, its purpose is to be used +// to build round_robin like balancers with complex picking algorithms. +// Balancers with more complicated logic should try to implement a balancer +// builder from scratch. +// +// All APIs in this package are experimental. +package base + +import ( + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" +) + +// PickerBuilder creates balancer.Picker. +type PickerBuilder interface { + // Build returns a picker that will be used by gRPC to pick a SubConn. + Build(info PickerBuildInfo) balancer.Picker +} + +// PickerBuildInfo contains information needed by the picker builder to +// construct a picker. +type PickerBuildInfo struct { + // ReadySCs is a map from all ready SubConns to the Addresses used to + // create them. + ReadySCs map[balancer.SubConn]SubConnInfo +} + +// SubConnInfo contains information about a SubConn created by the base +// balancer. +type SubConnInfo struct { + Address resolver.Address // the address used to create this SubConn +} + +// Config contains the config info about the base balancer builder. +type Config struct { + // HealthCheck indicates whether health checking should be enabled for this specific balancer. + HealthCheck bool +} + +// NewBalancerBuilder returns a base balancer builder configured by the provided config. +func NewBalancerBuilder(name string, pb PickerBuilder, config Config) balancer.Builder { + return &baseBuilder{ + name: name, + pickerBuilder: pb, + config: config, + } +} diff --git a/balancer/conn_state_evaluator.go b/balancer/conn_state_evaluator.go new file mode 100644 index 0000000..c334135 --- /dev/null +++ b/balancer/conn_state_evaluator.go @@ -0,0 +1,74 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package balancer + +import "google.golang.org/grpc/connectivity" + +// ConnectivityStateEvaluator takes the connectivity states of multiple SubConns +// and returns one aggregated connectivity state. +// +// It's not thread safe. +type ConnectivityStateEvaluator struct { + numReady uint64 // Number of addrConns in ready state. + numConnecting uint64 // Number of addrConns in connecting state. + numTransientFailure uint64 // Number of addrConns in transient failure state. + numIdle uint64 // Number of addrConns in idle state. +} + +// RecordTransition records state change happening in subConn and based on that +// it evaluates what aggregated state should be. +// +// - If at least one SubConn in Ready, the aggregated state is Ready; +// - Else if at least one SubConn in Connecting, the aggregated state is Connecting; +// - Else if at least one SubConn is Idle, the aggregated state is Idle; +// - Else if at least one SubConn is TransientFailure (or there are no SubConns), the aggregated state is Transient Failure. +// +// Shutdown is not considered. +func (cse *ConnectivityStateEvaluator) RecordTransition(oldState, newState connectivity.State) connectivity.State { + // Update counters. + for idx, state := range []connectivity.State{oldState, newState} { + updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. + switch state { + case connectivity.Ready: + cse.numReady += updateVal + case connectivity.Connecting: + cse.numConnecting += updateVal + case connectivity.TransientFailure: + cse.numTransientFailure += updateVal + case connectivity.Idle: + cse.numIdle += updateVal + } + } + return cse.CurrentState() +} + +// CurrentState returns the current aggregate conn state by evaluating the counters +func (cse *ConnectivityStateEvaluator) CurrentState() connectivity.State { + // Evaluate. + if cse.numReady > 0 { + return connectivity.Ready + } + if cse.numConnecting > 0 { + return connectivity.Connecting + } + if cse.numIdle > 0 { + return connectivity.Idle + } + return connectivity.TransientFailure +} diff --git a/balancer/conn_state_evaluator_test.go b/balancer/conn_state_evaluator_test.go new file mode 100644 index 0000000..d82ddf8 --- /dev/null +++ b/balancer/conn_state_evaluator_test.go @@ -0,0 +1,245 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package balancer + +import ( + "testing" + + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestRecordTransition_FirstStateChange tests the first call to +// RecordTransition where the `oldState` is usually set to `Shutdown` (a state +// that the ConnectivityStateEvaluator is set to ignore). +func (s) TestRecordTransition_FirstStateChange(t *testing.T) { + tests := []struct { + newState connectivity.State + wantState connectivity.State + }{ + { + newState: connectivity.Idle, + wantState: connectivity.Idle, + }, + { + newState: connectivity.Connecting, + wantState: connectivity.Connecting, + }, + { + newState: connectivity.Ready, + wantState: connectivity.Ready, + }, + { + newState: connectivity.TransientFailure, + wantState: connectivity.TransientFailure, + }, + { + newState: connectivity.Shutdown, + wantState: connectivity.TransientFailure, + }, + } + for _, test := range tests { + cse := &ConnectivityStateEvaluator{} + if gotState := cse.RecordTransition(connectivity.Shutdown, test.newState); gotState != test.wantState { + t.Fatalf("RecordTransition(%v, %v) = %v, want %v", connectivity.Shutdown, test.newState, gotState, test.wantState) + } + } +} + +// TestRecordTransition_SameState tests the scenario where state transitions to +// the same state are recorded multiple times. +func (s) TestRecordTransition_SameState(t *testing.T) { + tests := []struct { + newState connectivity.State + wantState connectivity.State + }{ + { + newState: connectivity.Idle, + wantState: connectivity.Idle, + }, + { + newState: connectivity.Connecting, + wantState: connectivity.Connecting, + }, + { + newState: connectivity.Ready, + wantState: connectivity.Ready, + }, + { + newState: connectivity.TransientFailure, + wantState: connectivity.TransientFailure, + }, + { + newState: connectivity.Shutdown, + wantState: connectivity.TransientFailure, + }, + } + const numStateChanges = 5 + for _, test := range tests { + cse := &ConnectivityStateEvaluator{} + var prevState, gotState connectivity.State + prevState = connectivity.Shutdown + for i := 0; i < numStateChanges; i++ { + gotState = cse.RecordTransition(prevState, test.newState) + prevState = test.newState + } + if gotState != test.wantState { + t.Fatalf("RecordTransition() = %v, want %v", gotState, test.wantState) + } + } +} + +// TestRecordTransition_SingleSubConn_DifferentStates tests some common +// connectivity state change scenarios, on a single subConn. +func (s) TestRecordTransition_SingleSubConn_DifferentStates(t *testing.T) { + tests := []struct { + name string + states []connectivity.State + wantState connectivity.State + }{ + { + name: "regular transition to ready", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.Ready}, + wantState: connectivity.Ready, + }, + { + name: "regular transition to transient failure", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.TransientFailure}, + wantState: connectivity.TransientFailure, + }, + { + name: "regular transition to ready", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.Idle}, + wantState: connectivity.Idle, + }, + { + name: "transition from ready to transient failure", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.TransientFailure}, + wantState: connectivity.TransientFailure, + }, + { + name: "transition from transient failure back to ready", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.TransientFailure, connectivity.Ready}, + wantState: connectivity.Ready, + }, + { + // This state transition is usually suppressed at the LB policy level, by + // not calling RecordTransition. + name: "transition from transient failure back to idle", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.TransientFailure, connectivity.Idle}, + wantState: connectivity.Idle, + }, + { + // This state transition is usually suppressed at the LB policy level, by + // not calling RecordTransition. + name: "transition from transient failure back to connecting", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.TransientFailure, connectivity.Connecting}, + wantState: connectivity.Connecting, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cse := &ConnectivityStateEvaluator{} + var prevState, gotState connectivity.State + prevState = connectivity.Shutdown + for _, newState := range test.states { + gotState = cse.RecordTransition(prevState, newState) + prevState = newState + } + if gotState != test.wantState { + t.Fatalf("RecordTransition() = %v, want %v", gotState, test.wantState) + } + }) + } +} + +// TestRecordTransition_MultipleSubConns_DifferentStates tests state transitions +// among multiple subConns, and verifies that the connectivity state aggregation +// algorithm produces the expected aggregate connectivity state. +func (s) TestRecordTransition_MultipleSubConns_DifferentStates(t *testing.T) { + tests := []struct { + name string + // Each entry in this slice corresponds to the state changes happening on an + // individual subConn. + subConnStates [][]connectivity.State + wantState connectivity.State + }{ + { + name: "atleast one ready", + subConnStates: [][]connectivity.State{ + {connectivity.Idle, connectivity.Connecting, connectivity.Ready}, + {connectivity.Idle}, + {connectivity.Idle, connectivity.Connecting}, + {connectivity.Idle, connectivity.Connecting, connectivity.TransientFailure}, + }, + wantState: connectivity.Ready, + }, + { + name: "atleast one connecting", + subConnStates: [][]connectivity.State{ + {connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.Connecting}, + {connectivity.Idle}, + {connectivity.Idle, connectivity.Connecting, connectivity.TransientFailure}, + }, + wantState: connectivity.Connecting, + }, + { + name: "atleast one idle", + subConnStates: [][]connectivity.State{ + {connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.Idle}, + {connectivity.Idle, connectivity.Connecting, connectivity.TransientFailure}, + }, + wantState: connectivity.Idle, + }, + { + name: "atleast one transient failure", + subConnStates: [][]connectivity.State{ + {connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.TransientFailure}, + {connectivity.TransientFailure}, + }, + wantState: connectivity.TransientFailure, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cse := &ConnectivityStateEvaluator{} + var prevState, gotState connectivity.State + for _, scStates := range test.subConnStates { + prevState = connectivity.Shutdown + for _, newState := range scStates { + gotState = cse.RecordTransition(prevState, newState) + prevState = newState + } + } + if gotState != test.wantState { + t.Fatalf("RecordTransition() = %v, want %v", gotState, test.wantState) + } + }) + } +} diff --git a/balancer/endpointsharding/endpointsharding.go b/balancer/endpointsharding/endpointsharding.go new file mode 100644 index 0000000..22dbc52 --- /dev/null +++ b/balancer/endpointsharding/endpointsharding.go @@ -0,0 +1,385 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package endpointsharding implements a load balancing policy that manages +// homogeneous child policies each owning a single endpoint. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +package endpointsharding + +import ( + "errors" + rand "math/rand/v2" + "sync" + "sync/atomic" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" +) + +var randIntN = rand.IntN + +// ChildState is the state of a child balancer. +type ChildState struct { + Endpoint resolver.Endpoint // Endpoint of the child balancer. + State balancer.State // State of the child balancer. + ExitIdle func() // Function to exit the child balancer from IDLE state. +} + +// Options configure the behaviour of the endpointsharding balancer. +type Options struct { + // DisableAutoReconnect allows the balancer to keep child balancer in the + // IDLE state until they are explicitly triggered to exit using the + // ChildState obtained from the endpointsharding picker. When set to false, + // the endpointsharding balancer will automatically call ExitIdle on child + // connections that report IDLE. + DisableAutoReconnect bool +} + +// ChildBuilderFunc creates a new balancer with the ClientConn. It has the same +// type as the balancer.Builder.Build method. +type ChildBuilderFunc func(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer + +// NewBalancer returns a load balancing policy that manages homogeneous child +// policies each owning a single endpoint. The endpointsharding balancer +// forwards the LoadBalancingConfig in ClientConn state updates to its children. +func NewBalancer(cc balancer.ClientConn, opts balancer.BuildOptions, childBuilder ChildBuilderFunc, esOpts Options) balancer.Balancer { + return &endpointSharding{ + cc: cc, + bOpts: opts, + esOpts: esOpts, + childBuilder: childBuilder, + endpoints: resolver.NewEndpointMap[*endpointState](), + } +} + +// endpointSharding is a balancer that wraps child balancers. It creates a child +// balancer with child config for every unique Endpoint received. It updates the +// child states on any update from parent or child. +type endpointSharding struct { + cc balancer.ClientConn + bOpts balancer.BuildOptions + esOpts Options + childBuilder ChildBuilderFunc + + // mu guards access to the below fields and guarantees mutual exclusion + // between top-down methods (like UpdateClientConnState, ResolverError etc, + // which are already serialized) and bottom-up methods (like UpdateState) + // that can be called concurrently. + // + // Lock ordering & deadlock prevention: + // Child callbacks (like UpdateState) acquire this mutex to push state updates + // to the parent. This establishes a strict lock ordering: + // [endpointState.childMu] -> [endpointSharding.mu] + // + // To prevent deadlocks, we must never invert this order. Therefore, we must + // never call any methods on a child balancer while holding this mutex. If we + // did, and that child balancer invoked UpdateState synchronously, it would + // attempt to re-acquire this mutex, causing a deadlock. + // + // Concurrency: + // Top-down operations need to iterate over all children. To ensure a + // single, clean aggregated update at the end of such operations, we inhibit + // intermediate updates from children. We grab this mutex briefly to set + // `inhibitChildUpdates = true` and immediately release it. This allows us + // to perform the potentially slow, top-down child updates without holding + // any parent locks. Once finished, we grab the mutex again to unset the + // flag and push the final aggregated state. An update from a child during + // this time will *only* update the child state, and will not access the + // endpoints map or push an aggregated state to the parent. + mu sync.Mutex + endpoints *resolver.EndpointMap[*endpointState] + inhibitChildUpdates bool +} + +// rotateEndpoints returns a slice of all the input endpoints rotated a random +// amount. +func rotateEndpoints(es []resolver.Endpoint) []resolver.Endpoint { + n := len(es) + if n == 0 { + return es + } + r := randIntN(n) + + // Make a copy to avoid mutating data beyond the end of es. + ret := make([]resolver.Endpoint, n) + copy(ret, es[r:]) + copy(ret[n-r:], es[:r]) + return ret +} + +// UpdateClientConnState creates a child for new endpoints and deletes children +// for endpoints that are no longer present. It also updates all the children, +// and sends a single synchronous update of the childrens' aggregated state at +// the end of the UpdateClientConnState operation. +// +// Returns the first error found from a child, but fully processes the update. +func (es *endpointSharding) UpdateClientConnState(state balancer.ClientConnState) error { + es.inhibitUpdatesFromChildren() + + // Update/create child balancers for each endpoint in the update. Note that we + // don't hold the mutex here, but this is fine because inhibitChildUpdates is + // true, and therefore UpdateState will not access es.endpoints. + var retErr error + newEndpoints := resolver.NewEndpointMap[*endpointState]() + for _, endpoint := range rotateEndpoints(state.ResolverState.Endpoints) { + if _, ok := newEndpoints.Get(endpoint); ok { + // Skip duplicate endpoints. + continue + } + epState, ok := es.endpoints.Get(endpoint) + if ok { + // Endpoint child already exists, update the stored endpoint. + epState.endpoint = endpoint + } else { + // Endpoint child does not exist, create a new one. + epState = &endpointState{ + ClientConn: es.cc, + parent: es, + endpoint: endpoint, + disableAutoReconnect: es.esOpts.DisableAutoReconnect, + } + epState.childLB = es.childBuilder(epState, es.bOpts) + } + // Update the endpoint state for the endpoint. + newEndpoints.Set(endpoint, epState) + + if err := epState.updateClientConnState(balancer.ClientConnState{ + BalancerConfig: state.BalancerConfig, + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{endpoint}, + Attributes: state.ResolverState.Attributes, + }, + }); err != nil && retErr == nil { + // Keep the first error found from any child. + retErr = err + } + } + + // Delete old children that are no longer present. + for e, child := range es.endpoints.All() { + if _, ok := newEndpoints.Get(e); !ok { + child.close() + } + } + + if newEndpoints.Len() == 0 { + retErr = balancer.ErrBadResolverState + } + + es.mu.Lock() + es.endpoints = newEndpoints + es.inhibitChildUpdates = false + es.updateStateLocked() + es.mu.Unlock() + + return retErr +} + +// ResolverError forwards the resolver error to all of the endpointSharding's +// children and sends a single synchronous update of the childStates at the end +// of the ResolverError operation. +func (es *endpointSharding) ResolverError(err error) { + es.inhibitUpdatesFromChildren() + for _, child := range es.endpoints.All() { + child.resolverError(err) + } + es.allowUpdatesFromChildren() +} + +func (es *endpointSharding) UpdateSubConnState(balancer.SubConn, balancer.SubConnState) { + // UpdateSubConnState is deprecated. +} + +func (es *endpointSharding) Close() { + es.inhibitUpdatesFromChildren() + for _, child := range es.endpoints.All() { + child.close() + } +} + +func (es *endpointSharding) ExitIdle() { + es.inhibitUpdatesFromChildren() + for _, child := range es.endpoints.All() { + child.exitIdle() + } + es.allowUpdatesFromChildren() +} + +func (es *endpointSharding) inhibitUpdatesFromChildren() { + es.mu.Lock() + es.inhibitChildUpdates = true + es.mu.Unlock() +} + +func (es *endpointSharding) allowUpdatesFromChildren() { + es.mu.Lock() + es.inhibitChildUpdates = false + es.updateStateLocked() + es.mu.Unlock() +} + +// updateStateLocked updates this component's state. It sends the aggregated +// state, and a picker with round robin behavior with all the child states +// present if needed. This method must only be called when inhibitChildUpdates +// is false. +// +// Caller must hold es.mu. +func (es *endpointSharding) updateStateLocked() { + var readyPickers, connectingPickers, idlePickers, transientFailurePickers []balancer.Picker + + childStates := make([]ChildState, 0, es.endpoints.Len()) + for _, epState := range es.endpoints.All() { + childState := ChildState{ + Endpoint: epState.endpoint, + State: epState.state, + ExitIdle: func() { go epState.exitIdle() }, + } + childStates = append(childStates, childState) + childPicker := childState.State.Picker + switch childState.State.ConnectivityState { + case connectivity.Ready: + readyPickers = append(readyPickers, childPicker) + case connectivity.Connecting: + connectingPickers = append(connectingPickers, childPicker) + case connectivity.Idle: + idlePickers = append(idlePickers, childPicker) + case connectivity.TransientFailure: + transientFailurePickers = append(transientFailurePickers, childPicker) + // connectivity.Shutdown shouldn't appear. + } + } + + // Construct the round robin picker based off the aggregated state. Whatever + // the aggregated state, use the pickers present that are currently in that + // state only. + var aggState connectivity.State + var pickers []balancer.Picker + if len(readyPickers) >= 1 { + aggState = connectivity.Ready + pickers = readyPickers + } else if len(connectingPickers) >= 1 { + aggState = connectivity.Connecting + pickers = connectingPickers + } else if len(idlePickers) >= 1 { + aggState = connectivity.Idle + pickers = idlePickers + } else if len(transientFailurePickers) >= 1 { + aggState = connectivity.TransientFailure + pickers = transientFailurePickers + } else { + aggState = connectivity.TransientFailure + pickers = []balancer.Picker{base.NewErrPicker(errors.New("no children to pick from"))} + } // No children (resolver error before valid update). + + es.cc.UpdateState(balancer.State{ + ConnectivityState: aggState, + Picker: &pickerWithChildStates{ + pickers: pickers, + childStates: childStates, + next: uint32(randIntN(len(pickers))), + }, + }) +} + +// pickerWithChildStates delegates to the pickers it holds in a round robin +// fashion. It also contains the childStates of all the endpointSharding's +// children. +type pickerWithChildStates struct { + pickers []balancer.Picker + childStates []ChildState + next uint32 +} + +func (p *pickerWithChildStates) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + nextIndex := atomic.AddUint32(&p.next, 1) + picker := p.pickers[nextIndex%uint32(len(p.pickers))] + return picker.Pick(info) +} + +// ChildStatesFromPicker returns the state of all the children managed by the +// endpoint sharding balancer that created this picker. +func ChildStatesFromPicker(picker balancer.Picker) []ChildState { + p, ok := picker.(*pickerWithChildStates) + if !ok { + return nil + } + return p.childStates +} + +// endpointState is the internal state maintained for each endpoint. +type endpointState struct { + balancer.ClientConn // Embedded to intercept UpdateState + + parent *endpointSharding // Parent endpointsharding balancer. + endpoint resolver.Endpoint // Endpoint of the child balancer. + state balancer.State // State of the child balancer. + disableAutoReconnect bool // Whether to disable auto reconnect for this child. + + childMu sync.Mutex // Guarantees mutual exclusion for Balancer API calls to the child balancer. + childLB balancer.Balancer // Child balancer. + closed bool // Tracks closure of the child balancer to ensure ExitIdle is not called after Close(). +} + +func (es *endpointState) UpdateState(state balancer.State) { + es.parent.mu.Lock() + es.state = state + if !es.parent.inhibitChildUpdates { + es.parent.updateStateLocked() + } + es.parent.mu.Unlock() + + if state.ConnectivityState == connectivity.Idle && !es.disableAutoReconnect { + go es.exitIdle() + } +} + +func (es *endpointState) updateClientConnState(state balancer.ClientConnState) error { + es.childMu.Lock() + err := es.childLB.UpdateClientConnState(state) + es.childMu.Unlock() + return err +} + +func (es *endpointState) resolverError(err error) { + es.childMu.Lock() + es.childLB.ResolverError(err) + es.childMu.Unlock() +} + +func (es *endpointState) close() { + es.childMu.Lock() + if !es.closed { + es.closed = true + es.childLB.Close() + } + es.childMu.Unlock() +} + +func (es *endpointState) exitIdle() { + es.childMu.Lock() + if !es.closed { + es.childLB.ExitIdle() + } + es.childMu.Unlock() +} diff --git a/balancer/endpointsharding/endpointsharding_ext_test.go b/balancer/endpointsharding/endpointsharding_ext_test.go new file mode 100644 index 0000000..db3d1f2 --- /dev/null +++ b/balancer/endpointsharding/endpointsharding_ext_test.go @@ -0,0 +1,353 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package endpointsharding_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/endpointsharding" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + defaultTestTimeout = time.Second * 10 + defaultTestShortTimeout = time.Millisecond * 10 +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +var logger = grpclog.Component("endpoint-sharding-test") + +func init() { + balancer.Register(fakePetioleBuilder{}) +} + +const fakePetioleName = "fake_petiole" + +type fakePetioleBuilder struct{} + +func (fakePetioleBuilder) Name() string { + return fakePetioleName +} + +func (fakePetioleBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + fp := &fakePetiole{ + ClientConn: cc, + bOpts: opts, + } + fp.Balancer = endpointsharding.NewBalancer(fp, opts, balancer.Get(pickfirst.Name).Build, endpointsharding.Options{}) + return fp +} + +func (fakePetioleBuilder) ParseConfig(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return nil, nil +} + +// fakePetiole is a load balancer that wraps the endpointShardingBalancer, and +// forwards ClientConnUpdates with a child config of graceful switch that wraps +// pick first. It also intercepts UpdateState to make sure it can access the +// child state maintained by EndpointSharding. +type fakePetiole struct { + balancer.Balancer + balancer.ClientConn + bOpts balancer.BuildOptions +} + +func (fp *fakePetiole) UpdateClientConnState(state balancer.ClientConnState) error { + if el := state.ResolverState.Endpoints; len(el) != 2 { + return fmt.Errorf("UpdateClientConnState wants two endpoints, got: %v", el) + } + + return fp.Balancer.UpdateClientConnState(state) +} + +func (fp *fakePetiole) UpdateState(state balancer.State) { + childStates := endpointsharding.ChildStatesFromPicker(state.Picker) + // Both child states should be present in the child picker. States and + // picker change over the lifecycle of test, but there should always be two. + if len(childStates) != 2 { + logger.Fatal(fmt.Errorf("length of child states received: %v, want 2", len(childStates))) + } + + fp.ClientConn.UpdateState(state) +} + +// TestEndpointShardingBasic tests the basic functionality of the endpoint +// sharding balancer. It specifies a petiole policy that is essentially a +// wrapper around the endpoint sharder. Two backends are started, with each +// backend's address specified in an endpoint. The petiole does not have a +// special picker, so it should fallback to the default behavior, which is to +// round_robin amongst the endpoint children that are in the aggregated state. +// It also verifies the petiole has access to the raw child state in case it +// wants to implement a custom picker. The test sends a resolver error to the +// endpointsharding balancer and verifies an error picker from the children +// is used while making an RPC. +func (s) TestEndpointShardingBasic(t *testing.T) { + backend1 := stubserver.StartTestService(t, nil) + defer backend1.Stop() + backend2 := stubserver.StartTestService(t, nil) + defer backend2.Stop() + + mr := manual.NewBuilderWithScheme("e2e-test") + defer mr.Close() + + json := fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, fakePetioleName) + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(json) + mr.InitialState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: backend1.Address}}}, + {Addresses: []resolver.Address{{Addr: backend2.Address}}}, + }, + ServiceConfig: sc, + }) + + dOpts := []grpc.DialOption{ + grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials()), + // Use a large backoff delay to avoid the error picker being updated + // too quickly. + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.Config{ + BaseDelay: 2 * defaultTestTimeout, + Multiplier: float64(0), + Jitter: float64(0), + MaxDelay: 2 * defaultTestTimeout, + }, + }), + } + cc, err := grpc.NewClient(mr.Scheme()+":///", dOpts...) + if err != nil { + t.Fatalf("Failed to create new client: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + // Assert a round robin distribution between the two spun up backends. This + // requires a poll and eventual consistency as both endpoint children do not + // start in state READY. + if err = roundrobin.CheckRoundRobinRPCs(ctx, client, []resolver.Address{{Addr: backend1.Address}, {Addr: backend2.Address}}); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // Stopping both the backends should make the channel enter + // TransientFailure. + backend1.Stop() + backend2.Stop() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // When the resolver reports an error, the picker should get updated to + // return the resolver error. + mr.CC().ReportError(errors.New("test error")) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Fatalf("EmptyCall succeeded when expected to fail with %q", "test error") + } + if strings.Contains(err.Error(), "test error") { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Context timed out waiting for picker with resolver error.") + } +} + +// Tests that endpointsharding doesn't automatically re-connect IDLE children. +// The test creates an endpoint with two servers and another with a single +// server. The active service in endpoint 1 is closed to make the child +// pickfirst enter IDLE state. The test verifies that the child pickfirst +// doesn't connect to the second address in the endpoint. +func (s) TestEndpointShardingReconnectDisabled(t *testing.T) { + backend1 := stubserver.StartTestService(t, nil) + defer backend1.Stop() + backend2 := stubserver.StartTestService(t, nil) + defer backend2.Stop() + backend3 := stubserver.StartTestService(t, nil) + defer backend3.Stop() + + mr := manual.NewBuilderWithScheme("e2e-test") + defer mr.Close() + + name := strings.ReplaceAll(strings.ToLower(t.Name()), "/", "") + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + epOpts := endpointsharding.Options{DisableAutoReconnect: true} + bd.ChildBalancer = endpointsharding.NewBalancer(bd.ClientConn, bd.BuildOptions, balancer.Get(pickfirst.Name).Build, epOpts) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + } + stub.Register(name, bf) + + json := fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, name) + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(json) + mr.InitialState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: backend1.Address}, {Addr: backend2.Address}}}, + {Addresses: []resolver.Address{{Addr: backend3.Address}}}, + }, + ServiceConfig: sc, + }) + + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create new client: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + // Assert a round robin distribution between the two spun up backends. This + // requires a poll and eventual consistency as both endpoint children do not + // start in state READY. + if err = roundrobin.CheckRoundRobinRPCs(ctx, client, []resolver.Address{{Addr: backend1.Address}, {Addr: backend3.Address}}); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // On closing the first server, the first child balancer should enter + // IDLE. Since endpointsharding is configured not to auto-reconnect, it will + // remain IDLE and will not try to connect to the second backend in the same + // endpoint. + backend1.Stop() + // CheckRoundRobinRPCs waits for all the backends to become reachable, we + // call it to ensure the picker no longer sends RPCs to closed backend. + if err = roundrobin.CheckRoundRobinRPCs(ctx, client, []resolver.Address{{Addr: backend3.Address}}); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // Verify requests go only to backend3 for a short time. + shortCtx, cancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer cancel() + for ; shortCtx.Err() == nil; <-time.After(time.Millisecond) { + var peer peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + if status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() returned unexpected error %v", err) + } + break + } + if got, want := peer.Addr.String(), backend3.Address; got != want { + t.Fatalf("EmptyCall() went to unexpected backend: got %q, want %q", got, want) + } + } +} + +// Tests that endpointsharding doesn't automatically re-connect IDLE children +// until cc.Connect() is called. The test creates an endpoint with a single +// address. The client is connected and the active server is closed to make the +// child pickfirst enter IDLE state. The test verifies that the child pickfirst +// doesn't re-connect automatically. The test calls cc.Connect() and verified +// that the balancer connects causing the channel to enter TransientFailure. +func (s) TestEndpointShardingExitIdle(t *testing.T) { + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + + mr := manual.NewBuilderWithScheme("e2e-test") + defer mr.Close() + + name := strings.ReplaceAll(strings.ToLower(t.Name()), "/", "") + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + epOpts := endpointsharding.Options{DisableAutoReconnect: true} + bd.ChildBalancer = endpointsharding.NewBalancer(bd.ClientConn, bd.BuildOptions, balancer.Get(pickfirst.Name).Build, epOpts) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + ExitIdle: func(bd *stub.BalancerData) { + bd.ChildBalancer.ExitIdle() + }, + } + stub.Register(name, bf) + + json := fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, name) + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(json) + mr.InitialState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: backend.Address}}}, + }, + ServiceConfig: sc, + }) + + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create new client: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("client.EmptyCall() returned unexpected error: %v", err) + } + + // On closing the first server, the first child balancer should enter + // IDLE. Since endpointsharding is configured not to auto-reconnect, it will + // remain IDLE and will not try to re-connect + backend.Stop() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer shortCancel() + testutils.AwaitNoStateChange(shortCtx, t, cc, connectivity.Idle) + + // The balancer should try to re-connect and fail. + cc.Connect() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) +} diff --git a/balancer/endpointsharding/endpointsharding_test.go b/balancer/endpointsharding/endpointsharding_test.go new file mode 100644 index 0000000..c4afe7a --- /dev/null +++ b/balancer/endpointsharding/endpointsharding_test.go @@ -0,0 +1,83 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package endpointsharding + +import ( + "fmt" + "testing" + + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/resolver" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestRotateEndpoints(t *testing.T) { + ep := func(addr string) resolver.Endpoint { + return resolver.Endpoint{Addresses: []resolver.Address{{Addr: addr}}} + } + endpoints := []resolver.Endpoint{ep("1"), ep("2"), ep("3"), ep("4"), ep("5")} + testCases := []struct { + rval int + want []resolver.Endpoint + }{ + { + rval: 0, + want: []resolver.Endpoint{ep("1"), ep("2"), ep("3"), ep("4"), ep("5")}, + }, + { + rval: 1, + want: []resolver.Endpoint{ep("2"), ep("3"), ep("4"), ep("5"), ep("1")}, + }, + { + rval: 2, + want: []resolver.Endpoint{ep("3"), ep("4"), ep("5"), ep("1"), ep("2")}, + }, + { + rval: 3, + want: []resolver.Endpoint{ep("4"), ep("5"), ep("1"), ep("2"), ep("3")}, + }, + { + rval: 4, + want: []resolver.Endpoint{ep("5"), ep("1"), ep("2"), ep("3"), ep("4")}, + }, + } + + defer func(r func(int) int) { + randIntN = r + }(randIntN) + + for _, tc := range testCases { + t.Run(fmt.Sprint(tc.rval), func(t *testing.T) { + randIntN = func(int) int { + return tc.rval + } + got := rotateEndpoints(endpoints) + if fmt.Sprint(got) != fmt.Sprint(tc.want) { + t.Fatalf("rand=%v; rotateEndpoints(%v) = %v; want %v", tc.rval, endpoints, got, tc.want) + } + }) + } +} diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go new file mode 100644 index 0000000..2004b12 --- /dev/null +++ b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go @@ -0,0 +1,763 @@ +// Copyright 2015 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines the GRPCLB LoadBalancing protocol. +// +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/lb/v1/load_balancer.proto + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/lb/v1/load_balancer.proto + +package grpc_lb_v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type LoadBalanceRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to LoadBalanceRequestType: + // + // *LoadBalanceRequest_InitialRequest + // *LoadBalanceRequest_ClientStats + LoadBalanceRequestType isLoadBalanceRequest_LoadBalanceRequestType `protobuf_oneof:"load_balance_request_type"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoadBalanceRequest) Reset() { + *x = LoadBalanceRequest{} + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoadBalanceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalanceRequest) ProtoMessage() {} + +func (x *LoadBalanceRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalanceRequest.ProtoReflect.Descriptor instead. +func (*LoadBalanceRequest) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{0} +} + +func (x *LoadBalanceRequest) GetLoadBalanceRequestType() isLoadBalanceRequest_LoadBalanceRequestType { + if x != nil { + return x.LoadBalanceRequestType + } + return nil +} + +func (x *LoadBalanceRequest) GetInitialRequest() *InitialLoadBalanceRequest { + if x != nil { + if x, ok := x.LoadBalanceRequestType.(*LoadBalanceRequest_InitialRequest); ok { + return x.InitialRequest + } + } + return nil +} + +func (x *LoadBalanceRequest) GetClientStats() *ClientStats { + if x != nil { + if x, ok := x.LoadBalanceRequestType.(*LoadBalanceRequest_ClientStats); ok { + return x.ClientStats + } + } + return nil +} + +type isLoadBalanceRequest_LoadBalanceRequestType interface { + isLoadBalanceRequest_LoadBalanceRequestType() +} + +type LoadBalanceRequest_InitialRequest struct { + // This message should be sent on the first request to the load balancer. + InitialRequest *InitialLoadBalanceRequest `protobuf:"bytes,1,opt,name=initial_request,json=initialRequest,proto3,oneof"` +} + +type LoadBalanceRequest_ClientStats struct { + // The client stats should be periodically reported to the load balancer + // based on the duration defined in the InitialLoadBalanceResponse. + ClientStats *ClientStats `protobuf:"bytes,2,opt,name=client_stats,json=clientStats,proto3,oneof"` +} + +func (*LoadBalanceRequest_InitialRequest) isLoadBalanceRequest_LoadBalanceRequestType() {} + +func (*LoadBalanceRequest_ClientStats) isLoadBalanceRequest_LoadBalanceRequestType() {} + +type InitialLoadBalanceRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The name of the load balanced service (e.g., service.googleapis.com). Its + // length should be less than 256 bytes. + // The name might include a port number. How to handle the port number is up + // to the balancer. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InitialLoadBalanceRequest) Reset() { + *x = InitialLoadBalanceRequest{} + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InitialLoadBalanceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InitialLoadBalanceRequest) ProtoMessage() {} + +func (x *InitialLoadBalanceRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InitialLoadBalanceRequest.ProtoReflect.Descriptor instead. +func (*InitialLoadBalanceRequest) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{1} +} + +func (x *InitialLoadBalanceRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +// Contains the number of calls finished for a particular load balance token. +type ClientStatsPerToken struct { + state protoimpl.MessageState `protogen:"open.v1"` + // See Server.load_balance_token. + LoadBalanceToken string `protobuf:"bytes,1,opt,name=load_balance_token,json=loadBalanceToken,proto3" json:"load_balance_token,omitempty"` + // The total number of RPCs that finished associated with the token. + NumCalls int64 `protobuf:"varint,2,opt,name=num_calls,json=numCalls,proto3" json:"num_calls,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClientStatsPerToken) Reset() { + *x = ClientStatsPerToken{} + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClientStatsPerToken) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientStatsPerToken) ProtoMessage() {} + +func (x *ClientStatsPerToken) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientStatsPerToken.ProtoReflect.Descriptor instead. +func (*ClientStatsPerToken) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{2} +} + +func (x *ClientStatsPerToken) GetLoadBalanceToken() string { + if x != nil { + return x.LoadBalanceToken + } + return "" +} + +func (x *ClientStatsPerToken) GetNumCalls() int64 { + if x != nil { + return x.NumCalls + } + return 0 +} + +// Contains client level statistics that are useful to load balancing. Each +// count except the timestamp should be reset to zero after reporting the stats. +type ClientStats struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The timestamp of generating the report. + Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + // The total number of RPCs that started. + NumCallsStarted int64 `protobuf:"varint,2,opt,name=num_calls_started,json=numCallsStarted,proto3" json:"num_calls_started,omitempty"` + // The total number of RPCs that finished. + NumCallsFinished int64 `protobuf:"varint,3,opt,name=num_calls_finished,json=numCallsFinished,proto3" json:"num_calls_finished,omitempty"` + // The total number of RPCs that failed to reach a server except dropped RPCs. + NumCallsFinishedWithClientFailedToSend int64 `protobuf:"varint,6,opt,name=num_calls_finished_with_client_failed_to_send,json=numCallsFinishedWithClientFailedToSend,proto3" json:"num_calls_finished_with_client_failed_to_send,omitempty"` + // The total number of RPCs that finished and are known to have been received + // by a server. + NumCallsFinishedKnownReceived int64 `protobuf:"varint,7,opt,name=num_calls_finished_known_received,json=numCallsFinishedKnownReceived,proto3" json:"num_calls_finished_known_received,omitempty"` + // The list of dropped calls. + CallsFinishedWithDrop []*ClientStatsPerToken `protobuf:"bytes,8,rep,name=calls_finished_with_drop,json=callsFinishedWithDrop,proto3" json:"calls_finished_with_drop,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClientStats) Reset() { + *x = ClientStats{} + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClientStats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientStats) ProtoMessage() {} + +func (x *ClientStats) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientStats.ProtoReflect.Descriptor instead. +func (*ClientStats) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{3} +} + +func (x *ClientStats) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp + } + return nil +} + +func (x *ClientStats) GetNumCallsStarted() int64 { + if x != nil { + return x.NumCallsStarted + } + return 0 +} + +func (x *ClientStats) GetNumCallsFinished() int64 { + if x != nil { + return x.NumCallsFinished + } + return 0 +} + +func (x *ClientStats) GetNumCallsFinishedWithClientFailedToSend() int64 { + if x != nil { + return x.NumCallsFinishedWithClientFailedToSend + } + return 0 +} + +func (x *ClientStats) GetNumCallsFinishedKnownReceived() int64 { + if x != nil { + return x.NumCallsFinishedKnownReceived + } + return 0 +} + +func (x *ClientStats) GetCallsFinishedWithDrop() []*ClientStatsPerToken { + if x != nil { + return x.CallsFinishedWithDrop + } + return nil +} + +type LoadBalanceResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to LoadBalanceResponseType: + // + // *LoadBalanceResponse_InitialResponse + // *LoadBalanceResponse_ServerList + // *LoadBalanceResponse_FallbackResponse + LoadBalanceResponseType isLoadBalanceResponse_LoadBalanceResponseType `protobuf_oneof:"load_balance_response_type"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoadBalanceResponse) Reset() { + *x = LoadBalanceResponse{} + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoadBalanceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalanceResponse) ProtoMessage() {} + +func (x *LoadBalanceResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalanceResponse.ProtoReflect.Descriptor instead. +func (*LoadBalanceResponse) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{4} +} + +func (x *LoadBalanceResponse) GetLoadBalanceResponseType() isLoadBalanceResponse_LoadBalanceResponseType { + if x != nil { + return x.LoadBalanceResponseType + } + return nil +} + +func (x *LoadBalanceResponse) GetInitialResponse() *InitialLoadBalanceResponse { + if x != nil { + if x, ok := x.LoadBalanceResponseType.(*LoadBalanceResponse_InitialResponse); ok { + return x.InitialResponse + } + } + return nil +} + +func (x *LoadBalanceResponse) GetServerList() *ServerList { + if x != nil { + if x, ok := x.LoadBalanceResponseType.(*LoadBalanceResponse_ServerList); ok { + return x.ServerList + } + } + return nil +} + +func (x *LoadBalanceResponse) GetFallbackResponse() *FallbackResponse { + if x != nil { + if x, ok := x.LoadBalanceResponseType.(*LoadBalanceResponse_FallbackResponse); ok { + return x.FallbackResponse + } + } + return nil +} + +type isLoadBalanceResponse_LoadBalanceResponseType interface { + isLoadBalanceResponse_LoadBalanceResponseType() +} + +type LoadBalanceResponse_InitialResponse struct { + // This message should be sent on the first response to the client. + InitialResponse *InitialLoadBalanceResponse `protobuf:"bytes,1,opt,name=initial_response,json=initialResponse,proto3,oneof"` +} + +type LoadBalanceResponse_ServerList struct { + // Contains the list of servers selected by the load balancer. The client + // should send requests to these servers in the specified order. + ServerList *ServerList `protobuf:"bytes,2,opt,name=server_list,json=serverList,proto3,oneof"` +} + +type LoadBalanceResponse_FallbackResponse struct { + // If this field is set, then the client should eagerly enter fallback + // mode (even if there are existing, healthy connections to backends). + FallbackResponse *FallbackResponse `protobuf:"bytes,3,opt,name=fallback_response,json=fallbackResponse,proto3,oneof"` +} + +func (*LoadBalanceResponse_InitialResponse) isLoadBalanceResponse_LoadBalanceResponseType() {} + +func (*LoadBalanceResponse_ServerList) isLoadBalanceResponse_LoadBalanceResponseType() {} + +func (*LoadBalanceResponse_FallbackResponse) isLoadBalanceResponse_LoadBalanceResponseType() {} + +type FallbackResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FallbackResponse) Reset() { + *x = FallbackResponse{} + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FallbackResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FallbackResponse) ProtoMessage() {} + +func (x *FallbackResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FallbackResponse.ProtoReflect.Descriptor instead. +func (*FallbackResponse) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{5} +} + +type InitialLoadBalanceResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // This interval defines how often the client should send the client stats + // to the load balancer. Stats should only be reported when the duration is + // positive. + ClientStatsReportInterval *durationpb.Duration `protobuf:"bytes,2,opt,name=client_stats_report_interval,json=clientStatsReportInterval,proto3" json:"client_stats_report_interval,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InitialLoadBalanceResponse) Reset() { + *x = InitialLoadBalanceResponse{} + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InitialLoadBalanceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InitialLoadBalanceResponse) ProtoMessage() {} + +func (x *InitialLoadBalanceResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InitialLoadBalanceResponse.ProtoReflect.Descriptor instead. +func (*InitialLoadBalanceResponse) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{6} +} + +func (x *InitialLoadBalanceResponse) GetClientStatsReportInterval() *durationpb.Duration { + if x != nil { + return x.ClientStatsReportInterval + } + return nil +} + +type ServerList struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Contains a list of servers selected by the load balancer. The list will + // be updated when server resolutions change or as needed to balance load + // across more servers. The client should consume the server list in order + // unless instructed otherwise via the client_config. + Servers []*Server `protobuf:"bytes,1,rep,name=servers,proto3" json:"servers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerList) Reset() { + *x = ServerList{} + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerList) ProtoMessage() {} + +func (x *ServerList) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerList.ProtoReflect.Descriptor instead. +func (*ServerList) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{7} +} + +func (x *ServerList) GetServers() []*Server { + if x != nil { + return x.Servers + } + return nil +} + +// Contains server information. When the drop field is not true, use the other +// fields. +type Server struct { + state protoimpl.MessageState `protogen:"open.v1"` + // A resolved address for the server, serialized in network-byte-order. It may + // either be an IPv4 or IPv6 address. + IpAddress []byte `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` + // A resolved port number for the server. + Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + // An opaque but printable token for load reporting. The client must include + // the token of the picked server into the initial metadata when it starts a + // call to that server. The token is used by the server to verify the request + // and to allow the server to report load to the gRPC LB system. The token is + // also used in client stats for reporting dropped calls. + // + // Its length can be variable but must be less than 50 bytes. + LoadBalanceToken string `protobuf:"bytes,3,opt,name=load_balance_token,json=loadBalanceToken,proto3" json:"load_balance_token,omitempty"` + // Indicates whether this particular request should be dropped by the client. + // If the request is dropped, there will be a corresponding entry in + // ClientStats.calls_finished_with_drop. + Drop bool `protobuf:"varint,4,opt,name=drop,proto3" json:"drop,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Server) Reset() { + *x = Server{} + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Server) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Server) ProtoMessage() {} + +func (x *Server) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Server.ProtoReflect.Descriptor instead. +func (*Server) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{8} +} + +func (x *Server) GetIpAddress() []byte { + if x != nil { + return x.IpAddress + } + return nil +} + +func (x *Server) GetPort() int32 { + if x != nil { + return x.Port + } + return 0 +} + +func (x *Server) GetLoadBalanceToken() string { + if x != nil { + return x.LoadBalanceToken + } + return "" +} + +func (x *Server) GetDrop() bool { + if x != nil { + return x.Drop + } + return false +} + +var File_grpc_lb_v1_load_balancer_proto protoreflect.FileDescriptor + +const file_grpc_lb_v1_load_balancer_proto_rawDesc = "" + + "\n" + + "\x1egrpc/lb/v1/load_balancer.proto\x12\n" + + "grpc.lb.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xc1\x01\n" + + "\x12LoadBalanceRequest\x12P\n" + + "\x0finitial_request\x18\x01 \x01(\v2%.grpc.lb.v1.InitialLoadBalanceRequestH\x00R\x0einitialRequest\x12<\n" + + "\fclient_stats\x18\x02 \x01(\v2\x17.grpc.lb.v1.ClientStatsH\x00R\vclientStatsB\x1b\n" + + "\x19load_balance_request_type\"/\n" + + "\x19InitialLoadBalanceRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\"`\n" + + "\x13ClientStatsPerToken\x12,\n" + + "\x12load_balance_token\x18\x01 \x01(\tR\x10loadBalanceToken\x12\x1b\n" + + "\tnum_calls\x18\x02 \x01(\x03R\bnumCalls\"\xb0\x03\n" + + "\vClientStats\x128\n" + + "\ttimestamp\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\x12*\n" + + "\x11num_calls_started\x18\x02 \x01(\x03R\x0fnumCallsStarted\x12,\n" + + "\x12num_calls_finished\x18\x03 \x01(\x03R\x10numCallsFinished\x12]\n" + + "-num_calls_finished_with_client_failed_to_send\x18\x06 \x01(\x03R&numCallsFinishedWithClientFailedToSend\x12H\n" + + "!num_calls_finished_known_received\x18\a \x01(\x03R\x1dnumCallsFinishedKnownReceived\x12X\n" + + "\x18calls_finished_with_drop\x18\b \x03(\v2\x1f.grpc.lb.v1.ClientStatsPerTokenR\x15callsFinishedWithDropJ\x04\b\x04\x10\x05J\x04\b\x05\x10\x06\"\x90\x02\n" + + "\x13LoadBalanceResponse\x12S\n" + + "\x10initial_response\x18\x01 \x01(\v2&.grpc.lb.v1.InitialLoadBalanceResponseH\x00R\x0finitialResponse\x129\n" + + "\vserver_list\x18\x02 \x01(\v2\x16.grpc.lb.v1.ServerListH\x00R\n" + + "serverList\x12K\n" + + "\x11fallback_response\x18\x03 \x01(\v2\x1c.grpc.lb.v1.FallbackResponseH\x00R\x10fallbackResponseB\x1c\n" + + "\x1aload_balance_response_type\"\x12\n" + + "\x10FallbackResponse\"~\n" + + "\x1aInitialLoadBalanceResponse\x12Z\n" + + "\x1cclient_stats_report_interval\x18\x02 \x01(\v2\x19.google.protobuf.DurationR\x19clientStatsReportIntervalJ\x04\b\x01\x10\x02\"@\n" + + "\n" + + "ServerList\x12,\n" + + "\aservers\x18\x01 \x03(\v2\x12.grpc.lb.v1.ServerR\aserversJ\x04\b\x03\x10\x04\"\x83\x01\n" + + "\x06Server\x12\x1d\n" + + "\n" + + "ip_address\x18\x01 \x01(\fR\tipAddress\x12\x12\n" + + "\x04port\x18\x02 \x01(\x05R\x04port\x12,\n" + + "\x12load_balance_token\x18\x03 \x01(\tR\x10loadBalanceToken\x12\x12\n" + + "\x04drop\x18\x04 \x01(\bR\x04dropJ\x04\b\x05\x10\x062b\n" + + "\fLoadBalancer\x12R\n" + + "\vBalanceLoad\x12\x1e.grpc.lb.v1.LoadBalanceRequest\x1a\x1f.grpc.lb.v1.LoadBalanceResponse(\x010\x01BW\n" + + "\rio.grpc.lb.v1B\x11LoadBalancerProtoP\x01Z1google.golang.org/grpc/balancer/grpclb/grpc_lb_v1b\x06proto3" + +var ( + file_grpc_lb_v1_load_balancer_proto_rawDescOnce sync.Once + file_grpc_lb_v1_load_balancer_proto_rawDescData []byte +) + +func file_grpc_lb_v1_load_balancer_proto_rawDescGZIP() []byte { + file_grpc_lb_v1_load_balancer_proto_rawDescOnce.Do(func() { + file_grpc_lb_v1_load_balancer_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_lb_v1_load_balancer_proto_rawDesc), len(file_grpc_lb_v1_load_balancer_proto_rawDesc))) + }) + return file_grpc_lb_v1_load_balancer_proto_rawDescData +} + +var file_grpc_lb_v1_load_balancer_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_grpc_lb_v1_load_balancer_proto_goTypes = []any{ + (*LoadBalanceRequest)(nil), // 0: grpc.lb.v1.LoadBalanceRequest + (*InitialLoadBalanceRequest)(nil), // 1: grpc.lb.v1.InitialLoadBalanceRequest + (*ClientStatsPerToken)(nil), // 2: grpc.lb.v1.ClientStatsPerToken + (*ClientStats)(nil), // 3: grpc.lb.v1.ClientStats + (*LoadBalanceResponse)(nil), // 4: grpc.lb.v1.LoadBalanceResponse + (*FallbackResponse)(nil), // 5: grpc.lb.v1.FallbackResponse + (*InitialLoadBalanceResponse)(nil), // 6: grpc.lb.v1.InitialLoadBalanceResponse + (*ServerList)(nil), // 7: grpc.lb.v1.ServerList + (*Server)(nil), // 8: grpc.lb.v1.Server + (*timestamppb.Timestamp)(nil), // 9: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 10: google.protobuf.Duration +} +var file_grpc_lb_v1_load_balancer_proto_depIdxs = []int32{ + 1, // 0: grpc.lb.v1.LoadBalanceRequest.initial_request:type_name -> grpc.lb.v1.InitialLoadBalanceRequest + 3, // 1: grpc.lb.v1.LoadBalanceRequest.client_stats:type_name -> grpc.lb.v1.ClientStats + 9, // 2: grpc.lb.v1.ClientStats.timestamp:type_name -> google.protobuf.Timestamp + 2, // 3: grpc.lb.v1.ClientStats.calls_finished_with_drop:type_name -> grpc.lb.v1.ClientStatsPerToken + 6, // 4: grpc.lb.v1.LoadBalanceResponse.initial_response:type_name -> grpc.lb.v1.InitialLoadBalanceResponse + 7, // 5: grpc.lb.v1.LoadBalanceResponse.server_list:type_name -> grpc.lb.v1.ServerList + 5, // 6: grpc.lb.v1.LoadBalanceResponse.fallback_response:type_name -> grpc.lb.v1.FallbackResponse + 10, // 7: grpc.lb.v1.InitialLoadBalanceResponse.client_stats_report_interval:type_name -> google.protobuf.Duration + 8, // 8: grpc.lb.v1.ServerList.servers:type_name -> grpc.lb.v1.Server + 0, // 9: grpc.lb.v1.LoadBalancer.BalanceLoad:input_type -> grpc.lb.v1.LoadBalanceRequest + 4, // 10: grpc.lb.v1.LoadBalancer.BalanceLoad:output_type -> grpc.lb.v1.LoadBalanceResponse + 10, // [10:11] is the sub-list for method output_type + 9, // [9:10] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name +} + +func init() { file_grpc_lb_v1_load_balancer_proto_init() } +func file_grpc_lb_v1_load_balancer_proto_init() { + if File_grpc_lb_v1_load_balancer_proto != nil { + return + } + file_grpc_lb_v1_load_balancer_proto_msgTypes[0].OneofWrappers = []any{ + (*LoadBalanceRequest_InitialRequest)(nil), + (*LoadBalanceRequest_ClientStats)(nil), + } + file_grpc_lb_v1_load_balancer_proto_msgTypes[4].OneofWrappers = []any{ + (*LoadBalanceResponse_InitialResponse)(nil), + (*LoadBalanceResponse_ServerList)(nil), + (*LoadBalanceResponse_FallbackResponse)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_lb_v1_load_balancer_proto_rawDesc), len(file_grpc_lb_v1_load_balancer_proto_rawDesc)), + NumEnums: 0, + NumMessages: 9, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_lb_v1_load_balancer_proto_goTypes, + DependencyIndexes: file_grpc_lb_v1_load_balancer_proto_depIdxs, + MessageInfos: file_grpc_lb_v1_load_balancer_proto_msgTypes, + }.Build() + File_grpc_lb_v1_load_balancer_proto = out.File + file_grpc_lb_v1_load_balancer_proto_goTypes = nil + file_grpc_lb_v1_load_balancer_proto_depIdxs = nil +} diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go new file mode 100644 index 0000000..6aeb819 --- /dev/null +++ b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go @@ -0,0 +1,134 @@ +// Copyright 2015 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines the GRPCLB LoadBalancing protocol. +// +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/lb/v1/load_balancer.proto + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: grpc/lb/v1/load_balancer.proto + +package grpc_lb_v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + LoadBalancer_BalanceLoad_FullMethodName = "/grpc.lb.v1.LoadBalancer/BalanceLoad" +) + +// LoadBalancerClient is the client API for LoadBalancer service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type LoadBalancerClient interface { + // Bidirectional rpc to get a list of servers. + BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[LoadBalanceRequest, LoadBalanceResponse], error) +} + +type loadBalancerClient struct { + cc grpc.ClientConnInterface +} + +func NewLoadBalancerClient(cc grpc.ClientConnInterface) LoadBalancerClient { + return &loadBalancerClient{cc} +} + +func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[LoadBalanceRequest, LoadBalanceResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &LoadBalancer_ServiceDesc.Streams[0], LoadBalancer_BalanceLoad_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[LoadBalanceRequest, LoadBalanceResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type LoadBalancer_BalanceLoadClient = grpc.BidiStreamingClient[LoadBalanceRequest, LoadBalanceResponse] + +// LoadBalancerServer is the server API for LoadBalancer service. +// All implementations should embed UnimplementedLoadBalancerServer +// for forward compatibility. +type LoadBalancerServer interface { + // Bidirectional rpc to get a list of servers. + BalanceLoad(grpc.BidiStreamingServer[LoadBalanceRequest, LoadBalanceResponse]) error +} + +// UnimplementedLoadBalancerServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedLoadBalancerServer struct{} + +func (UnimplementedLoadBalancerServer) BalanceLoad(grpc.BidiStreamingServer[LoadBalanceRequest, LoadBalanceResponse]) error { + return status.Error(codes.Unimplemented, "method BalanceLoad not implemented") +} +func (UnimplementedLoadBalancerServer) testEmbeddedByValue() {} + +// UnsafeLoadBalancerServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to LoadBalancerServer will +// result in compilation errors. +type UnsafeLoadBalancerServer interface { + mustEmbedUnimplementedLoadBalancerServer() +} + +func RegisterLoadBalancerServer(s grpc.ServiceRegistrar, srv LoadBalancerServer) { + // If the following call panics, it indicates UnimplementedLoadBalancerServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&LoadBalancer_ServiceDesc, srv) +} + +func _LoadBalancer_BalanceLoad_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(LoadBalancerServer).BalanceLoad(&grpc.GenericServerStream[LoadBalanceRequest, LoadBalanceResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type LoadBalancer_BalanceLoadServer = grpc.BidiStreamingServer[LoadBalanceRequest, LoadBalanceResponse] + +// LoadBalancer_ServiceDesc is the grpc.ServiceDesc for LoadBalancer service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var LoadBalancer_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.lb.v1.LoadBalancer", + HandlerType: (*LoadBalancerServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "BalanceLoad", + Handler: _LoadBalancer_BalanceLoad_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/lb/v1/load_balancer.proto", +} diff --git a/balancer/grpclb/grpclb.go b/balancer/grpclb/grpclb.go new file mode 100644 index 0000000..0770b88 --- /dev/null +++ b/balancer/grpclb/grpclb.go @@ -0,0 +1,535 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpclb defines a grpclb balancer. +// +// To install grpclb balancer, import this package as: +// +// import _ "google.golang.org/grpc/balancer/grpclb" +package grpclb + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + grpclbstate "google.golang.org/grpc/balancer/grpclb/state" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/backoff" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/resolver/dns" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/protobuf/types/known/durationpb" + + lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" +) + +const ( + lbTokenKey = "lb-token" + defaultFallbackTimeout = 10 * time.Second + grpclbName = "grpclb" +) + +var errServerTerminatedConnection = errors.New("grpclb: failed to recv server list: server terminated connection") +var logger = grpclog.Component("grpclb") + +func convertDuration(d *durationpb.Duration) time.Duration { + if d == nil { + return 0 + } + return time.Duration(d.Seconds)*time.Second + time.Duration(d.Nanos)*time.Nanosecond +} + +// Client API for LoadBalancer service. +// Mostly copied from generated pb.go file. +// To avoid circular dependency. +type loadBalancerClient struct { + cc *grpc.ClientConn +} + +func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (*balanceLoadClientStream, error) { + desc := &grpc.StreamDesc{ + StreamName: "BalanceLoad", + ServerStreams: true, + ClientStreams: true, + } + stream, err := c.cc.NewStream(ctx, desc, "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...) + if err != nil { + return nil, err + } + x := &balanceLoadClientStream{stream} + return x, nil +} + +type balanceLoadClientStream struct { + grpc.ClientStream +} + +func (x *balanceLoadClientStream) Send(m *lbpb.LoadBalanceRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *balanceLoadClientStream) Recv() (*lbpb.LoadBalanceResponse, error) { + m := new(lbpb.LoadBalanceResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func init() { + balancer.Register(newLBBuilder()) + dns.EnableSRVLookups = true +} + +// newLBBuilder creates a builder for grpclb. +func newLBBuilder() balancer.Builder { + return newLBBuilderWithFallbackTimeout(defaultFallbackTimeout) +} + +// newLBBuilderWithFallbackTimeout creates a grpclb builder with the given +// fallbackTimeout. If no response is received from the remote balancer within +// fallbackTimeout, the backend addresses from the resolved address list will be +// used. +// +// Only call this function when a non-default fallback timeout is needed. +func newLBBuilderWithFallbackTimeout(fallbackTimeout time.Duration) balancer.Builder { + return &lbBuilder{ + fallbackTimeout: fallbackTimeout, + } +} + +type lbBuilder struct { + fallbackTimeout time.Duration +} + +func (b *lbBuilder) Name() string { + return grpclbName +} + +func (b *lbBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { + // This generates a manual resolver builder with a fixed scheme. This + // scheme will be used to dial to remote LB, so we can send filtered + // address updates to remote LB ClientConn using this manual resolver. + mr := manual.NewBuilderWithScheme("grpclb-internal") + // ResolveNow() on this manual resolver is forwarded to the parent + // ClientConn, so when grpclb client loses contact with the remote balancer, + // the parent ClientConn's resolver will re-resolve. + mr.ResolveNowCallback = cc.ResolveNow + + lb := &lbBalancer{ + cc: newLBCacheClientConn(cc), + dialTarget: opt.Target.Endpoint(), + target: opt.Target.Endpoint(), + opt: opt, + fallbackTimeout: b.fallbackTimeout, + doneCh: make(chan struct{}), + + manualResolver: mr, + subConns: make(map[resolver.Address]balancer.SubConn), + scStates: make(map[balancer.SubConn]connectivity.State), + picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + clientStats: newRPCStats(), + backoff: backoff.DefaultExponential, // TODO: make backoff configurable. + } + lb.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[grpclb %p] ", lb)) + + var err error + if opt.CredsBundle != nil { + lb.grpclbClientConnCreds, err = opt.CredsBundle.NewWithMode(internal.CredsBundleModeBalancer) + if err != nil { + lb.logger.Warningf("Failed to create credentials used for connecting to grpclb: %v", err) + } + lb.grpclbBackendCreds, err = opt.CredsBundle.NewWithMode(internal.CredsBundleModeBackendFromBalancer) + if err != nil { + lb.logger.Warningf("Failed to create credentials used for connecting to backends returned by grpclb: %v", err) + } + } + + return lb +} + +type lbBalancer struct { + cc *lbCacheClientConn + dialTarget string // user's dial target + target string // same as dialTarget unless overridden in service config + opt balancer.BuildOptions + logger *internalgrpclog.PrefixLogger + + usePickFirst bool + + // grpclbClientConnCreds is the creds bundle to be used to connect to grpclb + // servers. If it's nil, use the TransportCredentials from BuildOptions + // instead. + grpclbClientConnCreds credentials.Bundle + // grpclbBackendCreds is the creds bundle to be used for addresses that are + // returned by grpclb server. If it's nil, don't set anything when creating + // SubConns. + grpclbBackendCreds credentials.Bundle + + fallbackTimeout time.Duration + doneCh chan struct{} + + // manualResolver is used in the remote LB ClientConn inside grpclb. When + // resolved address updates are received by grpclb, filtered updates will be + // sent to remote LB ClientConn through this resolver. + manualResolver *manual.Resolver + // The ClientConn to talk to the remote balancer. + ccRemoteLB *remoteBalancerCCWrapper + // backoff for calling remote balancer. + backoff backoff.Strategy + + // Support client side load reporting. Each picker gets a reference to this, + // and will update its content. + clientStats *rpcStats + + mu sync.Mutex // guards everything following. + // The full server list including drops, used to check if the newly received + // serverList contains anything new. Each generate picker will also have + // reference to this list to do the first layer pick. + fullServerList []*lbpb.Server + // Backend addresses. It's kept so the addresses are available when + // switching between round_robin and pickfirst. + backendAddrs []resolver.Address + // All backends addresses, with metadata set to nil. This list contains all + // backend addresses in the same order and with the same duplicates as in + // serverlist. When generating picker, a SubConn slice with the same order + // but with only READY SCs will be generated. + backendAddrsWithoutMetadata []resolver.Address + // Roundrobin functionalities. + state connectivity.State + subConns map[resolver.Address]balancer.SubConn // Used to new/shutdown SubConn. + scStates map[balancer.SubConn]connectivity.State // Used to filter READY SubConns. + picker balancer.Picker + // Support fallback to resolved backend addresses if there's no response + // from remote balancer within fallbackTimeout. + remoteBalancerConnected bool + serverListReceived bool + inFallback bool + // resolvedBackendAddrs is resolvedAddrs minus remote balancers. It's set + // when resolved address updates are received, and read in the goroutine + // handling fallback. + resolvedBackendAddrs []resolver.Address + connErr error // the last connection error +} + +// regeneratePicker takes a snapshot of the balancer, and generates a picker from +// it. The picker +// - always returns ErrTransientFailure if the balancer is in TransientFailure, +// - does two layer roundrobin pick otherwise. +// +// Caller must hold lb.mu. +func (lb *lbBalancer) regeneratePicker(resetDrop bool) { + if lb.state == connectivity.TransientFailure { + lb.picker = base.NewErrPicker(fmt.Errorf("all SubConns are in TransientFailure, last connection error: %v", lb.connErr)) + return + } + + if lb.state == connectivity.Connecting { + lb.picker = base.NewErrPicker(balancer.ErrNoSubConnAvailable) + return + } + + var readySCs []balancer.SubConn + if lb.usePickFirst { + for _, sc := range lb.subConns { + readySCs = append(readySCs, sc) + break + } + } else { + for _, a := range lb.backendAddrsWithoutMetadata { + if sc, ok := lb.subConns[a]; ok { + if st, ok := lb.scStates[sc]; ok && st == connectivity.Ready { + readySCs = append(readySCs, sc) + } + } + } + } + + if len(readySCs) <= 0 { + // If there's no ready SubConns, always re-pick. This is to avoid drops + // unless at least one SubConn is ready. Otherwise we may drop more + // often than want because of drops + re-picks(which become re-drops). + // + // This doesn't seem to be necessary after the connecting check above. + // Kept for safety. + lb.picker = base.NewErrPicker(balancer.ErrNoSubConnAvailable) + return + } + if lb.inFallback { + lb.picker = newRRPicker(readySCs) + return + } + if resetDrop { + lb.picker = newLBPicker(lb.fullServerList, readySCs, lb.clientStats) + return + } + prevLBPicker, ok := lb.picker.(*lbPicker) + if !ok { + lb.picker = newLBPicker(lb.fullServerList, readySCs, lb.clientStats) + return + } + prevLBPicker.updateReadySCs(readySCs) +} + +// aggregateSubConnStats calculate the aggregated state of SubConns in +// lb.SubConns. These SubConns are subconns in use (when switching between +// fallback and grpclb). lb.scState contains states for all SubConns, including +// those in cache (SubConns are cached for 10 seconds after shutdown). +// +// The aggregated state is: +// - If at least one SubConn in Ready, the aggregated state is Ready; +// - Else if at least one SubConn in Connecting or IDLE, the aggregated state is Connecting; +// - It's OK to consider IDLE as Connecting. SubConns never stay in IDLE, +// they start to connect immediately. But there's a race between the overall +// state is reported, and when the new SubConn state arrives. And SubConns +// never go back to IDLE. +// - Else the aggregated state is TransientFailure. +func (lb *lbBalancer) aggregateSubConnStates() connectivity.State { + var numConnecting uint64 + + for _, sc := range lb.subConns { + if state, ok := lb.scStates[sc]; ok { + switch state { + case connectivity.Ready: + return connectivity.Ready + case connectivity.Connecting, connectivity.Idle: + numConnecting++ + } + } + } + if numConnecting > 0 { + return connectivity.Connecting + } + return connectivity.TransientFailure +} + +// UpdateSubConnState is unused; NewSubConn's options always specifies +// updateSubConnState as the listener. +func (lb *lbBalancer) UpdateSubConnState(sc balancer.SubConn, scs balancer.SubConnState) { + lb.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, scs) +} + +func (lb *lbBalancer) updateSubConnState(sc balancer.SubConn, scs balancer.SubConnState) { + s := scs.ConnectivityState + if lb.logger.V(2) { + lb.logger.Infof("SubConn state change: %p, %v", sc, s) + } + lb.mu.Lock() + defer lb.mu.Unlock() + + oldS, ok := lb.scStates[sc] + if !ok { + if lb.logger.V(2) { + lb.logger.Infof("Received state change for an unknown SubConn: %p, %v", sc, s) + } + return + } + lb.scStates[sc] = s + switch s { + case connectivity.Idle: + sc.Connect() + case connectivity.Shutdown: + // When an address was removed by resolver, b called Shutdown but kept + // the sc's state in scStates. Remove state for this sc here. + delete(lb.scStates, sc) + case connectivity.TransientFailure: + lb.connErr = scs.ConnectionError + } + // Force regenerate picker if + // - this sc became ready from not-ready + // - this sc became not-ready from ready + lb.updateStateAndPicker((oldS == connectivity.Ready) != (s == connectivity.Ready), false) + + // Enter fallback when the aggregated state is not Ready and the connection + // to remote balancer is lost. + if lb.state != connectivity.Ready { + if !lb.inFallback && !lb.remoteBalancerConnected { + // Enter fallback. + lb.refreshSubConns(lb.resolvedBackendAddrs, true, lb.usePickFirst) + } + } +} + +// updateStateAndPicker re-calculate the aggregated state, and regenerate picker +// if overall state is changed. +// +// If forceRegeneratePicker is true, picker will be regenerated. +func (lb *lbBalancer) updateStateAndPicker(forceRegeneratePicker bool, resetDrop bool) { + oldAggrState := lb.state + lb.state = lb.aggregateSubConnStates() + // Regenerate picker when one of the following happens: + // - caller wants to regenerate + // - the aggregated state changed + if forceRegeneratePicker || (lb.state != oldAggrState) { + lb.regeneratePicker(resetDrop) + } + var cc balancer.ClientConn = lb.cc + if lb.usePickFirst { + // Bypass the caching layer that would wrap the picker. + cc = lb.cc.ClientConn + } + + cc.UpdateState(balancer.State{ConnectivityState: lb.state, Picker: lb.picker}) +} + +// fallbackToBackendsAfter blocks for fallbackTimeout and falls back to use +// resolved backends (backends received from resolver, not from remote balancer) +// if no connection to remote balancers was successful. +func (lb *lbBalancer) fallbackToBackendsAfter(fallbackTimeout time.Duration) { + timer := time.NewTimer(fallbackTimeout) + defer timer.Stop() + select { + case <-timer.C: + case <-lb.doneCh: + return + } + lb.mu.Lock() + if lb.inFallback || lb.serverListReceived { + lb.mu.Unlock() + return + } + // Enter fallback. + lb.refreshSubConns(lb.resolvedBackendAddrs, true, lb.usePickFirst) + lb.mu.Unlock() +} + +func (lb *lbBalancer) handleServiceConfig(gc *grpclbServiceConfig) { + lb.mu.Lock() + defer lb.mu.Unlock() + + // grpclb uses the user's dial target to populate the `Name` field of the + // `InitialLoadBalanceRequest` message sent to the remote balancer. But when + // grpclb is used a child policy in the context of RLS, we want the `Name` + // field to be populated with the value received from the RLS server. To + // support this use case, an optional "target_name" field has been added to + // the grpclb LB policy's config. If specified, it overrides the name of + // the target to be sent to the remote balancer; if not, the target to be + // sent to the balancer will continue to be obtained from the target URI + // passed to the gRPC client channel. Whenever that target to be sent to the + // balancer is updated, we need to restart the stream to the balancer as + // this target is sent in the first message on the stream. + if gc != nil { + target := lb.dialTarget + if gc.ServiceName != "" { + target = gc.ServiceName + } + if target != lb.target { + lb.target = target + if lb.ccRemoteLB != nil { + lb.ccRemoteLB.cancelRemoteBalancerCall() + } + } + } + + newUsePickFirst := childIsPickFirst(gc) + if lb.usePickFirst == newUsePickFirst { + return + } + if lb.logger.V(2) { + lb.logger.Infof("Switching mode. Is pick_first used for backends? %v", newUsePickFirst) + } + lb.refreshSubConns(lb.backendAddrs, lb.inFallback, newUsePickFirst) +} + +func (lb *lbBalancer) ResolverError(error) { + // Ignore resolver errors. GRPCLB is not selected unless the resolver + // works at least once. +} + +func (lb *lbBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { + if lb.logger.V(2) { + lb.logger.Infof("UpdateClientConnState: %s", pretty.ToJSON(ccs)) + } + gc, _ := ccs.BalancerConfig.(*grpclbServiceConfig) + lb.handleServiceConfig(gc) + + backendAddrs := ccs.ResolverState.Addresses + + var remoteBalancerAddrs []resolver.Address + if sd := grpclbstate.Get(ccs.ResolverState); sd != nil { + // Override any balancer addresses provided via + // ccs.ResolverState.Addresses. + remoteBalancerAddrs = sd.BalancerAddresses + } + + if len(backendAddrs)+len(remoteBalancerAddrs) == 0 { + // There should be at least one address, either grpclb server or + // fallback. Empty address is not valid. + return balancer.ErrBadResolverState + } + + if len(remoteBalancerAddrs) == 0 { + if lb.ccRemoteLB != nil { + lb.ccRemoteLB.close() + lb.ccRemoteLB = nil + } + } else if lb.ccRemoteLB == nil { + // First time receiving resolved addresses, create a cc to remote + // balancers. + if err := lb.newRemoteBalancerCCWrapper(); err != nil { + return err + } + // Start the fallback goroutine. + go lb.fallbackToBackendsAfter(lb.fallbackTimeout) + } + + if lb.ccRemoteLB != nil { + // cc to remote balancers uses lb.manualResolver. Send the updated remote + // balancer addresses to it through manualResolver. + lb.manualResolver.UpdateState(resolver.State{Addresses: remoteBalancerAddrs}) + } + + lb.mu.Lock() + lb.resolvedBackendAddrs = backendAddrs + if len(remoteBalancerAddrs) == 0 || lb.inFallback { + // If there's no remote balancer address in ClientConn update, grpclb + // enters fallback mode immediately. + // + // If a new update is received while grpclb is in fallback, update the + // list of backends being used to the new fallback backends. + lb.refreshSubConns(lb.resolvedBackendAddrs, true, lb.usePickFirst) + } + lb.mu.Unlock() + return nil +} + +func (lb *lbBalancer) Close() { + select { + case <-lb.doneCh: + return + default: + } + close(lb.doneCh) + if lb.ccRemoteLB != nil { + lb.ccRemoteLB.close() + } + lb.cc.close() +} + +func (lb *lbBalancer) ExitIdle() {} diff --git a/balancer/grpclb/grpclb_config.go b/balancer/grpclb/grpclb_config.go new file mode 100644 index 0000000..96a57c8 --- /dev/null +++ b/balancer/grpclb/grpclb_config.go @@ -0,0 +1,67 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpclb + +import ( + "encoding/json" + + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/serviceconfig" +) + +const ( + roundRobinName = roundrobin.Name + pickFirstName = pickfirst.Name +) + +type grpclbServiceConfig struct { + serviceconfig.LoadBalancingConfig + ChildPolicy *[]map[string]json.RawMessage + ServiceName string +} + +func (b *lbBuilder) ParseConfig(lbConfig json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + ret := &grpclbServiceConfig{} + if err := json.Unmarshal(lbConfig, ret); err != nil { + return nil, err + } + return ret, nil +} + +func childIsPickFirst(sc *grpclbServiceConfig) bool { + if sc == nil { + return false + } + childConfigs := sc.ChildPolicy + if childConfigs == nil { + return false + } + for _, childC := range *childConfigs { + // If round_robin exists before pick_first, return false + if _, ok := childC[roundRobinName]; ok { + return false + } + // If pick_first is before round_robin, return true + if _, ok := childC[pickFirstName]; ok { + return true + } + } + return false +} diff --git a/balancer/grpclb/grpclb_config_test.go b/balancer/grpclb/grpclb_config_test.go new file mode 100644 index 0000000..0409087 --- /dev/null +++ b/balancer/grpclb/grpclb_config_test.go @@ -0,0 +1,123 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpclb + +import ( + "encoding/json" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/serviceconfig" +) + +func (s) TestParse(t *testing.T) { + tests := []struct { + name string + sc string + want serviceconfig.LoadBalancingConfig + wantErr bool + }{ + { + name: "empty", + sc: "", + want: nil, + wantErr: true, + }, + { + name: "success1", + sc: ` +{ + "childPolicy": [ + {"pick_first":{}} + ], + "serviceName": "foo-service" +}`, + want: &grpclbServiceConfig{ + ChildPolicy: &[]map[string]json.RawMessage{ + {"pick_first": json.RawMessage("{}")}, + }, + ServiceName: "foo-service", + }, + }, + { + name: "success2", + sc: ` +{ + "childPolicy": [ + {"round_robin":{}}, + {"pick_first":{}} + ], + "serviceName": "foo-service" +}`, + want: &grpclbServiceConfig{ + ChildPolicy: &[]map[string]json.RawMessage{ + {"round_robin": json.RawMessage("{}")}, + {"pick_first": json.RawMessage("{}")}, + }, + ServiceName: "foo-service", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := (&lbBuilder{}).ParseConfig(json.RawMessage(tt.sc)) + if (err != nil) != (tt.wantErr) { + t.Fatalf("ParseConfig(%q) returned error: %v, wantErr: %v", tt.sc, err, tt.wantErr) + } + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatalf("ParseConfig(%q) returned unexpected difference (-want +got):\n%s", tt.sc, diff) + } + }) + } +} + +func (s) TestChildIsPickFirst(t *testing.T) { + tests := []struct { + name string + s string + want bool + }{ + { + name: "pickfirst_only", + s: `{"childPolicy":[{"pick_first":{}}]}`, + want: true, + }, + { + name: "pickfirst_before_rr", + s: `{"childPolicy":[{"pick_first":{}},{"round_robin":{}}]}`, + want: true, + }, + { + name: "rr_before_pickfirst", + s: `{"childPolicy":[{"round_robin":{}},{"pick_first":{}}]}`, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gc, err := (&lbBuilder{}).ParseConfig(json.RawMessage(tt.s)) + if err != nil { + t.Fatalf("Parse(%v) = _, %v; want _, nil", tt.s, err) + } + if got := childIsPickFirst(gc.(*grpclbServiceConfig)); got != tt.want { + t.Errorf("childIsPickFirst() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/balancer/grpclb/grpclb_picker.go b/balancer/grpclb/grpclb_picker.go new file mode 100644 index 0000000..9ff0752 --- /dev/null +++ b/balancer/grpclb/grpclb_picker.go @@ -0,0 +1,193 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpclb + +import ( + rand "math/rand/v2" + "sync" + "sync/atomic" + + "google.golang.org/grpc/balancer" + lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// rpcStats is same as lbpb.ClientStats, except that numCallsDropped is a map +// instead of a slice. +type rpcStats struct { + // Only access the following fields atomically. + numCallsStarted int64 + numCallsFinished int64 + numCallsFinishedWithClientFailedToSend int64 + numCallsFinishedKnownReceived int64 + + mu sync.Mutex + // map load_balance_token -> num_calls_dropped + numCallsDropped map[string]int64 +} + +func newRPCStats() *rpcStats { + return &rpcStats{ + numCallsDropped: make(map[string]int64), + } +} + +func isZeroStats(stats *lbpb.ClientStats) bool { + return len(stats.CallsFinishedWithDrop) == 0 && + stats.NumCallsStarted == 0 && + stats.NumCallsFinished == 0 && + stats.NumCallsFinishedWithClientFailedToSend == 0 && + stats.NumCallsFinishedKnownReceived == 0 +} + +// toClientStats converts rpcStats to lbpb.ClientStats, and clears rpcStats. +func (s *rpcStats) toClientStats() *lbpb.ClientStats { + stats := &lbpb.ClientStats{ + NumCallsStarted: atomic.SwapInt64(&s.numCallsStarted, 0), + NumCallsFinished: atomic.SwapInt64(&s.numCallsFinished, 0), + NumCallsFinishedWithClientFailedToSend: atomic.SwapInt64(&s.numCallsFinishedWithClientFailedToSend, 0), + NumCallsFinishedKnownReceived: atomic.SwapInt64(&s.numCallsFinishedKnownReceived, 0), + } + s.mu.Lock() + dropped := s.numCallsDropped + s.numCallsDropped = make(map[string]int64) + s.mu.Unlock() + for token, count := range dropped { + stats.CallsFinishedWithDrop = append(stats.CallsFinishedWithDrop, &lbpb.ClientStatsPerToken{ + LoadBalanceToken: token, + NumCalls: count, + }) + } + return stats +} + +func (s *rpcStats) drop(token string) { + atomic.AddInt64(&s.numCallsStarted, 1) + s.mu.Lock() + s.numCallsDropped[token]++ + s.mu.Unlock() + atomic.AddInt64(&s.numCallsFinished, 1) +} + +func (s *rpcStats) failedToSend() { + atomic.AddInt64(&s.numCallsStarted, 1) + atomic.AddInt64(&s.numCallsFinishedWithClientFailedToSend, 1) + atomic.AddInt64(&s.numCallsFinished, 1) +} + +func (s *rpcStats) knownReceived() { + atomic.AddInt64(&s.numCallsStarted, 1) + atomic.AddInt64(&s.numCallsFinishedKnownReceived, 1) + atomic.AddInt64(&s.numCallsFinished, 1) +} + +// rrPicker does roundrobin on subConns. It's typically used when there's no +// response from remote balancer, and grpclb falls back to the resolved +// backends. +// +// It guaranteed that len(subConns) > 0. +type rrPicker struct { + mu sync.Mutex + subConns []balancer.SubConn // The subConns that were READY when taking the snapshot. + subConnsNext int +} + +func newRRPicker(readySCs []balancer.SubConn) *rrPicker { + return &rrPicker{ + subConns: readySCs, + subConnsNext: rand.IntN(len(readySCs)), + } +} + +func (p *rrPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + sc := p.subConns[p.subConnsNext] + p.subConnsNext = (p.subConnsNext + 1) % len(p.subConns) + return balancer.PickResult{SubConn: sc}, nil +} + +// lbPicker does two layers of picks: +// +// First layer: roundrobin on all servers in serverList, including drops and backends. +// - If it picks a drop, the RPC will fail as being dropped. +// - If it picks a backend, do a second layer pick to pick the real backend. +// +// Second layer: roundrobin on all READY backends. +// +// It's guaranteed that len(serverList) > 0. +type lbPicker struct { + mu sync.Mutex + serverList []*lbpb.Server + serverListNext int + subConns []balancer.SubConn // The subConns that were READY when taking the snapshot. + subConnsNext int + + stats *rpcStats +} + +func newLBPicker(serverList []*lbpb.Server, readySCs []balancer.SubConn, stats *rpcStats) *lbPicker { + return &lbPicker{ + serverList: serverList, + subConns: readySCs, + subConnsNext: rand.IntN(len(readySCs)), + stats: stats, + } +} + +func (p *lbPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + + // Layer one roundrobin on serverList. + s := p.serverList[p.serverListNext] + p.serverListNext = (p.serverListNext + 1) % len(p.serverList) + + // If it's a drop, return an error and fail the RPC. + if s.Drop { + p.stats.drop(s.LoadBalanceToken) + return balancer.PickResult{}, status.Errorf(codes.Unavailable, "request dropped by grpclb") + } + + // If not a drop but there's no ready subConns. + if len(p.subConns) <= 0 { + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + } + + // Return the next ready subConn in the list, also collect rpc stats. + sc := p.subConns[p.subConnsNext] + p.subConnsNext = (p.subConnsNext + 1) % len(p.subConns) + done := func(info balancer.DoneInfo) { + if !info.BytesSent { + p.stats.failedToSend() + } else if info.BytesReceived { + p.stats.knownReceived() + } + } + return balancer.PickResult{SubConn: sc, Done: done}, nil +} + +func (p *lbPicker) updateReadySCs(readySCs []balancer.SubConn) { + p.mu.Lock() + defer p.mu.Unlock() + + p.subConns = readySCs + p.subConnsNext = p.subConnsNext % len(readySCs) +} diff --git a/balancer/grpclb/grpclb_remote_balancer.go b/balancer/grpclb/grpclb_remote_balancer.go new file mode 100644 index 0000000..baf9bc6 --- /dev/null +++ b/balancer/grpclb/grpclb_remote_balancer.go @@ -0,0 +1,459 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpclb + +import ( + "context" + "fmt" + "io" + "net" + "net/netip" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/backoff" + imetadata "google.golang.org/grpc/internal/metadata" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + + lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" +) + +func serverListEqual(a, b []*lbpb.Server) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if !proto.Equal(a[i], b[i]) { + return false + } + } + return true +} + +// processServerList updates balancer's internal state, create/remove SubConns +// and regenerates picker using the received serverList. +func (lb *lbBalancer) processServerList(l *lbpb.ServerList) { + if lb.logger.V(2) { + lb.logger.Infof("Processing server list: %#v", l) + } + lb.mu.Lock() + defer lb.mu.Unlock() + + // Set serverListReceived to true so fallback will not take effect if it has + // not hit timeout. + lb.serverListReceived = true + + // If the new server list == old server list, do nothing. + if serverListEqual(lb.fullServerList, l.Servers) { + if lb.logger.V(2) { + lb.logger.Infof("Ignoring new server list as it is the same as the previous one") + } + return + } + lb.fullServerList = l.Servers + + var backendAddrs []resolver.Address + for i, s := range l.Servers { + if s.Drop { + continue + } + + md := metadata.Pairs(lbTokenKey, s.LoadBalanceToken) + var ipStr string + if ip, ok := netip.AddrFromSlice(s.IpAddress); ok { + ipStr = ip.String() + } else { + ipStr = fmt.Sprintf("? %x", s.IpAddress) + } + addr := imetadata.Set(resolver.Address{Addr: net.JoinHostPort(ipStr, fmt.Sprintf("%d", s.Port))}, md) + if lb.logger.V(2) { + lb.logger.Infof("Server list entry:|%d|, ipStr:|%s|, port:|%d|, load balancer token:|%v|", i, ipStr, s.Port, s.LoadBalanceToken) + } + backendAddrs = append(backendAddrs, addr) + } + + // Call refreshSubConns to create/remove SubConns. If we are in fallback, + // this is also exiting fallback. + lb.refreshSubConns(backendAddrs, false, lb.usePickFirst) +} + +// refreshSubConns creates/removes SubConns with backendAddrs, and refreshes +// balancer state and picker. +// +// Caller must hold lb.mu. +func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fallback bool, pickFirst bool) { + opts := balancer.NewSubConnOptions{} + if !fallback { + opts.CredsBundle = lb.grpclbBackendCreds + } + + lb.backendAddrs = backendAddrs + lb.backendAddrsWithoutMetadata = nil + + fallbackModeChanged := lb.inFallback != fallback + lb.inFallback = fallback + if fallbackModeChanged && lb.inFallback { + // Clear previous received list when entering fallback, so if the server + // comes back and sends the same list again, the new addresses will be + // used. + lb.fullServerList = nil + } + + balancingPolicyChanged := lb.usePickFirst != pickFirst + lb.usePickFirst = pickFirst + + if fallbackModeChanged || balancingPolicyChanged { + // Remove all SubConns when switching balancing policy or switching + // fallback mode. + // + // For fallback mode switching with pickfirst, we want to recreate the + // SubConn because the creds could be different. + for a, sc := range lb.subConns { + sc.Shutdown() + delete(lb.subConns, a) + } + } + + if lb.usePickFirst { + var ( + scKey resolver.Address + sc balancer.SubConn + ) + for scKey, sc = range lb.subConns { + break + } + if sc != nil { + if len(backendAddrs) == 0 { + sc.Shutdown() + delete(lb.subConns, scKey) + return + } + lb.cc.ClientConn.UpdateAddresses(sc, backendAddrs) + sc.Connect() + return + } + opts.StateListener = func(scs balancer.SubConnState) { lb.updateSubConnState(sc, scs) } + // This bypasses the cc wrapper with SubConn cache. + sc, err := lb.cc.ClientConn.NewSubConn(backendAddrs, opts) + if err != nil { + lb.logger.Warningf("Failed to create new SubConn: %v", err) + return + } + sc.Connect() + lb.subConns[backendAddrs[0]] = sc + lb.scStates[sc] = connectivity.Idle + return + } + + // addrsSet is the set converted from backendAddrsWithoutMetadata, it's used to quick + // lookup for an address. + addrsSet := make(map[resolver.Address]struct{}) + // Create new SubConns. + for _, addr := range backendAddrs { + addrWithoutAttrs := addr + addrWithoutAttrs.Attributes = nil + addrsSet[addrWithoutAttrs] = struct{}{} + lb.backendAddrsWithoutMetadata = append(lb.backendAddrsWithoutMetadata, addrWithoutAttrs) + + if _, ok := lb.subConns[addrWithoutAttrs]; !ok { + // Use addrWithMD to create the SubConn. + var sc balancer.SubConn + opts.StateListener = func(scs balancer.SubConnState) { lb.updateSubConnState(sc, scs) } + sc, err := lb.cc.NewSubConn([]resolver.Address{addr}, opts) + if err != nil { + lb.logger.Warningf("Failed to create new SubConn: %v", err) + continue + } + lb.subConns[addrWithoutAttrs] = sc // Use the addr without MD as key for the map. + if _, ok := lb.scStates[sc]; !ok { + // Only set state of new sc to IDLE. The state could already be + // READY for cached SubConns. + lb.scStates[sc] = connectivity.Idle + } + sc.Connect() + } + } + + for a, sc := range lb.subConns { + // a was removed by resolver. + if _, ok := addrsSet[a]; !ok { + sc.Shutdown() + delete(lb.subConns, a) + // Keep the state of this sc in b.scStates until sc's state becomes Shutdown. + // The entry will be deleted in UpdateSubConnState. + } + } + + // Regenerate and update picker after refreshing subconns because with + // cache, even if SubConn was newed/removed, there might be no state + // changes (the subconn will be kept in cache, not actually + // newed/removed). + lb.updateStateAndPicker(true, true) +} + +type remoteBalancerCCWrapper struct { + cc *grpc.ClientConn + lb *lbBalancer + backoff backoff.Strategy + done chan struct{} + + streamMu sync.Mutex + streamCancel func() + + // waitgroup to wait for all goroutines to exit. + wg sync.WaitGroup +} + +func (lb *lbBalancer) newRemoteBalancerCCWrapper() error { + var dopts []grpc.DialOption + if creds := lb.opt.DialCreds; creds != nil { + dopts = append(dopts, grpc.WithTransportCredentials(creds)) + } else if bundle := lb.grpclbClientConnCreds; bundle != nil { + dopts = append(dopts, grpc.WithCredentialsBundle(bundle)) + } else { + dopts = append(dopts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + if lb.opt.Dialer != nil { + dopts = append(dopts, grpc.WithContextDialer(lb.opt.Dialer)) + } + if lb.opt.CustomUserAgent != "" { + dopts = append(dopts, grpc.WithUserAgent(lb.opt.CustomUserAgent)) + } + // Explicitly set pickfirst as the balancer. + dopts = append(dopts, grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"pick_first"}`)) + dopts = append(dopts, grpc.WithResolvers(lb.manualResolver)) + dopts = append(dopts, grpc.WithChannelzParentID(lb.opt.ChannelzParent)) + + // Enable Keepalive for grpclb client. + dopts = append(dopts, grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: 20 * time.Second, + Timeout: 10 * time.Second, + PermitWithoutStream: true, + })) + + // The dial target is not important. + // + // The grpclb server addresses will set field ServerName, and creds will + // receive ServerName as authority. + target := lb.manualResolver.Scheme() + ":///grpclb.subClientConn" + cc, err := grpc.NewClient(target, dopts...) + if err != nil { + return fmt.Errorf("grpc.NewClient(%s): %v", target, err) + } + cc.Connect() + ccw := &remoteBalancerCCWrapper{ + cc: cc, + lb: lb, + backoff: lb.backoff, + done: make(chan struct{}), + } + lb.ccRemoteLB = ccw + ccw.wg.Add(1) + go ccw.watchRemoteBalancer() + return nil +} + +// close closed the ClientConn to remote balancer, and waits until all +// goroutines to finish. +func (ccw *remoteBalancerCCWrapper) close() { + close(ccw.done) + ccw.cc.Close() + ccw.wg.Wait() +} + +func (ccw *remoteBalancerCCWrapper) readServerList(s *balanceLoadClientStream) error { + for { + reply, err := s.Recv() + if err != nil { + if err == io.EOF { + return errServerTerminatedConnection + } + return fmt.Errorf("grpclb: failed to recv server list: %v", err) + } + if serverList := reply.GetServerList(); serverList != nil { + ccw.lb.processServerList(serverList) + } + if reply.GetFallbackResponse() != nil { + // Eagerly enter fallback + ccw.lb.mu.Lock() + ccw.lb.refreshSubConns(ccw.lb.resolvedBackendAddrs, true, ccw.lb.usePickFirst) + ccw.lb.mu.Unlock() + } + } +} + +func (ccw *remoteBalancerCCWrapper) sendLoadReport(s *balanceLoadClientStream, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + lastZero := false + for { + select { + case <-ticker.C: + case <-s.Context().Done(): + return + } + stats := ccw.lb.clientStats.toClientStats() + zero := isZeroStats(stats) + if zero && lastZero { + // Quash redundant empty load reports. + continue + } + lastZero = zero + t := time.Now() + stats.Timestamp = ×tamppb.Timestamp{ + Seconds: t.Unix(), + Nanos: int32(t.Nanosecond()), + } + if err := s.Send(&lbpb.LoadBalanceRequest{ + LoadBalanceRequestType: &lbpb.LoadBalanceRequest_ClientStats{ + ClientStats: stats, + }, + }); err != nil { + return + } + } +} + +func (ccw *remoteBalancerCCWrapper) callRemoteBalancer(ctx context.Context) (backoff bool, _ error) { + lbClient := &loadBalancerClient{cc: ccw.cc} + stream, err := lbClient.BalanceLoad(ctx, grpc.WaitForReady(true)) + if err != nil { + return true, fmt.Errorf("grpclb: failed to perform RPC to the remote balancer: %v", err) + } + ccw.lb.mu.Lock() + ccw.lb.remoteBalancerConnected = true + ccw.lb.mu.Unlock() + + // grpclb handshake on the stream. + initReq := &lbpb.LoadBalanceRequest{ + LoadBalanceRequestType: &lbpb.LoadBalanceRequest_InitialRequest{ + InitialRequest: &lbpb.InitialLoadBalanceRequest{ + Name: ccw.lb.target, + }, + }, + } + if err := stream.Send(initReq); err != nil { + return true, fmt.Errorf("grpclb: failed to send init request: %v", err) + } + reply, err := stream.Recv() + if err != nil { + return true, fmt.Errorf("grpclb: failed to recv init response: %v", err) + } + initResp := reply.GetInitialResponse() + if initResp == nil { + return true, fmt.Errorf("grpclb: reply from remote balancer did not include initial response") + } + + ccw.wg.Add(1) + go func() { + defer ccw.wg.Done() + if d := convertDuration(initResp.ClientStatsReportInterval); d > 0 { + ccw.sendLoadReport(stream, d) + } + }() + // No backoff if init req/resp handshake was successful. + return false, ccw.readServerList(stream) +} + +// cancelRemoteBalancerCall cancels the context used by the stream to the remote +// balancer. watchRemoteBalancer() takes care of restarting this call after the +// stream fails. +func (ccw *remoteBalancerCCWrapper) cancelRemoteBalancerCall() { + ccw.streamMu.Lock() + if ccw.streamCancel != nil { + ccw.streamCancel() + ccw.streamCancel = nil + } + ccw.streamMu.Unlock() +} + +func (ccw *remoteBalancerCCWrapper) watchRemoteBalancer() { + defer func() { + ccw.wg.Done() + ccw.streamMu.Lock() + if ccw.streamCancel != nil { + // This is to make sure that we don't leak the context when we are + // directly returning from inside of the below `for` loop. + ccw.streamCancel() + ccw.streamCancel = nil + } + ccw.streamMu.Unlock() + }() + + var retryCount int + var ctx context.Context + for { + ccw.streamMu.Lock() + if ccw.streamCancel != nil { + ccw.streamCancel() + ccw.streamCancel = nil + } + ctx, ccw.streamCancel = context.WithCancel(context.Background()) + ccw.streamMu.Unlock() + + doBackoff, err := ccw.callRemoteBalancer(ctx) + select { + case <-ccw.done: + return + default: + if err != nil { + if err == errServerTerminatedConnection { + ccw.lb.logger.Infof("Call to remote balancer failed: %v", err) + } else { + ccw.lb.logger.Warningf("Call to remote balancer failed: %v", err) + } + } + } + // Trigger a re-resolve when the stream errors. + ccw.lb.cc.ClientConn.ResolveNow(resolver.ResolveNowOptions{}) + + ccw.lb.mu.Lock() + ccw.lb.remoteBalancerConnected = false + ccw.lb.fullServerList = nil + // Enter fallback when connection to remote balancer is lost, and the + // aggregated state is not Ready. + if !ccw.lb.inFallback && ccw.lb.state != connectivity.Ready { + // Entering fallback. + ccw.lb.refreshSubConns(ccw.lb.resolvedBackendAddrs, true, ccw.lb.usePickFirst) + } + ccw.lb.mu.Unlock() + + if !doBackoff { + retryCount = 0 + continue + } + + timer := time.NewTimer(ccw.backoff.Backoff(retryCount)) // Copy backoff + select { + case <-timer.C: + case <-ccw.done: + timer.Stop() + return + } + retryCount++ + } +} diff --git a/balancer/grpclb/grpclb_test.go b/balancer/grpclb/grpclb_test.go new file mode 100644 index 0000000..c575878 --- /dev/null +++ b/balancer/grpclb/grpclb_test.go @@ -0,0 +1,1649 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpclb + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/netip" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + grpclbstate "google.golang.org/grpc/balancer/grpclb/state" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/pickfirst" + "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" + + lbgrpc "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" + lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + lbServerName = "lb.server.com" + beServerName = "backends.com" + lbToken = "iamatoken" + + // Resolver replaces localhost with fakeName in Next(). + // Dialer replaces fakeName with localhost when dialing. + // This will test that custom dialer is passed from Dial to grpclb. + fakeName = "fake.Name" +) + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + testUserAgent = "test-user-agent" + grpclbConfig = `{"loadBalancingConfig": [{"grpclb": {}}]}` +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type serverNameCheckCreds struct { + mu sync.Mutex + sn string +} + +func (c *serverNameCheckCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + if _, err := io.WriteString(rawConn, c.sn); err != nil { + fmt.Printf("Failed to write the server name %s to the client %v", c.sn, err) + return nil, nil, err + } + return rawConn, nil, nil +} +func (c *serverNameCheckCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + c.mu.Lock() + defer c.mu.Unlock() + b := make([]byte, len(authority)) + errCh := make(chan error, 1) + go func() { + _, err := rawConn.Read(b) + errCh <- err + }() + select { + case err := <-errCh: + if err != nil { + fmt.Printf("test-creds: failed to read expected authority name from the server: %v\n", err) + return nil, nil, err + } + case <-ctx.Done(): + return nil, nil, ctx.Err() + } + if authority != string(b) { + fmt.Printf("test-creds: got authority from ClientConn %q, expected by server %q\n", authority, string(b)) + return nil, nil, errors.New("received unexpected server name") + } + return rawConn, nil, nil +} +func (c *serverNameCheckCreds) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{} +} +func (c *serverNameCheckCreds) Clone() credentials.TransportCredentials { + return &serverNameCheckCreds{} +} +func (c *serverNameCheckCreds) OverrideServerName(string) error { + return nil +} + +// fakeNameDialer replaces fakeName with localhost when dialing. +// This will test that custom dialer is passed from Dial to grpclb. +func fakeNameDialer(ctx context.Context, addr string) (net.Conn, error) { + addr = strings.Replace(addr, fakeName, "localhost", 1) + return (&net.Dialer{}).DialContext(ctx, "tcp", addr) +} + +// merge merges the new client stats into current stats. +// +// It's a test-only method. rpcStats is defined in grpclb_picker. +func (s *rpcStats) merge(cs *lbpb.ClientStats) { + atomic.AddInt64(&s.numCallsStarted, cs.NumCallsStarted) + atomic.AddInt64(&s.numCallsFinished, cs.NumCallsFinished) + atomic.AddInt64(&s.numCallsFinishedWithClientFailedToSend, cs.NumCallsFinishedWithClientFailedToSend) + atomic.AddInt64(&s.numCallsFinishedKnownReceived, cs.NumCallsFinishedKnownReceived) + s.mu.Lock() + for _, perToken := range cs.CallsFinishedWithDrop { + s.numCallsDropped[perToken.LoadBalanceToken] += perToken.NumCalls + } + s.mu.Unlock() +} + +func atomicEqual(a, b *int64) bool { + return atomic.LoadInt64(a) == atomic.LoadInt64(b) +} + +// equal compares two rpcStats. +// +// It's a test-only method. rpcStats is defined in grpclb_picker. +func (s *rpcStats) equal(o *rpcStats) bool { + if !atomicEqual(&s.numCallsStarted, &o.numCallsStarted) { + return false + } + if !atomicEqual(&s.numCallsFinished, &o.numCallsFinished) { + return false + } + if !atomicEqual(&s.numCallsFinishedWithClientFailedToSend, &o.numCallsFinishedWithClientFailedToSend) { + return false + } + if !atomicEqual(&s.numCallsFinishedKnownReceived, &o.numCallsFinishedKnownReceived) { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + o.mu.Lock() + defer o.mu.Unlock() + return cmp.Equal(s.numCallsDropped, o.numCallsDropped, cmpopts.EquateEmpty()) +} + +func (s *rpcStats) String() string { + s.mu.Lock() + defer s.mu.Unlock() + return fmt.Sprintf("Started: %v, Finished: %v, FinishedWithClientFailedToSend: %v, FinishedKnownReceived: %v, Dropped: %v", + atomic.LoadInt64(&s.numCallsStarted), + atomic.LoadInt64(&s.numCallsFinished), + atomic.LoadInt64(&s.numCallsFinishedWithClientFailedToSend), + atomic.LoadInt64(&s.numCallsFinishedKnownReceived), + s.numCallsDropped) +} + +type remoteBalancer struct { + lbgrpc.UnimplementedLoadBalancerServer + sls chan *lbpb.ServerList + statsDura time.Duration + done chan struct{} + stats *rpcStats + statsChan chan *lbpb.ClientStats + fbChan chan struct{} + balanceLoadCh chan struct{} // notify successful invocation of BalanceLoad + + wantUserAgent string // expected user-agent in metadata of BalancerLoad + wantServerName string // expected server name in InitialLoadBalanceRequest +} + +func newRemoteBalancer(wantUserAgent, wantServerName string, statsChan chan *lbpb.ClientStats) *remoteBalancer { + return &remoteBalancer{ + sls: make(chan *lbpb.ServerList, 1), + done: make(chan struct{}), + stats: newRPCStats(), + statsChan: statsChan, + fbChan: make(chan struct{}), + balanceLoadCh: make(chan struct{}, 1), + wantUserAgent: wantUserAgent, + wantServerName: wantServerName, + } +} + +func (b *remoteBalancer) stop() { + close(b.sls) + close(b.done) +} + +func (b *remoteBalancer) fallbackNow() { + b.fbChan <- struct{}{} +} + +func (b *remoteBalancer) updateServerName(name string) { + b.wantServerName = name +} + +func (b *remoteBalancer) BalanceLoad(stream lbgrpc.LoadBalancer_BalanceLoadServer) error { + md, ok := metadata.FromIncomingContext(stream.Context()) + if !ok { + return status.Error(codes.Internal, "failed to receive metadata") + } + if b.wantUserAgent != "" { + if ua := md["user-agent"]; len(ua) == 0 || !strings.HasPrefix(ua[0], b.wantUserAgent) { + return status.Errorf(codes.InvalidArgument, "received unexpected user-agent: %v, want prefix %q", ua, b.wantUserAgent) + } + } + + req, err := stream.Recv() + if err != nil { + return err + } + initReq := req.GetInitialRequest() + if initReq.Name != b.wantServerName { + return status.Errorf(codes.InvalidArgument, "invalid service name: %q, want: %q", initReq.Name, b.wantServerName) + } + b.balanceLoadCh <- struct{}{} + resp := &lbpb.LoadBalanceResponse{ + LoadBalanceResponseType: &lbpb.LoadBalanceResponse_InitialResponse{ + InitialResponse: &lbpb.InitialLoadBalanceResponse{ + ClientStatsReportInterval: &durationpb.Duration{ + Seconds: int64(b.statsDura.Seconds()), + Nanos: int32(b.statsDura.Nanoseconds() - int64(b.statsDura.Seconds())*1e9), + }, + }, + }, + } + if err := stream.Send(resp); err != nil { + return err + } + go func() { + for { + req, err := stream.Recv() + if err != nil { + return + } + b.stats.merge(req.GetClientStats()) + if b.statsChan != nil && req.GetClientStats() != nil { + b.statsChan <- req.GetClientStats() + } + } + }() + for { + select { + case v := <-b.sls: + resp = &lbpb.LoadBalanceResponse{ + LoadBalanceResponseType: &lbpb.LoadBalanceResponse_ServerList{ + ServerList: v, + }, + } + case <-b.fbChan: + resp = &lbpb.LoadBalanceResponse{ + LoadBalanceResponseType: &lbpb.LoadBalanceResponse_FallbackResponse{ + FallbackResponse: &lbpb.FallbackResponse{}, + }, + } + case <-stream.Context().Done(): + return stream.Context().Err() + } + if err := stream.Send(resp); err != nil { + return err + } + } +} + +const testmdkey = "testmd" + +func startBackends(t *testing.T, sn string, fallback bool, lis ...net.Listener) (servers []*grpc.Server) { + for _, l := range lis { + creds := &serverNameCheckCreds{ + sn: sn, + } + ss := &stubserver.StubServer{ + Listener: l, + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Error(codes.Internal, "failed to receive metadata") + } + if !fallback && (md == nil || len(md["lb-token"]) == 0 || md["lb-token"][0] != lbToken) { + return nil, status.Errorf(codes.Internal, "received unexpected metadata: %v", md) + } + grpc.SetTrailer(ctx, metadata.Pairs(testmdkey, l.Addr().String())) + return &testpb.Empty{}, nil + }, + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + return nil + }, + } + if err := ss.StartServer(grpc.Creds(creds)); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + servers = append(servers, ss.S.(*grpc.Server)) + t.Logf("Started backend server listening on %s", l.Addr().String()) + } + return +} + +func stopBackends(servers []*grpc.Server) { + for _, s := range servers { + s.Stop() + } +} + +type testServers struct { + lbAddr string + ls *remoteBalancer + lb *grpc.Server + backends []*grpc.Server + beIPs []netip.Addr + bePorts []int + + lbListener net.Listener + beListeners []net.Listener +} + +func startBackendsAndRemoteLoadBalancer(t *testing.T, numberOfBackends int, customUserAgent string, statsChan chan *lbpb.ClientStats) (tss *testServers, cleanup func(), err error) { + var ( + beListeners []net.Listener + ls *remoteBalancer + lb *grpc.Server + beIPs []netip.Addr + bePorts []int + ) + for i := 0; i < numberOfBackends; i++ { + beLis, e := net.Listen("tcp", "localhost:0") + if e != nil { + err = fmt.Errorf("failed to listen %v", err) + return + } + beIPs = append(beIPs, beLis.Addr().(*net.TCPAddr).AddrPort().Addr().Unmap()) + bePorts = append(bePorts, beLis.Addr().(*net.TCPAddr).Port) + + beListeners = append(beListeners, testutils.NewRestartableListener(beLis)) + } + backends := startBackends(t, beServerName, false, beListeners...) + + lbLis, err := net.Listen("tcp", "localhost:0") + if err != nil { + err = fmt.Errorf("failed to create the listener for the load balancer %v", err) + return + } + lbLis = testutils.NewRestartableListener(lbLis) + lbCreds := &serverNameCheckCreds{ + sn: lbServerName, + } + lb = grpc.NewServer(grpc.Creds(lbCreds)) + ls = newRemoteBalancer(customUserAgent, beServerName, statsChan) + lbgrpc.RegisterLoadBalancerServer(lb, ls) + go func() { + lb.Serve(lbLis) + }() + t.Logf("Started remote load balancer server listening on %s", lbLis.Addr().String()) + + tss = &testServers{ + lbAddr: net.JoinHostPort(fakeName, strconv.Itoa(lbLis.Addr().(*net.TCPAddr).Port)), + ls: ls, + lb: lb, + backends: backends, + beIPs: beIPs, + bePorts: bePorts, + + lbListener: lbLis, + beListeners: beListeners, + } + cleanup = func() { + defer stopBackends(backends) + defer func() { + ls.stop() + lb.Stop() + }() + } + return +} + +// TestGRPCLB_Basic tests the basic case of a channel being configured with +// grpclb as the load balancing policy. +func (s) TestGRPCLB_Basic(t *testing.T) { + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, testUserAgent, nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) + } + defer cleanup() + + // Push the test backend address to the remote balancer. + tss.ls.sls <- &lbpb.ServerList{ + Servers: []*lbpb.Server{ + { + IpAddress: tss.beIPs[0].AsSlice(), + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, + }, + } + + // Configure the manual resolver with an initial state containing a service + // config with grpclb as the load balancing policy and the remote balancer + // address specified via attributes. + r := manual.NewBuilderWithScheme("whatever") + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tss.lbAddr, + ServerName: lbServerName, + }, + }, + } + rs := grpclbstate.Set(resolver.State{ServiceConfig: internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(grpclbConfig)}, s) + r.InitialState(rs) + + // Connect to the test backend. + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + grpc.WithUserAgent(testUserAgent), + } + cc, err := grpc.NewClient(r.Scheme()+":///"+beServerName, dopts...) + if err != nil { + t.Fatalf("Failed to create a client for the backend %v", err) + } + defer cc.Close() + + // Make one successful RPC. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testC := testgrpc.NewTestServiceClient(cc) + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) + } +} + +// TestGRPCLB_Weighted tests weighted roundrobin. The remote balancer is +// configured to send a response with duplicate backend addresses (to simulate +// weights) to the grpclb client. The test verifies that RPCs are weighted +// roundrobin-ed across these backends. +func (s) TestGRPCLB_Weighted(t *testing.T) { + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 2, "", nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) + } + defer cleanup() + + beServers := []*lbpb.Server{{ + IpAddress: tss.beIPs[0].AsSlice(), + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, { + IpAddress: tss.beIPs[1].AsSlice(), + Port: int32(tss.bePorts[1]), + LoadBalanceToken: lbToken, + }} + + // Configure the manual resolver with an initial state containing a service + // config with grpclb as the load balancing policy and the remote balancer + // address specified via attributes. + r := manual.NewBuilderWithScheme("whatever") + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tss.lbAddr, + ServerName: lbServerName, + }, + }, + } + rs := grpclbstate.Set(resolver.State{ServiceConfig: internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(grpclbConfig)}, s) + r.InitialState(rs) + + // Connect to test backends. + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + } + cc, err := grpc.NewClient(r.Scheme()+":///"+beServerName, dopts...) + if err != nil { + t.Fatalf("Failed to create a client for the backend %v", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Sequence represents the sequence of backends to be returned from the + // remote load balancer. + sequences := [][]int{ + {0, 0, 1, 0, 1}, + {0, 0, 0, 1, 1}, + } + for _, seq := range sequences { + // Push the configured sequence of backend to the remote balancer, and + // compute the expected addresses to which RPCs should be routed. + var backends []*lbpb.Server + var wantAddrs []resolver.Address + for _, s := range seq { + backends = append(backends, beServers[s]) + wantAddrs = append(wantAddrs, resolver.Address{Addr: tss.beListeners[s].Addr().String()}) + } + tss.ls.sls <- &lbpb.ServerList{Servers: backends} + + testC := testgrpc.NewTestServiceClient(cc) + if err := roundrobin.CheckWeightedRoundRobinRPCs(ctx, t, testC, wantAddrs); err != nil { + t.Fatal(err) + } + } +} + +// TestGRPCLB_DropRequest tests grpclb support for dropping requests based on +// configuration received from the remote balancer. +// +// TODO: Rewrite this test to verify drop behavior using the +// ClientStats.CallsFinishedWithDrop field instead. +func (s) TestGRPCLB_DropRequest(t *testing.T) { + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 2, "", nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) + } + defer cleanup() + tss.ls.sls <- &lbpb.ServerList{ + Servers: []*lbpb.Server{{ + IpAddress: tss.beIPs[0].AsSlice(), + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + Drop: false, + }, { + IpAddress: tss.beIPs[1].AsSlice(), + Port: int32(tss.bePorts[1]), + LoadBalanceToken: lbToken, + Drop: false, + }, { + Drop: true, + }}, + } + + // Configure the manual resolver with an initial state containing a service + // config with grpclb as the load balancing policy and the remote balancer + // address specified via attributes. + r := manual.NewBuilderWithScheme("whatever") + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tss.lbAddr, + ServerName: lbServerName, + }, + }, + } + rs := grpclbstate.Set(resolver.State{ServiceConfig: internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(grpclbConfig)}, s) + r.InitialState(rs) + + // Connect to test backends. + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + } + cc, err := grpc.NewClient(r.Scheme()+":///"+beServerName, dopts...) + if err != nil { + t.Fatalf("Failed to create a client for the backend %v", err) + } + defer cc.Close() + testC := testgrpc.NewTestServiceClient(cc) + + var ( + i int + p peer.Peer + ) + const ( + // Poll to wait for something to happen. Total timeout 1 second. Sleep 1 + // ms each loop, and do at most 1000 loops. + sleepEachLoop = time.Millisecond + loopCount = int(time.Second / sleepEachLoop) + ) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Make a non-fail-fast RPC and wait for it to succeed. + for i = 0; i < loopCount; i++ { + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err == nil { + break + } + time.Sleep(sleepEachLoop) + } + if i >= loopCount { + t.Fatalf("timeout waiting for the first connection to become ready. EmptyCall(_, _) = _, %v, want _, ", err) + } + + // Make RPCs until the peer is different. So we know both connections are + // READY. + for i = 0; i < loopCount; i++ { + var temp peer.Peer + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&temp)); err == nil { + if temp.Addr.(*net.TCPAddr).Port != p.Addr.(*net.TCPAddr).Port { + break + } + } + time.Sleep(sleepEachLoop) + } + if i >= loopCount { + t.Fatalf("timeout waiting for the second connection to become ready") + } + + // More RPCs until drop happens. So we know the picker index, and the + // expected behavior of following RPCs. + for i = 0; i < loopCount; i++ { + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) == codes.Unavailable { + break + } + time.Sleep(sleepEachLoop) + } + if i >= loopCount { + t.Fatalf("timeout waiting for drop. EmptyCall(_, _) = _, %v, want _, ", err) + } + + select { + case <-ctx.Done(): + t.Fatal("timed out", ctx.Err()) + default: + } + for _, failfast := range []bool{true, false} { + for i := 0; i < 3; i++ { + // 1st RPCs pick the first item in server list. They should succeed + // since they choose the non-drop-request backend according to the + // round robin policy. + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(!failfast)); err != nil { + t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) + } + // 2nd RPCs pick the second item in server list. They should succeed + // since they choose the non-drop-request backend according to the + // round robin policy. + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(!failfast)); err != nil { + t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) + } + // 3rd RPCs should fail, because they pick last item in server list, + // with Drop set to true. + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(!failfast)); status.Code(err) != codes.Unavailable { + t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, %s", testC, err, codes.Unavailable) + } + } + } + + // Make one more RPC to move the picker index one step further, so it's not + // 0. The following RPCs will test that drop index is not reset. If picker + // index is at 0, we cannot tell whether it's reset or not. + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) + } + + tss.backends[0].Stop() + // This last pick was backend 0. Closing backend 0 doesn't reset drop index + // (for level 1 picking), so the following picks will be (backend1, drop, + // backend1), instead of (backend, backend, drop) if drop index was reset. + time.Sleep(time.Second) + for i := 0; i < 3; i++ { + var p peer.Peer + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) + } + if want := tss.bePorts[1]; p.Addr.(*net.TCPAddr).Port != want { + t.Errorf("got peer: %v, want peer port: %v", p.Addr, want) + } + + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.Unavailable { + t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, %s", testC, err, codes.Unavailable) + } + + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) + } + if want := tss.bePorts[1]; p.Addr.(*net.TCPAddr).Port != want { + t.Errorf("got peer: %v, want peer port: %v", p.Addr, want) + } + } +} + +// TestGRPCLB_BalancerDisconnects tests the case where the remote balancer in +// use disconnects. The test verifies that grpclb connects to the next remote +// balancer address specified in attributes, and RPCs get routed to the backends +// returned by the new balancer. +func (s) TestGRPCLB_BalancerDisconnects(t *testing.T) { + var ( + tests []*testServers + lbs []*grpc.Server + ) + for i := 0; i < 2; i++ { + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) + } + defer cleanup() + + tss.ls.sls <- &lbpb.ServerList{ + Servers: []*lbpb.Server{ + { + IpAddress: tss.beIPs[0].AsSlice(), + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, + }, + } + + tests = append(tests, tss) + lbs = append(lbs, tss.lb) + } + + // Configure the manual resolver with an initial state containing a service + // config with grpclb as the load balancing policy and the remote balancer + // addresses specified via attributes. + r := manual.NewBuilderWithScheme("whatever") + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tests[0].lbAddr, + ServerName: lbServerName, + }, + { + Addr: tests[1].lbAddr, + ServerName: lbServerName, + }, + }, + } + rs := grpclbstate.Set(resolver.State{ServiceConfig: internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(grpclbConfig)}, s) + r.InitialState(rs) + + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + } + cc, err := grpc.NewClient(r.Scheme()+":///"+beServerName, dopts...) + if err != nil { + t.Fatalf("Failed to create a client for the backend %v", err) + } + defer cc.Close() + testC := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tests[0].beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) + } + + // Stop balancer[0], balancer[1] should be used by grpclb. + // Check peer address to see if that happened. + lbs[0].Stop() + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tests[1].beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) + } +} + +// TestGRPCLB_Fallback tests the following fallback scenarios: +// - when the remote balancer address specified in attributes is invalid, the +// test verifies that RPCs are routed to the fallback backend. +// - when the remote balancer address specified in attributes is changed to a +// valid one, the test verifies that RPCs are routed to the backend returned +// by the remote balancer. +// - when the configured remote balancer goes down, the test verifies that +// RPCs are routed to the fallback backend. +func (s) TestGRPCLB_Fallback(t *testing.T) { + balancer.Register(newLBBuilderWithFallbackTimeout(100 * time.Millisecond)) + defer balancer.Register(newLBBuilder()) + + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) + } + defer cleanup() + sl := &lbpb.ServerList{ + Servers: []*lbpb.Server{ + { + IpAddress: tss.beIPs[0].AsSlice(), + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, + }, + } + // Push the backend address to the remote balancer. + tss.ls.sls <- sl + + // Start a standalone backend for fallback. + beLis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen %v", err) + } + defer beLis.Close() + standaloneBEs := startBackends(t, beServerName, true, beLis) + defer stopBackends(standaloneBEs) + + r := manual.NewBuilderWithScheme("whatever") + // Set the initial resolver state with fallback backend address stored in + // the `Addresses` field and an invalid remote balancer address stored in + // attributes, which will cause fallback behavior to be invoked. + rs := resolver.State{ + Addresses: []resolver.Address{{Addr: beLis.Addr().String()}}, + ServiceConfig: internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(grpclbConfig), + } + rs = grpclbstate.Set(rs, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: "invalid.address", ServerName: lbServerName}}}) + r.InitialState(rs) + + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + } + cc, err := grpc.NewClient(r.Scheme()+":///"+beServerName, dopts...) + if err != nil { + t.Fatalf("Failed to create new client to the backend %v", err) + } + defer cc.Close() + testC := testgrpc.NewTestServiceClient(cc) + + // Make an RPC and verify that it got routed to the fallback backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: beLis.Addr().String()}}); err != nil { + t.Fatal(err) + } + + // Push another update to the resolver, this time with a valid balancer + // address in the attributes field. + rs = resolver.State{ + ServiceConfig: r.CC().ParseServiceConfig(grpclbConfig), + Addresses: []resolver.Address{{Addr: beLis.Addr().String()}}, + } + rs = grpclbstate.Set(rs, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: tss.lbAddr, ServerName: lbServerName}}}) + r.UpdateState(rs) + select { + case <-ctx.Done(): + t.Fatalf("timeout when waiting for BalanceLoad RPC to be called on the remote balancer") + case <-tss.ls.balanceLoadCh: + } + + // Wait for RPCs to get routed to the backend behind the remote balancer. + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tss.beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) + } + + // Close backend and remote balancer connections, should use fallback. + tss.beListeners[0].(*testutils.RestartableListener).Stop() + tss.lbListener.(*testutils.RestartableListener).Stop() + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: beLis.Addr().String()}}); err != nil { + t.Fatal(err) + } + + // Restart backend and remote balancer, should not use fallback backend. + tss.beListeners[0].(*testutils.RestartableListener).Restart() + tss.lbListener.(*testutils.RestartableListener).Restart() + tss.ls.sls <- sl + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tss.beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) + } +} + +// TestGRPCLB_ExplicitFallback tests the case where the remote balancer sends an +// explicit fallback signal to the grpclb client, and the test verifies that +// RPCs are routed to the fallback backend. +func (s) TestGRPCLB_ExplicitFallback(t *testing.T) { + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) + } + defer cleanup() + sl := &lbpb.ServerList{ + Servers: []*lbpb.Server{ + { + IpAddress: tss.beIPs[0].AsSlice(), + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, + }, + } + // Push the backend address to the remote balancer. + tss.ls.sls <- sl + + // Start a standalone backend for fallback. + beLis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen %v", err) + } + defer beLis.Close() + standaloneBEs := startBackends(t, beServerName, true, beLis) + defer stopBackends(standaloneBEs) + + // Configure the manual resolver with an initial state containing a service + // config with grpclb as the load balancing policy and the address of the + // fallback backend. The remote balancer address is specified via + // attributes. + r := manual.NewBuilderWithScheme("whatever") + rs := resolver.State{ + Addresses: []resolver.Address{{Addr: beLis.Addr().String()}}, + ServiceConfig: internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(grpclbConfig), + } + rs = grpclbstate.Set(rs, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: tss.lbAddr, ServerName: lbServerName}}}) + r.InitialState(rs) + + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + } + cc, err := grpc.NewClient(r.Scheme()+":///"+beServerName, dopts...) + if err != nil { + t.Fatalf("Failed to create a client for the backend %v", err) + } + defer cc.Close() + testC := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tss.beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) + } + + // Send fallback signal from remote balancer; should use fallback. + tss.ls.fallbackNow() + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: beLis.Addr().String()}}); err != nil { + t.Fatal(err) + } + + // Send another server list; should use backends again. + tss.ls.sls <- sl + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tss.beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) + } +} + +// TestGRPCLB_FallBackWithNoServerAddress tests the fallback case where no +// backend addresses are returned by the remote balancer. +func (s) TestGRPCLB_FallBackWithNoServerAddress(t *testing.T) { + resolveNowCh := testutils.NewChannel() + r := manual.NewBuilderWithScheme("whatever") + r.ResolveNowCallback = func(resolver.ResolveNowOptions) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if err := resolveNowCh.SendContext(ctx, nil); err != nil { + t.Error("timeout when attempting to send on resolverNowCh") + } + } + + // Start a remote balancer and a backend. Don't push the backend address to + // the remote balancer yet. + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) + } + defer cleanup() + sl := &lbpb.ServerList{ + Servers: []*lbpb.Server{ + { + IpAddress: tss.beIPs[0].AsSlice(), + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, + }, + } + + // Start a standalone backend for fallback. + beLis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen %v", err) + } + defer beLis.Close() + standaloneBEs := startBackends(t, beServerName, true, beLis) + defer stopBackends(standaloneBEs) + + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + } + cc, err := grpc.NewClient(r.Scheme()+":///"+beServerName, dopts...) + if err != nil { + t.Fatalf("Failed to create a client for the backend %v", err) + } + defer cc.Close() + cc.Connect() + testC := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for i := 0; i < 2; i++ { + // Send an update with only backend address. grpclb should enter + // fallback and use the fallback backend. + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: beLis.Addr().String()}}, + ServiceConfig: r.CC().ParseServiceConfig(grpclbConfig), + }) + + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := resolveNowCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("unexpected resolveNow when grpclb gets no balancer address 1111, %d", i) + } + + var p peer.Peer + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { + t.Fatalf("_.EmptyCall(_, _) = _, %v, want _, ", err) + } + if p.Addr.String() != beLis.Addr().String() { + t.Fatalf("got peer: %v, want peer: %v", p.Addr, beLis.Addr()) + } + + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := resolveNowCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Errorf("unexpected resolveNow when grpclb gets no balancer address 2222, %d", i) + } + + tss.ls.sls <- sl + // Send an update with balancer address. The backends behind grpclb should + // be used. + rs := resolver.State{ + Addresses: []resolver.Address{{Addr: beLis.Addr().String()}}, + ServiceConfig: r.CC().ParseServiceConfig(grpclbConfig), + } + rs = grpclbstate.Set(rs, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: tss.lbAddr, ServerName: lbServerName}}}) + r.UpdateState(rs) + + select { + case <-ctx.Done(): + t.Fatalf("timeout when waiting for BalanceLoad RPC to be called on the remote balancer") + case <-tss.ls.balanceLoadCh: + } + + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tss.beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) + } + } +} + +// TestGRPCLB_PickFirst configures grpclb with pick_first as the child policy. +// The test changes the list of backend addresses returned by the remote +// balancer and verifies that RPCs are sent to the first address returned. +func (s) TestGRPCLB_PickFirst(t *testing.T) { + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 3, "", nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) + } + defer cleanup() + + beServers := []*lbpb.Server{{ + IpAddress: tss.beIPs[0].AsSlice(), + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, { + IpAddress: tss.beIPs[1].AsSlice(), + Port: int32(tss.bePorts[1]), + LoadBalanceToken: lbToken, + }, { + IpAddress: tss.beIPs[2].AsSlice(), + Port: int32(tss.bePorts[2]), + LoadBalanceToken: lbToken, + }} + beServerAddrs := []resolver.Address{} + for _, lis := range tss.beListeners { + beServerAddrs = append(beServerAddrs, resolver.Address{Addr: lis.Addr().String()}) + } + + // Connect to the test backends. + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + } + cc, err := grpc.NewClient(r.Scheme()+":///"+beServerName, dopts...) + if err != nil { + t.Fatalf("Failed to create a client for the backend: %v", err) + } + cc.Connect() + defer cc.Close() + + // Push a service config with grpclb as the load balancing policy and + // configure pick_first as its child policy. + rs := resolver.State{ServiceConfig: r.CC().ParseServiceConfig(`{"loadBalancingConfig":[{"grpclb":{"childPolicy":[{"pick_first":{}}]}}]}`)} + + // Push a resolver update with the remote balancer address specified via + // attributes. + r.UpdateState(grpclbstate.Set(rs, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: tss.lbAddr, ServerName: lbServerName}}})) + + // Push all three backend addresses to the remote balancer, and verify that + // RPCs are routed to the first backend. + tss.ls.sls <- &lbpb.ServerList{Servers: beServers[0:3]} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, beServerAddrs[0]); err != nil { + t.Fatal(err) + } + + // Update the address list with the remote balancer and verify pick_first + // behavior based on the new backends. + tss.ls.sls <- &lbpb.ServerList{Servers: beServers[2:]} + if err := pickfirst.CheckRPCsToBackend(ctx, cc, beServerAddrs[2]); err != nil { + t.Fatal(err) + } + + // Update the address list with the remote balancer and verify pick_first + // behavior based on the new backends. Since the currently connected backend + // is in the new list (even though it is not the first one on the list), + // pick_first will continue to use it. + tss.ls.sls <- &lbpb.ServerList{Servers: beServers[1:]} + if err := pickfirst.CheckRPCsToBackend(ctx, cc, beServerAddrs[2]); err != nil { + t.Fatal(err) + } + + // Switch child policy to roundrobin. + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tss.lbAddr, + ServerName: lbServerName, + }, + }, + } + rs = grpclbstate.Set(resolver.State{ServiceConfig: r.CC().ParseServiceConfig(grpclbConfig)}, s) + r.UpdateState(rs) + testC := testgrpc.NewTestServiceClient(cc) + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, beServerAddrs[1:]); err != nil { + t.Fatal(err) + } + + tss.ls.sls <- &lbpb.ServerList{Servers: beServers[0:3]} + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, beServerAddrs[0:3]); err != nil { + t.Fatal(err) + } +} + +// TestGRPCLB_BackendConnectionErrorPropagation tests the case where grpclb +// falls back to a backend which returns an error and the test verifies that the +// error is propagated to the RPC. +func (s) TestGRPCLB_BackendConnectionErrorPropagation(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + + // Start up an LB which will tells the client to fall back right away. + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 0, "", nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) + } + defer cleanup() + + // Start a standalone backend, to be used during fallback. The creds + // are intentionally misconfigured in order to simulate failure of a + // security handshake. + beLis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen %v", err) + } + defer beLis.Close() + standaloneBEs := startBackends(t, "arbitrary.invalid.name", true, beLis) + defer stopBackends(standaloneBEs) + + rs := resolver.State{ + Addresses: []resolver.Address{{Addr: beLis.Addr().String()}}, + ServiceConfig: internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(grpclbConfig), + } + rs = grpclbstate.Set(rs, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: tss.lbAddr, ServerName: lbServerName}}}) + r.InitialState(rs) + cc, err := grpc.NewClient(r.Scheme()+":///"+beServerName, + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer)) + if err != nil { + t.Fatalf("Failed to create a client for the backend: %v", err) + } + defer cc.Close() + testC := testgrpc.NewTestServiceClient(cc) + + // If https://github.com/grpc/grpc-go/blob/65cabd74d8e18d7347fecd414fa8d83a00035f5f/balancer/grpclb/grpclb_test.go#L103 + // changes, then expectedErrMsg may need to be updated. + const expectedErrMsg = "received unexpected server name" + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + var wg sync.WaitGroup + wg.Add(1) + go func() { + tss.ls.fallbackNow() + wg.Done() + }() + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}); err == nil || !strings.Contains(err.Error(), expectedErrMsg) { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, rpc error containing substring: %q", testC, err, expectedErrMsg) + } + wg.Wait() +} + +func testGRPCLBEmptyServerList(t *testing.T, svcfg string) { + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) + } + defer cleanup() + + beServers := []*lbpb.Server{{ + IpAddress: tss.beIPs[0].AsSlice(), + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }} + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + } + cc, err := grpc.NewClient(r.Scheme()+":///"+beServerName, dopts...) + if err != nil { + t.Fatalf("Failed to create a client for the backend %v", err) + } + cc.Connect() + defer cc.Close() + testC := testgrpc.NewTestServiceClient(cc) + + tss.ls.sls <- &lbpb.ServerList{Servers: beServers} + + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tss.lbAddr, + ServerName: lbServerName, + }, + }, + } + rs := grpclbstate.Set(resolver.State{ServiceConfig: r.CC().ParseServiceConfig(svcfg)}, s) + r.UpdateState(rs) + t.Log("Perform an initial RPC and expect it to succeed...") + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("Initial _.EmptyCall(_, _) = _, %v, want _, ", err) + } + t.Log("Now send an empty server list. Wait until we see an RPC failure to make sure the client got it...") + tss.ls.sls <- &lbpb.ServerList{} + gotError := false + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}); err != nil { + gotError = true + break + } + } + if !gotError { + t.Fatalf("Expected to eventually see an RPC fail after the grpclb sends an empty server list, but none did.") + } + t.Log("Now send a non-empty server list. A wait-for-ready RPC should now succeed...") + tss.ls.sls <- &lbpb.ServerList{Servers: beServers} + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("Final _.EmptyCall(_, _) = _, %v, want _, ", err) + } +} + +func (s) TestGRPCLBEmptyServerListRoundRobin(t *testing.T) { + testGRPCLBEmptyServerList(t, `{"loadBalancingConfig":[{"grpclb":{"childPolicy":[{"round_robin":{}}]}}]}`) +} + +func (s) TestGRPCLBEmptyServerListPickFirst(t *testing.T) { + testGRPCLBEmptyServerList(t, `{"loadBalancingConfig":[{"grpclb":{"childPolicy":[{"pick_first":{}}]}}]}`) +} + +func (s) TestGRPCLBWithTargetNameFieldInConfig(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) + } + defer cleanup() + sl := &lbpb.ServerList{ + Servers: []*lbpb.Server{ + { + IpAddress: tss.beIPs[0].AsSlice(), + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, + }, + } + // Push the backend address to the remote balancer. + tss.ls.sls <- sl + + cc, err := grpc.NewClient(r.Scheme()+":///"+beServerName, + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + grpc.WithUserAgent(testUserAgent)) + if err != nil { + t.Fatalf("Failed to create a client for the backend %v", err) + } + defer cc.Close() + cc.Connect() + testC := testgrpc.NewTestServiceClient(cc) + + // Push a resolver update with grpclb configuration which does not contain the + // target_name field. Our fake remote balancer is configured to always + // expect `beServerName` as the server name in the initial request. + rs := grpclbstate.Set(resolver.State{ServiceConfig: r.CC().ParseServiceConfig(grpclbConfig)}, + &grpclbstate.State{BalancerAddresses: []resolver.Address{{ + Addr: tss.lbAddr, + ServerName: lbServerName, + }}}) + r.UpdateState(rs) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout when waiting for BalanceLoad RPC to be called on the remote balancer") + case <-tss.ls.balanceLoadCh: + } + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) + } + + // When the value of target_field changes, grpclb will recreate the stream + // to the remote balancer. So, we need to update the fake remote balancer to + // expect a new server name in the initial request. + const newServerName = "new-server-name" + tss.ls.updateServerName(newServerName) + tss.ls.sls <- sl + + // Push the resolver update with target_field changed. + // Push a resolver update with grpclb configuration containing the + // target_name field. Our fake remote balancer has been updated above to expect the newServerName in the initial request. + lbCfg := fmt.Sprintf(`{"loadBalancingConfig": [{"grpclb": {"serviceName": "%s"}}]}`, newServerName) + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tss.lbAddr, + ServerName: lbServerName, + }, + }, + } + rs = grpclbstate.Set(resolver.State{ServiceConfig: r.CC().ParseServiceConfig(lbCfg)}, s) + r.UpdateState(rs) + select { + case <-ctx.Done(): + t.Fatalf("timeout when waiting for BalanceLoad RPC to be called on the remote balancer") + case <-tss.ls.balanceLoadCh: + } + + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) + } +} + +type failPreRPCCred struct{} + +func (failPreRPCCred) GetRequestMetadata(_ context.Context, uri ...string) (map[string]string, error) { + if strings.Contains(uri[0], failtosendURI) { + return nil, fmt.Errorf("rpc should fail to send") + } + return nil, nil +} + +func (failPreRPCCred) RequireTransportSecurity() bool { + return false +} + +func checkStats(stats, expected *rpcStats) error { + if !stats.equal(expected) { + return fmt.Errorf("stats not equal: got %+v, want %+v", stats, expected) + } + return nil +} + +func runAndCheckStats(t *testing.T, drop bool, statsChan chan *lbpb.ClientStats, runRPCs func(*grpc.ClientConn), statsWant *rpcStats) error { + r := manual.NewBuilderWithScheme("whatever") + + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", statsChan) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) + } + defer cleanup() + servers := []*lbpb.Server{{ + IpAddress: tss.beIPs[0].AsSlice(), + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }} + if drop { + servers = append(servers, &lbpb.Server{ + LoadBalanceToken: lbToken, + Drop: drop, + }) + } + tss.ls.sls <- &lbpb.ServerList{Servers: servers} + tss.ls.statsDura = 100 * time.Millisecond + creds := serverNameCheckCreds{} + + cc, err := grpc.NewClient(r.Scheme()+":///"+beServerName, grpc.WithResolvers(r), + grpc.WithTransportCredentials(&creds), + grpc.WithPerRPCCredentials(failPreRPCCred{}), + grpc.WithContextDialer(fakeNameDialer)) + if err != nil { + t.Fatalf("Failed to create a client for the backend %v", err) + } + cc.Connect() + defer cc.Close() + + rstate := resolver.State{ServiceConfig: r.CC().ParseServiceConfig(grpclbConfig)} + r.UpdateState(grpclbstate.Set(rstate, &grpclbstate.State{BalancerAddresses: []resolver.Address{{ + Addr: tss.lbAddr, + ServerName: lbServerName, + }}})) + + runRPCs(cc) + end := time.Now().Add(time.Second) + for time.Now().Before(end) { + if err := checkStats(tss.ls.stats, statsWant); err == nil { + time.Sleep(200 * time.Millisecond) // sleep for two intervals to make sure no new stats are reported. + break + } + } + return checkStats(tss.ls.stats, statsWant) +} + +const ( + countRPC = 40 + failtosendURI = "failtosend" +) + +func (s) TestGRPCLBStatsUnarySuccess(t *testing.T) { + if err := runAndCheckStats(t, false, nil, func(cc *grpc.ClientConn) { + testC := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout) + defer cancel() + // The first non-failfast RPC succeeds, all connections are up. + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) + } + for i := 0; i < countRPC-1; i++ { + testC.EmptyCall(ctx, &testpb.Empty{}) + } + }, &rpcStats{ + numCallsStarted: int64(countRPC), + numCallsFinished: int64(countRPC), + numCallsFinishedKnownReceived: int64(countRPC), + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestGRPCLBStatsUnaryDrop(t *testing.T) { + if err := runAndCheckStats(t, true, nil, func(cc *grpc.ClientConn) { + testC := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout) + defer cancel() + // The first non-failfast RPC succeeds, all connections are up. + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) + } + for i := 0; i < countRPC-1; i++ { + testC.EmptyCall(ctx, &testpb.Empty{}) + } + }, &rpcStats{ + numCallsStarted: int64(countRPC), + numCallsFinished: int64(countRPC), + numCallsFinishedKnownReceived: int64(countRPC) / 2, + numCallsDropped: map[string]int64{lbToken: int64(countRPC) / 2}, + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestGRPCLBStatsUnaryFailedToSend(t *testing.T) { + if err := runAndCheckStats(t, false, nil, func(cc *grpc.ClientConn) { + testC := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout) + defer cancel() + // The first non-failfast RPC succeeds, all connections are up. + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) + } + for i := 0; i < countRPC-1; i++ { + cc.Invoke(ctx, failtosendURI, &testpb.Empty{}, nil) + } + }, &rpcStats{ + numCallsStarted: int64(countRPC), + numCallsFinished: int64(countRPC), + numCallsFinishedWithClientFailedToSend: int64(countRPC) - 1, + numCallsFinishedKnownReceived: 1, + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestGRPCLBStatsStreamingSuccess(t *testing.T) { + if err := runAndCheckStats(t, false, nil, func(cc *grpc.ClientConn) { + testC := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout) + defer cancel() + // The first non-failfast RPC succeeds, all connections are up. + stream, err := testC.FullDuplexCall(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_, _) = _, %v, want _, ", testC, err) + } + for { + if _, err = stream.Recv(); err == io.EOF { + break + } + } + for i := 0; i < countRPC-1; i++ { + stream, err = testC.FullDuplexCall(ctx) + if err == nil { + // Wait for stream to end if err is nil. + for { + if _, err = stream.Recv(); err == io.EOF { + break + } + } + } + } + }, &rpcStats{ + numCallsStarted: int64(countRPC), + numCallsFinished: int64(countRPC), + numCallsFinishedKnownReceived: int64(countRPC), + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestGRPCLBStatsStreamingDrop(t *testing.T) { + if err := runAndCheckStats(t, true, nil, func(cc *grpc.ClientConn) { + testC := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout) + defer cancel() + // The first non-failfast RPC succeeds, all connections are up. + stream, err := testC.FullDuplexCall(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_, _) = _, %v, want _, ", testC, err) + } + for { + if _, err = stream.Recv(); err == io.EOF { + break + } + } + for i := 0; i < countRPC-1; i++ { + stream, err = testC.FullDuplexCall(ctx) + if err == nil { + // Wait for stream to end if err is nil. + for { + if _, err = stream.Recv(); err == io.EOF { + break + } + } + } + } + }, &rpcStats{ + numCallsStarted: int64(countRPC), + numCallsFinished: int64(countRPC), + numCallsFinishedKnownReceived: int64(countRPC) / 2, + numCallsDropped: map[string]int64{lbToken: int64(countRPC) / 2}, + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestGRPCLBStatsStreamingFailedToSend(t *testing.T) { + if err := runAndCheckStats(t, false, nil, func(cc *grpc.ClientConn) { + testC := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout) + defer cancel() + // The first non-failfast RPC succeeds, all connections are up. + stream, err := testC.FullDuplexCall(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_, _) = _, %v, want _, ", testC, err) + } + for { + if _, err = stream.Recv(); err == io.EOF { + break + } + } + for i := 0; i < countRPC-1; i++ { + cc.NewStream(ctx, &grpc.StreamDesc{}, failtosendURI) + } + }, &rpcStats{ + numCallsStarted: int64(countRPC), + numCallsFinished: int64(countRPC), + numCallsFinishedWithClientFailedToSend: int64(countRPC) - 1, + numCallsFinishedKnownReceived: 1, + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestGRPCLBStatsQuashEmpty(t *testing.T) { + ch := make(chan *lbpb.ClientStats) + defer close(ch) + if err := runAndCheckStats(t, false, ch, func(*grpc.ClientConn) { + // Perform no RPCs; wait for load reports to start, which should be + // zero, then expect no other load report within 5x the update + // interval. + select { + case st := <-ch: + if !isZeroStats(st) { + t.Errorf("got stats %v; want all zero", st) + } + case <-time.After(5 * time.Second): + t.Errorf("did not get initial stats report after 5 seconds") + return + } + + select { + case st := <-ch: + t.Errorf("got unexpected stats report: %v", st) + case <-time.After(500 * time.Millisecond): + // Success. + } + go func() { + for range ch { // Drain statsChan until it is closed. + } + }() + }, &rpcStats{ + numCallsStarted: 0, + numCallsFinished: 0, + numCallsFinishedKnownReceived: 0, + }); err != nil { + t.Fatal(err) + } +} diff --git a/balancer/grpclb/grpclb_util.go b/balancer/grpclb/grpclb_util.go new file mode 100644 index 0000000..c0f762c --- /dev/null +++ b/balancer/grpclb/grpclb_util.go @@ -0,0 +1,173 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpclb + +import ( + "fmt" + "sync" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" +) + +const subConnCacheTime = time.Second * 10 + +// lbCacheClientConn is a wrapper balancer.ClientConn with a SubConn cache. +// SubConns will be kept in cache for subConnCacheTime before being shut down. +// +// Its NewSubconn and SubConn.Shutdown methods are updated to do cache first. +type lbCacheClientConn struct { + balancer.ClientConn + + timeout time.Duration + + mu sync.Mutex + // subConnCache only keeps subConns that are being deleted. + subConnCache map[resolver.Address]*subConnCacheEntry + subConnToAddr map[balancer.SubConn]resolver.Address +} + +type subConnCacheEntry struct { + sc balancer.SubConn + + cancel func() + abortDeleting bool +} + +func newLBCacheClientConn(cc balancer.ClientConn) *lbCacheClientConn { + return &lbCacheClientConn{ + ClientConn: cc, + timeout: subConnCacheTime, + subConnCache: make(map[resolver.Address]*subConnCacheEntry), + subConnToAddr: make(map[balancer.SubConn]resolver.Address), + } +} + +func (ccc *lbCacheClientConn) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + if len(addrs) != 1 { + return nil, fmt.Errorf("grpclb calling NewSubConn with addrs of length %v", len(addrs)) + } + addrWithoutAttrs := addrs[0] + addrWithoutAttrs.Attributes = nil + + ccc.mu.Lock() + defer ccc.mu.Unlock() + if entry, ok := ccc.subConnCache[addrWithoutAttrs]; ok { + // If entry is in subConnCache, the SubConn was being deleted. + // cancel function will never be nil. + entry.cancel() + delete(ccc.subConnCache, addrWithoutAttrs) + return entry.sc, nil + } + + scNew, err := ccc.ClientConn.NewSubConn(addrs, opts) + if err != nil { + return nil, err + } + scNew = &lbCacheSubConn{SubConn: scNew, ccc: ccc} + + ccc.subConnToAddr[scNew] = addrWithoutAttrs + return scNew, nil +} + +func (ccc *lbCacheClientConn) RemoveSubConn(sc balancer.SubConn) { + logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc) +} + +type lbCacheSubConn struct { + balancer.SubConn + ccc *lbCacheClientConn +} + +func (sc *lbCacheSubConn) Shutdown() { + ccc := sc.ccc + ccc.mu.Lock() + defer ccc.mu.Unlock() + addr, ok := ccc.subConnToAddr[sc] + if !ok { + return + } + + if entry, ok := ccc.subConnCache[addr]; ok { + if entry.sc != sc { + // This could happen if NewSubConn was called multiple times for + // the same address, and those SubConns are all shut down. We + // remove sc immediately here. + delete(ccc.subConnToAddr, sc) + sc.SubConn.Shutdown() + } + return + } + + entry := &subConnCacheEntry{ + sc: sc, + } + ccc.subConnCache[addr] = entry + + timer := time.AfterFunc(ccc.timeout, func() { + ccc.mu.Lock() + defer ccc.mu.Unlock() + if entry.abortDeleting { + return + } + sc.SubConn.Shutdown() + delete(ccc.subConnToAddr, sc) + delete(ccc.subConnCache, addr) + }) + entry.cancel = func() { + if !timer.Stop() { + // If stop was not successful, the timer has fired (this can only + // happen in a race). But the deleting function is blocked on ccc.mu + // because the mutex was held by the caller of this function. + // + // Set abortDeleting to true to abort the deleting function. When + // the lock is released, the deleting function will acquire the + // lock, check the value of abortDeleting and return. + entry.abortDeleting = true + } + } +} + +func (ccc *lbCacheClientConn) UpdateState(s balancer.State) { + s.Picker = &lbCachePicker{Picker: s.Picker} + ccc.ClientConn.UpdateState(s) +} + +func (ccc *lbCacheClientConn) close() { + ccc.mu.Lock() + defer ccc.mu.Unlock() + // Only cancel all existing timers. There's no need to shut down SubConns. + for _, entry := range ccc.subConnCache { + entry.cancel() + } +} + +type lbCachePicker struct { + balancer.Picker +} + +func (cp *lbCachePicker) Pick(i balancer.PickInfo) (balancer.PickResult, error) { + res, err := cp.Picker.Pick(i) + if err != nil { + return res, err + } + res.SubConn = res.SubConn.(*lbCacheSubConn).SubConn + return res, nil +} diff --git a/balancer/grpclb/grpclb_util_test.go b/balancer/grpclb/grpclb_util_test.go new file mode 100644 index 0000000..d4217fb --- /dev/null +++ b/balancer/grpclb/grpclb_util_test.go @@ -0,0 +1,267 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpclb + +import ( + "fmt" + "sync" + "testing" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" +) + +type mockSubConn struct { + balancer.SubConn + mcc *mockClientConn +} + +func (msc *mockSubConn) Shutdown() { + msc.mcc.mu.Lock() + defer msc.mcc.mu.Unlock() + delete(msc.mcc.subConns, msc) +} + +type mockClientConn struct { + balancer.ClientConn + + mu sync.Mutex + subConns map[balancer.SubConn]resolver.Address +} + +func newMockClientConn() *mockClientConn { + return &mockClientConn{ + subConns: make(map[balancer.SubConn]resolver.Address), + } +} + +func (mcc *mockClientConn) NewSubConn(addrs []resolver.Address, _ balancer.NewSubConnOptions) (balancer.SubConn, error) { + sc := &mockSubConn{mcc: mcc} + mcc.mu.Lock() + defer mcc.mu.Unlock() + mcc.subConns[sc] = addrs[0] + return sc, nil +} + +func (mcc *mockClientConn) RemoveSubConn(sc balancer.SubConn) { + panic(fmt.Sprintf("RemoveSubConn(%v) called unexpectedly", sc)) +} + +const testCacheTimeout = 100 * time.Millisecond + +func checkMockCC(mcc *mockClientConn, scLen int) error { + mcc.mu.Lock() + defer mcc.mu.Unlock() + if len(mcc.subConns) != scLen { + return fmt.Errorf("mcc = %+v, want len(mcc.subConns) = %v", mcc.subConns, scLen) + } + return nil +} + +func checkCacheCC(ccc *lbCacheClientConn, sccLen, sctaLen int) error { + ccc.mu.Lock() + defer ccc.mu.Unlock() + if len(ccc.subConnCache) != sccLen { + return fmt.Errorf("ccc = %+v, want len(ccc.subConnCache) = %v", ccc.subConnCache, sccLen) + } + if len(ccc.subConnToAddr) != sctaLen { + return fmt.Errorf("ccc = %+v, want len(ccc.subConnToAddr) = %v", ccc.subConnToAddr, sctaLen) + } + return nil +} + +// Test that SubConn won't be immediately shut down. +func (s) TestLBCacheClientConnExpire(t *testing.T) { + mcc := newMockClientConn() + if err := checkMockCC(mcc, 0); err != nil { + t.Fatal(err) + } + + ccc := newLBCacheClientConn(mcc) + ccc.timeout = testCacheTimeout + if err := checkCacheCC(ccc, 0, 0); err != nil { + t.Fatal(err) + } + + sc, _ := ccc.NewSubConn([]resolver.Address{{Addr: "address1"}}, balancer.NewSubConnOptions{}) + // One subconn in MockCC. + if err := checkMockCC(mcc, 1); err != nil { + t.Fatal(err) + } + // No subconn being deleted, and one in CacheCC. + if err := checkCacheCC(ccc, 0, 1); err != nil { + t.Fatal(err) + } + + sc.Shutdown() + // One subconn in MockCC before timeout. + if err := checkMockCC(mcc, 1); err != nil { + t.Fatal(err) + } + // One subconn being deleted, and one in CacheCC. + if err := checkCacheCC(ccc, 1, 1); err != nil { + t.Fatal(err) + } + + // Should all become empty after timeout. + var err error + for i := 0; i < 2; i++ { + time.Sleep(testCacheTimeout) + err = checkMockCC(mcc, 0) + if err != nil { + continue + } + err = checkCacheCC(ccc, 0, 0) + if err != nil { + continue + } + } + if err != nil { + t.Fatal(err) + } +} + +// Test that NewSubConn with the same address of a SubConn being shut down will +// reuse the SubConn and cancel the removing. +func (s) TestLBCacheClientConnReuse(t *testing.T) { + mcc := newMockClientConn() + if err := checkMockCC(mcc, 0); err != nil { + t.Fatal(err) + } + + ccc := newLBCacheClientConn(mcc) + ccc.timeout = testCacheTimeout + if err := checkCacheCC(ccc, 0, 0); err != nil { + t.Fatal(err) + } + + sc, _ := ccc.NewSubConn([]resolver.Address{{Addr: "address1"}}, balancer.NewSubConnOptions{}) + // One subconn in MockCC. + if err := checkMockCC(mcc, 1); err != nil { + t.Fatal(err) + } + // No subconn being deleted, and one in CacheCC. + if err := checkCacheCC(ccc, 0, 1); err != nil { + t.Fatal(err) + } + + sc.Shutdown() + // One subconn in MockCC before timeout. + if err := checkMockCC(mcc, 1); err != nil { + t.Fatal(err) + } + // One subconn being deleted, and one in CacheCC. + if err := checkCacheCC(ccc, 1, 1); err != nil { + t.Fatal(err) + } + + // Recreate the old subconn, this should cancel the deleting process. + sc, _ = ccc.NewSubConn([]resolver.Address{{Addr: "address1"}}, balancer.NewSubConnOptions{}) + // One subconn in MockCC. + if err := checkMockCC(mcc, 1); err != nil { + t.Fatal(err) + } + // No subconn being deleted, and one in CacheCC. + if err := checkCacheCC(ccc, 0, 1); err != nil { + t.Fatal(err) + } + + var err error + // Should not become empty after 2*timeout. + time.Sleep(2 * testCacheTimeout) + err = checkMockCC(mcc, 1) + if err != nil { + t.Fatal(err) + } + err = checkCacheCC(ccc, 0, 1) + if err != nil { + t.Fatal(err) + } + + // Call Shutdown again, will delete after timeout. + sc.Shutdown() + // One subconn in MockCC before timeout. + if err := checkMockCC(mcc, 1); err != nil { + t.Fatal(err) + } + // One subconn being deleted, and one in CacheCC. + if err := checkCacheCC(ccc, 1, 1); err != nil { + t.Fatal(err) + } + + // Should all become empty after timeout. + for i := 0; i < 2; i++ { + time.Sleep(testCacheTimeout) + err = checkMockCC(mcc, 0) + if err != nil { + continue + } + err = checkCacheCC(ccc, 0, 0) + if err != nil { + continue + } + } + if err != nil { + t.Fatal(err) + } +} + +// Test that if the timer to shut down a SubConn fires at the same time +// NewSubConn cancels the timer, it doesn't cause deadlock. +func (s) TestLBCache_ShutdownTimer_New_Race(t *testing.T) { + mcc := newMockClientConn() + if err := checkMockCC(mcc, 0); err != nil { + t.Fatal(err) + } + + ccc := newLBCacheClientConn(mcc) + ccc.timeout = time.Nanosecond + if err := checkCacheCC(ccc, 0, 0); err != nil { + t.Fatal(err) + } + + sc, _ := ccc.NewSubConn([]resolver.Address{{Addr: "address1"}}, balancer.NewSubConnOptions{}) + // One subconn in MockCC. + if err := checkMockCC(mcc, 1); err != nil { + t.Fatal(err) + } + // No subconn being deleted, and one in CacheCC. + if err := checkCacheCC(ccc, 0, 1); err != nil { + t.Fatal(err) + } + + done := make(chan struct{}) + + go func() { + for i := 0; i < 1000; i++ { + // Shutdown starts a timer with 1 ns timeout, the NewSubConn will + // race with the timer. + sc.Shutdown() + sc, _ = ccc.NewSubConn([]resolver.Address{{Addr: "address1"}}, balancer.NewSubConnOptions{}) + } + close(done) + }() + + select { + case <-time.After(time.Second): + t.Fatalf("Test didn't finish within 1 second. Deadlock") + case <-done: + } +} diff --git a/balancer/grpclb/state/state.go b/balancer/grpclb/state/state.go new file mode 100644 index 0000000..4ecfa1c --- /dev/null +++ b/balancer/grpclb/state/state.go @@ -0,0 +1,51 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package state declares grpclb types to be set by resolvers wishing to pass +// information to grpclb via resolver.State Attributes. +package state + +import ( + "google.golang.org/grpc/resolver" +) + +// keyType is the key to use for storing State in Attributes. +type keyType string + +const key = keyType("grpc.grpclb.state") + +// State contains gRPCLB-relevant data passed from the name resolver. +type State struct { + // BalancerAddresses contains the remote load balancer address(es). If + // set, overrides any resolver-provided addresses with Type of GRPCLB. + BalancerAddresses []resolver.Address +} + +// Set returns a copy of the provided state with attributes containing s. s's +// data should not be mutated after calling Set. +func Set(state resolver.State, s *State) resolver.State { + state.Attributes = state.Attributes.WithValue(key, s) + return state +} + +// Get returns the grpclb State in the resolver.State, or nil if not present. +// The returned data should not be mutated. +func Get(state resolver.State) *State { + s, _ := state.Attributes.Value(key).(*State) + return s +} diff --git a/balancer/lazy/lazy.go b/balancer/lazy/lazy.go new file mode 100644 index 0000000..314ccd1 --- /dev/null +++ b/balancer/lazy/lazy.go @@ -0,0 +1,157 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package lazy contains a load balancer that starts in IDLE instead of +// CONNECTING. Once it starts connecting, it instantiates its delegate. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +package lazy + +import ( + "fmt" + "sync" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" + + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +var ( + logger = grpclog.Component("lazy-lb") +) + +const ( + logPrefix = "[lazy-lb %p] " +) + +// ChildBuilderFunc creates a new balancer with the ClientConn. It has the same +// type as the balancer.Builder.Build method. +type ChildBuilderFunc func(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer + +// NewBalancer is the constructor for the lazy balancer. +func NewBalancer(cc balancer.ClientConn, bOpts balancer.BuildOptions, childBuilder ChildBuilderFunc) balancer.Balancer { + b := &lazyBalancer{ + cc: cc, + buildOptions: bOpts, + childBuilder: childBuilder, + } + b.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(logPrefix, b)) + cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Idle, + Picker: &idlePicker{exitIdle: sync.OnceFunc(func() { + // Call ExitIdle in a new goroutine to avoid deadlocks while calling + // back into the channel synchronously. + go b.ExitIdle() + })}, + }) + return b +} + +type lazyBalancer struct { + // The following fields are initialized at build time and read-only after + // that and therefore do not need to be guarded by a mutex. + cc balancer.ClientConn + buildOptions balancer.BuildOptions + logger *internalgrpclog.PrefixLogger + childBuilder ChildBuilderFunc + + // The following fields are accessed while handling calls to the idlePicker + // and when handling ClientConn state updates. They are guarded by a mutex. + + mu sync.Mutex + delegate balancer.Balancer + latestClientConnState *balancer.ClientConnState + latestResolverError error +} + +func (lb *lazyBalancer) Close() { + lb.mu.Lock() + defer lb.mu.Unlock() + if lb.delegate != nil { + lb.delegate.Close() + lb.delegate = nil + } +} + +func (lb *lazyBalancer) ResolverError(err error) { + lb.mu.Lock() + defer lb.mu.Unlock() + if lb.delegate != nil { + lb.delegate.ResolverError(err) + return + } + lb.latestResolverError = err +} + +func (lb *lazyBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { + lb.mu.Lock() + defer lb.mu.Unlock() + if lb.delegate != nil { + return lb.delegate.UpdateClientConnState(ccs) + } + + lb.latestClientConnState = &ccs + lb.latestResolverError = nil + return nil +} + +// UpdateSubConnState implements balancer.Balancer. +func (lb *lazyBalancer) UpdateSubConnState(balancer.SubConn, balancer.SubConnState) { + // UpdateSubConnState is deprecated. +} + +func (lb *lazyBalancer) ExitIdle() { + lb.mu.Lock() + defer lb.mu.Unlock() + if lb.delegate != nil { + lb.delegate.ExitIdle() + return + } + lb.delegate = lb.childBuilder(lb.cc, lb.buildOptions) + if lb.latestClientConnState != nil { + if err := lb.delegate.UpdateClientConnState(*lb.latestClientConnState); err != nil { + if err == balancer.ErrBadResolverState { + lb.cc.ResolveNow(resolver.ResolveNowOptions{}) + } else { + lb.logger.Warningf("Error from child policy on receiving initial state: %v", err) + } + } + lb.latestClientConnState = nil + } + if lb.latestResolverError != nil { + lb.delegate.ResolverError(lb.latestResolverError) + lb.latestResolverError = nil + } +} + +// idlePicker is used when the SubConn is IDLE and kicks the SubConn into +// CONNECTING when Pick is called. +type idlePicker struct { + exitIdle func() +} + +func (i *idlePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + i.exitIdle() + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable +} diff --git a/balancer/lazy/lazy_ext_test.go b/balancer/lazy/lazy_ext_test.go new file mode 100644 index 0000000..dee0770 --- /dev/null +++ b/balancer/lazy/lazy_ext_test.go @@ -0,0 +1,466 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package lazy_test + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/lazy" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const ( + // Default timeout for tests in this package. + defaultTestTimeout = 10 * time.Second + // Default short timeout, to be used when waiting for events which are not + // expected to happen. + defaultTestShortTimeout = 100 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestExitIdle creates a lazy balancer than manages a pickfirst child. The test +// calls Connect() on the channel which in turn calls ExitIdle on the lazy +// balancer. The test verifies that the channel enters READY. +func (s) TestExitIdle(t *testing.T) { + backend1 := stubserver.StartTestService(t, nil) + defer backend1.Stop() + + mr := manual.NewBuilderWithScheme("e2e-test") + defer mr.Close() + + mr.InitialState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: backend1.Address}}}, + }, + }) + + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = lazy.NewBalancer(bd.ClientConn, bd.BuildOptions, balancer.Get(pickfirst.Name).Build) + }, + ExitIdle: func(bd *stub.BalancerData) { + bd.ChildBalancer.ExitIdle() + }, + ResolverError: func(bd *stub.BalancerData, err error) { + bd.ChildBalancer.ResolverError(err) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + } + stub.Register(t.Name(), bf) + json := fmt.Sprintf(`{"loadBalancingConfig": [{"%s": {}}]}`, t.Name()) + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(json), + grpc.WithResolvers(mr), + } + cc, err := grpc.NewClient(mr.Scheme()+":///", opts...) + if err != nil { + t.Fatalf("grpc.NewClient(_) failed: %v", err) + } + defer cc.Close() + + cc.Connect() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Send a resolver update to verify that the resolver state is correctly + // passed through to the leaf pickfirst balancer. + backend2 := stubserver.StartTestService(t, nil) + defer backend2.Stop() + + mr.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: backend2.Address}}}, + }, + }) + + var peer peer.Peer + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + t.Errorf("client.EmptyCall() returned unexpected error: %v", err) + } + if got, want := peer.Addr.String(), backend2.Address; got != want { + t.Errorf("EmptyCall() went to unexpected backend: got %q, want %q", got, want) + } +} + +// TestPicker creates a lazy balancer under a stub balancer which block all +// calls to ExitIdle. This ensures the only way to trigger lazy to exit idle is +// through the picker. The test makes an RPC and ensures it succeeds. +func (s) TestPicker(t *testing.T) { + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = lazy.NewBalancer(bd.ClientConn, bd.BuildOptions, balancer.Get(pickfirst.Name).Build) + }, + ExitIdle: func(*stub.BalancerData) { + t.Log("Ignoring call to ExitIdle, calling the picker should make the lazy balancer exit IDLE state.") + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + } + + name := strings.ReplaceAll(strings.ToLower(t.Name()), "/", "") + stub.Register(name, bf) + json := fmt.Sprintf(`{"loadBalancingConfig": [{%q: {}}]}`, name) + + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(json), + } + cc, err := grpc.NewClient(backend.Address, opts...) + if err != nil { + t.Fatalf("grpc.NewClient(_) failed: %v", err) + } + defer cc.Close() + + // The channel should remain in IDLE as the ExitIdle calls are not + // propagated to the lazy balancer from the stub balancer. + cc.Connect() + shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer shortCancel() + testutils.AwaitNoStateChange(shortCtx, t, cc, connectivity.Idle) + + // The picker from the lazy balancer should be send to the channel when the + // first resolver update is received by lazy. Making an RPC should trigger + // child creation. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("client.EmptyCall() returned unexpected error: %v", err) + } +} + +// Tests the scenario when a resolver produces a good state followed by a +// resolver error. The test verifies that the child balancer receives the good +// update followed by the error. +func (s) TestGoodUpdateThenResolverError(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + resolverStateReceived := false + resolverErrorReceived := grpcsync.NewEvent() + + childBF := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = balancer.Get(pickfirst.Name).Build(bd.ClientConn, bd.BuildOptions) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + if resolverErrorReceived.HasFired() { + t.Error("Received resolver error before resolver state.") + } + resolverStateReceived = true + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + ResolverError: func(bd *stub.BalancerData, err error) { + if !resolverStateReceived { + t.Error("Received resolver error before resolver state.") + } + resolverErrorReceived.Fire() + bd.ChildBalancer.ResolverError(err) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + } + + childBalName := strings.ReplaceAll(strings.ToLower(t.Name())+"_child", "/", "") + stub.Register(childBalName, childBF) + + topLevelBF := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = lazy.NewBalancer(bd.ClientConn, bd.BuildOptions, balancer.Get(childBalName).Build) + }, + ExitIdle: func(*stub.BalancerData) { + t.Log("Ignoring call to ExitIdle to delay lazy child creation until RPC time.") + }, + ResolverError: func(bd *stub.BalancerData, err error) { + bd.ChildBalancer.ResolverError(err) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + } + + topLevelBalName := strings.ReplaceAll(strings.ToLower(t.Name())+"_top_level", "/", "") + stub.Register(topLevelBalName, topLevelBF) + + json := fmt.Sprintf(`{"loadBalancingConfig": [{%q: {}}]}`, topLevelBalName) + + mr := manual.NewBuilderWithScheme("e2e-test") + defer mr.Close() + + mr.InitialState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: backend.Address}}}, + }, + }) + + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(mr), + grpc.WithDefaultServiceConfig(json), + } + cc, err := grpc.NewClient(mr.Scheme()+":///whatever", opts...) + if err != nil { + t.Fatalf("grpc.NewClient(_) failed: %v", err) + + } + + defer cc.Close() + cc.Connect() + + mr.CC().ReportError(errors.New("test error")) + // The channel should remain in IDLE as the ExitIdle calls are not + // propagated to the lazy balancer from the stub balancer. + shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer shortCancel() + testutils.AwaitNoStateChange(shortCtx, t, cc, connectivity.Idle) + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("client.EmptyCall() returned unexpected error: %v", err) + } + + if !resolverStateReceived { + t.Fatalf("Child balancer did not receive resolver state.") + } + + select { + case <-resolverErrorReceived.Done(): + case <-ctx.Done(): + t.Fatal("Context timed out waiting for resolver error to be delivered to child balancer.") + } +} + +// Tests the scenario when a resolver produces a list of endpoints followed by +// a resolver error. The test verifies that the child balancer receives only the +// good update. +func (s) TestResolverErrorThenGoodUpdate(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + + childBF := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = balancer.Get(pickfirst.Name).Build(bd.ClientConn, bd.BuildOptions) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + ResolverError: func(bd *stub.BalancerData, err error) { + t.Error("Received unexpected resolver error.") + bd.ChildBalancer.ResolverError(err) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + } + + childBalName := strings.ReplaceAll(strings.ToLower(t.Name())+"_child", "/", "") + stub.Register(childBalName, childBF) + + topLevelBF := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = lazy.NewBalancer(bd.ClientConn, bd.BuildOptions, balancer.Get(childBalName).Build) + }, + ExitIdle: func(*stub.BalancerData) { + t.Log("Ignoring call to ExitIdle to delay lazy child creation until RPC time.") + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + } + + topLevelBalName := strings.ReplaceAll(strings.ToLower(t.Name())+"_top_level", "/", "") + stub.Register(topLevelBalName, topLevelBF) + + json := fmt.Sprintf(`{"loadBalancingConfig": [{%q: {}}]}`, topLevelBalName) + + mr := manual.NewBuilderWithScheme("e2e-test") + defer mr.Close() + + mr.InitialState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: backend.Address}}}, + }, + }) + + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(mr), + grpc.WithDefaultServiceConfig(json), + } + cc, err := grpc.NewClient(mr.Scheme()+":///whatever", opts...) + if err != nil { + t.Fatalf("grpc.NewClient(_) failed: %v", err) + + } + + defer cc.Close() + cc.Connect() + + // Send an error followed by a good update. + mr.CC().ReportError(errors.New("test error")) + mr.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: backend.Address}}}, + }, + }) + + // The channel should remain in IDLE as the ExitIdle calls are not + // propagated to the lazy balancer from the stub balancer. + shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer shortCancel() + testutils.AwaitNoStateChange(shortCtx, t, cc, connectivity.Idle) + + // An RPC would succeed only if the leaf pickfirst receives the endpoint + // list. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("client.EmptyCall() returned unexpected error: %v", err) + } +} + +// Tests that ExitIdle calls are correctly passed through to the child balancer. +// It starts a backend and ensures the channel connects to it. The test then +// stops the backend, making the channel enter IDLE. The test calls Connect on +// the channel and verifies that the child balancer exits idle. +func (s) TestExitIdlePassthrough(t *testing.T) { + backend1 := stubserver.StartTestService(t, nil) + defer backend1.Stop() + + mr := manual.NewBuilderWithScheme("e2e-test") + defer mr.Close() + + mr.InitialState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: backend1.Address}}}, + }, + }) + + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = lazy.NewBalancer(bd.ClientConn, bd.BuildOptions, balancer.Get(pickfirst.Name).Build) + }, + ExitIdle: func(bd *stub.BalancerData) { + bd.ChildBalancer.ExitIdle() + }, + ResolverError: func(bd *stub.BalancerData, err error) { + bd.ChildBalancer.ResolverError(err) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + } + stub.Register(t.Name(), bf) + json := fmt.Sprintf(`{"loadBalancingConfig": [{"%s": {}}]}`, t.Name()) + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(json), + grpc.WithResolvers(mr), + } + cc, err := grpc.NewClient(mr.Scheme()+":///", opts...) + if err != nil { + t.Fatalf("grpc.NewClient(_) failed: %v", err) + + } + defer cc.Close() + + cc.Connect() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Stopping the active backend should put the channel in IDLE. + backend1.Stop() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Sending a new backend address should not kick the channel out of IDLE. + // On calling cc.Connect(), the channel should call ExitIdle on the lazy + // balancer which passes through the call to the leaf pickfirst. + backend2 := stubserver.StartTestService(t, nil) + defer backend2.Stop() + + mr.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: backend2.Address}}}, + }, + }) + + shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer shortCancel() + testutils.AwaitNoStateChange(shortCtx, t, cc, connectivity.Idle) + + cc.Connect() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) +} diff --git a/balancer/leastrequest/leastrequest.go b/balancer/leastrequest/leastrequest.go new file mode 100644 index 0000000..019d1a2 --- /dev/null +++ b/balancer/leastrequest/leastrequest.go @@ -0,0 +1,250 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package leastrequest implements a least request load balancer. +package leastrequest + +import ( + "encoding/json" + "fmt" + rand "math/rand/v2" + "sync" + "sync/atomic" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/endpointsharding" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Name is the name of the least request balancer. +const Name = "least_request_experimental" + +var ( + // randuint32 is a global to stub out in tests. + randuint32 = rand.Uint32 + logger = grpclog.Component("least-request") +) + +func init() { + balancer.Register(bb{}) +} + +// LBConfig is the balancer config for least_request_experimental balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + // ChoiceCount is the number of random SubConns to sample to find the one + // with the fewest outstanding requests. If unset, defaults to 2. If set to + // < 2, the config will be rejected, and if set to > 10, will become 10. + ChoiceCount uint32 `json:"choiceCount,omitempty"` +} + +type bb struct{} + +func (bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + lbConfig := &LBConfig{ + ChoiceCount: 2, + } + if err := json.Unmarshal(s, lbConfig); err != nil { + return nil, fmt.Errorf("least-request: unable to unmarshal LBConfig: %v", err) + } + // "If `choice_count < 2`, the config will be rejected." - A48 + if lbConfig.ChoiceCount < 2 { // sweet + return nil, fmt.Errorf("least-request: lbConfig.choiceCount: %v, must be >= 2", lbConfig.ChoiceCount) + } + // "If a LeastRequestLoadBalancingConfig with a choice_count > 10 is + // received, the least_request_experimental policy will set choice_count = + // 10." - A48 + if lbConfig.ChoiceCount > 10 { + lbConfig.ChoiceCount = 10 + } + return lbConfig, nil +} + +func (bb) Name() string { + return Name +} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &leastRequestBalancer{ + ClientConn: cc, + endpointRPCCounts: resolver.NewEndpointMap[*atomic.Int32](), + } + b.child = endpointsharding.NewBalancer(b, bOpts, balancer.Get(pickfirst.Name).Build, endpointsharding.Options{}) + b.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[%p] ", b)) + b.logger.Infof("Created") + return b +} + +type leastRequestBalancer struct { + // Embeds balancer.ClientConn because we need to intercept UpdateState + // calls from the child balancer. + balancer.ClientConn + child balancer.Balancer + logger *internalgrpclog.PrefixLogger + + mu sync.Mutex + choiceCount uint32 + // endpointRPCCounts holds RPC counts to keep track for subsequent picker + // updates. + endpointRPCCounts *resolver.EndpointMap[*atomic.Int32] +} + +func (lrb *leastRequestBalancer) Close() { + lrb.child.Close() + lrb.endpointRPCCounts = nil +} + +func (lrb *leastRequestBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + lrb.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (lrb *leastRequestBalancer) ResolverError(err error) { + // Will cause inline picker update from endpoint sharding. + lrb.child.ResolverError(err) +} + +func (lrb *leastRequestBalancer) ExitIdle() { + lrb.child.ExitIdle() +} + +func (lrb *leastRequestBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { + lrCfg, ok := ccs.BalancerConfig.(*LBConfig) + if !ok { + logger.Errorf("least-request: received config with unexpected type %T: %v", ccs.BalancerConfig, ccs.BalancerConfig) + return balancer.ErrBadResolverState + } + + lrb.mu.Lock() + lrb.choiceCount = lrCfg.ChoiceCount + lrb.mu.Unlock() + return lrb.child.UpdateClientConnState(balancer.ClientConnState{ + // Enable the health listener in pickfirst children for client side health + // checks and outlier detection, if configured. + ResolverState: pickfirst.EnableHealthListener(ccs.ResolverState), + }) +} + +type endpointState struct { + picker balancer.Picker + numRPCs *atomic.Int32 +} + +func (lrb *leastRequestBalancer) UpdateState(state balancer.State) { + var readyEndpoints []endpointsharding.ChildState + for _, child := range endpointsharding.ChildStatesFromPicker(state.Picker) { + if child.State.ConnectivityState == connectivity.Ready { + readyEndpoints = append(readyEndpoints, child) + } + } + + // If no ready pickers are present, simply defer to the round robin picker + // from endpoint sharding, which will round robin across the most relevant + // pick first children in the highest precedence connectivity state. + if len(readyEndpoints) == 0 { + lrb.ClientConn.UpdateState(state) + return + } + + lrb.mu.Lock() + defer lrb.mu.Unlock() + + if logger.V(2) { + lrb.logger.Infof("UpdateState called with ready endpoints: %v", readyEndpoints) + } + + // Reconcile endpoints. + newEndpoints := resolver.NewEndpointMap[any]() + for _, child := range readyEndpoints { + newEndpoints.Set(child.Endpoint, nil) + } + + // If endpoints are no longer ready, no need to count their active RPCs. + for endpoint := range lrb.endpointRPCCounts.All() { + if _, ok := newEndpoints.Get(endpoint); !ok { + lrb.endpointRPCCounts.Delete(endpoint) + } + } + + // Copy refs to counters into picker. + endpointStates := make([]endpointState, 0, len(readyEndpoints)) + for _, child := range readyEndpoints { + counter, ok := lrb.endpointRPCCounts.Get(child.Endpoint) + if !ok { + // Create new counts if needed. + counter = new(atomic.Int32) + lrb.endpointRPCCounts.Set(child.Endpoint, counter) + } + endpointStates = append(endpointStates, endpointState{ + picker: child.State.Picker, + numRPCs: counter, + }) + } + + lrb.ClientConn.UpdateState(balancer.State{ + Picker: &picker{ + choiceCount: lrb.choiceCount, + endpointStates: endpointStates, + }, + ConnectivityState: connectivity.Ready, + }) +} + +type picker struct { + // choiceCount is the number of random endpoints to sample for choosing the + // one with the least requests. + choiceCount uint32 + endpointStates []endpointState +} + +func (p *picker) Pick(pInfo balancer.PickInfo) (balancer.PickResult, error) { + var pickedEndpointState *endpointState + var pickedEndpointNumRPCs int32 + for i := 0; i < int(p.choiceCount); i++ { + index := randuint32() % uint32(len(p.endpointStates)) + endpointState := p.endpointStates[index] + n := endpointState.numRPCs.Load() + if pickedEndpointState == nil || n < pickedEndpointNumRPCs { + pickedEndpointState = &endpointState + pickedEndpointNumRPCs = n + } + } + result, err := pickedEndpointState.picker.Pick(pInfo) + if err != nil { + return result, err + } + // "The counter for a subchannel should be atomically incremented by one + // after it has been successfully picked by the picker." - A48 + pickedEndpointState.numRPCs.Add(1) + // "the picker should add a callback for atomically decrementing the + // subchannel counter once the RPC finishes (regardless of Status code)." - + // A48. + originalDone := result.Done + result.Done = func(info balancer.DoneInfo) { + pickedEndpointState.numRPCs.Add(-1) + if originalDone != nil { + originalDone(info) + } + } + return result, nil +} diff --git a/balancer/leastrequest/leastrequest_test.go b/balancer/leastrequest/leastrequest_test.go new file mode 100644 index 0000000..d807347 --- /dev/null +++ b/balancer/leastrequest/leastrequest_test.go @@ -0,0 +1,770 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package leastrequest + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestParseConfig(t *testing.T) { + parser := bb{} + tests := []struct { + name string + input string + wantCfg serviceconfig.LoadBalancingConfig + wantErr string + }{ + { + name: "happy-case-default", + input: `{}`, + wantCfg: &LBConfig{ + ChoiceCount: 2, + }, + }, + { + name: "happy-case-choice-count-set", + input: `{"choiceCount": 3}`, + wantCfg: &LBConfig{ + ChoiceCount: 3, + }, + }, + { + name: "happy-case-choice-count-greater-than-ten", + input: `{"choiceCount": 11}`, + wantCfg: &LBConfig{ + ChoiceCount: 10, + }, + }, + { + name: "choice-count-less-than-2", + input: `{"choiceCount": 1}`, + wantErr: "must be >= 2", + }, + { + name: "invalid-json", + input: "{{invalidjson{{", + wantErr: "invalid character", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotCfg, gotErr := parser.ParseConfig(json.RawMessage(test.input)) + // Substring match makes this very tightly coupled to the + // internalserviceconfig.BalancerConfig error strings. However, it + // is important to distinguish the different types of error messages + // possible as the parser has a few defined buckets of ways it can + // error out. + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if test.wantErr != "" { + return + } + if diff := cmp.Diff(gotCfg, test.wantCfg); diff != "" { + t.Fatalf("ParseConfig(%v) got unexpected output, diff (-got +want): %v", test.input, diff) + } + }) + } +} + +func startBackends(t *testing.T, numBackends int) []*stubserver.StubServer { + backends := make([]*stubserver.StubServer, 0, numBackends) + // Construct and start working backends. + for i := 0; i < numBackends; i++ { + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + <-stream.Context().Done() + return nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started good TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + backends = append(backends, backend) + } + return backends +} + +// setupBackends spins up three test backends, each listening on a port on +// localhost. The three backends always reply with an empty response with no +// error, and for streaming receive until hitting an EOF error. +func setupBackends(t *testing.T, numBackends int) []string { + t.Helper() + addresses := make([]string, numBackends) + backends := startBackends(t, numBackends) + // Construct and start working backends. + for i := 0; i < numBackends; i++ { + addresses[i] = backends[i].Address + } + return addresses +} + +// checkRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn, +// connected to a server exposing the test.grpc_testing.TestService, are +// roundrobined across the given backend addresses. +// +// Returns a non-nil error if context deadline expires before RPCs start to get +// roundrobined across the given backends. +func checkRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error { + wantAddrCount := make(map[string]int) + for _, addr := range addrs { + wantAddrCount[addr.Addr]++ + } + gotAddrCount := make(map[string]int) + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + gotAddrCount = make(map[string]int) + // Perform 3 iterations. + var iterations [][]string + for i := 0; i < 3; i++ { + iteration := make([]string, len(addrs)) + for c := 0; c < len(addrs); c++ { + var peer peer.Peer + client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)) + iteration[c] = peer.Addr.String() + } + iterations = append(iterations, iteration) + } + // Ensure the first iteration contains all addresses in addrs. + for _, addr := range iterations[0] { + gotAddrCount[addr]++ + } + if !cmp.Equal(gotAddrCount, wantAddrCount) { + continue + } + // Ensure all three iterations contain the same addresses. + if !cmp.Equal(iterations[0], iterations[1]) || !cmp.Equal(iterations[0], iterations[2]) { + continue + } + return nil + } + return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v; got: %v", addrs, gotAddrCount) +} + +// TestLeastRequestE2E tests the Least Request LB policy in an e2e style. The +// Least Request balancer is configured as the top level balancer of the +// channel, and is passed three addresses. Eventually, the test creates three +// streams, which should be on certain backends according to the least request +// algorithm. The randomness in the picker is injected in the test to be +// deterministic, allowing the test to make assertions on the distribution. +func (s) TestLeastRequestE2E(t *testing.T) { + defer func(u func() uint32) { + randuint32 = u + }(randuint32) + var index int + indexes := []uint32{ + 0, 0, 1, 1, 2, 2, // Triggers a round robin distribution. + } + randuint32 = func() uint32 { + ret := indexes[index%len(indexes)] + index++ + return ret + } + addresses := setupBackends(t, 3) + + mr := manual.NewBuilderWithScheme("lr-e2e") + defer mr.Close() + + // Configure least request as top level balancer of channel. + lrscJSON := ` +{ + "loadBalancingConfig": [ + { + "least_request_experimental": { + "choiceCount": 2 + } + } + ] +}` + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(lrscJSON) + firstThreeAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + {Addr: addresses[2]}, + } + mr.InitialState(resolver.State{ + Addresses: firstThreeAddresses, + ServiceConfig: sc, + }) + + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + + // Wait for all 3 backends to round robin across. The happens because a + // SubConn transitioning into READY causes a new picker update. Once the + // picker update with all 3 backends is present, this test can start to make + // assertions based on those backends. + if err := checkRoundRobinRPCs(ctx, testServiceClient, firstThreeAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // Map ordering of READY SubConns is non deterministic. Thus, perform 3 RPCs + // mocked from the random to each index to learn the addresses of SubConns + // at each index. + index = 0 + peerAtIndex := make([]string, 3) + var peer0 peer.Peer + if _, err := testServiceClient.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer0)); err != nil { + t.Fatalf("testServiceClient.EmptyCall failed: %v", err) + } + peerAtIndex[0] = peer0.Addr.String() + if _, err := testServiceClient.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer0)); err != nil { + t.Fatalf("testServiceClient.EmptyCall failed: %v", err) + } + peerAtIndex[1] = peer0.Addr.String() + if _, err := testServiceClient.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer0)); err != nil { + t.Fatalf("testServiceClient.EmptyCall failed: %v", err) + } + peerAtIndex[2] = peer0.Addr.String() + + // Start streaming RPCs, but do not finish them. Each subsequent stream + // should be started according to the least request algorithm, and chosen + // between the indexes provided. + index = 0 + indexes = []uint32{ + 0, 0, // Causes first stream to be on first address. + 0, 1, // Compares first address (one RPC) to second (no RPCs), so choose second. + 1, 2, // Compares second address (one RPC) to third (no RPCs), so choose third. + 0, 3, // Causes another stream on first address. + 1, 0, // Compares second address (one RPC) to first (two RPCs), so choose second. + 2, 0, // Compares third address (one RPC) to first (two RPCs), so choose third. + 0, 0, // Causes another stream on first address. + 2, 2, // Causes a stream on third address. + 2, 1, // Compares third address (three RPCs) to second (two RPCs), so choose third. + } + wantIndex := []uint32{0, 1, 2, 0, 1, 2, 0, 2, 1} + + // Start streaming RPC's, but do not finish them. Each created stream should + // be started based on the least request algorithm and injected randomness + // (see indexes slice above for exact expectations). + for _, wantIndex := range wantIndex { + stream, err := testServiceClient.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("testServiceClient.FullDuplexCall failed: %v", err) + } + p, ok := peer.FromContext(stream.Context()) + if !ok { + t.Fatalf("testServiceClient.FullDuplexCall has no Peer") + } + if p.Addr.String() != peerAtIndex[wantIndex] { + t.Fatalf("testServiceClient.FullDuplexCall's Peer got: %v, want: %v", p.Addr.String(), peerAtIndex[wantIndex]) + } + } +} + +// TestLeastRequestPersistsCounts tests that the Least Request Balancer persists +// counts once it gets a new picker update. It first updates the Least Request +// Balancer with two backends, and creates a bunch of streams on them. Then, it +// updates the Least Request Balancer with three backends, including the two +// previous. Any created streams should then be started on the new backend. +func (s) TestLeastRequestPersistsCounts(t *testing.T) { + defer func(u func() uint32) { + randuint32 = u + }(randuint32) + var index int + indexes := []uint32{ + 0, 0, 1, 1, + } + randuint32 = func() uint32 { + ret := indexes[index%len(indexes)] + index++ + return ret + } + addresses := setupBackends(t, 3) + + mr := manual.NewBuilderWithScheme("lr-e2e") + defer mr.Close() + + // Configure least request as top level balancer of channel. + lrscJSON := ` +{ + "loadBalancingConfig": [ + { + "least_request_experimental": { + "choiceCount": 2 + } + } + ] +}` + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(lrscJSON) + firstTwoAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + } + mr.InitialState(resolver.State{ + Addresses: firstTwoAddresses, + ServiceConfig: sc, + }) + + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + + // Wait for the two backends to round robin across. The happens because a + // SubConn transitioning into READY causes a new picker update. Once the + // picker update with the two backends is present, this test can start to + // populate those backends with streams. + if err := checkRoundRobinRPCs(ctx, testServiceClient, firstTwoAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // Start 50 streaming RPCs, and leave them unfinished for the duration of + // the test. This will populate the first two addresses with many active + // RPCs. + for i := 0; i < 50; i++ { + _, err := testServiceClient.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("testServiceClient.FullDuplexCall failed: %v", err) + } + } + + // Update the least request balancer to choice count 3. Also update the + // address list adding a third address. Alongside the injected randomness, + // this should trigger the least request balancer to search all created + // SubConns. Thus, since address 3 is the new address and the first two + // addresses are populated with RPCs, once the picker update of all 3 READY + // SubConns takes effect, all new streams should be started on address 3. + index = 0 + indexes = []uint32{ + 0, 1, 2, 3, 4, 5, + } + lrscJSON = ` +{ + "loadBalancingConfig": [ + { + "least_request_experimental": { + "choiceCount": 3 + } + } + ] +}` + sc = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(lrscJSON) + fullAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + {Addr: addresses[2]}, + } + mr.UpdateState(resolver.State{ + Addresses: fullAddresses, + ServiceConfig: sc, + }) + newAddress := fullAddresses[2] + // Poll for only address 3 to show up. This requires a polling loop because + // picker update with all three SubConns doesn't take into effect + // immediately, needs the third SubConn to become READY. + if err := checkRoundRobinRPCs(ctx, testServiceClient, []resolver.Address{newAddress}); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // Start 25 rpcs, but don't finish them. They should all start on address 3, + // since the first two addresses both have 25 RPCs (and randomness + // injection/choiceCount causes all 3 to be compared every iteration). + for i := 0; i < 25; i++ { + stream, err := testServiceClient.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("testServiceClient.FullDuplexCall failed: %v", err) + } + p, ok := peer.FromContext(stream.Context()) + if !ok { + t.Fatalf("testServiceClient.FullDuplexCall has no Peer") + } + if p.Addr.String() != addresses[2] { + t.Fatalf("testServiceClient.FullDuplexCall's Peer got: %v, want: %v", p.Addr.String(), addresses[2]) + } + } + + // Now 25 RPC's are active on each address, the next three RPC's should + // round robin, since choiceCount is three and the injected random indexes + // cause it to search all three addresses for fewest outstanding requests on + // each iteration. + wantAddrCount := map[string]int{ + addresses[0]: 1, + addresses[1]: 1, + addresses[2]: 1, + } + gotAddrCount := make(map[string]int) + for i := 0; i < len(addresses); i++ { + stream, err := testServiceClient.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("testServiceClient.FullDuplexCall failed: %v", err) + } + p, ok := peer.FromContext(stream.Context()) + if !ok { + t.Fatalf("testServiceClient.FullDuplexCall has no Peer") + } + if p.Addr != nil { + gotAddrCount[p.Addr.String()]++ + } + } + if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" { + t.Fatalf("addr count (-got:, +want): %v", diff) + } +} + +// TestConcurrentRPCs tests concurrent RPCs on the least request balancer. It +// configures a channel with a least request balancer as the top level balancer, +// and makes 100 RPCs asynchronously. This makes sure no race conditions happen +// in this scenario. +func (s) TestConcurrentRPCs(t *testing.T) { + addresses := setupBackends(t, 3) + + mr := manual.NewBuilderWithScheme("lr-e2e") + defer mr.Close() + + // Configure least request as top level balancer of channel. + lrscJSON := ` +{ + "loadBalancingConfig": [ + { + "least_request_experimental": { + "choiceCount": 2 + } + } + ] +}` + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(lrscJSON) + firstTwoAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + } + mr.InitialState(resolver.State{ + Addresses: firstTwoAddresses, + ServiceConfig: sc, + }) + + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 5; j++ { + testServiceClient.EmptyCall(ctx, &testpb.Empty{}) + } + }() + } + wg.Wait() +} + +// Test tests that the least request balancer persists RPC counts once it gets +// new picker updates and backends within an endpoint go down. It first updates +// the balancer with two endpoints having two addresses each. It verifies the +// requests are round robined across the first address of each endpoint. It then +// stops the active backend in endpoint[0]. It verified that the balancer starts +// using the second address in endpoint[0]. The test then creates a bunch of +// streams on two endpoints. Then, it updates the balancer with three endpoints, +// including the two previous. Any created streams should then be started on the +// new endpoint. The test shuts down the active backed in endpoint[1] and +// endpoint[2]. The test verifies that new RPCs are round robined across the +// active backends in endpoint[1] and endpoint[2]. +func (s) TestLeastRequestEndpoints_MultipleAddresses(t *testing.T) { + defer func(u func() uint32) { + randuint32 = u + }(randuint32) + var index int + indexes := []uint32{ + 0, 0, 1, 1, + } + randuint32 = func() uint32 { + ret := indexes[index%len(indexes)] + index++ + return ret + } + backends := startBackends(t, 6) + mr := manual.NewBuilderWithScheme("lr-e2e") + defer mr.Close() + + // Configure least request as top level balancer of channel. + lrscJSON := ` +{ + "loadBalancingConfig": [ + { + "least_request_experimental": { + "choiceCount": 2 + } + } + ] +}` + endpoints := []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: backends[0].Address}, {Addr: backends[1].Address}}}, + {Addresses: []resolver.Address{{Addr: backends[2].Address}, {Addr: backends[3].Address}}}, + {Addresses: []resolver.Address{{Addr: backends[4].Address}, {Addr: backends[5].Address}}}, + } + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(lrscJSON) + firstTwoEndpoints := []resolver.Endpoint{endpoints[0], endpoints[1]} + mr.InitialState(resolver.State{ + Endpoints: firstTwoEndpoints, + ServiceConfig: sc, + }) + + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + + // Wait for the two backends to round robin across. The happens because a + // child pickfirst transitioning into READY causes a new picker update. Once + // the picker update with the two backends is present, this test can start + // to populate those backends with streams. + wantAddrs := []resolver.Address{ + endpoints[0].Addresses[0], + endpoints[1].Addresses[0], + } + if err := checkRoundRobinRPCs(ctx, testServiceClient, wantAddrs); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // Shut down one of the addresses in endpoints[0], the child pickfirst + // should fallback to the next address in endpoints[0]. + backends[0].Stop() + wantAddrs = []resolver.Address{ + endpoints[0].Addresses[1], + endpoints[1].Addresses[0], + } + if err := checkRoundRobinRPCs(ctx, testServiceClient, wantAddrs); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // Start 50 streaming RPCs, and leave them unfinished for the duration of + // the test. This will populate the first two endpoints with many active + // RPCs. + for i := 0; i < 50; i++ { + _, err := testServiceClient.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("testServiceClient.FullDuplexCall failed: %v", err) + } + } + + // Update the least request balancer to choice count 3. Also update the + // address list adding a third endpoint. Alongside the injected randomness, + // this should trigger the least request balancer to search all created + // endpoints. Thus, since endpoint 3 is the new endpoint and the first two + // endpoint are populated with RPCs, once the picker update of all 3 READY + // pickfirsts takes effect, all new streams should be started on endpoint 3. + index = 0 + indexes = []uint32{ + 0, 1, 2, 3, 4, 5, + } + lrscJSON = ` +{ + "loadBalancingConfig": [ + { + "least_request_experimental": { + "choiceCount": 3 + } + } + ] +}` + sc = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(lrscJSON) + mr.UpdateState(resolver.State{ + Endpoints: endpoints, + ServiceConfig: sc, + }) + newAddress := endpoints[2].Addresses[0] + // Poll for only endpoint 3 to show up. This requires a polling loop because + // picker update with all three endpoints doesn't take into effect + // immediately, needs the third pickfirst to become READY. + if err := checkRoundRobinRPCs(ctx, testServiceClient, []resolver.Address{newAddress}); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // Start 25 rpcs, but don't finish them. They should all start on endpoint 3, + // since the first two endpoints both have 25 RPCs (and randomness + // injection/choiceCount causes all 3 to be compared every iteration). + for i := 0; i < 25; i++ { + stream, err := testServiceClient.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("testServiceClient.FullDuplexCall failed: %v", err) + } + p, ok := peer.FromContext(stream.Context()) + if !ok { + t.Fatalf("testServiceClient.FullDuplexCall has no Peer") + } + if p.Addr.String() != newAddress.Addr { + t.Fatalf("testServiceClient.FullDuplexCall's Peer got: %v, want: %v", p.Addr.String(), newAddress) + } + } + + // Now 25 RPC's are active on each endpoint, the next three RPC's should + // round robin, since choiceCount is three and the injected random indexes + // cause it to search all three endpoints for fewest outstanding requests on + // each iteration. + wantAddrCount := map[string]int{ + endpoints[0].Addresses[1].Addr: 1, + endpoints[1].Addresses[0].Addr: 1, + endpoints[2].Addresses[0].Addr: 1, + } + gotAddrCount := make(map[string]int) + for i := 0; i < len(endpoints); i++ { + stream, err := testServiceClient.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("testServiceClient.FullDuplexCall failed: %v", err) + } + p, ok := peer.FromContext(stream.Context()) + if !ok { + t.Fatalf("testServiceClient.FullDuplexCall has no Peer") + } + if p.Addr != nil { + gotAddrCount[p.Addr.String()]++ + } + } + if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" { + t.Fatalf("addr count (-got:, +want): %v", diff) + } + + // Shutdown the active address for endpoint[1] and endpoint[2]. This should + // result in their streams failing. Now the requests should roundrobin b/w + // endpoint[1] and endpoint[2]. + backends[2].Stop() + backends[4].Stop() + index = 0 + indexes = []uint32{ + 0, 1, 2, 2, 1, 0, + } + wantAddrs = []resolver.Address{ + endpoints[1].Addresses[1], + endpoints[2].Addresses[1], + } + if err := checkRoundRobinRPCs(ctx, testServiceClient, wantAddrs); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } +} + +// Test tests that the least request balancer properly surfaces resolver +// errors. +func (s) TestLeastRequestEndpoints_ResolverError(t *testing.T) { + const sc = `{"loadBalancingConfig": [{"least_request_experimental": {}}]}` + mr := manual.NewBuilderWithScheme("lr-e2e") + defer mr.Close() + + cc, err := grpc.NewClient( + mr.Scheme()+":///", + grpc.WithResolvers(mr), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(sc), + ) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + // We need to pass an endpoint with a valid address to the resolver before + // reporting an error - otherwise endpointsharding does not report the + // error through. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + // Act like a server that closes the connection without sending a server + // preface. + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("Unexpected error when accepting a connection: %v", err) + } + conn.Close() + }() + mr.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}}, + }) + cc.Connect() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Report an error through the resolver + resolverErr := fmt.Errorf("simulated resolver error") + mr.CC().ReportError(resolverErr) + + // Ensure the client returns the expected resolver error. + testServiceClient := testgrpc.NewTestServiceClient(cc) + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + _, err = testServiceClient.EmptyCall(ctx, &testpb.Empty{}) + if strings.Contains(err.Error(), resolverErr.Error()) { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for RPCs to fail with error containing %s. Last error: %v", resolverErr, err) + } +} diff --git a/balancer/pickfirst/internal/internal.go b/balancer/pickfirst/internal/internal.go new file mode 100644 index 0000000..cc902a4 --- /dev/null +++ b/balancer/pickfirst/internal/internal.go @@ -0,0 +1,37 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains code internal to the pickfirst package. +package internal + +import ( + rand "math/rand/v2" + "time" +) + +var ( + // RandShuffle pseudo-randomizes the order of addresses. + RandShuffle = rand.Shuffle + // RandFloat64 returns, as a float64, a pseudo-random number in [0.0,1.0). + RandFloat64 = rand.Float64 + // TimeAfterFunc allows mocking the timer for testing connection delay + // related functionality. + TimeAfterFunc = func(d time.Duration, f func()) func() { + timer := time.AfterFunc(d, f) + return func() { timer.Stop() } + } +) diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go new file mode 100644 index 0000000..ddd08f5 --- /dev/null +++ b/balancer/pickfirst/pickfirst.go @@ -0,0 +1,921 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package pickfirst contains the pick_first load balancing policy which +// is the universal leaf policy. +package pickfirst + +import ( + "cmp" + "encoding/json" + "errors" + "fmt" + "math" + "net" + "net/netip" + "slices" + "sync" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst/internal" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/experimental/balancer/weight" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/envconfig" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +func init() { + balancer.Register(pickfirstBuilder{}) +} + +// Name is the name of the pick_first balancer. +const Name = "pick_first" + +// enableHealthListenerKeyType is a unique key type used in resolver +// attributes to indicate whether the health listener usage is enabled. +type enableHealthListenerKeyType struct{} + +var logger = grpclog.Component("pick-first-leaf-lb") + +const ( + // TODO: change to pick-first when this becomes the default pick_first policy. + logPrefix = "[pick-first-leaf-lb %p] " + // connectionDelayInterval is the time to wait for during the happy eyeballs + // pass before starting the next connection attempt. + connectionDelayInterval = 250 * time.Millisecond +) + +type ipAddrFamily int + +const ( + // ipAddrFamilyUnknown represents strings that can't be parsed as an IP + // address. + ipAddrFamilyUnknown ipAddrFamily = iota + ipAddrFamilyV4 + ipAddrFamilyV6 +) + +type pickfirstBuilder struct{} + +func (pickfirstBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { + b := &pickfirstBalancer{ + cc: cc, + + subConns: resolver.NewAddressMapV2[*scData](), + state: connectivity.Connecting, + cancelConnectionTimer: func() {}, + } + b.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(logPrefix, b)) + return b +} + +func (b pickfirstBuilder) Name() string { + return Name +} + +func (pickfirstBuilder) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + var cfg pfConfig + if err := json.Unmarshal(js, &cfg); err != nil { + return nil, fmt.Errorf("pickfirst: unable to unmarshal LB policy config: %s, error: %v", string(js), err) + } + return cfg, nil +} + +// EnableHealthListener updates the state to configure pickfirst for using a +// generic health listener. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a later +// release. +func EnableHealthListener(state resolver.State) resolver.State { + state.Attributes = state.Attributes.WithValue(enableHealthListenerKeyType{}, true) + return state +} + +type pfConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + // If set to true, instructs the LB policy to shuffle the order of the list + // of endpoints received from the name resolver before attempting to + // connect to them. + ShuffleAddressList bool `json:"shuffleAddressList"` +} + +// scData keeps track of the current state of the subConn. +// It is not safe for concurrent access. +type scData struct { + // The following fields are initialized at build time and read-only after + // that. + subConn balancer.SubConn + addr resolver.Address + + rawConnectivityState connectivity.State + // The effective connectivity state based on raw connectivity, health state + // and after following sticky TransientFailure behaviour defined in A62. + effectiveState connectivity.State + lastErr error + connectionFailedInFirstPass bool +} + +func (b *pickfirstBalancer) newSCData(addr resolver.Address) (*scData, error) { + sd := &scData{ + rawConnectivityState: connectivity.Idle, + effectiveState: connectivity.Idle, + addr: addr, + } + sc, err := b.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + b.updateSubConnState(sd, state) + }, + }) + if err != nil { + return nil, err + } + sd.subConn = sc + return sd, nil +} + +type pickfirstBalancer struct { + // The following fields are initialized at build time and read-only after + // that and therefore do not need to be guarded by a mutex. + logger *internalgrpclog.PrefixLogger + cc balancer.ClientConn + + // The mutex is used to ensure synchronization of updates triggered + // from the idle picker and the already serialized resolver, + // SubConn state updates. + mu sync.Mutex + // State reported to the channel based on SubConn states and resolver + // updates. + state connectivity.State + // scData for active subonns mapped by address. + subConns *resolver.AddressMapV2[*scData] + addressList addressList + firstPass bool + numTF int + cancelConnectionTimer func() + healthCheckingEnabled bool +} + +// ResolverError is called by the ClientConn when the name resolver produces +// an error or when pickfirst determined the resolver update to be invalid. +func (b *pickfirstBalancer) ResolverError(err error) { + b.mu.Lock() + defer b.mu.Unlock() + b.resolverErrorLocked(err) +} + +func (b *pickfirstBalancer) resolverErrorLocked(err error) { + if b.logger.V(2) { + b.logger.Infof("Received error from the name resolver: %v", err) + } + + // The picker will not change since the balancer does not currently + // report an error. If the balancer hasn't received a single good resolver + // update yet, transition to TRANSIENT_FAILURE. + if b.state != connectivity.TransientFailure && b.addressList.size() > 0 { + if b.logger.V(2) { + b.logger.Infof("Ignoring resolver error because balancer is using a previous good update.") + } + return + } + + b.updateBalancerState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)}, + }) +} + +func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState) error { + b.mu.Lock() + defer b.mu.Unlock() + b.cancelConnectionTimer() + if len(state.ResolverState.Addresses) == 0 && len(state.ResolverState.Endpoints) == 0 { + // Cleanup state pertaining to the previous resolver state. + // Treat an empty address list like an error by calling b.ResolverError. + b.closeSubConnsLocked() + b.addressList.updateAddrs(nil) + b.resolverErrorLocked(errors.New("produced zero addresses")) + return balancer.ErrBadResolverState + } + b.healthCheckingEnabled = state.ResolverState.Attributes.Value(enableHealthListenerKeyType{}) != nil + cfg, ok := state.BalancerConfig.(pfConfig) + if state.BalancerConfig != nil && !ok { + return fmt.Errorf("pickfirst: received illegal BalancerConfig (type %T): %v: %w", state.BalancerConfig, state.BalancerConfig, balancer.ErrBadResolverState) + } + + if b.logger.V(2) { + b.logger.Infof("Received new config %s, resolver state %s", pretty.ToJSON(cfg), pretty.ToJSON(state.ResolverState)) + } + + var newAddrs []resolver.Address + if endpoints := state.ResolverState.Endpoints; len(endpoints) != 0 { + // Perform the optional shuffling described in gRFC A62. The shuffling + // will change the order of endpoints but not touch the order of the + // addresses within each endpoint. - A61 + if cfg.ShuffleAddressList { + if envconfig.PickFirstWeightedShuffling { + type weightedEndpoint struct { + endpoint resolver.Endpoint + weight float64 + } + + // For each endpoint, compute a key as described in A113 and + // https://utopia.duth.gr/~pefraimi/research/data/2007EncOfAlg.pdf: + var weightedEndpoints []weightedEndpoint + for _, endpoint := range endpoints { + u := internal.RandFloat64() // Random number in [0.0, 1.0) + weight := weightAttribute(endpoint) + weightedEndpoints = append(weightedEndpoints, weightedEndpoint{ + endpoint: endpoint, + weight: math.Pow(u, 1.0/float64(weight)), + }) + } + // Sort endpoints by key in descending order and reconstruct the + // endpoints slice. + slices.SortFunc(weightedEndpoints, func(a, b weightedEndpoint) int { + return cmp.Compare(b.weight, a.weight) + }) + + // Here, and in the "else" block below, we clone the endpoints + // slice to avoid mutating the resolver state. Doing the latter + // would lead to data races if the caller is accessing the same + // slice concurrently. + sortedEndpoints := make([]resolver.Endpoint, len(endpoints)) + for i, we := range weightedEndpoints { + sortedEndpoints[i] = we.endpoint + } + endpoints = sortedEndpoints + } else { + endpoints = slices.Clone(endpoints) + internal.RandShuffle(len(endpoints), func(i, j int) { endpoints[i], endpoints[j] = endpoints[j], endpoints[i] }) + } + } + + // "Flatten the list by concatenating the ordered list of addresses for + // each of the endpoints, in order." - A61 + for _, endpoint := range endpoints { + newAddrs = append(newAddrs, endpoint.Addresses...) + } + } else { + // Endpoints not set, process addresses until we migrate resolver + // emissions fully to Endpoints. The top channel does wrap emitted + // addresses with endpoints, however some balancers such as weighted + // target do not forward the corresponding correct endpoints down/split + // endpoints properly. Once all balancers correctly forward endpoints + // down, can delete this else conditional. + newAddrs = state.ResolverState.Addresses + if cfg.ShuffleAddressList { + newAddrs = append([]resolver.Address{}, newAddrs...) + internal.RandShuffle(len(newAddrs), func(i, j int) { newAddrs[i], newAddrs[j] = newAddrs[j], newAddrs[i] }) + } + } + + // If an address appears in multiple endpoints or in the same endpoint + // multiple times, we keep it only once. We will create only one SubConn + // for the address because an AddressMap is used to store SubConns. + // Not de-duplicating would result in attempting to connect to the same + // SubConn multiple times in the same pass. We don't want this. + newAddrs = deDupAddresses(newAddrs) + newAddrs = interleaveAddresses(newAddrs) + + prevAddr := b.addressList.currentAddress() + prevSCData, found := b.subConns.Get(prevAddr) + prevAddrsCount := b.addressList.size() + isPrevRawConnectivityStateReady := found && prevSCData.rawConnectivityState == connectivity.Ready + b.addressList.updateAddrs(newAddrs) + + // If the previous ready SubConn exists in new address list, + // keep this connection and don't create new SubConns. + if isPrevRawConnectivityStateReady && b.addressList.seekTo(prevAddr) { + return nil + } + + b.reconcileSubConnsLocked(newAddrs) + // If it's the first resolver update or the balancer was already READY + // (but the new address list does not contain the ready SubConn) or + // CONNECTING, enter CONNECTING. + // We may be in TRANSIENT_FAILURE due to a previous empty address list, + // we should still enter CONNECTING because the sticky TF behaviour + // mentioned in A62 applies only when the TRANSIENT_FAILURE is reported + // due to connectivity failures. + if isPrevRawConnectivityStateReady || b.state == connectivity.Connecting || prevAddrsCount == 0 { + // Start connection attempt at first address. + b.forceUpdateConcludedStateLocked(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + b.startFirstPassLocked() + } else if b.state == connectivity.TransientFailure { + // If we're in TRANSIENT_FAILURE, we stay in TRANSIENT_FAILURE until + // we're READY. See A62. + b.startFirstPassLocked() + } + return nil +} + +// UpdateSubConnState is unused as a StateListener is always registered when +// creating SubConns. +func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", subConn, state) +} + +func (b *pickfirstBalancer) Close() { + b.mu.Lock() + defer b.mu.Unlock() + b.closeSubConnsLocked() + b.cancelConnectionTimer() + b.state = connectivity.Shutdown +} + +// ExitIdle moves the balancer out of idle state. It can be called concurrently +// by the idlePicker and clientConn so access to variables should be +// synchronized. +func (b *pickfirstBalancer) ExitIdle() { + b.mu.Lock() + defer b.mu.Unlock() + if b.state == connectivity.Idle { + // Move the balancer into CONNECTING state immediately. This is done to + // avoid staying in IDLE if a resolver update arrives before the first + // SubConn reports CONNECTING. + b.updateBalancerState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + b.startFirstPassLocked() + } +} + +func (b *pickfirstBalancer) startFirstPassLocked() { + b.firstPass = true + b.numTF = 0 + // Reset the connection attempt record for existing SubConns. + for _, sd := range b.subConns.All() { + sd.connectionFailedInFirstPass = false + } + b.requestConnectionLocked() +} + +func (b *pickfirstBalancer) closeSubConnsLocked() { + for _, sd := range b.subConns.All() { + sd.subConn.Shutdown() + } + b.subConns = resolver.NewAddressMapV2[*scData]() +} + +// deDupAddresses ensures that each address appears only once in the slice. +func deDupAddresses(addrs []resolver.Address) []resolver.Address { + seenAddrs := resolver.NewAddressMapV2[bool]() + retAddrs := []resolver.Address{} + + for _, addr := range addrs { + if _, ok := seenAddrs.Get(addr); ok { + continue + } + seenAddrs.Set(addr, true) + retAddrs = append(retAddrs, addr) + } + return retAddrs +} + +// interleaveAddresses interleaves addresses of both families (IPv4 and IPv6) +// as per RFC-8305 section 4. +// Whichever address family is first in the list is followed by an address of +// the other address family; that is, if the first address in the list is IPv6, +// then the first IPv4 address should be moved up in the list to be second in +// the list. It doesn't support configuring "First Address Family Count", i.e. +// there will always be a single member of the first address family at the +// beginning of the interleaved list. +// Addresses that are neither IPv4 nor IPv6 are treated as part of a third +// "unknown" family for interleaving. +// See: https://datatracker.ietf.org/doc/html/rfc8305#autoid-6 +func interleaveAddresses(addrs []resolver.Address) []resolver.Address { + familyAddrsMap := map[ipAddrFamily][]resolver.Address{} + interleavingOrder := []ipAddrFamily{} + for _, addr := range addrs { + family := addressFamily(addr.Addr) + if _, found := familyAddrsMap[family]; !found { + interleavingOrder = append(interleavingOrder, family) + } + familyAddrsMap[family] = append(familyAddrsMap[family], addr) + } + + interleavedAddrs := make([]resolver.Address, 0, len(addrs)) + + for curFamilyIdx := 0; len(interleavedAddrs) < len(addrs); curFamilyIdx = (curFamilyIdx + 1) % len(interleavingOrder) { + // Some IP types may have fewer addresses than others, so we look for + // the next type that has a remaining member to add to the interleaved + // list. + family := interleavingOrder[curFamilyIdx] + remainingMembers := familyAddrsMap[family] + if len(remainingMembers) > 0 { + interleavedAddrs = append(interleavedAddrs, remainingMembers[0]) + familyAddrsMap[family] = remainingMembers[1:] + } + } + + return interleavedAddrs +} + +// addressFamily returns the ipAddrFamily after parsing the address string. +// If the address isn't of the format "ip-address:port", it returns +// ipAddrFamilyUnknown. The address may be valid even if it's not an IP when +// using a resolver like passthrough where the address may be a hostname in +// some format that the dialer can resolve. +func addressFamily(address string) ipAddrFamily { + // Parse the IP after removing the port. + host, _, err := net.SplitHostPort(address) + if err != nil { + return ipAddrFamilyUnknown + } + ip, err := netip.ParseAddr(host) + if err != nil { + return ipAddrFamilyUnknown + } + switch { + case ip.Is4() || ip.Is4In6(): + return ipAddrFamilyV4 + case ip.Is6(): + return ipAddrFamilyV6 + default: + return ipAddrFamilyUnknown + } +} + +// reconcileSubConnsLocked updates the active subchannels based on a new address +// list from the resolver. It does this by: +// - closing subchannels: any existing subchannels associated with addresses +// that are no longer in the updated list are shut down. +// - removing subchannels: entries for these closed subchannels are removed +// from the subchannel map. +// +// This ensures that the subchannel map accurately reflects the current set of +// addresses received from the name resolver. +func (b *pickfirstBalancer) reconcileSubConnsLocked(newAddrs []resolver.Address) { + newAddrsMap := resolver.NewAddressMapV2[bool]() + for _, addr := range newAddrs { + newAddrsMap.Set(addr, true) + } + + for oldAddr := range b.subConns.All() { + if _, ok := newAddrsMap.Get(oldAddr); ok { + continue + } + val, _ := b.subConns.Get(oldAddr) + val.subConn.Shutdown() + b.subConns.Delete(oldAddr) + } +} + +// shutdownRemainingLocked shuts down remaining subConns. Called when a subConn +// becomes ready, which means that all other subConn must be shutdown. +func (b *pickfirstBalancer) shutdownRemainingLocked(selected *scData) { + b.cancelConnectionTimer() + for _, sd := range b.subConns.All() { + if sd.subConn != selected.subConn { + sd.subConn.Shutdown() + } + } + b.subConns = resolver.NewAddressMapV2[*scData]() + b.subConns.Set(selected.addr, selected) +} + +// requestConnectionLocked starts connecting on the subchannel corresponding to +// the current address. If no subchannel exists, one is created. If the current +// subchannel is in TransientFailure, a connection to the next address is +// attempted until a subchannel is found. +func (b *pickfirstBalancer) requestConnectionLocked() { + if !b.addressList.isValid() { + return + } + var lastErr error + for valid := true; valid; valid = b.addressList.increment() { + curAddr := b.addressList.currentAddress() + sd, ok := b.subConns.Get(curAddr) + if !ok { + var err error + // We want to assign the new scData to sd from the outer scope, + // hence we can't use := below. + sd, err = b.newSCData(curAddr) + if err != nil { + // This should never happen, unless the clientConn is being shut + // down. + if b.logger.V(2) { + b.logger.Infof("Failed to create a subConn for address %v: %v", curAddr.String(), err) + } + // Do nothing, the LB policy will be closed soon. + return + } + b.subConns.Set(curAddr, sd) + } + + switch sd.rawConnectivityState { + case connectivity.Idle: + sd.subConn.Connect() + b.scheduleNextConnectionLocked() + return + case connectivity.TransientFailure: + // The SubConn is being re-used and failed during a previous pass + // over the addressList. It has not completed backoff yet. + // Mark it as having failed and try the next address. + sd.connectionFailedInFirstPass = true + lastErr = sd.lastErr + continue + case connectivity.Connecting: + // Wait for the connection attempt to complete or the timer to fire + // before attempting the next address. + b.scheduleNextConnectionLocked() + return + default: + b.logger.Errorf("SubConn with unexpected state %v present in SubConns map.", sd.rawConnectivityState) + return + + } + } + + // All the remaining addresses in the list are in TRANSIENT_FAILURE, end the + // first pass if possible. + b.endFirstPassIfPossibleLocked(lastErr) +} + +func (b *pickfirstBalancer) scheduleNextConnectionLocked() { + b.cancelConnectionTimer() + if !b.addressList.hasNext() { + return + } + curAddr := b.addressList.currentAddress() + cancelled := false // Access to this is protected by the balancer's mutex. + closeFn := internal.TimeAfterFunc(connectionDelayInterval, func() { + b.mu.Lock() + defer b.mu.Unlock() + // If the scheduled task is cancelled while acquiring the mutex, return. + if cancelled { + return + } + if b.logger.V(2) { + b.logger.Infof("Happy Eyeballs timer expired while waiting for connection to %q.", curAddr.Addr) + } + if b.addressList.increment() { + b.requestConnectionLocked() + } + }) + // Access to the cancellation callback held by the balancer is guarded by + // the balancer's mutex, so it's safe to set the boolean from the callback. + b.cancelConnectionTimer = sync.OnceFunc(func() { + cancelled = true + closeFn() + }) +} + +func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.SubConnState) { + b.mu.Lock() + defer b.mu.Unlock() + oldState := sd.rawConnectivityState + sd.rawConnectivityState = newState.ConnectivityState + // Previously relevant SubConns can still callback with state updates. + // To prevent pickers from returning these obsolete SubConns, this logic + // is included to check if the current list of active SubConns includes this + // SubConn. + if !b.isActiveSCData(sd) { + return + } + if newState.ConnectivityState == connectivity.Shutdown { + sd.effectiveState = connectivity.Shutdown + return + } + + if newState.ConnectivityState == connectivity.TransientFailure { + sd.connectionFailedInFirstPass = true + } + + if newState.ConnectivityState == connectivity.Ready { + b.shutdownRemainingLocked(sd) + if !b.addressList.seekTo(sd.addr) { + // This should not fail as we should have only one SubConn after + // entering READY. The SubConn should be present in the addressList. + b.logger.Errorf("Address %q not found address list in %v", sd.addr, b.addressList.addresses) + return + } + if !b.healthCheckingEnabled { + if b.logger.V(2) { + b.logger.Infof("SubConn %p reported connectivity state READY and the health listener is disabled. Transitioning SubConn to READY.", sd.subConn) + } + + sd.effectiveState = connectivity.Ready + b.updateBalancerState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &picker{result: balancer.PickResult{SubConn: sd.subConn}}, + }) + return + } + if b.logger.V(2) { + b.logger.Infof("SubConn %p reported connectivity state READY. Registering health listener.", sd.subConn) + } + // Send a CONNECTING update to take the SubConn out of sticky-TF if + // required. + sd.effectiveState = connectivity.Connecting + b.updateBalancerState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + sd.subConn.RegisterHealthListener(func(scs balancer.SubConnState) { + b.updateSubConnHealthState(sd, scs) + }) + return + } + + // If the LB policy is READY, and it receives a subchannel state change, + // it means that the READY subchannel has failed. + // A SubConn can also transition from CONNECTING directly to IDLE when + // a transport is successfully created, but the connection fails + // before the SubConn can send the notification for READY. We treat + // this as a successful connection and transition to IDLE. + // TODO: https://github.com/grpc/grpc-go/issues/7862 - Remove the second + // part of the if condition below once the issue is fixed. + if oldState == connectivity.Ready || (oldState == connectivity.Connecting && newState.ConnectivityState == connectivity.Idle) { + // Once a transport fails, the balancer enters IDLE and starts from + // the first address when the picker is used. + b.shutdownRemainingLocked(sd) + sd.effectiveState = newState.ConnectivityState + b.addressList.reset() + b.updateBalancerState(balancer.State{ + ConnectivityState: connectivity.Idle, + Picker: &idlePicker{exitIdle: sync.OnceFunc(b.ExitIdle)}, + }) + return + } + + if b.firstPass { + switch newState.ConnectivityState { + case connectivity.Connecting: + // The effective state can be in either IDLE, CONNECTING or + // TRANSIENT_FAILURE. If it's TRANSIENT_FAILURE, stay in + // TRANSIENT_FAILURE until it's READY. See A62. + if sd.effectiveState != connectivity.TransientFailure { + sd.effectiveState = connectivity.Connecting + b.updateBalancerState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + } + case connectivity.TransientFailure: + sd.lastErr = newState.ConnectionError + sd.effectiveState = connectivity.TransientFailure + // Since we're re-using common SubConns while handling resolver + // updates, we could receive an out of turn TRANSIENT_FAILURE from + // a pass over the previous address list. Happy Eyeballs will also + // cause out of order updates to arrive. + + if curAddr := b.addressList.currentAddress(); equalAddressIgnoringBalAttributes(&curAddr, &sd.addr) { + b.cancelConnectionTimer() + if b.addressList.increment() { + b.requestConnectionLocked() + return + } + } + + // End the first pass if we've seen a TRANSIENT_FAILURE from all + // SubConns once. + b.endFirstPassIfPossibleLocked(newState.ConnectionError) + } + return + } + + // We have finished the first pass, keep re-connecting failing SubConns. + switch newState.ConnectivityState { + case connectivity.TransientFailure: + b.numTF = (b.numTF + 1) % b.subConns.Len() + sd.lastErr = newState.ConnectionError + if b.numTF%b.subConns.Len() == 0 { + b.updateBalancerState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: newState.ConnectionError}, + }) + } + // We don't need to request re-resolution since the SubConn already + // does that before reporting TRANSIENT_FAILURE. + // TODO: #7534 - Move re-resolution requests from SubConn into + // pick_first. + case connectivity.Idle: + sd.subConn.Connect() + } +} + +// endFirstPassIfPossibleLocked ends the first happy-eyeballs pass if all the +// addresses are tried and their SubConns have reported a failure. +func (b *pickfirstBalancer) endFirstPassIfPossibleLocked(lastErr error) { + // An optimization to avoid iterating over the entire SubConn map. + if b.addressList.isValid() { + return + } + // Connect() has been called on all the SubConns. The first pass can be + // ended if all the SubConns have reported a failure. + for _, sd := range b.subConns.All() { + if !sd.connectionFailedInFirstPass { + return + } + } + b.firstPass = false + b.updateBalancerState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: lastErr}, + }) + // Start re-connecting all the SubConns that are already in IDLE. + for _, sd := range b.subConns.All() { + if sd.rawConnectivityState == connectivity.Idle { + sd.subConn.Connect() + } + } +} + +func (b *pickfirstBalancer) isActiveSCData(sd *scData) bool { + activeSD, found := b.subConns.Get(sd.addr) + return found && activeSD == sd +} + +func (b *pickfirstBalancer) updateSubConnHealthState(sd *scData, state balancer.SubConnState) { + b.mu.Lock() + defer b.mu.Unlock() + // Previously relevant SubConns can still callback with state updates. + // To prevent pickers from returning these obsolete SubConns, this logic + // is included to check if the current list of active SubConns includes + // this SubConn. + if !b.isActiveSCData(sd) { + return + } + sd.effectiveState = state.ConnectivityState + switch state.ConnectivityState { + case connectivity.Ready: + b.updateBalancerState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &picker{result: balancer.PickResult{SubConn: sd.subConn}}, + }) + case connectivity.TransientFailure: + b.updateBalancerState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("pickfirst: health check failure: %v", state.ConnectionError)}, + }) + case connectivity.Connecting: + b.updateBalancerState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + default: + b.logger.Errorf("Got unexpected health update for SubConn %p: %v", state) + } +} + +// updateBalancerState stores the state reported to the channel and calls +// ClientConn.UpdateState(). As an optimization, it avoids sending duplicate +// updates to the channel. +func (b *pickfirstBalancer) updateBalancerState(newState balancer.State) { + // In case of TransientFailures allow the picker to be updated to update + // the connectivity error, in all other cases don't send duplicate state + // updates. + if newState.ConnectivityState == b.state && b.state != connectivity.TransientFailure { + return + } + b.forceUpdateConcludedStateLocked(newState) +} + +// forceUpdateConcludedStateLocked stores the state reported to the channel and +// calls ClientConn.UpdateState(). +// A separate function is defined to force update the ClientConn state since the +// channel doesn't correctly assume that LB policies start in CONNECTING and +// relies on LB policy to send an initial CONNECTING update. +func (b *pickfirstBalancer) forceUpdateConcludedStateLocked(newState balancer.State) { + b.state = newState.ConnectivityState + b.cc.UpdateState(newState) +} + +type picker struct { + result balancer.PickResult + err error +} + +func (p *picker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + return p.result, p.err +} + +// idlePicker is used when the SubConn is IDLE and kicks the SubConn into +// CONNECTING when Pick is called. +type idlePicker struct { + exitIdle func() +} + +func (i *idlePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + i.exitIdle() + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable +} + +// addressList manages sequentially iterating over addresses present in a list +// of endpoints. It provides a 1 dimensional view of the addresses present in +// the endpoints. +// This type is not safe for concurrent access. +type addressList struct { + addresses []resolver.Address + idx int +} + +func (al *addressList) isValid() bool { + return al.idx < len(al.addresses) +} + +func (al *addressList) size() int { + return len(al.addresses) +} + +// increment moves to the next index in the address list. +// This method returns false if it went off the list, true otherwise. +func (al *addressList) increment() bool { + if !al.isValid() { + return false + } + al.idx++ + return al.idx < len(al.addresses) +} + +// currentAddress returns the current address pointed to in the addressList. +// If the list is in an invalid state, it returns an empty address instead. +func (al *addressList) currentAddress() resolver.Address { + if !al.isValid() { + return resolver.Address{} + } + return al.addresses[al.idx] +} + +func (al *addressList) reset() { + al.idx = 0 +} + +func (al *addressList) updateAddrs(addrs []resolver.Address) { + al.addresses = addrs + al.reset() +} + +// seekTo returns false if the needle was not found and the current index was +// left unchanged. +func (al *addressList) seekTo(needle resolver.Address) bool { + for ai, addr := range al.addresses { + if !equalAddressIgnoringBalAttributes(&addr, &needle) { + continue + } + al.idx = ai + return true + } + return false +} + +// hasNext returns whether incrementing the addressList will result in moving +// past the end of the list. If the list has already moved past the end, it +// returns false. +func (al *addressList) hasNext() bool { + if !al.isValid() { + return false + } + return al.idx+1 < len(al.addresses) +} + +// equalAddressIgnoringBalAttributes returns true is a and b are considered +// equal. This is different from the Equal method on the resolver.Address type +// which considers all fields to determine equality. Here, we only consider +// fields that are meaningful to the SubConn. +func equalAddressIgnoringBalAttributes(a, b *resolver.Address) bool { + return a.Addr == b.Addr && a.ServerName == b.ServerName && + a.Attributes.Equal(b.Attributes) +} + +// weightAttribute is a convenience function which returns the value of the +// weight endpoint Attribute. +// +// When used in the xDS context, the weight attribute is guaranteed to be +// non-zero. But, when used in a non-xDS context, the weight attribute could be +// unset. A Default of 1 is used in the latter case. +func weightAttribute(e resolver.Endpoint) uint32 { + w := weight.FromEndpoint(e).Weight + if w == 0 { + return 1 + } + return w +} diff --git a/balancer/pickfirst/pickfirst_ext_test.go b/balancer/pickfirst/pickfirst_ext_test.go new file mode 100644 index 0000000..107ca10 --- /dev/null +++ b/balancer/pickfirst/pickfirst_ext_test.go @@ -0,0 +1,2957 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package pickfirst_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/balancer" + pfbalancer "google.golang.org/grpc/balancer/pickfirst" + pfinternal "google.golang.org/grpc/balancer/pickfirst/internal" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/experimental/balancer/weight" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/pickfirst" + "google.golang.org/grpc/internal/testutils/stats" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const ( + pickFirstServiceConfig = `{"loadBalancingConfig": [{"pick_first":{}}]}` + // Default timeout for tests in this package. + defaultTestTimeout = 10 * time.Second + // Default short timeout, to be used when waiting for events which are not + // expected to happen. + defaultTestShortTimeout = 100 * time.Millisecond + stateStoringBalancerName = "state_storing" +) + +var ( + stateStoringServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateStoringBalancerName) + ignoreBalAttributesOpt = cmp.Transformer("IgnoreBalancerAttributes", func(a resolver.Address) resolver.Address { + a.BalancerAttributes = nil + return a + }) +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func init() { + channelz.TurnOn() +} + +// parseServiceConfig is a test helper which uses the manual resolver to parse +// the given service config. It calls t.Fatal() if service config parsing fails. +func parseServiceConfig(t *testing.T, r *manual.Resolver, sc string) *serviceconfig.ParseResult { + t.Helper() + + scpr := r.CC().ParseServiceConfig(sc) + if scpr.Err != nil { + t.Fatalf("Failed to parse service config %q: %v", sc, scpr.Err) + } + return scpr +} + +// setupPickFirst performs steps required for pick_first tests. It starts a +// bunch of backends exporting the TestService, creates a ClientConn to them +// with service config specifying the use of the pick_first LB policy. +func setupPickFirst(t *testing.T, backendCount int, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, []*stubserver.StubServer) { + t.Helper() + + r := manual.NewBuilderWithScheme("whatever") + + backends := make([]*stubserver.StubServer, backendCount) + addrs := make([]resolver.Address, backendCount) + for i := 0; i < backendCount; i++ { + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + + backends[i] = backend + addrs[i] = resolver.Address{Addr: backend.Address} + } + + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(pickFirstServiceConfig), + } + dopts = append(dopts, opts...) + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // At this point, the resolver has not returned any addresses to the channel. + // This RPC must block until the context expires. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + return cc, r, backends +} + +// stubBackendsToResolverAddrs converts from a set of stub server backends to +// resolver addresses. Useful when pushing addresses to the manual resolver. +func stubBackendsToResolverAddrs(backends []*stubserver.StubServer) []resolver.Address { + addrs := make([]resolver.Address, len(backends)) + for i, backend := range backends { + addrs[i] = resolver.Address{Addr: backend.Address} + } + return addrs +} + +// TestPickFirst_OneBackend tests the most basic scenario for pick_first. It +// brings up a single backend and verifies that all RPCs get routed to it. +func (s) TestPickFirst_OneBackend(t *testing.T) { + cc, r, backends := setupPickFirst(t, 1) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } +} + +// TestPickFirst_MultipleBackends tests the scenario with multiple backends and +// verifies that all RPCs get routed to the first one. +func (s) TestPickFirst_MultipleBackends(t *testing.T) { + cc, r, backends := setupPickFirst(t, 2) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } +} + +// TestPickFirst_OneServerDown tests the scenario where we have multiple +// backends and pick_first is working as expected. Verifies that RPCs get routed +// to the next backend in the list when the first one goes down. +func (s) TestPickFirst_OneServerDown(t *testing.T) { + cc, r, backends := setupPickFirst(t, 2) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Stop the backend which is currently being used. RPCs should get routed to + // the next backend in the list. + backends[0].Stop() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// TestPickFirst_AllServersDown tests the scenario where we have multiple +// backends and pick_first is working as expected. When all backends go down, +// the test verifies that RPCs fail with appropriate status code. +func (s) TestPickFirst_AllServersDown(t *testing.T) { + cc, r, backends := setupPickFirst(t, 2) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + for _, b := range backends { + b.Stop() + } + + client := testgrpc.NewTestServiceClient(cc) + for { + if ctx.Err() != nil { + t.Fatalf("channel failed to move to Unavailable after all backends were stopped: %v", ctx.Err()) + } + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) == codes.Unavailable { + return + } + time.Sleep(defaultTestShortTimeout) + } +} + +// TestPickFirst_AddressesRemoved tests the scenario where we have multiple +// backends and pick_first is working as expected. It then verifies that when +// addresses are removed by the name resolver, RPCs get routed appropriately. +func (s) TestPickFirst_AddressesRemoved(t *testing.T) { + cc, r, backends := setupPickFirst(t, 3) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Remove the first backend from the list of addresses originally pushed. + // RPCs should get routed to the first backend in the new list. + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[1], addrs[2]}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + // Append the backend that we just removed to the end of the list. + // Nothing should change. + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[1], addrs[2], addrs[0]}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + // Remove the first backend from the existing list of addresses. + // RPCs should get routed to the first backend in the new list. + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[2], addrs[0]}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[2]); err != nil { + t.Fatal(err) + } + + // Remove the first backend from the existing list of addresses. + // RPCs should get routed to the first backend in the new list. + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0]}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } +} + +// TestPickFirst_NewAddressWhileBlocking tests the case where pick_first is +// configured on a channel, things are working as expected and then a resolver +// updates removes all addresses. An RPC attempted at this point in time will be +// blocked because there are no valid backends. This test verifies that when new +// backends are added, the RPC is able to complete. +func (s) TestPickFirst_NewAddressWhileBlocking(t *testing.T) { + cc, r, backends := setupPickFirst(t, 2) + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Send a resolver update with no addresses. This should push the channel into + // TransientFailure. + r.UpdateState(resolver.State{}) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + doneCh := make(chan struct{}) + client := testgrpc.NewTestServiceClient(cc) + go func() { + // The channel is currently in TransientFailure and this RPC will block + // until the channel becomes Ready, which will only happen when we push a + // resolver update with a valid backend address. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Errorf("EmptyCall() = %v, want ", err) + } + close(doneCh) + }() + + // Make sure that there is one pending RPC on the ClientConn before attempting + // to push new addresses through the name resolver. If we don't do this, the + // resolver update can happen before the above goroutine gets to make the RPC. + for { + if err := ctx.Err(); err != nil { + t.Fatal(err) + } + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + t.Fatalf("there should only be one top channel, not %d", len(tcs)) + } + started := tcs[0].ChannelMetrics.CallsStarted.Load() + completed := tcs[0].ChannelMetrics.CallsSucceeded.Load() + tcs[0].ChannelMetrics.CallsFailed.Load() + if (started - completed) == 1 { + break + } + time.Sleep(defaultTestShortTimeout) + } + + // Send a resolver update with a valid backend to push the channel to Ready + // and unblock the above RPC. + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backends[0].Address}}}) + + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for blocked RPC to complete") + case <-doneCh: + } +} + +// TestPickFirst_StickyTransientFailure tests the case where pick_first is +// configured on a channel, and the backend is configured to close incoming +// connections as soon as they are accepted. The test verifies that the channel +// enters TransientFailure and stays there. The test also verifies that the +// pick_first LB policy is constantly trying to reconnect to the backend. +func (s) TestPickFirst_StickyTransientFailure(t *testing.T) { + // Spin up a local server which closes the connection as soon as it receives + // one. It also sends a signal on a channel whenever it received a connection. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + t.Cleanup(func() { lis.Close() }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + connCh := make(chan struct{}, 1) + go func() { + for { + conn, err := lis.Accept() + if err != nil { + return + } + select { + case connCh <- struct{}{}: + conn.Close() + case <-ctx.Done(): + return + } + } + }() + + // Dial the above server with a ConnectParams that does a constant backoff + // of defaultTestShortTimeout duration. + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(pickFirstServiceConfig), + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.Config{ + BaseDelay: defaultTestShortTimeout, + Multiplier: float64(0), + Jitter: float64(0), + MaxDelay: defaultTestShortTimeout, + }, + }), + } + cc, err := grpc.NewClient(lis.Addr().String(), dopts...) + if err != nil { + t.Fatalf("Failed to create new client: %v", err) + } + t.Cleanup(func() { cc.Close() }) + cc.Connect() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Spawn a goroutine to ensure that the channel stays in TransientFailure. + // The call to cc.WaitForStateChange will return false when the main + // goroutine exits and the context is cancelled. + go func() { + if cc.WaitForStateChange(ctx, connectivity.TransientFailure) { + if state := cc.GetState(); state != connectivity.Shutdown { + t.Errorf("Unexpected state change from TransientFailure to %s", cc.GetState()) + } + } + }() + + // Ensures that the pick_first LB policy is constantly trying to reconnect. + for i := 0; i < 10; i++ { + select { + case <-connCh: + case <-time.After(2 * defaultTestShortTimeout): + t.Error("Timeout when waiting for pick_first to reconnect") + } + } +} + +// Tests the PF LB policy with shuffling enabled. +func (s) TestPickFirst_ShuffleAddressList(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.PickFirstWeightedShuffling, false) + + const serviceConfig = `{"loadBalancingConfig": [{"pick_first":{ "shuffleAddressList": true }}]}` + + // Install a shuffler that always reverses two entries. + origShuf := pfinternal.RandShuffle + defer func() { pfinternal.RandShuffle = origShuf }() + pfinternal.RandShuffle = func(n int, f func(int, int)) { + if n != 2 { + t.Errorf("Shuffle called with n=%v; want 2", n) + return + } + f(0, 1) // reverse the two addresses + } + // Set up our backends. + cc, r, backends := setupPickFirst(t, 2) + addrs := stubBackendsToResolverAddrs(backends) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Push an update with both addresses and shuffling disabled. We should + // connect to backend 0. + r.UpdateState(resolver.State{Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{addrs[0]}}, + {Addresses: []resolver.Address{addrs[1]}}, + }}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Send a config with shuffling enabled. This will reverse the addresses, + // but the channel should still be connected to backend 0. + shufState := resolver.State{ + ServiceConfig: parseServiceConfig(t, r, serviceConfig), + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{addrs[0]}}, + {Addresses: []resolver.Address{addrs[1]}}, + }, + } + r.UpdateState(shufState) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Send a resolver update with no addresses. This should push the channel + // into TransientFailure. + r.UpdateState(resolver.State{}) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Send the same config as last time with shuffling enabled. Since we are + // not connected to backend 0, we should connect to backend 1. + r.UpdateState(shufState) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// Tests the PF LB policy with shuffling enabled. It explicitly unsets the +// Endpoints field in the resolver update to test the shuffling of the +// Addresses. +func (s) TestPickFirst_ShuffleAddressListNoEndpoints(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.PickFirstWeightedShuffling, false) + + // Install a shuffler that always reverses two entries. + origShuf := pfinternal.RandShuffle + defer func() { pfinternal.RandShuffle = origShuf }() + pfinternal.RandShuffle = func(n int, f func(int, int)) { + if n != 2 { + t.Errorf("Shuffle called with n=%v; want 2", n) + return + } + f(0, 1) // reverse the two addresses + } + + pfBuilder := balancer.Get(pfbalancer.Name) + shuffleConfig, err := pfBuilder.(balancer.ConfigParser).ParseConfig(json.RawMessage(`{ "shuffleAddressList": true }`)) + if err != nil { + t.Fatal(err) + } + noShuffleConfig, err := pfBuilder.(balancer.ConfigParser).ParseConfig(json.RawMessage(`{ "shuffleAddressList": false }`)) + if err != nil { + t.Fatal(err) + } + var activeCfg serviceconfig.LoadBalancingConfig + + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = pfBuilder.Build(bd.ClientConn, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + ccs.BalancerConfig = activeCfg + ccs.ResolverState.Endpoints = nil + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + } + + stub.Register(t.Name(), bf) + svcCfg := fmt.Sprintf(`{ "loadBalancingConfig": [{%q: {}}] }`, t.Name()) + // Set up our backends. + cc, r, backends := setupPickFirst(t, 2, grpc.WithDefaultServiceConfig(svcCfg)) + addrs := stubBackendsToResolverAddrs(backends) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Push an update with both addresses and shuffling disabled. We should + // connect to backend 0. + activeCfg = noShuffleConfig + resolverState := resolver.State{Addresses: addrs} + r.UpdateState(resolverState) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Send a config with shuffling enabled. This will reverse the addresses, + // but the channel should still be connected to backend 0. + activeCfg = shuffleConfig + r.UpdateState(resolverState) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Send a resolver update with no addresses. This should push the channel + // into TransientFailure. + r.UpdateState(resolver.State{}) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Send the same config as last time with shuffling enabled. Since we are + // not connected to backend 0, we should connect to backend 1. + r.UpdateState(resolverState) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// Tests the PF LB policy with weighted shuffling enabled. +func (s) TestPickFirst_ShuffleAddressList_WeightedShuffling(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.PickFirstWeightedShuffling, true) + + const serviceConfig = `{"loadBalancingConfig": [{"pick_first":{ "shuffleAddressList": true }}]}` + + // Install a rand func that returns a constant value. The test sets up three + // endpoints with increasing weights. This means that in the weighted + // shuffling algorithm, the endpoints will end up with increasing values for + // their keys. And since the algorithm sorts in descending order, the last + // endpoint should be the one that would get picked. + origRand := pfinternal.RandFloat64 + defer func() { pfinternal.RandFloat64 = origRand }() + pfinternal.RandFloat64 = func() float64 { + return 0.5 + } + + // Set up our backends. + cc, r, backends := setupPickFirst(t, 3) + addrs := stubBackendsToResolverAddrs(backends) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create endpoints for the above backends with increasing weights. + ep1 := resolver.Endpoint{Addresses: []resolver.Address{addrs[0]}} + ep1 = weight.Set(ep1, weight.EndpointInfo{Weight: 357913941}) // Normalized weight of 1/6 + ep2 := resolver.Endpoint{Addresses: []resolver.Address{addrs[1]}} + ep2 = weight.Set(ep2, weight.EndpointInfo{Weight: 715827882}) // Normalized weight of 2/6 + ep3 := resolver.Endpoint{Addresses: []resolver.Address{addrs[2]}} + ep3 = weight.Set(ep3, weight.EndpointInfo{Weight: 1073741824}) // Normalized weight of 3/6 + + // Push an update with all addresses and shuffling disabled. We should + // connect to backend 0. + r.UpdateState(resolver.State{Endpoints: []resolver.Endpoint{ep1, ep2, ep3}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Send a config with shuffling enabled. This will reverse the addresses, + // but the channel should still be connected to backend 0. + shufState := resolver.State{ + ServiceConfig: parseServiceConfig(t, r, serviceConfig), + Endpoints: []resolver.Endpoint{ep1, ep2, ep3}, + } + r.UpdateState(shufState) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Send a resolver update with no addresses. This should push the channel + // into TransientFailure. + r.UpdateState(resolver.State{}) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Send the same config as last time with shuffling enabled. Since we are + // not connected to backend 0, we should connect to backend 2. + r.UpdateState(shufState) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[2]); err != nil { + t.Fatal(err) + } +} + +// Test config parsing with the env var turned on and off for various scenarios. +func (s) TestPickFirst_ParseConfig_Success(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.PickFirstWeightedShuffling, false) + + // Install a shuffler that always reverses two entries. + origShuf := pfinternal.RandShuffle + defer func() { pfinternal.RandShuffle = origShuf }() + pfinternal.RandShuffle = func(n int, f func(int, int)) { + if n != 2 { + t.Errorf("Shuffle called with n=%v; want 2", n) + return + } + f(0, 1) // reverse the two addresses + } + + tests := []struct { + name string + serviceConfig string + wantFirstAddr bool + }{ + { + name: "empty pickfirst config", + serviceConfig: `{"loadBalancingConfig": [{"pick_first":{}}]}`, + wantFirstAddr: true, + }, + { + name: "empty good pickfirst config", + serviceConfig: `{"loadBalancingConfig": [{"pick_first":{ "shuffleAddressList": true }}]}`, + wantFirstAddr: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Set up our backends. + cc, r, backends := setupPickFirst(t, 2) + addrs := stubBackendsToResolverAddrs(backends) + + r.UpdateState(resolver.State{ + ServiceConfig: parseServiceConfig(t, r, test.serviceConfig), + Addresses: addrs, + }) + + // Some tests expect address shuffling to happen, and indicate that + // by setting wantFirstAddr to false (since our shuffling function + // defined at the top of this test, simply reverses the list of + // addresses provided to it). + wantAddr := addrs[0] + if !test.wantFirstAddr { + wantAddr = addrs[1] + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, wantAddr); err != nil { + t.Fatal(err) + } + }) + } +} + +// Test config parsing for a bad service config. +func (s) TestPickFirst_ParseConfig_Failure(t *testing.T) { + // Service config should fail with the below config. Name resolvers are + // expected to perform this parsing before they push the parsed service + // config to the channel. + const sc = `{"loadBalancingConfig": [{"pick_first":{ "shuffleAddressList": 666 }}]}` + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(sc) + if scpr.Err == nil { + t.Fatalf("ParseConfig() succeeded and returned %+v, when expected to fail", scpr) + } +} + +// setupPickFirstWithListenerWrapper is very similar to setupPickFirst, but uses +// a wrapped listener that the test can use to track accepted connections. +func setupPickFirstWithListenerWrapper(t *testing.T, backendCount int, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, []*stubserver.StubServer, []*testutils.ListenerWrapper) { + t.Helper() + + backends := make([]*stubserver.StubServer, backendCount) + addrs := make([]resolver.Address, backendCount) + listeners := make([]*testutils.ListenerWrapper, backendCount) + for i := 0; i < backendCount; i++ { + lis := testutils.NewListenerWrapper(t, nil) + backend := &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + + backends[i] = backend + addrs[i] = resolver.Address{Addr: backend.Address} + listeners[i] = lis + } + + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(pickFirstServiceConfig), + } + dopts = append(dopts, opts...) + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // At this point, the resolver has not returned any addresses to the channel. + // This RPC must block until the context expires. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + return cc, r, backends, listeners +} + +// TestPickFirst_AddressUpdateWithAttributes tests the case where an address +// update received by the pick_first LB policy differs in attributes. Addresses +// which differ in attributes are considered different from the perspective of +// subconn creation and connection establishment and the test verifies that new +// connections are created when attributes change. +func (s) TestPickFirst_AddressUpdateWithAttributes(t *testing.T) { + cc, r, backends, listeners := setupPickFirstWithListenerWrapper(t, 2) + + // Add a set of attributes to the addresses before pushing them to the + // pick_first LB policy through the manual resolver. + addrs := stubBackendsToResolverAddrs(backends) + for i := range addrs { + addrs[i].Attributes = addrs[i].Attributes.WithValue("test-attribute-1", fmt.Sprintf("%d", i)) + } + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that RPCs succeed to the first backend in the list. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Grab the wrapped connection from the listener wrapper. This will be used + // to verify the connection is closed. + val, err := listeners[0].NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) + } + conn := val.(*testutils.ConnWrapper) + + // Add another set of attributes to the addresses, and push them to the + // pick_first LB policy through the manual resolver. Leave the order of the + // addresses unchanged. + for i := range addrs { + addrs[i].Attributes = addrs[i].Attributes.WithValue("test-attribute-2", fmt.Sprintf("%d", i)) + } + r.UpdateState(resolver.State{Addresses: addrs}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // A change in the address attributes results in the new address being + // considered different to the current address. This will result in the old + // connection being closed and a new connection to the same backend (since + // address order is not modified). + if _, err := conn.CloseCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when expecting existing connection to be closed: %v", err) + } + val, err = listeners[0].NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) + } + conn = val.(*testutils.ConnWrapper) + + // Add another set of attributes to the addresses, and push them to the + // pick_first LB policy through the manual resolver. Reverse of the order + // of addresses. + for i := range addrs { + addrs[i].Attributes = addrs[i].Attributes.WithValue("test-attribute-3", fmt.Sprintf("%d", i)) + } + addrs[0], addrs[1] = addrs[1], addrs[0] + r.UpdateState(resolver.State{Addresses: addrs}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Ensure that the old connection is closed and a new connection is + // established to the first address in the new list. + if _, err := conn.CloseCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when expecting existing connection to be closed: %v", err) + } + _, err = listeners[1].NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) + } +} + +// TestPickFirst_AddressUpdateWithBalancerAttributes tests the case where an +// address update received by the pick_first LB policy differs in balancer +// attributes, which are meant only for consumption by LB policies. In this +// case, the test verifies that new connections are not created when the address +// update only changes the balancer attributes. +func (s) TestPickFirst_AddressUpdateWithBalancerAttributes(t *testing.T) { + cc, r, backends, listeners := setupPickFirstWithListenerWrapper(t, 2) + + // Add a set of balancer attributes to the addresses before pushing them to + // the pick_first LB policy through the manual resolver. + addrs := stubBackendsToResolverAddrs(backends) + for i := range addrs { + addrs[i].BalancerAttributes = addrs[i].BalancerAttributes.WithValue("test-attribute-1", fmt.Sprintf("%d", i)) + } + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that RPCs succeed to the expected backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Grab the wrapped connection from the listener wrapper. This will be used + // to verify the connection is not closed. + val, err := listeners[0].NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) + } + conn := val.(*testutils.ConnWrapper) + + // Add a set of balancer attributes to the addresses before pushing them to + // the pick_first LB policy through the manual resolver. Leave the order of + // the addresses unchanged. + for i := range addrs { + addrs[i].BalancerAttributes = addrs[i].BalancerAttributes.WithValue("test-attribute-2", fmt.Sprintf("%d", i)) + } + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that no new connection is established, and ensure that the old + // connection is not closed. + for i := range listeners { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := listeners[i].NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected error when expecting no new connection: %v", err) + } + } + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected error when expecting existing connection to stay active: %v", err) + } + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Add a set of balancer attributes to the addresses before pushing them to + // the pick_first LB policy through the manual resolver. Reverse of the + // order of addresses. + for i := range addrs { + addrs[i].BalancerAttributes = addrs[i].BalancerAttributes.WithValue("test-attribute-3", fmt.Sprintf("%d", i)) + } + addrs[0], addrs[1] = addrs[1], addrs[0] + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that no new connection is established, and ensure that the old + // connection is not closed. + for i := range listeners { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := listeners[i].NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected error when expecting no new connection: %v", err) + } + } + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected error when expecting existing connection to stay active: %v", err) + } + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// Tests the case where the pick_first LB policy receives an error from the name +// resolver without previously receiving a good update. Verifies that the +// channel moves to TRANSIENT_FAILURE and that error received from the name +// resolver is propagated to the caller of an RPC. +func (s) TestPickFirst_ResolverError_NoPreviousUpdate(t *testing.T) { + cc, r, _ := setupPickFirst(t, 0) + + nrErr := errors.New("error from name resolver") + r.CC().ReportError(nrErr) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + client := testgrpc.NewTestServiceClient(cc) + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Fatalf("EmptyCall() succeeded when expected to fail with error: %v", nrErr) + } + if !strings.Contains(err.Error(), nrErr.Error()) { + t.Fatalf("EmptyCall() failed with error: %v, want error: %v", err, nrErr) + } +} + +// Tests the case where the pick_first LB policy receives an error from the name +// resolver after receiving a good update (and the channel is currently READY). +// The test verifies that the channel continues to use the previously received +// good update. +func (s) TestPickFirst_ResolverError_WithPreviousUpdate_Ready(t *testing.T) { + cc, r, backends := setupPickFirst(t, 1) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + nrErr := errors.New("error from name resolver") + r.CC().ReportError(nrErr) + + // Ensure that RPCs continue to succeed for the next second. + client := testgrpc.NewTestServiceClient(cc) + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + } +} + +// Tests the case where the pick_first LB policy receives an error from the name +// resolver after receiving a good update (and the channel is currently in +// CONNECTING state). The test verifies that the channel continues to use the +// previously received good update, and that RPCs don't fail with the error +// received from the name resolver. +func (s) TestPickFirst_ResolverError_WithPreviousUpdate_Connecting(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + + // Listen on a local port and act like a server that blocks until the + // channel reaches CONNECTING and closes the connection without sending a + // server preface. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + waitForConnecting := make(chan struct{}) + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("Unexpected error when accepting a connection: %v", err) + } + defer conn.Close() + + select { + case <-waitForConnecting: + case <-ctx.Done(): + t.Error("Timeout when waiting for channel to move to CONNECTING state") + } + }() + + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(pickFirstServiceConfig), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + cc.Connect() + addrs := []resolver.Address{{Addr: lis.Addr().String()}} + r.UpdateState(resolver.State{Addresses: addrs}) + testutils.AwaitState(ctx, t, cc, connectivity.Connecting) + + nrErr := errors.New("error from name resolver") + r.CC().ReportError(nrErr) + + // RPCs should fail with deadline exceed error as long as they are in + // CONNECTING and not the error returned by the name resolver. + client := testgrpc.NewTestServiceClient(cc) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) { + t.Fatalf("EmptyCall() failed with error: %v, want error: %v", err, context.DeadlineExceeded) + } + + // Closing this channel leads to closing of the connection by our listener. + // gRPC should see this as a connection error. + close(waitForConnecting) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + checkForConnectionError(ctx, t, cc) +} + +// Tests the case where the pick_first LB policy receives an error from the name +// resolver after receiving a good update. The previous good update though has +// seen the channel move to TRANSIENT_FAILURE. The test verifies that the +// channel fails RPCs with the new error from the resolver. +func (s) TestPickFirst_ResolverError_WithPreviousUpdate_TransientFailure(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + + // Listen on a local port and act like a server that closes the connection + // without sending a server preface. + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("Unexpected error when accepting a connection: %v", err) + } + conn.Close() + }() + + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(pickFirstServiceConfig), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + cc.Connect() + addrs := []resolver.Address{{Addr: lis.Addr().String()}} + r.UpdateState(resolver.State{Addresses: addrs}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + checkForConnectionError(ctx, t, cc) + + // An error from the name resolver should result in RPCs failing with that + // error instead of the old error that caused the channel to move to + // TRANSIENT_FAILURE in the first place. + nrErr := errors.New("error from name resolver") + r.CC().ReportError(nrErr) + client := testgrpc.NewTestServiceClient(cc) + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); strings.Contains(err.Error(), nrErr.Error()) { + break + } + } + if ctx.Err() != nil { + t.Fatal("Timeout when waiting for RPCs to fail with error returned by the name resolver") + } +} + +func checkForConnectionError(ctx context.Context, t *testing.T, cc *grpc.ClientConn) { + t.Helper() + + // RPCs may fail on the client side in two ways, once the fake server closes + // the accepted connection: + // - writing the client preface succeeds, but not reading the server preface + // - writing the client preface fails + // In either case, we should see it fail with UNAVAILABLE. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable { + t.Fatalf("EmptyCall() failed with error: %v, want code %v", err, codes.Unavailable) + } +} + +// Tests the case where the pick_first LB policy receives an update from the +// name resolver with no addresses after receiving a good update. The test +// verifies that the channel fails RPCs with an error indicating the fact that +// the name resolver returned no addresses. +func (s) TestPickFirst_ResolverError_ZeroAddresses_WithPreviousUpdate(t *testing.T) { + cc, r, backends := setupPickFirst(t, 1) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + r.UpdateState(resolver.State{}) + wantErr := "produced zero addresses" + client := testgrpc.NewTestServiceClient(cc) + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); strings.Contains(err.Error(), wantErr) { + break + } + } + if ctx.Err() != nil { + t.Fatal("Timeout when waiting for RPCs to fail with error returned by the name resolver") + } +} + +// testServer is a server than can be stopped and resumed without closing +// the listener. This guarantees the same port number (and address) is used +// after restart. When a server is stopped, it accepts and closes all tcp +// connections from clients. +type testServer struct { + stubserver.StubServer + lis *testutils.RestartableListener +} + +func (s *testServer) stop() { + s.lis.Stop() +} + +func (s *testServer) resume() { + s.lis.Restart() +} + +func newTestServer(t *testing.T) *testServer { + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + rl := testutils.NewRestartableListener(l) + ss := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + Listener: rl, + } + return &testServer{ + StubServer: ss, + lis: rl, + } +} + +// setupPickFirstLeaf performs steps required for pick_first tests. It starts a +// bunch of backends exporting the TestService, and creates a ClientConn to them. +func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, *backendManager) { + t.Helper() + r := manual.NewBuilderWithScheme("whatever") + backends := make([]*testServer, backendCount) + addrs := make([]resolver.Address, backendCount) + + for i := 0; i < backendCount; i++ { + server := newTestServer(t) + backend := stubserver.StartTestService(t, &server.StubServer) + t.Cleanup(func() { + backend.Stop() + }) + backends[i] = server + addrs[i] = resolver.Address{Addr: backend.Address} + } + + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + } + dopts = append(dopts, opts...) + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // At this point, the resolver has not returned any addresses to the channel. + // This RPC must block until the context expires. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + return cc, r, &backendManager{backends} +} + +// TestPickFirstLeaf_SimpleResolverUpdate tests the behaviour of the pick first +// policy when given an list of addresses. The following steps are carried +// out in order: +// 1. A list of addresses are given through the resolver. Only one +// of the servers is running. +// 2. RPCs are sent to verify they reach the running server. +// +// The state transitions of the ClientConn and all the SubConns created are +// verified. +func (s) TestPickFirstLeaf_SimpleResolverUpdate_FirstServerReady(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + + cc, r, bm := setupPickFirstLeaf(t, 2, grpc.WithDefaultServiceConfig(stateStoringServiceConfig)) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + r.UpdateState(resolver.State{Addresses: addrs}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready}, + } + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions()); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_SimpleResolverUpdate_FirstServerUnReady(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + + cc, r, bm := setupPickFirstLeaf(t, 2, grpc.WithDefaultServiceConfig(stateStoringServiceConfig)) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + bm.stopAllExcept(1) + + r.UpdateState(resolver.State{Addresses: addrs}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions()); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_SimpleResolverUpdate_DuplicateAddrs(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + + cc, r, bm := setupPickFirstLeaf(t, 2, grpc.WithDefaultServiceConfig(stateStoringServiceConfig)) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + bm.stopAllExcept(1) + + // Add a duplicate entry in the addresslist + r.UpdateState(resolver.State{ + Addresses: append([]resolver.Address{addrs[0]}, addrs...), + }) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions()); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +// TestPickFirstLeaf_ResolverUpdates_DisjointLists tests the behaviour of the pick first +// policy when the following steps are carried out in order: +// 1. A list of addresses are given through the resolver. Only one +// of the servers is running. +// 2. RPCs are sent to verify they reach the running server. +// 3. A second resolver update is sent. Again, only one of the servers is +// running. This may not be the same server as before. +// 4. RPCs are sent to verify they reach the running server. +// +// The state transitions of the ClientConn and all the SubConns created are +// verified. +func (s) TestPickFirstLeaf_ResolverUpdates_DisjointLists(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 4, grpc.WithDefaultServiceConfig(stateStoringServiceConfig)) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + bm.backends[0].stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + bm.backends[2].stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[2], addrs[3]}}) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[3]); err != nil { + t.Fatal(err) + } + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[2]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[3]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions()); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_ResolverUpdates_ActiveBackendInUpdatedList(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 3, grpc.WithDefaultServiceConfig(stateStoringServiceConfig)) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + bm.backends[0].stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + bm.backends[2].stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[2], addrs[1]}}) + + // Verify that the ClientConn stays in READY. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions()); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_ResolverUpdates_InActiveBackendInUpdatedList(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 3, grpc.WithDefaultServiceConfig(stateStoringServiceConfig)) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + bm.backends[0].stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + bm.backends[2].stop() + bm.backends[0].resume() + + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[2]}}) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions()); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_ResolverUpdates_IdenticalLists(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 2, grpc.WithDefaultServiceConfig(stateStoringServiceConfig)) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + bm.backends[0].stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) + + // Verify that the ClientConn stays in READY. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions()); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +// TestPickFirstLeaf_StopConnectedServer tests the behaviour of the pick first +// policy when the connected server is shut down. It carries out the following +// steps in order: +// 1. A list of addresses are given through the resolver. Only one +// of the servers is running. +// 2. The running server is stopped, causing the ClientConn to enter IDLE. +// 3. A (possibly different) server is started. +// 4. RPCs are made to kick the ClientConn out of IDLE. The test verifies that +// the RPCs reach the running server. +// +// The test verifies the ClientConn state transitions. +func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerRestart(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 2, grpc.WithDefaultServiceConfig(stateStoringServiceConfig)) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + // shutdown all active backends except the target. + bm.stopAllExcept(0) + + r.UpdateState(resolver.State{Addresses: addrs}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + // Shut down the connected server. + bm.backends[0].stop() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Start the new target server. + bm.backends[0].resume() + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions()); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerRestart(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 2, grpc.WithDefaultServiceConfig(stateStoringServiceConfig)) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + // shutdown all active backends except the target. + bm.stopAllExcept(1) + + r.UpdateState(resolver.State{Addresses: addrs}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + // Shut down the connected server. + bm.backends[1].stop() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Start the new target server. + bm.backends[1].resume() + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions()); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerToFirst(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 2, grpc.WithDefaultServiceConfig(stateStoringServiceConfig)) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + // shutdown all active backends except the target. + bm.stopAllExcept(1) + + r.UpdateState(resolver.State{Addresses: addrs}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + // Shut down the connected server. + bm.backends[1].stop() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Start the new target server. + bm.backends[0].resume() + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions()); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerToSecond(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 2, grpc.WithDefaultServiceConfig(stateStoringServiceConfig)) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + // shutdown all active backends except the target. + bm.stopAllExcept(0) + + r.UpdateState(resolver.State{Addresses: addrs}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + // Shut down the connected server. + bm.backends[0].stop() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Start the new target server. + bm.backends[1].resume() + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates(), ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions()); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +// TestPickFirstLeaf_EmptyAddressList carries out the following steps in order: +// 1. Send a resolver update with one running backend. +// 2. Send an empty address list causing the balancer to enter TRANSIENT_FAILURE. +// 3. Send a resolver update with one running backend. +// The test verifies the ClientConn state transitions. +func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + balChan := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balChan}) + cc, r, bm := setupPickFirstLeaf(t, 1, grpc.WithDefaultServiceConfig(stateStoringServiceConfig)) + addrs := bm.resolverAddrs() + + stateSubscriber := &ccStateSubscriber{} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + r.UpdateState(resolver.State{Addresses: addrs}) + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + r.UpdateState(resolver.State{}) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // The balancer should have entered transient failure. + // It should transition to CONNECTING from TRANSIENT_FAILURE as sticky TF + // only applies when the initial TF is reported due to connection failures + // and not bad resolver states. + r.UpdateState(resolver.State{Addresses: addrs}) + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + wantTransitions := []connectivity.State{ + // From first resolver update. + connectivity.Connecting, + connectivity.Ready, + // From second update. + connectivity.TransientFailure, + // From third update. + connectivity.Connecting, + connectivity.Ready, + } + + if diff := cmp.Diff(wantTransitions, stateSubscriber.transitions()); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +// Test verifies that pickfirst correctly detects the end of the first happy +// eyeballs pass when the timer causes pickfirst to reach the end of the address +// list and failures are reported out of order. +func (s) TestPickFirstLeaf_HappyEyeballs_TF_AfterEndOfList(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + originalTimer := pfinternal.TimeAfterFunc + defer func() { + pfinternal.TimeAfterFunc = originalTimer + }() + triggerTimer, timeAfter := mockTimer() + pfinternal.TimeAfterFunc = timeAfter + + tmr := stats.NewTestMetricsRecorder() + dialer := testutils.NewBlockingDialer() + opts := []grpc.DialOption{ + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pfbalancer.Name)), + grpc.WithContextDialer(dialer.DialContext), + grpc.WithStatsHandler(tmr), + } + cc, rb, bm := setupPickFirstLeaf(t, 3, opts...) + addrs := bm.resolverAddrs() + holds := bm.holds(dialer) + rb.UpdateState(resolver.State{Addresses: addrs}) + cc.Connect() + + testutils.AwaitState(ctx, t, cc, connectivity.Connecting) + + // Verify that only the first server is contacted. + if holds[0].Wait(ctx) != true { + t.Fatalf("Timeout waiting for server %d with address %q to be contacted", 0, addrs[0]) + } + if holds[1].IsStarted() != false { + t.Fatalf("Server %d with address %q contacted unexpectedly", 1, addrs[1]) + } + if holds[2].IsStarted() != false { + t.Fatalf("Server %d with address %q contacted unexpectedly", 2, addrs[2]) + } + + // Make the happy eyeballs timer fire once and verify that the + // second server is contacted, but the third isn't. + triggerTimer() + if holds[1].Wait(ctx) != true { + t.Fatalf("Timeout waiting for server %d with address %q to be contacted", 1, addrs[1]) + } + if holds[2].IsStarted() != false { + t.Fatalf("Server %d with address %q contacted unexpectedly", 2, addrs[2]) + } + + // Make the happy eyeballs timer fire once more and verify that the + // third server is contacted. + triggerTimer() + if holds[2].Wait(ctx) != true { + t.Fatalf("Timeout waiting for server %d with address %q to be contacted", 2, addrs[2]) + } + + // First SubConn Fails. + holds[0].Fail(fmt.Errorf("test error")) + tmr.WaitForInt64CountIncr(ctx, 1) + + // No TF should be reported until the first pass is complete. + shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer shortCancel() + testutils.AwaitNotState(shortCtx, t, cc, connectivity.TransientFailure) + + // Third SubConn fails. + shortCtx, shortCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer shortCancel() + holds[2].Fail(fmt.Errorf("test error")) + tmr.WaitForInt64CountIncr(ctx, 1) + testutils.AwaitNotState(shortCtx, t, cc, connectivity.TransientFailure) + + // Last SubConn fails, this should result in a TF update. + holds[1].Fail(fmt.Errorf("test error")) + tmr.WaitForInt64CountIncr(ctx, 1) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + waitForMetric(ctx, t, tmr, "grpc.subchannel.connection_attempts_failed") + + // Only connection attempt fails in this test. + if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_failed"); got != 1 { + t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_failed", got, 1) + } + if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_succeeded"); got != 0 { + t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_succeeded", got, 0) + } +} + +// Test verifies that pickfirst attempts to connect to the second backend once +// the happy eyeballs timer expires. +func (s) TestPickFirstLeaf_HappyEyeballs_TriggerConnectionDelay(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + originalTimer := pfinternal.TimeAfterFunc + defer func() { + pfinternal.TimeAfterFunc = originalTimer + }() + triggerTimer, timeAfter := mockTimer() + pfinternal.TimeAfterFunc = timeAfter + + tmr := stats.NewTestMetricsRecorder() + dialer := testutils.NewBlockingDialer() + opts := []grpc.DialOption{ + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pfbalancer.Name)), + grpc.WithContextDialer(dialer.DialContext), + grpc.WithStatsHandler(tmr), + } + cc, rb, bm := setupPickFirstLeaf(t, 2, opts...) + addrs := bm.resolverAddrs() + holds := bm.holds(dialer) + rb.UpdateState(resolver.State{Addresses: addrs}) + cc.Connect() + + testutils.AwaitState(ctx, t, cc, connectivity.Connecting) + + // Verify that only the first server is contacted. + if holds[0].Wait(ctx) != true { + t.Fatalf("Timeout waiting for server %d with address %q to be contacted", 0, addrs[0]) + } + if holds[1].IsStarted() != false { + t.Fatalf("Server %d with address %q contacted unexpectedly", 1, addrs[1]) + } + + // Make the happy eyeballs timer fire once and verify that the + // second server is contacted. + triggerTimer() + if holds[1].Wait(ctx) != true { + t.Fatalf("Timeout waiting for server %d with address %q to be contacted", 1, addrs[1]) + } + + // Get the connection attempt to the second server to succeed and verify + // that the channel becomes READY. + holds[1].Resume() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + waitForMetric(ctx, t, tmr, "grpc.subchannel.connection_attempts_succeeded") + // Only connection attempt successes in this test. + if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_succeeded"); got != 1 { + t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_succeeded", got, 1) + } + if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_failed"); got != 0 { + t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_failed", got, 0) + } +} + +func waitForMetric(ctx context.Context, t *testing.T, tmr *stats.TestMetricsRecorder, metricName string) { + for { + if _, ok := tmr.Metric(metricName); ok { + break + } + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for metric emission: %s", metricName) + case <-time.After(10 * time.Millisecond): + continue + } + } +} + +// Test tests the pickfirst balancer by causing a SubConn to fail and then +// jumping to the 3rd SubConn after the happy eyeballs timer expires. +func (s) TestPickFirstLeaf_HappyEyeballs_TF_ThenTimerFires(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + originalTimer := pfinternal.TimeAfterFunc + defer func() { + pfinternal.TimeAfterFunc = originalTimer + }() + triggerTimer, timeAfter := mockTimer() + pfinternal.TimeAfterFunc = timeAfter + + tmr := stats.NewTestMetricsRecorder() + dialer := testutils.NewBlockingDialer() + opts := []grpc.DialOption{ + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pfbalancer.Name)), + grpc.WithContextDialer(dialer.DialContext), + grpc.WithStatsHandler(tmr), + } + cc, rb, bm := setupPickFirstLeaf(t, 3, opts...) + addrs := bm.resolverAddrs() + holds := bm.holds(dialer) + rb.UpdateState(resolver.State{Addresses: addrs}) + cc.Connect() + + testutils.AwaitState(ctx, t, cc, connectivity.Connecting) + + // Verify that only the first server is contacted. + if holds[0].Wait(ctx) != true { + t.Fatalf("Timeout waiting for server %d with address %q to be contacted", 0, addrs[0]) + } + if holds[1].IsStarted() != false { + t.Fatalf("Server %d with address %q contacted unexpectedly", 1, addrs[1]) + } + if holds[2].IsStarted() != false { + t.Fatalf("Server %d with address %q contacted unexpectedly", 2, addrs[2]) + } + + // First SubConn Fails. + holds[0].Fail(fmt.Errorf("test error")) + + // Verify that only the second server is contacted. + if holds[1].Wait(ctx) != true { + t.Fatalf("Timeout waiting for server %d with address %q to be contacted", 1, addrs[1]) + } + if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_failed"); got != 1 { + t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_failed", got, 1) + } + if holds[2].IsStarted() != false { + t.Fatalf("Server %d with address %q contacted unexpectedly", 2, addrs[2]) + } + + // The happy eyeballs timer expires, pickfirst should stop waiting for + // server[1] to report a failure/success and request the creation of a third + // SubConn. + triggerTimer() + if holds[2].Wait(ctx) != true { + t.Fatalf("Timeout waiting for server %d with address %q to be contacted", 2, addrs[2]) + } + + // Get the connection attempt to the second server to succeed and verify + // that the channel becomes READY. + holds[1].Resume() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + waitForMetric(ctx, t, tmr, "grpc.subchannel.connection_attempts_succeeded") + if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_succeeded"); got != 1 { + t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_succeeded", got, 1) + } + if got, _ := tmr.Metric("grpc.subchannel.disconnections"); got != 0 { + t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.disconnections", got, 0) + } +} + +// Test verifies that when a subchannel is shut down by the LB (because another +// subchannel won) while its dial is still in-flight, it records exactly one +// successful attempt. +func (s) TestPickFirstLeaf_HappyEyeballs_Ignore_Inflight_Cancellations(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + originalTimer := pfinternal.TimeAfterFunc + defer func() { + pfinternal.TimeAfterFunc = originalTimer + }() + triggerTimer, timeAfter := mockTimer() + pfinternal.TimeAfterFunc = timeAfter + + tmr := stats.NewTestMetricsRecorder() + dialer := testutils.NewBlockingDialer() + opts := []grpc.DialOption{ + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pfbalancer.Name)), + grpc.WithContextDialer(dialer.DialContext), + grpc.WithStatsHandler(tmr), + } + + // Setup 2 backend addresses + cc, rb, bm := setupPickFirstLeaf(t, 2, opts...) + addrs := bm.resolverAddrs() + holds := bm.holds(dialer) + rb.UpdateState(resolver.State{Addresses: addrs}) + cc.Connect() + + // Make sure we connect to second subconn + testutils.AwaitState(ctx, t, cc, connectivity.Connecting) + if holds[0].Wait(ctx) != true { + t.Fatalf("Timeout waiting for server %d to be contacted", 0) + } + triggerTimer() + if holds[1].Wait(ctx) != true { + t.Fatalf("Timeout waiting for server %d to be contacted", 1) + } + holds[1].Resume() + + // Wait for Channel to become READY. + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Unblock the First SubConn. + // Since the LB has already closed this subchannel, the context passed to Dial + // is canceled. This will lead to an inflight attempt to be cancelled. + // No success or failure metric should be recorded for this. + holds[0].Resume() + + // --- Assertions --- + + // Wait for the SUCCESS metric to ensure recording logic has processed. + waitForMetric(ctx, t, tmr, "grpc.subchannel.connection_attempts_succeeded") + + // Verify Success: Exactly 1 (The Winner). + if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_succeeded"); got != 1 { + t.Errorf("Unexpected data for metric %v, got: %v, want: 1", "grpc.subchannel.connection_attempts_succeeded", got) + } + + // Verify Failure: Exactly 0 (The Loser was ignored). + // We poll briefly to ensure no delayed failure metric appears. + sCtx, sCancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer sCancel() + for ; sCtx.Err() == nil; <-time.After(time.Millisecond) { + if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_failed"); got != 0 { + t.Fatalf("Unexpected failure recorded for shutdown subchannel, got: %v, want: 0", got) + } + } + +} + +func (s) TestPickFirstLeaf_InterleavingIPV4Preferred(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc := testutils.NewBalancerClientConn(t) + bal := balancer.Get(pfbalancer.Name).Build(cc, balancer.BuildOptions{}) + defer bal.Close() + ccState := balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1111"}}}, + {Addresses: []resolver.Address{{Addr: "2.2.2.2:2"}}}, + {Addresses: []resolver.Address{{Addr: "3.3.3.3:3"}}}, + // IPv4-mapped IPv6 address, considered as an IPv4 for + // interleaving. + {Addresses: []resolver.Address{{Addr: "[::FFFF:192.168.0.1]:2222"}}}, + {Addresses: []resolver.Address{{Addr: "[0001:0001:0001:0001:0001:0001:0001:0001]:8080"}}}, + {Addresses: []resolver.Address{{Addr: "[0002:0002:0002:0002:0002:0002:0002:0002]:8080"}}}, + {Addresses: []resolver.Address{{Addr: "[fe80::1%eth0]:3333"}}}, + {Addresses: []resolver.Address{{Addr: "grpc.io:80"}}}, // not an IP. + }, + }, + } + if err := bal.UpdateClientConnState(ccState); err != nil { + t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) + } + + wantAddrs := []resolver.Address{ + {Addr: "1.1.1.1:1111"}, + {Addr: "[0001:0001:0001:0001:0001:0001:0001:0001]:8080"}, + {Addr: "grpc.io:80"}, + {Addr: "2.2.2.2:2"}, + {Addr: "[0002:0002:0002:0002:0002:0002:0002:0002]:8080"}, + {Addr: "3.3.3.3:3"}, + {Addr: "[fe80::1%eth0]:3333"}, + {Addr: "[::FFFF:192.168.0.1]:2222"}, + } + + gotAddrs, err := subConnAddresses(ctx, cc, 8) + if err != nil { + t.Fatalf("%v", err) + } + if diff := cmp.Diff(wantAddrs, gotAddrs, ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn creation order mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_InterleavingIPv6Preferred(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc := testutils.NewBalancerClientConn(t) + bal := balancer.Get(pfbalancer.Name).Build(cc, balancer.BuildOptions{}) + defer bal.Close() + ccState := balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "[0001:0001:0001:0001:0001:0001:0001:0001]:8080"}}}, + {Addresses: []resolver.Address{{Addr: "[0001:0001:0001:0001:0001:0001:0001:0001]:8080"}}}, // duplicate, should be ignored. + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1111"}}}, + {Addresses: []resolver.Address{{Addr: "2.2.2.2:2"}}}, + {Addresses: []resolver.Address{{Addr: "3.3.3.3:3"}}}, + {Addresses: []resolver.Address{{Addr: "[::FFFF:192.168.0.1]:2222"}}}, + {Addresses: []resolver.Address{{Addr: "[0002:0002:0002:0002:0002:0002:0002:0002]:2222"}}}, + {Addresses: []resolver.Address{{Addr: "[fe80::1%eth0]:3333"}}}, + {Addresses: []resolver.Address{{Addr: "grpc.io:80"}}}, // not an IP. + }, + }, + } + if err := bal.UpdateClientConnState(ccState); err != nil { + t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) + } + + wantAddrs := []resolver.Address{ + {Addr: "[0001:0001:0001:0001:0001:0001:0001:0001]:8080"}, + {Addr: "1.1.1.1:1111"}, + {Addr: "grpc.io:80"}, + {Addr: "[0002:0002:0002:0002:0002:0002:0002:0002]:2222"}, + {Addr: "2.2.2.2:2"}, + {Addr: "[fe80::1%eth0]:3333"}, + {Addr: "3.3.3.3:3"}, + {Addr: "[::FFFF:192.168.0.1]:2222"}, + } + + gotAddrs, err := subConnAddresses(ctx, cc, 8) + if err != nil { + t.Fatalf("%v", err) + } + if diff := cmp.Diff(wantAddrs, gotAddrs, ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn creation order mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_InterleavingUnknownPreferred(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc := testutils.NewBalancerClientConn(t) + bal := balancer.Get(pfbalancer.Name).Build(cc, balancer.BuildOptions{}) + defer bal.Close() + ccState := balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "grpc.io:80"}}}, // not an IP. + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1111"}}}, + {Addresses: []resolver.Address{{Addr: "2.2.2.2:2"}}}, + {Addresses: []resolver.Address{{Addr: "3.3.3.3:3"}}}, + {Addresses: []resolver.Address{{Addr: "[::FFFF:192.168.0.1]:2222"}}}, + {Addresses: []resolver.Address{{Addr: "[0001:0001:0001:0001:0001:0001:0001:0001]:8080"}}}, + {Addresses: []resolver.Address{{Addr: "[0002:0002:0002:0002:0002:0002:0002:0002]:8080"}}}, + {Addresses: []resolver.Address{{Addr: "[fe80::1%eth0]:3333"}}}, + {Addresses: []resolver.Address{{Addr: "example.com:80"}}}, // not an IP. + }, + }, + } + if err := bal.UpdateClientConnState(ccState); err != nil { + t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) + } + + wantAddrs := []resolver.Address{ + {Addr: "grpc.io:80"}, + {Addr: "1.1.1.1:1111"}, + {Addr: "[0001:0001:0001:0001:0001:0001:0001:0001]:8080"}, + {Addr: "example.com:80"}, + {Addr: "2.2.2.2:2"}, + {Addr: "[0002:0002:0002:0002:0002:0002:0002:0002]:8080"}, + {Addr: "3.3.3.3:3"}, + {Addr: "[fe80::1%eth0]:3333"}, + {Addr: "[::FFFF:192.168.0.1]:2222"}, + } + + gotAddrs, err := subConnAddresses(ctx, cc, 9) + if err != nil { + t.Fatalf("%v", err) + } + if diff := cmp.Diff(wantAddrs, gotAddrs, ignoreBalAttributesOpt); diff != "" { + t.Errorf("SubConn creation order mismatch (-want +got):\n%s", diff) + } +} + +// Test verifies that pickfirst balancer transitions to READY when the health +// listener is enabled. Since client side health checking is not enabled in +// the service config, the health listener will send a health update for READY +// after registering the listener. +func (s) TestPickFirstLeaf_HealthListenerEnabled(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = balancer.Get(pfbalancer.Name).Build(bd.ClientConn, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + ccs.ResolverState = pfbalancer.EnableHealthListener(ccs.ResolverState) + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + } + + stub.Register(t.Name(), bf) + svcCfg := fmt.Sprintf(`{ "loadBalancingConfig": [{%q: {}}] }`, t.Name()) + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(svcCfg), + } + cc, err := grpc.NewClient(backend.Address, opts...) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", backend.Address, err) + + } + defer cc.Close() + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, resolver.Address{Addr: backend.Address}); err != nil { + t.Fatal(err) + } +} + +// Test verifies that a health listener is not registered when pickfirst is not +// under a petiole policy. +func (s) TestPickFirstLeaf_HealthListenerNotEnabled(t *testing.T) { + // Wrap the clientconn to intercept NewSubConn. + // Capture the health list by wrapping the SC. + // Wrap the picker to unwrap the SC. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + healthListenerCh := make(chan func(balancer.SubConnState)) + + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + ccw := &healthListenerCapturingCCWrapper{ + ClientConn: bd.ClientConn, + healthListenerCh: healthListenerCh, + subConnStateCh: make(chan balancer.SubConnState, 5), + } + bd.ChildBalancer = balancer.Get(pfbalancer.Name).Build(ccw, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + // Functions like a non-petiole policy by not configuring the use + // of health listeners. + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + } + + stub.Register(t.Name(), bf) + svcCfg := fmt.Sprintf(`{ "loadBalancingConfig": [{%q: {}}] }`, t.Name()) + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(svcCfg), + } + cc, err := grpc.NewClient(backend.Address, opts...) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", backend.Address, err) + + } + defer cc.Close() + cc.Connect() + + select { + case <-healthListenerCh: + t.Fatal("Health listener registered when not enabled.") + case <-time.After(defaultTestShortTimeout): + } + + testutils.AwaitState(ctx, t, cc, connectivity.Ready) +} + +// Test mocks the updates sent to the health listener and verifies that the +// balancer correctly reports the health state once the SubConn's connectivity +// state becomes READY. +func (s) TestPickFirstLeaf_HealthUpdates(t *testing.T) { + // Wrap the clientconn to intercept NewSubConn. + // Capture the health list by wrapping the SC. + // Wrap the picker to unwrap the SC. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + healthListenerCh := make(chan func(balancer.SubConnState)) + scConnectivityStateCh := make(chan balancer.SubConnState, 5) + + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + ccw := &healthListenerCapturingCCWrapper{ + ClientConn: bd.ClientConn, + healthListenerCh: healthListenerCh, + subConnStateCh: scConnectivityStateCh, + } + bd.ChildBalancer = balancer.Get(pfbalancer.Name).Build(ccw, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + ccs.ResolverState = pfbalancer.EnableHealthListener(ccs.ResolverState) + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + } + + stub.Register(t.Name(), bf) + svcCfg := fmt.Sprintf(`{ "loadBalancingConfig": [{%q: {}}] }`, t.Name()) + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(svcCfg), + } + cc, err := grpc.NewClient(backend.Address, opts...) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", backend.Address, err) + + } + defer cc.Close() + cc.Connect() + + var healthListener func(balancer.SubConnState) + select { + case healthListener = <-healthListenerCh: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for health listener to be registered.") + } + + // Wait for the raw connectivity state to become READY. The LB policy should + // wait for the health updates before transitioning the channel to READY. + for { + var scs balancer.SubConnState + select { + case scs = <-scConnectivityStateCh: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for the SubConn connectivity state to become READY.") + } + if scs.ConnectivityState == connectivity.Ready { + break + } + } + + shortCtx, cancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer cancel() + testutils.AwaitNoStateChange(shortCtx, t, cc, connectivity.Connecting) + + // The LB policy should update the channel state based on the health state. + healthListener(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: fmt.Errorf("test health check failure"), + }) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + healthListener(balancer.SubConnState{ + ConnectivityState: connectivity.Connecting, + ConnectionError: balancer.ErrNoSubConnAvailable, + }) + testutils.AwaitState(ctx, t, cc, connectivity.Connecting) + + healthListener(balancer.SubConnState{ + ConnectivityState: connectivity.Ready, + }) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, resolver.Address{Addr: backend.Address}); err != nil { + t.Fatal(err) + } + + // When the health check fails, the channel should transition to TF. + healthListener(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: fmt.Errorf("test health check failure"), + }) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) +} + +// Tests the case where an address update received by the pick_first LB policy +// differs in metadata which should be ignored by the LB policy. In this case, +// the test verifies that new connections are not created when the address +// update only changes the metadata. +func (s) TestPickFirstLeaf_AddressUpdateWithMetadata(t *testing.T) { + dialer := testutils.NewBlockingDialer() + dopts := []grpc.DialOption{ + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pfbalancer.Name)), + grpc.WithContextDialer(dialer.DialContext), + } + cc, r, backends := setupPickFirstLeaf(t, 2, dopts...) + + // Add a metadata to the addresses before pushing them to the pick_first LB + // policy through the manual resolver. + addrs := backends.resolverAddrs() + for i := range addrs { + addrs[i].Metadata = &metadata.MD{ + "test-metadata-1": []string{fmt.Sprintf("%d", i)}, + } + } + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that RPCs succeed to the expected backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Create holds for each backend. This will be used to verify the connection + // is not re-established. + holds := backends.holds(dialer) + + // Add metadata to the addresses before pushing them to the pick_first LB + // policy through the manual resolver. Leave the order of the addresses + // unchanged. + for i := range addrs { + addrs[i].Metadata = &metadata.MD{ + "test-metadata-2": []string{fmt.Sprintf("%d", i)}, + } + } + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that no new connection is established. + for i := range holds { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if holds[i].Wait(sCtx) { + t.Fatalf("Unexpected connection attempt to backend: %s", addrs[i]) + } + } + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Add metadata to the addresses before pushing them to the pick_first LB + // policy through the manual resolver. Reverse of the order of addresses. + for i := range addrs { + addrs[i].Metadata = &metadata.MD{ + "test-metadata-3": []string{fmt.Sprintf("%d", i)}, + } + } + addrs[0], addrs[1] = addrs[1], addrs[0] + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that no new connection is established. + for i := range holds { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if holds[i].Wait(sCtx) { + t.Fatalf("Unexpected connection attempt to backend: %s", addrs[i]) + } + } + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// Tests the scenario where a connection is established and then breaks, leading +// to a reconnection attempt. While the reconnection is in progress, a resolver +// update with a new address is received. The test verifies that the balancer +// creates a new SubConn for the new address and that the ClientConn eventually +// becomes READY. +func (s) TestPickFirstLeaf_Reconnection(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc := testutils.NewBalancerClientConn(t) + bal := balancer.Get(pfbalancer.Name).Build(cc, balancer.BuildOptions{}) + defer bal.Close() + ccState := balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, + }, + }, + } + if err := bal.UpdateClientConnState(ccState); err != nil { + t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) + } + + select { + case state := <-cc.NewStateCh: + if got, want := state, connectivity.Connecting; got != want { + t.Fatalf("Received unexpected ClientConn sate: got %v, want %v", got, want) + } + case <-ctx.Done(): + t.Fatal("Context timed out waiting for ClientConn state update.") + } + + sc1 := <-cc.NewSubConnCh + select { + case <-sc1.ConnectCh: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for Connect() to be called on sc1.") + } + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + if err := cc.WaitForConnectivityState(ctx, connectivity.Ready); err != nil { + t.Fatalf("Context timed out waiting for ClientConn to become READY.") + } + + // Simulate a connection breakage, this should result the channel + // transitioning to IDLE. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + if err := cc.WaitForConnectivityState(ctx, connectivity.Idle); err != nil { + t.Fatalf("Context timed out waiting for ClientConn to enter IDLE.") + } + + // Calling the idle picker should result in the SubConn being re-connected. + picker := <-cc.NewPickerCh + if _, err := picker.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("picker.Pick() returned error: %v, want %v", err, balancer.ErrNoSubConnAvailable) + } + + select { + case <-sc1.ConnectCh: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for Connect() to be called on sc1.") + } + + // Send a resolver update, removing the existing SubConn. Since the balancer + // is connecting, it should create a new SubConn for the new backend + // address. + ccState = balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "2.2.2.2:2"}}}, + }, + }, + } + if err := bal.UpdateClientConnState(ccState); err != nil { + t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) + } + + var sc2 *testutils.TestSubConn + select { + case sc2 = <-cc.NewSubConnCh: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for new SubConn to be created.") + } + + select { + case <-sc2.ConnectCh: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for Connect() to be called on sc2.") + } + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + if err := cc.WaitForConnectivityState(ctx, connectivity.Ready); err != nil { + t.Fatalf("Context timed out waiting for ClientConn to become READY.") + } +} + +// healthListenerCapturingCCWrapper is used to capture the health listener so +// that health updates can be mocked for testing. +type healthListenerCapturingCCWrapper struct { + balancer.ClientConn + healthListenerCh chan func(balancer.SubConnState) + subConnStateCh chan balancer.SubConnState +} + +func (ccw *healthListenerCapturingCCWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + oldListener := opts.StateListener + opts.StateListener = func(scs balancer.SubConnState) { + ccw.subConnStateCh <- scs + if oldListener != nil { + oldListener(scs) + } + } + sc, err := ccw.ClientConn.NewSubConn(addrs, opts) + if err != nil { + return nil, err + } + return &healthListenerCapturingSCWrapper{ + SubConn: sc, + listenerCh: ccw.healthListenerCh, + }, nil +} + +func (ccw *healthListenerCapturingCCWrapper) UpdateState(state balancer.State) { + state.Picker = &unwrappingPicker{state.Picker} + ccw.ClientConn.UpdateState(state) +} + +type healthListenerCapturingSCWrapper struct { + balancer.SubConn + listenerCh chan func(balancer.SubConnState) +} + +func (scw *healthListenerCapturingSCWrapper) RegisterHealthListener(listener func(balancer.SubConnState)) { + scw.listenerCh <- listener +} + +// unwrappingPicker unwraps SubConns because the channel expects SubConns to be +// addrConns. +type unwrappingPicker struct { + balancer.Picker +} + +func (pw *unwrappingPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + pr, err := pw.Picker.Pick(info) + if pr.SubConn != nil { + pr.SubConn = pr.SubConn.(*healthListenerCapturingSCWrapper).SubConn + } + return pr, err +} + +// subConnAddresses makes the pickfirst balancer create the requested number of +// SubConns by triggering transient failures. The function returns the +// addresses of the created SubConns. +func subConnAddresses(ctx context.Context, cc *testutils.BalancerClientConn, subConnCount int) ([]resolver.Address, error) { + addresses := []resolver.Address{} + for i := 0; i < subConnCount; i++ { + select { + case <-ctx.Done(): + return nil, fmt.Errorf("test timed out after creating %d subchannels, want %d", i, subConnCount) + case sc := <-cc.NewSubConnCh: + if len(sc.Addresses) != 1 { + return nil, fmt.Errorf("new subchannel created with %d addresses, want 1", len(sc.Addresses)) + } + addresses = append(addresses, sc.Addresses[0]) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + }) + } + } + return addresses, nil +} + +// stateStoringBalancer stores the state of the SubConns being created. +type stateStoringBalancer struct { + balancer.Balancer + mu sync.Mutex + scStates []*scState +} + +func (b *stateStoringBalancer) Close() { + b.Balancer.Close() +} + +type stateStoringBalancerBuilder struct { + balancer chan *stateStoringBalancer +} + +func (b *stateStoringBalancerBuilder) Name() string { + return stateStoringBalancerName +} + +func (b *stateStoringBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + bal := &stateStoringBalancer{} + bal.Balancer = balancer.Get(pfbalancer.Name).Build(&stateStoringCCWrapper{cc, bal}, opts) + b.balancer <- bal + return bal +} + +func (b *stateStoringBalancer) subConnStates() []scState { + b.mu.Lock() + defer b.mu.Unlock() + ret := []scState{} + for _, s := range b.scStates { + ret = append(ret, *s) + } + return ret +} + +func (b *stateStoringBalancer) addSCState(state *scState) { + b.mu.Lock() + b.scStates = append(b.scStates, state) + b.mu.Unlock() +} + +type stateStoringCCWrapper struct { + balancer.ClientConn + b *stateStoringBalancer +} + +func (ccw *stateStoringCCWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + oldListener := opts.StateListener + scs := &scState{ + State: connectivity.Idle, + Addrs: addrs, + } + ccw.b.addSCState(scs) + opts.StateListener = func(s balancer.SubConnState) { + ccw.b.mu.Lock() + scs.State = s.ConnectivityState + ccw.b.mu.Unlock() + oldListener(s) + } + return ccw.ClientConn.NewSubConn(addrs, opts) +} + +type scState struct { + State connectivity.State + Addrs []resolver.Address +} + +type backendManager struct { + backends []*testServer +} + +func (b *backendManager) stopAllExcept(index int) { + for idx, b := range b.backends { + if idx != index { + b.stop() + } + } +} + +// resolverAddrs returns a list of resolver addresses for the stub server +// backends. Useful when pushing addresses to the manual resolver. +func (b *backendManager) resolverAddrs() []resolver.Address { + addrs := make([]resolver.Address, len(b.backends)) + for i, backend := range b.backends { + addrs[i] = resolver.Address{Addr: backend.Address} + } + return addrs +} + +func (b *backendManager) holds(dialer *testutils.BlockingDialer) []*testutils.Hold { + holds := []*testutils.Hold{} + for _, addr := range b.resolverAddrs() { + holds = append(holds, dialer.Hold(addr.Addr)) + } + return holds +} + +type ccStateSubscriber struct { + mu sync.Mutex + states []connectivity.State +} + +// transitions returns all the states that ccStateSubscriber recorded. +// Without this a race condition occurs when the test compares the states +// and the subscriber at the same time receives a connectivity.Shutdown. +func (c *ccStateSubscriber) transitions() []connectivity.State { + c.mu.Lock() + defer c.mu.Unlock() + return c.states +} + +func (c *ccStateSubscriber) OnMessage(msg any) { + c.mu.Lock() + defer c.mu.Unlock() + c.states = append(c.states, msg.(connectivity.State)) +} + +// mockTimer returns a fake timeAfterFunc that will not trigger automatically. +// It returns a function that can be called to manually trigger the execution +// of the scheduled callback. +func mockTimer() (triggerFunc func(), timerFunc func(_ time.Duration, f func()) func()) { + timerCh := make(chan struct{}) + triggerFunc = func() { + timerCh <- struct{}{} + } + return triggerFunc, func(_ time.Duration, f func()) func() { + stopCh := make(chan struct{}) + go func() { + select { + case <-timerCh: + f() + case <-stopCh: + } + }() + return sync.OnceFunc(func() { + close(stopCh) + }) + } +} diff --git a/balancer/pickfirst/pickfirst_test.go b/balancer/pickfirst/pickfirst_test.go new file mode 100644 index 0000000..9052c1f --- /dev/null +++ b/balancer/pickfirst/pickfirst_test.go @@ -0,0 +1,376 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package pickfirst + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" +) + +const ( + // Default timeout for tests in this package. + defaultTestTimeout = 10 * time.Second + // Default short timeout, to be used when waiting for events which are not + // expected to happen. + defaultTestShortTimeout = 100 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestPickFirst_InitialResolverError sends a resolver error to the balancer +// before a valid resolver update. It verifies that the clientconn state is +// updated to TRANSIENT_FAILURE. +func (s) TestPickFirst_InitialResolverError(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc := testutils.NewBalancerClientConn(t) + bal := balancer.Get(Name).Build(cc, balancer.BuildOptions{}) + defer bal.Close() + bal.ResolverError(errors.New("resolution failed: test error")) + + if err := cc.WaitForConnectivityState(ctx, connectivity.TransientFailure); err != nil { + t.Fatalf("cc.WaitForConnectivityState(%v) returned error: %v", connectivity.TransientFailure, err) + } + + // After sending a valid update, the LB policy should report CONNECTING. + ccState := balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, + {Addresses: []resolver.Address{{Addr: "2.2.2.2:2"}}}, + }, + }, + } + if err := bal.UpdateClientConnState(ccState); err != nil { + t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) + } + + if err := cc.WaitForConnectivityState(ctx, connectivity.Connecting); err != nil { + t.Fatalf("cc.WaitForConnectivityState(%v) returned error: %v", connectivity.Connecting, err) + } +} + +// TestPickFirst_ResolverErrorinTF sends a resolver error to the balancer +// before when it's attempting to connect to a SubConn TRANSIENT_FAILURE. It +// verifies that the picker is updated and the SubConn is not closed. +func (s) TestPickFirst_ResolverErrorinTF(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc := testutils.NewBalancerClientConn(t) + bal := balancer.Get(Name).Build(cc, balancer.BuildOptions{}) + defer bal.Close() + + // After sending a valid update, the LB policy should report CONNECTING. + ccState := balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, + }, + }, + } + + if err := bal.UpdateClientConnState(ccState); err != nil { + t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) + } + + sc1 := <-cc.NewSubConnCh + if err := cc.WaitForConnectivityState(ctx, connectivity.Connecting); err != nil { + t.Fatalf("cc.WaitForConnectivityState(%v) returned error: %v", connectivity.Connecting, err) + } + + scErr := fmt.Errorf("test error: connection refused") + sc1.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: scErr, + }) + + if err := cc.WaitForPickerWithErr(ctx, scErr); err != nil { + t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", scErr, err) + } + + bal.ResolverError(errors.New("resolution failed: test error")) + if err := cc.WaitForErrPicker(ctx); err != nil { + t.Fatalf("cc.WaitForPickerWithErr() returned error: %v", err) + } + + select { + case <-time.After(defaultTestShortTimeout): + case sc := <-cc.ShutdownSubConnCh: + t.Fatalf("Unexpected SubConn shutdown: %v", sc) + } +} + +// TestAddressList_Iteration verifies the behaviour of the addressList while +// iterating through the entries. +func (s) TestAddressList_Iteration(t *testing.T) { + addrs := []resolver.Address{ + { + Addr: "192.168.1.1", + ServerName: "test-host-1", + Attributes: attributes.New("key-1", "val-1"), + BalancerAttributes: attributes.New("bal-key-1", "bal-val-1"), + }, + { + Addr: "192.168.1.2", + ServerName: "test-host-2", + Attributes: attributes.New("key-2", "val-2"), + BalancerAttributes: attributes.New("bal-key-2", "bal-val-2"), + }, + { + Addr: "192.168.1.3", + ServerName: "test-host-3", + Attributes: attributes.New("key-3", "val-3"), + BalancerAttributes: attributes.New("bal-key-3", "bal-val-3"), + }, + } + + addressList := addressList{} + addressList.updateAddrs(addrs) + + for i := 0; i < len(addrs); i++ { + if got, want := addressList.isValid(), true; got != want { + t.Fatalf("addressList.isValid() = %t, want %t", got, want) + } + if got, want := addressList.currentAddress(), addrs[i]; !want.Equal(got) { + t.Errorf("addressList.currentAddress() = %v, want %v", got, want) + } + if got, want := addressList.increment(), i+1 < len(addrs); got != want { + t.Fatalf("addressList.increment() = %t, want %t", got, want) + } + } + + if got, want := addressList.isValid(), false; got != want { + t.Fatalf("addressList.isValid() = %t, want %t", got, want) + } + + // increment an invalid address list. + if got, want := addressList.increment(), false; got != want { + t.Errorf("addressList.increment() = %t, want %t", got, want) + } + + if got, want := addressList.isValid(), false; got != want { + t.Errorf("addressList.isValid() = %t, want %t", got, want) + } + + addressList.reset() + for i := 0; i < len(addrs); i++ { + if got, want := addressList.isValid(), true; got != want { + t.Fatalf("addressList.isValid() = %t, want %t", got, want) + } + if got, want := addressList.currentAddress(), addrs[i]; !want.Equal(got) { + t.Errorf("addressList.currentAddress() = %v, want %v", got, want) + } + if got, want := addressList.increment(), i+1 < len(addrs); got != want { + t.Fatalf("addressList.increment() = %t, want %t", got, want) + } + } +} + +// TestAddressList_SeekTo verifies the behaviour of addressList.seekTo. +func (s) TestAddressList_SeekTo(t *testing.T) { + addrs := []resolver.Address{ + { + Addr: "192.168.1.1", + ServerName: "test-host-1", + Attributes: attributes.New("key-1", "val-1"), + BalancerAttributes: attributes.New("bal-key-1", "bal-val-1"), + }, + { + Addr: "192.168.1.2", + ServerName: "test-host-2", + Attributes: attributes.New("key-2", "val-2"), + BalancerAttributes: attributes.New("bal-key-2", "bal-val-2"), + }, + { + Addr: "192.168.1.3", + ServerName: "test-host-3", + Attributes: attributes.New("key-3", "val-3"), + BalancerAttributes: attributes.New("bal-key-3", "bal-val-3"), + }, + } + + addressList := addressList{} + addressList.updateAddrs(addrs) + + // Try finding an address in the list. + key := resolver.Address{ + Addr: "192.168.1.2", + ServerName: "test-host-2", + Attributes: attributes.New("key-2", "val-2"), + BalancerAttributes: attributes.New("ignored", "bal-val-2"), + } + + if got, want := addressList.seekTo(key), true; got != want { + t.Errorf("addressList.seekTo(%v) = %t, want %t", key, got, want) + } + + // It should be possible to increment once more now that the pointer has advanced. + if got, want := addressList.increment(), true; got != want { + t.Errorf("addressList.increment() = %t, want %t", got, want) + } + + if got, want := addressList.increment(), false; got != want { + t.Errorf("addressList.increment() = %t, want %t", got, want) + } + + // Seek to the key again, it is behind the pointer now. + if got, want := addressList.seekTo(key), true; got != want { + t.Errorf("addressList.seekTo(%v) = %t, want %t", key, got, want) + } + + // Seek to a key not in the list. + key = resolver.Address{ + Addr: "192.168.1.5", + ServerName: "test-host-5", + Attributes: attributes.New("key-5", "val-5"), + BalancerAttributes: attributes.New("ignored", "bal-val-5"), + } + + if got, want := addressList.seekTo(key), false; got != want { + t.Errorf("addressList.seekTo(%v) = %t, want %t", key, got, want) + } + + // It should be possible to increment once more since the pointer has not advanced. + if got, want := addressList.increment(), true; got != want { + t.Errorf("addressList.increment() = %t, want %t", got, want) + } + + if got, want := addressList.increment(), false; got != want { + t.Errorf("addressList.increment() = %t, want %t", got, want) + } +} + +// TestPickFirstLeaf_TFPickerUpdate sends TRANSIENT_FAILURE SubConn state updates +// for each SubConn managed by a pickfirst balancer. It verifies that the picker +// is updated with the expected frequency. +func (s) TestPickFirstLeaf_TFPickerUpdate(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc := testutils.NewBalancerClientConn(t) + bal := pickfirstBuilder{}.Build(cc, balancer.BuildOptions{}) + defer bal.Close() + ccState := balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, // duplicate, should be ignored. + {Addresses: []resolver.Address{{Addr: "2.2.2.2:2"}}}, + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, // duplicate, should be ignored. + }, + }, + } + if err := bal.UpdateClientConnState(ccState); err != nil { + t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) + } + + // PF should report TRANSIENT_FAILURE only once all the sunbconns have failed + // once. + tfErr := fmt.Errorf("test err: connection refused") + sc1 := <-cc.NewSubConnCh + select { + case <-sc1.ConnectCh: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for Connect() to be called on sc1.") + } + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: tfErr}) + + // Move the subconn back to IDLE, it should not be re-connected until the + // first pass is complete. + shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer shortCancel() + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + select { + case <-sc1.ConnectCh: + t.Fatal("Connect() unexpectedly called on sc1.") + case <-shortCtx.Done(): + } + + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", balancer.ErrNoSubConnAvailable, err) + } + + sc2 := <-cc.NewSubConnCh + select { + case <-sc2.ConnectCh: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for Connect() to be called on sc2.") + } + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: tfErr}) + + if err := cc.WaitForPickerWithErr(ctx, tfErr); err != nil { + t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", tfErr, err) + } + + // Subsequent TRANSIENT_FAILUREs should be reported only after seeing "# of SubConns" + // TRANSIENT_FAILUREs. + + // Both the subconns should be connected in parallel. + select { + case <-sc1.ConnectCh: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for Connect() to be called on sc1.") + } + + shortCtx, shortCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer shortCancel() + select { + case <-sc2.ConnectCh: + t.Fatal("Connect() called on sc2 before it completed backing-off.") + case <-shortCtx.Done(): + } + + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + select { + case <-sc2.ConnectCh: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for Connect() to be called on sc2.") + } + + newTfErr := fmt.Errorf("test err: unreachable") + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: newTfErr}) + select { + case <-time.After(defaultTestShortTimeout): + case p := <-cc.NewPickerCh: + sc, err := p.Pick(balancer.PickInfo{}) + t.Fatalf("Unexpected picker update: %v, %v", sc, err) + } + + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: newTfErr}) + if err := cc.WaitForPickerWithErr(ctx, newTfErr); err != nil { + t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", newTfErr, err) + } +} diff --git a/balancer/randomsubsetting/randomsubsetting.go b/balancer/randomsubsetting/randomsubsetting.go new file mode 100644 index 0000000..8cad8a3 --- /dev/null +++ b/balancer/randomsubsetting/randomsubsetting.go @@ -0,0 +1,201 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package randomsubsetting implements the random_subsetting LB policy specified +// here: https://github.com/grpc/proposal/blob/master/A68-random-subsetting.md +// +// To install the LB policy, import this package as: +// +// import _ "google.golang.org/grpc/balancer/randomsubsetting" +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +package randomsubsetting + +import ( + "cmp" + "encoding/json" + "fmt" + "math/rand/v2" + "slices" + + xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/balancer/gracefulswitch" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Name is the name of the random subsetting load balancer. +const Name = "random_subsetting_experimental" + +var ( + logger = grpclog.Component(Name) + randUint64 = rand.Uint64 +) + +func prefixLogger(p *subsettingBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[random-subsetting-lb %p] ", p)) +} + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &subsettingBalancer{ + Balancer: gracefulswitch.NewBalancer(cc, bOpts), + hashSeed: randUint64(), + hashDigest: xxhash.New(), + } + b.logger = prefixLogger(b) + b.logger.Infof("Created") + return b +} + +type lbConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + SubsetSize uint32 `json:"subsetSize,omitempty"` + ChildPolicy *iserviceconfig.BalancerConfig `json:"childPolicy,omitempty"` +} + +func (bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + lbCfg := &lbConfig{} + + // Ensure that the specified child policy is registered and validates its + // config, if present. + if err := json.Unmarshal(s, lbCfg); err != nil { + return nil, fmt.Errorf("randomsubsetting: json.Unmarshal failed for configuration: %s with error: %v", string(s), err) + } + if lbCfg.SubsetSize == 0 { + return nil, fmt.Errorf("randomsubsetting: SubsetSize must be greater than 0") + } + if lbCfg.ChildPolicy == nil { + return nil, fmt.Errorf("randomsubsetting: ChildPolicy must be specified") + } + + return lbCfg, nil +} + +func (bb) Name() string { + return Name +} + +type subsettingBalancer struct { + *gracefulswitch.Balancer + + logger *internalgrpclog.PrefixLogger + cfg *lbConfig + hashSeed uint64 + hashDigest *xxhash.Digest +} + +func (b *subsettingBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + lbCfg, ok := s.BalancerConfig.(*lbConfig) + if !ok { + b.logger.Warningf("Received config with unexpected type %T: %v", s.BalancerConfig, s.BalancerConfig) + return balancer.ErrBadResolverState + } + + // Build config for the gracefulswitch balancer. It is safe to ignore + // JSON marshaling errors here, since the config was already validated + // as part of ParseConfig(). + cfg := []map[string]any{{lbCfg.ChildPolicy.Name: lbCfg.ChildPolicy.Config}} + cfgJSON, _ := json.Marshal(cfg) + parsedCfg, err := gracefulswitch.ParseConfig(cfgJSON) + if err != nil { + return fmt.Errorf("randomsubsetting: error switching to child of type %q: %v", lbCfg.ChildPolicy.Name, err) + } + b.cfg = lbCfg + endpoints := resolver.State{ + Endpoints: b.calculateSubset(s.ResolverState.Endpoints), + ServiceConfig: s.ResolverState.ServiceConfig, + Attributes: s.ResolverState.Attributes, + } + + return b.Balancer.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: endpoints, + BalancerConfig: parsedCfg, + }) +} + +// calculateSubset implements the subsetting algorithm, as described in A68: +// https://github.com/grpc/proposal/blob/master/A68-random-subsetting.md#subsetting-algorithm +func (b *subsettingBalancer) calculateSubset(endpoints []resolver.Endpoint) []resolver.Endpoint { + // A helper struct to hold an endpoint and its hash. + type endpointWithHash struct { + hash uint64 + ep resolver.Endpoint + } + + // Empty endpoints cannot be hashed and are ignored by child policies. + // Remove them here so they do not count toward the requested subset size. + isEmpty := func(endpoint resolver.Endpoint) bool { + return len(endpoint.Addresses) == 0 + } + if slices.ContainsFunc(endpoints, isEmpty) { + endpoints = slices.DeleteFunc(slices.Clone(endpoints), isEmpty) + } + + subsetSize := b.cfg.SubsetSize + if len(endpoints) <= int(subsetSize) { + return endpoints + } + + hashedEndpoints := make([]endpointWithHash, len(endpoints)) + for i, endpoint := range endpoints { + // For every endpoint in the list, compute a hash with previously + // generated seed - A68. + // + // The xxhash package's Sum64() function does not allow setting a seed. + // This means that we need to reset the digest with the seed for every + // endpoint. Without this, an endpoint will not retain the same hash + // across resolver updates. + // + // Note that we only hash the first address of the endpoint, as per A68. + b.hashDigest.ResetWithSeed(b.hashSeed) + b.hashDigest.WriteString(endpoint.Addresses[0].String()) + hashedEndpoints[i] = endpointWithHash{ + hash: b.hashDigest.Sum64(), + ep: endpoint, + } + } + + slices.SortFunc(hashedEndpoints, func(a, b endpointWithHash) int { + // Note: This uses the standard library cmp package, not the + // github.com/google/go-cmp/cmp package. The latter is intended for + // testing purposes only. + return cmp.Compare(a.hash, b.hash) + }) + + // Convert back to resolver.Endpoints + endpointSubset := make([]resolver.Endpoint, subsetSize) + for i, endpoint := range hashedEndpoints[:subsetSize] { + endpointSubset[i] = endpoint.ep + } + + return endpointSubset +} diff --git a/balancer/randomsubsetting/randomsubsetting_test.go b/balancer/randomsubsetting/randomsubsetting_test.go new file mode 100644 index 0000000..ae5fbd0 --- /dev/null +++ b/balancer/randomsubsetting/randomsubsetting_test.go @@ -0,0 +1,281 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package randomsubsetting + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + xxhash "github.com/cespare/xxhash/v2" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + + _ "google.golang.org/grpc/balancer/roundrobin" // For round_robin LB policy in tests +) + +var defaultTestTimeout = 5 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestParseConfig(t *testing.T) { + parser := bb{} + tests := []struct { + name string + input string + wantCfg serviceconfig.LoadBalancingConfig + wantErr string + }{ + { + name: "invalid_json", + input: "{{invalidjson{{", + wantErr: "json.Unmarshal failed for configuration", + }, + { + name: "empty_config", + input: `{}`, + wantErr: "SubsetSize must be greater than 0", + }, + { + name: "subset_size_zero", + input: `{ "subsetSize": 0 }`, + wantErr: "SubsetSize must be greater than 0", + }, + { + name: "child_policy_missing", + input: `{ "subsetSize": 1 }`, + wantErr: "ChildPolicy must be specified", + }, + { + name: "child_policy_not_registered", + input: `{ "subsetSize": 1 , "childPolicy": [{"unregistered_lb": {}}] }`, + wantErr: "no supported policies found", + }, + { + name: "success", + input: `{ "subsetSize": 3, "childPolicy": [{"round_robin": {}}]}`, + wantCfg: &lbConfig{ + SubsetSize: 3, + ChildPolicy: &iserviceconfig.BalancerConfig{Name: "round_robin"}, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotCfg, gotErr := parser.ParseConfig(json.RawMessage(test.input)) + // Substring match makes this very tightly coupled to the + // internalserviceconfig.BalancerConfig error strings. However, it + // is important to distinguish the different types of error messages + // possible as the parser has a few defined buckets of ways it can + // error out. + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if test.wantErr != "" { + return + } + if diff := cmp.Diff(test.wantCfg, gotCfg); diff != "" { + t.Fatalf("ParseConfig(%s) got unexpected output, diff (-want +got):\n%s", test.input, diff) + } + }) + } +} + +func makeEndpoints(n int) []resolver.Endpoint { + endpoints := make([]resolver.Endpoint, n) + for i := 0; i < n; i++ { + endpoints[i] = resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: fmt.Sprintf("endpoint-%d", i)}}, + } + } + return endpoints +} + +func (s) TestCalculateSubset_Simple(t *testing.T) { + tests := []struct { + name string + endpoints []resolver.Endpoint + subsetSize uint32 + want []resolver.Endpoint + }{ + { + name: "NoEndpoints", + endpoints: []resolver.Endpoint{}, + subsetSize: 3, + want: []resolver.Endpoint{}, + }, + { + name: "SubsetSizeLargerThanNumberOfEndpoints", + endpoints: makeEndpoints(5), + subsetSize: 10, + want: makeEndpoints(5), + }, + { + name: "SubsetSizeEqualToNumberOfEndpoints", + endpoints: makeEndpoints(5), + subsetSize: 5, + want: makeEndpoints(5), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &subsettingBalancer{ + cfg: &lbConfig{SubsetSize: tt.subsetSize}, + hashSeed: 0, + hashDigest: xxhash.New(), + } + got := b.calculateSubset(tt.endpoints) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("calculateSubset() returned diff (-want +got):\n%s", diff) + } + }) + } +} + +func (s) TestCalculateSubset_IgnoresEndpointsWithoutAddresses(t *testing.T) { + emptyEndpoint := resolver.Endpoint{Addresses: []resolver.Address{}} + endpoints := append([]resolver.Endpoint{emptyEndpoint}, makeEndpoints(3)...) + b := &subsettingBalancer{ + cfg: &lbConfig{SubsetSize: 2}, + hashSeed: 0, + hashDigest: xxhash.New(), + } + + got := b.calculateSubset(endpoints) + if len(got) != 2 { + t.Fatalf("calculateSubset() returned %d endpoints, want 2", len(got)) + } + for _, endpoint := range got { + if len(endpoint.Addresses) == 0 { + t.Errorf("calculateSubset() returned an endpoint without addresses") + } + } +} + +func (s) TestCalculateSubset_EndpointsRetainHashValues(t *testing.T) { + endpoints := makeEndpoints(10) + const subsetSize = 5 + // The subset is deterministic based on the hash, so we can hardcode + // the expected output. + want := []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "endpoint-6"}}}, + {Addresses: []resolver.Address{{Addr: "endpoint-0"}}}, + {Addresses: []resolver.Address{{Addr: "endpoint-1"}}}, + {Addresses: []resolver.Address{{Addr: "endpoint-7"}}}, + {Addresses: []resolver.Address{{Addr: "endpoint-3"}}}, + } + + b := &subsettingBalancer{ + cfg: &lbConfig{SubsetSize: subsetSize}, + hashSeed: 0, + hashDigest: xxhash.New(), + } + for range 10 { + got := b.calculateSubset(endpoints) + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("calculateSubset() returned diff (-want +got):\n%s", diff) + } + } +} + +func (s) TestSubsettingBalancer_DeterministicSubset(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Register the stub balancer builder, which will be used as the child + // policy in the random_subsetting balancer. + updateCh := make(chan balancer.ClientConnState, 1) + stub.Register("stub-child-balancer", stub.BalancerFuncs{ + UpdateClientConnState: func(_ *stub.BalancerData, s balancer.ClientConnState) error { + select { + case <-ctx.Done(): + case updateCh <- s: + } + return nil + }, + }) + + // Create a random_subsetting balancer. + tcc := testutils.NewBalancerClientConn(t) + rsb := balancer.Get(Name).Build(tcc, balancer.BuildOptions{}) + defer rsb.Close() + + // Prepare the configuration and resolver state to be passed to the + // random_subsetting balancer. + endpoints := makeEndpoints(10) + state := balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: endpoints}, + BalancerConfig: &lbConfig{ + SubsetSize: 5, + ChildPolicy: &iserviceconfig.BalancerConfig{Name: "stub-child-balancer"}, + }, + } + + // Send the resolver state to the random_subsetting balancer and verify that + // the child policy receives the expected number of endpoints. + if err := rsb.UpdateClientConnState(state); err != nil { + t.Fatalf("UpdateClientConnState failed: %v", err) + } + + var wantEndpoints []resolver.Endpoint + select { + case s := <-updateCh: + if len(s.ResolverState.Endpoints) != 5 { + t.Fatalf("Child policy received %d endpoints, want 5", len(s.ResolverState.Endpoints)) + } + // Store the subset for the next comparison. + wantEndpoints = s.ResolverState.Endpoints + case <-ctx.Done(): + t.Fatal("Timed out waiting for child policy to receive an update") + } + + // Call UpdateClientConnState again with the same configuration. + if err := rsb.UpdateClientConnState(state); err != nil { + t.Fatalf("Second UpdateClientConnState failed: %v", err) + } + + // Verify that the child policy receives the same subset of endpoints. + select { + case s := <-updateCh: + if diff := cmp.Diff(wantEndpoints, s.ResolverState.Endpoints); diff != "" { + t.Fatalf("Child policy received a different subset of endpoints on second update, diff (-want +got):\n%s", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for second child policy update") + } +} diff --git a/balancer/ringhash/config.go b/balancer/ringhash/config.go new file mode 100644 index 0000000..65d9e8e --- /dev/null +++ b/balancer/ringhash/config.go @@ -0,0 +1,77 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "encoding/json" + "fmt" + "strings" + + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/metadata" + iringhash "google.golang.org/grpc/internal/ringhash" +) + +const ( + defaultMinSize = 1024 + defaultMaxSize = 4096 + ringHashSizeUpperBound = 8 * 1024 * 1024 // 8M +) + +func parseConfig(c json.RawMessage) (*iringhash.LBConfig, error) { + var cfg iringhash.LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + if cfg.MinRingSize > ringHashSizeUpperBound { + return nil, fmt.Errorf("min_ring_size value of %d is greater than max supported value %d for this field", cfg.MinRingSize, ringHashSizeUpperBound) + } + if cfg.MaxRingSize > ringHashSizeUpperBound { + return nil, fmt.Errorf("max_ring_size value of %d is greater than max supported value %d for this field", cfg.MaxRingSize, ringHashSizeUpperBound) + } + if cfg.MinRingSize == 0 { + cfg.MinRingSize = defaultMinSize + } + if cfg.MaxRingSize == 0 { + cfg.MaxRingSize = defaultMaxSize + } + if cfg.MinRingSize > cfg.MaxRingSize { + return nil, fmt.Errorf("min %v is greater than max %v", cfg.MinRingSize, cfg.MaxRingSize) + } + if cfg.MinRingSize > envconfig.RingHashCap { + cfg.MinRingSize = envconfig.RingHashCap + } + if cfg.MaxRingSize > envconfig.RingHashCap { + cfg.MaxRingSize = envconfig.RingHashCap + } + if !envconfig.RingHashSetRequestHashKey { + cfg.RequestHashHeader = "" + } + if cfg.RequestHashHeader != "" { + cfg.RequestHashHeader = strings.ToLower(cfg.RequestHashHeader) + // See rules in https://github.com/grpc/proposal/blob/master/A76-ring-hash-improvements.md#explicitly-setting-the-request-hash-key + if err := metadata.ValidateKey(cfg.RequestHashHeader); err != nil { + return nil, fmt.Errorf("invalid requestHashHeader %q: %v", cfg.RequestHashHeader, err) + } + if strings.HasSuffix(cfg.RequestHashHeader, "-bin") { + return nil, fmt.Errorf("invalid requestHashHeader %q: key must not end with \"-bin\"", cfg.RequestHashHeader) + } + } + return &cfg, nil +} diff --git a/balancer/ringhash/config_test.go b/balancer/ringhash/config_test.go new file mode 100644 index 0000000..2030148 --- /dev/null +++ b/balancer/ringhash/config_test.go @@ -0,0 +1,173 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "encoding/json" + "testing" + + "github.com/google/go-cmp/cmp" + + "google.golang.org/grpc/internal/envconfig" + iringhash "google.golang.org/grpc/internal/ringhash" + "google.golang.org/grpc/internal/testutils" +) + +func (s) TestParseConfig(t *testing.T) { + tests := []struct { + name string + js string + envConfigCap uint64 + requestHeaderEnvVar bool + want *iringhash.LBConfig + wantErr bool + }{ + { + name: "OK", + js: `{"minRingSize": 1, "maxRingSize": 2}`, + requestHeaderEnvVar: true, + want: &iringhash.LBConfig{MinRingSize: 1, MaxRingSize: 2}, + }, + { + name: "OK with default min", + js: `{"maxRingSize": 2000}`, + requestHeaderEnvVar: true, + want: &iringhash.LBConfig{MinRingSize: defaultMinSize, MaxRingSize: 2000}, + }, + { + name: "OK with default max", + js: `{"minRingSize": 2000}`, + requestHeaderEnvVar: true, + want: &iringhash.LBConfig{MinRingSize: 2000, MaxRingSize: defaultMaxSize}, + }, + { + name: "min greater than max", + js: `{"minRingSize": 10, "maxRingSize": 2}`, + requestHeaderEnvVar: true, + want: nil, + wantErr: true, + }, + { + name: "min greater than max greater than global limit", + js: `{"minRingSize": 6000, "maxRingSize": 5000}`, + requestHeaderEnvVar: true, + want: nil, + wantErr: true, + }, + { + name: "max greater than global limit", + js: `{"minRingSize": 1, "maxRingSize": 6000}`, + requestHeaderEnvVar: true, + want: &iringhash.LBConfig{MinRingSize: 1, MaxRingSize: 4096}, + }, + { + name: "min and max greater than global limit", + js: `{"minRingSize": 5000, "maxRingSize": 6000}`, + requestHeaderEnvVar: true, + want: &iringhash.LBConfig{MinRingSize: 4096, MaxRingSize: 4096}, + }, + { + name: "min and max less than raised global limit", + js: `{"minRingSize": 5000, "maxRingSize": 6000}`, + envConfigCap: 8000, + requestHeaderEnvVar: true, + want: &iringhash.LBConfig{MinRingSize: 5000, MaxRingSize: 6000}, + }, + { + name: "min and max greater than raised global limit", + js: `{"minRingSize": 10000, "maxRingSize": 10000}`, + envConfigCap: 8000, + requestHeaderEnvVar: true, + want: &iringhash.LBConfig{MinRingSize: 8000, MaxRingSize: 8000}, + }, + { + name: "min greater than upper bound", + js: `{"minRingSize": 8388610, "maxRingSize": 10}`, + requestHeaderEnvVar: true, + want: nil, + wantErr: true, + }, + { + name: "max greater than upper bound", + js: `{"minRingSize": 10, "maxRingSize": 8388610}`, + requestHeaderEnvVar: true, + want: nil, + wantErr: true, + }, + { + name: "request metadata key set", + js: `{"requestHashHeader": "x-foo"}`, + requestHeaderEnvVar: true, + want: &iringhash.LBConfig{ + MinRingSize: defaultMinSize, + MaxRingSize: defaultMaxSize, + RequestHashHeader: "x-foo", + }, + }, + { + name: "request metadata key set with uppercase letters", + js: `{"requestHashHeader": "x-FOO"}`, + requestHeaderEnvVar: true, + want: &iringhash.LBConfig{ + MinRingSize: defaultMinSize, + MaxRingSize: defaultMaxSize, + RequestHashHeader: "x-foo", + }, + }, + { + name: "invalid request hash header", + js: `{"requestHashHeader": "!invalid"}`, + requestHeaderEnvVar: true, + want: nil, + wantErr: true, + }, + { + name: "binary request hash header", + js: `{"requestHashHeader": "header-with-bin"}`, + requestHeaderEnvVar: true, + want: nil, + wantErr: true, + }, + { + name: "request hash header cleared when RingHashSetRequestHashKey env var is false", + js: `{"requestHashHeader": "x-foo"}`, + requestHeaderEnvVar: false, + want: &iringhash.LBConfig{ + MinRingSize: defaultMinSize, + MaxRingSize: defaultMaxSize, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.envConfigCap != 0 { + testutils.SetEnvConfig(t, &envconfig.RingHashCap, tt.envConfigCap) + } + testutils.SetEnvConfig(t, &envconfig.RingHashSetRequestHashKey, tt.requestHeaderEnvVar) + got, err := parseConfig(json.RawMessage(tt.js)) + if (err != nil) != tt.wantErr { + t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("parseConfig() got unexpected output, diff (-got +want): %v", diff) + } + }) + } +} diff --git a/balancer/ringhash/logging.go b/balancer/ringhash/logging.go new file mode 100644 index 0000000..64a1d46 --- /dev/null +++ b/balancer/ringhash/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[ring-hash-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *ringhashBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/balancer/ringhash/picker.go b/balancer/ringhash/picker.go new file mode 100644 index 0000000..3f76917 --- /dev/null +++ b/balancer/ringhash/picker.go @@ -0,0 +1,124 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "fmt" + "strings" + + xxhash "github.com/cespare/xxhash/v2" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + iringhash "google.golang.org/grpc/internal/ringhash" + "google.golang.org/grpc/metadata" +) + +type picker struct { + ring *ring + + // endpointStates is a cache of endpoint states. + // The ringhash balancer stores endpoint states in a `resolver.EndpointMap`, + // with access guarded by `ringhashBalancer.mu`. The `endpointStates` cache + // in the picker helps avoid locking the ringhash balancer's mutex when + // reading the latest state at RPC time. + endpointStates map[string]endpointState // endpointState.hashKey -> endpointState + + // requestHashHeader is the header key to look for the request hash. If it's + // empty, the request hash is expected to be set in the context via xDS. + // See gRFC A76. + requestHashHeader string + + // hasEndpointInConnectingState is true if any of the endpoints is in + // CONNECTING. + hasEndpointInConnectingState bool + + randUint64 func() uint64 +} + +func (p *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + usingRandomHash := false + var requestHash uint64 + if p.requestHashHeader == "" { + var ok bool + if requestHash, ok = iringhash.XDSRequestHash(info.Ctx); !ok { + return balancer.PickResult{}, fmt.Errorf("ringhash: expected xDS config selector to set the request hash") + } + } else { + md, ok := metadata.FromOutgoingContext(info.Ctx) + if !ok || len(md.Get(p.requestHashHeader)) == 0 { + requestHash = p.randUint64() + usingRandomHash = true + } else { + values := strings.Join(md.Get(p.requestHashHeader), ",") + requestHash = xxhash.Sum64String(values) + } + } + + e := p.ring.pick(requestHash) + ringSize := len(p.ring.items) + if !usingRandomHash { + // Per gRFC A61, because of sticky-TF with PickFirst's auto reconnect on TF, + // we ignore all TF subchannels and find the first ring entry in READY, + // CONNECTING or IDLE. If that entry is in IDLE, we need to initiate a + // connection. The idlePicker returned by the LazyLB or the new Pickfirst + // should do this automatically. + for i := 0; i < ringSize; i++ { + index := (e.idx + i) % ringSize + es := p.endpointState(p.ring.items[index]) + switch es.state.ConnectivityState { + case connectivity.Ready, connectivity.Connecting, connectivity.Idle: + return es.state.Picker.Pick(info) + case connectivity.TransientFailure: + default: + panic(fmt.Sprintf("Found child balancer in unknown state: %v", es.state.ConnectivityState)) + } + } + } else { + // If the picker has generated a random hash, it will walk the ring from + // this hash, and pick the first READY endpoint. If no endpoint is + // currently in CONNECTING state, it will trigger a connection attempt + // on at most one endpoint that is in IDLE state along the way. - A76 + requestedConnection := p.hasEndpointInConnectingState + for i := 0; i < ringSize; i++ { + index := (e.idx + i) % ringSize + es := p.endpointState(p.ring.items[index]) + if es.state.ConnectivityState == connectivity.Ready { + return es.state.Picker.Pick(info) + } + if !requestedConnection && es.state.ConnectivityState == connectivity.Idle { + requestedConnection = true + // If the SubChannel is in idle state, initiate a connection but + // continue to check other pickers to see if there is one in + // ready state. + es.exitIdle() + } + } + if requestedConnection { + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + } + } + + // All children are in transient failure. Return the first failure. + return p.endpointState(e).state.Picker.Pick(info) +} + +func (p *picker) endpointState(e *ringEntry) endpointState { + return p.endpointStates[e.hashKey] +} diff --git a/balancer/ringhash/picker_test.go b/balancer/ringhash/picker_test.go new file mode 100644 index 0000000..f5caa24 --- /dev/null +++ b/balancer/ringhash/picker_test.go @@ -0,0 +1,301 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "context" + "errors" + "fmt" + "math" + "testing" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + iringhash "google.golang.org/grpc/internal/ringhash" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/metadata" +) + +var ( + testSubConns []*testutils.TestSubConn + errPicker = errors.New("picker in TransientFailure") +) + +func init() { + for i := 0; i < 8; i++ { + testSubConns = append(testSubConns, testutils.NewTestSubConn(fmt.Sprint(i))) + } +} + +// fakeChildPicker is used to mock pickers from child pickfirst balancers. +type fakeChildPicker struct { + connectivityState connectivity.State + subConn *testutils.TestSubConn + tfError error +} + +func (p *fakeChildPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + switch p.connectivityState { + case connectivity.Idle: + p.subConn.Connect() + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + case connectivity.Connecting: + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + case connectivity.Ready: + return balancer.PickResult{SubConn: p.subConn}, nil + default: + return balancer.PickResult{}, p.tfError + } +} + +func testRingAndEndpointStates(states []connectivity.State) (*ring, map[string]endpointState) { + var items []*ringEntry + epStates := map[string]endpointState{} + for i, st := range states { + testSC := testSubConns[i] + items = append(items, &ringEntry{ + idx: i, + hash: math.MaxUint64 / uint64(len(states)) * uint64(i), + hashKey: testSC.String(), + }) + epState := endpointState{ + state: balancer.State{ + ConnectivityState: st, + Picker: &fakeChildPicker{ + connectivityState: st, + tfError: fmt.Errorf("%d: %w", i, errPicker), + subConn: testSC, + }, + }, + exitIdle: func() { testSC.Connect() }, + } + epStates[testSC.String()] = epState + } + return &ring{items: items}, epStates +} + +func (s) TestPickerPickFirstTwo(t *testing.T) { + tests := []struct { + name string + connectivityStates []connectivity.State + wantSC balancer.SubConn + wantErr error + wantSCToConnect balancer.SubConn + }{ + { + name: "picked is Ready", + connectivityStates: []connectivity.State{connectivity.Ready, connectivity.Idle}, + wantSC: testSubConns[0], + }, + { + name: "picked is connecting, queue", + connectivityStates: []connectivity.State{connectivity.Connecting, connectivity.Idle}, + wantErr: balancer.ErrNoSubConnAvailable, + }, + { + name: "picked is Idle, connect and queue", + connectivityStates: []connectivity.State{connectivity.Idle, connectivity.Idle}, + wantErr: balancer.ErrNoSubConnAvailable, + wantSCToConnect: testSubConns[0], + }, + { + name: "picked is TransientFailure, next is ready, return", + connectivityStates: []connectivity.State{connectivity.TransientFailure, connectivity.Ready}, + wantSC: testSubConns[1], + }, + { + name: "picked is TransientFailure, next is connecting, queue", + connectivityStates: []connectivity.State{connectivity.TransientFailure, connectivity.Connecting}, + wantErr: balancer.ErrNoSubConnAvailable, + }, + { + name: "picked is TransientFailure, next is Idle, connect and queue", + connectivityStates: []connectivity.State{connectivity.TransientFailure, connectivity.Idle}, + wantErr: balancer.ErrNoSubConnAvailable, + wantSCToConnect: testSubConns[1], + }, + { + name: "all are in TransientFailure, return picked failure", + connectivityStates: []connectivity.State{connectivity.TransientFailure, connectivity.TransientFailure}, + wantErr: errPicker, + }, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ring, epStates := testRingAndEndpointStates(tt.connectivityStates) + p := &picker{ + ring: ring, + endpointStates: epStates, + } + got, err := p.Pick(balancer.PickInfo{ + Ctx: iringhash.SetXDSRequestHash(ctx, 0), // always pick the first endpoint on the ring. + }) + if (err != nil || tt.wantErr != nil) && !errors.Is(err, tt.wantErr) { + t.Errorf("Pick() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got.SubConn != tt.wantSC { + t.Errorf("Pick() got = %v, want picked SubConn: %v", got, tt.wantSC) + } + if sc := tt.wantSCToConnect; sc != nil { + select { + case <-sc.(*testutils.TestSubConn).ConnectCh: + case <-time.After(defaultTestShortTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc) + } + } + }) + } +} + +func (s) TestPickerNoRequestHash(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + ring, epStates := testRingAndEndpointStates([]connectivity.State{connectivity.Ready}) + p := &picker{ + ring: ring, + endpointStates: epStates, + } + if _, err := p.Pick(balancer.PickInfo{Ctx: ctx}); err == nil { + t.Errorf("Pick() should have failed with no request hash") + } +} + +func (s) TestPickerRequestHashKey(t *testing.T) { + tests := []struct { + name string + headerValues []string + expectedPick int + }{ + { + name: "header not set", + expectedPick: 0, // Random hash set to 0, which is within (MaxUint64 / 3 * 2, 0] + }, + { + name: "header empty", + headerValues: []string{""}, + expectedPick: 0, // xxhash.Sum64String("value1,value2") is within (MaxUint64 / 3 * 2, 0] + }, + { + name: "header set to one value", + headerValues: []string{"some-value"}, + expectedPick: 1, // xxhash.Sum64String("some-value") is within (0, MaxUint64 / 3] + }, + { + name: "header set to multiple values", + headerValues: []string{"value1", "value2"}, + expectedPick: 2, // xxhash.Sum64String("value1,value2") is within (MaxUint64 / 3, MaxUint64 / 3 * 2] + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + ring, epStates := testRingAndEndpointStates( + []connectivity.State{ + connectivity.Ready, + connectivity.Ready, + connectivity.Ready, + }) + headerName := "some-header" + p := &picker{ + ring: ring, + endpointStates: epStates, + requestHashHeader: headerName, + randUint64: func() uint64 { return 0 }, + } + for _, v := range tt.headerValues { + ctx = metadata.AppendToOutgoingContext(ctx, headerName, v) + } + if res, err := p.Pick(balancer.PickInfo{Ctx: ctx}); err != nil { + t.Errorf("Pick() failed: %v", err) + } else if res.SubConn != testSubConns[tt.expectedPick] { + t.Errorf("Pick() got = %v, want SubConn: %v", res.SubConn, testSubConns[tt.expectedPick]) + } + }) + } +} + +func (s) TestPickerRandomHash(t *testing.T) { + tests := []struct { + name string + hash uint64 + connectivityStates []connectivity.State + wantSC balancer.SubConn + wantErr error + wantSCToConnect balancer.SubConn + hasEndpointInConnectingState bool + }{ + { + name: "header not set, picked is Ready", + connectivityStates: []connectivity.State{connectivity.Ready, connectivity.Idle}, + wantSC: testSubConns[0], + }, + { + name: "header not set, picked is Idle, another is Ready. Connect and pick Ready", + connectivityStates: []connectivity.State{connectivity.Idle, connectivity.Ready}, + wantSC: testSubConns[1], + wantSCToConnect: testSubConns[0], + }, + { + name: "header not set, picked is Idle, there is at least one Connecting", + connectivityStates: []connectivity.State{connectivity.Connecting, connectivity.Idle}, + wantErr: balancer.ErrNoSubConnAvailable, + hasEndpointInConnectingState: true, + }, + { + name: "header not set, all Idle or TransientFailure, connect", + connectivityStates: []connectivity.State{connectivity.TransientFailure, connectivity.Idle}, + wantErr: balancer.ErrNoSubConnAvailable, + wantSCToConnect: testSubConns[1], + }, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ring, epStates := testRingAndEndpointStates(tt.connectivityStates) + p := &picker{ + ring: ring, + endpointStates: epStates, + requestHashHeader: "some-header", + hasEndpointInConnectingState: tt.hasEndpointInConnectingState, + randUint64: func() uint64 { return 0 }, // always return the first endpoint on the ring. + } + if got, err := p.Pick(balancer.PickInfo{Ctx: ctx}); err != tt.wantErr { + t.Errorf("Pick() error = %v, wantErr %v", err, tt.wantErr) + return + } else if got.SubConn != tt.wantSC { + t.Errorf("Pick() got = %v, want picked SubConn: %v", got, tt.wantSC) + } + if sc := tt.wantSCToConnect; sc != nil { + select { + case <-sc.(*testutils.TestSubConn).ConnectCh: + case <-time.After(defaultTestShortTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc) + } + } + }) + } +} diff --git a/balancer/ringhash/ring.go b/balancer/ringhash/ring.go new file mode 100644 index 0000000..d2e0034 --- /dev/null +++ b/balancer/ringhash/ring.go @@ -0,0 +1,189 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "math" + "sort" + "strconv" + + xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/resolver" +) + +type ring struct { + items []*ringEntry +} + +type endpointInfo struct { + hashKey string + scaledWeight float64 + originalWeight uint32 +} + +type ringEntry struct { + idx int + hash uint64 + hashKey string + weight uint32 +} + +// newRing creates a ring from the endpoints stored in the EndpointMap. The ring +// size is limited by the passed in max/min. +// +// ring entries will be created for each endpoint, and endpoints with high +// weight (specified by the endpoint) may have multiple entries. +// +// For example, for endpoints with weights {a:3, b:3, c:4}, a generated ring of +// size 10 could be: +// - {idx:0 hash:3689675255460411075 b} +// - {idx:1 hash:4262906501694543955 c} +// - {idx:2 hash:5712155492001633497 c} +// - {idx:3 hash:8050519350657643659 b} +// - {idx:4 hash:8723022065838381142 b} +// - {idx:5 hash:11532782514799973195 a} +// - {idx:6 hash:13157034721563383607 c} +// - {idx:7 hash:14468677667651225770 c} +// - {idx:8 hash:17336016884672388720 a} +// - {idx:9 hash:18151002094784932496 a} +// +// To pick from a ring, a binary search will be done for the given target hash, +// and first item with hash >= given hash will be returned. +// +// Must be called with a non-empty endpoints map. +func newRing(endpoints *resolver.EndpointMap[*endpointState], minRingSize, maxRingSize uint64, logger *grpclog.PrefixLogger) *ring { + if logger.V(2) { + logger.Infof("newRing: number of endpoints is %d, minRingSize is %d, maxRingSize is %d", endpoints.Len(), minRingSize, maxRingSize) + } + + // https://github.com/envoyproxy/envoy/blob/765c970f06a4c962961a0e03a467e165b276d50f/source/common/upstream/ring_hash_lb.cc#L114 + normalizedWeights, minWeight := normalizeWeights(endpoints) + if logger.V(2) { + logger.Infof("newRing: normalized endpoint weights is %v", normalizedWeights) + } + + // Normalized weights for {3,3,4} is {0.3,0.3,0.4}. + + // Scale up the size of the ring such that the least-weighted host gets a + // whole number of hashes on the ring. + // + // Note that size is limited by the input max/min. + scale := math.Min(math.Ceil(minWeight*float64(minRingSize))/minWeight, float64(maxRingSize)) + ringSize := math.Ceil(scale) + items := make([]*ringEntry, 0, int(ringSize)) + if logger.V(2) { + logger.Infof("newRing: creating new ring of size %v", ringSize) + } + + // For each entry, scale*weight nodes are generated in the ring. + // + // Not all of these are whole numbers. E.g. for weights {a:3,b:3,c:4}, if + // ring size is 7, scale is 6.66. The numbers of nodes will be + // {a,a,b,b,c,c,c}. + // + // A hash is generated for each item, and later the results will be sorted + // based on the hash. + var currentHashes, targetHashes float64 + for _, epInfo := range normalizedWeights { + targetHashes += scale * epInfo.scaledWeight + // This index ensures that ring entries corresponding to the same + // endpoint hash to different values. And since this index is + // per-endpoint, these entries hash to the same value across address + // updates. + idx := 0 + for currentHashes < targetHashes { + h := xxhash.Sum64String(epInfo.hashKey + "_" + strconv.Itoa(idx)) + items = append(items, &ringEntry{hash: h, hashKey: epInfo.hashKey, weight: epInfo.originalWeight}) + idx++ + currentHashes++ + } + } + + // Sort items based on hash, to prepare for binary search. + sort.Slice(items, func(i, j int) bool { return items[i].hash < items[j].hash }) + for i, ii := range items { + ii.idx = i + } + return &ring{items: items} +} + +// normalizeWeights calculates the normalized weights for each endpoint in the +// given endpoints map. It returns a slice of endpointWithState structs, where +// each struct contains the picker for an endpoint and its corresponding weight. +// The function also returns the minimum weight among all endpoints. +// +// The normalized weight of each endpoint is calculated by dividing its weight +// attribute by the sum of all endpoint weights. If the weight attribute is not +// found on the endpoint, a default weight of 1 is used. +// +// The endpoints are sorted in ascending order to ensure consistent results. +// +// Must be called with a non-empty endpoints map. +func normalizeWeights(endpoints *resolver.EndpointMap[*endpointState]) ([]endpointInfo, float64) { + var weightSum uint32 + // Since attributes are explicitly ignored in the EndpointMap key, we need + // to iterate over the values to get the weights. + endpointVals := endpoints.Values() + for _, epState := range endpointVals { + weightSum += epState.weight + } + ret := make([]endpointInfo, 0, endpoints.Len()) + min := 1.0 + for _, epState := range endpointVals { + // (*endpointState).weight is set to 1 if the weight attribute is not + // found on the endpoint. And since this function is guaranteed to be + // called with a non-empty endpoints map, weightSum is guaranteed to be + // non-zero. So, we need not worry about divide by zero error here. + nw := float64(epState.weight) / float64(weightSum) + ret = append(ret, endpointInfo{ + hashKey: epState.hashKey, + scaledWeight: nw, + originalWeight: epState.weight, + }) + min = math.Min(min, nw) + } + // Sort the endpoints to return consistent results. + // + // Note: this might not be necessary, but this makes sure the ring is + // consistent as long as the endpoints are the same, for example, in cases + // where an endpoint is added and then removed, the RPCs will still pick the + // same old endpoint. + sort.Slice(ret, func(i, j int) bool { + return ret[i].hashKey < ret[j].hashKey + }) + return ret, min +} + +// pick does a binary search. It returns the item with smallest index i that +// r.items[i].hash >= h. +func (r *ring) pick(h uint64) *ringEntry { + i := sort.Search(len(r.items), func(i int) bool { return r.items[i].hash >= h }) + if i == len(r.items) { + // If not found, and h is greater than the largest hash, return the + // first item. + i = 0 + } + return r.items[i] +} + +// next returns the next entry. +func (r *ring) next(e *ringEntry) *ringEntry { + return r.items[(e.idx+1)%len(r.items)] +} diff --git a/balancer/ringhash/ring_test.go b/balancer/ringhash/ring_test.go new file mode 100644 index 0000000..3b6d2c9 --- /dev/null +++ b/balancer/ringhash/ring_test.go @@ -0,0 +1,112 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "fmt" + "math" + "testing" + + xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/experimental/balancer/weight" + "google.golang.org/grpc/resolver" +) + +var testEndpoints []resolver.Endpoint +var testEndpointStateMap *resolver.EndpointMap[*endpointState] + +func init() { + testEndpoints = []resolver.Endpoint{ + testEndpoint("a", 3), + testEndpoint("b", 3), + testEndpoint("c", 4), + } + testEndpointStateMap = resolver.NewEndpointMap[*endpointState]() + testEndpointStateMap.Set(testEndpoints[0], &endpointState{hashKey: "a", weight: 3}) + testEndpointStateMap.Set(testEndpoints[1], &endpointState{hashKey: "b", weight: 3}) + testEndpointStateMap.Set(testEndpoints[2], &endpointState{hashKey: "c", weight: 4}) +} + +func testEndpoint(addr string, endpointWeight uint32) resolver.Endpoint { + ep := resolver.Endpoint{Addresses: []resolver.Address{{Addr: addr}}} + return weight.Set(ep, weight.EndpointInfo{Weight: endpointWeight}) +} + +func (s) TestRingNew(t *testing.T) { + var totalWeight float64 = 10 + for _, min := range []uint64{3, 4, 6, 8} { + for _, max := range []uint64{20, 8} { + t.Run(fmt.Sprintf("size-min-%v-max-%v", min, max), func(t *testing.T) { + r := newRing(testEndpointStateMap, min, max, nil) + totalCount := len(r.items) + if totalCount < int(min) || totalCount > int(max) { + t.Fatalf("unexpected size %v, want min %v, max %v", totalCount, min, max) + } + for _, e := range testEndpoints { + var count int + for _, ii := range r.items { + if ii.hashKey == hashKey(e) { + count++ + } + } + got := float64(count) / float64(totalCount) + want := float64(getWeightAttribute(e)) / totalWeight + if !equalApproximately(got, want) { + t.Fatalf("unexpected item weight in ring: %v != %v", got, want) + } + } + }) + } + } +} + +func equalApproximately(x, y float64) bool { + delta := math.Abs(x - y) + mean := math.Abs(x+y) / 2.0 + return delta/mean < 0.25 +} + +func (s) TestRingPick(t *testing.T) { + r := newRing(testEndpointStateMap, 10, 20, nil) + for _, h := range []uint64{xxhash.Sum64String("1"), xxhash.Sum64String("2"), xxhash.Sum64String("3"), xxhash.Sum64String("4")} { + t.Run(fmt.Sprintf("picking-hash-%v", h), func(t *testing.T) { + e := r.pick(h) + var low uint64 + if e.idx > 0 { + low = r.items[e.idx-1].hash + } + high := e.hash + // h should be in [low, high). + if h < low || h >= high { + t.Fatalf("unexpected item picked, hash: %v, low: %v, high: %v", h, low, high) + } + }) + } +} + +func (s) TestRingNext(t *testing.T) { + r := newRing(testEndpointStateMap, 10, 20, nil) + + for _, e := range r.items { + ne := r.next(e) + if ne.idx != (e.idx+1)%len(r.items) { + t.Fatalf("next(%+v) returned unexpected %+v", e, ne) + } + } +} diff --git a/balancer/ringhash/ringhash.go b/balancer/ringhash/ringhash.go new file mode 100644 index 0000000..384dedb --- /dev/null +++ b/balancer/ringhash/ringhash.go @@ -0,0 +1,410 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package ringhash implements the ringhash balancer. See the following +// gRFCs for details: +// - https://github.com/grpc/proposal/blob/master/A42-xds-ring-hash-lb-policy.md +// - https://github.com/grpc/proposal/blob/master/A61-IPv4-IPv6-dualstack-backends.md#ring-hash +// - https://github.com/grpc/proposal/blob/master/A76-ring-hash-improvements.md +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +package ringhash + +import ( + "encoding/json" + "errors" + "fmt" + "math/rand/v2" + "sort" + "sync" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/balancer/endpointsharding" + "google.golang.org/grpc/balancer/lazy" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/experimental/balancer/weight" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/pretty" + iringhash "google.golang.org/grpc/internal/ringhash" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/ringhash" + "google.golang.org/grpc/serviceconfig" +) + +// Name is the name of the ring_hash balancer. +const Name = "ring_hash_experimental" + +func lazyPickFirstBuilder(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + return lazy.NewBalancer(cc, opts, balancer.Get(pickfirst.Name).Build) +} + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + b := &ringhashBalancer{ + ClientConn: cc, + endpointStates: resolver.NewEndpointMap[*endpointState](), + } + esOpts := endpointsharding.Options{DisableAutoReconnect: true} + b.child = endpointsharding.NewBalancer(b, opts, lazyPickFirstBuilder, esOpts) + b.logger = prefixLogger(b) + b.logger.Infof("Created") + return b +} + +func (bb) Name() string { + return Name +} + +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(c) +} + +type ringhashBalancer struct { + // The following fields are initialized at build time and read-only after + // that and therefore do not need to be guarded by a mutex. + + // ClientConn is embedded to intercept UpdateState calls from the child + // endpointsharding balancer. + balancer.ClientConn + logger *grpclog.PrefixLogger + child balancer.Balancer + + mu sync.Mutex + config *iringhash.LBConfig + inhibitChildUpdates bool + shouldRegenerateRing bool + endpointStates *resolver.EndpointMap[*endpointState] + + // ring is always in sync with endpoints. When endpoints change, a new ring + // is generated. Note that address weights updates also regenerates the + // ring. + ring *ring +} + +// hashKey returns the hash key to use for an endpoint. Per gRFC A61, each entry +// in the ring is a hash of the endpoint's hash key concatenated with a +// per-entry unique suffix. +func hashKey(endpoint resolver.Endpoint) string { + if hk := ringhash.HashKey(endpoint); hk != "" { + return hk + } + // If no hash key is set, use the endpoint's first address as the hash key. + // This is the default behavior when no hash key is set. + return endpoint.Addresses[0].Addr +} + +// UpdateState intercepts child balancer state updates. It updates the +// per-endpoint state stored in the ring, and also the aggregated state based on +// the child picker. It also reconciles the endpoint list. It sets +// `b.shouldRegenerateRing` to true if the new endpoint list is different from +// the previous, i.e. any of the following is true: +// - an endpoint was added +// - an endpoint was removed +// - an endpoint's weight was updated +// - the first addresses of the endpoint has changed +func (b *ringhashBalancer) UpdateState(state balancer.State) { + b.mu.Lock() + defer b.mu.Unlock() + childStates := endpointsharding.ChildStatesFromPicker(state.Picker) + // endpointsSet is the set converted from endpoints, used for quick lookup. + endpointsSet := resolver.NewEndpointMap[bool]() + + for _, childState := range childStates { + endpoint := childState.Endpoint + endpointsSet.Set(endpoint, true) + newWeight := getWeightAttribute(endpoint) + hk := hashKey(endpoint) + es, ok := b.endpointStates.Get(endpoint) + if !ok { + es := &endpointState{ + hashKey: hk, + weight: newWeight, + state: childState.State, + exitIdle: childState.ExitIdle, + } + b.endpointStates.Set(endpoint, es) + b.shouldRegenerateRing = true + } else { + // We have seen this endpoint before and created a `endpointState` + // object for it. If the weight or the hash key of the endpoint has + // changed, update the endpoint state map with the new weight or + // hash key. This will be used when a new ring is created. + if oldWeight := es.weight; oldWeight != newWeight { + b.shouldRegenerateRing = true + es.weight = newWeight + } + if es.hashKey != hk { + b.shouldRegenerateRing = true + es.hashKey = hk + } + es.state = childState.State + } + } + + for endpoint := range b.endpointStates.All() { + if _, ok := endpointsSet.Get(endpoint); ok { + continue + } + // endpoint was removed by resolver. + b.endpointStates.Delete(endpoint) + b.shouldRegenerateRing = true + } + + b.updatePickerLocked() +} + +func (b *ringhashBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { + if b.logger.V(2) { + b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(ccs.BalancerConfig)) + } + + newConfig, ok := ccs.BalancerConfig.(*iringhash.LBConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", ccs.BalancerConfig) + } + + b.mu.Lock() + b.inhibitChildUpdates = true + b.mu.Unlock() + + defer func() { + b.mu.Lock() + b.inhibitChildUpdates = false + b.updatePickerLocked() + b.mu.Unlock() + }() + + if err := b.child.UpdateClientConnState(balancer.ClientConnState{ + // Make pickfirst children use health listeners for outlier detection + // and health checking to work. + ResolverState: pickfirst.EnableHealthListener(ccs.ResolverState), + }); err != nil { + return err + } + + b.mu.Lock() + // Ring updates can happen due to the following: + // 1. Addition or deletion of endpoints: The synchronous picker update from + // the child endpointsharding balancer would contain the list of updated + // endpoints. Updates triggered by the child after handling the + // `UpdateClientConnState` call will not change the endpoint list. + // 2. Change in the `LoadBalancerConfig`: Ring config such as max/min ring + // size. + // To avoid extra ring updates, a boolean is used to track the need for a + // ring update and the update is done only once at the end. + // + // If the ring configuration has changed, we need to regenerate the ring + // while sending a new picker. + if b.config == nil || b.config.MinRingSize != newConfig.MinRingSize || b.config.MaxRingSize != newConfig.MaxRingSize { + b.shouldRegenerateRing = true + } + b.config = newConfig + b.mu.Unlock() + return nil +} + +func (b *ringhashBalancer) ResolverError(err error) { + b.child.ResolverError(err) +} + +func (b *ringhashBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *ringhashBalancer) updatePickerLocked() { + state := b.aggregatedStateLocked() + // Start connecting to new endpoints if necessary. + if state == connectivity.Connecting || state == connectivity.TransientFailure { + // When overall state is TransientFailure, we need to make sure at least + // one endpoint is attempting to connect, otherwise this balancer may + // never get picks if the parent is priority. + // + // Because we report Connecting as the overall state when only one + // endpoint is in TransientFailure, we do the same check for Connecting + // here. + // + // Note that this check also covers deleting endpoints. E.g. if the + // endpoint attempting to connect is deleted, and the overall state is + // TF. Since there must be at least one endpoint attempting to connect, + // we need to trigger one. + // + // After calling `ExitIdle` on a child balancer, the child will send a + // picker update asynchronously. A race condition may occur if another + // picker update from endpointsharding arrives before the child's + // picker update. The received picker may trigger a re-execution of the + // loop below to find an idle child. Since map iteration order is + // non-deterministic, the list of `endpointState`s must be sorted to + // ensure `ExitIdle` is called on the same child, preventing unnecessary + // connections. + endpointStates := make([]*endpointState, 0, b.endpointStates.Len()) + for _, s := range b.endpointStates.All() { + endpointStates = append(endpointStates, s) + } + sort.Slice(endpointStates, func(i, j int) bool { + return endpointStates[i].hashKey < endpointStates[j].hashKey + }) + + // Store the function to ExitIdle on the first IDLE endpoint. + var exitIdle func() + for _, es := range endpointStates { + connState := es.state.ConnectivityState + if connState == connectivity.Connecting { + exitIdle = nil + break + } + if exitIdle == nil && connState == connectivity.Idle { + exitIdle = es.exitIdle + } + } + if exitIdle != nil { + exitIdle() + } + } + + if b.inhibitChildUpdates { + return + } + + // Update the channel. + if b.endpointStates.Len() > 0 && b.shouldRegenerateRing { + // with a non-empty list of endpoints. + b.ring = newRing(b.endpointStates, b.config.MinRingSize, b.config.MaxRingSize, b.logger) + } + b.shouldRegenerateRing = false + var newPicker balancer.Picker + if b.endpointStates.Len() == 0 { + newPicker = base.NewErrPicker(errors.New("produced zero addresses")) + } else { + newPicker = b.newPickerLocked() + } + b.ClientConn.UpdateState(balancer.State{ + ConnectivityState: state, + Picker: newPicker, + }) +} + +func (b *ringhashBalancer) Close() { + b.logger.Infof("Shutdown") + b.child.Close() +} + +func (b *ringhashBalancer) ExitIdle() { + // ExitIdle implementation is a no-op because connections are either + // triggers from picks or from child balancer state changes. +} + +// newPickerLocked generates a picker. The picker copies the endpoint states +// over to avoid locking the mutex at RPC time. The picker should be +// re-generated every time an endpoint state is updated. +func (b *ringhashBalancer) newPickerLocked() *picker { + states := make(map[string]endpointState) + hasEndpointConnecting := false + for _, epState := range b.endpointStates.All() { + // Copy the endpoint state to avoid races, since ring hash + // mutates the state, weight and hash key in place. + states[epState.hashKey] = *epState + if epState.state.ConnectivityState == connectivity.Connecting { + hasEndpointConnecting = true + } + } + return &picker{ + ring: b.ring, + endpointStates: states, + requestHashHeader: b.config.RequestHashHeader, + hasEndpointInConnectingState: hasEndpointConnecting, + randUint64: rand.Uint64, + } +} + +// aggregatedStateLocked returns the aggregated child balancers state +// based on the following rules. +// - If there is at least one endpoint in READY state, report READY. +// - If there are 2 or more endpoints in TRANSIENT_FAILURE state, report +// TRANSIENT_FAILURE. +// - If there is at least one endpoint in CONNECTING state, report CONNECTING. +// - If there is one endpoint in TRANSIENT_FAILURE and there is more than one +// endpoint, report state CONNECTING. +// - If there is at least one endpoint in Idle state, report Idle. +// - Otherwise, report TRANSIENT_FAILURE. +// +// Note that if there are 1 connecting, 2 transient failure, the overall state +// is transient failure. This is because the second transient failure is a +// fallback of the first failing endpoint, and we want to report transient +// failure to failover to the lower priority. +func (b *ringhashBalancer) aggregatedStateLocked() connectivity.State { + var nums [5]int + for _, es := range b.endpointStates.All() { + nums[es.state.ConnectivityState]++ + } + + if nums[connectivity.Ready] > 0 { + return connectivity.Ready + } + if nums[connectivity.TransientFailure] > 1 { + return connectivity.TransientFailure + } + if nums[connectivity.Connecting] > 0 { + return connectivity.Connecting + } + if nums[connectivity.TransientFailure] == 1 && b.endpointStates.Len() > 1 { + return connectivity.Connecting + } + if nums[connectivity.Idle] > 0 { + return connectivity.Idle + } + return connectivity.TransientFailure +} + +// getWeightAttribute is a convenience function which returns the value of the +// weight endpoint Attribute. +// +// When used in the xDS context, the weight attribute is guaranteed to be +// non-zero. But, when used in a non-xDS context, the weight attribute could be +// unset. A Default of 1 is used in the latter case. +func getWeightAttribute(e resolver.Endpoint) uint32 { + w := weight.FromEndpoint(e).Weight + if w == 0 { + return 1 + } + return w +} + +type endpointState struct { + // hashKey is the hash key of the endpoint. Per gRFC A61, each entry in the + // ring is an endpoint, positioned based on the hash of the endpoint's first + // address by default. Per gRFC A76, the hash key of an endpoint may be + // overridden, for example based on EDS endpoint metadata. + hashKey string + weight uint32 + exitIdle func() + + // state is updated by the balancer while receiving resolver updates from + // the channel and picker updates from its children. Access to it is guarded + // by ringhashBalancer.mu. + state balancer.State +} diff --git a/balancer/ringhash/ringhash_e2e_test.go b/balancer/ringhash/ringhash_e2e_test.go new file mode 100644 index 0000000..915db5a --- /dev/null +++ b/balancer/ringhash/ringhash_e2e_test.go @@ -0,0 +1,2944 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash_test + +import ( + "context" + "errors" + "fmt" + "math" + rand "math/rand/v2" + "net" + "slices" + "strconv" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + iringhash "google.golang.org/grpc/internal/ringhash" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/protobuf/types/known/wrapperspb" + + _ "google.golang.org/grpc/xds" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + + errorTolerance = .05 // For tests that rely on statistical significance. + + virtualHostName = "test.server" + + // minRingSize is the minimum ring size to use when testing randomly a + // backend for each request. It lowers the skew that may occur from + // an imbalanced ring. + minRingSize = 10000 +) + +// fastConnectParams disables connection attempts backoffs and lowers delays. +// This speeds up tests that rely on subchannel to move to transient failure. +var fastConnectParams = grpc.ConnectParams{ + Backoff: backoff.Config{ + BaseDelay: 10 * time.Millisecond, + }, + MinConnectTimeout: 100 * time.Millisecond, +} + +// Tests the case where the ring contains a single subConn, and verifies that +// when the server goes down, the LB policy on the client automatically +// reconnects until the subChannel moves out of TRANSIENT_FAILURE. +func (s) TestRingHash_ReconnectToMoveOutOfTransientFailure(t *testing.T) { + // Create a restartable listener to simulate server being down. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + srv := stubserver.StartTestService(t, &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + }) + defer srv.Stop() + + // Create a clientConn with a manual resolver (which is used to push the + // address of the test backend), and a default service config pointing to + // the use of the ring_hash_experimental LB policy. + const ringHashServiceConfig = `{"loadBalancingConfig": [{"ring_hash_experimental":{}}]}` + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(ringHashServiceConfig), + grpc.WithConnectParams(fastConnectParams), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + + // Push the address of the test backend through the manual resolver. + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + ctx = iringhash.SetXDSRequestHash(ctx, 0) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // Stopping the server listener will close the transport on the client, + // which will lead to the channel eventually moving to IDLE. The ring_hash + // LB policy is not expected to reconnect by itself at this point. + lis.Stop() + + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Make an RPC to get the ring_hash LB policy to reconnect and thereby move + // to TRANSIENT_FAILURE upon connection failure. + client.EmptyCall(ctx, &testpb.Empty{}) + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // An RPC at this point is expected to fail. + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Fatal("EmptyCall RPC succeeded when the channel is in TRANSIENT_FAILURE") + } + + // Restart the server listener. The ring_hash LB policy is expected to + // attempt to reconnect on its own and come out of TRANSIENT_FAILURE, even + // without an RPC attempt. + lis.Restart() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // An RPC at this point is expected to succeed. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} + +// startTestServiceBackends starts num stub servers. It returns the list of +// stubservers. Servers are closed when the test is stopped. +func startTestServiceBackends(t *testing.T, num int) []*stubserver.StubServer { + t.Helper() + + servers := make([]*stubserver.StubServer, 0, num) + for i := 0; i < num; i++ { + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + servers = append(servers, server) + } + return servers +} + +// backendAddrs returns a list of address strings for the given stubservers. +func backendAddrs(servers []*stubserver.StubServer) []string { + addrs := make([]string, 0, len(servers)) + for _, s := range servers { + addrs = append(addrs, s.Address) + } + return addrs +} + +// backendOptions returns a slice of e2e.BackendOptions for the given server +// addresses. +func backendOptions(t *testing.T, serverAddrs []string) []e2e.BackendOptions { + t.Helper() + backendAddrs := [][]string{} + for _, addr := range serverAddrs { + backendAddrs = append(backendAddrs, []string{addr}) + } + return backendOptionsForEndpointsWithMultipleAddrs(t, backendAddrs) +} + +// backendOptions returns a slice of e2e.BackendOptions for the given server +// addresses. Each endpoint can have multiple addresses. +func backendOptionsForEndpointsWithMultipleAddrs(t *testing.T, backendAddrs [][]string) []e2e.BackendOptions { + t.Helper() + + var backendOpts []e2e.BackendOptions + for _, backend := range backendAddrs { + ports := []uint32{} + for _, addr := range backend { + ports = append(ports, testutils.ParsePort(t, addr)) + } + backendOpts = append(backendOpts, e2e.BackendOptions{Ports: ports}) + } + return backendOpts +} + +// channelIDHashRoute returns a RouteConfiguration with a hash policy that +// hashes based on the channel ID. +func channelIDHashRoute(routeName, virtualHostDomain, clusterName string) *v3routepb.RouteConfiguration { + route := e2e.DefaultRouteConfig(routeName, virtualHostDomain, clusterName) + hashPolicy := v3routepb.RouteAction_HashPolicy{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_FilterState_{ + FilterState: &v3routepb.RouteAction_HashPolicy_FilterState{ + Key: "io.grpc.channel_id", + }, + }, + } + action := route.VirtualHosts[0].Routes[0].Action.(*v3routepb.Route_Route) + action.Route.HashPolicy = []*v3routepb.RouteAction_HashPolicy{&hashPolicy} + return route +} + +// checkRPCSendOK sends num RPCs to the client. It returns a map of backend +// addresses as keys and number of RPCs sent to this address as value. Abort the +// test if any RPC fails. +func checkRPCSendOK(ctx context.Context, t *testing.T, client testgrpc.TestServiceClient, num int) map[string]int { + t.Helper() + + backendCount := make(map[string]int) + for i := 0; i < num; i++ { + var remote peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&remote)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + backendCount[remote.Addr.String()]++ + } + return backendCount +} + +// makeUnreachableBackends returns a slice of addresses of backends that close +// connections as soon as they are established. Useful to simulate servers that +// are unreachable. +func makeUnreachableBackends(t *testing.T, num int) []string { + t.Helper() + + addrs := make([]string, 0, num) + for i := 0; i < num; i++ { + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + addrs = append(addrs, lis.Addr().String()) + + // It is enough to fail the first connection attempt to put the subchannel + // in TRANSIENT_FAILURE. + go func() { lis.Accept() }() + + // We don't close these listeners here, to make sure ports are + // not reused across them, and across tests. + lis.Stop() + t.Cleanup(func() { lis.Close() }) + } + return addrs +} + +// setupManagementServerAndResolver sets up an xDS management server, creates +// bootstrap configuration pointing to that server and creates an xDS resolver +// using that configuration. +// +// Registers a cleanup function on t to stop the management server. +// +// Returns the management server, node ID and the xDS resolver builder. +func setupManagementServerAndResolver(t *testing.T) (*e2e.ManagementServer, string, resolver.Builder) { + t.Helper() + + // Start an xDS management server. + xdsServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, xdsServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + return xdsServer, nodeID, r +} + +// xdsUpdateOpts returns an e2e.UpdateOptions for the given node ID with the given xDS resources. +func xdsUpdateOpts(nodeID string, endpoints *v3endpointpb.ClusterLoadAssignment, cluster *v3clusterpb.Cluster, route *v3routepb.RouteConfiguration, listener *v3listenerpb.Listener) e2e.UpdateOptions { + return e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{endpoints}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Routes: []*v3routepb.RouteConfiguration{route}, + Listeners: []*v3listenerpb.Listener{listener}, + } +} + +// Tests that when an aggregate cluster is configured with ring hash policy, and +// the first cluster is in transient failure, all RPCs are sent to the second +// cluster using the ring hash policy. +func (s) TestRingHash_AggregateClusterFallBackFromRingHashAtStartup(t *testing.T) { + addrs := backendAddrs(startTestServiceBackends(t, 2)) + + const primaryClusterName = "new_cluster_1" + const primaryServiceName = "new_eds_service_1" + const secondaryClusterName = "new_cluster_2" + const secondaryServiceName = "new_eds_service_2" + const clusterName = "aggregate_cluster" + + ep1 := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: primaryServiceName, + Localities: []e2e.LocalityOptions{{ + Name: "locality0", + Weight: 1, + Backends: backendOptions(t, makeUnreachableBackends(t, 2)), + }}, + }) + ep2 := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: secondaryServiceName, + Localities: []e2e.LocalityOptions{{ + Name: "locality0", + Weight: 1, + Backends: backendOptions(t, addrs), + }}, + }) + primaryCluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: primaryClusterName, + ServiceName: primaryServiceName, + }) + secondaryCluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: secondaryClusterName, + ServiceName: secondaryServiceName, + }) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + Type: e2e.ClusterTypeAggregate, + // TODO: when "A75: xDS Aggregate Cluster Behavior Fixes" is implemented, the + // policy will have to be set on the child clusters. + Policy: e2e.LoadBalancingPolicyRingHash, + ChildNames: []string{primaryClusterName, secondaryClusterName}, + }) + route := channelIDHashRoute("new_route", virtualHostName, clusterName) + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + updateOpts := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ep1, ep2}, + Clusters: []*v3clusterpb.Cluster{cluster, primaryCluster, secondaryCluster}, + Routes: []*v3routepb.RouteConfiguration{route}, + Listeners: []*v3listenerpb.Listener{listener}, + } + if err := xdsServer.Update(ctx, updateOpts); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + const numRPCs = 100 + gotPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + + // Since this is using ring hash with the channel ID as the key, all RPCs + // are routed to the same backend of the secondary locality. + if len(gotPerBackend) != 1 { + t.Errorf("Got RPCs routed to %v backends, want %v", len(gotPerBackend), 1) + } + + var backend string + var got int + for backend, got = range gotPerBackend { + } + if !slices.Contains(addrs, backend) { + t.Errorf("Got RPCs routed to an unexpected backend: %v, want one of %v", backend, addrs) + } + if got != numRPCs { + t.Errorf("Got %v RPCs routed to a backend, want %v", got, 100) + } +} + +func replaceDNSResolver(t *testing.T) *manual.Resolver { + mr := manual.NewBuilderWithScheme("dns") + + dnsResolverBuilder := resolver.Get("dns") + resolver.Register(mr) + + t.Cleanup(func() { resolver.Register(dnsResolverBuilder) }) + return mr +} + +// Tests that when an aggregate cluster is configured with ring hash policy, and +// the first is an EDS cluster in transient failure, and the fallback is a +// logical DNS cluster, all RPCs are sent to the second cluster using the ring +// hash policy. +func (s) TestRingHash_AggregateClusterFallBackFromRingHashToLogicalDnsAtStartup(t *testing.T) { + const edsClusterName = "eds_cluster" + const logicalDNSClusterName = "logical_dns_cluster" + const clusterName = "aggregate_cluster" + + backends := backendAddrs(startTestServiceBackends(t, 1)) + + endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: edsClusterName, + Localities: []e2e.LocalityOptions{{ + Name: "locality0", + Weight: 1, + Backends: backendOptions(t, makeUnreachableBackends(t, 1)), + Priority: 0, + }}, + }) + edsCluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: edsClusterName, + ServiceName: edsClusterName, + }) + + logicalDNSCluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + Type: e2e.ClusterTypeLogicalDNS, + ClusterName: logicalDNSClusterName, + // The DNS values are not used because we fake DNS later on, but they + // are required to be present for the resource to be valid. + DNSHostName: "server.example.com", + DNSPort: 443, + }) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + Type: e2e.ClusterTypeAggregate, + // TODO: when "A75: xDS Aggregate Cluster Behavior Fixes" is merged, the + // policy will have to be set on the child clusters. + Policy: e2e.LoadBalancingPolicyRingHash, + ChildNames: []string{edsClusterName, logicalDNSClusterName}, + }) + route := channelIDHashRoute("new_route", virtualHostName, clusterName) + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + updateOpts := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{endpoints}, + Clusters: []*v3clusterpb.Cluster{cluster, edsCluster, logicalDNSCluster}, + Routes: []*v3routepb.RouteConfiguration{route}, + Listeners: []*v3listenerpb.Listener{listener}, + } + + dnsR := replaceDNSResolver(t) + dnsR.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: backends[0]}}}}, + }) + + if err := xdsServer.Update(ctx, updateOpts); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + gotPerBackend := checkRPCSendOK(ctx, t, client, 1) + var got string + for got = range gotPerBackend { + } + if want := backends[0]; got != want { + t.Errorf("Got RPCs routed to an unexpected got: %v, want %v", got, want) + } +} + +// Tests that when an aggregate cluster is configured with ring hash policy, and +// it's first child is in transient failure, and the fallback is a logical DNS, +// the later recovers from transient failure when its backend becomes available. +func (s) TestRingHash_AggregateClusterFallBackFromRingHashToLogicalDnsAtStartupNoFailedRPCs(t *testing.T) { + const edsClusterName = "eds_cluster" + const logicalDNSClusterName = "logical_dns_cluster" + const clusterName = "aggregate_cluster" + + backends := backendAddrs(startTestServiceBackends(t, 1)) + + endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: edsClusterName, + Localities: []e2e.LocalityOptions{{ + Name: "locality0", + Weight: 1, + Backends: backendOptions(t, makeUnreachableBackends(t, 1)), + Priority: 0, + }}, + }) + edsCluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: edsClusterName, + ServiceName: edsClusterName, + }) + + logicalDNSCluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + Type: e2e.ClusterTypeLogicalDNS, + ClusterName: logicalDNSClusterName, + // The DNS values are not used because we fake DNS later on, but they + // are required to be present for the resource to be valid. + DNSHostName: "server.example.com", + DNSPort: 443, + }) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + Type: e2e.ClusterTypeAggregate, + // TODO: when "A75: xDS Aggregate Cluster Behavior Fixes" is merged, the + // policy will have to be set on the child clusters. + Policy: e2e.LoadBalancingPolicyRingHash, + ChildNames: []string{edsClusterName, logicalDNSClusterName}, + }) + route := channelIDHashRoute("new_route", virtualHostName, clusterName) + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + updateOpts := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{endpoints}, + Clusters: []*v3clusterpb.Cluster{cluster, edsCluster, logicalDNSCluster}, + Routes: []*v3routepb.RouteConfiguration{route}, + Listeners: []*v3listenerpb.Listener{listener}, + } + + dnsR := replaceDNSResolver(t) + dnsR.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: backends[0]}}}}, + }) + + if err := xdsServer.Update(ctx, updateOpts); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + dialer := testutils.NewBlockingDialer() + cp := grpc.ConnectParams{ + // Increase backoff time, so that subconns stay in TRANSIENT_FAILURE + // for long enough to trigger potential problems. + Backoff: backoff.Config{ + BaseDelay: defaultTestTimeout, + }, + MinConnectTimeout: 0, + } + dopts := []grpc.DialOption{ + grpc.WithResolvers(xdsResolver), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(dialer.DialContext), + grpc.WithConnectParams(cp)} + conn, err := grpc.NewClient("xds:///test.server", dopts...) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + hold := dialer.Hold(backends[0]) + + errCh := make(chan error, 2) + go func() { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + errCh <- fmt.Errorf("first rpc UnaryCall() failed: %v", err) + return + } + errCh <- nil + }() + + testutils.AwaitState(ctx, t, conn, connectivity.Connecting) + + go func() { + // Start a second RPC at this point, which should be queued as well. + // This will fail if the priority policy fails to update the picker to + // point to the LOGICAL_DNS child; if it leaves it pointing to the EDS + // priority 1, then the RPC will fail, because all subchannels are in + // transient failure. + // + // Note that sending only the first RPC does not catch this case, + // because if the priority policy fails to update the picker, then the + // pick for the first RPC will not be retried. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + errCh <- fmt.Errorf("second UnaryCall() failed: %v", err) + return + } + errCh <- nil + }() + + // Wait for a connection attempt to backends[0]. + if !hold.Wait(ctx) { + t.Fatalf("Timeout while waiting for a connection attempt to %s", backends[0]) + } + // Allow the connection attempts to complete. + hold.Resume() + + // RPCs should complete successfully. + for range []int{0, 1} { + select { + case err := <-errCh: + if err != nil { + t.Errorf("Expected 2 rpc to succeed, but at least one failed: %v", err) + } + case <-ctx.Done(): + t.Fatalf("Timed out waiting for RPCs to complete") + } + } +} + +// endpointResource creates a ClusterLoadAssignment containing a single locality +// with the given addresses. +func endpointResource(t *testing.T, clusterName string, addrs []string) *v3endpointpb.ClusterLoadAssignment { + t.Helper() + backendAddrs := [][]string{} + for _, addr := range addrs { + backendAddrs = append(backendAddrs, []string{addr}) + } + return endpointResourceForBackendsWithMultipleAddrs(t, clusterName, backendAddrs) +} + +// endpointResourceForBackendsWithMultipleAddrs creates a ClusterLoadAssignment +// containing a single locality with the given addresses. +func endpointResourceForBackendsWithMultipleAddrs(t *testing.T, clusterName string, addrs [][]string) *v3endpointpb.ClusterLoadAssignment { + t.Helper() + + // We must set the host name socket address in EDS, as the ring hash policy + // uses it to construct the ring. + host, _, err := net.SplitHostPort(addrs[0][0]) + if err != nil { + t.Fatalf("Failed to split host and port from stubserver: %v", err) + } + + return e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: clusterName, + Host: host, + Localities: []e2e.LocalityOptions{{ + Backends: backendOptionsForEndpointsWithMultipleAddrs(t, addrs), + Weight: 1, + }}, + }) +} + +// Tests that ring hash policy that hashes using channel id ensures all RPCs to +// go 1 particular backend. +func (s) TestRingHash_ChannelIdHashing(t *testing.T) { + backends := backendAddrs(startTestServiceBackends(t, 4)) + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + + const clusterName = "cluster" + endpoints := endpointResource(t, clusterName, backends) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := channelIDHashRoute("new_route", virtualHostName, clusterName) + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + const numRPCs = 100 + received := checkRPCSendOK(ctx, t, client, numRPCs) + if len(received) != 1 { + t.Errorf("Got RPCs routed to %v backends, want %v", len(received), 1) + } + var got int + for _, got = range received { + } + if got != numRPCs { + t.Errorf("Got %v RPCs routed to a backend, want %v", got, numRPCs) + } +} + +// headerHashRoute creates a RouteConfiguration with a hash policy that uses the +// provided header. +func headerHashRoute(routeName, virtualHostName, clusterName, header string) *v3routepb.RouteConfiguration { + route := e2e.DefaultRouteConfig(routeName, virtualHostName, clusterName) + hashPolicy := v3routepb.RouteAction_HashPolicy{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: header, + }, + }, + } + action := route.VirtualHosts[0].Routes[0].Action.(*v3routepb.Route_Route) + action.Route.HashPolicy = []*v3routepb.RouteAction_HashPolicy{&hashPolicy} + return route +} + +// Tests that ring hash policy that hashes using a header value can send RPCs +// to specific backends based on their hash. +func (s) TestRingHash_HeaderHashing(t *testing.T) { + backends := backendAddrs(startTestServiceBackends(t, 4)) + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + + const clusterName = "cluster" + endpoints := endpointResource(t, clusterName, backends) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := headerHashRoute("new_route", virtualHostName, clusterName, "address_hash") + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + // Note each type of RPC contains a header value that will always be hashed + // to a specific backend as the header value matches the value used to + // create the entry in the ring. + for _, backend := range backends { + ctx := metadata.NewOutgoingContext(ctx, metadata.Pairs("address_hash", backend+"_0")) + numRPCs := 10 + reqPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + if reqPerBackend[backend] != numRPCs { + t.Errorf("Got RPC routed to addresses %v, want all RPCs routed to %v", reqPerBackend, backend) + } + } +} + +// Tests that ring hash policy that hashes using a header value and regex +// rewrite to aggregate RPCs to 1 backend. +func (s) TestRingHash_HeaderHashingWithRegexRewrite(t *testing.T) { + backends := backendAddrs(startTestServiceBackends(t, 4)) + + clusterName := "cluster" + endpoints := endpointResource(t, clusterName, backends) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := headerHashRoute("new_route", virtualHostName, clusterName, "address_hash") + action := route.VirtualHosts[0].Routes[0].Action.(*v3routepb.Route_Route) + action.Route.HashPolicy[0].GetHeader().RegexRewrite = &v3matcherpb.RegexMatchAndSubstitute{ + Pattern: &v3matcherpb.RegexMatcher{ + EngineType: &v3matcherpb.RegexMatcher_GoogleRe2{}, + Regex: "[0-9]+", + }, + Substitution: "foo", + } + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + // Note each type of RPC contains a header value that would always be hashed + // to a specific backend as the header value matches the value used to + // create the entry in the ring. However, the regex rewrites all numbers to + // "foo", and header values only differ by numbers, so they all end up + // hashing to the same value. + gotPerBackend := make(map[string]int) + for _, backend := range backends { + ctx := metadata.NewOutgoingContext(ctx, metadata.Pairs("address_hash", backend+"_0")) + res := checkRPCSendOK(ctx, t, client, 100) + for addr, count := range res { + gotPerBackend[addr] += count + } + } + if want := 1; len(gotPerBackend) != want { + t.Errorf("Got RPCs routed to %v backends, want %v", len(gotPerBackend), want) + } + var got int + for _, got = range gotPerBackend { + } + if want := 400; got != want { + t.Errorf("Got %v RPCs routed to a backend, want %v", got, want) + } +} + +// computeIdealNumberOfRPCs computes the ideal number of RPCs to send so that +// we can observe an event happening with probability p, and the result will +// have value p with the given error tolerance. +// +// See https://github.com/grpc/grpc/blob/4f6e13bdda9e8c26d6027af97db4b368ca2b3069/test/cpp/end2end/xds/xds_end2end_test_lib.h#L941 +// for an explanation of the formula. +func computeIdealNumberOfRPCs(t *testing.T, p, errorTolerance float64) int { + if p < 0 || p > 1 { + t.Fatal("p must be in (0, 1)") + } + numRPCs := math.Ceil(p * (1 - p) * 5. * 5. / errorTolerance / errorTolerance) + return int(numRPCs + 1000.) // add 1k as a buffer to avoid flakiness. +} + +// setRingHashLBPolicyWithHighMinRingSize sets the ring hash policy with a high +// minimum ring size to ensure that the ring is large enough to distribute +// requests more uniformly across endpoints when a random hash is used. +func setRingHashLBPolicyWithHighMinRingSize(t *testing.T, cluster *v3clusterpb.Cluster) { + testutils.SetEnvConfig(t, &envconfig.RingHashCap, minRingSize) + + // Increasing min ring size for random distribution. + config := testutils.MarshalAny(t, &v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_XX_HASH, + MinimumRingSize: &wrapperspb.UInt64Value{Value: minRingSize}, + }) + cluster.LoadBalancingPolicy = &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{{ + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + Name: "envoy.load_balancing_policies.ring_hash", + TypedConfig: config, + }, + }}, + } +} + +// Tests that ring hash policy that hashes using a random value. +func (s) TestRingHash_NoHashPolicy(t *testing.T) { + backends := backendAddrs(startTestServiceBackends(t, 2)) + numRPCs := computeIdealNumberOfRPCs(t, .5, errorTolerance) + + const clusterName = "cluster" + endpoints := endpointResource(t, clusterName, backends) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + }) + setRingHashLBPolicyWithHighMinRingSize(t, cluster) + route := e2e.DefaultRouteConfig("new_route", virtualHostName, clusterName) + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + // Send a large number of RPCs and check that they are distributed randomly. + gotPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + for _, backend := range backends { + got := float64(gotPerBackend[backend]) / float64(numRPCs) + want := .5 + if !cmp.Equal(got, want, cmpopts.EquateApprox(0, errorTolerance)) { + t.Errorf("Fraction of RPCs to backend %s: got %v, want %v (margin: +-%v)", backend, got, want, errorTolerance) + } + } +} + +// Tests that we observe endpoint weights. +func (s) TestRingHash_EndpointWeights(t *testing.T) { + backends := backendAddrs(startTestServiceBackends(t, 3)) + + const clusterName = "cluster" + backendOpts := []e2e.BackendOptions{ + {Ports: []uint32{testutils.ParsePort(t, backends[0])}}, + {Ports: []uint32{testutils.ParsePort(t, backends[1])}}, + {Ports: []uint32{testutils.ParsePort(t, backends[2])}, Weight: 2}, + } + + endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: clusterName, + Localities: []e2e.LocalityOptions{{ + Backends: backendOpts, + Weight: 1, + }}, + }) + endpoints.Endpoints[0].LbEndpoints[0].LoadBalancingWeight = wrapperspb.UInt32(uint32(1)) + endpoints.Endpoints[0].LbEndpoints[1].LoadBalancingWeight = wrapperspb.UInt32(uint32(1)) + endpoints.Endpoints[0].LbEndpoints[2].LoadBalancingWeight = wrapperspb.UInt32(uint32(2)) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + }) + // Increasing min ring size for random distribution. + setRingHashLBPolicyWithHighMinRingSize(t, cluster) + route := e2e.DefaultRouteConfig("new_route", virtualHostName, clusterName) + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + // Send a large number of RPCs and check that they are distributed randomly. + numRPCs := computeIdealNumberOfRPCs(t, .25, errorTolerance) + gotPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + + got := float64(gotPerBackend[backends[0]]) / float64(numRPCs) + want := .25 + if !cmp.Equal(got, want, cmpopts.EquateApprox(0, errorTolerance)) { + t.Errorf("Fraction of RPCs to backend %s: got %v, want %v (margin: +-%v)", backends[0], got, want, errorTolerance) + } + got = float64(gotPerBackend[backends[1]]) / float64(numRPCs) + if !cmp.Equal(got, want, cmpopts.EquateApprox(0, errorTolerance)) { + t.Errorf("Fraction of RPCs to backend %s: got %v, want %v (margin: +-%v)", backends[1], got, want, errorTolerance) + } + got = float64(gotPerBackend[backends[2]]) / float64(numRPCs) + want = .50 + if !cmp.Equal(got, want, cmpopts.EquateApprox(0, errorTolerance)) { + t.Errorf("Fraction of RPCs to backend %s: got %v, want %v (margin: +-%v)", backends[2], got, want, errorTolerance) + } +} + +// Tests that ring hash policy evaluation will continue past the terminal hash +// policy if no results are produced yet. +func (s) TestRingHash_ContinuesPastTerminalPolicyThatDoesNotProduceResult(t *testing.T) { + backends := backendAddrs(startTestServiceBackends(t, 2)) + + const clusterName = "cluster" + endpoints := endpointResource(t, clusterName, backends) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + + route := e2e.DefaultRouteConfig("new_route", "test.server", clusterName) + + // Even though this hash policy is terminal, since it produces no result, we + // continue past it to find a policy that produces results. + hashPolicy := v3routepb.RouteAction_HashPolicy{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: "header_not_present", + }, + }, + Terminal: true, + } + hashPolicy2 := v3routepb.RouteAction_HashPolicy{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: "address_hash", + }, + }, + } + action := route.VirtualHosts[0].Routes[0].Action.(*v3routepb.Route_Route) + action.Route.HashPolicy = []*v3routepb.RouteAction_HashPolicy{&hashPolicy, &hashPolicy2} + + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + // - The first hash policy does not match because the header is not present. + // If this hash policy was applied, it would spread the load across + // backend 0 and 1, since a random hash would be used. + // - In the second hash policy, each type of RPC contains a header + // value that always hashes to backend 0, as the header value + // matches the value used to create the entry in the ring. + // We verify that the second hash policy is used by checking that all RPCs + // are being routed to backend 0. + wantBackend := backends[0] + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("address_hash", wantBackend+"_0")) + const numRPCs = 100 + gotPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + if got := gotPerBackend[wantBackend]; got != numRPCs { + t.Errorf("Got %v RPCs routed to backend %v, want %v", got, wantBackend, numRPCs) + } +} + +// Tests that a random hash is used when header hashing policy specified a +// header field that the RPC did not have. +func (s) TestRingHash_HashOnHeaderThatIsNotPresent(t *testing.T) { + backends := backendAddrs(startTestServiceBackends(t, 2)) + wantFractionPerBackend := .5 + numRPCs := computeIdealNumberOfRPCs(t, wantFractionPerBackend, errorTolerance) + + const clusterName = "cluster" + endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: clusterName, + Localities: []e2e.LocalityOptions{{ + Backends: backendOptions(t, backends), + Weight: 1, + }}, + }) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + }) + setRingHashLBPolicyWithHighMinRingSize(t, cluster) + route := headerHashRoute("new_route", virtualHostName, clusterName, "header_not_present") + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + // The first hash policy does not apply because the header is not present in + // the RPCs that we are about to send. As a result, a random hash should be + // used instead, resulting in a random request distribution. + // We verify this by checking that the RPCs are distributed randomly. + gotPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + for _, backend := range backends { + got := float64(gotPerBackend[backend]) / float64(numRPCs) + if !cmp.Equal(got, wantFractionPerBackend, cmpopts.EquateApprox(0, errorTolerance)) { + t.Errorf("fraction of RPCs to backend %s: got %v, want %v (margin: +-%v)", backend, got, wantFractionPerBackend, errorTolerance) + } + } +} + +// Tests that a random hash is used when only unsupported hash policies are +// configured. +func (s) TestRingHash_UnsupportedHashPolicyDefaultToRandomHashing(t *testing.T) { + backends := backendAddrs(startTestServiceBackends(t, 2)) + wantFractionPerBackend := .5 + numRPCs := computeIdealNumberOfRPCs(t, wantFractionPerBackend, errorTolerance) + + const clusterName = "cluster" + endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: clusterName, + Localities: []e2e.LocalityOptions{{ + Backends: backendOptions(t, backends), + Weight: 1, + }}, + }) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + }) + setRingHashLBPolicyWithHighMinRingSize(t, cluster) + route := e2e.DefaultRouteConfig("new_route", "test.server", clusterName) + unsupportedHashPolicy1 := v3routepb.RouteAction_HashPolicy{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Cookie_{ + Cookie: &v3routepb.RouteAction_HashPolicy_Cookie{Name: "cookie"}, + }, + } + unsupportedHashPolicy2 := v3routepb.RouteAction_HashPolicy{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_ConnectionProperties_{ + ConnectionProperties: &v3routepb.RouteAction_HashPolicy_ConnectionProperties{SourceIp: true}, + }, + } + unsupportedHashPolicy3 := v3routepb.RouteAction_HashPolicy{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_QueryParameter_{ + QueryParameter: &v3routepb.RouteAction_HashPolicy_QueryParameter{Name: "query_parameter"}, + }, + } + action := route.VirtualHosts[0].Routes[0].Action.(*v3routepb.Route_Route) + action.Route.HashPolicy = []*v3routepb.RouteAction_HashPolicy{&unsupportedHashPolicy1, &unsupportedHashPolicy2, &unsupportedHashPolicy3} + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + // Since none of the hash policy are supported, a random hash should be + // generated for every request. + // We verify this by checking that the RPCs are distributed randomly. + gotPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + for _, backend := range backends { + got := float64(gotPerBackend[backend]) / float64(numRPCs) + if !cmp.Equal(got, wantFractionPerBackend, cmpopts.EquateApprox(0, errorTolerance)) { + t.Errorf("Fraction of RPCs to backend %s: got %v, want %v (margin: +-%v)", backend, got, wantFractionPerBackend, errorTolerance) + } + } +} + +// Tests that unsupported hash policy types are all ignored before a supported +// hash policy. +func (s) TestRingHash_UnsupportedHashPolicyUntilChannelIdHashing(t *testing.T) { + backends := backendAddrs(startTestServiceBackends(t, 2)) + + const clusterName = "cluster" + endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: clusterName, + Localities: []e2e.LocalityOptions{{ + Backends: backendOptions(t, backends), + Weight: 1, + }}, + }) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + }) + setRingHashLBPolicyWithHighMinRingSize(t, cluster) + route := e2e.DefaultRouteConfig("new_route", "test.server", clusterName) + unsupportedHashPolicy1 := v3routepb.RouteAction_HashPolicy{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Cookie_{ + Cookie: &v3routepb.RouteAction_HashPolicy_Cookie{Name: "cookie"}, + }, + } + unsupportedHashPolicy2 := v3routepb.RouteAction_HashPolicy{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_ConnectionProperties_{ + ConnectionProperties: &v3routepb.RouteAction_HashPolicy_ConnectionProperties{SourceIp: true}, + }, + } + unsupportedHashPolicy3 := v3routepb.RouteAction_HashPolicy{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_QueryParameter_{ + QueryParameter: &v3routepb.RouteAction_HashPolicy_QueryParameter{Name: "query_parameter"}, + }, + } + channelIDhashPolicy := v3routepb.RouteAction_HashPolicy{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_FilterState_{ + FilterState: &v3routepb.RouteAction_HashPolicy_FilterState{ + Key: "io.grpc.channel_id", + }, + }, + } + action := route.VirtualHosts[0].Routes[0].Action.(*v3routepb.Route_Route) + action.Route.HashPolicy = []*v3routepb.RouteAction_HashPolicy{&unsupportedHashPolicy1, &unsupportedHashPolicy2, &unsupportedHashPolicy3, &channelIDhashPolicy} + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + // Since only unsupported policies are present except for the last one + // which is using the channel ID hashing policy, all requests should be + // routed to the same backend. + const numRPCs = 100 + gotPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + if len(gotPerBackend) != 1 { + t.Errorf("Got RPCs routed to %v backends, want 1", len(gotPerBackend)) + } + var got int + for _, got = range gotPerBackend { + } + if got != numRPCs { + t.Errorf("Got %v RPCs routed to a backend, want %v", got, numRPCs) + } +} + +// Tests that ring hash policy that hashes using a random value can spread RPCs +// across all the backends according to locality weight. +func (s) TestRingHash_RandomHashingDistributionAccordingToLocalityAndEndpointWeight(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.PickFirstWeightedShuffling, true) + backends := backendAddrs(startTestServiceBackends(t, 2)) + + const clusterName = "cluster" + const locality0Weight = uint32(1) + const endpoint0Weight = uint32(1) + const locality1Weight = uint32(2) + const endpoint1Weight = uint32(2) + endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: clusterName, + Localities: []e2e.LocalityOptions{ + { + Backends: []e2e.BackendOptions{{ + Ports: []uint32{testutils.ParsePort(t, backends[0])}, + Weight: endpoint0Weight, + }}, + Weight: locality0Weight, + }, + { + Backends: []e2e.BackendOptions{{ + Ports: []uint32{testutils.ParsePort(t, backends[1])}, + Weight: endpoint1Weight, + }}, + Weight: locality1Weight, + }, + }, + }) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + }) + setRingHashLBPolicyWithHighMinRingSize(t, cluster) + route := e2e.DefaultRouteConfig("new_route", "test.server", clusterName) + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + // The target fraction of RPCs to each backend is computed as the product of + // the probability of selecting the locality and the probability of + // selecting the endpoint within the locality. The probability of selecting + // locality0 is 1/3 and locality1 is 2/3. Since there is only one endpoint + // in each locality, the probability of selecting the endpoint within the + // locality is 1. Therefore, the target fractions end up as 1/3 and 2/3 + // respectively. + const wantRPCs0 = float64(1) / float64(3) + const wantRPCs1 = float64(2) / float64(3) + numRPCs := computeIdealNumberOfRPCs(t, math.Min(wantRPCs0, wantRPCs1), errorTolerance) + + // Send a large number of RPCs and check that they are distributed randomly. + gotPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + got := float64(gotPerBackend[backends[0]]) / float64(numRPCs) + if !cmp.Equal(got, wantRPCs0, cmpopts.EquateApprox(0, errorTolerance)) { + t.Errorf("Fraction of RPCs to backend %s: got %v, want %v (margin: +-%v)", backends[0], got, wantRPCs0, errorTolerance) + } + got = float64(gotPerBackend[backends[1]]) / float64(numRPCs) + if !cmp.Equal(got, wantRPCs1, cmpopts.EquateApprox(0, errorTolerance)) { + t.Errorf("Fraction of RPCs to backend %s: got %v, want %v (margin: +-%v)", backends[1], got, wantRPCs1, errorTolerance) + } +} + +// Tests that ring hash policy that hashes using a fixed string ensures all RPCs +// to go 1 particular backend; and that subsequent hashing policies are ignored +// due to the setting of terminal. +func (s) TestRingHash_FixedHashingTerminalPolicy(t *testing.T) { + backends := backendAddrs(startTestServiceBackends(t, 2)) + const clusterName = "cluster" + endpoints := endpointResource(t, clusterName, backends) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + + route := e2e.DefaultRouteConfig("new_route", "test.server", clusterName) + + hashPolicy := v3routepb.RouteAction_HashPolicy{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: "fixed_string", + }, + }, + Terminal: true, + } + hashPolicy2 := v3routepb.RouteAction_HashPolicy{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: "random_string", + }, + }, + } + action := route.VirtualHosts[0].Routes[0].Action.(*v3routepb.Route_Route) + action.Route.HashPolicy = []*v3routepb.RouteAction_HashPolicy{&hashPolicy, &hashPolicy2} + + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + // Check that despite the matching random string header, since the fixed + // string hash policy is terminal, only the fixed string hash policy applies + // and requests all get routed to the same host. + gotPerBackend := make(map[string]int) + const numRPCs = 100 + for i := 0; i < numRPCs; i++ { + ctx := metadata.NewOutgoingContext(ctx, metadata.Pairs( + "fixed_string", backends[0]+"_0", + "random_string", fmt.Sprintf("%d", rand.Int())), + ) + var remote peer.Peer + _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&remote)) + if err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + gotPerBackend[remote.Addr.String()]++ + } + + if len(gotPerBackend) != 1 { + t.Error("Got RPCs routed to multiple backends, want a single backend") + } + if got := gotPerBackend[backends[0]]; got != numRPCs { + t.Errorf("Got %v RPCs routed to %v, want %v", got, backends[0], numRPCs) + } +} + +// TestRingHash_IdleToReady tests that the channel will go from idle to ready +// via connecting; (though it is not possible to catch the connecting state +// before moving to ready via the public API). +// TODO: we should be able to catch all state transitions by using the internal.SubscribeToConnectivityStateChanges API. +func (s) TestRingHash_IdleToReady(t *testing.T) { + backends := backendAddrs(startTestServiceBackends(t, 1)) + + const clusterName = "cluster" + endpoints := endpointResource(t, clusterName, backends) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := channelIDHashRoute("new_route", virtualHostName, clusterName) + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + testutils.AwaitState(ctx, t, conn, connectivity.Idle) + + client := testgrpc.NewTestServiceClient(conn) + checkRPCSendOK(ctx, t, client, 1) + testutils.AwaitState(ctx, t, conn, connectivity.Ready) +} + +// Test that the channel will transition to READY once it starts +// connecting even if there are no RPCs being sent to the picker. +func (s) TestRingHash_ContinuesConnectingWithoutPicks(t *testing.T) { + backend := stubserver.StartTestService(t, &stubserver.StubServer{ + // We expect the server EmptyCall to not be call here because the + // aggregated channel state is never READY when the call is pending. + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + t.Errorf("EmptyCall() should not have been called") + return &testpb.Empty{}, nil + }, + }) + defer backend.Stop() + + unReachableServerAddr := makeUnreachableBackends(t, 1)[0] + + const clusterName = "cluster" + endpoints := endpointResource(t, clusterName, []string{backend.Address, unReachableServerAddr}) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := headerHashRoute("new_route", virtualHostName, clusterName, "address_hash") + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + dialer := testutils.NewBlockingDialer() + dopts := []grpc.DialOption{ + grpc.WithResolvers(xdsResolver), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(dialer.DialContext), + } + conn, err := grpc.NewClient("xds:///test.server", dopts...) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + hold := dialer.Hold(backend.Address) + + rpcCtx, rpcCancel := context.WithCancel(ctx) + go func() { + rpcCtx = metadata.NewOutgoingContext(rpcCtx, metadata.Pairs("address_hash", unReachableServerAddr+"_0")) + _, err := client.EmptyCall(rpcCtx, &testpb.Empty{}) + if status.Code(err) != codes.Canceled { + t.Errorf("Expected RPC to be canceled, got error: %v", err) + } + }() + + // Wait for the connection attempt to the real backend. + if !hold.Wait(ctx) { + t.Fatalf("Timeout waiting for connection attempt to backend %v.", backend.Address) + } + // Now cancel the RPC while we are still connecting. + rpcCancel() + + // This allows the connection attempts to continue. The RPC was cancelled + // before the backend was connected, but the backend is up. The conn + // becomes Ready due to the connection attempt to the existing backend + // succeeding, despite no new RPC being sent. + hold.Resume() + + testutils.AwaitState(ctx, t, conn, connectivity.Ready) +} + +// Tests that when the first pick is down leading to a transient failure, we +// will move on to the next ring hash entry. +func (s) TestRingHash_TransientFailureCheckNextOne(t *testing.T) { + backends := backendAddrs(startTestServiceBackends(t, 1)) + unReachableBackends := makeUnreachableBackends(t, 1) + + const clusterName = "cluster" + endpoints := endpointResource(t, clusterName, append(unReachableBackends, backends...)) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := headerHashRoute("new_route", virtualHostName, clusterName, "address_hash") + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + // Note each type of RPC contains a header value that will always be hashed + // the value that was used to place the non-existent endpoint on the ring, + // but it still gets routed to the backend that is up. + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("address_hash", unReachableBackends[0]+"_0")) + reqPerBackend := checkRPCSendOK(ctx, t, client, 1) + var got string + for got = range reqPerBackend { + } + if want := backends[0]; got != want { + t.Errorf("Got RPC routed to addr %v, want %v", got, want) + } +} + +// Tests for a bug seen in the wild in c-core, where ring_hash started with no +// endpoints and reported TRANSIENT_FAILURE, then got an update with endpoints +// and reported IDLE, but the picker update was squelched, so it failed to ever +// get reconnected. +func (s) TestRingHash_ReattemptWhenGoingFromTransientFailureToIdle(t *testing.T) { + const clusterName = "cluster" + endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: clusterName, + Localities: []e2e.LocalityOptions{{}}, // note the empty locality (no endpoint). + }) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := e2e.DefaultRouteConfig("new_route", virtualHostName, clusterName) + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + testutils.AwaitState(ctx, t, conn, connectivity.Idle) + + // There are no endpoints in EDS. RPCs should fail and the channel should + // transition to transient failure. + client := testgrpc.NewTestServiceClient(conn) + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Errorf("rpc EmptyCall() succeeded, want error") + } + testutils.AwaitState(ctx, t, conn, connectivity.TransientFailure) + + t.Log("Updating EDS with a new backend endpoint.") + backends := backendAddrs(startTestServiceBackends(t, 1)) + endpoints = e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: clusterName, + Localities: []e2e.LocalityOptions{{ + Backends: backendOptions(t, backends), + Weight: 1, + }}, + }) + if err = xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + // A WaitForReady RPC should succeed, and the channel should report READY. + if _, err = client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Errorf("rpc EmptyCall() failed: %v", err) + } + testutils.AwaitState(ctx, t, conn, connectivity.Ready) +} + +// Tests that when all backends are down and then up, we may pick a TF backend +// and we will then jump to ready backend. +func (s) TestRingHash_TransientFailureSkipToAvailableReady(t *testing.T) { + emptyCallF := func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + } + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + restartableListener1 := testutils.NewRestartableListener(lis) + restartableServer1 := stubserver.StartTestService(t, &stubserver.StubServer{ + Listener: restartableListener1, + EmptyCallF: emptyCallF, + }) + defer restartableServer1.Stop() + + lis, err = testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + restartableListener2 := testutils.NewRestartableListener(lis) + restartableServer2 := stubserver.StartTestService(t, &stubserver.StubServer{ + Listener: restartableListener2, + EmptyCallF: emptyCallF, + }) + defer restartableServer2.Stop() + + unReachableBackends := makeUnreachableBackends(t, 2) + + const clusterName = "cluster" + backends := []string{restartableServer1.Address, restartableServer2.Address} + backends = append(backends, unReachableBackends...) + endpoints := endpointResource(t, clusterName, backends) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := headerHashRoute("new_route", virtualHostName, clusterName, "address_hash") + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + opts := []grpc.DialOption{ + grpc.WithConnectParams(grpc.ConnectParams{ + // Disable backoff to speed up the test. + MinConnectTimeout: 100 * time.Millisecond, + }), + grpc.WithResolvers(xdsResolver), + grpc.WithTransportCredentials(insecure.NewCredentials()), + } + conn, err := grpc.NewClient("xds:///test.server", opts...) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + testutils.AwaitState(ctx, t, conn, connectivity.Idle) + + // Test starts with backends not listening. + restartableListener1.Stop() + restartableListener2.Stop() + + // Send a request with a hash that should go to restartableServer1. + // Because it is not accepting connections, and no other backend is + // listening, the RPC fails. + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("address_hash", restartableServer1.Address+"_0")) + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Fatalf("rpc EmptyCall() succeeded, want error") + } + + testutils.AwaitState(ctx, t, conn, connectivity.TransientFailure) + + // Bring up first backend. The channel should become Ready without any + // picks, because in TF, we are always trying to connect to at least one + // backend at all times. + restartableListener1.Restart() + testutils.AwaitState(ctx, t, conn, connectivity.Ready) + + // Bring down backend 1 and bring up backend 2. + // Note the RPC contains a header value that will always be hashed to + // backend 1. So by purposely bringing down backend 1 and bringing up + // another backend, this will ensure Picker's first choice of backend 1 + // fails and it will go through the remaining subchannels to find one in + // READY. Since the entries in the ring are pretty distributed and we have + // unused ports to fill the ring, it is almost guaranteed that the Picker + // will go through some non-READY entries and skip them as per design. + t.Logf("bringing down backend 1") + restartableListener1.Stop() + + testutils.AwaitState(ctx, t, conn, connectivity.TransientFailure) + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Fatalf("rpc EmptyCall() succeeded, want error") + } + + t.Logf("bringing up backend 2") + restartableListener2.Restart() + testutils.AwaitState(ctx, t, conn, connectivity.Ready) + + wantPeerAddr := "" + for wantPeerAddr != restartableServer2.Address { + p := peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&p)); errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Timed out waiting for rpc EmptyCall() to be routed to the expected backend") + } + wantPeerAddr = p.Addr.String() + } +} + +// Tests that when all backends are down, we keep reattempting. +func (s) TestRingHash_ReattemptWhenAllEndpointsUnreachable(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + restartableListener := testutils.NewRestartableListener(lis) + restartableServer := stubserver.StartTestService(t, &stubserver.StubServer{ + Listener: restartableListener, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + }) + defer restartableServer.Stop() + + const clusterName = "cluster" + endpoints := endpointResource(t, clusterName, []string{restartableServer.Address}) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := headerHashRoute("new_route", virtualHostName, clusterName, "address_hash") + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + dopts := []grpc.DialOption{ + grpc.WithResolvers(xdsResolver), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithConnectParams(fastConnectParams), + } + conn, err := grpc.NewClient("xds:///test.server", dopts...) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + testutils.AwaitState(ctx, t, conn, connectivity.Idle) + + t.Log("Stopping the backend server") + restartableListener.Stop() + + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable { + t.Fatalf("rpc EmptyCall() succeeded, want Unavailable error") + } + + // Wait for channel to fail. + testutils.AwaitState(ctx, t, conn, connectivity.TransientFailure) + + t.Log("Restarting the backend server") + restartableListener.Restart() + + // Wait for channel to become READY without any pending RPC. + testutils.AwaitState(ctx, t, conn, connectivity.Ready) +} + +// Tests that when a backend goes down, we will move on to the next subchannel +// (with a lower priority). When the backend comes back up, traffic will move +// back. +func (s) TestRingHash_SwitchToLowerPriorityAndThenBack(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + restartableListener := testutils.NewRestartableListener(lis) + restartableServer := stubserver.StartTestService(t, &stubserver.StubServer{ + Listener: restartableListener, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + }) + defer restartableServer.Stop() + + otherBackend := backendAddrs(startTestServiceBackends(t, 1))[0] + + // We must set the host name socket address in EDS, as the ring hash policy + // uses it to construct the ring. + host, _, err := net.SplitHostPort(otherBackend) + if err != nil { + t.Fatalf("Failed to split host and port from stubserver: %v", err) + } + + const clusterName = "cluster" + endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: clusterName, + Host: host, + Localities: []e2e.LocalityOptions{{ + Backends: backendOptions(t, []string{restartableServer.Address}), + Weight: 1, + }, { + Backends: backendOptions(t, []string{otherBackend}), + Weight: 1, + Priority: 1, + }}}) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := headerHashRoute("new_route", virtualHostName, clusterName, "address_hash") + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + dopts := []grpc.DialOption{ + grpc.WithResolvers(xdsResolver), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithConnectParams(fastConnectParams), + } + conn, err := grpc.NewClient("xds:///test.server", dopts...) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + // Note each type of RPC contains a header value that will always be hashed + // to the value that was used to place the non-existent endpoint on the ring. + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("address_hash", restartableServer.Address+"_0")) + var got string + for got = range checkRPCSendOK(ctx, t, client, 1) { + } + if want := restartableServer.Address; got != want { + t.Fatalf("Got RPC routed to addr %v, want %v", got, want) + } + + // Trigger failure with the existing backend, which should cause the + // balancer to go in transient failure and the priority balancer to move + // to the lower priority. + restartableListener.Stop() + + for { + p := peer.Peer{} + _, err = client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)) + + // Ignore errors: we may need to attempt to send an RPC to detect the + // failure (the next write on connection fails). + if err == nil { + if got, want := p.Addr.String(), otherBackend; got != want { + t.Fatalf("Got RPC routed to addr %v, want %v", got, want) + } + break + } + } + + // Now we start the backend with the address hash that is used in the + // metadata, so eventually RPCs should be routed to it, since it is in a + // locality with higher priority. + peerAddr := "" + restartableListener.Restart() + for peerAddr != restartableServer.Address { + p := peer.Peer{} + _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&p)) + if errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Timed out waiting for rpc EmptyCall() to be routed to the expected backend") + } + peerAddr = p.Addr.String() + } +} + +// Tests that when we trigger internal connection attempts without picks, we +// keep retrying all the SubConns that have reported TF previously. +func (s) TestRingHash_ContinuesConnectingWithoutPicksToMultipleSubConnsConcurrently(t *testing.T) { + const backendsCount = 4 + backends := backendAddrs(startTestServiceBackends(t, backendsCount)) + + const clusterName = "cluster" + + endpoints := endpointResource(t, clusterName, backends) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := headerHashRoute("new_route", virtualHostName, clusterName, "address_hash") + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + dialer := testutils.NewBlockingDialer() + dialOpts := []grpc.DialOption{ + grpc.WithResolvers(xdsResolver), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(dialer.DialContext), + grpc.WithConnectParams(fastConnectParams), + } + conn, err := grpc.NewClient("xds:///test.server", dialOpts...) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + + // Create holds for each backend address to delay a successful connection + // until the end of the test. + holds := make([]*testutils.Hold, backendsCount) + for i := 0; i < len(backends); i++ { + holds[i] = dialer.Hold(backends[i]) + } + + client := testgrpc.NewTestServiceClient(conn) + + rpcCtx, rpcCancel := context.WithCancel(ctx) + errCh := make(chan error, 1) + go func() { + rpcCtx = metadata.NewOutgoingContext(rpcCtx, metadata.Pairs("address_hash", backends[0]+"_0")) + _, err := client.EmptyCall(rpcCtx, &testpb.Empty{}) + if status.Code(err) == codes.Canceled { + errCh <- nil + return + } + errCh <- err + }() + + // Wait for the RPC to trigger a connection attempt to the first address, + // then cancel the RPC. No other connection attempts should be started yet. + if !holds[0].Wait(ctx) { + t.Fatalf("Timeout waiting for connection attempt to backend 0") + } + rpcCancel() + if err := <-errCh; err != nil { + t.Fatalf("Expected RPC to fail be canceled, got %v", err) + } + + // In every iteration of the following loop, we count the number of backends + // that are dialed. After counting, we fail all the connection attempts. + // This should cause the number of dialed backends to increase by 1 in every + // iteration of the loop as ringhash tries to exit TRANSIENT_FAILURE. + activeAddrs := map[string]bool{} + for wantBackendCount := 1; wantBackendCount <= backendsCount; wantBackendCount++ { + newAddrIdx := -1 + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + for i, hold := range holds { + if !hold.IsStarted() { + continue + } + if _, ok := activeAddrs[backends[i]]; ok { + continue + } + activeAddrs[backends[i]] = true + newAddrIdx = i + } + if len(activeAddrs) > wantBackendCount { + t.Fatalf("More backends dialed than expected: got %d, want %d", len(activeAddrs), wantBackendCount) + } + if len(activeAddrs) == wantBackendCount { + break + } + } + + // Wait for a short time and verify no more backends are contacted. + <-time.After(defaultTestShortTimeout) + for i, hold := range holds { + if !hold.IsStarted() { + continue + } + activeAddrs[backends[i]] = true + } + if len(activeAddrs) != wantBackendCount { + t.Fatalf("Unexpected number of backends dialed: got %d, want %d", len(activeAddrs), wantBackendCount) + } + + // Create a new hold for the address dialed in this iteration and fail + // the existing hold. + hold := holds[newAddrIdx] + holds[newAddrIdx] = dialer.Hold(backends[newAddrIdx]) + hold.Fail(errors.New("Test error")) + } + + // Allow the request to a backend to succeed. + if !holds[1].Wait(ctx) { + t.Fatalf("Context timed out waiting %q to be dialed again.", backends[1]) + } + holds[1].Resume() + + // Wait for channel to become READY without any pending RPC. + testutils.AwaitState(ctx, t, conn, connectivity.Ready) +} + +// Tests that first address of an endpoint is used to generate the ring. The +// test sends a request to a random endpoint. The test then reverses the +// addresses of every endpoint and verifies that an RPC with header pointing to +// the second address of the endpoint is sent to the initial address. The test +// then swaps the second and third address of the endpoint and verifies that an +// RPC with the header used earlier still reaches the same backend. +func (s) TestRingHash_ReorderAddressessWithinEndpoint(t *testing.T) { + origDualstackEndpointsEnabled := envconfig.XDSDualstackEndpointsEnabled + defer func() { + envconfig.XDSDualstackEndpointsEnabled = origDualstackEndpointsEnabled + }() + envconfig.XDSDualstackEndpointsEnabled = true + backends := backendAddrs(startTestServiceBackends(t, 6)) + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + + const clusterName = "cluster" + addrGroups := [][]string{ + {backends[0], backends[1], backends[2]}, + {backends[3], backends[4], backends[5]}, + } + endpoints := endpointResourceForBackendsWithMultipleAddrs(t, clusterName, addrGroups) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := headerHashRoute("new_route", virtualHostName, clusterName, "address_hash") + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + rpcCtx := metadata.NewOutgoingContext(ctx, metadata.Pairs( + "address_hash", fmt.Sprintf("%d", rand.Int()), + )) + var remote peer.Peer + if _, err := client.EmptyCall(rpcCtx, &testpb.Empty{}, grpc.Peer(&remote)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + initialFirstAddr := "" + newFirstAddr := "" + switch remote.Addr.String() { + case addrGroups[0][0]: + initialFirstAddr = addrGroups[0][0] + newFirstAddr = addrGroups[0][2] + case addrGroups[1][0]: + initialFirstAddr = addrGroups[1][0] + newFirstAddr = addrGroups[1][2] + default: + t.Fatalf("Request went to unexpected address: %q", remote.Addr) + } + + t.Log("Reversing addresses within each endpoint.") + addrGroups1 := [][]string{ + {addrGroups[0][2], addrGroups[0][1], addrGroups[0][0]}, + {addrGroups[1][2], addrGroups[1][1], addrGroups[1][0]}, + } + endpoints = endpointResourceForBackendsWithMultipleAddrs(t, clusterName, addrGroups1) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + // The first address of an endpoint is used to create the ring. This means + // that requests should continue to go to the first address, but the hash + // should be computed based on the last address in the original list. + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + rpcCtx := metadata.NewOutgoingContext(ctx, metadata.Pairs( + "address_hash", newFirstAddr+"_0", + )) + if _, err := client.EmptyCall(rpcCtx, &testpb.Empty{}, grpc.Peer(&remote)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + if remote.Addr.String() == initialFirstAddr { + break + } + } + + if ctx.Err() != nil { + t.Fatalf("Context timed out waiting for request to be sent to %q, last request went to %q", initialFirstAddr, remote.Addr) + } + + t.Log("Swapping the second and third addresses within each endpoint.") + // This should not effect the ring, since only the first address is used + // by the ring. + addrGroups2 := [][]string{ + {addrGroups1[0][0], addrGroups[0][2], addrGroups[0][1]}, + {addrGroups1[1][0], addrGroups[1][2], addrGroups[1][1]}, + } + endpoints = endpointResourceForBackendsWithMultipleAddrs(t, clusterName, addrGroups2) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + // Verify that requests with the hash of the last address in chosenAddrGroup + // continue reaching the first address in chosenAddrGroup. + shortCtx, cancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer cancel() + for ; shortCtx.Err() == nil; <-time.After(time.Millisecond) { + rpcCtx := metadata.NewOutgoingContext(ctx, metadata.Pairs( + "address_hash", newFirstAddr+"_0", + )) + if _, err := client.EmptyCall(rpcCtx, &testpb.Empty{}, grpc.Peer(&remote)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + if remote.Addr.String() == initialFirstAddr { + continue + } + t.Fatalf("Request went to unexpected backend %q, want backend %q", remote.Addr, initialFirstAddr) + } +} + +// Tests that requests are sent to the next address within the same endpoint +// after the first address becomes unreachable. +func (s) TestRingHash_FallBackWithinEndpoint(t *testing.T) { + origDualstackEndpointsEnabled := envconfig.XDSDualstackEndpointsEnabled + defer func() { + envconfig.XDSDualstackEndpointsEnabled = origDualstackEndpointsEnabled + }() + envconfig.XDSDualstackEndpointsEnabled = true + backends := startTestServiceBackends(t, 4) + backendAddrs := backendAddrs(backends) + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + + const clusterName = "cluster" + endpoints := endpointResourceForBackendsWithMultipleAddrs(t, clusterName, [][]string{{backendAddrs[0], backendAddrs[1]}, {backendAddrs[2], backendAddrs[3]}}) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := channelIDHashRoute("new_route", virtualHostName, clusterName) + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient("xds:///test.server", grpc.WithResolvers(xdsResolver), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + const numRPCs = 5 + received := checkRPCSendOK(ctx, t, client, numRPCs) + if len(received) != 1 { + t.Errorf("Got RPCs routed to %v backends, want %v", len(received), 1) + } + var got int + var initialAddr string + for initialAddr, got = range received { + } + if got != numRPCs { + t.Errorf("Got %v RPCs routed to a backend, want %v", got, numRPCs) + } + + // Due to the channel ID hashing policy, the request could go to the first + // address of either endpoint. + var backendIdx int + switch initialAddr { + case backendAddrs[0]: + backendIdx = 0 + case backendAddrs[2]: + backendIdx = 2 + default: + t.Fatalf("Request sent to unexpected backend: %q", initialAddr) + } + otherEndpointAddr := backendAddrs[backendIdx+1] + + // Shut down the previously used backend. + backends[backendIdx].Stop() + testutils.AwaitState(ctx, t, conn, connectivity.Idle) + + // Verify that the requests go to the remaining address in the same + // endpoint. + received = checkRPCSendOK(ctx, t, client, numRPCs) + if len(received) != 1 { + t.Errorf("Got RPCs routed to %v backends, want %v", len(received), 1) + } + var newAddr string + for newAddr, got = range received { + } + if got != numRPCs { + t.Errorf("Got %v RPCs routed to a backend, want %v", got, numRPCs) + } + + if newAddr != otherEndpointAddr { + t.Errorf("Requests went to unexpected address, got=%q, want=%q", newAddr, otherEndpointAddr) + } +} + +// Tests that ringhash is able to recover automatically in situations when a +// READY endpoint enters IDLE making the aggregated state TRANSIENT_FAILURE. The +// test creates 4 endpoints in the following connectivity states: [TF, TF, +// READY, IDLE]. The test fails the READY backend and verifies that the last +// IDLE endopint is dialed and the channel enters READY. +func (s) TestRingHash_RecoverWhenEndpointEntersIdle(t *testing.T) { + const backendsCount = 4 + backends := startTestServiceBackends(t, backendsCount) + backendAddrs := backendAddrs(backends) + + const clusterName = "cluster" + + endpoints := endpointResource(t, clusterName, backendAddrs) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := headerHashRoute("new_route", virtualHostName, clusterName, "address_hash") + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + dialer := testutils.NewBlockingDialer() + dialOpts := []grpc.DialOption{ + grpc.WithResolvers(xdsResolver), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(dialer.DialContext), + grpc.WithConnectParams(fastConnectParams), + } + conn, err := grpc.NewClient("xds:///test.server", dialOpts...) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + + // Create holds for each backend address to delay a successful connection + // until the end of the test. + holds := make([]*testutils.Hold, backendsCount) + for i := 0; i < len(backendAddrs); i++ { + holds[i] = dialer.Hold(backendAddrs[i]) + } + + client := testgrpc.NewTestServiceClient(conn) + + rpcCtx, rpcCancel := context.WithCancel(ctx) + errCh := make(chan error, 1) + go func() { + rpcCtx = metadata.NewOutgoingContext(rpcCtx, metadata.Pairs("address_hash", backendAddrs[0]+"_0")) + _, err := client.EmptyCall(rpcCtx, &testpb.Empty{}) + if status.Code(err) == codes.Canceled { + errCh <- nil + return + } + errCh <- err + }() + + // Wait for the RPC to trigger a connection attempt to the first address, + // then cancel the RPC. No other connection attempts should be started yet. + if !holds[0].Wait(ctx) { + t.Fatalf("Timeout waiting for connection attempt to backend 0") + } + rpcCancel() + if err := <-errCh; err != nil { + t.Fatalf("Expected RPC to fail be canceled, got %v", err) + } + + // The number of dialed backends increases by 1 in every iteration of the + // loop as ringhash tries to exit TRANSIENT_FAILURE. Run the loop twice to + // get two endpoints in TRANSIENT_FAILURE. + activeAddrs := map[string]bool{} + for wantFailingBackendCount := 1; wantFailingBackendCount <= 2; wantFailingBackendCount++ { + newAddrIdx := -1 + for ; ctx.Err() == nil && len(activeAddrs) < wantFailingBackendCount; <-time.After(time.Millisecond) { + for i, hold := range holds { + if !hold.IsStarted() { + continue + } + if _, ok := activeAddrs[backendAddrs[i]]; ok { + continue + } + activeAddrs[backendAddrs[i]] = true + newAddrIdx = i + } + } + + if ctx.Err() != nil { + t.Fatal("Context timed out waiting for new backneds to be dialed.") + } + if len(activeAddrs) > wantFailingBackendCount { + t.Fatalf("More backends dialed than expected: got %d, want %d", len(activeAddrs), wantFailingBackendCount) + } + + // Create a new hold for the address dialed in this iteration and fail + // the existing hold. + hold := holds[newAddrIdx] + holds[newAddrIdx] = dialer.Hold(backendAddrs[newAddrIdx]) + hold.Fail(errors.New("Test error")) + } + + // Current state of endpoints: [TF, TF, READY, IDLE]. + // Two endpoints failing should cause the channel to enter + // TRANSIENT_FAILURE. + testutils.AwaitState(ctx, t, conn, connectivity.TransientFailure) + + // Allow the request to the backend dialed next to succeed. + readyBackendIdx := -1 + for ; ctx.Err() == nil && readyBackendIdx == -1; <-time.After(time.Millisecond) { + for i, addr := range backendAddrs { + if _, ok := activeAddrs[addr]; ok || !holds[i].IsStarted() { + continue + } + readyBackendIdx = i + activeAddrs[addr] = true + holds[i].Resume() + break + } + } + + if ctx.Err() != nil { + t.Fatal("Context timed out waiting for the next backend to be contacted.") + } + + // Wait for channel to become READY without any pending RPC. + testutils.AwaitState(ctx, t, conn, connectivity.Ready) + + // Current state of endpoints: [TF, TF, READY, IDLE]. + // Stopping the READY backend should cause the channel to re-enter + // TRANSIENT_FAILURE. + backends[readyBackendIdx].Stop() + testutils.AwaitState(ctx, t, conn, connectivity.TransientFailure) + + // To recover from TRANSIENT_FAILURE, ringhash should automatically try to + // connect to the final endpoint. + readyBackendIdx = -1 + for ; ctx.Err() == nil && readyBackendIdx == -1; <-time.After(time.Millisecond) { + for i, addr := range backendAddrs { + if _, ok := activeAddrs[addr]; ok || !holds[i].IsStarted() { + continue + } + readyBackendIdx = i + activeAddrs[addr] = true + holds[i].Resume() + break + } + } + + if ctx.Err() != nil { + t.Fatal("Context timed out waiting for next backend to be contacted.") + } + + // Wait for channel to become READY without any pending RPC. + testutils.AwaitState(ctx, t, conn, connectivity.Ready) +} + +// Tests that ringhash is able to recover automatically in situations when a +// READY endpoint is removed by the resolver making the aggregated state +// TRANSIENT_FAILURE. The test creates 4 endpoints in the following +// connectivity states: [TF, TF, READY, IDLE]. The test removes the +// READY endpoint and verifies that the last IDLE endopint is dialed and the +// channel enters READY. +func (s) TestRingHash_RecoverWhenResolverRemovesEndpoint(t *testing.T) { + const backendsCount = 4 + backends := startTestServiceBackends(t, backendsCount) + backendAddrs := backendAddrs(backends) + + const clusterName = "cluster" + + endpoints := endpointResource(t, clusterName, backendAddrs) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := headerHashRoute("new_route", virtualHostName, clusterName, "address_hash") + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + dialer := testutils.NewBlockingDialer() + dialOpts := []grpc.DialOption{ + grpc.WithResolvers(xdsResolver), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(dialer.DialContext), + grpc.WithConnectParams(fastConnectParams), + } + conn, err := grpc.NewClient("xds:///test.server", dialOpts...) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + + // Create holds for each backend address to delay a successful connection + // until the end of the test. + holds := make([]*testutils.Hold, backendsCount) + for i := 0; i < len(backendAddrs); i++ { + holds[i] = dialer.Hold(backendAddrs[i]) + } + + client := testgrpc.NewTestServiceClient(conn) + + rpcCtx, rpcCancel := context.WithCancel(ctx) + errCh := make(chan error, 1) + go func() { + rpcCtx = metadata.NewOutgoingContext(rpcCtx, metadata.Pairs("address_hash", backendAddrs[0]+"_0")) + _, err := client.EmptyCall(rpcCtx, &testpb.Empty{}) + if status.Code(err) == codes.Canceled { + errCh <- nil + return + } + errCh <- err + }() + + // Wait for the RPC to trigger a connection attempt to the first address, + // then cancel the RPC. No other connection attempts should be started yet. + if !holds[0].Wait(ctx) { + t.Fatalf("Timeout waiting for connection attempt to backend 0") + } + rpcCancel() + if err := <-errCh; err != nil { + t.Fatalf("Expected RPC to fail be canceled, got %v", err) + } + + // The number of dialed backends increases by 1 in every iteration of the + // loop as ringhash tries to exit TRANSIENT_FAILURE. Run the loop twice to + // get two endpoints in TRANSIENT_FAILURE. + activeAddrs := map[string]bool{} + for wantFailingBackendCount := 1; wantFailingBackendCount <= 2; wantFailingBackendCount++ { + newAddrIdx := -1 + for ; ctx.Err() == nil && len(activeAddrs) < wantFailingBackendCount; <-time.After(time.Millisecond) { + for i, hold := range holds { + if !hold.IsStarted() { + continue + } + if _, ok := activeAddrs[backendAddrs[i]]; ok { + continue + } + activeAddrs[backendAddrs[i]] = true + newAddrIdx = i + } + } + + if ctx.Err() != nil { + t.Fatal("Context timed out waiting for new backneds to be dialed.") + } + if len(activeAddrs) > wantFailingBackendCount { + t.Fatalf("More backends dialed than expected: got %d, want %d", len(activeAddrs), wantFailingBackendCount) + } + + // Create a new hold for the address dialed in this iteration and fail + // the existing hold. + hold := holds[newAddrIdx] + holds[newAddrIdx] = dialer.Hold(backendAddrs[newAddrIdx]) + hold.Fail(errors.New("Test error")) + } + + // Current state of endpoints: [TF, TF, READY, IDLE]. + // Two endpoints failing should cause the channel to enter + // TRANSIENT_FAILURE. + testutils.AwaitState(ctx, t, conn, connectivity.TransientFailure) + + // Allow the request to the backend dialed next to succeed. + readyBackendIdx := -1 + for ; ctx.Err() == nil && readyBackendIdx == -1; <-time.After(time.Millisecond) { + for i, addr := range backendAddrs { + if _, ok := activeAddrs[addr]; ok || !holds[i].IsStarted() { + continue + } + readyBackendIdx = i + activeAddrs[addr] = true + holds[i].Resume() + break + } + } + + if ctx.Err() != nil { + t.Fatal("Context timed out waiting for the next backend to be contacted.") + } + + // Wait for channel to become READY without any pending RPC. + testutils.AwaitState(ctx, t, conn, connectivity.Ready) + + // Current state of endpoints: [TF, TF, READY, IDLE]. + // Removing the READY backend should cause the channel to re-enter + // TRANSIENT_FAILURE. + updatedAddrs := append([]string{}, backendAddrs[:readyBackendIdx]...) + updatedAddrs = append(updatedAddrs, backendAddrs[readyBackendIdx+1:]...) + updatedEndpoints := endpointResource(t, clusterName, updatedAddrs) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, updatedEndpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + testutils.AwaitState(ctx, t, conn, connectivity.TransientFailure) + + // To recover from TRANSIENT_FAILURE, ringhash should automatically try to + // connect to the final endpoint. + readyBackendIdx = -1 + for ; ctx.Err() == nil && readyBackendIdx == -1; <-time.After(time.Millisecond) { + for i, addr := range backendAddrs { + if _, ok := activeAddrs[addr]; ok || !holds[i].IsStarted() { + continue + } + readyBackendIdx = i + activeAddrs[addr] = true + holds[i].Resume() + break + } + } + + if ctx.Err() != nil { + t.Fatal("Context timed out waiting for next backend to be contacted.") + } + + // Wait for channel to become READY without any pending RPC. + testutils.AwaitState(ctx, t, conn, connectivity.Ready) +} + +// Tests that RPCs are routed according to endpoint hash key rather than +// endpoint first address if it is set in EDS endpoint metadata. +func (s) TestRingHash_EndpointHashKey(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSEndpointHashKeyBackwardCompat, false) + + backends := backendAddrs(startTestServiceBackends(t, 4)) + + const clusterName = "cluster" + var backendOpts []e2e.BackendOptions + for i, addr := range backends { + var ports []uint32 + ports = append(ports, testutils.ParsePort(t, addr)) + backendOpts = append(backendOpts, e2e.BackendOptions{ + Ports: ports, + Metadata: map[string]any{"hash_key": strconv.Itoa(i)}, + }) + } + endpoints := e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: clusterName, + Host: "localhost", + Localities: []e2e.LocalityOptions{{ + Backends: backendOpts, + Weight: 1, + }}, + }) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: clusterName, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + route := headerHashRoute("new_route", virtualHostName, clusterName, "address_hash") + listener := e2e.DefaultClientListener(virtualHostName, route.Name) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + xdsServer, nodeID, xdsResolver := setupManagementServerAndResolver(t) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + opts := []grpc.DialOption{ + grpc.WithResolvers(xdsResolver), + grpc.WithTransportCredentials(insecure.NewCredentials()), + } + conn, err := grpc.NewClient("xds:///test.server", opts...) + if err != nil { + t.Fatalf("Failed to create client: %s", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + + // Make sure RPCs are routed to backends according to the endpoint metadata + // rather than their address. Note each type of RPC contains a header value + // that will always be hashed to a specific backend as the header value + // matches the endpoint metadata hash key. + for i, backend := range backends { + ctx := metadata.NewOutgoingContext(ctx, metadata.Pairs("address_hash", strconv.Itoa(i)+"_0")) + numRPCs := 10 + reqPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + if reqPerBackend[backend] != numRPCs { + t.Errorf("Got RPC routed to addresses %v, want all RPCs routed to %v", reqPerBackend, backend) + } + } + + // Update the endpoints to swap the metadata hash key. + for i := range backendOpts { + backendOpts[i].Metadata = map[string]any{"hash_key": strconv.Itoa(len(backends) - i - 1)} + } + endpoints = e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: clusterName, + Host: "localhost", + Localities: []e2e.LocalityOptions{{ + Backends: backendOpts, + Weight: 1, + }}, + }) + if err := xdsServer.Update(ctx, xdsUpdateOpts(nodeID, endpoints, cluster, route, listener)); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + // Wait for the resolver update to make it to the balancer. This RPC should + // be routed to backend 3 with the reverse numbering of the hash_key + // attribute delivered above. + for { + ctx := metadata.NewOutgoingContext(ctx, metadata.Pairs("address_hash", "0_0")) + var remote peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&remote)); err != nil { + t.Fatalf("Unexpected RPC error waiting for EDS update propagation: %s", err) + } + if remote.Addr.String() == backends[3] { + break + } + } + + // Now that the balancer has the new endpoint attributes, make sure RPCs are + // routed to backends according to the new endpoint metadata. + for i, backend := range backends { + ctx := metadata.NewOutgoingContext(ctx, metadata.Pairs("address_hash", strconv.Itoa(len(backends)-i-1)+"_0")) + numRPCs := 10 + reqPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + if reqPerBackend[backend] != numRPCs { + t.Errorf("Got RPC routed to addresses %v, want all RPCs routed to %v", reqPerBackend, backend) + } + } +} + +// Tests that when a request hash key is set in the balancer configuration via +// service config, this header is used to route to a specific backend. +func (s) TestRingHash_RequestHashKey(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.RingHashSetRequestHashKey, true) + + backends := backendAddrs(startTestServiceBackends(t, 4)) + + // Create a clientConn with a manual resolver (which is used to push the + // address of the test backend), and a default service config pointing to + // the use of the ring_hash_experimental LB policy with an explicit hash + // header. + const ringHashServiceConfig = `{"loadBalancingConfig": [{"ring_hash_experimental":{"requestHashHeader":"address_hash"}}]}` + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(ringHashServiceConfig), + grpc.WithConnectParams(fastConnectParams), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + var endpoints []resolver.Endpoint + for _, backend := range backends { + endpoints = append(endpoints, resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: backend}}, + }) + } + r.UpdateState(resolver.State{ + Endpoints: endpoints, + }) + client := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Note each type of RPC contains a header value that will always be hashed + // to a specific backend as the header value matches the value used to + // create the entry in the ring. + for _, backend := range backends { + ctx := metadata.NewOutgoingContext(ctx, metadata.Pairs("address_hash", backend+"_0")) + numRPCs := 10 + reqPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + if reqPerBackend[backend] != numRPCs { + t.Errorf("Got RPC routed to addresses %v, want all RPCs routed to %v", reqPerBackend, backend) + } + } + + const ringHashServiceConfigUpdate = `{"loadBalancingConfig": [{"ring_hash_experimental":{"requestHashHeader":"other_header"}}]}` + r.UpdateState(resolver.State{ + Endpoints: endpoints, + ServiceConfig: (&testutils.ResolverClientConn{}).ParseServiceConfig(ringHashServiceConfigUpdate), + }) + + // Make sure that requests with the new hash are sent to the right backend. + for _, backend := range backends { + ctx := metadata.NewOutgoingContext(ctx, metadata.Pairs("other_header", backend+"_0")) + numRPCs := 10 + reqPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + if reqPerBackend[backend] != numRPCs { + t.Errorf("Got RPC routed to addresses %v, want all RPCs routed to %v", reqPerBackend, backend) + } + } +} + +func highRingSizeServiceConfig(t *testing.T) string { + t.Helper() + testutils.SetEnvConfig(t, &envconfig.RingHashCap, minRingSize) + + return fmt.Sprintf(`{ + "loadBalancingConfig": [{"ring_hash_experimental":{ + "requestHashHeader": "address_hash", + "minRingSize": %d, + "maxRingSize": %d + } +}]}`, minRingSize, minRingSize) +} + +// Tests that when a request hash key is set in the balancer configuration via +// service config, and the header is not set in the outgoing request, then it +// is sent to a random backend. +func (s) TestRingHash_RequestHashKeyRandom(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.RingHashSetRequestHashKey, true) + + backends := backendAddrs(startTestServiceBackends(t, 4)) + + // Create a clientConn with a manual resolver (which is used to push the + // address of the test backend), and a default service config pointing to + // the use of the ring_hash_experimental LB policy with an explicit hash + // header. + ringHashServiceConfig := highRingSizeServiceConfig(t) + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(ringHashServiceConfig), + grpc.WithConnectParams(fastConnectParams), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + var endpoints []resolver.Endpoint + for _, backend := range backends { + endpoints = append(endpoints, resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: backend}}, + }) + } + r.UpdateState(resolver.State{ + Endpoints: endpoints, + }) + client := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Due to the way that ring hash lazily establishes connections when using a + // random hash, request distribution is skewed towards the order in which we + // connected. The test send RPCs until we are connected to all backends, so + // we can later assert that the distribution is uniform. + seen := make(map[string]bool) + for len(seen) != 4 { + var remote peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&remote)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + seen[remote.String()] = true + } + + // Make sure that requests with the old hash are sent to random backends. + const want = 1.0 / 4 + numRPCs := computeIdealNumberOfRPCs(t, want, errorTolerance) + gotPerBackend := checkRPCSendOK(ctx, t, client, numRPCs) + for _, backend := range backends { + got := float64(gotPerBackend[backend]) / float64(numRPCs) + if !cmp.Equal(got, want, cmpopts.EquateApprox(0, errorTolerance)) { + t.Errorf("Fraction of RPCs to backend %s: got %v, want %v (margin: +-%v)", backend, got, want, errorTolerance) + } + } +} + +// Tests that when a request hash key is set in the balancer configuration via +// service config, and the header is not set in the outgoing request (random +// behavior), then each RPC wakes up at most one SubChannel, and, if there are +// SubChannels in Ready state, RPCs are routed to them. +func (s) TestRingHash_RequestHashKeyConnecting(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.RingHashSetRequestHashKey, true) + + backends := backendAddrs(startTestServiceBackends(t, 20)) + + // Create a clientConn with a manual resolver (which is used to push the + // address of the test backend), and a default service config pointing to + // the use of the ring_hash_experimental LB policy with an explicit hash + // header. Use a blocking dialer to control connection attempts. + const ringHashServiceConfig = `{"loadBalancingConfig": [ + {"ring_hash_experimental":{"requestHashHeader":"address_hash"}} + ]}` + r := manual.NewBuilderWithScheme("whatever") + blockingDialer := testutils.NewBlockingDialer() + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(ringHashServiceConfig), + grpc.WithConnectParams(fastConnectParams), + grpc.WithContextDialer(blockingDialer.DialContext), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + var endpoints []resolver.Endpoint + for _, backend := range backends { + endpoints = append(endpoints, resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: backend}}, + }) + } + r.UpdateState(resolver.State{ + Endpoints: endpoints, + }) + client := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Intercept all connection attempts to the backends. + var holds []*testutils.Hold + for i := 0; i < len(backends); i++ { + holds = append(holds, blockingDialer.Hold(backends[i])) + } + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + // Send 1 RPC and make sure this triggers at most 1 connection attempt. + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err != nil { + t.Errorf("EmptyCall(): got %v, want success", err) + } + wg.Done() + }() + + // Wait for at least one connection attempt. + nConn := 0 + for nConn == 0 { + if ctx.Err() != nil { + t.Fatal("Test timed out waiting for a connection attempt") + } + time.Sleep(1 * time.Millisecond) + for _, hold := range holds { + if hold.IsStarted() { + nConn++ + } + } + } + if wantMaxConn := 1; nConn > wantMaxConn { + t.Fatalf("Got %d connection attempts, want at most %d", nConn, wantMaxConn) + } + + // Do a second RPC. Since there should already be a SubChannel in + // Connecting state, this should not trigger a connection attempt. + wg.Add(1) + go func() { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err != nil { + t.Errorf("EmptyCall(): got %v, want success", err) + } + wg.Done() + }() + + // Give extra time for more connections to be attempted. + time.Sleep(defaultTestShortTimeout) + + var firstConnectedBackend string + nConn = 0 + for i, hold := range holds { + if hold.IsStarted() { + // Unblock the connection attempt. The SubChannel (and hence the + // channel) should transition to Ready. RPCs should succeed and + // be routed to this backend. + hold.Resume() + holds[i] = nil + firstConnectedBackend = backends[i] + nConn++ + } + } + if wantMaxConn := 1; nConn > wantMaxConn { + t.Fatalf("Got %d connection attempts, want at most %d", nConn, wantMaxConn) + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + wg.Wait() // Make sure we're done with the 2 previous RPCs. + + // Now send RPCs until we have at least one more connection attempt, that + // is, the random hash did not land on the same backend on every pick (the + // chances are low, but we don't want this to be flaky). Make sure no RPC + // fails and that we route all of them to the only subchannel in ready + // state. + nConn = 0 + for nConn == 0 { + p := peer.Peer{} + _, err = client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&p)) + if status.Code(err) == codes.DeadlineExceeded { + t.Fatal("EmptyCall(): test timed out while waiting for more connection attempts") + } + if err != nil { + t.Fatalf("EmptyCall(): got %v, want success", err) + } + if p.Addr.String() != firstConnectedBackend { + t.Errorf("RPC sent to backend %q, want %q", p.Addr.String(), firstConnectedBackend) + } + for _, hold := range holds { + if hold != nil && hold.IsStarted() { + nConn++ + } + } + } +} diff --git a/balancer/ringhash/ringhash_test.go b/balancer/ringhash/ringhash_test.go new file mode 100644 index 0000000..6155649 --- /dev/null +++ b/balancer/ringhash/ringhash_test.go @@ -0,0 +1,739 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "context" + "fmt" + "testing" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/experimental/balancer/weight" + "google.golang.org/grpc/internal/grpctest" + iringhash "google.golang.org/grpc/internal/ringhash" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" +) + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + + testBackendAddrsCount = 12 +) + +var ( + testBackendAddrStrs []string + testConfig = &iringhash.LBConfig{MinRingSize: 1, MaxRingSize: 10} +) + +func init() { + for i := 0; i < testBackendAddrsCount; i++ { + testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)) + } +} + +// setupTest creates the balancer, and does an initial sanity check. +func setupTest(t *testing.T, endpoints []resolver.Endpoint) (*testutils.BalancerClientConn, balancer.Balancer, balancer.Picker) { + t.Helper() + cc := testutils.NewBalancerClientConn(t) + builder := balancer.Get(Name) + b := builder.Build(cc, balancer.BuildOptions{}) + if b == nil { + t.Fatalf("builder.Build(%s) failed and returned nil", Name) + } + t.Cleanup(func() { b.Close() }) + + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: endpoints}, + BalancerConfig: testConfig, + }); err != nil { + t.Fatalf("UpdateClientConnState returned err: %v", err) + } + + // The leaf pickfirst are created lazily, only when their endpoint is picked + // or other endpoints are in TF. No SubConns should be created immediately. + select { + case sc := <-cc.NewSubConnCh: + t.Errorf("unexpected SubConn creation: %v", sc) + case <-time.After(defaultTestShortTimeout): + } + + // Should also have a picker, with all endpoints in Idle. + p1 := <-cc.NewPickerCh + + ringHashPicker := p1.(*picker) + if got, want := len(ringHashPicker.endpointStates), len(endpoints); got != want { + t.Errorf("Number of child balancers = %d, want = %d", got, want) + } + for firstAddr, bs := range ringHashPicker.endpointStates { + if got, want := bs.state.ConnectivityState, connectivity.Idle; got != want { + t.Errorf("Child balancer connectivity state for address %q = %v, want = %v", firstAddr, got, want) + } + } + return cc, b, p1 +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestUpdateClientConnState_NewRingSize tests the scenario where the ringhash +// LB policy receives new configuration which specifies new values for the ring +// min and max sizes. The test verifies that a new ring is created and a new +// picker is sent to the ClientConn. +func (s) TestUpdateClientConnState_NewRingSize(t *testing.T) { + origMinRingSize, origMaxRingSize := 1, 10 // Configured from `testConfig` in `setupTest` + newMinRingSize, newMaxRingSize := 20, 100 + + endpoints := []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}} + cc, b, p1 := setupTest(t, endpoints) + ring1 := p1.(*picker).ring + if ringSize := len(ring1.items); ringSize < origMinRingSize || ringSize > origMaxRingSize { + t.Fatalf("Ring created with size %d, want between [%d, %d]", ringSize, origMinRingSize, origMaxRingSize) + } + + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: endpoints}, + BalancerConfig: &iringhash.LBConfig{ + MinRingSize: uint64(newMinRingSize), + MaxRingSize: uint64(newMaxRingSize), + }, + }); err != nil { + t.Fatalf("UpdateClientConnState returned err: %v", err) + } + + var ring2 *ring + select { + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout when waiting for a picker update after a configuration update") + case p2 := <-cc.NewPickerCh: + ring2 = p2.(*picker).ring + } + if ringSize := len(ring2.items); ringSize < newMinRingSize || ringSize > newMaxRingSize { + t.Fatalf("Ring created with size %d, want between [%d, %d]", ringSize, newMinRingSize, newMaxRingSize) + } +} + +func (s) TestOneEndpoint(t *testing.T) { + wantAddr1 := resolver.Address{Addr: testBackendAddrStrs[0]} + cc, _, p0 := setupTest(t, []resolver.Endpoint{{Addresses: []resolver.Address{wantAddr1}}}) + ring0 := p0.(*picker).ring + + firstHash := ring0.items[0].hash + // firstHash-1 will pick the first (and only) SubConn from the ring. + testHash := firstHash - 1 + // The first pick should be queued, and should trigger a connection to the + // only Endpoint which has a single address. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := p0.Pick(balancer.PickInfo{Ctx: iringhash.SetXDSRequestHash(ctx, testHash)}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("first pick returned err %v, want %v", err, balancer.ErrNoSubConnAvailable) + } + var sc0 *testutils.TestSubConn + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for SubConn creation.") + case sc0 = <-cc.NewSubConnCh: + } + if got, want := sc0.Addresses[0].Addr, wantAddr1.Addr; got != want { + t.Fatalf("SubConn.Addresses = %v, want = %v", got, want) + } + select { + case <-sc0.ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc0) + } + + // Send state updates to Ready. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + if err := cc.WaitForConnectivityState(ctx, connectivity.Ready); err != nil { + t.Fatal(err) + } + + // Test pick with one backend. + p1 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, _ := p1.Pick(balancer.PickInfo{Ctx: iringhash.SetXDSRequestHash(ctx, testHash)}) + if gotSCSt.SubConn != sc0 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc0) + } + } +} + +// TestThreeBackendsAffinity covers that there are 3 SubConns, RPCs with the +// same hash always pick the same SubConn. When the one picked is down, another +// one will be picked. +func (s) TestThreeSubConnsAffinity(t *testing.T) { + endpoints := []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, + {Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, + {Addresses: []resolver.Address{{Addr: testBackendAddrStrs[2]}}}, + } + remainingAddrs := map[string]bool{ + testBackendAddrStrs[0]: true, + testBackendAddrStrs[1]: true, + testBackendAddrStrs[2]: true, + } + cc, _, p0 := setupTest(t, endpoints) + // This test doesn't update addresses, so this ring will be used by all the + // pickers. + ring := p0.(*picker).ring + + firstHash := ring.items[0].hash + // firstHash+1 will pick the second endpoint from the ring. + testHash := firstHash + 1 + // The first pick should be queued, and should trigger Connect() on the only + // SubConn. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := p0.Pick(balancer.PickInfo{Ctx: iringhash.SetXDSRequestHash(ctx, testHash)}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("first pick returned err %v, want %v", err, balancer.ErrNoSubConnAvailable) + } + + // The picked endpoint should be the second in the ring. + var subConns [3]*testutils.TestSubConn + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for SubConn creation.") + case subConns[1] = <-cc.NewSubConnCh: + } + if got, want := subConns[1].Addresses[0].Addr, ring.items[1].hashKey; got != want { + t.Fatalf("SubConn.Address = %v, want = %v", got, want) + } + select { + case <-subConns[1].ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", subConns[1]) + } + delete(remainingAddrs, ring.items[1].hashKey) + + // Turn down the subConn in use. + subConns[1].UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + subConns[1].UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + + // This should trigger a connection to a new endpoint. + <-cc.NewPickerCh + var sc *testutils.TestSubConn + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for SubConn creation.") + case sc = <-cc.NewSubConnCh: + } + scAddr := sc.Addresses[0].Addr + if _, ok := remainingAddrs[scAddr]; !ok { + t.Fatalf("New SubConn created with previously used address: %q", scAddr) + } + delete(remainingAddrs, scAddr) + select { + case <-sc.ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", subConns[1]) + } + if scAddr == ring.items[0].hashKey { + subConns[0] = sc + } else if scAddr == ring.items[2].hashKey { + subConns[2] = sc + } + + // Turning down the SubConn should cause creation of a connection to the + // final endpoint. + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for SubConn creation.") + case sc = <-cc.NewSubConnCh: + } + scAddr = sc.Addresses[0].Addr + if _, ok := remainingAddrs[scAddr]; !ok { + t.Fatalf("New SubConn created with previously used address: %q", scAddr) + } + delete(remainingAddrs, scAddr) + select { + case <-sc.ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", subConns[1]) + } + if scAddr == ring.items[0].hashKey { + subConns[0] = sc + } else if scAddr == ring.items[2].hashKey { + subConns[2] = sc + } + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + + // All endpoints are in TransientFailure. Make the first endpoint in the + // ring report Ready. All picks should go to this endpoint which is two + // indexes away from the endpoint with the chosen hash. + subConns[0].UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + subConns[0].UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + subConns[0].UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + if err := cc.WaitForConnectivityState(ctx, connectivity.Ready); err != nil { + t.Fatalf("Context timed out while waiting for channel to report Ready.") + } + p1 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, _ := p1.Pick(balancer.PickInfo{Ctx: iringhash.SetXDSRequestHash(ctx, testHash)}) + if gotSCSt.SubConn != subConns[0] { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, subConns[0]) + } + } + + // Make the last endpoint in the ring report Ready. All picks should go to + // this endpoint since it is one index away from the chosen hash. + subConns[2].UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + subConns[2].UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + subConns[2].UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p2 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, _ := p2.Pick(balancer.PickInfo{Ctx: iringhash.SetXDSRequestHash(ctx, testHash)}) + if gotSCSt.SubConn != subConns[2] { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, subConns[2]) + } + } + + // Make the second endpoint in the ring report Ready. All picks should go to + // this endpoint as it is the one with the chosen hash. + subConns[1].UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + subConns[1].UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + subConns[1].UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p3 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, _ := p3.Pick(balancer.PickInfo{Ctx: iringhash.SetXDSRequestHash(ctx, testHash)}) + if gotSCSt.SubConn != subConns[1] { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, subConns[1]) + } + } +} + +// TestThreeBackendsAffinity covers that there are 3 SubConns, RPCs with the +// same hash always pick the same SubConn. Then try different hash to pick +// another backend, and verify the first hash still picks the first backend. +func (s) TestThreeBackendsAffinityMultiple(t *testing.T) { + wantEndpoints := []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, + {Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, + {Addresses: []resolver.Address{{Addr: testBackendAddrStrs[2]}}}, + } + cc, _, p0 := setupTest(t, wantEndpoints) + // This test doesn't update addresses, so this ring will be used by all the + // pickers. + ring0 := p0.(*picker).ring + + firstHash := ring0.items[0].hash + // firstHash+1 will pick the second SubConn from the ring. + testHash := firstHash + 1 + // The first pick should be queued, and should trigger Connect() on the only + // SubConn. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := p0.Pick(balancer.PickInfo{Ctx: iringhash.SetXDSRequestHash(ctx, testHash)}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("first pick returned err %v, want %v", err, balancer.ErrNoSubConnAvailable) + } + // The picked SubConn should be the second in the ring. + var sc0 *testutils.TestSubConn + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for SubConn creation.") + case sc0 = <-cc.NewSubConnCh: + } + if got, want := sc0.Addresses[0].Addr, ring0.items[1].hashKey; got != want { + t.Fatalf("SubConn.Address = %v, want = %v", got, want) + } + select { + case <-sc0.ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc0) + } + + // Send state updates to Ready. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + if err := cc.WaitForConnectivityState(ctx, connectivity.Ready); err != nil { + t.Fatal(err) + } + + // First hash should always pick sc0. + p1 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, _ := p1.Pick(balancer.PickInfo{Ctx: iringhash.SetXDSRequestHash(ctx, testHash)}) + if gotSCSt.SubConn != sc0 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc0) + } + } + + secondHash := ring0.items[1].hash + // secondHash+1 will pick the third SubConn from the ring. + testHash2 := secondHash + 1 + if _, err := p0.Pick(balancer.PickInfo{Ctx: iringhash.SetXDSRequestHash(ctx, testHash2)}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("first pick returned err %v, want %v", err, balancer.ErrNoSubConnAvailable) + } + var sc1 *testutils.TestSubConn + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for SubConn creation.") + case sc1 = <-cc.NewSubConnCh: + } + if got, want := sc1.Addresses[0].Addr, ring0.items[2].hashKey; got != want { + t.Fatalf("SubConn.Address = %v, want = %v", got, want) + } + select { + case <-sc1.ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc1) + } + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // With the new generated picker, hash2 always picks sc1. + p2 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, _ := p2.Pick(balancer.PickInfo{Ctx: iringhash.SetXDSRequestHash(ctx, testHash2)}) + if gotSCSt.SubConn != sc1 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) + } + } + // But the first hash still picks sc0. + for i := 0; i < 5; i++ { + gotSCSt, _ := p2.Pick(balancer.PickInfo{Ctx: iringhash.SetXDSRequestHash(ctx, testHash)}) + if gotSCSt.SubConn != sc0 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc0) + } + } +} + +// TestAddrWeightChange covers the following scenarios after setting up the +// balancer with 3 addresses [A, B, C]: +// - updates balancer with [A, B, C], a new Picker should not be sent. +// - updates balancer with [A, B] (C removed), a new Picker is sent and the +// ring is updated. +// - updates balancer with [A, B], but B has a weight of 2, a new Picker is +// sent. And the new ring should contain the correct number of entries +// and weights. +func (s) TestAddrWeightChange(t *testing.T) { + endpoints := []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, + {Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, + {Addresses: []resolver.Address{{Addr: testBackendAddrStrs[2]}}}, + } + cc, b, p0 := setupTest(t, endpoints) + ring0 := p0.(*picker).ring + + // Update with the same addresses, it will result in a new picker, but with + // the same ring. + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: endpoints}, + BalancerConfig: testConfig, + }); err != nil { + t.Fatalf("UpdateClientConnState returned err: %v", err) + } + var p1 balancer.Picker + select { + case p1 = <-cc.NewPickerCh: + case <-time.After(defaultTestTimeout): + t.Fatalf("timeout waiting for picker after UpdateClientConn with same addresses") + } + ring1 := p1.(*picker).ring + if ring1 != ring0 { + t.Fatalf("new picker with same address has a different ring than before, want same") + } + + // Delete an address, should send a new Picker. + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: endpoints[:2]}, + BalancerConfig: testConfig, + }); err != nil { + t.Fatalf("UpdateClientConnState returned err: %v", err) + } + var p2 balancer.Picker + select { + case p2 = <-cc.NewPickerCh: + case <-time.After(defaultTestTimeout): + t.Fatalf("timeout waiting for picker after UpdateClientConn with different addresses") + } + ring2 := p2.(*picker).ring + if ring2 == ring0 { + t.Fatalf("new picker after removing address has the same ring as before, want different") + } + + // Another update with the same addresses, but different weight. + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + endpoints[0], + weight.Set(endpoints[1], weight.EndpointInfo{Weight: 2}), + }}, + BalancerConfig: testConfig, + }); err != nil { + t.Fatalf("UpdateClientConnState returned err: %v", err) + } + var p3 balancer.Picker + select { + case p3 = <-cc.NewPickerCh: + case <-time.After(defaultTestTimeout): + t.Fatalf("timeout waiting for picker after UpdateClientConn with different addresses") + } + if p3.(*picker).ring == ring2 { + t.Fatalf("new picker after changing address weight has the same ring as before, want different") + } + // With the new update, the ring must look like this: + // [ + // {idx:0 endpoint: {addr: testBackendAddrStrs[0], weight: 1}}, + // {idx:1 endpoint: {addr: testBackendAddrStrs[1], weight: 2}}, + // {idx:2 endpoint: {addr: testBackendAddrStrs[2], weight: 1}}, + // ]. + if len(p3.(*picker).ring.items) != 3 { + t.Fatalf("new picker after changing address weight has %d entries, want 3", len(p3.(*picker).ring.items)) + } + for _, i := range p3.(*picker).ring.items { + if i.hashKey == testBackendAddrStrs[0] { + if i.weight != 1 { + t.Fatalf("new picker after changing address weight has weight %d for %v, want 1", i.weight, i.hashKey) + } + } + if i.hashKey == testBackendAddrStrs[1] { + if i.weight != 2 { + t.Fatalf("new picker after changing address weight has weight %d for %v, want 2", i.weight, i.hashKey) + } + } + } +} + +// TestAutoConnectEndpointOnTransientFailure covers the situation when an +// endpoint fails. It verifies that a new endpoint is automatically tried +// (without a pick) when there is no endpoint already in Connecting state. +func (s) TestAutoConnectEndpointOnTransientFailure(t *testing.T) { + wantEndpoints := []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, + {Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, + {Addresses: []resolver.Address{{Addr: testBackendAddrStrs[2]}}}, + {Addresses: []resolver.Address{{Addr: testBackendAddrStrs[3]}}}, + } + cc, _, p0 := setupTest(t, wantEndpoints) + + // ringhash won't tell SCs to connect until there is an RPC, so simulate + // one now. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + ctx = iringhash.SetXDSRequestHash(ctx, 0) + defer cancel() + p0.Pick(balancer.PickInfo{Ctx: ctx}) + + // The picked SubConn should be the second in the ring. + var sc0 *testutils.TestSubConn + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for SubConn creation.") + case sc0 = <-cc.NewSubConnCh: + } + select { + case <-sc0.ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc0) + } + + // Turn the first subconn to transient failure. This should set the overall + // connectivity state to CONNECTING. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + cc.WaitForConnectivityState(ctx, connectivity.Connecting) + + // It will trigger the second subconn to connect since there is only one + // endpoint, which is in TF. + var sc1 *testutils.TestSubConn + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for SubConn creation.") + case sc1 = <-cc.NewSubConnCh: + } + select { + case <-sc1.ConnectCh: + case <-time.After(defaultTestShortTimeout): + t.Fatalf("timeout waiting for Connect() from SubConn %v", sc1) + } + + // Turn the second subconn to TF. This will set the overall state to TF. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + cc.WaitForConnectivityState(ctx, connectivity.TransientFailure) + + // It will trigger the third subconn to connect. + var sc2 *testutils.TestSubConn + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for SubConn creation.") + case sc2 = <-cc.NewSubConnCh: + } + select { + case <-sc2.ConnectCh: + case <-time.After(defaultTestShortTimeout): + t.Fatalf("timeout waiting for Connect() from SubConn %v", sc2) + } + + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + + // Send the first SubConn into CONNECTING. To do this, first make it READY, + // then CONNECTING. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + cc.WaitForConnectivityState(ctx, connectivity.Ready) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + // Since one endpoint is in TF and one in CONNECTING, the aggregated state + // will be CONNECTING. + cc.WaitForConnectivityState(ctx, connectivity.Connecting) + p1 := <-cc.NewPickerCh + p1.Pick(balancer.PickInfo{Ctx: ctx}) + select { + case <-sc0.ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc0) + } + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + + // This will not trigger any new SubCOnns to be created, because sc0 is + // still attempting to connect, and we only need one SubConn to connect. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + + select { + case sc := <-cc.NewSubConnCh: + t.Fatalf("unexpected SubConn creation: %v", sc) + case <-sc0.ConnectCh: + t.Fatalf("unexpected Connect() from SubConn %v", sc0) + case <-sc1.ConnectCh: + t.Fatalf("unexpected Connect() from SubConn %v", sc1) + case <-sc2.ConnectCh: + t.Fatalf("unexpected Connect() from SubConn %v", sc2) + case <-time.After(defaultTestShortTimeout): + } +} + +func (s) TestAggregatedConnectivityState(t *testing.T) { + tests := []struct { + name string + endpointStates []connectivity.State + want connectivity.State + }{ + { + name: "one ready", + endpointStates: []connectivity.State{connectivity.Ready}, + want: connectivity.Ready, + }, + { + name: "one connecting", + endpointStates: []connectivity.State{connectivity.Connecting}, + want: connectivity.Connecting, + }, + { + name: "one ready one transient failure", + endpointStates: []connectivity.State{connectivity.Ready, connectivity.TransientFailure}, + want: connectivity.Ready, + }, + { + name: "one connecting one transient failure", + endpointStates: []connectivity.State{connectivity.Connecting, connectivity.TransientFailure}, + want: connectivity.Connecting, + }, + { + name: "one connecting two transient failure", + endpointStates: []connectivity.State{connectivity.Connecting, connectivity.TransientFailure, connectivity.TransientFailure}, + want: connectivity.TransientFailure, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + bal := &ringhashBalancer{endpointStates: resolver.NewEndpointMap[*endpointState]()} + for i, cs := range tt.endpointStates { + es := &endpointState{ + state: balancer.State{ConnectivityState: cs}, + } + ep := resolver.Endpoint{Addresses: []resolver.Address{{Addr: fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)}}} + bal.endpointStates.Set(ep, es) + } + if got := bal.aggregatedStateLocked(); got != tt.want { + t.Errorf("recordTransition() = %v, want %v", got, tt.want) + } + }) + } +} + +type testKeyType string + +const testKey testKeyType = "grpc.lb.ringhash.testKey" + +type testAttribute struct { + content string +} + +func setTestAttrAddr(addr resolver.Address, content string) resolver.Address { + addr.BalancerAttributes = addr.BalancerAttributes.WithValue(testKey, testAttribute{content}) + return addr +} + +func setTestAttrEndpoint(endpoint resolver.Endpoint, content string) resolver.Endpoint { + endpoint.Attributes = endpoint.Attributes.WithValue(testKey, testAttribute{content}) + return endpoint +} + +// TestAddrBalancerAttributesChange tests the case where the ringhash balancer +// receives a ClientConnUpdate with the same config and addresses as received in +// the previous update. Although the `BalancerAttributes` and endpoint +// attributes contents are the same, the pointers are different. This test +// verifies that subConns are not recreated in this scenario. +func (s) TestAddrBalancerAttributesChange(t *testing.T) { + content := "test" + addrs1 := []resolver.Address{setTestAttrAddr(resolver.Address{Addr: testBackendAddrStrs[0]}, content)} + wantEndpoints1 := []resolver.Endpoint{ + setTestAttrEndpoint(resolver.Endpoint{Addresses: addrs1}, "content"), + } + cc, b, p0 := setupTest(t, wantEndpoints1) + ring0 := p0.(*picker).ring + + firstHash := ring0.items[0].hash + // The first pick should be queued, and should trigger a connection to the + // only Endpoint which has a single address. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := p0.Pick(balancer.PickInfo{Ctx: iringhash.SetXDSRequestHash(ctx, firstHash)}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("first pick returned err %v, want %v", err, balancer.ErrNoSubConnAvailable) + } + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for SubConn creation.") + case <-cc.NewSubConnCh: + } + + addrs2 := []resolver.Address{setTestAttrAddr(resolver.Address{Addr: testBackendAddrStrs[0]}, content)} + wantEndpoints2 := []resolver.Endpoint{setTestAttrEndpoint(resolver.Endpoint{Addresses: addrs2}, content)} + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: wantEndpoints2}, + BalancerConfig: testConfig, + }); err != nil { + t.Fatalf("UpdateClientConnState returned err: %v", err) + } + select { + case <-cc.NewSubConnCh: + t.Fatal("new subConn created for an update with the same addresses") + case <-time.After(defaultTestShortTimeout): + } +} diff --git a/balancer/rls/balancer.go b/balancer/rls/balancer.go new file mode 100644 index 0000000..6d414dd --- /dev/null +++ b/balancer/rls/balancer.go @@ -0,0 +1,733 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package rls implements the RLS LB policy. +package rls + +import ( + "encoding/json" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + "unsafe" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/balancergroup" + "google.golang.org/grpc/internal/buffer" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" +) + +const ( + // Name is the name of the RLS LB policy. + // + // It currently has an experimental suffix which would be removed once + // end-to-end testing of the policy is completed. + Name = internal.RLSLoadBalancingPolicyName + // Default frequency for data cache purging. + periodicCachePurgeFreq = time.Minute +) + +var ( + logger = grpclog.Component("rls") + errBalancerClosed = errors.New("rls LB policy is closed") + + // Below defined vars for overriding in unit tests. + + // Default exponential backoff strategy for data cache entries. + defaultBackoffStrategy = backoff.Strategy(backoff.DefaultExponential) + // Ticker used for periodic data cache purging. + dataCachePurgeTicker = func() *time.Ticker { return time.NewTicker(periodicCachePurgeFreq) } + // We want every cache entry to live in the cache for at least this + // duration. If we encounter a cache entry whose minimum expiration time is + // in the future, we abort the LRU pass, which may temporarily leave the + // cache being too large. This is necessary to ensure that in cases where + // the cache is too small, when we receive an RLS Response, we keep the + // resulting cache entry around long enough for the pending incoming + // requests to be re-processed through the new Picker. If we didn't do this, + // then we'd risk throwing away each RLS response as we receive it, in which + // case we would fail to actually route any of our incoming requests. + minEvictDuration = 5 * time.Second + + // Following functions are no-ops in actual code, but can be overridden in + // tests to give tests visibility into exactly when certain events happen. + clientConnUpdateHook = func() {} + dataCachePurgeHook = func() {} + resetBackoffHook = func() {} + + cacheEntriesMetric = estats.RegisterInt64AsyncGauge(estats.MetricDescriptor{ + Name: "grpc.lb.rls.cache_entries", + Description: "EXPERIMENTAL. Number of entries in the RLS cache.", + Unit: "{entry}", + Labels: []string{"grpc.target", "grpc.lb.rls.server_target", "grpc.lb.rls.instance_uuid"}, + Default: false, + }) + cacheSizeMetric = estats.RegisterInt64AsyncGauge(estats.MetricDescriptor{ + Name: "grpc.lb.rls.cache_size", + Description: "EXPERIMENTAL. The current size of the RLS cache.", + Unit: "By", + Labels: []string{"grpc.target", "grpc.lb.rls.server_target", "grpc.lb.rls.instance_uuid"}, + Default: false, + }) + defaultTargetPicksMetric = estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "grpc.lb.rls.default_target_picks", + Description: "EXPERIMENTAL. Number of LB picks sent to the default target.", + Unit: "{pick}", + Labels: []string{"grpc.target", "grpc.lb.rls.server_target", "grpc.lb.rls.data_plane_target", "grpc.lb.pick_result"}, + OptionalLabels: []string{"grpc.client.call.custom"}, + Default: false, + }) + targetPicksMetric = estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "grpc.lb.rls.target_picks", + Description: "EXPERIMENTAL. Number of LB picks sent to each RLS target. Note that if the default target is also returned by the RLS server, RPCs sent to that target from the cache will be counted in this metric, not in grpc.rls.default_target_picks.", + Unit: "{pick}", + Labels: []string{"grpc.target", "grpc.lb.rls.server_target", "grpc.lb.rls.data_plane_target", "grpc.lb.pick_result"}, + OptionalLabels: []string{"grpc.client.call.custom"}, + Default: false, + }) + failedPicksMetric = estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "grpc.lb.rls.failed_picks", + Description: "EXPERIMENTAL. Number of LB picks failed due to either a failed RLS request or the RLS channel being throttled.", + Unit: "{pick}", + Labels: []string{"grpc.target", "grpc.lb.rls.server_target"}, + OptionalLabels: []string{"grpc.client.call.custom"}, + Default: false, + }) +) + +func init() { + balancer.Register(&rlsBB{}) +} + +type rlsBB struct{} + +func (rlsBB) Name() string { + return Name +} + +func (rlsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + lb := &rlsBalancer{ + closed: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + cc: cc, + bopts: opts, + purgeTicker: dataCachePurgeTicker(), + dataCachePurgeHook: dataCachePurgeHook, + lbCfg: &lbConfig{}, + pendingMap: make(map[cacheKey]*backoffState), + childPolicies: make(map[string]*childPolicyWrapper), + updateCh: buffer.NewUnbounded[any](), + } + lb.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[rls-experimental-lb %p] ", lb)) + lb.dataCache = newDataCache(maxCacheSize, lb.logger, opts.Target.String()) + metricsRecorder := cc.MetricsRecorder() + lb.unregisterMetricHandler = metricsRecorder.RegisterAsyncReporter(lb, cacheEntriesMetric, cacheSizeMetric) + lb.bg = balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: opts, + StateAggregator: lb, + Logger: lb.logger, + SubBalancerCloseTimeout: time.Duration(0), // Disable caching of removed child policies + }) + go lb.run() + return lb +} + +// rlsBalancer implements the RLS LB policy. +type rlsBalancer struct { + closed *grpcsync.Event // Fires when Close() is invoked. Guarded by stateMu. + done *grpcsync.Event // Fires when Close() is done. + cc balancer.ClientConn + bopts balancer.BuildOptions + purgeTicker *time.Ticker + dataCachePurgeHook func() + logger *internalgrpclog.PrefixLogger + + // unregisterMetricHandler is the function to deregister the async metric reporter. + unregisterMetricHandler func() + + // If both cacheMu and stateMu need to be acquired, the former must be + // acquired first to prevent a deadlock. This order restriction is due to the + // fact that in places where we need to acquire both the locks, we always + // start off reading the cache. + + // cacheMu guards access to the data cache and pending requests map. We + // cannot use an RWMutex here since even an operation like + // dataCache.getEntry() modifies the underlying LRU, which is implemented as + // a doubly linked list. + cacheMu sync.Mutex + dataCache *dataCache // Cache of RLS data. + pendingMap map[cacheKey]*backoffState // Map of pending RLS requests. + + // stateMu guards access to all LB policy state. + stateMu sync.Mutex + lbCfg *lbConfig // Most recently received service config. + childPolicyBuilder balancer.Builder // Cached child policy builder. + resolverState resolver.State // Cached resolver state. + ctrlCh *controlChannel // Control channel to the RLS server. + bg *balancergroup.BalancerGroup + childPolicies map[string]*childPolicyWrapper + defaultPolicy *childPolicyWrapper + // A reference to the most recent picker sent to gRPC as part of a state + // update is cached in this field so that we can release the reference to the + // default child policy wrapper when a new picker is created. See + // sendNewPickerLocked() for details. + lastPicker *rlsPicker + // Set during UpdateClientConnState when pushing updates to child policies. + // Prevents state updates from child policies causing new pickers to be sent + // up the channel. Cleared after all child policies have processed the + // updates sent to them, after which a new picker is sent up the channel. + inhibitPickerUpdates bool + + // Channel on which all updates are pushed. Processed in run(). + updateCh *buffer.Unbounded[any] +} + +type resumePickerUpdates struct { + done chan struct{} +} + +// childPolicyIDAndState wraps a child policy id and its state update. +type childPolicyIDAndState struct { + id string + state balancer.State +} + +type controlChannelReady struct{} + +// run is a long-running goroutine which handles all the updates that the +// balancer wishes to handle. The appropriate updateHandler will push the update +// on to a channel that this goroutine will select on, thereby the handling of +// the update will happen asynchronously. +func (b *rlsBalancer) run() { + // We exit out of the for loop below only after `Close()` has been invoked. + // Firing the done event here will ensure that Close() returns only after + // all goroutines are done. + defer func() { b.done.Fire() }() + + // Wait for purgeDataCache() goroutine to exit before returning from here. + doneCh := make(chan struct{}) + defer func() { + <-doneCh + }() + go b.purgeDataCache(doneCh) + + for { + select { + case u, ok := <-b.updateCh.Get(): + if !ok { + return + } + b.updateCh.Load() + switch update := u.(type) { + case childPolicyIDAndState: + b.handleChildPolicyStateUpdate(update.id, update.state) + case controlChannelReady: + b.logger.Infof("Resetting backoff state after control channel getting back to READY") + b.cacheMu.Lock() + updatePicker := b.dataCache.resetBackoffState(&backoffState{bs: defaultBackoffStrategy}) + b.cacheMu.Unlock() + if updatePicker { + b.sendNewPicker() + } + resetBackoffHook() + case resumePickerUpdates: + b.stateMu.Lock() + b.logger.Infof("Resuming picker updates after config propagation to child policies") + b.inhibitPickerUpdates = false + b.sendNewPickerLocked() + close(update.done) + b.stateMu.Unlock() + default: + b.logger.Errorf("Unsupported update type %T", update) + } + case <-b.closed.Done(): + return + } + } +} + +// purgeDataCache is a long-running goroutine which periodically deletes expired +// entries. An expired entry is one for which both the expiryTime and +// backoffExpiryTime are in the past. +func (b *rlsBalancer) purgeDataCache(doneCh chan struct{}) { + defer close(doneCh) + + for { + select { + case <-b.closed.Done(): + return + case <-b.purgeTicker.C: + b.cacheMu.Lock() + updatePicker := b.dataCache.evictExpiredEntries() + b.cacheMu.Unlock() + if updatePicker { + b.sendNewPicker() + } + b.dataCachePurgeHook() + } + } +} + +func (b *rlsBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { + defer clientConnUpdateHook() + + b.stateMu.Lock() + if b.closed.HasFired() { + b.stateMu.Unlock() + b.logger.Warningf("Received service config after balancer close: %s", pretty.ToJSON(ccs.BalancerConfig)) + return errBalancerClosed + } + + newCfg := ccs.BalancerConfig.(*lbConfig) + if b.lbCfg.Equal(newCfg) { + b.stateMu.Unlock() + b.logger.Infof("New service config matches existing config") + return nil + } + + b.logger.Infof("Delaying picker updates until config is propagated to and processed by child policies") + b.inhibitPickerUpdates = true + + // When the RLS server name changes, the old control channel needs to be + // swapped out for a new one. All state associated with the throttling + // algorithm is stored on a per-control-channel basis; when we swap out + // channels, we also swap out the throttling state. + b.handleControlChannelUpdate(newCfg) + + // Any changes to child policy name or configuration needs to be handled by + // either creating new child policies or pushing updates to existing ones. + b.resolverState = ccs.ResolverState + b.handleChildPolicyConfigUpdate(newCfg, &ccs) + + // Resize the cache if the size in the config has changed. + resizeCache := newCfg.cacheSizeBytes != b.lbCfg.cacheSizeBytes + + // Update the copy of the config in the LB policy before releasing the lock. + b.lbCfg = newCfg + b.stateMu.Unlock() + + // We cannot do cache operations above because `cacheMu` needs to be grabbed + // before `stateMu` if we are to hold both locks at the same time. + b.cacheMu.Lock() + b.dataCache.updateRLSServerTarget(newCfg.lookupService) + if resizeCache { + // If the new config changes reduces the size of the data cache, we + // might have to evict entries to get the cache size down to the newly + // specified size. If we do evict an entry with valid backoff timer, + // the new picker needs to be sent to the channel to re-process any + // RPCs queued as a result of this backoff timer. + b.dataCache.resize(newCfg.cacheSizeBytes) + } + b.cacheMu.Unlock() + // Enqueue an event which will notify us when the above update has been + // propagated to all child policies, and the child policies have all + // processed their updates, and we have sent a picker update. + done := make(chan struct{}) + b.updateCh.Put(resumePickerUpdates{done: done}) + <-done + return nil +} + +// handleControlChannelUpdate handles updates to service config fields which +// influence the control channel to the RLS server. +// +// Caller must hold lb.stateMu. +func (b *rlsBalancer) handleControlChannelUpdate(newCfg *lbConfig) { + if newCfg.lookupService == b.lbCfg.lookupService && newCfg.lookupServiceTimeout == b.lbCfg.lookupServiceTimeout { + return + } + + // Create a new control channel and close the existing one. + b.logger.Infof("Creating control channel to RLS server at: %v", newCfg.lookupService) + backToReadyFn := func() { + b.updateCh.Put(controlChannelReady{}) + } + ctrlCh, err := newControlChannel(newCfg.lookupService, newCfg.controlChannelServiceConfig, newCfg.lookupServiceTimeout, b.bopts, backToReadyFn) + if err != nil { + // This is very uncommon and usually represents a non-transient error. + // There is not much we can do here other than wait for another update + // which might fix things. + b.logger.Errorf("Failed to create control channel to %q: %v", newCfg.lookupService, err) + return + } + if b.ctrlCh != nil { + b.ctrlCh.close() + } + b.ctrlCh = ctrlCh +} + +// handleChildPolicyConfigUpdate handles updates to service config fields which +// influence child policy configuration. +// +// Caller must hold lb.stateMu. +func (b *rlsBalancer) handleChildPolicyConfigUpdate(newCfg *lbConfig, ccs *balancer.ClientConnState) { + // Update child policy builder first since other steps are dependent on this. + if b.childPolicyBuilder == nil || b.childPolicyBuilder.Name() != newCfg.childPolicyName { + b.logger.Infof("Child policy changed to %q", newCfg.childPolicyName) + b.childPolicyBuilder = balancer.Get(newCfg.childPolicyName) + for _, cpw := range b.childPolicies { + // If the child policy has changed, we need to remove the old policy + // from the BalancerGroup and add a new one. The BalancerGroup takes + // care of closing the old one in this case. + b.bg.Remove(cpw.target) + b.bg.Add(cpw.target, b.childPolicyBuilder) + } + } + + configSentToDefault := false + if b.lbCfg.defaultTarget != newCfg.defaultTarget { + // If the default target has changed, create a new childPolicyWrapper for + // the new target if required. If a new wrapper is created, add it to the + // childPolicies map and the BalancerGroup. + b.logger.Infof("Default target in LB config changing from %q to %q", b.lbCfg.defaultTarget, newCfg.defaultTarget) + cpw := b.childPolicies[newCfg.defaultTarget] + if cpw == nil { + cpw = newChildPolicyWrapper(newCfg.defaultTarget) + b.childPolicies[newCfg.defaultTarget] = cpw + b.bg.Add(newCfg.defaultTarget, b.childPolicyBuilder) + b.logger.Infof("Child policy %q added to BalancerGroup", newCfg.defaultTarget) + } + if err := b.buildAndPushChildPolicyConfigs(newCfg.defaultTarget, newCfg, ccs); err != nil { + cpw.lamify(err) + } + + // If an old default exists, release its reference. If this was the last + // reference, remove the child policy from the BalancerGroup and remove the + // corresponding entry the childPolicies map. + if b.defaultPolicy != nil { + if b.defaultPolicy.releaseRef() { + delete(b.childPolicies, b.lbCfg.defaultTarget) + b.bg.Remove(b.defaultPolicy.target) + } + } + b.defaultPolicy = cpw + configSentToDefault = true + } + + // No change in configuration affecting child policies. Return early. + if b.lbCfg.childPolicyName == newCfg.childPolicyName && b.lbCfg.childPolicyTargetField == newCfg.childPolicyTargetField && childPolicyConfigEqual(b.lbCfg.childPolicyConfig, newCfg.childPolicyConfig) { + return + } + + // If fields affecting child policy configuration have changed, the changes + // are pushed to the childPolicyWrapper which handles them appropriately. + for _, cpw := range b.childPolicies { + if configSentToDefault && cpw.target == newCfg.defaultTarget { + // Default target has already been taken care of. + continue + } + if err := b.buildAndPushChildPolicyConfigs(cpw.target, newCfg, ccs); err != nil { + cpw.lamify(err) + } + } +} + +// buildAndPushChildPolicyConfigs builds the final child policy configuration by +// adding the `targetField` to the base child policy configuration received in +// RLS LB policy configuration. The `targetField` is set to target and +// configuration is pushed to the child policy through the BalancerGroup. +// +// Caller must hold lb.stateMu. +func (b *rlsBalancer) buildAndPushChildPolicyConfigs(target string, newCfg *lbConfig, ccs *balancer.ClientConnState) error { + jsonTarget, err := json.Marshal(target) + if err != nil { + return fmt.Errorf("failed to marshal child policy target %q: %v", target, err) + } + + config := newCfg.childPolicyConfig + targetField := newCfg.childPolicyTargetField + config[targetField] = jsonTarget + jsonCfg, err := json.Marshal(config) + if err != nil { + return fmt.Errorf("failed to marshal child policy config %+v: %v", config, err) + } + + parser, _ := b.childPolicyBuilder.(balancer.ConfigParser) + parsedCfg, err := parser.ParseConfig(jsonCfg) + if err != nil { + return fmt.Errorf("childPolicy config parsing failed: %v", err) + } + + state := balancer.ClientConnState{ResolverState: ccs.ResolverState, BalancerConfig: parsedCfg} + b.logger.Infof("Pushing new state to child policy %q: %+v", target, state) + if err := b.bg.UpdateClientConnState(target, state); err != nil { + b.logger.Warningf("UpdateClientConnState(%q, %+v) failed : %v", target, ccs, err) + } + return nil +} + +func (b *rlsBalancer) ResolverError(err error) { + b.bg.ResolverError(err) +} + +func (b *rlsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *rlsBalancer) Close() { + b.stateMu.Lock() + b.closed.Fire() + b.purgeTicker.Stop() + if b.ctrlCh != nil { + b.ctrlCh.close() + } + b.unregisterMetricHandler() + b.bg.Close() + b.stateMu.Unlock() + + b.cacheMu.Lock() + b.dataCache.stop() + b.cacheMu.Unlock() + + b.updateCh.Close() + + <-b.done.Done() +} + +func (b *rlsBalancer) ExitIdle() { + b.bg.ExitIdle() +} + +// sendNewPickerLocked pushes a new picker on to the channel. +// +// Note that regardless of what connectivity state is reported, the policy will +// return its own picker, and not a picker that unconditionally queues +// (typically used for IDLE or CONNECTING) or a picker that unconditionally +// fails (typically used for TRANSIENT_FAILURE). This is required because, +// irrespective of the connectivity state, we need to able to perform RLS +// lookups for incoming RPCs and affect the status of queued RPCs based on the +// receipt of RLS responses. +// +// Caller must hold lb.stateMu. +func (b *rlsBalancer) sendNewPickerLocked() { + aggregatedState := b.aggregatedConnectivityState() + + // Acquire a separate reference for the picker. This is required to ensure + // that the wrapper held by the old picker is not closed when the default + // target changes in the config, and a new wrapper is created for the new + // default target. See handleChildPolicyConfigUpdate() for how config changes + // affecting the default target are handled. + if b.defaultPolicy != nil { + b.defaultPolicy.acquireRef() + } + + picker := &rlsPicker{ + kbm: b.lbCfg.kbMap, + origEndpoint: b.bopts.Target.Endpoint(), + lb: b, + defaultPolicy: b.defaultPolicy, + ctrlCh: b.ctrlCh, + maxAge: b.lbCfg.maxAge, + staleAge: b.lbCfg.staleAge, + bg: b.bg, + rlsServerTarget: b.lbCfg.lookupService, + grpcTarget: b.bopts.Target.String(), + metricsRecorder: b.cc.MetricsRecorder(), + } + picker.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[rls-picker %p] ", picker)) + state := balancer.State{ + ConnectivityState: aggregatedState, + Picker: picker, + } + + if !b.inhibitPickerUpdates { + b.logger.Infof("New balancer.State: %+v", state) + b.cc.UpdateState(state) + } else { + b.logger.Infof("Delaying picker update: %+v", state) + } + + if b.lastPicker != nil { + if b.defaultPolicy != nil { + b.defaultPolicy.releaseRef() + } + } + b.lastPicker = picker +} + +func (b *rlsBalancer) sendNewPicker() { + b.stateMu.Lock() + defer b.stateMu.Unlock() + if b.closed.HasFired() { + return + } + b.sendNewPickerLocked() +} + +// The aggregated connectivity state reported is determined as follows: +// - If there is at least one child policy in state READY, the connectivity +// state is READY. +// - Otherwise, if there is at least one child policy in state CONNECTING, the +// connectivity state is CONNECTING. +// - Otherwise, if there is at least one child policy in state IDLE, the +// connectivity state is IDLE. +// - Otherwise, all child policies are in TRANSIENT_FAILURE, and the +// connectivity state is TRANSIENT_FAILURE. +// +// If the RLS policy has no child policies and no configured default target, +// then we will report connectivity state IDLE. +// +// Caller must hold lb.stateMu. +func (b *rlsBalancer) aggregatedConnectivityState() connectivity.State { + if len(b.childPolicies) == 0 && b.lbCfg.defaultTarget == "" { + return connectivity.Idle + } + + var readyN, connectingN, idleN int + for _, cpw := range b.childPolicies { + state := (*balancer.State)(atomic.LoadPointer(&cpw.state)) + switch state.ConnectivityState { + case connectivity.Ready: + readyN++ + case connectivity.Connecting: + connectingN++ + case connectivity.Idle: + idleN++ + } + } + + switch { + case readyN > 0: + return connectivity.Ready + case connectingN > 0: + return connectivity.Connecting + case idleN > 0: + return connectivity.Idle + default: + return connectivity.TransientFailure + } +} + +// UpdateState is a implementation of the balancergroup.BalancerStateAggregator +// interface. The actual state aggregation functionality is handled +// asynchronously. This method only pushes the state update on to channel read +// and dispatched by the run() goroutine. +func (b *rlsBalancer) UpdateState(id string, state balancer.State) { + b.updateCh.Put(childPolicyIDAndState{id: id, state: state}) +} + +// handleChildPolicyStateUpdate provides the state aggregator functionality for +// the BalancerGroup. +// +// This method is invoked by the BalancerGroup whenever a child policy sends a +// state update. We cache the child policy's connectivity state and picker for +// two reasons: +// - to suppress connectivity state transitions from TRANSIENT_FAILURE to states +// other than READY +// - to delegate picks to child policies +func (b *rlsBalancer) handleChildPolicyStateUpdate(id string, newState balancer.State) { + b.stateMu.Lock() + defer b.stateMu.Unlock() + + cpw := b.childPolicies[id] + if cpw == nil { + // All child policies start with an entry in the map. If ID is not in + // map, it's either been removed, or never existed. + b.logger.Warningf("Received state update %+v for missing child policy %q", newState, id) + return + } + + oldState := (*balancer.State)(atomic.LoadPointer(&cpw.state)) + if oldState.ConnectivityState == connectivity.TransientFailure && newState.ConnectivityState == connectivity.Connecting { + // Ignore state transitions from TRANSIENT_FAILURE to CONNECTING, and thus + // fail pending RPCs instead of queuing them indefinitely when all + // subChannels are failing, even if the subChannels are bouncing back and + // forth between CONNECTING and TRANSIENT_FAILURE. + return + } + atomic.StorePointer(&cpw.state, unsafe.Pointer(&newState)) + b.logger.Infof("Child policy %q has new state %+v", id, newState) + b.sendNewPickerLocked() +} + +// acquireChildPolicyReferences attempts to acquire references to +// childPolicyWrappers corresponding to the passed in targets. If there is no +// childPolicyWrapper corresponding to one of the targets, a new one is created +// and added to the BalancerGroup. +func (b *rlsBalancer) acquireChildPolicyReferences(targets []string) []*childPolicyWrapper { + b.stateMu.Lock() + var newChildPolicies []*childPolicyWrapper + for _, target := range targets { + // If the target exists in the LB policy's childPolicies map. a new + // reference is taken here and added to the new list. + if cpw := b.childPolicies[target]; cpw != nil { + cpw.acquireRef() + newChildPolicies = append(newChildPolicies, cpw) + continue + } + + // If the target does not exist in the child policy map, then a new + // child policy wrapper is created and added to the new list. + cpw := newChildPolicyWrapper(target) + b.childPolicies[target] = cpw + b.bg.Add(target, b.childPolicyBuilder) + b.logger.Infof("Child policy %q added to BalancerGroup", target) + newChildPolicies = append(newChildPolicies, cpw) + if err := b.buildAndPushChildPolicyConfigs(target, b.lbCfg, &balancer.ClientConnState{ + ResolverState: b.resolverState, + }); err != nil { + cpw.lamify(err) + } + } + b.stateMu.Unlock() + return newChildPolicies +} + +// releaseChildPolicyReferences releases references to childPolicyWrappers +// corresponding to the passed in targets. If the release reference was the last +// one, the child policy is removed from the BalancerGroup. +func (b *rlsBalancer) releaseChildPolicyReferences(targets []string) { + b.stateMu.Lock() + for _, target := range targets { + if cpw := b.childPolicies[target]; cpw.releaseRef() { + delete(b.childPolicies, cpw.target) + b.bg.Remove(cpw.target) + } + } + b.stateMu.Unlock() +} + +// Report reports the metrics data to the provided recorder. +func (b *rlsBalancer) Report(r estats.AsyncMetricsRecorder) error { + b.cacheMu.Lock() + currentSize := b.dataCache.currentSize + entriesLen := int64(len(b.dataCache.entries)) + rlsServerTarget := b.dataCache.rlsServerTarget + grpcTarget := b.dataCache.grpcTarget + uuid := b.dataCache.uuid + shutdown := b.dataCache.shutdown.HasFired() + b.cacheMu.Unlock() + + if shutdown { + return nil + } + + cacheSizeMetric.Record(r, currentSize, grpcTarget, rlsServerTarget, uuid) + cacheEntriesMetric.Record(r, entriesLen, grpcTarget, rlsServerTarget, uuid) + return nil +} diff --git a/balancer/rls/balancer_test.go b/balancer/rls/balancer_test.go new file mode 100644 index 0000000..4a9e08c --- /dev/null +++ b/balancer/rls/balancer_test.go @@ -0,0 +1,1536 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/balancer/rls/internal/test/e2e" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/internal/grpcsync" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + rlstest "google.golang.org/grpc/internal/testutils/rls" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/testdata" + + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/protobuf/types/known/durationpb" +) + +// TestConfigUpdate_ControlChannel tests the scenario where a config update +// changes the RLS server name. Verifies that the new control channel is created +// and the old one is closed. +func (s) TestConfigUpdate_ControlChannel(t *testing.T) { + // Start two RLS servers. + lis1 := testutils.NewListenerWrapper(t, nil) + rlsServer1, rlsReqCh1 := rlstest.SetupFakeRLSServer(t, lis1) + lis2 := testutils.NewListenerWrapper(t, nil) + rlsServer2, rlsReqCh2 := rlstest.SetupFakeRLSServer(t, lis2) + + // Build RLS service config with the RLS server pointing to the first one. + // Set a very low value for maxAge to ensure that the entry expires soon. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer1.Address) + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(defaultTestShortTimeout) + + // Start a couple of test backends, and set up the fake RLS servers to return + // these as a target in the RLS response. + backendCh1, backendAddress1 := startBackend(t) + rlsServer1.SetResponseCallback(func(_ context.Context, _ *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress1}}} + }) + backendCh2, backendAddress2 := startBackend(t) + rlsServer2.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress2}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh1) + + // Ensure a connection is established to the first RLS server. + val, err := lis1.NewConnCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout expired when waiting for LB policy to create control channel") + } + conn1 := val.(*testutils.ConnWrapper) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh1, true) + + // Change lookup_service field of the RLS config to point to the second one. + rlsConfig.RouteLookupConfig.LookupService = rlsServer2.Address + + // Push the config update through the manual resolver. + scJSON, err := rlsConfig.ServiceConfigJSON() + if err != nil { + t.Fatal(err) + } + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.UpdateState(resolver.State{ServiceConfig: sc}) + + // Ensure a connection is established to the second RLS server. + if _, err := lis2.NewConnCh.Receive(ctx); err != nil { + t.Fatal("Timeout expired when waiting for LB policy to create control channel") + } + + // Ensure the connection to the old one is closed. + if _, err := conn1.CloseCh.Receive(ctx); err != nil { + t.Fatal("Timeout expired when waiting for LB policy to close control channel") + } + + // Make an RPC and expect it to get routed to the second test backend through + // the second RLS server. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh2) + verifyRLSRequest(t, rlsReqCh2, true) +} + +// TestConfigUpdate_ControlChannelWithCreds tests the scenario where a config +// update specified an RLS server name, and the parent ClientConn specifies +// transport credentials. The RLS server and the test backend are configured to +// accept those transport credentials. This test verifies that the parent +// channel credentials are correctly propagated to the control channel. +func (s) TestConfigUpdate_ControlChannelWithCreds(t *testing.T) { + serverCreds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("credentials.NewServerTLSFromFile(server1.pem, server1.key) = %v", err) + } + clientCreds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "") + if err != nil { + t.Fatalf("credentials.NewClientTLSFromFile(ca.pem) = %v", err) + } + + // Start an RLS server with the wrapped listener and credentials. + lis := testutils.NewListenerWrapper(t, nil) + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, lis, grpc.Creds(serverCreds)) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build RLS service config. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + + // Start a test backend which uses the same credentials as the RLS server, + // and set up the fake RLS server to return this as the target in the RLS + // response. + backendCh, backendAddress := startBackend(t, grpc.Creds(serverCreds)) + rlsServer.SetResponseCallback(func(_ context.Context, _ *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Dial with credentials and expect the RLS server to receive the same. The + // server certificate used for the RLS server and the backend specifies a + // DNS SAN of "*.test.example.com". Hence we use a dial target which is a + // subdomain of the same here. + cc, err := grpc.NewClient(r.Scheme()+":///rls.test.example.com", grpc.WithResolvers(r), grpc.WithTransportCredentials(clientCreds)) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Ensure a connection is established to the first RLS server. + if _, err := lis.NewConnCh.Receive(ctx); err != nil { + t.Fatal("Timeout expired when waiting for LB policy to create control channel") + } +} + +// TestConfigUpdate_ControlChannelServiceConfig tests the scenario where RLS LB +// policy's configuration specifies the service config for the control channel +// via the `routeLookupChannelServiceConfig` field. This test verifies that the +// provided service config is applied for the control channel. +func (s) TestConfigUpdate_ControlChannelServiceConfig(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Register a balancer to be used for the control channel, and set up a + // callback to get notified when the balancer receives a clientConn updates. + ccUpdateCh := testutils.NewChannel() + bf := &e2e.BalancerFuncs{ + UpdateClientConnState: func(cfg *e2e.RLSChildPolicyConfig) error { + if cfg.Backend != rlsServer.Address { + return fmt.Errorf("control channel LB policy received config with backend %q, want %q", cfg.Backend, rlsServer.Address) + } + ccUpdateCh.Replace(nil) + return nil + }, + } + controlChannelPolicyName := "test-control-channel-" + t.Name() + e2e.RegisterRLSChildPolicy(controlChannelPolicyName, bf) + t.Logf("Registered child policy with name %q", controlChannelPolicyName) + + // Build RLS service config and set the `routeLookupChannelServiceConfig` + // field to a service config which uses the above balancer. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + rlsConfig.RouteLookupChannelServiceConfig = fmt.Sprintf(`{"loadBalancingConfig" : [{%q: {"backend": %q} }]}`, controlChannelPolicyName, rlsServer.Address) + + // Start a test backend, and set up the fake RLS server to return this as a + // target in the RLS response. + backendCh, backendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, _ *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.NewClient(r.Scheme()+":///rls.test.example.com", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Verify that the control channel is using the LB policy we injected via the + // routeLookupChannelServiceConfig field. + if _, err := ccUpdateCh.Receive(ctx); err != nil { + t.Fatalf("timeout when waiting for control channel LB policy to receive a clientConn update") + } +} + +// TestConfigUpdate_DefaultTarget tests the scenario where a config update +// changes the default target. Verifies that RPCs get routed to the new default +// target after the config has been applied. +func (s) TestConfigUpdate_DefaultTarget(t *testing.T) { + // Start an RLS server and set the throttler to always throttle requests. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, alwaysThrottlingThrottler()) + + // Build RLS service config with a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + backendCh1, backendAddress1 := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = backendAddress1 + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the default target. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh1) + + // Change default_target field of the RLS config. + backendCh2, backendAddress2 := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = backendAddress2 + + // Push the config update through the manual resolver. + scJSON, err := rlsConfig.ServiceConfigJSON() + if err != nil { + t.Fatal(err) + } + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.UpdateState(resolver.State{ServiceConfig: sc}) + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh2) +} + +// TestConfigUpdate_ChildPolicyConfigs verifies that config changes which affect +// child policy configuration are propagated correctly. +func (s) TestConfigUpdate_ChildPolicyConfigs(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Start a default backend and a test backend. + _, defBackendAddress := startBackend(t) + testBackendCh, testBackendAddress := startBackend(t) + + // Set up the RLS server to respond with the test backend. + rlsServer.SetResponseCallback(func(_ context.Context, _ *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Set up a test balancer callback to push configs received by child policies. + defBackendConfigsCh := make(chan *e2e.RLSChildPolicyConfig, 1) + testBackendConfigsCh := make(chan *e2e.RLSChildPolicyConfig, 1) + bf := &e2e.BalancerFuncs{ + UpdateClientConnState: func(cfg *e2e.RLSChildPolicyConfig) error { + switch cfg.Backend { + case defBackendAddress: + defBackendConfigsCh <- cfg + case testBackendAddress: + testBackendConfigsCh <- cfg + default: + t.Errorf("Received child policy configs for unknown target %q", cfg.Backend) + } + return nil + }, + } + + // Register an LB policy to act as the child policy for RLS LB policy. + childPolicyName := "test-child-policy" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName, bf) + t.Logf("Registered child policy with name %q", childPolicyName) + + // Build RLS service config with default target. + rlsConfig := buildBasicRLSConfig(childPolicyName, rlsServer.Address) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + + // At this point, the RLS LB policy should have received its config, and + // should have created a child policy for the default target. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantCfg := &e2e.RLSChildPolicyConfig{Backend: defBackendAddress} + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the default target child policy to receive its config") + case gotCfg := <-defBackendConfigsCh: + if !cmp.Equal(gotCfg, wantCfg) { + t.Fatalf("Default target child policy received config %+v, want %+v", gotCfg, wantCfg) + } + } + + // Make an RPC and ensure it gets routed to the test backend. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // As part of handling the above RPC, the RLS LB policy should have created + // a child policy for the test target. + wantCfg = &e2e.RLSChildPolicyConfig{Backend: testBackendAddress} + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the test target child policy to receive its config") + case gotCfg := <-testBackendConfigsCh: + if !cmp.Equal(gotCfg, wantCfg) { + t.Fatalf("Test target child policy received config %+v, want %+v", gotCfg, wantCfg) + } + } + + // Push an RLS config update with a change in the child policy config. + childPolicyBuilder := balancer.Get(childPolicyName) + childPolicyParser := childPolicyBuilder.(balancer.ConfigParser) + lbCfg, err := childPolicyParser.ParseConfig([]byte(`{"Random": "random"}`)) + if err != nil { + t.Fatal(err) + } + rlsConfig.ChildPolicy.Config = lbCfg + scJSON, err := rlsConfig.ServiceConfigJSON() + if err != nil { + t.Fatal(err) + } + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.UpdateState(resolver.State{ServiceConfig: sc}) + + // Expect the child policy for the test backend to receive the update. + wantCfg = &e2e.RLSChildPolicyConfig{ + Backend: testBackendAddress, + Random: "random", + } + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the test target child policy to receive its config") + case gotCfg := <-testBackendConfigsCh: + if !cmp.Equal(gotCfg, wantCfg) { + t.Fatalf("Test target child policy received config %+v, want %+v", gotCfg, wantCfg) + } + } + + // Expect the child policy for the default backend to receive the update. + wantCfg = &e2e.RLSChildPolicyConfig{ + Backend: defBackendAddress, + Random: "random", + } + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the default target child policy to receive its config") + case gotCfg := <-defBackendConfigsCh: + if !cmp.Equal(gotCfg, wantCfg) { + t.Fatalf("Default target child policy received config %+v, want %+v", gotCfg, wantCfg) + } + } +} + +// TestConfigUpdate_ChildPolicyChange verifies that a child policy change is +// handled by closing the old balancer and creating a new one. +func (s) TestConfigUpdate_ChildPolicyChange(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Set up balancer callbacks. + configsCh1 := make(chan *e2e.RLSChildPolicyConfig, 1) + closeCh1 := make(chan struct{}, 1) + bf := &e2e.BalancerFuncs{ + UpdateClientConnState: func(cfg *e2e.RLSChildPolicyConfig) error { + configsCh1 <- cfg + return nil + }, + Close: func() { + closeCh1 <- struct{}{} + }, + } + + // Register an LB policy to act as the child policy for RLS LB policy. + childPolicyName1 := "test-child-policy-1" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName1, bf) + t.Logf("Registered child policy with name %q", childPolicyName1) + + // Build RLS service config with a dummy default target. + const defaultBackend = "default-backend" + rlsConfig := buildBasicRLSConfig(childPolicyName1, rlsServer.Address) + rlsConfig.RouteLookupConfig.DefaultTarget = defaultBackend + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + + // At this point, the RLS LB policy should have received its config, and + // should have created a child policy for the default target. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantCfg := &e2e.RLSChildPolicyConfig{Backend: defaultBackend} + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the first child policy to receive its config") + case gotCfg := <-configsCh1: + if !cmp.Equal(gotCfg, wantCfg) { + t.Fatalf("First child policy received config %+v, want %+v", gotCfg, wantCfg) + } + } + + // Set up balancer callbacks for the second policy. + configsCh2 := make(chan *e2e.RLSChildPolicyConfig, 1) + bf = &e2e.BalancerFuncs{ + UpdateClientConnState: func(cfg *e2e.RLSChildPolicyConfig) error { + configsCh2 <- cfg + return nil + }, + } + + // Register a second LB policy to act as the child policy for RLS LB policy. + childPolicyName2 := "test-child-policy-2" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName2, bf) + t.Logf("Registered child policy with name %q", childPolicyName2) + + // Push an RLS config update with a change in the child policy name. + rlsConfig.ChildPolicy = &internalserviceconfig.BalancerConfig{Name: childPolicyName2} + scJSON, err := rlsConfig.ServiceConfigJSON() + if err != nil { + t.Fatal(err) + } + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.UpdateState(resolver.State{ServiceConfig: sc}) + + // The above update should result in the first LB policy being shutdown and + // the second LB policy receiving a config update. + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the first child policy to be shutdown") + case <-closeCh1: + } + + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the second child policy to receive its config") + case gotCfg := <-configsCh2: + if !cmp.Equal(gotCfg, wantCfg) { + t.Fatalf("First child policy received config %+v, want %+v", gotCfg, wantCfg) + } + } +} + +// TestConfigUpdate_BadChildPolicyConfigs tests the scenario where a config +// update is rejected by the child policy. Verifies that the child policy +// wrapper goes "lame" and the error from the child policy is reported back to +// the caller of the RPC. +func (s) TestConfigUpdate_BadChildPolicyConfigs(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Set up the RLS server to respond with a bad target field which is expected + // to cause the child policy's ParseTarget to fail and should result in the LB + // policy creating a lame child policy wrapper. + rlsServer.SetResponseCallback(func(_ context.Context, _ *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{e2e.RLSChildPolicyBadTarget}}} + }) + + // Build RLS service config with a default target. This default backend is + // expected to be healthy (even though we don't attempt to route RPCs to it) + // and ensures that the overall connectivity state of the RLS LB policy is not + // TRANSIENT_FAILURE. This is required to make sure that the pick for the bad + // child policy actually gets delegated to the child policy picker. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + _, addr := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = addr + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure that if fails with the expected error. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndVerifyError(ctx, t, cc, codes.Unavailable, e2e.ErrParseConfigBadTarget) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) +} + +// TestConfigUpdate_DataCacheSizeDecrease tests the scenario where a config +// update decreases the data cache size. Verifies that entries are evicted from +// the cache. +func (s) TestConfigUpdate_DataCacheSizeDecrease(t *testing.T) { + // Override the clientConn update hook to get notified. + clientConnUpdateDone := make(chan struct{}, 1) + origClientConnUpdateHook := clientConnUpdateHook + clientConnUpdateHook = func() { clientConnUpdateDone <- struct{}{} } + defer func() { clientConnUpdateHook = origClientConnUpdateHook }() + + // Override the cache entry size func, and always return 1. + origEntrySizeFunc := computeDataCacheEntrySize + computeDataCacheEntrySize = func(cacheKey, *cacheEntry) int64 { return 1 } + defer func() { computeDataCacheEntrySize = origEntrySizeFunc }() + + // Override the minEvictionDuration to ensure that when the config update + // reduces the cache size, the resize operation is not stopped because + // we find an entry whose minExpiryDuration has not elapsed. + origMinEvictDuration := minEvictDuration + minEvictDuration = time.Duration(0) + defer func() { minEvictDuration = origMinEvictDuration }() + + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Register an LB policy to act as the child policy for RLS LB policy. + childPolicyName := "test-child-policy" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName, nil) + t.Logf("Registered child policy with name %q", childPolicyName) + + // Build RLS service config with header matchers. + rlsConfig := buildBasicRLSConfig(childPolicyName, rlsServer.Address) + + // Start a couple of test backends, and set up the fake RLS server to return + // these as targets in the RLS response, based on request keys. + backendCh1, backendAddress1 := startBackend(t) + backendCh2, backendAddress2 := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + if req.KeyMap["k1"] == "v1" { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress1}}} + } + if req.KeyMap["k2"] == "v2" { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress2}}} + } + return &rlstest.RouteLookupResponse{Err: errors.New("no keys in request metadata")} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + + <-clientConnUpdateDone + + // Make an RPC and ensure it gets routed to the first backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctxOutgoing := metadata.AppendToOutgoingContext(ctx, "n1", "v1") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh1) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Make another RPC with a different set of headers. This will force the LB + // policy to send out a new RLS request, resulting in a new data cache + // entry. + ctxOutgoing = metadata.AppendToOutgoingContext(ctx, "n2", "v2") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh2) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // We currently have two cache entries. Setting the size to 1, will cause + // the entry corresponding to backend1 to be evicted. + rlsConfig.RouteLookupConfig.CacheSizeBytes = 1 + + // Push the config update through the manual resolver. + scJSON, err := rlsConfig.ServiceConfigJSON() + if err != nil { + t.Fatal(err) + } + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.UpdateState(resolver.State{ServiceConfig: sc}) + + <-clientConnUpdateDone + + // Make an RPC to match the cache entry which got evicted above, and expect + // an RLS request to be made to fetch the targets. + ctxOutgoing = metadata.AppendToOutgoingContext(ctx, "n1", "v1") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh1) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) +} + +// stateCapturingCC wraps a balancer.ClientConn, overrides UpdateState, pushes +// the update on to a channel, and delegates to the wrapped balancer.ClientConn. +type stateCapturingCC struct { + balancer.ClientConn + stateCh chan balancer.State +} + +func (cc *stateCapturingCC) UpdateState(bs balancer.State) { + cc.stateCh <- bs + cc.ClientConn.UpdateState(bs) +} + +func newStateCapturingCC(cc balancer.ClientConn) *stateCapturingCC { + return &stateCapturingCC{ + ClientConn: cc, + stateCh: make(chan balancer.State, 10), // Some operations result in multiple UpdateState calls. + } +} + +// Test that when a data cache entry is evicted due to config change +// in cache size, the picker is updated accordingly. +func (s) TestPickerUpdateOnDataCacheSizeDecrease(t *testing.T) { + // Override the clientConn update hook to get notified. + clientConnUpdateDone := make(chan struct{}, 1) + origClientConnUpdateHook := clientConnUpdateHook + clientConnUpdateHook = func() { clientConnUpdateDone <- struct{}{} } + defer func() { clientConnUpdateHook = origClientConnUpdateHook }() + + // Override the cache entry size func, and always return 1. + origEntrySizeFunc := computeDataCacheEntrySize + computeDataCacheEntrySize = func(cacheKey, *cacheEntry) int64 { return 1 } + defer func() { computeDataCacheEntrySize = origEntrySizeFunc }() + + // Override the backoff strategy to return a large backoff which + // will make sure the date cache entry remains in backoff for the + // duration of the test. + origBackoffStrategy := defaultBackoffStrategy + defaultBackoffStrategy = &fakeBackoffStrategy{backoff: defaultTestTimeout} + defer func() { defaultBackoffStrategy = origBackoffStrategy }() + + // Override the minEvictionDuration to ensure that when the config update + // reduces the cache size, the resize operation is not stopped because + // we find an entry whose minExpiryDuration has not elapsed. + origMinEvictDuration := minEvictDuration + minEvictDuration = time.Duration(0) + defer func() { minEvictDuration = origMinEvictDuration }() + + // Register the top-level wrapping balancer which forwards calls to RLS. + topLevelBalancerName := t.Name() + "top-level" + var ccWrapper *stateCapturingCC + stub.Register(topLevelBalancerName, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + ccWrapper = newStateCapturingCC(bd.ClientConn) + bd.ChildBalancer = balancer.Get(Name).Build(ccWrapper, bd.BuildOptions) + }, + ParseConfig: func(sc json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + parser := balancer.Get(Name).(balancer.ConfigParser) + return parser.ParseConfig(sc) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + }) + + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Register an LB policy to act as the child policy for RLS LB policy. + childPolicyName := "test-child-policy" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName, nil) + t.Logf("Registered child policy with name %q", childPolicyName) + + // Start a couple of test backends, and set up the fake RLS server to return + // these as targets in the RLS response, based on request keys. + // Start a couple of test backends, and set up the fake RLS server to return + // these as targets in the RLS response, based on request keys. + backendCh1, backendAddress1 := startBackend(t) + backendCh2, backendAddress2 := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + if req.KeyMap["k1"] == "v1" { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress1}}} + } + if req.KeyMap["k2"] == "v2" { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress2}}} + } + return &rlstest.RouteLookupResponse{Err: errors.New("no keys in request metadata")} + }) + + // Register a manual resolver and push the RLS service config through it. + r := manual.NewBuilderWithScheme("rls-e2e") + headers := ` + [ + { + "key": "k1", + "names": [ + "n1" + ] + }, + { + "key": "k2", + "names": [ + "n2" + ] + } + ] + ` + + configJSON := ` + { + "loadBalancingConfig": [ + { + "%s": { + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "grpc.testing.TestService"}], + "headers": %s + }], + "lookupService": "%s", + "cacheSizeBytes": %d + }, + "childPolicy": [{"%s": {}}], + "childPolicyConfigTargetFieldName": "Backend" + } + } + ] + }` + scJSON := fmt.Sprintf(configJSON, topLevelBalancerName, headers, rlsServer.Address, 1000, childPolicyName) + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.InitialState(resolver.State{ServiceConfig: sc}) + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("create grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + + // Wait for the clientconn update to be processed by the RLS LB policy. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for RLS LB policy to process the initial clientconn update") + case <-clientConnUpdateDone: + } + + // RLS LB policy starts off in IDLE state. + gotStates := waitForStateTransitions(ctx, t, ccWrapper.stateCh, 1) + if gotStates[0] != connectivity.Idle { + t.Fatalf("RLS LB policy in state %s, want IDLE", gotStates[0]) + } + + // Make an RPC call with empty metadata, which will eventually throw + // the error as no metadata will match from rlsServer response + // callback defined above. This will cause the control channel to + // throw the error and cause the item to get into backoff. + makeTestRPCAndVerifyError(ctx, t, cc, codes.Unavailable, nil) + + // RLS LB policy sends a picker update when it receives the RLS response, but + // continues to remain in IDLE state, as no child policy is created yet + gotStates = waitForStateTransitions(ctx, t, ccWrapper.stateCh, 1) + if gotStates[0] != connectivity.Idle { + t.Fatalf("RLS LB policy in state %s, want IDLE", gotStates[0]) + } + + ctxOutgoing := metadata.AppendToOutgoingContext(ctx, "n1", "v1") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh1) + verifyRLSRequest(t, rlsReqCh, true) + + // We expect three state updates as the LB policy transitions to READY. Two of + // them correspond to the child policy's state updates (pick_first reports + // CONNECTING and READY), and one corresponds to the picker update that is + // sent upon receiving the RLS response (this could be CONNECTING or READY + // based on when this runs relative to the update from the child policy). + gotStates = waitForStateTransitions(ctx, t, ccWrapper.stateCh, 3) + gotStates = slices.Compact(gotStates) + wantStates := []connectivity.State{connectivity.Connecting, connectivity.Ready} + if !cmp.Equal(gotStates, wantStates) { + t.Fatalf("RLS LB policy in states %v, want %v", gotStates, wantStates) + } + + ctxOutgoing = metadata.AppendToOutgoingContext(ctx, "n2", "v2") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh2) + verifyRLSRequest(t, rlsReqCh, true) + + // We expect three state updates as the LB policy stays in READY. Two of + // them correspond to the child policy's state updates (pick_first reports + // CONNECTING and READY), and one corresponds to the picker update that is + // sent upon receiving the RLS response. + gotStates = waitForStateTransitions(ctx, t, ccWrapper.stateCh, 3) + gotStates = slices.Compact(gotStates) + wantStates = []connectivity.State{connectivity.Ready} + if !cmp.Equal(gotStates, wantStates) { + t.Fatalf("RLS LB policy in states %v, want %v", gotStates, wantStates) + } + + // Setting the size to 2 will cause the entry corresponding to the first RPC + // to be evicted from the cache. This entry has an ongoing backoff, and so + // the picker needs to be updated to reflect this change. + scJSON1 := fmt.Sprintf(` +{ + "loadBalancingConfig": [ + { + "%s": { + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "grpc.testing.TestService"}], + "headers": %s + }], + "lookupService": "%s", + "cacheSizeBytes": 2 + }, + "childPolicy": [{"%s": {}}], + "childPolicyConfigTargetFieldName": "Backend" + } + } + ] +}`, topLevelBalancerName, headers, rlsServer.Address, childPolicyName) + sc1 := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON1) + r.UpdateState(resolver.State{ServiceConfig: sc1}) + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for RLS LB policy to process the subsequent clientconn update") + case <-clientConnUpdateDone: + } + + gotStates = waitForStateTransitions(ctx, t, ccWrapper.stateCh, 1) + if gotStates[0] != connectivity.Ready { + t.Fatalf("RLS LB policy in state %s, want Ready", gotStates[0]) + } +} + +// TestDataCachePurging verifies that the LB policy periodically evicts expired +// entries from the data cache. +func (s) TestDataCachePurging(t *testing.T) { + // Override the frequency of the data cache purger to a small one. + origDataCachePurgeTicker := dataCachePurgeTicker + ticker := time.NewTicker(defaultTestShortTimeout) + defer ticker.Stop() + dataCachePurgeTicker = func() *time.Ticker { return ticker } + defer func() { dataCachePurgeTicker = origDataCachePurgeTicker }() + + // Override the data cache purge hook to get notified. + dataCachePurgeDone := make(chan struct{}, 1) + origDataCachePurgeHook := dataCachePurgeHook + dataCachePurgeHook = func() { dataCachePurgeDone <- struct{}{} } + defer func() { dataCachePurgeHook = origDataCachePurgeHook }() + + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Register an LB policy to act as the child policy for RLS LB policy. + childPolicyName := "test-child-policy" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName, nil) + t.Logf("Registered child policy with name %q", childPolicyName) + + // Build RLS service config with header matchers and lookupService pointing to + // the fake RLS server created above. Set a very low value for maxAge to + // ensure that the entry expires soon. + rlsConfig := buildBasicRLSConfig(childPolicyName, rlsServer.Address) + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(time.Millisecond) + + // Start a test backend, and set up the fake RLS server to return this as a + // target in the RLS response. + backendCh, backendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, _ *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctxOutgoing := metadata.AppendToOutgoingContext(ctx, "n1", "v1") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Make another RPC with different headers. This will force the LB policy to + // send out a new RLS request, resulting in a new data cache entry. + ctxOutgoing = metadata.AppendToOutgoingContext(ctx, "n2", "v2") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Wait for the data cache purging to happen before proceeding. + <-dataCachePurgeDone + + // Perform the same RPCs again and verify that they result in RLS requests. + ctxOutgoing = metadata.AppendToOutgoingContext(ctx, "n1", "v1") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Make another RPC with different headers. This will force the LB policy to + // send out a new RLS request, resulting in a new data cache entry. + ctxOutgoing = metadata.AppendToOutgoingContext(ctx, "n2", "v2") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) +} + +// wrappingConnectivityStateSubscriber wraps a grpcsync.Subscriber and forwards +// connectivity state updates to both the delegate and a channel for testing. +type wrappingConnectivityStateSubscriber struct { + delegate grpcsync.Subscriber + connStateCh *buffer.Unbounded[connectivity.State] +} + +func (w *wrappingConnectivityStateSubscriber) OnMessage(msg any) { + w.delegate.OnMessage(msg) + w.connStateCh.Put(msg.(connectivity.State)) +} + +// waitForConnectivityState waits for one of the specified connectivity states +// to appear on the channel, returning the state that matched first. It skips +// intermediate states that do not match any of the wanted states, and fails +// the test if the context expires before any of the desired states are reached. +func waitForConnectivityState(ctx context.Context, t *testing.T, ch *buffer.Unbounded[connectivity.State], wants ...connectivity.State) connectivity.State { + t.Helper() + for { + select { + case gotState, ok := <-ch.Get(): + ch.Load() + if !ok { + t.Fatalf("Channel closed while waiting for RLS control channel to become one of %v", wants) + } + for _, want := range wants { + if gotState == want { + return gotState + } + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for RLS control channel to become one of %v", wants) + } + } +} + +// TestControlChannelConnectivityStateMonitoring tests the scenario where the +// control channel goes down and comes back up again and verifies that backoff +// state is reset for cache entries in this scenario. It also verifies that: +// - Backoff is NOT reset when the control channel first becomes READY (i.e., +// the initial CONNECTING → READY transition should not trigger a backoff reset) +// - Backoff is reset for READY → TRANSIENT_FAILURE → READY transitions +func (s) TestControlChannelConnectivityStateMonitoring(t *testing.T) { + // Create a restartable listener which can close existing connections. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + + // Start an RLS server with the restartable listener and set the throttler to + // never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, lis) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Override the reset backoff hook to get notified. + resetBackoffDone := make(chan struct{}, 1) + origResetBackoffHook := resetBackoffHook + resetBackoffHook = func() { resetBackoffDone <- struct{}{} } + defer func() { resetBackoffHook = origResetBackoffHook }() + + // Override the backoff strategy to return a large backoff which + // will make sure the date cache entry remains in backoff for the + // duration of the test. + origBackoffStrategy := defaultBackoffStrategy + defaultBackoffStrategy = &fakeBackoffStrategy{backoff: defaultTestTimeout} + defer func() { defaultBackoffStrategy = origBackoffStrategy }() + + // Override the connectivity state subscriber to wrap the original and + // make connectivity state changes visible to the test. + wrappedSubscriber := &wrappingConnectivityStateSubscriber{connStateCh: buffer.NewUnbounded[connectivity.State]()} + origConnectivityStateSubscriber := newConnectivityStateSubscriber + newConnectivityStateSubscriber = func(delegate grpcsync.Subscriber) grpcsync.Subscriber { + return &wrappingConnectivityStateSubscriber{ + delegate: delegate, + connStateCh: wrappedSubscriber.connStateCh, + } + } + defer func() { newConnectivityStateSubscriber = origConnectivityStateSubscriber }() + + // Register an LB policy to act as the child policy for RLS LB policy. + childPolicyName := "test-child-policy" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName, nil) + t.Logf("Registered child policy with name %q", childPolicyName) + + // Build RLS service config with header matchers, and a very low value for + // maxAge to ensure that cache entries become invalid very soon. + rlsConfig := buildBasicRLSConfig(childPolicyName, rlsServer.Address) + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(defaultTestShortTimeout) + + // Start a test backend, and set up the fake RLS server to return this as a + // target in the RLS response. + backendCh, backendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, _ *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Wait for the control channel to move to CONNECTING and then to READY. + waitForConnectivityState(ctx, t, wrappedSubscriber.connStateCh, connectivity.Connecting) + waitForConnectivityState(ctx, t, wrappedSubscriber.connStateCh, connectivity.Ready) + + // Verify that the initial READY state of the control channel did NOT trigger + // a backoff reset. The resetBackoffHook should only be called when + // transitioning from TRANSIENT_FAILURE to READY, not for the initial + // CONNECTING → READY transition. + select { + case <-resetBackoffDone: + t.Fatal("Backoff reset was triggered for initial READY state, want no reset") + case <-time.After(defaultTestShortTimeout): + } + + // Stop the RLS server. + lis.Stop() + + // When the server is stopped, the control channel connection is closed, + // causing it to transition to IDLE or TRANSIENT_FAILURE. + // If it transitions to IDLE, we need to send a new RLS request to force + // it to attempt reconnection, which will fail and move it to TRANSIENT_FAILURE. + state := waitForConnectivityState(ctx, t, wrappedSubscriber.connStateCh, connectivity.Idle, connectivity.TransientFailure) + + if state == connectivity.Idle { + // Make another RPC with different headers to bypass any cached backoff entries + // and force the control channel to attempt a connection, which will fail + // and move it to TRANSIENT_FAILURE. + ctxFailed := metadata.AppendToOutgoingContext(ctx, "n1", "v1") + makeTestRPCAndVerifyError(ctxFailed, t, cc, codes.Unavailable, nil) + + // Wait for the control channel to move to TRANSIENT_FAILURE. + waitForConnectivityState(ctx, t, wrappedSubscriber.connStateCh, connectivity.TransientFailure) + } + + // Restart the RLS server. + lis.Restart() + + // When we closed the RLS server earlier, the existing transport to the RLS + // server would have closed, and the RLS control channel would have moved to + // TRANSIENT_FAILURE with a subConn backoff before moving to IDLE. This + // backoff will last for about a second. We need to keep retrying RPCs for the + // subConn to eventually come out of backoff and attempt to reconnect. + // + // Make this RPC with a different set of headers leading to the creation of + // a new cache entry and a new RLS request. This RLS request will also fail + // till the control channel comes moves back to READY. So, override the + // backoff strategy to perform a small backoff on this entry. + defaultBackoffStrategy = &fakeBackoffStrategy{backoff: defaultTestShortTimeout} + ctxOutgoing := metadata.AppendToOutgoingContext(ctx, "n2", "v2") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh) + + // Wait for the control channel to move back to READY. + waitForConnectivityState(ctx, t, wrappedSubscriber.connStateCh, connectivity.Ready) + + // Verify that backoff was reset when transitioning from TRANSIENT_FAILURE to READY. + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for resetBackoffDone") + case <-resetBackoffDone: + } + + // The fact that the above RPC succeeded indicates that the control channel + // has moved back to READY. The connectivity state monitoring code should have + // realized this and should have reset all backoff timers (which in this case + // is the cache entry corresponding to the first RPC). Retrying that RPC now + // should succeed with an RLS request being sent out. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh) + verifyRLSRequest(t, rlsReqCh, true) +} + +// TestControlChannelIdleTransitionNoBackoffReset tests that READY → IDLE → READY +// transitions do not trigger backoff resets. This is a benign state change that +// should not affect cache entry backoff state. +func (s) TestControlChannelIdleTransitionNoBackoffReset(t *testing.T) { + // Create a restartable listener which can close existing connections. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + + // Start an RLS server with the restartable listener and set the throttler to + // never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, lis) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Override the reset backoff hook to get notified. + resetBackoffCalled := make(chan struct{}, 1) + origResetBackoffHook := resetBackoffHook + resetBackoffHook = func() { resetBackoffCalled <- struct{}{} } + defer func() { resetBackoffHook = origResetBackoffHook }() + + // Override the backoff strategy to return a large backoff which + // will make sure the data cache entry remains in backoff for the + // duration of the test. + origBackoffStrategy := defaultBackoffStrategy + defaultBackoffStrategy = &fakeBackoffStrategy{backoff: defaultTestTimeout} + defer func() { defaultBackoffStrategy = origBackoffStrategy }() + + // Override the connectivity state subscriber to wrap the original and + // make connectivity state changes visible to the test. + wrappedSubscriber := &wrappingConnectivityStateSubscriber{connStateCh: buffer.NewUnbounded[connectivity.State]()} + origConnectivityStateSubscriber := newConnectivityStateSubscriber + newConnectivityStateSubscriber = func(delegate grpcsync.Subscriber) grpcsync.Subscriber { + return &wrappingConnectivityStateSubscriber{ + delegate: delegate, + connStateCh: wrappedSubscriber.connStateCh, + } + } + defer func() { newConnectivityStateSubscriber = origConnectivityStateSubscriber }() + + // Register an LB policy to act as the child policy for RLS LB policy. + childPolicyName := "test-child-policy" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName, nil) + t.Logf("Registered child policy with name %q", childPolicyName) + + // Build RLS service config with header matchers, and a very low value for + // maxAge to ensure that cache entries become invalid very soon. + rlsConfig := buildBasicRLSConfig(childPolicyName, rlsServer.Address) + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(defaultTestShortTimeout) + + // Start a test backend, and set up the fake RLS server to return this as a + // target in the RLS response. + backendCh, backendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, _ *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Wait for the control channel to move to READY. + waitForConnectivityState(ctx, t, wrappedSubscriber.connStateCh, connectivity.Ready) + + // Verify that the initial READY state did NOT trigger a backoff reset. + select { + case <-resetBackoffCalled: + t.Fatal("Backoff reset was triggered for initial READY state, want no reset") + case <-time.After(defaultTestShortTimeout): + } + + // Stop the RLS server to force the control channel to go IDLE. Stop() + // closes all existing connections on the listener. Since there are no active + // RPCs on the control channel, the subchannel transitions to IDLE instead of + // TRANSIENT_FAILURE. + lis.Stop() + + // Wait for the control channel to move to IDLE. + waitForConnectivityState(ctx, t, wrappedSubscriber.connStateCh, connectivity.Idle) + + // Restart the RLS server. + lis.Restart() + + // Make another RPC to trigger reconnection. Use different headers to create + // a new cache entry. + ctxOutgoing := metadata.AppendToOutgoingContext(ctx, "n1", "v1") + defaultBackoffStrategy = &fakeBackoffStrategy{backoff: defaultTestShortTimeout} + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh) + + // Wait for the control channel to move back to READY. + waitForConnectivityState(ctx, t, wrappedSubscriber.connStateCh, connectivity.Ready) + + // Verify that the READY → IDLE → READY transition did NOT trigger a backoff reset. + select { + case <-resetBackoffCalled: + t.Fatal("Backoff reset was triggered for READY → IDLE → READY transition, want no reset") + case <-time.After(defaultTestShortTimeout): + } +} + +// testCCWrapper wraps a balancer.ClientConn and overrides UpdateState and +// stores all state updates pushed by the RLS LB policy. +type testCCWrapper struct { + balancer.ClientConn + + mu sync.Mutex + states []balancer.State +} + +func (t *testCCWrapper) UpdateState(bs balancer.State) { + t.mu.Lock() + t.states = append(t.states, bs) + t.mu.Unlock() + t.ClientConn.UpdateState(bs) +} + +func (t *testCCWrapper) getStates() []balancer.State { + t.mu.Lock() + defer t.mu.Unlock() + + states := make([]balancer.State, len(t.states)) + copy(states, t.states) + return states +} + +// TestUpdateStatePauses tests the scenario where a config update received by +// the RLS LB policy results in multiple UpdateState calls from the child +// policies. This test verifies that picker updates are paused when the config +// update is being processed by RLS LB policy and its child policies. +// +// The test uses a wrapping balancer as the top-level LB policy on the channel. +// The wrapping balancer wraps an RLS LB policy as a child policy and forwards +// all calls to it. It also records the UpdateState() calls from the RLS LB +// policy and makes it available for inspection by the test. +// +// The test uses another wrapped balancer (which wraps a pickfirst balancer) as +// the child policy of the RLS LB policy. This balancer makes multiple +// UpdateState calls when handling an update from its parent in +// UpdateClientConnState. +func (s) TestUpdateStatePauses(t *testing.T) { + // Override the hook to get notified when UpdateClientConnState is done. + clientConnUpdateDone := make(chan struct{}, 1) + origClientConnUpdateHook := clientConnUpdateHook + clientConnUpdateHook = func() { clientConnUpdateDone <- struct{}{} } + defer func() { clientConnUpdateHook = origClientConnUpdateHook }() + + // Register the top-level wrapping balancer which forwards calls to RLS. + topLevelBalancerName := t.Name() + "top-level" + var ccWrapper *testCCWrapper + stub.Register(topLevelBalancerName, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + ccWrapper = &testCCWrapper{ClientConn: bd.ClientConn} + bd.ChildBalancer = balancer.Get(Name).Build(ccWrapper, bd.BuildOptions) + }, + ParseConfig: func(sc json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + parser := balancer.Get(Name).(balancer.ConfigParser) + return parser.ParseConfig(sc) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + }) + + // Register a child policy that wraps a pickfirst balancer and makes multiple calls + // to UpdateState when handling a config update in UpdateClientConnState. When + // this policy is used as a child policy of the RLS LB policy, it is expected + // that the latter suppress these updates and push a single picker update on the + // channel (after the config has been processed by all child policies). + childPolicyName := t.Name() + "child" + type childPolicyConfig struct { + serviceconfig.LoadBalancingConfig + Backend string // `json:"backend,omitempty"` + } + stub.Register(childPolicyName, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = balancer.Get(pickfirst.Name).Build(bd.ClientConn, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + ParseConfig: func(sc json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + cfg := &childPolicyConfig{} + if err := json.Unmarshal(sc, cfg); err != nil { + return nil, err + } + return cfg, nil + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + bal := bd.ChildBalancer + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Idle, Picker: &testutils.TestConstPicker{Err: balancer.ErrNoSubConnAvailable}}) + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: &testutils.TestConstPicker{Err: balancer.ErrNoSubConnAvailable}}) + + cfg := ccs.BalancerConfig.(*childPolicyConfig) + return bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{{Addr: cfg.Backend}}}, + }) + }, + }) + + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Start a test backend and set the RLS server to respond with it. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, _ *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := manual.NewBuilderWithScheme("rls-e2e") + scJSON := fmt.Sprintf(` +{ + "loadBalancingConfig": [ + { + "%s": { + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "grpc.testing.TestService"}] + }], + "lookupService": "%s", + "cacheSizeBytes": 1000 + }, + "childPolicy": [{"%s": {}}], + "childPolicyConfigTargetFieldName": "Backend" + } + } + ] +}`, topLevelBalancerName, rlsServer.Address, childPolicyName) + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.InitialState(resolver.State{ServiceConfig: sc}) + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + + // Wait for the clientconn update to be processed by the RLS LB policy. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + case <-clientConnUpdateDone: + } + + // It is important to note that at this point no child policies have been + // created because we have not attempted any RPC so far. When we attempt an + // RPC (below), child policies will be created and their configs will be + // pushed to them. But this config update will not happen in the context of + // a config update on the parent. + + // Make an RPC and ensure it gets routed to the test backend. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Wait for the control channel to become READY, before reading the states + // out of the wrapping top-level balancer. + // + // makeTestRPCAndExpectItToReachBackend repeatedly sends RPCs with short + // deadlines until one succeeds. See its docstring for details. + // + // The following sequence of events is possible: + // 1. When the first RPC is attempted above, a pending cache entry is + // created, an RLS request is sent out, and the pick is queued. The + // channel is in CONNECTING state. + // 2. When the RLS response arrives, the pending cache entry is moved to the + // data cache, a child policy is created for the target specified in the + // response and a new picker is returned. The channel is still in + // CONNECTING, and retried pick is again queued. + // 3. The child policy moves through the standard set of states, IDLE --> + // CONNECTING --> READY. And for each of these state changes, a new + // picker is sent on the channel. But the overall connectivity state of + // the channel is still CONNECTING. + // 4. Right around the time when the child policy becomes READY, the + // deadline associated with the first RPC made by + // makeTestRPCAndExpectItToReachBackend() could expire, and it could send + // a new one. And because the internal state of the LB policy now + // contains a child policy which is READY, this RPC will succeed. But the + // RLS LB policy has yet to push a new picker on the channel. + // 5. If we read the states seen by the top-level wrapping LB policy without + // waiting for the channel to become READY, there is a possibility that we + // might not see the READY state in there. And if that happens, we will + // see two extra states in the last check made in the test, and thereby + // the test would fail. Waiting for the channel to become READY here + // ensures that the test does not flake because of this rare sequence of + // events. + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Cache the state changes seen up to this point. + states0 := ccWrapper.getStates() + + // Push an updated service config. As mentioned earlier, the previous config + // updates on the child policies did not happen in the context of a config + // update on the parent. Hence, this update is required to force the + // scenario which we are interesting in testing here, i.e child policies get + // config updates as part of the parent policy getting its config update. + scJSON = fmt.Sprintf(` +{ + "loadBalancingConfig": [ + { + "%s": { + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [ + {"service": "grpc.testing.TestService"}, + {"service": "grpc.health.v1.Health"} + ] + }], + "lookupService": "%s", + "cacheSizeBytes": 1000 + }, + "childPolicy": [{"%s": {}}], + "childPolicyConfigTargetFieldName": "Backend" + } + } + ] +}`, topLevelBalancerName, rlsServer.Address, childPolicyName) + sc = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.UpdateState(resolver.State{ServiceConfig: sc}) + + // Wait for the clientconn update to be processed by the RLS LB policy. + select { + case <-ctx.Done(): + case <-clientConnUpdateDone: + } + + // Even though the child policies used in this test make multiple calls to + // UpdateState as part of handling their configs, we expect the RLS policy + // to inhibit picker updates during this time frame, and send a single + // picker once the config update is completely handled. + states1 := ccWrapper.getStates() + if len(states1) != len(states0)+1 { + t.Fatalf("more than one state update seen. before %v, after %v", states0, states1) + } +} + +// waitForStateTransitions waits for the given number of state updates on the +// channel and returns the sequence of connectivity states received. +func waitForStateTransitions(ctx context.Context, t *testing.T, stateCh <-chan balancer.State, wantNum int) []connectivity.State { + t.Helper() + + var gotStates []connectivity.State + for i := 0; i < wantNum; i++ { + select { + case state := <-stateCh: + gotStates = append(gotStates, state.ConnectivityState) + case <-ctx.Done(): + t.Fatalf("Timeout waiting for %d balancer state updates, got %d", wantNum, len(gotStates)) + } + } + return gotStates +} diff --git a/balancer/rls/cache.go b/balancer/rls/cache.go new file mode 100644 index 0000000..2f48d85 --- /dev/null +++ b/balancer/rls/cache.go @@ -0,0 +1,378 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "container/list" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc/internal/backoff" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" +) + +// cacheKey represents the key used to uniquely identify an entry in the data +// cache and in the pending requests map. +type cacheKey struct { + // path is the full path of the incoming RPC request. + path string + // keys is a stringified version of the RLS request key map built using the + // RLS keyBuilder. Since maps are not a type which is comparable in Go, it + // cannot be part of the key for another map (entries in the data cache and + // pending requests map are stored in maps). + keys string +} + +// cacheEntry wraps all the data to be stored in a data cache entry. +type cacheEntry struct { + // childPolicyWrappers contains the list of child policy wrappers + // corresponding to the targets returned by the RLS server for this entry. + childPolicyWrappers []*childPolicyWrapper + // headerData is received in the RLS response and is to be sent in the + // X-Google-RLS-Data header for matching RPCs. + headerData string + // expiryTime is the absolute time at which this cache entry stops + // being valid. When an RLS request succeeds, this is set to the current + // time plus the max_age field from the LB policy config. + expiryTime time.Time + // staleTime is the absolute time after which this cache entry will be + // proactively refreshed if an incoming RPC matches this entry. When an RLS + // request succeeds, this is set to the current time plus the stale_age from + // the LB policy config. + staleTime time.Time + // earliestEvictTime is the absolute time before which this entry should not + // be evicted from the cache. When a cache entry is created, this is set to + // the current time plus a default value of 5 seconds. This is required to + // make sure that a new entry added to the cache is not evicted before the + // RLS response arrives (usually when the cache is too small). + earliestEvictTime time.Time + + // status stores the RPC status of the previous RLS request for this + // entry. Picks for entries with a non-nil value for this field are failed + // with the error stored here. + status error + // backoffState contains all backoff related state. When an RLS request + // succeeds, backoffState is reset. This state moves between the data cache + // and the pending requests map. + backoffState *backoffState + // backoffTime is the absolute time at which the backoff period for this + // entry ends. When an RLS request fails, this is set to the current time + // plus the backoff value returned by the backoffState. The backoff timer is + // also setup with this value. No new RLS requests are sent out for this + // entry until the backoff period ends. + // + // Set to zero time instant upon a successful RLS response. + backoffTime time.Time + // backoffExpiryTime is the absolute time at which an entry which has gone + // through backoff stops being valid. When an RLS request fails, this is + // set to the current time plus twice the backoff time. The cache expiry + // timer will only delete entries for which both expiryTime and + // backoffExpiryTime are in the past. + // + // Set to zero time instant upon a successful RLS response. + backoffExpiryTime time.Time + + // size stores the size of this cache entry. Used to enforce the cache size + // specified in the LB policy configuration. + size int64 +} + +// backoffState wraps all backoff related state associated with a cache entry. +type backoffState struct { + // retries keeps track of the number of RLS failures, to be able to + // determine the amount of time to backoff before the next attempt. + retries int + // bs is the exponential backoff implementation which returns the amount of + // time to backoff, given the number of retries. + bs backoff.Strategy + // timer fires when the backoff period ends and incoming requests after this + // will trigger a new RLS request. + timer *time.Timer +} + +// lru is a cache implementation with a least recently used eviction policy. +// Internally it uses a doubly linked list, with the least recently used element +// at the front of the list and the most recently used element at the back of +// the list. The value stored in this cache will be of type `cacheKey`. +// +// It is not safe for concurrent access. +type lru struct { + ll *list.List + + // A map from the value stored in the lru to its underlying list element is + // maintained to have a clean API. Without this, a subset of the lru's API + // would accept/return cacheKey while another subset would accept/return + // list elements. + m map[cacheKey]*list.Element +} + +// newLRU creates a new cache with a least recently used eviction policy. +func newLRU() *lru { + return &lru{ + ll: list.New(), + m: make(map[cacheKey]*list.Element), + } +} + +func (l *lru) addEntry(key cacheKey) { + e := l.ll.PushBack(key) + l.m[key] = e +} + +func (l *lru) makeRecent(key cacheKey) { + e := l.m[key] + l.ll.MoveToBack(e) +} + +func (l *lru) removeEntry(key cacheKey) { + e := l.m[key] + l.ll.Remove(e) + delete(l.m, key) +} + +func (l *lru) getLeastRecentlyUsed() cacheKey { + e := l.ll.Front() + if e == nil { + return cacheKey{} + } + return e.Value.(cacheKey) +} + +// dataCache contains a cache of RLS data used by the LB policy to make routing +// decisions. +// +// The dataCache will be keyed by the request's path and keys, represented by +// the `cacheKey` type. It will maintain the cache keys in an `lru` and the +// cache data, represented by the `cacheEntry` type, in a native map. +// +// It is not safe for concurrent access. +type dataCache struct { + maxSize int64 // Maximum allowed size. + currentSize int64 // Current size. + keys *lru // Cache keys maintained in lru order. + entries map[cacheKey]*cacheEntry + logger *internalgrpclog.PrefixLogger + shutdown *grpcsync.Event + rlsServerTarget string + + // Read only after initialization. + grpcTarget string + uuid string +} + +func newDataCache(size int64, logger *internalgrpclog.PrefixLogger, grpcTarget string) *dataCache { + return &dataCache{ + maxSize: size, + keys: newLRU(), + entries: make(map[cacheKey]*cacheEntry), + logger: logger, + shutdown: grpcsync.NewEvent(), + grpcTarget: grpcTarget, + uuid: uuid.New().String(), + } +} + +// updateRLSServerTarget updates the RLS Server Target the RLS Balancer is +// configured with. +func (dc *dataCache) updateRLSServerTarget(rlsServerTarget string) { + dc.rlsServerTarget = rlsServerTarget +} + +// resize changes the maximum allowed size of the data cache. +// +// The return value indicates if an entry with a valid backoff timer was +// evicted. This is important to the RLS LB policy which would send a new picker +// on the channel to re-process any RPCs queued as a result of this backoff +// timer. +func (dc *dataCache) resize(size int64) (backoffCancelled bool) { + if dc.shutdown.HasFired() { + return false + } + + backoffCancelled = false + for dc.currentSize > size { + key := dc.keys.getLeastRecentlyUsed() + entry, ok := dc.entries[key] + if !ok { + // This should never happen. + dc.logger.Errorf("cacheKey %+v not found in the cache while attempting to resize it", key) + break + } + + // When we encounter a cache entry whose minimum expiration time is in + // the future, we abort the LRU pass, which may temporarily leave the + // cache being too large. This is necessary to ensure that in cases + // where the cache is too small, when we receive an RLS Response, we + // keep the resulting cache entry around long enough for the pending + // incoming requests to be re-processed through the new Picker. If we + // didn't do this, then we'd risk throwing away each RLS response as we + // receive it, in which case we would fail to actually route any of our + // incoming requests. + if entry.earliestEvictTime.After(time.Now()) { + dc.logger.Warningf("cachekey %+v is too recent to be evicted. Stopping cache resizing for now", key) + break + } + + // Stop the backoff timer before evicting the entry. + if entry.backoffState != nil && entry.backoffState.timer != nil { + if entry.backoffState.timer.Stop() { + entry.backoffState.timer = nil + backoffCancelled = true + } + } + dc.deleteAndCleanup(key, entry) + } + dc.maxSize = size + return backoffCancelled +} + +// evictExpiredEntries sweeps through the cache and deletes expired entries. An +// expired entry is one for which both the `expiryTime` and `backoffExpiryTime` +// fields are in the past. +// +// The return value indicates if any expired entries were evicted. +// +// The LB policy invokes this method periodically to purge expired entries. +func (dc *dataCache) evictExpiredEntries() bool { + if dc.shutdown.HasFired() { + return false + } + + evicted := false + for key, entry := range dc.entries { + // Only evict entries for which both the data expiration time and + // backoff expiration time fields are in the past. + now := time.Now() + if entry.expiryTime.After(now) || entry.backoffExpiryTime.After(now) { + continue + } + dc.deleteAndCleanup(key, entry) + evicted = true + } + return evicted +} + +// resetBackoffState sweeps through the cache and for entries with a backoff +// state, the backoff timer is cancelled and the backoff state is reset. The +// return value indicates if any entries were mutated in this fashion. +// +// The LB policy invokes this method when the control channel moves from READY +// to TRANSIENT_FAILURE back to READY. See `monitorConnectivityState` method on +// the `controlChannel` type for more details. +func (dc *dataCache) resetBackoffState(newBackoffState *backoffState) bool { + if dc.shutdown.HasFired() { + return false + } + + backoffReset := false + for _, entry := range dc.entries { + if entry.backoffState == nil { + continue + } + if entry.backoffState.timer != nil { + entry.backoffState.timer.Stop() + entry.backoffState.timer = nil + } + entry.backoffState = &backoffState{bs: newBackoffState.bs} + entry.backoffTime = time.Time{} + entry.backoffExpiryTime = time.Time{} + backoffReset = true + } + return backoffReset +} + +// addEntry adds a cache entry for the given key. +// +// Return value backoffCancelled indicates if a cache entry with a valid backoff +// timer was evicted to make space for the current entry. This is important to +// the RLS LB policy which would send a new picker on the channel to re-process +// any RPCs queued as a result of this backoff timer. +// +// Return value ok indicates if entry was successfully added to the cache. +func (dc *dataCache) addEntry(key cacheKey, entry *cacheEntry) (backoffCancelled bool, ok bool) { + if dc.shutdown.HasFired() { + return false, false + } + + // Handle the extremely unlikely case that a single entry is bigger than the + // size of the cache. + if entry.size > dc.maxSize { + return false, false + } + dc.entries[key] = entry + dc.currentSize += entry.size + dc.keys.addEntry(key) + // If the new entry makes the cache go over its configured size, remove some + // old entries. + if dc.currentSize > dc.maxSize { + backoffCancelled = dc.resize(dc.maxSize) + } + + return backoffCancelled, true +} + +// updateEntrySize updates the size of a cache entry and the current size of the +// data cache. An entry's size can change upon receipt of an RLS response. +func (dc *dataCache) updateEntrySize(entry *cacheEntry, newSize int64) { + dc.currentSize -= entry.size + entry.size = newSize + dc.currentSize += entry.size + +} + +func (dc *dataCache) getEntry(key cacheKey) *cacheEntry { + if dc.shutdown.HasFired() { + return nil + } + + entry, ok := dc.entries[key] + if !ok { + return nil + } + dc.keys.makeRecent(key) + return entry +} + +func (dc *dataCache) removeEntryForTesting(key cacheKey) { + entry, ok := dc.entries[key] + if !ok { + return + } + dc.deleteAndCleanup(key, entry) +} + +// deleteAndCleanup performs actions required at the time of deleting an entry +// from the data cache. +// - the entry is removed from the map of entries +// - current size of the data cache is update +// - the key is removed from the LRU +func (dc *dataCache) deleteAndCleanup(key cacheKey, entry *cacheEntry) { + delete(dc.entries, key) + dc.currentSize -= entry.size + dc.keys.removeEntry(key) + +} + +func (dc *dataCache) stop() { + for key, entry := range dc.entries { + dc.deleteAndCleanup(key, entry) + } + dc.shutdown.Fire() +} diff --git a/balancer/rls/cache_test.go b/balancer/rls/cache_test.go new file mode 100644 index 0000000..5a35665 --- /dev/null +++ b/balancer/rls/cache_test.go @@ -0,0 +1,243 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/backoff" +) + +var ( + cacheKeys = []cacheKey{ + {path: "0", keys: "a"}, + {path: "1", keys: "b"}, + {path: "2", keys: "c"}, + {path: "3", keys: "d"}, + {path: "4", keys: "e"}, + } + + longDuration = 10 * time.Minute + shortDuration = 1 * time.Millisecond + cacheEntries []*cacheEntry +) + +func initCacheEntries() { + // All entries have a dummy size of 1 to simplify resize operations. + cacheEntries = []*cacheEntry{ + { + // Entry is valid and minimum expiry time has not expired. + expiryTime: time.Now().Add(longDuration), + earliestEvictTime: time.Now().Add(longDuration), + size: 1, + }, + { + // Entry is valid and is in backoff. + expiryTime: time.Now().Add(longDuration), + backoffTime: time.Now().Add(longDuration), + backoffState: &backoffState{timer: time.NewTimer(longDuration)}, + size: 1, + }, + { + // Entry is valid, and not in backoff. + expiryTime: time.Now().Add(longDuration), + size: 1, + }, + { + // Entry is invalid. + expiryTime: time.Time{}.Add(shortDuration), + size: 1, + }, + { + // Entry is invalid valid and backoff has expired. + expiryTime: time.Time{}.Add(shortDuration), + backoffExpiryTime: time.Time{}.Add(shortDuration), + size: 1, + }, + } +} + +func (s) TestLRU_BasicOperations(t *testing.T) { + initCacheEntries() + // Create an LRU and add some entries to it. + lru := newLRU() + for _, k := range cacheKeys { + lru.addEntry(k) + } + + // Get the least recent entry. This should be the first entry we added. + if got, want := lru.getLeastRecentlyUsed(), cacheKeys[0]; got != want { + t.Fatalf("lru.getLeastRecentlyUsed() = %v, want %v", got, want) + } + + // Iterate through the slice of keys we added earlier, making them the most + // recent entry, one at a time. The least recent entry at that point should + // be the next entry from our slice of keys. + for i, k := range cacheKeys { + lru.makeRecent(k) + + lruIndex := (i + 1) % len(cacheKeys) + if got, want := lru.getLeastRecentlyUsed(), cacheKeys[lruIndex]; got != want { + t.Fatalf("lru.getLeastRecentlyUsed() = %v, want %v", got, want) + } + } + + // Iterate through the slice of keys we added earlier, removing them one at + // a time The least recent entry at that point should be the next entry from + // our slice of keys, except for the last one because the lru will be empty. + for i, k := range cacheKeys { + lru.removeEntry(k) + + var want cacheKey + if i < len(cacheKeys)-1 { + want = cacheKeys[i+1] + } + if got := lru.getLeastRecentlyUsed(); got != want { + t.Fatalf("lru.getLeastRecentlyUsed() = %v, want %v", got, want) + } + } +} + +func (s) TestDataCache_BasicOperations(t *testing.T) { + initCacheEntries() + dc := newDataCache(5, nil, "") + for i, k := range cacheKeys { + dc.addEntry(k, cacheEntries[i]) + } + for i, k := range cacheKeys { + entry := dc.getEntry(k) + if !cmp.Equal(entry, cacheEntries[i], cmp.AllowUnexported(cacheEntry{}, backoffState{}), cmpopts.IgnoreUnexported(time.Timer{})) { + t.Fatalf("Data cache lookup for key %v returned entry %v, want %v", k, entry, cacheEntries[i]) + } + } +} + +func (s) TestDataCache_AddForcesResize(t *testing.T) { + initCacheEntries() + dc := newDataCache(1, nil, "") + + // The first entry in cacheEntries has a minimum expiry time in the future. + // This entry would stop the resize operation since we do not evict entries + // whose minimum expiration time is in the future. So, we do not use that + // entry in this test. The entry being added has a running backoff timer. + evicted, ok := dc.addEntry(cacheKeys[1], cacheEntries[1]) + if evicted || !ok { + t.Fatalf("dataCache.addEntry() returned (%v, %v) want (false, true)", evicted, ok) + } + + // Add another entry leading to the eviction of the above entry which has a + // running backoff timer. The first return value is expected to be true. + backoffCancelled, ok := dc.addEntry(cacheKeys[2], cacheEntries[2]) + if !backoffCancelled || !ok { + t.Fatalf("dataCache.addEntry() returned (%v, %v) want (true, true)", backoffCancelled, ok) + } + + // Add another entry leading to the eviction of the above entry which does not + // have a running backoff timer. This should evict the above entry, but the + // first return value is expected to be false. + backoffCancelled, ok = dc.addEntry(cacheKeys[3], cacheEntries[3]) + if backoffCancelled || !ok { + t.Fatalf("dataCache.addEntry() returned (%v, %v) want (false, true)", backoffCancelled, ok) + } +} + +func (s) TestDataCache_Resize(t *testing.T) { + initCacheEntries() + dc := newDataCache(5, nil, "") + for i, k := range cacheKeys { + dc.addEntry(k, cacheEntries[i]) + } + + // The first cache entry (with a key of cacheKeys[0]) that we added has an + // earliestEvictTime in the future. As part of the resize operation, we + // traverse the cache in least recently used order, and this will be first + // entry that we will encounter. And since the earliestEvictTime is in the + // future, the resize operation will stop, leaving the cache bigger than + // what was asked for. + if dc.resize(1) { + t.Fatalf("dataCache.resize() returned true, want false") + } + if dc.currentSize != 5 { + t.Fatalf("dataCache.size is %d, want 5", dc.currentSize) + } + + // Remove the entry with earliestEvictTime in the future and retry the + // resize operation. + dc.removeEntryForTesting(cacheKeys[0]) + if !dc.resize(1) { + t.Fatalf("dataCache.resize() returned false, want true") + } + if dc.currentSize != 1 { + t.Fatalf("dataCache.size is %d, want 1", dc.currentSize) + } +} + +func (s) TestDataCache_EvictExpiredEntries(t *testing.T) { + initCacheEntries() + dc := newDataCache(5, nil, "") + for i, k := range cacheKeys { + dc.addEntry(k, cacheEntries[i]) + } + + // The last two entries in the cacheEntries list have expired, and will be + // evicted. The first three should still remain in the cache. + if !dc.evictExpiredEntries() { + t.Fatal("dataCache.evictExpiredEntries() returned false, want true") + } + if dc.currentSize != 3 { + t.Fatalf("dataCache.size is %d, want 3", dc.currentSize) + } + for i := 0; i < 3; i++ { + entry := dc.getEntry(cacheKeys[i]) + if !cmp.Equal(entry, cacheEntries[i], cmp.AllowUnexported(cacheEntry{}, backoffState{}), cmpopts.IgnoreUnexported(time.Timer{})) { + t.Fatalf("Data cache lookup for key %v returned entry %v, want %v", cacheKeys[i], entry, cacheEntries[i]) + } + } +} + +func (s) TestDataCache_ResetBackoffState(t *testing.T) { + type fakeBackoff struct { + backoff.Strategy + } + + initCacheEntries() + dc := newDataCache(5, nil, "") + for i, k := range cacheKeys { + dc.addEntry(k, cacheEntries[i]) + } + + newBackoffState := &backoffState{bs: &fakeBackoff{}} + if updatePicker := dc.resetBackoffState(newBackoffState); !updatePicker { + t.Fatal("dataCache.resetBackoffState() returned updatePicker is false, want true") + } + + // Make sure that the entry with no backoff state was not touched. + if entry := dc.getEntry(cacheKeys[0]); cmp.Equal(entry.backoffState, newBackoffState, cmp.AllowUnexported(backoffState{})) { + t.Fatal("dataCache.resetBackoffState() touched entries without a valid backoffState") + } + + // Make sure that the entry with a valid backoff state was reset. + entry := dc.getEntry(cacheKeys[1]) + if diff := cmp.Diff(entry.backoffState, newBackoffState, cmp.AllowUnexported(backoffState{})); diff != "" { + t.Fatalf("unexpected diff in backoffState for cache entry after dataCache.resetBackoffState(): %s", diff) + } +} diff --git a/balancer/rls/child_policy.go b/balancer/rls/child_policy.go new file mode 100644 index 0000000..c74184c --- /dev/null +++ b/balancer/rls/child_policy.go @@ -0,0 +1,109 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "fmt" + "sync/atomic" + "unsafe" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +// childPolicyWrapper is a reference counted wrapper around a child policy. +// +// The LB policy maintains a map of these wrappers keyed by the target returned +// by RLS. When a target is seen for the first time, a child policy wrapper is +// created for it and the wrapper is added to the child policy map. Each entry +// in the data cache holds references to the corresponding child policy +// wrappers. The LB policy also holds a reference to the child policy wrapper +// for the default target specified in the LB Policy Configuration +// +// When a cache entry is evicted, it releases references to the child policy +// wrappers that it contains. When all references have been released, the +// wrapper is removed from the child policy map and is destroyed. +// +// The child policy wrapper also caches the connectivity state and most recent +// picker from the child policy. Once the child policy wrapper reports +// TRANSIENT_FAILURE, it will continue reporting that state until it goes READY; +// transitions from TRANSIENT_FAILURE to CONNECTING are ignored. +// +// Whenever a child policy wrapper changes its connectivity state, the LB policy +// returns a new picker to the channel, since the channel may need to re-process +// the picks for queued RPCs. +// +// It is not safe for concurrent access. +type childPolicyWrapper struct { + logger *internalgrpclog.PrefixLogger + target string // RLS target corresponding to this child policy. + refCnt int // Reference count. + + // Balancer state reported by the child policy. The RLS LB policy maintains + // these child policies in a BalancerGroup. The state reported by the child + // policy is pushed to the state aggregator (which is also implemented by the + // RLS LB policy) and cached here. See handleChildPolicyStateUpdate() for + // details on how the state aggregation is performed. + // + // While this field is written to by the LB policy, it is read by the picker + // at Pick time. Making this an atomic to enable the picker to read this value + // without a mutex. + state unsafe.Pointer // *balancer.State +} + +// newChildPolicyWrapper creates a child policy wrapper for the given target, +// and is initialized with one reference and starts off in CONNECTING state. +func newChildPolicyWrapper(target string) *childPolicyWrapper { + c := &childPolicyWrapper{ + target: target, + refCnt: 1, + state: unsafe.Pointer(&balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + }), + } + c.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[rls-child-policy-wrapper %s %p] ", c.target, c)) + c.logger.Infof("Created") + return c +} + +// acquireRef increments the reference count on the child policy wrapper. +func (c *childPolicyWrapper) acquireRef() { + c.refCnt++ +} + +// releaseRef decrements the reference count on the child policy wrapper. The +// return value indicates whether the released reference was the last one. +func (c *childPolicyWrapper) releaseRef() bool { + c.refCnt-- + return c.refCnt == 0 +} + +// lamify causes the child policy wrapper to return a picker which will always +// fail requests. This is used when the wrapper runs into errors when trying to +// build and parse the child policy configuration. +func (c *childPolicyWrapper) lamify(err error) { + c.logger.Warningf("Entering lame mode: %v", err) + atomic.StorePointer(&c.state, unsafe.Pointer(&balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(err), + })) +} diff --git a/balancer/rls/config.go b/balancer/rls/config.go new file mode 100644 index 0000000..9693c8b --- /dev/null +++ b/balancer/rls/config.go @@ -0,0 +1,330 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "bytes" + "encoding/json" + "fmt" + "net/url" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/rls/internal/keys" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/pretty" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/durationpb" +) + +const ( + // Default max_age if not specified (or greater than this value) in the + // service config. + maxMaxAge = 5 * time.Minute + // Upper limit for cache_size since we don't fully trust the service config. + maxCacheSize = 5 * 1024 * 1024 * 8 // 5MB in bytes + // Default lookup_service_timeout if not specified in the service config. + defaultLookupServiceTimeout = 10 * time.Second + // Default value for targetNameField in the child policy config during + // service config validation. + dummyChildPolicyTarget = "target_name_to_be_filled_in_later" +) + +// lbConfig is the internal representation of the RLS LB policy's config. +type lbConfig struct { + serviceconfig.LoadBalancingConfig + + cacheSizeBytes int64 // Keep this field 64-bit aligned. + kbMap keys.BuilderMap + lookupService string + lookupServiceTimeout time.Duration + maxAge time.Duration + staleAge time.Duration + defaultTarget string + + childPolicyName string + childPolicyConfig map[string]json.RawMessage + childPolicyTargetField string + controlChannelServiceConfig string +} + +func (lbCfg *lbConfig) Equal(other *lbConfig) bool { + return lbCfg.kbMap.Equal(other.kbMap) && + lbCfg.lookupService == other.lookupService && + lbCfg.lookupServiceTimeout == other.lookupServiceTimeout && + lbCfg.maxAge == other.maxAge && + lbCfg.staleAge == other.staleAge && + lbCfg.cacheSizeBytes == other.cacheSizeBytes && + lbCfg.defaultTarget == other.defaultTarget && + lbCfg.childPolicyName == other.childPolicyName && + lbCfg.childPolicyTargetField == other.childPolicyTargetField && + lbCfg.controlChannelServiceConfig == other.controlChannelServiceConfig && + childPolicyConfigEqual(lbCfg.childPolicyConfig, other.childPolicyConfig) +} + +func childPolicyConfigEqual(a, b map[string]json.RawMessage) bool { + if (b == nil) != (a == nil) { + return false + } + if len(b) != len(a) { + return false + } + for k, jsonA := range a { + jsonB, ok := b[k] + if !ok { + return false + } + if !bytes.Equal(jsonA, jsonB) { + return false + } + } + return true +} + +// This struct resembles the JSON representation of the loadBalancing config +// and makes it easier to unmarshal. +type lbConfigJSON struct { + RouteLookupConfig json.RawMessage + RouteLookupChannelServiceConfig json.RawMessage + ChildPolicy []map[string]json.RawMessage + ChildPolicyConfigTargetFieldName string +} + +// ParseConfig parses the JSON load balancer config provided into an +// internal form or returns an error if the config is invalid. +// +// When parsing a config update, the following validations are performed: +// - routeLookupConfig: +// - grpc_keybuilders field: +// - must have at least one entry +// - must not have two entries with the same `Name` +// - within each entry: +// - must have at least one `Name` +// - must not have a `Name` with the `service` field unset or empty +// - within each `headers` entry: +// - must not have `required_match` set +// - must not have `key` unset or empty +// - across all `headers`, `constant_keys` and `extra_keys` fields: +// - must not have the same `key` specified twice +// - no `key` must be the empty string +// - `lookup_service` field must be set and must parse as a target URI +// - if `max_age` > 5m, it should be set to 5 minutes +// - if `stale_age` > `max_age`, ignore it +// - if `stale_age` is set, then `max_age` must also be set +// - ignore `valid_targets` field +// - `cache_size_bytes` field must have a value greater than 0, and if its +// value is greater than 5M, we cap it at 5M +// +// - routeLookupChannelServiceConfig: +// - if specified, must parse as valid service config +// +// - childPolicy: +// - must find a valid child policy with a valid config +// +// - childPolicyConfigTargetFieldName: +// - must be set and non-empty +func (rlsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + if logger.V(2) { + logger.Infof("Received JSON service config: %v", pretty.ToJSON(c)) + } + + cfgJSON := &lbConfigJSON{} + if err := json.Unmarshal(c, cfgJSON); err != nil { + return nil, fmt.Errorf("rls: json unmarshal failed for service config %+v: %v", string(c), err) + } + + m := protojson.UnmarshalOptions{DiscardUnknown: true} + rlsProto := &rlspb.RouteLookupConfig{} + if err := m.Unmarshal(cfgJSON.RouteLookupConfig, rlsProto); err != nil { + return nil, fmt.Errorf("rls: bad RouteLookupConfig proto %+v: %v", string(cfgJSON.RouteLookupConfig), err) + } + lbCfg, err := parseRLSProto(rlsProto) + if err != nil { + return nil, err + } + + if sc := string(cfgJSON.RouteLookupChannelServiceConfig); sc != "" { + parsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(sc) + if parsed.Err != nil { + return nil, fmt.Errorf("rls: bad control channel service config %q: %v", sc, parsed.Err) + } + lbCfg.controlChannelServiceConfig = sc + } + + if cfgJSON.ChildPolicyConfigTargetFieldName == "" { + return nil, fmt.Errorf("rls: childPolicyConfigTargetFieldName field is not set in service config %+v", string(c)) + } + name, config, err := parseChildPolicyConfigs(cfgJSON.ChildPolicy, cfgJSON.ChildPolicyConfigTargetFieldName) + if err != nil { + return nil, err + } + lbCfg.childPolicyName = name + lbCfg.childPolicyConfig = config + lbCfg.childPolicyTargetField = cfgJSON.ChildPolicyConfigTargetFieldName + return lbCfg, nil +} + +func parseRLSProto(rlsProto *rlspb.RouteLookupConfig) (*lbConfig, error) { + // Validations specified on the `grpc_keybuilders` field are performed here. + kbMap, err := keys.MakeBuilderMap(rlsProto) + if err != nil { + return nil, err + } + + // `lookup_service` field must be set and must parse as a target URI. + lookupService := rlsProto.GetLookupService() + if lookupService == "" { + return nil, fmt.Errorf("rls: empty lookup_service in route lookup config %+v", rlsProto) + } + parsedTarget, err := url.Parse(lookupService) + if err != nil { + // url.Parse() fails if scheme is missing. Retry with default scheme. + parsedTarget, err = url.Parse(resolver.GetDefaultScheme() + ":///" + lookupService) + if err != nil { + return nil, fmt.Errorf("rls: invalid target URI in lookup_service %s", lookupService) + } + } + if parsedTarget.Scheme == "" { + parsedTarget.Scheme = resolver.GetDefaultScheme() + } + if resolver.Get(parsedTarget.Scheme) == nil { + return nil, fmt.Errorf("rls: unregistered scheme in lookup_service %s", lookupService) + } + + lookupServiceTimeout, err := convertDuration(rlsProto.GetLookupServiceTimeout()) + if err != nil { + return nil, fmt.Errorf("rls: failed to parse lookup_service_timeout in route lookup config %+v: %v", rlsProto, err) + } + if lookupServiceTimeout == 0 { + lookupServiceTimeout = defaultLookupServiceTimeout + } + + // Validations performed here: + // - if `max_age` > 5m, it should be set to 5 minutes + // only if stale age is not set + // - if `stale_age` > `max_age`, ignore it + // - if `stale_age` is set, then `max_age` must also be set + maxAgeSet := false + maxAge, err := convertDuration(rlsProto.GetMaxAge()) + if err != nil { + return nil, fmt.Errorf("rls: failed to parse max_age in route lookup config %+v: %v", rlsProto, err) + } + if maxAge == 0 { + maxAge = maxMaxAge + } else { + maxAgeSet = true + } + + staleAgeSet := false + staleAge, err := convertDuration(rlsProto.GetStaleAge()) + if err != nil { + return nil, fmt.Errorf("rls: failed to parse staleAge in route lookup config %+v: %v", rlsProto, err) + } + if staleAge == 0 { + staleAge = maxMaxAge + } else { + staleAgeSet = true + } + + if staleAgeSet && !maxAgeSet { + return nil, fmt.Errorf("rls: stale_age is set, but max_age is not in route lookup config %+v", rlsProto) + } + if staleAge > maxMaxAge { + staleAge = maxMaxAge + } + if !staleAgeSet && maxAge > maxMaxAge { + maxAge = maxMaxAge + } + if staleAge > maxAge { + staleAge = maxAge + } + + // `cache_size_bytes` field must have a value greater than 0, and if its + // value is greater than 5M, we cap it at 5M + cacheSizeBytes := rlsProto.GetCacheSizeBytes() + if cacheSizeBytes <= 0 { + return nil, fmt.Errorf("rls: cache_size_bytes must be set to a non-zero value: %+v", rlsProto) + } + if cacheSizeBytes > maxCacheSize { + logger.Infof("rls: cache_size_bytes %v is too large, setting it to: %v", cacheSizeBytes, maxCacheSize) + cacheSizeBytes = maxCacheSize + } + return &lbConfig{ + kbMap: kbMap, + lookupService: lookupService, + lookupServiceTimeout: lookupServiceTimeout, + maxAge: maxAge, + staleAge: staleAge, + cacheSizeBytes: cacheSizeBytes, + defaultTarget: rlsProto.GetDefaultTarget(), + }, nil +} + +// parseChildPolicyConfigs iterates through the list of child policies and picks +// the first registered policy and validates its config. +func parseChildPolicyConfigs(childPolicies []map[string]json.RawMessage, targetFieldName string) (string, map[string]json.RawMessage, error) { + for i, config := range childPolicies { + if len(config) != 1 { + return "", nil, fmt.Errorf("rls: invalid childPolicy: entry %v does not contain exactly 1 policy/config pair: %q", i, config) + } + + var name string + var rawCfg json.RawMessage + for name, rawCfg = range config { + } + builder := balancer.Get(name) + if builder == nil { + continue + } + parser, ok := builder.(balancer.ConfigParser) + if !ok { + return "", nil, fmt.Errorf("rls: childPolicy %q with config %q does not support config parsing", name, string(rawCfg)) + } + + // To validate child policy configs we do the following: + // - unmarshal the raw JSON bytes of the child policy config into a map + // - add an entry with key set to `target_field_name` and a dummy value + // - marshal the map back to JSON and parse the config using the parser + // retrieved previously + var childConfig map[string]json.RawMessage + if err := json.Unmarshal(rawCfg, &childConfig); err != nil { + return "", nil, fmt.Errorf("rls: json unmarshal failed for child policy config %q: %v", string(rawCfg), err) + } + childConfig[targetFieldName], _ = json.Marshal(dummyChildPolicyTarget) + jsonCfg, err := json.Marshal(childConfig) + if err != nil { + return "", nil, fmt.Errorf("rls: json marshal failed for child policy config {%+v}: %v", childConfig, err) + } + if _, err := parser.ParseConfig(jsonCfg); err != nil { + return "", nil, fmt.Errorf("rls: childPolicy config validation failed: %v", err) + } + return name, childConfig, nil + } + return "", nil, fmt.Errorf("rls: invalid childPolicy config: no supported policies found in %+v", childPolicies) +} + +func convertDuration(d *durationpb.Duration) (time.Duration, error) { + if d == nil { + return 0, nil + } + return d.AsDuration(), d.CheckValid() +} diff --git a/balancer/rls/config_test.go b/balancer/rls/config_test.go new file mode 100644 index 0000000..c1aff0c --- /dev/null +++ b/balancer/rls/config_test.go @@ -0,0 +1,515 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + _ "google.golang.org/grpc/balancer/grpclb" // grpclb for config parsing. + _ "google.golang.org/grpc/internal/resolver/passthrough" // passthrough resolver. +) + +// testEqual reports whether the lbCfgs a and b are equal. This is to be used +// only from tests. This ignores the keyBuilderMap field because its internals +// are not exported, and hence not possible to specify in the want section of +// the test. This is fine because we already have tests to make sure that the +// keyBuilder is parsed properly from the service config. +func testEqual(a, b *lbConfig) bool { + return a.lookupService == b.lookupService && + a.lookupServiceTimeout == b.lookupServiceTimeout && + a.maxAge == b.maxAge && + a.staleAge == b.staleAge && + a.cacheSizeBytes == b.cacheSizeBytes && + a.defaultTarget == b.defaultTarget && + a.controlChannelServiceConfig == b.controlChannelServiceConfig && + a.childPolicyName == b.childPolicyName && + a.childPolicyTargetField == b.childPolicyTargetField && + childPolicyConfigEqual(a.childPolicyConfig, b.childPolicyConfig) +} + +// TestParseConfig verifies successful config parsing scenarios. +func (s) TestParseConfig(t *testing.T) { + childPolicyTargetFieldVal, _ := json.Marshal(dummyChildPolicyTarget) + tests := []struct { + desc string + input []byte + wantCfg *lbConfig + }{ + { + // This input validates a few cases: + // - A top-level unknown field should not fail. + // - An unknown field in routeLookupConfig proto should not fail. + // - lookupServiceTimeout is set to its default value, since it is not specified in the input. + // - maxAge is clamped to maxMaxAge if staleAge is not set. + // - staleAge is ignored because it is higher than maxAge in the input. + // - cacheSizeBytes is greater than the hard upper limit of 5MB + desc: "with transformations 1", + input: []byte(`{ + "top-level-unknown-field": "unknown-value", + "routeLookupConfig": { + "unknown-field": "unknown-value", + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": ":///target", + "maxAge" : "500s", + "staleAge": "600s", + "cacheSizeBytes": 100000000, + "defaultTarget": "passthrough:///default" + }, + "childPolicy": [ + {"cds_experimental": {"Cluster": "my-fav-cluster"}}, + {"unknown-policy": {"unknown-field": "unknown-value"}}, + {"grpclb": {"childPolicy": [{"pickfirst": {}}]}} + ], + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantCfg: &lbConfig{ + lookupService: ":///target", + lookupServiceTimeout: 10 * time.Second, // This is the default value. + maxAge: 500 * time.Second, // Max age is not clamped when stale age is set. + staleAge: 300 * time.Second, // StaleAge is clamped because it was higher than maxMaxAge. + cacheSizeBytes: maxCacheSize, + defaultTarget: "passthrough:///default", + childPolicyName: "grpclb", + childPolicyTargetField: "serviceName", + childPolicyConfig: map[string]json.RawMessage{ + "childPolicy": json.RawMessage(`[{"pickfirst": {}}]`), + "serviceName": json.RawMessage(childPolicyTargetFieldVal), + }, + }, + }, + { + desc: "maxAge not clamped when staleAge is set", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": ":///target", + "maxAge" : "500s", + "staleAge": "200s", + "cacheSizeBytes": 100000000 + }, + "childPolicy": [ + {"grpclb": {"childPolicy": [{"pickfirst": {}}]}} + ], + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantCfg: &lbConfig{ + lookupService: ":///target", + lookupServiceTimeout: 10 * time.Second, // This is the default value. + maxAge: 500 * time.Second, // Max age is not clamped when stale age is set. + staleAge: 200 * time.Second, // This is stale age within maxMaxAge. + cacheSizeBytes: maxCacheSize, + childPolicyName: "grpclb", + childPolicyTargetField: "serviceName", + childPolicyConfig: map[string]json.RawMessage{ + "childPolicy": json.RawMessage(`[{"pickfirst": {}}]`), + "serviceName": json.RawMessage(childPolicyTargetFieldVal), + }, + }, + }, + { + desc: "maxAge clamped when staleAge is not set", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": ":///target", + "maxAge" : "500s", + "cacheSizeBytes": 100000000 + }, + "childPolicy": [ + {"grpclb": {"childPolicy": [{"pickfirst": {}}]}} + ], + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantCfg: &lbConfig{ + lookupService: ":///target", + lookupServiceTimeout: 10 * time.Second, // This is the default value. + maxAge: 300 * time.Second, // Max age is clamped when stale age is not set. + staleAge: 300 * time.Second, + cacheSizeBytes: maxCacheSize, + childPolicyName: "grpclb", + childPolicyTargetField: "serviceName", + childPolicyConfig: map[string]json.RawMessage{ + "childPolicy": json.RawMessage(`[{"pickfirst": {}}]`), + "serviceName": json.RawMessage(childPolicyTargetFieldVal), + }, + }, + }, + { + desc: "without transformations", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "target", + "lookupServiceTimeout" : "100s", + "maxAge": "60s", + "staleAge" : "50s", + "cacheSizeBytes": 1000, + "defaultTarget": "passthrough:///default" + }, + "routeLookupChannelServiceConfig": {"loadBalancingConfig": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}]}, + "childPolicy": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}], + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantCfg: &lbConfig{ + lookupService: "target", + lookupServiceTimeout: 100 * time.Second, + maxAge: 60 * time.Second, + staleAge: 50 * time.Second, + cacheSizeBytes: 1000, + defaultTarget: "passthrough:///default", + controlChannelServiceConfig: `{"loadBalancingConfig": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}]}`, + childPolicyName: "grpclb", + childPolicyTargetField: "serviceName", + childPolicyConfig: map[string]json.RawMessage{ + "childPolicy": json.RawMessage(`[{"pickfirst": {}}]`), + "serviceName": json.RawMessage(childPolicyTargetFieldVal), + }, + }, + }, + } + + builder := rlsBB{} + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + lbCfg, err := builder.ParseConfig(test.input) + if err != nil || !testEqual(lbCfg.(*lbConfig), test.wantCfg) { + t.Errorf("ParseConfig(%s) = {%+v, %v}, want {%+v, nil}", string(test.input), lbCfg, err, test.wantCfg) + } + }) + } +} + +// TestParseConfigErrors verifies config parsing failure scenarios. +func (s) TestParseConfigErrors(t *testing.T) { + tests := []struct { + desc string + input []byte + wantErr string + }{ + { + desc: "empty input", + input: nil, + wantErr: "rls: json unmarshal failed for service config", + }, + { + desc: "bad json", + input: []byte(`bad bad json`), + wantErr: "rls: json unmarshal failed for service config", + }, + { + desc: "bad grpcKeyBuilder", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "requiredMatch": true, "names": ["v1"]}] + }] + } + }`), + wantErr: "rls: GrpcKeyBuilder in RouteLookupConfig has required_match field set", + }, + { + desc: "empty lookup service", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }] + } + }`), + wantErr: "rls: empty lookup_service in route lookup config", + }, + { + desc: "unregistered scheme in lookup service URI", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "badScheme:///target" + } + }`), + wantErr: "rls: unregistered scheme in lookup_service", + }, + { + desc: "invalid lookup service timeout", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "passthrough:///target", + "lookupServiceTimeout" : "315576000001s" + } + }`), + wantErr: "google.protobuf.Duration value out of range", + }, + { + desc: "invalid max age", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "passthrough:///target", + "lookupServiceTimeout" : "10s", + "maxAge" : "315576000001s" + } + }`), + wantErr: "google.protobuf.Duration value out of range", + }, + { + desc: "invalid stale age", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "passthrough:///target", + "lookupServiceTimeout" : "10s", + "maxAge" : "10s", + "staleAge" : "315576000001s" + } + }`), + wantErr: "google.protobuf.Duration value out of range", + }, + { + desc: "invalid max age stale age combo", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "passthrough:///target", + "lookupServiceTimeout" : "10s", + "staleAge" : "10s" + } + }`), + wantErr: "rls: stale_age is set, but max_age is not in route lookup config", + }, + { + desc: "cache_size_bytes field is not set", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "passthrough:///target", + "lookupServiceTimeout" : "10s", + "maxAge": "30s", + "staleAge" : "25s", + "defaultTarget": "passthrough:///default" + }, + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantErr: "rls: cache_size_bytes must be set to a non-zero value", + }, + { + desc: "routeLookupChannelServiceConfig is not in service config format", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "target", + "lookupServiceTimeout" : "100s", + "maxAge": "60s", + "staleAge" : "50s", + "cacheSizeBytes": 1000, + "defaultTarget": "passthrough:///default" + }, + "routeLookupChannelServiceConfig": "unknown", + "childPolicy": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}], + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantErr: "cannot unmarshal string into Go value of type grpc.jsonSC", + }, + { + desc: "routeLookupChannelServiceConfig contains unknown LB policy", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "target", + "lookupServiceTimeout" : "100s", + "maxAge": "60s", + "staleAge" : "50s", + "cacheSizeBytes": 1000, + "defaultTarget": "passthrough:///default" + }, + "routeLookupChannelServiceConfig": { + "loadBalancingConfig": [{"not_a_balancer1": {} }, {"not_a_balancer2": {}}] + }, + "childPolicy": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}], + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantErr: "no supported policies found in config", + }, + { + desc: "no child policy", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "passthrough:///target", + "lookupServiceTimeout" : "10s", + "maxAge": "30s", + "staleAge" : "25s", + "cacheSizeBytes": 1000, + "defaultTarget": "passthrough:///default" + }, + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantErr: "rls: invalid childPolicy config: no supported policies found", + }, + { + desc: "no known child policy", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "passthrough:///target", + "lookupServiceTimeout" : "10s", + "maxAge": "30s", + "staleAge" : "25s", + "cacheSizeBytes": 1000, + "defaultTarget": "passthrough:///default" + }, + "childPolicy": [ + {"cds_experimental": {"Cluster": "my-fav-cluster"}}, + {"unknown-policy": {"unknown-field": "unknown-value"}} + ], + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantErr: "rls: invalid childPolicy config: no supported policies found", + }, + { + desc: "invalid child policy config - more than one entry in map", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "passthrough:///target", + "lookupServiceTimeout" : "10s", + "maxAge": "30s", + "staleAge" : "25s", + "cacheSizeBytes": 1000, + "defaultTarget": "passthrough:///default" + }, + "childPolicy": [ + { + "cds_experimental": {"Cluster": "my-fav-cluster"}, + "unknown-policy": {"unknown-field": "unknown-value"} + } + ], + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantErr: "does not contain exactly 1 policy/config pair", + }, + { + desc: "no childPolicyConfigTargetFieldName", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "passthrough:///target", + "lookupServiceTimeout" : "10s", + "maxAge": "30s", + "staleAge" : "25s", + "cacheSizeBytes": 1000, + "defaultTarget": "passthrough:///default" + }, + "childPolicy": [ + {"cds_experimental": {"Cluster": "my-fav-cluster"}}, + {"unknown-policy": {"unknown-field": "unknown-value"}}, + {"grpclb": {}} + ] + }`), + wantErr: "rls: childPolicyConfigTargetFieldName field is not set in service config", + }, + { + desc: "child policy config validation failure", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "passthrough:///target", + "lookupServiceTimeout" : "10s", + "maxAge": "30s", + "staleAge" : "25s", + "cacheSizeBytes": 1000, + "defaultTarget": "passthrough:///default" + }, + "childPolicy": [ + {"cds_experimental": {"Cluster": "my-fav-cluster"}}, + {"unknown-policy": {"unknown-field": "unknown-value"}}, + {"grpclb": {"childPolicy": "not-an-array"}} + ], + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantErr: "rls: childPolicy config validation failed", + }, + } + + builder := rlsBB{} + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + lbCfg, err := builder.ParseConfig(test.input) + if lbCfg != nil || !strings.Contains(fmt.Sprint(err), test.wantErr) { + t.Errorf("ParseConfig(%s) = {%+v, %v}, want {nil, %s}", string(test.input), lbCfg, err, test.wantErr) + } + }) + } +} diff --git a/balancer/rls/control_channel.go b/balancer/rls/control_channel.go new file mode 100644 index 0000000..844cfc3 --- /dev/null +++ b/balancer/rls/control_channel.go @@ -0,0 +1,219 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "context" + "fmt" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/rls/internal/adaptive" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/pretty" + rlsgrpc "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" +) + +var newAdaptiveThrottler = func() adaptiveThrottler { return adaptive.New() } + +// newConnectivityStateSubscriber is a variable that can be overridden in tests +// to wrap the connectivity state subscriber for testing purposes. +var newConnectivityStateSubscriber = func(sub grpcsync.Subscriber) grpcsync.Subscriber { + return sub +} + +type adaptiveThrottler interface { + ShouldThrottle() bool + RegisterBackendResponse(throttled bool) +} + +// controlChannel is a wrapper around the gRPC channel to the RLS server +// specified in the service config. +type controlChannel struct { + // rpcTimeout specifies the timeout for the RouteLookup RPC call. The LB + // policy receives this value in its service config. + rpcTimeout time.Duration + // backToReadyFunc is a callback to be invoked when the connectivity state + // changes from READY --> TRANSIENT_FAILURE --> READY. + backToReadyFunc func() + // throttler in an adaptive throttling implementation used to avoid + // hammering the RLS service while it is overloaded or down. + throttler adaptiveThrottler + + cc *grpc.ClientConn + client rlsgrpc.RouteLookupServiceClient + logger *internalgrpclog.PrefixLogger + dropConnStateSubscriber func() + seenTransientFailure bool +} + +// newControlChannel creates a controlChannel to rlsServerName and uses +// serviceConfig, if non-empty, as the default service config for the underlying +// gRPC channel. +func newControlChannel(rlsServerName, serviceConfig string, rpcTimeout time.Duration, bOpts balancer.BuildOptions, backToReadyFunc func()) (*controlChannel, error) { + ctrlCh := &controlChannel{ + rpcTimeout: rpcTimeout, + backToReadyFunc: backToReadyFunc, + throttler: newAdaptiveThrottler(), + } + ctrlCh.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[rls-control-channel %p] ", ctrlCh)) + + dopts, err := ctrlCh.dialOpts(bOpts, serviceConfig) + if err != nil { + return nil, err + } + ctrlCh.cc, err = grpc.NewClient(rlsServerName, dopts...) + if err != nil { + return nil, err + } + // Subscribe to connectivity state before connecting to avoid missing initial + // updates, which are only delivered to active subscribers. + subscribe := internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func()) + ctrlCh.dropConnStateSubscriber = subscribe(ctrlCh.cc, newConnectivityStateSubscriber(ctrlCh)) + ctrlCh.cc.Connect() + ctrlCh.client = rlsgrpc.NewRouteLookupServiceClient(ctrlCh.cc) + ctrlCh.logger.Infof("Control channel created to RLS server at: %v", rlsServerName) + return ctrlCh, nil +} + +func (cc *controlChannel) OnMessage(msg any) { + st, ok := msg.(connectivity.State) + if !ok { + panic(fmt.Sprintf("Unexpected message type %T , wanted connectectivity.State type", msg)) + } + + switch st { + case connectivity.Ready: + // Only reset backoff when transitioning from TRANSIENT_FAILURE to READY. + // This indicates the RLS server has recovered from being unreachable, so + // we reset backoff state in all cache entries to allow pending RPCs to + // proceed immediately. We skip benign transitions like READY → IDLE → READY + // since those don't represent actual failures. + if cc.seenTransientFailure { + if cc.logger.V(2) { + cc.logger.Infof("Control channel back to READY after TRANSIENT_FAILURE") + } + cc.seenTransientFailure = false + if cc.backToReadyFunc != nil { + cc.backToReadyFunc() + } + } else { + if cc.logger.V(2) { + cc.logger.Infof("Control channel is READY") + } + } + case connectivity.TransientFailure: + // Track that we've entered TRANSIENT_FAILURE state so we know to reset + // backoffs when we recover to READY. + cc.logger.Warningf("Control channel is TRANSIENT_FAILURE") + cc.seenTransientFailure = true + default: + if cc.logger.V(2) { + cc.logger.Infof("Control channel connectivity state is %s", st) + } + } +} + +// dialOpts constructs the dial options for the control plane channel. +func (cc *controlChannel) dialOpts(bOpts balancer.BuildOptions, serviceConfig string) ([]grpc.DialOption, error) { + // The control plane channel will use the same authority as the parent + // channel for server authorization. This ensures that the identity of the + // RLS server and the identity of the backends is the same, so if the RLS + // config is injected by an attacker, it cannot cause leakage of private + // information contained in headers set by the application. + dopts := []grpc.DialOption{grpc.WithAuthority(bOpts.Authority)} + if bOpts.Dialer != nil { + dopts = append(dopts, grpc.WithContextDialer(bOpts.Dialer)) + } + // The control channel will use the channel credentials from the parent + // channel, including any call creds associated with the channel creds. + var credsOpt grpc.DialOption + switch { + case bOpts.DialCreds != nil: + credsOpt = grpc.WithTransportCredentials(bOpts.DialCreds.Clone()) + case bOpts.CredsBundle != nil: + // The "fallback" mode in google default credentials (which is the only + // type of credentials we expect to be used with RLS) uses TLS/ALTS + // creds for transport and uses the same call creds as that on the + // parent bundle. + bundle, err := bOpts.CredsBundle.NewWithMode(internal.CredsBundleModeFallback) + if err != nil { + return nil, err + } + credsOpt = grpc.WithCredentialsBundle(bundle) + default: + cc.logger.Warningf("no credentials available, using Insecure") + credsOpt = grpc.WithTransportCredentials(insecure.NewCredentials()) + } + dopts = append(dopts, credsOpt) + + // If the RLS LB policy's configuration specified a service config for the + // control channel, use that and disable service config fetching via the name + // resolver for the control channel. + if serviceConfig != "" { + cc.logger.Infof("Disabling service config from the name resolver and instead using: %s", serviceConfig) + dopts = append(dopts, grpc.WithDisableServiceConfig(), grpc.WithDefaultServiceConfig(serviceConfig)) + } + + return dopts, nil +} + +func (cc *controlChannel) close() { + cc.dropConnStateSubscriber() + cc.cc.Close() + cc.logger.Infof("Shutdown") +} + +type lookupCallback func(targets []string, headerData string, err error) + +// lookup starts a RouteLookup RPC in a separate goroutine and returns the +// results (and error, if any) in the provided callback. +// +// The returned boolean indicates whether the request was throttled by the +// client-side adaptive throttling algorithm in which case the provided callback +// will not be invoked. +func (cc *controlChannel) lookup(reqKeys map[string]string, reason rlspb.RouteLookupRequest_Reason, staleHeaders string, cb lookupCallback) (throttled bool) { + if cc.throttler.ShouldThrottle() { + cc.logger.Infof("RLS request throttled by client-side adaptive throttling") + return true + } + go func() { + req := &rlspb.RouteLookupRequest{ + TargetType: "grpc", + KeyMap: reqKeys, + Reason: reason, + StaleHeaderData: staleHeaders, + } + if cc.logger.V(2) { + cc.logger.Infof("Sending RLS request %+v", pretty.ToJSON(req)) + } + + ctx, cancel := context.WithTimeout(context.Background(), cc.rpcTimeout) + defer cancel() + resp, err := cc.client.RouteLookup(ctx, req) + cb(resp.GetTargets(), resp.GetHeaderData(), err) + }() + return false +} diff --git a/balancer/rls/control_channel_test.go b/balancer/rls/control_channel_test.go new file mode 100644 index 0000000..2ddd863 --- /dev/null +++ b/balancer/rls/control_channel_test.go @@ -0,0 +1,465 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "os" + "regexp" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + rlstest "google.golang.org/grpc/internal/testutils/rls" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" + "google.golang.org/protobuf/proto" +) + +// TestControlChannelThrottled tests the case where the adaptive throttler +// indicates that the control channel needs to be throttled. +func (s) TestControlChannelThrottled(t *testing.T) { + // Start an RLS server and set the throttler to always throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, alwaysThrottlingThrottler()) + + // Create a control channel to the fake RLS server. + ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, balancer.BuildOptions{}, nil) + if err != nil { + t.Fatalf("Failed to create control channel to RLS server: %v", err) + } + defer ctrlCh.close() + + // Perform the lookup and expect the attempt to be throttled. + ctrlCh.lookup(nil, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, nil) + + select { + case <-rlsReqCh: + t.Fatal("RouteLookup RPC invoked when control channel is throttled") + case <-time.After(defaultTestShortTimeout): + } +} + +// TestLookupFailure tests the case where the RLS server responds with an error. +func (s) TestLookupFailure(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Setup the RLS server to respond with errors. + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Err: errors.New("rls failure")} + }) + + // Create a control channel to the fake RLS server. + ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, balancer.BuildOptions{}, nil) + if err != nil { + t.Fatalf("Failed to create control channel to RLS server: %v", err) + } + defer ctrlCh.close() + + // Perform the lookup and expect the callback to be invoked with an error. + errCh := make(chan error, 1) + ctrlCh.lookup(nil, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, func(_ []string, _ string, err error) { + if err == nil { + errCh <- errors.New("rlsClient.lookup() succeeded, should have failed") + return + } + errCh <- nil + }) + + select { + case <-time.After(defaultTestTimeout): + t.Fatal("timeout when waiting for lookup callback to be invoked") + case err := <-errCh: + if err != nil { + t.Fatal(err) + } + } +} + +// TestLookupDeadlineExceeded tests the case where the RLS server does not +// respond within the configured rpc timeout. +func (s) TestLookupDeadlineExceeded(t *testing.T) { + // A unary interceptor which returns a status error with DeadlineExceeded. + interceptor := func(context.Context, any, *grpc.UnaryServerInfo, grpc.UnaryHandler) (resp any, err error) { + return nil, status.Error(codes.DeadlineExceeded, "deadline exceeded") + } + + // Start an RLS server and set the throttler to never throttle. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, grpc.UnaryInterceptor(interceptor)) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Create a control channel with a small deadline. + ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestShortTimeout, balancer.BuildOptions{}, nil) + if err != nil { + t.Fatalf("Failed to create control channel to RLS server: %v", err) + } + defer ctrlCh.close() + + // Perform the lookup and expect the callback to be invoked with an error. + errCh := make(chan error, 1) + ctrlCh.lookup(nil, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, func(_ []string, _ string, err error) { + if st, ok := status.FromError(err); !ok || st.Code() != codes.DeadlineExceeded { + errCh <- fmt.Errorf("rlsClient.lookup() returned error: %v, want %v", err, codes.DeadlineExceeded) + return + } + errCh <- nil + }) + + select { + case <-time.After(defaultTestTimeout): + t.Fatal("timeout when waiting for lookup callback to be invoked") + case err := <-errCh: + if err != nil { + t.Fatal(err) + } + } +} + +// testCredsBundle wraps a test call creds and real transport creds. +type testCredsBundle struct { + transportCreds credentials.TransportCredentials + callCreds credentials.PerRPCCredentials +} + +func (f *testCredsBundle) TransportCredentials() credentials.TransportCredentials { + return f.transportCreds +} + +func (f *testCredsBundle) PerRPCCredentials() credentials.PerRPCCredentials { + return f.callCreds +} + +func (f *testCredsBundle) NewWithMode(mode string) (credentials.Bundle, error) { + if mode != internal.CredsBundleModeFallback { + return nil, fmt.Errorf("unsupported mode: %v", mode) + } + return &testCredsBundle{ + transportCreds: f.transportCreds, + callCreds: f.callCreds, + }, nil +} + +var ( + // Call creds sent by the testPerRPCCredentials on the client, and verified + // by an interceptor on the server. + perRPCCredsData = map[string]string{ + "test-key": "test-value", + "test-key-bin": string([]byte{1, 2, 3}), + } +) + +type testPerRPCCredentials struct { + callCreds map[string]string +} + +func (f *testPerRPCCredentials) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { + return f.callCreds, nil +} + +func (f *testPerRPCCredentials) RequireTransportSecurity() bool { + return true +} + +// Unary server interceptor which validates if the RPC contains call credentials +// which match `perRPCCredsData +func callCredsValidatingServerInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Error(codes.PermissionDenied, "didn't find metadata in context") + } + for k, want := range perRPCCredsData { + got, ok := md[k] + if !ok { + return ctx, status.Errorf(codes.PermissionDenied, "didn't find call creds key %v in context", k) + } + if got[0] != want { + return ctx, status.Errorf(codes.PermissionDenied, "for key %v, got value %v, want %v", k, got, want) + } + } + return handler(ctx, req) +} + +// makeTLSCreds is a test helper which creates a TLS based transport credentials +// from files specified in the arguments. +func makeTLSCreds(t *testing.T, certPath, keyPath, rootsPath string) credentials.TransportCredentials { + cert, err := tls.LoadX509KeyPair(testdata.Path(certPath), testdata.Path(keyPath)) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(%q, %q) failed: %v", certPath, keyPath, err) + } + b, err := os.ReadFile(testdata.Path(rootsPath)) + if err != nil { + t.Fatalf("os.ReadFile(%q) failed: %v", rootsPath, err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(b) { + t.Fatal("failed to append certificates") + } + return credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: roots, + }) +} + +const ( + wantHeaderData = "headerData" + staleHeaderData = "staleHeaderData" +) + +var ( + keyMap = map[string]string{ + "k1": "v1", + "k2": "v2", + } + wantTargets = []string{"us_east_1.firestore.googleapis.com"} + lookupRequest = &rlspb.RouteLookupRequest{ + TargetType: "grpc", + KeyMap: keyMap, + Reason: rlspb.RouteLookupRequest_REASON_MISS, + StaleHeaderData: staleHeaderData, + } + lookupResponse = &rlstest.RouteLookupResponse{ + Resp: &rlspb.RouteLookupResponse{ + Targets: wantTargets, + HeaderData: wantHeaderData, + }, + } +) + +func testControlChannelCredsSuccess(t *testing.T, sopts []grpc.ServerOption, bopts balancer.BuildOptions) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, sopts...) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Setup the RLS server to respond with a valid response. + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return lookupResponse + }) + + // Verify that the request received by the RLS matches the expected one. + rlsServer.SetRequestCallback(func(got *rlspb.RouteLookupRequest) { + if diff := cmp.Diff(lookupRequest, got, cmp.Comparer(proto.Equal)); diff != "" { + t.Errorf("RouteLookupRequest diff (-want, +got):\n%s", diff) + } + }) + + // Create a control channel to the fake server. + ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, bopts, nil) + if err != nil { + t.Fatalf("Failed to create control channel to RLS server: %v", err) + } + defer ctrlCh.close() + + // Perform the lookup and expect a successful callback invocation. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + errCh := make(chan error, 1) + ctrlCh.lookup(keyMap, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, func(targets []string, headerData string, err error) { + if err != nil { + errCh <- fmt.Errorf("rlsClient.lookup() failed with err: %v", err) + return + } + if !cmp.Equal(targets, wantTargets) || headerData != wantHeaderData { + errCh <- fmt.Errorf("rlsClient.lookup() = (%v, %s), want (%v, %s)", targets, headerData, wantTargets, wantHeaderData) + return + } + errCh <- nil + }) + + select { + case <-ctx.Done(): + t.Fatal("timeout when waiting for lookup callback to be invoked") + case err := <-errCh: + if err != nil { + t.Fatal(err) + } + } +} + +// TestControlChannelCredsSuccess tests creation of the control channel with +// different credentials, which are expected to succeed. +func (s) TestControlChannelCredsSuccess(t *testing.T) { + serverCreds := makeTLSCreds(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + clientCreds := makeTLSCreds(t, "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem") + + tests := []struct { + name string + sopts []grpc.ServerOption + bopts balancer.BuildOptions + }{ + { + name: "insecure", + sopts: nil, + bopts: balancer.BuildOptions{}, + }, + { + name: "transport creds only", + sopts: []grpc.ServerOption{grpc.Creds(serverCreds)}, + bopts: balancer.BuildOptions{ + DialCreds: clientCreds, + Authority: "x.test.example.com", + }, + }, + { + name: "creds bundle", + sopts: []grpc.ServerOption{ + grpc.Creds(serverCreds), + grpc.UnaryInterceptor(callCredsValidatingServerInterceptor), + }, + bopts: balancer.BuildOptions{ + CredsBundle: &testCredsBundle{ + transportCreds: clientCreds, + callCreds: &testPerRPCCredentials{callCreds: perRPCCredsData}, + }, + Authority: "x.test.example.com", + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testControlChannelCredsSuccess(t, test.sopts, test.bopts) + }) + } +} + +func testControlChannelCredsFailure(t *testing.T, sopts []grpc.ServerOption, bopts balancer.BuildOptions, wantCode codes.Code, wantErrRegex *regexp.Regexp) { + // StartFakeRouteLookupServer a fake server. + // + // Start an RLS server and set the throttler to never throttle requests. The + // creds failures happen before the RPC handler on the server is invoked. + // So, there is need to setup the request and responses on the fake server. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, sopts...) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Create the control channel to the fake server. + ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, bopts, nil) + if err != nil { + t.Fatalf("Failed to create control channel to RLS server: %v", err) + } + defer ctrlCh.close() + + // Perform the lookup and expect the callback to be invoked with an error. + errCh := make(chan error, 1) + ctrlCh.lookup(nil, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, func(_ []string, _ string, err error) { + if st, ok := status.FromError(err); !ok || st.Code() != wantCode || !wantErrRegex.MatchString(st.String()) { + errCh <- fmt.Errorf("rlsClient.lookup() returned error: %v, wantCode: %v, wantErr: %s", err, wantCode, wantErrRegex.String()) + return + } + errCh <- nil + }) + + select { + case <-time.After(defaultTestTimeout): + t.Fatal("timeout when waiting for lookup callback to be invoked") + case err := <-errCh: + if err != nil { + t.Fatal(err) + } + } +} + +// TestControlChannelCredsFailure tests creation of the control channel with +// different credentials, which are expected to fail. +func (s) TestControlChannelCredsFailure(t *testing.T) { + serverCreds := makeTLSCreds(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + clientCreds := makeTLSCreds(t, "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem") + + tests := []struct { + name string + sopts []grpc.ServerOption + bopts balancer.BuildOptions + wantCode codes.Code + wantErrRegex *regexp.Regexp + }{ + { + name: "transport creds authority mismatch", + sopts: []grpc.ServerOption{grpc.Creds(serverCreds)}, + bopts: balancer.BuildOptions{ + DialCreds: clientCreds, + Authority: "authority-mismatch", + }, + wantCode: codes.Unavailable, + wantErrRegex: regexp.MustCompile(`transport: authentication handshake failed: .* \*\.test\.example\.com.*authority-mismatch`), + }, + { + name: "transport creds handshake failure", + sopts: nil, // server expects insecure connection + bopts: balancer.BuildOptions{ + DialCreds: clientCreds, + Authority: "x.test.example.com", + }, + wantCode: codes.Unavailable, + wantErrRegex: regexp.MustCompile("transport: authentication handshake failed: .*"), + }, + { + name: "call creds mismatch", + sopts: []grpc.ServerOption{ + grpc.Creds(serverCreds), + grpc.UnaryInterceptor(callCredsValidatingServerInterceptor), // server expects call creds + }, + bopts: balancer.BuildOptions{ + CredsBundle: &testCredsBundle{ + transportCreds: clientCreds, + callCreds: &testPerRPCCredentials{}, // sends no call creds + }, + Authority: "x.test.example.com", + }, + wantCode: codes.PermissionDenied, + wantErrRegex: regexp.MustCompile("didn't find call creds"), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testControlChannelCredsFailure(t, test.sopts, test.bopts, test.wantCode, test.wantErrRegex) + }) + } +} + +type unsupportedCredsBundle struct { + credentials.Bundle +} + +func (*unsupportedCredsBundle) NewWithMode(mode string) (credentials.Bundle, error) { + return nil, fmt.Errorf("unsupported mode: %v", mode) +} + +// TestNewControlChannelUnsupportedCredsBundle tests the case where the control +// channel is configured with a bundle which does not support the mode we use. +func (s) TestNewControlChannelUnsupportedCredsBundle(t *testing.T) { + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + + // Create the control channel to the fake server. + ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, balancer.BuildOptions{CredsBundle: &unsupportedCredsBundle{}}, nil) + if err == nil { + ctrlCh.close() + t.Fatal("newControlChannel succeeded when expected to fail") + } +} diff --git a/balancer/rls/helpers_test.go b/balancer/rls/helpers_test.go new file mode 100644 index 0000000..cc23ed8 --- /dev/null +++ b/balancer/rls/helpers_test.go @@ -0,0 +1,299 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "context" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer/rls/internal/test/e2e" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/stubserver" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 100 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// fakeBackoffStrategy is a fake implementation of the backoff.Strategy +// interface, for tests to inject the backoff duration. +type fakeBackoffStrategy struct { + backoff time.Duration +} + +func (f *fakeBackoffStrategy) Backoff(int) time.Duration { + return f.backoff +} + +// fakeThrottler is a fake implementation of the adaptiveThrottler interface. +type fakeThrottler struct { + throttleFunc func() bool // Fake throttler implementation. + throttleCh chan struct{} // Invocation of ShouldThrottle signals here. +} + +func (f *fakeThrottler) ShouldThrottle() bool { + select { + case <-f.throttleCh: + default: + } + f.throttleCh <- struct{}{} + + return f.throttleFunc() +} + +func (f *fakeThrottler) RegisterBackendResponse(bool) {} + +// alwaysThrottlingThrottler returns a fake throttler which always throttles. +func alwaysThrottlingThrottler() *fakeThrottler { + return &fakeThrottler{ + throttleFunc: func() bool { return true }, + throttleCh: make(chan struct{}, 1), + } +} + +// neverThrottlingThrottler returns a fake throttler which never throttles. +func neverThrottlingThrottler() *fakeThrottler { + return &fakeThrottler{ + throttleFunc: func() bool { return false }, + throttleCh: make(chan struct{}, 1), + } +} + +// oneTimeAllowingThrottler returns a fake throttler which does not throttle +// requests until the client RPC succeeds, but throttles everything that comes +// after. This is useful for tests which need to set up a valid cache entry +// before testing other cases. +func oneTimeAllowingThrottler(firstRPCDone *grpcsync.Event) *fakeThrottler { + return &fakeThrottler{ + throttleFunc: firstRPCDone.HasFired, + throttleCh: make(chan struct{}, 1), + } +} + +func overrideAdaptiveThrottler(t *testing.T, f *fakeThrottler) { + origAdaptiveThrottler := newAdaptiveThrottler + newAdaptiveThrottler = func() adaptiveThrottler { return f } + t.Cleanup(func() { newAdaptiveThrottler = origAdaptiveThrottler }) +} + +// buildBasicRLSConfig constructs a basic service config for the RLS LB policy +// with header matching rules. This expects the passed child policy name to +// have been registered by the caller. +func buildBasicRLSConfig(childPolicyName, rlsServerAddress string) *e2e.RLSConfig { + return &e2e.RLSConfig{ + RouteLookupConfig: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "grpc.testing.TestService"}}, + Headers: []*rlspb.NameMatcher{ + {Key: "k1", Names: []string{"n1"}}, + {Key: "k2", Names: []string{"n2"}}, + }, + }, + }, + LookupService: rlsServerAddress, + LookupServiceTimeout: durationpb.New(defaultTestTimeout), + CacheSizeBytes: 1024, + }, + RouteLookupChannelServiceConfig: `{"loadBalancingConfig": [{"pick_first": {}}]}`, + ChildPolicy: &internalserviceconfig.BalancerConfig{Name: childPolicyName}, + ChildPolicyConfigTargetFieldName: e2e.RLSChildPolicyTargetNameField, + } +} + +// buildBasicRLSConfigWithChildPolicy constructs a very basic service config for +// the RLS LB policy. It also registers a test LB policy which is capable of +// being a child of the RLS LB policy. +func buildBasicRLSConfigWithChildPolicy(t *testing.T, childPolicyName, rlsServerAddress string) *e2e.RLSConfig { + childPolicyName = "test-child-policy" + childPolicyName + e2e.RegisterRLSChildPolicy(childPolicyName, nil) + t.Logf("Registered child policy with name %q", childPolicyName) + + return &e2e.RLSConfig{ + RouteLookupConfig: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{{Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "grpc.testing.TestService"}}}}, + LookupService: rlsServerAddress, + LookupServiceTimeout: durationpb.New(defaultTestTimeout), + CacheSizeBytes: 1024, + }, + RouteLookupChannelServiceConfig: `{"loadBalancingConfig": [{"pick_first": {}}]}`, + ChildPolicy: &internalserviceconfig.BalancerConfig{Name: childPolicyName}, + ChildPolicyConfigTargetFieldName: e2e.RLSChildPolicyTargetNameField, + } +} + +// startBackend starts a backend implementing the TestService on a local port. +// It returns a channel for tests to get notified whenever an RPC is invoked on +// the backend. This allows tests to ensure that RPCs reach expected backends. +// Also returns the address of the backend. +func startBackend(t *testing.T, sopts ...grpc.ServerOption) (rpcCh chan struct{}, address string) { + t.Helper() + + rpcCh = make(chan struct{}, 1) + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + select { + case rpcCh <- struct{}{}: + default: + } + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(sopts...); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + return rpcCh, backend.Address +} + +// startManualResolverWithConfig registers and returns a manual resolver which +// pushes the RLS LB policy's service config on the channel. +func startManualResolverWithConfig(t *testing.T, rlsConfig *e2e.RLSConfig) *manual.Resolver { + t.Helper() + + scJSON, err := rlsConfig.ServiceConfigJSON() + if err != nil { + t.Fatal(err) + } + + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r := manual.NewBuilderWithScheme("rls-e2e") + r.InitialState(resolver.State{ServiceConfig: sc}) + t.Cleanup(r.Close) + return r +} + +// makeTestRPCAndExpectItToReachBackend is a test helper function which makes +// the EmptyCall RPC on the given ClientConn and verifies that it reaches a +// backend. The latter is accomplished by listening on the provided channel +// which gets pushed to whenever the backend in question gets an RPC. +// +// There are many instances where it can take a while before the attempted RPC +// reaches the expected backend. Examples include, but are not limited to: +// - control channel is changed in a config update. The RLS LB policy creates a +// new control channel, and sends a new picker to gRPC. But it takes a while +// before gRPC actually starts using the new picker. +// - test is waiting for a cache entry to expire after which we expect a +// different behavior because we have configured the fake RLS server to return +// different backends. +// +// Therefore, we do not return an error when the RPC fails. Instead, we wait for +// the context to expire before failing. +func makeTestRPCAndExpectItToReachBackend(ctx context.Context, t *testing.T, cc *grpc.ClientConn, ch chan struct{}) { + t.Helper() + + // Drain the backend channel before performing the RPC to remove any + // notifications from previous RPCs. + select { + case <-ch: + default: + } + + for { + if err := ctx.Err(); err != nil { + t.Fatalf("Timeout when waiting for RPCs to be routed to the given target: %v", err) + } + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + client := testgrpc.NewTestServiceClient(cc) + client.EmptyCall(sCtx, &testpb.Empty{}) + + select { + case <-sCtx.Done(): + case <-ch: + sCancel() + return + } + } +} + +// makeTestRPCAndVerifyError is a test helper function which makes the EmptyCall +// RPC on the given ClientConn and verifies that the RPC fails with the given +// status code and error. +// +// Similar to makeTestRPCAndExpectItToReachBackend, retries until expected +// outcome is reached or the provided context has expired. +func makeTestRPCAndVerifyError(ctx context.Context, t *testing.T, cc *grpc.ClientConn, wantCode codes.Code, wantErr error) { + t.Helper() + + for { + if err := ctx.Err(); err != nil { + t.Fatalf("Timeout when waiting for RPCs to fail with given error: %v", err) + } + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + client := testgrpc.NewTestServiceClient(cc) + _, err := client.EmptyCall(sCtx, &testpb.Empty{}) + + // If the RPC fails with the expected code and expected error message (if + // one was provided), we return. Else we retry after blocking for a little + // while to ensure that we don't keep blasting away with RPCs. + if code := status.Code(err); code == wantCode { + if wantErr == nil || strings.Contains(err.Error(), wantErr.Error()) { + sCancel() + return + } + } + <-sCtx.Done() + } +} + +// verifyRLSRequest is a test helper which listens on a channel to see if an RLS +// request was received by the fake RLS server. Based on whether the test +// expects a request to be sent out or not, it uses a different timeout. +func verifyRLSRequest(t *testing.T, ch chan struct{}, wantRequest bool) { + t.Helper() + + if wantRequest { + select { + case <-time.After(defaultTestTimeout): + t.Fatalf("Timeout when waiting for an RLS request to be sent out") + case <-ch: + } + } else { + select { + case <-time.After(defaultTestShortTimeout): + case <-ch: + t.Fatalf("RLS request sent out when not expecting one") + } + } +} diff --git a/balancer/rls/internal/adaptive/adaptive.go b/balancer/rls/internal/adaptive/adaptive.go new file mode 100644 index 0000000..6249948 --- /dev/null +++ b/balancer/rls/internal/adaptive/adaptive.go @@ -0,0 +1,131 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package adaptive provides functionality for adaptive client-side throttling. +package adaptive + +import ( + rand "math/rand/v2" + "sync" + "time" +) + +// For overriding in unittests. +var ( + timeNowFunc = time.Now + randFunc = rand.Float64 +) + +const ( + defaultDuration = 30 * time.Second + defaultBins = 100 + defaultRatioForAccepts = 2.0 + defaultRequestsPadding = 8.0 +) + +// Throttler implements a client-side throttling recommendation system. All +// methods are safe for concurrent use by multiple goroutines. +// +// The throttler has the following knobs for which we will use defaults for +// now. If there is a need to make them configurable at a later point in time, +// support for the same will be added. +// - Duration: amount of recent history that will be taken into account for +// making client-side throttling decisions. A default of 30 seconds is used. +// - Bins: number of bins to be used for bucketing historical data. A default +// of 100 is used. +// - RatioForAccepts: ratio by which accepts are multiplied, typically a value +// slightly larger than 1.0. This is used to make the throttler behave as if +// the backend had accepted more requests than it actually has, which lets us +// err on the side of sending to the backend more requests than we think it +// will accept for the sake of speeding up the propagation of state. A +// default of 2.0 is used. +// - RequestsPadding: is used to decrease the (client-side) throttling +// probability in the low QPS regime (to speed up propagation of state), as +// well as to safeguard against hitting a client-side throttling probability +// of 100%. The weight of this value decreases as the number of requests in +// recent history grows. A default of 8 is used. +// +// The adaptive throttler attempts to estimate the probability that a request +// will be throttled using recent history. Server requests (both throttled and +// accepted) are registered with the throttler (via the RegisterBackendResponse +// method), which then recommends client-side throttling (via the +// ShouldThrottle method) with probability given by: +// (requests - RatioForAccepts * accepts) / (requests + RequestsPadding) +type Throttler struct { + ratioForAccepts float64 + requestsPadding float64 + + // Number of total accepts and throttles in the lookback period. + mu sync.Mutex + accepts *lookback + throttles *lookback +} + +// New initializes a new adaptive throttler with the default values. +func New() *Throttler { + return newWithArgs(defaultDuration, defaultBins, defaultRatioForAccepts, defaultRequestsPadding) +} + +// newWithArgs initializes a new adaptive throttler with the provided values. +// Used only in unittests. +func newWithArgs(duration time.Duration, bins int64, ratioForAccepts, requestsPadding float64) *Throttler { + return &Throttler{ + ratioForAccepts: ratioForAccepts, + requestsPadding: requestsPadding, + accepts: newLookback(bins, duration), + throttles: newLookback(bins, duration), + } +} + +// ShouldThrottle returns a probabilistic estimate of whether the server would +// throttle the next request. This should be called for every request before +// allowing it to hit the network. If the returned value is true, the request +// should be aborted immediately (as if it had been throttled by the server). +func (t *Throttler) ShouldThrottle() bool { + randomProbability := randFunc() + now := timeNowFunc() + + t.mu.Lock() + defer t.mu.Unlock() + + accepts, throttles := float64(t.accepts.sum(now)), float64(t.throttles.sum(now)) + requests := accepts + throttles + throttleProbability := (requests - t.ratioForAccepts*accepts) / (requests + t.requestsPadding) + if throttleProbability <= randomProbability { + return false + } + + t.throttles.add(now, 1) + return true +} + +// RegisterBackendResponse registers a response received from the backend for a +// request allowed by ShouldThrottle. This should be called for every response +// received from the backend (i.e., once for each request for which +// ShouldThrottle returned false). +func (t *Throttler) RegisterBackendResponse(throttled bool) { + now := timeNowFunc() + + t.mu.Lock() + if throttled { + t.throttles.add(now, 1) + } else { + t.accepts.add(now, 1) + } + t.mu.Unlock() +} diff --git a/balancer/rls/internal/adaptive/adaptive_test.go b/balancer/rls/internal/adaptive/adaptive_test.go new file mode 100644 index 0000000..8b74a7c --- /dev/null +++ b/balancer/rls/internal/adaptive/adaptive_test.go @@ -0,0 +1,231 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package adaptive + +import ( + "sync" + "testing" + "time" +) + +// stats returns a tuple with accepts, throttles for the current time. +func (t *Throttler) stats() (int64, int64) { + now := timeNowFunc() + + t.mu.Lock() + a, th := t.accepts.sum(now), t.throttles.sum(now) + t.mu.Unlock() + return a, th +} + +// Enums for responses. +const ( + E = iota // No response + A // Accepted + T // Throttled +) + +func TestRegisterBackendResponse(t *testing.T) { + testcases := []struct { + desc string + bins int64 + ticks []int64 + responses []int64 + wantAccepts []int64 + wantThrottled []int64 + }{ + { + "Accumulate", + 3, + []int64{0, 1, 2}, // Ticks + []int64{A, T, E}, // Responses + []int64{1, 1, 1}, // Accepts + []int64{0, 1, 1}, // Throttled + }, + { + "LightTimeTravel", + 3, + []int64{1, 0, 2}, // Ticks + []int64{A, T, E}, // Response + []int64{1, 1, 1}, // Accepts + []int64{0, 1, 1}, // Throttled + }, + { + "HeavyTimeTravel", + 3, + []int64{8, 0, 9}, // Ticks + []int64{A, A, A}, // Response + []int64{1, 1, 2}, // Accepts + []int64{0, 0, 0}, // Throttled + }, + { + "Rollover", + 1, + []int64{0, 1, 2}, // Ticks + []int64{A, T, E}, // Responses + []int64{1, 0, 0}, // Accepts + []int64{0, 1, 0}, // Throttled + }, + } + + m := mockClock{} + oldTimeNowFunc := timeNowFunc + timeNowFunc = m.Now + defer func() { timeNowFunc = oldTimeNowFunc }() + + for _, test := range testcases { + t.Run(test.desc, func(t *testing.T) { + th := newWithArgs(time.Duration(test.bins), test.bins, 2.0, 8) + for i, tick := range test.ticks { + m.SetNanos(tick) + + if test.responses[i] != E { + th.RegisterBackendResponse(test.responses[i] == T) + } + + if gotAccepts, gotThrottled := th.stats(); gotAccepts != test.wantAccepts[i] || gotThrottled != test.wantThrottled[i] { + t.Errorf("th.stats() = {%d, %d} for index %d, want {%d, %d}", i, gotAccepts, gotThrottled, test.wantAccepts[i], test.wantThrottled[i]) + } + } + }) + } +} + +func TestShouldThrottleOptions(t *testing.T) { + // ShouldThrottle should return true iff + // (requests - RatioForAccepts * accepts) / (requests + RequestsPadding) <= p + // where p is a random number. For the purposes of this test it's fixed + // to 0.5. + responses := []int64{T, T, T, T, T, T, T, T, T, A, A, A, A, A, A, T, T, T, T} + + n := false + y := true + + testcases := []struct { + desc string + ratioForAccepts float64 + requestsPadding float64 + want []bool + }{ + { + "Baseline", + 1.1, + 8, + []bool{n, n, n, n, n, n, n, n, y, y, y, y, y, n, n, n, y, y, y}, + }, + { + "ChangePadding", + 1.1, + 7, + []bool{n, n, n, n, n, n, n, y, y, y, y, y, y, y, y, y, y, y, y}, + }, + { + "ChangeRatioForAccepts", + 1.4, + 8, + []bool{n, n, n, n, n, n, n, n, y, y, n, n, n, n, n, n, n, n, n}, + }, + } + + m := mockClock{} + oldTimeNowFunc := timeNowFunc + timeNowFunc = m.Now + oldRandFunc := randFunc + randFunc = func() float64 { return 0.5 } + defer func() { + timeNowFunc = oldTimeNowFunc + randFunc = oldRandFunc + }() + + for _, test := range testcases { + t.Run(test.desc, func(t *testing.T) { + m.SetNanos(0) + th := newWithArgs(time.Nanosecond, 1, test.ratioForAccepts, test.requestsPadding) + for i, response := range responses { + if response != E { + th.RegisterBackendResponse(response == T) + } + if got := th.ShouldThrottle(); got != test.want[i] { + t.Errorf("ShouldThrottle for index %d: got %v, want %v", i, got, test.want[i]) + } + } + }) + } +} + +func TestParallel(t *testing.T) { + // Uses all the defaults which comes with a 30 second duration. + th := New() + + testDuration := 2 * time.Second + numRoutines := 10 + accepts := make([]int64, numRoutines) + throttles := make([]int64, numRoutines) + var wg sync.WaitGroup + for i := 0; i < numRoutines; i++ { + wg.Add(1) + go func(num int) { + defer wg.Done() + + ticker := time.NewTicker(testDuration) + var accept int64 + var throttle int64 + for i := 0; ; i++ { + select { + case <-ticker.C: + ticker.Stop() + accepts[num] = accept + throttles[num] = throttle + return + default: + if i%2 == 0 { + th.RegisterBackendResponse(true) + throttle++ + } else { + th.RegisterBackendResponse(false) + accept++ + } + } + } + }(i) + } + wg.Wait() + + var wantAccepts, wantThrottles int64 + for i := 0; i < numRoutines; i++ { + wantAccepts += accepts[i] + wantThrottles += throttles[i] + } + + if gotAccepts, gotThrottles := th.stats(); gotAccepts != wantAccepts || gotThrottles != wantThrottles { + t.Errorf("th.stats() = {%d, %d}, want {%d, %d}", gotAccepts, gotThrottles, wantAccepts, wantThrottles) + } +} + +type mockClock struct { + t time.Time +} + +func (m *mockClock) Now() time.Time { + return m.t +} + +func (m *mockClock) SetNanos(n int64) { + m.t = time.Unix(0, n) +} diff --git a/balancer/rls/internal/adaptive/lookback.go b/balancer/rls/internal/adaptive/lookback.go new file mode 100644 index 0000000..1ab874c3 --- /dev/null +++ b/balancer/rls/internal/adaptive/lookback.go @@ -0,0 +1,84 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package adaptive + +import "time" + +// lookback implements a moving sum over an int64 timeline. +type lookback struct { + bins int64 // Number of bins to use for lookback. + width time.Duration // Width of each bin. + + head int64 // Absolute bin index (time * bins / duration) of the current head bin. + total int64 // Sum over all the values in buf, within the lookback window behind head. + buf []int64 // Ring buffer for keeping track of the sum elements. +} + +// newLookback creates a new lookback for the given duration with a set number +// of bins. +func newLookback(bins int64, duration time.Duration) *lookback { + return &lookback{ + bins: bins, + width: duration / time.Duration(bins), + buf: make([]int64, bins), + } +} + +// add is used to increment the lookback sum. +func (l *lookback) add(t time.Time, v int64) { + pos := l.advance(t) + + if (l.head - pos) >= l.bins { + // Do not increment counters if pos is more than bins behind head. + return + } + l.buf[pos%l.bins] += v + l.total += v +} + +// sum returns the sum of the lookback buffer at the given time or head, +// whichever is greater. +func (l *lookback) sum(t time.Time) int64 { + l.advance(t) + return l.total +} + +// advance prepares the lookback buffer for calls to add() or sum() at time t. +// If head is greater than t then the lookback buffer will be untouched. The +// absolute bin index corresponding to t is returned. It will always be less +// than or equal to head. +func (l *lookback) advance(t time.Time) int64 { + ch := l.head // Current head bin index. + nh := t.UnixNano() / l.width.Nanoseconds() // New head bin index. + + if nh <= ch { + // Either head unchanged or clock jitter (time has moved backwards). Do + // not advance. + return nh + } + + jmax := min(l.bins, nh-ch) + for j := int64(0); j < jmax; j++ { + i := (ch + j + 1) % l.bins + l.total -= l.buf[i] + l.buf[i] = 0 + } + l.head = nh + return nh +} diff --git a/balancer/rls/internal/adaptive/lookback_test.go b/balancer/rls/internal/adaptive/lookback_test.go new file mode 100644 index 0000000..3e441f5 --- /dev/null +++ b/balancer/rls/internal/adaptive/lookback_test.go @@ -0,0 +1,85 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package adaptive + +import ( + "testing" + "time" +) + +func TestLookback(t *testing.T) { + makeTicks := func(offsets []int64) []time.Time { + var ticks []time.Time + now := time.Now() + for _, offset := range offsets { + ticks = append(ticks, now.Add(time.Duration(offset))) + } + return ticks + } + + // lookback.add and lookback.sum behave correctly. + testcases := []struct { + desc string + bins int64 + ticks []time.Time + values []int64 + want []int64 + }{ + { + "Accumulate", + 3, + makeTicks([]int64{0, 1, 2}), // Ticks + []int64{1, 2, 3}, // Values + []int64{1, 3, 6}, // Want + }, + { + "LightTimeTravel", + 3, + makeTicks([]int64{1, 0, 2}), // Ticks + []int64{1, 2, 3}, // Values + []int64{1, 3, 6}, // Want + }, + { + "HeavyTimeTravel", + 3, + makeTicks([]int64{8, 0, 9}), // Ticks + []int64{1, 2, 3}, // Values + []int64{1, 1, 4}, // Want + }, + { + "Rollover", + 1, + makeTicks([]int64{0, 1, 2}), // Ticks + []int64{1, 2, 3}, // Values + []int64{1, 2, 3}, // Want + }, + } + + for _, test := range testcases { + t.Run(test.desc, func(t *testing.T) { + lb := newLookback(test.bins, time.Duration(test.bins)) + for i, tick := range test.ticks { + lb.add(tick, test.values[i]) + if got := lb.sum(tick); got != test.want[i] { + t.Errorf("sum for index %d got %d, want %d", i, got, test.want[i]) + } + } + }) + } +} diff --git a/balancer/rls/internal/keys/builder.go b/balancer/rls/internal/keys/builder.go new file mode 100644 index 0000000..7deb7dc --- /dev/null +++ b/balancer/rls/internal/keys/builder.go @@ -0,0 +1,267 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package keys provides functionality required to build RLS request keys. +package keys + +import ( + "errors" + "fmt" + "sort" + "strings" + + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/grpc/metadata" +) + +// BuilderMap maps from request path to the key builder for that path. +type BuilderMap map[string]builder + +// MakeBuilderMap parses the provided RouteLookupConfig proto and returns a map +// from paths to key builders. +func MakeBuilderMap(cfg *rlspb.RouteLookupConfig) (BuilderMap, error) { + kbs := cfg.GetGrpcKeybuilders() + if len(kbs) == 0 { + return nil, errors.New("rls: RouteLookupConfig does not contain any GrpcKeyBuilder") + } + + bm := make(map[string]builder) + for _, kb := range kbs { + // Extract keys from `headers`, `constant_keys` and `extra_keys` fields + // and populate appropriate values in the builder struct. Also ensure + // that keys are not repeated. + var matchers []matcher + seenKeys := make(map[string]bool) + constantKeys := kb.GetConstantKeys() + for k := range kb.GetConstantKeys() { + seenKeys[k] = true + } + for _, h := range kb.GetHeaders() { + if h.GetRequiredMatch() { + return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig has required_match field set {%+v}", kbs) + } + key := h.GetKey() + if seenKeys[key] { + return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key %q across headers, constant_keys and extra_keys {%+v}", key, kbs) + } + seenKeys[key] = true + matchers = append(matchers, matcher{key: h.GetKey(), names: h.GetNames()}) + } + if seenKeys[kb.GetExtraKeys().GetHost()] { + return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key %q in extra_keys from constant_keys or headers {%+v}", kb.GetExtraKeys().GetHost(), kbs) + } + if seenKeys[kb.GetExtraKeys().GetService()] { + return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key %q in extra_keys from constant_keys or headers {%+v}", kb.GetExtraKeys().GetService(), kbs) + } + if seenKeys[kb.GetExtraKeys().GetMethod()] { + return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key %q in extra_keys from constant_keys or headers {%+v}", kb.GetExtraKeys().GetMethod(), kbs) + } + b := builder{ + headerKeys: matchers, + constantKeys: constantKeys, + hostKey: kb.GetExtraKeys().GetHost(), + serviceKey: kb.GetExtraKeys().GetService(), + methodKey: kb.GetExtraKeys().GetMethod(), + } + + // Store the builder created above in the BuilderMap based on the value + // of the `Names` field, which wraps incoming request's service and + // method. Also, ensure that there are no repeated `Names` field. + names := kb.GetNames() + if len(names) == 0 { + return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig does not contain any Name {%+v}", kbs) + } + for _, name := range names { + if name.GetService() == "" { + return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains a Name field with no Service {%+v}", kbs) + } + if strings.Contains(name.GetMethod(), `/`) { + return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains a method with a slash {%+v}", kbs) + } + path := "/" + name.GetService() + "/" + name.GetMethod() + if _, ok := bm[path]; ok { + return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated Name field {%+v}", kbs) + } + bm[path] = b + } + } + return bm, nil +} + +// KeyMap represents the RLS keys to be used for a request. +type KeyMap struct { + // Map is the representation of an RLS key as a Go map. This is used when + // an actual RLS request is to be sent out on the wire, since the + // RouteLookupRequest proto expects a Go map. + Map map[string]string + // Str is the representation of an RLS key as a string, sorted by keys. + // Since the RLS keys are part of the cache key in the request cache + // maintained by the RLS balancer, and Go maps cannot be used as keys for + // Go maps (the cache is implemented as a map), we need a stringified + // version of it. + Str string +} + +// RLSKey builds the RLS keys to be used for the given request, identified by +// the request path and the request headers stored in metadata. +func (bm BuilderMap) RLSKey(md metadata.MD, host, path string) KeyMap { + // The path passed in is of the form "/service/method". The keyBuilderMap is + // indexed with keys of the form "/service/" or "/service/method". The service + // that we set in the keyMap (to be sent out in the RLS request) should not + // include any slashes though. + i := strings.LastIndex(path, "/") + service, method := path[:i+1], path[i+1:] + b, ok := bm[path] + if !ok { + b, ok = bm[service] + if !ok { + return KeyMap{} + } + } + + kvMap := b.buildHeaderKeys(md) + if b.hostKey != "" { + kvMap[b.hostKey] = host + } + if b.serviceKey != "" { + kvMap[b.serviceKey] = strings.Trim(service, "/") + } + if b.methodKey != "" { + kvMap[b.methodKey] = method + } + for k, v := range b.constantKeys { + kvMap[k] = v + } + return KeyMap{Map: kvMap, Str: mapToString(kvMap)} +} + +// Equal reports whether bm and am represent equivalent BuilderMaps. +func (bm BuilderMap) Equal(am BuilderMap) bool { + if (bm == nil) != (am == nil) { + return false + } + if len(bm) != len(am) { + return false + } + + for key, bBuilder := range bm { + aBuilder, ok := am[key] + if !ok { + return false + } + if !bBuilder.Equal(aBuilder) { + return false + } + } + return true +} + +// builder provides the actual functionality of building RLS keys. +type builder struct { + headerKeys []matcher + constantKeys map[string]string + // The following keys mirror corresponding fields in `extra_keys`. + hostKey string + serviceKey string + methodKey string +} + +// Equal reports whether b and a represent equivalent key builders. +func (b builder) Equal(a builder) bool { + if len(b.headerKeys) != len(a.headerKeys) { + return false + } + // Protobuf serialization maintains the order of repeated fields. Matchers + // are specified as a repeated field inside the KeyBuilder proto. If the + // order changes, it means that the order in the protobuf changed. We report + // this case as not being equal even though the builders could possibly be + // functionally equal. + for i, bMatcher := range b.headerKeys { + aMatcher := a.headerKeys[i] + if !bMatcher.Equal(aMatcher) { + return false + } + } + + if len(b.constantKeys) != len(a.constantKeys) { + return false + } + for k, v := range b.constantKeys { + if a.constantKeys[k] != v { + return false + } + } + + return b.hostKey == a.hostKey && b.serviceKey == a.serviceKey && b.methodKey == a.methodKey +} + +// matcher helps extract a key from request headers based on a given name. +type matcher struct { + // The key used in the keyMap sent as part of the RLS request. + key string + // List of header names which can supply the value for this key. + names []string +} + +// Equal reports if m and a are equivalent headerKeys. +func (m matcher) Equal(a matcher) bool { + if m.key != a.key { + return false + } + if len(m.names) != len(a.names) { + return false + } + for i := 0; i < len(m.names); i++ { + if m.names[i] != a.names[i] { + return false + } + } + return true +} + +func (b builder) buildHeaderKeys(md metadata.MD) map[string]string { + kvMap := make(map[string]string) + if len(md) == 0 { + return kvMap + } + for _, m := range b.headerKeys { + for _, name := range m.names { + if vals := md.Get(name); vals != nil { + kvMap[m.key] = strings.Join(vals, ",") + break + } + } + } + return kvMap +} + +func mapToString(kv map[string]string) string { + keys := make([]string, 0, len(kv)) + for k := range kv { + keys = append(keys, k) + } + sort.Strings(keys) + var sb strings.Builder + for i, k := range keys { + if i != 0 { + fmt.Fprint(&sb, ",") + } + fmt.Fprintf(&sb, "%s=%s", k, kv[k]) + } + return sb.String() +} diff --git a/balancer/rls/internal/keys/builder_test.go b/balancer/rls/internal/keys/builder_test.go new file mode 100644 index 0000000..90c132b --- /dev/null +++ b/balancer/rls/internal/keys/builder_test.go @@ -0,0 +1,650 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package keys + +import ( + "fmt" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/grpc/metadata" +) + +var ( + goodKeyBuilder1 = &rlspb.GrpcKeyBuilder{ + Names: []*rlspb.GrpcKeyBuilder_Name{ + {Service: "gFoo"}, + }, + Headers: []*rlspb.NameMatcher{ + {Key: "k1", Names: []string{"n1"}}, + {Key: "k2", Names: []string{"n1"}}, + }, + ExtraKeys: &rlspb.GrpcKeyBuilder_ExtraKeys{ + Host: "host", + Service: "service", + Method: "method", + }, + ConstantKeys: map[string]string{ + "const-key-1": "const-val-1", + "const-key-2": "const-val-2", + }, + } + goodKeyBuilder2 = &rlspb.GrpcKeyBuilder{ + Names: []*rlspb.GrpcKeyBuilder_Name{ + {Service: "gBar", Method: "method1"}, + {Service: "gFoobar"}, + }, + Headers: []*rlspb.NameMatcher{ + {Key: "k1", Names: []string{"n1", "n2"}}, + }, + } +) + +func TestMakeBuilderMap(t *testing.T) { + gFooBuilder := builder{ + headerKeys: []matcher{{key: "k1", names: []string{"n1"}}, {key: "k2", names: []string{"n1"}}}, + constantKeys: map[string]string{ + "const-key-1": "const-val-1", + "const-key-2": "const-val-2", + }, + hostKey: "host", + serviceKey: "service", + methodKey: "method", + } + wantBuilderMap1 := map[string]builder{"/gFoo/": gFooBuilder} + wantBuilderMap2 := map[string]builder{ + "/gFoo/": gFooBuilder, + "/gBar/method1": {headerKeys: []matcher{{key: "k1", names: []string{"n1", "n2"}}}}, + "/gFoobar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1", "n2"}}}}, + } + + tests := []struct { + desc string + cfg *rlspb.RouteLookupConfig + wantBuilderMap BuilderMap + }{ + { + desc: "One good GrpcKeyBuilder", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{goodKeyBuilder1}, + }, + wantBuilderMap: wantBuilderMap1, + }, + { + desc: "Two good GrpcKeyBuilders", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{goodKeyBuilder1, goodKeyBuilder2}, + }, + wantBuilderMap: wantBuilderMap2, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + builderMap, err := MakeBuilderMap(test.cfg) + if err != nil || !builderMap.Equal(test.wantBuilderMap) { + t.Errorf("MakeBuilderMap(%+v) returned {%v, %v}, want: {%v, nil}", test.cfg, builderMap, err, test.wantBuilderMap) + } + }) + } +} + +func TestMakeBuilderMapErrors(t *testing.T) { + tests := []struct { + desc string + cfg *rlspb.RouteLookupConfig + wantErrPrefix string + }{ + { + desc: "No GrpcKeyBuilder", + cfg: &rlspb.RouteLookupConfig{}, + wantErrPrefix: "rls: RouteLookupConfig does not contain any GrpcKeyBuilder", + }, + { + desc: "Two GrpcKeyBuilders with same Name", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{goodKeyBuilder1, goodKeyBuilder1}, + }, + wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated Name field", + }, + { + desc: "GrpcKeyBuilder with empty Service field", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + {Service: "bFoo", Method: "method1"}, + {Service: "bBar"}, + {Method: "method1"}, + }, + Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}}, + }, + goodKeyBuilder1, + }, + }, + wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains a Name field with no Service", + }, + { + desc: "GrpcKeyBuilder with no Name", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{{}, goodKeyBuilder1}, + }, + wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig does not contain any Name", + }, + { + desc: "GrpcKeyBuilder with requiredMatch field set", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "bFoo", Method: "method1"}}, + Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}, RequiredMatch: true}}, + }, + goodKeyBuilder1, + }, + }, + wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig has required_match field set", + }, + { + desc: "GrpcKeyBuilder two headers with same key", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + {Service: "gBar", Method: "method1"}, + {Service: "gFoobar"}, + }, + Headers: []*rlspb.NameMatcher{ + {Key: "k1", Names: []string{"n1", "n2"}}, + {Key: "k1", Names: []string{"n1", "n2"}}, + }, + }, + goodKeyBuilder1, + }, + }, + wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key \"k1\" across headers, constant_keys and extra_keys", + }, + { + desc: "GrpcKeyBuilder repeated keys across headers and constant_keys", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + {Service: "gBar", Method: "method1"}, + {Service: "gFoobar"}, + }, + Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}}, + ConstantKeys: map[string]string{"k1": "v1"}, + }, + goodKeyBuilder1, + }, + }, + wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key \"k1\" across headers, constant_keys and extra_keys", + }, + { + desc: "GrpcKeyBuilder repeated keys across headers and extra_keys", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + {Service: "gBar", Method: "method1"}, + {Service: "gFoobar"}, + }, + Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}}, + ExtraKeys: &rlspb.GrpcKeyBuilder_ExtraKeys{Method: "k1"}, + }, + goodKeyBuilder1, + }, + }, + wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key \"k1\" in extra_keys from constant_keys or headers", + }, + { + desc: "GrpcKeyBuilder repeated keys across constant_keys and extra_keys", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + {Service: "gBar", Method: "method1"}, + {Service: "gFoobar"}, + }, + Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}}, + ConstantKeys: map[string]string{"host": "v1"}, + ExtraKeys: &rlspb.GrpcKeyBuilder_ExtraKeys{Host: "host"}, + }, + goodKeyBuilder1, + }, + }, + wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key \"host\" in extra_keys from constant_keys or headers", + }, + { + desc: "GrpcKeyBuilder with slash in method name", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "gBar", Method: "method1/foo"}}, + Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}}, + }, + }, + }, + wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains a method with a slash", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + builderMap, err := MakeBuilderMap(test.cfg) + if builderMap != nil || !strings.HasPrefix(fmt.Sprint(err), test.wantErrPrefix) { + t.Errorf("MakeBuilderMap(%+v) returned {%v, %v}, want: {nil, %v}", test.cfg, builderMap, err, test.wantErrPrefix) + } + }) + } +} + +func TestRLSKey(t *testing.T) { + bm, err := MakeBuilderMap(&rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{goodKeyBuilder1, goodKeyBuilder2}, + }) + if err != nil { + t.Fatalf("MakeBuilderMap() failed: %v", err) + } + + tests := []struct { + desc string + path string + md metadata.MD + wantKM KeyMap + }{ + { + // No keyBuilder is found for the provided service. + desc: "service not found in key builder map", + path: "/notFoundService/method", + md: metadata.Pairs("n1", "v1", "n2", "v2"), + wantKM: KeyMap{}, + }, + { + // No keyBuilder is found for the provided method. + desc: "method not found in key builder map", + path: "/gBar/notFoundMethod", + md: metadata.Pairs("n1", "v1", "n2", "v2"), + wantKM: KeyMap{}, + }, + { + // A keyBuilder is found, but none of the headers match. + desc: "directPathMatch-NoMatchingKey", + path: "/gBar/method1", + md: metadata.Pairs("notMatchingKey", "v1"), + wantKM: KeyMap{Map: map[string]string{}, Str: ""}, + }, + { + // A keyBuilder is found, and a single headers matches. + desc: "directPathMatch-SingleKey", + path: "/gBar/method1", + md: metadata.Pairs("n1", "v1"), + wantKM: KeyMap{Map: map[string]string{"k1": "v1"}, Str: "k1=v1"}, + }, + { + // A keyBuilder is found, and multiple headers match, but the first + // match is chosen. + desc: "directPathMatch-FirstMatchingKey", + path: "/gBar/method1", + md: metadata.Pairs("n2", "v2", "n1", "v1"), + wantKM: KeyMap{Map: map[string]string{"k1": "v1"}, Str: "k1=v1"}, + }, + { + // A keyBuilder is found as a wildcard match, but none of the + // headers match. + desc: "wildcardPathMatch-NoMatchingKey", + path: "/gFoobar/method1", + md: metadata.Pairs("notMatchingKey", "v1"), + wantKM: KeyMap{Map: map[string]string{}, Str: ""}, + }, + { + // A keyBuilder is found as a wildcard match, and a single headers + // matches. + desc: "wildcardPathMatch-SingleKey", + path: "/gFoobar/method1", + md: metadata.Pairs("n1", "v1"), + wantKM: KeyMap{Map: map[string]string{"k1": "v1"}, Str: "k1=v1"}, + }, + { + // A keyBuilder is found as a wildcard match, and multiple headers + // match, but the first match is chosen. + desc: "wildcardPathMatch-FirstMatchingKey", + path: "/gFoobar/method1", + md: metadata.Pairs("n2", "v2", "n1", "v1"), + wantKM: KeyMap{Map: map[string]string{"k1": "v1"}, Str: "k1=v1"}, + }, + { + // Multiple headerKeys find hits in the provided request headers. + desc: "multipleMatchers", + path: "/gFoo/method1", + md: metadata.Pairs("n2", "v2", "n1", "v1"), + wantKM: KeyMap{ + Map: map[string]string{ + "const-key-1": "const-val-1", + "const-key-2": "const-val-2", + "host": "dummy-host", + "service": "gFoo", + "method": "method1", + "k1": "v1", + "k2": "v1", + }, + Str: "const-key-1=const-val-1,const-key-2=const-val-2,host=dummy-host,k1=v1,k2=v1,method=method1,service=gFoo", + }, + }, + { + // A match is found for a header which is specified multiple times. + // So, the values are joined with commas separating them. + desc: "commaSeparated", + path: "/gBar/method1", + md: metadata.Pairs("n1", "v1", "n1", "v2", "n1", "v3"), + wantKM: KeyMap{Map: map[string]string{"k1": "v1,v2,v3"}, Str: "k1=v1,v2,v3"}, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if gotKM := bm.RLSKey(test.md, "dummy-host", test.path); !cmp.Equal(gotKM, test.wantKM) { + t.Errorf("RLSKey(%+v, %s) = %+v, want %+v", test.md, test.path, gotKM, test.wantKM) + } + }) + } +} + +func TestMapToString(t *testing.T) { + tests := []struct { + desc string + input map[string]string + wantStr string + }{ + { + desc: "empty map", + input: nil, + wantStr: "", + }, + { + desc: "one key", + input: map[string]string{ + "k1": "v1", + }, + wantStr: "k1=v1", + }, + { + desc: "sorted keys", + input: map[string]string{ + "k1": "v1", + "k2": "v2", + "k3": "v3", + }, + wantStr: "k1=v1,k2=v2,k3=v3", + }, + { + desc: "unsorted keys", + input: map[string]string{ + "k3": "v3", + "k1": "v1", + "k2": "v2", + }, + wantStr: "k1=v1,k2=v2,k3=v3", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if gotStr := mapToString(test.input); gotStr != test.wantStr { + t.Errorf("mapToString(%v) = %s, want %s", test.input, gotStr, test.wantStr) + } + }) + } +} + +func TestBuilderMapEqual(t *testing.T) { + tests := []struct { + desc string + a BuilderMap + b BuilderMap + wantEqual bool + }{ + { + desc: "nil builder maps", + a: nil, + b: nil, + wantEqual: true, + }, + { + desc: "empty builder maps", + a: make(map[string]builder), + b: make(map[string]builder), + wantEqual: true, + }, + { + desc: "nil and non-nil builder maps", + a: nil, + b: map[string]builder{"/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}}, + wantEqual: false, + }, + { + desc: "empty and non-empty builder maps", + a: make(map[string]builder), + b: map[string]builder{"/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}}, + wantEqual: false, + }, + { + desc: "different number of map keys", + a: map[string]builder{ + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + }, + b: map[string]builder{ + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + }, + wantEqual: false, + }, + { + desc: "different map keys", + a: map[string]builder{ + "/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + }, + b: map[string]builder{ + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + }, + wantEqual: false, + }, + { + desc: "equal keys different values", + a: map[string]builder{ + "/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1", "n2"}}}}, + }, + b: map[string]builder{ + "/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + }, + wantEqual: false, + }, + { + desc: "good match", + a: map[string]builder{ + "/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + }, + b: map[string]builder{ + "/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + }, + wantEqual: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if gotEqual := test.a.Equal(test.b); gotEqual != test.wantEqual { + t.Errorf("BuilderMap.Equal(%v, %v) = %v, want %v", test.a, test.b, gotEqual, test.wantEqual) + } + }) + } +} + +func TestBuilderEqual(t *testing.T) { + tests := []struct { + desc string + a builder + b builder + wantEqual bool + }{ + { + desc: "nil builders", + a: builder{headerKeys: nil}, + b: builder{headerKeys: nil}, + wantEqual: true, + }, + { + desc: "empty builders", + a: builder{headerKeys: []matcher{}}, + b: builder{headerKeys: []matcher{}}, + wantEqual: true, + }, + { + desc: "empty and non-empty builders", + a: builder{headerKeys: []matcher{}}, + b: builder{headerKeys: []matcher{{key: "foo"}}}, + wantEqual: false, + }, + { + desc: "different number of headerKeys", + a: builder{headerKeys: []matcher{{key: "foo"}, {key: "bar"}}}, + b: builder{headerKeys: []matcher{{key: "foo"}}}, + wantEqual: false, + }, + { + desc: "equal number but differing headerKeys", + a: builder{headerKeys: []matcher{{key: "bar"}}}, + b: builder{headerKeys: []matcher{{key: "foo"}}}, + wantEqual: false, + }, + { + desc: "different number of constantKeys", + a: builder{constantKeys: map[string]string{"k1": "v1"}}, + b: builder{constantKeys: map[string]string{"k1": "v1", "k2": "v2"}}, + wantEqual: false, + }, + { + desc: "equal number but differing constantKeys", + a: builder{constantKeys: map[string]string{"k1": "v1"}}, + b: builder{constantKeys: map[string]string{"k2": "v2"}}, + wantEqual: false, + }, + { + desc: "different hostKey", + a: builder{hostKey: "host1"}, + b: builder{hostKey: "host2"}, + wantEqual: false, + }, + { + desc: "different serviceKey", + a: builder{hostKey: "service1"}, + b: builder{hostKey: "service2"}, + wantEqual: false, + }, + { + desc: "different methodKey", + a: builder{hostKey: "method1"}, + b: builder{hostKey: "method2"}, + wantEqual: false, + }, + { + desc: "equal", + a: builder{ + headerKeys: []matcher{{key: "foo"}}, + constantKeys: map[string]string{"k1": "v1"}, + hostKey: "host", + serviceKey: "/service/", + methodKey: "method", + }, + b: builder{ + headerKeys: []matcher{{key: "foo"}}, + constantKeys: map[string]string{"k1": "v1"}, + hostKey: "host", + serviceKey: "/service/", + methodKey: "method", + }, + wantEqual: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + t.Run(test.desc, func(t *testing.T) { + if gotEqual := test.a.Equal(test.b); gotEqual != test.wantEqual { + t.Errorf("builder.Equal(%v, %v) = %v, want %v", test.a, test.b, gotEqual, test.wantEqual) + } + }) + }) + } +} + +// matcher helps extract a key from request headers based on a given name. +func TestMatcherEqual(t *testing.T) { + tests := []struct { + desc string + a matcher + b matcher + wantEqual bool + }{ + { + desc: "different keys", + a: matcher{key: "foo"}, + b: matcher{key: "bar"}, + wantEqual: false, + }, + { + desc: "different number of names", + a: matcher{key: "foo", names: []string{"v1", "v2"}}, + b: matcher{key: "foo", names: []string{"v1"}}, + wantEqual: false, + }, + { + desc: "equal number but differing names", + a: matcher{key: "foo", names: []string{"v1", "v2"}}, + b: matcher{key: "foo", names: []string{"v1", "v22"}}, + wantEqual: false, + }, + { + desc: "same names in different order", + a: matcher{key: "foo", names: []string{"v2", "v1"}}, + b: matcher{key: "foo", names: []string{"v1", "v3"}}, + wantEqual: false, + }, + { + desc: "good match", + a: matcher{key: "foo", names: []string{"v1", "v2"}}, + b: matcher{key: "foo", names: []string{"v1", "v2"}}, + wantEqual: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if gotEqual := test.a.Equal(test.b); gotEqual != test.wantEqual { + t.Errorf("matcher.Equal(%v, %v) = %v, want %v", test.a, test.b, gotEqual, test.wantEqual) + } + }) + } +} diff --git a/balancer/rls/internal/test/e2e/e2e.go b/balancer/rls/internal/test/e2e/e2e.go new file mode 100644 index 0000000..7b8a8bb --- /dev/null +++ b/balancer/rls/internal/test/e2e/e2e.go @@ -0,0 +1,20 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package e2e contains utilities for end-to-end RouteLookupService tests. +package e2e diff --git a/balancer/rls/internal/test/e2e/rls_child_policy.go b/balancer/rls/internal/test/e2e/rls_child_policy.go new file mode 100644 index 0000000..2bae34b --- /dev/null +++ b/balancer/rls/internal/test/e2e/rls_child_policy.go @@ -0,0 +1,131 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +import ( + "encoding/json" + "errors" + "fmt" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const ( + // RLSChildPolicyTargetNameField is a top-level field name to add to the child + // policy's config, whose value is set to the target for the child policy. + RLSChildPolicyTargetNameField = "Backend" + // RLSChildPolicyBadTarget is a value which is considered a bad target by the + // child policy. This is useful to test bad child policy configuration. + RLSChildPolicyBadTarget = "bad-target" +) + +// ErrParseConfigBadTarget is the error returned from ParseConfig when the +// backend field is set to RLSChildPolicyBadTarget. +var ErrParseConfigBadTarget = errors.New("backend field set to RLSChildPolicyBadTarget") + +// BalancerFuncs is a set of callbacks which get invoked when the corresponding +// method on the child policy is invoked. +type BalancerFuncs struct { + UpdateClientConnState func(cfg *RLSChildPolicyConfig) error + Close func() +} + +// RegisterRLSChildPolicy registers a balancer builder with the given name, to +// be used as a child policy for the RLS LB policy. +// +// The child policy uses a pickfirst balancer under the hood to send all traffic +// to the single backend specified by the `RLSChildPolicyTargetNameField` field +// in its configuration which looks like: {"Backend": "Backend-address"}. +func RegisterRLSChildPolicy(name string, bf *BalancerFuncs) { + balancer.Register(bb{name: name, bf: bf}) +} + +type bb struct { + name string + bf *BalancerFuncs +} + +func (bb bb) Name() string { return bb.name } + +func (bb bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + pf := balancer.Get(pickfirst.Name) + b := &bal{ + Balancer: pf.Build(cc, opts), + bf: bb.bf, + done: grpcsync.NewEvent(), + } + go b.run() + return b +} + +func (bb bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + cfg := &RLSChildPolicyConfig{} + if err := json.Unmarshal(c, cfg); err != nil { + return nil, err + } + if cfg.Backend == RLSChildPolicyBadTarget { + return nil, ErrParseConfigBadTarget + } + return cfg, nil +} + +type bal struct { + balancer.Balancer + bf *BalancerFuncs + done *grpcsync.Event +} + +// RLSChildPolicyConfig is the LB config for the test child policy. +type RLSChildPolicyConfig struct { + serviceconfig.LoadBalancingConfig + Backend string // The target for which this child policy was created. + Random string // A random field to test child policy config changes. +} + +func (b *bal) UpdateClientConnState(c balancer.ClientConnState) error { + cfg, ok := c.BalancerConfig.(*RLSChildPolicyConfig) + if !ok { + return fmt.Errorf("received balancer config of type %T, want %T", c.BalancerConfig, &RLSChildPolicyConfig{}) + } + if b.bf != nil && b.bf.UpdateClientConnState != nil { + b.bf.UpdateClientConnState(cfg) + } + return b.Balancer.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{{Addr: cfg.Backend}}}, + }) +} + +func (b *bal) Close() { + b.Balancer.Close() + if b.bf != nil && b.bf.Close != nil { + b.bf.Close() + } + b.done.Fire() +} + +// run is a dummy goroutine to make sure that child policies are closed at the +// end of tests. If they are not closed, these goroutines will be picked up by +// the leak checker and tests will fail. +func (b *bal) run() { + <-b.done.Done() +} diff --git a/balancer/rls/internal/test/e2e/rls_lb_config.go b/balancer/rls/internal/test/e2e/rls_lb_config.go new file mode 100644 index 0000000..0a5993d --- /dev/null +++ b/balancer/rls/internal/test/e2e/rls_lb_config.go @@ -0,0 +1,103 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +import ( + "errors" + "fmt" + + "google.golang.org/grpc/balancer" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" + + "google.golang.org/protobuf/encoding/protojson" +) + +// RLSConfig is a utility type to build service config for the RLS LB policy. +type RLSConfig struct { + RouteLookupConfig *rlspb.RouteLookupConfig + RouteLookupChannelServiceConfig string + ChildPolicy *internalserviceconfig.BalancerConfig + ChildPolicyConfigTargetFieldName string +} + +// ServiceConfigJSON generates service config with a load balancing config +// corresponding to the RLS LB policy. +func (c *RLSConfig) ServiceConfigJSON() (string, error) { + m := protojson.MarshalOptions{ + Multiline: true, + Indent: " ", + UseProtoNames: true, + } + routeLookupCfg, err := m.Marshal(c.RouteLookupConfig) + if err != nil { + return "", err + } + childPolicy, err := c.ChildPolicy.MarshalJSON() + if err != nil { + return "", err + } + + return fmt.Sprintf(` +{ + "loadBalancingConfig": [ + { + "rls_experimental": { + "routeLookupConfig": %s, + "routeLookupChannelServiceConfig": %s, + "childPolicy": %s, + "childPolicyConfigTargetFieldName": %q + } + } + ] +}`, string(routeLookupCfg), c.RouteLookupChannelServiceConfig, string(childPolicy), c.ChildPolicyConfigTargetFieldName), nil +} + +// LoadBalancingConfig generates load balancing config which can used as part of +// a ClientConnState update to the RLS LB policy. +func (c *RLSConfig) LoadBalancingConfig() (serviceconfig.LoadBalancingConfig, error) { + m := protojson.MarshalOptions{ + Multiline: true, + Indent: " ", + UseProtoNames: true, + } + routeLookupCfg, err := m.Marshal(c.RouteLookupConfig) + if err != nil { + return nil, err + } + childPolicy, err := c.ChildPolicy.MarshalJSON() + if err != nil { + return nil, err + } + lbConfigJSON := fmt.Sprintf(` +{ + "routeLookupConfig": %s, + "routeLookupChannelServiceConfig": %s, + "childPolicy": %s, + "childPolicyConfigTargetFieldName": %q +}`, string(routeLookupCfg), c.RouteLookupChannelServiceConfig, string(childPolicy), c.ChildPolicyConfigTargetFieldName) + + builder := balancer.Get("rls_experimental") + if builder == nil { + return nil, errors.New("balancer builder not found for RLS LB policy") + } + parser := builder.(balancer.ConfigParser) + return parser.ParseConfig([]byte(lbConfigJSON)) +} diff --git a/balancer/rls/metrics_test.go b/balancer/rls/metrics_test.go new file mode 100644 index 0000000..1a5de6a --- /dev/null +++ b/balancer/rls/metrics_test.go @@ -0,0 +1,375 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rls + +import ( + "context" + "math/rand" + "testing" + + "github.com/google/uuid" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + estats "google.golang.org/grpc/experimental/stats" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/grpc/internal/stubserver" + rlstest "google.golang.org/grpc/internal/testutils/rls" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/stats/opentelemetry" +) + +func metricsDataFromReader(ctx context.Context, reader *metric.ManualReader) map[string]metricdata.Metrics { + rm := &metricdata.ResourceMetrics{} + reader.Collect(ctx, rm) + gotMetrics := map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + gotMetrics[m.Name] = m + } + } + return gotMetrics +} + +// TestRLSTargetPickMetric tests RLS Metrics in the case an RLS Balancer picks a +// target from an RLS Response for a RPC. This should emit a +// "grpc.lb.rls.target_picks" with certain labels and cache metrics with certain +// labels. +func (s) TestRLSTargetPickMetric(t *testing.T) { + // Overwrite the uuid random number generator to be deterministic. + uuid.SetRand(rand.New(rand.NewSource(1))) + defer uuid.SetRand(nil) + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + defer backend.Stop() + + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backend.Address}}} + }) + r := startManualResolverWithConfig(t, rlsConfig) + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + mo := opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics().Add("grpc.lb.rls.cache_entries", "grpc.lb.rls.cache_size", "grpc.lb.rls.default_target_picks", "grpc.lb.rls.target_picks", "grpc.lb.rls.failed_picks"), + OptionalLabels: []string{"grpc.client.call.custom"}, + } + grpcTarget := r.Scheme() + ":///" + cc, err := grpc.NewClient(grpcTarget, grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials()), opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo})) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + + wantMetrics := []metricdata.Metrics{ + { + Name: "grpc.lb.rls.target_picks", + Description: "EXPERIMENTAL. Number of LB picks sent to each RLS target. Note that if the default target is also returned by the RLS server, RPCs sent to that target from the cache will be counted in this metric, not in grpc.rls.default_target_picks.", + Unit: "{pick}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.data_plane_target", backend.Address), attribute.String("grpc.lb.pick_result", "complete"), attribute.String("grpc.client.call.custom", "target-pick-custom")), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + + // Receives an empty RLS Response, so a single cache entry with no size. + { + Name: "grpc.lb.rls.cache_entries", + Description: "EXPERIMENTAL. Number of entries in the RLS cache.", + Unit: "{entry}", + Data: metricdata.Gauge[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.instance_uuid", "52fdfc07-2182-454f-963f-5f0f9a621d72")), + Value: 1, + }, + }, + }, + }, + { + Name: "grpc.lb.rls.cache_size", + Description: "EXPERIMENTAL. The current size of the RLS cache.", + Unit: "By", + Data: metricdata.Gauge[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.instance_uuid", "52fdfc07-2182-454f-963f-5f0f9a621d72")), + Value: 35, + }, + }, + }, + }, + } + client := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = estats.NewContextWithCustomLabel(ctx, "target-pick-custom") + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if err != nil { + t.Fatalf("client.EmptyCall failed with error: %v", err) + } + + gotMetrics := metricsDataFromReader(ctx, reader) + for _, metric := range wantMetrics { + val, ok := gotMetrics[metric.Name] + if !ok { + t.Fatalf("Metric %v not present in recorded metrics", metric.Name) + } + if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) { + t.Fatalf("Metrics data type not equal for metric: %v", metric.Name) + } + } + + // Only one pick was made, which was a target pick, so no default target + // pick or failed pick metric should emit. + for _, metric := range []string{"grpc.lb.rls.default_target_picks", "grpc.lb.rls.failed_picks"} { + if _, ok := gotMetrics[metric]; ok { + t.Fatalf("Metric %v present in recorded metrics", metric) + } + } +} + +// TestRLSDefaultTargetPickMetric tests RLS Metrics in the case an RLS Balancer +// falls back to the default target for an RPC. This should emit a +// "grpc.lb.rls.default_target_picks" with certain labels and cache metrics with +// certain labels. +func (s) TestRLSDefaultTargetPickMetric(t *testing.T) { + // Overwrite the uuid random number generator to be deterministic. + uuid.SetRand(rand.New(rand.NewSource(1))) + defer uuid.SetRand(nil) + + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + // Build RLS service config with a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + defer backend.Stop() + rlsConfig.RouteLookupConfig.DefaultTarget = backend.Address + + r := startManualResolverWithConfig(t, rlsConfig) + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + mo := opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics().Add("grpc.lb.rls.cache_entries", "grpc.lb.rls.cache_size", "grpc.lb.rls.default_target_picks", "grpc.lb.rls.target_picks", "grpc.lb.rls.failed_picks"), + OptionalLabels: []string{"grpc.client.call.custom"}, + } + grpcTarget := r.Scheme() + ":///" + cc, err := grpc.NewClient(grpcTarget, grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials()), opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo})) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + + wantMetrics := []metricdata.Metrics{ + { + Name: "grpc.lb.rls.default_target_picks", + Description: "EXPERIMENTAL. Number of LB picks sent to the default target.", + Unit: "{pick}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.data_plane_target", backend.Address), attribute.String("grpc.lb.pick_result", "complete"), attribute.String("grpc.client.call.custom", "default-target-pick-custom")), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + // Receives a RLS Response with target information, so a single cache + // entry with a certain size. + { + Name: "grpc.lb.rls.cache_entries", + Description: "EXPERIMENTAL. Number of entries in the RLS cache.", + Unit: "{entry}", + Data: metricdata.Gauge[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.instance_uuid", "52fdfc07-2182-454f-963f-5f0f9a621d72")), + Value: 1, + }, + }, + }, + }, + { + Name: "grpc.lb.rls.cache_size", + Description: "EXPERIMENTAL. The current size of the RLS cache.", + Unit: "By", + Data: metricdata.Gauge[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.instance_uuid", "52fdfc07-2182-454f-963f-5f0f9a621d72")), + Value: 0, + }, + }, + }, + }, + } + client := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = estats.NewContextWithCustomLabel(ctx, "default-target-pick-custom") + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("client.EmptyCall failed with error: %v", err) + } + + gotMetrics := metricsDataFromReader(ctx, reader) + for _, metric := range wantMetrics { + val, ok := gotMetrics[metric.Name] + if !ok { + t.Fatalf("Metric %v not present in recorded metrics", metric.Name) + } + if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) { + t.Fatalf("Metrics data type not equal for metric: %v", metric.Name) + } + } + // No target picks and failed pick metrics should be emitted, as the test + // made only one RPC which recorded as a default target pick. + for _, metric := range []string{"grpc.lb.rls.target_picks", "grpc.lb.rls.failed_picks"} { + if _, ok := gotMetrics[metric]; ok { + t.Fatalf("Metric %v present in recorded metrics", metric) + } + } +} + +// TestRLSFailedRPCMetric tests RLS Metrics in the case an RLS Balancer fails an +// RPC due to an RLS failure. This should emit a +// "grpc.lb.rls.default_target_picks" with certain labels and cache metrics with +// certain labels. +func (s) TestRLSFailedRPCMetric(t *testing.T) { + // Overwrite the uuid random number generator to be deterministic. + uuid.SetRand(rand.New(rand.NewSource(1))) + defer uuid.SetRand(nil) + + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + // Build an RLS config without a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + mo := opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics().Add("grpc.lb.rls.cache_entries", "grpc.lb.rls.cache_size", "grpc.lb.rls.default_target_picks", "grpc.lb.rls.target_picks", "grpc.lb.rls.failed_picks"), + OptionalLabels: []string{"grpc.client.call.custom"}, + } + grpcTarget := r.Scheme() + ":///" + cc, err := grpc.NewClient(grpcTarget, grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials()), opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo})) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + + wantMetrics := []metricdata.Metrics{ + { + Name: "grpc.lb.rls.failed_picks", + Description: "EXPERIMENTAL. Number of LB picks failed due to either a failed RLS request or the RLS channel being throttled.", + Unit: "{pick}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.client.call.custom", "failed-pick-custom")), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + // Receives an empty RLS Response, so a single cache entry with no size. + { + Name: "grpc.lb.rls.cache_entries", + Description: "EXPERIMENTAL. Number of entries in the RLS cache.", + Unit: "{entry}", + Data: metricdata.Gauge[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.instance_uuid", "52fdfc07-2182-454f-963f-5f0f9a621d72")), + Value: 1, + }, + }, + }, + }, + { + Name: "grpc.lb.rls.cache_size", + Description: "EXPERIMENTAL. The current size of the RLS cache.", + Unit: "By", + Data: metricdata.Gauge[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget), attribute.String("grpc.lb.rls.server_target", rlsServer.Address), attribute.String("grpc.lb.rls.instance_uuid", "52fdfc07-2182-454f-963f-5f0f9a621d72")), + Value: 0, + }, + }, + }, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = estats.NewContextWithCustomLabel(ctx, "failed-pick-custom") + client := testgrpc.NewTestServiceClient(cc) + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Fatalf("client.EmptyCall error = %v, expected a non nil error", err) + } + + gotMetrics := metricsDataFromReader(ctx, reader) + for _, metric := range wantMetrics { + val, ok := gotMetrics[metric.Name] + if !ok { + t.Fatalf("Metric %v not present in recorded metrics", metric.Name) + } + if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) { + t.Fatalf("Metrics data type not equal for metric: %v", metric.Name) + } + } + // Only one RPC was made, which was a failed pick due to an RLS failure, so + // no metrics for target picks or default target picks should have emitted. + for _, metric := range []string{"grpc.lb.rls.target_picks", "grpc.lb.rls.default_target_picks"} { + if _, ok := gotMetrics[metric]; ok { + t.Fatalf("Metric %v present in recorded metrics", metric) + } + } +} diff --git a/balancer/rls/picker.go b/balancer/rls/picker.go new file mode 100644 index 0000000..855fcc3 --- /dev/null +++ b/balancer/rls/picker.go @@ -0,0 +1,401 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "errors" + "fmt" + "strings" + "sync/atomic" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/rls/internal/keys" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + estats "google.golang.org/grpc/experimental/stats" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +var ( + errRLSThrottled = errors.New("RLS call throttled at client side") + + // Function to compute data cache entry size. + computeDataCacheEntrySize = dcEntrySize +) + +// exitIdler wraps the only method on the BalancerGroup that the picker calls. +type exitIdler interface { + ExitIdleOne(id string) +} + +// rlsPicker selects the subConn to be used for a particular RPC. It does not +// manage subConns directly and delegates to pickers provided by child policies. +type rlsPicker struct { + // The keyBuilder map used to generate RLS keys for the RPC. This is built + // by the LB policy based on the received ServiceConfig. + kbm keys.BuilderMap + // Endpoint from the user's original dial target. Used to set the `host_key` + // field in `extra_keys`. + origEndpoint string + + lb *rlsBalancer + + // The picker is given its own copy of the below fields from the RLS LB policy + // to avoid having to grab the mutex on the latter. + rlsServerTarget string + grpcTarget string + metricsRecorder estats.MetricsRecorder + defaultPolicy *childPolicyWrapper // Child policy for the default target. + ctrlCh *controlChannel // Control channel to the RLS server. + maxAge time.Duration // Cache max age from LB config. + staleAge time.Duration // Cache stale age from LB config. + bg exitIdler + logger *internalgrpclog.PrefixLogger +} + +// isFullMethodNameValid return true if name is of the form `/service/method`. +func isFullMethodNameValid(name string) bool { + return strings.HasPrefix(name, "/") && strings.Count(name, "/") == 2 +} + +// Pick makes the routing decision for every outbound RPC. +func (p *rlsPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + if name := info.FullMethodName; !isFullMethodNameValid(name) { + return balancer.PickResult{}, fmt.Errorf("rls: method name %q is not of the form '/service/method", name) + } + + // Build the request's keys using the key builders from LB config. + md, _ := metadata.FromOutgoingContext(info.Ctx) + reqKeys := p.kbm.RLSKey(md, p.origEndpoint, info.FullMethodName) + + p.lb.cacheMu.Lock() + var pr balancer.PickResult + var err error + + // Record metrics without the cache mutex held, to prevent lock contention + // between concurrent RPC's and their Pick calls. Metrics Recording can + // potentially be expensive. + metricsCallback := func() {} + defer func() { + p.lb.cacheMu.Unlock() + metricsCallback() + }() + + // Lookup data cache and pending request map using request path and keys. + cacheKey := cacheKey{path: info.FullMethodName, keys: reqKeys.Str} + dcEntry := p.lb.dataCache.getEntry(cacheKey) + pendingEntry := p.lb.pendingMap[cacheKey] + now := time.Now() + + switch { + // No data cache entry. No pending request. + case dcEntry == nil && pendingEntry == nil: + throttled := p.sendRouteLookupRequestLocked(cacheKey, &backoffState{bs: defaultBackoffStrategy}, reqKeys.Map, rlspb.RouteLookupRequest_REASON_MISS, "") + if throttled { + pr, metricsCallback, err = p.useDefaultPickIfPossible(info, errRLSThrottled) + return pr, err + } + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + + // No data cache entry. Pending request exits. + case dcEntry == nil && pendingEntry != nil: + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + + // Data cache hit. No pending request. + case dcEntry != nil && pendingEntry == nil: + if dcEntry.expiryTime.After(now) { + if !dcEntry.staleTime.IsZero() && dcEntry.staleTime.Before(now) && dcEntry.backoffTime.Before(now) { + p.sendRouteLookupRequestLocked(cacheKey, dcEntry.backoffState, reqKeys.Map, rlspb.RouteLookupRequest_REASON_STALE, dcEntry.headerData) + } + // Delegate to child policies. + pr, metricsCallback, err = p.delegateToChildPoliciesLocked(dcEntry, info) + return pr, err + } + + // We get here only if the data cache entry has expired. If entry is in + // backoff, delegate to default target or fail the pick. + if dcEntry.backoffState != nil && dcEntry.backoffTime.After(now) { + // Avoid propagating the status code received on control plane RPCs to the + // data plane which can lead to unexpected outcomes as we do not control + // the status code sent by the control plane. Propagating the status + // message received from the control plane is still fine, as it could be + // useful for debugging purposes. + st := dcEntry.status + pr, metricsCallback, err = p.useDefaultPickIfPossible(info, status.Error(codes.Unavailable, fmt.Sprintf("most recent error from RLS server: %v", st.Error()))) + return pr, err + } + + // We get here only if the entry has expired and is not in backoff. + throttled := p.sendRouteLookupRequestLocked(cacheKey, dcEntry.backoffState, reqKeys.Map, rlspb.RouteLookupRequest_REASON_MISS, "") + if throttled { + pr, metricsCallback, err = p.useDefaultPickIfPossible(info, errRLSThrottled) + return pr, err + } + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + + // Data cache hit. Pending request exists. + default: + if dcEntry.expiryTime.After(now) { + pr, metricsCallback, err = p.delegateToChildPoliciesLocked(dcEntry, info) + return pr, err + } + // Data cache entry has expired and pending request exists. Queue pick. + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + } +} + +// errToPickResult is a helper function which converts the error value returned +// by Pick() to a string that represents the pick result. +func errToPickResult(err error) string { + if err == nil { + return "complete" + } + if errors.Is(err, balancer.ErrNoSubConnAvailable) { + return "queue" + } + if _, ok := status.FromError(err); ok { + return "drop" + } + return "fail" +} + +// delegateToChildPoliciesLocked is a helper function which iterates through the +// list of child policy wrappers in a cache entry and attempts to find a child +// policy to which this RPC can be routed to. If all child policies are in +// TRANSIENT_FAILURE, we delegate to the last child policy arbitrarily. Returns +// a function to be invoked to record metrics. +func (p *rlsPicker) delegateToChildPoliciesLocked(dcEntry *cacheEntry, info balancer.PickInfo) (balancer.PickResult, func(), error) { + const rlsDataHeaderName = "x-google-rls-data" + for i, cpw := range dcEntry.childPolicyWrappers { + state := (*balancer.State)(atomic.LoadPointer(&cpw.state)) + // Delegate to the child policy if it is not in TRANSIENT_FAILURE, or if + // it is the last one (which handles the case of delegating to the last + // child picker if all child policies are in TRANSIENT_FAILURE). + if state.ConnectivityState != connectivity.TransientFailure || i == len(dcEntry.childPolicyWrappers)-1 { + // Any header data received from the RLS server is stored in the + // cache entry and needs to be sent to the actual backend in the + // X-Google-RLS-Data header. + res, err := state.Picker.Pick(info) + if err != nil { + pr := errToPickResult(err) + customLabel := estats.CustomLabelFromContext(info.Ctx) + return res, func() { + if pr == "queue" { + // Don't record metrics for queued Picks. + return + } + targetPicksMetric.Record(p.metricsRecorder, 1, p.grpcTarget, p.rlsServerTarget, cpw.target, pr, customLabel) + }, err + } + + if res.Metadata == nil { + res.Metadata = metadata.Pairs(rlsDataHeaderName, dcEntry.headerData) + } else { + res.Metadata.Append(rlsDataHeaderName, dcEntry.headerData) + } + return res, func() { + customLabel := estats.CustomLabelFromContext(info.Ctx) + targetPicksMetric.Record(p.metricsRecorder, 1, p.grpcTarget, p.rlsServerTarget, cpw.target, "complete", customLabel) + }, nil + } + } + + // In the unlikely event that we have a cache entry with no targets, we end up + // queueing the RPC. + return balancer.PickResult{}, func() {}, balancer.ErrNoSubConnAvailable +} + +// useDefaultPickIfPossible is a helper method which delegates to the default +// target if one is configured, or fails the pick with the given error. Returns +// a function to be invoked to record metrics. +func (p *rlsPicker) useDefaultPickIfPossible(info balancer.PickInfo, errOnNoDefault error) (balancer.PickResult, func(), error) { + customLabel := estats.CustomLabelFromContext(info.Ctx) + + if p.defaultPolicy != nil { + state := (*balancer.State)(atomic.LoadPointer(&p.defaultPolicy.state)) + res, err := state.Picker.Pick(info) + pr := errToPickResult(err) + return res, func() { + if pr == "queue" { + // Don't record metrics for queued Picks. + return + } + defaultTargetPicksMetric.Record(p.metricsRecorder, 1, p.grpcTarget, p.rlsServerTarget, p.defaultPolicy.target, pr, customLabel) + }, err + } + + return balancer.PickResult{}, func() { + failedPicksMetric.Record(p.metricsRecorder, 1, p.grpcTarget, p.rlsServerTarget, customLabel) + }, errOnNoDefault +} + +// sendRouteLookupRequestLocked adds an entry to the pending request map and +// sends out an RLS request using the passed in arguments. Returns a value +// indicating if the request was throttled by the client-side adaptive +// throttler. +func (p *rlsPicker) sendRouteLookupRequestLocked(cacheKey cacheKey, bs *backoffState, reqKeys map[string]string, reason rlspb.RouteLookupRequest_Reason, staleHeaders string) bool { + if p.lb.pendingMap[cacheKey] != nil { + return false + } + + p.lb.pendingMap[cacheKey] = bs + throttled := p.ctrlCh.lookup(reqKeys, reason, staleHeaders, func(targets []string, headerData string, err error) { + p.handleRouteLookupResponse(cacheKey, targets, headerData, err) + }) + if throttled { + delete(p.lb.pendingMap, cacheKey) + } + return throttled +} + +// handleRouteLookupResponse is the callback invoked by the control channel upon +// receipt of an RLS response. Modifies the data cache and pending requests map +// and sends a new picker. +// +// Acquires the write-lock on the cache. Caller must not hold p.lb.cacheMu. +func (p *rlsPicker) handleRouteLookupResponse(cacheKey cacheKey, targets []string, headerData string, err error) { + p.logger.Infof("Received RLS response for key %+v with targets %+v, headerData %q, err: %v", cacheKey, targets, headerData, err) + + p.lb.cacheMu.Lock() + defer func() { + // Pending request map entry is unconditionally deleted since the request is + // no longer pending. + p.logger.Infof("Removing pending request entry for key %+v", cacheKey) + delete(p.lb.pendingMap, cacheKey) + p.lb.sendNewPicker() + p.lb.cacheMu.Unlock() + }() + + // Lookup the data cache entry or create a new one. + dcEntry := p.lb.dataCache.getEntry(cacheKey) + if dcEntry == nil { + dcEntry = &cacheEntry{} + if _, ok := p.lb.dataCache.addEntry(cacheKey, dcEntry); !ok { + // This is a very unlikely case where we are unable to add a + // data cache entry. Log and leave. + p.logger.Warningf("Failed to add data cache entry for %+v", cacheKey) + return + } + } + + // For failed requests, the data cache entry is modified as follows: + // - status is set to error returned from the control channel + // - current backoff state is available in the pending entry + // - `retries` field is incremented and + // - backoff state is moved to the data cache + // - backoffTime is set to the time indicated by the backoff state + // - backoffExpirationTime is set to twice the backoff time + // - backoffTimer is set to fire after backoffTime + // + // When a proactive cache refresh fails, this would leave the targets and the + // expiry time from the old entry unchanged. And this mean that the old valid + // entry would be used until expiration, and a new picker would be sent upon + // backoff expiry. + now := time.Now() + + // "An RLS request is considered to have failed if it returns a non-OK + // status or the RLS response's targets list is non-empty." - RLS LB Policy + // design. + if len(targets) == 0 && err == nil { + err = fmt.Errorf("RLS response's target list does not contain any entries for key %+v", cacheKey) + // If err is set, rpc error from the control plane and no control plane + // configuration is why no targets were passed into this helper, no need + // to specify and tell the user this information. + } + if err != nil { + dcEntry.status = err + pendingEntry := p.lb.pendingMap[cacheKey] + pendingEntry.retries++ + backoffTime := pendingEntry.bs.Backoff(pendingEntry.retries) + dcEntry.backoffState = pendingEntry + dcEntry.backoffTime = now.Add(backoffTime) + dcEntry.backoffExpiryTime = now.Add(2 * backoffTime) + if dcEntry.backoffState.timer != nil { + dcEntry.backoffState.timer.Stop() + } + dcEntry.backoffState.timer = time.AfterFunc(backoffTime, p.lb.sendNewPicker) + return + } + + // For successful requests, the cache entry is modified as follows: + // - childPolicyWrappers is set to point to the child policy wrappers + // associated with the targets specified in the received response + // - headerData is set to the value received in the response + // - expiryTime, stateTime and earliestEvictionTime are set + // - status is set to nil (OK status) + // - backoff state is cleared + p.setChildPolicyWrappersInCacheEntry(dcEntry, targets) + dcEntry.headerData = headerData + dcEntry.expiryTime = now.Add(p.maxAge) + if p.staleAge != 0 { + dcEntry.staleTime = now.Add(p.staleAge) + } + dcEntry.earliestEvictTime = now.Add(minEvictDuration) + dcEntry.status = nil + dcEntry.backoffState = &backoffState{bs: defaultBackoffStrategy} + dcEntry.backoffTime = time.Time{} + dcEntry.backoffExpiryTime = time.Time{} + p.lb.dataCache.updateEntrySize(dcEntry, computeDataCacheEntrySize(cacheKey, dcEntry)) +} + +// setChildPolicyWrappersInCacheEntry sets up the childPolicyWrappers field in +// the cache entry to point to the child policy wrappers for the targets +// specified in the RLS response. +// +// Caller must hold a write-lock on p.lb.cacheMu. +func (p *rlsPicker) setChildPolicyWrappersInCacheEntry(dcEntry *cacheEntry, newTargets []string) { + // If the childPolicyWrappers field is already pointing to the right targets, + // then the field's value does not need to change. + targetsChanged := true + func() { + if cpws := dcEntry.childPolicyWrappers; cpws != nil { + if len(newTargets) != len(cpws) { + return + } + for i, target := range newTargets { + if cpws[i].target != target { + return + } + } + targetsChanged = false + } + }() + if !targetsChanged { + return + } + + // If the childPolicyWrappers field is not already set to the right targets, + // then it must be reset. We construct a new list of child policies and + // then swap out the old list for the new one. + newChildPolicies := p.lb.acquireChildPolicyReferences(newTargets) + oldChildPolicyTargets := make([]string, len(dcEntry.childPolicyWrappers)) + for i, cpw := range dcEntry.childPolicyWrappers { + oldChildPolicyTargets[i] = cpw.target + } + p.lb.releaseChildPolicyReferences(oldChildPolicyTargets) + dcEntry.childPolicyWrappers = newChildPolicies +} + +func dcEntrySize(key cacheKey, entry *cacheEntry) int64 { + return int64(len(key.path) + len(key.keys) + len(entry.headerData)) +} diff --git a/balancer/rls/picker_test.go b/balancer/rls/picker_test.go new file mode 100644 index 0000000..a0bdbc8 --- /dev/null +++ b/balancer/rls/picker_test.go @@ -0,0 +1,1049 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/stubserver" + rlstest "google.golang.org/grpc/internal/testutils/rls" + "google.golang.org/grpc/internal/testutils/stats" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" + + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestNoNonEmptyTargetsReturnsError tests the case where the RLS Server returns +// a response with no non empty targets. This should be treated as an Control +// Plane RPC failure, and thus fail Data Plane RPC's with an error with the +// appropriate information specifying data plane sent a response with no non +// empty targets. +func (s) TestNoNonEmptyTargetsReturnsError(t *testing.T) { + // Setup RLS Server to return a response with an empty target string. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{}} + }) + + // Register a manual resolver and push the RLS service config through it. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + r := startManualResolverWithConfig(t, rlsConfig) + + // Create new client. + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and expect it to fail with an error specifying RLS response's + // target list does not contain any non empty entries. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndVerifyError(ctx, t, cc, codes.Unavailable, errors.New("RLS response's target list does not contain any entries for key")) + + // Make sure an RLS request is sent out. Even though the RLS Server will + // return no targets, the request should still hit the server. + verifyRLSRequest(t, rlsReqCh, true) +} + +// Test verifies the scenario where there is no matching entry in the data cache +// and no pending request either, and the ensuing RLS request is throttled. +func (s) TestPick_DataCacheMiss_NoPendingEntry_ThrottledWithDefaultTarget(t *testing.T) { + // Start an RLS server and set the throttler to always throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, alwaysThrottlingThrottler()) + + // Build RLS service config with a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + defBackendCh, defBackendAddress := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the default target. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, defBackendCh) + + // Make sure no RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, false) +} + +// Test verifies the scenario where there is no matching entry in the data cache +// and no pending request either, and the ensuing RLS request is throttled. +// There is no default target configured in the service config, so the RPC is +// expected to fail with an RLS throttled error. +func (s) TestPick_DataCacheMiss_NoPendingEntry_ThrottledWithoutDefaultTarget(t *testing.T) { + // Start an RLS server and set the throttler to always throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, alwaysThrottlingThrottler()) + + // Build an RLS config without a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Create new client. + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and expect it to fail with RLS throttled error. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndVerifyError(ctx, t, cc, codes.Unavailable, errRLSThrottled) + + // Make sure no RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, false) +} + +// Test verifies the scenario where there is no matching entry in the data cache +// and no pending request either, and the ensuing RLS request is not throttled. +// The RLS response does not contain any backends, so the RPC fails with a +// unavailable error. +func (s) TestPick_DataCacheMiss_NoPendingEntry_NotThrottled(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build an RLS config without a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Create new client. + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and expect it to fail with deadline exceeded error. We use a + // smaller timeout to ensure that the test doesn't run very long. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + makeTestRPCAndVerifyError(ctx, t, cc, codes.Unavailable, errors.New("RLS response's target list does not contain any entries for key")) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) +} + +// Test verifies the scenario where there is no matching entry in the data +// cache, but there is a pending request. So, we expect no RLS request to be +// sent out. The pick should be queued and not delegated to the default target. +func (s) TestPick_DataCacheMiss_PendingEntryExists(t *testing.T) { + tests := []struct { + name string + withDefaultTarget bool + }{ + { + name: "withDefaultTarget", + withDefaultTarget: true, + }, + { + name: "withoutDefaultTarget", + withDefaultTarget: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // A unary interceptor which blocks the RouteLookup RPC on the fake + // RLS server until the test is done. The first RPC by the client + // will cause the LB policy to send out an RLS request. This will + // also lead to creation of a pending entry, and further RPCs by the + // client should not result in RLS requests being sent out. + rlsReqCh := make(chan struct{}, 1) + interceptor := func(ctx context.Context, _ any, _ *grpc.UnaryServerInfo, _ grpc.UnaryHandler) (resp any, err error) { + rlsReqCh <- struct{}{} + <-ctx.Done() + return nil, ctx.Err() + } + + // Start an RLS server and set the throttler to never throttle. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, grpc.UnaryInterceptor(interceptor)) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build RLS service config with an optional default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + if test.withDefaultTarget { + _, defBackendAddress := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + } + + // Register a manual resolver and push the RLS service config + // through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Create new client. + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC that results in the RLS request being sent out. And + // since the RLS server is configured to block on the first request, + // this RPC will block until its context expires. This ensures that + // we have a pending cache entry for the duration of the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + go func() { + client := testgrpc.NewTestServiceClient(cc) + client.EmptyCall(ctx, &testpb.Empty{}) + }() + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Make another RPC and expect it to fail the same way. + ctx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + makeTestRPCAndVerifyError(ctx, t, cc, codes.DeadlineExceeded, context.DeadlineExceeded) + + // Make sure no RLS request is sent out this time around. + verifyRLSRequest(t, rlsReqCh, false) + }) + } +} + +// Test_RLSDefaultTargetPicksMetric tests the default target picks metric. It +// configures an RLS Balancer which specifies to route to the default target in +// the RLS Configuration, and makes an RPC on a Channel containing this RLS +// Balancer. This test then asserts a default target picks metric is emitted, +// and target pick or failed pick metric is not emitted. +func (s) Test_RLSDefaultTargetPicksMetric(t *testing.T) { + // Start an RLS server and set the throttler to always throttle requests. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, alwaysThrottlingThrottler()) + + // Build RLS service config with a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + defBackendCh, defBackendAddress := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + tmr := stats.NewTestMetricsRecorder() + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithStatsHandler(tmr)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the default target. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, defBackendCh) + + if got, _ := tmr.Metric("grpc.lb.rls.default_target_picks"); got != 1 { + t.Fatalf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.rls.default_target_picks", got, 1) + } + if _, ok := tmr.Metric("grpc.lb.rls.target_picks"); ok { + t.Fatalf("Data is present for metric %v", "grpc.lb.rls.target_picks") + } + if _, ok := tmr.Metric("grpc.lb.rls.failed_picks"); ok { + t.Fatalf("Data is present for metric %v", "grpc.lb.rls.failed_picks") + } +} + +// Test_RLSTargetPicksMetric tests the target picks metric. It configures an RLS +// Balancer which specifies to route to a target through a RouteLookupResponse, +// and makes an RPC on a Channel containing this RLS Balancer. This test then +// asserts a target picks metric is emitted, and default target pick or failed +// pick metric is not emitted. +func (s) Test_RLSTargetPicksMetric(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build the RLS config without a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + + // Start a test backend, and setup the fake RLS server to return this as a + // target in the RLS response. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + tmr := stats.NewTestMetricsRecorder() + // Dial the backend. + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithStatsHandler(tmr)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + if got, _ := tmr.Metric("grpc.lb.rls.target_picks"); got != 1 { + t.Fatalf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.rls.target_picks", got, 1) + } + if _, ok := tmr.Metric("grpc.lb.rls.default_target_picks"); ok { + t.Fatalf("Data is present for metric %v", "grpc.lb.rls.default_target_picks") + } + if _, ok := tmr.Metric("grpc.lb.rls.failed_picks"); ok { + t.Fatalf("Data is present for metric %v", "grpc.lb.rls.failed_picks") + } +} + +// Test_RLSFailedPicksMetric tests the failed picks metric. It configures an RLS +// Balancer to fail a pick with unavailable, and makes an RPC on a Channel +// containing this RLS Balancer. This test then asserts a failed picks metric is +// emitted, and default target pick or target pick metric is not emitted. +func (s) Test_RLSFailedPicksMetric(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build an RLS config without a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + tmr := stats.NewTestMetricsRecorder() + // Dial the backend. + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithStatsHandler(tmr)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and expect it to fail with deadline exceeded error. We use a + // smaller timeout to ensure that the test doesn't run very long. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + makeTestRPCAndVerifyError(ctx, t, cc, codes.Unavailable, errors.New("RLS response's target list does not contain any entries for key")) + + if got, _ := tmr.Metric("grpc.lb.rls.failed_picks"); got != 1 { + t.Fatalf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.rls.failed_picks", got, 1) + } + if _, ok := tmr.Metric("grpc.lb.rls.target_picks"); ok { + t.Fatalf("Data is present for metric %v", "grpc.lb.rls.target_picks") + } + if _, ok := tmr.Metric("grpc.lb.rls.default_target_picks"); ok { + t.Fatalf("Data is present for metric %v", "grpc.lb.rls.default_target_picks") + } +} + +// Test verifies the scenario where there is a matching entry in the data cache +// which is valid and there is no pending request. The pick is expected to be +// delegated to the child policy. +func (s) TestPick_DataCacheHit_NoPendingEntry_ValidEntry(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build the RLS config without a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + // Start a test backend, and setup the fake RLS server to return this as a + // target in the RLS response. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Create new client. + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Make another RPC and expect it to find the target in the data cache. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure no RLS request is sent out this time around. + verifyRLSRequest(t, rlsReqCh, false) +} + +// Test verifies the scenario where there is a matching entry in the data cache +// which is valid and there is no pending request. The pick is expected to be +// delegated to the child policy. +func (s) TestPick_DataCacheHit_NoPendingEntry_ValidEntry_WithHeaderData(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build the RLS config without a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + + // Start a test backend which expects the header data contents sent from the + // RLS server to be part of RPC metadata as X-Google-RLS-Data header. + const headerDataContents = "foo,bar,baz" + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + gotHeaderData := metadata.ValueFromIncomingContext(ctx, "x-google-rls-data") + if len(gotHeaderData) != 1 || gotHeaderData[0] != headerDataContents { + return nil, fmt.Errorf("got metadata in `X-Google-RLS-Data` is %v, want %s", gotHeaderData, headerDataContents) + } + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + defer backend.Stop() + + // Setup the fake RLS server to return the above backend as a target in the + // RLS response. Also, populate the header data field in the response. + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{ + Targets: []string{backend.Address}, + HeaderData: headerDataContents, + }} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Create new client. + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend with the header + // data sent by the RLS server. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() RPC: %v", err) + } +} + +// Test verifies the scenario where there is a matching entry in the data cache +// which is stale and there is no pending request. The pick is expected to be +// delegated to the child policy with a proactive cache refresh. +func (s) TestPick_DataCacheHit_NoPendingEntry_StaleEntry(t *testing.T) { + // We expect the same pick behavior (i.e delegated to the child policy) for + // a proactive refresh whether the control channel is throttled or not. + tests := []struct { + name string + throttled bool + }{ + { + name: "throttled", + throttled: true, + }, + { + name: "notThrottled", + throttled: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Start an RLS server and setup the throttler appropriately. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + var throttler *fakeThrottler + firstRPCDone := grpcsync.NewEvent() + if test.throttled { + throttler = oneTimeAllowingThrottler(firstRPCDone) + overrideAdaptiveThrottler(t, throttler) + } else { + throttler = neverThrottlingThrottler() + overrideAdaptiveThrottler(t, throttler) + } + + // Build the RLS config without a default target. Set the stale age + // to a very low value to force entries to become stale quickly. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(time.Minute) + rlsConfig.RouteLookupConfig.StaleAge = durationpb.New(defaultTestShortTimeout) + + // Start a test backend, and setup the fake RLS server to return + // this as a target in the RLS response. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config + // through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Create new client. + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + firstRPCDone.Fire() + + // The cache entry has a large maxAge, but a small stateAge. We keep + // retrying until the cache entry becomes stale, in which case we expect a + // proactive cache refresh. + // + // If the control channel is not throttled, then we expect an RLS request + // to be sent out. If the control channel is throttled, we expect the fake + // throttler's channel to be signalled. + for { + // Make another RPC and expect it to find the target in the data cache. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + if !test.throttled { + select { + case <-time.After(defaultTestShortTimeout): + // Go back and retry the RPC. + case <-rlsReqCh: + return + } + } else { + select { + case <-time.After(defaultTestShortTimeout): + // Go back and retry the RPC. + case <-throttler.throttleCh: + return + } + } + } + }) + } +} + +// Test verifies scenarios where there is a matching entry in the data cache +// which has expired and there is no pending request. +func (s) TestPick_DataCacheHit_NoPendingEntry_ExpiredEntry(t *testing.T) { + tests := []struct { + name string + throttled bool + withDefaultTarget bool + }{ + { + name: "throttledWithDefaultTarget", + throttled: true, + withDefaultTarget: true, + }, + { + name: "throttledWithoutDefaultTarget", + throttled: true, + withDefaultTarget: false, + }, + { + name: "notThrottled", + throttled: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Start an RLS server and setup the throttler appropriately. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + var throttler *fakeThrottler + firstRPCDone := grpcsync.NewEvent() + if test.throttled { + throttler = oneTimeAllowingThrottler(firstRPCDone) + overrideAdaptiveThrottler(t, throttler) + } else { + throttler = neverThrottlingThrottler() + overrideAdaptiveThrottler(t, throttler) + } + + // Build the RLS config with a very low value for maxAge. This will + // ensure that cache entries become invalid very soon. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(defaultTestShortTimeout) + + // Start a default backend if needed. + var defBackendCh chan struct{} + if test.withDefaultTarget { + var defBackendAddress string + defBackendCh, defBackendAddress = startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + } + + // Start a test backend, and setup the fake RLS server to return + // this as a target in the RLS response. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config + // through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Create new client. + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + firstRPCDone.Fire() + + // Keep retrying the RPC until the cache entry expires. Expected behavior + // is dependent on the scenario being tested. + switch { + case test.throttled && test.withDefaultTarget: + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, defBackendCh) + <-throttler.throttleCh + case test.throttled && !test.withDefaultTarget: + makeTestRPCAndVerifyError(ctx, t, cc, codes.Unavailable, errRLSThrottled) + <-throttler.throttleCh + case !test.throttled: + for { + // The backend to which the RPC is routed does not change after the + // cache entry expires because the control channel is not throttled. + // So, we need to keep retrying until the cache entry expires, at + // which point we expect an RLS request to be sent out and the RPC to + // get routed to the same testBackend. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + select { + case <-time.After(defaultTestShortTimeout): + // Go back and retry the RPC. + case <-rlsReqCh: + return + } + } + } + }) + } +} + +// Test verifies scenarios where there is a matching entry in the data cache +// which has expired and is in backoff and there is no pending request. +func (s) TestPick_DataCacheHit_NoPendingEntry_ExpiredEntryInBackoff(t *testing.T) { + tests := []struct { + name string + withDefaultTarget bool + }{ + { + name: "withDefaultTarget", + withDefaultTarget: true, + }, + { + name: "withoutDefaultTarget", + withDefaultTarget: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Override the backoff strategy to return a large backoff which + // will make sure the date cache entry remains in backoff for the + // duration of the test. + origBackoffStrategy := defaultBackoffStrategy + defaultBackoffStrategy = &fakeBackoffStrategy{backoff: defaultTestTimeout} + defer func() { defaultBackoffStrategy = origBackoffStrategy }() + + // Build the RLS config with a very low value for maxAge. This will + // ensure that cache entries become invalid very soon. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(defaultTestShortTimeout) + + // Start a default backend if needed. + var defBackendCh chan struct{} + if test.withDefaultTarget { + var defBackendAddress string + defBackendCh, defBackendAddress = startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + } + + // Start a test backend, and set up the fake RLS server to return this as + // a target in the RLS response. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Create new client. + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Set up the fake RLS server to return errors. This will push the cache + // entry into backoff. + var rlsLastErr = status.Error(codes.DeadlineExceeded, "last RLS request failed") + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Err: rlsLastErr} + }) + + // Since the RLS server is now configured to return errors, this will push + // the cache entry into backoff. The pick will be delegated to the default + // backend if one exits, and will fail with the error returned by the RLS + // server otherwise. + if test.withDefaultTarget { + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, defBackendCh) + } else { + makeTestRPCAndVerifyError(ctx, t, cc, codes.Unavailable, rlsLastErr) + } + }) + } +} + +// Test verifies scenarios where there is a matching entry in the data cache +// which is stale and there is a pending request. +func (s) TestPick_DataCacheHit_PendingEntryExists_StaleEntry(t *testing.T) { + tests := []struct { + name string + withDefaultTarget bool + }{ + { + name: "withDefaultTarget", + withDefaultTarget: true, + }, + { + name: "withoutDefaultTarget", + withDefaultTarget: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // A unary interceptor which simply calls the underlying handler + // until the first client RPC is done. We want one client RPC to + // succeed to ensure that a data cache entry is created. For + // subsequent client RPCs which result in RLS requests, this + // interceptor blocks until the test's context expires. And since we + // configure the RLS LB policy with a really low value for max age, + // this allows us to simulate the condition where the it has an + // expired entry and a pending entry in the cache. + rlsReqCh := make(chan struct{}, 1) + firstRPCDone := grpcsync.NewEvent() + interceptor := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + select { + case rlsReqCh <- struct{}{}: + default: + } + if firstRPCDone.HasFired() { + <-ctx.Done() + return nil, ctx.Err() + } + return handler(ctx, req) + } + + // Start an RLS server and set the throttler to never throttle. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, grpc.UnaryInterceptor(interceptor)) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build RLS service config with an optional default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + if test.withDefaultTarget { + _, defBackendAddress := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + } + + // Low value for stale age to force entries to become stale quickly. + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(time.Minute) + rlsConfig.RouteLookupConfig.StaleAge = durationpb.New(defaultTestShortTimeout) + + // Start a test backend, and setup the fake RLS server to return + // this as a target in the RLS response. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config + // through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Create new client. + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + firstRPCDone.Fire() + + // The cache entry has a large maxAge, but a small stateAge. We keep + // retrying until the cache entry becomes stale, in which case we expect a + // proactive cache refresh. + for { + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + select { + case <-time.After(defaultTestShortTimeout): + // Go back and retry the RPC. + case <-rlsReqCh: + return + } + } + }) + } +} + +// Test verifies scenarios where there is a matching entry in the data cache +// which is expired and there is a pending request. +func (s) TestPick_DataCacheHit_PendingEntryExists_ExpiredEntry(t *testing.T) { + tests := []struct { + name string + withDefaultTarget bool + }{ + { + name: "withDefaultTarget", + withDefaultTarget: true, + }, + { + name: "withoutDefaultTarget", + withDefaultTarget: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // A unary interceptor which simply calls the underlying handler + // until the first client RPC is done. We want one client RPC to + // succeed to ensure that a data cache entry is created. For + // subsequent client RPCs which result in RLS requests, this + // interceptor blocks until the test's context expires. And since we + // configure the RLS LB policy with a really low value for max age, + // this allows us to simulate the condition where the it has an + // expired entry and a pending entry in the cache. + rlsReqCh := make(chan struct{}, 1) + firstRPCDone := grpcsync.NewEvent() + interceptor := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + select { + case rlsReqCh <- struct{}{}: + default: + } + if firstRPCDone.HasFired() { + <-ctx.Done() + return nil, ctx.Err() + } + return handler(ctx, req) + } + + // Start an RLS server and set the throttler to never throttle. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, grpc.UnaryInterceptor(interceptor)) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build RLS service config with an optional default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + if test.withDefaultTarget { + _, defBackendAddress := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + } + // Set a low value for maxAge to ensure cache entries expire soon. + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(defaultTestShortTimeout) + + // Start a test backend, and setup the fake RLS server to return + // this as a target in the RLS response. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config + // through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Create new client. + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + firstRPCDone.Fire() + + // At this point, we have a cache entry with a small maxAge, and the + // RLS server is configured to block on further RLS requests. As we + // retry the RPC, at some point the cache entry would expire and + // force us to send an RLS request which would block on the server, + // giving us a pending cache entry for the duration of the test. + go func() { + for client := testgrpc.NewTestServiceClient(cc); ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + client.EmptyCall(ctx, &testpb.Empty{}) + } + }() + verifyRLSRequest(t, rlsReqCh, true) + + // Another RPC at this point should find the pending entry and be queued. + // But since we pass a small deadline, this RPC should fail with a + // deadline exceeded error since the pending request does not return until + // the test is done. And since we have a pending entry, we expect no RLS + // request to be sent out. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + makeTestRPCAndVerifyError(sCtx, t, cc, codes.DeadlineExceeded, context.DeadlineExceeded) + verifyRLSRequest(t, rlsReqCh, false) + }) + } +} + +func TestIsFullMethodNameValid(t *testing.T) { + tests := []struct { + desc string + methodName string + want bool + }{ + { + desc: "does not start with a slash", + methodName: "service/method", + want: false, + }, + { + desc: "does not contain a method", + methodName: "/service", + want: false, + }, + { + desc: "path has more elements", + methodName: "/service/path/to/method", + want: false, + }, + { + desc: "valid", + methodName: "/service/method", + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if got := isFullMethodNameValid(test.methodName); got != test.want { + t.Fatalf("isFullMethodNameValid(%q) = %v, want %v", test.methodName, got, test.want) + } + }) + } +} + +// Tests the conversion of the child pickers error to the pick result attribute. +func (s) TestChildPickResultError(t *testing.T) { + tests := []struct { + name string + err error + want string + }{ + { + name: "nil", + err: nil, + want: "complete", + }, + { + name: "errNoSubConnAvailable", + err: balancer.ErrNoSubConnAvailable, + want: "queue", + }, + { + name: "status error", + err: status.Error(codes.Unimplemented, "unimplemented"), + want: "drop", + }, + { + name: "other error", + err: errors.New("some error"), + want: "fail", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := errToPickResult(test.err); got != test.want { + t.Fatalf("errToPickResult(%q) = %v, want %v", test.err, got, test.want) + } + }) + } +} diff --git a/balancer/roundrobin/roundrobin.go b/balancer/roundrobin/roundrobin.go new file mode 100644 index 0000000..22e6e32 --- /dev/null +++ b/balancer/roundrobin/roundrobin.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package roundrobin defines a roundrobin balancer. Roundrobin balancer is +// installed as one of the default balancers in gRPC, users don't need to +// explicitly install this balancer. +package roundrobin + +import ( + "fmt" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/endpointsharding" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +// Name is the name of round_robin balancer. +const Name = "round_robin" + +var logger = grpclog.Component("roundrobin") + +func init() { + balancer.Register(builder{}) +} + +type builder struct{} + +func (bb builder) Name() string { + return Name +} + +func (bb builder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + childBuilder := balancer.Get(pickfirst.Name).Build + bal := &rrBalancer{ + cc: cc, + Balancer: endpointsharding.NewBalancer(cc, opts, childBuilder, endpointsharding.Options{}), + } + bal.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[%p] ", bal)) + bal.logger.Infof("Created") + return bal +} + +type rrBalancer struct { + balancer.Balancer + cc balancer.ClientConn + logger *internalgrpclog.PrefixLogger +} + +func (b *rrBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { + return b.Balancer.UpdateClientConnState(balancer.ClientConnState{ + // Enable the health listener in pickfirst children for client side health + // checks and outlier detection, if configured. + ResolverState: pickfirst.EnableHealthListener(ccs.ResolverState), + }) +} diff --git a/balancer/subconn.go b/balancer/subconn.go new file mode 100644 index 0000000..c1ca7c9 --- /dev/null +++ b/balancer/subconn.go @@ -0,0 +1,120 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package balancer + +import ( + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/resolver" +) + +// A SubConn represents a single connection to a gRPC backend service. +// +// All SubConns start in IDLE, and will not try to connect. To trigger a +// connection attempt, Balancers must call Connect. +// +// If the connection attempt fails, the SubConn will transition to +// TRANSIENT_FAILURE for a backoff period, and then return to IDLE. If the +// connection attempt succeeds, it will transition to READY. +// +// If a READY SubConn becomes disconnected, the SubConn will transition to IDLE. +// +// If a connection re-enters IDLE, Balancers must call Connect again to trigger +// a new connection attempt. +// +// Each SubConn contains a list of addresses. gRPC will try to connect to the +// addresses in sequence, and stop trying the remainder once the first +// connection is successful. However, this behavior is deprecated. SubConns +// should only use a single address. +// +// NOTICE: This interface is intended to be implemented by gRPC, or intercepted +// by custom load balancing polices. Users should not need their own complete +// implementation of this interface -- they should always delegate to a SubConn +// returned by ClientConn.NewSubConn() by embedding it in their implementations. +// An embedded SubConn must never be nil, or runtime panics will occur. +type SubConn interface { + // UpdateAddresses updates the addresses used in this SubConn. + // gRPC checks if currently-connected address is still in the new list. + // If it's in the list, the connection will be kept. + // If it's not in the list, the connection will gracefully close, and + // a new connection will be created. + // + // This will trigger a state transition for the SubConn. + // + // Deprecated: this method will be removed. Create new SubConns for new + // addresses instead. + UpdateAddresses([]resolver.Address) + // Connect starts the connecting for this SubConn. + Connect() + // GetOrBuildProducer returns a reference to the existing Producer for this + // ProducerBuilder in this SubConn, or, if one does not currently exist, + // creates a new one and returns it. Returns a close function which may be + // called when the Producer is no longer needed. Otherwise the producer + // will automatically be closed upon connection loss or subchannel close. + // Should only be called on a SubConn in state Ready. Otherwise the + // producer will be unable to create streams. + GetOrBuildProducer(ProducerBuilder) (p Producer, close func()) + // Shutdown shuts down the SubConn gracefully. Any started RPCs will be + // allowed to complete. No future calls should be made on the SubConn. + // One final state update will be delivered to the StateListener (or + // UpdateSubConnState; deprecated) with ConnectivityState of Shutdown to + // indicate the shutdown operation. This may be delivered before + // in-progress RPCs are complete and the actual connection is closed. + Shutdown() + // RegisterHealthListener registers a health listener that receives health + // updates for a Ready SubConn. Only one health listener can be registered + // at a time. A health listener should be registered each time the SubConn's + // connectivity state changes to READY. Registering a health listener when + // the connectivity state is not READY may result in undefined behaviour. + // This method must not be called synchronously while handling an update + // from a previously registered health listener. + RegisterHealthListener(func(SubConnState)) + // EnforceSubConnEmbedding is included to force implementers to embed + // another implementation of this interface, allowing gRPC to add methods + // without breaking users. + internal.EnforceSubConnEmbedding +} + +// A ProducerBuilder is a simple constructor for a Producer. It is used by the +// SubConn to create producers when needed. +type ProducerBuilder interface { + // Build creates a Producer. The first parameter is always a + // grpc.ClientConnInterface (a type to allow creating RPCs/streams on the + // associated SubConn), but is declared as `any` to avoid a dependency + // cycle. Build also returns a close function that will be called when all + // references to the Producer have been given up for a SubConn, or when a + // connectivity state change occurs on the SubConn. The close function + // should always block until all asynchronous cleanup work is completed. + Build(grpcClientConnInterface any) (p Producer, close func()) +} + +// SubConnState describes the state of a SubConn. +type SubConnState struct { + // ConnectivityState is the connectivity state of the SubConn. + ConnectivityState connectivity.State + // ConnectionError is set if the ConnectivityState is TransientFailure, + // describing the reason the SubConn failed. Otherwise, it is nil. + ConnectionError error +} + +// A Producer is a type shared among potentially many consumers. It is +// associated with a SubConn, and an implementation will typically contain +// other methods to provide additional functionality, e.g. configuration or +// subscription registration. +type Producer any diff --git a/balancer/weightedroundrobin/balancer.go b/balancer/weightedroundrobin/balancer.go new file mode 100644 index 0000000..483b51e --- /dev/null +++ b/balancer/weightedroundrobin/balancer.go @@ -0,0 +1,674 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package weightedroundrobin provides an implementation of the weighted round +// robin LB policy, as defined in [gRFC A58]. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +// +// [gRFC A58]: https://github.com/grpc/proposal/blob/master/A58-client-side-weighted-round-robin-lb-policy.md +package weightedroundrobin + +import ( + "encoding/json" + "fmt" + rand "math/rand/v2" + "sync" + "sync/atomic" + "time" + "unsafe" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/endpointsharding" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/balancer/weightedroundrobin/internal" + "google.golang.org/grpc/balancer/weightedtarget" + "google.golang.org/grpc/connectivity" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" +) + +// Name is the name of the weighted round robin balancer. +const Name = "weighted_round_robin" + +var ( + rrFallbackMetric = estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "grpc.lb.wrr.rr_fallback", + Description: "EXPERIMENTAL. Number of scheduler updates in which there were not enough endpoints with valid weight, which caused the WRR policy to fall back to RR behavior.", + Unit: "{update}", + Labels: []string{"grpc.target"}, + OptionalLabels: []string{"grpc.lb.locality", "grpc.lb.backend_service"}, + Default: false, + }) + + endpointWeightNotYetUsableMetric = estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "grpc.lb.wrr.endpoint_weight_not_yet_usable", + Description: "EXPERIMENTAL. Number of endpoints from each scheduler update that don't yet have usable weight information (i.e., either the load report has not yet been received, or it is within the blackout period).", + Unit: "{endpoint}", + Labels: []string{"grpc.target"}, + OptionalLabels: []string{"grpc.lb.locality", "grpc.lb.backend_service"}, + Default: false, + }) + + endpointWeightStaleMetric = estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "grpc.lb.wrr.endpoint_weight_stale", + Description: "EXPERIMENTAL. Number of endpoints from each scheduler update whose latest weight is older than the expiration period.", + Unit: "{endpoint}", + Labels: []string{"grpc.target"}, + OptionalLabels: []string{"grpc.lb.locality", "grpc.lb.backend_service"}, + Default: false, + }) + endpointWeightsMetric = estats.RegisterFloat64Histo(estats.MetricDescriptor{ + Name: "grpc.lb.wrr.endpoint_weights", + Description: "EXPERIMENTAL. Weight of each endpoint, recorded on every scheduler update. Endpoints without usable weights will be recorded as weight 0.", + Unit: "{endpoint}", + Labels: []string{"grpc.target"}, + OptionalLabels: []string{"grpc.lb.locality", "grpc.lb.backend_service"}, + Default: false, + }) +) + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &wrrBalancer{ + ClientConn: cc, + target: bOpts.Target.String(), + metricsRecorder: cc.MetricsRecorder(), + addressWeights: resolver.NewAddressMapV2[*endpointWeight](), + endpointToWeight: resolver.NewEndpointMap[*endpointWeight](), + scToWeight: make(map[balancer.SubConn]*endpointWeight), + } + + b.child = endpointsharding.NewBalancer(b, bOpts, balancer.Get(pickfirst.Name).Build, endpointsharding.Options{}) + b.logger = prefixLogger(b) + if b.logger.V(2) { + b.logger.Infof("Created") + } + return b +} + +func (bb) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + lbCfg := &lbConfig{ + // Default values as documented in A58. + OOBReportingPeriod: iserviceconfig.Duration(10 * time.Second), + BlackoutPeriod: iserviceconfig.Duration(10 * time.Second), + WeightExpirationPeriod: iserviceconfig.Duration(3 * time.Minute), + WeightUpdatePeriod: iserviceconfig.Duration(time.Second), + ErrorUtilizationPenalty: 1, + } + if err := json.Unmarshal(js, lbCfg); err != nil { + return nil, fmt.Errorf("wrr: unable to unmarshal LB policy config: %s, error: %v", string(js), err) + } + + if lbCfg.ErrorUtilizationPenalty < 0 { + return nil, fmt.Errorf("wrr: errorUtilizationPenalty must be non-negative") + } + + // For easier comparisons later, ensure the OOB reporting period is unset + // (0s) when OOB reports are disabled. + if !lbCfg.EnableOOBLoadReport { + lbCfg.OOBReportingPeriod = 0 + } + + // Impose lower bound of 100ms on weightUpdatePeriod. + if !internal.AllowAnyWeightUpdatePeriod && lbCfg.WeightUpdatePeriod < iserviceconfig.Duration(100*time.Millisecond) { + lbCfg.WeightUpdatePeriod = iserviceconfig.Duration(100 * time.Millisecond) + } + + return lbCfg, nil +} + +func (bb) Name() string { + return Name +} + +// updateEndpointsLocked updates endpoint weight state based off new update, by +// starting and clearing any endpoint weights needed. +// +// Caller must hold b.mu. +func (b *wrrBalancer) updateEndpointsLocked(endpoints []resolver.Endpoint) { + endpointSet := resolver.NewEndpointMap[*endpointWeight]() + addressSet := resolver.NewAddressMapV2[*endpointWeight]() + for _, endpoint := range endpoints { + endpointSet.Set(endpoint, nil) + for _, addr := range endpoint.Addresses { + addressSet.Set(addr, nil) + } + ew, ok := b.endpointToWeight.Get(endpoint) + if !ok { + ew = &endpointWeight{ + logger: b.logger, + connectivityState: connectivity.Connecting, + // Initially, we set load reports to off, because they are not + // running upon initial endpointWeight creation. + cfg: &lbConfig{EnableOOBLoadReport: false}, + metricsRecorder: b.metricsRecorder, + target: b.target, + locality: b.locality, + clusterName: b.clusterName, + } + for _, addr := range endpoint.Addresses { + b.addressWeights.Set(addr, ew) + } + b.endpointToWeight.Set(endpoint, ew) + } + ew.updateConfig(b.cfg) + } + + for endpoint := range b.endpointToWeight.All() { + if _, ok := endpointSet.Get(endpoint); ok { + // Existing endpoint also in new endpoint list; skip. + continue + } + b.endpointToWeight.Delete(endpoint) + for _, addr := range endpoint.Addresses { + if _, ok := addressSet.Get(addr); !ok { // old endpoints to be deleted can share addresses with new endpoints, so only delete if necessary + b.addressWeights.Delete(addr) + } + } + // SubConn map will get handled in updateSubConnState + // when receives SHUTDOWN signal. + } +} + +// wrrBalancer implements the weighted round robin LB policy. +type wrrBalancer struct { + // The following fields are set at initialization time and read only after that, + // so they do not need to be protected by a mutex. + child balancer.Balancer + balancer.ClientConn // Embed to intercept NewSubConn operation + logger *grpclog.PrefixLogger + target string + metricsRecorder estats.MetricsRecorder + + mu sync.Mutex + cfg *lbConfig // active config + locality string + clusterName string + stopPicker *grpcsync.Event + addressWeights *resolver.AddressMapV2[*endpointWeight] + endpointToWeight *resolver.EndpointMap[*endpointWeight] + scToWeight map[balancer.SubConn]*endpointWeight +} + +func (b *wrrBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { + if b.logger.V(2) { + b.logger.Infof("Received update from resolver with state: %+v", ccs) + } + cfg, ok := ccs.BalancerConfig.(*lbConfig) + if !ok { + return fmt.Errorf("wrr: received nil or illegal BalancerConfig (type %T): %v", ccs.BalancerConfig, ccs.BalancerConfig) + } + + // Note: empty endpoints and duplicate addresses across endpoints won't + // explicitly error but will have undefined behavior. + b.mu.Lock() + b.cfg = cfg + b.locality = weightedtarget.LocalityFromResolverState(ccs.ResolverState) + b.clusterName = backendServiceFromState(ccs.ResolverState) + b.updateEndpointsLocked(ccs.ResolverState.Endpoints) + b.mu.Unlock() + + // This causes child to update picker inline and will thus cause inline + // picker update. + return b.child.UpdateClientConnState(balancer.ClientConnState{ + // Make pickfirst children use health listeners for outlier detection to + // work. + ResolverState: pickfirst.EnableHealthListener(ccs.ResolverState), + }) +} + +func (b *wrrBalancer) UpdateState(state balancer.State) { + if b.logger.V(2) { + b.logger.Infof("Received update from child policy with state: %+v", state) + } + + b.mu.Lock() + defer b.mu.Unlock() + + if b.stopPicker != nil { + b.stopPicker.Fire() + b.stopPicker = nil + } + + childStates := endpointsharding.ChildStatesFromPicker(state.Picker) + + var readyPickersWeight []pickerWeightedEndpoint + + for _, childState := range childStates { + if childState.State.ConnectivityState == connectivity.Ready { + ew, ok := b.endpointToWeight.Get(childState.Endpoint) + if !ok { + // Should never happen, simply continue and ignore this endpoint + // for READY pickers. + continue + } + readyPickersWeight = append(readyPickersWeight, pickerWeightedEndpoint{ + picker: childState.State.Picker, + weightedEndpoint: ew, + }) + } + } + // If no ready pickers are present, simply defer to the round robin picker + // from endpoint sharding, which will round robin across the most relevant + // pick first children in the highest precedence connectivity state. + if len(readyPickersWeight) == 0 { + b.ClientConn.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: state.Picker, + }) + return + } + + p := &picker{ + cfg: b.cfg, + weightedPickers: readyPickersWeight, + metricsRecorder: b.metricsRecorder, + locality: b.locality, + target: b.target, + clusterName: b.clusterName, + } + p.idx.Store(rand.Uint32()) // start the scheduler at a random point + + b.stopPicker = grpcsync.NewEvent() + p.start(b.stopPicker) + + b.ClientConn.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: p, + }) +} + +type pickerWeightedEndpoint struct { + picker balancer.Picker + weightedEndpoint *endpointWeight +} + +func (b *wrrBalancer) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + addr := addrs[0] // The new pick first policy for DualStack will only ever create a SubConn with one address. + var sc balancer.SubConn + + oldListener := opts.StateListener + opts.StateListener = func(state balancer.SubConnState) { + b.updateSubConnState(sc, state) + oldListener(state) + } + + b.mu.Lock() + defer b.mu.Unlock() + ewi, ok := b.addressWeights.Get(addr) + if !ok { + // SubConn state updates can come in for a no longer relevant endpoint + // weight (from the old system after a new config update is applied). + return nil, fmt.Errorf("wrr: balancer is being closed; no new SubConns allowed") + } + sc, err := b.ClientConn.NewSubConn([]resolver.Address{addr}, opts) + if err != nil { + return nil, err + } + b.scToWeight[sc] = ewi + return sc, nil +} + +func (b *wrrBalancer) ResolverError(err error) { + if b.logger.V(2) { + b.logger.Infof("Received error from resolver: %v", err) + } + // Will cause inline picker update from endpoint sharding. + b.child.ResolverError(err) +} + +func (b *wrrBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *wrrBalancer) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + if b.logger.V(2) { + b.logger.Infof("Received update from SubConn %v with state: %+v", sc, state) + } + + b.mu.Lock() + ew := b.scToWeight[sc] + // updates from a no longer relevant SubConn update, nothing to do here but + // forward state to state listener, which happens in wrapped listener. Will + // eventually get cleared from scMap once receives Shutdown signal. + if ew == nil { + b.mu.Unlock() + return + } + if state.ConnectivityState == connectivity.Shutdown { + delete(b.scToWeight, sc) + } + b.mu.Unlock() + + // On the first READY SubConn/Transition for an endpoint, set pickedSC, + // clear endpoint tracking weight state, and potentially start an OOB watch. + if state.ConnectivityState == connectivity.Ready && ew.pickedSC == nil { + ew.pickedSC = sc + ew.mu.Lock() + ew.nonEmptySince = time.Time{} + ew.lastUpdated = time.Time{} + cfg := ew.cfg + ew.mu.Unlock() + ew.updateORCAListener(cfg) + return + } + + // If the pickedSC (the one pick first uses for an endpoint) transitions out + // of READY, stop OOB listener if needed and clear pickedSC so the next + // created SubConn for the endpoint that goes READY will be chosen for + // endpoint as the active SubConn. + if state.ConnectivityState != connectivity.Ready && ew.pickedSC == sc { + // The first SubConn that goes READY for an endpoint is what pick first + // will pick. Only once that SubConn goes not ready will pick first + // restart this cycle of creating SubConns and using the first READY + // one. The lower level endpoint sharding will ping the Pick First once + // this occurs to ExitIdle which will trigger a connection attempt. + if ew.stopORCAListener != nil { + ew.stopORCAListener() + } + ew.pickedSC = nil + } +} + +// Close stops the balancer. It cancels any ongoing scheduler updates and +// stops any ORCA listeners. +func (b *wrrBalancer) Close() { + b.mu.Lock() + if b.stopPicker != nil { + b.stopPicker.Fire() + b.stopPicker = nil + } + b.mu.Unlock() + + // Ensure any lingering OOB watchers are stopped. + for _, ew := range b.endpointToWeight.All() { + if ew.stopORCAListener != nil { + ew.stopORCAListener() + } + } + b.child.Close() + if b.logger.V(2) { + b.logger.Infof("Shutdown") + } +} + +func (b *wrrBalancer) ExitIdle() { + b.child.ExitIdle() +} + +// picker is the WRR policy's picker. It uses live-updating backend weights to +// update the scheduler periodically and ensure picks are routed proportional +// to those weights. +type picker struct { + scheduler unsafe.Pointer // *scheduler; accessed atomically + idx atomic.Uint32 // incrementing value used by the scheduler + cfg *lbConfig // active config when picker created + + weightedPickers []pickerWeightedEndpoint // all READY pickers + + // The following fields are immutable. + target string + locality string + clusterName string + metricsRecorder estats.MetricsRecorder +} + +func (p *picker) endpointWeights(recordMetrics bool) []float64 { + wp := make([]float64, len(p.weightedPickers)) + now := internal.TimeNow() + for i, wpi := range p.weightedPickers { + wp[i] = wpi.weightedEndpoint.weight(now, time.Duration(p.cfg.WeightExpirationPeriod), time.Duration(p.cfg.BlackoutPeriod), recordMetrics) + } + return wp +} + +func (p *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + // Read the scheduler atomically. All scheduler operations are threadsafe, + // and if the scheduler is replaced during this usage, we want to use the + // scheduler that was live when the pick started. + sched := *(*scheduler)(atomic.LoadPointer(&p.scheduler)) + + pickedPicker := p.weightedPickers[sched.nextIndex()] + pr, err := pickedPicker.picker.Pick(info) + if err != nil { + return balancer.PickResult{}, err + } + if !p.cfg.EnableOOBLoadReport { + oldDone := pr.Done + pr.Done = func(info balancer.DoneInfo) { + if load, ok := info.ServerLoad.(*v3orcapb.OrcaLoadReport); ok && load != nil { + pickedPicker.weightedEndpoint.OnLoadReport(load) + } + if oldDone != nil { + oldDone(info) + } + } + } + return pr, nil +} + +func (p *picker) inc() uint32 { + return p.idx.Add(1) +} + +func (p *picker) regenerateScheduler() { + s := p.newScheduler(true) + atomic.StorePointer(&p.scheduler, unsafe.Pointer(&s)) +} + +func (p *picker) start(stopPicker *grpcsync.Event) { + p.regenerateScheduler() + if len(p.weightedPickers) == 1 { + // No need to regenerate weights with only one backend. + return + } + + go func() { + ticker := time.NewTicker(time.Duration(p.cfg.WeightUpdatePeriod)) + defer ticker.Stop() + for { + select { + case <-stopPicker.Done(): + return + case <-ticker.C: + p.regenerateScheduler() + } + } + }() +} + +// endpointWeight is the weight for an endpoint. It tracks the SubConn that will +// be picked for the endpoint, and other parameters relevant to computing the +// effective weight. When needed, it also tracks connectivity state, listens for +// metrics updates by implementing the orca.OOBListener interface and manages +// that listener. +type endpointWeight struct { + // The following fields are immutable. + logger *grpclog.PrefixLogger + target string + metricsRecorder estats.MetricsRecorder + locality string + clusterName string + + // The following fields are only accessed on calls into the LB policy, and + // do not need a mutex. + connectivityState connectivity.State + stopORCAListener func() + // The first SubConn for the endpoint that goes READY when endpoint has no + // READY SubConns yet, cleared on that sc disconnecting (i.e. going out of + // READY). Represents what pick first will use as it's picked SubConn for + // this endpoint. + pickedSC balancer.SubConn + + // The following fields are accessed asynchronously and are protected by + // mu. Note that mu may not be held when calling into the stopORCAListener + // or when registering a new listener, as those calls require the ORCA + // producer mu which is held when calling the listener, and the listener + // holds mu. + mu sync.Mutex + weightVal float64 + nonEmptySince time.Time + lastUpdated time.Time + cfg *lbConfig +} + +func (w *endpointWeight) OnLoadReport(load *v3orcapb.OrcaLoadReport) { + if w.logger.V(2) { + w.logger.Infof("Received load report for subchannel %v: %v", w.pickedSC, load) + } + // Update weights of this endpoint according to the reported load. + utilization := load.ApplicationUtilization + if utilization == 0 { + utilization = load.CpuUtilization + } + if utilization == 0 || load.RpsFractional == 0 { + if w.logger.V(2) { + w.logger.Infof("Ignoring empty load report for subchannel %v", w.pickedSC) + } + return + } + + w.mu.Lock() + defer w.mu.Unlock() + + errorRate := load.Eps / load.RpsFractional + w.weightVal = load.RpsFractional / (utilization + errorRate*w.cfg.ErrorUtilizationPenalty) + if w.logger.V(2) { + w.logger.Infof("New weight for subchannel %v: %v", w.pickedSC, w.weightVal) + } + + w.lastUpdated = internal.TimeNow() + if w.nonEmptySince.Equal(time.Time{}) { + w.nonEmptySince = w.lastUpdated + } +} + +// updateConfig updates the parameters of the WRR policy and +// stops/starts/restarts the ORCA OOB listener. +func (w *endpointWeight) updateConfig(cfg *lbConfig) { + w.mu.Lock() + oldCfg := w.cfg + w.cfg = cfg + w.mu.Unlock() + + if cfg.EnableOOBLoadReport == oldCfg.EnableOOBLoadReport && + cfg.OOBReportingPeriod == oldCfg.OOBReportingPeriod { + // Load reporting wasn't enabled before or after, or load reporting was + // enabled before and after, and had the same period. (Note that with + // load reporting disabled, OOBReportingPeriod is always 0.) + return + } + // (Re)start the listener to use the new config's settings for OOB + // reporting. + w.updateORCAListener(cfg) +} + +func (w *endpointWeight) updateORCAListener(cfg *lbConfig) { + if w.stopORCAListener != nil { + w.stopORCAListener() + } + if !cfg.EnableOOBLoadReport { + w.stopORCAListener = nil + return + } + if w.pickedSC == nil { // No picked SC for this endpoint yet, nothing to listen on. + return + } + if w.logger.V(2) { + w.logger.Infof("Registering ORCA listener for %v with interval %v", w.pickedSC, cfg.OOBReportingPeriod) + } + opts := orca.OOBListenerOptions{ReportInterval: time.Duration(cfg.OOBReportingPeriod)} + w.stopORCAListener = orca.RegisterOOBListener(w.pickedSC, w, opts) +} + +// weight returns the current effective weight of the endpoint, taking into +// account the parameters. Returns 0 for blacked out or expired data, which +// will cause the backend weight to be treated as the mean of the weights of the +// other backends. If forScheduler is set to true, this function will emit +// metrics through the metrics registry. +func (w *endpointWeight) weight(now time.Time, weightExpirationPeriod, blackoutPeriod time.Duration, recordMetrics bool) (weight float64) { + w.mu.Lock() + defer w.mu.Unlock() + + if recordMetrics { + defer func() { + endpointWeightsMetric.Record(w.metricsRecorder, weight, w.target, w.locality, w.clusterName) + }() + } + + // The endpoint has not received a load report (i.e. just turned READY with + // no load report). + if w.lastUpdated.Equal(time.Time{}) { + endpointWeightNotYetUsableMetric.Record(w.metricsRecorder, 1, w.target, w.locality, w.clusterName) + return 0 + } + + // If the most recent update was longer ago than the expiration period, + // reset nonEmptySince so that we apply the blackout period again if we + // start getting data again in the future, and return 0. + if now.Sub(w.lastUpdated) >= weightExpirationPeriod { + if recordMetrics { + endpointWeightStaleMetric.Record(w.metricsRecorder, 1, w.target, w.locality, w.clusterName) + } + w.nonEmptySince = time.Time{} + return 0 + } + + // If we don't have at least blackoutPeriod worth of data, return 0. + if blackoutPeriod != 0 && (w.nonEmptySince.Equal(time.Time{}) || now.Sub(w.nonEmptySince) < blackoutPeriod) { + if recordMetrics { + endpointWeightNotYetUsableMetric.Record(w.metricsRecorder, 1, w.target, w.locality, w.clusterName) + } + return 0 + } + + return w.weightVal +} + +type backendServiceKey struct{} + +// SetBackendService stores the backendService on the resolver state so +// that it can be used later as a label in wrr metrics. +func SetBackendService(state resolver.State, backendService string) resolver.State { + state.Attributes = state.Attributes.WithValue(backendServiceKey{}, backendService) + return state +} + +// getBackendServiceFromState retrieves the cluster name stored as an attribute +// in the resolver state. +func backendServiceFromState(state resolver.State) string { + v := state.Attributes.Value(backendServiceKey{}) + name, _ := v.(string) + return name +} diff --git a/balancer/weightedroundrobin/balancer_test.go b/balancer/weightedroundrobin/balancer_test.go new file mode 100644 index 0000000..c455b9c --- /dev/null +++ b/balancer/weightedroundrobin/balancer_test.go @@ -0,0 +1,869 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedroundrobin_test + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/internal/testutils/stats" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + + wrr "google.golang.org/grpc/balancer/weightedroundrobin" + iwrr "google.golang.org/grpc/balancer/weightedroundrobin/internal" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const defaultTestTimeout = 10 * time.Second +const weightUpdatePeriod = 50 * time.Millisecond +const weightExpirationPeriod = time.Minute +const oobReportingInterval = 10 * time.Millisecond + +func init() { + iwrr.AllowAnyWeightUpdatePeriod = true +} + +func boolp(b bool) *bool { return &b } +func float64p(f float64) *float64 { return &f } +func stringp(s string) *string { return &s } + +var ( + perCallConfig = iwrr.LBConfig{ + EnableOOBLoadReport: boolp(false), + OOBReportingPeriod: stringp("0.005s"), + BlackoutPeriod: stringp("0s"), + WeightExpirationPeriod: stringp("60s"), + WeightUpdatePeriod: stringp(".050s"), + ErrorUtilizationPenalty: float64p(0), + } + oobConfig = iwrr.LBConfig{ + EnableOOBLoadReport: boolp(true), + OOBReportingPeriod: stringp("0.005s"), + BlackoutPeriod: stringp("0s"), + WeightExpirationPeriod: stringp("60s"), + WeightUpdatePeriod: stringp(".050s"), + ErrorUtilizationPenalty: float64p(0), + } + testMetricsConfig = iwrr.LBConfig{ + EnableOOBLoadReport: boolp(false), + OOBReportingPeriod: stringp("0.005s"), + BlackoutPeriod: stringp("0s"), + WeightExpirationPeriod: stringp("60s"), + WeightUpdatePeriod: stringp("30s"), + ErrorUtilizationPenalty: float64p(0), + } +) + +type testServer struct { + *stubserver.StubServer + + oobMetrics orca.ServerMetricsRecorder // Attached to the OOB stream. + callMetrics orca.CallMetricsRecorder // Attached to per-call metrics. +} + +type reportType int + +const ( + reportNone reportType = iota + reportOOB + reportCall + reportBoth +) + +func startServer(t *testing.T, r reportType) *testServer { + t.Helper() + + smr := orca.NewServerMetricsRecorder() + cmr := orca.NewServerMetricsRecorder().(orca.CallMetricsRecorder) + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + if r := orca.CallMetricsRecorderFromContext(ctx); r != nil { + // Copy metrics from what the test set in cmr into r. + sm := cmr.(orca.ServerMetricsProvider).ServerMetrics() + r.SetApplicationUtilization(sm.AppUtilization) + r.SetQPS(sm.QPS) + r.SetEPS(sm.EPS) + } + return &testpb.Empty{}, nil + }, + } + + var sopts []grpc.ServerOption + if r == reportCall || r == reportBoth { + sopts = append(sopts, orca.CallMetricsServerOption(nil)) + } + + if r == reportOOB || r == reportBoth { + oso := orca.ServiceOptions{ + ServerMetricsProvider: smr, + MinReportingInterval: 10 * time.Millisecond, + } + internal.ORCAAllowAnyMinReportingInterval.(func(so *orca.ServiceOptions))(&oso) + sopts = append(sopts, stubserver.RegisterServiceServerOption(func(s grpc.ServiceRegistrar) { + if err := orca.Register(s, oso); err != nil { + t.Fatalf("Failed to register orca service: %v", err) + } + })) + } + + if err := ss.StartServer(sopts...); err != nil { + t.Fatalf("Error starting server: %v", err) + } + t.Cleanup(ss.Stop) + + return &testServer{ + StubServer: ss, + oobMetrics: smr, + callMetrics: cmr, + } +} + +func svcConfig(t *testing.T, wrrCfg iwrr.LBConfig) string { + t.Helper() + m, err := json.Marshal(wrrCfg) + if err != nil { + t.Fatalf("Error marshaling JSON %v: %v", wrrCfg, err) + } + sc := fmt.Sprintf(`{"loadBalancingConfig": [ {%q:%v} ] }`, wrr.Name, string(m)) + t.Logf("Marshaled service config: %v", sc) + return sc +} + +// Tests basic functionality with one address. With only one address, load +// reporting doesn't affect routing at all. +func (s) TestBalancer_OneAddress(t *testing.T) { + testCases := []struct { + rt reportType + cfg iwrr.LBConfig + }{ + {rt: reportNone, cfg: perCallConfig}, + {rt: reportCall, cfg: perCallConfig}, + {rt: reportOOB, cfg: oobConfig}, + } + + for _, tc := range testCases { + t.Run(fmt.Sprintf("reportType:%v", tc.rt), func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv := startServer(t, tc.rt) + + sc := svcConfig(t, tc.cfg) + if err := srv.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + + // Perform many RPCs to ensure the LB policy works with 1 address. + for i := 0; i < 100; i++ { + srv.callMetrics.SetQPS(float64(i)) + srv.oobMetrics.SetQPS(float64(i)) + if _, err := srv.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("Error from EmptyCall: %v", err) + } + time.Sleep(time.Millisecond) // Delay; test will run 100ms and should perform ~10 weight updates + } + }) + } +} + +// TestWRRMetricsBasic tests metrics emitted from the WRR balancer. It +// configures a weighted round robin balancer as the top level balancer of a +// ClientConn, and configures a fake stats handler on the ClientConn to receive +// metrics. It verifies stats emitted from the Weighted Round Robin Balancer on +// balancer startup case which triggers the first picker and scheduler update +// before any load reports are received. +// +// Note that this test and others, metrics emission assertions are a snapshot +// of the most recently emitted metrics. This is due to the nondeterminism of +// scheduler updates with respect to test bodies, so the assertions made are +// from the most recently synced state of the system (picker/scheduler) from the +// test body. +func (s) TestWRRMetricsBasic(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv := startServer(t, reportCall) + sc := svcConfig(t, testMetricsConfig) + + tmr := stats.NewTestMetricsRecorder() + if err := srv.StartClient(grpc.WithDefaultServiceConfig(sc), grpc.WithStatsHandler(tmr)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + srv.callMetrics.SetQPS(float64(1)) + + if _, err := srv.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("Error from EmptyCall: %v", err) + } + + if got, _ := tmr.Metric("grpc.lb.wrr.rr_fallback"); got != 1 { + t.Fatalf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.wrr.rr_fallback", got, 1) + } + if got, _ := tmr.Metric("grpc.lb.wrr.endpoint_weight_stale"); got != 0 { + t.Fatalf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.wrr.endpoint_weight_stale", got, 0) + } + if got, _ := tmr.Metric("grpc.lb.wrr.endpoint_weight_not_yet_usable"); got != 1 { + t.Fatalf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.wrr.endpoint_weight_not_yet_usable", got, 1) + } + // Unusable, so no endpoint weight. Due to only one SubConn, this will never + // update the weight. Thus, this will stay 0. + if got, _ := tmr.Metric("grpc.lb.wrr.endpoint_weight_stale"); got != 0 { + t.Fatalf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.wrr.endpoint_weight_stale", got, 0) + } +} + +// Tests two addresses with ORCA reporting disabled (should fall back to pure +// RR). +func (s) TestBalancer_TwoAddresses_ReportingDisabled(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportNone) + srv2 := startServer(t, reportNone) + + sc := svcConfig(t, perCallConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Perform many RPCs to ensure the LB policy works with 2 addresses. + for i := 0; i < 20; i++ { + roundrobin.CheckRoundRobinRPCs(ctx, srv1.Client, addrs) + } +} + +// Tests two addresses with per-call ORCA reporting enabled. Checks the +// backends are called in the appropriate ratios. +func (s) TestBalancer_TwoAddresses_ReportingEnabledPerCall(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportCall) + srv2 := startServer(t, reportCall) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). + srv1.callMetrics.SetQPS(10.0) + srv1.callMetrics.SetApplicationUtilization(1.0) + + srv2.callMetrics.SetQPS(10.0) + srv2.callMetrics.SetApplicationUtilization(.1) + + sc := svcConfig(t, perCallConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) +} + +// Tests two addresses with OOB ORCA reporting enabled. Checks the backends +// are called in the appropriate ratios. +func (s) TestBalancer_TwoAddresses_ReportingEnabledOOB(t *testing.T) { + testCases := []struct { + name string + utilSetter func(orca.ServerMetricsRecorder, float64) + }{{ + name: "application_utilization", + utilSetter: func(smr orca.ServerMetricsRecorder, val float64) { + smr.SetApplicationUtilization(val) + }, + }, { + name: "cpu_utilization", + utilSetter: func(smr orca.ServerMetricsRecorder, val float64) { + smr.SetCPUUtilization(val) + }, + }, { + name: "application over cpu", + utilSetter: func(smr orca.ServerMetricsRecorder, val float64) { + smr.SetApplicationUtilization(val) + smr.SetCPUUtilization(2.0) // ignored because ApplicationUtilization is set + }, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportOOB) + srv2 := startServer(t, reportOOB) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). + srv1.oobMetrics.SetQPS(10.0) + tc.utilSetter(srv1.oobMetrics, 1.0) + + srv2.oobMetrics.SetQPS(10.0) + tc.utilSetter(srv2.oobMetrics, 0.1) + + sc := svcConfig(t, oobConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + }) + } +} + +// Tests two addresses with OOB ORCA reporting enabled, where the reports +// change over time. Checks the backends are called in the appropriate ratios +// before and after modifying the reports. +func (s) TestBalancer_TwoAddresses_UpdateLoads(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportOOB) + srv2 := startServer(t, reportOOB) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(1.0) + + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(.1) + + sc := svcConfig(t, oobConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + + // Update the loads so srv2 is loaded and srv1 is not; ensure RPCs are + // routed disproportionately to srv1. + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(.1) + + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(1.0) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod + oobReportingInterval) + checkWeights(ctx, t, srvWeight{srv1, 10}, srvWeight{srv2, 1}) +} + +// Tests two addresses with OOB ORCA reporting enabled, then with switching to +// per-call reporting. Checks the backends are called in the appropriate +// ratios before and after the change. +func (s) TestBalancer_TwoAddresses_OOBThenPerCall(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportBoth) + srv2 := startServer(t, reportBoth) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(1.0) + + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(.1) + + // For per-call metrics (not used initially), srv2 reports that it is + // loaded and srv1 reports low load. After confirming OOB works, switch to + // per-call and confirm the new routing weights are applied. + srv1.callMetrics.SetQPS(10.0) + srv1.callMetrics.SetApplicationUtilization(.1) + + srv2.callMetrics.SetQPS(10.0) + srv2.callMetrics.SetApplicationUtilization(1.0) + + sc := svcConfig(t, oobConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + + // Update to per-call weights. + c := svcConfig(t, perCallConfig) + parsedCfg := srv1.R.CC().ParseServiceConfig(c) + if parsedCfg.Err != nil { + panic(fmt.Sprintf("Error parsing config %q: %v", c, parsedCfg.Err)) + } + srv1.R.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: parsedCfg}) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 10}, srvWeight{srv2, 1}) +} + +// TestEndpoints_SharedAddress tests the case where two endpoints have the same +// address. The expected behavior is undefined, however the program should not +// crash. +func (s) TestEndpoints_SharedAddress(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv := startServer(t, reportCall) + sc := svcConfig(t, perCallConfig) + if err := srv.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + + endpointsSharedAddress := []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: srv.Address}}}, {Addresses: []resolver.Address{{Addr: srv.Address}}}} + srv.R.UpdateState(resolver.State{Endpoints: endpointsSharedAddress}) + + // Make some RPC's and make sure doesn't crash. It should go to one of the + // endpoints addresses, it's undefined which one it will choose and the load + // reporting might not work, but it should be able to make an RPC. + for i := 0; i < 10; i++ { + if _, err := srv.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall failed with err: %v", err) + } + } +} + +// TestEndpoints_MultipleAddresses tests WRR on endpoints with numerous +// addresses. It configures WRR with two endpoints with one bad address followed +// by a good address. It configures two backends that each report per call +// metrics, each corresponding to the two endpoints good address. It then +// asserts load is distributed as expected corresponding to the call metrics +// received. +func (s) TestEndpoints_MultipleAddresses(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + srv1 := startServer(t, reportCall) + srv2 := startServer(t, reportCall) + + srv1.callMetrics.SetQPS(10.0) + srv1.callMetrics.SetApplicationUtilization(.1) + + srv2.callMetrics.SetQPS(10.0) + srv2.callMetrics.SetApplicationUtilization(1.0) + + sc := svcConfig(t, perCallConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + + twoEndpoints := []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "bad-address-1"}, {Addr: srv1.Address}}}, {Addresses: []resolver.Address{{Addr: "bad-address-2"}, {Addr: srv2.Address}}}} + srv1.R.UpdateState(resolver.State{Endpoints: twoEndpoints}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 10}, srvWeight{srv2, 1}) +} + +// Tests two addresses with OOB ORCA reporting enabled and a non-zero error +// penalty applied. +func (s) TestBalancer_TwoAddresses_ErrorPenalty(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportOOB) + srv2 := startServer(t, reportOOB) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). EPS values are set (but ignored + // initially due to ErrorUtilizationPenalty=0). Later EUP will be updated + // to 0.9 which will cause the weights to be equal and RPCs to be routed + // 50/50. + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(1.0) + srv1.oobMetrics.SetEPS(0) + // srv1 weight before: 10.0 / 1.0 = 10.0 + // srv1 weight after: 10.0 / 1.0 = 10.0 + + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(.1) + srv2.oobMetrics.SetEPS(10.0) + // srv2 weight before: 10.0 / 0.1 = 100.0 + // srv2 weight after: 10.0 / 1.0 = 10.0 + + sc := svcConfig(t, oobConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + + // Update to include an error penalty in the weights. + newCfg := oobConfig + newCfg.ErrorUtilizationPenalty = float64p(0.9) + c := svcConfig(t, newCfg) + parsedCfg := srv1.R.CC().ParseServiceConfig(c) + if parsedCfg.Err != nil { + panic(fmt.Sprintf("Error parsing config %q: %v", c, parsedCfg.Err)) + } + srv1.R.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: parsedCfg}) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod + oobReportingInterval) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 1}) +} + +// Tests that the blackout period causes backends to use 0 as their weight +// (meaning to use the average weight) until the blackout period elapses. +func (s) TestBalancer_TwoAddresses_BlackoutPeriod(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + var mu sync.Mutex + start := time.Now() + now := start + setNow := func(t time.Time) { + mu.Lock() + defer mu.Unlock() + now = t + } + + setTimeNow(func() time.Time { + mu.Lock() + defer mu.Unlock() + return now + }) + t.Cleanup(func() { setTimeNow(time.Now) }) + + testCases := []struct { + blackoutPeriodCfg *string + blackoutPeriod time.Duration + }{{ + blackoutPeriodCfg: stringp("1s"), + blackoutPeriod: time.Second, + }, { + blackoutPeriodCfg: nil, + blackoutPeriod: 10 * time.Second, // the default + }} + for _, tc := range testCases { + setNow(start) + srv1 := startServer(t, reportOOB) + srv2 := startServer(t, reportOOB) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(1.0) + + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(.1) + + cfg := oobConfig + cfg.BlackoutPeriod = tc.blackoutPeriodCfg + sc := svcConfig(t, cfg) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + // During the blackout period (1s) we should route roughly 50/50. + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 1}) + + // Advance time to right before the blackout period ends and the weights + // should still be zero. + setNow(start.Add(tc.blackoutPeriod - time.Nanosecond)) + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 1}) + + // Advance time to right after the blackout period ends and the weights + // should now activate. + setNow(start.Add(tc.blackoutPeriod)) + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + } +} + +// Tests that the weight expiration period causes backends to use 0 as their +// weight (meaning to use the average weight) once the expiration period +// elapses. +func (s) TestBalancer_TwoAddresses_WeightExpiration(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + var mu sync.Mutex + start := time.Now() + now := start + setNow := func(t time.Time) { + mu.Lock() + defer mu.Unlock() + now = t + } + setTimeNow(func() time.Time { + mu.Lock() + defer mu.Unlock() + return now + }) + t.Cleanup(func() { setTimeNow(time.Now) }) + + srv1 := startServer(t, reportBoth) + srv2 := startServer(t, reportBoth) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). Because the OOB reporting interval + // is 1 minute but the weights expire in 1 second, routing will go to 50/50 + // after the weights expire. + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(1.0) + + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(.1) + + cfg := oobConfig + cfg.OOBReportingPeriod = stringp("60s") + sc := svcConfig(t, cfg) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + + // Advance what time.Now returns to the weight expiration time minus 1s to + // ensure all weights are still honored. + setNow(start.Add(weightExpirationPeriod - time.Second)) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + + // Advance what time.Now returns to the weight expiration time plus 1s to + // ensure all weights expired and addresses are routed evenly. + setNow(start.Add(weightExpirationPeriod + time.Second)) + + // Wait for the weight expiration period so the weights have expired. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 1}) +} + +// Tests logic surrounding subchannel management. +func (s) TestBalancer_AddressesChanging(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportBoth) + srv2 := startServer(t, reportBoth) + srv3 := startServer(t, reportBoth) + srv4 := startServer(t, reportBoth) + + // srv1: weight 10 + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(1.0) + // srv2: weight 100 + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(.1) + // srv3: weight 20 + srv3.oobMetrics.SetQPS(20.0) + srv3.oobMetrics.SetApplicationUtilization(1.0) + // srv4: weight 200 + srv4.oobMetrics.SetQPS(20.0) + srv4.oobMetrics.SetApplicationUtilization(.1) + + sc := svcConfig(t, oobConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + srv2.Client = srv1.Client + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}, {Addr: srv3.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 3) + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}, srvWeight{srv3, 2}) + + // Add backend 4 + addrs = append(addrs, resolver.Address{Addr: srv4.Address}) + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}, srvWeight{srv3, 2}, srvWeight{srv4, 20}) + + // Shutdown backend 3. RPCs will no longer be routed to it. + srv3.Stop() + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}, srvWeight{srv4, 20}) + + // Remove addresses 2 and 3. RPCs will no longer be routed to 2 either. + addrs = []resolver.Address{{Addr: srv1.Address}, {Addr: srv4.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv4, 20}) + + // Re-add 2 and remove the rest. + addrs = []resolver.Address{{Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv2, 10}) + + // Re-add 4. + addrs = append(addrs, resolver.Address{Addr: srv4.Address}) + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv2, 10}, srvWeight{srv4, 20}) +} + +func ensureReached(ctx context.Context, t *testing.T, c testgrpc.TestServiceClient, n int) { + t.Helper() + reached := make(map[string]struct{}) + for len(reached) != n { + var peer peer.Peer + if _, err := c.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + t.Fatalf("Error from EmptyCall: %v", err) + } + reached[peer.Addr.String()] = struct{}{} + } +} + +type srvWeight struct { + srv *testServer + w int +} + +const rrIterations = 100 + +// checkWeights does rrIterations RPCs and expects the different backends to be +// routed in a ratio as determined by the srvWeights passed in. Allows for +// some variance (+/- 2 RPCs per backend). +func checkWeights(ctx context.Context, t *testing.T, sws ...srvWeight) { + t.Helper() + + c := sws[0].srv.Client + + // Replace the weights with approximate counts of RPCs wanted given the + // iterations performed. + weightSum := 0 + for _, sw := range sws { + weightSum += sw.w + } + for i := range sws { + sws[i].w = rrIterations * sws[i].w / weightSum + } + + for attempts := 0; attempts < 10; attempts++ { + serverCounts := make(map[string]int) + for i := 0; i < rrIterations; i++ { + var peer peer.Peer + if _, err := c.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + t.Fatalf("Error from EmptyCall: %v; timed out waiting for weighted RR behavior?", err) + } + serverCounts[peer.Addr.String()]++ + } + if len(serverCounts) != len(sws) { + continue + } + success := true + for _, sw := range sws { + c := serverCounts[sw.srv.Address] + if c < sw.w-2 || c > sw.w+2 { + success = false + break + } + } + if success { + t.Logf("Passed iteration %v; counts: %v", attempts, serverCounts) + return + } + t.Logf("Failed iteration %v; counts: %v; want %+v", attempts, serverCounts, sws) + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("Failed to route RPCs with proper ratio") +} + +func init() { + setTimeNow(time.Now) + iwrr.TimeNow = timeNow +} + +var timeNowFunc atomic.Value // func() time.Time + +func timeNow() time.Time { + return timeNowFunc.Load().(func() time.Time)() +} + +func setTimeNow(f func() time.Time) { + timeNowFunc.Store(f) +} diff --git a/balancer/weightedroundrobin/config.go b/balancer/weightedroundrobin/config.go new file mode 100644 index 0000000..38f89d3 --- /dev/null +++ b/balancer/weightedroundrobin/config.go @@ -0,0 +1,59 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedroundrobin + +import ( + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +type lbConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + // Whether to enable out-of-band utilization reporting collection from the + // endpoints. By default, per-request utilization reporting is used. + EnableOOBLoadReport bool `json:"enableOobLoadReport,omitempty"` + + // Load reporting interval to request from the server. Note that the + // server may not provide reports as frequently as the client requests. + // Used only when enable_oob_load_report is true. Default is 10 seconds. + OOBReportingPeriod iserviceconfig.Duration `json:"oobReportingPeriod,omitempty"` + + // A given endpoint must report load metrics continuously for at least this + // long before the endpoint weight will be used. This avoids churn when + // the set of endpoint addresses changes. Takes effect both immediately + // after we establish a connection to an endpoint and after + // weight_expiration_period has caused us to stop using the most recent + // load metrics. Default is 10 seconds. + BlackoutPeriod iserviceconfig.Duration `json:"blackoutPeriod,omitempty"` + + // If a given endpoint has not reported load metrics in this long, + // then we stop using the reported weight. This ensures that we do + // not continue to use very stale weights. Once we stop using a stale + // value, if we later start seeing fresh reports again, the + // blackout_period applies. Defaults to 3 minutes. + WeightExpirationPeriod iserviceconfig.Duration `json:"weightExpirationPeriod,omitempty"` + + // How often endpoint weights are recalculated. Default is 1 second. + WeightUpdatePeriod iserviceconfig.Duration `json:"weightUpdatePeriod,omitempty"` + + // The multiplier used to adjust endpoint weights with the error rate + // calculated as eps/qps. Default is 1.0. + ErrorUtilizationPenalty float64 `json:"errorUtilizationPenalty,omitempty"` +} diff --git a/balancer/weightedroundrobin/internal/internal.go b/balancer/weightedroundrobin/internal/internal.go new file mode 100644 index 0000000..7b64fbf --- /dev/null +++ b/balancer/weightedroundrobin/internal/internal.go @@ -0,0 +1,44 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal allows for easier testing of the weightedroundrobin +// package. +package internal + +import ( + "time" +) + +// AllowAnyWeightUpdatePeriod permits any setting of WeightUpdatePeriod for +// testing. Normally a minimum of 100ms is applied. +var AllowAnyWeightUpdatePeriod bool + +// LBConfig allows tests to produce a JSON form of the config from the struct +// instead of using a string. +type LBConfig struct { + EnableOOBLoadReport *bool `json:"enableOobLoadReport,omitempty"` + OOBReportingPeriod *string `json:"oobReportingPeriod,omitempty"` + BlackoutPeriod *string `json:"blackoutPeriod,omitempty"` + WeightExpirationPeriod *string `json:"weightExpirationPeriod,omitempty"` + WeightUpdatePeriod *string `json:"weightUpdatePeriod,omitempty"` + ErrorUtilizationPenalty *float64 `json:"errorUtilizationPenalty,omitempty"` +} + +// TimeNow can be overridden by tests to return a different value for the +// current iserviceconfig. +var TimeNow = time.Now diff --git a/balancer/weightedroundrobin/logging.go b/balancer/weightedroundrobin/logging.go new file mode 100644 index 0000000..43184ca --- /dev/null +++ b/balancer/weightedroundrobin/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedroundrobin + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[%p] " + +var logger = grpclog.Component("weighted-round-robin") + +func prefixLogger(p *wrrBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/balancer/weightedroundrobin/metrics_test.go b/balancer/weightedroundrobin/metrics_test.go new file mode 100644 index 0000000..79e4d0a --- /dev/null +++ b/balancer/weightedroundrobin/metrics_test.go @@ -0,0 +1,173 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedroundrobin + +import ( + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils/stats" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestWRR_Metrics_SubConnWeight tests different scenarios for the weight call +// on a weighted SubConn, and expects certain metrics for each of these +// scenarios. +func (s) TestWRR_Metrics_SubConnWeight(t *testing.T) { + tests := []struct { + name string + weightExpirationPeriod time.Duration + blackoutPeriod time.Duration + lastUpdated time.Time + nonEmpty time.Time + nowTime time.Time + endpointWeightStaleWant float64 + endpointWeightNotYetUsableWant float64 + endpointWeightWant float64 + }{ + // The weighted SubConn's lastUpdated field hasn't been set, so this + // SubConn's weight is not yet usable. Thus, should emit that endpoint + // weight is not yet usable, and 0 for weight. + { + name: "no weight set", + weightExpirationPeriod: time.Second, + blackoutPeriod: time.Second, + nowTime: time.Now(), + endpointWeightStaleWant: 0, + endpointWeightNotYetUsableWant: 1, + endpointWeightWant: 0, + }, + { + name: "weight expiration", + lastUpdated: time.Now(), + weightExpirationPeriod: 2 * time.Second, + blackoutPeriod: time.Second, + nowTime: time.Now().Add(100 * time.Second), + endpointWeightStaleWant: 1, + endpointWeightNotYetUsableWant: 0, + endpointWeightWant: 0, + }, + { + name: "in blackout period", + lastUpdated: time.Now(), + weightExpirationPeriod: time.Minute, + blackoutPeriod: 10 * time.Second, + nowTime: time.Now(), + endpointWeightStaleWant: 0, + endpointWeightNotYetUsableWant: 1, + endpointWeightWant: 0, + }, + { + name: "normal weight", + lastUpdated: time.Now(), + nonEmpty: time.Now(), + weightExpirationPeriod: time.Minute, + blackoutPeriod: time.Second, + nowTime: time.Now().Add(10 * time.Second), + endpointWeightStaleWant: 0, + endpointWeightNotYetUsableWant: 0, + endpointWeightWant: 3, + }, + { + name: "weight expiration takes precdedence over blackout", + lastUpdated: time.Now(), + nonEmpty: time.Now(), + weightExpirationPeriod: time.Second, + blackoutPeriod: time.Minute, + nowTime: time.Now().Add(10 * time.Second), + endpointWeightStaleWant: 1, + endpointWeightNotYetUsableWant: 0, + endpointWeightWant: 0, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tmr := stats.NewTestMetricsRecorder() + wsc := &endpointWeight{ + metricsRecorder: tmr, + weightVal: 3, + lastUpdated: test.lastUpdated, + nonEmptySince: test.nonEmpty, + } + wsc.weight(test.nowTime, test.weightExpirationPeriod, test.blackoutPeriod, true) + + if got, _ := tmr.Metric("grpc.lb.wrr.endpoint_weight_stale"); got != test.endpointWeightStaleWant { + t.Fatalf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.wrr.endpoint_weight_stale", got, test.endpointWeightStaleWant) + } + if got, _ := tmr.Metric("grpc.lb.wrr.endpoint_weight_not_yet_usable"); got != test.endpointWeightNotYetUsableWant { + t.Fatalf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.wrr.endpoint_weight_not_yet_usable", got, test.endpointWeightNotYetUsableWant) + } + if got, _ := tmr.Metric("grpc.lb.wrr.endpoint_weight_stale"); got != test.endpointWeightStaleWant { + t.Fatalf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.wrr.endpoint_weight_stale", got, test.endpointWeightStaleWant) + } + }) + } + +} + +// TestWRR_Metrics_Scheduler_RR_Fallback tests the round robin fallback metric +// for scheduler updates. It tests the case with one SubConn, and two SubConns +// with no weights. Both of these should emit a count metric for round robin +// fallback. +func (s) TestWRR_Metrics_Scheduler_RR_Fallback(t *testing.T) { + tmr := stats.NewTestMetricsRecorder() + ew := &endpointWeight{ + metricsRecorder: tmr, + weightVal: 0, + } + + p := &picker{ + cfg: &lbConfig{ + BlackoutPeriod: iserviceconfig.Duration(10 * time.Second), + WeightExpirationPeriod: iserviceconfig.Duration(3 * time.Minute), + }, + weightedPickers: []pickerWeightedEndpoint{{weightedEndpoint: ew}}, + metricsRecorder: tmr, + } + // There is only one SubConn, so no matter if the SubConn has a weight or + // not will fallback to round robin. + p.regenerateScheduler() + if got, _ := tmr.Metric("grpc.lb.wrr.rr_fallback"); got != 1 { + t.Fatalf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.wrr.rr_fallback", got, 1) + } + tmr.ClearMetrics() + + // With two SubConns, if neither of them have weights, it will also fallback + // to round robin. + ew2 := &endpointWeight{ + target: "target", + metricsRecorder: tmr, + weightVal: 0, + } + p.weightedPickers = append(p.weightedPickers, pickerWeightedEndpoint{weightedEndpoint: ew2}) + p.regenerateScheduler() + if got, _ := tmr.Metric("grpc.lb.wrr.rr_fallback"); got != 1 { + t.Fatalf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.wrr.rr_fallback", got, 1) + } +} diff --git a/balancer/weightedroundrobin/scheduler.go b/balancer/weightedroundrobin/scheduler.go new file mode 100644 index 0000000..8ed8d92 --- /dev/null +++ b/balancer/weightedroundrobin/scheduler.go @@ -0,0 +1,146 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedroundrobin + +import ( + "math" +) + +type scheduler interface { + nextIndex() int +} + +// newScheduler uses scWeights to create a new scheduler for selecting endpoints +// in a picker. It will return a round robin implementation if at least +// len(scWeights)-1 are zero or there is only a single endpoint, otherwise it +// will return an Earliest Deadline First (EDF) scheduler implementation that +// selects the endpoints according to their weights. +func (p *picker) newScheduler(recordMetrics bool) scheduler { + epWeights := p.endpointWeights(recordMetrics) + n := len(epWeights) + if n == 0 { + return nil + } + if n == 1 { + if recordMetrics { + rrFallbackMetric.Record(p.metricsRecorder, 1, p.target, p.locality, p.clusterName) + } + return &rrScheduler{numSCs: 1, inc: p.inc} + } + sum := float64(0) + numZero := 0 + max := float64(0) + for _, w := range epWeights { + sum += w + if w > max { + max = w + } + if w == 0 { + numZero++ + } + } + + if numZero >= n-1 { + if recordMetrics { + rrFallbackMetric.Record(p.metricsRecorder, 1, p.target, p.locality, p.clusterName) + } + return &rrScheduler{numSCs: uint32(n), inc: p.inc} + } + unscaledMean := sum / float64(n-numZero) + scalingFactor := maxWeight / max + mean := uint16(math.Round(scalingFactor * unscaledMean)) + + weights := make([]uint16, n) + allEqual := true + for i, w := range epWeights { + if w == 0 { + // Backends with weight = 0 use the mean. + weights[i] = mean + } else { + scaledWeight := uint16(math.Round(scalingFactor * w)) + weights[i] = scaledWeight + if scaledWeight != mean { + allEqual = false + } + } + } + + if allEqual { + return &rrScheduler{numSCs: uint32(n), inc: p.inc} + } + + logger.Infof("using edf scheduler with weights: %v", weights) + return &edfScheduler{weights: weights, inc: p.inc} +} + +const maxWeight = math.MaxUint16 + +// edfScheduler implements EDF using the same algorithm as grpc-c++ here: +// +// https://github.com/grpc/grpc/blob/master/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/static_stride_scheduler.cc +type edfScheduler struct { + inc func() uint32 + weights []uint16 +} + +// Returns the index in s.weights for the picker to choose. +func (s *edfScheduler) nextIndex() int { + const offset = maxWeight / 2 + + for { + idx := uint64(s.inc()) + + // The sequence number (idx) is split in two: the lower %n gives the + // index of the backend, and the rest gives the number of times we've + // iterated through all backends. `generation` is used to + // deterministically decide whether we pick or skip the backend on this + // iteration, in proportion to the backend's weight. + + backendIndex := idx % uint64(len(s.weights)) + generation := idx / uint64(len(s.weights)) + weight := uint64(s.weights[backendIndex]) + + // We pick a backend `weight` times per `maxWeight` generations. The + // multiply and modulus ~evenly spread out the picks for a given + // backend between different generations. The offset by `backendIndex` + // helps to reduce the chance of multiple consecutive non-picks: if we + // have two consecutive backends with an equal, say, 80% weight of the + // max, with no offset we would see 1/5 generations that skipped both. + // TODO(b/190488683): add test for offset efficacy. + mod := uint64(weight*generation+backendIndex*offset) % maxWeight + + if mod < maxWeight-weight { + continue + } + return int(backendIndex) + } +} + +// A simple RR scheduler to use for fallback when fewer than two backends have +// non-zero weights, or all backends have the same weight, or when only one +// subconn exists. +type rrScheduler struct { + inc func() uint32 + numSCs uint32 +} + +func (s *rrScheduler) nextIndex() int { + idx := s.inc() + return int(idx % s.numSCs) +} diff --git a/balancer/weightedtarget/logging.go b/balancer/weightedtarget/logging.go new file mode 100644 index 0000000..e406139 --- /dev/null +++ b/balancer/weightedtarget/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedtarget + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[weighted-target-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *weightedTargetBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/balancer/weightedtarget/weightedaggregator/aggregator.go b/balancer/weightedtarget/weightedaggregator/aggregator.go new file mode 100644 index 0000000..fc929a2 --- /dev/null +++ b/balancer/weightedtarget/weightedaggregator/aggregator.go @@ -0,0 +1,318 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package weightedaggregator implements state aggregator for weighted_target +// balancer. +// +// This is a separate package so it can be shared by weighted_target and eds. +// The eds balancer will be refactored to use weighted_target directly. After +// that, all functions and structs in this package can be moved to package +// weightedtarget and unexported. +package weightedaggregator + +import ( + "errors" + "fmt" + "sync" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/wrr" +) + +type weightedPickerState struct { + weight uint32 + state balancer.State + // stateToAggregate is the connectivity state used only for state + // aggregation. It could be different from state.ConnectivityState. For + // example when a sub-balancer transitions from TransientFailure to + // connecting, state.ConnectivityState is Connecting, but stateToAggregate + // is still TransientFailure. + stateToAggregate connectivity.State +} + +func (s *weightedPickerState) String() string { + return fmt.Sprintf("weight:%v,picker:%p,state:%v,stateToAggregate:%v", s.weight, s.state.Picker, s.state.ConnectivityState, s.stateToAggregate) +} + +// Aggregator is the weighted balancer state aggregator. +type Aggregator struct { + cc balancer.ClientConn + logger *grpclog.PrefixLogger + newWRR func() wrr.WRR + + csEvltr *balancer.ConnectivityStateEvaluator + + mu sync.Mutex + // If started is false, no updates should be sent to the parent cc. A closed + // sub-balancer could still send pickers to this aggregator. This makes sure + // that no updates will be forwarded to parent when the whole balancer group + // and states aggregator is closed. + started bool + // All balancer IDs exist as keys in this map, even if balancer group is not + // started. + // + // If an ID is not in map, it's either removed or never added. + idToPickerState map[string]*weightedPickerState + // Set when UpdateState call propagation is paused. + pauseUpdateState bool + // Set when UpdateState call propagation is paused and an UpdateState call + // is suppressed. + needUpdateStateOnResume bool +} + +// New creates a new weighted balancer state aggregator. +func New(cc balancer.ClientConn, logger *grpclog.PrefixLogger, newWRR func() wrr.WRR) *Aggregator { + return &Aggregator{ + cc: cc, + logger: logger, + newWRR: newWRR, + csEvltr: &balancer.ConnectivityStateEvaluator{}, + idToPickerState: make(map[string]*weightedPickerState), + } +} + +// Start starts the aggregator. It can be called after Stop to restart the +// aggregator. +func (wbsa *Aggregator) Start() { + wbsa.mu.Lock() + defer wbsa.mu.Unlock() + wbsa.started = true +} + +// Stop stops the aggregator. When the aggregator is stopped, it won't call +// parent ClientConn to update balancer state. +func (wbsa *Aggregator) Stop() { + wbsa.mu.Lock() + defer wbsa.mu.Unlock() + wbsa.started = false + wbsa.clearStates() +} + +// Add adds a sub-balancer state with weight. It adds a place holder, and waits for +// the real sub-balancer to update state. +func (wbsa *Aggregator) Add(id string, weight uint32) { + wbsa.mu.Lock() + defer wbsa.mu.Unlock() + wbsa.idToPickerState[id] = &weightedPickerState{ + weight: weight, + // Start everything in CONNECTING, so if one of the sub-balancers + // reports TransientFailure, the RPCs will still wait for the other + // sub-balancers. + state: balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + }, + stateToAggregate: connectivity.Connecting, + } + wbsa.csEvltr.RecordTransition(connectivity.Shutdown, connectivity.Connecting) + + wbsa.buildAndUpdateLocked() +} + +// Remove removes the sub-balancer state. Future updates from this sub-balancer, +// if any, will be ignored. +func (wbsa *Aggregator) Remove(id string) { + wbsa.mu.Lock() + defer wbsa.mu.Unlock() + if _, ok := wbsa.idToPickerState[id]; !ok { + return + } + // Setting the state of the deleted sub-balancer to Shutdown will get csEvltr + // to remove the previous state for any aggregated state evaluations. + // transitions to and from connectivity.Shutdown are ignored by csEvltr. + wbsa.csEvltr.RecordTransition(wbsa.idToPickerState[id].stateToAggregate, connectivity.Shutdown) + // Remove id and picker from picker map. This also results in future updates + // for this ID to be ignored. + delete(wbsa.idToPickerState, id) + wbsa.buildAndUpdateLocked() +} + +// UpdateWeight updates the weight for the given id. Note that this doesn't +// trigger an update to the parent ClientConn. The caller should decide when +// it's necessary, and call BuildAndUpdate. +func (wbsa *Aggregator) UpdateWeight(id string, newWeight uint32) { + wbsa.mu.Lock() + defer wbsa.mu.Unlock() + pState, ok := wbsa.idToPickerState[id] + if !ok { + return + } + pState.weight = newWeight +} + +// PauseStateUpdates causes UpdateState calls to not propagate to the parent +// ClientConn. The last state will be remembered and propagated when +// ResumeStateUpdates is called. +func (wbsa *Aggregator) PauseStateUpdates() { + wbsa.mu.Lock() + defer wbsa.mu.Unlock() + wbsa.pauseUpdateState = true + wbsa.needUpdateStateOnResume = false +} + +// ResumeStateUpdates will resume propagating UpdateState calls to the parent, +// and call UpdateState on the parent if any UpdateState call was suppressed. +func (wbsa *Aggregator) ResumeStateUpdates() { + wbsa.mu.Lock() + defer wbsa.mu.Unlock() + wbsa.pauseUpdateState = false + if wbsa.needUpdateStateOnResume { + wbsa.cc.UpdateState(wbsa.build()) + } +} + +// NeedUpdateStateOnResume sets the UpdateStateOnResume bool to true, letting a +// picker update be sent once ResumeStateUpdates is called. +func (wbsa *Aggregator) NeedUpdateStateOnResume() { + wbsa.mu.Lock() + defer wbsa.mu.Unlock() + wbsa.needUpdateStateOnResume = true +} + +// UpdateState is called to report a balancer state change from sub-balancer. +// It's usually called by the balancer group. +// +// It calls parent ClientConn's UpdateState with the new aggregated state. +func (wbsa *Aggregator) UpdateState(id string, newState balancer.State) { + wbsa.mu.Lock() + defer wbsa.mu.Unlock() + state, ok := wbsa.idToPickerState[id] + if !ok { + // All state starts with an entry in pickStateMap. If ID is not in map, + // it's either removed, or never existed. + return + } + + if !(state.state.ConnectivityState == connectivity.TransientFailure && newState.ConnectivityState == connectivity.Connecting) { + // If old state is TransientFailure, and new state is Connecting, don't + // update the state, to prevent the aggregated state from being always + // CONNECTING. Otherwise, stateToAggregate is the same as + // state.ConnectivityState. + wbsa.csEvltr.RecordTransition(state.stateToAggregate, newState.ConnectivityState) + state.stateToAggregate = newState.ConnectivityState + } + state.state = newState + + wbsa.buildAndUpdateLocked() +} + +// clearState Reset everything to init state (Connecting) but keep the entry in +// map (to keep the weight). +// +// Caller must hold wbsa.mu. +func (wbsa *Aggregator) clearStates() { + for _, pState := range wbsa.idToPickerState { + pState.state = balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + } + pState.stateToAggregate = connectivity.Connecting + } +} + +// buildAndUpdateLocked aggregates the connectivity states of the sub-balancers, +// builds a new picker and sends an update to the parent ClientConn. +// +// Caller must hold wbsa.mu. +func (wbsa *Aggregator) buildAndUpdateLocked() { + if !wbsa.started { + return + } + if wbsa.pauseUpdateState { + // If updates are paused, do not call UpdateState, but remember that we + // need to call it when they are resumed. + wbsa.needUpdateStateOnResume = true + return + } + + wbsa.cc.UpdateState(wbsa.build()) +} + +// build combines sub-states into one. +// +// Caller must hold wbsa.mu. +func (wbsa *Aggregator) build() balancer.State { + wbsa.logger.Infof("Child pickers with config: %+v", wbsa.idToPickerState) + + if len(wbsa.idToPickerState) == 0 { + // This is the case when all sub-balancers are removed. + return balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(errors.New("weighted-target: no targets to pick from")), + } + } + + // Make sure picker's return error is consistent with the aggregatedState. + pickers := make([]weightedPickerState, 0, len(wbsa.idToPickerState)) + + switch aggState := wbsa.csEvltr.CurrentState(); aggState { + case connectivity.Connecting: + return balancer.State{ + ConnectivityState: aggState, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable)} + case connectivity.TransientFailure: + // this means that all sub-balancers are now in TransientFailure. + for _, ps := range wbsa.idToPickerState { + pickers = append(pickers, *ps) + } + return balancer.State{ + ConnectivityState: aggState, + Picker: newWeightedPickerGroup(pickers, wbsa.newWRR)} + default: + for _, ps := range wbsa.idToPickerState { + if ps.stateToAggregate == connectivity.Ready { + pickers = append(pickers, *ps) + } + } + return balancer.State{ + ConnectivityState: aggState, + Picker: newWeightedPickerGroup(pickers, wbsa.newWRR)} + } + +} + +type weightedPickerGroup struct { + w wrr.WRR +} + +// newWeightedPickerGroup takes pickers with weights, and groups them into one +// picker. +// +// Note it only takes ready pickers. The map shouldn't contain non-ready +// pickers. +func newWeightedPickerGroup(readyWeightedPickers []weightedPickerState, newWRR func() wrr.WRR) *weightedPickerGroup { + w := newWRR() + for _, ps := range readyWeightedPickers { + w.Add(ps.state.Picker, int64(ps.weight)) + } + + return &weightedPickerGroup{ + w: w, + } +} + +func (pg *weightedPickerGroup) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + p, ok := pg.w.Next().(balancer.Picker) + if !ok { + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + } + return p.Pick(info) +} diff --git a/balancer/weightedtarget/weightedtarget.go b/balancer/weightedtarget/weightedtarget.go new file mode 100644 index 0000000..02230d1 --- /dev/null +++ b/balancer/weightedtarget/weightedtarget.go @@ -0,0 +1,196 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package weightedtarget implements the weighted_target balancer. +// +// All APIs in this package are experimental. +package weightedtarget + +import ( + "encoding/json" + "fmt" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/weightedtarget/weightedaggregator" + "google.golang.org/grpc/internal/balancergroup" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/hierarchy" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/wrr" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Name is the name of the weighted_target balancer. +const Name = "weighted_target_experimental" + +// NewRandomWRR is the WRR constructor used to pick sub-pickers from +// sub-balancers. It's to be modified in tests. +var NewRandomWRR = wrr.NewRandom + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &weightedTargetBalancer{} + b.logger = prefixLogger(b) + b.stateAggregator = weightedaggregator.New(cc, b.logger, NewRandomWRR) + b.stateAggregator.Start() + b.bg = balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: bOpts, + StateAggregator: b.stateAggregator, + Logger: b.logger, + SubBalancerCloseTimeout: time.Duration(0), // Disable caching of removed child policies + }) + b.logger.Infof("Created") + return b +} + +func (bb) Name() string { + return Name +} + +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(c) +} + +type weightedTargetBalancer struct { + logger *grpclog.PrefixLogger + + bg *balancergroup.BalancerGroup + stateAggregator *weightedaggregator.Aggregator + + targets map[string]Target +} + +type localityKeyType string + +const localityKey = localityKeyType("locality") + +// LocalityFromResolverState returns the locality from the resolver.State +// provided, or an empty string if not present. +func LocalityFromResolverState(state resolver.State) string { + locality, _ := state.Attributes.Value(localityKey).(string) + return locality +} + +// UpdateClientConnState takes the new targets in balancer group, +// creates/deletes sub-balancers and sends them update. addresses are split into +// groups based on hierarchy path. +func (b *weightedTargetBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + if b.logger.V(2) { + b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig)) + } + + newConfig, ok := s.BalancerConfig.(*LBConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) + } + endpointsSplit := hierarchy.Group(s.ResolverState.Endpoints) + + b.stateAggregator.PauseStateUpdates() + defer b.stateAggregator.ResumeStateUpdates() + + // Remove sub-pickers and sub-balancers that are not in the new config. + for name := range b.targets { + if _, ok := newConfig.Targets[name]; !ok { + b.stateAggregator.Remove(name) + b.bg.Remove(name) + } + } + + // For sub-balancers in the new config + // - if it's new. add to balancer group, + // - if it's old, but has a new weight, update weight in balancer group. + // + // For all sub-balancers, forward the address/balancer config update. + for name, newT := range newConfig.Targets { + oldT, ok := b.targets[name] + if !ok { + // If this is a new sub-balancer, add weights to the picker map. + b.stateAggregator.Add(name, newT.Weight) + // Then add to the balancer group. + b.bg.Add(name, balancer.Get(newT.ChildPolicy.Name)) + // Not trigger a state/picker update. Wait for the new sub-balancer + // to send its updates. + } else if newT.ChildPolicy.Name != oldT.ChildPolicy.Name { + // If the child policy name is different, remove from balancer group + // and re-add. + b.stateAggregator.Remove(name) + b.bg.Remove(name) + b.stateAggregator.Add(name, newT.Weight) + b.bg.Add(name, balancer.Get(newT.ChildPolicy.Name)) + } else if newT.Weight != oldT.Weight { + // If this is an existing sub-balancer, update weight if necessary. + b.stateAggregator.UpdateWeight(name, newT.Weight) + } + + // Forwards all the update: + // - addresses are from the map after splitting with hierarchy path, + // - Top level service config and attributes are the same, + // - Balancer config comes from the targets map. + // + // TODO: handle error? How to aggregate errors and return? + _ = b.bg.UpdateClientConnState(name, balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: endpointsSplit[name], + ServiceConfig: s.ResolverState.ServiceConfig, + Attributes: s.ResolverState.Attributes.WithValue(localityKey, name), + }, + BalancerConfig: newT.ChildPolicy.Config, + }) + } + + b.targets = newConfig.Targets + + // If the targets length is zero, it means we have removed all child + // policies from the balancer group and aggregator. + // At the start of this UpdateClientConnState() operation, a call to + // b.stateAggregator.ResumeStateUpdates() is deferred. Thus, setting the + // needUpdateStateOnResume bool to true here will ensure a new picker is + // built as part of that deferred function. Since there are now no child + // policies, the aggregated connectivity state reported form the Aggregator + // will be TRANSIENT_FAILURE. + if len(b.targets) == 0 { + b.stateAggregator.NeedUpdateStateOnResume() + } + + return nil +} + +func (b *weightedTargetBalancer) ResolverError(err error) { + b.bg.ResolverError(err) +} + +func (b *weightedTargetBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *weightedTargetBalancer) Close() { + b.stateAggregator.Stop() + b.bg.Close() +} + +func (b *weightedTargetBalancer) ExitIdle() { + b.bg.ExitIdle() +} diff --git a/balancer/weightedtarget/weightedtarget_config.go b/balancer/weightedtarget/weightedtarget_config.go new file mode 100644 index 0000000..52090cd --- /dev/null +++ b/balancer/weightedtarget/weightedtarget_config.go @@ -0,0 +1,49 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedtarget + +import ( + "encoding/json" + + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +// Target represents one target with the weight and the child policy. +type Target struct { + // Weight is the weight of the child policy. + Weight uint32 `json:"weight,omitempty"` + // ChildPolicy is the child policy and it's config. + ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"` +} + +// LBConfig is the balancer config for weighted_target. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + Targets map[string]Target `json:"targets,omitempty"` +} + +func parseConfig(c json.RawMessage) (*LBConfig, error) { + var cfg LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} diff --git a/balancer/weightedtarget/weightedtarget_config_test.go b/balancer/weightedtarget/weightedtarget_config_test.go new file mode 100644 index 0000000..25bbee8 --- /dev/null +++ b/balancer/weightedtarget/weightedtarget_config_test.go @@ -0,0 +1,104 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedtarget + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + _ "google.golang.org/grpc/balancer/grpclb" + "google.golang.org/grpc/balancer/roundrobin" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" +) + +const ( + testJSONConfig = `{ + "targets": { + "cluster_1": { + "weight": 75, + "childPolicy": [{ + "grpclb": { + "childPolicy": [{"pick_first":{}}], + "targetName": "foo-service" + } + }] + }, + "cluster_2": { + "weight": 25, + "childPolicy": [{"round_robin": ""}] + } + } +}` +) + +var ( + grpclbConfigParser = balancer.Get("grpclb").(balancer.ConfigParser) + grpclbConfigJSON = `{"childPolicy": [{"pick_first":{}}], "targetName": "foo-service"}` + grpclbConfig, _ = grpclbConfigParser.ParseConfig([]byte(grpclbConfigJSON)) +) + +func (s) TestParseConfig(t *testing.T) { + tests := []struct { + name string + js string + want *LBConfig + wantErr bool + }{ + { + name: "empty json", + js: "", + want: nil, + wantErr: true, + }, + { + name: "OK", + js: testJSONConfig, + want: &LBConfig{ + Targets: map[string]Target{ + "cluster_1": { + Weight: 75, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: "grpclb", + Config: grpclbConfig, + }, + }, + "cluster_2": { + Weight: 25, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseConfig([]byte(tt.js)) + if (err != nil) != tt.wantErr { + t.Fatalf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + } + if !cmp.Equal(got, tt.want) { + t.Errorf("parseConfig() got unexpected result, diff: %v", cmp.Diff(got, tt.want)) + } + }) + } +} diff --git a/balancer/weightedtarget/weightedtarget_test.go b/balancer/weightedtarget/weightedtarget_test.go new file mode 100644 index 0000000..8468892 --- /dev/null +++ b/balancer/weightedtarget/weightedtarget_test.go @@ -0,0 +1,1519 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedtarget + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/hierarchy" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const ( + defaultTestTimeout = 5 * time.Second +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type testConfigBalancerBuilder struct { + balancer.Builder +} + +func newTestConfigBalancerBuilder() *testConfigBalancerBuilder { + return &testConfigBalancerBuilder{ + Builder: balancer.Get(roundrobin.Name), + } +} + +// pickAndCheckError returns a function which takes a picker, invokes the Pick() method +// multiple times and ensures that the error returned by the picker matches the provided error. +func pickAndCheckError(want error) func(balancer.Picker) error { + const rpcCount = 5 + return func(p balancer.Picker) error { + for i := 0; i < rpcCount; i++ { + if _, err := p.Pick(balancer.PickInfo{}); err == nil || !strings.Contains(err.Error(), want.Error()) { + return fmt.Errorf("picker.Pick() returned error: %v, want: %v", err, want) + } + } + return nil + } +} + +func (t *testConfigBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + rr := t.Builder.Build(cc, opts) + return &testConfigBalancer{ + Balancer: rr, + } +} + +const testConfigBalancerName = "test_config_balancer" + +func (t *testConfigBalancerBuilder) Name() string { + return testConfigBalancerName +} + +type stringBalancerConfig struct { + serviceconfig.LoadBalancingConfig + configStr string +} + +func (t *testConfigBalancerBuilder) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + var cfg string + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal config in %q: %v", testConfigBalancerName, err) + } + return stringBalancerConfig{configStr: cfg}, nil +} + +// testConfigBalancer is a roundrobin balancer, but it takes the balancer config +// string and adds it as an address attribute to the backend addresses. +type testConfigBalancer struct { + balancer.Balancer +} + +// configKey is the type used as the key to store balancer config in the +// Attributes field of resolver.Address. +type configKey struct{} + +func setConfigKey(addr resolver.Address, config string) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(configKey{}, config) + return addr +} + +func getConfigKey(attr *attributes.Attributes) (string, bool) { + v := attr.Value(configKey{}) + name, ok := v.(string) + return name, ok +} + +func (b *testConfigBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + c, ok := s.BalancerConfig.(stringBalancerConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type %T", s.BalancerConfig) + } + + for i, ep := range s.ResolverState.Endpoints { + addrsWithAttr := make([]resolver.Address, len(ep.Addresses)) + for j, addr := range ep.Addresses { + addrsWithAttr[j] = setConfigKey(addr, c.configStr) + } + s.ResolverState.Endpoints[i].Addresses = addrsWithAttr + } + s.BalancerConfig = nil + return b.Balancer.UpdateClientConnState(s) +} + +func (b *testConfigBalancer) Close() { + b.Balancer.Close() +} + +var ( + wtbBuilder balancer.Builder + wtbParser balancer.ConfigParser + testBackendAddrStrs []string +) + +const testBackendAddrsCount = 12 + +func init() { + balancer.Register(newTestConfigBalancerBuilder()) + for i := 0; i < testBackendAddrsCount; i++ { + testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)) + } + wtbBuilder = balancer.Get(Name) + wtbParser = wtbBuilder.(balancer.ConfigParser) + + NewRandomWRR = testutils.NewTestWRR +} + +// Tests the behavior of the weighted_target LB policy when there are no targets +// configured. It verifies that the LB policy sets the overall channel state to +// TRANSIENT_FAILURE and fails RPCs with an expected status code and message. +func (s) TestWeightedTarget_NoTargets(t *testing.T) { + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"weighted_target_experimental":{}}]}`), + } + cc, err := grpc.NewClient("passthrough:///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Error("EmptyCall() succeeded, want failure") + } + if gotCode, wantCode := status.Code(err), codes.Unavailable; gotCode != wantCode { + t.Errorf("EmptyCall() failed with code = %v, want %s", gotCode, wantCode) + } + if gotMsg, wantMsg := err.Error(), "no targets to pick from"; !strings.Contains(gotMsg, wantMsg) { + t.Errorf("EmptyCall() failed with message = %q, want to contain %q", gotMsg, wantMsg) + } + if gotState, wantState := cc.GetState(), connectivity.TransientFailure; gotState != wantState { + t.Errorf("cc.GetState() = %v, want %v", gotState, wantState) + } +} + +// TestWeightedTarget covers the cases that a sub-balancer is added and a +// sub-balancer is removed. It verifies that the addresses and balancer configs +// are forwarded to the right sub-balancer. This test is intended to test the +// glue code in weighted_target. It also tests an empty target config update, +// which should trigger a transient failure state update. +func (s) TestWeightedTarget(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with "cluster_1: round_robin". + config1, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"round_robin": ""}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1], Attributes: nil} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr1}}, []string{"cluster_1"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + verifyAddressInNewSubConn(t, cc, addr1) + + // Send subconn state change. + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + // Test pick with one backend. + for i := 0; i < 5; i++ { + gotSCSt, _ := p.Pick(balancer.PickInfo{}) + if gotSCSt.SubConn != sc1 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) + } + } + + // Remove cluster_1, and add "cluster_2: test_config_balancer". The + // test_config_balancer adds an address attribute whose value is set to the + // config that is passed to it. + config2, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_2": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and one address with hierarchy path "cluster_2". + addr2 := resolver.Address{Addr: testBackendAddrStrs[2], Attributes: nil} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr2}}, []string{"cluster_2"}), + }}, + BalancerConfig: config2, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Expect a new subConn from the test_config_balancer which has an address + // attribute set to the config that was passed to it. + verifyAddressInNewSubConn(t, cc, setConfigKey(addr2, "cluster_2")) + + // The subconn for cluster_1 should be shut down. + scShutdown := <-cc.ShutdownSubConnCh + // The same SubConn is closed by gracefulswitch and pickfirstleaf when they + // are closed. Remove duplicate events. + // TODO: https://github.com/grpc/grpc-go/issues/6472 - Remove this + // workaround once pickfirst is the only leaf policy and responsible for + // shutting down SubConns. + <-cc.ShutdownSubConnCh + if scShutdown != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scShutdown) + } + scShutdown.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) + + sc2 := <-cc.NewSubConnCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p = <-cc.NewPickerCh + + // Test pick with one backend. + for i := 0; i < 5; i++ { + gotSCSt, _ := p.Pick(balancer.PickInfo{}) + if gotSCSt.SubConn != sc2 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2) + } + } + + // Replace child policy of "cluster_1" to "round_robin". + config3, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_2": { + "weight":1, + "childPolicy": [{"round_robin": ""}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_2"]. + addr3 := resolver.Address{Addr: testBackendAddrStrs[3], Attributes: nil} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr3}}, []string{"cluster_2"}), + }}, + BalancerConfig: config3, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + verifyAddressInNewSubConn(t, cc, addr3) + + // The subconn from the test_config_balancer should be shut down. + scShutdown = <-cc.ShutdownSubConnCh + // The same SubConn is closed by gracefulswitch and pickfirstleaf when they + // are closed. Remove duplicate events. + // TODO: https://github.com/grpc/grpc-go/issues/6472 - Remove this + // workaround once pickfirst is the only leaf policy and responsible for + // shutting down SubConns. + <-cc.ShutdownSubConnCh + + if scShutdown != sc2 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc2, scShutdown) + } + scShutdown.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) + + // Send subconn state change. + sc3 := <-cc.NewSubConnCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p = <-cc.NewPickerCh + + // Test pick with one backend. + for i := 0; i < 5; i++ { + gotSCSt, _ := p.Pick(balancer.PickInfo{}) + if gotSCSt.SubConn != sc3 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc3) + } + } + + // Update the Weighted Target Balancer with an empty address list and no + // targets. This should cause a Transient Failure State update to the Client + // Conn. + emptyConfig, err := wtbParser.ParseConfig([]byte(`{}`)) + if err != nil { + t.Fatalf("Failed to parse balancer config: %v", err) + } + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{}, + BalancerConfig: emptyConfig, + }); err != nil { + t.Fatalf("Failed to update ClientConn state: %v", err) + } + + state := <-cc.NewStateCh + if state != connectivity.TransientFailure { + t.Fatalf("Empty target update should have triggered a TF state update, got: %v", state) + } + p = <-cc.NewPickerCh + const wantErr = "no targets to pick from" + if _, err := p.Pick(balancer.PickInfo{}); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("Pick() returned error: %v, want: %v", err, wantErr) + } +} + +// TestWeightedTarget_OneSubBalancer_AddRemoveBackend tests the case where we +// have a weighted target balancer will one sub-balancer, and we add and remove +// backends from the subBalancer. +func (s) TestWeightedTarget_OneSubBalancer_AddRemoveBackend(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with "cluster_1: round_robin". + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"round_robin": ""}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr1}}, []string{"cluster_1"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + verifyAddressInNewSubConn(t, cc, addr1) + + // Expect one SubConn, and move it to READY. + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + // Test pick with one backend. + for i := 0; i < 5; i++ { + gotSCSt, _ := p.Pick(balancer.PickInfo{}) + if gotSCSt.SubConn != sc1 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) + } + } + + // Send two addresses. + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr1}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr2}}, []string{"cluster_1"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + verifyAddressInNewSubConn(t, cc, addr2) + + // Expect one new SubConn, and move it to READY. + sc2 := <-cc.NewSubConnCh + // Update the SubConn to become READY. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p = <-cc.NewPickerCh + + // Test round robin pick. + want := []balancer.SubConn{sc1, sc2} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Remove the first address. + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr2}}, []string{"cluster_1"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Expect one SubConn to be shut down. + scShutdown := <-cc.ShutdownSubConnCh + if scShutdown != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scShutdown) + } + scShutdown.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) + p = <-cc.NewPickerCh + + // Test pick with only the second SubConn. + for i := 0; i < 5; i++ { + gotSC, _ := p.Pick(balancer.PickInfo{}) + if gotSC.SubConn != sc2 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSC, sc2) + } + } +} + +// TestWeightedTarget_TwoSubBalancers_OneBackend tests the case where we have a +// weighted target balancer with two sub-balancers, each with one backend. +func (s) TestWeightedTarget_TwoSubBalancers_OneBackend(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with "cluster_1: test_config_balancer, cluster_2: test_config_balancer". + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with one address for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr1}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr2}}, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + scs := waitForNewSubConns(ctx, t, cc, 2) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1}, + "cluster_2": {addr2}, + }) + + // We expect a single subConn on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + sc2 := scs["cluster_2"][0].sc.(*testutils.TestSubConn) + + // The CONNECTING picker should be sent by all leaf pickfirst policies on + // receiving the first resolver update. + <-cc.NewPickerCh + // Send state changes for both SubConns, and wait for the picker. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + // Test roundrobin on the last picker. + want := []balancer.SubConn{sc1, sc2} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } +} + +// TestWeightedTarget_TwoSubBalancers_MoreBackends tests the case where we have +// a weighted target balancer with two sub-balancers, each with more than one +// backend. +func (s) TestWeightedTarget_TwoSubBalancers_MoreBackends(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with "cluster_1: round_robin, cluster_2: round_robin". + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with two backends for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + addr3 := resolver.Address{Addr: testBackendAddrStrs[3]} + addr4 := resolver.Address{Addr: testBackendAddrStrs[4]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr1}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr2}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr3}}, []string{"cluster_2"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr4}}, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + scs := waitForNewSubConns(ctx, t, cc, 4) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1, addr2}, + "cluster_2": {addr3, addr4}, + }) + + // We expect two subConns on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + sc2 := scs["cluster_1"][1].sc.(*testutils.TestSubConn) + sc3 := scs["cluster_2"][0].sc.(*testutils.TestSubConn) + sc4 := scs["cluster_2"][1].sc.(*testutils.TestSubConn) + + // Due to connection order randomization in RR, and the assumed order in the + // remainder of this test, adjust the scs according to the addrs if needed. + if sc1.Addresses[0].Addr != addr1.Addr { + sc1, sc2 = sc2, sc1 + } + if sc3.Addresses[0].Addr != addr3.Addr { + sc3, sc4 = sc4, sc3 + } + + // The CONNECTING picker should be sent by all leaf pickfirst policies on + // receiving the first resolver update. + <-cc.NewPickerCh + + // Send state changes for all SubConns, and wait for the picker. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + // Test roundrobin on the last picker. RPCs should be sent equally to all + // backends. + want := []balancer.SubConn{sc1, sc2, sc3, sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Turn sc2's connection down, should be RR between balancers. + wantSubConnErr := errors.New("subConn connection error") + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: wantSubConnErr, + }) + p = <-cc.NewPickerCh + want = []balancer.SubConn{sc1, sc1, sc3, sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Shut down subConn corresponding to addr3. + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr1}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr2}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr4}}, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + scShutdown := <-cc.ShutdownSubConnCh + if scShutdown != sc3 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc3, scShutdown) + } + scShutdown.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) + p = <-cc.NewPickerCh + want = []balancer.SubConn{sc1, sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Turn sc1's connection down. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: wantSubConnErr, + }) + p = <-cc.NewPickerCh + want = []balancer.SubConn{sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Turn last connection to connecting. + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + p = <-cc.NewPickerCh + for i := 0; i < 5; i++ { + if _, err := p.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err) + } + } + + // Turn all connections down. + sc4.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: wantSubConnErr, + }) + + if err := cc.WaitForPicker(ctx, pickAndCheckError(wantSubConnErr)); err != nil { + t.Fatal(err) + } +} + +// TestWeightedTarget_TwoSubBalancers_DifferentWeight_MoreBackends tests the +// case where we have a weighted target balancer with two sub-balancers of +// differing weights. +func (s) TestWeightedTarget_TwoSubBalancers_DifferentWeight_MoreBackends(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with two subBalancers, one with twice the weight of the other. + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight": 2, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with two backends for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + addr3 := resolver.Address{Addr: testBackendAddrStrs[3]} + addr4 := resolver.Address{Addr: testBackendAddrStrs[4]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr1}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr2}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr3}}, []string{"cluster_2"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr4}}, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + scs := waitForNewSubConns(ctx, t, cc, 4) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1, addr2}, + "cluster_2": {addr3, addr4}, + }) + + // We expect two subConns on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + sc2 := scs["cluster_1"][1].sc.(*testutils.TestSubConn) + sc3 := scs["cluster_2"][0].sc.(*testutils.TestSubConn) + sc4 := scs["cluster_2"][1].sc.(*testutils.TestSubConn) + + // The CONNECTING picker should be sent by all leaf pickfirst policies on + // receiving the first resolver update. + <-cc.NewPickerCh + + // Send state changes for all SubConns, and wait for the picker. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + // Test roundrobin on the last picker. Twice the number of RPCs should be + // sent to cluster_1 when compared to cluster_2. + want := []balancer.SubConn{sc1, sc1, sc2, sc2, sc3, sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } +} + +// TestWeightedTarget_ThreeSubBalancers_RemoveBalancer tests the case where we +// have a weighted target balancer with three sub-balancers and we remove one of +// the subBalancers. +func (s) TestWeightedTarget_ThreeSubBalancers_RemoveBalancer(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with two subBalancers, one with twice the weight of the other. + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + }, + "cluster_3": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_3"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with one backend for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + addr3 := resolver.Address{Addr: testBackendAddrStrs[3]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr1}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr2}}, []string{"cluster_2"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr3}}, []string{"cluster_3"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + scs := waitForNewSubConns(ctx, t, cc, 3) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1}, + "cluster_2": {addr2}, + "cluster_3": {addr3}, + }) + + // We expect one subConn on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + sc2 := scs["cluster_2"][0].sc.(*testutils.TestSubConn) + sc3 := scs["cluster_3"][0].sc.(*testutils.TestSubConn) + + // Send state changes for all SubConns, and wait for the picker. + // The CONNECTING picker should be sent by all leaf pickfirst policies on + // receiving the first resolver update. + <-cc.NewPickerCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + <-sc3.ConnectCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + want := []balancer.SubConn{sc1, sc2, sc3} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Remove the second balancer, while the others two are ready. + config, err = wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_3": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_3"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr1}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr3}}, []string{"cluster_3"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Removing a subBalancer causes the weighted target LB policy to push a new + // picker which ensures that the removed subBalancer is not picked for RPCs. + p = <-cc.NewPickerCh + + scShutdown := <-cc.ShutdownSubConnCh + // The same SubConn is closed by gracefulswitch and pickfirstleaf when they + // are closed. Remove duplicate events. + // TODO: https://github.com/grpc/grpc-go/issues/6472 - Remove this + // workaround once pickfirst is the only leaf policy and responsible for + // shutting down SubConns. + <-cc.ShutdownSubConnCh + if scShutdown != sc2 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc2, scShutdown) + } + want = []balancer.SubConn{sc1, sc3} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Move balancer 3 into transient failure. + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + <-sc3.ConnectCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + wantSubConnErr := errors.New("subConn connection error") + sc3.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: wantSubConnErr, + }) + <-cc.NewPickerCh + + // Remove the first balancer, while the third is transient failure. + config, err = wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_3": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_3"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr3}}, []string{"cluster_3"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Removing a subBalancer causes the weighted target LB policy to push a new + // picker which ensures that the removed subBalancer is not picked for RPCs. + scShutdown = <-cc.ShutdownSubConnCh + // The same SubConn is closed by gracefulswitch and pickfirstleaf when they + // are closed. Remove duplicate events. + // TODO: https://github.com/grpc/grpc-go/issues/6472 - Remove this + // workaround once pickfirst is the only leaf policy and responsible for + // shutting down SubConns. + <-cc.ShutdownSubConnCh + if scShutdown != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scShutdown) + } + + if err := cc.WaitForPicker(ctx, pickAndCheckError(wantSubConnErr)); err != nil { + t.Fatal(err) + } +} + +// TestWeightedTarget_TwoSubBalancers_ChangeWeight_MoreBackends tests the case +// where we have a weighted target balancer with two sub-balancers, and we +// change the weight of these subBalancers. +func (s) TestWeightedTarget_TwoSubBalancers_ChangeWeight_MoreBackends(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with two subBalancers, one with twice the weight of the other. + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight": 2, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with two backends for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + addr3 := resolver.Address{Addr: testBackendAddrStrs[3]} + addr4 := resolver.Address{Addr: testBackendAddrStrs[4]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr1}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr2}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr3}}, []string{"cluster_2"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr4}}, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + scs := waitForNewSubConns(ctx, t, cc, 4) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1, addr2}, + "cluster_2": {addr3, addr4}, + }) + + // We expect two subConns on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + sc2 := scs["cluster_1"][1].sc.(*testutils.TestSubConn) + sc3 := scs["cluster_2"][0].sc.(*testutils.TestSubConn) + sc4 := scs["cluster_2"][1].sc.(*testutils.TestSubConn) + + // The CONNECTING picker should be sent by all leaf pickfirst policies on + // receiving the first resolver update. + <-cc.NewPickerCh + + // Send state changes for all SubConns, and wait for the picker. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + // Test roundrobin on the last picker. Twice the number of RPCs should be + // sent to cluster_1 when compared to cluster_2. + want := []balancer.SubConn{sc1, sc1, sc2, sc2, sc3, sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Change the weight of cluster_1. + config, err = wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight": 3, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr1}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr2}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr3}}, []string{"cluster_2"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr4}}, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Weight change causes a new picker to be pushed to the channel. + p = <-cc.NewPickerCh + want = []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc2, sc3, sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } +} + +// TestWeightedTarget_InitOneSubBalancerTransientFailure tests that at init +// time, with two sub-balancers, if one sub-balancer reports transient_failure, +// the picks won't fail with transient_failure, and should instead wait for the +// other sub-balancer. +func (s) TestWeightedTarget_InitOneSubBalancerTransientFailure(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with "cluster_1: test_config_balancer, cluster_2: test_config_balancer". + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with one address for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr1}}, []string{"cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr2}}, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + scs := waitForNewSubConns(ctx, t, cc, 2) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1}, + "cluster_2": {addr2}, + }) + + // We expect a single subConn on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + _ = scs["cluster_2"][0].sc + + // Set one subconn to TransientFailure, this will trigger one sub-balancer + // to report transient failure. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + + p := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + r, err := p.Pick(balancer.PickInfo{}) + if err != balancer.ErrNoSubConnAvailable { + t.Fatalf("want pick to fail with %v, got result %v, err %v", balancer.ErrNoSubConnAvailable, r, err) + } + } +} + +// Test that with two sub-balancers, both in transient_failure, if one turns +// connecting, the overall state stays in transient_failure, and all picks +// return transient failure error. +func (s) TestBalancerGroup_SubBalancerTurnsConnectingFromTransientFailure(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with "cluster_1: test_config_balancer, cluster_2: test_config_balancer". + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with one address for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + ep1 := resolver.Endpoint{Addresses: []resolver.Address{addr1}} + ep2 := resolver.Endpoint{Addresses: []resolver.Address{addr2}} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(ep1, []string{"cluster_1"}), + hierarchy.SetInEndpoint(ep2, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + scs := waitForNewSubConns(ctx, t, cc, 2) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1}, + "cluster_2": {addr2}, + }) + + // We expect a single subConn on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + sc2 := scs["cluster_2"][0].sc.(*testutils.TestSubConn) + + // Set both subconn to TransientFailure, this will put both sub-balancers in + // transient failure. + wantSubConnErr := errors.New("subConn connection error") + sc1.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: wantSubConnErr, + }) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: wantSubConnErr, + }) + p := <-cc.NewPickerCh + + for i := 0; i < 5; i++ { + if _, err := p.Pick(balancer.PickInfo{}); err == nil || !strings.Contains(err.Error(), wantSubConnErr.Error()) { + t.Fatalf("picker.Pick() returned error: %v, want: %v", err, wantSubConnErr) + } + } + + // Set one subconn to Connecting, it shouldn't change the overall state. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + select { + case <-time.After(100 * time.Millisecond): + case <-cc.NewPickerCh: + t.Fatal("received new picker from the LB policy when expecting none") + } + + for i := 0; i < 5; i++ { + if _, err := p.Pick(balancer.PickInfo{}); err == nil || !strings.Contains(err.Error(), wantSubConnErr.Error()) { + t.Fatalf("picker.Pick() returned error: %v, want: %v", err, wantSubConnErr) + } + } +} + +// Verify that a SubConn is created with the expected address. +func verifyAddressInNewSubConn(t *testing.T, cc *testutils.BalancerClientConn, addr resolver.Address) { + t.Helper() + + gotAddr := <-cc.NewSubConnAddrsCh + wantAddr := []resolver.Address{addr} + gotAddr[0].BalancerAttributes = nil + if diff := cmp.Diff(gotAddr, wantAddr, cmp.AllowUnexported(attributes.Attributes{})); diff != "" { + t.Fatalf("got unexpected new subconn addrs: %v", diff) + } +} + +// subConnWithAddr wraps a subConn and the address for which it was created. +type subConnWithAddr struct { + sc balancer.SubConn + addr resolver.Address +} + +// waitForNewSubConns waits for `num` number of subConns to be created. This is +// expected to be used from tests using the "test_config_balancer" LB policy, +// which adds an address attribute with value set to the balancer config. +// +// Returned value is a map from subBalancer (identified by its config) to +// subConns created by it. +func waitForNewSubConns(ctx context.Context, t *testing.T, cc *testutils.BalancerClientConn, num int) map[string][]subConnWithAddr { + t.Helper() + + scs := make(map[string][]subConnWithAddr) + for i := 0; i < num; i++ { + var addrs []resolver.Address + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for addresses for new SubConn.") + case addrs = <-cc.NewSubConnAddrsCh: + } + if len(addrs) != 1 { + t.Fatalf("received subConns with %d addresses, want 1", len(addrs)) + } + cfg, ok := getConfigKey(addrs[0].Attributes) + if !ok { + t.Fatalf("received subConn address %v contains no attribute for balancer config", addrs[0]) + } + var sc balancer.SubConn + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for new SubConn.") + case sc = <-cc.NewSubConnCh: + } + scWithAddr := subConnWithAddr{sc: sc, addr: addrs[0]} + scs[cfg] = append(scs[cfg], scWithAddr) + } + return scs +} + +func verifySubConnAddrs(t *testing.T, scs map[string][]subConnWithAddr, wantSubConnAddrs map[string][]resolver.Address) { + t.Helper() + + if len(scs) != len(wantSubConnAddrs) { + t.Fatalf("got new subConns %+v, want %v", scs, wantSubConnAddrs) + } + for cfg, scsWithAddr := range scs { + if len(scsWithAddr) != len(wantSubConnAddrs[cfg]) { + t.Fatalf("got new subConns %+v, want %v", scs, wantSubConnAddrs) + } + wantAddrs := wantSubConnAddrs[cfg] + if diff := cmp.Diff(addressesToAddrs(wantAddrs), scwasToAddrs(scsWithAddr), + cmpopts.SortSlices(func(a, b string) bool { + return a < b + }), + ); diff != "" { + t.Fatalf("got unexpected new subconn addrs: %v", diff) + } + } +} + +func scwasToAddrs(ss []subConnWithAddr) []string { + ret := make([]string, len(ss)) + for i, s := range ss { + ret[i] = s.addr.Addr + } + return ret +} + +func addressesToAddrs(as []resolver.Address) []string { + ret := make([]string, len(as)) + for i, a := range as { + ret[i] = a.Addr + } + return ret +} + +const initIdleBalancerName = "test-init-idle-balancer" + +var errTestInitIdle = fmt.Errorf("init Idle balancer error 0") + +func init() { + stub.Register(initIdleBalancerName, stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, opts balancer.ClientConnState) error { + sc, err := bd.ClientConn.NewSubConn(opts.ResolverState.Addresses, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + err := fmt.Errorf("wrong picker error") + if state.ConnectivityState == connectivity.Idle { + err = errTestInitIdle + } + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &testutils.TestConstPicker{Err: err}, + }) + }, + }) + if err != nil { + return err + } + sc.Connect() + return nil + }, + }) +} + +// TestInitialIdle covers the case that if the child reports Idle, the overall +// state will be Idle. +func (s) TestInitialIdle(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"test-init-idle-balancer": ""}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + addrs := []resolver.Address{{Addr: testBackendAddrStrs[0], Attributes: nil}} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addrs[0]}}, []string{"cds:cluster_1"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Verify that a subconn is created with the address, and the hierarchy path + // in the address is cleared. + for range addrs { + sc := <-cc.NewSubConnCh + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + } + + if state := <-cc.NewStateCh; state != connectivity.Idle { + t.Fatalf("Received aggregated state: %v, want Idle", state) + } +} + +// TestIgnoreSubBalancerStateTransitions covers the case that if the child reports a +// transition from TF to Connecting, the overall state will still be TF. +func (s) TestIgnoreSubBalancerStateTransitions(t *testing.T) { + cc := &tcc{BalancerClientConn: testutils.NewBalancerClientConn(t)} + + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"round_robin": ""}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + addr := resolver.Address{Addr: testBackendAddrStrs[0], Attributes: nil} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addr}}, []string{"cluster_1"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + sc := <-cc.NewSubConnCh + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + + // Verify that the SubConnState update from TF to Connecting is ignored. + if len(cc.states) != 2 || cc.states[0].ConnectivityState != connectivity.Connecting || cc.states[1].ConnectivityState != connectivity.TransientFailure { + t.Fatalf("cc.states = %v; want [Connecting, TransientFailure]", cc.states) + } +} + +// tcc wraps a testutils.TestClientConn but stores all state transitions in a +// slice. +type tcc struct { + *testutils.BalancerClientConn + states []balancer.State +} + +func (t *tcc) UpdateState(bs balancer.State) { + t.states = append(t.states, bs) + t.BalancerClientConn.UpdateState(bs) +} + +func (s) TestUpdateStatePauses(t *testing.T) { + cc := &tcc{BalancerClientConn: testutils.NewBalancerClientConn(t)} + + balFuncs := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: nil}) + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: nil}) + return nil + }, + } + stub.Register("update_state_balancer", balFuncs) + + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"update_state_balancer": ""}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + addrs := []resolver.Address{{Addr: testBackendAddrStrs[0], Attributes: nil}} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{addrs[0]}}, []string{"cds:cluster_1"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Verify that the only state update is the second one called by the child. + if len(cc.states) != 1 || cc.states[0].ConnectivityState != connectivity.Ready { + t.Fatalf("cc.states = %v; want [connectivity.Ready]", cc.states) + } +} diff --git a/balancer_wrapper.go b/balancer_wrapper.go new file mode 100644 index 0000000..a1e56a3 --- /dev/null +++ b/balancer_wrapper.go @@ -0,0 +1,517 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "fmt" + "sync" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/gracefulswitch" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" +) + +var ( + // noOpRegisterHealthListenerFn is used when client side health checking is + // disabled. It sends a single READY update on the registered listener. + noOpRegisterHealthListenerFn = func(_ context.Context, listener func(balancer.SubConnState)) func() { + listener(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + return func() {} + } +) + +// ccBalancerWrapper sits between the ClientConn and the Balancer. +// +// ccBalancerWrapper implements methods corresponding to the ones on the +// balancer.Balancer interface. The ClientConn is free to call these methods +// concurrently and the ccBalancerWrapper ensures that calls from the ClientConn +// to the Balancer happen in order by performing them in the serializer, without +// any mutexes held. +// +// ccBalancerWrapper also implements the balancer.ClientConn interface and is +// passed to the Balancer implementations. It invokes unexported methods on the +// ClientConn to handle these calls from the Balancer. +// +// It uses the gracefulswitch.Balancer internally to ensure that balancer +// switches happen in a graceful manner. +type ccBalancerWrapper struct { + internal.EnforceClientConnEmbedding + // The following fields are initialized when the wrapper is created and are + // read-only afterwards, and therefore can be accessed without a mutex. + cc *ClientConn + opts balancer.BuildOptions + serializer *grpcsync.CallbackSerializer + serializerCancel context.CancelFunc + + // The following fields are only accessed within the serializer or during + // initialization. + curBalancerName string + balancer *gracefulswitch.Balancer + + // The following field is protected by mu. Caller must take cc.mu before + // taking mu. + mu sync.Mutex + closed bool +} + +// newCCBalancerWrapper creates a new balancer wrapper in idle state. The +// underlying balancer is not created until the updateClientConnState() method +// is invoked. +func newCCBalancerWrapper(cc *ClientConn) *ccBalancerWrapper { + ctx, cancel := context.WithCancel(cc.ctx) + ccb := &ccBalancerWrapper{ + cc: cc, + opts: balancer.BuildOptions{ + DialCreds: cc.dopts.copts.TransportCredentials, + CredsBundle: cc.dopts.copts.CredsBundle, + Dialer: cc.dopts.copts.Dialer, + Authority: cc.authority, + CustomUserAgent: cc.dopts.copts.UserAgent, + ChannelzParent: cc.channelz, + Target: cc.parsedTarget, + }, + serializer: grpcsync.NewCallbackSerializer(ctx), + serializerCancel: cancel, + } + ccb.balancer = gracefulswitch.NewBalancer(ccb, ccb.opts) + return ccb +} + +func (ccb *ccBalancerWrapper) MetricsRecorder() stats.MetricsRecorder { + return ccb.cc.metricsRecorderList +} + +// updateClientConnState is invoked by grpc to push a ClientConnState update to +// the underlying balancer. This is always executed from the serializer, so +// it is safe to call into the balancer here. +func (ccb *ccBalancerWrapper) updateClientConnState(ccs *balancer.ClientConnState) error { + errCh := make(chan error) + uccs := func(ctx context.Context) { + defer close(errCh) + if ctx.Err() != nil || ccb.balancer == nil { + return + } + name := gracefulswitch.ChildName(ccs.BalancerConfig) + if ccb.curBalancerName != name { + ccb.curBalancerName = name + channelz.Infof(logger, ccb.cc.channelz, "Channel switches to new LB policy %q", name) + } + err := ccb.balancer.UpdateClientConnState(*ccs) + if logger.V(2) && err != nil { + logger.Infof("error from balancer.UpdateClientConnState: %v", err) + } + errCh <- err + } + onFailure := func() { close(errCh) } + + // UpdateClientConnState can race with Close, and when the latter wins, the + // serializer is closed, and the attempt to schedule the callback will fail. + // It is acceptable to ignore this failure. But since we want to handle the + // state update in a blocking fashion (when we successfully schedule the + // callback), we have to use the ScheduleOr method and not the MaybeSchedule + // method on the serializer. + ccb.serializer.ScheduleOr(uccs, onFailure) + return <-errCh +} + +// resolverError is invoked by grpc to push a resolver error to the underlying +// balancer. The call to the balancer is executed from the serializer. +func (ccb *ccBalancerWrapper) resolverError(err error) { + ccb.serializer.TrySchedule(func(ctx context.Context) { + if ctx.Err() != nil || ccb.balancer == nil { + return + } + ccb.balancer.ResolverError(err) + }) +} + +// close initiates async shutdown of the wrapper. cc.mu must be held when +// calling this function. To determine the wrapper has finished shutting down, +// the channel should block on ccb.serializer.Done() without cc.mu held. +func (ccb *ccBalancerWrapper) close() { + ccb.mu.Lock() + ccb.closed = true + ccb.mu.Unlock() + channelz.Info(logger, ccb.cc.channelz, "ccBalancerWrapper: closing") + ccb.serializer.TrySchedule(func(context.Context) { + if ccb.balancer == nil { + return + } + ccb.balancer.Close() + ccb.balancer = nil + }) + ccb.serializerCancel() +} + +// exitIdle invokes the balancer's exitIdle method in the serializer. +func (ccb *ccBalancerWrapper) exitIdle() { + ccb.serializer.TrySchedule(func(ctx context.Context) { + if ctx.Err() != nil || ccb.balancer == nil { + return + } + ccb.balancer.ExitIdle() + }) +} + +func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + ccb.cc.mu.Lock() + defer ccb.cc.mu.Unlock() + + ccb.mu.Lock() + if ccb.closed { + ccb.mu.Unlock() + return nil, fmt.Errorf("balancer is being closed; no new SubConns allowed") + } + ccb.mu.Unlock() + + if len(addrs) == 0 { + return nil, fmt.Errorf("grpc: cannot create SubConn with empty address list") + } + ac, err := ccb.cc.newAddrConnLocked(addrs, opts) + if err != nil { + channelz.Warningf(logger, ccb.cc.channelz, "acBalancerWrapper: NewSubConn: failed to newAddrConn: %v", err) + return nil, err + } + acbw := &acBalancerWrapper{ + ccb: ccb, + ac: ac, + producers: make(map[balancer.ProducerBuilder]*refCountedProducer), + stateListener: opts.StateListener, + healthData: newHealthData(connectivity.Idle), + } + ac.acbw = acbw + return acbw, nil +} + +func (ccb *ccBalancerWrapper) RemoveSubConn(balancer.SubConn) { + // The graceful switch balancer will never call this. + logger.Errorf("ccb RemoveSubConn(%v) called unexpectedly, sc") +} + +func (ccb *ccBalancerWrapper) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + acbw, ok := sc.(*acBalancerWrapper) + if !ok { + return + } + acbw.UpdateAddresses(addrs) +} + +func (ccb *ccBalancerWrapper) UpdateState(s balancer.State) { + ccb.cc.mu.Lock() + defer ccb.cc.mu.Unlock() + if ccb.cc.conns == nil { + // The CC has been closed; ignore this update. + return + } + + ccb.mu.Lock() + if ccb.closed { + ccb.mu.Unlock() + return + } + ccb.mu.Unlock() + // Update picker before updating state. Even though the ordering here does + // not matter, it can lead to multiple calls of Pick in the common start-up + // case where we wait for ready and then perform an RPC. If the picker is + // updated later, we could call the "connecting" picker when the state is + // updated, and then call the "ready" picker after the picker gets updated. + + // Note that there is no need to check if the balancer wrapper was closed, + // as we know the graceful switch LB policy will not call cc if it has been + // closed. + ccb.cc.pickerWrapper.updatePicker(s.Picker) + ccb.cc.csMgr.updateState(s.ConnectivityState) +} + +func (ccb *ccBalancerWrapper) ResolveNow(o resolver.ResolveNowOptions) { + ccb.cc.mu.RLock() + defer ccb.cc.mu.RUnlock() + + ccb.mu.Lock() + if ccb.closed { + ccb.mu.Unlock() + return + } + ccb.mu.Unlock() + ccb.cc.resolveNowLocked(o) +} + +func (ccb *ccBalancerWrapper) Target() string { + return ccb.cc.target +} + +// acBalancerWrapper is a wrapper on top of ac for balancers. +// It implements balancer.SubConn interface. +type acBalancerWrapper struct { + internal.EnforceSubConnEmbedding + ac *addrConn // read-only + ccb *ccBalancerWrapper // read-only + stateListener func(balancer.SubConnState) + + producersMu sync.Mutex + producers map[balancer.ProducerBuilder]*refCountedProducer + + // Access to healthData is protected by healthMu. + healthMu sync.Mutex + // healthData is stored as a pointer to detect when the health listener is + // dropped or updated. This is required as closures can't be compared for + // equality. + healthData *healthData +} + +// healthData holds data related to health state reporting. +type healthData struct { + // connectivityState stores the most recent connectivity state delivered + // to the LB policy. This is stored to avoid sending updates when the + // SubConn has already exited connectivity state READY. + connectivityState connectivity.State + // closeHealthProducer stores function to close the ref counted health + // producer. The health producer is automatically closed when the SubConn + // state changes. + closeHealthProducer func() +} + +func newHealthData(s connectivity.State) *healthData { + return &healthData{ + connectivityState: s, + closeHealthProducer: func() {}, + } +} + +// updateState is invoked by grpc to push a subConn state update to the +// underlying balancer. +func (acbw *acBalancerWrapper) updateState(s connectivity.State, err error) { + acbw.ccb.serializer.TrySchedule(func(ctx context.Context) { + if ctx.Err() != nil || acbw.ccb.balancer == nil { + return + } + // Invalidate all producers on any state change. + acbw.closeProducers() + + // Even though it is optional for balancers, gracefulswitch ensures + // opts.StateListener is set, so this cannot ever be nil. + // TODO: delete this comment when UpdateSubConnState is removed. + scs := balancer.SubConnState{ConnectivityState: s, ConnectionError: err} + // Invalidate the health listener by updating the healthData. + acbw.healthMu.Lock() + // A race may occur if a health listener is registered soon after the + // connectivity state is set but before the stateListener is called. + // Two cases may arise: + // 1. The new state is not READY: RegisterHealthListener has checks to + // ensure no updates are sent when the connectivity state is not + // READY. + // 2. The new state is READY: This means that the old state wasn't Ready. + // The RegisterHealthListener API mentions that a health listener + // must not be registered when a SubConn is not ready to avoid such + // races. When this happens, the LB policy would get health updates + // on the old listener. When the LB policy registers a new listener + // on receiving the connectivity update, the health updates will be + // sent to the new health listener. + acbw.healthData = newHealthData(scs.ConnectivityState) + acbw.healthMu.Unlock() + + acbw.stateListener(scs) + }) +} + +func (acbw *acBalancerWrapper) String() string { + return fmt.Sprintf("SubConn(id:%d)", acbw.ac.channelz.ID) +} + +func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) { + acbw.ac.updateAddrs(addrs) +} + +func (acbw *acBalancerWrapper) Connect() { + go acbw.ac.connect() +} + +func (acbw *acBalancerWrapper) Shutdown() { + acbw.closeProducers() + acbw.ccb.cc.removeAddrConn(acbw.ac, errConnDrain) +} + +// NewStream begins a streaming RPC on the addrConn. If the addrConn is not +// ready, blocks until it is or ctx expires. Returns an error when the context +// expires or the addrConn is shut down. +func (acbw *acBalancerWrapper) NewStream(ctx context.Context, desc *StreamDesc, method string, opts ...CallOption) (ClientStream, error) { + transport := acbw.ac.getReadyTransport() + if transport == nil { + return nil, status.Errorf(codes.Unavailable, "SubConn state is not Ready") + + } + return newNonRetryClientStream(ctx, desc, method, transport, acbw.ac, opts...) +} + +// Invoke performs a unary RPC. If the addrConn is not ready, returns +// errSubConnNotReady. +func (acbw *acBalancerWrapper) Invoke(ctx context.Context, method string, args any, reply any, opts ...CallOption) error { + cs, err := acbw.NewStream(ctx, unaryStreamDesc, method, opts...) + if err != nil { + return err + } + if err := cs.SendMsg(args); err != nil { + return err + } + return cs.RecvMsg(reply) +} + +type refCountedProducer struct { + producer balancer.Producer + refs int // number of current refs to the producer + close func() // underlying producer's close function +} + +func (acbw *acBalancerWrapper) GetOrBuildProducer(pb balancer.ProducerBuilder) (balancer.Producer, func()) { + acbw.producersMu.Lock() + defer acbw.producersMu.Unlock() + + // Look up existing producer from this builder. + pData := acbw.producers[pb] + if pData == nil { + // Not found; create a new one and add it to the producers map. + p, closeFn := pb.Build(acbw) + pData = &refCountedProducer{producer: p, close: closeFn} + acbw.producers[pb] = pData + } + // Account for this new reference. + pData.refs++ + + // Return a cleanup function wrapped in a OnceFunc to remove this reference + // and delete the refCountedProducer from the map if the total reference + // count goes to zero. + unref := func() { + acbw.producersMu.Lock() + // If closeProducers has already closed this producer instance, refs is + // set to 0, so the check after decrementing will never pass, and the + // producer will not be double-closed. + pData.refs-- + if pData.refs == 0 { + defer pData.close() // Run outside the acbw mutex + delete(acbw.producers, pb) + } + acbw.producersMu.Unlock() + } + return pData.producer, sync.OnceFunc(unref) +} + +func (acbw *acBalancerWrapper) closeProducers() { + acbw.producersMu.Lock() + defer acbw.producersMu.Unlock() + for pb, pData := range acbw.producers { + pData.refs = 0 + pData.close() + delete(acbw.producers, pb) + } +} + +// healthProducerRegisterFn is a type alias for the health producer's function +// for registering listeners. +type healthProducerRegisterFn = func(context.Context, balancer.SubConn, string, func(balancer.SubConnState)) func() + +// healthListenerRegFn returns a function to register a listener for health +// updates. If client side health checks are disabled, the registered listener +// will get a single READY (raw connectivity state) update. +// +// Client side health checking is enabled when all the following +// conditions are satisfied: +// 1. Health checking is not disabled using the dial option. +// 2. The health package is imported. +// 3. The health check config is present in the service config. +func (acbw *acBalancerWrapper) healthListenerRegFn() func(context.Context, func(balancer.SubConnState)) func() { + if acbw.ccb.cc.dopts.disableHealthCheck { + return noOpRegisterHealthListenerFn + } + cfg := acbw.ac.cc.healthCheckConfig() + if cfg == nil { + return noOpRegisterHealthListenerFn + } + regHealthLisFn := internal.RegisterClientHealthCheckListener + if regHealthLisFn == nil { + // The health package is not imported. + channelz.Error(logger, acbw.ac.channelz, "Health check is requested but health package is not imported.") + return noOpRegisterHealthListenerFn + } + return func(ctx context.Context, listener func(balancer.SubConnState)) func() { + return regHealthLisFn.(healthProducerRegisterFn)(ctx, acbw, cfg.ServiceName, listener) + } +} + +// RegisterHealthListener accepts a health listener from the LB policy. It sends +// updates to the health listener as long as the SubConn's connectivity state +// doesn't change and a new health listener is not registered. To invalidate +// the currently registered health listener, acbw updates the healthData. If a +// nil listener is registered, the active health listener is dropped. +func (acbw *acBalancerWrapper) RegisterHealthListener(listener func(balancer.SubConnState)) { + acbw.healthMu.Lock() + defer acbw.healthMu.Unlock() + acbw.healthData.closeHealthProducer() + // listeners should not be registered when the connectivity state + // isn't Ready. This may happen when the balancer registers a listener + // after the connectivityState is updated, but before it is notified + // of the update. + if acbw.healthData.connectivityState != connectivity.Ready { + return + } + // Replace the health data to stop sending updates to any previously + // registered health listeners. + hd := newHealthData(connectivity.Ready) + acbw.healthData = hd + if listener == nil { + return + } + + registerFn := acbw.healthListenerRegFn() + acbw.ccb.serializer.TrySchedule(func(ctx context.Context) { + if ctx.Err() != nil || acbw.ccb.balancer == nil { + return + } + // Don't send updates if a new listener is registered. + acbw.healthMu.Lock() + defer acbw.healthMu.Unlock() + if acbw.healthData != hd { + return + } + // Serialize the health updates from the health producer with + // other calls into the LB policy. + listenerWrapper := func(scs balancer.SubConnState) { + acbw.ccb.serializer.TrySchedule(func(ctx context.Context) { + if ctx.Err() != nil || acbw.ccb.balancer == nil { + return + } + acbw.healthMu.Lock() + defer acbw.healthMu.Unlock() + if acbw.healthData != hd { + return + } + listener(scs) + }) + } + + hd.closeHealthProducer = registerFn(ctx, listenerWrapper) + }) +} diff --git a/balancer_wrapper_test.go b/balancer_wrapper_test.go new file mode 100644 index 0000000..fd02462 --- /dev/null +++ b/balancer_wrapper_test.go @@ -0,0 +1,82 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "fmt" + "strings" + "sync" + "testing" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpcsync" +) + +// TestBalancer_StateListenerBeforeConnect tries to stimulate a race between +// NewSubConn and ClientConn.Close. In no cases should the SubConn's +// StateListener be invoked, because Connect was never called. +func (s) TestBalancer_StateListenerBeforeConnect(t *testing.T) { + // started is fired after cc is set so cc can be used in the balancer. + started := grpcsync.NewEvent() + var cc *ClientConn + + wg := sync.WaitGroup{} + wg.Add(2) + + // Create a balancer that calls NewSubConn and cc.Close at approximately the + // same time. + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + go func() { + // Wait for cc to be valid after the channel is created. + <-started.Done() + // In a goroutine, create the subconn. + go func() { + _, err := bd.ClientConn.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{ + StateListener: func(scs balancer.SubConnState) { + t.Error("Unexpected call to StateListener with:", scs) + }, + }) + if err != nil && !strings.Contains(err.Error(), "connection is closing") && !strings.Contains(err.Error(), "is deleted") && !strings.Contains(err.Error(), "is closed or idle") && !strings.Contains(err.Error(), "balancer is being closed") { + t.Error("Unexpected error creating subconn:", err) + } + wg.Done() + }() + // At approximately the same time, close the channel. + cc.Close() + wg.Done() + }() + return nil + }, + } + stub.Register(t.Name(), bf) + svcCfg := fmt.Sprintf(`{ "loadBalancingConfig": [{%q: {}}] }`, t.Name()) + + cc, err := NewClient("passthrough:///test.server", WithTransportCredentials(insecure.NewCredentials()), WithDefaultServiceConfig(svcCfg)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + started.Fire() + + // Wait for the LB policy to call NewSubConn and cc.Close. + wg.Wait() +} diff --git a/benchmark/benchmain/main.go b/benchmark/benchmain/main.go new file mode 100644 index 0000000..6115f21 --- /dev/null +++ b/benchmark/benchmain/main.go @@ -0,0 +1,1022 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* +Package main provides benchmark with setting flags. + +An example to run some benchmarks with profiling enabled: + + go run benchmark/benchmain/main.go -benchtime=10s -workloads=all \ + -compression=gzip -maxConcurrentCalls=1 -trace=off \ + -reqSizeBytes=1,1048576 -respSizeBytes=1,1048576 -networkMode=Local \ + -cpuProfile=cpuProf -memProfile=memProf -memProfileRate=10000 -resultFile=result + +As a suggestion, when creating a branch, you can run this benchmark and save the result +file "-resultFile=basePerf", and later when you at the middle of the work or finish the +work, you can get the benchmark result and compare it with the base anytime. + +Assume there are two result files names as "basePerf" and "curPerf" created by adding +-resultFile=basePerf and -resultFile=curPerf. + + To format the curPerf, run: + go run benchmark/benchresult/main.go curPerf + To observe how the performance changes based on a base result, run: + go run benchmark/benchresult/main.go basePerf curPerf +*/ +package main + +import ( + "context" + "encoding/gob" + "flag" + "fmt" + "io" + "log" + rand "math/rand/v2" + "net" + "os" + "reflect" + "runtime" + "runtime/pprof" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/benchmark" + "google.golang.org/grpc/benchmark/flags" + "google.golang.org/grpc/benchmark/latency" + "google.golang.org/grpc/benchmark/stats" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/encoding/gzip" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/test/bufconn" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + workloads = flags.StringWithAllowedValues("workloads", workloadsAll, + fmt.Sprintf("Workloads to execute - One of: %v", strings.Join(allWorkloads, ", ")), allWorkloads) + traceMode = flags.StringWithAllowedValues("trace", toggleModeOff, + fmt.Sprintf("Trace mode - One of: %v", strings.Join(allToggleModes, ", ")), allToggleModes) + preloaderMode = flags.StringWithAllowedValues("preloader", toggleModeOff, + fmt.Sprintf("Preloader mode - One of: %v, preloader works only in streaming and unconstrained modes and will be ignored in unary mode", + strings.Join(allToggleModes, ", ")), allToggleModes) + channelzOn = flags.StringWithAllowedValues("channelz", toggleModeOff, + fmt.Sprintf("Channelz mode - One of: %v", strings.Join(allToggleModes, ", ")), allToggleModes) + compressorMode = flags.StringWithAllowedValues("compression", compModeOff, + fmt.Sprintf("Compression mode - One of: %v", strings.Join(allCompModes, ", ")), allCompModes) + networkMode = flags.StringWithAllowedValues("networkMode", networkModeNone, + "Network mode includes LAN, WAN, Local and Longhaul", allNetworkModes) + readLatency = flags.DurationSlice("latency", defaultReadLatency, "Simulated one-way network latency - may be a comma-separated list") + readKbps = flags.IntSlice("kbps", defaultReadKbps, "Simulated network throughput (in kbps) - may be a comma-separated list") + readMTU = flags.IntSlice("mtu", defaultReadMTU, "Simulated network MTU (Maximum Transmission Unit) - may be a comma-separated list") + maxConcurrentCalls = flags.IntSlice("maxConcurrentCalls", defaultMaxConcurrentCalls, "Number of concurrent RPCs during benchmarks") + readReqSizeBytes = flags.IntSlice("reqSizeBytes", nil, "Request size in bytes - may be a comma-separated list") + readRespSizeBytes = flags.IntSlice("respSizeBytes", nil, "Response size in bytes - may be a comma-separated list") + reqPayloadCurveFiles = flags.StringSlice("reqPayloadCurveFiles", nil, "comma-separated list of CSV files describing the shape a random distribution of request payload sizes") + respPayloadCurveFiles = flags.StringSlice("respPayloadCurveFiles", nil, "comma-separated list of CSV files describing the shape a random distribution of response payload sizes") + benchTime = flag.Duration("benchtime", time.Second, "Configures the amount of time to run each benchmark") + memProfile = flag.String("memProfile", "", "Enables memory profiling output to the filename provided.") + memProfileRate = flag.Int("memProfileRate", 512*1024, "Configures the memory profiling rate. \n"+ + "memProfile should be set before setting profile rate. To include every allocated block in the profile, "+ + "set MemProfileRate to 1. To turn off profiling entirely, set MemProfileRate to 0. 512 * 1024 by default.") + cpuProfile = flag.String("cpuProfile", "", "Enables CPU profiling output to the filename provided") + benchmarkResultFile = flag.String("resultFile", "", "Save the benchmark result into a binary file") + useBufconn = flag.Bool("bufconn", false, "Use in-memory connection instead of system network I/O") + enableKeepalive = flag.Bool("enable_keepalive", false, "Enable client keepalive. \n"+ + "Keepalive.Time is set to 10s, Keepalive.Timeout is set to 1s, Keepalive.PermitWithoutStream is set to true.") + clientReadBufferSize = flags.IntSlice("clientReadBufferSize", []int{-1}, "Configures the client read buffer size in bytes. If negative, use the default - may be a comma-separated list") + clientWriteBufferSize = flags.IntSlice("clientWriteBufferSize", []int{-1}, "Configures the client write buffer size in bytes. If negative, use the default - may be a comma-separated list") + serverReadBufferSize = flags.IntSlice("serverReadBufferSize", []int{-1}, "Configures the server read buffer size in bytes. If negative, use the default - may be a comma-separated list") + serverWriteBufferSize = flags.IntSlice("serverWriteBufferSize", []int{-1}, "Configures the server write buffer size in bytes. If negative, use the default - may be a comma-separated list") + sleepBetweenRPCs = flags.DurationSlice("sleepBetweenRPCs", []time.Duration{0}, "Configures the maximum amount of time the client should sleep between consecutive RPCs - may be a comma-separated list") + connections = flag.Int("connections", 1, "The number of connections. Each connection will handle maxConcurrentCalls RPC streams") + recvBufferPool = flags.StringWithAllowedValues("recvBufferPool", recvBufferPoolSimple, "Configures the shared receive buffer pool. One of: nil, simple, all", allRecvBufferPools) + sharedWriteBuffer = flags.StringWithAllowedValues("sharedWriteBuffer", toggleModeOn, + fmt.Sprintf("Configures both client and server to share write buffer - One of: %v", strings.Join(allToggleModes, ", ")), allToggleModes) + + logger = grpclog.Component("benchmark") +) + +const ( + workloadsUnary = "unary" + workloadsStreaming = "streaming" + workloadsUnconstrained = "unconstrained" + workloadsAll = "all" + // Compression modes. + compModeOff = "off" + compModeGzip = "gzip" + compModeNop = "nop" + compModeAll = "all" + // Toggle modes. + toggleModeOff = "off" + toggleModeOn = "on" + toggleModeBoth = "both" + // Network modes. + networkModeNone = "none" + networkModeLocal = "Local" + networkModeLAN = "LAN" + networkModeWAN = "WAN" + networkLongHaul = "Longhaul" + // Shared recv buffer pool + recvBufferPoolNil = "nil" + recvBufferPoolSimple = "simple" + recvBufferPoolAll = "all" + + numStatsBuckets = 10 + warmupCallCount = 10 + warmuptime = time.Second +) + +var useNopBufferPool atomic.Bool + +type swappableBufferPool struct { + mem.BufferPool +} + +func (p swappableBufferPool) Get(length int) *[]byte { + var pool mem.BufferPool + if useNopBufferPool.Load() { + pool = mem.NopBufferPool{} + } else { + pool = p.BufferPool + } + return pool.Get(length) +} + +func (p swappableBufferPool) Put(i *[]byte) { + if useNopBufferPool.Load() { + return + } + p.BufferPool.Put(i) +} + +func init() { + internal.SetDefaultBufferPool.(func(mem.BufferPool))(swappableBufferPool{mem.DefaultBufferPool()}) +} + +var ( + allWorkloads = []string{workloadsUnary, workloadsStreaming, workloadsUnconstrained, workloadsAll} + allCompModes = []string{compModeOff, compModeGzip, compModeNop, compModeAll} + allToggleModes = []string{toggleModeOff, toggleModeOn, toggleModeBoth} + allNetworkModes = []string{networkModeNone, networkModeLocal, networkModeLAN, networkModeWAN, networkLongHaul} + allRecvBufferPools = []string{recvBufferPoolNil, recvBufferPoolSimple, recvBufferPoolAll} + defaultReadLatency = []time.Duration{0, 40 * time.Millisecond} // if non-positive, no delay. + defaultReadKbps = []int{0, 10240} // if non-positive, infinite + defaultReadMTU = []int{0} // if non-positive, infinite + defaultMaxConcurrentCalls = []int{1, 8, 64, 512} + defaultReqSizeBytes = []int{1, 1024, 1024 * 1024} + defaultRespSizeBytes = []int{1, 1024, 1024 * 1024} + networks = map[string]latency.Network{ + networkModeLocal: latency.Local, + networkModeLAN: latency.LAN, + networkModeWAN: latency.WAN, + networkLongHaul: latency.Longhaul, + } + keepaliveTime = 10 * time.Second + keepaliveTimeout = 1 * time.Second + // This is 0.8*keepaliveTime to prevent connection issues because of server + // keepalive enforcement. + keepaliveMinTime = 8 * time.Second +) + +// runModes indicates the workloads to run. This is initialized with a call to +// `runModesFromWorkloads`, passing the workloads flag set by the user. +type runModes struct { + unary, streaming, unconstrained bool +} + +// runModesFromWorkloads determines the runModes based on the value of +// workloads flag set by the user. +func runModesFromWorkloads(workload string) runModes { + r := runModes{} + switch workload { + case workloadsUnary: + r.unary = true + case workloadsStreaming: + r.streaming = true + case workloadsUnconstrained: + r.unconstrained = true + case workloadsAll: + r.unary = true + r.streaming = true + r.unconstrained = true + default: + log.Fatalf("Unknown workloads setting: %v (want one of: %v)", + workloads, strings.Join(allWorkloads, ", ")) + } + return r +} + +type startFunc func(mode string, bf stats.Features) +type stopFunc func(count uint64) +type ucStopFunc func(req uint64, resp uint64) +type rpcCallFunc func(cn, pos int) +type rpcSendFunc func(cn, pos int) +type rpcRecvFunc func(cn, pos int) +type rpcCleanupFunc func() + +func unaryBenchmark(start startFunc, stop stopFunc, bf stats.Features, s *stats.Stats) { + caller, cleanup := makeFuncUnary(bf) + defer cleanup() + runBenchmark(caller, start, stop, bf, s, workloadsUnary) +} + +func streamBenchmark(start startFunc, stop stopFunc, bf stats.Features, s *stats.Stats) { + caller, cleanup := makeFuncStream(bf) + defer cleanup() + runBenchmark(caller, start, stop, bf, s, workloadsStreaming) +} + +func unconstrainedStreamBenchmark(start startFunc, stop ucStopFunc, bf stats.Features) { + var sender rpcSendFunc + var recver rpcRecvFunc + var cleanup rpcCleanupFunc + if bf.EnablePreloader { + sender, recver, cleanup = makeFuncUnconstrainedStreamPreloaded(bf) + } else { + sender, recver, cleanup = makeFuncUnconstrainedStream(bf) + } + defer cleanup() + + var req, resp uint64 + go func() { + // Resets the counters once warmed up + <-time.NewTimer(warmuptime).C + atomic.StoreUint64(&req, 0) + atomic.StoreUint64(&resp, 0) + start(workloadsUnconstrained, bf) + }() + + bmEnd := time.Now().Add(bf.BenchTime + warmuptime) + var wg sync.WaitGroup + wg.Add(2 * bf.Connections * bf.MaxConcurrentCalls) + maxSleep := int(bf.SleepBetweenRPCs) + for cn := 0; cn < bf.Connections; cn++ { + for pos := 0; pos < bf.MaxConcurrentCalls; pos++ { + go func(cn, pos int) { + defer wg.Done() + for { + if maxSleep > 0 { + time.Sleep(time.Duration(rand.IntN(maxSleep))) + } + t := time.Now() + if t.After(bmEnd) { + return + } + sender(cn, pos) + atomic.AddUint64(&req, 1) + } + }(cn, pos) + go func(cn, pos int) { + defer wg.Done() + for { + t := time.Now() + if t.After(bmEnd) { + return + } + recver(cn, pos) + atomic.AddUint64(&resp, 1) + } + }(cn, pos) + } + } + wg.Wait() + stop(req, resp) +} + +// makeClients returns a gRPC client (or multiple clients) for the grpc.testing.BenchmarkService +// service. The client is configured using the different options in the passed +// 'bf'. Also returns a cleanup function to close the client and release +// resources. +func makeClients(bf stats.Features) ([]testgrpc.BenchmarkServiceClient, func()) { + nw := &latency.Network{Kbps: bf.Kbps, Latency: bf.Latency, MTU: bf.MTU} + opts := []grpc.DialOption{} + sopts := []grpc.ServerOption{} + if bf.ModeCompressor == compModeNop { + sopts = append(sopts, + grpc.RPCCompressor(nopCompressor{}), + grpc.RPCDecompressor(nopDecompressor{}), + ) + opts = append(opts, + grpc.WithCompressor(nopCompressor{}), + grpc.WithDecompressor(nopDecompressor{}), + ) + } + if bf.ModeCompressor == compModeGzip { + opts = append(opts, + grpc.WithDefaultCallOptions(grpc.UseCompressor(gzip.Name)), + ) + } + if bf.EnableKeepalive { + sopts = append(sopts, + grpc.KeepaliveParams(keepalive.ServerParameters{ + Time: keepaliveTime, + Timeout: keepaliveTimeout, + }), + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: keepaliveMinTime, + PermitWithoutStream: true, + }), + ) + opts = append(opts, + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: keepaliveTime, + Timeout: keepaliveTimeout, + PermitWithoutStream: true, + }), + ) + } + if bf.ClientReadBufferSize >= 0 { + opts = append(opts, grpc.WithReadBufferSize(bf.ClientReadBufferSize)) + } + if bf.ClientWriteBufferSize >= 0 { + opts = append(opts, grpc.WithWriteBufferSize(bf.ClientWriteBufferSize)) + } + if bf.ServerReadBufferSize >= 0 { + sopts = append(sopts, grpc.ReadBufferSize(bf.ServerReadBufferSize)) + } + if bf.SharedWriteBuffer { + opts = append(opts, grpc.WithSharedWriteBuffer(true)) + sopts = append(sopts, grpc.SharedWriteBuffer(true)) + } + if bf.ServerWriteBufferSize >= 0 { + sopts = append(sopts, grpc.WriteBufferSize(bf.ServerWriteBufferSize)) + } + switch bf.RecvBufferPool { + case recvBufferPoolNil: + useNopBufferPool.Store(true) + case recvBufferPoolSimple: + // Do nothing as buffering is enabled by default. + default: + logger.Fatalf("Unknown shared recv buffer pool type: %v", bf.RecvBufferPool) + } + + sopts = append(sopts, grpc.MaxConcurrentStreams(uint32(bf.MaxConcurrentCalls+1))) + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + + var lis net.Listener + if bf.UseBufConn { + bcLis := bufconn.Listen(256 * 1024) + lis = bcLis + opts = append(opts, grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return nw.ContextDialer(func(context.Context, string, string) (net.Conn, error) { + return bcLis.Dial() + })(ctx, "", "") + })) + } else { + var err error + lis, err = net.Listen("tcp", "localhost:0") + if err != nil { + logger.Fatalf("Failed to listen: %v", err) + } + opts = append(opts, grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return nw.ContextDialer((internal.NetDialerWithTCPKeepalive().DialContext))(ctx, "tcp", lis.Addr().String()) + })) + } + lis = nw.Listener(lis) + stopper := benchmark.StartServer(benchmark.ServerInfo{Type: "protobuf", Listener: lis}, sopts...) + conns := make([]*grpc.ClientConn, bf.Connections) + clients := make([]testgrpc.BenchmarkServiceClient, bf.Connections) + for cn := 0; cn < bf.Connections; cn++ { + conns[cn] = benchmark.NewClientConn("passthrough://" /* target not used */, opts...) + clients[cn] = testgrpc.NewBenchmarkServiceClient(conns[cn]) + } + + return clients, func() { + for _, conn := range conns { + conn.Close() + } + stopper() + } +} + +func makeFuncUnary(bf stats.Features) (rpcCallFunc, rpcCleanupFunc) { + clients, cleanup := makeClients(bf) + return func(cn, _ int) { + reqSizeBytes := bf.ReqSizeBytes + respSizeBytes := bf.RespSizeBytes + if bf.ReqPayloadCurve != nil { + reqSizeBytes = bf.ReqPayloadCurve.ChooseRandom() + } + if bf.RespPayloadCurve != nil { + respSizeBytes = bf.RespPayloadCurve.ChooseRandom() + } + unaryCaller(clients[cn], reqSizeBytes, respSizeBytes) + }, cleanup +} + +func makeFuncStream(bf stats.Features) (rpcCallFunc, rpcCleanupFunc) { + streams, req, cleanup := setupStream(bf, false) + + var preparedMsg [][]*grpc.PreparedMsg + if bf.EnablePreloader { + preparedMsg = prepareMessages(streams, req) + } + + return func(cn, pos int) { + reqSizeBytes := bf.ReqSizeBytes + respSizeBytes := bf.RespSizeBytes + if bf.ReqPayloadCurve != nil { + reqSizeBytes = bf.ReqPayloadCurve.ChooseRandom() + } + if bf.RespPayloadCurve != nil { + respSizeBytes = bf.RespPayloadCurve.ChooseRandom() + } + var req any + if bf.EnablePreloader { + req = preparedMsg[cn][pos] + } else { + pl := benchmark.NewPayload(testpb.PayloadType_COMPRESSABLE, reqSizeBytes) + req = &testpb.SimpleRequest{ + ResponseType: pl.Type, + ResponseSize: int32(respSizeBytes), + Payload: pl, + } + } + streamCaller(streams[cn][pos], req) + }, cleanup +} + +func makeFuncUnconstrainedStreamPreloaded(bf stats.Features) (rpcSendFunc, rpcRecvFunc, rpcCleanupFunc) { + streams, req, cleanup := setupStream(bf, true) + + preparedMsg := prepareMessages(streams, req) + + return func(cn, pos int) { + streams[cn][pos].SendMsg(preparedMsg[cn][pos]) + }, func(cn, pos int) { + streams[cn][pos].Recv() + }, cleanup +} + +func makeFuncUnconstrainedStream(bf stats.Features) (rpcSendFunc, rpcRecvFunc, rpcCleanupFunc) { + streams, req, cleanup := setupStream(bf, true) + + return func(cn, pos int) { + streams[cn][pos].Send(req) + }, func(cn, pos int) { + streams[cn][pos].Recv() + }, cleanup +} + +func setupStream(bf stats.Features, unconstrained bool) ([][]testgrpc.BenchmarkService_StreamingCallClient, *testpb.SimpleRequest, rpcCleanupFunc) { + clients, cleanup := makeClients(bf) + + streams := make([][]testgrpc.BenchmarkService_StreamingCallClient, bf.Connections) + ctx := context.Background() + if unconstrained { + md := metadata.Pairs(benchmark.UnconstrainedStreamingHeader, "1", benchmark.UnconstrainedStreamingDelayHeader, bf.SleepBetweenRPCs.String()) + ctx = metadata.NewOutgoingContext(ctx, md) + } + if bf.EnablePreloader { + md := metadata.Pairs(benchmark.PreloadMsgSizeHeader, strconv.Itoa(bf.RespSizeBytes), benchmark.UnconstrainedStreamingDelayHeader, bf.SleepBetweenRPCs.String()) + ctx = metadata.NewOutgoingContext(ctx, md) + } + for cn := 0; cn < bf.Connections; cn++ { + tc := clients[cn] + streams[cn] = make([]testgrpc.BenchmarkService_StreamingCallClient, bf.MaxConcurrentCalls) + for pos := 0; pos < bf.MaxConcurrentCalls; pos++ { + stream, err := tc.StreamingCall(ctx) + if err != nil { + logger.Fatalf("%v.StreamingCall(_) = _, %v", tc, err) + } + streams[cn][pos] = stream + } + } + + pl := benchmark.NewPayload(testpb.PayloadType_COMPRESSABLE, bf.ReqSizeBytes) + req := &testpb.SimpleRequest{ + ResponseType: pl.Type, + ResponseSize: int32(bf.RespSizeBytes), + Payload: pl, + } + + return streams, req, cleanup +} + +func prepareMessages(streams [][]testgrpc.BenchmarkService_StreamingCallClient, req *testpb.SimpleRequest) [][]*grpc.PreparedMsg { + preparedMsg := make([][]*grpc.PreparedMsg, len(streams)) + for cn, connStreams := range streams { + preparedMsg[cn] = make([]*grpc.PreparedMsg, len(connStreams)) + for pos, stream := range connStreams { + preparedMsg[cn][pos] = &grpc.PreparedMsg{} + if err := preparedMsg[cn][pos].Encode(stream, req); err != nil { + logger.Fatalf("%v.Encode(%v, %v) = %v", preparedMsg[cn][pos], req, stream, err) + } + } + } + return preparedMsg +} + +// Makes a UnaryCall gRPC request using the given BenchmarkServiceClient and +// request and response sizes. +func unaryCaller(client testgrpc.BenchmarkServiceClient, reqSize, respSize int) { + if err := benchmark.DoUnaryCall(client, reqSize, respSize); err != nil { + logger.Fatalf("DoUnaryCall failed: %v", err) + } +} + +func streamCaller(stream testgrpc.BenchmarkService_StreamingCallClient, req any) { + if err := benchmark.DoStreamingRoundTripPreloaded(stream, req); err != nil { + logger.Fatalf("DoStreamingRoundTrip failed: %v", err) + } +} + +func runBenchmark(caller rpcCallFunc, start startFunc, stop stopFunc, bf stats.Features, s *stats.Stats, mode string) { + // if SleepBetweenRPCs > 0 we skip the warmup because otherwise + // we are going to send a set of simultaneous requests on every connection, + // which is something we are trying to avoid when using SleepBetweenRPCs. + if bf.SleepBetweenRPCs == 0 { + // Warm up connections. + for i := 0; i < warmupCallCount; i++ { + for cn := 0; cn < bf.Connections; cn++ { + caller(cn, 0) + } + } + } + + // Run benchmark. + start(mode, bf) + var wg sync.WaitGroup + wg.Add(bf.Connections * bf.MaxConcurrentCalls) + bmEnd := time.Now().Add(bf.BenchTime) + maxSleep := int(bf.SleepBetweenRPCs) + var count uint64 + for cn := 0; cn < bf.Connections; cn++ { + for pos := 0; pos < bf.MaxConcurrentCalls; pos++ { + go func(cn, pos int) { + defer wg.Done() + for { + if maxSleep > 0 { + time.Sleep(time.Duration(rand.IntN(maxSleep))) + } + t := time.Now() + if t.After(bmEnd) { + return + } + start := time.Now() + caller(cn, pos) + elapse := time.Since(start) + atomic.AddUint64(&count, 1) + s.AddDuration(elapse) + } + }(cn, pos) + } + } + wg.Wait() + stop(count) +} + +// benchOpts represents all configurable options available while running this +// benchmark. This is built from the values passed as flags. +type benchOpts struct { + rModes runModes + benchTime time.Duration + memProfileRate int + memProfile string + cpuProfile string + networkMode string + benchmarkResultFile string + useBufconn bool + enableKeepalive bool + connections int + features *featureOpts +} + +// featureOpts represents options which can have multiple values. The user +// usually provides a comma-separated list of options for each of these +// features through command line flags. We generate all possible combinations +// for the provided values and run the benchmarks for each combination. +type featureOpts struct { + enableTrace []bool + readLatencies []time.Duration + readKbps []int + readMTU []int + maxConcurrentCalls []int + reqSizeBytes []int + respSizeBytes []int + reqPayloadCurves []*stats.PayloadCurve + respPayloadCurves []*stats.PayloadCurve + compModes []string + enableChannelz []bool + enablePreloader []bool + clientReadBufferSize []int + clientWriteBufferSize []int + serverReadBufferSize []int + serverWriteBufferSize []int + sleepBetweenRPCs []time.Duration + recvBufferPools []string + sharedWriteBuffer []bool +} + +// makeFeaturesNum returns a slice of ints of size 'maxFeatureIndex' where each +// element of the slice (indexed by 'featuresIndex' enum) contains the number +// of features to be exercised by the benchmark code. +// For example: Index 0 of the returned slice contains the number of values for +// enableTrace feature, while index 1 contains the number of value of +// readLatencies feature and so on. +func makeFeaturesNum(b *benchOpts) []int { + featuresNum := make([]int, stats.MaxFeatureIndex) + for i := 0; i < len(featuresNum); i++ { + switch stats.FeatureIndex(i) { + case stats.EnableTraceIndex: + featuresNum[i] = len(b.features.enableTrace) + case stats.ReadLatenciesIndex: + featuresNum[i] = len(b.features.readLatencies) + case stats.ReadKbpsIndex: + featuresNum[i] = len(b.features.readKbps) + case stats.ReadMTUIndex: + featuresNum[i] = len(b.features.readMTU) + case stats.MaxConcurrentCallsIndex: + featuresNum[i] = len(b.features.maxConcurrentCalls) + case stats.ReqSizeBytesIndex: + featuresNum[i] = len(b.features.reqSizeBytes) + case stats.RespSizeBytesIndex: + featuresNum[i] = len(b.features.respSizeBytes) + case stats.ReqPayloadCurveIndex: + featuresNum[i] = len(b.features.reqPayloadCurves) + case stats.RespPayloadCurveIndex: + featuresNum[i] = len(b.features.respPayloadCurves) + case stats.CompModesIndex: + featuresNum[i] = len(b.features.compModes) + case stats.EnableChannelzIndex: + featuresNum[i] = len(b.features.enableChannelz) + case stats.EnablePreloaderIndex: + featuresNum[i] = len(b.features.enablePreloader) + case stats.ClientReadBufferSize: + featuresNum[i] = len(b.features.clientReadBufferSize) + case stats.ClientWriteBufferSize: + featuresNum[i] = len(b.features.clientWriteBufferSize) + case stats.ServerReadBufferSize: + featuresNum[i] = len(b.features.serverReadBufferSize) + case stats.ServerWriteBufferSize: + featuresNum[i] = len(b.features.serverWriteBufferSize) + case stats.SleepBetweenRPCs: + featuresNum[i] = len(b.features.sleepBetweenRPCs) + case stats.RecvBufferPool: + featuresNum[i] = len(b.features.recvBufferPools) + case stats.SharedWriteBuffer: + featuresNum[i] = len(b.features.sharedWriteBuffer) + default: + log.Fatalf("Unknown feature index %v in generateFeatures. maxFeatureIndex is %v", i, stats.MaxFeatureIndex) + } + } + return featuresNum +} + +// sharedFeatures returns a bool slice which acts as a bitmask. Each item in +// the slice represents a feature, indexed by 'featureIndex' enum. The bit is +// set to 1 if the corresponding feature does not have multiple value, so is +// shared amongst all benchmarks. +func sharedFeatures(featuresNum []int) []bool { + result := make([]bool, len(featuresNum)) + for i, num := range featuresNum { + if num <= 1 { + result[i] = true + } + } + return result +} + +// generateFeatures generates all combinations of the provided feature options. +// While all the feature options are stored in the benchOpts struct, the input +// parameter 'featuresNum' is a slice indexed by 'featureIndex' enum containing +// the number of values for each feature. +// For example, let's say the user sets -workloads=all and +// -maxConcurrentCalls=1,100, this would end up with the following +// combinations: +// [workloads: unary, maxConcurrentCalls=1] +// [workloads: unary, maxConcurrentCalls=1] +// [workloads: streaming, maxConcurrentCalls=100] +// [workloads: streaming, maxConcurrentCalls=100] +// [workloads: unconstrained, maxConcurrentCalls=1] +// [workloads: unconstrained, maxConcurrentCalls=100] +func (b *benchOpts) generateFeatures(featuresNum []int) []stats.Features { + // curPos and initialPos are two slices where each value acts as an index + // into the appropriate feature slice maintained in benchOpts.features. This + // loop generates all possible combinations of features by changing one value + // at a time, and once curPos becomes equal to initialPos, we have explored + // all options. + var result []stats.Features + var curPos []int + initialPos := make([]int, stats.MaxFeatureIndex) + for !reflect.DeepEqual(initialPos, curPos) { + if curPos == nil { + curPos = make([]int, stats.MaxFeatureIndex) + } + f := stats.Features{ + // These features stay the same for each iteration. + NetworkMode: b.networkMode, + UseBufConn: b.useBufconn, + EnableKeepalive: b.enableKeepalive, + BenchTime: b.benchTime, + Connections: b.connections, + // These features can potentially change for each iteration. + EnableTrace: b.features.enableTrace[curPos[stats.EnableTraceIndex]], + Latency: b.features.readLatencies[curPos[stats.ReadLatenciesIndex]], + Kbps: b.features.readKbps[curPos[stats.ReadKbpsIndex]], + MTU: b.features.readMTU[curPos[stats.ReadMTUIndex]], + MaxConcurrentCalls: b.features.maxConcurrentCalls[curPos[stats.MaxConcurrentCallsIndex]], + ModeCompressor: b.features.compModes[curPos[stats.CompModesIndex]], + EnableChannelz: b.features.enableChannelz[curPos[stats.EnableChannelzIndex]], + EnablePreloader: b.features.enablePreloader[curPos[stats.EnablePreloaderIndex]], + ClientReadBufferSize: b.features.clientReadBufferSize[curPos[stats.ClientReadBufferSize]], + ClientWriteBufferSize: b.features.clientWriteBufferSize[curPos[stats.ClientWriteBufferSize]], + ServerReadBufferSize: b.features.serverReadBufferSize[curPos[stats.ServerReadBufferSize]], + ServerWriteBufferSize: b.features.serverWriteBufferSize[curPos[stats.ServerWriteBufferSize]], + SleepBetweenRPCs: b.features.sleepBetweenRPCs[curPos[stats.SleepBetweenRPCs]], + RecvBufferPool: b.features.recvBufferPools[curPos[stats.RecvBufferPool]], + SharedWriteBuffer: b.features.sharedWriteBuffer[curPos[stats.SharedWriteBuffer]], + } + if len(b.features.reqPayloadCurves) == 0 { + f.ReqSizeBytes = b.features.reqSizeBytes[curPos[stats.ReqSizeBytesIndex]] + } else { + f.ReqPayloadCurve = b.features.reqPayloadCurves[curPos[stats.ReqPayloadCurveIndex]] + } + if len(b.features.respPayloadCurves) == 0 { + f.RespSizeBytes = b.features.respSizeBytes[curPos[stats.RespSizeBytesIndex]] + } else { + f.RespPayloadCurve = b.features.respPayloadCurves[curPos[stats.RespPayloadCurveIndex]] + } + result = append(result, f) + addOne(curPos, featuresNum) + } + return result +} + +// addOne mutates the input slice 'features' by changing one feature, thus +// arriving at the next combination of feature values. 'featuresMaxPosition' +// provides the numbers of allowed values for each feature, indexed by +// 'featureIndex' enum. +func addOne(features []int, featuresMaxPosition []int) { + for i := len(features) - 1; i >= 0; i-- { + if featuresMaxPosition[i] == 0 { + continue + } + features[i] = (features[i] + 1) + if features[i]/featuresMaxPosition[i] == 0 { + break + } + features[i] = features[i] % featuresMaxPosition[i] + } +} + +// processFlags reads the command line flags and builds benchOpts. Specifying +// invalid values for certain flags will cause flag.Parse() to fail, and the +// program to terminate. +// This *SHOULD* be the only place where the flags are accessed. All other +// parts of the benchmark code should rely on the returned benchOpts. +func processFlags() *benchOpts { + flag.Parse() + if flag.NArg() != 0 { + log.Fatal("Error: unparsed arguments: ", flag.Args()) + } + + opts := &benchOpts{ + rModes: runModesFromWorkloads(*workloads), + benchTime: *benchTime, + memProfileRate: *memProfileRate, + memProfile: *memProfile, + cpuProfile: *cpuProfile, + networkMode: *networkMode, + benchmarkResultFile: *benchmarkResultFile, + useBufconn: *useBufconn, + enableKeepalive: *enableKeepalive, + connections: *connections, + features: &featureOpts{ + enableTrace: setToggleMode(*traceMode), + readLatencies: append([]time.Duration(nil), *readLatency...), + readKbps: append([]int(nil), *readKbps...), + readMTU: append([]int(nil), *readMTU...), + maxConcurrentCalls: append([]int(nil), *maxConcurrentCalls...), + reqSizeBytes: append([]int(nil), *readReqSizeBytes...), + respSizeBytes: append([]int(nil), *readRespSizeBytes...), + compModes: setCompressorMode(*compressorMode), + enableChannelz: setToggleMode(*channelzOn), + enablePreloader: setToggleMode(*preloaderMode), + clientReadBufferSize: append([]int(nil), *clientReadBufferSize...), + clientWriteBufferSize: append([]int(nil), *clientWriteBufferSize...), + serverReadBufferSize: append([]int(nil), *serverReadBufferSize...), + serverWriteBufferSize: append([]int(nil), *serverWriteBufferSize...), + sleepBetweenRPCs: append([]time.Duration(nil), *sleepBetweenRPCs...), + recvBufferPools: setRecvBufferPool(*recvBufferPool), + sharedWriteBuffer: setToggleMode(*sharedWriteBuffer), + }, + } + + if len(*reqPayloadCurveFiles) == 0 { + if len(opts.features.reqSizeBytes) == 0 { + opts.features.reqSizeBytes = defaultReqSizeBytes + } + } else { + if len(opts.features.reqSizeBytes) != 0 { + log.Fatalf("you may not specify -reqPayloadCurveFiles and -reqSizeBytes at the same time") + } + if len(opts.features.enablePreloader) != 0 { + log.Fatalf("you may not specify -reqPayloadCurveFiles and -preloader at the same time") + } + for _, file := range *reqPayloadCurveFiles { + pc, err := stats.NewPayloadCurve(file) + if err != nil { + log.Fatalf("cannot load payload curve file %s: %v", file, err) + } + opts.features.reqPayloadCurves = append(opts.features.reqPayloadCurves, pc) + } + opts.features.reqSizeBytes = nil + } + if len(*respPayloadCurveFiles) == 0 { + if len(opts.features.respSizeBytes) == 0 { + opts.features.respSizeBytes = defaultRespSizeBytes + } + } else { + if len(opts.features.respSizeBytes) != 0 { + log.Fatalf("you may not specify -respPayloadCurveFiles and -respSizeBytes at the same time") + } + if len(opts.features.enablePreloader) != 0 { + log.Fatalf("you may not specify -respPayloadCurveFiles and -preloader at the same time") + } + for _, file := range *respPayloadCurveFiles { + pc, err := stats.NewPayloadCurve(file) + if err != nil { + log.Fatalf("cannot load payload curve file %s: %v", file, err) + } + opts.features.respPayloadCurves = append(opts.features.respPayloadCurves, pc) + } + opts.features.respSizeBytes = nil + } + + // Re-write latency, kpbs and mtu if network mode is set. + if network, ok := networks[opts.networkMode]; ok { + opts.features.readLatencies = []time.Duration{network.Latency} + opts.features.readKbps = []int{network.Kbps} + opts.features.readMTU = []int{network.MTU} + } + return opts +} + +func setToggleMode(val string) []bool { + switch val { + case toggleModeOn: + return []bool{true} + case toggleModeOff: + return []bool{false} + case toggleModeBoth: + return []bool{false, true} + default: + // This should never happen because a wrong value passed to this flag would + // be caught during flag.Parse(). + return []bool{} + } +} + +func setCompressorMode(val string) []string { + switch val { + case compModeNop, compModeGzip, compModeOff: + return []string{val} + case compModeAll: + return []string{compModeNop, compModeGzip, compModeOff} + default: + // This should never happen because a wrong value passed to this flag would + // be caught during flag.Parse(). + return []string{} + } +} + +func setRecvBufferPool(val string) []string { + switch val { + case recvBufferPoolNil, recvBufferPoolSimple: + return []string{val} + case recvBufferPoolAll: + return []string{recvBufferPoolNil, recvBufferPoolSimple} + default: + // This should never happen because a wrong value passed to this flag would + // be caught during flag.Parse(). + return []string{} + } +} + +func main() { + opts := processFlags() + before(opts) + + s := stats.NewStats(numStatsBuckets) + featuresNum := makeFeaturesNum(opts) + sf := sharedFeatures(featuresNum) + + var ( + start = func(mode string, bf stats.Features) { s.StartRun(mode, bf, sf) } + stop = func(count uint64) { s.EndRun(count) } + ucStop = func(req uint64, resp uint64) { s.EndUnconstrainedRun(req, resp) } + ) + + for _, bf := range opts.generateFeatures(featuresNum) { + grpc.EnableTracing = bf.EnableTrace + if bf.EnableChannelz { + channelz.TurnOn() + } + if opts.rModes.unary { + unaryBenchmark(start, stop, bf, s) + } + if opts.rModes.streaming { + streamBenchmark(start, stop, bf, s) + } + if opts.rModes.unconstrained { + unconstrainedStreamBenchmark(start, ucStop, bf) + } + } + after(opts, s.GetResults()) +} + +func before(opts *benchOpts) { + if opts.memProfile != "" { + runtime.MemProfileRate = opts.memProfileRate + } + if opts.cpuProfile != "" { + f, err := os.Create(opts.cpuProfile) + if err != nil { + fmt.Fprintf(os.Stderr, "testing: %s\n", err) + return + } + if err := pprof.StartCPUProfile(f); err != nil { + fmt.Fprintf(os.Stderr, "testing: can't start cpu profile: %s\n", err) + f.Close() + return + } + } +} + +func after(opts *benchOpts, data []stats.BenchResults) { + if opts.cpuProfile != "" { + pprof.StopCPUProfile() // flushes profile to disk + } + if opts.memProfile != "" { + f, err := os.Create(opts.memProfile) + if err != nil { + fmt.Fprintf(os.Stderr, "testing: %s\n", err) + os.Exit(2) + } + runtime.GC() // materialize all statistics + if err = pprof.WriteHeapProfile(f); err != nil { + fmt.Fprintf(os.Stderr, "testing: can't write heap profile %s: %s\n", opts.memProfile, err) + os.Exit(2) + } + f.Close() + } + if opts.benchmarkResultFile != "" { + f, err := os.Create(opts.benchmarkResultFile) + if err != nil { + log.Fatalf("testing: can't write benchmark result %s: %s\n", opts.benchmarkResultFile, err) + } + dataEncoder := gob.NewEncoder(f) + dataEncoder.Encode(data) + f.Close() + } +} + +// nopCompressor is a compressor that just copies data. +type nopCompressor struct{} + +func (nopCompressor) Do(w io.Writer, p []byte) error { + n, err := w.Write(p) + if err != nil { + return err + } + if n != len(p) { + return fmt.Errorf("nopCompressor.Write: wrote %d bytes; want %d", n, len(p)) + } + return nil +} + +func (nopCompressor) Type() string { return compModeNop } + +// nopDecompressor is a decompressor that just copies data. +type nopDecompressor struct{} + +func (nopDecompressor) Do(r io.Reader) ([]byte, error) { return io.ReadAll(r) } +func (nopDecompressor) Type() string { return compModeNop } diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go new file mode 100644 index 0000000..5c3dcd5 --- /dev/null +++ b/benchmark/benchmark.go @@ -0,0 +1,349 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* +Package benchmark implements the building blocks to setup end-to-end gRPC benchmarks. +*/ +package benchmark + +import ( + "context" + "fmt" + "io" + "log" + rand "math/rand/v2" + "net" + "strconv" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var logger = grpclog.Component("benchmark") + +// Allows reuse of the same testpb.Payload object. +func setPayload(p *testpb.Payload, t testpb.PayloadType, size int) { + if size < 0 { + logger.Fatalf("Requested a response with invalid length %d", size) + } + body := make([]byte, size) + switch t { + case testpb.PayloadType_COMPRESSABLE: + default: + logger.Fatalf("Unsupported payload type: %d", t) + } + p.Type = t + p.Body = body +} + +// NewPayload creates a payload with the given type and size. +func NewPayload(t testpb.PayloadType, size int) *testpb.Payload { + p := new(testpb.Payload) + setPayload(p, t, size) + return p +} + +type testServer struct { + testgrpc.UnimplementedBenchmarkServiceServer +} + +func (s *testServer) UnaryCall(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{ + Payload: NewPayload(in.ResponseType, int(in.ResponseSize)), + }, nil +} + +// UnconstrainedStreamingHeader indicates to the StreamingCall handler that its +// behavior should be unconstrained (constant send/receive in parallel) instead +// of ping-pong. +const UnconstrainedStreamingHeader = "unconstrained-streaming" + +// UnconstrainedStreamingDelayHeader is used to pass the maximum amount of time +// the server should sleep between consecutive RPC responses. +const UnconstrainedStreamingDelayHeader = "unconstrained-streaming-delay" + +// PreloadMsgSizeHeader indicates that the client is going to ask for +// a fixed response size and passes this size to the server. +// The server is expected to preload the response on startup. +const PreloadMsgSizeHeader = "preload-msg-size" + +func (s *testServer) StreamingCall(stream testgrpc.BenchmarkService_StreamingCallServer) error { + preloadMsgSize := 0 + if md, ok := metadata.FromIncomingContext(stream.Context()); ok && len(md[PreloadMsgSizeHeader]) != 0 { + val := md[PreloadMsgSizeHeader][0] + var err error + preloadMsgSize, err = strconv.Atoi(val) + if err != nil { + return fmt.Errorf("%q header value is not an integer: %s", PreloadMsgSizeHeader, err) + } + } + + if md, ok := metadata.FromIncomingContext(stream.Context()); ok && len(md[UnconstrainedStreamingHeader]) != 0 { + return s.UnconstrainedStreamingCall(stream, preloadMsgSize) + } + response := &testpb.SimpleResponse{ + Payload: new(testpb.Payload), + } + preloadedResponse := &grpc.PreparedMsg{} + if preloadMsgSize > 0 { + setPayload(response.Payload, testpb.PayloadType_COMPRESSABLE, preloadMsgSize) + if err := preloadedResponse.Encode(stream, response); err != nil { + return err + } + } + in := new(testpb.SimpleRequest) + for { + // use ServerStream directly to reuse the same testpb.SimpleRequest object + err := stream.(grpc.ServerStream).RecvMsg(in) + if err == io.EOF { + // read done. + return nil + } + if err != nil { + return err + } + if preloadMsgSize > 0 { + err = stream.SendMsg(preloadedResponse) + } else { + setPayload(response.Payload, in.ResponseType, int(in.ResponseSize)) + err = stream.Send(response) + } + if err != nil { + return err + } + } +} + +func (s *testServer) UnconstrainedStreamingCall(stream testgrpc.BenchmarkService_StreamingCallServer, preloadMsgSize int) error { + maxSleep := 0 + if md, ok := metadata.FromIncomingContext(stream.Context()); ok && len(md[UnconstrainedStreamingDelayHeader]) != 0 { + val := md[UnconstrainedStreamingDelayHeader][0] + d, err := time.ParseDuration(val) + if err != nil { + return fmt.Errorf("can't parse %q header: %s", UnconstrainedStreamingDelayHeader, err) + } + maxSleep = int(d) + } + + in := new(testpb.SimpleRequest) + // Receive a message to learn response type and size. + err := stream.RecvMsg(in) + if err == io.EOF { + // read done. + return nil + } + if err != nil { + return err + } + + response := &testpb.SimpleResponse{ + Payload: new(testpb.Payload), + } + setPayload(response.Payload, in.ResponseType, int(in.ResponseSize)) + + preloadedResponse := &grpc.PreparedMsg{} + if preloadMsgSize > 0 { + if err := preloadedResponse.Encode(stream, response); err != nil { + return err + } + } + + go func() { + for { + // Using RecvMsg rather than Recv to prevent reallocation of SimpleRequest. + err := stream.RecvMsg(in) + switch status.Code(err) { + case codes.Canceled: + return + case codes.OK: + default: + log.Fatalf("server recv error: %v", err) + } + } + }() + + go func() { + for { + if maxSleep > 0 { + time.Sleep(time.Duration(rand.IntN(maxSleep))) + } + var err error + if preloadMsgSize > 0 { + err = stream.SendMsg(preloadedResponse) + } else { + err = stream.Send(response) + } + switch status.Code(err) { + case codes.Unavailable, codes.Canceled: + return + case codes.OK: + default: + log.Fatalf("server send error: %v", err) + } + } + }() + + <-stream.Context().Done() + return stream.Context().Err() +} + +// byteBufServer is a gRPC server that sends and receives byte buffer. +// The purpose is to benchmark the gRPC performance without protobuf serialization/deserialization overhead. +type byteBufServer struct { + testgrpc.UnimplementedBenchmarkServiceServer + respSize int32 +} + +// UnaryCall is an empty function and is not used for benchmark. +// If bytebuf UnaryCall benchmark is needed later, the function body needs to be updated. +func (s *byteBufServer) UnaryCall(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil +} + +func (s *byteBufServer) StreamingCall(stream testgrpc.BenchmarkService_StreamingCallServer) error { + for { + var in []byte + err := stream.(grpc.ServerStream).RecvMsg(&in) + if err == io.EOF { + return nil + } + if err != nil { + return err + } + out := make([]byte, s.respSize) + if err := stream.(grpc.ServerStream).SendMsg(&out); err != nil { + return err + } + } +} + +// ServerInfo contains the information to create a gRPC benchmark server. +type ServerInfo struct { + // Type is the type of the server. + // It should be "protobuf" or "bytebuf". + Type string + + // Metadata is an optional configuration. + // For "protobuf", it's ignored. + // For "bytebuf", it should be an int representing response size. + Metadata any + + // Listener is the network listener for the server to use + Listener net.Listener +} + +// StartServer starts a gRPC server serving a benchmark service according to info. +// It returns a function to stop the server. +func StartServer(info ServerInfo, opts ...grpc.ServerOption) func() { + s := grpc.NewServer(opts...) + switch info.Type { + case "protobuf": + testgrpc.RegisterBenchmarkServiceServer(s, &testServer{}) + case "bytebuf": + respSize, ok := info.Metadata.(int32) + if !ok { + logger.Fatalf("failed to StartServer, invalid metadata: %v, for Type: %v", info.Metadata, info.Type) + } + testgrpc.RegisterBenchmarkServiceServer(s, &byteBufServer{respSize: respSize}) + default: + logger.Fatalf("failed to StartServer, unknown Type: %v", info.Type) + } + go s.Serve(info.Listener) + return func() { + s.Stop() + } +} + +// DoUnaryCall performs a unary RPC with given stub and request and response sizes. +func DoUnaryCall(tc testgrpc.BenchmarkServiceClient, reqSize, respSize int) error { + pl := NewPayload(testpb.PayloadType_COMPRESSABLE, reqSize) + req := &testpb.SimpleRequest{ + ResponseType: pl.Type, + ResponseSize: int32(respSize), + Payload: pl, + } + if _, err := tc.UnaryCall(context.Background(), req); err != nil { + return fmt.Errorf("/BenchmarkService/UnaryCall(_, _) = _, %v, want _, ", err) + } + return nil +} + +// DoStreamingRoundTrip performs a round trip for a single streaming rpc. +func DoStreamingRoundTrip(stream testgrpc.BenchmarkService_StreamingCallClient, reqSize, respSize int) error { + pl := NewPayload(testpb.PayloadType_COMPRESSABLE, reqSize) + req := &testpb.SimpleRequest{ + ResponseType: pl.Type, + ResponseSize: int32(respSize), + Payload: pl, + } + return DoStreamingRoundTripPreloaded(stream, req) +} + +// DoStreamingRoundTripPreloaded performs a round trip for a single streaming rpc with preloaded payload. +func DoStreamingRoundTripPreloaded(stream testgrpc.BenchmarkService_StreamingCallClient, req any) error { + // req could be either *testpb.SimpleRequest or *grpc.PreparedMsg + if err := stream.SendMsg(req); err != nil { + return fmt.Errorf("/BenchmarkService/StreamingCall.Send(_) = %v, want ", err) + } + if _, err := stream.Recv(); err != nil { + // EOF is a valid error here. + if err == io.EOF { + return nil + } + return fmt.Errorf("/BenchmarkService/StreamingCall.Recv(_) = %v, want ", err) + } + return nil +} + +// DoByteBufStreamingRoundTrip performs a round trip for a single streaming rpc, using a custom codec for byte buffer. +func DoByteBufStreamingRoundTrip(stream testgrpc.BenchmarkService_StreamingCallClient, reqSize, _ int) error { + out := make([]byte, reqSize) + if err := stream.(grpc.ClientStream).SendMsg(&out); err != nil { + return fmt.Errorf("/BenchmarkService/StreamingCall.(ClientStream).SendMsg(_) = %v, want ", err) + } + var in []byte + if err := stream.(grpc.ClientStream).RecvMsg(&in); err != nil { + // EOF is a valid error here. + if err == io.EOF { + return nil + } + return fmt.Errorf("/BenchmarkService/StreamingCall.(ClientStream).RecvMsg(_) = %v, want ", err) + } + return nil +} + +// NewClientConn creates a gRPC client connection to addr. +func NewClientConn(addr string, opts ...grpc.DialOption) *grpc.ClientConn { + return NewClientConnWithContext(context.Background(), addr, opts...) +} + +// NewClientConnWithContext creates a gRPC client connection to addr using ctx. +func NewClientConnWithContext(_ context.Context, addr string, opts ...grpc.DialOption) *grpc.ClientConn { + conn, err := grpc.NewClient(addr, opts...) + if err != nil { + logger.Fatalf("grpc.NewClient(%q) = %v", addr, err) + } + return conn +} diff --git a/benchmark/benchresult/main.go b/benchmark/benchresult/main.go new file mode 100644 index 0000000..8959855 --- /dev/null +++ b/benchmark/benchresult/main.go @@ -0,0 +1,152 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* +To format the benchmark result: + + go run benchmark/benchresult/main.go resultfile + +To see the performance change based on an old result: + + go run benchmark/benchresult/main.go resultfile_old resultfile + +It will print the comparison result of intersection benchmarks between two files. +*/ +package main + +import ( + "encoding/gob" + "fmt" + "log" + "os" + "strings" + "time" + + "google.golang.org/grpc/benchmark/stats" +) + +func createMap(fileName string) map[string]stats.BenchResults { + f, err := os.Open(fileName) + if err != nil { + log.Fatalf("Read file %s error: %s\n", fileName, err) + } + defer f.Close() + var data []stats.BenchResults + decoder := gob.NewDecoder(f) + if err = decoder.Decode(&data); err != nil { + log.Fatalf("Decode file %s error: %s\n", fileName, err) + } + m := make(map[string]stats.BenchResults) + for _, d := range data { + m[d.RunMode+"-"+d.Features.String()] = d + } + return m +} + +func intChange(title string, val1, val2 uint64) string { + return fmt.Sprintf("%20s %12d %12d %8.2f%%\n", title, val1, val2, float64(int64(val2)-int64(val1))*100/float64(val1)) +} + +func floatChange(title string, val1, val2 float64) string { + return fmt.Sprintf("%20s %12.2f %12.2f %8.2f%%\n", title, val1, val2, float64(int64(val2)-int64(val1))*100/float64(val1)) +} +func timeChange(title string, val1, val2 time.Duration) string { + return fmt.Sprintf("%20s %12s %12s %8.2f%%\n", title, val1.String(), + val2.String(), float64(val2-val1)*100/float64(val1)) +} + +func strDiff(title, val1, val2 string) string { + return fmt.Sprintf("%20s %12s %12s\n", title, val1, val2) +} + +func compareTwoMap(m1, m2 map[string]stats.BenchResults) { + for k2, v2 := range m2 { + if v1, ok := m1[k2]; ok { + changes := k2 + "\n" + changes += fmt.Sprintf("%20s %12s %12s %8s\n", "Title", "Before", "After", "Percentage") + changes += intChange("TotalOps", v1.Data.TotalOps, v2.Data.TotalOps) + changes += intChange("SendOps", v1.Data.SendOps, v2.Data.SendOps) + changes += intChange("RecvOps", v1.Data.RecvOps, v2.Data.RecvOps) + changes += floatChange("Bytes/op", v1.Data.AllocedBytes, v2.Data.AllocedBytes) + changes += floatChange("Allocs/op", v1.Data.Allocs, v2.Data.Allocs) + changes += floatChange("ReqT/op", v1.Data.ReqT, v2.Data.ReqT) + changes += floatChange("RespT/op", v1.Data.RespT, v2.Data.RespT) + changes += timeChange("50th-Lat", v1.Data.Fiftieth, v2.Data.Fiftieth) + changes += timeChange("90th-Lat", v1.Data.Ninetieth, v2.Data.Ninetieth) + changes += timeChange("99th-Lat", v1.Data.NinetyNinth, v2.Data.NinetyNinth) + changes += timeChange("Avg-Lat", v1.Data.Average, v2.Data.Average) + changes += strDiff("GoVersion", v1.GoVersion, v2.GoVersion) + changes += strDiff("GrpcVersion", v1.GrpcVersion, v2.GrpcVersion) + fmt.Printf("%s\n", changes) + } + } +} + +func compareBenchmark(file1, file2 string) { + compareTwoMap(createMap(file1), createMap(file2)) +} + +func printHeader() { + fmt.Printf("%-80s%12s%12s%12s%18s%18s%18s%18s%12s%12s%12s%12s\n", + "Name", "TotalOps", "SendOps", "RecvOps", "Bytes/op (B)", "Allocs/op (#)", + "RequestT", "ResponseT", "L-50", "L-90", "L-99", "L-Avg") +} + +func printline(benchName string, d stats.RunData) { + fmt.Printf("%-80s%12d%12d%12d%18.2f%18.2f%18.2f%18.2f%12v%12v%12v%12v\n", + benchName, d.TotalOps, d.SendOps, d.RecvOps, d.AllocedBytes, d.Allocs, + d.ReqT, d.RespT, d.Fiftieth, d.Ninetieth, d.NinetyNinth, d.Average) +} + +func formatBenchmark(fileName string) { + f, err := os.Open(fileName) + if err != nil { + log.Fatalf("Read file %s error: %s\n", fileName, err) + } + defer f.Close() + var results []stats.BenchResults + decoder := gob.NewDecoder(f) + if err = decoder.Decode(&results); err != nil { + log.Fatalf("Decode file %s error: %s\n", fileName, err) + } + if len(results) == 0 { + log.Fatalf("No benchmark results in file %s\n", fileName) + } + + fmt.Println("\nShared features:\n" + strings.Repeat("-", 20)) + fmt.Print(results[0].Features.SharedFeatures(results[0].SharedFeatures)) + fmt.Println(strings.Repeat("-", 35)) + + wantFeatures := results[0].SharedFeatures + for i := 0; i < len(results[0].SharedFeatures); i++ { + wantFeatures[i] = !wantFeatures[i] + } + + printHeader() + for _, r := range results { + printline(r.RunMode+r.Features.PrintableName(wantFeatures), r.Data) + } +} + +func main() { + if len(os.Args) == 2 { + formatBenchmark(os.Args[1]) + } else { + compareBenchmark(os.Args[1], os.Args[2]) + } +} diff --git a/benchmark/client/main.go b/benchmark/client/main.go new file mode 100644 index 0000000..d742e1a --- /dev/null +++ b/benchmark/client/main.go @@ -0,0 +1,217 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* +Package main provides a client used for benchmarking. Before running the +client, the user would need to launch the grpc server. + +To start the server before running the client, you can run look for the command +under the following file: + + benchmark/server/main.go + +After starting the server, the client can be run. An example of how to run this +command is: + +go run benchmark/client/main.go -test_name=grpc_test + +If the server is running on a different port than 50051, then use the port flag +for the client to hit the server on the correct port. +An example for how to run this command on a different port can be found here: + +go run benchmark/client/main.go -test_name=grpc_test -port=8080 +*/ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "runtime" + "runtime/pprof" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/benchmark" + "google.golang.org/grpc/benchmark/stats" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/syscall" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + port = flag.String("port", "50051", "Localhost port to connect to.") + numRPC = flag.Int("r", 1, "The number of concurrent RPCs on each connection.") + numConn = flag.Int("c", 1, "The number of parallel connections.") + warmupDur = flag.Int("w", 10, "Warm-up duration in seconds") + duration = flag.Int("d", 60, "Benchmark duration in seconds") + rqSize = flag.Int("req", 1, "Request message size in bytes.") + rspSize = flag.Int("resp", 1, "Response message size in bytes.") + rpcType = flag.String("rpc_type", "unary", + `Configure different client rpc type. Valid options are: + unary; + streaming.`) + testName = flag.String("test_name", "", "Name of the test used for creating profiles.") + wg sync.WaitGroup + hopts = stats.HistogramOptions{ + NumBuckets: 2495, + GrowthFactor: .01, + } + mu sync.Mutex + hists []*stats.Histogram + + logger = grpclog.Component("benchmark") +) + +func main() { + flag.Parse() + if *testName == "" { + logger.Fatal("-test_name not set") + } + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(*rspSize), + Payload: &testpb.Payload{ + Type: testpb.PayloadType_COMPRESSABLE, + Body: make([]byte, *rqSize), + }, + } + connectCtx, connectCancel := context.WithDeadline(context.Background(), time.Now().Add(5*time.Second)) + defer connectCancel() + ccs := buildConnections(connectCtx) + warmDeadline := time.Now().Add(time.Duration(*warmupDur) * time.Second) + endDeadline := warmDeadline.Add(time.Duration(*duration) * time.Second) + cf, err := os.Create("/tmp/" + *testName + ".cpu") + if err != nil { + logger.Fatalf("Error creating file: %v", err) + } + defer cf.Close() + pprof.StartCPUProfile(cf) + cpuBeg := syscall.GetCPUTime() + for _, cc := range ccs { + runWithConn(cc, req, warmDeadline, endDeadline) + } + wg.Wait() + cpu := time.Duration(syscall.GetCPUTime() - cpuBeg) + pprof.StopCPUProfile() + mf, err := os.Create("/tmp/" + *testName + ".mem") + if err != nil { + logger.Fatalf("Error creating file: %v", err) + } + defer mf.Close() + runtime.GC() // materialize all statistics + if err := pprof.WriteHeapProfile(mf); err != nil { + logger.Fatalf("Error writing memory profile: %v", err) + } + hist := stats.NewHistogram(hopts) + for _, h := range hists { + hist.Merge(h) + } + parseHist(hist) + fmt.Println("Client CPU utilization:", cpu) + fmt.Println("Client CPU profile:", cf.Name()) + fmt.Println("Client Mem Profile:", mf.Name()) +} + +func buildConnections(ctx context.Context) []*grpc.ClientConn { + ccs := make([]*grpc.ClientConn, *numConn) + for i := range ccs { + ccs[i] = benchmark.NewClientConnWithContext(ctx, "localhost:"+*port, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithBlock(), + grpc.WithWriteBufferSize(128*1024), + grpc.WithReadBufferSize(128*1024), + ) + } + return ccs +} + +func runWithConn(cc *grpc.ClientConn, req *testpb.SimpleRequest, warmDeadline, endDeadline time.Time) { + for i := 0; i < *numRPC; i++ { + wg.Add(1) + go func() { + defer wg.Done() + caller := makeCaller(cc, req) + hist := stats.NewHistogram(hopts) + for { + start := time.Now() + if start.After(endDeadline) { + mu.Lock() + hists = append(hists, hist) + mu.Unlock() + return + } + caller() + elapsed := time.Since(start) + if start.After(warmDeadline) { + hist.Add(elapsed.Nanoseconds()) + } + } + }() + } +} + +func makeCaller(cc *grpc.ClientConn, req *testpb.SimpleRequest) func() { + client := testgrpc.NewBenchmarkServiceClient(cc) + if *rpcType == "unary" { + return func() { + if _, err := client.UnaryCall(context.Background(), req); err != nil { + logger.Fatalf("RPC failed: %v", err) + } + } + } + stream, err := client.StreamingCall(context.Background()) + if err != nil { + logger.Fatalf("RPC failed: %v", err) + } + return func() { + if err := stream.Send(req); err != nil { + logger.Fatalf("Streaming RPC failed to send: %v", err) + } + if _, err := stream.Recv(); err != nil { + logger.Fatalf("Streaming RPC failed to read: %v", err) + } + } +} + +func parseHist(hist *stats.Histogram) { + fmt.Println("qps:", float64(hist.Count)/float64(*duration)) + fmt.Printf("Latency: (50/90/99 %%ile): %v/%v/%v\n", + time.Duration(median(.5, hist)), + time.Duration(median(.9, hist)), + time.Duration(median(.99, hist))) +} + +func median(percentile float64, h *stats.Histogram) int64 { + need := int64(float64(h.Count) * percentile) + have := int64(0) + for _, bucket := range h.Buckets { + count := bucket.Count + if have+count >= need { + percent := float64(need-have) / float64(count) + return int64((1.0-percent)*bucket.LowBound + percent*bucket.LowBound*(1.0+hopts.GrowthFactor)) + } + have += bucket.Count + } + panic("should have found a bound") +} diff --git a/benchmark/flags/flags.go b/benchmark/flags/flags.go new file mode 100644 index 0000000..67ff5fc --- /dev/null +++ b/benchmark/flags/flags.go @@ -0,0 +1,178 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* +Package flags provide convenience types and routines to accept specific types +of flag values on the command line. +*/ +package flags + +import ( + "bytes" + "encoding/csv" + "flag" + "fmt" + "strconv" + "strings" + "time" +) + +// stringFlagWithAllowedValues represents a string flag which can only take a +// predefined set of values. +type stringFlagWithAllowedValues struct { + val string + allowed []string +} + +// StringWithAllowedValues returns a flag variable of type +// stringFlagWithAllowedValues configured with the provided parameters. +// 'allowed` is the set of values that this flag can be set to. +func StringWithAllowedValues(name, defaultVal, usage string, allowed []string) *string { + as := &stringFlagWithAllowedValues{defaultVal, allowed} + flag.CommandLine.Var(as, name, usage) + return &as.val +} + +// String implements the flag.Value interface. +func (as *stringFlagWithAllowedValues) String() string { + return as.val +} + +// Set implements the flag.Value interface. +func (as *stringFlagWithAllowedValues) Set(val string) error { + for _, a := range as.allowed { + if a == val { + as.val = val + return nil + } + } + return fmt.Errorf("want one of: %v", strings.Join(as.allowed, ", ")) +} + +type durationSliceValue []time.Duration + +// DurationSlice returns a flag representing a slice of time.Duration objects. +func DurationSlice(name string, defaultVal []time.Duration, usage string) *[]time.Duration { + ds := make([]time.Duration, len(defaultVal)) + copy(ds, defaultVal) + dsv := (*durationSliceValue)(&ds) + flag.CommandLine.Var(dsv, name, usage) + return &ds +} + +// Set implements the flag.Value interface. +func (dsv *durationSliceValue) Set(s string) error { + ds := strings.Split(s, ",") + var dd []time.Duration + for _, n := range ds { + d, err := time.ParseDuration(n) + if err != nil { + return err + } + dd = append(dd, d) + } + *dsv = durationSliceValue(dd) + return nil +} + +// String implements the flag.Value interface. +func (dsv *durationSliceValue) String() string { + var b bytes.Buffer + for i, d := range *dsv { + if i > 0 { + b.WriteRune(',') + } + b.WriteString(d.String()) + } + return b.String() +} + +type intSliceValue []int + +// IntSlice returns a flag representing a slice of ints. +func IntSlice(name string, defaultVal []int, usage string) *[]int { + is := make([]int, len(defaultVal)) + copy(is, defaultVal) + isv := (*intSliceValue)(&is) + flag.CommandLine.Var(isv, name, usage) + return &is +} + +// Set implements the flag.Value interface. +func (isv *intSliceValue) Set(s string) error { + is := strings.Split(s, ",") + var ret []int + for _, n := range is { + i, err := strconv.Atoi(n) + if err != nil { + return err + } + ret = append(ret, i) + } + *isv = intSliceValue(ret) + return nil +} + +// String implements the flag.Value interface. +func (isv *intSliceValue) String() string { + var b bytes.Buffer + for i, n := range *isv { + if i > 0 { + b.WriteRune(',') + } + b.WriteString(strconv.Itoa(n)) + } + return b.String() +} + +type stringSliceValue []string + +// StringSlice returns a flag representing a slice of strings. +func StringSlice(name string, defaultVal []string, usage string) *[]string { + ss := make([]string, len(defaultVal)) + copy(ss, defaultVal) + ssv := (*stringSliceValue)(&ss) + flag.CommandLine.Var(ssv, name, usage) + return &ss +} + +// escapedCommaSplit splits a comma-separated list of strings in the same way +// CSV files work (escaping a comma requires double-quotes). +func escapedCommaSplit(str string) ([]string, error) { + r := csv.NewReader(strings.NewReader(str)) + ret, err := r.Read() + if err != nil { + return nil, err + } + return ret, nil +} + +// Set implements the flag.Value interface. +func (ss *stringSliceValue) Set(str string) error { + var err error + *ss, err = escapedCommaSplit(str) + if err != nil { + return err + } + return nil +} + +// String implements the flag.Value interface. +func (ss *stringSliceValue) String() string { + return strings.Join(*ss, ",") +} diff --git a/benchmark/flags/flags_test.go b/benchmark/flags/flags_test.go new file mode 100644 index 0000000..115b492 --- /dev/null +++ b/benchmark/flags/flags_test.go @@ -0,0 +1,153 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package flags + +import ( + "flag" + "reflect" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestStringWithAllowedValues(t *testing.T) { + const defaultVal = "default" + tests := []struct { + args string + allowed []string + wantVal string + wantErr bool + }{ + {"-workloads=all", []string{"unary", "streaming", "all"}, "all", false}, + {"-workloads=disallowed", []string{"unary", "streaming", "all"}, defaultVal, true}, + } + + for _, test := range tests { + flag.CommandLine = flag.NewFlagSet("test", flag.ContinueOnError) + var w = StringWithAllowedValues("workloads", defaultVal, "usage", test.allowed) + err := flag.CommandLine.Parse([]string{test.args}) + switch { + case !test.wantErr && err != nil: + t.Errorf("failed to parse command line args {%v}: %v", test.args, err) + case test.wantErr && err == nil: + t.Errorf("flag.Parse(%v) = nil, want non-nil error", test.args) + default: + if *w != test.wantVal { + t.Errorf("flag value is %v, want %v", *w, test.wantVal) + } + } + } +} + +func (s) TestDurationSlice(t *testing.T) { + defaultVal := []time.Duration{time.Second, time.Nanosecond} + tests := []struct { + args string + wantVal []time.Duration + wantErr bool + }{ + {"-latencies=1s", []time.Duration{time.Second}, false}, + {"-latencies=1s,2s,3s", []time.Duration{time.Second, 2 * time.Second, 3 * time.Second}, false}, + {"-latencies=bad", defaultVal, true}, + } + + for _, test := range tests { + flag.CommandLine = flag.NewFlagSet("test", flag.ContinueOnError) + var w = DurationSlice("latencies", defaultVal, "usage") + err := flag.CommandLine.Parse([]string{test.args}) + switch { + case !test.wantErr && err != nil: + t.Errorf("failed to parse command line args {%v}: %v", test.args, err) + case test.wantErr && err == nil: + t.Errorf("flag.Parse(%v) = nil, want non-nil error", test.args) + default: + if !reflect.DeepEqual(*w, test.wantVal) { + t.Errorf("flag value is %v, want %v", *w, test.wantVal) + } + } + } +} + +func (s) TestIntSlice(t *testing.T) { + defaultVal := []int{1, 1024} + tests := []struct { + args string + wantVal []int + wantErr bool + }{ + {"-kbps=1", []int{1}, false}, + {"-kbps=1,2,3", []int{1, 2, 3}, false}, + {"-kbps=20e4", defaultVal, true}, + } + + for _, test := range tests { + flag.CommandLine = flag.NewFlagSet("test", flag.ContinueOnError) + var w = IntSlice("kbps", defaultVal, "usage") + err := flag.CommandLine.Parse([]string{test.args}) + switch { + case !test.wantErr && err != nil: + t.Errorf("failed to parse command line args {%v}: %v", test.args, err) + case test.wantErr && err == nil: + t.Errorf("flag.Parse(%v) = nil, want non-nil error", test.args) + default: + if !reflect.DeepEqual(*w, test.wantVal) { + t.Errorf("flag value is %v, want %v", *w, test.wantVal) + } + } + } +} + +func (s) TestStringSlice(t *testing.T) { + defaultVal := []string{"bar", "baz"} + tests := []struct { + args string + wantVal []string + wantErr bool + }{ + {"-name=foobar", []string{"foobar"}, false}, + {"-name=foo,bar", []string{"foo", "bar"}, false}, + {`-name="foo,bar",baz`, []string{"foo,bar", "baz"}, false}, + {`-name="foo,bar""",baz`, []string{`foo,bar"`, "baz"}, false}, + } + + for _, test := range tests { + flag.CommandLine = flag.NewFlagSet("test", flag.ContinueOnError) + var w = StringSlice("name", defaultVal, "usage") + err := flag.CommandLine.Parse([]string{test.args}) + switch { + case !test.wantErr && err != nil: + t.Errorf("failed to parse command line args {%v}: %v", test.args, err) + case test.wantErr && err == nil: + t.Errorf("flag.Parse(%v) = nil, want non-nil error", test.args) + default: + if !reflect.DeepEqual(*w, test.wantVal) { + t.Errorf("flag value is %v, want %v", *w, test.wantVal) + } + } + } +} diff --git a/benchmark/latency/latency.go b/benchmark/latency/latency.go new file mode 100644 index 0000000..4fae637 --- /dev/null +++ b/benchmark/latency/latency.go @@ -0,0 +1,334 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package latency provides wrappers for net.Conn, net.Listener, and +// net.Dialers, designed to interoperate to inject real-world latency into +// network connections. +package latency + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "io" + "net" + "time" +) + +// Dialer is a function matching the signature of net.Dial. +type Dialer func(network, address string) (net.Conn, error) + +// TimeoutDialer is a function matching the signature of net.DialTimeout. +type TimeoutDialer func(network, address string, timeout time.Duration) (net.Conn, error) + +// ContextDialer is a function matching the signature of +// net.Dialer.DialContext. +type ContextDialer func(ctx context.Context, network, address string) (net.Conn, error) + +// Network represents a network with the given bandwidth, latency, and MTU +// (Maximum Transmission Unit) configuration, and can produce wrappers of +// net.Listeners, net.Conn, and various forms of dialing functions. The +// Listeners and Dialers/Conns on both sides of connections must come from this +// package, but need not be created from the same Network. Latency is computed +// when sending (in Write), and is injected when receiving (in Read). This +// allows senders' Write calls to be non-blocking, as in real-world +// applications. +// +// Note: Latency is injected by the sender specifying the absolute time data +// should be available, and the reader delaying until that time arrives to +// provide the data. This package attempts to counter-act the effects of clock +// drift and existing network latency by measuring the delay between the +// sender's transmission time and the receiver's reception time during startup. +// No attempt is made to measure the existing bandwidth of the connection. +type Network struct { + Kbps int // Kilobits per second; if non-positive, infinite + Latency time.Duration // One-way latency (sending); if non-positive, no delay + MTU int // Bytes per packet; if non-positive, infinite +} + +var ( + // Local simulates local network. + Local = Network{0, 0, 0} + // LAN simulates local area network. + LAN = Network{100 * 1024, 2 * time.Millisecond, 1500} + // WAN simulates wide area network. + WAN = Network{20 * 1024, 30 * time.Millisecond, 1500} + // Longhaul simulates bad network. + Longhaul = Network{1000 * 1024, 200 * time.Millisecond, 9000} +) + +func (n *Network) isLocal() bool { + return *n == Local +} + +// Conn returns a net.Conn that wraps c and injects n's latency into that +// connection. This function also imposes latency for connection creation. +// If n's Latency is lower than the measured latency in c, an error is +// returned. +func (n *Network) Conn(c net.Conn) (net.Conn, error) { + if n.isLocal() { + return c, nil + } + start := now() + nc := &conn{Conn: c, network: n, readBuf: new(bytes.Buffer)} + if err := nc.sync(); err != nil { + return nil, err + } + sleep(start.Add(nc.delay).Sub(now())) + return nc, nil +} + +type conn struct { + net.Conn + network *Network + + readBuf *bytes.Buffer // one packet worth of data received + lastSendEnd time.Time // time the previous Write should be fully on the wire + delay time.Duration // desired latency - measured latency +} + +// header is sent before all data transmitted by the application. +type header struct { + ReadTime int64 // Time the reader is allowed to read this packet (UnixNano) + Sz int32 // Size of the data in the packet +} + +func (c *conn) Write(p []byte) (n int, err error) { + tNow := now() + if c.lastSendEnd.Before(tNow) { + c.lastSendEnd = tNow + } + for len(p) > 0 { + pkt := p + if c.network.MTU > 0 && len(pkt) > c.network.MTU { + pkt = pkt[:c.network.MTU] + p = p[c.network.MTU:] + } else { + p = nil + } + if c.network.Kbps > 0 { + if congestion := c.lastSendEnd.Sub(tNow) - c.delay; congestion > 0 { + // The network is full; sleep until this packet can be sent. + sleep(congestion) + tNow = tNow.Add(congestion) + } + } + c.lastSendEnd = c.lastSendEnd.Add(c.network.pktTime(len(pkt))) + hdr := header{ReadTime: c.lastSendEnd.Add(c.delay).UnixNano(), Sz: int32(len(pkt))} + if err := binary.Write(c.Conn, binary.BigEndian, hdr); err != nil { + return n, err + } + x, err := c.Conn.Write(pkt) + n += x + if err != nil { + return n, err + } + } + return n, nil +} + +func (c *conn) Read(p []byte) (n int, err error) { + if c.readBuf.Len() == 0 { + var hdr header + if err := binary.Read(c.Conn, binary.BigEndian, &hdr); err != nil { + return 0, err + } + defer func() { sleep(time.Unix(0, hdr.ReadTime).Sub(now())) }() + + if _, err := io.CopyN(c.readBuf, c.Conn, int64(hdr.Sz)); err != nil { + return 0, err + } + } + // Read from readBuf. + return c.readBuf.Read(p) +} + +// sync does a handshake and then measures the latency on the network in +// coordination with the other side. +func (c *conn) sync() error { + const ( + pingMsg = "syncPing" + warmup = 10 // minimum number of iterations to measure latency + giveUp = 50 // maximum number of iterations to measure latency + accuracy = time.Millisecond // req'd accuracy to stop early + goodRun = 3 // stop early if latency within accuracy this many times + ) + + type syncMsg struct { + SendT int64 // Time sent. If zero, stop. + RecvT int64 // Time received. If zero, fill in and respond. + } + + // A trivial handshake + if err := binary.Write(c.Conn, binary.BigEndian, []byte(pingMsg)); err != nil { + return err + } + var ping [8]byte + if err := binary.Read(c.Conn, binary.BigEndian, &ping); err != nil { + return err + } else if string(ping[:]) != pingMsg { + return fmt.Errorf("malformed handshake message: %v (want %q)", ping, pingMsg) + } + + // Both sides are alive and syncing. Calculate network delay / clock skew. + att := 0 + good := 0 + var latency time.Duration + localDone, remoteDone := false, false + send := true + for !localDone || !remoteDone { + if send { + if err := binary.Write(c.Conn, binary.BigEndian, syncMsg{SendT: now().UnixNano()}); err != nil { + return err + } + att++ + send = false + } + + // Block until we get a syncMsg + m := syncMsg{} + if err := binary.Read(c.Conn, binary.BigEndian, &m); err != nil { + return err + } + + if m.RecvT == 0 { + // Message initiated from other side. + if m.SendT == 0 { + remoteDone = true + continue + } + // Send response. + m.RecvT = now().UnixNano() + if err := binary.Write(c.Conn, binary.BigEndian, m); err != nil { + return err + } + continue + } + + lag := time.Duration(m.RecvT - m.SendT) + latency += lag + avgLatency := latency / time.Duration(att) + if e := lag - avgLatency; e > -accuracy && e < accuracy { + good++ + } else { + good = 0 + } + if att < giveUp && (att < warmup || good < goodRun) { + send = true + continue + } + localDone = true + latency = avgLatency + // Tell the other side we're done. + if err := binary.Write(c.Conn, binary.BigEndian, syncMsg{}); err != nil { + return err + } + } + if c.network.Latency <= 0 { + return nil + } + c.delay = c.network.Latency - latency + if c.delay < 0 { + return fmt.Errorf("measured network latency (%v) higher than desired latency (%v)", latency, c.network.Latency) + } + return nil +} + +// Listener returns a net.Listener that wraps l and injects n's latency in its +// connections. +func (n *Network) Listener(l net.Listener) net.Listener { + if n.isLocal() { + return l + } + return &listener{Listener: l, network: n} +} + +type listener struct { + net.Listener + network *Network +} + +func (l *listener) Accept() (net.Conn, error) { + c, err := l.Listener.Accept() + if err != nil { + return nil, err + } + return l.network.Conn(c) +} + +// Dialer returns a Dialer that wraps d and injects n's latency in its +// connections. n's Latency is also injected to the connection's creation. +func (n *Network) Dialer(d Dialer) Dialer { + if n.isLocal() { + return d + } + return func(network, address string) (net.Conn, error) { + conn, err := d(network, address) + if err != nil { + return nil, err + } + return n.Conn(conn) + } +} + +// TimeoutDialer returns a TimeoutDialer that wraps d and injects n's latency +// in its connections. n's Latency is also injected to the connection's +// creation. +func (n *Network) TimeoutDialer(d TimeoutDialer) TimeoutDialer { + if n.isLocal() { + return d + } + return func(network, address string, timeout time.Duration) (net.Conn, error) { + conn, err := d(network, address, timeout) + if err != nil { + return nil, err + } + return n.Conn(conn) + } +} + +// ContextDialer returns a ContextDialer that wraps d and injects n's latency +// in its connections. n's Latency is also injected to the connection's +// creation. +func (n *Network) ContextDialer(d ContextDialer) ContextDialer { + if n.isLocal() { + return d + } + return func(ctx context.Context, network, address string) (net.Conn, error) { + conn, err := d(ctx, network, address) + if err != nil { + return nil, err + } + return n.Conn(conn) + } +} + +// pktTime returns the time it takes to transmit one packet of data of size b +// in bytes. +func (n *Network) pktTime(b int) time.Duration { + if n.Kbps <= 0 { + return time.Duration(0) + } + return time.Duration(b) * time.Second / time.Duration(n.Kbps*(1024/8)) +} + +// Wrappers for testing + +var now = time.Now +var sleep = time.Sleep diff --git a/benchmark/latency/latency_test.go b/benchmark/latency/latency_test.go new file mode 100644 index 0000000..dadd845 --- /dev/null +++ b/benchmark/latency/latency_test.go @@ -0,0 +1,363 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package latency + +import ( + "bytes" + "fmt" + "net" + "reflect" + "sync" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// bufConn is a net.Conn implemented by a bytes.Buffer (which is a ReadWriter). +type bufConn struct { + *bytes.Buffer +} + +func (bufConn) Close() error { panic("unimplemented") } +func (bufConn) LocalAddr() net.Addr { panic("unimplemented") } +func (bufConn) RemoteAddr() net.Addr { panic("unimplemented") } +func (bufConn) SetDeadline(time.Time) error { panic("unimplemented") } +func (bufConn) SetReadDeadline(time.Time) error { panic("unimplemented") } +func (bufConn) SetWriteDeadline(time.Time) error { panic("unimplemented") } + +func restoreHooks() func() { + s := sleep + n := now + return func() { + sleep = s + now = n + } +} + +func (s) TestConn(t *testing.T) { + defer restoreHooks()() + + // Constant time. + now = func() time.Time { return time.Unix(123, 456) } + + // Capture sleep times for checking later. + var sleepTimes []time.Duration + sleep = func(t time.Duration) { sleepTimes = append(sleepTimes, t) } + + wantSleeps := func(want ...time.Duration) { + if !reflect.DeepEqual(want, sleepTimes) { + t.Fatalf("sleepTimes = %v; want %v", sleepTimes, want) + } + sleepTimes = nil + } + + // Use a fairly high latency to cause a large BDP and avoid sleeps while + // writing due to simulation of full buffers. + latency := 1 * time.Second + c, err := (&Network{Kbps: 1, Latency: latency, MTU: 5}).Conn(bufConn{&bytes.Buffer{}}) + if err != nil { + t.Fatalf("Unexpected error creating connection: %v", err) + } + wantSleeps(latency) // Connection creation delay. + + // 1 kbps = 128 Bps. Divides evenly by 1 second using nanos. + byteLatency := time.Second / 128 + + write := func(b []byte) { + n, err := c.Write(b) + if n != len(b) || err != nil { + t.Fatalf("c.Write(%v) = %v, %v; want %v, nil", b, n, err, len(b)) + } + } + + write([]byte{1, 2, 3, 4, 5}) // One full packet + pkt1Time := latency + byteLatency*5 + write([]byte{6}) // One partial packet + pkt2Time := pkt1Time + byteLatency + write([]byte{7, 8, 9, 10, 11, 12, 13}) // Two packets + pkt3Time := pkt2Time + byteLatency*5 + pkt4Time := pkt3Time + byteLatency*2 + + // No reads, so no sleeps yet. + wantSleeps() + + read := func(n int, want []byte) { + b := make([]byte, n) + if rd, err := c.Read(b); err != nil || rd != len(want) { + t.Fatalf("c.Read(<%v bytes>) = %v, %v; want %v, nil", n, rd, err, len(want)) + } + if !reflect.DeepEqual(b[:len(want)], want) { + t.Fatalf("read %v; want %v", b, want) + } + } + + read(1, []byte{1}) + wantSleeps(pkt1Time) + read(1, []byte{2}) + wantSleeps() + read(3, []byte{3, 4, 5}) + wantSleeps() + read(2, []byte{6}) + wantSleeps(pkt2Time) + read(2, []byte{7, 8}) + wantSleeps(pkt3Time) + read(10, []byte{9, 10, 11}) + wantSleeps() + read(10, []byte{12, 13}) + wantSleeps(pkt4Time) +} + +func (s) TestSync(t *testing.T) { + defer restoreHooks()() + + // Infinitely fast CPU: time doesn't pass unless sleep is called. + tn := time.Unix(123, 0) + now = func() time.Time { return tn } + sleep = func(d time.Duration) { tn = tn.Add(d) } + + // Simulate a 20ms latency network, then run sync across that and expect to + // measure 20ms latency, or 10ms additional delay for a 30ms network. + slowConn, err := (&Network{Kbps: 0, Latency: 20 * time.Millisecond, MTU: 5}).Conn(bufConn{&bytes.Buffer{}}) + if err != nil { + t.Fatalf("Unexpected error creating connection: %v", err) + } + c, err := (&Network{Latency: 30 * time.Millisecond}).Conn(slowConn) + if err != nil { + t.Fatalf("Unexpected error creating connection: %v", err) + } + if c.(*conn).delay != 10*time.Millisecond { + t.Fatalf("c.delay = %v; want 10ms", c.(*conn).delay) + } +} + +func (s) TestSyncTooSlow(t *testing.T) { + defer restoreHooks()() + + // Infinitely fast CPU: time doesn't pass unless sleep is called. + tn := time.Unix(123, 0) + now = func() time.Time { return tn } + sleep = func(d time.Duration) { tn = tn.Add(d) } + + // Simulate a 10ms latency network, then attempt to simulate a 5ms latency + // network and expect an error. + slowConn, err := (&Network{Kbps: 0, Latency: 10 * time.Millisecond, MTU: 5}).Conn(bufConn{&bytes.Buffer{}}) + if err != nil { + t.Fatalf("Unexpected error creating connection: %v", err) + } + + errWant := "measured network latency (10ms) higher than desired latency (5ms)" + if _, err := (&Network{Latency: 5 * time.Millisecond}).Conn(slowConn); err == nil || err.Error() != errWant { + t.Fatalf("Conn() = _, %q; want _, %q", err, errWant) + } +} + +func (s) TestListenerAndDialer(t *testing.T) { + defer restoreHooks()() + + tn := time.Unix(123, 0) + startTime := tn + mu := &sync.Mutex{} + now = func() time.Time { + mu.Lock() + defer mu.Unlock() + return tn + } + + // Use a fairly high latency to cause a large BDP and avoid sleeps while + // writing due to simulation of full buffers. + n := &Network{Kbps: 2, Latency: 1 * time.Second, MTU: 10} + // 2 kbps = .25 kBps = 256 Bps + byteLatency := func(n int) time.Duration { + return time.Duration(n) * time.Second / 256 + } + + // Create a real listener and wrap it. + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Unexpected error creating listener: %v", err) + } + defer l.Close() + l = n.Listener(l) + + var serverConn net.Conn + var scErr error + scDone := make(chan struct{}) + go func() { + serverConn, scErr = l.Accept() + close(scDone) + }() + + // Create a dialer and use it. + clientConn, err := n.TimeoutDialer(net.DialTimeout)("tcp", l.Addr().String(), 2*time.Second) + if err != nil { + t.Fatalf("Unexpected error dialing: %v", err) + } + defer clientConn.Close() + + // Block until server's Conn is available. + <-scDone + if scErr != nil { + t.Fatalf("Unexpected error listening: %v", scErr) + } + defer serverConn.Close() + + // sleep (only) advances tn. Done after connections established so sync detects zero delay. + sleep = func(d time.Duration) { + mu.Lock() + defer mu.Unlock() + if d > 0 { + tn = tn.Add(d) + } + } + + seq := func(a, b int) []byte { + buf := make([]byte, b-a) + for i := 0; i < b-a; i++ { + buf[i] = byte(i + a) + } + return buf + } + + pkt1 := seq(0, 10) + pkt2 := seq(10, 30) + pkt3 := seq(30, 35) + + write := func(c net.Conn, b []byte) { + n, err := c.Write(b) + if n != len(b) || err != nil { + t.Fatalf("c.Write(%v) = %v, %v; want %v, nil", b, n, err, len(b)) + } + } + + write(serverConn, pkt1) + write(serverConn, pkt2) + write(serverConn, pkt3) + write(clientConn, pkt3) + write(clientConn, pkt1) + write(clientConn, pkt2) + + if tn != startTime { + t.Fatalf("unexpected sleep in write; tn = %v; want %v", tn, startTime) + } + + read := func(c net.Conn, n int, want []byte, timeWant time.Time) { + b := make([]byte, n) + if rd, err := c.Read(b); err != nil || rd != len(want) { + t.Fatalf("c.Read(<%v bytes>) = %v, %v; want %v, nil (read: %v)", n, rd, err, len(want), b[:rd]) + } + if !reflect.DeepEqual(b[:len(want)], want) { + t.Fatalf("read %v; want %v", b, want) + } + if !tn.Equal(timeWant) { + t.Errorf("tn after read(%v) = %v; want %v", want, tn, timeWant) + } + } + + read(clientConn, len(pkt1)+1, pkt1, startTime.Add(n.Latency+byteLatency(len(pkt1)))) + read(serverConn, len(pkt3)+1, pkt3, tn) // tn was advanced by the above read; pkt3 is shorter than pkt1 + + read(clientConn, len(pkt2), pkt2[:10], startTime.Add(n.Latency+byteLatency(len(pkt1)+10))) + read(clientConn, len(pkt2), pkt2[10:], startTime.Add(n.Latency+byteLatency(len(pkt1)+len(pkt2)))) + read(clientConn, len(pkt3), pkt3, startTime.Add(n.Latency+byteLatency(len(pkt1)+len(pkt2)+len(pkt3)))) + + read(serverConn, len(pkt1), pkt1, tn) // tn already past the arrival time due to prior reads + read(serverConn, len(pkt2), pkt2[:10], tn) + read(serverConn, len(pkt2), pkt2[10:], tn) + + // Sleep awhile and make sure the read happens disregarding previous writes + // (lastSendEnd handling). + sleep(10 * time.Second) + write(clientConn, pkt1) + read(serverConn, len(pkt1), pkt1, tn.Add(n.Latency+byteLatency(len(pkt1)))) + + // Send, sleep longer than the network delay, then make sure the read happens + // instantly. + write(serverConn, pkt1) + sleep(10 * time.Second) + read(clientConn, len(pkt1), pkt1, tn) +} + +func (s) TestBufferBloat(t *testing.T) { + defer restoreHooks()() + + // Infinitely fast CPU: time doesn't pass unless sleep is called. + tn := time.Unix(123, 0) + now = func() time.Time { return tn } + // Capture sleep times for checking later. + var sleepTimes []time.Duration + sleep = func(d time.Duration) { + sleepTimes = append(sleepTimes, d) + tn = tn.Add(d) + } + + wantSleeps := func(want ...time.Duration) error { + if !reflect.DeepEqual(want, sleepTimes) { + return fmt.Errorf("sleepTimes = %v; want %v", sleepTimes, want) + } + sleepTimes = nil + return nil + } + + n := &Network{Kbps: 8 /* 1KBps */, Latency: time.Second, MTU: 8} + bdpBytes := (n.Kbps * 1024 / 8) * int(n.Latency/time.Second) // 1024 + c, err := n.Conn(bufConn{&bytes.Buffer{}}) + if err != nil { + t.Fatalf("Unexpected error creating connection: %v", err) + } + wantSleeps(n.Latency) // Connection creation delay. + + write := func(n int, sleeps ...time.Duration) { + if wt, err := c.Write(make([]byte, n)); err != nil || wt != n { + t.Fatalf("c.Write(<%v bytes>) = %v, %v; want %v, nil", n, wt, err, n) + } + if err := wantSleeps(sleeps...); err != nil { + t.Fatalf("After writing %v bytes: %v", n, err) + } + } + + read := func(n int, sleeps ...time.Duration) { + if rd, err := c.Read(make([]byte, n)); err != nil || rd != n { + t.Fatalf("c.Read(_) = %v, %v; want %v, nil", rd, err, n) + } + if err := wantSleeps(sleeps...); err != nil { + t.Fatalf("After reading %v bytes: %v", n, err) + } + } + + write(8) // No reads and buffer not full, so no sleeps yet. + read(8, time.Second+n.pktTime(8)) + + write(bdpBytes) // Fill the buffer. + write(1) // We can send one extra packet even when the buffer is full. + write(n.MTU, n.pktTime(1)) // Make sure we sleep to clear the previous write. + write(1, n.pktTime(n.MTU)) + write(n.MTU+1, n.pktTime(1), n.pktTime(n.MTU)) + + tn = tn.Add(10 * time.Second) // Wait long enough for the buffer to clear. + write(bdpBytes) // No sleeps required. +} diff --git a/benchmark/primitives/code_string_test.go b/benchmark/primitives/code_string_test.go new file mode 100644 index 0000000..d5270ee --- /dev/null +++ b/benchmark/primitives/code_string_test.go @@ -0,0 +1,125 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package primitives_test + +import ( + "strconv" + "testing" + + "google.golang.org/grpc/codes" +) + +type codeBench uint32 + +const ( + OK codeBench = iota + Canceled + Unknown + InvalidArgument + DeadlineExceeded + NotFound + AlreadyExists + PermissionDenied + ResourceExhausted + FailedPrecondition + Aborted + OutOfRange + Unimplemented + Internal + Unavailable + DataLoss + Unauthenticated +) + +// The following String() function was generated by stringer. +const codeName = "OKCanceledUnknownInvalidArgumentDeadlineExceededNotFoundAlreadyExistsPermissionDeniedResourceExhaustedFailedPreconditionAbortedOutOfRangeUnimplementedInternalUnavailableDataLossUnauthenticated" + +var codeIndex = [...]uint8{0, 2, 10, 17, 32, 48, 56, 69, 85, 102, 120, 127, 137, 150, 158, 169, 177, 192} + +func (i codeBench) String() string { + if i >= codeBench(len(codeIndex)-1) { + return "Code(" + strconv.FormatInt(int64(i), 10) + ")" + } + return codeName[codeIndex[i]:codeIndex[i+1]] +} + +var nameMap = map[codeBench]string{ + OK: "OK", + Canceled: "Canceled", + Unknown: "Unknown", + InvalidArgument: "InvalidArgument", + DeadlineExceeded: "DeadlineExceeded", + NotFound: "NotFound", + AlreadyExists: "AlreadyExists", + PermissionDenied: "PermissionDenied", + ResourceExhausted: "ResourceExhausted", + FailedPrecondition: "FailedPrecondition", + Aborted: "Aborted", + OutOfRange: "OutOfRange", + Unimplemented: "Unimplemented", + Internal: "Internal", + Unavailable: "Unavailable", + DataLoss: "DataLoss", + Unauthenticated: "Unauthenticated", +} + +func (i codeBench) StringUsingMap() string { + if s, ok := nameMap[i]; ok { + return s + } + return "Code(" + strconv.FormatInt(int64(i), 10) + ")" +} + +func BenchmarkCodeStringStringer(b *testing.B) { + for i := 0; i < b.N; i++ { + c := codeBench(uint32(i % 17)) + _ = c.String() + } +} + +func BenchmarkCodeStringMap(b *testing.B) { + for i := 0; i < b.N; i++ { + c := codeBench(uint32(i % 17)) + _ = c.StringUsingMap() + } +} + +// codes.Code.String() does a switch. +func BenchmarkCodeStringSwitch(b *testing.B) { + for i := 0; i < b.N; i++ { + c := codes.Code(uint32(i % 17)) + _ = c.String() + } +} + +// Testing all codes (0<=c<=16) and also one overflow (17). +func BenchmarkCodeStringStringerWithOverflow(b *testing.B) { + for i := 0; i < b.N; i++ { + c := codeBench(uint32(i % 18)) + _ = c.String() + } +} + +// Testing all codes (0<=c<=16) and also one overflow (17). +func BenchmarkCodeStringSwitchWithOverflow(b *testing.B) { + for i := 0; i < b.N; i++ { + c := codes.Code(uint32(i % 18)) + _ = c.String() + } +} diff --git a/benchmark/primitives/context_test.go b/benchmark/primitives/context_test.go new file mode 100644 index 0000000..faae507 --- /dev/null +++ b/benchmark/primitives/context_test.go @@ -0,0 +1,152 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package primitives_test + +import ( + "context" + "testing" + "time" +) + +const defaultTestTimeout = 10 * time.Second + +func BenchmarkCancelContextErrNoErr(b *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + for i := 0; i < b.N; i++ { + if err := ctx.Err(); err != nil { + b.Fatal("error") + } + } + cancel() +} + +func BenchmarkCancelContextErrGotErr(b *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + for i := 0; i < b.N; i++ { + if err := ctx.Err(); err == nil { + b.Fatal("error") + } + } +} + +func BenchmarkCancelContextChannelNoErr(b *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + for i := 0; i < b.N; i++ { + select { + case <-ctx.Done(): + b.Fatal("error: ctx.Done():", ctx.Err()) + default: + } + } + cancel() +} + +func BenchmarkCancelContextChannelGotErr(b *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + for i := 0; i < b.N; i++ { + select { + case <-ctx.Done(): + if err := ctx.Err(); err == nil { + b.Fatal("error") + } + default: + b.Fatal("error: !ctx.Done()") + } + } +} + +func BenchmarkTimerContextErrNoErr(b *testing.B) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + for i := 0; i < b.N; i++ { + if err := ctx.Err(); err != nil { + b.Fatal("error") + } + } + cancel() +} + +func BenchmarkTimerContextErrGotErr(b *testing.B) { + ctx, cancel := context.WithTimeout(context.Background(), time.Microsecond) + cancel() + for i := 0; i < b.N; i++ { + if err := ctx.Err(); err == nil { + b.Fatal("error") + } + } +} + +func BenchmarkTimerContextChannelNoErr(b *testing.B) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + for i := 0; i < b.N; i++ { + select { + case <-ctx.Done(): + b.Fatal("error: ctx.Done():", ctx.Err()) + default: + } + } + cancel() +} + +func BenchmarkTimerContextChannelGotErr(b *testing.B) { + ctx, cancel := context.WithTimeout(context.Background(), time.Microsecond) + cancel() + for i := 0; i < b.N; i++ { + select { + case <-ctx.Done(): + if err := ctx.Err(); err == nil { + b.Fatal("error") + } + default: + b.Fatal("error: !ctx.Done()") + } + } +} + +type ctxKey struct{} + +func newContextWithLocalKey(parent context.Context) context.Context { + return context.WithValue(parent, ctxKey{}, nil) +} + +var ck = ctxKey{} + +func newContextWithGlobalKey(parent context.Context) context.Context { + return context.WithValue(parent, ck, nil) +} + +func BenchmarkContextWithValue(b *testing.B) { + benches := []struct { + name string + f func(context.Context) context.Context + }{ + {"newContextWithLocalKey", newContextWithLocalKey}, + {"newContextWithGlobalKey", newContextWithGlobalKey}, + } + + pCtx := context.Background() + for _, bench := range benches { + b.Run(bench.name, func(b *testing.B) { + for j := 0; j < b.N; j++ { + bench.f(pCtx) + } + }) + } +} diff --git a/benchmark/primitives/primitives_test.go b/benchmark/primitives/primitives_test.go new file mode 100644 index 0000000..5d7c810 --- /dev/null +++ b/benchmark/primitives/primitives_test.go @@ -0,0 +1,466 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package primitives_test contains benchmarks for various synchronization primitives +// available in Go. +package primitives_test + +import ( + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + "unsafe" +) + +func BenchmarkSelectClosed(b *testing.B) { + c := make(chan struct{}) + close(c) + x := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + select { + case <-c: + x++ + default: + } + } + b.StopTimer() + if x != b.N { + b.Fatal("error") + } +} + +func BenchmarkSelectOpen(b *testing.B) { + c := make(chan struct{}) + x := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + select { + case <-c: + default: + x++ + } + } + b.StopTimer() + if x != b.N { + b.Fatal("error") + } +} + +func BenchmarkAtomicBool(b *testing.B) { + c := int32(0) + x := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + if atomic.LoadInt32(&c) == 0 { + x++ + } + } + b.StopTimer() + if x != b.N { + b.Fatal("error") + } +} + +func BenchmarkAtomicValueLoad(b *testing.B) { + c := atomic.Value{} + c.Store(0) + x := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + if c.Load().(int) == 0 { + x++ + } + } + b.StopTimer() + if x != b.N { + b.Fatal("error") + } +} + +func BenchmarkAtomicValueStore(b *testing.B) { + c := atomic.Value{} + v := 123 + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.Store(v) + } + b.StopTimer() +} + +func BenchmarkMutex(b *testing.B) { + c := sync.Mutex{} + x := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.Lock() + x++ + c.Unlock() + } + b.StopTimer() + if x != b.N { + b.Fatal("error") + } +} + +func BenchmarkRWMutex(b *testing.B) { + c := sync.RWMutex{} + x := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.RLock() + x++ + c.RUnlock() + } + b.StopTimer() + if x != b.N { + b.Fatal("error") + } +} + +func BenchmarkRWMutexW(b *testing.B) { + c := sync.RWMutex{} + x := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.Lock() + x++ + c.Unlock() + } + b.StopTimer() + if x != b.N { + b.Fatal("error") + } +} + +func BenchmarkMutexWithDefer(b *testing.B) { + c := sync.Mutex{} + x := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + func() { + c.Lock() + defer c.Unlock() + x++ + }() + } + b.StopTimer() + if x != b.N { + b.Fatal("error") + } +} + +func BenchmarkMutexWithClosureDefer(b *testing.B) { + c := sync.Mutex{} + x := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + func() { + c.Lock() + defer func() { c.Unlock() }() + x++ + }() + } + b.StopTimer() + if x != b.N { + b.Fatal("error") + } +} + +func BenchmarkMutexWithoutDefer(b *testing.B) { + c := sync.Mutex{} + x := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + func() { + c.Lock() + x++ + c.Unlock() + }() + } + b.StopTimer() + if x != b.N { + b.Fatal("error") + } +} + +func BenchmarkAtomicAddInt64(b *testing.B) { + var c int64 + b.ResetTimer() + for i := 0; i < b.N; i++ { + atomic.AddInt64(&c, 1) + } + b.StopTimer() + if c != int64(b.N) { + b.Fatal("error") + } +} + +func BenchmarkAtomicTimeValueStore(b *testing.B) { + var c atomic.Value + t := time.Now() + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.Store(t) + } + b.StopTimer() +} + +func BenchmarkAtomic16BValueStore(b *testing.B) { + var c atomic.Value + t := struct { + a int64 + b int64 + }{ + 123, 123, + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.Store(t) + } + b.StopTimer() +} + +func BenchmarkAtomic32BValueStore(b *testing.B) { + var c atomic.Value + t := struct { + a int64 + b int64 + c int64 + d int64 + }{ + 123, 123, 123, 123, + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.Store(t) + } + b.StopTimer() +} + +func BenchmarkAtomicPointerStore(b *testing.B) { + t := 123 + var up unsafe.Pointer + b.ResetTimer() + for i := 0; i < b.N; i++ { + atomic.StorePointer(&up, unsafe.Pointer(&t)) + } + b.StopTimer() +} + +func BenchmarkAtomicTimePointerStore(b *testing.B) { + t := time.Now() + var up unsafe.Pointer + b.ResetTimer() + for i := 0; i < b.N; i++ { + atomic.StorePointer(&up, unsafe.Pointer(&t)) + } + b.StopTimer() +} + +func BenchmarkStoreContentionWithAtomic(b *testing.B) { + t := 123 + var c unsafe.Pointer + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + atomic.StorePointer(&c, unsafe.Pointer(&t)) + } + }) +} + +func BenchmarkStoreContentionWithMutex(b *testing.B) { + t := 123 + var mu sync.Mutex + var c int + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + mu.Lock() + c = t + mu.Unlock() + } + }) + _ = c +} + +type dummyStruct struct { + a int64 + b time.Time +} + +func BenchmarkStructStoreContention(b *testing.B) { + d := dummyStruct{} + dp := unsafe.Pointer(&d) + t := time.Now() + for _, j := range []int{100000000, 10000, 0} { + for _, i := range []int{100000, 10} { + b.Run(fmt.Sprintf("CAS/%v/%v", j, i), func(b *testing.B) { + b.SetParallelism(i) + b.RunParallel(func(pb *testing.PB) { + n := &dummyStruct{ + b: t, + } + for pb.Next() { + for y := 0; y < j; y++ { + } + for { + v := (*dummyStruct)(atomic.LoadPointer(&dp)) + n.a = v.a + 1 + if atomic.CompareAndSwapPointer(&dp, unsafe.Pointer(v), unsafe.Pointer(n)) { + n = v + break + } + } + } + }) + }) + } + } + + var mu sync.Mutex + for _, j := range []int{100000000, 10000, 0} { + for _, i := range []int{100000, 10} { + b.Run(fmt.Sprintf("Mutex/%v/%v", j, i), func(b *testing.B) { + b.SetParallelism(i) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + for y := 0; y < j; y++ { + } + mu.Lock() + d.a++ + d.b = t + mu.Unlock() + } + }) + }) + } + } +} + +type myFooer struct{} + +func (myFooer) Foo() {} + +type fooer interface { + Foo() +} + +func BenchmarkInterfaceTypeAssertion(b *testing.B) { + // Call a separate function to avoid compiler optimizations. + runInterfaceTypeAssertion(b, myFooer{}) +} + +func runInterfaceTypeAssertion(b *testing.B, fer any) { + x := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, ok := fer.(fooer); ok { + x++ + } + } + b.StopTimer() + if x != b.N { + b.Fatal("error") + } +} + +func BenchmarkStructTypeAssertion(b *testing.B) { + // Call a separate function to avoid compiler optimizations. + runStructTypeAssertion(b, myFooer{}) +} + +func runStructTypeAssertion(b *testing.B, fer any) { + x := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, ok := fer.(myFooer); ok { + x++ + } + } + b.StopTimer() + if x != b.N { + b.Fatal("error") + } +} + +func BenchmarkWaitGroupAddDone(b *testing.B) { + wg := sync.WaitGroup{} + b.RunParallel(func(pb *testing.PB) { + i := 0 + for ; pb.Next(); i++ { + wg.Add(1) + } + for ; i > 0; i-- { + wg.Done() + } + }) +} + +func BenchmarkRLockUnlock(b *testing.B) { + mu := sync.RWMutex{} + b.RunParallel(func(pb *testing.PB) { + i := 0 + for ; pb.Next(); i++ { + mu.RLock() + } + for ; i > 0; i-- { + mu.RUnlock() + } + }) +} + +type ifNop interface { + nop() +} + +type alwaysNop struct{} + +func (alwaysNop) nop() {} + +type concreteNop struct { + isNop atomic.Bool + i int +} + +func (c *concreteNop) nop() { + if c.isNop.Load() { + return + } + c.i++ +} + +func BenchmarkInterfaceNop(b *testing.B) { + n := ifNop(alwaysNop{}) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + n.nop() + } + }) +} + +func BenchmarkConcreteNop(b *testing.B) { + n := &concreteNop{} + n.isNop.Store(true) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + n.nop() + } + }) +} diff --git a/benchmark/primitives/safe_config_selector_test.go b/benchmark/primitives/safe_config_selector_test.go new file mode 100644 index 0000000..3d368d8 --- /dev/null +++ b/benchmark/primitives/safe_config_selector_test.go @@ -0,0 +1,114 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Benchmark options for safe config selector type. + +package primitives_test + +import ( + "sync" + "sync/atomic" + "testing" + "time" + "unsafe" +) + +type safeUpdaterAtomicAndCounter struct { + ptr unsafe.Pointer // *countingFunc +} + +type countingFunc struct { + mu sync.RWMutex + f func() +} + +func (s *safeUpdaterAtomicAndCounter) call() { + cfPtr := atomic.LoadPointer(&s.ptr) + var cf *countingFunc + for { + cf = (*countingFunc)(cfPtr) + cf.mu.RLock() + cfPtr2 := atomic.LoadPointer(&s.ptr) + if cfPtr == cfPtr2 { + // Use cf with confidence! + break + } + // cf changed; try to use the new one instead, because the old one is + // no longer valid to use. + cf.mu.RUnlock() + cfPtr = cfPtr2 + } + defer cf.mu.RUnlock() + cf.f() +} + +func (s *safeUpdaterAtomicAndCounter) update(f func()) { + newCF := &countingFunc{f: f} + oldCFPtr := atomic.SwapPointer(&s.ptr, unsafe.Pointer(newCF)) + if oldCFPtr == nil { + return + } + (*countingFunc)(oldCFPtr).mu.Lock() + (*countingFunc)(oldCFPtr).mu.Unlock() //lint:ignore SA2001 necessary to unlock after locking to unblock any RLocks +} + +type safeUpdaterRWMutex struct { + mu sync.RWMutex + f func() +} + +func (s *safeUpdaterRWMutex) call() { + s.mu.RLock() + defer s.mu.RUnlock() + s.f() +} + +func (s *safeUpdaterRWMutex) update(f func()) { + s.mu.Lock() + defer s.mu.Unlock() + s.f = f +} + +type updater interface { + call() + update(f func()) +} + +func benchmarkSafeUpdater(b *testing.B, u updater) { + t := time.NewTicker(time.Second) + go func() { + for range t.C { + u.update(func() {}) + } + }() + b.RunParallel(func(pb *testing.PB) { + u.update(func() {}) + for pb.Next() { + u.call() + } + }) + t.Stop() +} + +func BenchmarkSafeUpdaterAtomicAndCounter(b *testing.B) { + benchmarkSafeUpdater(b, &safeUpdaterAtomicAndCounter{}) +} + +func BenchmarkSafeUpdaterRWMutex(b *testing.B) { + benchmarkSafeUpdater(b, &safeUpdaterRWMutex{}) +} diff --git a/benchmark/primitives/syncmap_test.go b/benchmark/primitives/syncmap_test.go new file mode 100644 index 0000000..37f9e2c --- /dev/null +++ b/benchmark/primitives/syncmap_test.go @@ -0,0 +1,165 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package primitives_test + +import ( + "sync" + "sync/atomic" + "testing" +) + +type incrementUint64Map interface { + increment(string) + result(string) uint64 +} + +type mapWithLock struct { + mu sync.Mutex + m map[string]uint64 +} + +func newMapWithLock() incrementUint64Map { + return &mapWithLock{ + m: make(map[string]uint64), + } +} + +func (mwl *mapWithLock) increment(c string) { + mwl.mu.Lock() + mwl.m[c]++ + mwl.mu.Unlock() +} + +func (mwl *mapWithLock) result(c string) uint64 { + return mwl.m[c] +} + +type mapWithAtomicFastpath struct { + mu sync.RWMutex + m map[string]*uint64 +} + +func newMapWithAtomicFastpath() incrementUint64Map { + return &mapWithAtomicFastpath{ + m: make(map[string]*uint64), + } +} + +func (mwaf *mapWithAtomicFastpath) increment(c string) { + mwaf.mu.RLock() + if p, ok := mwaf.m[c]; ok { + atomic.AddUint64(p, 1) + mwaf.mu.RUnlock() + return + } + mwaf.mu.RUnlock() + + mwaf.mu.Lock() + if p, ok := mwaf.m[c]; ok { + atomic.AddUint64(p, 1) + mwaf.mu.Unlock() + return + } + var temp uint64 = 1 + mwaf.m[c] = &temp + mwaf.mu.Unlock() +} + +func (mwaf *mapWithAtomicFastpath) result(c string) uint64 { + return atomic.LoadUint64(mwaf.m[c]) +} + +type mapWithSyncMap struct { + m sync.Map +} + +func newMapWithSyncMap() incrementUint64Map { + return &mapWithSyncMap{} +} + +func (mwsm *mapWithSyncMap) increment(c string) { + p, ok := mwsm.m.Load(c) + if !ok { + tp := new(uint64) + p, _ = mwsm.m.LoadOrStore(c, tp) + } + atomic.AddUint64(p.(*uint64), 1) +} + +func (mwsm *mapWithSyncMap) result(c string) uint64 { + p, _ := mwsm.m.Load(c) + return atomic.LoadUint64(p.(*uint64)) +} + +func benchmarkIncrementUint64Map(b *testing.B, f func() incrementUint64Map) { + const cat = "cat" + benches := []struct { + name string + goroutineCount int + }{ + { + name: " 1", + goroutineCount: 1, + }, + { + name: " 10", + goroutineCount: 10, + }, + { + name: " 100", + goroutineCount: 100, + }, + { + name: "1000", + goroutineCount: 1000, + }, + } + for _, bb := range benches { + b.Run(bb.name, func(b *testing.B) { + m := f() + var wg sync.WaitGroup + wg.Add(bb.goroutineCount) + b.ResetTimer() + for i := 0; i < bb.goroutineCount; i++ { + go func() { + for j := 0; j < b.N; j++ { + m.increment(cat) + } + wg.Done() + }() + } + wg.Wait() + b.StopTimer() + if m.result(cat) != uint64(bb.goroutineCount*b.N) { + b.Fatalf("result is %d, want %d", m.result(cat), b.N) + } + }) + } +} + +func BenchmarkMapWithSyncMutexContention(b *testing.B) { + benchmarkIncrementUint64Map(b, newMapWithLock) +} + +func BenchmarkMapWithAtomicFastpath(b *testing.B) { + benchmarkIncrementUint64Map(b, newMapWithAtomicFastpath) +} + +func BenchmarkMapWithSyncMap(b *testing.B) { + benchmarkIncrementUint64Map(b, newMapWithSyncMap) +} diff --git a/benchmark/run_bench.sh b/benchmark/run_bench.sh new file mode 100755 index 0000000..965a135 --- /dev/null +++ b/benchmark/run_bench.sh @@ -0,0 +1,188 @@ +#!/bin/bash + +rpcs=(1) +conns=(1) +warmup=10 +dur=10 +reqs=(1) +resps=(1) +rpc_types=(unary) + +# idx[0] = idx value for rpcs +# idx[1] = idx value for conns +# idx[2] = idx value for reqs +# idx[3] = idx value for resps +# idx[4] = idx value for rpc_types +idx=(0 0 0 0 0) +idx_max=(1 1 1 1 1) + +inc() +{ + for i in $(seq $((${#idx[@]}-1)) -1 0); do + idx[${i}]=$((${idx[${i}]}+1)) + if [ ${idx[${i}]} == ${idx_max[${i}]} ]; then + idx[${i}]=0 + else + break + fi + done + local fin + fin=1 + # Check to see if we have looped back to the beginning. + for v in ${idx[@]}; do + if [ ${v} != 0 ]; then + fin=0 + break + fi + done + if [ ${fin} == 1 ]; then + rm -Rf ${out_dir} + clean_and_die 0 + fi +} + +clean_and_die() { + rm -Rf ${out_dir} + exit $1 +} + +run(){ + local nr + nr=${rpcs[${idx[0]}]} + local nc + nc=${conns[${idx[1]}]} + req_sz=${reqs[${idx[2]}]} + resp_sz=${resps[${idx[3]}]} + r_type=${rpc_types[${idx[4]}]} + # Following runs one benchmark + base_port=50051 + delta=0 + test_name="r_"${nr}"_c_"${nc}"_req_"${req_sz}"_resp_"${resp_sz}"_"${r_type}"_"$(date +%s) + echo "================================================================================" + echo ${test_name} + while : + do + port=$((${base_port}+${delta})) + + # Launch the server in background + ${out_dir}/server --port=${port} --test_name="Server_"${test_name}& + server_pid=$(echo $!) + + # Launch the client + ${out_dir}/client --port=${port} --d=${dur} --w=${warmup} --r=${nr} --c=${nc} --req=${req_sz} --resp=${resp_sz} --rpc_type=${r_type} --test_name="client_"${test_name} + client_status=$(echo $?) + + kill -INT ${server_pid} + wait ${server_pid} + + if [ ${client_status} == 0 ]; then + break + fi + + delta=$((${delta}+1)) + if [ ${delta} == 10 ]; then + echo "Continuous 10 failed runs. Exiting now." + rm -Rf ${out_dir} + clean_and_die 1 + fi + done + +} + +set_param(){ + local argname=$1 + shift + local idx=$1 + shift + if [ $# -eq 0 ]; then + echo "${argname} not specified" + exit 1 + fi + PARAM=($(echo $1 | sed 's/,/ /g')) + if [ ${idx} -lt 0 ]; then + return + fi + idx_max[${idx}]=${#PARAM[@]} +} + +while [ $# -gt 0 ]; do + case "$1" in + -r) + shift + set_param "number of rpcs" 0 $1 + rpcs=(${PARAM[@]}) + shift + ;; + -c) + shift + set_param "number of connections" 1 $1 + conns=(${PARAM[@]}) + shift + ;; + -w) + shift + set_param "warm-up period" -1 $1 + warmup=${PARAM} + shift + ;; + -d) + shift + set_param "duration" -1 $1 + dur=${PARAM} + shift + ;; + -req) + shift + set_param "request size" 2 $1 + reqs=(${PARAM[@]}) + shift + ;; + -resp) + shift + set_param "response size" 3 $1 + resps=(${PARAM[@]}) + shift + ;; + -rpc_type) + shift + set_param "rpc type" 4 $1 + rpc_types=(${PARAM[@]}) + shift + ;; + -h|--help) + echo "Following are valid options:" + echo + echo "-h, --help show brief help" + echo "-w warm-up duration in seconds, default value is 10" + echo "-d benchmark duration in seconds, default value is 60" + echo "" + echo "Each of the following can have multiple comma separated values." + echo "" + echo "-r number of RPCs, default value is 1" + echo "-c number of Connections, default value is 1" + echo "-req req size in bytes, default value is 1" + echo "-resp resp size in bytes, default value is 1" + echo "-rpc_type valid values are unary|streaming, default is unary" + exit 0 + ;; + *) + echo "Incorrect option $1" + exit 1 + ;; + esac +done + +# Build server and client +out_dir=$(mktemp -d oss_benchXXX) + +go build -o ${out_dir}/server $GOPATH/src/google.golang.org/grpc/benchmark/server/main.go && go build -o ${out_dir}/client $GOPATH/src/google.golang.org/grpc/benchmark/client/main.go +if [ $? != 0 ]; then + clean_and_die 1 +fi + + +while : +do + run + inc +done diff --git a/benchmark/server/main.go b/benchmark/server/main.go new file mode 100644 index 0000000..1093ee9 --- /dev/null +++ b/benchmark/server/main.go @@ -0,0 +1,97 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* +Package main provides a server used for benchmarking. It launches a server +which is listening on port 50051. An example to start the server can be found +at: + + go run benchmark/server/main.go -test_name=grpc_test + +After starting the server, the client can be run separately and used to test +qps and latency. +*/ +package main + +import ( + "flag" + "fmt" + "net" + _ "net/http/pprof" + "os" + "os/signal" + "runtime" + "runtime/pprof" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/benchmark" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/syscall" +) + +var ( + port = flag.String("port", "50051", "Localhost port to listen on.") + testName = flag.String("test_name", "", "Name of the test used for creating profiles.") + + logger = grpclog.Component("benchmark") +) + +func main() { + flag.Parse() + if *testName == "" { + logger.Fatal("-test_name not set") + } + lis, err := net.Listen("tcp", ":"+*port) + if err != nil { + logger.Fatalf("Failed to listen: %v", err) + } + defer lis.Close() + + cf, err := os.Create("/tmp/" + *testName + ".cpu") + if err != nil { + logger.Fatalf("Failed to create file: %v", err) + } + defer cf.Close() + pprof.StartCPUProfile(cf) + cpuBeg := syscall.GetCPUTime() + // Launch server in a separate goroutine. + stop := benchmark.StartServer(benchmark.ServerInfo{Type: "protobuf", Listener: lis}, + grpc.WriteBufferSize(128*1024), + grpc.ReadBufferSize(128*1024), + ) + // Wait on OS terminate signal. + ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt) + <-ch + cpu := time.Duration(syscall.GetCPUTime() - cpuBeg) + stop() + pprof.StopCPUProfile() + mf, err := os.Create("/tmp/" + *testName + ".mem") + if err != nil { + logger.Fatalf("Failed to create file: %v", err) + } + defer mf.Close() + runtime.GC() // materialize all statistics + if err := pprof.WriteHeapProfile(mf); err != nil { + logger.Fatalf("Failed to write memory profile: %v", err) + } + fmt.Println("Server CPU utilization:", cpu) + fmt.Println("Server CPU profile:", cf.Name()) + fmt.Println("Server Mem Profile:", mf.Name()) +} diff --git a/benchmark/stats/curve.go b/benchmark/stats/curve.go new file mode 100644 index 0000000..8014031 --- /dev/null +++ b/benchmark/stats/curve.go @@ -0,0 +1,179 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package stats + +import ( + "crypto/sha256" + "encoding/csv" + "encoding/hex" + "fmt" + "math" + rand "math/rand/v2" + "os" + "sort" + "strconv" +) + +// payloadCurveRange represents a line within a payload curve CSV file. +type payloadCurveRange struct { + from, to int32 + weight float64 +} + +// newPayloadCurveRange receives a line from a payload curve CSV file and +// returns a *payloadCurveRange if the values are acceptable. +func newPayloadCurveRange(line []string) (*payloadCurveRange, error) { + if len(line) != 3 { + return nil, fmt.Errorf("invalid number of entries in line %v (expected 3)", line) + } + + var from, to int64 + var weight float64 + var err error + if from, err = strconv.ParseInt(line[0], 10, 32); err != nil { + return nil, err + } + if from <= 0 { + return nil, fmt.Errorf("line %v: field (%d) must be in (0, %d]", line, from, math.MaxInt32) + } + if to, err = strconv.ParseInt(line[1], 10, 32); err != nil { + return nil, err + } + if to <= 0 { + return nil, fmt.Errorf("line %v: field %d must be in (0, %d]", line, to, math.MaxInt32) + } + if from > to { + return nil, fmt.Errorf("line %v: from (%d) > to (%d)", line, from, to) + } + if weight, err = strconv.ParseFloat(line[2], 64); err != nil { + return nil, err + } + return &payloadCurveRange{from: int32(from), to: int32(to), weight: weight}, nil +} + +// chooseRandom picks a payload size (in bytes) for a particular range. This is +// done with a uniform distribution. +func (pcr *payloadCurveRange) chooseRandom() int { + if pcr.from == pcr.to { // fast path + return int(pcr.from) + } + + return int(rand.Int32N(pcr.to-pcr.from+1) + pcr.from) +} + +// sha256file is a helper function that returns a hex string matching the +// SHA-256 sum of the input file. +func sha256file(file string) (string, error) { + data, err := os.ReadFile(file) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// PayloadCurve is an internal representation of a weighted random distribution +// CSV file. Once a *PayloadCurve is created with NewPayloadCurve, the +// ChooseRandom function should be called to generate random payload sizes. +type PayloadCurve struct { + pcrs []*payloadCurveRange + // Sha256 must be a public field so that the gob encoder can write it to + // disk. This will be needed at decode-time by the Hash function. + Sha256 string +} + +// NewPayloadCurve parses a .csv file and returns a *PayloadCurve if no errors +// were encountered in parsing and initialization. +func NewPayloadCurve(file string) (*PayloadCurve, error) { + f, err := os.Open(file) + if err != nil { + return nil, err + } + defer f.Close() + + r := csv.NewReader(f) + lines, err := r.ReadAll() + if err != nil { + return nil, err + } + + ret := &PayloadCurve{} + var total float64 + for _, line := range lines { + pcr, err := newPayloadCurveRange(line) + if err != nil { + return nil, err + } + + ret.pcrs = append(ret.pcrs, pcr) + total += pcr.weight + } + + ret.Sha256, err = sha256file(file) + if err != nil { + return nil, err + } + for _, pcr := range ret.pcrs { + pcr.weight /= total + } + + sort.Slice(ret.pcrs, func(i, j int) bool { + if ret.pcrs[i].from == ret.pcrs[j].from { + return ret.pcrs[i].to < ret.pcrs[j].to + } + return ret.pcrs[i].from < ret.pcrs[j].from + }) + + var lastTo int32 + for _, pcr := range ret.pcrs { + if lastTo >= pcr.from { + return nil, fmt.Errorf("[%d, %d] overlaps with a different line", pcr.from, pcr.to) + } + lastTo = pcr.to + } + + return ret, nil +} + +// ChooseRandom picks a random payload size (in bytes) that follows the +// underlying weighted random distribution. +func (pc *PayloadCurve) ChooseRandom() int { + target := rand.Float64() + var seen float64 + for _, pcr := range pc.pcrs { + seen += pcr.weight + if seen >= target { + return pcr.chooseRandom() + } + } + + // This should never happen, but if it does, return a sane default. + return 1 +} + +// Hash returns a string uniquely identifying a payload curve file for feature +// matching purposes. +func (pc *PayloadCurve) Hash() string { + return pc.Sha256 +} + +// ShortHash returns a shortened version of Hash for display purposes. +func (pc *PayloadCurve) ShortHash() string { + return pc.Sha256[:8] +} diff --git a/benchmark/stats/histogram.go b/benchmark/stats/histogram.go new file mode 100644 index 0000000..461135f --- /dev/null +++ b/benchmark/stats/histogram.go @@ -0,0 +1,221 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package stats + +import ( + "bytes" + "fmt" + "io" + "log" + "math" + "strconv" + "strings" +) + +// Histogram accumulates values in the form of a histogram with +// exponentially increased bucket sizes. +type Histogram struct { + // Count is the total number of values added to the histogram. + Count int64 + // Sum is the sum of all the values added to the histogram. + Sum int64 + // SumOfSquares is the sum of squares of all values. + SumOfSquares int64 + // Min is the minimum of all the values added to the histogram. + Min int64 + // Max is the maximum of all the values added to the histogram. + Max int64 + // Buckets contains all the buckets of the histogram. + Buckets []HistogramBucket + + opts HistogramOptions + logBaseBucketSize float64 + oneOverLogOnePlusGrowthFactor float64 +} + +// HistogramOptions contains the parameters that define the histogram's buckets. +// The first bucket of the created histogram (with index 0) contains [min, min+n) +// where n = BaseBucketSize, min = MinValue. +// Bucket i (i>=1) contains [min + n * m^(i-1), min + n * m^i), where m = 1+GrowthFactor. +// The type of the values is int64. +type HistogramOptions struct { + // NumBuckets is the number of buckets. + NumBuckets int + // GrowthFactor is the growth factor of the buckets. A value of 0.1 + // indicates that bucket N+1 will be 10% larger than bucket N. + GrowthFactor float64 + // BaseBucketSize is the size of the first bucket. + BaseBucketSize float64 + // MinValue is the lower bound of the first bucket. + MinValue int64 +} + +// HistogramBucket represents one histogram bucket. +type HistogramBucket struct { + // LowBound is the lower bound of the bucket. + LowBound float64 + // Count is the number of values in the bucket. + Count int64 +} + +// NewHistogram returns a pointer to a new Histogram object that was created +// with the provided options. +func NewHistogram(opts HistogramOptions) *Histogram { + if opts.NumBuckets == 0 { + opts.NumBuckets = 32 + } + if opts.BaseBucketSize == 0.0 { + opts.BaseBucketSize = 1.0 + } + h := Histogram{ + Buckets: make([]HistogramBucket, opts.NumBuckets), + Min: math.MaxInt64, + Max: math.MinInt64, + + opts: opts, + logBaseBucketSize: math.Log(opts.BaseBucketSize), + oneOverLogOnePlusGrowthFactor: 1 / math.Log(1+opts.GrowthFactor), + } + m := 1.0 + opts.GrowthFactor + delta := opts.BaseBucketSize + h.Buckets[0].LowBound = float64(opts.MinValue) + for i := 1; i < opts.NumBuckets; i++ { + h.Buckets[i].LowBound = float64(opts.MinValue) + delta + delta = delta * m + } + return &h +} + +// Print writes textual output of the histogram values. +func (h *Histogram) Print(w io.Writer) { + h.PrintWithUnit(w, 1) +} + +// PrintWithUnit writes textual output of the histogram values . +// Data in histogram is divided by a Unit before print. +func (h *Histogram) PrintWithUnit(w io.Writer, unit float64) { + avg := float64(h.Sum) / float64(h.Count) + fmt.Fprintf(w, "Count: %d Min: %5.1f Max: %5.1f Avg: %.2f\n", h.Count, float64(h.Min)/unit, float64(h.Max)/unit, avg/unit) + fmt.Fprintf(w, "%s\n", strings.Repeat("-", 60)) + if h.Count <= 0 { + return + } + + maxBucketDigitLen := len(strconv.FormatFloat(h.Buckets[len(h.Buckets)-1].LowBound, 'f', 6, 64)) + maxCountDigitLen := len(strconv.FormatInt(h.Count, 10)) + percentMulti := 100 / float64(h.Count) + + accCount := int64(0) + for i, b := range h.Buckets { + fmt.Fprintf(w, "[%*f, ", maxBucketDigitLen, b.LowBound/unit) + if i+1 < len(h.Buckets) { + fmt.Fprintf(w, "%*f)", maxBucketDigitLen, h.Buckets[i+1].LowBound/unit) + } else { + upperBound := float64(h.opts.MinValue) + (b.LowBound-float64(h.opts.MinValue))*(1.0+h.opts.GrowthFactor) + fmt.Fprintf(w, "%*f)", maxBucketDigitLen, upperBound/unit) + } + accCount += b.Count + fmt.Fprintf(w, " %*d %5.1f%% %5.1f%%", maxCountDigitLen, b.Count, float64(b.Count)*percentMulti, float64(accCount)*percentMulti) + + const barScale = 0.1 + barLength := int(float64(b.Count)*percentMulti*barScale + 0.5) + fmt.Fprintf(w, " %s\n", strings.Repeat("#", barLength)) + } +} + +// String returns the textual output of the histogram values as string. +func (h *Histogram) String() string { + var b bytes.Buffer + h.Print(&b) + return b.String() +} + +// Clear resets all the content of histogram. +func (h *Histogram) Clear() { + h.Count = 0 + h.Sum = 0 + h.SumOfSquares = 0 + h.Min = math.MaxInt64 + h.Max = math.MinInt64 + for i := range h.Buckets { + h.Buckets[i].Count = 0 + } +} + +// Opts returns a copy of the options used to create the Histogram. +func (h *Histogram) Opts() HistogramOptions { + return h.opts +} + +// Add adds a value to the histogram. +func (h *Histogram) Add(value int64) error { + bucket, err := h.findBucket(value) + if err != nil { + return err + } + h.Buckets[bucket].Count++ + h.Count++ + h.Sum += value + h.SumOfSquares += value * value + if value < h.Min { + h.Min = value + } + if value > h.Max { + h.Max = value + } + return nil +} + +func (h *Histogram) findBucket(value int64) (int, error) { + delta := float64(value - h.opts.MinValue) + if delta < 0 { + return 0, fmt.Errorf("no bucket for value: %d", value) + } + var b int + if delta >= h.opts.BaseBucketSize { + // b = log_{1+growthFactor} (delta / baseBucketSize) + 1 + // = log(delta / baseBucketSize) / log(1+growthFactor) + 1 + // = (log(delta) - log(baseBucketSize)) * (1 / log(1+growthFactor)) + 1 + b = int((math.Log(delta)-h.logBaseBucketSize)*h.oneOverLogOnePlusGrowthFactor + 1) + } + if b >= len(h.Buckets) { + return 0, fmt.Errorf("no bucket for value: %d", value) + } + return b, nil +} + +// Merge takes another histogram h2, and merges its content into h. +// The two histograms must be created by equivalent HistogramOptions. +func (h *Histogram) Merge(h2 *Histogram) { + if h.opts != h2.opts { + log.Fatalf("failed to merge histograms, created by inequivalent options") + } + h.Count += h2.Count + h.Sum += h2.Sum + h.SumOfSquares += h2.SumOfSquares + if h2.Min < h.Min { + h.Min = h2.Min + } + if h2.Max > h.Max { + h.Max = h2.Max + } + for i, b := range h2.Buckets { + h.Buckets[i].Count += b.Count + } +} diff --git a/benchmark/stats/stats.go b/benchmark/stats/stats.go new file mode 100644 index 0000000..ade627c --- /dev/null +++ b/benchmark/stats/stats.go @@ -0,0 +1,496 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package stats tracks the statistics associated with benchmark runs. +package stats + +import ( + "bytes" + "fmt" + "log" + "math" + "runtime" + "sort" + "strconv" + "sync" + "time" + + "google.golang.org/grpc" +) + +// FeatureIndex is an enum for features that usually differ across individual +// benchmark runs in a single execution. These are usually configured by the +// user through command line flags. +type FeatureIndex int + +// FeatureIndex enum values corresponding to individually settable features. +const ( + EnableTraceIndex FeatureIndex = iota + ReadLatenciesIndex + ReadKbpsIndex + ReadMTUIndex + MaxConcurrentCallsIndex + ReqSizeBytesIndex + RespSizeBytesIndex + ReqPayloadCurveIndex + RespPayloadCurveIndex + CompModesIndex + EnableChannelzIndex + EnablePreloaderIndex + ClientReadBufferSize + ClientWriteBufferSize + ServerReadBufferSize + ServerWriteBufferSize + SleepBetweenRPCs + RecvBufferPool + SharedWriteBuffer + + // MaxFeatureIndex is a place holder to indicate the total number of feature + // indices we have. Any new feature indices should be added above this. + MaxFeatureIndex +) + +// Features represent configured options for a specific benchmark run. This is +// usually constructed from command line arguments passed by the caller. See +// benchmark/benchmain/main.go for defined command line flags. This is also +// part of the BenchResults struct which is serialized and written to a file. +type Features struct { + // Network mode used for this benchmark run. Could be one of Local, LAN, WAN + // or Longhaul. + NetworkMode string + // UseBufCon indicates whether an in-memory connection was used for this + // benchmark run instead of system network I/O. + UseBufConn bool + // EnableKeepalive indicates if keepalives were enabled on the connections + // used in this benchmark run. + EnableKeepalive bool + // BenchTime indicates the duration of the benchmark run. + BenchTime time.Duration + // Connections configures the number of grpc connections between client and server. + Connections int + + // Features defined above are usually the same for all benchmark runs in a + // particular invocation, while the features defined below could vary from + // run to run based on the configured command line. These features have a + // corresponding featureIndex value which is used for a variety of reasons. + + // EnableTrace indicates if tracing was enabled. + EnableTrace bool + // Latency is the simulated one-way network latency used. + Latency time.Duration + // Kbps is the simulated network throughput used. + Kbps int + // MTU is the simulated network MTU used. + MTU int + // MaxConcurrentCalls is the number of concurrent RPCs made during this + // benchmark run. + MaxConcurrentCalls int + // ReqSizeBytes is the request size in bytes used in this benchmark run. + // Unused if ReqPayloadCurve is non-nil. + ReqSizeBytes int + // RespSizeBytes is the response size in bytes used in this benchmark run. + // Unused if RespPayloadCurve is non-nil. + RespSizeBytes int + // ReqPayloadCurve is a histogram representing the shape a random + // distribution request payloads should take. + ReqPayloadCurve *PayloadCurve + // RespPayloadCurve is a histogram representing the shape a random + // distribution request payloads should take. + RespPayloadCurve *PayloadCurve + // ModeCompressor represents the compressor mode used. + ModeCompressor string + // EnableChannelz indicates if channelz was turned on. + EnableChannelz bool + // EnablePreloader indicates if preloading was turned on. + EnablePreloader bool + // ClientReadBufferSize is the size of the client read buffer in bytes. If negative, use the default buffer size. + ClientReadBufferSize int + // ClientWriteBufferSize is the size of the client write buffer in bytes. If negative, use the default buffer size. + ClientWriteBufferSize int + // ServerReadBufferSize is the size of the server read buffer in bytes. If negative, use the default buffer size. + ServerReadBufferSize int + // ServerWriteBufferSize is the size of the server write buffer in bytes. If negative, use the default buffer size. + ServerWriteBufferSize int + // SleepBetweenRPCs configures optional delay between RPCs. + SleepBetweenRPCs time.Duration + // RecvBufferPool represents the shared recv buffer pool used. + RecvBufferPool string + // SharedWriteBuffer configures whether both client and server share per-connection write buffer + SharedWriteBuffer bool +} + +// String returns all the feature values as a string. +func (f Features) String() string { + var reqPayloadString, respPayloadString string + if f.ReqPayloadCurve != nil { + reqPayloadString = fmt.Sprintf("reqPayloadCurve_%s", f.ReqPayloadCurve.ShortHash()) + } else { + reqPayloadString = fmt.Sprintf("reqSize_%vB", f.ReqSizeBytes) + } + if f.RespPayloadCurve != nil { + respPayloadString = fmt.Sprintf("respPayloadCurve_%s", f.RespPayloadCurve.ShortHash()) + } else { + respPayloadString = fmt.Sprintf("respSize_%vB", f.RespSizeBytes) + } + return fmt.Sprintf("networkMode_%v-bufConn_%v-keepalive_%v-benchTime_%v-"+ + "trace_%v-latency_%v-kbps_%v-MTU_%v-maxConcurrentCalls_%v-%s-%s-"+ + "compressor_%v-channelz_%v-preloader_%v-clientReadBufferSize_%v-"+ + "clientWriteBufferSize_%v-serverReadBufferSize_%v-serverWriteBufferSize_%v-"+ + "sleepBetweenRPCs_%v-connections_%v-recvBufferPool_%v-sharedWriteBuffer_%v", + f.NetworkMode, f.UseBufConn, f.EnableKeepalive, f.BenchTime, f.EnableTrace, + f.Latency, f.Kbps, f.MTU, f.MaxConcurrentCalls, reqPayloadString, + respPayloadString, f.ModeCompressor, f.EnableChannelz, f.EnablePreloader, + f.ClientReadBufferSize, f.ClientWriteBufferSize, f.ServerReadBufferSize, + f.ServerWriteBufferSize, f.SleepBetweenRPCs, f.Connections, + f.RecvBufferPool, f.SharedWriteBuffer) +} + +// SharedFeatures returns the shared features as a pretty printable string. +// 'wantFeatures' is a bitmask of wanted features, indexed by FeaturesIndex. +func (f Features) SharedFeatures(wantFeatures []bool) string { + var b bytes.Buffer + if f.NetworkMode != "" { + b.WriteString(fmt.Sprintf("Network: %v\n", f.NetworkMode)) + } + if f.UseBufConn { + b.WriteString(fmt.Sprintf("UseBufConn: %v\n", f.UseBufConn)) + } + if f.EnableKeepalive { + b.WriteString(fmt.Sprintf("EnableKeepalive: %v\n", f.EnableKeepalive)) + } + b.WriteString(fmt.Sprintf("BenchTime: %v\n", f.BenchTime)) + f.partialString(&b, wantFeatures, ": ", "\n") + return b.String() +} + +// PrintableName returns a one line name which includes the features specified +// by 'wantFeatures' which is a bitmask of wanted features, indexed by +// FeaturesIndex. +func (f Features) PrintableName(wantFeatures []bool) string { + var b bytes.Buffer + f.partialString(&b, wantFeatures, "_", "-") + return b.String() +} + +// partialString writes features specified by 'wantFeatures' to the provided +// bytes.Buffer. +func (f Features) partialString(b *bytes.Buffer, wantFeatures []bool, sep, delim string) { + for i, sf := range wantFeatures { + if sf { + switch FeatureIndex(i) { + case EnableTraceIndex: + b.WriteString(fmt.Sprintf("Trace%v%v%v", sep, f.EnableTrace, delim)) + case ReadLatenciesIndex: + b.WriteString(fmt.Sprintf("Latency%v%v%v", sep, f.Latency, delim)) + case ReadKbpsIndex: + b.WriteString(fmt.Sprintf("Kbps%v%v%v", sep, f.Kbps, delim)) + case ReadMTUIndex: + b.WriteString(fmt.Sprintf("MTU%v%v%v", sep, f.MTU, delim)) + case MaxConcurrentCallsIndex: + b.WriteString(fmt.Sprintf("Callers%v%v%v", sep, f.MaxConcurrentCalls, delim)) + case ReqSizeBytesIndex: + b.WriteString(fmt.Sprintf("ReqSize%v%vB%v", sep, f.ReqSizeBytes, delim)) + case RespSizeBytesIndex: + b.WriteString(fmt.Sprintf("RespSize%v%vB%v", sep, f.RespSizeBytes, delim)) + case ReqPayloadCurveIndex: + if f.ReqPayloadCurve != nil { + b.WriteString(fmt.Sprintf("ReqPayloadCurve%vSHA-256:%v%v", sep, f.ReqPayloadCurve.Hash(), delim)) + } + case RespPayloadCurveIndex: + if f.RespPayloadCurve != nil { + b.WriteString(fmt.Sprintf("RespPayloadCurve%vSHA-256:%v%v", sep, f.RespPayloadCurve.Hash(), delim)) + } + case CompModesIndex: + b.WriteString(fmt.Sprintf("Compressor%v%v%v", sep, f.ModeCompressor, delim)) + case EnableChannelzIndex: + b.WriteString(fmt.Sprintf("Channelz%v%v%v", sep, f.EnableChannelz, delim)) + case EnablePreloaderIndex: + b.WriteString(fmt.Sprintf("Preloader%v%v%v", sep, f.EnablePreloader, delim)) + case ClientReadBufferSize: + b.WriteString(fmt.Sprintf("ClientReadBufferSize%v%v%v", sep, f.ClientReadBufferSize, delim)) + case ClientWriteBufferSize: + b.WriteString(fmt.Sprintf("ClientWriteBufferSize%v%v%v", sep, f.ClientWriteBufferSize, delim)) + case ServerReadBufferSize: + b.WriteString(fmt.Sprintf("ServerReadBufferSize%v%v%v", sep, f.ServerReadBufferSize, delim)) + case ServerWriteBufferSize: + b.WriteString(fmt.Sprintf("ServerWriteBufferSize%v%v%v", sep, f.ServerWriteBufferSize, delim)) + case SleepBetweenRPCs: + b.WriteString(fmt.Sprintf("SleepBetweenRPCs%v%v%v", sep, f.SleepBetweenRPCs, delim)) + case RecvBufferPool: + b.WriteString(fmt.Sprintf("RecvBufferPool%v%v%v", sep, f.RecvBufferPool, delim)) + case SharedWriteBuffer: + b.WriteString(fmt.Sprintf("SharedWriteBuffer%v%v%v", sep, f.SharedWriteBuffer, delim)) + default: + log.Fatalf("Unknown feature index %v. maxFeatureIndex is %v", i, MaxFeatureIndex) + } + } + } +} + +// BenchResults records features and results of a benchmark run. A collection +// of these structs is usually serialized and written to a file after a +// benchmark execution, and could later be read for pretty-printing or +// comparison with other benchmark results. +type BenchResults struct { + // GoVersion is the version of the compiler the benchmark was compiled with. + GoVersion string + // GrpcVersion is the gRPC version being benchmarked. + GrpcVersion string + // RunMode is the workload mode for this benchmark run. This could be unary, + // stream or unconstrained. + RunMode string + // Features represents the configured feature options for this run. + Features Features + // SharedFeatures represents the features which were shared across all + // benchmark runs during one execution. It is a slice indexed by + // 'FeaturesIndex' and a value of true indicates that the associated + // feature is shared across all runs. + SharedFeatures []bool + // Data contains the statistical data of interest from the benchmark run. + Data RunData +} + +// RunData contains statistical data of interest from a benchmark run. +type RunData struct { + // TotalOps is the number of operations executed during this benchmark run. + // Only makes sense for unary and streaming workloads. + TotalOps uint64 + // SendOps is the number of send operations executed during this benchmark + // run. Only makes sense for unconstrained workloads. + SendOps uint64 + // RecvOps is the number of receive operations executed during this benchmark + // run. Only makes sense for unconstrained workloads. + RecvOps uint64 + // AllocedBytes is the average memory allocation in bytes per operation. + AllocedBytes float64 + // Allocs is the average number of memory allocations per operation. + Allocs float64 + // ReqT is the average request throughput associated with this run. + ReqT float64 + // RespT is the average response throughput associated with this run. + RespT float64 + + // We store different latencies associated with each run. These latencies are + // only computed for unary and stream workloads as they are not very useful + // for unconstrained workloads. + + // Fiftieth is the 50th percentile latency. + Fiftieth time.Duration + // Ninetieth is the 90th percentile latency. + Ninetieth time.Duration + // NinetyNinth is the 99th percentile latency. + NinetyNinth time.Duration + // Average is the average latency. + Average time.Duration +} + +type durationSlice []time.Duration + +func (a durationSlice) Len() int { return len(a) } +func (a durationSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a durationSlice) Less(i, j int) bool { return a[i] < a[j] } + +// Stats is a helper for gathering statistics about individual benchmark runs. +type Stats struct { + mu sync.Mutex + numBuckets int + hw *histWrapper + results []BenchResults + startMS runtime.MemStats + stopMS runtime.MemStats +} + +type histWrapper struct { + unit time.Duration + histogram *Histogram + durations durationSlice +} + +// NewStats creates a new Stats instance. If numBuckets is not positive, the +// default value (16) will be used. +func NewStats(numBuckets int) *Stats { + if numBuckets <= 0 { + numBuckets = 16 + } + // Use one more bucket for the last unbounded bucket. + s := &Stats{numBuckets: numBuckets + 1} + s.hw = &histWrapper{} + return s +} + +// StartRun is to be invoked to indicate the start of a new benchmark run. +func (s *Stats) StartRun(mode string, f Features, sf []bool) { + s.mu.Lock() + defer s.mu.Unlock() + + runtime.ReadMemStats(&s.startMS) + s.results = append(s.results, BenchResults{ + GoVersion: runtime.Version(), + GrpcVersion: grpc.Version, + RunMode: mode, + Features: f, + SharedFeatures: sf, + }) +} + +// EndRun is to be invoked to indicate the end of the ongoing benchmark run. It +// computes a bunch of stats and dumps them to stdout. +func (s *Stats) EndRun(count uint64) { + s.mu.Lock() + defer s.mu.Unlock() + + runtime.ReadMemStats(&s.stopMS) + r := &s.results[len(s.results)-1] + r.Data = RunData{ + TotalOps: count, + AllocedBytes: float64(s.stopMS.TotalAlloc-s.startMS.TotalAlloc) / float64(count), + Allocs: float64(s.stopMS.Mallocs-s.startMS.Mallocs) / float64(count), + ReqT: float64(count) * float64(r.Features.ReqSizeBytes) * 8 / r.Features.BenchTime.Seconds(), + RespT: float64(count) * float64(r.Features.RespSizeBytes) * 8 / r.Features.BenchTime.Seconds(), + } + s.computeLatencies(r) + s.dump(r) + s.hw = &histWrapper{} +} + +// EndUnconstrainedRun is similar to EndRun, but is to be used for +// unconstrained workloads. +func (s *Stats) EndUnconstrainedRun(req uint64, resp uint64) { + s.mu.Lock() + defer s.mu.Unlock() + + runtime.ReadMemStats(&s.stopMS) + r := &s.results[len(s.results)-1] + r.Data = RunData{ + SendOps: req, + RecvOps: resp, + AllocedBytes: float64(s.stopMS.TotalAlloc-s.startMS.TotalAlloc) / float64((req+resp)/2), + Allocs: float64(s.stopMS.Mallocs-s.startMS.Mallocs) / float64((req+resp)/2), + ReqT: float64(req) * float64(r.Features.ReqSizeBytes) * 8 / r.Features.BenchTime.Seconds(), + RespT: float64(resp) * float64(r.Features.RespSizeBytes) * 8 / r.Features.BenchTime.Seconds(), + } + s.computeLatencies(r) + s.dump(r) + s.hw = &histWrapper{} +} + +// AddDuration adds an elapsed duration per operation to the stats. This is +// used by unary and stream modes where request and response stats are equal. +func (s *Stats) AddDuration(d time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + + s.hw.durations = append(s.hw.durations, d) +} + +// GetResults returns the results from all benchmark runs. +func (s *Stats) GetResults() []BenchResults { + s.mu.Lock() + defer s.mu.Unlock() + + return s.results +} + +// computeLatencies computes percentile latencies based on durations stored in +// the stats object and updates the corresponding fields in the result object. +func (s *Stats) computeLatencies(result *BenchResults) { + if len(s.hw.durations) == 0 { + return + } + sort.Sort(s.hw.durations) + minDuration := int64(s.hw.durations[0]) + maxDuration := int64(s.hw.durations[len(s.hw.durations)-1]) + + // Use the largest unit that can represent the minimum time duration. + s.hw.unit = time.Nanosecond + for _, u := range []time.Duration{time.Microsecond, time.Millisecond, time.Second} { + if minDuration <= int64(u) { + break + } + s.hw.unit = u + } + + numBuckets := s.numBuckets + if n := int(maxDuration - minDuration + 1); n < numBuckets { + numBuckets = n + } + s.hw.histogram = NewHistogram(HistogramOptions{ + NumBuckets: numBuckets, + // max-min(lower bound of last bucket) = (1 + growthFactor)^(numBuckets-2) * baseBucketSize. + GrowthFactor: math.Pow(float64(maxDuration-minDuration), 1/float64(numBuckets-2)) - 1, + BaseBucketSize: 1.0, + MinValue: minDuration, + }) + for _, d := range s.hw.durations { + s.hw.histogram.Add(int64(d)) + } + result.Data.Fiftieth = s.hw.durations[max(s.hw.histogram.Count*int64(50)/100-1, 0)] + result.Data.Ninetieth = s.hw.durations[max(s.hw.histogram.Count*int64(90)/100-1, 0)] + result.Data.NinetyNinth = s.hw.durations[max(s.hw.histogram.Count*int64(99)/100-1, 0)] + result.Data.Average = time.Duration(float64(s.hw.histogram.Sum) / float64(s.hw.histogram.Count)) +} + +// dump returns a printable version. +func (s *Stats) dump(result *BenchResults) { + var b bytes.Buffer + + // Go and gRPC version information. + b.WriteString(fmt.Sprintf("%s/grpc%s\n", result.GoVersion, result.GrpcVersion)) + + // This prints the run mode and all features of the bench on a line. + b.WriteString(fmt.Sprintf("%s-%s:\n", result.RunMode, result.Features.String())) + + unit := s.hw.unit + tUnit := fmt.Sprintf("%v", unit)[1:] // stores one of s, ms, μs, ns + + if l := result.Data.Fiftieth; l != 0 { + b.WriteString(fmt.Sprintf("50_Latency: %s%s\t", strconv.FormatFloat(float64(l)/float64(unit), 'f', 4, 64), tUnit)) + } + if l := result.Data.Ninetieth; l != 0 { + b.WriteString(fmt.Sprintf("90_Latency: %s%s\t", strconv.FormatFloat(float64(l)/float64(unit), 'f', 4, 64), tUnit)) + } + if l := result.Data.NinetyNinth; l != 0 { + b.WriteString(fmt.Sprintf("99_Latency: %s%s\t", strconv.FormatFloat(float64(l)/float64(unit), 'f', 4, 64), tUnit)) + } + if l := result.Data.Average; l != 0 { + b.WriteString(fmt.Sprintf("Avg_Latency: %s%s\t", strconv.FormatFloat(float64(l)/float64(unit), 'f', 4, 64), tUnit)) + } + b.WriteString(fmt.Sprintf("Bytes/op: %v\t", result.Data.AllocedBytes)) + b.WriteString(fmt.Sprintf("Allocs/op: %v\t\n", result.Data.Allocs)) + + // This prints the histogram stats for the latency. + if s.hw.histogram == nil { + b.WriteString("Histogram (empty)\n") + } else { + b.WriteString(fmt.Sprintf("Histogram (unit: %s)\n", tUnit)) + s.hw.histogram.PrintWithUnit(&b, float64(unit)) + } + + // Print throughput data. + req := result.Data.SendOps + if req == 0 { + req = result.Data.TotalOps + } + resp := result.Data.RecvOps + if resp == 0 { + resp = result.Data.TotalOps + } + b.WriteString(fmt.Sprintf("Number of requests: %v\tRequest throughput: %v bit/s\n", req, result.Data.ReqT)) + b.WriteString(fmt.Sprintf("Number of responses: %v\tResponse throughput: %v bit/s\n", resp, result.Data.RespT)) + fmt.Println(b.String()) +} diff --git a/benchmark/worker/benchmark_client.go b/benchmark/worker/benchmark_client.go new file mode 100644 index 0000000..af81445 --- /dev/null +++ b/benchmark/worker/benchmark_client.go @@ -0,0 +1,428 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "context" + "flag" + "math" + rand "math/rand/v2" + "runtime" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/benchmark" + "google.golang.org/grpc/benchmark/stats" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/syscall" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/xds" // To install the xds resolvers and balancers. +) + +var caFile = flag.String("ca_file", "", "The file containing the CA root cert file") + +type lockingHistogram struct { + mu sync.Mutex + histogram *stats.Histogram +} + +func (h *lockingHistogram) add(value int64) { + h.mu.Lock() + defer h.mu.Unlock() + h.histogram.Add(value) +} + +// swap sets h.histogram to o and returns its old value. +func (h *lockingHistogram) swap(o *stats.Histogram) *stats.Histogram { + h.mu.Lock() + defer h.mu.Unlock() + old := h.histogram + h.histogram = o + return old +} + +func (h *lockingHistogram) mergeInto(merged *stats.Histogram) { + h.mu.Lock() + defer h.mu.Unlock() + merged.Merge(h.histogram) +} + +type benchmarkClient struct { + closeConns func() + lastResetTime time.Time + histogramOptions stats.HistogramOptions + lockingHistograms []lockingHistogram + rusageLastReset *syscall.Rusage +} + +func printClientConfig(config *testpb.ClientConfig) { + // Some config options are ignored: + // - client type: + // will always create sync client + // - async client threads. + // - core list + logger.Infof(" * client type: %v (ignored, always creates sync client)", config.ClientType) + logger.Infof(" * async client threads: %v (ignored)", config.AsyncClientThreads) + // TODO: use cores specified by CoreList when setting list of cores is supported in go. + logger.Infof(" * core list: %v (ignored)", config.CoreList) + + logger.Infof(" - security params: %v", config.SecurityParams) + logger.Infof(" - core limit: %v", config.CoreLimit) + logger.Infof(" - payload config: %v", config.PayloadConfig) + logger.Infof(" - rpcs per chann: %v", config.OutstandingRpcsPerChannel) + logger.Infof(" - channel number: %v", config.ClientChannels) + logger.Infof(" - load params: %v", config.LoadParams) + logger.Infof(" - rpc type: %v", config.RpcType) + logger.Infof(" - histogram params: %v", config.HistogramParams) + logger.Infof(" - server targets: %v", config.ServerTargets) +} + +func setupClientEnv(config *testpb.ClientConfig) { + // Use all cpu cores available on machine by default. + // TODO: Revisit this for the optimal default setup. + if config.CoreLimit > 0 { + runtime.GOMAXPROCS(int(config.CoreLimit)) + } else { + runtime.GOMAXPROCS(runtime.NumCPU()) + } +} + +// createConns creates connections according to given config. +// It returns the connections and corresponding function to close them. +// It returns non-nil error if there is anything wrong. +func createConns(config *testpb.ClientConfig) ([]*grpc.ClientConn, func(), error) { + opts := []grpc.DialOption{ + grpc.WithWriteBufferSize(128 * 1024), + grpc.WithReadBufferSize(128 * 1024), + } + + // Sanity check for client type. + switch config.ClientType { + case testpb.ClientType_SYNC_CLIENT: + case testpb.ClientType_ASYNC_CLIENT: + default: + return nil, nil, status.Errorf(codes.InvalidArgument, "unknown client type: %v", config.ClientType) + } + + // Check and set security options. + if config.SecurityParams != nil { + if *caFile == "" { + *caFile = testdata.Path("ca.pem") + } + creds, err := credentials.NewClientTLSFromFile(*caFile, config.SecurityParams.ServerHostOverride) + if err != nil { + return nil, nil, status.Errorf(codes.InvalidArgument, "failed to create TLS credentials: %v", err) + } + opts = append(opts, grpc.WithTransportCredentials(creds)) + } else { + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + + // Use byteBufCodec if it is required. + if config.PayloadConfig != nil { + switch config.PayloadConfig.Payload.(type) { + case *testpb.PayloadConfig_BytebufParams: + opts = append(opts, grpc.WithDefaultCallOptions(grpc.CallCustomCodec(byteBufCodec{}))) + case *testpb.PayloadConfig_SimpleParams: + default: + return nil, nil, status.Errorf(codes.InvalidArgument, "unknown payload config: %v", config.PayloadConfig) + } + } + + // Create connections. + connCount := int(config.ClientChannels) + conns := make([]*grpc.ClientConn, connCount) + for connIndex := 0; connIndex < connCount; connIndex++ { + conns[connIndex] = benchmark.NewClientConn(config.ServerTargets[connIndex%len(config.ServerTargets)], opts...) + } + + return conns, func() { + for _, conn := range conns { + conn.Close() + } + }, nil +} + +func performRPCs(ctx context.Context, config *testpb.ClientConfig, conns []*grpc.ClientConn, bc *benchmarkClient) error { + // Read payload size and type from config. + var ( + payloadReqSize, payloadRespSize int + payloadType string + ) + if config.PayloadConfig != nil { + switch c := config.PayloadConfig.Payload.(type) { + case *testpb.PayloadConfig_BytebufParams: + payloadReqSize = int(c.BytebufParams.ReqSize) + payloadRespSize = int(c.BytebufParams.RespSize) + payloadType = "bytebuf" + case *testpb.PayloadConfig_SimpleParams: + payloadReqSize = int(c.SimpleParams.ReqSize) + payloadRespSize = int(c.SimpleParams.RespSize) + payloadType = "protobuf" + default: + return status.Errorf(codes.InvalidArgument, "unknown payload config: %v", config.PayloadConfig) + } + } + + // If set, perform an open loop, if not perform a closed loop. An open loop + // asynchronously starts RPCs based on random start times derived from a + // Poisson distribution. A closed loop performs RPCs in a blocking manner, + // and runs the next RPC after the previous RPC completes and returns. + var poissonLambda *float64 + switch t := config.LoadParams.Load.(type) { + case *testpb.LoadParams_ClosedLoop: + case *testpb.LoadParams_Poisson: + if t.Poisson == nil { + return status.Errorf(codes.InvalidArgument, "poisson is nil, needs to be set") + } + if t.Poisson.OfferedLoad <= 0 { + return status.Errorf(codes.InvalidArgument, "poisson.offered is <= 0: %v, needs to be >0", t.Poisson.OfferedLoad) + } + poissonLambda = &t.Poisson.OfferedLoad + default: + return status.Errorf(codes.InvalidArgument, "unknown load params: %v", config.LoadParams) + } + + rpcCountPerConn := int(config.OutstandingRpcsPerChannel) + + switch config.RpcType { + case testpb.RpcType_UNARY: + bc.unaryLoop(ctx, conns, rpcCountPerConn, payloadReqSize, payloadRespSize, poissonLambda) + case testpb.RpcType_STREAMING: + bc.streamingLoop(ctx, conns, rpcCountPerConn, payloadReqSize, payloadRespSize, payloadType, poissonLambda) + default: + return status.Errorf(codes.InvalidArgument, "unknown rpc type: %v", config.RpcType) + } + + return nil +} + +func startBenchmarkClient(ctx context.Context, config *testpb.ClientConfig) (*benchmarkClient, error) { + printClientConfig(config) + + // Set running environment like how many cores to use. + setupClientEnv(config) + + conns, closeConns, err := createConns(config) + if err != nil { + return nil, err + } + + rpcCountPerConn := int(config.OutstandingRpcsPerChannel) + bc := &benchmarkClient{ + histogramOptions: stats.HistogramOptions{ + NumBuckets: int(math.Log(config.HistogramParams.MaxPossible)/math.Log(1+config.HistogramParams.Resolution)) + 1, + GrowthFactor: config.HistogramParams.Resolution, + BaseBucketSize: (1 + config.HistogramParams.Resolution), + MinValue: 0, + }, + lockingHistograms: make([]lockingHistogram, rpcCountPerConn*len(conns)), + + lastResetTime: time.Now(), + closeConns: closeConns, + rusageLastReset: syscall.GetRusage(), + } + + if err = performRPCs(ctx, config, conns, bc); err != nil { + // Close all connections if performRPCs failed. + closeConns() + return nil, err + } + + return bc, nil +} + +func (bc *benchmarkClient) unaryLoop(ctx context.Context, conns []*grpc.ClientConn, rpcCountPerConn int, reqSize int, respSize int, poissonLambda *float64) { + for ic, conn := range conns { + client := testgrpc.NewBenchmarkServiceClient(conn) + // For each connection, create rpcCountPerConn goroutines to do rpc. + for j := range rpcCountPerConn { + // Create histogram for each goroutine. + idx := ic*rpcCountPerConn + j + bc.lockingHistograms[idx].histogram = stats.NewHistogram(bc.histogramOptions) + // Start goroutine on the created mutex and histogram. + go func(idx int) { + // TODO: do warm up if necessary. + // Now relying on worker client to reserve time to do warm up. + // The worker client needs to wait for some time after client is created, + // before starting benchmark. + if poissonLambda == nil { // Closed loop. + for { + if ctx.Err() != nil { + break + } + start := time.Now() + if err := benchmark.DoUnaryCall(client, reqSize, respSize); err != nil { + continue + } + elapse := time.Since(start) + bc.lockingHistograms[idx].add(int64(elapse)) + } + } else { // Open loop. + timeBetweenRPCs := time.Duration((rand.ExpFloat64() / *poissonLambda) * float64(time.Second)) + time.AfterFunc(timeBetweenRPCs, func() { + bc.poissonUnary(client, idx, reqSize, respSize, *poissonLambda) + }) + } + }(idx) + } + } +} + +func (bc *benchmarkClient) streamingLoop(ctx context.Context, conns []*grpc.ClientConn, rpcCountPerConn int, reqSize int, respSize int, payloadType string, poissonLambda *float64) { + var doRPC func(testgrpc.BenchmarkService_StreamingCallClient, int, int) error + if payloadType == "bytebuf" { + doRPC = benchmark.DoByteBufStreamingRoundTrip + } else { + doRPC = benchmark.DoStreamingRoundTrip + } + for ic, conn := range conns { + // For each connection, create rpcCountPerConn goroutines to do rpc. + for j := 0; j < rpcCountPerConn; j++ { + c := testgrpc.NewBenchmarkServiceClient(conn) + stream, err := c.StreamingCall(context.Background()) + if err != nil { + logger.Fatalf("%v.StreamingCall(_) = _, %v", c, err) + } + idx := ic*rpcCountPerConn + j + bc.lockingHistograms[idx].histogram = stats.NewHistogram(bc.histogramOptions) + if poissonLambda == nil { // Closed loop. + // Start goroutine on the created mutex and histogram. + go func(idx int) { + // TODO: do warm up if necessary. + // Now relying on worker client to reserve time to do warm up. + // The worker client needs to wait for some time after client is created, + // before starting benchmark. + for { + start := time.Now() + if err := doRPC(stream, reqSize, respSize); err != nil { + return + } + elapse := time.Since(start) + bc.lockingHistograms[idx].add(int64(elapse)) + if ctx.Err() != nil { + return + } + } + }(idx) + } else { // Open loop. + timeBetweenRPCs := time.Duration((rand.ExpFloat64() / *poissonLambda) * float64(time.Second)) + time.AfterFunc(timeBetweenRPCs, func() { + bc.poissonStreaming(stream, idx, reqSize, respSize, *poissonLambda, doRPC) + }) + } + } + } +} + +func (bc *benchmarkClient) poissonUnary(client testgrpc.BenchmarkServiceClient, idx int, reqSize int, respSize int, lambda float64) { + go func() { + start := time.Now() + if err := benchmark.DoUnaryCall(client, reqSize, respSize); err != nil { + return + } + elapse := time.Since(start) + bc.lockingHistograms[idx].add(int64(elapse)) + }() + timeBetweenRPCs := time.Duration((rand.ExpFloat64() / lambda) * float64(time.Second)) + time.AfterFunc(timeBetweenRPCs, func() { + bc.poissonUnary(client, idx, reqSize, respSize, lambda) + }) +} + +func (bc *benchmarkClient) poissonStreaming(stream testgrpc.BenchmarkService_StreamingCallClient, idx int, reqSize int, respSize int, lambda float64, doRPC func(testgrpc.BenchmarkService_StreamingCallClient, int, int) error) { + go func() { + start := time.Now() + + if err := doRPC(stream, reqSize, respSize); err != nil { + return + } + elapse := time.Since(start) + bc.lockingHistograms[idx].add(int64(elapse)) + }() + timeBetweenRPCs := time.Duration((rand.ExpFloat64() / lambda) * float64(time.Second)) + time.AfterFunc(timeBetweenRPCs, func() { + bc.poissonStreaming(stream, idx, reqSize, respSize, lambda, doRPC) + }) +} + +// getStats returns the stats for benchmark client. +// It resets lastResetTime and all histograms if argument reset is true. +func (bc *benchmarkClient) getStats(reset bool) *testpb.ClientStats { + var wallTimeElapsed, uTimeElapsed, sTimeElapsed float64 + mergedHistogram := stats.NewHistogram(bc.histogramOptions) + + if reset { + // Merging histogram may take some time. + // Put all histograms aside and merge later. + toMerge := make([]*stats.Histogram, len(bc.lockingHistograms)) + for i := range bc.lockingHistograms { + toMerge[i] = bc.lockingHistograms[i].swap(stats.NewHistogram(bc.histogramOptions)) + } + + for i := 0; i < len(toMerge); i++ { + mergedHistogram.Merge(toMerge[i]) + } + + wallTimeElapsed = time.Since(bc.lastResetTime).Seconds() + latestRusage := syscall.GetRusage() + uTimeElapsed, sTimeElapsed = syscall.CPUTimeDiff(bc.rusageLastReset, latestRusage) + + bc.rusageLastReset = latestRusage + bc.lastResetTime = time.Now() + } else { + // Merge only, not reset. + for i := range bc.lockingHistograms { + bc.lockingHistograms[i].mergeInto(mergedHistogram) + } + + wallTimeElapsed = time.Since(bc.lastResetTime).Seconds() + uTimeElapsed, sTimeElapsed = syscall.CPUTimeDiff(bc.rusageLastReset, syscall.GetRusage()) + } + + b := make([]uint32, len(mergedHistogram.Buckets)) + for i, v := range mergedHistogram.Buckets { + b[i] = uint32(v.Count) + } + return &testpb.ClientStats{ + Latencies: &testpb.HistogramData{ + Bucket: b, + MinSeen: float64(mergedHistogram.Min), + MaxSeen: float64(mergedHistogram.Max), + Sum: float64(mergedHistogram.Sum), + SumOfSquares: float64(mergedHistogram.SumOfSquares), + Count: float64(mergedHistogram.Count), + }, + TimeElapsed: wallTimeElapsed, + TimeUser: uTimeElapsed, + TimeSystem: sTimeElapsed, + } +} + +func (bc *benchmarkClient) shutdown() { + bc.closeConns() +} diff --git a/benchmark/worker/benchmark_server.go b/benchmark/worker/benchmark_server.go new file mode 100644 index 0000000..fcf5027 --- /dev/null +++ b/benchmark/worker/benchmark_server.go @@ -0,0 +1,187 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "flag" + "fmt" + "net" + "runtime" + "strconv" + "strings" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/benchmark" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/syscall" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" +) + +var ( + certFile = flag.String("tls_cert_file", "", "The TLS cert file") + keyFile = flag.String("tls_key_file", "", "The TLS key file") +) + +type benchmarkServer struct { + port int + cores int + closeFunc func() + + mu sync.Mutex + lastResetTime time.Time + rusageLastReset *syscall.Rusage +} + +func printServerConfig(config *testpb.ServerConfig) { + // Some config options are ignored: + // - server type: + // will always start sync server + // - async server threads + // - core list + logger.Infof(" * server type: %v (ignored, always starts sync server)", config.ServerType) + logger.Infof(" * async server threads: %v (ignored)", config.AsyncServerThreads) + // TODO: use cores specified by CoreList when setting list of cores is supported in go. + logger.Infof(" * core list: %v (ignored)", config.CoreList) + + logger.Infof(" - security params: %v", config.SecurityParams) + logger.Infof(" - core limit: %v", config.CoreLimit) + logger.Infof(" - port: %v", config.Port) + logger.Infof(" - payload config: %v", config.PayloadConfig) +} + +func startBenchmarkServer(config *testpb.ServerConfig, serverPort int) (*benchmarkServer, error) { + printServerConfig(config) + + // Use all cpu cores available on machine by default. + // TODO: Revisit this for the optimal default setup. + numOfCores := runtime.NumCPU() + if config.CoreLimit > 0 { + numOfCores = int(config.CoreLimit) + } + runtime.GOMAXPROCS(numOfCores) + + opts := []grpc.ServerOption{ + grpc.WriteBufferSize(128 * 1024), + grpc.ReadBufferSize(128 * 1024), + } + + // Sanity check for server type. + switch config.ServerType { + case testpb.ServerType_SYNC_SERVER: + case testpb.ServerType_ASYNC_SERVER: + case testpb.ServerType_ASYNC_GENERIC_SERVER: + default: + return nil, status.Errorf(codes.InvalidArgument, "unknown server type: %v", config.ServerType) + } + + // Set security options. + if config.SecurityParams != nil { + if *certFile == "" { + *certFile = testdata.Path("server1.pem") + } + if *keyFile == "" { + *keyFile = testdata.Path("server1.key") + } + creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile) + if err != nil { + logger.Fatalf("failed to generate credentials: %v", err) + } + opts = append(opts, grpc.Creds(creds)) + } + + // Priority: config.Port > serverPort > default (0). + port := int(config.Port) + if port == 0 { + port = serverPort + } + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + logger.Fatalf("Failed to listen: %v", err) + } + addr := lis.Addr().String() + + // Create different benchmark server according to config. + var closeFunc func() + if config.PayloadConfig != nil { + switch payload := config.PayloadConfig.Payload.(type) { + case *testpb.PayloadConfig_BytebufParams: + opts = append(opts, grpc.CustomCodec(byteBufCodec{})) + closeFunc = benchmark.StartServer(benchmark.ServerInfo{ + Type: "bytebuf", + Metadata: payload.BytebufParams.RespSize, + Listener: lis, + }, opts...) + case *testpb.PayloadConfig_SimpleParams: + closeFunc = benchmark.StartServer(benchmark.ServerInfo{ + Type: "protobuf", + Listener: lis, + }, opts...) + case *testpb.PayloadConfig_ComplexParams: + return nil, status.Errorf(codes.Unimplemented, "unsupported payload config: %v", config.PayloadConfig) + default: + return nil, status.Errorf(codes.InvalidArgument, "unknown payload config: %v", config.PayloadConfig) + } + } else { + // Start protobuf server if payload config is nil. + closeFunc = benchmark.StartServer(benchmark.ServerInfo{ + Type: "protobuf", + Listener: lis, + }, opts...) + } + + logger.Infof("benchmark server listening at %v", addr) + addrSplitted := strings.Split(addr, ":") + p, err := strconv.Atoi(addrSplitted[len(addrSplitted)-1]) + if err != nil { + logger.Fatalf("failed to get port number from server address: %v", err) + } + + return &benchmarkServer{ + port: p, + cores: numOfCores, + closeFunc: closeFunc, + lastResetTime: time.Now(), + rusageLastReset: syscall.GetRusage(), + }, nil +} + +// getStats returns the stats for benchmark server. +// It resets lastResetTime if argument reset is true. +func (bs *benchmarkServer) getStats(reset bool) *testpb.ServerStats { + bs.mu.Lock() + defer bs.mu.Unlock() + wallTimeElapsed := time.Since(bs.lastResetTime).Seconds() + rusageLatest := syscall.GetRusage() + uTimeElapsed, sTimeElapsed := syscall.CPUTimeDiff(bs.rusageLastReset, rusageLatest) + + if reset { + bs.lastResetTime = time.Now() + bs.rusageLastReset = rusageLatest + } + return &testpb.ServerStats{ + TimeElapsed: wallTimeElapsed, + TimeUser: uTimeElapsed, + TimeSystem: sTimeElapsed, + } +} diff --git a/benchmark/worker/main.go b/benchmark/worker/main.go new file mode 100644 index 0000000..0a6dd96 --- /dev/null +++ b/benchmark/worker/main.go @@ -0,0 +1,239 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary worker implements the benchmark worker that can turn into a benchmark +// client or server. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "net" + "net/http" + _ "net/http/pprof" + "runtime" + "strconv" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + driverPort = flag.Int("driver_port", 10000, "port for communication with driver") + serverPort = flag.Int("server_port", 0, "port for benchmark server if not specified by server config message") + pprofPort = flag.Int("pprof_port", -1, "Port for pprof debug server to listen on. Pprof server doesn't start if unset") + blockProfRate = flag.Int("block_prof_rate", 0, "fraction of goroutine blocking events to report in blocking profile") + + logger = grpclog.Component("benchmark") +) + +type byteBufCodec struct { +} + +func (byteBufCodec) Marshal(v any) ([]byte, error) { + b, ok := v.(*[]byte) + if !ok { + return nil, fmt.Errorf("failed to marshal: %v is not type of *[]byte", v) + } + return *b, nil +} + +func (byteBufCodec) Unmarshal(data []byte, v any) error { + b, ok := v.(*[]byte) + if !ok { + return fmt.Errorf("failed to marshal: %v is not type of *[]byte", v) + } + *b = data + return nil +} + +func (byteBufCodec) String() string { + return "bytebuffer" +} + +// workerServer implements WorkerService rpc handlers. +// It can create benchmarkServer or benchmarkClient on demand. +type workerServer struct { + testgrpc.UnimplementedWorkerServiceServer + stop chan<- bool + serverPort int +} + +func (s *workerServer) RunServer(stream testgrpc.WorkerService_RunServerServer) error { + var bs *benchmarkServer + defer func() { + // Close benchmark server when stream ends. + logger.Infof("closing benchmark server") + if bs != nil { + bs.closeFunc() + } + }() + for { + in, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + var out *testpb.ServerStatus + switch argtype := in.Argtype.(type) { + case *testpb.ServerArgs_Setup: + logger.Infof("server setup received:") + if bs != nil { + logger.Infof("server setup received when server already exists, closing the existing server") + bs.closeFunc() + } + bs, err = startBenchmarkServer(argtype.Setup, s.serverPort) + if err != nil { + return err + } + out = &testpb.ServerStatus{ + Stats: bs.getStats(false), + Port: int32(bs.port), + Cores: int32(bs.cores), + } + + case *testpb.ServerArgs_Mark: + logger.Infof("server mark received:") + logger.Infof(" - %v", argtype) + if bs == nil { + return status.Error(codes.InvalidArgument, "server does not exist when mark received") + } + out = &testpb.ServerStatus{ + Stats: bs.getStats(argtype.Mark.Reset_), + Port: int32(bs.port), + Cores: int32(bs.cores), + } + } + + if err := stream.Send(out); err != nil { + return err + } + } +} + +func (s *workerServer) RunClient(stream testgrpc.WorkerService_RunClientServer) error { + var bc *benchmarkClient + ctx, cancel := context.WithCancel(stream.Context()) + defer func() { + cancel() + // Shut down benchmark client when stream ends. + logger.Infof("shutting down benchmark client") + if bc != nil { + bc.shutdown() + } + }() + for { + in, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + var out *testpb.ClientStatus + switch t := in.Argtype.(type) { + case *testpb.ClientArgs_Setup: + logger.Infof("client setup received:") + if bc != nil { + logger.Infof("client setup received when client already exists, shutting down the existing client") + bc.shutdown() + } + bc, err = startBenchmarkClient(ctx, t.Setup) + if err != nil { + return err + } + out = &testpb.ClientStatus{ + Stats: bc.getStats(false), + } + + case *testpb.ClientArgs_Mark: + logger.Infof("client mark received:") + logger.Infof(" - %v", t) + if bc == nil { + return status.Error(codes.InvalidArgument, "client does not exist when mark received") + } + out = &testpb.ClientStatus{ + Stats: bc.getStats(t.Mark.Reset_), + } + } + + if err := stream.Send(out); err != nil { + return err + } + } +} + +func (s *workerServer) CoreCount(context.Context, *testpb.CoreRequest) (*testpb.CoreResponse, error) { + logger.Infof("core count: %v", runtime.NumCPU()) + return &testpb.CoreResponse{Cores: int32(runtime.NumCPU())}, nil +} + +func (s *workerServer) QuitWorker(context.Context, *testpb.Void) (*testpb.Void, error) { + logger.Infof("quitting worker") + s.stop <- true + return &testpb.Void{}, nil +} + +func main() { + grpc.EnableTracing = false + + flag.Parse() + lis, err := net.Listen("tcp", ":"+strconv.Itoa(*driverPort)) + if err != nil { + logger.Fatalf("failed to listen: %v", err) + } + logger.Infof("worker listening at port %v", *driverPort) + + s := grpc.NewServer() + stop := make(chan bool) + testgrpc.RegisterWorkerServiceServer(s, &workerServer{ + stop: stop, + serverPort: *serverPort, + }) + + go func() { + <-stop + // Wait for 1 second before stopping the server to make sure the return value of QuitWorker is sent to client. + // TODO revise this once server graceful stop is supported in gRPC. + time.Sleep(time.Second) + s.Stop() + }() + + runtime.SetBlockProfileRate(*blockProfRate) + + if *pprofPort >= 0 { + go func() { + logger.Infoln("Starting pprof server on port " + strconv.Itoa(*pprofPort)) + logger.Infoln(http.ListenAndServe("localhost:"+strconv.Itoa(*pprofPort), nil)) + }() + } + + s.Serve(lis) +} diff --git a/binarylog/binarylog_end2end_test.go b/binarylog/binarylog_end2end_test.go new file mode 100644 index 0000000..434d7b7 --- /dev/null +++ b/binarylog/binarylog_end2end_test.go @@ -0,0 +1,1053 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog_test + +import ( + "context" + "fmt" + "io" + "net" + "net/netip" + "sort" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/binarylog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + iblog "google.golang.org/grpc/internal/binarylog" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var grpclogLogger = grpclog.Component("binarylog") + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func init() { + // Setting environment variable in tests doesn't work because of the init + // orders. Set the loggers directly here. + iblog.SetLogger(iblog.AllLogger) + binarylog.SetSink(testSink) +} + +var testSink = &testBinLogSink{} + +type testBinLogSink struct { + mu sync.Mutex + buf []*binlogpb.GrpcLogEntry +} + +func (s *testBinLogSink) Write(e *binlogpb.GrpcLogEntry) error { + s.mu.Lock() + s.buf = append(s.buf, e) + s.mu.Unlock() + return nil +} + +func (s *testBinLogSink) Close() error { return nil } + +// Returns all client entries if client is true, otherwise return all server +// entries. +func (s *testBinLogSink) logEntries(client bool) []*binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_SERVER + if client { + logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT + } + var ret []*binlogpb.GrpcLogEntry + s.mu.Lock() + for _, e := range s.buf { + if e.Logger == logger { + ret = append(ret, e) + } + } + s.mu.Unlock() + return ret +} + +func (s *testBinLogSink) clear() { + s.mu.Lock() + s.buf = nil + s.mu.Unlock() +} + +var ( + // For headers: + testMetadata = metadata.MD{ + "key1": []string{"value1"}, + "key2": []string{"value2"}, + } + // For trailers: + testTrailerMetadata = metadata.MD{ + "tkey1": []string{"trailerValue1"}, + "tkey2": []string{"trailerValue2"}, + } + // The id for which the service handler should return error. + errorID int32 = 32202 + + globalRPCID uint64 // RPC id starts with 1, but we do ++ at the beginning of each test. +) + +func idToPayload(id int32) *testpb.Payload { + return &testpb.Payload{Body: []byte{byte(id), byte(id >> 8), byte(id >> 16), byte(id >> 24)}} +} + +func payloadToID(p *testpb.Payload) int32 { + if p == nil || len(p.Body) != 4 { + panic("invalid payload") + } + return int32(p.Body[0]) + int32(p.Body[1])<<8 + int32(p.Body[2])<<16 + int32(p.Body[3])<<24 +} + +// test is an end-to-end test. It should be created with the newTest +// func, modified as needed, and then started with its startServer method. +// It should be cleaned up with the tearDown method. +type test struct { + t *testing.T + + // ss and srvAddr are set once startServer is called. + ss *stubserver.StubServer + srvAddr string // Server IP without port. + srvIP netip.Addr + srvPort int + + // Fields for client address. Set by the service handler. + clientAddrMu sync.Mutex + clientIP netip.Addr + clientPort int +} + +// newTest returns a new test using the provided testing.T and +// environment. It is returned with default values. Tests should +// modify it before calling its startServer and clientConn methods. +func newTest(t *testing.T) *test { + te := &test{ + t: t, + } + return te +} + +type listenerWrapper struct { + net.Listener + te *test +} + +func (lw *listenerWrapper) Accept() (net.Conn, error) { + conn, err := lw.Listener.Accept() + if err != nil { + return nil, err + } + lw.te.clientAddrMu.Lock() + lw.te.clientIP = conn.RemoteAddr().(*net.TCPAddr).AddrPort().Addr().Unmap() + lw.te.clientPort = conn.RemoteAddr().(*net.TCPAddr).Port + lw.te.clientAddrMu.Unlock() + return conn, nil +} + +// startServer starts a gRPC server listening. Callers should defer a +// call to te.tearDown to clean up. +func (te *test) startServer() { + lis, err := net.Listen("tcp", "localhost:0") + + lis = &listenerWrapper{ + Listener: lis, + te: te, + } + + if err != nil { + te.t.Fatalf("Failed to listen: %v", err) + } + + te.ss = &stubserver.StubServer{ + Listener: lis, + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + md, ok := metadata.FromIncomingContext(ctx) + if ok { + if err := grpc.SendHeader(ctx, md); err != nil { + return nil, status.Errorf(status.Code(err), "grpc.SendHeader(_, %v) = %v, want ", md, err) + } + if err := grpc.SetTrailer(ctx, testTrailerMetadata); err != nil { + return nil, status.Errorf(status.Code(err), "grpc.SetTrailer(_, %v) = %v, want ", testTrailerMetadata, err) + } + } + if id := payloadToID(in.Payload); id == errorID { + return nil, fmt.Errorf("got error id: %v", id) + } + return &testpb.SimpleResponse{Payload: in.Payload}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + md, ok := metadata.FromIncomingContext(stream.Context()) + if ok { + if err := stream.SendHeader(md); err != nil { + return status.Errorf(status.Code(err), "stream.SendHeader(%v) = %v, want %v", md, err, nil) + } + stream.SetTrailer(testTrailerMetadata) + } + for { + in, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if id := payloadToID(in.Payload); id == errorID { + return fmt.Errorf("got error id: %v", id) + } + if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: in.Payload}); err != nil { + return err + } + } + }, + StreamingInputCallF: func(stream testgrpc.TestService_StreamingInputCallServer) error { + md, ok := metadata.FromIncomingContext(stream.Context()) + if ok { + if err := stream.SendHeader(md); err != nil { + return status.Errorf(status.Code(err), "stream.SendHeader(%v) = %v, want %v", md, err, nil) + } + stream.SetTrailer(testTrailerMetadata) + } + for { + in, err := stream.Recv() + if err == io.EOF { + return stream.SendAndClose(&testpb.StreamingInputCallResponse{AggregatedPayloadSize: 0}) + } + if err != nil { + return err + } + if id := payloadToID(in.Payload); id == errorID { + return fmt.Errorf("got error id: %v", id) + } + } + }, + StreamingOutputCallF: func(in *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error { + md, ok := metadata.FromIncomingContext(stream.Context()) + if ok { + if err := stream.SendHeader(md); err != nil { + return status.Errorf(status.Code(err), "stream.SendHeader(%v) = %v, want %v", md, err, nil) + } + stream.SetTrailer(testTrailerMetadata) + } + if id := payloadToID(in.Payload); id == errorID { + return fmt.Errorf("got error id: %v", id) + } + for i := 0; i < 5; i++ { + if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: in.Payload}); err != nil { + return err + } + } + return nil + }, + } + if err := te.ss.Start(nil); err != nil { + te.t.Fatalf("Failed to start server: %v", err) + } + te.srvAddr = lis.Addr().String() + te.srvIP = lis.Addr().(*net.TCPAddr).AddrPort().Addr().Unmap() + te.srvPort = lis.Addr().(*net.TCPAddr).Port +} + +type rpcType int + +const ( + unaryRPC rpcType = iota + clientStreamRPC + serverStreamRPC + fullDuplexStreamRPC + cancelRPC +) + +type rpcConfig struct { + count int // Number of requests and responses for streaming RPCs. + success bool // Whether the RPC should succeed or return error. + callType rpcType // Type of RPC. +} + +func (te *test) doUnaryCall(c *rpcConfig) (*testpb.SimpleRequest, *testpb.SimpleResponse, error) { + var ( + resp *testpb.SimpleResponse + req *testpb.SimpleRequest + err error + ) + tc := testgrpc.NewTestServiceClient(te.ss.CC) + if c.success { + req = &testpb.SimpleRequest{Payload: idToPayload(errorID + 1)} + } else { + req = &testpb.SimpleRequest{Payload: idToPayload(errorID)} + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) + + resp, err = tc.UnaryCall(ctx, req) + return req, resp, err +} + +func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]proto.Message, []proto.Message, error) { + var ( + reqs []proto.Message + resps []proto.Message + err error + ) + tc := testgrpc.NewTestServiceClient(te.ss.CC) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) + + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + return reqs, resps, err + } + + if c.callType == cancelRPC { + cancel() + return reqs, resps, context.Canceled + } + + var startID int32 + if !c.success { + startID = errorID + } + for i := 0; i < c.count; i++ { + req := &testpb.StreamingOutputCallRequest{ + Payload: idToPayload(int32(i) + startID), + } + reqs = append(reqs, req) + if err = stream.Send(req); err != nil { + return reqs, resps, err + } + var resp *testpb.StreamingOutputCallResponse + if resp, err = stream.Recv(); err != nil { + return reqs, resps, err + } + resps = append(resps, resp) + } + if err = stream.CloseSend(); err != nil && err != io.EOF { + return reqs, resps, err + } + if _, err = stream.Recv(); err != io.EOF { + return reqs, resps, err + } + + return reqs, resps, nil +} + +func (te *test) doClientStreamCall(c *rpcConfig) ([]proto.Message, proto.Message, error) { + var ( + reqs []proto.Message + resp *testpb.StreamingInputCallResponse + err error + ) + tc := testgrpc.NewTestServiceClient(te.ss.CC) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) + + stream, err := tc.StreamingInputCall(ctx) + if err != nil { + return reqs, resp, err + } + var startID int32 + if !c.success { + startID = errorID + } + for i := 0; i < c.count; i++ { + req := &testpb.StreamingInputCallRequest{ + Payload: idToPayload(int32(i) + startID), + } + reqs = append(reqs, req) + if err = stream.Send(req); err != nil { + return reqs, resp, err + } + } + resp, err = stream.CloseAndRecv() + return reqs, resp, err +} + +func (te *test) doServerStreamCall(c *rpcConfig) (proto.Message, []proto.Message, error) { + var ( + req *testpb.StreamingOutputCallRequest + resps []proto.Message + err error + ) + + tc := testgrpc.NewTestServiceClient(te.ss.CC) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) + + var startID int32 + if !c.success { + startID = errorID + } + req = &testpb.StreamingOutputCallRequest{Payload: idToPayload(startID)} + stream, err := tc.StreamingOutputCall(ctx, req) + if err != nil { + return req, resps, err + } + for { + var resp *testpb.StreamingOutputCallResponse + resp, err := stream.Recv() + if err == io.EOF { + return req, resps, nil + } else if err != nil { + return req, resps, err + } + resps = append(resps, resp) + } +} + +type expectedData struct { + te *test + cc *rpcConfig + + method string + requests []proto.Message + responses []proto.Message + err error +} + +func (ed *expectedData) newClientHeaderEntry(client bool, rpcID, inRPCID uint64) *binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_CLIENT + var peer *binlogpb.Address + if !client { + logger = binlogpb.GrpcLogEntry_LOGGER_SERVER + ed.te.clientAddrMu.Lock() + peer = &binlogpb.Address{ + Address: ed.te.clientIP.String(), + IpPort: uint32(ed.te.clientPort), + } + if ed.te.clientIP.Is4() { + peer.Type = binlogpb.Address_TYPE_IPV4 + } else { + peer.Type = binlogpb.Address_TYPE_IPV6 + } + ed.te.clientAddrMu.Unlock() + } + return &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: rpcID, + SequenceIdWithinCall: inRPCID, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, + Logger: logger, + Payload: &binlogpb.GrpcLogEntry_ClientHeader{ + ClientHeader: &binlogpb.ClientHeader{ + Metadata: iblog.MdToMetadataProto(testMetadata), + MethodName: ed.method, + Authority: ed.te.srvAddr, + }, + }, + Peer: peer, + } +} + +func (ed *expectedData) newServerHeaderEntry(client bool, rpcID, inRPCID uint64) *binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_SERVER + var peer *binlogpb.Address + if client { + logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT + peer = &binlogpb.Address{ + Address: ed.te.srvIP.String(), + IpPort: uint32(ed.te.srvPort), + } + if ed.te.srvIP.Is4() { + peer.Type = binlogpb.Address_TYPE_IPV4 + } else { + peer.Type = binlogpb.Address_TYPE_IPV6 + } + } + return &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: rpcID, + SequenceIdWithinCall: inRPCID, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, + Logger: logger, + Payload: &binlogpb.GrpcLogEntry_ServerHeader{ + ServerHeader: &binlogpb.ServerHeader{ + Metadata: iblog.MdToMetadataProto(testMetadata), + }, + }, + Peer: peer, + } +} + +func (ed *expectedData) newClientMessageEntry(client bool, rpcID, inRPCID uint64, msg proto.Message) *binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_CLIENT + if !client { + logger = binlogpb.GrpcLogEntry_LOGGER_SERVER + } + data, err := proto.Marshal(msg) + if err != nil { + grpclogLogger.Infof("binarylogging_testing: failed to marshal proto message: %v", err) + } + return &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: rpcID, + SequenceIdWithinCall: inRPCID, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE, + Logger: logger, + Payload: &binlogpb.GrpcLogEntry_Message{ + Message: &binlogpb.Message{ + Length: uint32(len(data)), + Data: data, + }, + }, + } +} + +func (ed *expectedData) newServerMessageEntry(client bool, rpcID, inRPCID uint64, msg proto.Message) *binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_CLIENT + if !client { + logger = binlogpb.GrpcLogEntry_LOGGER_SERVER + } + data, err := proto.Marshal(msg) + if err != nil { + grpclogLogger.Infof("binarylogging_testing: failed to marshal proto message: %v", err) + } + return &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: rpcID, + SequenceIdWithinCall: inRPCID, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE, + Logger: logger, + Payload: &binlogpb.GrpcLogEntry_Message{ + Message: &binlogpb.Message{ + Length: uint32(len(data)), + Data: data, + }, + }, + } +} + +func (ed *expectedData) newHalfCloseEntry(client bool, rpcID, inRPCID uint64) *binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_CLIENT + if !client { + logger = binlogpb.GrpcLogEntry_LOGGER_SERVER + } + return &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: rpcID, + SequenceIdWithinCall: inRPCID, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE, + Payload: nil, // No payload here. + Logger: logger, + } +} + +func (ed *expectedData) newServerTrailerEntry(client bool, rpcID, inRPCID uint64, stErr error) *binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_SERVER + var peer *binlogpb.Address + if client { + logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT + peer = &binlogpb.Address{ + Address: ed.te.srvIP.String(), + IpPort: uint32(ed.te.srvPort), + } + if ed.te.srvIP.Is4() { + peer.Type = binlogpb.Address_TYPE_IPV4 + } else { + peer.Type = binlogpb.Address_TYPE_IPV6 + } + } + st, ok := status.FromError(stErr) + if !ok { + grpclogLogger.Info("binarylogging: error in trailer is not a status error") + } + stProto := st.Proto() + var ( + detailsBytes []byte + err error + ) + if stProto != nil && len(stProto.Details) != 0 { + detailsBytes, err = proto.Marshal(stProto) + if err != nil { + grpclogLogger.Infof("binarylogging: failed to marshal status proto: %v", err) + } + } + return &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: rpcID, + SequenceIdWithinCall: inRPCID, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, + Logger: logger, + Payload: &binlogpb.GrpcLogEntry_Trailer{ + Trailer: &binlogpb.Trailer{ + Metadata: iblog.MdToMetadataProto(testTrailerMetadata), + // st will be nil if err was not a status error, but nil is ok. + StatusCode: uint32(st.Code()), + StatusMessage: st.Message(), + StatusDetails: detailsBytes, + }, + }, + Peer: peer, + } +} + +func (ed *expectedData) newCancelEntry(rpcID, inRPCID uint64) *binlogpb.GrpcLogEntry { + return &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: rpcID, + SequenceIdWithinCall: inRPCID, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CANCEL, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, + Payload: nil, + } +} + +func (ed *expectedData) toClientLogEntries() []*binlogpb.GrpcLogEntry { + var ( + ret []*binlogpb.GrpcLogEntry + idInRPC uint64 = 1 + ) + ret = append(ret, ed.newClientHeaderEntry(true, globalRPCID, idInRPC)) + idInRPC++ + + switch ed.cc.callType { + case unaryRPC, fullDuplexStreamRPC: + for i := 0; i < len(ed.requests); i++ { + ret = append(ret, ed.newClientMessageEntry(true, globalRPCID, idInRPC, ed.requests[i])) + idInRPC++ + if i == 0 { + // First message, append ServerHeader. + ret = append(ret, ed.newServerHeaderEntry(true, globalRPCID, idInRPC)) + idInRPC++ + } + if !ed.cc.success { + // There is no response in the RPC error case. + continue + } + ret = append(ret, ed.newServerMessageEntry(true, globalRPCID, idInRPC, ed.responses[i])) + idInRPC++ + } + if ed.cc.success && ed.cc.callType == fullDuplexStreamRPC { + ret = append(ret, ed.newHalfCloseEntry(true, globalRPCID, idInRPC)) + idInRPC++ + } + case clientStreamRPC, serverStreamRPC: + for i := 0; i < len(ed.requests); i++ { + ret = append(ret, ed.newClientMessageEntry(true, globalRPCID, idInRPC, ed.requests[i])) + idInRPC++ + } + if ed.cc.callType == clientStreamRPC { + ret = append(ret, ed.newHalfCloseEntry(true, globalRPCID, idInRPC)) + idInRPC++ + } + ret = append(ret, ed.newServerHeaderEntry(true, globalRPCID, idInRPC)) + idInRPC++ + if ed.cc.success { + for i := 0; i < len(ed.responses); i++ { + ret = append(ret, ed.newServerMessageEntry(true, globalRPCID, idInRPC, ed.responses[0])) + idInRPC++ + } + } + } + + if ed.cc.callType == cancelRPC { + ret = append(ret, ed.newCancelEntry(globalRPCID, idInRPC)) + idInRPC++ + } else { + ret = append(ret, ed.newServerTrailerEntry(true, globalRPCID, idInRPC, ed.err)) + idInRPC++ + } + return ret +} + +func (ed *expectedData) toServerLogEntries() []*binlogpb.GrpcLogEntry { + var ( + ret []*binlogpb.GrpcLogEntry + idInRPC uint64 = 1 + ) + ret = append(ret, ed.newClientHeaderEntry(false, globalRPCID, idInRPC)) + idInRPC++ + + switch ed.cc.callType { + case unaryRPC: + ret = append(ret, ed.newClientMessageEntry(false, globalRPCID, idInRPC, ed.requests[0])) + idInRPC++ + ret = append(ret, ed.newServerHeaderEntry(false, globalRPCID, idInRPC)) + idInRPC++ + if ed.cc.success { + ret = append(ret, ed.newServerMessageEntry(false, globalRPCID, idInRPC, ed.responses[0])) + idInRPC++ + } + case fullDuplexStreamRPC: + ret = append(ret, ed.newServerHeaderEntry(false, globalRPCID, idInRPC)) + idInRPC++ + for i := 0; i < len(ed.requests); i++ { + ret = append(ret, ed.newClientMessageEntry(false, globalRPCID, idInRPC, ed.requests[i])) + idInRPC++ + if !ed.cc.success { + // There is no response in the RPC error case. + continue + } + ret = append(ret, ed.newServerMessageEntry(false, globalRPCID, idInRPC, ed.responses[i])) + idInRPC++ + } + + if ed.cc.success && ed.cc.callType == fullDuplexStreamRPC { + ret = append(ret, ed.newHalfCloseEntry(false, globalRPCID, idInRPC)) + idInRPC++ + } + case clientStreamRPC: + ret = append(ret, ed.newServerHeaderEntry(false, globalRPCID, idInRPC)) + idInRPC++ + for i := 0; i < len(ed.requests); i++ { + ret = append(ret, ed.newClientMessageEntry(false, globalRPCID, idInRPC, ed.requests[i])) + idInRPC++ + } + if ed.cc.success { + ret = append(ret, ed.newHalfCloseEntry(false, globalRPCID, idInRPC)) + idInRPC++ + ret = append(ret, ed.newServerMessageEntry(false, globalRPCID, idInRPC, ed.responses[0])) + idInRPC++ + } + case serverStreamRPC: + ret = append(ret, ed.newClientMessageEntry(false, globalRPCID, idInRPC, ed.requests[0])) + idInRPC++ + ret = append(ret, ed.newServerHeaderEntry(false, globalRPCID, idInRPC)) + idInRPC++ + for i := 0; i < len(ed.responses); i++ { + ret = append(ret, ed.newServerMessageEntry(false, globalRPCID, idInRPC, ed.responses[0])) + idInRPC++ + } + } + + ret = append(ret, ed.newServerTrailerEntry(false, globalRPCID, idInRPC, ed.err)) + idInRPC++ + + return ret +} + +func runRPCs(t *testing.T, cc *rpcConfig) *expectedData { + te := newTest(t) + te.startServer() + defer te.ss.Stop() + + expect := &expectedData{ + te: te, + cc: cc, + } + + switch cc.callType { + case unaryRPC: + expect.method = "/grpc.testing.TestService/UnaryCall" + req, resp, err := te.doUnaryCall(cc) + expect.requests = []proto.Message{req} + expect.responses = []proto.Message{resp} + expect.err = err + case clientStreamRPC: + expect.method = "/grpc.testing.TestService/StreamingInputCall" + reqs, resp, err := te.doClientStreamCall(cc) + expect.requests = reqs + expect.responses = []proto.Message{resp} + expect.err = err + case serverStreamRPC: + expect.method = "/grpc.testing.TestService/StreamingOutputCall" + req, resps, err := te.doServerStreamCall(cc) + expect.responses = resps + expect.requests = []proto.Message{req} + expect.err = err + case fullDuplexStreamRPC, cancelRPC: + expect.method = "/grpc.testing.TestService/FullDuplexCall" + expect.requests, expect.responses, expect.err = te.doFullDuplexCallRoundtrip(cc) + } + if cc.success != (expect.err == nil) { + t.Fatalf("cc.success: %v, got error: %v", cc.success, expect.err) + } + te.ss.S.GracefulStop() // Wait for the server to stop. + + return expect +} + +// equalLogEntry sorts the metadata entries by key (to compare metadata). +// +// This function is typically called with only two entries. It's written in this +// way so the code can be put in a for loop instead of copied twice. +func equalLogEntry(entries ...*binlogpb.GrpcLogEntry) (equal bool) { + for i, e := range entries { + // Clear out some fields we don't compare. + e.Timestamp = nil + e.CallId = 0 // CallID is global to the binary, hard to compare. + if h := e.GetClientHeader(); h != nil { + h.Timeout = nil + tmp := append(h.Metadata.Entry[:0], h.Metadata.Entry...) + h.Metadata.Entry = tmp + sort.Slice(h.Metadata.Entry, func(i, j int) bool { return h.Metadata.Entry[i].Key < h.Metadata.Entry[j].Key }) + } + if h := e.GetServerHeader(); h != nil { + tmp := append(h.Metadata.Entry[:0], h.Metadata.Entry...) + h.Metadata.Entry = tmp + sort.Slice(h.Metadata.Entry, func(i, j int) bool { return h.Metadata.Entry[i].Key < h.Metadata.Entry[j].Key }) + } + if h := e.GetTrailer(); h != nil { + sort.Slice(h.Metadata.Entry, func(i, j int) bool { return h.Metadata.Entry[i].Key < h.Metadata.Entry[j].Key }) + } + + if i > 0 && !proto.Equal(e, entries[i-1]) { + return false + } + } + return true +} + +func testClientBinaryLog(t *testing.T, c *rpcConfig) error { + defer testSink.clear() + expect := runRPCs(t, c) + want := expect.toClientLogEntries() + var got []*binlogpb.GrpcLogEntry + // In racy cases, some entries are not logged when the RPC is finished (e.g. + // context.Cancel). + // + // Check 10 times, with a sleep of 1/100 seconds between each check. Makes + // it an 1-second wait in total. + for i := 0; i < 10; i++ { + got = testSink.logEntries(true) // all client entries. + if len(want) == len(got) { + break + } + time.Sleep(100 * time.Millisecond) + } + if len(want) != len(got) { + for i, e := range want { + t.Errorf("in want: %d, %s", i, e.GetType()) + } + for i, e := range got { + t.Errorf("in got: %d, %s", i, e.GetType()) + } + return fmt.Errorf("didn't get same amount of log entries, want: %d, got: %d", len(want), len(got)) + } + var errored bool + for i := 0; i < len(got); i++ { + if !equalLogEntry(want[i], got[i]) { + t.Errorf("entry: %d, want %+v, got %+v", i, want[i], got[i]) + errored = true + } + } + if errored { + return fmt.Errorf("test failed") + } + return nil +} + +func (s) TestClientBinaryLogUnaryRPC(t *testing.T) { + if err := testClientBinaryLog(t, &rpcConfig{success: true, callType: unaryRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestClientBinaryLogUnaryRPCError(t *testing.T) { + if err := testClientBinaryLog(t, &rpcConfig{success: false, callType: unaryRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestClientBinaryLogClientStreamRPC(t *testing.T) { + count := 5 + if err := testClientBinaryLog(t, &rpcConfig{count: count, success: true, callType: clientStreamRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestClientBinaryLogClientStreamRPCError(t *testing.T) { + count := 1 + if err := testClientBinaryLog(t, &rpcConfig{count: count, success: false, callType: clientStreamRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestClientBinaryLogServerStreamRPC(t *testing.T) { + count := 5 + if err := testClientBinaryLog(t, &rpcConfig{count: count, success: true, callType: serverStreamRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestClientBinaryLogServerStreamRPCError(t *testing.T) { + count := 5 + if err := testClientBinaryLog(t, &rpcConfig{count: count, success: false, callType: serverStreamRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestClientBinaryLogFullDuplexRPC(t *testing.T) { + count := 5 + if err := testClientBinaryLog(t, &rpcConfig{count: count, success: true, callType: fullDuplexStreamRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestClientBinaryLogFullDuplexRPCError(t *testing.T) { + count := 5 + if err := testClientBinaryLog(t, &rpcConfig{count: count, success: false, callType: fullDuplexStreamRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestClientBinaryLogCancel(t *testing.T) { + count := 5 + if err := testClientBinaryLog(t, &rpcConfig{count: count, success: false, callType: cancelRPC}); err != nil { + t.Fatal(err) + } +} + +func testServerBinaryLog(t *testing.T, c *rpcConfig) error { + defer testSink.clear() + expect := runRPCs(t, c) + want := expect.toServerLogEntries() + var got []*binlogpb.GrpcLogEntry + // In racy cases, some entries are not logged when the RPC is finished (e.g. + // context.Cancel). This is unlikely to happen on server side, but it does + // no harm to retry. + // + // Check 10 times, with a sleep of 1/100 seconds between each check. Makes + // it an 1-second wait in total. + for i := 0; i < 10; i++ { + got = testSink.logEntries(false) // all server entries. + if len(want) == len(got) { + break + } + time.Sleep(100 * time.Millisecond) + } + + if len(want) != len(got) { + for i, e := range want { + t.Errorf("in want: %d, %s", i, e.GetType()) + } + for i, e := range got { + t.Errorf("in got: %d, %s", i, e.GetType()) + } + return fmt.Errorf("didn't get same amount of log entries, want: %d, got: %d", len(want), len(got)) + } + var errored bool + for i := 0; i < len(got); i++ { + if !equalLogEntry(want[i], got[i]) { + t.Errorf("entry: %d, want %+v, got %+v", i, want[i], got[i]) + errored = true + } + } + if errored { + return fmt.Errorf("test failed") + } + return nil +} + +func (s) TestServerBinaryLogUnaryRPC(t *testing.T) { + if err := testServerBinaryLog(t, &rpcConfig{success: true, callType: unaryRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestServerBinaryLogUnaryRPCError(t *testing.T) { + if err := testServerBinaryLog(t, &rpcConfig{success: false, callType: unaryRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestServerBinaryLogClientStreamRPC(t *testing.T) { + count := 5 + if err := testServerBinaryLog(t, &rpcConfig{count: count, success: true, callType: clientStreamRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestServerBinaryLogClientStreamRPCError(t *testing.T) { + count := 1 + if err := testServerBinaryLog(t, &rpcConfig{count: count, success: false, callType: clientStreamRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestServerBinaryLogServerStreamRPC(t *testing.T) { + count := 5 + if err := testServerBinaryLog(t, &rpcConfig{count: count, success: true, callType: serverStreamRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestServerBinaryLogServerStreamRPCError(t *testing.T) { + count := 5 + if err := testServerBinaryLog(t, &rpcConfig{count: count, success: false, callType: serverStreamRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestServerBinaryLogFullDuplex(t *testing.T) { + count := 5 + if err := testServerBinaryLog(t, &rpcConfig{count: count, success: true, callType: fullDuplexStreamRPC}); err != nil { + t.Fatal(err) + } +} + +func (s) TestServerBinaryLogFullDuplexError(t *testing.T) { + count := 5 + if err := testServerBinaryLog(t, &rpcConfig{count: count, success: false, callType: fullDuplexStreamRPC}); err != nil { + t.Fatal(err) + } +} + +// TestCanceledStatus ensures a server that responds with a Canceled status has +// its trailers logged appropriately and is not treated as a canceled RPC. +func (s) TestCanceledStatus(t *testing.T) { + defer testSink.clear() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + const statusMsgWant = "server returned Canceled" + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + grpc.SetTrailer(ctx, metadata.Pairs("key", "value")) + return nil, status.Error(codes.Canceled, statusMsgWant) + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Canceled { + t.Fatalf("Received unexpected error from UnaryCall: %v; want Canceled", err) + } + + got := testSink.logEntries(true) + last := got[len(got)-1] + if last.Type != binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER || + last.GetTrailer().GetStatusCode() != uint32(codes.Canceled) || + last.GetTrailer().GetStatusMessage() != statusMsgWant || + len(last.GetTrailer().GetMetadata().GetEntry()) != 1 || + last.GetTrailer().GetMetadata().GetEntry()[0].GetKey() != "key" || + string(last.GetTrailer().GetMetadata().GetEntry()[0].GetValue()) != "value" { + t.Fatalf("Got binary log: %+v; want last entry is server trailing with status Canceled", got) + } +} diff --git a/binarylog/grpc_binarylog_v1/binarylog.pb.go b/binarylog/grpc_binarylog_v1/binarylog.pb.go new file mode 100644 index 0000000..d22d1c3 --- /dev/null +++ b/binarylog/grpc_binarylog_v1/binarylog.pb.go @@ -0,0 +1,1006 @@ +// Copyright 2018 The gRPC Authors +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/binlog/v1/binarylog.proto + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/binlog/v1/binarylog.proto + +package grpc_binarylog_v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Enumerates the type of event +// Note the terminology is different from the RPC semantics +// definition, but the same meaning is expressed here. +type GrpcLogEntry_EventType int32 + +const ( + GrpcLogEntry_EVENT_TYPE_UNKNOWN GrpcLogEntry_EventType = 0 + // Header sent from client to server + GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER GrpcLogEntry_EventType = 1 + // Header sent from server to client + GrpcLogEntry_EVENT_TYPE_SERVER_HEADER GrpcLogEntry_EventType = 2 + // Message sent from client to server + GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE GrpcLogEntry_EventType = 3 + // Message sent from server to client + GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE GrpcLogEntry_EventType = 4 + // A signal that client is done sending + GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE GrpcLogEntry_EventType = 5 + // Trailer indicates the end of the RPC. + // On client side, this event means a trailer was either received + // from the network or the gRPC library locally generated a status + // to inform the application about a failure. + // On server side, this event means the server application requested + // to send a trailer. Note: EVENT_TYPE_CANCEL may still arrive after + // this due to races on server side. + GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER GrpcLogEntry_EventType = 6 + // A signal that the RPC is cancelled. On client side, this + // indicates the client application requests a cancellation. + // On server side, this indicates that cancellation was detected. + // Note: This marks the end of the RPC. Events may arrive after + // this due to races. For example, on client side a trailer + // may arrive even though the application requested to cancel the RPC. + GrpcLogEntry_EVENT_TYPE_CANCEL GrpcLogEntry_EventType = 7 +) + +// Enum value maps for GrpcLogEntry_EventType. +var ( + GrpcLogEntry_EventType_name = map[int32]string{ + 0: "EVENT_TYPE_UNKNOWN", + 1: "EVENT_TYPE_CLIENT_HEADER", + 2: "EVENT_TYPE_SERVER_HEADER", + 3: "EVENT_TYPE_CLIENT_MESSAGE", + 4: "EVENT_TYPE_SERVER_MESSAGE", + 5: "EVENT_TYPE_CLIENT_HALF_CLOSE", + 6: "EVENT_TYPE_SERVER_TRAILER", + 7: "EVENT_TYPE_CANCEL", + } + GrpcLogEntry_EventType_value = map[string]int32{ + "EVENT_TYPE_UNKNOWN": 0, + "EVENT_TYPE_CLIENT_HEADER": 1, + "EVENT_TYPE_SERVER_HEADER": 2, + "EVENT_TYPE_CLIENT_MESSAGE": 3, + "EVENT_TYPE_SERVER_MESSAGE": 4, + "EVENT_TYPE_CLIENT_HALF_CLOSE": 5, + "EVENT_TYPE_SERVER_TRAILER": 6, + "EVENT_TYPE_CANCEL": 7, + } +) + +func (x GrpcLogEntry_EventType) Enum() *GrpcLogEntry_EventType { + p := new(GrpcLogEntry_EventType) + *p = x + return p +} + +func (x GrpcLogEntry_EventType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GrpcLogEntry_EventType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_binlog_v1_binarylog_proto_enumTypes[0].Descriptor() +} + +func (GrpcLogEntry_EventType) Type() protoreflect.EnumType { + return &file_grpc_binlog_v1_binarylog_proto_enumTypes[0] +} + +func (x GrpcLogEntry_EventType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GrpcLogEntry_EventType.Descriptor instead. +func (GrpcLogEntry_EventType) EnumDescriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{0, 0} +} + +// Enumerates the entity that generates the log entry +type GrpcLogEntry_Logger int32 + +const ( + GrpcLogEntry_LOGGER_UNKNOWN GrpcLogEntry_Logger = 0 + GrpcLogEntry_LOGGER_CLIENT GrpcLogEntry_Logger = 1 + GrpcLogEntry_LOGGER_SERVER GrpcLogEntry_Logger = 2 +) + +// Enum value maps for GrpcLogEntry_Logger. +var ( + GrpcLogEntry_Logger_name = map[int32]string{ + 0: "LOGGER_UNKNOWN", + 1: "LOGGER_CLIENT", + 2: "LOGGER_SERVER", + } + GrpcLogEntry_Logger_value = map[string]int32{ + "LOGGER_UNKNOWN": 0, + "LOGGER_CLIENT": 1, + "LOGGER_SERVER": 2, + } +) + +func (x GrpcLogEntry_Logger) Enum() *GrpcLogEntry_Logger { + p := new(GrpcLogEntry_Logger) + *p = x + return p +} + +func (x GrpcLogEntry_Logger) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GrpcLogEntry_Logger) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_binlog_v1_binarylog_proto_enumTypes[1].Descriptor() +} + +func (GrpcLogEntry_Logger) Type() protoreflect.EnumType { + return &file_grpc_binlog_v1_binarylog_proto_enumTypes[1] +} + +func (x GrpcLogEntry_Logger) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GrpcLogEntry_Logger.Descriptor instead. +func (GrpcLogEntry_Logger) EnumDescriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{0, 1} +} + +type Address_Type int32 + +const ( + Address_TYPE_UNKNOWN Address_Type = 0 + // address is in 1.2.3.4 form + Address_TYPE_IPV4 Address_Type = 1 + // address is in IPv6 canonical form (RFC5952 section 4) + // The scope is NOT included in the address string. + Address_TYPE_IPV6 Address_Type = 2 + // address is UDS string + Address_TYPE_UNIX Address_Type = 3 +) + +// Enum value maps for Address_Type. +var ( + Address_Type_name = map[int32]string{ + 0: "TYPE_UNKNOWN", + 1: "TYPE_IPV4", + 2: "TYPE_IPV6", + 3: "TYPE_UNIX", + } + Address_Type_value = map[string]int32{ + "TYPE_UNKNOWN": 0, + "TYPE_IPV4": 1, + "TYPE_IPV6": 2, + "TYPE_UNIX": 3, + } +) + +func (x Address_Type) Enum() *Address_Type { + p := new(Address_Type) + *p = x + return p +} + +func (x Address_Type) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Address_Type) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_binlog_v1_binarylog_proto_enumTypes[2].Descriptor() +} + +func (Address_Type) Type() protoreflect.EnumType { + return &file_grpc_binlog_v1_binarylog_proto_enumTypes[2] +} + +func (x Address_Type) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Address_Type.Descriptor instead. +func (Address_Type) EnumDescriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{7, 0} +} + +// Log entry we store in binary logs +type GrpcLogEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The timestamp of the binary log message + Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + // Uniquely identifies a call. The value must not be 0 in order to disambiguate + // from an unset value. + // Each call may have several log entries, they will all have the same call_id. + // Nothing is guaranteed about their value other than they are unique across + // different RPCs in the same gRPC process. + CallId uint64 `protobuf:"varint,2,opt,name=call_id,json=callId,proto3" json:"call_id,omitempty"` + // The entry sequence id for this call. The first GrpcLogEntry has a + // value of 1, to disambiguate from an unset value. The purpose of + // this field is to detect missing entries in environments where + // durability or ordering is not guaranteed. + SequenceIdWithinCall uint64 `protobuf:"varint,3,opt,name=sequence_id_within_call,json=sequenceIdWithinCall,proto3" json:"sequence_id_within_call,omitempty"` + Type GrpcLogEntry_EventType `protobuf:"varint,4,opt,name=type,proto3,enum=grpc.binarylog.v1.GrpcLogEntry_EventType" json:"type,omitempty"` + Logger GrpcLogEntry_Logger `protobuf:"varint,5,opt,name=logger,proto3,enum=grpc.binarylog.v1.GrpcLogEntry_Logger" json:"logger,omitempty"` // One of the above Logger enum + // The logger uses one of the following fields to record the payload, + // according to the type of the log entry. + // + // Types that are valid to be assigned to Payload: + // + // *GrpcLogEntry_ClientHeader + // *GrpcLogEntry_ServerHeader + // *GrpcLogEntry_Message + // *GrpcLogEntry_Trailer + Payload isGrpcLogEntry_Payload `protobuf_oneof:"payload"` + // true if payload does not represent the full message or metadata. + PayloadTruncated bool `protobuf:"varint,10,opt,name=payload_truncated,json=payloadTruncated,proto3" json:"payload_truncated,omitempty"` + // Peer address information, will only be recorded on the first + // incoming event. On client side, peer is logged on + // EVENT_TYPE_SERVER_HEADER normally or EVENT_TYPE_SERVER_TRAILER in + // the case of trailers-only. On server side, peer is always + // logged on EVENT_TYPE_CLIENT_HEADER. + Peer *Address `protobuf:"bytes,11,opt,name=peer,proto3" json:"peer,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GrpcLogEntry) Reset() { + *x = GrpcLogEntry{} + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GrpcLogEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GrpcLogEntry) ProtoMessage() {} + +func (x *GrpcLogEntry) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GrpcLogEntry.ProtoReflect.Descriptor instead. +func (*GrpcLogEntry) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{0} +} + +func (x *GrpcLogEntry) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp + } + return nil +} + +func (x *GrpcLogEntry) GetCallId() uint64 { + if x != nil { + return x.CallId + } + return 0 +} + +func (x *GrpcLogEntry) GetSequenceIdWithinCall() uint64 { + if x != nil { + return x.SequenceIdWithinCall + } + return 0 +} + +func (x *GrpcLogEntry) GetType() GrpcLogEntry_EventType { + if x != nil { + return x.Type + } + return GrpcLogEntry_EVENT_TYPE_UNKNOWN +} + +func (x *GrpcLogEntry) GetLogger() GrpcLogEntry_Logger { + if x != nil { + return x.Logger + } + return GrpcLogEntry_LOGGER_UNKNOWN +} + +func (x *GrpcLogEntry) GetPayload() isGrpcLogEntry_Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *GrpcLogEntry) GetClientHeader() *ClientHeader { + if x != nil { + if x, ok := x.Payload.(*GrpcLogEntry_ClientHeader); ok { + return x.ClientHeader + } + } + return nil +} + +func (x *GrpcLogEntry) GetServerHeader() *ServerHeader { + if x != nil { + if x, ok := x.Payload.(*GrpcLogEntry_ServerHeader); ok { + return x.ServerHeader + } + } + return nil +} + +func (x *GrpcLogEntry) GetMessage() *Message { + if x != nil { + if x, ok := x.Payload.(*GrpcLogEntry_Message); ok { + return x.Message + } + } + return nil +} + +func (x *GrpcLogEntry) GetTrailer() *Trailer { + if x != nil { + if x, ok := x.Payload.(*GrpcLogEntry_Trailer); ok { + return x.Trailer + } + } + return nil +} + +func (x *GrpcLogEntry) GetPayloadTruncated() bool { + if x != nil { + return x.PayloadTruncated + } + return false +} + +func (x *GrpcLogEntry) GetPeer() *Address { + if x != nil { + return x.Peer + } + return nil +} + +type isGrpcLogEntry_Payload interface { + isGrpcLogEntry_Payload() +} + +type GrpcLogEntry_ClientHeader struct { + ClientHeader *ClientHeader `protobuf:"bytes,6,opt,name=client_header,json=clientHeader,proto3,oneof"` +} + +type GrpcLogEntry_ServerHeader struct { + ServerHeader *ServerHeader `protobuf:"bytes,7,opt,name=server_header,json=serverHeader,proto3,oneof"` +} + +type GrpcLogEntry_Message struct { + // Used by EVENT_TYPE_CLIENT_MESSAGE, EVENT_TYPE_SERVER_MESSAGE + Message *Message `protobuf:"bytes,8,opt,name=message,proto3,oneof"` +} + +type GrpcLogEntry_Trailer struct { + Trailer *Trailer `protobuf:"bytes,9,opt,name=trailer,proto3,oneof"` +} + +func (*GrpcLogEntry_ClientHeader) isGrpcLogEntry_Payload() {} + +func (*GrpcLogEntry_ServerHeader) isGrpcLogEntry_Payload() {} + +func (*GrpcLogEntry_Message) isGrpcLogEntry_Payload() {} + +func (*GrpcLogEntry_Trailer) isGrpcLogEntry_Payload() {} + +type ClientHeader struct { + state protoimpl.MessageState `protogen:"open.v1"` + // This contains only the metadata from the application. + Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + // The name of the RPC method, which looks something like: + // // + // Note the leading "/" character. + MethodName string `protobuf:"bytes,2,opt,name=method_name,json=methodName,proto3" json:"method_name,omitempty"` + // A single process may be used to run multiple virtual + // servers with different identities. + // The authority is the name of such a server identity. + // It is typically a portion of the URI in the form of + // or : . + Authority string `protobuf:"bytes,3,opt,name=authority,proto3" json:"authority,omitempty"` + // the RPC timeout + Timeout *durationpb.Duration `protobuf:"bytes,4,opt,name=timeout,proto3" json:"timeout,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClientHeader) Reset() { + *x = ClientHeader{} + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClientHeader) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientHeader) ProtoMessage() {} + +func (x *ClientHeader) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientHeader.ProtoReflect.Descriptor instead. +func (*ClientHeader) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{1} +} + +func (x *ClientHeader) GetMetadata() *Metadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *ClientHeader) GetMethodName() string { + if x != nil { + return x.MethodName + } + return "" +} + +func (x *ClientHeader) GetAuthority() string { + if x != nil { + return x.Authority + } + return "" +} + +func (x *ClientHeader) GetTimeout() *durationpb.Duration { + if x != nil { + return x.Timeout + } + return nil +} + +type ServerHeader struct { + state protoimpl.MessageState `protogen:"open.v1"` + // This contains only the metadata from the application. + Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerHeader) Reset() { + *x = ServerHeader{} + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerHeader) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerHeader) ProtoMessage() {} + +func (x *ServerHeader) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerHeader.ProtoReflect.Descriptor instead. +func (*ServerHeader) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{2} +} + +func (x *ServerHeader) GetMetadata() *Metadata { + if x != nil { + return x.Metadata + } + return nil +} + +type Trailer struct { + state protoimpl.MessageState `protogen:"open.v1"` + // This contains only the metadata from the application. + Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + // The gRPC status code. + StatusCode uint32 `protobuf:"varint,2,opt,name=status_code,json=statusCode,proto3" json:"status_code,omitempty"` + // An original status message before any transport specific + // encoding. + StatusMessage string `protobuf:"bytes,3,opt,name=status_message,json=statusMessage,proto3" json:"status_message,omitempty"` + // The value of the 'grpc-status-details-bin' metadata key. If + // present, this is always an encoded 'google.rpc.Status' message. + StatusDetails []byte `protobuf:"bytes,4,opt,name=status_details,json=statusDetails,proto3" json:"status_details,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Trailer) Reset() { + *x = Trailer{} + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Trailer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Trailer) ProtoMessage() {} + +func (x *Trailer) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Trailer.ProtoReflect.Descriptor instead. +func (*Trailer) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{3} +} + +func (x *Trailer) GetMetadata() *Metadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *Trailer) GetStatusCode() uint32 { + if x != nil { + return x.StatusCode + } + return 0 +} + +func (x *Trailer) GetStatusMessage() string { + if x != nil { + return x.StatusMessage + } + return "" +} + +func (x *Trailer) GetStatusDetails() []byte { + if x != nil { + return x.StatusDetails + } + return nil +} + +// Message payload, used by CLIENT_MESSAGE and SERVER_MESSAGE +type Message struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Length of the message. It may not be the same as the length of the + // data field, as the logging payload can be truncated or omitted. + Length uint32 `protobuf:"varint,1,opt,name=length,proto3" json:"length,omitempty"` + // May be truncated or omitted. + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Message) Reset() { + *x = Message{} + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Message) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Message) ProtoMessage() {} + +func (x *Message) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Message.ProtoReflect.Descriptor instead. +func (*Message) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{4} +} + +func (x *Message) GetLength() uint32 { + if x != nil { + return x.Length + } + return 0 +} + +func (x *Message) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +// A list of metadata pairs, used in the payload of client header, +// server header, and server trailer. +// Implementations may omit some entries to honor the header limits +// of GRPC_BINARY_LOG_FILTER. +// +// Header keys added by gRPC are omitted. To be more specific, +// implementations will not log the following entries, and this is +// not to be treated as a truncation: +// - entries handled by grpc that are not user visible, such as those +// that begin with 'grpc-' (with exception of grpc-trace-bin) +// or keys like 'lb-token' +// - transport specific entries, including but not limited to: +// ':path', ':authority', 'content-encoding', 'user-agent', 'te', etc +// - entries added for call credentials (on client side, excludes all +// such headers; on server side, excludes only the "authorization" +// header) +// +// Implementations must always log grpc-trace-bin if it is present. +// Practically speaking it will only be visible on server side because +// grpc-trace-bin is managed by low level client side mechanisms +// inaccessible from the application level. On server side, the +// header is just a normal metadata key. +// The pair will not count towards the size limit. +type Metadata struct { + state protoimpl.MessageState `protogen:"open.v1"` + Entry []*MetadataEntry `protobuf:"bytes,1,rep,name=entry,proto3" json:"entry,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Metadata) Reset() { + *x = Metadata{} + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Metadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Metadata) ProtoMessage() {} + +func (x *Metadata) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Metadata.ProtoReflect.Descriptor instead. +func (*Metadata) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{5} +} + +func (x *Metadata) GetEntry() []*MetadataEntry { + if x != nil { + return x.Entry + } + return nil +} + +// A metadata key value pair +type MetadataEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MetadataEntry) Reset() { + *x = MetadataEntry{} + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MetadataEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetadataEntry) ProtoMessage() {} + +func (x *MetadataEntry) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetadataEntry.ProtoReflect.Descriptor instead. +func (*MetadataEntry) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{6} +} + +func (x *MetadataEntry) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *MetadataEntry) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +// Address information +type Address struct { + state protoimpl.MessageState `protogen:"open.v1"` + Type Address_Type `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.binarylog.v1.Address_Type" json:"type,omitempty"` + Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"` + // only for TYPE_IPV4 and TYPE_IPV6 + IpPort uint32 `protobuf:"varint,3,opt,name=ip_port,json=ipPort,proto3" json:"ip_port,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Address) Reset() { + *x = Address{} + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Address) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Address) ProtoMessage() {} + +func (x *Address) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Address.ProtoReflect.Descriptor instead. +func (*Address) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{7} +} + +func (x *Address) GetType() Address_Type { + if x != nil { + return x.Type + } + return Address_TYPE_UNKNOWN +} + +func (x *Address) GetAddress() string { + if x != nil { + return x.Address + } + return "" +} + +func (x *Address) GetIpPort() uint32 { + if x != nil { + return x.IpPort + } + return 0 +} + +var File_grpc_binlog_v1_binarylog_proto protoreflect.FileDescriptor + +const file_grpc_binlog_v1_binarylog_proto_rawDesc = "" + + "\n" + + "\x1egrpc/binlog/v1/binarylog.proto\x12\x11grpc.binarylog.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xbb\a\n" + + "\fGrpcLogEntry\x128\n" + + "\ttimestamp\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\x12\x17\n" + + "\acall_id\x18\x02 \x01(\x04R\x06callId\x125\n" + + "\x17sequence_id_within_call\x18\x03 \x01(\x04R\x14sequenceIdWithinCall\x12=\n" + + "\x04type\x18\x04 \x01(\x0e2).grpc.binarylog.v1.GrpcLogEntry.EventTypeR\x04type\x12>\n" + + "\x06logger\x18\x05 \x01(\x0e2&.grpc.binarylog.v1.GrpcLogEntry.LoggerR\x06logger\x12F\n" + + "\rclient_header\x18\x06 \x01(\v2\x1f.grpc.binarylog.v1.ClientHeaderH\x00R\fclientHeader\x12F\n" + + "\rserver_header\x18\a \x01(\v2\x1f.grpc.binarylog.v1.ServerHeaderH\x00R\fserverHeader\x126\n" + + "\amessage\x18\b \x01(\v2\x1a.grpc.binarylog.v1.MessageH\x00R\amessage\x126\n" + + "\atrailer\x18\t \x01(\v2\x1a.grpc.binarylog.v1.TrailerH\x00R\atrailer\x12+\n" + + "\x11payload_truncated\x18\n" + + " \x01(\bR\x10payloadTruncated\x12.\n" + + "\x04peer\x18\v \x01(\v2\x1a.grpc.binarylog.v1.AddressR\x04peer\"\xf5\x01\n" + + "\tEventType\x12\x16\n" + + "\x12EVENT_TYPE_UNKNOWN\x10\x00\x12\x1c\n" + + "\x18EVENT_TYPE_CLIENT_HEADER\x10\x01\x12\x1c\n" + + "\x18EVENT_TYPE_SERVER_HEADER\x10\x02\x12\x1d\n" + + "\x19EVENT_TYPE_CLIENT_MESSAGE\x10\x03\x12\x1d\n" + + "\x19EVENT_TYPE_SERVER_MESSAGE\x10\x04\x12 \n" + + "\x1cEVENT_TYPE_CLIENT_HALF_CLOSE\x10\x05\x12\x1d\n" + + "\x19EVENT_TYPE_SERVER_TRAILER\x10\x06\x12\x15\n" + + "\x11EVENT_TYPE_CANCEL\x10\a\"B\n" + + "\x06Logger\x12\x12\n" + + "\x0eLOGGER_UNKNOWN\x10\x00\x12\x11\n" + + "\rLOGGER_CLIENT\x10\x01\x12\x11\n" + + "\rLOGGER_SERVER\x10\x02B\t\n" + + "\apayload\"\xbb\x01\n" + + "\fClientHeader\x127\n" + + "\bmetadata\x18\x01 \x01(\v2\x1b.grpc.binarylog.v1.MetadataR\bmetadata\x12\x1f\n" + + "\vmethod_name\x18\x02 \x01(\tR\n" + + "methodName\x12\x1c\n" + + "\tauthority\x18\x03 \x01(\tR\tauthority\x123\n" + + "\atimeout\x18\x04 \x01(\v2\x19.google.protobuf.DurationR\atimeout\"G\n" + + "\fServerHeader\x127\n" + + "\bmetadata\x18\x01 \x01(\v2\x1b.grpc.binarylog.v1.MetadataR\bmetadata\"\xb1\x01\n" + + "\aTrailer\x127\n" + + "\bmetadata\x18\x01 \x01(\v2\x1b.grpc.binarylog.v1.MetadataR\bmetadata\x12\x1f\n" + + "\vstatus_code\x18\x02 \x01(\rR\n" + + "statusCode\x12%\n" + + "\x0estatus_message\x18\x03 \x01(\tR\rstatusMessage\x12%\n" + + "\x0estatus_details\x18\x04 \x01(\fR\rstatusDetails\"5\n" + + "\aMessage\x12\x16\n" + + "\x06length\x18\x01 \x01(\rR\x06length\x12\x12\n" + + "\x04data\x18\x02 \x01(\fR\x04data\"B\n" + + "\bMetadata\x126\n" + + "\x05entry\x18\x01 \x03(\v2 .grpc.binarylog.v1.MetadataEntryR\x05entry\"7\n" + + "\rMetadataEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\fR\x05value\"\xb8\x01\n" + + "\aAddress\x123\n" + + "\x04type\x18\x01 \x01(\x0e2\x1f.grpc.binarylog.v1.Address.TypeR\x04type\x12\x18\n" + + "\aaddress\x18\x02 \x01(\tR\aaddress\x12\x17\n" + + "\aip_port\x18\x03 \x01(\rR\x06ipPort\"E\n" + + "\x04Type\x12\x10\n" + + "\fTYPE_UNKNOWN\x10\x00\x12\r\n" + + "\tTYPE_IPV4\x10\x01\x12\r\n" + + "\tTYPE_IPV6\x10\x02\x12\r\n" + + "\tTYPE_UNIX\x10\x03B\\\n" + + "\x14io.grpc.binarylog.v1B\x0eBinaryLogProtoP\x01Z2google.golang.org/grpc/binarylog/grpc_binarylog_v1b\x06proto3" + +var ( + file_grpc_binlog_v1_binarylog_proto_rawDescOnce sync.Once + file_grpc_binlog_v1_binarylog_proto_rawDescData []byte +) + +func file_grpc_binlog_v1_binarylog_proto_rawDescGZIP() []byte { + file_grpc_binlog_v1_binarylog_proto_rawDescOnce.Do(func() { + file_grpc_binlog_v1_binarylog_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_binlog_v1_binarylog_proto_rawDesc), len(file_grpc_binlog_v1_binarylog_proto_rawDesc))) + }) + return file_grpc_binlog_v1_binarylog_proto_rawDescData +} + +var file_grpc_binlog_v1_binarylog_proto_enumTypes = make([]protoimpl.EnumInfo, 3) +var file_grpc_binlog_v1_binarylog_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_grpc_binlog_v1_binarylog_proto_goTypes = []any{ + (GrpcLogEntry_EventType)(0), // 0: grpc.binarylog.v1.GrpcLogEntry.EventType + (GrpcLogEntry_Logger)(0), // 1: grpc.binarylog.v1.GrpcLogEntry.Logger + (Address_Type)(0), // 2: grpc.binarylog.v1.Address.Type + (*GrpcLogEntry)(nil), // 3: grpc.binarylog.v1.GrpcLogEntry + (*ClientHeader)(nil), // 4: grpc.binarylog.v1.ClientHeader + (*ServerHeader)(nil), // 5: grpc.binarylog.v1.ServerHeader + (*Trailer)(nil), // 6: grpc.binarylog.v1.Trailer + (*Message)(nil), // 7: grpc.binarylog.v1.Message + (*Metadata)(nil), // 8: grpc.binarylog.v1.Metadata + (*MetadataEntry)(nil), // 9: grpc.binarylog.v1.MetadataEntry + (*Address)(nil), // 10: grpc.binarylog.v1.Address + (*timestamppb.Timestamp)(nil), // 11: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 12: google.protobuf.Duration +} +var file_grpc_binlog_v1_binarylog_proto_depIdxs = []int32{ + 11, // 0: grpc.binarylog.v1.GrpcLogEntry.timestamp:type_name -> google.protobuf.Timestamp + 0, // 1: grpc.binarylog.v1.GrpcLogEntry.type:type_name -> grpc.binarylog.v1.GrpcLogEntry.EventType + 1, // 2: grpc.binarylog.v1.GrpcLogEntry.logger:type_name -> grpc.binarylog.v1.GrpcLogEntry.Logger + 4, // 3: grpc.binarylog.v1.GrpcLogEntry.client_header:type_name -> grpc.binarylog.v1.ClientHeader + 5, // 4: grpc.binarylog.v1.GrpcLogEntry.server_header:type_name -> grpc.binarylog.v1.ServerHeader + 7, // 5: grpc.binarylog.v1.GrpcLogEntry.message:type_name -> grpc.binarylog.v1.Message + 6, // 6: grpc.binarylog.v1.GrpcLogEntry.trailer:type_name -> grpc.binarylog.v1.Trailer + 10, // 7: grpc.binarylog.v1.GrpcLogEntry.peer:type_name -> grpc.binarylog.v1.Address + 8, // 8: grpc.binarylog.v1.ClientHeader.metadata:type_name -> grpc.binarylog.v1.Metadata + 12, // 9: grpc.binarylog.v1.ClientHeader.timeout:type_name -> google.protobuf.Duration + 8, // 10: grpc.binarylog.v1.ServerHeader.metadata:type_name -> grpc.binarylog.v1.Metadata + 8, // 11: grpc.binarylog.v1.Trailer.metadata:type_name -> grpc.binarylog.v1.Metadata + 9, // 12: grpc.binarylog.v1.Metadata.entry:type_name -> grpc.binarylog.v1.MetadataEntry + 2, // 13: grpc.binarylog.v1.Address.type:type_name -> grpc.binarylog.v1.Address.Type + 14, // [14:14] is the sub-list for method output_type + 14, // [14:14] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name +} + +func init() { file_grpc_binlog_v1_binarylog_proto_init() } +func file_grpc_binlog_v1_binarylog_proto_init() { + if File_grpc_binlog_v1_binarylog_proto != nil { + return + } + file_grpc_binlog_v1_binarylog_proto_msgTypes[0].OneofWrappers = []any{ + (*GrpcLogEntry_ClientHeader)(nil), + (*GrpcLogEntry_ServerHeader)(nil), + (*GrpcLogEntry_Message)(nil), + (*GrpcLogEntry_Trailer)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_binlog_v1_binarylog_proto_rawDesc), len(file_grpc_binlog_v1_binarylog_proto_rawDesc)), + NumEnums: 3, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_binlog_v1_binarylog_proto_goTypes, + DependencyIndexes: file_grpc_binlog_v1_binarylog_proto_depIdxs, + EnumInfos: file_grpc_binlog_v1_binarylog_proto_enumTypes, + MessageInfos: file_grpc_binlog_v1_binarylog_proto_msgTypes, + }.Build() + File_grpc_binlog_v1_binarylog_proto = out.File + file_grpc_binlog_v1_binarylog_proto_goTypes = nil + file_grpc_binlog_v1_binarylog_proto_depIdxs = nil +} diff --git a/binarylog/sink.go b/binarylog/sink.go new file mode 100644 index 0000000..d924e4c --- /dev/null +++ b/binarylog/sink.go @@ -0,0 +1,68 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package binarylog implementation binary logging as defined in +// https://github.com/grpc/proposal/blob/master/A16-binary-logging.md. +// +// Notice: All APIs in this package are experimental. +package binarylog + +import ( + "fmt" + "os" + + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + iblog "google.golang.org/grpc/internal/binarylog" +) + +// SetSink sets the destination for the binary log entries. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. +func SetSink(s Sink) { + if iblog.DefaultSink != nil { + iblog.DefaultSink.Close() + } + iblog.DefaultSink = s +} + +// Sink represents the destination for the binary log entries. +type Sink interface { + // Write marshals the log entry and writes it to the destination. The format + // is not specified, but should have sufficient information to rebuild the + // entry. Some options are: proto bytes, or proto json. + // + // Note this function needs to be thread-safe. + Write(*binlogpb.GrpcLogEntry) error + // Close closes this sink and cleans up resources (e.g. the flushing + // goroutine). + Close() error +} + +// NewTempFileSink creates a temp file and returns a Sink that writes to this +// file. +func NewTempFileSink() (Sink, error) { + // Two other options to replace this function: + // 1. take filename as input. + // 2. export NewBufferedSink(). + tempFile, err := os.CreateTemp("/tmp", "grpcgo_binarylog_*.txt") + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %v", err) + } + return iblog.NewBufferedSink(tempFile), nil +} diff --git a/call.go b/call.go new file mode 100644 index 0000000..2079bf4 --- /dev/null +++ b/call.go @@ -0,0 +1,97 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "io" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Invoke sends the RPC request on the wire and returns after response is +// received. This is typically called by generated code. +// +// All errors returned by Invoke are compatible with the status package. +func (cc *ClientConn) Invoke(ctx context.Context, method string, args, reply any, opts ...CallOption) error { + // allow interceptor to see all applicable call options, which means those + // configured as defaults from dial option as well as per-call options + opts = combine(cc.dopts.callOptions, opts) + + if cc.dopts.unaryInt != nil { + return cc.dopts.unaryInt(ctx, method, args, reply, cc, invoke, opts...) + } + return invoke(ctx, method, args, reply, cc, opts...) +} + +func combine(o1 []CallOption, o2 []CallOption) []CallOption { + // we don't use append because o1 could have extra capacity whose + // elements would be overwritten, which could cause inadvertent + // sharing (and race conditions) between concurrent calls + if len(o1) == 0 { + return o2 + } else if len(o2) == 0 { + return o1 + } + ret := make([]CallOption, len(o1)+len(o2)) + copy(ret, o1) + copy(ret[len(o1):], o2) + return ret +} + +// Invoke sends the RPC request on the wire and returns after response is +// received. This is typically called by generated code. +// +// DEPRECATED: Use ClientConn.Invoke instead. +func Invoke(ctx context.Context, method string, args, reply any, cc *ClientConn, opts ...CallOption) error { + return cc.Invoke(ctx, method, args, reply, opts...) +} + +var unaryStreamDesc = &StreamDesc{ServerStreams: false, ClientStreams: false} + +func invoke(ctx context.Context, method string, req, reply any, cc *ClientConn, opts ...CallOption) error { + cs, err := newClientStream(ctx, unaryStreamDesc, cc, method, opts...) + if err != nil { + return err + } + // In case of nil or io.EOF error, call RecvMsg. + if err := cs.SendMsg(req); err != nil && err != io.EOF { + return err + } + // CloseSend is needed because in some scenarios (e.g., xDS), the same + // interceptors are used to process both unary and streaming RPCs. Calling + // CloseSend signals to those interceptors that no more messages are on the + // way. + if err := cs.CloseSend(); err != nil && err != io.EOF { + return err + } + if err := cs.RecvMsg(reply); err != nil { + return err + } + // Call RecvMsg again to get the trailers. + err = cs.RecvMsg(reply) + if err == io.EOF { + return nil + } + if err == nil { + return status.Error(codes.Internal, "cardinality violation: expected for non server-streaming RPCs, but received another message") + } + return err +} diff --git a/channelz/channelz.go b/channelz/channelz.go new file mode 100644 index 0000000..32b7fa5 --- /dev/null +++ b/channelz/channelz.go @@ -0,0 +1,36 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package channelz exports internals of the channelz implementation as required +// by other gRPC packages. +// +// The implementation of the channelz spec as defined in +// https://github.com/grpc/proposal/blob/master/A14-channelz.md, is provided by +// the `internal/channelz` package. +// +// # Experimental +// +// Notice: All APIs in this package are experimental and may be removed in a +// later release. +package channelz + +import "google.golang.org/grpc/internal/channelz" + +// Identifier is an opaque identifier which uniquely identifies an entity in the +// channelz database. +type Identifier = channelz.Identifier diff --git a/channelz/grpc_channelz_v1/channelz.pb.go b/channelz/grpc_channelz_v1/channelz.pb.go new file mode 100644 index 0000000..d623144 --- /dev/null +++ b/channelz/grpc_channelz_v1/channelz.pb.go @@ -0,0 +1,3418 @@ +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines an interface for exporting monitoring information +// out of gRPC servers. See the full design at +// https://github.com/grpc/proposal/blob/master/A14-channelz.md +// +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/channelz/v1/channelz.proto + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/channelz/v1/channelz.proto + +package grpc_channelz_v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + anypb "google.golang.org/protobuf/types/known/anypb" + durationpb "google.golang.org/protobuf/types/known/durationpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + wrapperspb "google.golang.org/protobuf/types/known/wrapperspb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ChannelConnectivityState_State int32 + +const ( + ChannelConnectivityState_UNKNOWN ChannelConnectivityState_State = 0 + ChannelConnectivityState_IDLE ChannelConnectivityState_State = 1 + ChannelConnectivityState_CONNECTING ChannelConnectivityState_State = 2 + ChannelConnectivityState_READY ChannelConnectivityState_State = 3 + ChannelConnectivityState_TRANSIENT_FAILURE ChannelConnectivityState_State = 4 + ChannelConnectivityState_SHUTDOWN ChannelConnectivityState_State = 5 +) + +// Enum value maps for ChannelConnectivityState_State. +var ( + ChannelConnectivityState_State_name = map[int32]string{ + 0: "UNKNOWN", + 1: "IDLE", + 2: "CONNECTING", + 3: "READY", + 4: "TRANSIENT_FAILURE", + 5: "SHUTDOWN", + } + ChannelConnectivityState_State_value = map[string]int32{ + "UNKNOWN": 0, + "IDLE": 1, + "CONNECTING": 2, + "READY": 3, + "TRANSIENT_FAILURE": 4, + "SHUTDOWN": 5, + } +) + +func (x ChannelConnectivityState_State) Enum() *ChannelConnectivityState_State { + p := new(ChannelConnectivityState_State) + *p = x + return p +} + +func (x ChannelConnectivityState_State) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ChannelConnectivityState_State) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_channelz_v1_channelz_proto_enumTypes[0].Descriptor() +} + +func (ChannelConnectivityState_State) Type() protoreflect.EnumType { + return &file_grpc_channelz_v1_channelz_proto_enumTypes[0] +} + +func (x ChannelConnectivityState_State) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ChannelConnectivityState_State.Descriptor instead. +func (ChannelConnectivityState_State) EnumDescriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{2, 0} +} + +// The supported severity levels of trace events. +type ChannelTraceEvent_Severity int32 + +const ( + ChannelTraceEvent_CT_UNKNOWN ChannelTraceEvent_Severity = 0 + ChannelTraceEvent_CT_INFO ChannelTraceEvent_Severity = 1 + ChannelTraceEvent_CT_WARNING ChannelTraceEvent_Severity = 2 + ChannelTraceEvent_CT_ERROR ChannelTraceEvent_Severity = 3 +) + +// Enum value maps for ChannelTraceEvent_Severity. +var ( + ChannelTraceEvent_Severity_name = map[int32]string{ + 0: "CT_UNKNOWN", + 1: "CT_INFO", + 2: "CT_WARNING", + 3: "CT_ERROR", + } + ChannelTraceEvent_Severity_value = map[string]int32{ + "CT_UNKNOWN": 0, + "CT_INFO": 1, + "CT_WARNING": 2, + "CT_ERROR": 3, + } +) + +func (x ChannelTraceEvent_Severity) Enum() *ChannelTraceEvent_Severity { + p := new(ChannelTraceEvent_Severity) + *p = x + return p +} + +func (x ChannelTraceEvent_Severity) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ChannelTraceEvent_Severity) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_channelz_v1_channelz_proto_enumTypes[1].Descriptor() +} + +func (ChannelTraceEvent_Severity) Type() protoreflect.EnumType { + return &file_grpc_channelz_v1_channelz_proto_enumTypes[1] +} + +func (x ChannelTraceEvent_Severity) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ChannelTraceEvent_Severity.Descriptor instead. +func (ChannelTraceEvent_Severity) EnumDescriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{4, 0} +} + +// Channel is a logical grouping of channels, subchannels, and sockets. +type Channel struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The identifier for this channel. This should be set. + Ref *ChannelRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` + // Data specific to this channel. + Data *ChannelData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` // At most one of 'channel_ref+subchannel_ref' and 'socket' is set. + // There are no ordering guarantees on the order of channel refs. + // There may not be cycles in the ref graph. + // A channel ref may be present in more than one channel or subchannel. + ChannelRef []*ChannelRef `protobuf:"bytes,3,rep,name=channel_ref,json=channelRef,proto3" json:"channel_ref,omitempty"` + // At most one of 'channel_ref+subchannel_ref' and 'socket' is set. + // There are no ordering guarantees on the order of subchannel refs. + // There may not be cycles in the ref graph. + // A sub channel ref may be present in more than one channel or subchannel. + SubchannelRef []*SubchannelRef `protobuf:"bytes,4,rep,name=subchannel_ref,json=subchannelRef,proto3" json:"subchannel_ref,omitempty"` + // There are no ordering guarantees on the order of sockets. + SocketRef []*SocketRef `protobuf:"bytes,5,rep,name=socket_ref,json=socketRef,proto3" json:"socket_ref,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Channel) Reset() { + *x = Channel{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Channel) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Channel) ProtoMessage() {} + +func (x *Channel) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Channel.ProtoReflect.Descriptor instead. +func (*Channel) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{0} +} + +func (x *Channel) GetRef() *ChannelRef { + if x != nil { + return x.Ref + } + return nil +} + +func (x *Channel) GetData() *ChannelData { + if x != nil { + return x.Data + } + return nil +} + +func (x *Channel) GetChannelRef() []*ChannelRef { + if x != nil { + return x.ChannelRef + } + return nil +} + +func (x *Channel) GetSubchannelRef() []*SubchannelRef { + if x != nil { + return x.SubchannelRef + } + return nil +} + +func (x *Channel) GetSocketRef() []*SocketRef { + if x != nil { + return x.SocketRef + } + return nil +} + +// Subchannel is a logical grouping of channels, subchannels, and sockets. +// A subchannel is load balanced over by its ancestor +type Subchannel struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The identifier for this channel. + Ref *SubchannelRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` + // Data specific to this channel. + Data *ChannelData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` // At most one of 'channel_ref+subchannel_ref' and 'socket' is set. + // There are no ordering guarantees on the order of channel refs. + // There may not be cycles in the ref graph. + // A channel ref may be present in more than one channel or subchannel. + ChannelRef []*ChannelRef `protobuf:"bytes,3,rep,name=channel_ref,json=channelRef,proto3" json:"channel_ref,omitempty"` + // At most one of 'channel_ref+subchannel_ref' and 'socket' is set. + // There are no ordering guarantees on the order of subchannel refs. + // There may not be cycles in the ref graph. + // A sub channel ref may be present in more than one channel or subchannel. + SubchannelRef []*SubchannelRef `protobuf:"bytes,4,rep,name=subchannel_ref,json=subchannelRef,proto3" json:"subchannel_ref,omitempty"` + // There are no ordering guarantees on the order of sockets. + SocketRef []*SocketRef `protobuf:"bytes,5,rep,name=socket_ref,json=socketRef,proto3" json:"socket_ref,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Subchannel) Reset() { + *x = Subchannel{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Subchannel) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Subchannel) ProtoMessage() {} + +func (x *Subchannel) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Subchannel.ProtoReflect.Descriptor instead. +func (*Subchannel) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{1} +} + +func (x *Subchannel) GetRef() *SubchannelRef { + if x != nil { + return x.Ref + } + return nil +} + +func (x *Subchannel) GetData() *ChannelData { + if x != nil { + return x.Data + } + return nil +} + +func (x *Subchannel) GetChannelRef() []*ChannelRef { + if x != nil { + return x.ChannelRef + } + return nil +} + +func (x *Subchannel) GetSubchannelRef() []*SubchannelRef { + if x != nil { + return x.SubchannelRef + } + return nil +} + +func (x *Subchannel) GetSocketRef() []*SocketRef { + if x != nil { + return x.SocketRef + } + return nil +} + +// These come from the specified states in this document: +// https://github.com/grpc/grpc/blob/master/doc/connectivity-semantics-and-api.md +type ChannelConnectivityState struct { + state protoimpl.MessageState `protogen:"open.v1"` + State ChannelConnectivityState_State `protobuf:"varint,1,opt,name=state,proto3,enum=grpc.channelz.v1.ChannelConnectivityState_State" json:"state,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ChannelConnectivityState) Reset() { + *x = ChannelConnectivityState{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChannelConnectivityState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChannelConnectivityState) ProtoMessage() {} + +func (x *ChannelConnectivityState) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChannelConnectivityState.ProtoReflect.Descriptor instead. +func (*ChannelConnectivityState) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{2} +} + +func (x *ChannelConnectivityState) GetState() ChannelConnectivityState_State { + if x != nil { + return x.State + } + return ChannelConnectivityState_UNKNOWN +} + +// Channel data is data related to a specific Channel or Subchannel. +type ChannelData struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The connectivity state of the channel or subchannel. Implementations + // should always set this. + State *ChannelConnectivityState `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` + // The target this channel originally tried to connect to. May be absent + Target string `protobuf:"bytes,2,opt,name=target,proto3" json:"target,omitempty"` + // A trace of recent events on the channel. May be absent. + Trace *ChannelTrace `protobuf:"bytes,3,opt,name=trace,proto3" json:"trace,omitempty"` + // The number of calls started on the channel + CallsStarted int64 `protobuf:"varint,4,opt,name=calls_started,json=callsStarted,proto3" json:"calls_started,omitempty"` + // The number of calls that have completed with an OK status + CallsSucceeded int64 `protobuf:"varint,5,opt,name=calls_succeeded,json=callsSucceeded,proto3" json:"calls_succeeded,omitempty"` + // The number of calls that have completed with a non-OK status + CallsFailed int64 `protobuf:"varint,6,opt,name=calls_failed,json=callsFailed,proto3" json:"calls_failed,omitempty"` + // The last time a call was started on the channel. + LastCallStartedTimestamp *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=last_call_started_timestamp,json=lastCallStartedTimestamp,proto3" json:"last_call_started_timestamp,omitempty"` + // Populated for subchannels only. + MaxConnectionsPerSubchannel uint32 `protobuf:"varint,8,opt,name=max_connections_per_subchannel,json=maxConnectionsPerSubchannel,proto3" json:"max_connections_per_subchannel,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ChannelData) Reset() { + *x = ChannelData{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChannelData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChannelData) ProtoMessage() {} + +func (x *ChannelData) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChannelData.ProtoReflect.Descriptor instead. +func (*ChannelData) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{3} +} + +func (x *ChannelData) GetState() *ChannelConnectivityState { + if x != nil { + return x.State + } + return nil +} + +func (x *ChannelData) GetTarget() string { + if x != nil { + return x.Target + } + return "" +} + +func (x *ChannelData) GetTrace() *ChannelTrace { + if x != nil { + return x.Trace + } + return nil +} + +func (x *ChannelData) GetCallsStarted() int64 { + if x != nil { + return x.CallsStarted + } + return 0 +} + +func (x *ChannelData) GetCallsSucceeded() int64 { + if x != nil { + return x.CallsSucceeded + } + return 0 +} + +func (x *ChannelData) GetCallsFailed() int64 { + if x != nil { + return x.CallsFailed + } + return 0 +} + +func (x *ChannelData) GetLastCallStartedTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.LastCallStartedTimestamp + } + return nil +} + +func (x *ChannelData) GetMaxConnectionsPerSubchannel() uint32 { + if x != nil { + return x.MaxConnectionsPerSubchannel + } + return 0 +} + +// A trace event is an interesting thing that happened to a channel or +// subchannel, such as creation, address resolution, subchannel creation, etc. +type ChannelTraceEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // High level description of the event. + Description string `protobuf:"bytes,1,opt,name=description,proto3" json:"description,omitempty"` + // the severity of the trace event + Severity ChannelTraceEvent_Severity `protobuf:"varint,2,opt,name=severity,proto3,enum=grpc.channelz.v1.ChannelTraceEvent_Severity" json:"severity,omitempty"` + // When this event occurred. + Timestamp *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + // ref of referenced channel or subchannel. + // Optional, only present if this event refers to a child object. For example, + // this field would be filled if this trace event was for a subchannel being + // created. + // + // Types that are valid to be assigned to ChildRef: + // + // *ChannelTraceEvent_ChannelRef + // *ChannelTraceEvent_SubchannelRef + ChildRef isChannelTraceEvent_ChildRef `protobuf_oneof:"child_ref"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ChannelTraceEvent) Reset() { + *x = ChannelTraceEvent{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChannelTraceEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChannelTraceEvent) ProtoMessage() {} + +func (x *ChannelTraceEvent) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChannelTraceEvent.ProtoReflect.Descriptor instead. +func (*ChannelTraceEvent) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{4} +} + +func (x *ChannelTraceEvent) GetDescription() string { + if x != nil { + return x.Description + } + return "" +} + +func (x *ChannelTraceEvent) GetSeverity() ChannelTraceEvent_Severity { + if x != nil { + return x.Severity + } + return ChannelTraceEvent_CT_UNKNOWN +} + +func (x *ChannelTraceEvent) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp + } + return nil +} + +func (x *ChannelTraceEvent) GetChildRef() isChannelTraceEvent_ChildRef { + if x != nil { + return x.ChildRef + } + return nil +} + +func (x *ChannelTraceEvent) GetChannelRef() *ChannelRef { + if x != nil { + if x, ok := x.ChildRef.(*ChannelTraceEvent_ChannelRef); ok { + return x.ChannelRef + } + } + return nil +} + +func (x *ChannelTraceEvent) GetSubchannelRef() *SubchannelRef { + if x != nil { + if x, ok := x.ChildRef.(*ChannelTraceEvent_SubchannelRef); ok { + return x.SubchannelRef + } + } + return nil +} + +type isChannelTraceEvent_ChildRef interface { + isChannelTraceEvent_ChildRef() +} + +type ChannelTraceEvent_ChannelRef struct { + ChannelRef *ChannelRef `protobuf:"bytes,4,opt,name=channel_ref,json=channelRef,proto3,oneof"` +} + +type ChannelTraceEvent_SubchannelRef struct { + SubchannelRef *SubchannelRef `protobuf:"bytes,5,opt,name=subchannel_ref,json=subchannelRef,proto3,oneof"` +} + +func (*ChannelTraceEvent_ChannelRef) isChannelTraceEvent_ChildRef() {} + +func (*ChannelTraceEvent_SubchannelRef) isChannelTraceEvent_ChildRef() {} + +// ChannelTrace represents the recent events that have occurred on the channel. +type ChannelTrace struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Number of events ever logged in this tracing object. This can differ from + // events.size() because events can be overwritten or garbage collected by + // implementations. + NumEventsLogged int64 `protobuf:"varint,1,opt,name=num_events_logged,json=numEventsLogged,proto3" json:"num_events_logged,omitempty"` + // Time that this channel was created. + CreationTimestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=creation_timestamp,json=creationTimestamp,proto3" json:"creation_timestamp,omitempty"` + // List of events that have occurred on this channel. + Events []*ChannelTraceEvent `protobuf:"bytes,3,rep,name=events,proto3" json:"events,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ChannelTrace) Reset() { + *x = ChannelTrace{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChannelTrace) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChannelTrace) ProtoMessage() {} + +func (x *ChannelTrace) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChannelTrace.ProtoReflect.Descriptor instead. +func (*ChannelTrace) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{5} +} + +func (x *ChannelTrace) GetNumEventsLogged() int64 { + if x != nil { + return x.NumEventsLogged + } + return 0 +} + +func (x *ChannelTrace) GetCreationTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.CreationTimestamp + } + return nil +} + +func (x *ChannelTrace) GetEvents() []*ChannelTraceEvent { + if x != nil { + return x.Events + } + return nil +} + +// ChannelRef is a reference to a Channel. +type ChannelRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The globally unique id for this channel. Must be a positive number. + ChannelId int64 `protobuf:"varint,1,opt,name=channel_id,json=channelId,proto3" json:"channel_id,omitempty"` + // An optional name associated with the channel. + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ChannelRef) Reset() { + *x = ChannelRef{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChannelRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChannelRef) ProtoMessage() {} + +func (x *ChannelRef) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChannelRef.ProtoReflect.Descriptor instead. +func (*ChannelRef) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{6} +} + +func (x *ChannelRef) GetChannelId() int64 { + if x != nil { + return x.ChannelId + } + return 0 +} + +func (x *ChannelRef) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +// SubchannelRef is a reference to a Subchannel. +type SubchannelRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The globally unique id for this subchannel. Must be a positive number. + SubchannelId int64 `protobuf:"varint,7,opt,name=subchannel_id,json=subchannelId,proto3" json:"subchannel_id,omitempty"` + // An optional name associated with the subchannel. + Name string `protobuf:"bytes,8,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubchannelRef) Reset() { + *x = SubchannelRef{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubchannelRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubchannelRef) ProtoMessage() {} + +func (x *SubchannelRef) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubchannelRef.ProtoReflect.Descriptor instead. +func (*SubchannelRef) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{7} +} + +func (x *SubchannelRef) GetSubchannelId() int64 { + if x != nil { + return x.SubchannelId + } + return 0 +} + +func (x *SubchannelRef) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +// SocketRef is a reference to a Socket. +type SocketRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The globally unique id for this socket. Must be a positive number. + SocketId int64 `protobuf:"varint,3,opt,name=socket_id,json=socketId,proto3" json:"socket_id,omitempty"` + // An optional name associated with the socket. + Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SocketRef) Reset() { + *x = SocketRef{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SocketRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SocketRef) ProtoMessage() {} + +func (x *SocketRef) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SocketRef.ProtoReflect.Descriptor instead. +func (*SocketRef) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{8} +} + +func (x *SocketRef) GetSocketId() int64 { + if x != nil { + return x.SocketId + } + return 0 +} + +func (x *SocketRef) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +// ServerRef is a reference to a Server. +type ServerRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + // A globally unique identifier for this server. Must be a positive number. + ServerId int64 `protobuf:"varint,5,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"` + // An optional name associated with the server. + Name string `protobuf:"bytes,6,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerRef) Reset() { + *x = ServerRef{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerRef) ProtoMessage() {} + +func (x *ServerRef) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerRef.ProtoReflect.Descriptor instead. +func (*ServerRef) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{9} +} + +func (x *ServerRef) GetServerId() int64 { + if x != nil { + return x.ServerId + } + return 0 +} + +func (x *ServerRef) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +// Server represents a single server. There may be multiple servers in a single +// program. +type Server struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The identifier for a Server. This should be set. + Ref *ServerRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` + // The associated data of the Server. + Data *ServerData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + // The sockets that the server is listening on. There are no ordering + // guarantees. This may be absent. + ListenSocket []*SocketRef `protobuf:"bytes,3,rep,name=listen_socket,json=listenSocket,proto3" json:"listen_socket,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Server) Reset() { + *x = Server{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Server) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Server) ProtoMessage() {} + +func (x *Server) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Server.ProtoReflect.Descriptor instead. +func (*Server) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{10} +} + +func (x *Server) GetRef() *ServerRef { + if x != nil { + return x.Ref + } + return nil +} + +func (x *Server) GetData() *ServerData { + if x != nil { + return x.Data + } + return nil +} + +func (x *Server) GetListenSocket() []*SocketRef { + if x != nil { + return x.ListenSocket + } + return nil +} + +// ServerData is data for a specific Server. +type ServerData struct { + state protoimpl.MessageState `protogen:"open.v1"` + // A trace of recent events on the server. May be absent. + Trace *ChannelTrace `protobuf:"bytes,1,opt,name=trace,proto3" json:"trace,omitempty"` + // The number of incoming calls started on the server + CallsStarted int64 `protobuf:"varint,2,opt,name=calls_started,json=callsStarted,proto3" json:"calls_started,omitempty"` + // The number of incoming calls that have completed with an OK status + CallsSucceeded int64 `protobuf:"varint,3,opt,name=calls_succeeded,json=callsSucceeded,proto3" json:"calls_succeeded,omitempty"` + // The number of incoming calls that have a completed with a non-OK status + CallsFailed int64 `protobuf:"varint,4,opt,name=calls_failed,json=callsFailed,proto3" json:"calls_failed,omitempty"` + // The last time a call was started on the server. + LastCallStartedTimestamp *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=last_call_started_timestamp,json=lastCallStartedTimestamp,proto3" json:"last_call_started_timestamp,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerData) Reset() { + *x = ServerData{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerData) ProtoMessage() {} + +func (x *ServerData) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerData.ProtoReflect.Descriptor instead. +func (*ServerData) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{11} +} + +func (x *ServerData) GetTrace() *ChannelTrace { + if x != nil { + return x.Trace + } + return nil +} + +func (x *ServerData) GetCallsStarted() int64 { + if x != nil { + return x.CallsStarted + } + return 0 +} + +func (x *ServerData) GetCallsSucceeded() int64 { + if x != nil { + return x.CallsSucceeded + } + return 0 +} + +func (x *ServerData) GetCallsFailed() int64 { + if x != nil { + return x.CallsFailed + } + return 0 +} + +func (x *ServerData) GetLastCallStartedTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.LastCallStartedTimestamp + } + return nil +} + +// Information about an actual connection. Pronounced "sock-ay". +type Socket struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The identifier for the Socket. + Ref *SocketRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` + // Data specific to this Socket. + Data *SocketData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + // The locally bound address. + Local *Address `protobuf:"bytes,3,opt,name=local,proto3" json:"local,omitempty"` + // The remote bound address. May be absent. + Remote *Address `protobuf:"bytes,4,opt,name=remote,proto3" json:"remote,omitempty"` + // Security details for this socket. May be absent if not available, or + // there is no security on the socket. + Security *Security `protobuf:"bytes,5,opt,name=security,proto3" json:"security,omitempty"` + // Optional, represents the name of the remote endpoint, if different than + // the original target name. + RemoteName string `protobuf:"bytes,6,opt,name=remote_name,json=remoteName,proto3" json:"remote_name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Socket) Reset() { + *x = Socket{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Socket) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Socket) ProtoMessage() {} + +func (x *Socket) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Socket.ProtoReflect.Descriptor instead. +func (*Socket) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{12} +} + +func (x *Socket) GetRef() *SocketRef { + if x != nil { + return x.Ref + } + return nil +} + +func (x *Socket) GetData() *SocketData { + if x != nil { + return x.Data + } + return nil +} + +func (x *Socket) GetLocal() *Address { + if x != nil { + return x.Local + } + return nil +} + +func (x *Socket) GetRemote() *Address { + if x != nil { + return x.Remote + } + return nil +} + +func (x *Socket) GetSecurity() *Security { + if x != nil { + return x.Security + } + return nil +} + +func (x *Socket) GetRemoteName() string { + if x != nil { + return x.RemoteName + } + return "" +} + +// SocketData is data associated for a specific Socket. The fields present +// are specific to the implementation, so there may be minor differences in +// the semantics. (e.g. flow control windows) +type SocketData struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The number of streams that have been started. + StreamsStarted int64 `protobuf:"varint,1,opt,name=streams_started,json=streamsStarted,proto3" json:"streams_started,omitempty"` + // The number of streams that have ended successfully: + // On client side, received frame with eos bit set; + // On server side, sent frame with eos bit set. + StreamsSucceeded int64 `protobuf:"varint,2,opt,name=streams_succeeded,json=streamsSucceeded,proto3" json:"streams_succeeded,omitempty"` + // The number of streams that have ended unsuccessfully: + // On client side, ended without receiving frame with eos bit set; + // On server side, ended without sending frame with eos bit set. + StreamsFailed int64 `protobuf:"varint,3,opt,name=streams_failed,json=streamsFailed,proto3" json:"streams_failed,omitempty"` + // The number of grpc messages successfully sent on this socket. + MessagesSent int64 `protobuf:"varint,4,opt,name=messages_sent,json=messagesSent,proto3" json:"messages_sent,omitempty"` + // The number of grpc messages received on this socket. + MessagesReceived int64 `protobuf:"varint,5,opt,name=messages_received,json=messagesReceived,proto3" json:"messages_received,omitempty"` + // The number of keep alives sent. This is typically implemented with HTTP/2 + // ping messages. + KeepAlivesSent int64 `protobuf:"varint,6,opt,name=keep_alives_sent,json=keepAlivesSent,proto3" json:"keep_alives_sent,omitempty"` + // The last time a stream was created by this endpoint. Usually unset for + // servers. + LastLocalStreamCreatedTimestamp *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=last_local_stream_created_timestamp,json=lastLocalStreamCreatedTimestamp,proto3" json:"last_local_stream_created_timestamp,omitempty"` + // The last time a stream was created by the remote endpoint. Usually unset + // for clients. + LastRemoteStreamCreatedTimestamp *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=last_remote_stream_created_timestamp,json=lastRemoteStreamCreatedTimestamp,proto3" json:"last_remote_stream_created_timestamp,omitempty"` + // The last time a message was sent by this endpoint. + LastMessageSentTimestamp *timestamppb.Timestamp `protobuf:"bytes,9,opt,name=last_message_sent_timestamp,json=lastMessageSentTimestamp,proto3" json:"last_message_sent_timestamp,omitempty"` + // The last time a message was received by this endpoint. + LastMessageReceivedTimestamp *timestamppb.Timestamp `protobuf:"bytes,10,opt,name=last_message_received_timestamp,json=lastMessageReceivedTimestamp,proto3" json:"last_message_received_timestamp,omitempty"` + // The amount of window, granted to the local endpoint by the remote endpoint. + // This may be slightly out of date due to network latency. This does NOT + // include stream level or TCP level flow control info. + LocalFlowControlWindow *wrapperspb.Int64Value `protobuf:"bytes,11,opt,name=local_flow_control_window,json=localFlowControlWindow,proto3" json:"local_flow_control_window,omitempty"` + // The amount of window, granted to the remote endpoint by the local endpoint. + // This may be slightly out of date due to network latency. This does NOT + // include stream level or TCP level flow control info. + RemoteFlowControlWindow *wrapperspb.Int64Value `protobuf:"bytes,12,opt,name=remote_flow_control_window,json=remoteFlowControlWindow,proto3" json:"remote_flow_control_window,omitempty"` + // Socket options set on this socket. May be absent if 'summary' is set + // on GetSocketRequest. + Option []*SocketOption `protobuf:"bytes,13,rep,name=option,proto3" json:"option,omitempty"` + // Populated if a GOAWAY has been received. The value will be the + // HTTP/2 error code from the GOAWAY. + ReceivedGoawayError *wrapperspb.UInt32Value `protobuf:"bytes,14,opt,name=received_goaway_error,json=receivedGoawayError,proto3" json:"received_goaway_error,omitempty"` + // The value of MAX_CONCURRENT_STREAMS set by the peer. Populated on + // client side only. + PeerMaxConcurrentStreams uint32 `protobuf:"varint,15,opt,name=peer_max_concurrent_streams,json=peerMaxConcurrentStreams,proto3" json:"peer_max_concurrent_streams,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SocketData) Reset() { + *x = SocketData{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SocketData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SocketData) ProtoMessage() {} + +func (x *SocketData) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SocketData.ProtoReflect.Descriptor instead. +func (*SocketData) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{13} +} + +func (x *SocketData) GetStreamsStarted() int64 { + if x != nil { + return x.StreamsStarted + } + return 0 +} + +func (x *SocketData) GetStreamsSucceeded() int64 { + if x != nil { + return x.StreamsSucceeded + } + return 0 +} + +func (x *SocketData) GetStreamsFailed() int64 { + if x != nil { + return x.StreamsFailed + } + return 0 +} + +func (x *SocketData) GetMessagesSent() int64 { + if x != nil { + return x.MessagesSent + } + return 0 +} + +func (x *SocketData) GetMessagesReceived() int64 { + if x != nil { + return x.MessagesReceived + } + return 0 +} + +func (x *SocketData) GetKeepAlivesSent() int64 { + if x != nil { + return x.KeepAlivesSent + } + return 0 +} + +func (x *SocketData) GetLastLocalStreamCreatedTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.LastLocalStreamCreatedTimestamp + } + return nil +} + +func (x *SocketData) GetLastRemoteStreamCreatedTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.LastRemoteStreamCreatedTimestamp + } + return nil +} + +func (x *SocketData) GetLastMessageSentTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.LastMessageSentTimestamp + } + return nil +} + +func (x *SocketData) GetLastMessageReceivedTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.LastMessageReceivedTimestamp + } + return nil +} + +func (x *SocketData) GetLocalFlowControlWindow() *wrapperspb.Int64Value { + if x != nil { + return x.LocalFlowControlWindow + } + return nil +} + +func (x *SocketData) GetRemoteFlowControlWindow() *wrapperspb.Int64Value { + if x != nil { + return x.RemoteFlowControlWindow + } + return nil +} + +func (x *SocketData) GetOption() []*SocketOption { + if x != nil { + return x.Option + } + return nil +} + +func (x *SocketData) GetReceivedGoawayError() *wrapperspb.UInt32Value { + if x != nil { + return x.ReceivedGoawayError + } + return nil +} + +func (x *SocketData) GetPeerMaxConcurrentStreams() uint32 { + if x != nil { + return x.PeerMaxConcurrentStreams + } + return 0 +} + +// Address represents the address used to create the socket. +type Address struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Address: + // + // *Address_TcpipAddress + // *Address_UdsAddress_ + // *Address_OtherAddress_ + Address isAddress_Address `protobuf_oneof:"address"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Address) Reset() { + *x = Address{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Address) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Address) ProtoMessage() {} + +func (x *Address) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Address.ProtoReflect.Descriptor instead. +func (*Address) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{14} +} + +func (x *Address) GetAddress() isAddress_Address { + if x != nil { + return x.Address + } + return nil +} + +func (x *Address) GetTcpipAddress() *Address_TcpIpAddress { + if x != nil { + if x, ok := x.Address.(*Address_TcpipAddress); ok { + return x.TcpipAddress + } + } + return nil +} + +func (x *Address) GetUdsAddress() *Address_UdsAddress { + if x != nil { + if x, ok := x.Address.(*Address_UdsAddress_); ok { + return x.UdsAddress + } + } + return nil +} + +func (x *Address) GetOtherAddress() *Address_OtherAddress { + if x != nil { + if x, ok := x.Address.(*Address_OtherAddress_); ok { + return x.OtherAddress + } + } + return nil +} + +type isAddress_Address interface { + isAddress_Address() +} + +type Address_TcpipAddress struct { + TcpipAddress *Address_TcpIpAddress `protobuf:"bytes,1,opt,name=tcpip_address,json=tcpipAddress,proto3,oneof"` +} + +type Address_UdsAddress_ struct { + UdsAddress *Address_UdsAddress `protobuf:"bytes,2,opt,name=uds_address,json=udsAddress,proto3,oneof"` +} + +type Address_OtherAddress_ struct { + OtherAddress *Address_OtherAddress `protobuf:"bytes,3,opt,name=other_address,json=otherAddress,proto3,oneof"` +} + +func (*Address_TcpipAddress) isAddress_Address() {} + +func (*Address_UdsAddress_) isAddress_Address() {} + +func (*Address_OtherAddress_) isAddress_Address() {} + +// Security represents details about how secure the socket is. +type Security struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Model: + // + // *Security_Tls_ + // *Security_Other + Model isSecurity_Model `protobuf_oneof:"model"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Security) Reset() { + *x = Security{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Security) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Security) ProtoMessage() {} + +func (x *Security) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Security.ProtoReflect.Descriptor instead. +func (*Security) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{15} +} + +func (x *Security) GetModel() isSecurity_Model { + if x != nil { + return x.Model + } + return nil +} + +func (x *Security) GetTls() *Security_Tls { + if x != nil { + if x, ok := x.Model.(*Security_Tls_); ok { + return x.Tls + } + } + return nil +} + +func (x *Security) GetOther() *Security_OtherSecurity { + if x != nil { + if x, ok := x.Model.(*Security_Other); ok { + return x.Other + } + } + return nil +} + +type isSecurity_Model interface { + isSecurity_Model() +} + +type Security_Tls_ struct { + Tls *Security_Tls `protobuf:"bytes,1,opt,name=tls,proto3,oneof"` +} + +type Security_Other struct { + Other *Security_OtherSecurity `protobuf:"bytes,2,opt,name=other,proto3,oneof"` +} + +func (*Security_Tls_) isSecurity_Model() {} + +func (*Security_Other) isSecurity_Model() {} + +// SocketOption represents socket options for a socket. Specifically, these +// are the options returned by getsockopt(). +type SocketOption struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The full name of the socket option. Typically this will be the upper case + // name, such as "SO_REUSEPORT". + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // The human readable value of this socket option. At least one of value or + // additional will be set. + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + // Additional data associated with the socket option. At least one of value + // or additional will be set. + Additional *anypb.Any `protobuf:"bytes,3,opt,name=additional,proto3" json:"additional,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SocketOption) Reset() { + *x = SocketOption{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SocketOption) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SocketOption) ProtoMessage() {} + +func (x *SocketOption) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SocketOption.ProtoReflect.Descriptor instead. +func (*SocketOption) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{16} +} + +func (x *SocketOption) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *SocketOption) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +func (x *SocketOption) GetAdditional() *anypb.Any { + if x != nil { + return x.Additional + } + return nil +} + +// For use with SocketOption's additional field. This is primarily used for +// SO_RCVTIMEO and SO_SNDTIMEO +type SocketOptionTimeout struct { + state protoimpl.MessageState `protogen:"open.v1"` + Duration *durationpb.Duration `protobuf:"bytes,1,opt,name=duration,proto3" json:"duration,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SocketOptionTimeout) Reset() { + *x = SocketOptionTimeout{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SocketOptionTimeout) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SocketOptionTimeout) ProtoMessage() {} + +func (x *SocketOptionTimeout) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SocketOptionTimeout.ProtoReflect.Descriptor instead. +func (*SocketOptionTimeout) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{17} +} + +func (x *SocketOptionTimeout) GetDuration() *durationpb.Duration { + if x != nil { + return x.Duration + } + return nil +} + +// For use with SocketOption's additional field. This is primarily used for +// SO_LINGER. +type SocketOptionLinger struct { + state protoimpl.MessageState `protogen:"open.v1"` + // active maps to `struct linger.l_onoff` + Active bool `protobuf:"varint,1,opt,name=active,proto3" json:"active,omitempty"` + // duration maps to `struct linger.l_linger` + Duration *durationpb.Duration `protobuf:"bytes,2,opt,name=duration,proto3" json:"duration,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SocketOptionLinger) Reset() { + *x = SocketOptionLinger{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SocketOptionLinger) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SocketOptionLinger) ProtoMessage() {} + +func (x *SocketOptionLinger) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SocketOptionLinger.ProtoReflect.Descriptor instead. +func (*SocketOptionLinger) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{18} +} + +func (x *SocketOptionLinger) GetActive() bool { + if x != nil { + return x.Active + } + return false +} + +func (x *SocketOptionLinger) GetDuration() *durationpb.Duration { + if x != nil { + return x.Duration + } + return nil +} + +// For use with SocketOption's additional field. Tcp info for +// SOL_TCP and TCP_INFO. +type SocketOptionTcpInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + TcpiState uint32 `protobuf:"varint,1,opt,name=tcpi_state,json=tcpiState,proto3" json:"tcpi_state,omitempty"` + TcpiCaState uint32 `protobuf:"varint,2,opt,name=tcpi_ca_state,json=tcpiCaState,proto3" json:"tcpi_ca_state,omitempty"` + TcpiRetransmits uint32 `protobuf:"varint,3,opt,name=tcpi_retransmits,json=tcpiRetransmits,proto3" json:"tcpi_retransmits,omitempty"` + TcpiProbes uint32 `protobuf:"varint,4,opt,name=tcpi_probes,json=tcpiProbes,proto3" json:"tcpi_probes,omitempty"` + TcpiBackoff uint32 `protobuf:"varint,5,opt,name=tcpi_backoff,json=tcpiBackoff,proto3" json:"tcpi_backoff,omitempty"` + TcpiOptions uint32 `protobuf:"varint,6,opt,name=tcpi_options,json=tcpiOptions,proto3" json:"tcpi_options,omitempty"` + TcpiSndWscale uint32 `protobuf:"varint,7,opt,name=tcpi_snd_wscale,json=tcpiSndWscale,proto3" json:"tcpi_snd_wscale,omitempty"` + TcpiRcvWscale uint32 `protobuf:"varint,8,opt,name=tcpi_rcv_wscale,json=tcpiRcvWscale,proto3" json:"tcpi_rcv_wscale,omitempty"` + TcpiRto uint32 `protobuf:"varint,9,opt,name=tcpi_rto,json=tcpiRto,proto3" json:"tcpi_rto,omitempty"` + TcpiAto uint32 `protobuf:"varint,10,opt,name=tcpi_ato,json=tcpiAto,proto3" json:"tcpi_ato,omitempty"` + TcpiSndMss uint32 `protobuf:"varint,11,opt,name=tcpi_snd_mss,json=tcpiSndMss,proto3" json:"tcpi_snd_mss,omitempty"` + TcpiRcvMss uint32 `protobuf:"varint,12,opt,name=tcpi_rcv_mss,json=tcpiRcvMss,proto3" json:"tcpi_rcv_mss,omitempty"` + TcpiUnacked uint32 `protobuf:"varint,13,opt,name=tcpi_unacked,json=tcpiUnacked,proto3" json:"tcpi_unacked,omitempty"` + TcpiSacked uint32 `protobuf:"varint,14,opt,name=tcpi_sacked,json=tcpiSacked,proto3" json:"tcpi_sacked,omitempty"` + TcpiLost uint32 `protobuf:"varint,15,opt,name=tcpi_lost,json=tcpiLost,proto3" json:"tcpi_lost,omitempty"` + TcpiRetrans uint32 `protobuf:"varint,16,opt,name=tcpi_retrans,json=tcpiRetrans,proto3" json:"tcpi_retrans,omitempty"` + TcpiFackets uint32 `protobuf:"varint,17,opt,name=tcpi_fackets,json=tcpiFackets,proto3" json:"tcpi_fackets,omitempty"` + TcpiLastDataSent uint32 `protobuf:"varint,18,opt,name=tcpi_last_data_sent,json=tcpiLastDataSent,proto3" json:"tcpi_last_data_sent,omitempty"` + TcpiLastAckSent uint32 `protobuf:"varint,19,opt,name=tcpi_last_ack_sent,json=tcpiLastAckSent,proto3" json:"tcpi_last_ack_sent,omitempty"` + TcpiLastDataRecv uint32 `protobuf:"varint,20,opt,name=tcpi_last_data_recv,json=tcpiLastDataRecv,proto3" json:"tcpi_last_data_recv,omitempty"` + TcpiLastAckRecv uint32 `protobuf:"varint,21,opt,name=tcpi_last_ack_recv,json=tcpiLastAckRecv,proto3" json:"tcpi_last_ack_recv,omitempty"` + TcpiPmtu uint32 `protobuf:"varint,22,opt,name=tcpi_pmtu,json=tcpiPmtu,proto3" json:"tcpi_pmtu,omitempty"` + TcpiRcvSsthresh uint32 `protobuf:"varint,23,opt,name=tcpi_rcv_ssthresh,json=tcpiRcvSsthresh,proto3" json:"tcpi_rcv_ssthresh,omitempty"` + TcpiRtt uint32 `protobuf:"varint,24,opt,name=tcpi_rtt,json=tcpiRtt,proto3" json:"tcpi_rtt,omitempty"` + TcpiRttvar uint32 `protobuf:"varint,25,opt,name=tcpi_rttvar,json=tcpiRttvar,proto3" json:"tcpi_rttvar,omitempty"` + TcpiSndSsthresh uint32 `protobuf:"varint,26,opt,name=tcpi_snd_ssthresh,json=tcpiSndSsthresh,proto3" json:"tcpi_snd_ssthresh,omitempty"` + TcpiSndCwnd uint32 `protobuf:"varint,27,opt,name=tcpi_snd_cwnd,json=tcpiSndCwnd,proto3" json:"tcpi_snd_cwnd,omitempty"` + TcpiAdvmss uint32 `protobuf:"varint,28,opt,name=tcpi_advmss,json=tcpiAdvmss,proto3" json:"tcpi_advmss,omitempty"` + TcpiReordering uint32 `protobuf:"varint,29,opt,name=tcpi_reordering,json=tcpiReordering,proto3" json:"tcpi_reordering,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SocketOptionTcpInfo) Reset() { + *x = SocketOptionTcpInfo{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SocketOptionTcpInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SocketOptionTcpInfo) ProtoMessage() {} + +func (x *SocketOptionTcpInfo) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SocketOptionTcpInfo.ProtoReflect.Descriptor instead. +func (*SocketOptionTcpInfo) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{19} +} + +func (x *SocketOptionTcpInfo) GetTcpiState() uint32 { + if x != nil { + return x.TcpiState + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiCaState() uint32 { + if x != nil { + return x.TcpiCaState + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiRetransmits() uint32 { + if x != nil { + return x.TcpiRetransmits + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiProbes() uint32 { + if x != nil { + return x.TcpiProbes + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiBackoff() uint32 { + if x != nil { + return x.TcpiBackoff + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiOptions() uint32 { + if x != nil { + return x.TcpiOptions + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiSndWscale() uint32 { + if x != nil { + return x.TcpiSndWscale + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiRcvWscale() uint32 { + if x != nil { + return x.TcpiRcvWscale + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiRto() uint32 { + if x != nil { + return x.TcpiRto + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiAto() uint32 { + if x != nil { + return x.TcpiAto + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiSndMss() uint32 { + if x != nil { + return x.TcpiSndMss + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiRcvMss() uint32 { + if x != nil { + return x.TcpiRcvMss + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiUnacked() uint32 { + if x != nil { + return x.TcpiUnacked + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiSacked() uint32 { + if x != nil { + return x.TcpiSacked + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiLost() uint32 { + if x != nil { + return x.TcpiLost + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiRetrans() uint32 { + if x != nil { + return x.TcpiRetrans + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiFackets() uint32 { + if x != nil { + return x.TcpiFackets + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiLastDataSent() uint32 { + if x != nil { + return x.TcpiLastDataSent + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiLastAckSent() uint32 { + if x != nil { + return x.TcpiLastAckSent + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiLastDataRecv() uint32 { + if x != nil { + return x.TcpiLastDataRecv + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiLastAckRecv() uint32 { + if x != nil { + return x.TcpiLastAckRecv + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiPmtu() uint32 { + if x != nil { + return x.TcpiPmtu + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiRcvSsthresh() uint32 { + if x != nil { + return x.TcpiRcvSsthresh + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiRtt() uint32 { + if x != nil { + return x.TcpiRtt + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiRttvar() uint32 { + if x != nil { + return x.TcpiRttvar + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiSndSsthresh() uint32 { + if x != nil { + return x.TcpiSndSsthresh + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiSndCwnd() uint32 { + if x != nil { + return x.TcpiSndCwnd + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiAdvmss() uint32 { + if x != nil { + return x.TcpiAdvmss + } + return 0 +} + +func (x *SocketOptionTcpInfo) GetTcpiReordering() uint32 { + if x != nil { + return x.TcpiReordering + } + return 0 +} + +type GetTopChannelsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // start_channel_id indicates that only channels at or above this id should be + // included in the results. + // To request the first page, this should be set to 0. To request + // subsequent pages, the client generates this value by adding 1 to + // the highest seen result ID. + StartChannelId int64 `protobuf:"varint,1,opt,name=start_channel_id,json=startChannelId,proto3" json:"start_channel_id,omitempty"` + // If non-zero, the server will return a page of results containing + // at most this many items. If zero, the server will choose a + // reasonable page size. Must never be negative. + MaxResults int64 `protobuf:"varint,2,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetTopChannelsRequest) Reset() { + *x = GetTopChannelsRequest{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetTopChannelsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetTopChannelsRequest) ProtoMessage() {} + +func (x *GetTopChannelsRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetTopChannelsRequest.ProtoReflect.Descriptor instead. +func (*GetTopChannelsRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{20} +} + +func (x *GetTopChannelsRequest) GetStartChannelId() int64 { + if x != nil { + return x.StartChannelId + } + return 0 +} + +func (x *GetTopChannelsRequest) GetMaxResults() int64 { + if x != nil { + return x.MaxResults + } + return 0 +} + +type GetTopChannelsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // list of channels that the connection detail service knows about. Sorted in + // ascending channel_id order. + // Must contain at least 1 result, otherwise 'end' must be true. + Channel []*Channel `protobuf:"bytes,1,rep,name=channel,proto3" json:"channel,omitempty"` + // If set, indicates that the list of channels is the final list. Requesting + // more channels can only return more if they are created after this RPC + // completes. + End bool `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetTopChannelsResponse) Reset() { + *x = GetTopChannelsResponse{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetTopChannelsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetTopChannelsResponse) ProtoMessage() {} + +func (x *GetTopChannelsResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetTopChannelsResponse.ProtoReflect.Descriptor instead. +func (*GetTopChannelsResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{21} +} + +func (x *GetTopChannelsResponse) GetChannel() []*Channel { + if x != nil { + return x.Channel + } + return nil +} + +func (x *GetTopChannelsResponse) GetEnd() bool { + if x != nil { + return x.End + } + return false +} + +type GetServersRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // start_server_id indicates that only servers at or above this id should be + // included in the results. + // To request the first page, this must be set to 0. To request + // subsequent pages, the client generates this value by adding 1 to + // the highest seen result ID. + StartServerId int64 `protobuf:"varint,1,opt,name=start_server_id,json=startServerId,proto3" json:"start_server_id,omitempty"` + // If non-zero, the server will return a page of results containing + // at most this many items. If zero, the server will choose a + // reasonable page size. Must never be negative. + MaxResults int64 `protobuf:"varint,2,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetServersRequest) Reset() { + *x = GetServersRequest{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetServersRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetServersRequest) ProtoMessage() {} + +func (x *GetServersRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetServersRequest.ProtoReflect.Descriptor instead. +func (*GetServersRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{22} +} + +func (x *GetServersRequest) GetStartServerId() int64 { + if x != nil { + return x.StartServerId + } + return 0 +} + +func (x *GetServersRequest) GetMaxResults() int64 { + if x != nil { + return x.MaxResults + } + return 0 +} + +type GetServersResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // list of servers that the connection detail service knows about. Sorted in + // ascending server_id order. + // Must contain at least 1 result, otherwise 'end' must be true. + Server []*Server `protobuf:"bytes,1,rep,name=server,proto3" json:"server,omitempty"` + // If set, indicates that the list of servers is the final list. Requesting + // more servers will only return more if they are created after this RPC + // completes. + End bool `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetServersResponse) Reset() { + *x = GetServersResponse{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetServersResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetServersResponse) ProtoMessage() {} + +func (x *GetServersResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetServersResponse.ProtoReflect.Descriptor instead. +func (*GetServersResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{23} +} + +func (x *GetServersResponse) GetServer() []*Server { + if x != nil { + return x.Server + } + return nil +} + +func (x *GetServersResponse) GetEnd() bool { + if x != nil { + return x.End + } + return false +} + +type GetServerRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // server_id is the identifier of the specific server to get. + ServerId int64 `protobuf:"varint,1,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetServerRequest) Reset() { + *x = GetServerRequest{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetServerRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetServerRequest) ProtoMessage() {} + +func (x *GetServerRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[24] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetServerRequest.ProtoReflect.Descriptor instead. +func (*GetServerRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{24} +} + +func (x *GetServerRequest) GetServerId() int64 { + if x != nil { + return x.ServerId + } + return 0 +} + +type GetServerResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The Server that corresponds to the requested server_id. This field + // should be set. + Server *Server `protobuf:"bytes,1,opt,name=server,proto3" json:"server,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetServerResponse) Reset() { + *x = GetServerResponse{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetServerResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetServerResponse) ProtoMessage() {} + +func (x *GetServerResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[25] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetServerResponse.ProtoReflect.Descriptor instead. +func (*GetServerResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{25} +} + +func (x *GetServerResponse) GetServer() *Server { + if x != nil { + return x.Server + } + return nil +} + +type GetServerSocketsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ServerId int64 `protobuf:"varint,1,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"` + // start_socket_id indicates that only sockets at or above this id should be + // included in the results. + // To request the first page, this must be set to 0. To request + // subsequent pages, the client generates this value by adding 1 to + // the highest seen result ID. + StartSocketId int64 `protobuf:"varint,2,opt,name=start_socket_id,json=startSocketId,proto3" json:"start_socket_id,omitempty"` + // If non-zero, the server will return a page of results containing + // at most this many items. If zero, the server will choose a + // reasonable page size. Must never be negative. + MaxResults int64 `protobuf:"varint,3,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetServerSocketsRequest) Reset() { + *x = GetServerSocketsRequest{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetServerSocketsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetServerSocketsRequest) ProtoMessage() {} + +func (x *GetServerSocketsRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[26] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetServerSocketsRequest.ProtoReflect.Descriptor instead. +func (*GetServerSocketsRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{26} +} + +func (x *GetServerSocketsRequest) GetServerId() int64 { + if x != nil { + return x.ServerId + } + return 0 +} + +func (x *GetServerSocketsRequest) GetStartSocketId() int64 { + if x != nil { + return x.StartSocketId + } + return 0 +} + +func (x *GetServerSocketsRequest) GetMaxResults() int64 { + if x != nil { + return x.MaxResults + } + return 0 +} + +type GetServerSocketsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // list of socket refs that the connection detail service knows about. Sorted in + // ascending socket_id order. + // Must contain at least 1 result, otherwise 'end' must be true. + SocketRef []*SocketRef `protobuf:"bytes,1,rep,name=socket_ref,json=socketRef,proto3" json:"socket_ref,omitempty"` + // If set, indicates that the list of sockets is the final list. Requesting + // more sockets will only return more if they are created after this RPC + // completes. + End bool `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetServerSocketsResponse) Reset() { + *x = GetServerSocketsResponse{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetServerSocketsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetServerSocketsResponse) ProtoMessage() {} + +func (x *GetServerSocketsResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetServerSocketsResponse.ProtoReflect.Descriptor instead. +func (*GetServerSocketsResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{27} +} + +func (x *GetServerSocketsResponse) GetSocketRef() []*SocketRef { + if x != nil { + return x.SocketRef + } + return nil +} + +func (x *GetServerSocketsResponse) GetEnd() bool { + if x != nil { + return x.End + } + return false +} + +type GetChannelRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // channel_id is the identifier of the specific channel to get. + ChannelId int64 `protobuf:"varint,1,opt,name=channel_id,json=channelId,proto3" json:"channel_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetChannelRequest) Reset() { + *x = GetChannelRequest{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetChannelRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetChannelRequest) ProtoMessage() {} + +func (x *GetChannelRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[28] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetChannelRequest.ProtoReflect.Descriptor instead. +func (*GetChannelRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{28} +} + +func (x *GetChannelRequest) GetChannelId() int64 { + if x != nil { + return x.ChannelId + } + return 0 +} + +type GetChannelResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The Channel that corresponds to the requested channel_id. This field + // should be set. + Channel *Channel `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetChannelResponse) Reset() { + *x = GetChannelResponse{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetChannelResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetChannelResponse) ProtoMessage() {} + +func (x *GetChannelResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[29] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetChannelResponse.ProtoReflect.Descriptor instead. +func (*GetChannelResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{29} +} + +func (x *GetChannelResponse) GetChannel() *Channel { + if x != nil { + return x.Channel + } + return nil +} + +type GetSubchannelRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // subchannel_id is the identifier of the specific subchannel to get. + SubchannelId int64 `protobuf:"varint,1,opt,name=subchannel_id,json=subchannelId,proto3" json:"subchannel_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetSubchannelRequest) Reset() { + *x = GetSubchannelRequest{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetSubchannelRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetSubchannelRequest) ProtoMessage() {} + +func (x *GetSubchannelRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[30] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetSubchannelRequest.ProtoReflect.Descriptor instead. +func (*GetSubchannelRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{30} +} + +func (x *GetSubchannelRequest) GetSubchannelId() int64 { + if x != nil { + return x.SubchannelId + } + return 0 +} + +type GetSubchannelResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The Subchannel that corresponds to the requested subchannel_id. This + // field should be set. + Subchannel *Subchannel `protobuf:"bytes,1,opt,name=subchannel,proto3" json:"subchannel,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetSubchannelResponse) Reset() { + *x = GetSubchannelResponse{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetSubchannelResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetSubchannelResponse) ProtoMessage() {} + +func (x *GetSubchannelResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[31] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetSubchannelResponse.ProtoReflect.Descriptor instead. +func (*GetSubchannelResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{31} +} + +func (x *GetSubchannelResponse) GetSubchannel() *Subchannel { + if x != nil { + return x.Subchannel + } + return nil +} + +type GetSocketRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // socket_id is the identifier of the specific socket to get. + SocketId int64 `protobuf:"varint,1,opt,name=socket_id,json=socketId,proto3" json:"socket_id,omitempty"` + // If true, the response will contain only high level information + // that is inexpensive to obtain. Fields that may be omitted are + // documented. + Summary bool `protobuf:"varint,2,opt,name=summary,proto3" json:"summary,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetSocketRequest) Reset() { + *x = GetSocketRequest{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetSocketRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetSocketRequest) ProtoMessage() {} + +func (x *GetSocketRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[32] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetSocketRequest.ProtoReflect.Descriptor instead. +func (*GetSocketRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{32} +} + +func (x *GetSocketRequest) GetSocketId() int64 { + if x != nil { + return x.SocketId + } + return 0 +} + +func (x *GetSocketRequest) GetSummary() bool { + if x != nil { + return x.Summary + } + return false +} + +type GetSocketResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The Socket that corresponds to the requested socket_id. This field + // should be set. + Socket *Socket `protobuf:"bytes,1,opt,name=socket,proto3" json:"socket,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetSocketResponse) Reset() { + *x = GetSocketResponse{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetSocketResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetSocketResponse) ProtoMessage() {} + +func (x *GetSocketResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[33] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetSocketResponse.ProtoReflect.Descriptor instead. +func (*GetSocketResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{33} +} + +func (x *GetSocketResponse) GetSocket() *Socket { + if x != nil { + return x.Socket + } + return nil +} + +type Address_TcpIpAddress struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Either the IPv4 or IPv6 address in bytes. Will be either 4 bytes or 16 + // bytes in length. + IpAddress []byte `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` + // 0-64k, or -1 if not appropriate. + Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Address_TcpIpAddress) Reset() { + *x = Address_TcpIpAddress{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Address_TcpIpAddress) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Address_TcpIpAddress) ProtoMessage() {} + +func (x *Address_TcpIpAddress) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[34] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Address_TcpIpAddress.ProtoReflect.Descriptor instead. +func (*Address_TcpIpAddress) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{14, 0} +} + +func (x *Address_TcpIpAddress) GetIpAddress() []byte { + if x != nil { + return x.IpAddress + } + return nil +} + +func (x *Address_TcpIpAddress) GetPort() int32 { + if x != nil { + return x.Port + } + return 0 +} + +// A Unix Domain Socket address. +type Address_UdsAddress struct { + state protoimpl.MessageState `protogen:"open.v1"` + Filename string `protobuf:"bytes,1,opt,name=filename,proto3" json:"filename,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Address_UdsAddress) Reset() { + *x = Address_UdsAddress{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[35] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Address_UdsAddress) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Address_UdsAddress) ProtoMessage() {} + +func (x *Address_UdsAddress) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[35] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Address_UdsAddress.ProtoReflect.Descriptor instead. +func (*Address_UdsAddress) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{14, 1} +} + +func (x *Address_UdsAddress) GetFilename() string { + if x != nil { + return x.Filename + } + return "" +} + +// An address type not included above. +type Address_OtherAddress struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The human readable version of the value. This value should be set. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // The actual address message. + Value *anypb.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Address_OtherAddress) Reset() { + *x = Address_OtherAddress{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[36] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Address_OtherAddress) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Address_OtherAddress) ProtoMessage() {} + +func (x *Address_OtherAddress) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[36] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Address_OtherAddress.ProtoReflect.Descriptor instead. +func (*Address_OtherAddress) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{14, 2} +} + +func (x *Address_OtherAddress) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Address_OtherAddress) GetValue() *anypb.Any { + if x != nil { + return x.Value + } + return nil +} + +type Security_Tls struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to CipherSuite: + // + // *Security_Tls_StandardName + // *Security_Tls_OtherName + CipherSuite isSecurity_Tls_CipherSuite `protobuf_oneof:"cipher_suite"` + // the certificate used by this endpoint. + LocalCertificate []byte `protobuf:"bytes,3,opt,name=local_certificate,json=localCertificate,proto3" json:"local_certificate,omitempty"` + // the certificate used by the remote endpoint. + RemoteCertificate []byte `protobuf:"bytes,4,opt,name=remote_certificate,json=remoteCertificate,proto3" json:"remote_certificate,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Security_Tls) Reset() { + *x = Security_Tls{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Security_Tls) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Security_Tls) ProtoMessage() {} + +func (x *Security_Tls) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[37] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Security_Tls.ProtoReflect.Descriptor instead. +func (*Security_Tls) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{15, 0} +} + +func (x *Security_Tls) GetCipherSuite() isSecurity_Tls_CipherSuite { + if x != nil { + return x.CipherSuite + } + return nil +} + +func (x *Security_Tls) GetStandardName() string { + if x != nil { + if x, ok := x.CipherSuite.(*Security_Tls_StandardName); ok { + return x.StandardName + } + } + return "" +} + +func (x *Security_Tls) GetOtherName() string { + if x != nil { + if x, ok := x.CipherSuite.(*Security_Tls_OtherName); ok { + return x.OtherName + } + } + return "" +} + +func (x *Security_Tls) GetLocalCertificate() []byte { + if x != nil { + return x.LocalCertificate + } + return nil +} + +func (x *Security_Tls) GetRemoteCertificate() []byte { + if x != nil { + return x.RemoteCertificate + } + return nil +} + +type isSecurity_Tls_CipherSuite interface { + isSecurity_Tls_CipherSuite() +} + +type Security_Tls_StandardName struct { + // The cipher suite name in the RFC 4346 format: + // https://tools.ietf.org/html/rfc4346#appendix-C + StandardName string `protobuf:"bytes,1,opt,name=standard_name,json=standardName,proto3,oneof"` +} + +type Security_Tls_OtherName struct { + // Some other way to describe the cipher suite if + // the RFC 4346 name is not available. + OtherName string `protobuf:"bytes,2,opt,name=other_name,json=otherName,proto3,oneof"` +} + +func (*Security_Tls_StandardName) isSecurity_Tls_CipherSuite() {} + +func (*Security_Tls_OtherName) isSecurity_Tls_CipherSuite() {} + +type Security_OtherSecurity struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The human readable version of the value. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // The actual security details message. + Value *anypb.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Security_OtherSecurity) Reset() { + *x = Security_OtherSecurity{} + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[38] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Security_OtherSecurity) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Security_OtherSecurity) ProtoMessage() {} + +func (x *Security_OtherSecurity) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[38] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Security_OtherSecurity.ProtoReflect.Descriptor instead. +func (*Security_OtherSecurity) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{15, 1} +} + +func (x *Security_OtherSecurity) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Security_OtherSecurity) GetValue() *anypb.Any { + if x != nil { + return x.Value + } + return nil +} + +var File_grpc_channelz_v1_channelz_proto protoreflect.FileDescriptor + +const file_grpc_channelz_v1_channelz_proto_rawDesc = "" + + "\n" + + "\x1fgrpc/channelz/v1/channelz.proto\x12\x10grpc.channelz.v1\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\"\xaf\x02\n" + + "\aChannel\x12.\n" + + "\x03ref\x18\x01 \x01(\v2\x1c.grpc.channelz.v1.ChannelRefR\x03ref\x121\n" + + "\x04data\x18\x02 \x01(\v2\x1d.grpc.channelz.v1.ChannelDataR\x04data\x12=\n" + + "\vchannel_ref\x18\x03 \x03(\v2\x1c.grpc.channelz.v1.ChannelRefR\n" + + "channelRef\x12F\n" + + "\x0esubchannel_ref\x18\x04 \x03(\v2\x1f.grpc.channelz.v1.SubchannelRefR\rsubchannelRef\x12:\n" + + "\n" + + "socket_ref\x18\x05 \x03(\v2\x1b.grpc.channelz.v1.SocketRefR\tsocketRef\"\xb5\x02\n" + + "\n" + + "Subchannel\x121\n" + + "\x03ref\x18\x01 \x01(\v2\x1f.grpc.channelz.v1.SubchannelRefR\x03ref\x121\n" + + "\x04data\x18\x02 \x01(\v2\x1d.grpc.channelz.v1.ChannelDataR\x04data\x12=\n" + + "\vchannel_ref\x18\x03 \x03(\v2\x1c.grpc.channelz.v1.ChannelRefR\n" + + "channelRef\x12F\n" + + "\x0esubchannel_ref\x18\x04 \x03(\v2\x1f.grpc.channelz.v1.SubchannelRefR\rsubchannelRef\x12:\n" + + "\n" + + "socket_ref\x18\x05 \x03(\v2\x1b.grpc.channelz.v1.SocketRefR\tsocketRef\"\xc2\x01\n" + + "\x18ChannelConnectivityState\x12F\n" + + "\x05state\x18\x01 \x01(\x0e20.grpc.channelz.v1.ChannelConnectivityState.StateR\x05state\"^\n" + + "\x05State\x12\v\n" + + "\aUNKNOWN\x10\x00\x12\b\n" + + "\x04IDLE\x10\x01\x12\x0e\n" + + "\n" + + "CONNECTING\x10\x02\x12\t\n" + + "\x05READY\x10\x03\x12\x15\n" + + "\x11TRANSIENT_FAILURE\x10\x04\x12\f\n" + + "\bSHUTDOWN\x10\x05\"\xae\x03\n" + + "\vChannelData\x12@\n" + + "\x05state\x18\x01 \x01(\v2*.grpc.channelz.v1.ChannelConnectivityStateR\x05state\x12\x16\n" + + "\x06target\x18\x02 \x01(\tR\x06target\x124\n" + + "\x05trace\x18\x03 \x01(\v2\x1e.grpc.channelz.v1.ChannelTraceR\x05trace\x12#\n" + + "\rcalls_started\x18\x04 \x01(\x03R\fcallsStarted\x12'\n" + + "\x0fcalls_succeeded\x18\x05 \x01(\x03R\x0ecallsSucceeded\x12!\n" + + "\fcalls_failed\x18\x06 \x01(\x03R\vcallsFailed\x12Y\n" + + "\x1blast_call_started_timestamp\x18\a \x01(\v2\x1a.google.protobuf.TimestampR\x18lastCallStartedTimestamp\x12C\n" + + "\x1emax_connections_per_subchannel\x18\b \x01(\rR\x1bmaxConnectionsPerSubchannel\"\x98\x03\n" + + "\x11ChannelTraceEvent\x12 \n" + + "\vdescription\x18\x01 \x01(\tR\vdescription\x12H\n" + + "\bseverity\x18\x02 \x01(\x0e2,.grpc.channelz.v1.ChannelTraceEvent.SeverityR\bseverity\x128\n" + + "\ttimestamp\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\x12?\n" + + "\vchannel_ref\x18\x04 \x01(\v2\x1c.grpc.channelz.v1.ChannelRefH\x00R\n" + + "channelRef\x12H\n" + + "\x0esubchannel_ref\x18\x05 \x01(\v2\x1f.grpc.channelz.v1.SubchannelRefH\x00R\rsubchannelRef\"E\n" + + "\bSeverity\x12\x0e\n" + + "\n" + + "CT_UNKNOWN\x10\x00\x12\v\n" + + "\aCT_INFO\x10\x01\x12\x0e\n" + + "\n" + + "CT_WARNING\x10\x02\x12\f\n" + + "\bCT_ERROR\x10\x03B\v\n" + + "\tchild_ref\"\xc2\x01\n" + + "\fChannelTrace\x12*\n" + + "\x11num_events_logged\x18\x01 \x01(\x03R\x0fnumEventsLogged\x12I\n" + + "\x12creation_timestamp\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x11creationTimestamp\x12;\n" + + "\x06events\x18\x03 \x03(\v2#.grpc.channelz.v1.ChannelTraceEventR\x06events\"c\n" + + "\n" + + "ChannelRef\x12\x1d\n" + + "\n" + + "channel_id\x18\x01 \x01(\x03R\tchannelId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04nameJ\x04\b\x03\x10\x04J\x04\b\x04\x10\x05J\x04\b\x05\x10\x06J\x04\b\x06\x10\aJ\x04\b\a\x10\bJ\x04\b\b\x10\t\"l\n" + + "\rSubchannelRef\x12#\n" + + "\rsubchannel_id\x18\a \x01(\x03R\fsubchannelId\x12\x12\n" + + "\x04name\x18\b \x01(\tR\x04nameJ\x04\b\x01\x10\x02J\x04\b\x02\x10\x03J\x04\b\x03\x10\x04J\x04\b\x04\x10\x05J\x04\b\x05\x10\x06J\x04\b\x06\x10\a\"`\n" + + "\tSocketRef\x12\x1b\n" + + "\tsocket_id\x18\x03 \x01(\x03R\bsocketId\x12\x12\n" + + "\x04name\x18\x04 \x01(\tR\x04nameJ\x04\b\x01\x10\x02J\x04\b\x02\x10\x03J\x04\b\x05\x10\x06J\x04\b\x06\x10\aJ\x04\b\a\x10\bJ\x04\b\b\x10\t\"`\n" + + "\tServerRef\x12\x1b\n" + + "\tserver_id\x18\x05 \x01(\x03R\bserverId\x12\x12\n" + + "\x04name\x18\x06 \x01(\tR\x04nameJ\x04\b\x01\x10\x02J\x04\b\x02\x10\x03J\x04\b\x03\x10\x04J\x04\b\x04\x10\x05J\x04\b\a\x10\bJ\x04\b\b\x10\t\"\xab\x01\n" + + "\x06Server\x12-\n" + + "\x03ref\x18\x01 \x01(\v2\x1b.grpc.channelz.v1.ServerRefR\x03ref\x120\n" + + "\x04data\x18\x02 \x01(\v2\x1c.grpc.channelz.v1.ServerDataR\x04data\x12@\n" + + "\rlisten_socket\x18\x03 \x03(\v2\x1b.grpc.channelz.v1.SocketRefR\flistenSocket\"\x8e\x02\n" + + "\n" + + "ServerData\x124\n" + + "\x05trace\x18\x01 \x01(\v2\x1e.grpc.channelz.v1.ChannelTraceR\x05trace\x12#\n" + + "\rcalls_started\x18\x02 \x01(\x03R\fcallsStarted\x12'\n" + + "\x0fcalls_succeeded\x18\x03 \x01(\x03R\x0ecallsSucceeded\x12!\n" + + "\fcalls_failed\x18\x04 \x01(\x03R\vcallsFailed\x12Y\n" + + "\x1blast_call_started_timestamp\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\x18lastCallStartedTimestamp\"\xa6\x02\n" + + "\x06Socket\x12-\n" + + "\x03ref\x18\x01 \x01(\v2\x1b.grpc.channelz.v1.SocketRefR\x03ref\x120\n" + + "\x04data\x18\x02 \x01(\v2\x1c.grpc.channelz.v1.SocketDataR\x04data\x12/\n" + + "\x05local\x18\x03 \x01(\v2\x19.grpc.channelz.v1.AddressR\x05local\x121\n" + + "\x06remote\x18\x04 \x01(\v2\x19.grpc.channelz.v1.AddressR\x06remote\x126\n" + + "\bsecurity\x18\x05 \x01(\v2\x1a.grpc.channelz.v1.SecurityR\bsecurity\x12\x1f\n" + + "\vremote_name\x18\x06 \x01(\tR\n" + + "remoteName\"\x94\b\n" + + "\n" + + "SocketData\x12'\n" + + "\x0fstreams_started\x18\x01 \x01(\x03R\x0estreamsStarted\x12+\n" + + "\x11streams_succeeded\x18\x02 \x01(\x03R\x10streamsSucceeded\x12%\n" + + "\x0estreams_failed\x18\x03 \x01(\x03R\rstreamsFailed\x12#\n" + + "\rmessages_sent\x18\x04 \x01(\x03R\fmessagesSent\x12+\n" + + "\x11messages_received\x18\x05 \x01(\x03R\x10messagesReceived\x12(\n" + + "\x10keep_alives_sent\x18\x06 \x01(\x03R\x0ekeepAlivesSent\x12h\n" + + "#last_local_stream_created_timestamp\x18\a \x01(\v2\x1a.google.protobuf.TimestampR\x1flastLocalStreamCreatedTimestamp\x12j\n" + + "$last_remote_stream_created_timestamp\x18\b \x01(\v2\x1a.google.protobuf.TimestampR lastRemoteStreamCreatedTimestamp\x12Y\n" + + "\x1blast_message_sent_timestamp\x18\t \x01(\v2\x1a.google.protobuf.TimestampR\x18lastMessageSentTimestamp\x12a\n" + + "\x1flast_message_received_timestamp\x18\n" + + " \x01(\v2\x1a.google.protobuf.TimestampR\x1clastMessageReceivedTimestamp\x12V\n" + + "\x19local_flow_control_window\x18\v \x01(\v2\x1b.google.protobuf.Int64ValueR\x16localFlowControlWindow\x12X\n" + + "\x1aremote_flow_control_window\x18\f \x01(\v2\x1b.google.protobuf.Int64ValueR\x17remoteFlowControlWindow\x126\n" + + "\x06option\x18\r \x03(\v2\x1e.grpc.channelz.v1.SocketOptionR\x06option\x12P\n" + + "\x15received_goaway_error\x18\x0e \x01(\v2\x1c.google.protobuf.UInt32ValueR\x13receivedGoawayError\x12=\n" + + "\x1bpeer_max_concurrent_streams\x18\x0f \x01(\rR\x18peerMaxConcurrentStreams\"\xb8\x03\n" + + "\aAddress\x12M\n" + + "\rtcpip_address\x18\x01 \x01(\v2&.grpc.channelz.v1.Address.TcpIpAddressH\x00R\ftcpipAddress\x12G\n" + + "\vuds_address\x18\x02 \x01(\v2$.grpc.channelz.v1.Address.UdsAddressH\x00R\n" + + "udsAddress\x12M\n" + + "\rother_address\x18\x03 \x01(\v2&.grpc.channelz.v1.Address.OtherAddressH\x00R\fotherAddress\x1aA\n" + + "\fTcpIpAddress\x12\x1d\n" + + "\n" + + "ip_address\x18\x01 \x01(\fR\tipAddress\x12\x12\n" + + "\x04port\x18\x02 \x01(\x05R\x04port\x1a(\n" + + "\n" + + "UdsAddress\x12\x1a\n" + + "\bfilename\x18\x01 \x01(\tR\bfilename\x1aN\n" + + "\fOtherAddress\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12*\n" + + "\x05value\x18\x02 \x01(\v2\x14.google.protobuf.AnyR\x05valueB\t\n" + + "\aaddress\"\x96\x03\n" + + "\bSecurity\x122\n" + + "\x03tls\x18\x01 \x01(\v2\x1e.grpc.channelz.v1.Security.TlsH\x00R\x03tls\x12@\n" + + "\x05other\x18\x02 \x01(\v2(.grpc.channelz.v1.Security.OtherSecurityH\x00R\x05other\x1a\xb9\x01\n" + + "\x03Tls\x12%\n" + + "\rstandard_name\x18\x01 \x01(\tH\x00R\fstandardName\x12\x1f\n" + + "\n" + + "other_name\x18\x02 \x01(\tH\x00R\totherName\x12+\n" + + "\x11local_certificate\x18\x03 \x01(\fR\x10localCertificate\x12-\n" + + "\x12remote_certificate\x18\x04 \x01(\fR\x11remoteCertificateB\x0e\n" + + "\fcipher_suite\x1aO\n" + + "\rOtherSecurity\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12*\n" + + "\x05value\x18\x02 \x01(\v2\x14.google.protobuf.AnyR\x05valueB\a\n" + + "\x05model\"n\n" + + "\fSocketOption\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value\x124\n" + + "\n" + + "additional\x18\x03 \x01(\v2\x14.google.protobuf.AnyR\n" + + "additional\"L\n" + + "\x13SocketOptionTimeout\x125\n" + + "\bduration\x18\x01 \x01(\v2\x19.google.protobuf.DurationR\bduration\"c\n" + + "\x12SocketOptionLinger\x12\x16\n" + + "\x06active\x18\x01 \x01(\bR\x06active\x125\n" + + "\bduration\x18\x02 \x01(\v2\x19.google.protobuf.DurationR\bduration\"\xb2\b\n" + + "\x13SocketOptionTcpInfo\x12\x1d\n" + + "\n" + + "tcpi_state\x18\x01 \x01(\rR\ttcpiState\x12\"\n" + + "\rtcpi_ca_state\x18\x02 \x01(\rR\vtcpiCaState\x12)\n" + + "\x10tcpi_retransmits\x18\x03 \x01(\rR\x0ftcpiRetransmits\x12\x1f\n" + + "\vtcpi_probes\x18\x04 \x01(\rR\n" + + "tcpiProbes\x12!\n" + + "\ftcpi_backoff\x18\x05 \x01(\rR\vtcpiBackoff\x12!\n" + + "\ftcpi_options\x18\x06 \x01(\rR\vtcpiOptions\x12&\n" + + "\x0ftcpi_snd_wscale\x18\a \x01(\rR\rtcpiSndWscale\x12&\n" + + "\x0ftcpi_rcv_wscale\x18\b \x01(\rR\rtcpiRcvWscale\x12\x19\n" + + "\btcpi_rto\x18\t \x01(\rR\atcpiRto\x12\x19\n" + + "\btcpi_ato\x18\n" + + " \x01(\rR\atcpiAto\x12 \n" + + "\ftcpi_snd_mss\x18\v \x01(\rR\n" + + "tcpiSndMss\x12 \n" + + "\ftcpi_rcv_mss\x18\f \x01(\rR\n" + + "tcpiRcvMss\x12!\n" + + "\ftcpi_unacked\x18\r \x01(\rR\vtcpiUnacked\x12\x1f\n" + + "\vtcpi_sacked\x18\x0e \x01(\rR\n" + + "tcpiSacked\x12\x1b\n" + + "\ttcpi_lost\x18\x0f \x01(\rR\btcpiLost\x12!\n" + + "\ftcpi_retrans\x18\x10 \x01(\rR\vtcpiRetrans\x12!\n" + + "\ftcpi_fackets\x18\x11 \x01(\rR\vtcpiFackets\x12-\n" + + "\x13tcpi_last_data_sent\x18\x12 \x01(\rR\x10tcpiLastDataSent\x12+\n" + + "\x12tcpi_last_ack_sent\x18\x13 \x01(\rR\x0ftcpiLastAckSent\x12-\n" + + "\x13tcpi_last_data_recv\x18\x14 \x01(\rR\x10tcpiLastDataRecv\x12+\n" + + "\x12tcpi_last_ack_recv\x18\x15 \x01(\rR\x0ftcpiLastAckRecv\x12\x1b\n" + + "\ttcpi_pmtu\x18\x16 \x01(\rR\btcpiPmtu\x12*\n" + + "\x11tcpi_rcv_ssthresh\x18\x17 \x01(\rR\x0ftcpiRcvSsthresh\x12\x19\n" + + "\btcpi_rtt\x18\x18 \x01(\rR\atcpiRtt\x12\x1f\n" + + "\vtcpi_rttvar\x18\x19 \x01(\rR\n" + + "tcpiRttvar\x12*\n" + + "\x11tcpi_snd_ssthresh\x18\x1a \x01(\rR\x0ftcpiSndSsthresh\x12\"\n" + + "\rtcpi_snd_cwnd\x18\x1b \x01(\rR\vtcpiSndCwnd\x12\x1f\n" + + "\vtcpi_advmss\x18\x1c \x01(\rR\n" + + "tcpiAdvmss\x12'\n" + + "\x0ftcpi_reordering\x18\x1d \x01(\rR\x0etcpiReordering\"b\n" + + "\x15GetTopChannelsRequest\x12(\n" + + "\x10start_channel_id\x18\x01 \x01(\x03R\x0estartChannelId\x12\x1f\n" + + "\vmax_results\x18\x02 \x01(\x03R\n" + + "maxResults\"_\n" + + "\x16GetTopChannelsResponse\x123\n" + + "\achannel\x18\x01 \x03(\v2\x19.grpc.channelz.v1.ChannelR\achannel\x12\x10\n" + + "\x03end\x18\x02 \x01(\bR\x03end\"\\\n" + + "\x11GetServersRequest\x12&\n" + + "\x0fstart_server_id\x18\x01 \x01(\x03R\rstartServerId\x12\x1f\n" + + "\vmax_results\x18\x02 \x01(\x03R\n" + + "maxResults\"X\n" + + "\x12GetServersResponse\x120\n" + + "\x06server\x18\x01 \x03(\v2\x18.grpc.channelz.v1.ServerR\x06server\x12\x10\n" + + "\x03end\x18\x02 \x01(\bR\x03end\"/\n" + + "\x10GetServerRequest\x12\x1b\n" + + "\tserver_id\x18\x01 \x01(\x03R\bserverId\"E\n" + + "\x11GetServerResponse\x120\n" + + "\x06server\x18\x01 \x01(\v2\x18.grpc.channelz.v1.ServerR\x06server\"\x7f\n" + + "\x17GetServerSocketsRequest\x12\x1b\n" + + "\tserver_id\x18\x01 \x01(\x03R\bserverId\x12&\n" + + "\x0fstart_socket_id\x18\x02 \x01(\x03R\rstartSocketId\x12\x1f\n" + + "\vmax_results\x18\x03 \x01(\x03R\n" + + "maxResults\"h\n" + + "\x18GetServerSocketsResponse\x12:\n" + + "\n" + + "socket_ref\x18\x01 \x03(\v2\x1b.grpc.channelz.v1.SocketRefR\tsocketRef\x12\x10\n" + + "\x03end\x18\x02 \x01(\bR\x03end\"2\n" + + "\x11GetChannelRequest\x12\x1d\n" + + "\n" + + "channel_id\x18\x01 \x01(\x03R\tchannelId\"I\n" + + "\x12GetChannelResponse\x123\n" + + "\achannel\x18\x01 \x01(\v2\x19.grpc.channelz.v1.ChannelR\achannel\";\n" + + "\x14GetSubchannelRequest\x12#\n" + + "\rsubchannel_id\x18\x01 \x01(\x03R\fsubchannelId\"U\n" + + "\x15GetSubchannelResponse\x12<\n" + + "\n" + + "subchannel\x18\x01 \x01(\v2\x1c.grpc.channelz.v1.SubchannelR\n" + + "subchannel\"I\n" + + "\x10GetSocketRequest\x12\x1b\n" + + "\tsocket_id\x18\x01 \x01(\x03R\bsocketId\x12\x18\n" + + "\asummary\x18\x02 \x01(\bR\asummary\"E\n" + + "\x11GetSocketResponse\x120\n" + + "\x06socket\x18\x01 \x01(\v2\x18.grpc.channelz.v1.SocketR\x06socket2\x9a\x05\n" + + "\bChannelz\x12c\n" + + "\x0eGetTopChannels\x12'.grpc.channelz.v1.GetTopChannelsRequest\x1a(.grpc.channelz.v1.GetTopChannelsResponse\x12W\n" + + "\n" + + "GetServers\x12#.grpc.channelz.v1.GetServersRequest\x1a$.grpc.channelz.v1.GetServersResponse\x12T\n" + + "\tGetServer\x12\".grpc.channelz.v1.GetServerRequest\x1a#.grpc.channelz.v1.GetServerResponse\x12i\n" + + "\x10GetServerSockets\x12).grpc.channelz.v1.GetServerSocketsRequest\x1a*.grpc.channelz.v1.GetServerSocketsResponse\x12W\n" + + "\n" + + "GetChannel\x12#.grpc.channelz.v1.GetChannelRequest\x1a$.grpc.channelz.v1.GetChannelResponse\x12`\n" + + "\rGetSubchannel\x12&.grpc.channelz.v1.GetSubchannelRequest\x1a'.grpc.channelz.v1.GetSubchannelResponse\x12T\n" + + "\tGetSocket\x12\".grpc.channelz.v1.GetSocketRequest\x1a#.grpc.channelz.v1.GetSocketResponseBX\n" + + "\x13io.grpc.channelz.v1B\rChannelzProtoP\x01Z0google.golang.org/grpc/channelz/grpc_channelz_v1b\x06proto3" + +var ( + file_grpc_channelz_v1_channelz_proto_rawDescOnce sync.Once + file_grpc_channelz_v1_channelz_proto_rawDescData []byte +) + +func file_grpc_channelz_v1_channelz_proto_rawDescGZIP() []byte { + file_grpc_channelz_v1_channelz_proto_rawDescOnce.Do(func() { + file_grpc_channelz_v1_channelz_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_channelz_v1_channelz_proto_rawDesc), len(file_grpc_channelz_v1_channelz_proto_rawDesc))) + }) + return file_grpc_channelz_v1_channelz_proto_rawDescData +} + +var file_grpc_channelz_v1_channelz_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_grpc_channelz_v1_channelz_proto_msgTypes = make([]protoimpl.MessageInfo, 39) +var file_grpc_channelz_v1_channelz_proto_goTypes = []any{ + (ChannelConnectivityState_State)(0), // 0: grpc.channelz.v1.ChannelConnectivityState.State + (ChannelTraceEvent_Severity)(0), // 1: grpc.channelz.v1.ChannelTraceEvent.Severity + (*Channel)(nil), // 2: grpc.channelz.v1.Channel + (*Subchannel)(nil), // 3: grpc.channelz.v1.Subchannel + (*ChannelConnectivityState)(nil), // 4: grpc.channelz.v1.ChannelConnectivityState + (*ChannelData)(nil), // 5: grpc.channelz.v1.ChannelData + (*ChannelTraceEvent)(nil), // 6: grpc.channelz.v1.ChannelTraceEvent + (*ChannelTrace)(nil), // 7: grpc.channelz.v1.ChannelTrace + (*ChannelRef)(nil), // 8: grpc.channelz.v1.ChannelRef + (*SubchannelRef)(nil), // 9: grpc.channelz.v1.SubchannelRef + (*SocketRef)(nil), // 10: grpc.channelz.v1.SocketRef + (*ServerRef)(nil), // 11: grpc.channelz.v1.ServerRef + (*Server)(nil), // 12: grpc.channelz.v1.Server + (*ServerData)(nil), // 13: grpc.channelz.v1.ServerData + (*Socket)(nil), // 14: grpc.channelz.v1.Socket + (*SocketData)(nil), // 15: grpc.channelz.v1.SocketData + (*Address)(nil), // 16: grpc.channelz.v1.Address + (*Security)(nil), // 17: grpc.channelz.v1.Security + (*SocketOption)(nil), // 18: grpc.channelz.v1.SocketOption + (*SocketOptionTimeout)(nil), // 19: grpc.channelz.v1.SocketOptionTimeout + (*SocketOptionLinger)(nil), // 20: grpc.channelz.v1.SocketOptionLinger + (*SocketOptionTcpInfo)(nil), // 21: grpc.channelz.v1.SocketOptionTcpInfo + (*GetTopChannelsRequest)(nil), // 22: grpc.channelz.v1.GetTopChannelsRequest + (*GetTopChannelsResponse)(nil), // 23: grpc.channelz.v1.GetTopChannelsResponse + (*GetServersRequest)(nil), // 24: grpc.channelz.v1.GetServersRequest + (*GetServersResponse)(nil), // 25: grpc.channelz.v1.GetServersResponse + (*GetServerRequest)(nil), // 26: grpc.channelz.v1.GetServerRequest + (*GetServerResponse)(nil), // 27: grpc.channelz.v1.GetServerResponse + (*GetServerSocketsRequest)(nil), // 28: grpc.channelz.v1.GetServerSocketsRequest + (*GetServerSocketsResponse)(nil), // 29: grpc.channelz.v1.GetServerSocketsResponse + (*GetChannelRequest)(nil), // 30: grpc.channelz.v1.GetChannelRequest + (*GetChannelResponse)(nil), // 31: grpc.channelz.v1.GetChannelResponse + (*GetSubchannelRequest)(nil), // 32: grpc.channelz.v1.GetSubchannelRequest + (*GetSubchannelResponse)(nil), // 33: grpc.channelz.v1.GetSubchannelResponse + (*GetSocketRequest)(nil), // 34: grpc.channelz.v1.GetSocketRequest + (*GetSocketResponse)(nil), // 35: grpc.channelz.v1.GetSocketResponse + (*Address_TcpIpAddress)(nil), // 36: grpc.channelz.v1.Address.TcpIpAddress + (*Address_UdsAddress)(nil), // 37: grpc.channelz.v1.Address.UdsAddress + (*Address_OtherAddress)(nil), // 38: grpc.channelz.v1.Address.OtherAddress + (*Security_Tls)(nil), // 39: grpc.channelz.v1.Security.Tls + (*Security_OtherSecurity)(nil), // 40: grpc.channelz.v1.Security.OtherSecurity + (*timestamppb.Timestamp)(nil), // 41: google.protobuf.Timestamp + (*wrapperspb.Int64Value)(nil), // 42: google.protobuf.Int64Value + (*wrapperspb.UInt32Value)(nil), // 43: google.protobuf.UInt32Value + (*anypb.Any)(nil), // 44: google.protobuf.Any + (*durationpb.Duration)(nil), // 45: google.protobuf.Duration +} +var file_grpc_channelz_v1_channelz_proto_depIdxs = []int32{ + 8, // 0: grpc.channelz.v1.Channel.ref:type_name -> grpc.channelz.v1.ChannelRef + 5, // 1: grpc.channelz.v1.Channel.data:type_name -> grpc.channelz.v1.ChannelData + 8, // 2: grpc.channelz.v1.Channel.channel_ref:type_name -> grpc.channelz.v1.ChannelRef + 9, // 3: grpc.channelz.v1.Channel.subchannel_ref:type_name -> grpc.channelz.v1.SubchannelRef + 10, // 4: grpc.channelz.v1.Channel.socket_ref:type_name -> grpc.channelz.v1.SocketRef + 9, // 5: grpc.channelz.v1.Subchannel.ref:type_name -> grpc.channelz.v1.SubchannelRef + 5, // 6: grpc.channelz.v1.Subchannel.data:type_name -> grpc.channelz.v1.ChannelData + 8, // 7: grpc.channelz.v1.Subchannel.channel_ref:type_name -> grpc.channelz.v1.ChannelRef + 9, // 8: grpc.channelz.v1.Subchannel.subchannel_ref:type_name -> grpc.channelz.v1.SubchannelRef + 10, // 9: grpc.channelz.v1.Subchannel.socket_ref:type_name -> grpc.channelz.v1.SocketRef + 0, // 10: grpc.channelz.v1.ChannelConnectivityState.state:type_name -> grpc.channelz.v1.ChannelConnectivityState.State + 4, // 11: grpc.channelz.v1.ChannelData.state:type_name -> grpc.channelz.v1.ChannelConnectivityState + 7, // 12: grpc.channelz.v1.ChannelData.trace:type_name -> grpc.channelz.v1.ChannelTrace + 41, // 13: grpc.channelz.v1.ChannelData.last_call_started_timestamp:type_name -> google.protobuf.Timestamp + 1, // 14: grpc.channelz.v1.ChannelTraceEvent.severity:type_name -> grpc.channelz.v1.ChannelTraceEvent.Severity + 41, // 15: grpc.channelz.v1.ChannelTraceEvent.timestamp:type_name -> google.protobuf.Timestamp + 8, // 16: grpc.channelz.v1.ChannelTraceEvent.channel_ref:type_name -> grpc.channelz.v1.ChannelRef + 9, // 17: grpc.channelz.v1.ChannelTraceEvent.subchannel_ref:type_name -> grpc.channelz.v1.SubchannelRef + 41, // 18: grpc.channelz.v1.ChannelTrace.creation_timestamp:type_name -> google.protobuf.Timestamp + 6, // 19: grpc.channelz.v1.ChannelTrace.events:type_name -> grpc.channelz.v1.ChannelTraceEvent + 11, // 20: grpc.channelz.v1.Server.ref:type_name -> grpc.channelz.v1.ServerRef + 13, // 21: grpc.channelz.v1.Server.data:type_name -> grpc.channelz.v1.ServerData + 10, // 22: grpc.channelz.v1.Server.listen_socket:type_name -> grpc.channelz.v1.SocketRef + 7, // 23: grpc.channelz.v1.ServerData.trace:type_name -> grpc.channelz.v1.ChannelTrace + 41, // 24: grpc.channelz.v1.ServerData.last_call_started_timestamp:type_name -> google.protobuf.Timestamp + 10, // 25: grpc.channelz.v1.Socket.ref:type_name -> grpc.channelz.v1.SocketRef + 15, // 26: grpc.channelz.v1.Socket.data:type_name -> grpc.channelz.v1.SocketData + 16, // 27: grpc.channelz.v1.Socket.local:type_name -> grpc.channelz.v1.Address + 16, // 28: grpc.channelz.v1.Socket.remote:type_name -> grpc.channelz.v1.Address + 17, // 29: grpc.channelz.v1.Socket.security:type_name -> grpc.channelz.v1.Security + 41, // 30: grpc.channelz.v1.SocketData.last_local_stream_created_timestamp:type_name -> google.protobuf.Timestamp + 41, // 31: grpc.channelz.v1.SocketData.last_remote_stream_created_timestamp:type_name -> google.protobuf.Timestamp + 41, // 32: grpc.channelz.v1.SocketData.last_message_sent_timestamp:type_name -> google.protobuf.Timestamp + 41, // 33: grpc.channelz.v1.SocketData.last_message_received_timestamp:type_name -> google.protobuf.Timestamp + 42, // 34: grpc.channelz.v1.SocketData.local_flow_control_window:type_name -> google.protobuf.Int64Value + 42, // 35: grpc.channelz.v1.SocketData.remote_flow_control_window:type_name -> google.protobuf.Int64Value + 18, // 36: grpc.channelz.v1.SocketData.option:type_name -> grpc.channelz.v1.SocketOption + 43, // 37: grpc.channelz.v1.SocketData.received_goaway_error:type_name -> google.protobuf.UInt32Value + 36, // 38: grpc.channelz.v1.Address.tcpip_address:type_name -> grpc.channelz.v1.Address.TcpIpAddress + 37, // 39: grpc.channelz.v1.Address.uds_address:type_name -> grpc.channelz.v1.Address.UdsAddress + 38, // 40: grpc.channelz.v1.Address.other_address:type_name -> grpc.channelz.v1.Address.OtherAddress + 39, // 41: grpc.channelz.v1.Security.tls:type_name -> grpc.channelz.v1.Security.Tls + 40, // 42: grpc.channelz.v1.Security.other:type_name -> grpc.channelz.v1.Security.OtherSecurity + 44, // 43: grpc.channelz.v1.SocketOption.additional:type_name -> google.protobuf.Any + 45, // 44: grpc.channelz.v1.SocketOptionTimeout.duration:type_name -> google.protobuf.Duration + 45, // 45: grpc.channelz.v1.SocketOptionLinger.duration:type_name -> google.protobuf.Duration + 2, // 46: grpc.channelz.v1.GetTopChannelsResponse.channel:type_name -> grpc.channelz.v1.Channel + 12, // 47: grpc.channelz.v1.GetServersResponse.server:type_name -> grpc.channelz.v1.Server + 12, // 48: grpc.channelz.v1.GetServerResponse.server:type_name -> grpc.channelz.v1.Server + 10, // 49: grpc.channelz.v1.GetServerSocketsResponse.socket_ref:type_name -> grpc.channelz.v1.SocketRef + 2, // 50: grpc.channelz.v1.GetChannelResponse.channel:type_name -> grpc.channelz.v1.Channel + 3, // 51: grpc.channelz.v1.GetSubchannelResponse.subchannel:type_name -> grpc.channelz.v1.Subchannel + 14, // 52: grpc.channelz.v1.GetSocketResponse.socket:type_name -> grpc.channelz.v1.Socket + 44, // 53: grpc.channelz.v1.Address.OtherAddress.value:type_name -> google.protobuf.Any + 44, // 54: grpc.channelz.v1.Security.OtherSecurity.value:type_name -> google.protobuf.Any + 22, // 55: grpc.channelz.v1.Channelz.GetTopChannels:input_type -> grpc.channelz.v1.GetTopChannelsRequest + 24, // 56: grpc.channelz.v1.Channelz.GetServers:input_type -> grpc.channelz.v1.GetServersRequest + 26, // 57: grpc.channelz.v1.Channelz.GetServer:input_type -> grpc.channelz.v1.GetServerRequest + 28, // 58: grpc.channelz.v1.Channelz.GetServerSockets:input_type -> grpc.channelz.v1.GetServerSocketsRequest + 30, // 59: grpc.channelz.v1.Channelz.GetChannel:input_type -> grpc.channelz.v1.GetChannelRequest + 32, // 60: grpc.channelz.v1.Channelz.GetSubchannel:input_type -> grpc.channelz.v1.GetSubchannelRequest + 34, // 61: grpc.channelz.v1.Channelz.GetSocket:input_type -> grpc.channelz.v1.GetSocketRequest + 23, // 62: grpc.channelz.v1.Channelz.GetTopChannels:output_type -> grpc.channelz.v1.GetTopChannelsResponse + 25, // 63: grpc.channelz.v1.Channelz.GetServers:output_type -> grpc.channelz.v1.GetServersResponse + 27, // 64: grpc.channelz.v1.Channelz.GetServer:output_type -> grpc.channelz.v1.GetServerResponse + 29, // 65: grpc.channelz.v1.Channelz.GetServerSockets:output_type -> grpc.channelz.v1.GetServerSocketsResponse + 31, // 66: grpc.channelz.v1.Channelz.GetChannel:output_type -> grpc.channelz.v1.GetChannelResponse + 33, // 67: grpc.channelz.v1.Channelz.GetSubchannel:output_type -> grpc.channelz.v1.GetSubchannelResponse + 35, // 68: grpc.channelz.v1.Channelz.GetSocket:output_type -> grpc.channelz.v1.GetSocketResponse + 62, // [62:69] is the sub-list for method output_type + 55, // [55:62] is the sub-list for method input_type + 55, // [55:55] is the sub-list for extension type_name + 55, // [55:55] is the sub-list for extension extendee + 0, // [0:55] is the sub-list for field type_name +} + +func init() { file_grpc_channelz_v1_channelz_proto_init() } +func file_grpc_channelz_v1_channelz_proto_init() { + if File_grpc_channelz_v1_channelz_proto != nil { + return + } + file_grpc_channelz_v1_channelz_proto_msgTypes[4].OneofWrappers = []any{ + (*ChannelTraceEvent_ChannelRef)(nil), + (*ChannelTraceEvent_SubchannelRef)(nil), + } + file_grpc_channelz_v1_channelz_proto_msgTypes[14].OneofWrappers = []any{ + (*Address_TcpipAddress)(nil), + (*Address_UdsAddress_)(nil), + (*Address_OtherAddress_)(nil), + } + file_grpc_channelz_v1_channelz_proto_msgTypes[15].OneofWrappers = []any{ + (*Security_Tls_)(nil), + (*Security_Other)(nil), + } + file_grpc_channelz_v1_channelz_proto_msgTypes[37].OneofWrappers = []any{ + (*Security_Tls_StandardName)(nil), + (*Security_Tls_OtherName)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_channelz_v1_channelz_proto_rawDesc), len(file_grpc_channelz_v1_channelz_proto_rawDesc)), + NumEnums: 2, + NumMessages: 39, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_channelz_v1_channelz_proto_goTypes, + DependencyIndexes: file_grpc_channelz_v1_channelz_proto_depIdxs, + EnumInfos: file_grpc_channelz_v1_channelz_proto_enumTypes, + MessageInfos: file_grpc_channelz_v1_channelz_proto_msgTypes, + }.Build() + File_grpc_channelz_v1_channelz_proto = out.File + file_grpc_channelz_v1_channelz_proto_goTypes = nil + file_grpc_channelz_v1_channelz_proto_depIdxs = nil +} diff --git a/channelz/grpc_channelz_v1/channelz_grpc.pb.go b/channelz/grpc_channelz_v1/channelz_grpc.pb.go new file mode 100644 index 0000000..bb418c2 --- /dev/null +++ b/channelz/grpc_channelz_v1/channelz_grpc.pb.go @@ -0,0 +1,390 @@ +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines an interface for exporting monitoring information +// out of gRPC servers. See the full design at +// https://github.com/grpc/proposal/blob/master/A14-channelz.md +// +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/channelz/v1/channelz.proto + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: grpc/channelz/v1/channelz.proto + +package grpc_channelz_v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Channelz_GetTopChannels_FullMethodName = "/grpc.channelz.v1.Channelz/GetTopChannels" + Channelz_GetServers_FullMethodName = "/grpc.channelz.v1.Channelz/GetServers" + Channelz_GetServer_FullMethodName = "/grpc.channelz.v1.Channelz/GetServer" + Channelz_GetServerSockets_FullMethodName = "/grpc.channelz.v1.Channelz/GetServerSockets" + Channelz_GetChannel_FullMethodName = "/grpc.channelz.v1.Channelz/GetChannel" + Channelz_GetSubchannel_FullMethodName = "/grpc.channelz.v1.Channelz/GetSubchannel" + Channelz_GetSocket_FullMethodName = "/grpc.channelz.v1.Channelz/GetSocket" +) + +// ChannelzClient is the client API for Channelz service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Channelz is a service exposed by gRPC servers that provides detailed debug +// information. +type ChannelzClient interface { + // Gets all root channels (i.e. channels the application has directly + // created). This does not include subchannels nor non-top level channels. + GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error) + // Gets all servers that exist in the process. + GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error) + // Returns a single Server, or else a NOT_FOUND code. + GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error) + // Gets all server sockets that exist in the process. + GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error) + // Returns a single Channel, or else a NOT_FOUND code. + GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error) + // Returns a single Subchannel, or else a NOT_FOUND code. + GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error) + // Returns a single Socket or else a NOT_FOUND code. + GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error) +} + +type channelzClient struct { + cc grpc.ClientConnInterface +} + +func NewChannelzClient(cc grpc.ClientConnInterface) ChannelzClient { + return &channelzClient{cc} +} + +func (c *channelzClient) GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetTopChannelsResponse) + err := c.cc.Invoke(ctx, Channelz_GetTopChannels_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *channelzClient) GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetServersResponse) + err := c.cc.Invoke(ctx, Channelz_GetServers_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *channelzClient) GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetServerResponse) + err := c.cc.Invoke(ctx, Channelz_GetServer_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *channelzClient) GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetServerSocketsResponse) + err := c.cc.Invoke(ctx, Channelz_GetServerSockets_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *channelzClient) GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetChannelResponse) + err := c.cc.Invoke(ctx, Channelz_GetChannel_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *channelzClient) GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetSubchannelResponse) + err := c.cc.Invoke(ctx, Channelz_GetSubchannel_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *channelzClient) GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetSocketResponse) + err := c.cc.Invoke(ctx, Channelz_GetSocket_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ChannelzServer is the server API for Channelz service. +// All implementations should embed UnimplementedChannelzServer +// for forward compatibility. +// +// Channelz is a service exposed by gRPC servers that provides detailed debug +// information. +type ChannelzServer interface { + // Gets all root channels (i.e. channels the application has directly + // created). This does not include subchannels nor non-top level channels. + GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) + // Gets all servers that exist in the process. + GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) + // Returns a single Server, or else a NOT_FOUND code. + GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) + // Gets all server sockets that exist in the process. + GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) + // Returns a single Channel, or else a NOT_FOUND code. + GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) + // Returns a single Subchannel, or else a NOT_FOUND code. + GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) + // Returns a single Socket or else a NOT_FOUND code. + GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) +} + +// UnimplementedChannelzServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedChannelzServer struct{} + +func (UnimplementedChannelzServer) GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetTopChannels not implemented") +} +func (UnimplementedChannelzServer) GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetServers not implemented") +} +func (UnimplementedChannelzServer) GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetServer not implemented") +} +func (UnimplementedChannelzServer) GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetServerSockets not implemented") +} +func (UnimplementedChannelzServer) GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetChannel not implemented") +} +func (UnimplementedChannelzServer) GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetSubchannel not implemented") +} +func (UnimplementedChannelzServer) GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetSocket not implemented") +} +func (UnimplementedChannelzServer) testEmbeddedByValue() {} + +// UnsafeChannelzServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ChannelzServer will +// result in compilation errors. +type UnsafeChannelzServer interface { + mustEmbedUnimplementedChannelzServer() +} + +func RegisterChannelzServer(s grpc.ServiceRegistrar, srv ChannelzServer) { + // If the following call panics, it indicates UnimplementedChannelzServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Channelz_ServiceDesc, srv) +} + +func _Channelz_GetTopChannels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetTopChannelsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ChannelzServer).GetTopChannels(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Channelz_GetTopChannels_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ChannelzServer).GetTopChannels(ctx, req.(*GetTopChannelsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Channelz_GetServers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetServersRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ChannelzServer).GetServers(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Channelz_GetServers_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ChannelzServer).GetServers(ctx, req.(*GetServersRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Channelz_GetServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetServerRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ChannelzServer).GetServer(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Channelz_GetServer_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ChannelzServer).GetServer(ctx, req.(*GetServerRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Channelz_GetServerSockets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetServerSocketsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ChannelzServer).GetServerSockets(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Channelz_GetServerSockets_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ChannelzServer).GetServerSockets(ctx, req.(*GetServerSocketsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Channelz_GetChannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetChannelRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ChannelzServer).GetChannel(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Channelz_GetChannel_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ChannelzServer).GetChannel(ctx, req.(*GetChannelRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Channelz_GetSubchannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetSubchannelRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ChannelzServer).GetSubchannel(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Channelz_GetSubchannel_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ChannelzServer).GetSubchannel(ctx, req.(*GetSubchannelRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Channelz_GetSocket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetSocketRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ChannelzServer).GetSocket(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Channelz_GetSocket_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ChannelzServer).GetSocket(ctx, req.(*GetSocketRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// Channelz_ServiceDesc is the grpc.ServiceDesc for Channelz service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Channelz_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.channelz.v1.Channelz", + HandlerType: (*ChannelzServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetTopChannels", + Handler: _Channelz_GetTopChannels_Handler, + }, + { + MethodName: "GetServers", + Handler: _Channelz_GetServers_Handler, + }, + { + MethodName: "GetServer", + Handler: _Channelz_GetServer_Handler, + }, + { + MethodName: "GetServerSockets", + Handler: _Channelz_GetServerSockets_Handler, + }, + { + MethodName: "GetChannel", + Handler: _Channelz_GetChannel_Handler, + }, + { + MethodName: "GetSubchannel", + Handler: _Channelz_GetSubchannel_Handler, + }, + { + MethodName: "GetSocket", + Handler: _Channelz_GetSocket_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/channelz/v1/channelz.proto", +} diff --git a/channelz/internal/protoconv/channel.go b/channelz/internal/protoconv/channel.go new file mode 100644 index 0000000..751608b --- /dev/null +++ b/channelz/internal/protoconv/channel.go @@ -0,0 +1,136 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package protoconv + +import ( + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" + + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" +) + +func connectivityStateToProto(s *connectivity.State) *channelzpb.ChannelConnectivityState { + if s == nil { + return &channelzpb.ChannelConnectivityState{State: channelzpb.ChannelConnectivityState_UNKNOWN} + } + switch *s { + case connectivity.Idle: + return &channelzpb.ChannelConnectivityState{State: channelzpb.ChannelConnectivityState_IDLE} + case connectivity.Connecting: + return &channelzpb.ChannelConnectivityState{State: channelzpb.ChannelConnectivityState_CONNECTING} + case connectivity.Ready: + return &channelzpb.ChannelConnectivityState{State: channelzpb.ChannelConnectivityState_READY} + case connectivity.TransientFailure: + return &channelzpb.ChannelConnectivityState{State: channelzpb.ChannelConnectivityState_TRANSIENT_FAILURE} + case connectivity.Shutdown: + return &channelzpb.ChannelConnectivityState{State: channelzpb.ChannelConnectivityState_SHUTDOWN} + default: + return &channelzpb.ChannelConnectivityState{State: channelzpb.ChannelConnectivityState_UNKNOWN} + } +} + +func channelTraceToProto(ct *channelz.ChannelTrace) *channelzpb.ChannelTrace { + pbt := &channelzpb.ChannelTrace{} + if ct == nil { + return pbt + } + pbt.NumEventsLogged = ct.EventNum + if ts := timestamppb.New(ct.CreationTime); ts.IsValid() { + pbt.CreationTimestamp = ts + } + events := make([]*channelzpb.ChannelTraceEvent, 0, len(ct.Events)) + for _, e := range ct.Events { + cte := &channelzpb.ChannelTraceEvent{ + Description: e.Desc, + Severity: channelzpb.ChannelTraceEvent_Severity(e.Severity), + } + if ts := timestamppb.New(e.Timestamp); ts.IsValid() { + cte.Timestamp = ts + } + if e.RefID != 0 { + switch e.RefType { + case channelz.RefChannel: + cte.ChildRef = &channelzpb.ChannelTraceEvent_ChannelRef{ChannelRef: &channelzpb.ChannelRef{ChannelId: e.RefID, Name: e.RefName}} + case channelz.RefSubChannel: + cte.ChildRef = &channelzpb.ChannelTraceEvent_SubchannelRef{SubchannelRef: &channelzpb.SubchannelRef{SubchannelId: e.RefID, Name: e.RefName}} + } + } + events = append(events, cte) + } + pbt.Events = events + return pbt +} + +func channelToProto(cm *channelz.Channel) *channelzpb.Channel { + c := &channelzpb.Channel{} + c.Ref = &channelzpb.ChannelRef{ChannelId: cm.ID, Name: cm.RefName} + + c.Data = &channelzpb.ChannelData{ + State: connectivityStateToProto(cm.ChannelMetrics.State.Load()), + Target: strFromPointer(cm.ChannelMetrics.Target.Load()), + CallsStarted: cm.ChannelMetrics.CallsStarted.Load(), + CallsSucceeded: cm.ChannelMetrics.CallsSucceeded.Load(), + CallsFailed: cm.ChannelMetrics.CallsFailed.Load(), + } + if ts := timestamppb.New(time.Unix(0, cm.ChannelMetrics.LastCallStartedTimestamp.Load())); ts.IsValid() { + c.Data.LastCallStartedTimestamp = ts + } + ncs := cm.NestedChans() + nestedChans := make([]*channelzpb.ChannelRef, 0, len(ncs)) + for id, ref := range ncs { + nestedChans = append(nestedChans, &channelzpb.ChannelRef{ChannelId: id, Name: ref}) + } + c.ChannelRef = nestedChans + + scs := cm.SubChans() + subChans := make([]*channelzpb.SubchannelRef, 0, len(scs)) + for id, ref := range scs { + subChans = append(subChans, &channelzpb.SubchannelRef{SubchannelId: id, Name: ref}) + } + c.SubchannelRef = subChans + + c.Data.Trace = channelTraceToProto(cm.Trace()) + return c +} + +// GetTopChannels returns the protobuf representation of the channels starting +// at startID (max of len), and returns end=true if no top channels exist with +// higher IDs. +func GetTopChannels(startID int64, len int) (channels []*channelzpb.Channel, end bool) { + chans, end := channelz.GetTopChannels(startID, len) + for _, ch := range chans { + channels = append(channels, channelToProto(ch)) + } + return channels, end +} + +// GetChannel returns the protobuf representation of the channel with the given +// ID. +func GetChannel(id int64) (*channelzpb.Channel, error) { + ch := channelz.GetChannel(id) + if ch == nil { + return nil, status.Errorf(codes.NotFound, "requested channel %d not found", id) + } + return channelToProto(ch), nil +} diff --git a/channelz/internal/protoconv/server.go b/channelz/internal/protoconv/server.go new file mode 100644 index 0000000..af9ff36 --- /dev/null +++ b/channelz/internal/protoconv/server.go @@ -0,0 +1,73 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package protoconv + +import ( + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" + + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" +) + +func serverToProto(sm *channelz.Server) *channelzpb.Server { + s := &channelzpb.Server{} + s.Ref = &channelzpb.ServerRef{ServerId: sm.ID, Name: sm.RefName} + + s.Data = &channelzpb.ServerData{ + CallsStarted: sm.ServerMetrics.CallsStarted.Load(), + CallsSucceeded: sm.ServerMetrics.CallsSucceeded.Load(), + CallsFailed: sm.ServerMetrics.CallsFailed.Load(), + } + + if ts := timestamppb.New(time.Unix(0, sm.ServerMetrics.LastCallStartedTimestamp.Load())); ts.IsValid() { + s.Data.LastCallStartedTimestamp = ts + } + lss := sm.ListenSockets() + sockets := make([]*channelzpb.SocketRef, 0, len(lss)) + for id, ref := range lss { + sockets = append(sockets, &channelzpb.SocketRef{SocketId: id, Name: ref}) + } + s.ListenSocket = sockets + return s +} + +// GetServers returns the protobuf representation of the servers starting at +// startID (max of len), and returns end=true if no servers exist with higher +// IDs. +func GetServers(startID int64, len int) (servers []*channelzpb.Server, end bool) { + srvs, end := channelz.GetServers(startID, len) + for _, srv := range srvs { + servers = append(servers, serverToProto(srv)) + } + return servers, end +} + +// GetServer returns the protobuf representation of the server with the given +// ID. +func GetServer(id int64) (*channelzpb.Server, error) { + srv := channelz.GetServer(id) + if srv == nil { + return nil, status.Errorf(codes.NotFound, "requested server %d not found", id) + } + return serverToProto(srv), nil +} diff --git a/channelz/internal/protoconv/socket.go b/channelz/internal/protoconv/socket.go new file mode 100644 index 0000000..0998f0a --- /dev/null +++ b/channelz/internal/protoconv/socket.go @@ -0,0 +1,136 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package protoconv + +import ( + "net" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" + + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" +) + +func securityToProto(se credentials.ChannelzSecurityValue) *channelzpb.Security { + switch v := se.(type) { + case *credentials.TLSChannelzSecurityValue: + return &channelzpb.Security{Model: &channelzpb.Security_Tls_{Tls: &channelzpb.Security_Tls{ + CipherSuite: &channelzpb.Security_Tls_StandardName{StandardName: v.StandardName}, + LocalCertificate: v.LocalCertificate, + RemoteCertificate: v.RemoteCertificate, + }}} + case *credentials.OtherChannelzSecurityValue: + otherSecurity := &channelzpb.Security_OtherSecurity{ + Name: v.Name, + } + if anyval, err := anypb.New(v.Value); err == nil { + otherSecurity.Value = anyval + } + return &channelzpb.Security{Model: &channelzpb.Security_Other{Other: otherSecurity}} + } + return nil +} + +func addrToProto(a net.Addr) *channelzpb.Address { + if a == nil { + return nil + } + switch a.Network() { + case "udp": + // TODO: Address_OtherAddress{}. Need proto def for Value. + case "ip": + // Note zone info is discarded through the conversion. + return &channelzpb.Address{Address: &channelzpb.Address_TcpipAddress{TcpipAddress: &channelzpb.Address_TcpIpAddress{IpAddress: a.(*net.IPAddr).IP}}} + case "ip+net": + // Note mask info is discarded through the conversion. + return &channelzpb.Address{Address: &channelzpb.Address_TcpipAddress{TcpipAddress: &channelzpb.Address_TcpIpAddress{IpAddress: a.(*net.IPNet).IP}}} + case "tcp": + // Note zone info is discarded through the conversion. + return &channelzpb.Address{Address: &channelzpb.Address_TcpipAddress{TcpipAddress: &channelzpb.Address_TcpIpAddress{IpAddress: a.(*net.TCPAddr).IP, Port: int32(a.(*net.TCPAddr).Port)}}} + case "unix", "unixgram", "unixpacket": + return &channelzpb.Address{Address: &channelzpb.Address_UdsAddress_{UdsAddress: &channelzpb.Address_UdsAddress{Filename: a.String()}}} + default: + } + return &channelzpb.Address{} +} + +func socketToProto(skt *channelz.Socket) *channelzpb.Socket { + s := &channelzpb.Socket{} + s.Ref = &channelzpb.SocketRef{SocketId: skt.ID, Name: skt.RefName} + + s.Data = &channelzpb.SocketData{ + StreamsStarted: skt.SocketMetrics.StreamsStarted.Load(), + StreamsSucceeded: skt.SocketMetrics.StreamsSucceeded.Load(), + StreamsFailed: skt.SocketMetrics.StreamsFailed.Load(), + MessagesSent: skt.SocketMetrics.MessagesSent.Load(), + MessagesReceived: skt.SocketMetrics.MessagesReceived.Load(), + KeepAlivesSent: skt.SocketMetrics.KeepAlivesSent.Load(), + } + if ts := timestamppb.New(time.Unix(0, skt.SocketMetrics.LastLocalStreamCreatedTimestamp.Load())); ts.IsValid() { + s.Data.LastLocalStreamCreatedTimestamp = ts + } + if ts := timestamppb.New(time.Unix(0, skt.SocketMetrics.LastRemoteStreamCreatedTimestamp.Load())); ts.IsValid() { + s.Data.LastRemoteStreamCreatedTimestamp = ts + } + if ts := timestamppb.New(time.Unix(0, skt.SocketMetrics.LastMessageSentTimestamp.Load())); ts.IsValid() { + s.Data.LastMessageSentTimestamp = ts + } + if ts := timestamppb.New(time.Unix(0, skt.SocketMetrics.LastMessageReceivedTimestamp.Load())); ts.IsValid() { + s.Data.LastMessageReceivedTimestamp = ts + } + if skt.EphemeralMetrics != nil { + e := skt.EphemeralMetrics() + s.Data.LocalFlowControlWindow = wrapperspb.Int64(e.LocalFlowControlWindow) + s.Data.RemoteFlowControlWindow = wrapperspb.Int64(e.RemoteFlowControlWindow) + } + + s.Data.Option = sockoptToProto(skt.SocketOptions) + s.Security = securityToProto(skt.Security) + s.Local = addrToProto(skt.LocalAddr) + s.Remote = addrToProto(skt.RemoteAddr) + s.RemoteName = skt.RemoteName + return s +} + +// GetServerSockets returns the protobuf representation of the server (listen) +// sockets starting at startID (max of len), and returns end=true if no server +// sockets exist with higher IDs. +func GetServerSockets(serverID, startID int64, len int) (sockets []*channelzpb.SocketRef, end bool) { + skts, end := channelz.GetServerSockets(serverID, startID, len) + for _, m := range skts { + sockets = append(sockets, &channelzpb.SocketRef{SocketId: m.ID, Name: m.RefName}) + } + return sockets, end +} + +// GetSocket returns the protobuf representation of the socket with the given +// ID. +func GetSocket(id int64) (*channelzpb.Socket, error) { + skt := channelz.GetSocket(id) + if skt == nil { + return nil, status.Errorf(codes.NotFound, "requested socket %d not found", id) + } + return socketToProto(skt), nil +} diff --git a/channelz/internal/protoconv/sockopt_linux.go b/channelz/internal/protoconv/sockopt_linux.go new file mode 100644 index 0000000..5b62773 --- /dev/null +++ b/channelz/internal/protoconv/sockopt_linux.go @@ -0,0 +1,125 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package protoconv + +import ( + "time" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/channelz" + + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/durationpb" +) + +var logger = grpclog.Component("channelz") + +func convertToPbDuration(sec int64, usec int64) *durationpb.Duration { + return durationpb.New(time.Duration(sec*1e9 + usec*1e3)) +} + +func sockoptToProto(skopts *channelz.SocketOptionData) []*channelzpb.SocketOption { + if skopts == nil { + return nil + } + var opts []*channelzpb.SocketOption + if skopts.Linger != nil { + additional, err := anypb.New(&channelzpb.SocketOptionLinger{ + Active: skopts.Linger.Onoff != 0, + Duration: convertToPbDuration(int64(skopts.Linger.Linger), 0), + }) + if err == nil { + opts = append(opts, &channelzpb.SocketOption{ + Name: "SO_LINGER", + Additional: additional, + }) + } else { + logger.Warningf("Failed to marshal socket options linger %+v: %v", skopts.Linger, err) + } + } + if skopts.RecvTimeout != nil { + additional, err := anypb.New(&channelzpb.SocketOptionTimeout{ + Duration: convertToPbDuration(int64(skopts.RecvTimeout.Sec), int64(skopts.RecvTimeout.Usec)), + }) + if err == nil { + opts = append(opts, &channelzpb.SocketOption{ + Name: "SO_RCVTIMEO", + Additional: additional, + }) + } else { + logger.Warningf("Failed to marshal socket options receive timeout %+v: %v", skopts.RecvTimeout, err) + } + } + if skopts.SendTimeout != nil { + additional, err := anypb.New(&channelzpb.SocketOptionTimeout{ + Duration: convertToPbDuration(int64(skopts.SendTimeout.Sec), int64(skopts.SendTimeout.Usec)), + }) + if err == nil { + opts = append(opts, &channelzpb.SocketOption{ + Name: "SO_SNDTIMEO", + Additional: additional, + }) + } else { + logger.Warningf("Failed to marshal socket options send timeout %+v: %v", skopts.SendTimeout, err) + } + } + if skopts.TCPInfo != nil { + additional, err := anypb.New(&channelzpb.SocketOptionTcpInfo{ + TcpiState: uint32(skopts.TCPInfo.State), + TcpiCaState: uint32(skopts.TCPInfo.Ca_state), + TcpiRetransmits: uint32(skopts.TCPInfo.Retransmits), + TcpiProbes: uint32(skopts.TCPInfo.Probes), + TcpiBackoff: uint32(skopts.TCPInfo.Backoff), + TcpiOptions: uint32(skopts.TCPInfo.Options), + // https://golang.org/pkg/syscall/#TCPInfo + // TCPInfo struct does not contain info about TcpiSndWscale and TcpiRcvWscale. + TcpiRto: skopts.TCPInfo.Rto, + TcpiAto: skopts.TCPInfo.Ato, + TcpiSndMss: skopts.TCPInfo.Snd_mss, + TcpiRcvMss: skopts.TCPInfo.Rcv_mss, + TcpiUnacked: skopts.TCPInfo.Unacked, + TcpiSacked: skopts.TCPInfo.Sacked, + TcpiLost: skopts.TCPInfo.Lost, + TcpiRetrans: skopts.TCPInfo.Retrans, + TcpiFackets: skopts.TCPInfo.Fackets, + TcpiLastDataSent: skopts.TCPInfo.Last_data_sent, + TcpiLastAckSent: skopts.TCPInfo.Last_ack_sent, + TcpiLastDataRecv: skopts.TCPInfo.Last_data_recv, + TcpiLastAckRecv: skopts.TCPInfo.Last_ack_recv, + TcpiPmtu: skopts.TCPInfo.Pmtu, + TcpiRcvSsthresh: skopts.TCPInfo.Rcv_ssthresh, + TcpiRtt: skopts.TCPInfo.Rtt, + TcpiRttvar: skopts.TCPInfo.Rttvar, + TcpiSndSsthresh: skopts.TCPInfo.Snd_ssthresh, + TcpiSndCwnd: skopts.TCPInfo.Snd_cwnd, + TcpiAdvmss: skopts.TCPInfo.Advmss, + TcpiReordering: skopts.TCPInfo.Reordering, + }) + if err == nil { + opts = append(opts, &channelzpb.SocketOption{ + Name: "TCP_INFO", + Additional: additional, + }) + } else { + logger.Warningf("Failed to marshal socket options TCP info %+v: %v", skopts.TCPInfo, err) + } + } + return opts +} diff --git a/channelz/internal/protoconv/sockopt_nonlinux.go b/channelz/internal/protoconv/sockopt_nonlinux.go new file mode 100644 index 0000000..cf5d4ee --- /dev/null +++ b/channelz/internal/protoconv/sockopt_nonlinux.go @@ -0,0 +1,31 @@ +//go:build !linux +// +build !linux + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package protoconv + +import ( + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" + "google.golang.org/grpc/internal/channelz" +) + +func sockoptToProto(_ *channelz.SocketOptionData) []*channelzpb.SocketOption { + return nil +} diff --git a/channelz/internal/protoconv/subchannel.go b/channelz/internal/protoconv/subchannel.go new file mode 100644 index 0000000..6b78105 --- /dev/null +++ b/channelz/internal/protoconv/subchannel.go @@ -0,0 +1,65 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package protoconv + +import ( + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" + + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" +) + +func subChannelToProto(cm *channelz.SubChannel) *channelzpb.Subchannel { + sc := &channelzpb.Subchannel{} + sc.Ref = &channelzpb.SubchannelRef{SubchannelId: cm.ID, Name: cm.RefName} + + sc.Data = &channelzpb.ChannelData{ + State: connectivityStateToProto(cm.ChannelMetrics.State.Load()), + Target: strFromPointer(cm.ChannelMetrics.Target.Load()), + CallsStarted: cm.ChannelMetrics.CallsStarted.Load(), + CallsSucceeded: cm.ChannelMetrics.CallsSucceeded.Load(), + CallsFailed: cm.ChannelMetrics.CallsFailed.Load(), + } + if ts := timestamppb.New(time.Unix(0, cm.ChannelMetrics.LastCallStartedTimestamp.Load())); ts.IsValid() { + sc.Data.LastCallStartedTimestamp = ts + } + + skts := cm.Sockets() + sockets := make([]*channelzpb.SocketRef, 0, len(skts)) + for id, ref := range skts { + sockets = append(sockets, &channelzpb.SocketRef{SocketId: id, Name: ref}) + } + sc.SocketRef = sockets + sc.Data.Trace = channelTraceToProto(cm.Trace()) + return sc +} + +// GetSubChannel returns the protobuf representation of the subchannel with the +// given ID. +func GetSubChannel(id int64) (*channelzpb.Subchannel, error) { + subChan := channelz.GetSubChannel(id) + if subChan == nil { + return nil, status.Errorf(codes.NotFound, "requested sub channel %d not found", id) + } + return subChannelToProto(subChan), nil +} diff --git a/channelz/internal/protoconv/util.go b/channelz/internal/protoconv/util.go new file mode 100644 index 0000000..a4c14ec --- /dev/null +++ b/channelz/internal/protoconv/util.go @@ -0,0 +1,28 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package protoconv supports converting between the internal channelz +// implementation and the protobuf representation of all the entities. +package protoconv + +func strFromPointer(s *string) string { + if s == nil { + return "" + } + return *s +} diff --git a/channelz/service/service.go b/channelz/service/service.go new file mode 100644 index 0000000..3b1f603 --- /dev/null +++ b/channelz/service/service.go @@ -0,0 +1,102 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package service provides an implementation for channelz service server. +package service + +import ( + "context" + + channelzgrpc "google.golang.org/grpc/channelz/grpc_channelz_v1" + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" + + "google.golang.org/grpc" + "google.golang.org/grpc/channelz/internal/protoconv" + "google.golang.org/grpc/internal/channelz" +) + +func init() { + channelz.TurnOn() +} + +// RegisterChannelzServiceToServer registers the channelz service to the given server. +// +// Note: it is preferred to use the admin API +// (https://pkg.go.dev/google.golang.org/grpc/admin#Register) instead to +// register Channelz and other administrative services. +func RegisterChannelzServiceToServer(s grpc.ServiceRegistrar) { + channelzgrpc.RegisterChannelzServer(s, newCZServer()) +} + +func newCZServer() channelzgrpc.ChannelzServer { + return &serverImpl{} +} + +type serverImpl struct { + channelzgrpc.UnimplementedChannelzServer +} + +func (s *serverImpl) GetChannel(_ context.Context, req *channelzpb.GetChannelRequest) (*channelzpb.GetChannelResponse, error) { + ch, err := protoconv.GetChannel(req.GetChannelId()) + if err != nil { + return nil, err + } + return &channelzpb.GetChannelResponse{Channel: ch}, nil +} + +func (s *serverImpl) GetTopChannels(_ context.Context, req *channelzpb.GetTopChannelsRequest) (*channelzpb.GetTopChannelsResponse, error) { + resp := &channelzpb.GetTopChannelsResponse{} + resp.Channel, resp.End = protoconv.GetTopChannels(req.GetStartChannelId(), int(req.GetMaxResults())) + return resp, nil +} + +func (s *serverImpl) GetServer(_ context.Context, req *channelzpb.GetServerRequest) (*channelzpb.GetServerResponse, error) { + srv, err := protoconv.GetServer(req.GetServerId()) + if err != nil { + return nil, err + } + return &channelzpb.GetServerResponse{Server: srv}, nil +} + +func (s *serverImpl) GetServers(_ context.Context, req *channelzpb.GetServersRequest) (*channelzpb.GetServersResponse, error) { + resp := &channelzpb.GetServersResponse{} + resp.Server, resp.End = protoconv.GetServers(req.GetStartServerId(), int(req.GetMaxResults())) + return resp, nil +} + +func (s *serverImpl) GetSubchannel(_ context.Context, req *channelzpb.GetSubchannelRequest) (*channelzpb.GetSubchannelResponse, error) { + subChan, err := protoconv.GetSubChannel(req.GetSubchannelId()) + if err != nil { + return nil, err + } + return &channelzpb.GetSubchannelResponse{Subchannel: subChan}, nil +} + +func (s *serverImpl) GetServerSockets(_ context.Context, req *channelzpb.GetServerSocketsRequest) (*channelzpb.GetServerSocketsResponse, error) { + resp := &channelzpb.GetServerSocketsResponse{} + resp.SocketRef, resp.End = protoconv.GetServerSockets(req.GetServerId(), req.GetStartSocketId(), int(req.GetMaxResults())) + return resp, nil +} + +func (s *serverImpl) GetSocket(_ context.Context, req *channelzpb.GetSocketRequest) (*channelzpb.GetSocketResponse, error) { + socket, err := protoconv.GetSocket(req.GetSocketId()) + if err != nil { + return nil, err + } + return &channelzpb.GetSocketResponse{Socket: socket}, nil +} diff --git a/channelz/service/service_sktopt_test.go b/channelz/service/service_sktopt_test.go new file mode 100644 index 0000000..8c6473e --- /dev/null +++ b/channelz/service/service_sktopt_test.go @@ -0,0 +1,98 @@ +//go:build linux && (386 || amd64) +// +build linux +// +build 386 amd64 + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// SocketOptions is only supported on linux system. The functions defined in +// this file are to parse the socket option field and the test is specifically +// to verify the behavior of socket option parsing. + +package service + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "golang.org/x/sys/unix" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/durationpb" + + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" +) + +func (s) TestGetSocketOptions(t *testing.T) { + ss := &channelz.Socket{ + SocketOptions: &channelz.SocketOptionData{ + Linger: &unix.Linger{Onoff: 1, Linger: 2}, + RecvTimeout: &unix.Timeval{Sec: 10, Usec: 1}, + SendTimeout: &unix.Timeval{}, + TCPInfo: &unix.TCPInfo{State: 1}, + }, + } + svr := newCZServer() + czServer := channelz.RegisterServer("test svr") + defer channelz.RemoveEntry(czServer.ID) + id := channelz.RegisterSocket(&channelz.Socket{SocketType: channelz.SocketTypeNormal, RefName: "0", Parent: czServer, SocketOptions: ss.SocketOptions}) + defer channelz.RemoveEntry(id.ID) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, _ := svr.GetSocket(ctx, &channelzpb.GetSocketRequest{SocketId: id.ID}) + { + got, want := resp.GetSocket().GetRef(), &channelzpb.SocketRef{SocketId: id.ID, Name: "0"} + if diff := cmp.Diff(got, want, protocmp.Transform()); diff != "" { + t.Fatal("resp.GetSocket() ref (-got +want): ", diff) + } + } + { + got := resp.GetSocket().GetData().GetOption() + want := []*channelzpb.SocketOption{{ + Name: "SO_LINGER", + Additional: testutils.MarshalAny( + t, + &channelzpb.SocketOptionLinger{Active: true, Duration: durationpb.New(2 * time.Second)}, + ), + }, { + Name: "SO_RCVTIMEO", + Additional: testutils.MarshalAny( + t, + &channelzpb.SocketOptionTimeout{Duration: durationpb.New(10*time.Second + time.Microsecond)}, + ), + }, { + Name: "SO_SNDTIMEO", + Additional: testutils.MarshalAny( + t, + &channelzpb.SocketOptionTimeout{Duration: durationpb.New(0)}, + ), + }, { + Name: "TCP_INFO", + Additional: testutils.MarshalAny( + t, + &channelzpb.SocketOptionTcpInfo{TcpiState: 1}, + ), + }} + if diff := cmp.Diff(got, want, protocmp.Transform()); diff != "" { + t.Fatal("resp.GetSocket() options (-got +want): ", diff) + } + } +} diff --git a/channelz/service/service_test.go b/channelz/service/service_test.go new file mode 100644 index 0000000..111e5d1 --- /dev/null +++ b/channelz/service/service_test.go @@ -0,0 +1,803 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package service + +import ( + "context" + "fmt" + "net" + "net/netip" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/reflect/protodesc" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/descriptorpb" + "google.golang.org/protobuf/types/dynamicpb" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/timestamppb" + + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" +) + +func init() { + channelz.TurnOn() +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const defaultTestTimeout = 10 * time.Second + +func channelProtoToStruct(c *channelzpb.Channel) (*channelz.ChannelMetrics, error) { + cm := &channelz.ChannelMetrics{} + pdata := c.GetData() + var s connectivity.State + switch pdata.GetState().GetState() { + case channelzpb.ChannelConnectivityState_UNKNOWN: + // TODO: what should we set here? + case channelzpb.ChannelConnectivityState_IDLE: + s = connectivity.Idle + case channelzpb.ChannelConnectivityState_CONNECTING: + s = connectivity.Connecting + case channelzpb.ChannelConnectivityState_READY: + s = connectivity.Ready + case channelzpb.ChannelConnectivityState_TRANSIENT_FAILURE: + s = connectivity.TransientFailure + case channelzpb.ChannelConnectivityState_SHUTDOWN: + s = connectivity.Shutdown + } + cm.State.Store(&s) + tgt := pdata.GetTarget() + cm.Target.Store(&tgt) + cm.CallsStarted.Store(pdata.CallsStarted) + cm.CallsSucceeded.Store(pdata.CallsSucceeded) + cm.CallsFailed.Store(pdata.CallsFailed) + if err := pdata.GetLastCallStartedTimestamp().CheckValid(); err != nil { + return nil, err + } + cm.LastCallStartedTimestamp.Store(int64(pdata.GetLastCallStartedTimestamp().AsTime().UnixNano())) + return cm, nil +} + +func convertSocketRefSliceToMap(sktRefs []*channelzpb.SocketRef) map[int64]string { + m := make(map[int64]string) + for _, sr := range sktRefs { + m[sr.SocketId] = sr.Name + } + return m +} + +func (s) TestGetTopChannels(t *testing.T) { + tcs := []*channelz.ChannelMetrics{ + channelz.NewChannelMetricForTesting( + connectivity.Connecting, + "test.channelz:1234", + 6, + 2, + 3, + time.Now().UTC().UnixNano(), + ), + channelz.NewChannelMetricForTesting( + connectivity.Connecting, + "test.channelz:1234", + 1, + 2, + 3, + time.Now().UTC().UnixNano(), + ), + channelz.NewChannelMetricForTesting( + connectivity.Shutdown, + "test.channelz:8888", + 0, + 0, + 0, + 0, + ), + } + + for _, c := range tcs { + cz := channelz.RegisterChannel(nil, "test channel") + cz.ChannelMetrics.CopyFrom(c) + defer channelz.RemoveEntry(cz.ID) + } + s := newCZServer() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, _ := s.GetTopChannels(ctx, &channelzpb.GetTopChannelsRequest{StartChannelId: 0}) + if !resp.GetEnd() { + t.Fatalf("resp.GetEnd() want true, got %v", resp.GetEnd()) + } + for i, c := range resp.GetChannel() { + channel, err := channelProtoToStruct(c) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tcs[i], channel, protocmp.Transform()); diff != "" { + t.Fatalf("unexpected channel, diff (-want +got):\n%s", diff) + } + } + for i := 0; i < 50; i++ { + cz := channelz.RegisterChannel(nil, "") + defer channelz.RemoveEntry(cz.ID) + } + resp, _ = s.GetTopChannels(ctx, &channelzpb.GetTopChannelsRequest{StartChannelId: 0}) + if resp.GetEnd() { + t.Fatalf("resp.GetEnd() want false, got %v", resp.GetEnd()) + } +} + +func (s) TestGetServers(t *testing.T) { + ss := []*channelz.ServerMetrics{ + channelz.NewServerMetricsForTesting( + 6, + 2, + 3, + time.Now().UnixNano(), + ), + channelz.NewServerMetricsForTesting( + 1, + 2, + 3, + time.Now().UnixNano(), + ), + channelz.NewServerMetricsForTesting( + 1, + 0, + 0, + time.Now().UnixNano(), + ), + } + + firstID := int64(0) + for i, s := range ss { + svr := channelz.RegisterServer("") + if i == 0 { + firstID = svr.ID + } + svr.ServerMetrics.CopyFrom(s) + defer channelz.RemoveEntry(svr.ID) + } + svr := newCZServer() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, _ := svr.GetServers(ctx, &channelzpb.GetServersRequest{StartServerId: 0}) + if !resp.GetEnd() { + t.Fatalf("resp.GetEnd() want true, got %v", resp.GetEnd()) + } + serversWant := []*channelzpb.Server{ + { + Ref: &channelzpb.ServerRef{ServerId: firstID, Name: ""}, + Data: &channelzpb.ServerData{ + CallsStarted: 6, + CallsSucceeded: 2, + CallsFailed: 3, + LastCallStartedTimestamp: timestamppb.New(time.Unix(0, ss[0].LastCallStartedTimestamp.Load())), + }, + }, + { + Ref: &channelzpb.ServerRef{ServerId: firstID + 1, Name: ""}, + Data: &channelzpb.ServerData{ + CallsStarted: 1, + CallsSucceeded: 2, + CallsFailed: 3, + LastCallStartedTimestamp: timestamppb.New(time.Unix(0, ss[1].LastCallStartedTimestamp.Load())), + }, + }, + { + Ref: &channelzpb.ServerRef{ServerId: firstID + 2, Name: ""}, + Data: &channelzpb.ServerData{ + CallsStarted: 1, + CallsSucceeded: 0, + CallsFailed: 0, + LastCallStartedTimestamp: timestamppb.New(time.Unix(0, ss[2].LastCallStartedTimestamp.Load())), + }, + }, + } + if diff := cmp.Diff(serversWant, resp.GetServer(), protocmp.Transform()); diff != "" { + t.Fatalf("unexpected server, diff (-want +got):\n%s", diff) + } + for i := 0; i < 50; i++ { + id := channelz.RegisterServer("").ID + defer channelz.RemoveEntry(id) + } + resp, _ = svr.GetServers(ctx, &channelzpb.GetServersRequest{StartServerId: 0}) + if resp.GetEnd() { + t.Fatalf("resp.GetEnd() want false, got %v", resp.GetEnd()) + } +} + +func (s) TestGetServerSockets(t *testing.T) { + svrID := channelz.RegisterServer("") + defer channelz.RemoveEntry(svrID.ID) + refNames := []string{"listen socket 1", "normal socket 1", "normal socket 2"} + ids := make([]int64, 3) + ids[0] = channelz.RegisterSocket(&channelz.Socket{SocketType: channelz.SocketTypeListen, Parent: svrID, RefName: refNames[0]}).ID + ids[1] = channelz.RegisterSocket(&channelz.Socket{SocketType: channelz.SocketTypeNormal, Parent: svrID, RefName: refNames[1]}).ID + ids[2] = channelz.RegisterSocket(&channelz.Socket{SocketType: channelz.SocketTypeNormal, Parent: svrID, RefName: refNames[2]}).ID + for _, id := range ids { + defer channelz.RemoveEntry(id) + } + svr := newCZServer() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, _ := svr.GetServerSockets(ctx, &channelzpb.GetServerSocketsRequest{ServerId: svrID.ID, StartSocketId: 0}) + if !resp.GetEnd() { + t.Fatalf("resp.GetEnd() want: true, got: %v", resp.GetEnd()) + } + // GetServerSockets only return normal sockets. + want := map[int64]string{ + ids[1]: refNames[1], + ids[2]: refNames[2], + } + if got := convertSocketRefSliceToMap(resp.GetSocketRef()); !cmp.Equal(got, want) { + t.Fatalf("GetServerSockets want: %#v, got: %#v (resp=%v)", want, got, prototext.Format(resp)) + } + + for i := 0; i < 50; i++ { + id := channelz.RegisterSocket(&channelz.Socket{SocketType: channelz.SocketTypeNormal, Parent: svrID}) + defer channelz.RemoveEntry(id.ID) + } + resp, _ = svr.GetServerSockets(ctx, &channelzpb.GetServerSocketsRequest{ServerId: svrID.ID, StartSocketId: 0}) + if resp.GetEnd() { + t.Fatalf("resp.GetEnd() want false, got %v", resp.GetEnd()) + } +} + +// This test makes a GetServerSockets with a non-zero start ID, and expect only +// sockets with ID >= the given start ID. +func (s) TestGetServerSocketsNonZeroStartID(t *testing.T) { + svrID := channelz.RegisterServer("test server") + defer channelz.RemoveEntry(svrID.ID) + refNames := []string{"listen socket 1", "normal socket 1", "normal socket 2"} + ids := make([]int64, 3) + ids[0] = channelz.RegisterSocket(&channelz.Socket{SocketType: channelz.SocketTypeListen, Parent: svrID, RefName: refNames[0]}).ID + ids[1] = channelz.RegisterSocket(&channelz.Socket{SocketType: channelz.SocketTypeNormal, Parent: svrID, RefName: refNames[1]}).ID + ids[2] = channelz.RegisterSocket(&channelz.Socket{SocketType: channelz.SocketTypeNormal, Parent: svrID, RefName: refNames[2]}).ID + for _, id := range ids { + defer channelz.RemoveEntry(id) + } + svr := newCZServer() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Make GetServerSockets with startID = ids[1]+1, so socket-1 won't be + // included in the response. + resp, _ := svr.GetServerSockets(ctx, &channelzpb.GetServerSocketsRequest{ServerId: svrID.ID, StartSocketId: ids[1] + 1}) + if !resp.GetEnd() { + t.Fatalf("resp.GetEnd() want: true, got: %v", resp.GetEnd()) + } + // GetServerSockets only return normal socket-2, socket-1 should be + // filtered by start ID. + want := map[int64]string{ + ids[2]: refNames[2], + } + if !cmp.Equal(convertSocketRefSliceToMap(resp.GetSocketRef()), want) { + t.Fatalf("GetServerSockets want: %#v, got: %#v", want, resp.GetSocketRef()) + } +} + +var logger = grpclog.Component("channelz") + +func (s) TestGetChannel(t *testing.T) { + refNames := []string{"top channel 1", "nested channel 1", "sub channel 2", "nested channel 3"} + cids := make([]*channelz.Channel, 3) + cids[0] = channelz.RegisterChannel(nil, refNames[0]) + channelz.AddTraceEvent(logger, cids[0], 0, &channelz.TraceEvent{ + Desc: "Channel Created", + Severity: channelz.CtInfo, + }) + + cids[1] = channelz.RegisterChannel(cids[0], refNames[1]) + channelz.AddTraceEvent(logger, cids[1], 0, &channelz.TraceEvent{ + Desc: "Channel Created", + Severity: channelz.CtInfo, + Parent: &channelz.TraceEvent{ + Desc: fmt.Sprintf("Nested Channel(id:%d) created", cids[1].ID), + Severity: channelz.CtInfo, + }, + }) + + subChan := channelz.RegisterSubChannel(cids[0], refNames[2]) + channelz.AddTraceEvent(logger, subChan, 0, &channelz.TraceEvent{ + Desc: "SubChannel Created", + Severity: channelz.CtInfo, + Parent: &channelz.TraceEvent{ + Desc: fmt.Sprintf("SubChannel(id:%d) created", subChan.ID), + Severity: channelz.CtInfo, + }, + }) + defer channelz.RemoveEntry(subChan.ID) + + cids[2] = channelz.RegisterChannel(cids[1], refNames[3]) + channelz.AddTraceEvent(logger, cids[2], 0, &channelz.TraceEvent{ + Desc: "Channel Created", + Severity: channelz.CtInfo, + Parent: &channelz.TraceEvent{ + Desc: fmt.Sprintf("Nested Channel(id:%d) created", cids[2].ID), + Severity: channelz.CtInfo, + }, + }) + channelz.AddTraceEvent(logger, cids[0], 0, &channelz.TraceEvent{ + Desc: fmt.Sprintf("Channel Connectivity change to %v", connectivity.Ready), + Severity: channelz.CtInfo, + }) + channelz.AddTraceEvent(logger, cids[0], 0, &channelz.TraceEvent{ + Desc: "Resolver returns an empty address list", + Severity: channelz.CtWarning, + }) + + for _, id := range cids { + defer channelz.RemoveEntry(id.ID) + } + + svr := newCZServer() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, _ := svr.GetChannel(ctx, &channelzpb.GetChannelRequest{ChannelId: cids[0].ID}) + metrics := resp.GetChannel() + subChans := metrics.GetSubchannelRef() + if len(subChans) != 1 || subChans[0].GetName() != refNames[2] || subChans[0].GetSubchannelId() != subChan.ID { + t.Fatalf("metrics.GetSubChannelRef() want %#v, got %#v", []*channelzpb.SubchannelRef{{SubchannelId: subChan.ID, Name: refNames[2]}}, subChans) + } + nestedChans := metrics.GetChannelRef() + if len(nestedChans) != 1 || nestedChans[0].GetName() != refNames[1] || nestedChans[0].GetChannelId() != cids[1].ID { + t.Fatalf("metrics.GetChannelRef() want %#v, got %#v", []*channelzpb.ChannelRef{{ChannelId: cids[1].ID, Name: refNames[1]}}, nestedChans) + } + trace := metrics.GetData().GetTrace() + want := []struct { + desc string + severity channelzpb.ChannelTraceEvent_Severity + childID int64 + childRef string + }{ + {desc: "Channel Created", severity: channelzpb.ChannelTraceEvent_CT_INFO}, + {desc: fmt.Sprintf("Nested Channel(id:%d) created", cids[1].ID), severity: channelzpb.ChannelTraceEvent_CT_INFO, childID: cids[1].ID, childRef: refNames[1]}, + {desc: fmt.Sprintf("SubChannel(id:%d) created", subChan.ID), severity: channelzpb.ChannelTraceEvent_CT_INFO, childID: subChan.ID, childRef: refNames[2]}, + {desc: fmt.Sprintf("Channel Connectivity change to %v", connectivity.Ready), severity: channelzpb.ChannelTraceEvent_CT_INFO}, + {desc: "Resolver returns an empty address list", severity: channelzpb.ChannelTraceEvent_CT_WARNING}, + } + + for i, e := range trace.Events { + if !strings.Contains(e.GetDescription(), want[i].desc) { + t.Fatalf("trace: GetDescription want %#v, got %#v", want[i].desc, e.GetDescription()) + } + if e.GetSeverity() != want[i].severity { + t.Fatalf("trace: GetSeverity want %#v, got %#v", want[i].severity, e.GetSeverity()) + } + if want[i].childID == 0 && (e.GetChannelRef() != nil || e.GetSubchannelRef() != nil) { + t.Fatalf("trace: GetChannelRef() should return nil, as there is no reference") + } + if e.GetChannelRef().GetChannelId() != want[i].childID || e.GetChannelRef().GetName() != want[i].childRef { + if e.GetSubchannelRef().GetSubchannelId() != want[i].childID || e.GetSubchannelRef().GetName() != want[i].childRef { + t.Fatalf("trace: GetChannelRef/GetSubchannelRef want (child ID: %d, child name: %q), got %#v and %#v", want[i].childID, want[i].childRef, e.GetChannelRef(), e.GetSubchannelRef()) + } + } + } + resp, _ = svr.GetChannel(ctx, &channelzpb.GetChannelRequest{ChannelId: cids[1].ID}) + metrics = resp.GetChannel() + nestedChans = metrics.GetChannelRef() + if len(nestedChans) != 1 || nestedChans[0].GetName() != refNames[3] || nestedChans[0].GetChannelId() != cids[2].ID { + t.Fatalf("metrics.GetChannelRef() want %#v, got %#v", []*channelzpb.ChannelRef{{ChannelId: cids[2].ID, Name: refNames[3]}}, nestedChans) + } +} + +func (s) TestGetSubChannel(t *testing.T) { + var ( + subchanCreated = "SubChannel Created" + subchanConnectivityChange = fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.Ready) + subChanPickNewAddress = fmt.Sprintf("Subchannel picks a new address %q to connect", "0.0.0.0") + ) + + refNames := []string{"top channel 1", "sub channel 1", "socket 1", "socket 2"} + chann := channelz.RegisterChannel(nil, refNames[0]) + defer channelz.RemoveEntry(chann.ID) + channelz.AddTraceEvent(logger, chann, 0, &channelz.TraceEvent{ + Desc: "Channel Created", + Severity: channelz.CtInfo, + }) + subChan := channelz.RegisterSubChannel(chann, refNames[1]) + defer channelz.RemoveEntry(subChan.ID) + channelz.AddTraceEvent(logger, subChan, 0, &channelz.TraceEvent{ + Desc: subchanCreated, + Severity: channelz.CtInfo, + Parent: &channelz.TraceEvent{ + Desc: fmt.Sprintf("Nested Channel(id:%d) created", chann.ID), + Severity: channelz.CtInfo, + }, + }) + skt1 := channelz.RegisterSocket(&channelz.Socket{SocketType: channelz.SocketTypeNormal, Parent: subChan, RefName: refNames[2]}) + defer channelz.RemoveEntry(skt1.ID) + skt2 := channelz.RegisterSocket(&channelz.Socket{SocketType: channelz.SocketTypeNormal, Parent: subChan, RefName: refNames[3]}) + defer channelz.RemoveEntry(skt2.ID) + channelz.AddTraceEvent(logger, subChan, 0, &channelz.TraceEvent{ + Desc: subchanConnectivityChange, + Severity: channelz.CtInfo, + }) + channelz.AddTraceEvent(logger, subChan, 0, &channelz.TraceEvent{ + Desc: subChanPickNewAddress, + Severity: channelz.CtInfo, + }) + svr := newCZServer() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, _ := svr.GetSubchannel(ctx, &channelzpb.GetSubchannelRequest{SubchannelId: subChan.ID}) + metrics := resp.GetSubchannel() + want := map[int64]string{ + skt1.ID: refNames[2], + skt2.ID: refNames[3], + } + if !cmp.Equal(convertSocketRefSliceToMap(metrics.GetSocketRef()), want) { + t.Fatalf("metrics.GetSocketRef() want %#v: got: %#v", want, metrics.GetSocketRef()) + } + + trace := metrics.GetData().GetTrace() + wantTrace := []struct { + desc string + severity channelzpb.ChannelTraceEvent_Severity + childID int64 + childRef string + }{ + {desc: subchanCreated, severity: channelzpb.ChannelTraceEvent_CT_INFO}, + {desc: subchanConnectivityChange, severity: channelzpb.ChannelTraceEvent_CT_INFO}, + {desc: subChanPickNewAddress, severity: channelzpb.ChannelTraceEvent_CT_INFO}, + } + for i, e := range trace.Events { + if e.GetDescription() != wantTrace[i].desc { + t.Fatalf("trace: GetDescription want %#v, got %#v", wantTrace[i].desc, e.GetDescription()) + } + if e.GetSeverity() != wantTrace[i].severity { + t.Fatalf("trace: GetSeverity want %#v, got %#v", wantTrace[i].severity, e.GetSeverity()) + } + if wantTrace[i].childID == 0 && (e.GetChannelRef() != nil || e.GetSubchannelRef() != nil) { + t.Fatalf("trace: GetChannelRef() should return nil, as there is no reference") + } + if e.GetChannelRef().GetChannelId() != wantTrace[i].childID || e.GetChannelRef().GetName() != wantTrace[i].childRef { + if e.GetSubchannelRef().GetSubchannelId() != wantTrace[i].childID || e.GetSubchannelRef().GetName() != wantTrace[i].childRef { + t.Fatalf("trace: GetChannelRef/GetSubchannelRef want (child ID: %d, child name: %q), got %#v and %#v", wantTrace[i].childID, wantTrace[i].childRef, e.GetChannelRef(), e.GetSubchannelRef()) + } + } + } +} + +type czSocket struct { + streamsStarted int64 + streamsSucceeded int64 + streamsFailed int64 + messagesSent int64 + messagesReceived int64 + keepAlivesSent int64 + lastLocalStreamCreatedTimestamp time.Time + lastRemoteStreamCreatedTimestamp time.Time + lastMessageSentTimestamp time.Time + lastMessageReceivedTimestamp time.Time + localFlowControlWindow int64 + remoteFlowControlWindow int64 + + localAddr net.Addr + remoteAddr net.Addr + remoteName string + socketOptions *channelz.SocketOptionData + security credentials.ChannelzSecurityValue +} + +func newSocket(cs czSocket) *channelz.Socket { + if cs.lastLocalStreamCreatedTimestamp.IsZero() { + cs.lastLocalStreamCreatedTimestamp = time.Unix(0, 0) + } + if cs.lastRemoteStreamCreatedTimestamp.IsZero() { + cs.lastRemoteStreamCreatedTimestamp = time.Unix(0, 0) + } + if cs.lastMessageSentTimestamp.IsZero() { + cs.lastMessageSentTimestamp = time.Unix(0, 0) + } + if cs.lastMessageReceivedTimestamp.IsZero() { + cs.lastMessageReceivedTimestamp = time.Unix(0, 0) + } + + s := &channelz.Socket{ + LocalAddr: cs.localAddr, + RemoteAddr: cs.remoteAddr, + RemoteName: cs.remoteName, + SocketOptions: cs.socketOptions, + Security: cs.security, + } + s.SocketMetrics.StreamsStarted.Store(cs.streamsStarted) + s.SocketMetrics.StreamsSucceeded.Store(cs.streamsSucceeded) + s.SocketMetrics.StreamsFailed.Store(cs.streamsFailed) + s.SocketMetrics.MessagesSent.Store(cs.messagesSent) + s.SocketMetrics.MessagesReceived.Store(cs.messagesReceived) + s.SocketMetrics.KeepAlivesSent.Store(cs.keepAlivesSent) + s.SocketMetrics.LastLocalStreamCreatedTimestamp.Store(cs.lastLocalStreamCreatedTimestamp.UnixNano()) + s.SocketMetrics.LastRemoteStreamCreatedTimestamp.Store(cs.lastRemoteStreamCreatedTimestamp.UnixNano()) + s.SocketMetrics.LastMessageSentTimestamp.Store(cs.lastMessageSentTimestamp.UnixNano()) + s.SocketMetrics.LastMessageReceivedTimestamp.Store(cs.lastMessageReceivedTimestamp.UnixNano()) + s.EphemeralMetrics = func() *channelz.EphemeralSocketMetrics { + return &channelz.EphemeralSocketMetrics{ + LocalFlowControlWindow: cs.localFlowControlWindow, + RemoteFlowControlWindow: cs.remoteFlowControlWindow, + } + } + return s +} + +type OtherChannelzSecurityValue struct { + LocalCertificate []byte `protobuf:"bytes,1,opt,name=local_certificate,json=localCertificate,proto3" json:"local_certificate,omitempty"` + RemoteCertificate []byte `protobuf:"bytes,2,opt,name=remote_certificate,json=remoteCertificate,proto3" json:"remote_certificate,omitempty"` +} + +func (x *OtherChannelzSecurityValue) Reset() { + *x = OtherChannelzSecurityValue{} +} + +func (x *OtherChannelzSecurityValue) String() string { + return prototext.Format(x) +} + +func (*OtherChannelzSecurityValue) ProtoMessage() {} + +func (x OtherChannelzSecurityValue) ProtoReflect() protoreflect.Message { + const s = ` + name: "service_test.proto" + syntax: "proto3" + package: "grpc.credentials", + message_type: [{ + name: "OtherChannelzSecurityValue" + field: [ + {name:"local_certificate" number:1 type:TYPE_BYTES}, + {name:"remote_certificate" number:2 type:TYPE_BYTES} + ] + }] + ` + pb := new(descriptorpb.FileDescriptorProto) + if err := prototext.Unmarshal([]byte(s), pb); err != nil { + panic(err) + } + fd, err := protodesc.NewFile(pb, nil) + if err != nil { + panic(err) + } + md := fd.Messages().Get(0) + mt := dynamicpb.NewMessageType(md) + return mt.New() +} + +func (s) TestGetSocket(t *testing.T) { + ss := []*channelz.Socket{newSocket(czSocket{ + streamsStarted: 10, + streamsSucceeded: 2, + streamsFailed: 3, + messagesSent: 20, + messagesReceived: 10, + keepAlivesSent: 2, + lastLocalStreamCreatedTimestamp: time.Unix(0, 0), + lastRemoteStreamCreatedTimestamp: time.Unix(1, 0), + lastMessageSentTimestamp: time.Unix(2, 0), + lastMessageReceivedTimestamp: time.Unix(3, 0), + localFlowControlWindow: 65536, + remoteFlowControlWindow: 1024, + localAddr: &net.TCPAddr{IP: netip.MustParseAddr("1.0.0.1").AsSlice(), Port: 10001}, + remoteAddr: &net.TCPAddr{IP: netip.MustParseAddr("12.0.0.1").AsSlice(), Port: 10002}, + remoteName: "remote.remote", + }), newSocket(czSocket{ + streamsStarted: 10, + streamsSucceeded: 2, + streamsFailed: 3, + messagesSent: 20, + messagesReceived: 10, + keepAlivesSent: 2, + lastLocalStreamCreatedTimestamp: time.Unix(0, 0), + lastRemoteStreamCreatedTimestamp: time.Unix(5, 0), + lastMessageSentTimestamp: time.Unix(6, 0), + lastMessageReceivedTimestamp: time.Unix(7, 0), + localFlowControlWindow: 65536, + remoteFlowControlWindow: 1024, + localAddr: &net.UnixAddr{Name: "file.path", Net: "unix"}, + remoteAddr: &net.UnixAddr{Name: "another.path", Net: "unix"}, + remoteName: "remote.remote", + }), newSocket(czSocket{ + streamsStarted: 5, + streamsSucceeded: 2, + streamsFailed: 3, + messagesSent: 20, + messagesReceived: 10, + keepAlivesSent: 2, + lastLocalStreamCreatedTimestamp: time.Unix(10, 10), + lastRemoteStreamCreatedTimestamp: time.Unix(0, 0), + lastMessageSentTimestamp: time.Unix(0, 0), + lastMessageReceivedTimestamp: time.Unix(0, 0), + localFlowControlWindow: 65536, + remoteFlowControlWindow: 10240, + localAddr: &net.IPAddr{IP: netip.MustParseAddr("1.0.0.1").AsSlice()}, + remoteAddr: &net.IPAddr{IP: netip.MustParseAddr("9.0.0.1").AsSlice()}, + remoteName: "", + }), newSocket(czSocket{ + localAddr: &net.TCPAddr{IP: netip.MustParseAddr("127.0.0.1").AsSlice(), Port: 10001}, + }), newSocket(czSocket{ + security: &credentials.TLSChannelzSecurityValue{ + StandardName: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + RemoteCertificate: []byte{48, 130, 2, 156, 48, 130, 2, 5, 160}, + }, + }), newSocket(czSocket{ + security: &credentials.OtherChannelzSecurityValue{ + Name: "XXXX", + }, + }), newSocket(czSocket{ + security: &credentials.OtherChannelzSecurityValue{ + Name: "YYYY", + Value: OtherChannelzSecurityValue{ + LocalCertificate: []byte{1, 2, 3}, + RemoteCertificate: []byte{4, 5, 6}, + }, + }, + }), + } + otherSecVal, err := anypb.New(ss[6].Security.(*credentials.OtherChannelzSecurityValue).Value) + if err != nil { + t.Fatal("Error marshalling proto:", err) + } + + svr := newCZServer() + skts := make([]*channelz.Socket, len(ss)) + svrID := channelz.RegisterServer("") + defer channelz.RemoveEntry(svrID.ID) + for i, s := range ss { + s.Parent = svrID + s.RefName = strconv.Itoa(i) + skts[i] = channelz.RegisterSocket(s) + defer channelz.RemoveEntry(skts[i].ID) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + emptyData := `data: { + last_local_stream_created_timestamp: {seconds: 0 nanos: 0} + last_remote_stream_created_timestamp: {seconds: 0 nanos: 0} + last_message_sent_timestamp: {seconds: 0 nanos: 0} + last_message_received_timestamp: {seconds: 0 nanos: 0} + local_flow_control_window: { value: 0 } + remote_flow_control_window: { value: 0 } + }` + want := []string{` + ref: {socket_id: ` + fmt.Sprint(skts[0].ID) + ` name: "0" } + data: { + streams_started: 10 + streams_succeeded: 2 + streams_failed: 3 + messages_sent: 20 + messages_received: 10 + keep_alives_sent: 2 + last_local_stream_created_timestamp: {seconds: 0 nanos: 0} + last_remote_stream_created_timestamp: {seconds: 1 nanos: 0} + last_message_sent_timestamp: {seconds: 2 nanos: 0} + last_message_received_timestamp: {seconds: 3 nanos: 0} + local_flow_control_window: { value: 65536 } + remote_flow_control_window: { value: 1024 } + } + local: { tcpip_address: { ip_address: "` + addr(skts[0].LocalAddr) + `" port: 10001 } } + remote: { tcpip_address: { ip_address: "` + addr(skts[0].RemoteAddr) + `" port: 10002 } } + remote_name: "remote.remote"`, + ` + ref: {socket_id: ` + fmt.Sprint(skts[1].ID) + ` name: "1" } + data: { + streams_started: 10 + streams_succeeded: 2 + streams_failed: 3 + messages_sent: 20 + messages_received: 10 + keep_alives_sent: 2 + last_local_stream_created_timestamp: {seconds: 0 nanos: 0} + last_remote_stream_created_timestamp: {seconds: 5 nanos: 0} + last_message_sent_timestamp: {seconds: 6 nanos: 0} + last_message_received_timestamp: {seconds: 7 nanos: 0} + local_flow_control_window: { value: 65536 } + remote_flow_control_window: { value: 1024 } + } + local: { uds_address { filename: "file.path" } } + remote: { uds_address { filename: "another.path" } } + remote_name: "remote.remote"`, + ` + ref: {socket_id: ` + fmt.Sprint(skts[2].ID) + ` name: "2" } + data: { + streams_started: 5 + streams_succeeded: 2 + streams_failed: 3 + messages_sent: 20 + messages_received: 10 + keep_alives_sent: 2 + last_local_stream_created_timestamp: {seconds: 10 nanos: 10} + last_remote_stream_created_timestamp: {seconds: 0 nanos: 0} + last_message_sent_timestamp: {seconds: 0 nanos: 0} + last_message_received_timestamp: {seconds: 0 nanos: 0} + local_flow_control_window: { value: 65536 } + remote_flow_control_window: { value: 10240 } + } + local: { tcpip_address: { ip_address: "` + addr(skts[2].LocalAddr) + `" } } + remote: { tcpip_address: { ip_address: "` + addr(skts[2].RemoteAddr) + `" } } + remote_name: ""`, + ` + ref: {socket_id: ` + fmt.Sprint(skts[3].ID) + ` name: "3" } + local: { tcpip_address: { ip_address: "` + addr(skts[3].LocalAddr) + `" port: 10001 } } + ` + emptyData, + ` + ref: {socket_id: ` + fmt.Sprint(skts[4].ID) + ` name: "4" } + security: { tls: { + standard_name: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" + remote_certificate: "\x30\x82\x02\x9c\x30\x82\x02\x05\xa0" + } } + ` + emptyData, + ` + ref: {socket_id: ` + fmt.Sprint(skts[5].ID) + ` name: "5" } + security: { other: { name: "XXXX" } } + ` + emptyData, + ` + ref: {socket_id: ` + fmt.Sprint(skts[6].ID) + ` name: "6" } + security: { other: { + name: "YYYY" + value: { + type_url: "type.googleapis.com/grpc.credentials.OtherChannelzSecurityValue" + value: "` + escape(otherSecVal.Value) + `" + } + } } + ` + emptyData, + } + + for i := range ss { + resp, _ := svr.GetSocket(ctx, &channelzpb.GetSocketRequest{SocketId: skts[i].ID}) + w := &channelzpb.Socket{} + if err := prototext.Unmarshal([]byte(want[i]), w); err != nil { + t.Fatalf("Error unmarshalling %q: %v", want[i], err) + } + if diff := cmp.Diff(resp.GetSocket(), w, protocmp.Transform()); diff != "" { + t.Fatalf("Socket %v did not match expected. -got +want: %v", i, diff) + } + } +} + +func escape(bs []byte) string { + ret := "" + for _, b := range bs { + ret += fmt.Sprintf("\\x%02x", b) + } + return ret +} + +func addr(a net.Addr) string { + switch a := a.(type) { + case *net.TCPAddr: + return escape([]byte(a.IP)) + case *net.IPAddr: + return escape([]byte(a.IP)) + } + return "" +} diff --git a/clientconn.go b/clientconn.go new file mode 100644 index 0000000..fc50b44 --- /dev/null +++ b/clientconn.go @@ -0,0 +1,1981 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "errors" + "fmt" + "math" + "net/url" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + expstats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/idle" + iresolver "google.golang.org/grpc/internal/resolver" + istats "google.golang.org/grpc/internal/stats" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" + + _ "google.golang.org/grpc/balancer/roundrobin" // To register roundrobin. + _ "google.golang.org/grpc/internal/resolver/passthrough" // To register passthrough resolver. + _ "google.golang.org/grpc/internal/resolver/unix" // To register unix resolver. + _ "google.golang.org/grpc/resolver/dns" // To register dns resolver. +) + +const ( + // minimum time to give a connection to complete + minConnectTimeout = 20 * time.Second +) + +var ( + // ErrClientConnClosing indicates that the operation is illegal because + // the ClientConn is closing. + // + // Deprecated: this error should not be relied upon by users; use the status + // code of Canceled instead. + ErrClientConnClosing = status.Error(codes.Canceled, "grpc: the client connection is closing") + // errConnDrain indicates that the connection starts to be drained and does not accept any new RPCs. + errConnDrain = errors.New("grpc: the connection is drained") + // errConnClosing indicates that the connection is closing. + errConnClosing = errors.New("grpc: the connection is closing") + // errConnIdling indicates the connection is being closed as the channel + // is moving to an idle mode due to inactivity. + errConnIdling = errors.New("grpc: the connection is closing due to channel idleness") + // invalidDefaultServiceConfigErrPrefix is used to prefix the json parsing error for the default + // service config. + invalidDefaultServiceConfigErrPrefix = "grpc: the provided default service config is invalid" + // PickFirstBalancerName is the name of the pick_first balancer. + PickFirstBalancerName = pickfirst.Name +) + +// The following errors are returned from Dial and DialContext +var ( + // errNoTransportSecurity indicates that there is no transport security + // being set for ClientConn. Users should either set one or explicitly + // call WithInsecure DialOption to disable security. + errNoTransportSecurity = errors.New("grpc: no transport security set (use grpc.WithTransportCredentials(insecure.NewCredentials()) explicitly or set credentials)") + // errTransportCredsAndBundle indicates that creds bundle is used together + // with other individual Transport Credentials. + errTransportCredsAndBundle = errors.New("grpc: credentials.Bundle may not be used with individual TransportCredentials") + // errNoTransportCredsInBundle indicated that the configured creds bundle + // returned a transport credentials which was nil. + errNoTransportCredsInBundle = errors.New("grpc: credentials.Bundle must return non-nil transport credentials") + // errTransportCredentialsMissing indicates that users want to transmit + // security information (e.g., OAuth2 token) which requires secure + // connection on an insecure connection. + errTransportCredentialsMissing = errors.New("grpc: the credentials require transport level security (use grpc.WithTransportCredentials() to set)") +) + +var ( + disconnectionsMetric = expstats.RegisterInt64Count(expstats.MetricDescriptor{ + Name: "grpc.subchannel.disconnections", + Description: "EXPERIMENTAL. Number of times the selected subchannel becomes disconnected.", + Unit: "{disconnection}", + Labels: []string{"grpc.target"}, + OptionalLabels: []string{"grpc.lb.backend_service", "grpc.lb.locality", "grpc.disconnect_error"}, + Default: false, + }) + connectionAttemptsSucceededMetric = expstats.RegisterInt64Count(expstats.MetricDescriptor{ + Name: "grpc.subchannel.connection_attempts_succeeded", + Description: "EXPERIMENTAL. Number of successful connection attempts.", + Unit: "{attempt}", + Labels: []string{"grpc.target"}, + OptionalLabels: []string{"grpc.lb.backend_service", "grpc.lb.locality"}, + Default: false, + }) + connectionAttemptsFailedMetric = expstats.RegisterInt64Count(expstats.MetricDescriptor{ + Name: "grpc.subchannel.connection_attempts_failed", + Description: "EXPERIMENTAL. Number of failed connection attempts.", + Unit: "{attempt}", + Labels: []string{"grpc.target"}, + OptionalLabels: []string{"grpc.lb.backend_service", "grpc.lb.locality"}, + Default: false, + }) + openConnectionsMetric = expstats.RegisterInt64UpDownCount(expstats.MetricDescriptor{ + Name: "grpc.subchannel.open_connections", + Description: "EXPERIMENTAL. Number of open connections.", + Unit: "{attempt}", + Labels: []string{"grpc.target"}, + OptionalLabels: []string{"grpc.lb.backend_service", "grpc.security_level", "grpc.lb.locality"}, + Default: false, + }) +) + +const ( + defaultClientMaxReceiveMessageSize = 1024 * 1024 * 4 + defaultClientMaxSendMessageSize = math.MaxInt32 + // http2IOBufSize specifies the buffer size for sending frames. + defaultWriteBufSize = 32 * 1024 + defaultReadBufSize = 32 * 1024 +) + +type defaultConfigSelector struct { + sc *ServiceConfig +} + +func (dcs *defaultConfigSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RPCConfig, error) { + return &iresolver.RPCConfig{ + Context: rpcInfo.Context, + MethodConfig: getMethodConfig(dcs.sc, rpcInfo.Method), + }, nil +} + +// NewClient creates a new gRPC "channel" for the target URI provided. No I/O +// is performed. Use of the ClientConn for RPCs will automatically cause it to +// connect. The Connect method may be called to manually create a connection, +// but for most users this should be unnecessary. +// +// The target name syntax is defined in +// https://github.com/grpc/grpc/blob/master/doc/naming.md. E.g. to use the dns +// name resolver, a "dns:///" prefix may be applied to the target. The default +// name resolver will be used if no scheme is detected, or if the parsed scheme +// is not a registered name resolver. The default resolver is "dns" but can be +// overridden using the resolver package's SetDefaultScheme. +// +// Examples: +// +// - "foo.googleapis.com:8080" +// - "dns:///foo.googleapis.com:8080" +// - "dns:///foo.googleapis.com" +// - "dns:///10.0.0.213:8080" +// - "dns:///%5B2001:db8:85a3:8d3:1319:8a2e:370:7348%5D:443" +// - "dns://8.8.8.8/foo.googleapis.com:8080" +// - "dns://8.8.8.8/foo.googleapis.com" +// - "zookeeper://zk.example.com:9900/example_service" +// +// The DialOptions returned by WithBlock, WithTimeout, +// WithReturnConnectionError, and FailOnNonTempDialError are ignored by this +// function. +func NewClient(target string, opts ...DialOption) (conn *ClientConn, err error) { + cc := &ClientConn{ + target: target, + conns: make(map[*addrConn]struct{}), + dopts: defaultDialOptions(), + } + + cc.retryThrottler.Store((*retryThrottler)(nil)) + cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{nil}) + cc.ctx, cc.cancel = context.WithCancel(context.Background()) + + // Apply dial options. + disableGlobalOpts := false + for _, opt := range opts { + if _, ok := opt.(*disableGlobalDialOptions); ok { + disableGlobalOpts = true + break + } + } + + if !disableGlobalOpts { + for _, opt := range globalDialOptions { + opt.apply(&cc.dopts) + } + } + + for _, opt := range opts { + opt.apply(&cc.dopts) + } + + // Determine the resolver to use. + if err := cc.initParsedTargetAndResolverBuilder(); err != nil { + return nil, err + } + + for _, opt := range globalPerTargetDialOptions { + opt.DialOptionForTarget(cc.parsedTarget.URL).apply(&cc.dopts) + } + + chainUnaryClientInterceptors(cc) + chainStreamClientInterceptors(cc) + + if err := cc.validateTransportCredentials(); err != nil { + return nil, err + } + + if cc.dopts.defaultServiceConfigRawJSON != nil { + scpr := parseServiceConfig(*cc.dopts.defaultServiceConfigRawJSON, cc.dopts.maxCallAttempts) + if scpr.Err != nil { + return nil, fmt.Errorf("%s: %v", invalidDefaultServiceConfigErrPrefix, scpr.Err) + } + cc.dopts.defaultServiceConfig, _ = scpr.Config.(*ServiceConfig) + } + cc.keepaliveParams = cc.dopts.copts.KeepaliveParams + + if err = cc.initAuthority(); err != nil { + return nil, err + } + + // Register ClientConn with channelz. Note that this is only done after + // channel creation cannot fail. + cc.channelzRegistration(target) + channelz.Infof(logger, cc.channelz, "parsed dial target is: %#v", cc.parsedTarget) + channelz.Infof(logger, cc.channelz, "Channel authority set to %q", cc.authority) + + cc.csMgr = newConnectivityStateManager(cc.ctx, cc.channelz) + cc.pickerWrapper = newPickerWrapper() + + cc.metricsRecorderList = istats.NewMetricsRecorderList(cc.dopts.copts.StatsHandlers) + cc.statsHandler = istats.NewCombinedHandler(cc.dopts.copts.StatsHandlers...) + + cc.initIdleStateLocked() // Safe to call without the lock, since nothing else has a reference to cc. + cc.idlenessMgr = idle.NewManager((*idler)(cc), cc.dopts.idleTimeout) + + return cc, nil +} + +// Dial calls DialContext(context.Background(), target, opts...). +// +// Deprecated: use NewClient instead. Will be supported throughout 1.x. +func Dial(target string, opts ...DialOption) (*ClientConn, error) { + return DialContext(context.Background(), target, opts...) +} + +// DialContext calls NewClient and then exits idle mode. If WithBlock(true) is +// used, it calls Connect and WaitForStateChange until either the context +// expires or the state of the ClientConn is Ready. +// +// One subtle difference between NewClient and Dial and DialContext is that the +// former uses "dns" as the default name resolver, while the latter use +// "passthrough" for backward compatibility. This distinction should not matter +// to most users, but could matter to legacy users that specify a custom dialer +// and expect it to receive the target string directly. +// +// Deprecated: use NewClient instead. Will be supported throughout 1.x. +func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *ClientConn, err error) { + // At the end of this method, we kick the channel out of idle, rather than + // waiting for the first rpc. + // + // WithLocalDNSResolution dial option in `grpc.Dial` ensures that it + // preserves behavior: when default scheme passthrough is used, skip + // hostname resolution, when "dns" is used for resolution, perform + // resolution on the client. + opts = append([]DialOption{withDefaultScheme("passthrough"), WithLocalDNSResolution()}, opts...) + cc, err := NewClient(target, opts...) + if err != nil { + return nil, err + } + + // We start the channel off in idle mode, but kick it out of idle now, + // instead of waiting for the first RPC. This is the legacy behavior of + // Dial. + defer func() { + if err != nil { + cc.Close() + } + }() + + // This creates the name resolver, load balancer, etc. + if err := cc.exitIdleMode(); err != nil { + return nil, fmt.Errorf("failed to exit idle mode: %w", err) + } + cc.idlenessMgr.UnsafeSetNotIdle() + + // Return now for non-blocking dials. + if !cc.dopts.block { + return cc, nil + } + + if cc.dopts.timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, cc.dopts.timeout) + defer cancel() + } + defer func() { + select { + case <-ctx.Done(): + switch { + case ctx.Err() == err: + conn = nil + case err == nil || !cc.dopts.returnLastError: + conn, err = nil, ctx.Err() + default: + conn, err = nil, fmt.Errorf("%v: %v", ctx.Err(), err) + } + default: + } + }() + + // A blocking dial blocks until the clientConn is ready. + for { + s := cc.GetState() + if s == connectivity.Idle { + cc.Connect() + } + if s == connectivity.Ready { + return cc, nil + } else if cc.dopts.copts.FailOnNonTempDialError && s == connectivity.TransientFailure { + if err = cc.connectionError(); err != nil { + terr, ok := err.(interface { + Temporary() bool + }) + if ok && !terr.Temporary() { + return nil, err + } + } + } + if !cc.WaitForStateChange(ctx, s) { + // ctx got timeout or canceled. + if err = cc.connectionError(); err != nil && cc.dopts.returnLastError { + return nil, err + } + return nil, ctx.Err() + } + } +} + +// addTraceEvent is a helper method to add a trace event on the channel. If the +// channel is a nested one, the same event is also added on the parent channel. +func (cc *ClientConn) addTraceEvent(msg string) { + ted := &channelz.TraceEvent{ + Desc: fmt.Sprintf("Channel %s", msg), + Severity: channelz.CtInfo, + } + if cc.dopts.channelzParent != nil { + ted.Parent = &channelz.TraceEvent{ + Desc: fmt.Sprintf("Nested channel(id:%d) %s", cc.channelz.ID, msg), + Severity: channelz.CtInfo, + } + } + channelz.AddTraceEvent(logger, cc.channelz, 1, ted) +} + +type idler ClientConn + +func (i *idler) EnterIdleMode() { + (*ClientConn)(i).enterIdleMode() +} + +func (i *idler) ExitIdleMode() { + // Ignore the error returned from this method, because from the perspective + // of the caller (idleness manager), the channel would have always moved out + // of IDLE by the time this method returns. + (*ClientConn)(i).exitIdleMode() +} + +// exitIdleMode moves the channel out of idle mode by recreating the name +// resolver and load balancer. This should never be called directly; use +// cc.idlenessMgr.ExitIdleMode instead. +func (cc *ClientConn) exitIdleMode() error { + cc.mu.Lock() + if cc.conns == nil { + cc.mu.Unlock() + return errConnClosing + } + cc.mu.Unlock() + + // Set state to CONNECTING before building the name resolver + // so the channel does not remain in IDLE. + cc.csMgr.updateState(connectivity.Connecting) + + // This needs to be called without cc.mu because this builds a new resolver + // which might update state or report error inline, which would then need to + // acquire cc.mu. + if err := cc.resolverWrapper.start(); err != nil { + // If resolver creation fails, treat it like an error reported by the + // resolver before any valid updates. Set channel's state to + // TransientFailure, and set an erroring picker with the resolver build + // error, which will returned as part of any subsequent RPCs. + logger.Warningf("Failed to start resolver: %v", err) + cc.csMgr.updateState(connectivity.TransientFailure) + cc.mu.Lock() + cc.updateResolverStateAndUnlock(resolver.State{}, err) + return fmt.Errorf("failed to start resolver: %w", err) + } + + cc.addTraceEvent("exiting idle mode") + return nil +} + +// initIdleStateLocked initializes common state to how it should be while idle. +func (cc *ClientConn) initIdleStateLocked() { + cc.resolverWrapper = newCCResolverWrapper(cc) + cc.balancerWrapper = newCCBalancerWrapper(cc) + cc.firstResolveEvent = grpcsync.NewEvent() + // cc.conns == nil is a proxy for the ClientConn being closed. So, instead + // of setting it to nil here, we recreate the map. This also means that we + // don't have to do this when exiting idle mode. + cc.conns = make(map[*addrConn]struct{}) +} + +// enterIdleMode puts the channel in idle mode, and as part of it shuts down the +// name resolver, load balancer, and any subchannels. This should never be +// called directly; use cc.idlenessMgr.EnterIdleMode instead. +func (cc *ClientConn) enterIdleMode() { + cc.mu.Lock() + + if cc.conns == nil { + cc.mu.Unlock() + return + } + + conns := cc.conns + + rWrapper := cc.resolverWrapper + rWrapper.close() + cc.pickerWrapper.reset() + bWrapper := cc.balancerWrapper + bWrapper.close() + cc.csMgr.updateState(connectivity.Idle) + cc.addTraceEvent("entering idle mode") + + cc.initIdleStateLocked() + + cc.mu.Unlock() + + // Block until the name resolver and LB policy are closed. + <-rWrapper.serializer.Done() + <-bWrapper.serializer.Done() + + // Close all subchannels after the LB policy is closed. + for ac := range conns { + ac.tearDown(errConnIdling) + } +} + +// validateTransportCredentials performs a series of checks on the configured +// transport credentials. It returns a non-nil error if any of these conditions +// are met: +// - no transport creds and no creds bundle is configured +// - both transport creds and creds bundle are configured +// - creds bundle is configured, but it lacks a transport credentials +// - insecure transport creds configured alongside call creds that require +// transport level security +// +// If none of the above conditions are met, the configured credentials are +// deemed valid and a nil error is returned. +func (cc *ClientConn) validateTransportCredentials() error { + if cc.dopts.copts.TransportCredentials == nil && cc.dopts.copts.CredsBundle == nil { + return errNoTransportSecurity + } + if cc.dopts.copts.TransportCredentials != nil && cc.dopts.copts.CredsBundle != nil { + return errTransportCredsAndBundle + } + if cc.dopts.copts.CredsBundle != nil && cc.dopts.copts.CredsBundle.TransportCredentials() == nil { + return errNoTransportCredsInBundle + } + transportCreds := cc.dopts.copts.TransportCredentials + if transportCreds == nil { + transportCreds = cc.dopts.copts.CredsBundle.TransportCredentials() + } + if transportCreds.Info().SecurityProtocol == "insecure" { + for _, cd := range cc.dopts.copts.PerRPCCredentials { + if cd.RequireTransportSecurity() { + return errTransportCredentialsMissing + } + } + } + return nil +} + +// channelzRegistration registers the newly created ClientConn with channelz and +// stores the returned identifier in `cc.channelz`. A channelz trace event is +// emitted for ClientConn creation. If the newly created ClientConn is a nested +// one, i.e a valid parent ClientConn ID is specified via a dial option, the +// trace event is also added to the parent. +// +// Doesn't grab cc.mu as this method is expected to be called only at Dial time. +func (cc *ClientConn) channelzRegistration(target string) { + parentChannel, _ := cc.dopts.channelzParent.(*channelz.Channel) + cc.channelz = channelz.RegisterChannel(parentChannel, target) + cc.addTraceEvent(fmt.Sprintf("created for target %q", target)) +} + +// chainUnaryClientInterceptors chains all unary client interceptors into one. +func chainUnaryClientInterceptors(cc *ClientConn) { + interceptors := cc.dopts.chainUnaryInts + // Prepend dopts.unaryInt to the chaining interceptors if it exists, since unaryInt will + // be executed before any other chained interceptors. + if cc.dopts.unaryInt != nil { + interceptors = append([]UnaryClientInterceptor{cc.dopts.unaryInt}, interceptors...) + } + var chainedInt UnaryClientInterceptor + if len(interceptors) == 0 { + chainedInt = nil + } else if len(interceptors) == 1 { + chainedInt = interceptors[0] + } else { + chainedInt = func(ctx context.Context, method string, req, reply any, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error { + return interceptors[0](ctx, method, req, reply, cc, getChainUnaryInvoker(interceptors, 0, invoker), opts...) + } + } + cc.dopts.unaryInt = chainedInt +} + +// getChainUnaryInvoker recursively generate the chained unary invoker. +func getChainUnaryInvoker(interceptors []UnaryClientInterceptor, curr int, finalInvoker UnaryInvoker) UnaryInvoker { + if curr == len(interceptors)-1 { + return finalInvoker + } + return func(ctx context.Context, method string, req, reply any, cc *ClientConn, opts ...CallOption) error { + return interceptors[curr+1](ctx, method, req, reply, cc, getChainUnaryInvoker(interceptors, curr+1, finalInvoker), opts...) + } +} + +// chainStreamClientInterceptors chains all stream client interceptors into one. +func chainStreamClientInterceptors(cc *ClientConn) { + interceptors := cc.dopts.chainStreamInts + // Prepend dopts.streamInt to the chaining interceptors if it exists, since streamInt will + // be executed before any other chained interceptors. + if cc.dopts.streamInt != nil { + interceptors = append([]StreamClientInterceptor{cc.dopts.streamInt}, interceptors...) + } + interceptors = append(interceptors, defaultStreamInterceptor) + var chainedInt StreamClientInterceptor + if len(interceptors) == 0 { + chainedInt = nil + } else if len(interceptors) == 1 { + chainedInt = interceptors[0] + } else { + chainedInt = func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error) { + return interceptors[0](ctx, desc, cc, method, getChainStreamer(interceptors, 0, streamer), opts...) + } + } + cc.dopts.streamInt = chainedInt +} + +// getChainStreamer recursively generate the chained client stream constructor. +func getChainStreamer(interceptors []StreamClientInterceptor, curr int, finalStreamer Streamer) Streamer { + if curr == len(interceptors)-1 { + return finalStreamer + } + return func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) { + return interceptors[curr+1](ctx, desc, cc, method, getChainStreamer(interceptors, curr+1, finalStreamer), opts...) + } +} + +// newConnectivityStateManager creates an connectivityStateManager with +// the specified channel. +func newConnectivityStateManager(ctx context.Context, channel *channelz.Channel) *connectivityStateManager { + return &connectivityStateManager{ + channelz: channel, + pubSub: grpcsync.NewPubSub(ctx), + } +} + +// connectivityStateManager keeps the connectivity.State of ClientConn. +// This struct will eventually be exported so the balancers can access it. +// +// TODO: If possible, get rid of the `connectivityStateManager` type, and +// provide this functionality using the `PubSub`, to avoid keeping track of +// the connectivity state at two places. +type connectivityStateManager struct { + mu sync.Mutex + state connectivity.State + notifyChan chan struct{} + channelz *channelz.Channel + pubSub *grpcsync.PubSub +} + +// updateState updates the connectivity.State of ClientConn. +// If there's a change it notifies goroutines waiting on state change to +// happen. +func (csm *connectivityStateManager) updateState(state connectivity.State) { + csm.mu.Lock() + defer csm.mu.Unlock() + if csm.state == connectivity.Shutdown { + return + } + if csm.state == state { + return + } + csm.state = state + csm.channelz.ChannelMetrics.State.Store(&state) + csm.pubSub.Publish(state) + + channelz.Infof(logger, csm.channelz, "Channel Connectivity change to %v", state) + if csm.notifyChan != nil { + // There are other goroutines waiting on this channel. + close(csm.notifyChan) + csm.notifyChan = nil + } +} + +func (csm *connectivityStateManager) getState() connectivity.State { + csm.mu.Lock() + defer csm.mu.Unlock() + return csm.state +} + +func (csm *connectivityStateManager) getNotifyChan() <-chan struct{} { + csm.mu.Lock() + defer csm.mu.Unlock() + if csm.notifyChan == nil { + csm.notifyChan = make(chan struct{}) + } + return csm.notifyChan +} + +// ClientConnInterface defines the functions clients need to perform unary and +// streaming RPCs. It is implemented by *ClientConn, and is only intended to +// be referenced by generated code. +type ClientConnInterface interface { + // Invoke performs a unary RPC and returns after the response is received + // into reply. + Invoke(ctx context.Context, method string, args any, reply any, opts ...CallOption) error + // NewStream begins a streaming RPC. + NewStream(ctx context.Context, desc *StreamDesc, method string, opts ...CallOption) (ClientStream, error) +} + +// Assert *ClientConn implements ClientConnInterface. +var _ ClientConnInterface = (*ClientConn)(nil) + +// ClientConn represents a virtual connection to a conceptual endpoint, to +// perform RPCs. +// +// A ClientConn is free to have zero or more actual connections to the endpoint +// based on configuration, load, etc. It is also free to determine which actual +// endpoints to use and may change it every RPC, permitting client-side load +// balancing. +// +// A ClientConn encapsulates a range of functionality including name +// resolution, TCP connection establishment (with retries and backoff) and TLS +// handshakes. It also handles errors on established connections by +// re-resolving the name and reconnecting. +type ClientConn struct { + ctx context.Context // Initialized using the background context at dial time. + cancel context.CancelFunc // Cancelled on close. + + // The following are initialized at dial time, and are read-only after that. + target string // User's dial target. + parsedTarget resolver.Target // See initParsedTargetAndResolverBuilder(). + authority string // See initAuthority(). + dopts dialOptions // Default and user specified dial options. + channelz *channelz.Channel // Channelz object. + resolverBuilder resolver.Builder // See initParsedTargetAndResolverBuilder(). + idlenessMgr *idle.Manager + metricsRecorderList *istats.MetricsRecorderList + statsHandler stats.Handler + + // The following provide their own synchronization, and therefore don't + // require cc.mu to be held to access them. + csMgr *connectivityStateManager + pickerWrapper *pickerWrapper + safeConfigSelector iresolver.SafeConfigSelector + retryThrottler atomic.Value // Updated from service config. + + // mu protects the following fields. + // TODO: split mu so the same mutex isn't used for everything. + mu sync.RWMutex + resolverWrapper *ccResolverWrapper // Always recreated whenever entering idle to simplify Close. + balancerWrapper *ccBalancerWrapper // Always recreated whenever entering idle to simplify Close. + sc *ServiceConfig // Latest service config received from the resolver. + conns map[*addrConn]struct{} // Set to nil on close. + keepaliveParams keepalive.ClientParameters // May be updated upon receipt of a GoAway. + // firstResolveEvent is used to track whether the name resolver sent us at + // least one update. RPCs block on this event. May be accessed without mu + // if we know we cannot be asked to enter idle mode while accessing it (e.g. + // when the idle manager has already been closed, or if we are already + // entering idle mode). + firstResolveEvent *grpcsync.Event + + lceMu sync.Mutex // protects lastConnectionError + lastConnectionError error +} + +// WaitForStateChange waits until the connectivity.State of ClientConn changes from sourceState or +// ctx expires. A true value is returned in former case and false in latter. +func (cc *ClientConn) WaitForStateChange(ctx context.Context, sourceState connectivity.State) bool { + ch := cc.csMgr.getNotifyChan() + if cc.csMgr.getState() != sourceState { + return true + } + select { + case <-ctx.Done(): + return false + case <-ch: + return true + } +} + +// GetState returns the connectivity.State of ClientConn. +func (cc *ClientConn) GetState() connectivity.State { + return cc.csMgr.getState() +} + +// Connect causes all subchannels in the ClientConn to attempt to connect if +// the channel is idle. Does not wait for the connection attempts to begin +// before returning. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a later +// release. +func (cc *ClientConn) Connect() { + cc.idlenessMgr.ExitIdleMode() + + // If the ClientConn was not in idle mode, we need to call ExitIdle on the + // LB policy so that connections can be created. + cc.mu.Lock() + cc.balancerWrapper.exitIdle() + cc.mu.Unlock() +} + +// waitForResolvedAddrs blocks until the resolver provides addresses or the +// context expires, whichever happens first. +// +// Error is nil unless the context expires first; otherwise returns a status +// error based on the context. +// +// The returned boolean indicates whether it did block or not. If the +// resolution has already happened once before, it returns false without +// blocking. Otherwise, it wait for the resolution and return true if +// resolution has succeeded or return false along with error if resolution has +// failed. +func (cc *ClientConn) waitForResolvedAddrs(ctx context.Context) (bool, error) { + // This is on the RPC path, so we use a fast path to avoid the + // more-expensive "select" below after the resolver has returned once. + if cc.firstResolveEvent.HasFired() { + return false, nil + } + internal.NewStreamWaitingForResolver() + select { + case <-cc.firstResolveEvent.Done(): + return true, nil + case <-ctx.Done(): + return false, status.FromContextError(ctx.Err()).Err() + case <-cc.ctx.Done(): + return false, ErrClientConnClosing + } +} + +var emptyServiceConfig *ServiceConfig + +func init() { + cfg := parseServiceConfig("{}", defaultMaxCallAttempts) + if cfg.Err != nil { + panic(fmt.Sprintf("impossible error parsing empty service config: %v", cfg.Err)) + } + emptyServiceConfig = cfg.Config.(*ServiceConfig) + + internal.SubscribeToConnectivityStateChanges = func(cc *ClientConn, s grpcsync.Subscriber) func() { + return cc.csMgr.pubSub.Subscribe(s) + } + internal.EnterIdleModeForTesting = func(cc *ClientConn) { + cc.idlenessMgr.EnterIdleModeForTesting() + } + internal.ExitIdleModeForTesting = func(cc *ClientConn) { + cc.idlenessMgr.ExitIdleMode() + } +} + +func (cc *ClientConn) maybeApplyDefaultServiceConfig() { + if cc.sc != nil { + cc.applyServiceConfigAndBalancer(cc.sc, nil) + return + } + if cc.dopts.defaultServiceConfig != nil { + cc.applyServiceConfigAndBalancer(cc.dopts.defaultServiceConfig, &defaultConfigSelector{cc.dopts.defaultServiceConfig}) + } else { + cc.applyServiceConfigAndBalancer(emptyServiceConfig, &defaultConfigSelector{emptyServiceConfig}) + } +} + +func (cc *ClientConn) updateResolverStateAndUnlock(s resolver.State, err error) error { + defer cc.firstResolveEvent.Fire() + // Check if the ClientConn is already closed. Some fields (e.g. + // balancerWrapper) are set to nil when closing the ClientConn, and could + // cause nil pointer panic if we don't have this check. + if cc.conns == nil { + cc.mu.Unlock() + return nil + } + + if err != nil { + // May need to apply the initial service config in case the resolver + // doesn't support service configs, or doesn't provide a service config + // with the new addresses. + cc.maybeApplyDefaultServiceConfig() + + cc.balancerWrapper.resolverError(err) + + // No addresses are valid with err set; return early. + cc.mu.Unlock() + return balancer.ErrBadResolverState + } + + var ret error + if cc.dopts.disableServiceConfig { + channelz.Infof(logger, cc.channelz, "ignoring service config from resolver (%v) and applying the default because service config is disabled", s.ServiceConfig) + cc.maybeApplyDefaultServiceConfig() + } else if s.ServiceConfig == nil { + cc.maybeApplyDefaultServiceConfig() + // TODO: do we need to apply a failing LB policy if there is no + // default, per the error handling design? + } else { + if sc, ok := s.ServiceConfig.Config.(*ServiceConfig); s.ServiceConfig.Err == nil && ok { + configSelector := iresolver.GetConfigSelector(s) + if configSelector != nil { + if len(s.ServiceConfig.Config.(*ServiceConfig).Methods) != 0 { + channelz.Infof(logger, cc.channelz, "method configs in service config will be ignored due to presence of config selector") + } + } else { + configSelector = &defaultConfigSelector{sc} + } + cc.applyServiceConfigAndBalancer(sc, configSelector) + } else { + ret = balancer.ErrBadResolverState + if cc.sc == nil { + // Apply the failing LB only if we haven't received valid service config + // from the name resolver in the past. + cc.applyFailingLBLocked(s.ServiceConfig) + cc.mu.Unlock() + return ret + } + } + } + + balCfg := cc.sc.lbConfig + bw := cc.balancerWrapper + cc.mu.Unlock() + + uccsErr := bw.updateClientConnState(&balancer.ClientConnState{ResolverState: s, BalancerConfig: balCfg}) + if ret == nil { + ret = uccsErr // prefer ErrBadResolver state since any other error is + // currently meaningless to the caller. + } + return ret +} + +// applyFailingLBLocked is akin to configuring an LB policy on the channel which +// always fails RPCs. Here, an actual LB policy is not configured, but an always +// erroring picker is configured, which returns errors with information about +// what was invalid in the received service config. A config selector with no +// service config is configured, and the connectivity state of the channel is +// set to TransientFailure. +func (cc *ClientConn) applyFailingLBLocked(sc *serviceconfig.ParseResult) { + var err error + if sc.Err != nil { + err = status.Errorf(codes.Unavailable, "error parsing service config: %v", sc.Err) + } else { + err = status.Errorf(codes.Unavailable, "illegal service config type: %T", sc.Config) + } + cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{nil}) + cc.pickerWrapper.updatePicker(base.NewErrPicker(err)) + cc.csMgr.updateState(connectivity.TransientFailure) +} + +// Makes a copy of the input addresses slice. Addresses are passed during +// subconn creation and address update operations. +func copyAddresses(in []resolver.Address) []resolver.Address { + out := make([]resolver.Address, len(in)) + copy(out, in) + return out +} + +// newAddrConnLocked creates an addrConn for addrs and adds it to cc.conns. +// +// Caller needs to make sure len(addrs) > 0. +func (cc *ClientConn) newAddrConnLocked(addrs []resolver.Address, opts balancer.NewSubConnOptions) (*addrConn, error) { + if cc.conns == nil { + return nil, ErrClientConnClosing + } + + ac := &addrConn{ + state: connectivity.Idle, + cc: cc, + addrs: copyAddresses(addrs), + scopts: opts, + dopts: cc.dopts, + channelz: channelz.RegisterSubChannel(cc.channelz, ""), + resetBackoff: make(chan struct{}), + } + ac.updateTelemetryLabelsLocked() + ac.ctx, ac.cancel = context.WithCancel(cc.ctx) + // Start with our address set to the first address; this may be updated if + // we connect to different addresses. + ac.channelz.ChannelMetrics.Target.Store(&addrs[0].Addr) + + channelz.AddTraceEvent(logger, ac.channelz, 0, &channelz.TraceEvent{ + Desc: "Subchannel created", + Severity: channelz.CtInfo, + Parent: &channelz.TraceEvent{ + Desc: fmt.Sprintf("Subchannel(id:%d) created", ac.channelz.ID), + Severity: channelz.CtInfo, + }, + }) + + // Track ac in cc. This needs to be done before any getTransport(...) is called. + cc.conns[ac] = struct{}{} + return ac, nil +} + +// removeAddrConn removes the addrConn in the subConn from clientConn. +// It also tears down the ac with the given error. +func (cc *ClientConn) removeAddrConn(ac *addrConn, err error) { + cc.mu.Lock() + if cc.conns == nil { + cc.mu.Unlock() + return + } + delete(cc.conns, ac) + cc.mu.Unlock() + ac.tearDown(err) +} + +// Target returns the target string of the ClientConn. +func (cc *ClientConn) Target() string { + return cc.target +} + +// CanonicalTarget returns the canonical target string used when creating cc. +// +// This always has the form "://[authority]/". For example: +// +// - "dns:///example.com:42" +// - "dns://8.8.8.8/example.com:42" +// - "unix:///path/to/socket" +func (cc *ClientConn) CanonicalTarget() string { + return cc.parsedTarget.String() +} + +func (cc *ClientConn) incrCallsStarted() { + cc.channelz.ChannelMetrics.CallsStarted.Add(1) + cc.channelz.ChannelMetrics.LastCallStartedTimestamp.Store(time.Now().UnixNano()) +} + +func (cc *ClientConn) incrCallsSucceeded() { + cc.channelz.ChannelMetrics.CallsSucceeded.Add(1) +} + +func (cc *ClientConn) incrCallsFailed() { + cc.channelz.ChannelMetrics.CallsFailed.Add(1) +} + +// connect starts creating a transport. +// It does nothing if the ac is not IDLE. +// TODO(bar) Move this to the addrConn section. +func (ac *addrConn) connect() { + ac.mu.Lock() + if ac.state == connectivity.Shutdown { + if logger.V(2) { + logger.Infof("connect called on shutdown addrConn; ignoring.") + } + ac.mu.Unlock() + return + } + if ac.state != connectivity.Idle { + if logger.V(2) { + logger.Infof("connect called on addrConn in non-idle state (%v); ignoring.", ac.state) + } + ac.mu.Unlock() + return + } + + ac.resetTransportAndUnlock() +} + +// equalAddressIgnoringBalAttributes returns true is a and b are considered equal. +// This is different from the Equal method on the resolver.Address type which +// considers all fields to determine equality. Here, we only consider fields +// that are meaningful to the subConn. +func equalAddressIgnoringBalAttributes(a, b *resolver.Address) bool { + return a.Addr == b.Addr && a.ServerName == b.ServerName && + a.Attributes.Equal(b.Attributes) && + a.Metadata == b.Metadata +} + +func equalAddressesIgnoringBalAttributes(a, b []resolver.Address) bool { + return slices.EqualFunc(a, b, func(a, b resolver.Address) bool { return equalAddressIgnoringBalAttributes(&a, &b) }) +} + +// updateAddrs updates ac.addrs with the new addresses list and handles active +// connections or connection attempts. +func (ac *addrConn) updateAddrs(addrs []resolver.Address) { + addrs = copyAddresses(addrs) + limit := len(addrs) + if limit > 5 { + limit = 5 + } + channelz.Infof(logger, ac.channelz, "addrConn: updateAddrs addrs (%d of %d): %v", limit, len(addrs), addrs[:limit]) + + ac.mu.Lock() + if equalAddressesIgnoringBalAttributes(ac.addrs, addrs) { + ac.mu.Unlock() + return + } + + ac.addrs = addrs + ac.updateTelemetryLabelsLocked() + if ac.state == connectivity.Shutdown || + ac.state == connectivity.TransientFailure || + ac.state == connectivity.Idle { + // We were not connecting, so do nothing but update the addresses. + ac.mu.Unlock() + return + } + + if ac.state == connectivity.Ready { + // Try to find the connected address. + for _, a := range addrs { + a.ServerName = ac.cc.getServerName(a) + if equalAddressIgnoringBalAttributes(&a, &ac.curAddr) { + // We are connected to a valid address, so do nothing but + // update the addresses. + ac.mu.Unlock() + return + } + } + } + + // We are either connected to the wrong address or currently connecting. + // Stop the current iteration and restart. + + ac.cancel() + ac.ctx, ac.cancel = context.WithCancel(ac.cc.ctx) + + // We have to defer here because GracefulClose => onClose, which requires + // locking ac.mu. + if ac.transport != nil { + defer ac.transport.GracefulClose() + ac.transport = nil + } + + if len(addrs) == 0 { + ac.updateConnectivityState(connectivity.Idle, nil) + } + + // Since we were connecting/connected, we should start a new connection + // attempt. + go ac.resetTransportAndUnlock() +} + +// getServerName determines the serverName to be used in the connection +// handshake. The default value for the serverName is the authority on the +// ClientConn, which either comes from the user's dial target or through an +// authority override specified using the WithAuthority dial option. Name +// resolvers can specify a per-address override for the serverName through the +// resolver.Address.ServerName field which is used only if the WithAuthority +// dial option was not used. The rationale is that per-address authority +// overrides specified by the name resolver can represent a security risk, while +// an override specified by the user is more dependable since they probably know +// what they are doing. +func (cc *ClientConn) getServerName(addr resolver.Address) string { + if cc.dopts.authority != "" { + return cc.dopts.authority + } + if addr.ServerName != "" { + return addr.ServerName + } + return cc.authority +} + +func getMethodConfig(sc *ServiceConfig, method string) MethodConfig { + if sc == nil { + return MethodConfig{} + } + if m, ok := sc.Methods[method]; ok { + return m + } + i := strings.LastIndex(method, "/") + if m, ok := sc.Methods[method[:i+1]]; ok { + return m + } + return sc.Methods[""] +} + +// GetMethodConfig gets the method config of the input method. +// If there's an exact match for input method (i.e. /service/method), we return +// the corresponding MethodConfig. +// If there isn't an exact match for the input method, we look for the service's default +// config under the service (i.e /service/) and then for the default for all services (empty string). +// +// If there is a default MethodConfig for the service, we return it. +// Otherwise, we return an empty MethodConfig. +func (cc *ClientConn) GetMethodConfig(method string) MethodConfig { + // TODO: Avoid the locking here. + cc.mu.RLock() + defer cc.mu.RUnlock() + return getMethodConfig(cc.sc, method) +} + +func (cc *ClientConn) healthCheckConfig() *healthCheckConfig { + cc.mu.RLock() + defer cc.mu.RUnlock() + if cc.sc == nil { + return nil + } + return cc.sc.healthCheckConfig +} + +func (cc *ClientConn) applyServiceConfigAndBalancer(sc *ServiceConfig, configSelector iresolver.ConfigSelector) { + if sc == nil { + // should never reach here. + return + } + cc.sc = sc + if configSelector != nil { + cc.safeConfigSelector.UpdateConfigSelector(configSelector) + } + + if cc.sc.retryThrottling != nil { + newThrottler := &retryThrottler{ + tokens: cc.sc.retryThrottling.MaxTokens, + max: cc.sc.retryThrottling.MaxTokens, + thresh: cc.sc.retryThrottling.MaxTokens / 2, + ratio: cc.sc.retryThrottling.TokenRatio, + } + cc.retryThrottler.Store(newThrottler) + } else { + cc.retryThrottler.Store((*retryThrottler)(nil)) + } +} + +func (cc *ClientConn) resolveNow(o resolver.ResolveNowOptions) { + cc.mu.RLock() + cc.resolverWrapper.resolveNow(o) + cc.mu.RUnlock() +} + +func (cc *ClientConn) resolveNowLocked(o resolver.ResolveNowOptions) { + cc.resolverWrapper.resolveNow(o) +} + +// ResetConnectBackoff wakes up all subchannels in transient failure and causes +// them to attempt another connection immediately. It also resets the backoff +// times used for subsequent attempts regardless of the current state. +// +// In general, this function should not be used. Typical service or network +// outages result in a reasonable client reconnection strategy by default. +// However, if a previously unavailable network becomes available, this may be +// used to trigger an immediate reconnect. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func (cc *ClientConn) ResetConnectBackoff() { + cc.mu.Lock() + conns := cc.conns + cc.mu.Unlock() + for ac := range conns { + ac.resetConnectBackoff() + } +} + +// Close tears down the ClientConn and all underlying connections. +func (cc *ClientConn) Close() error { + defer func() { + cc.cancel() + <-cc.csMgr.pubSub.Done() + }() + + // Prevent calls to enter/exit idle immediately, and ensure we are not + // currently entering/exiting idle mode. + cc.idlenessMgr.Close() + + cc.mu.Lock() + if cc.conns == nil { + cc.mu.Unlock() + return ErrClientConnClosing + } + + conns := cc.conns + cc.conns = nil + cc.csMgr.updateState(connectivity.Shutdown) + + // We can safely unlock and continue to access all fields now as + // cc.conns==nil, preventing any further operations on cc. + cc.mu.Unlock() + + cc.resolverWrapper.close() + // Swap in a default ConfigSelector so the resolver-provided one (which may + // transitively retain resolver-owned state such as parsed service configs + // and cluster maps) is released once in-flight RPCs finish selecting on + // it. Done after the resolver has closed so no further UpdateState calls + // race with this swap. + cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{nil}) + // The order of closing matters here since the balancer wrapper assumes the + // picker is closed before it is closed. + cc.pickerWrapper.close() + cc.balancerWrapper.close() + + <-cc.resolverWrapper.serializer.Done() + <-cc.balancerWrapper.serializer.Done() + var wg sync.WaitGroup + for ac := range conns { + wg.Add(1) + go func(ac *addrConn) { + defer wg.Done() + ac.tearDown(ErrClientConnClosing) + }(ac) + } + wg.Wait() + cc.addTraceEvent("deleted") + // TraceEvent needs to be called before RemoveEntry, as TraceEvent may add + // trace reference to the entity being deleted, and thus prevent it from being + // deleted right away. + channelz.RemoveEntry(cc.channelz.ID) + + return nil +} + +// addrConn is a network connection to a given address. +type addrConn struct { + ctx context.Context + cancel context.CancelFunc + + cc *ClientConn + dopts dialOptions + acbw *acBalancerWrapper + scopts balancer.NewSubConnOptions + + // transport is set when there's a viable transport (note: ac state may not be READY as LB channel + // health checking may require server to report healthy to set ac to READY), and is reset + // to nil when the current transport should no longer be used to create a stream (e.g. after GoAway + // is received, transport is closed, ac has been torn down). + transport transport.ClientTransport // The current transport. + + // This mutex is used on the RPC path, so its usage should be minimized as + // much as possible. + // TODO: Find a lock-free way to retrieve the transport and state from the + // addrConn. + mu sync.Mutex + curAddr resolver.Address // The current address. + addrs []resolver.Address // All addresses that the resolver resolved to. + + // Use updateConnectivityState for updating addrConn's connectivity state. + state connectivity.State + + backoffIdx int // Needs to be stateful for resetConnectBackoff. + resetBackoff chan struct{} + + channelz *channelz.SubChannel + + localityLabel string + backendServiceLabel string + disconnectErrorLabel string +} + +// Note: this requires a lock on ac.mu. +func (ac *addrConn) updateConnectivityState(s connectivity.State, lastErr error) { + if ac.state == s { + return + } + + // If we are transitioning out of Ready, it means there is a disconnection. + // A SubConn can also transition from CONNECTING directly to IDLE when + // a transport is successfully created, but the connection fails + // before the SubConn can send the notification for READY. We treat + // this as a successful connection and transition to IDLE. + // TODO: https://github.com/grpc/grpc-go/issues/7862 - Remove the second + // part of the if condition below once the issue is fixed. + if ac.state == connectivity.Ready || (ac.state == connectivity.Connecting && s == connectivity.Idle) { + disconnectError := ac.disconnectErrorLabel + if disconnectError == "" { + disconnectError = "unknown" + } + disconnectionsMetric.Record(ac.cc.metricsRecorderList, 1, ac.cc.target, ac.backendServiceLabel, ac.localityLabel, disconnectError) + openConnectionsMetric.Record(ac.cc.metricsRecorderList, -1, ac.cc.target, ac.backendServiceLabel, ac.securityLevelLocked(), ac.localityLabel) + } + ac.disconnectErrorLabel = "" // Reset for next time + ac.state = s + ac.channelz.ChannelMetrics.State.Store(&s) + if lastErr == nil { + channelz.Infof(logger, ac.channelz, "Subchannel Connectivity change to %v", s) + } else { + channelz.Infof(logger, ac.channelz, "Subchannel Connectivity change to %v, last error: %s", s, lastErr) + } + ac.acbw.updateState(s, lastErr) +} + +// adjustParams updates parameters used to create transports upon +// receiving a GoAway. +func (ac *addrConn) adjustParams(r transport.GoAwayReason) { + if r == transport.GoAwayTooManyPings { + v := 2 * ac.dopts.copts.KeepaliveParams.Time + ac.cc.mu.Lock() + if v > ac.cc.keepaliveParams.Time { + ac.cc.keepaliveParams.Time = v + } + ac.cc.mu.Unlock() + } +} + +// resetTransportAndUnlock unconditionally connects the addrConn. +// +// ac.mu must be held by the caller, and this function will guarantee it is released. +func (ac *addrConn) resetTransportAndUnlock() { + acCtx := ac.ctx + if acCtx.Err() != nil { + ac.mu.Unlock() + return + } + + addrs := ac.addrs + backoffFor := ac.dopts.bs.Backoff(ac.backoffIdx) + // This will be the duration that dial gets to finish. + dialDuration := minConnectTimeout + if ac.dopts.minConnectTimeout != nil { + dialDuration = ac.dopts.minConnectTimeout() + } + + if dialDuration < backoffFor { + // Give dial more time as we keep failing to connect. + dialDuration = backoffFor + } + // We can potentially spend all the time trying the first address, and + // if the server accepts the connection and then hangs, the following + // addresses will never be tried. + // + // The spec doesn't mention what should be done for multiple addresses. + // https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md#proposed-backoff-algorithm + connectDeadline := time.Now().Add(dialDuration) + + ac.updateConnectivityState(connectivity.Connecting, nil) + ac.mu.Unlock() + + if err := ac.tryAllAddrs(acCtx, addrs, connectDeadline); err != nil { + if !errors.Is(err, context.Canceled) { + connectionAttemptsFailedMetric.Record(ac.cc.metricsRecorderList, 1, ac.cc.target, ac.backendServiceLabel, ac.localityLabel) + } else { + if logger.V(2) { + // This records cancelled connection attempts which can be later + // replaced by a metric. + logger.Infof("Context cancellation detected; not recording this as a failed connection attempt.") + } + } + // TODO: #7534 - Move re-resolution requests into the pick_first LB policy + // to ensure one resolution request per pass instead of per subconn failure. + ac.cc.resolveNow(resolver.ResolveNowOptions{}) + ac.mu.Lock() + if acCtx.Err() != nil { + // addrConn was torn down. + ac.mu.Unlock() + return + } + // After exhausting all addresses, the addrConn enters + // TRANSIENT_FAILURE. + ac.updateConnectivityState(connectivity.TransientFailure, err) + + // Backoff. + b := ac.resetBackoff + ac.mu.Unlock() + + timer := time.NewTimer(backoffFor) + select { + case <-timer.C: + ac.mu.Lock() + ac.backoffIdx++ + ac.mu.Unlock() + case <-b: + timer.Stop() + case <-acCtx.Done(): + timer.Stop() + return + } + + ac.mu.Lock() + if acCtx.Err() == nil { + ac.updateConnectivityState(connectivity.Idle, err) + } + ac.mu.Unlock() + return + } + // Success; reset backoff. + ac.mu.Lock() + connectionAttemptsSucceededMetric.Record(ac.cc.metricsRecorderList, 1, ac.cc.target, ac.backendServiceLabel, ac.localityLabel) + openConnectionsMetric.Record(ac.cc.metricsRecorderList, 1, ac.cc.target, ac.backendServiceLabel, ac.securityLevelLocked(), ac.localityLabel) + ac.backoffIdx = 0 + ac.mu.Unlock() +} + +// updateTelemetryLabelsLocked calculates and caches the telemetry labels based on the +// first address in addrConn. +func (ac *addrConn) updateTelemetryLabelsLocked() { + labelsFunc, ok := internal.AddressToTelemetryLabels.(func(resolver.Address) map[string]string) + if !ok || len(ac.addrs) == 0 { + // Reset defaults + ac.localityLabel = "" + ac.backendServiceLabel = "" + return + } + labels := labelsFunc(ac.addrs[0]) + ac.localityLabel = labels["grpc.lb.locality"] + ac.backendServiceLabel = labels["grpc.lb.backend_service"] +} + +type securityLevelKey struct{} + +func (ac *addrConn) securityLevelLocked() string { + var secLevel string + // During disconnection, ac.transport is nil. Fall back to the security level + // stored in the current address during connection. + if ac.transport == nil { + secLevel, _ = ac.curAddr.Attributes.Value(securityLevelKey{}).(string) + return secLevel + } + authInfo := ac.transport.Peer().AuthInfo + if ci, ok := authInfo.(interface { + GetCommonAuthInfo() credentials.CommonAuthInfo + }); ok { + secLevel = ci.GetCommonAuthInfo().SecurityLevel.String() + // Store the security level in the current address' attributes so + // that it remains available for disconnection metrics after the + // transport is closed. + ac.curAddr.Attributes = ac.curAddr.Attributes.WithValue(securityLevelKey{}, secLevel) + } + return secLevel +} + +// tryAllAddrs tries to create a connection to the addresses, and stop when at +// the first successful one. It returns an error if no address was successfully +// connected, or updates ac appropriately with the new transport. +func (ac *addrConn) tryAllAddrs(ctx context.Context, addrs []resolver.Address, connectDeadline time.Time) error { + var firstConnErr error + for _, addr := range addrs { + ac.channelz.ChannelMetrics.Target.Store(&addr.Addr) + if ctx.Err() != nil { + return errConnClosing + } + ac.mu.Lock() + + ac.cc.mu.RLock() + ac.dopts.copts.KeepaliveParams = ac.cc.keepaliveParams + ac.cc.mu.RUnlock() + + copts := ac.dopts.copts + if ac.scopts.CredsBundle != nil { + copts.CredsBundle = ac.scopts.CredsBundle + } + ac.mu.Unlock() + + channelz.Infof(logger, ac.channelz, "Subchannel picks a new address %q to connect", addr.Addr) + + err := ac.createTransport(ctx, addr, copts, connectDeadline) + if err == nil { + return nil + } + if firstConnErr == nil { + firstConnErr = err + } + ac.cc.updateConnectionError(err) + } + + // Couldn't connect to any address. + return firstConnErr +} + +// createTransport creates a connection to addr. It returns an error if the +// address was not successfully connected, or updates ac appropriately with the +// new transport. +func (ac *addrConn) createTransport(ctx context.Context, addr resolver.Address, copts transport.ConnectOptions, connectDeadline time.Time) error { + addr.ServerName = ac.cc.getServerName(addr) + hctx, hcancel := context.WithCancel(ctx) + + onClose := func(info transport.GoAwayInfo) { + ac.mu.Lock() + defer ac.mu.Unlock() + // adjust params based on GoAwayReason + ac.adjustParams(info.Reason) + if ctx.Err() != nil { + // Already shut down or connection attempt canceled. tearDown() or + // updateAddrs() already cleared the transport and canceled hctx + // via ac.ctx, and we expected this connection to be closed, so do + // nothing here. + return + } + hcancel() + if ac.transport == nil { + // We're still connecting to this address, which could error. Do + // not update the connectivity state or resolve; these will happen + // at the end of the tryAllAddrs connection loop in the event of an + // error. + return + } + ac.transport = nil + ac.disconnectErrorLabel = disconnectErrorString(info) + // Refresh the name resolver on any connection loss. + ac.cc.resolveNow(resolver.ResolveNowOptions{}) + // Always go idle and wait for the LB policy to initiate a new + // connection attempt. + ac.updateConnectivityState(connectivity.Idle, nil) + } + + connectCtx, cancel := context.WithDeadline(ctx, connectDeadline) + defer cancel() + copts.ChannelzParent = ac.channelz + + newTr, err := transport.NewHTTP2Client(connectCtx, ac.cc.ctx, addr, copts, onClose) + if err != nil { + if logger.V(2) { + logger.Infof("Creating new client transport to %q: %v", addr, err) + } + // newTr is either nil, or closed. + hcancel() + channelz.Warningf(logger, ac.channelz, "grpc: addrConn.createTransport failed to connect to %s. Err: %v", addr, err) + return err + } + + ac.mu.Lock() + if ctx.Err() != nil { + // This can happen if the subConn was removed while in `Connecting` + // state. tearDown() would have set the state to `Shutdown`, but + // would not have closed the transport since ac.transport would not + // have been set at that point. + + // We unlock ac.mu because newTr.Close() calls onClose() + // inline, which requires locking ac.mu. + ac.mu.Unlock() + + // The error we pass to Close() is immaterial since there are no open + // streams at this point, so no trailers with error details will be sent + // out. We just need to pass a non-nil error. + // + // This can also happen when updateAddrs is called during a connection + // attempt. + newTr.Close(transport.ErrConnClosing) + return nil + } + defer ac.mu.Unlock() + if hctx.Err() != nil { + // onClose was already called for this connection, but the connection + // was successfully established first. Consider it a success and set + // the new state to Idle. + ac.updateConnectivityState(connectivity.Idle, nil) + return nil + } + ac.curAddr = addr + ac.transport = newTr + ac.startHealthCheck(hctx) // Will set state to READY if appropriate. + return nil +} + +// disconnectErrorString returns the grpc.disconnect_error metric label corresponding +// to the provided transport.GoAwayInfo, as specified by gRFC A94: +// https://github.com/grpc/proposal/blob/master/A94-grpc-subchannel-disconnections-metrics.md +func disconnectErrorString(info transport.GoAwayInfo) string { + if info.Reason != transport.GoAwayInvalid { + return fmt.Sprintf("GOAWAY %s", info.GoAwayCode.String()) + } + if info.Err == nil { + return "unknown" + } + return disconnectErrorLabel(info.Err) +} + +// startHealthCheck starts the health checking stream (RPC) to watch the health +// stats of this connection if health checking is requested and configured. +// +// LB channel health checking is enabled when all requirements below are met: +// 1. it is not disabled by the user with the WithDisableHealthCheck DialOption +// 2. internal.HealthCheckFunc is set by importing the grpc/health package +// 3. a service config with non-empty healthCheckConfig field is provided +// 4. the load balancer requests it +// +// It sets addrConn to READY if the health checking stream is not started. +// +// Caller must hold ac.mu. +func (ac *addrConn) startHealthCheck(ctx context.Context) { + var healthcheckManagingState bool + defer func() { + if !healthcheckManagingState { + ac.updateConnectivityState(connectivity.Ready, nil) + } + }() + + if ac.cc.dopts.disableHealthCheck { + return + } + healthCheckConfig := ac.cc.healthCheckConfig() + if healthCheckConfig == nil { + return + } + if !ac.scopts.HealthCheckEnabled { + return + } + healthCheckFunc := internal.HealthCheckFunc + if healthCheckFunc == nil { + // The health package is not imported to set health check function. + // + // TODO: add a link to the health check doc in the error message. + channelz.Error(logger, ac.channelz, "Health check is requested but health check function is not set.") + return + } + + healthcheckManagingState = true + + // Set up the health check helper functions. + currentTr := ac.transport + newStream := func(method string) (any, error) { + ac.mu.Lock() + if ac.transport != currentTr { + ac.mu.Unlock() + return nil, status.Error(codes.Canceled, "the provided transport is no longer valid to use") + } + ac.mu.Unlock() + return newNonRetryClientStream(ctx, &StreamDesc{ServerStreams: true}, method, currentTr, ac) + } + setConnectivityState := func(s connectivity.State, lastErr error) { + ac.mu.Lock() + defer ac.mu.Unlock() + if ac.transport != currentTr { + return + } + ac.updateConnectivityState(s, lastErr) + } + // Start the health checking stream. + go func() { + err := healthCheckFunc(ctx, newStream, setConnectivityState, healthCheckConfig.ServiceName) + if err != nil { + if status.Code(err) == codes.Unimplemented { + channelz.Error(logger, ac.channelz, "Subchannel health check is unimplemented at server side, thus health check is disabled") + } else { + channelz.Errorf(logger, ac.channelz, "Health checking failed: %v", err) + } + } + }() +} + +func (ac *addrConn) resetConnectBackoff() { + ac.mu.Lock() + close(ac.resetBackoff) + ac.backoffIdx = 0 + ac.resetBackoff = make(chan struct{}) + ac.mu.Unlock() +} + +// getReadyTransport returns the transport if ac's state is READY or nil if not. +func (ac *addrConn) getReadyTransport() transport.ClientTransport { + ac.mu.Lock() + defer ac.mu.Unlock() + if ac.state == connectivity.Ready { + return ac.transport + } + return nil +} + +// tearDown starts to tear down the addrConn. +// +// Note that tearDown doesn't remove ac from ac.cc.conns, so the addrConn struct +// will leak. In most cases, call cc.removeAddrConn() instead. +func (ac *addrConn) tearDown(err error) { + ac.mu.Lock() + if ac.state == connectivity.Shutdown { + ac.mu.Unlock() + return + } + curTr := ac.transport + ac.transport = nil + if ac.disconnectErrorLabel == "" { + ac.disconnectErrorLabel = "subchannel shutdown" + } + // We have to set the state to Shutdown before anything else to prevent races + // between setting the state and logic that waits on context cancellation / etc. + ac.updateConnectivityState(connectivity.Shutdown, nil) + ac.cancel() + ac.curAddr = resolver.Address{} + + channelz.AddTraceEvent(logger, ac.channelz, 0, &channelz.TraceEvent{ + Desc: "Subchannel deleted", + Severity: channelz.CtInfo, + Parent: &channelz.TraceEvent{ + Desc: fmt.Sprintf("Subchannel(id:%d) deleted", ac.channelz.ID), + Severity: channelz.CtInfo, + }, + }) + // TraceEvent needs to be called before RemoveEntry, as TraceEvent may add + // trace reference to the entity being deleted, and thus prevent it from + // being deleted right away. + channelz.RemoveEntry(ac.channelz.ID) + ac.mu.Unlock() + + // We have to release the lock before the call to GracefulClose/Close here + // because both of them call onClose(), which requires locking ac.mu. + if curTr != nil { + if err == errConnDrain { + // Close the transport gracefully when the subConn is being shutdown. + // + // GracefulClose() may be executed multiple times if: + // - multiple GoAway frames are received from the server + // - there are concurrent name resolver or balancer triggered + // address removal and GoAway + curTr.GracefulClose() + } else { + // Hard close the transport when the channel is entering idle or is + // being shutdown. In the case where the channel is being shutdown, + // closing of transports is also taken care of by cancellation of cc.ctx. + // But in the case where the channel is entering idle, we need to + // explicitly close the transports here. Instead of distinguishing + // between these two cases, it is simpler to close the transport + // unconditionally here. + curTr.Close(err) + } + } +} + +type retryThrottler struct { + max float64 + thresh float64 + ratio float64 + + mu sync.Mutex + tokens float64 // TODO(dfawley): replace with atomic and remove lock. +} + +// throttle subtracts a retry token from the pool and returns whether a retry +// should be throttled (disallowed) based upon the retry throttling policy in +// the service config. +func (rt *retryThrottler) throttle() bool { + if rt == nil { + return false + } + rt.mu.Lock() + defer rt.mu.Unlock() + rt.tokens-- + if rt.tokens < 0 { + rt.tokens = 0 + } + return rt.tokens <= rt.thresh +} + +func (rt *retryThrottler) successfulRPC() { + if rt == nil { + return + } + rt.mu.Lock() + defer rt.mu.Unlock() + rt.tokens += rt.ratio + if rt.tokens > rt.max { + rt.tokens = rt.max + } +} + +func (ac *addrConn) incrCallsStarted() { + ac.channelz.ChannelMetrics.CallsStarted.Add(1) + ac.channelz.ChannelMetrics.LastCallStartedTimestamp.Store(time.Now().UnixNano()) +} + +func (ac *addrConn) incrCallsSucceeded() { + ac.channelz.ChannelMetrics.CallsSucceeded.Add(1) +} + +func (ac *addrConn) incrCallsFailed() { + ac.channelz.ChannelMetrics.CallsFailed.Add(1) +} + +// ErrClientConnTimeout indicates that the ClientConn cannot establish the +// underlying connections within the specified timeout. +// +// Deprecated: This error is never returned by grpc and should not be +// referenced by users. +var ErrClientConnTimeout = errors.New("grpc: timed out when dialing") + +// getResolver finds the scheme in the cc's resolvers or the global registry. +// scheme should always be lowercase (typically by virtue of url.Parse() +// performing proper RFC3986 behavior). +func (cc *ClientConn) getResolver(scheme string) resolver.Builder { + for _, rb := range cc.dopts.resolvers { + if scheme == rb.Scheme() { + return rb + } + } + return resolver.Get(scheme) +} + +func (cc *ClientConn) updateConnectionError(err error) { + cc.lceMu.Lock() + cc.lastConnectionError = err + cc.lceMu.Unlock() +} + +func (cc *ClientConn) connectionError() error { + cc.lceMu.Lock() + defer cc.lceMu.Unlock() + return cc.lastConnectionError +} + +// initParsedTargetAndResolverBuilder parses the user's dial target and stores +// the parsed target in `cc.parsedTarget`. +// +// The resolver to use is determined based on the scheme in the parsed target +// and the same is stored in `cc.resolverBuilder`. +// +// Doesn't grab cc.mu as this method is expected to be called only at Dial time. +func (cc *ClientConn) initParsedTargetAndResolverBuilder() error { + logger.Infof("original dial target is: %q", cc.target) + + var rb resolver.Builder + parsedTarget, err := parseTarget(cc.target) + if err == nil { + rb = cc.getResolver(parsedTarget.URL.Scheme) + if rb != nil { + cc.parsedTarget = parsedTarget + cc.resolverBuilder = rb + return nil + } + } + + // We are here because the user's dial target did not contain a scheme or + // specified an unregistered scheme. We should fallback to the default + // scheme, except when a custom dialer is specified in which case, we should + // always use passthrough scheme. For either case, we need to respect any overridden + // global defaults set by the user. + defScheme := cc.dopts.defaultScheme + if internal.UserSetDefaultScheme { + defScheme = resolver.GetDefaultScheme() + } + + canonicalTarget := defScheme + ":///" + cc.target + + parsedTarget, err = parseTarget(canonicalTarget) + if err != nil { + return err + } + rb = cc.getResolver(parsedTarget.URL.Scheme) + if rb == nil { + return fmt.Errorf("could not get resolver for default scheme: %q", parsedTarget.URL.Scheme) + } + cc.parsedTarget = parsedTarget + cc.resolverBuilder = rb + return nil +} + +// parseTarget uses RFC 3986 semantics to parse the given target into a +// resolver.Target struct containing url. Query params are stripped from the +// endpoint. +func parseTarget(target string) (resolver.Target, error) { + u, err := url.Parse(target) + if err != nil { + return resolver.Target{}, err + } + + return resolver.Target{URL: *u}, nil +} + +// encodeAuthority escapes the authority string based on valid chars defined in +// https://datatracker.ietf.org/doc/html/rfc3986#section-3.2. +func encodeAuthority(authority string) string { + const upperhex = "0123456789ABCDEF" + + // Return for characters that must be escaped as per + // Valid chars are mentioned here: + // https://datatracker.ietf.org/doc/html/rfc3986#section-3.2 + shouldEscape := func(c byte) bool { + // Alphanum are always allowed. + if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' { + return false + } + switch c { + case '-', '_', '.', '~': // Unreserved characters + return false + case '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=': // Subdelim characters + return false + case ':', '[', ']', '@': // Authority related delimiters + return false + } + // Everything else must be escaped. + return true + } + + hexCount := 0 + for i := 0; i < len(authority); i++ { + c := authority[i] + if shouldEscape(c) { + hexCount++ + } + } + + if hexCount == 0 { + return authority + } + + required := len(authority) + 2*hexCount + t := make([]byte, required) + + j := 0 + // This logic is a barebones version of escape in the go net/url library. + for i := 0; i < len(authority); i++ { + switch c := authority[i]; { + case shouldEscape(c): + t[j] = '%' + t[j+1] = upperhex[c>>4] + t[j+2] = upperhex[c&15] + j += 3 + default: + t[j] = authority[i] + j++ + } + } + return string(t) +} + +// Determine channel authority. The order of precedence is as follows: +// - user specified authority override using `WithAuthority` dial option +// - creds' notion of server name for the authentication handshake +// - endpoint from dial target of the form "scheme://[authority]/endpoint" +// +// Stores the determined authority in `cc.authority`. +// +// Returns a non-nil error if the authority returned by the transport +// credentials do not match the authority configured through the dial option. +// +// Doesn't grab cc.mu as this method is expected to be called only at Dial time. +func (cc *ClientConn) initAuthority() error { + dopts := cc.dopts + // Historically, we had two options for users to specify the serverName or + // authority for a channel. One was through the transport credentials + // (either in its constructor, or through the OverrideServerName() method). + // The other option (for cases where WithInsecure() dial option was used) + // was to use the WithAuthority() dial option. + // + // A few things have changed since: + // - `insecure` package with an implementation of the `TransportCredentials` + // interface for the insecure case + // - WithAuthority() dial option support for secure credentials + authorityFromCreds := "" + if creds := dopts.copts.TransportCredentials; creds != nil && creds.Info().ServerName != "" { + authorityFromCreds = creds.Info().ServerName + } + authorityFromDialOption := dopts.authority + if (authorityFromCreds != "" && authorityFromDialOption != "") && authorityFromCreds != authorityFromDialOption { + return fmt.Errorf("ClientConn's authority from transport creds %q and dial option %q don't match", authorityFromCreds, authorityFromDialOption) + } + + endpoint := cc.parsedTarget.Endpoint() + if authorityFromDialOption != "" { + cc.authority = authorityFromDialOption + } else if authorityFromCreds != "" { + cc.authority = authorityFromCreds + } else if auth, ok := cc.resolverBuilder.(resolver.AuthorityOverrider); ok { + cc.authority = auth.OverrideAuthority(cc.parsedTarget) + } else if strings.HasPrefix(endpoint, ":") { + cc.authority = "localhost" + encodeAuthority(endpoint) + } else { + cc.authority = encodeAuthority(endpoint) + } + return nil +} diff --git a/clientconn_authority_test.go b/clientconn_authority_test.go new file mode 100644 index 0000000..4c45461 --- /dev/null +++ b/clientconn_authority_test.go @@ -0,0 +1,135 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "net" + "testing" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/testdata" +) + +func (s) TestClientConnAuthority(t *testing.T) { + serverNameOverride := "over.write.server.name" + creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), serverNameOverride) + if err != nil { + t.Fatalf("credentials.NewClientTLSFromFile(_, %q) failed: %v", err, serverNameOverride) + } + + tests := []struct { + name string + target string + opts []DialOption + wantAuthority string + }{ + { + name: "default", + target: "Non-Existent.Server:8080", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials())}, + wantAuthority: "Non-Existent.Server:8080", + }, + { + name: "override-via-creds", + target: "Non-Existent.Server:8080", + opts: []DialOption{WithTransportCredentials(creds)}, + wantAuthority: serverNameOverride, + }, + { + name: "override-via-WithAuthority", + target: "Non-Existent.Server:8080", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials()), WithAuthority("authority-override")}, + wantAuthority: "authority-override", + }, + { + name: "override-via-creds-and-WithAuthority", + target: "Non-Existent.Server:8080", + opts: []DialOption{WithTransportCredentials(creds), WithAuthority(serverNameOverride)}, + wantAuthority: serverNameOverride, + }, + { + name: "unix relative", + target: "unix:sock.sock", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials())}, + wantAuthority: "localhost", + }, + { + name: "unix relative with custom dialer", + target: "unix:sock.sock", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials()), WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "", addr) + })}, + wantAuthority: "localhost", + }, + { + name: "unix absolute", + target: "unix:/sock.sock", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials())}, + wantAuthority: "localhost", + }, + { + name: "unix absolute with custom dialer", + target: "unix:///sock.sock", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials()), WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "", addr) + })}, + wantAuthority: "localhost", + }, + { + name: "localhost colon port", + target: "localhost:50051", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials())}, + wantAuthority: "localhost:50051", + }, + { + name: "colon port", + target: ":50051", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials())}, + wantAuthority: "localhost:50051", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cc, err := NewClient(test.target, test.opts...) + if err != nil { + t.Fatalf("NewClient(%q) failed: %v", test.target, err) + } + defer cc.Close() + if cc.authority != test.wantAuthority { + t.Fatalf("cc.authority = %q, want %q", cc.authority, test.wantAuthority) + } + }) + } +} + +func (s) TestClientConnAuthority_CredsAndDialOptionMismatch(t *testing.T) { + serverNameOverride := "over.write.server.name" + creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), serverNameOverride) + if err != nil { + t.Fatalf("credentials.NewClientTLSFromFile(_, %q) failed: %v", err, serverNameOverride) + } + opts := []DialOption{WithTransportCredentials(creds), WithAuthority("authority-override")} + if cc, err := NewClient("Non-Existent.Server:8000", opts...); err == nil { + cc.Close() + t.Fatal("NewClient() succeeded when expected to fail") + } +} diff --git a/clientconn_disconnect_reason_noplan9.go b/clientconn_disconnect_reason_noplan9.go new file mode 100644 index 0000000..f0fcd88 --- /dev/null +++ b/clientconn_disconnect_reason_noplan9.go @@ -0,0 +1,48 @@ +//go:build !plan9 + +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "errors" + "os" + "syscall" +) + +// disconnectErrorLabel returns the grpc.disconnect_error metric label for a +// transport error, as specified by gRFC A94. +func disconnectErrorLabel(err error) string { + var sysErr syscall.Errno + switch { + case errors.Is(err, context.Canceled): + return "subchannel shutdown" + case errors.Is(err, syscall.ECONNRESET): + return "connection reset" + case errors.Is(err, syscall.ETIMEDOUT), errors.Is(err, context.DeadlineExceeded), errors.Is(err, os.ErrDeadlineExceeded): + return "connection timed out" + case errors.Is(err, syscall.ECONNABORTED): + return "connection aborted" + case errors.As(err, &sysErr): + return "socket error" + default: + return "unknown" + } +} diff --git a/clientconn_disconnect_reason_plan9.go b/clientconn_disconnect_reason_plan9.go new file mode 100644 index 0000000..930b126 --- /dev/null +++ b/clientconn_disconnect_reason_plan9.go @@ -0,0 +1,39 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "errors" + "os" +) + +// disconnectErrorLabel returns the grpc.disconnect_error metric label for a +// transport error, as specified by gRFC A94. syscall.Errno does not exist on +// plan9, so only the portable classifications are available. +func disconnectErrorLabel(err error) string { + switch { + case errors.Is(err, context.Canceled): + return "subchannel shutdown" + case errors.Is(err, context.DeadlineExceeded), errors.Is(err, os.ErrDeadlineExceeded): + return "connection timed out" + default: + return "unknown" + } +} diff --git a/clientconn_parsed_target_test.go b/clientconn_parsed_target_test.go new file mode 100644 index 0000000..57ae3ad --- /dev/null +++ b/clientconn_parsed_target_test.go @@ -0,0 +1,332 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" +) + +func generateTarget(target string) resolver.Target { + return resolver.Target{URL: *testutils.MustParseURL(target)} +} + +// Resets the default scheme as though it was never set by the user. +func resetInitialResolverState() { + resolver.SetDefaultScheme("passthrough") + internal.UserSetDefaultScheme = false +} + +type testResolverForParser struct { + resolver.Resolver +} + +func (testResolverForParser) Build(resolver.Target, resolver.ClientConn, resolver.BuildOptions) (resolver.Resolver, error) { + return testResolverForParser{}, nil +} + +func (testResolverForParser) Close() {} + +func (testResolverForParser) Scheme() string { + return "testresolverforparser" +} + +func init() { resolver.Register(testResolverForParser{}) } + +func (s) TestParsedTarget_Success_WithoutCustomDialer(t *testing.T) { + tests := []struct { + target string + wantDialParse resolver.Target + wantNewClientParse resolver.Target + wantCustomParse resolver.Target + }{ + // No scheme is specified. + { + target: "://a/b", + wantDialParse: generateTarget("passthrough:///://a/b"), + wantNewClientParse: generateTarget("dns:///://a/b"), + wantCustomParse: generateTarget("testresolverforparser:///://a/b"), + }, + { + target: "a//b", + wantDialParse: generateTarget("passthrough:///a//b"), + wantNewClientParse: generateTarget("dns:///a//b"), + wantCustomParse: generateTarget("testresolverforparser:///a//b"), + }, + + // An unregistered scheme is specified. + { + target: "a:///", + wantDialParse: generateTarget("passthrough:///a:///"), + wantNewClientParse: generateTarget("dns:///a:///"), + wantCustomParse: generateTarget("testresolverforparser:///a:///"), + }, + { + target: "a:b", + wantDialParse: generateTarget("passthrough:///a:b"), + wantNewClientParse: generateTarget("dns:///a:b"), + wantCustomParse: generateTarget("testresolverforparser:///a:b"), + }, + + // A registered scheme is specified. + { + target: "dns://a.server.com/google.com", + wantDialParse: generateTarget("dns://a.server.com/google.com"), + wantNewClientParse: generateTarget("dns://a.server.com/google.com"), + wantCustomParse: generateTarget("dns://a.server.com/google.com"), + }, + { + target: "unix-abstract:/ a///://::!@#$%25^&*()b", + wantDialParse: generateTarget("unix-abstract:/ a///://::!@#$%25^&*()b"), + wantNewClientParse: generateTarget("unix-abstract:/ a///://::!@#$%25^&*()b"), + wantCustomParse: generateTarget("unix-abstract:/ a///://::!@#$%25^&*()b"), + }, + { + target: "unix-abstract:passthrough:abc", + wantDialParse: generateTarget("unix-abstract:passthrough:abc"), + wantNewClientParse: generateTarget("unix-abstract:passthrough:abc"), + wantCustomParse: generateTarget("unix-abstract:passthrough:abc"), + }, + { + target: "passthrough:///unix:///a/b/c", + wantDialParse: generateTarget("passthrough:///unix:///a/b/c"), + wantNewClientParse: generateTarget("passthrough:///unix:///a/b/c"), + wantCustomParse: generateTarget("passthrough:///unix:///a/b/c"), + }, + + // Cases for `scheme:absolute-path`. + { + target: "dns:/a/b/c", + wantDialParse: generateTarget("dns:/a/b/c"), + wantNewClientParse: generateTarget("dns:/a/b/c"), + wantCustomParse: generateTarget("dns:/a/b/c"), + }, + { + target: "unregistered:/a/b/c", + wantDialParse: generateTarget("passthrough:///unregistered:/a/b/c"), + wantNewClientParse: generateTarget("dns:///unregistered:/a/b/c"), + wantCustomParse: generateTarget("testresolverforparser:///unregistered:/a/b/c"), + }, + } + + for _, test := range tests { + t.Run(test.target, func(t *testing.T) { + resetInitialResolverState() + cc, err := Dial(test.target, WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Dial(%q) failed: %v", test.target, err) + } + cc.Close() + + if !cmp.Equal(cc.parsedTarget, test.wantDialParse) { + t.Errorf("cc.parsedTarget for dial target %q = %+v, want %+v", test.target, cc.parsedTarget, test.wantDialParse) + } + + cc, err = NewClient(test.target, WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("NewClient(%q) failed: %v", test.target, err) + } + cc.Close() + + if !cmp.Equal(cc.parsedTarget, test.wantNewClientParse) { + t.Errorf("cc.parsedTarget for newClient target %q = %+v, want %+v", test.target, cc.parsedTarget, test.wantNewClientParse) + } + + resolver.SetDefaultScheme("testresolverforparser") + cc, err = Dial(test.target, WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Dial(%q) failed: %v", test.target, err) + } + cc.Close() + + if !cmp.Equal(cc.parsedTarget, test.wantCustomParse) { + t.Errorf("cc.parsedTarget for dial target %q = %+v, want %+v", test.target, cc.parsedTarget, test.wantDialParse) + } + + cc, err = NewClient(test.target, WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("NewClient(%q) failed: %v", test.target, err) + } + cc.Close() + + if !cmp.Equal(cc.parsedTarget, test.wantCustomParse) { + t.Errorf("cc.parsedTarget for newClient target %q = %+v, want %+v", test.target, cc.parsedTarget, test.wantNewClientParse) + } + + }) + } + resetInitialResolverState() +} + +func (s) TestParsedTarget_Failure_WithoutCustomDialer(t *testing.T) { + targets := []string{ + "", + "unix://a/b/c", + "unix://authority", + "unix-abstract://authority/a/b/c", + "unix-abstract://authority", + } + + for _, target := range targets { + t.Run(target, func(t *testing.T) { + if cc, err := Dial(target, WithTransportCredentials(insecure.NewCredentials())); err == nil { + defer cc.Close() + t.Fatalf("Dial(%q) succeeded cc.parsedTarget = %+v, expected to fail", target, cc.parsedTarget) + } + }) + } +} + +func (s) TestParsedTarget_Failure_WithoutCustomDialer_WithNewClient(t *testing.T) { + tests := []struct { + target string + wantErrSubstr string + }{ + + {target: "", wantErrSubstr: "invalid target address"}, + {target: "unix://a/b/c", wantErrSubstr: "invalid (non-empty) authority"}, + {target: "unix://authority", wantErrSubstr: "invalid (non-empty) authority"}, + {target: "unix-abstract://authority/a/b/c", wantErrSubstr: "invalid (non-empty) authority"}, + {target: "unix-abstract://authority", wantErrSubstr: "invalid (non-empty) authority"}, + } + + for _, test := range tests { + t.Run(test.target, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc, err := NewClient(test.target, WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("NewClient(%q) failed: %v", test, err) + } + defer cc.Close() + if _, err := cc.NewStream(ctx, &StreamDesc{}, "/my.service.v1.MyService/UnaryCall"); err == nil { + t.Fatalf("NewStream() succeeded with target = %q, cc.parsedTarget = %+v, expected to fail", test, cc.parsedTarget) + } else if !strings.Contains(err.Error(), test.wantErrSubstr) { + t.Fatalf("NewStream() with target = %q returned unexpected error: got %v, want substring %q", test, err, test.wantErrSubstr) + } + }) + } +} + +func (s) TestParsedTarget_WithCustomDialer(t *testing.T) { + resetInitialResolverState() + defScheme := resolver.GetDefaultScheme() + tests := []struct { + target string + wantParsed resolver.Target + wantDialerAddress string + }{ + // unix:[local_path], unix:[/absolute], and unix://[/absolute] have + // different behaviors with a custom dialer. + { + target: "unix:a/b/c", + wantParsed: resolver.Target{URL: *testutils.MustParseURL("unix:a/b/c")}, + wantDialerAddress: "unix:a/b/c", + }, + { + target: "unix:/a/b/c", + wantParsed: resolver.Target{URL: *testutils.MustParseURL("unix:/a/b/c")}, + wantDialerAddress: "unix:///a/b/c", + }, + { + target: "unix:///a/b/c", + wantParsed: resolver.Target{URL: *testutils.MustParseURL("unix:///a/b/c")}, + wantDialerAddress: "unix:///a/b/c", + }, + { + target: "dns:///127.0.0.1:50051", + wantParsed: resolver.Target{URL: *testutils.MustParseURL("dns:///127.0.0.1:50051")}, + wantDialerAddress: "127.0.0.1:50051", + }, + { + target: ":///127.0.0.1:50051", + wantParsed: resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("%s:///%s", defScheme, ":///127.0.0.1:50051"))}, + wantDialerAddress: ":///127.0.0.1:50051", + }, + { + target: "dns://authority/127.0.0.1:50051", + wantParsed: resolver.Target{URL: *testutils.MustParseURL("dns://authority/127.0.0.1:50051")}, + wantDialerAddress: "127.0.0.1:50051", + }, + { + target: "://authority/127.0.0.1:50051", + wantParsed: resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("%s:///%s", defScheme, "://authority/127.0.0.1:50051"))}, + wantDialerAddress: "://authority/127.0.0.1:50051", + }, + { + target: "/unix/socket/address", + wantParsed: resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("%s:///%s", defScheme, "/unix/socket/address"))}, + wantDialerAddress: "/unix/socket/address", + }, + { + target: "", + wantParsed: resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("%s:///%s", defScheme, ""))}, + wantDialerAddress: "", + }, + { + target: "passthrough://a.server.com/google.com", + wantParsed: resolver.Target{URL: *testutils.MustParseURL("passthrough://a.server.com/google.com")}, + wantDialerAddress: "google.com", + }, + } + + for _, test := range tests { + t.Run(test.target, func(t *testing.T) { + addrCh := make(chan string, 1) + dialer := func(ctx context.Context, address string) (net.Conn, error) { + select { + case addrCh <- address: + return nil, errors.New("dialer error") + case <-ctx.Done(): + return nil, ctx.Err() + } + } + + cc, err := NewClient(test.target, WithTransportCredentials(insecure.NewCredentials()), withDefaultScheme(defScheme), WithContextDialer(dialer)) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", test.target, err) + } + defer cc.Close() + cc.Connect() + + select { + case addr := <-addrCh: + if addr != test.wantDialerAddress { + t.Fatalf("address in custom dialer is %q, want %q", addr, test.wantDialerAddress) + } + case <-time.After(time.Second): + t.Fatal("timeout when waiting for custom dialer to be invoked") + } + if !cmp.Equal(cc.parsedTarget, test.wantParsed) { + t.Errorf("cc.parsedTarget for dial target %q = %+v, want %+v", test.target, cc.parsedTarget, test.wantParsed) + } + }) + } +} diff --git a/clientconn_test.go b/clientconn_test.go new file mode 100644 index 0000000..f1e9771 --- /dev/null +++ b/clientconn_test.go @@ -0,0 +1,1079 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "errors" + "fmt" + "math" + "net" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "golang.org/x/net/http2" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + internalbackoff "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/testdata" +) + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + stateRecordingBalancerName = "state_recording_balancer" + grpclbServiceConfig = `{"loadBalancingConfig": [{"grpclb": {}}]}` + rrServiceConfig = `{"loadBalancingPolicy": [{"round_robin": {}}]}` +) + +var testBalancerBuilder = newStateRecordingBalancerBuilder() + +func init() { + balancer.Register(testBalancerBuilder) +} + +func parseCfg(r *manual.Resolver, s string) *serviceconfig.ParseResult { + scpr := r.CC().ParseServiceConfig(s) + if scpr.Err != nil { + panic(fmt.Sprintf("Error parsing config %q: %v", s, scpr.Err)) + } + return scpr +} + +func (s) TestNewClientWithMultipleBackendsNotSendingServerPreface(t *testing.T) { + lis1, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis1.Close() + lis1Addr := resolver.Address{Addr: lis1.Addr().String()} + lis1Done := make(chan struct{}) + // 1st listener accepts the connection and immediately closes it. + go func() { + defer close(lis1Done) + conn, err := lis1.Accept() + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + conn.Close() + }() + + lis2, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis2.Close() + lis2Done := make(chan struct{}) + lis2Addr := resolver.Address{Addr: lis2.Addr().String()} + // 2nd listener should get a connection attempt since the first one failed. + go func() { + defer close(lis2Done) + _, err := lis2.Accept() // Closing the client will clean up this conn. + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + }() + + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{lis1Addr, lis2Addr}}) + client, err := NewClient(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + client.Connect() + defer client.Close() + timeout := time.After(5 * time.Second) + select { + case <-timeout: + t.Fatal("timed out waiting for server 1 to finish") + case <-lis1Done: + } + select { + case <-timeout: + t.Fatal("timed out waiting for server 2 to finish") + case <-lis2Done: + } +} + +// 1. Client connects to a server that doesn't send preface. +// 2. After minConnectTimeout(500 ms here), client disconnects and retries. +// 3. The new server sends its preface. +// 4. Client doesn't kill the connection this time. +func (s) TestCloseConnectionWhenServerPrefaceNotReceived(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + var ( + conn2 net.Conn + over uint32 + ) + defer func() { + lis.Close() + // conn2 shouldn't be closed until the client has + // observed a successful test. + if conn2 != nil { + conn2.Close() + } + }() + done := make(chan struct{}) + accepted := make(chan struct{}) + go func() { // Launch the server. + defer close(done) + conn1, err := lis.Accept() + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + defer conn1.Close() + // Don't send server settings and the client should close the connection and try again. + conn2, err = lis.Accept() // Accept a reconnection request from client. + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + close(accepted) + framer := http2.NewFramer(conn2, conn2) + if err = framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings. Err: %v", err) + return + } + b := make([]byte, 8) + for { + _, err = conn2.Read(b) + if err == nil { + continue + } + if atomic.LoadUint32(&over) == 1 { + // The connection stayed alive for the timer. + // Success. + return + } + t.Errorf("Unexpected error while reading. Err: %v, want timeout error", err) + break + } + }() + client, err := NewClient(lis.Addr().String(), WithTransportCredentials(insecure.NewCredentials()), withMinConnectDeadline(func() time.Duration { return time.Millisecond * 500 })) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err) + } + + go stayConnected(client) + + // wait for connection to be accepted on the server. + timer := time.NewTimer(time.Second * 10) + select { + case <-accepted: + case <-timer.C: + t.Fatalf("Client didn't make another connection request in time.") + } + // Make sure the connection stays alive for some time. + time.Sleep(time.Second) + atomic.StoreUint32(&over, 1) + client.Close() + <-done +} + +func (s) TestBackoffWhenNoServerPrefaceReceived(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Unexpected error from net.Listen(%q, %q): %v", "tcp", "localhost:0", err) + } + defer lis.Close() + done := make(chan struct{}) + go func() { // Launch the server. + defer close(done) + conn, err := lis.Accept() // Accept the connection only to close it immediately. + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + prevAt := time.Now() + conn.Close() + var prevDuration time.Duration + // Make sure the retry attempts are backed off properly. + for i := 0; i < 3; i++ { + conn, err := lis.Accept() + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + meow := time.Now() + conn.Close() + dr := meow.Sub(prevAt) + if dr <= prevDuration { + t.Errorf("Client backoff did not increase with retries. Previous duration: %v, current duration: %v", prevDuration, dr) + return + } + prevDuration = dr + prevAt = meow + } + }() + bc := backoff.Config{ + BaseDelay: 200 * time.Millisecond, + Multiplier: 2.0, + Jitter: 0, + MaxDelay: 120 * time.Second, + } + cp := ConnectParams{ + Backoff: bc, + MinConnectTimeout: 1 * time.Second, + } + cc, err := NewClient(lis.Addr().String(), WithTransportCredentials(insecure.NewCredentials()), WithConnectParams(cp)) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err) + } + defer cc.Close() + go stayConnected(cc) + <-done +} + +// When creating a transport configured with n addresses, only calculate the +// backoff once per "round" of attempts instead of once per address (n times +// per "round" of attempts) for old pickfirst and once per address for new pickfirst. +func (s) TestNewClient_BackoffCountPerRetryGroup(t *testing.T) { + var attempts uint32 + wantBackoffs := uint32(2) + getMinConnectTimeout := func() time.Duration { + if atomic.AddUint32(&attempts, 1) <= wantBackoffs { + // Once all addresses are exhausted, hang around and wait for the + // client.Close to happen rather than re-starting a new round of + // attempts. + return time.Hour + } + t.Errorf("only %d attempt backoff calculation, but got more", wantBackoffs) + return 0 + } + + lis1, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis1.Close() + + lis2, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis2.Close() + + server1Done := make(chan struct{}) + server2Done := make(chan struct{}) + + // Launch server 1. + go func() { + conn, err := lis1.Accept() + if err != nil { + t.Error(err) + return + } + + conn.Close() + close(server1Done) + }() + // Launch server 2. + go func() { + conn, err := lis2.Accept() + if err != nil { + t.Error(err) + return + } + conn.Close() + close(server2Done) + }() + + rb := manual.NewBuilderWithScheme("whatever") + rb.InitialState(resolver.State{Addresses: []resolver.Address{ + {Addr: lis1.Addr().String()}, + {Addr: lis2.Addr().String()}, + }}) + client, err := NewClient("whatever:///this-gets-overwritten", + WithTransportCredentials(insecure.NewCredentials()), + WithResolvers(rb), + withMinConnectDeadline(getMinConnectTimeout)) + if err != nil { + t.Fatal(err) + } + defer client.Close() + client.Connect() + timeout := time.After(15 * time.Second) + + select { + case <-timeout: + t.Fatal("timed out waiting for test to finish") + case <-server1Done: + } + + select { + case <-timeout: + t.Fatal("timed out waiting for test to finish") + case <-server2Done: + } + + if got, want := atomic.LoadUint32(&attempts), wantBackoffs; got != want { + t.Errorf("attempts = %d, want %d", got, want) + } +} + +// securePerRPCCredentials always requires transport security. +type securePerRPCCredentials struct { + credentials.PerRPCCredentials +} + +func (c securePerRPCCredentials) RequireTransportSecurity() bool { + return true +} + +type fakeBundleCreds struct { + credentials.Bundle + transportCreds credentials.TransportCredentials +} + +func (b *fakeBundleCreds) TransportCredentials() credentials.TransportCredentials { + return b.transportCreds +} + +func (s) TestCredentialsMisuse(t *testing.T) { + // Use of no transport creds and no creds bundle must fail. + if _, err := NewClient("passthrough:///Non-Existent.Server:80"); err != errNoTransportSecurity { + t.Fatalf("grpc.NewClient() failed with error: %v, want: %v", err, errNoTransportSecurity) + } + + // Use of both transport creds and creds bundle must fail. + creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") + if err != nil { + t.Fatalf("Failed to create authenticator %v", err) + } + dopts := []DialOption{ + WithTransportCredentials(creds), + WithCredentialsBundle(&fakeBundleCreds{transportCreds: creds}), + } + if _, err := NewClient("passthrough:///Non-Existent.Server:80", dopts...); err != errTransportCredsAndBundle { + t.Fatalf("grpc.NewClient() failed with error: %v, want: %v", err, errTransportCredsAndBundle) + } + + // Use of perRPC creds requiring transport security over an insecure + // transport must fail. + if _, err := NewClient("passthrough:///Non-Existent.Server:80", WithPerRPCCredentials(securePerRPCCredentials{}), WithTransportCredentials(insecure.NewCredentials())); err != errTransportCredentialsMissing { + t.Fatalf("grpc.NewClient() failed with error: %v, want: %v", err, errTransportCredentialsMissing) + } + + // Use of a creds bundle with nil transport credentials must fail. + if _, err := NewClient("passthrough:///Non-Existent.Server:80", WithCredentialsBundle(&fakeBundleCreds{})); err != errNoTransportCredsInBundle { + t.Fatalf("grpc.NewClient() failed with error: %v, want: %v", err, errTransportCredsAndBundle) + } +} + +func (s) TestWithBackoffConfigDefault(t *testing.T) { + testBackoffConfigSet(t, internalbackoff.DefaultExponential) +} + +func (s) TestWithBackoffConfig(t *testing.T) { + b := BackoffConfig{MaxDelay: DefaultBackoffConfig.MaxDelay / 2} + bc := backoff.DefaultConfig + bc.MaxDelay = b.MaxDelay + wantBackoff := internalbackoff.Exponential{Config: bc} + testBackoffConfigSet(t, wantBackoff, WithBackoffConfig(b)) +} + +func (s) TestWithBackoffMaxDelay(t *testing.T) { + md := DefaultBackoffConfig.MaxDelay / 2 + bc := backoff.DefaultConfig + bc.MaxDelay = md + wantBackoff := internalbackoff.Exponential{Config: bc} + testBackoffConfigSet(t, wantBackoff, WithBackoffMaxDelay(md)) +} + +func (s) TestWithConnectParams(t *testing.T) { + bd := 2 * time.Second + mltpr := 2.0 + jitter := 0.0 + bc := backoff.Config{BaseDelay: bd, Multiplier: mltpr, Jitter: jitter} + + crt := ConnectParams{Backoff: bc} + // MaxDelay is not set in the ConnectParams. So it should not be set on + // internalbackoff.Exponential as well. + wantBackoff := internalbackoff.Exponential{Config: bc} + testBackoffConfigSet(t, wantBackoff, WithConnectParams(crt)) +} + +func testBackoffConfigSet(t *testing.T, wantBackoff internalbackoff.Exponential, opts ...DialOption) { + opts = append(opts, WithTransportCredentials(insecure.NewCredentials())) + conn, err := NewClient("passthrough:///foo:80", opts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer conn.Close() + + if conn.dopts.bs == nil { + t.Fatalf("backoff config not set") + } + + gotBackoff, ok := conn.dopts.bs.(internalbackoff.Exponential) + if !ok { + t.Fatalf("unexpected type of backoff config: %#v", conn.dopts.bs) + } + + if gotBackoff != wantBackoff { + t.Fatalf("unexpected backoff config on connection: %v, want %v", gotBackoff, wantBackoff) + } +} + +func (s) TestConnectParamsWithMinConnectTimeout(t *testing.T) { + // Default value specified for minConnectTimeout in the spec is 20 seconds. + mct := 1 * time.Minute + conn, err := NewClient("passthrough:///foo:80", WithTransportCredentials(insecure.NewCredentials()), WithConnectParams(ConnectParams{MinConnectTimeout: mct})) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer conn.Close() + + if got := conn.dopts.minConnectTimeout(); got != mct { + t.Errorf("unexpected minConnectTimeout on the connection: %v, want %v", got, mct) + } +} + +func (s) TestResolverServiceConfigBeforeAddressNotPanic(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + + cc, err := NewClient(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + // SwitchBalancer before NewAddress. There was no balancer created, this + // makes sure we don't call close on nil balancerWrapper. + r.UpdateState(resolver.State{ServiceConfig: r.CC().ParseServiceConfig(grpclbServiceConfig)}) // This should not panic. + + time.Sleep(time.Second) // Sleep to make sure the service config is handled by ClientConn. +} + +func (s) TestResolverServiceConfigWhileClosingNotPanic(t *testing.T) { + for i := 0; i < 10; i++ { // Run this multiple times to make sure it doesn't panic. + r := manual.NewBuilderWithScheme(fmt.Sprintf("whatever-%d", i)) + cc, err := NewClient(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + // Send a new service config while closing the ClientConn. + go cc.Close() + go r.UpdateState(resolver.State{ServiceConfig: r.CC().ParseServiceConfig(rrServiceConfig)}) // This should not panic. + } +} + +func (s) TestResolverEmptyUpdateNotPanic(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + + cc, err := NewClient(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + // This make sure we don't create addrConn with empty address list. + r.UpdateState(resolver.State{}) // This should not panic. + + time.Sleep(time.Second) // Sleep to make sure the service config is handled by ClientConn. +} + +func (s) TestDisableServiceConfigOption(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + addr := r.Scheme() + ":///non.existent" + cc, err := NewClient(addr, WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithDisableServiceConfig()) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v, want: nil", addr, err) + } + defer cc.Close() + cc.Connect() + r.UpdateState(resolver.State{ServiceConfig: r.CC().ParseServiceConfig(`{ + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "waitForReady": true + } + ] +}`)}) + time.Sleep(1 * time.Second) + m := cc.GetMethodConfig("/foo/Bar") + if m.WaitForReady != nil { + t.Fatalf("want: method (\"/foo/bar/\") config to be empty, got: %+v", m) + } +} + +func (s) TestMethodConfigDefaultService(t *testing.T) { + addr := "passthrough:///non.existent" + cc, err := NewClient(addr, WithTransportCredentials(insecure.NewCredentials()), WithDefaultServiceConfig(`{ + "methodConfig": [{ + "name": [ + { + "service": "" + } + ], + "waitForReady": true + }] +}`)) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v, want: nil", addr, err) + } + cc.Connect() + defer cc.Close() + + m := cc.GetMethodConfig("/foo/Bar") + if m.WaitForReady == nil { + t.Fatalf("want: method (%q) config to fallback to the default service", "/foo/Bar") + } +} + +func (s) TestClientConnCanonicalTarget(t *testing.T) { + tests := []struct { + name string + addr string + canonicalTargetWant string + }{ + { + name: "normal-case", + addr: "dns://a.server.com/google.com", + canonicalTargetWant: "dns://a.server.com/google.com", + }, + { + name: "canonical-target-not-specified", + addr: "no.scheme", + canonicalTargetWant: "dns:///no.scheme", + }, + { + name: "canonical-target-nonexistent", + addr: "nonexist:///non.existent", + canonicalTargetWant: "dns:///nonexist:///non.existent", + }, + { + name: "canonical-target-add-colon-slash", + addr: "dns:hostname:port", + canonicalTargetWant: "dns:///hostname:port", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cc, err := NewClient(test.addr, WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v, want: nil", test.addr, err) + } + defer cc.Close() + if cc.Target() != test.addr { + t.Fatalf("Target() = %s, want %s", cc.Target(), test.addr) + } + if cc.CanonicalTarget() != test.canonicalTargetWant { + t.Fatalf("CanonicalTarget() = %s, want %s", cc.CanonicalTarget(), test.canonicalTargetWant) + } + }) + } +} + +type backoffForever struct{} + +func (b backoffForever) Backoff(int) time.Duration { return time.Duration(math.MaxInt64) } + +func (s) TestResetConnectBackoff(t *testing.T) { + dials := make(chan struct{}) + defer func() { // If we fail, let the http2client break out of dialing. + select { + case <-dials: + default: + } + }() + dialer := func(string, time.Duration) (net.Conn, error) { + dials <- struct{}{} + return nil, errors.New("failed to fake dial") + } + cc, err := NewClient("passthrough:///", WithTransportCredentials(insecure.NewCredentials()), WithDialer(dialer), withBackoff(backoffForever{})) + if err != nil { + t.Fatalf("grpc.NewClient() failed with error: %v, want: nil", err) + } + defer cc.Close() + go stayConnected(cc) + select { + case <-dials: + case <-time.NewTimer(10 * time.Second).C: + t.Fatal("Failed to call dial within 10s") + } + + select { + case <-dials: + t.Fatal("Dial called unexpectedly before resetting backoff") + case <-time.NewTimer(100 * time.Millisecond).C: + } + + cc.ResetConnectBackoff() + + select { + case <-dials: + case <-time.NewTimer(10 * time.Second).C: + t.Fatal("Failed to call dial within 10s after resetting backoff") + } +} + +func (s) TestBackoffCancel(t *testing.T) { + dialStrCh := make(chan string) + cc, err := NewClient("passthrough:///", WithTransportCredentials(insecure.NewCredentials()), WithDialer(func(t string, _ time.Duration) (net.Conn, error) { + dialStrCh <- t + return nil, fmt.Errorf("test dialer, always error") + })) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + defer cc.Close() + + select { + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout when waiting for custom dialer to be invoked during Connect()") + case <-dialStrCh: + } +} + +// TestUpdateAddresses_NoopIfCalledWithSameAddresses tests that UpdateAddresses +// should be noop if UpdateAddresses is called with the same list of addresses, +// even when the SubConn is in Connecting and doesn't have a current address. +func (s) TestUpdateAddresses_NoopIfCalledWithSameAddresses(t *testing.T) { + lis1, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis1.Close() + + lis2, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis2.Close() + + lis3, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis3.Close() + + closeServer2 := make(chan struct{}) + exitCh := make(chan struct{}) + server1ContactedFirstTime := make(chan struct{}) + server1ContactedSecondTime := make(chan struct{}) + server2ContactedFirstTime := make(chan struct{}) + server2ContactedSecondTime := make(chan struct{}) + server3Contacted := make(chan struct{}) + + defer close(exitCh) + + // Launch server 1. + go func() { + // First, let's allow the initial connection to go READY. We need to do + // this because tryUpdateAddrs only works after there's some non-nil + // address on the ac, and curAddress is only set after READY. + conn1, err := lis1.Accept() + if err != nil { + t.Error(err) + return + } + go keepReading(conn1) + + framer := http2.NewFramer(conn1, conn1) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings frame. %v", err) + return + } + + // nextStateNotifier() is updated after balancerBuilder.Build(), which + // is called by ClientConn.Connect in stayConnected. It's safe to do it + // here because lis1.Accept blocks until ClientConn.Connect is called + // and the balancer is built to process the addresses. + stateNotifications := testBalancerBuilder.nextStateNotifier() + // Wait for the transport to become ready. + for { + select { + case st := <-stateNotifications: + if st == connectivity.Ready { + goto ready + } + case <-exitCh: + return + } + } + + ready: + // Once it's ready, curAddress has been set. So let's close this + // connection prompting the first reconnect cycle. + conn1.Close() + + // Accept and immediately close, causing it to go to server2. + conn2, err := lis1.Accept() + if err != nil { + t.Error(err) + return + } + close(server1ContactedFirstTime) + conn2.Close() + + // Hopefully it picks this server after tryUpdateAddrs. + lis1.Accept() + close(server1ContactedSecondTime) + }() + // Launch server 2. + go func() { + // Accept and then hang waiting for the test call tryUpdateAddrs and + // then signal to this server to close. After this server closes, it + // should start from the top instead of trying server2 or continuing + // to server3. + conn, err := lis2.Accept() + if err != nil { + t.Error(err) + return + } + + close(server2ContactedFirstTime) + <-closeServer2 + conn.Close() + + // After tryUpdateAddrs, it should NOT try server2. + lis2.Accept() + close(server2ContactedSecondTime) + }() + // Launch server 3. + go func() { + // After tryUpdateAddrs, it should NOT try server3. (or any other time) + lis3.Accept() + close(server3Contacted) + }() + + addrsList := []resolver.Address{ + {Addr: lis1.Addr().String()}, + {Addr: lis2.Addr().String()}, + {Addr: lis3.Addr().String()}, + } + rb := manual.NewBuilderWithScheme("whatever") + rb.InitialState(resolver.State{Addresses: addrsList}) + + client, err := NewClient("whatever:///this-gets-overwritten", + WithTransportCredentials(insecure.NewCredentials()), + WithResolvers(rb), + WithConnectParams(ConnectParams{ + Backoff: backoff.Config{}, + MinConnectTimeout: time.Hour, + }), + WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerName))) + if err != nil { + t.Fatal(err) + } + defer client.Close() + go stayConnected(client) + + timeout := time.After(5 * time.Second) + + // Wait for server1 to be contacted (which will immediately fail), then + // server2 (which will hang waiting for our signal). + select { + case <-server1ContactedFirstTime: + case <-timeout: + t.Fatal("timed out waiting for server1 to be contacted") + } + select { + case <-server2ContactedFirstTime: + case <-timeout: + t.Fatal("timed out waiting for server2 to be contacted") + } + + // Grab the addrConn and call tryUpdateAddrs. + client.mu.Lock() + for clientAC := range client.conns { + // Call UpdateAddresses with the same list of addresses, it should be a noop + // (even when the SubConn is Connecting, and doesn't have a curAddr). + clientAC.acbw.UpdateAddresses(clientAC.addrs) + } + client.mu.Unlock() + + // We've called tryUpdateAddrs - now let's make server2 close the + // connection and check that it continues to server3. + close(closeServer2) + + select { + case <-server1ContactedSecondTime: + t.Fatal("server1 was contacted a second time, but it should have continued to server 3") + case <-server2ContactedSecondTime: + t.Fatal("server2 was contacted a second time, but it should have continued to server 3") + case <-server3Contacted: + case <-timeout: + t.Fatal("timed out waiting for any server to be contacted after tryUpdateAddrs") + } +} + +func (s) TestDefaultServiceConfig(t *testing.T) { + const defaultSC = ` +{ + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "bar" + } + ], + "waitForReady": true + } + ] +}` + tests := []struct { + name string + testF func(t *testing.T, r *manual.Resolver, addr, sc string) + sc string + }{ + { + name: "invalid-service-config", + testF: testInvalidDefaultServiceConfig, + sc: "", + }, + { + name: "resolver-service-config-disabled", + testF: testDefaultServiceConfigWhenResolverServiceConfigDisabled, + sc: defaultSC, + }, + { + name: "resolver-does-not-return-service-config", + testF: testDefaultServiceConfigWhenResolverDoesNotReturnServiceConfig, + sc: defaultSC, + }, + { + name: "resolver-returns-invalid-service-config", + testF: testDefaultServiceConfigWhenResolverReturnInvalidServiceConfig, + sc: defaultSC, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r := manual.NewBuilderWithScheme(test.name) + addr := r.Scheme() + ":///non.existent" + test.testF(t, r, addr, test.sc) + }) + } +} + +func verifyWaitForReadyEqualsTrue(cc *ClientConn) bool { + var i int + for i = 0; i < 10; i++ { + mc := cc.GetMethodConfig("/foo/bar") + if mc.WaitForReady != nil && *mc.WaitForReady == true { + break + } + time.Sleep(100 * time.Millisecond) + } + return i != 10 +} + +func testInvalidDefaultServiceConfig(t *testing.T, r *manual.Resolver, addr, sc string) { + _, err := NewClient(addr, WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithDefaultServiceConfig(sc)) + if !strings.Contains(err.Error(), invalidDefaultServiceConfigErrPrefix) { + t.Fatalf("grpc.NewClient() got err: %v, want err contains: %v", err, invalidDefaultServiceConfigErrPrefix) + } +} + +func testDefaultServiceConfigWhenResolverServiceConfigDisabled(t *testing.T, r *manual.Resolver, addr string, js string) { + cc, err := NewClient(addr, WithTransportCredentials(insecure.NewCredentials()), WithDisableServiceConfig(), WithResolvers(r), WithDefaultServiceConfig(js)) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v, want: nil", addr, err) + } + cc.Connect() + defer cc.Close() + // Resolver service config gets ignored since resolver service config is disabled. + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: addr}}, + ServiceConfig: parseCfg(r, "{}"), + }) + if !verifyWaitForReadyEqualsTrue(cc) { + t.Fatal("default service config failed to be applied after 1s") + } +} + +func testDefaultServiceConfigWhenResolverDoesNotReturnServiceConfig(t *testing.T, r *manual.Resolver, addr string, js string) { + cc, err := NewClient(addr, WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithDefaultServiceConfig(js)) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v, want: nil", addr, err) + } + cc.Connect() + defer cc.Close() + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: addr}}, + }) + if !verifyWaitForReadyEqualsTrue(cc) { + t.Fatal("default service config failed to be applied after 1s") + } +} + +func testDefaultServiceConfigWhenResolverReturnInvalidServiceConfig(t *testing.T, r *manual.Resolver, addr string, js string) { + cc, err := NewClient(addr, WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithDefaultServiceConfig(js)) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v, want: nil", addr, err) + } + cc.Connect() + defer cc.Close() + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: addr}}, + }) + if !verifyWaitForReadyEqualsTrue(cc) { + t.Fatal("default service config failed to be applied after 1s") + } +} + +type stateRecordingBalancer struct { + balancer.Balancer +} + +func (b *stateRecordingBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) { + panic(fmt.Sprintf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, s)) +} + +func (b *stateRecordingBalancer) Close() { + b.Balancer.Close() +} + +type stateRecordingBalancerBuilder struct { + mu sync.Mutex + notifier chan connectivity.State // The notifier used in the last Balancer. +} + +func newStateRecordingBalancerBuilder() *stateRecordingBalancerBuilder { + return &stateRecordingBalancerBuilder{} +} + +func (b *stateRecordingBalancerBuilder) Name() string { + return stateRecordingBalancerName +} + +func (b *stateRecordingBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + stateNotifications := make(chan connectivity.State, 10) + b.mu.Lock() + b.notifier = stateNotifications + b.mu.Unlock() + return &stateRecordingBalancer{ + Balancer: balancer.Get("pick_first").Build(&stateRecordingCCWrapper{cc, stateNotifications}, opts), + } +} + +func (b *stateRecordingBalancerBuilder) nextStateNotifier() <-chan connectivity.State { + b.mu.Lock() + defer b.mu.Unlock() + ret := b.notifier + b.notifier = nil + return ret +} + +type stateRecordingCCWrapper struct { + balancer.ClientConn + notifier chan<- connectivity.State +} + +func (ccw *stateRecordingCCWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + oldListener := opts.StateListener + opts.StateListener = func(s balancer.SubConnState) { + ccw.notifier <- s.ConnectivityState + oldListener(s) + } + return ccw.ClientConn.NewSubConn(addrs, opts) +} + +// Keep reading until something causes the connection to die (EOF, server +// closed, etc). Useful as a tool for mindlessly keeping the connection +// healthy, since the client will error if things like client prefaces are not +// accepted in a timely fashion. +func keepReading(conn net.Conn) { + buf := make([]byte, 1024) + for _, err := conn.Read(buf); err == nil; _, err = conn.Read(buf) { + } +} + +// stayConnected makes cc stay connected by repeatedly calling cc.Connect() +// until the state becomes Shutdown or until 10 seconds elapses. +func stayConnected(cc *ClientConn) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + for { + state := cc.GetState() + switch state { + case connectivity.Idle: + cc.Connect() + case connectivity.Shutdown: + return + } + if !cc.WaitForStateChange(ctx, state) { + return + } + } +} + +func (s) TestURLAuthorityEscape(t *testing.T) { + tests := []struct { + name string + authority string + want string + }{ + { + name: "ipv6_authority", + authority: "[::1]", + want: "[::1]", + }, + { + name: "with_user_and_host", + authority: "userinfo@host:10001", + want: "userinfo@host:10001", + }, + { + name: "with_multiple_slashes", + authority: "projects/123/network/abc/service", + want: "projects%2F123%2Fnetwork%2Fabc%2Fservice", + }, + { + name: "all_possible_allowed_chars", + authority: "abc123-._~!$&'()*+,;=@:[]", + want: "abc123-._~!$&'()*+,;=@:[]", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got, want := encodeAuthority(test.authority), test.want; got != want { + t.Errorf("encodeAuthority(%s) = %s, want %s", test.authority, got, test.want) + } + }) + } +} diff --git a/cmd/protoc-gen-go-grpc/README.md b/cmd/protoc-gen-go-grpc/README.md new file mode 100644 index 0000000..711d20e --- /dev/null +++ b/cmd/protoc-gen-go-grpc/README.md @@ -0,0 +1,28 @@ +# protoc-gen-go-grpc + +This tool generates Go language bindings of `service`s in protobuf definition +files for gRPC. For usage information, please see our [quick start +guide](https://grpc.io/docs/languages/go/quickstart/). + +## Future-proofing services + +By default, to register services using the methods generated by this tool, the +service implementations must embed the corresponding +`UnimplementedServer` for future compatibility. This is a behavior +change from the grpc code generator previously included with `protoc-gen-go`. +To restore this behavior, set the option `require_unimplemented_servers=false`. +E.g.: + +```sh + protoc --go-grpc_out=. --go-grpc_opt=require_unimplemented_servers=false[,other options...] \ +``` + +Note that this is not recommended, and the option is only provided to restore +backward compatibility with previously-generated code. + +When embedding the `UnimplementedServer` in a struct that +implements the service, it should be embedded by _value_ instead of as a +_pointer_. If it is embedded as a pointer, it must be assigned to a valid, +non-nil pointer or else unimplemented methods would panic when called. This is +tested at service registration time, and will lead to a panic in +`RegisterServer` if it is not embedded properly. diff --git a/cmd/protoc-gen-go-grpc/go.mod b/cmd/protoc-gen-go-grpc/go.mod new file mode 100644 index 0000000..68ebf33 --- /dev/null +++ b/cmd/protoc-gen-go-grpc/go.mod @@ -0,0 +1,15 @@ +module google.golang.org/grpc/cmd/protoc-gen-go-grpc + +go 1.25.0 + +require ( + google.golang.org/grpc v1.82.1 + google.golang.org/protobuf v1.36.12 +) + +require ( + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect +) diff --git a/cmd/protoc-gen-go-grpc/go.sum b/cmd/protoc-gen-go-grpc/go.sum new file mode 100644 index 0000000..9a53e40 --- /dev/null +++ b/cmd/protoc-gen-go-grpc/go.sum @@ -0,0 +1,38 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 h1:qEHAMpSaUhtD0p3NbEEI83HwNGFxEwaSJ1G9PLnCBZE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/cmd/protoc-gen-go-grpc/grpc.go b/cmd/protoc-gen-go-grpc/grpc.go new file mode 100644 index 0000000..31344f1 --- /dev/null +++ b/cmd/protoc-gen-go-grpc/grpc.go @@ -0,0 +1,513 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "fmt" + "strconv" + "strings" + + "google.golang.org/protobuf/compiler/protogen" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/descriptorpb" +) + +const ( + contextPackage = protogen.GoImportPath("context") + grpcPackage = protogen.GoImportPath("google.golang.org/grpc") + codesPackage = protogen.GoImportPath("google.golang.org/grpc/codes") + statusPackage = protogen.GoImportPath("google.golang.org/grpc/status") +) + +type serviceGenerateHelperInterface interface { + formatFullMethodSymbol(service *protogen.Service, method *protogen.Method) string + genFullMethods(g *protogen.GeneratedFile, service *protogen.Service) + generateClientStruct(g *protogen.GeneratedFile, clientName string) + generateNewClientDefinitions(g *protogen.GeneratedFile, service *protogen.Service, clientName string) + generateUnimplementedServerType(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) + generateServerFunctions(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service, serverType string, serviceDescVar string) + formatHandlerFuncName(service *protogen.Service, hname string) string +} + +type serviceGenerateHelper struct{} + +func (serviceGenerateHelper) formatFullMethodSymbol(service *protogen.Service, method *protogen.Method) string { + return fmt.Sprintf("%s_%s_FullMethodName", service.GoName, method.GoName) +} + +func (serviceGenerateHelper) genFullMethods(g *protogen.GeneratedFile, service *protogen.Service) { + if len(service.Methods) == 0 { + return + } + + g.P("const (") + for _, method := range service.Methods { + fmSymbol := helper.formatFullMethodSymbol(service, method) + fmName := fmt.Sprintf("/%s/%s", service.Desc.FullName(), method.Desc.Name()) + g.P(fmSymbol, ` = "`, fmName, `"`) + } + g.P(")") + g.P() +} + +func (serviceGenerateHelper) generateClientStruct(g *protogen.GeneratedFile, clientName string) { + g.P("type ", unexport(clientName), " struct {") + g.P("cc ", grpcPackage.Ident("ClientConnInterface")) + g.P("}") + g.P() +} + +func (serviceGenerateHelper) generateNewClientDefinitions(g *protogen.GeneratedFile, _ *protogen.Service, clientName string) { + g.P("return &", unexport(clientName), "{cc}") +} + +func (serviceGenerateHelper) generateUnimplementedServerType(_ *protogen.Plugin, _ *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) { + serverType := service.GoName + "Server" + mustOrShould := "must" + if !*requireUnimplemented { + mustOrShould = "should" + } + // Server Unimplemented struct for forward compatibility. + g.P("// Unimplemented", serverType, " ", mustOrShould, " be embedded to have") + g.P("// forward compatible implementations.") + g.P("//") + g.P("// NOTE: this should be embedded by value instead of pointer to avoid a nil") + g.P("// pointer dereference when methods are called.") + g.P("type Unimplemented", serverType, " struct {}") + g.P() + for _, method := range service.Methods { + nilArg := "" + if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() { + nilArg = "nil," + } + g.P("func (Unimplemented", serverType, ") ", serverSignature(g, method), "{") + g.P("return ", nilArg, statusPackage.Ident("Error"), "(", codesPackage.Ident("Unimplemented"), `, "method `, method.GoName, ` not implemented")`) + g.P("}") + } + if *requireUnimplemented { + g.P("func (Unimplemented", serverType, ") mustEmbedUnimplemented", serverType, "() {}") + } + g.P("func (Unimplemented", serverType, ") testEmbeddedByValue() {}") + g.P() +} + +func (serviceGenerateHelper) generateServerFunctions(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service, serverType string, serviceDescVar string) { + // Server handler implementations. + handlerNames := make([]string, 0, len(service.Methods)) + for _, method := range service.Methods { + hname := genServerMethod(gen, file, g, method, func(hname string) string { + return hname + }) + handlerNames = append(handlerNames, hname) + } + genServiceDesc(file, g, serviceDescVar, serverType, service, handlerNames) +} + +func (serviceGenerateHelper) formatHandlerFuncName(_ *protogen.Service, hname string) string { + return hname +} + +var helper serviceGenerateHelperInterface = serviceGenerateHelper{} + +// FileDescriptorProto.package field number +const fileDescriptorProtoPackageFieldNumber = 2 + +// FileDescriptorProto.syntax field number +const fileDescriptorProtoSyntaxFieldNumber = 12 + +// generateFile generates a _grpc.pb.go file containing gRPC service definitions. +func generateFile(gen *protogen.Plugin, file *protogen.File) *protogen.GeneratedFile { + if len(file.Services) == 0 { + return nil + } + filename := file.GeneratedFilenamePrefix + "_grpc.pb.go" + g := gen.NewGeneratedFile(filename, file.GoImportPath) + // Attach all comments associated with the syntax field. + genLeadingComments(g, file.Desc.SourceLocations().ByPath(protoreflect.SourcePath{fileDescriptorProtoSyntaxFieldNumber})) + g.P("// Code generated by protoc-gen-go-grpc. DO NOT EDIT.") + g.P("// versions:") + g.P("// - protoc-gen-go-grpc v", version) + g.P("// - protoc ", protocVersion(gen)) + if file.Proto.GetOptions().GetDeprecated() { + g.P("// ", file.Desc.Path(), " is a deprecated file.") + } else { + g.P("// source: ", file.Desc.Path()) + } + g.P() + // Attach all comments associated with the package field. + genLeadingComments(g, file.Desc.SourceLocations().ByPath(protoreflect.SourcePath{fileDescriptorProtoPackageFieldNumber})) + g.P("package ", file.GoPackageName) + g.P() + generateFileContent(gen, file, g) + return g +} + +func protocVersion(gen *protogen.Plugin) string { + v := gen.Request.GetCompilerVersion() + if v == nil { + return "(unknown)" + } + var suffix string + if s := v.GetSuffix(); s != "" { + suffix = "-" + s + } + return fmt.Sprintf("v%d.%d.%d%s", v.GetMajor(), v.GetMinor(), v.GetPatch(), suffix) +} + +// generateFileContent generates the gRPC service definitions, excluding the package statement. +func generateFileContent(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile) { + if len(file.Services) == 0 { + return + } + + g.P("// This is a compile-time assertion to ensure that this generated file") + g.P("// is compatible with the grpc package it is being compiled against.") + g.P("// Requires gRPC-Go v1.64.0 or later.") + g.P("const _ = ", grpcPackage.Ident("SupportPackageIsVersion9")) + g.P() + for _, service := range file.Services { + genService(gen, file, g, service) + } +} + +// genServiceComments copies the comments from the RPC proto definitions +// to the corresponding generated interface file. +func genServiceComments(g *protogen.GeneratedFile, service *protogen.Service) { + if service.Comments.Leading != "" { + // Add empty comment line to attach this service's comments to + // the godoc comments previously output for all services. + g.P("//") + g.P(strings.TrimSpace(service.Comments.Leading.String())) + } +} + +func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) { + // Full methods constants. + helper.genFullMethods(g, service) + + // Client interface. + clientName := service.GoName + "Client" + + g.P("// ", clientName, " is the client API for ", service.GoName, " service.") + g.P("//") + g.P("// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.") + + // Copy comments from proto file. + genServiceComments(g, service) + + if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() { + g.P("//") + g.P(deprecationComment) + } + g.AnnotateSymbol(clientName, protogen.Annotation{Location: service.Location}) + g.P("type ", clientName, " interface {") + for _, method := range service.Methods { + g.AnnotateSymbol(clientName+"."+method.GoName, protogen.Annotation{Location: method.Location}) + if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() { + g.P(deprecationComment) + } + g.P(method.Comments.Leading, + clientSignature(g, method)) + } + g.P("}") + g.P() + + // Client structure. + helper.generateClientStruct(g, clientName) + + // NewClient factory. + if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() { + g.P(deprecationComment) + } + g.P("func New", clientName, " (cc ", grpcPackage.Ident("ClientConnInterface"), ") ", clientName, " {") + helper.generateNewClientDefinitions(g, service, clientName) + g.P("}") + g.P() + + var methodIndex, streamIndex int + // Client method implementations. + for _, method := range service.Methods { + if !method.Desc.IsStreamingServer() && !method.Desc.IsStreamingClient() { + // Unary RPC method + genClientMethod(gen, file, g, method, methodIndex) + methodIndex++ + } else { + // Streaming RPC method + genClientMethod(gen, file, g, method, streamIndex) + streamIndex++ + } + } + + mustOrShould := "must" + if !*requireUnimplemented { + mustOrShould = "should" + } + + // Server interface. + serverType := service.GoName + "Server" + g.P("// ", serverType, " is the server API for ", service.GoName, " service.") + g.P("// All implementations ", mustOrShould, " embed Unimplemented", serverType) + g.P("// for forward compatibility.") + + // Copy comments from proto file. + genServiceComments(g, service) + + if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() { + g.P("//") + g.P(deprecationComment) + } + g.AnnotateSymbol(serverType, protogen.Annotation{Location: service.Location}) + g.P("type ", serverType, " interface {") + for _, method := range service.Methods { + g.AnnotateSymbol(serverType+"."+method.GoName, protogen.Annotation{Location: method.Location}) + if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() { + g.P(deprecationComment) + } + g.P(method.Comments.Leading, + serverSignature(g, method)) + } + if *requireUnimplemented { + g.P("mustEmbedUnimplemented", serverType, "()") + } + g.P("}") + g.P() + + // Server Unimplemented struct for forward compatibility. + helper.generateUnimplementedServerType(gen, file, g, service) + + // Unsafe Server interface to opt-out of forward compatibility. + g.P("// Unsafe", serverType, " may be embedded to opt out of forward compatibility for this service.") + g.P("// Use of this interface is not recommended, as added methods to ", serverType, " will") + g.P("// result in compilation errors.") + g.P("type Unsafe", serverType, " interface {") + g.P("mustEmbedUnimplemented", serverType, "()") + g.P("}") + + // Server registration. + if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() { + g.P(deprecationComment) + } + serviceDescVar := service.GoName + "_ServiceDesc" + g.P("func Register", service.GoName, "Server(s ", grpcPackage.Ident("ServiceRegistrar"), ", srv ", serverType, ") {") + g.P("// If the following call panics, it indicates Unimplemented", serverType, " was") + g.P("// embedded by pointer and is nil. This will cause panics if an") + g.P("// unimplemented method is ever invoked, so we test this at initialization") + g.P("// time to prevent it from happening at runtime later due to I/O.") + g.P("if t, ok := srv.(interface { testEmbeddedByValue() }); ok {") + g.P("t.testEmbeddedByValue()") + g.P("}") + g.P("s.RegisterService(&", serviceDescVar, `, srv)`) + g.P("}") + g.P() + + helper.generateServerFunctions(gen, file, g, service, serverType, serviceDescVar) +} + +func clientSignature(g *protogen.GeneratedFile, method *protogen.Method) string { + s := method.GoName + "(ctx " + g.QualifiedGoIdent(contextPackage.Ident("Context")) + if !method.Desc.IsStreamingClient() { + s += ", in *" + g.QualifiedGoIdent(method.Input.GoIdent) + } + s += ", opts ..." + g.QualifiedGoIdent(grpcPackage.Ident("CallOption")) + ") (" + if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() { + s += "*" + g.QualifiedGoIdent(method.Output.GoIdent) + } else { + s += clientStreamInterface(g, method) + } + s += ", error)" + return s +} + +func clientStreamInterface(g *protogen.GeneratedFile, method *protogen.Method) string { + typeParam := g.QualifiedGoIdent(method.Input.GoIdent) + ", " + g.QualifiedGoIdent(method.Output.GoIdent) + if method.Desc.IsStreamingClient() && method.Desc.IsStreamingServer() { + return g.QualifiedGoIdent(grpcPackage.Ident("BidiStreamingClient")) + "[" + typeParam + "]" + } + if method.Desc.IsStreamingClient() { + return g.QualifiedGoIdent(grpcPackage.Ident("ClientStreamingClient")) + "[" + typeParam + "]" + } + return g.QualifiedGoIdent(grpcPackage.Ident("ServerStreamingClient")) + "[" + g.QualifiedGoIdent(method.Output.GoIdent) + "]" +} + +func genClientMethod(_ *protogen.Plugin, _ *protogen.File, g *protogen.GeneratedFile, method *protogen.Method, index int) { + service := method.Parent + fmSymbol := helper.formatFullMethodSymbol(service, method) + + if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() { + g.P(deprecationComment) + } + g.P("func (c *", unexport(service.GoName), "Client) ", clientSignature(g, method), "{") + g.P("cOpts := append([]", grpcPackage.Ident("CallOption"), "{", grpcPackage.Ident("StaticMethod()"), "}, opts...)") + if !method.Desc.IsStreamingServer() && !method.Desc.IsStreamingClient() { + g.P("out := new(", method.Output.GoIdent, ")") + g.P(`err := c.cc.Invoke(ctx, `, fmSymbol, `, in, out, cOpts...)`) + g.P("if err != nil { return nil, err }") + g.P("return out, nil") + g.P("}") + g.P() + return + } + + typeParam := g.QualifiedGoIdent(method.Input.GoIdent) + ", " + g.QualifiedGoIdent(method.Output.GoIdent) + streamImpl := g.QualifiedGoIdent(grpcPackage.Ident("GenericClientStream")) + "[" + typeParam + "]" + serviceDescVar := service.GoName + "_ServiceDesc" + g.P("stream, err := c.cc.NewStream(ctx, &", serviceDescVar, ".Streams[", index, `], `, fmSymbol, `, cOpts...)`) + g.P("if err != nil { return nil, err }") + g.P("x := &", streamImpl, "{ClientStream: stream}") + if !method.Desc.IsStreamingClient() { + g.P("if err := x.ClientStream.SendMsg(in); err != nil { return nil, err }") + g.P("if err := x.ClientStream.CloseSend(); err != nil { return nil, err }") + } + g.P("return x, nil") + g.P("}") + g.P() + + // Auxiliary types aliases, for backwards compatibility. + g.P("// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.") + g.P("type ", service.GoName, "_", method.GoName, "Client = ", clientStreamInterface(g, method)) + g.P() +} + +func serverSignature(g *protogen.GeneratedFile, method *protogen.Method) string { + var reqArgs []string + ret := "error" + if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() { + reqArgs = append(reqArgs, g.QualifiedGoIdent(contextPackage.Ident("Context"))) + ret = "(*" + g.QualifiedGoIdent(method.Output.GoIdent) + ", error)" + } + if !method.Desc.IsStreamingClient() { + reqArgs = append(reqArgs, "*"+g.QualifiedGoIdent(method.Input.GoIdent)) + } + if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() { + reqArgs = append(reqArgs, serverStreamInterface(g, method)) + } + return method.GoName + "(" + strings.Join(reqArgs, ", ") + ") " + ret +} + +func genServiceDesc(file *protogen.File, g *protogen.GeneratedFile, serviceDescVar string, serverType string, service *protogen.Service, handlerNames []string) { + // Service descriptor. + g.P("// ", serviceDescVar, " is the ", grpcPackage.Ident("ServiceDesc"), " for ", service.GoName, " service.") + g.P("// It's only intended for direct use with ", grpcPackage.Ident("RegisterService"), ",") + g.P("// and not to be introspected or modified (even as a copy)") + g.P("var ", serviceDescVar, " = ", grpcPackage.Ident("ServiceDesc"), " {") + g.P("ServiceName: ", strconv.Quote(string(service.Desc.FullName())), ",") + g.P("HandlerType: (*", serverType, ")(nil),") + g.P("Methods: []", grpcPackage.Ident("MethodDesc"), "{") + for i, method := range service.Methods { + if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() { + continue + } + g.P("{") + g.P("MethodName: ", strconv.Quote(string(method.Desc.Name())), ",") + g.P("Handler: ", handlerNames[i], ",") + g.P("},") + } + g.P("},") + g.P("Streams: []", grpcPackage.Ident("StreamDesc"), "{") + for i, method := range service.Methods { + if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() { + continue + } + g.P("{") + g.P("StreamName: ", strconv.Quote(string(method.Desc.Name())), ",") + g.P("Handler: ", handlerNames[i], ",") + if method.Desc.IsStreamingServer() { + g.P("ServerStreams: true,") + } + if method.Desc.IsStreamingClient() { + g.P("ClientStreams: true,") + } + g.P("},") + } + g.P("},") + g.P("Metadata: \"", file.Desc.Path(), "\",") + g.P("}") + g.P() +} + +func serverStreamInterface(g *protogen.GeneratedFile, method *protogen.Method) string { + typeParam := g.QualifiedGoIdent(method.Input.GoIdent) + ", " + g.QualifiedGoIdent(method.Output.GoIdent) + if method.Desc.IsStreamingClient() && method.Desc.IsStreamingServer() { + return g.QualifiedGoIdent(grpcPackage.Ident("BidiStreamingServer")) + "[" + typeParam + "]" + } + if method.Desc.IsStreamingClient() { + return g.QualifiedGoIdent(grpcPackage.Ident("ClientStreamingServer")) + "[" + typeParam + "]" + } + + return g.QualifiedGoIdent(grpcPackage.Ident("ServerStreamingServer")) + "[" + g.QualifiedGoIdent(method.Output.GoIdent) + "]" +} + +func genServerMethod(_ *protogen.Plugin, _ *protogen.File, g *protogen.GeneratedFile, method *protogen.Method, hnameFuncNameFormatter func(string) string) string { + service := method.Parent + hname := fmt.Sprintf("_%s_%s_Handler", service.GoName, method.GoName) + + if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() { + g.P("func ", hnameFuncNameFormatter(hname), "(srv interface{}, ctx ", contextPackage.Ident("Context"), ", dec func(interface{}) error, interceptor ", grpcPackage.Ident("UnaryServerInterceptor"), ") (interface{}, error) {") + g.P("in := new(", method.Input.GoIdent, ")") + g.P("if err := dec(in); err != nil { return nil, err }") + g.P("if interceptor == nil { return srv.(", service.GoName, "Server).", method.GoName, "(ctx, in) }") + g.P("info := &", grpcPackage.Ident("UnaryServerInfo"), "{") + g.P("Server: srv,") + fmSymbol := helper.formatFullMethodSymbol(service, method) + g.P("FullMethod: ", fmSymbol, ",") + g.P("}") + g.P("handler := func(ctx ", contextPackage.Ident("Context"), ", req interface{}) (interface{}, error) {") + g.P("return srv.(", service.GoName, "Server).", method.GoName, "(ctx, req.(*", method.Input.GoIdent, "))") + g.P("}") + g.P("return interceptor(ctx, in, info, handler)") + g.P("}") + g.P() + return hname + } + + typeParam := g.QualifiedGoIdent(method.Input.GoIdent) + ", " + g.QualifiedGoIdent(method.Output.GoIdent) + streamImpl := g.QualifiedGoIdent(grpcPackage.Ident("GenericServerStream")) + "[" + typeParam + "]" + + g.P("func ", hnameFuncNameFormatter(hname), "(srv interface{}, stream ", grpcPackage.Ident("ServerStream"), ") error {") + if !method.Desc.IsStreamingClient() { + g.P("m := new(", method.Input.GoIdent, ")") + g.P("if err := stream.RecvMsg(m); err != nil { return err }") + g.P("return srv.(", service.GoName, "Server).", method.GoName, "(m, &", streamImpl, "{ServerStream: stream})") + } else { + g.P("return srv.(", service.GoName, "Server).", method.GoName, "(&", streamImpl, "{ServerStream: stream})") + } + g.P("}") + g.P() + + // Auxiliary types aliases, for backwards compatibility. + g.P("// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.") + g.P("type ", service.GoName, "_", method.GoName, "Server = ", serverStreamInterface(g, method)) + g.P() + return hname +} + +func genLeadingComments(g *protogen.GeneratedFile, loc protoreflect.SourceLocation) { + for _, s := range loc.LeadingDetachedComments { + g.P(protogen.Comments(s)) + g.P() + } + if s := loc.LeadingComments; s != "" { + g.P(protogen.Comments(s)) + g.P() + } +} + +const deprecationComment = "// Deprecated: Do not use." + +func unexport(s string) string { return strings.ToLower(s[:1]) + s[1:] } diff --git a/cmd/protoc-gen-go-grpc/main.go b/cmd/protoc-gen-go-grpc/main.go new file mode 100644 index 0000000..94e7ffc --- /dev/null +++ b/cmd/protoc-gen-go-grpc/main.go @@ -0,0 +1,74 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// protoc-gen-go-grpc is a plugin for the Google protocol buffer compiler to +// generate Go code. Install it by building this program and making it +// accessible within your PATH with the name: +// +// protoc-gen-go-grpc +// +// The 'go-grpc' suffix becomes part of the argument for the protocol compiler, +// such that it can be invoked as: +// +// protoc --go-grpc_out=. path/to/file.proto +// +// This generates Go service definitions for the protocol buffer defined by +// file.proto. With that input, the output will be written to: +// +// path/to/file_grpc.pb.go +package main + +import ( + "flag" + "fmt" + + "google.golang.org/protobuf/compiler/protogen" + "google.golang.org/protobuf/types/descriptorpb" + "google.golang.org/protobuf/types/pluginpb" +) + +const version = "1.6.2" + +var requireUnimplemented *bool + +func main() { + showVersion := flag.Bool("version", false, "print the version and exit") + flag.Parse() + if *showVersion { + fmt.Printf("protoc-gen-go-grpc %v\n", version) + return + } + + var flags flag.FlagSet + requireUnimplemented = flags.Bool("require_unimplemented_servers", true, "set to false to match legacy behavior") + + protogen.Options{ + ParamFunc: flags.Set, + }.Run(func(gen *protogen.Plugin) error { + gen.SupportedFeatures = uint64(pluginpb.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL) | uint64(pluginpb.CodeGeneratorResponse_FEATURE_SUPPORTS_EDITIONS) + gen.SupportedEditionsMinimum = descriptorpb.Edition_EDITION_PROTO2 + gen.SupportedEditionsMaximum = descriptorpb.Edition_EDITION_2024 + for _, f := range gen.Files { + if !f.Generate { + continue + } + generateFile(gen, f) + } + return nil + }) +} diff --git a/cmd/protoc-gen-go-grpc/protoc-gen-go-grpc_test.sh b/cmd/protoc-gen-go-grpc/protoc-gen-go-grpc_test.sh new file mode 100755 index 0000000..fb67b1f --- /dev/null +++ b/cmd/protoc-gen-go-grpc/protoc-gen-go-grpc_test.sh @@ -0,0 +1,50 @@ +#!/bin/bash -e + +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Uncomment to enable debugging. +# set -x + +WORKDIR="$(dirname $0)" +TEMPDIR=$(mktemp -d) + +trap "rm -rf ${TEMPDIR}" EXIT + +# Build protoc-gen-go-grpc binary and add to $PATH. +pushd "${WORKDIR}" +go build -o "${TEMPDIR}" . +PATH="${TEMPDIR}:${PATH}" +popd + +protoc \ + --go-grpc_out="${TEMPDIR}" \ + --go-grpc_opt=paths=source_relative \ + "examples/route_guide/routeguide/route_guide.proto" + +GOLDENFILE="examples/route_guide/routeguide/route_guide_grpc.pb.go" +GENFILE="${TEMPDIR}/examples/route_guide/routeguide/route_guide_grpc.pb.go" + +# diff is piped to [[ $? == 1 ]] to avoid exiting on diff but exit on error +# (like if the file was not found). See man diff for more info. +DIFF=$(diff "${GOLDENFILE}" "${GENFILE}" || [[ $? == 1 ]]) +if [[ -n "${DIFF}" ]]; then + echo -e "ERROR: Generated file differs from golden file:\n${DIFF}" + echo -e "If you have made recent changes to protoc-gen-go-grpc," \ + "please regenerate the golden files by running:" \ + "\n\t go generate google.golang.org/grpc/..." >&2 + exit 1 +fi + +echo SUCCESS diff --git a/cmd/protoc-gen-go-grpc/unimpl_test.go b/cmd/protoc-gen-go-grpc/unimpl_test.go new file mode 100644 index 0000000..f46b59c --- /dev/null +++ b/cmd/protoc-gen-go-grpc/unimpl_test.go @@ -0,0 +1,47 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main_test + +import ( + "testing" + + "google.golang.org/grpc" + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +type unimplEmbeddedByPointer struct { + *testgrpc.UnimplementedTestServiceServer +} + +type unimplEmbeddedByValue struct { + testgrpc.UnimplementedTestServiceServer +} + +func TestUnimplementedEmbedding(t *testing.T) { + // Embedded by value, this should succeed. + testgrpc.RegisterTestServiceServer(grpc.NewServer(), &unimplEmbeddedByValue{}) + defer func() { + if recover() == nil { + t.Fatalf("Expected panic; received none") + } + }() + + // Embedded by pointer, this should panic. + testgrpc.RegisterTestServiceServer(grpc.NewServer(), &unimplEmbeddedByPointer{}) +} diff --git a/codec.go b/codec.go new file mode 100644 index 0000000..959c2f9 --- /dev/null +++ b/codec.go @@ -0,0 +1,105 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "google.golang.org/grpc/encoding" + _ "google.golang.org/grpc/encoding/proto" // to register the Codec for "proto" + "google.golang.org/grpc/mem" +) + +// baseCodec captures the new encoding.CodecV2 interface without the Name +// function, allowing it to be implemented by older Codec and encoding.Codec +// implementations. The omitted Name function is only needed for the register in +// the encoding package and is not part of the core functionality. +type baseCodec interface { + Marshal(v any) (mem.BufferSlice, error) + Unmarshal(data mem.BufferSlice, v any) error +} + +// getCodec returns an encoding.CodecV2 for the codec of the given name (if +// registered). Initially checks the V2 registry with encoding.GetCodecV2 and +// returns the V2 codec if it is registered. Otherwise, it checks the V1 registry +// with encoding.GetCodec and if it is registered wraps it with newCodecV1Bridge +// to turn it into an encoding.CodecV2. Returns nil otherwise. +func getCodec(name string) encoding.CodecV2 { + if codecV1 := encoding.GetCodec(name); codecV1 != nil { + return newCodecV1Bridge(codecV1) + } + + return encoding.GetCodecV2(name) +} + +func newCodecV0Bridge(c Codec) baseCodec { + return codecV0Bridge{codec: c} +} + +func newCodecV1Bridge(c encoding.Codec) encoding.CodecV2 { + return codecV1Bridge{ + codecV0Bridge: codecV0Bridge{codec: c}, + name: c.Name(), + } +} + +var _ baseCodec = codecV0Bridge{} + +type codecV0Bridge struct { + codec interface { + Marshal(v any) ([]byte, error) + Unmarshal(data []byte, v any) error + } +} + +func (c codecV0Bridge) Marshal(v any) (mem.BufferSlice, error) { + data, err := c.codec.Marshal(v) + if err != nil { + return nil, err + } + return mem.BufferSlice{mem.SliceBuffer(data)}, nil +} + +func (c codecV0Bridge) Unmarshal(data mem.BufferSlice, v any) (err error) { + return c.codec.Unmarshal(data.Materialize(), v) +} + +var _ encoding.CodecV2 = codecV1Bridge{} + +type codecV1Bridge struct { + codecV0Bridge + name string +} + +func (c codecV1Bridge) Name() string { + return c.name +} + +// Codec defines the interface gRPC uses to encode and decode messages. +// Note that implementations of this interface must be thread safe; +// a Codec's methods can be called from concurrent goroutines. +// +// Deprecated: use encoding.Codec instead. +type Codec interface { + // Marshal returns the wire format of v. + Marshal(v any) ([]byte, error) + // Unmarshal parses the wire format into v. + Unmarshal(data []byte, v any) error + // String returns the name of the Codec implementation. This is unused by + // gRPC. + String() string +} diff --git a/codec_test.go b/codec_test.go new file mode 100644 index 0000000..f2f88e9 --- /dev/null +++ b/codec_test.go @@ -0,0 +1,32 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "testing" + + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/encoding/proto" +) + +func (s) TestGetCodecForProtoIsNotNil(t *testing.T) { + if encoding.GetCodecV2(proto.Name) == nil { + t.Fatalf("encoding.GetCodec(%q) must not be nil by default", proto.Name) + } +} diff --git a/codes/code_string.go b/codes/code_string.go new file mode 100644 index 0000000..934fac2 --- /dev/null +++ b/codes/code_string.go @@ -0,0 +1,111 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package codes + +import ( + "strconv" + + "google.golang.org/grpc/internal" +) + +func init() { + internal.CanonicalString = canonicalString +} + +func (c Code) String() string { + switch c { + case OK: + return "OK" + case Canceled: + return "Canceled" + case Unknown: + return "Unknown" + case InvalidArgument: + return "InvalidArgument" + case DeadlineExceeded: + return "DeadlineExceeded" + case NotFound: + return "NotFound" + case AlreadyExists: + return "AlreadyExists" + case PermissionDenied: + return "PermissionDenied" + case ResourceExhausted: + return "ResourceExhausted" + case FailedPrecondition: + return "FailedPrecondition" + case Aborted: + return "Aborted" + case OutOfRange: + return "OutOfRange" + case Unimplemented: + return "Unimplemented" + case Internal: + return "Internal" + case Unavailable: + return "Unavailable" + case DataLoss: + return "DataLoss" + case Unauthenticated: + return "Unauthenticated" + default: + return "Code(" + strconv.FormatInt(int64(c), 10) + ")" + } +} + +func canonicalString(c Code) string { + switch c { + case OK: + return "OK" + case Canceled: + return "CANCELLED" + case Unknown: + return "UNKNOWN" + case InvalidArgument: + return "INVALID_ARGUMENT" + case DeadlineExceeded: + return "DEADLINE_EXCEEDED" + case NotFound: + return "NOT_FOUND" + case AlreadyExists: + return "ALREADY_EXISTS" + case PermissionDenied: + return "PERMISSION_DENIED" + case ResourceExhausted: + return "RESOURCE_EXHAUSTED" + case FailedPrecondition: + return "FAILED_PRECONDITION" + case Aborted: + return "ABORTED" + case OutOfRange: + return "OUT_OF_RANGE" + case Unimplemented: + return "UNIMPLEMENTED" + case Internal: + return "INTERNAL" + case Unavailable: + return "UNAVAILABLE" + case DataLoss: + return "DATA_LOSS" + case Unauthenticated: + return "UNAUTHENTICATED" + default: + return "CODE(" + strconv.FormatInt(int64(c), 10) + ")" + } +} diff --git a/codes/codes.go b/codes/codes.go new file mode 100644 index 0000000..0b42c30 --- /dev/null +++ b/codes/codes.go @@ -0,0 +1,250 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package codes defines the canonical error codes used by gRPC. It is +// consistent across various languages. +package codes // import "google.golang.org/grpc/codes" + +import ( + "fmt" + "strconv" +) + +// A Code is a status code defined according to the [gRPC documentation]. +// +// Only the codes defined as consts in this package are valid codes. Do not use +// other code values. Behavior of other codes is implementation-specific and +// interoperability between implementations is not guaranteed. +// +// [gRPC documentation]: https://github.com/grpc/grpc/blob/master/doc/statuscodes.md +type Code uint32 + +const ( + // OK is returned on success. + OK Code = 0 + + // Canceled indicates the operation was canceled (typically by the caller). + // + // The gRPC framework will generate this error code when cancellation + // is requested. + Canceled Code = 1 + + // Unknown error. An example of where this error may be returned is + // if a Status value received from another address space belongs to + // an error-space that is not known in this address space. Also + // errors raised by APIs that do not return enough error information + // may be converted to this error. + // + // The gRPC framework will generate this error code in the above two + // mentioned cases. + Unknown Code = 2 + + // InvalidArgument indicates client specified an invalid argument. + // Note that this differs from FailedPrecondition. It indicates arguments + // that are problematic regardless of the state of the system + // (e.g., a malformed file name). + // + // This error code will not be generated by the gRPC framework. + InvalidArgument Code = 3 + + // DeadlineExceeded means operation expired before completion. + // For operations that change the state of the system, this error may be + // returned even if the operation has completed successfully. For + // example, a successful response from a server could have been delayed + // long enough for the deadline to expire. + // + // The gRPC framework will generate this error code when the deadline is + // exceeded. + DeadlineExceeded Code = 4 + + // NotFound means some requested entity (e.g., file or directory) was + // not found. + // + // This error code will not be generated by the gRPC framework. + NotFound Code = 5 + + // AlreadyExists means an attempt to create an entity failed because one + // already exists. + // + // This error code will not be generated by the gRPC framework. + AlreadyExists Code = 6 + + // PermissionDenied indicates the caller does not have permission to + // execute the specified operation. It must not be used for rejections + // caused by exhausting some resource (use ResourceExhausted + // instead for those errors). It must not be + // used if the caller cannot be identified (use Unauthenticated + // instead for those errors). + // + // This error code will not be generated by the gRPC core framework, + // but expect authentication middleware to use it. + PermissionDenied Code = 7 + + // ResourceExhausted indicates some resource has been exhausted, perhaps + // a per-user quota, or perhaps the entire file system is out of space. + // + // This error code will be generated by the gRPC framework in + // out-of-memory and server overload situations, or when a message is + // larger than the configured maximum size. + ResourceExhausted Code = 8 + + // FailedPrecondition indicates operation was rejected because the + // system is not in a state required for the operation's execution. + // For example, directory to be deleted may be non-empty, an rmdir + // operation is applied to a non-directory, etc. + // + // A litmus test that may help a service implementor in deciding + // between FailedPrecondition, Aborted, and Unavailable: + // (a) Use Unavailable if the client can retry just the failing call. + // (b) Use Aborted if the client should retry at a higher-level + // (e.g., restarting a read-modify-write sequence). + // (c) Use FailedPrecondition if the client should not retry until + // the system state has been explicitly fixed. E.g., if an "rmdir" + // fails because the directory is non-empty, FailedPrecondition + // should be returned since the client should not retry unless + // they have first fixed up the directory by deleting files from it. + // (d) Use FailedPrecondition if the client performs conditional + // REST Get/Update/Delete on a resource and the resource on the + // server does not match the condition. E.g., conflicting + // read-modify-write on the same resource. + // + // This error code will not be generated by the gRPC framework. + FailedPrecondition Code = 9 + + // Aborted indicates the operation was aborted, typically due to a + // concurrency issue like sequencer check failures, transaction aborts, + // etc. + // + // See litmus test above for deciding between FailedPrecondition, + // Aborted, and Unavailable. + // + // This error code will not be generated by the gRPC framework. + Aborted Code = 10 + + // OutOfRange means operation was attempted past the valid range. + // E.g., seeking or reading past end of file. + // + // Unlike InvalidArgument, this error indicates a problem that may + // be fixed if the system state changes. For example, a 32-bit file + // system will generate InvalidArgument if asked to read at an + // offset that is not in the range [0,2^32-1], but it will generate + // OutOfRange if asked to read from an offset past the current + // file size. + // + // There is a fair bit of overlap between FailedPrecondition and + // OutOfRange. We recommend using OutOfRange (the more specific + // error) when it applies so that callers who are iterating through + // a space can easily look for an OutOfRange error to detect when + // they are done. + // + // This error code will not be generated by the gRPC framework. + OutOfRange Code = 11 + + // Unimplemented indicates operation is not implemented or not + // supported/enabled in this service. + // + // This error code will be generated by the gRPC framework. Most + // commonly, you will see this error code when a method implementation + // is missing on the server. It can also be generated for unknown + // compression algorithms or a disagreement as to whether an RPC should + // be streaming. + Unimplemented Code = 12 + + // Internal errors. Means some invariants expected by underlying + // system has been broken. If you see one of these errors, + // something is very broken. + // + // This error code will be generated by the gRPC framework in several + // internal error conditions. + Internal Code = 13 + + // Unavailable indicates the service is currently unavailable. + // This is a most likely a transient condition and may be corrected + // by retrying with a backoff. Note that it is not always safe to retry + // non-idempotent operations. + // + // See litmus test above for deciding between FailedPrecondition, + // Aborted, and Unavailable. + // + // This error code will be generated by the gRPC framework during + // abrupt shutdown of a server process or network connection. + Unavailable Code = 14 + + // DataLoss indicates unrecoverable data loss or corruption. + // + // This error code will not be generated by the gRPC framework. + DataLoss Code = 15 + + // Unauthenticated indicates the request does not have valid + // authentication credentials for the operation. + // + // The gRPC framework will generate this error code when the + // authentication metadata is invalid or a Credentials callback fails, + // but also expect authentication middleware to generate it. + Unauthenticated Code = 16 + + _maxCode = 17 +) + +var strToCode = map[string]Code{ + `"OK"`: OK, + `"CANCELLED"`:/* [sic] */ Canceled, + `"UNKNOWN"`: Unknown, + `"INVALID_ARGUMENT"`: InvalidArgument, + `"DEADLINE_EXCEEDED"`: DeadlineExceeded, + `"NOT_FOUND"`: NotFound, + `"ALREADY_EXISTS"`: AlreadyExists, + `"PERMISSION_DENIED"`: PermissionDenied, + `"RESOURCE_EXHAUSTED"`: ResourceExhausted, + `"FAILED_PRECONDITION"`: FailedPrecondition, + `"ABORTED"`: Aborted, + `"OUT_OF_RANGE"`: OutOfRange, + `"UNIMPLEMENTED"`: Unimplemented, + `"INTERNAL"`: Internal, + `"UNAVAILABLE"`: Unavailable, + `"DATA_LOSS"`: DataLoss, + `"UNAUTHENTICATED"`: Unauthenticated, +} + +// UnmarshalJSON unmarshals b into the Code. +func (c *Code) UnmarshalJSON(b []byte) error { + // From json.Unmarshaler: By convention, to approximate the behavior of + // Unmarshal itself, Unmarshalers implement UnmarshalJSON([]byte("null")) as + // a no-op. + if string(b) == "null" { + return nil + } + if c == nil { + return fmt.Errorf("nil receiver passed to UnmarshalJSON") + } + + if ci, err := strconv.ParseUint(string(b), 10, 32); err == nil { + if ci >= _maxCode { + return fmt.Errorf("invalid code: %d", ci) + } + + *c = Code(ci) + return nil + } + + if jc, ok := strToCode[string(b)]; ok { + *c = jc + return nil + } + return fmt.Errorf("invalid code: %q", string(b)) +} diff --git a/codes/codes_test.go b/codes/codes_test.go new file mode 100644 index 0000000..e27ff61 --- /dev/null +++ b/codes/codes_test.go @@ -0,0 +1,103 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package codes + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + cpb "google.golang.org/genproto/googleapis/rpc/code" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestUnmarshalJSON(t *testing.T) { + for s, v := range cpb.Code_value { + want := Code(v) + var got Code + if err := got.UnmarshalJSON([]byte(`"` + s + `"`)); err != nil || got != want { + t.Errorf("got.UnmarshalJSON(%q) = %v; want . got=%v; want %v", s, err, got, want) + } + } +} + +func (s) TestJSONUnmarshal(t *testing.T) { + var got []Code + want := []Code{OK, NotFound, Internal, Canceled} + in := `["OK", "NOT_FOUND", "INTERNAL", "CANCELLED"]` + err := json.Unmarshal([]byte(in), &got) + if err != nil || !cmp.Equal(got, want) { + t.Fatalf("json.Unmarshal(%q, &got) = %v; want . got=%v; want %v", in, err, got, want) + } +} + +func (s) TestUnmarshalJSON_NilReceiver(t *testing.T) { + var got *Code + in := OK.String() + if err := got.UnmarshalJSON([]byte(in)); err == nil { + t.Errorf("got.UnmarshalJSON(%q) = nil; want . got=%v", in, got) + } +} + +func (s) TestUnmarshalJSON_UnknownInput(t *testing.T) { + var got Code + for _, in := range [][]byte{[]byte(""), []byte("xxx"), []byte("Code(17)"), nil} { + if err := got.UnmarshalJSON([]byte(in)); err == nil { + t.Errorf("got.UnmarshalJSON(%q) = nil; want . got=%v", in, got) + } + } +} + +func (s) TestUnmarshalJSON_MarshalUnmarshal(t *testing.T) { + for i := 0; i < _maxCode; i++ { + var cUnMarshaled Code + c := Code(i) + + cJSON, err := json.Marshal(c) + if err != nil { + t.Errorf("marshalling %q failed: %v", c, err) + } + + if err := json.Unmarshal(cJSON, &cUnMarshaled); err != nil { + t.Errorf("unmarshalling code failed: %s", err) + } + + if c != cUnMarshaled { + t.Errorf("code is %q after marshalling/unmarshalling, expected %q", cUnMarshaled, c) + } + } +} + +func (s) TestUnmarshalJSON_InvalidIntegerCode(t *testing.T) { + const wantErr = "invalid code: 200" // for integer invalid code, expect integer value in error message + + var got Code + if err := got.UnmarshalJSON([]byte("200")); !strings.Contains(err.Error(), wantErr) { + t.Errorf("got.UnmarshalJSON(200) = %v; wantErr: %v", err, wantErr) + } +} diff --git a/connectivity/connectivity.go b/connectivity/connectivity.go new file mode 100644 index 0000000..4a89926 --- /dev/null +++ b/connectivity/connectivity.go @@ -0,0 +1,94 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package connectivity defines connectivity semantics. +// For details, see https://github.com/grpc/grpc/blob/master/doc/connectivity-semantics-and-api.md. +package connectivity + +import ( + "google.golang.org/grpc/grpclog" +) + +var logger = grpclog.Component("core") + +// State indicates the state of connectivity. +// It can be the state of a ClientConn or SubConn. +type State int + +func (s State) String() string { + switch s { + case Idle: + return "IDLE" + case Connecting: + return "CONNECTING" + case Ready: + return "READY" + case TransientFailure: + return "TRANSIENT_FAILURE" + case Shutdown: + return "SHUTDOWN" + default: + logger.Errorf("unknown connectivity state: %d", s) + return "INVALID_STATE" + } +} + +const ( + // Idle indicates the ClientConn is idle. + Idle State = iota + // Connecting indicates the ClientConn is connecting. + Connecting + // Ready indicates the ClientConn is ready for work. + Ready + // TransientFailure indicates the ClientConn has seen a failure but expects to recover. + TransientFailure + // Shutdown indicates the ClientConn has started shutting down. + Shutdown +) + +// ServingMode indicates the current mode of operation of the server. +// +// Only xDS enabled gRPC servers currently report their serving mode. +type ServingMode int + +const ( + // ServingModeStarting indicates that the server is starting up. + ServingModeStarting ServingMode = iota + // ServingModeServing indicates that the server contains all required + // configuration and is serving RPCs. + ServingModeServing + // ServingModeNotServing indicates that the server is not accepting new + // connections. Existing connections will be closed gracefully, allowing + // in-progress RPCs to complete. A server enters this mode when it does not + // contain the required configuration to serve RPCs. + ServingModeNotServing +) + +func (s ServingMode) String() string { + switch s { + case ServingModeStarting: + return "STARTING" + case ServingModeServing: + return "SERVING" + case ServingModeNotServing: + return "NOT_SERVING" + default: + logger.Errorf("unknown serving mode: %d", s) + return "INVALID_MODE" + } +} diff --git a/credentials/alts/alts.go b/credentials/alts/alts.go new file mode 100644 index 0000000..9fb7170 --- /dev/null +++ b/credentials/alts/alts.go @@ -0,0 +1,322 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package alts implements the ALTS credential support by gRPC library, which +// encapsulates all the state needed by a client to authenticate with a server +// using ALTS and make various assertions, e.g., about the client's identity, +// role, or whether it is authorized to make a particular call. +// This package is experimental. +package alts + +import ( + "context" + "errors" + "fmt" + "net" + "sync" + "time" + + "google.golang.org/grpc/credentials" + core "google.golang.org/grpc/credentials/alts/internal" + "google.golang.org/grpc/credentials/alts/internal/handshaker" + "google.golang.org/grpc/credentials/alts/internal/handshaker/service" + altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/googlecloud" +) + +const ( + // hypervisorHandshakerServiceAddress represents the default ALTS gRPC + // handshaker service address in the hypervisor. + hypervisorHandshakerServiceAddress = "dns:///metadata.google.internal.:8080" + // defaultTimeout specifies the server handshake timeout. + defaultTimeout = 30.0 * time.Second + // The following constants specify the minimum and maximum acceptable + // protocol versions. + protocolVersionMaxMajor = 2 + protocolVersionMaxMinor = 1 + protocolVersionMinMajor = 2 + protocolVersionMinMinor = 1 +) + +var ( + vmOnGCP bool + once sync.Once + maxRPCVersion = &altspb.RpcProtocolVersions_Version{ + Major: protocolVersionMaxMajor, + Minor: protocolVersionMaxMinor, + } + minRPCVersion = &altspb.RpcProtocolVersions_Version{ + Major: protocolVersionMinMajor, + Minor: protocolVersionMinMinor, + } + // ErrUntrustedPlatform is returned from ClientHandshake and + // ServerHandshake is running on a platform where the trustworthiness of + // the handshaker service is not guaranteed. + ErrUntrustedPlatform = errors.New("ALTS: untrusted platform. ALTS is only supported on GCP") + logger = grpclog.Component("alts") +) + +// AuthInfo exposes security information from the ALTS handshake to the +// application. This interface is to be implemented by ALTS. Users should not +// need a brand new implementation of this interface. For situations like +// testing, any new implementation should embed this interface. This allows +// ALTS to add new methods to this interface. +type AuthInfo interface { + // ApplicationProtocol returns application protocol negotiated for the + // ALTS connection. + ApplicationProtocol() string + // RecordProtocol returns the record protocol negotiated for the ALTS + // connection. + RecordProtocol() string + // SecurityLevel returns the security level of the created ALTS secure + // channel. + SecurityLevel() altspb.SecurityLevel + // PeerServiceAccount returns the peer service account. + PeerServiceAccount() string + // LocalServiceAccount returns the local service account. + LocalServiceAccount() string + // PeerRPCVersions returns the RPC version supported by the peer. + PeerRPCVersions() *altspb.RpcProtocolVersions +} + +// ClientOptions contains the client-side options of an ALTS channel. These +// options will be passed to the underlying ALTS handshaker. +type ClientOptions struct { + // TargetServiceAccounts contains a list of expected target service + // accounts. + TargetServiceAccounts []string + // HandshakerServiceAddress represents the ALTS handshaker gRPC service + // address to connect to. + HandshakerServiceAddress string +} + +// DefaultClientOptions creates a new ClientOptions object with the default +// values. +func DefaultClientOptions() *ClientOptions { + return &ClientOptions{ + HandshakerServiceAddress: hypervisorHandshakerServiceAddress, + } +} + +// ServerOptions contains the server-side options of an ALTS channel. These +// options will be passed to the underlying ALTS handshaker. +type ServerOptions struct { + // HandshakerServiceAddress represents the ALTS handshaker gRPC service + // address to connect to. + HandshakerServiceAddress string +} + +// DefaultServerOptions creates a new ServerOptions object with the default +// values. +func DefaultServerOptions() *ServerOptions { + return &ServerOptions{ + HandshakerServiceAddress: hypervisorHandshakerServiceAddress, + } +} + +// altsTC is the credentials required for authenticating a connection using ALTS. +// It implements credentials.TransportCredentials interface. +type altsTC struct { + info *credentials.ProtocolInfo + side core.Side + accounts []string + hsAddress string + boundAccessToken string +} + +// NewClientCreds constructs a client-side ALTS TransportCredentials object. +func NewClientCreds(opts *ClientOptions) credentials.TransportCredentials { + return newALTS(core.ClientSide, opts.TargetServiceAccounts, opts.HandshakerServiceAddress) +} + +// NewServerCreds constructs a server-side ALTS TransportCredentials object. +func NewServerCreds(opts *ServerOptions) credentials.TransportCredentials { + return newALTS(core.ServerSide, nil, opts.HandshakerServiceAddress) +} + +func newALTS(side core.Side, accounts []string, hsAddress string) credentials.TransportCredentials { + once.Do(func() { + vmOnGCP = googlecloud.OnGCE() + }) + if hsAddress == "" { + hsAddress = hypervisorHandshakerServiceAddress + } + return &altsTC{ + info: &credentials.ProtocolInfo{ + SecurityProtocol: "alts", + SecurityVersion: "1.0", + }, + side: side, + accounts: accounts, + hsAddress: hsAddress, + } +} + +// ClientHandshake implements the client side handshake protocol. +func (g *altsTC) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (_ net.Conn, _ credentials.AuthInfo, err error) { + if !vmOnGCP { + return nil, nil, ErrUntrustedPlatform + } + + // Connecting to ALTS handshaker service. + hsConn, err := service.Dial(g.hsAddress) + if err != nil { + return nil, nil, err + } + // Do not close hsConn since it is shared with other handshakes. + + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + defer cancel() + + opts := handshaker.DefaultClientHandshakerOptions() + opts.TargetName = addr + opts.TargetServiceAccounts = g.accounts + opts.RPCVersions = &altspb.RpcProtocolVersions{ + MaxRpcVersion: maxRPCVersion, + MinRpcVersion: minRPCVersion, + } + opts.BoundAccessToken = g.boundAccessToken + chs, err := handshaker.NewClientHandshaker(ctx, hsConn, rawConn, opts) + if err != nil { + return nil, nil, err + } + // Close the handshaker since we have obtained a connection. + defer chs.Close() + secConn, authInfo, err := chs.ClientHandshake(ctx) + if err != nil { + return nil, nil, err + } + altsAuthInfo, ok := authInfo.(AuthInfo) + if !ok { + return nil, nil, errors.New("client-side auth info is not of type alts.AuthInfo") + } + match, _ := checkRPCVersions(opts.RPCVersions, altsAuthInfo.PeerRPCVersions()) + if !match { + return nil, nil, fmt.Errorf("server-side RPC versions are not compatible with this client, local versions: %v, peer versions: %v", opts.RPCVersions, altsAuthInfo.PeerRPCVersions()) + } + return secConn, authInfo, nil +} + +// ServerHandshake implements the server side ALTS handshaker. +func (g *altsTC) ServerHandshake(rawConn net.Conn) (_ net.Conn, _ credentials.AuthInfo, err error) { + if !vmOnGCP { + return nil, nil, ErrUntrustedPlatform + } + // Connecting to ALTS handshaker service. + hsConn, err := service.Dial(g.hsAddress) + if err != nil { + return nil, nil, err + } + // Do not close hsConn since it's shared with other handshakes. + + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + opts := handshaker.DefaultServerHandshakerOptions() + opts.RPCVersions = &altspb.RpcProtocolVersions{ + MaxRpcVersion: maxRPCVersion, + MinRpcVersion: minRPCVersion, + } + shs, err := handshaker.NewServerHandshaker(ctx, hsConn, rawConn, opts) + if err != nil { + return nil, nil, err + } + secConn, authInfo, err := shs.ServerHandshake(ctx) + if err != nil { + return nil, nil, err + } + // Close the handshaker since we have obtained a connection. + defer shs.Close() + altsAuthInfo, ok := authInfo.(AuthInfo) + if !ok { + return nil, nil, errors.New("server-side auth info is not of type alts.AuthInfo") + } + match, _ := checkRPCVersions(opts.RPCVersions, altsAuthInfo.PeerRPCVersions()) + if !match { + return nil, nil, fmt.Errorf("client-side RPC versions is not compatible with this server, local versions: %v, peer versions: %v", opts.RPCVersions, altsAuthInfo.PeerRPCVersions()) + } + return secConn, authInfo, nil +} + +func (g *altsTC) Info() credentials.ProtocolInfo { + return *g.info +} + +func (g *altsTC) Clone() credentials.TransportCredentials { + info := *g.info + var accounts []string + if g.accounts != nil { + accounts = make([]string, len(g.accounts)) + copy(accounts, g.accounts) + } + return &altsTC{ + info: &info, + side: g.side, + hsAddress: g.hsAddress, + accounts: accounts, + boundAccessToken: g.boundAccessToken, + } +} + +func (g *altsTC) OverrideServerName(serverNameOverride string) error { + g.info.ServerName = serverNameOverride + return nil +} + +// compareRPCVersion returns 0 if v1 == v2, 1 if v1 > v2 and -1 if v1 < v2. +func compareRPCVersions(v1, v2 *altspb.RpcProtocolVersions_Version) int { + switch { + case v1.GetMajor() > v2.GetMajor(), + v1.GetMajor() == v2.GetMajor() && v1.GetMinor() > v2.GetMinor(): + return 1 + case v1.GetMajor() < v2.GetMajor(), + v1.GetMajor() == v2.GetMajor() && v1.GetMinor() < v2.GetMinor(): + return -1 + } + return 0 +} + +// checkRPCVersions performs a version check between local and peer rpc protocol +// versions. This function returns true if the check passes which means both +// parties agreed on a common rpc protocol to use, and false otherwise. The +// function also returns the highest common RPC protocol version both parties +// agreed on. +func checkRPCVersions(local, peer *altspb.RpcProtocolVersions) (bool, *altspb.RpcProtocolVersions_Version) { + if local == nil || peer == nil { + logger.Error("invalid checkRPCVersions argument, either local or peer is nil.") + return false, nil + } + + // maxCommonVersion is MIN(local.max, peer.max). + maxCommonVersion := local.GetMaxRpcVersion() + if compareRPCVersions(local.GetMaxRpcVersion(), peer.GetMaxRpcVersion()) > 0 { + maxCommonVersion = peer.GetMaxRpcVersion() + } + + // minCommonVersion is MAX(local.min, peer.min). + minCommonVersion := peer.GetMinRpcVersion() + if compareRPCVersions(local.GetMinRpcVersion(), peer.GetMinRpcVersion()) > 0 { + minCommonVersion = local.GetMinRpcVersion() + } + + if compareRPCVersions(maxCommonVersion, minCommonVersion) < 0 { + return false, nil + } + return true, maxCommonVersion +} diff --git a/credentials/alts/alts_test.go b/credentials/alts/alts_test.go new file mode 100644 index 0000000..cb751ce --- /dev/null +++ b/credentials/alts/alts_test.go @@ -0,0 +1,505 @@ +//go:build linux || windows +// +build linux windows + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package alts + +import ( + "context" + "reflect" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/alts/internal/handshaker" + "google.golang.org/grpc/credentials/alts/internal/handshaker/service" + altsgrpc "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" + altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" + "google.golang.org/grpc/credentials/alts/internal/testutil" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +const ( + defaultTestLongTimeout = 60 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func init() { + // The vmOnGCP global variable MUST be forced to true. Otherwise, if + // this test is run anywhere except on a GCP VM, then an ALTS handshake + // will immediately fail. + once.Do(func() {}) + vmOnGCP = true +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestInfoServerName(t *testing.T) { + // This is not testing any handshaker functionality, so it's fine to only + // use NewServerCreds and not NewClientCreds. + alts := NewServerCreds(DefaultServerOptions()) + if got, want := alts.Info().ServerName, ""; got != want { + t.Fatalf("%v.Info().ServerName = %v, want %v", alts, got, want) + } +} + +func (s) TestOverrideServerName(t *testing.T) { + wantServerName := "server.name" + // This is not testing any handshaker functionality, so it's fine to only + // use NewServerCreds and not NewClientCreds. + c := NewServerCreds(DefaultServerOptions()) + c.OverrideServerName(wantServerName) + if got, want := c.Info().ServerName, wantServerName; got != want { + t.Fatalf("c.Info().ServerName = %v, want %v", got, want) + } +} + +func (s) TestCloneClient(t *testing.T) { + wantServerName := "server.name" + opt := DefaultClientOptions() + opt.TargetServiceAccounts = []string{"not", "empty"} + c := NewClientCreds(opt) + c.(*altsTC).boundAccessToken = "bound-access-token" + c.OverrideServerName(wantServerName) + cc := c.Clone() + if got, want := cc.Info().ServerName, wantServerName; got != want { + t.Fatalf("cc.Info().ServerName = %v, want %v", got, want) + } + cc.OverrideServerName("") + if got, want := c.Info().ServerName, wantServerName; got != want { + t.Fatalf("Change in clone should not affect the original, c.Info().ServerName = %v, want %v", got, want) + } + if got, want := cc.Info().ServerName, ""; got != want { + t.Fatalf("cc.Info().ServerName = %v, want %v", got, want) + } + + ct := c.(*altsTC) + cct := cc.(*altsTC) + + if ct.side != cct.side { + t.Errorf("cc.side = %q, want %q", cct.side, ct.side) + } + if ct.hsAddress != cct.hsAddress { + t.Errorf("cc.hsAddress = %q, want %q", cct.hsAddress, ct.hsAddress) + } + if !reflect.DeepEqual(ct.accounts, cct.accounts) { + t.Errorf("cc.accounts = %q, want %q", cct.accounts, ct.accounts) + } + if ct.boundAccessToken != cct.boundAccessToken { + t.Errorf("cc.boundAccessToken = %q, want %q", cct.boundAccessToken, ct.boundAccessToken) + } +} + +func (s) TestCloneServer(t *testing.T) { + wantServerName := "server.name" + c := NewServerCreds(DefaultServerOptions()) + c.OverrideServerName(wantServerName) + cc := c.Clone() + if got, want := cc.Info().ServerName, wantServerName; got != want { + t.Fatalf("cc.Info().ServerName = %v, want %v", got, want) + } + cc.OverrideServerName("") + if got, want := c.Info().ServerName, wantServerName; got != want { + t.Fatalf("Change in clone should not affect the original, c.Info().ServerName = %v, want %v", got, want) + } + if got, want := cc.Info().ServerName, ""; got != want { + t.Fatalf("cc.Info().ServerName = %v, want %v", got, want) + } + + ct := c.(*altsTC) + cct := cc.(*altsTC) + + if ct.side != cct.side { + t.Errorf("cc.side = %q, want %q", cct.side, ct.side) + } + if ct.hsAddress != cct.hsAddress { + t.Errorf("cc.hsAddress = %q, want %q", cct.hsAddress, ct.hsAddress) + } + if !reflect.DeepEqual(ct.accounts, cct.accounts) { + t.Errorf("cc.accounts = %q, want %q", cct.accounts, ct.accounts) + } +} + +func (s) TestInfo(t *testing.T) { + // This is not testing any handshaker functionality, so it's fine to only + // use NewServerCreds and not NewClientCreds. + c := NewServerCreds(DefaultServerOptions()) + info := c.Info() + if got, want := info.SecurityProtocol, "alts"; got != want { + t.Errorf("info.SecurityProtocol=%v, want %v", got, want) + } + if got, want := info.SecurityVersion, "1.0"; got != want { + t.Errorf("info.SecurityVersion=%v, want %v", got, want) + } + if got, want := info.ServerName, ""; got != want { + t.Errorf("info.ServerName=%v, want %v", got, want) + } +} + +func (s) TestCompareRPCVersions(t *testing.T) { + for _, tc := range []struct { + v1 *altspb.RpcProtocolVersions_Version + v2 *altspb.RpcProtocolVersions_Version + output int + }{ + { + version(3, 2), + version(2, 1), + 1, + }, + { + version(3, 2), + version(3, 1), + 1, + }, + { + version(2, 1), + version(3, 2), + -1, + }, + { + version(3, 1), + version(3, 2), + -1, + }, + { + version(3, 2), + version(3, 2), + 0, + }, + } { + if got, want := compareRPCVersions(tc.v1, tc.v2), tc.output; got != want { + t.Errorf("compareRPCVersions(%v, %v)=%v, want %v", tc.v1, tc.v2, got, want) + } + } +} + +func (s) TestCheckRPCVersions(t *testing.T) { + for _, tc := range []struct { + desc string + local *altspb.RpcProtocolVersions + peer *altspb.RpcProtocolVersions + output bool + maxCommonVersion *altspb.RpcProtocolVersions_Version + }{ + { + "local.max > peer.max and local.min > peer.min", + versions(2, 1, 3, 2), + versions(1, 2, 2, 1), + true, + version(2, 1), + }, + { + "local.max > peer.max and local.min < peer.min", + versions(1, 2, 3, 2), + versions(2, 1, 2, 1), + true, + version(2, 1), + }, + { + "local.max > peer.max and local.min = peer.min", + versions(2, 1, 3, 2), + versions(2, 1, 2, 1), + true, + version(2, 1), + }, + { + "local.max < peer.max and local.min > peer.min", + versions(2, 1, 2, 1), + versions(1, 2, 3, 2), + true, + version(2, 1), + }, + { + "local.max = peer.max and local.min > peer.min", + versions(2, 1, 2, 1), + versions(1, 2, 2, 1), + true, + version(2, 1), + }, + { + "local.max < peer.max and local.min < peer.min", + versions(1, 2, 2, 1), + versions(2, 1, 3, 2), + true, + version(2, 1), + }, + { + "local.max < peer.max and local.min = peer.min", + versions(1, 2, 2, 1), + versions(1, 2, 3, 2), + true, + version(2, 1), + }, + { + "local.max = peer.max and local.min < peer.min", + versions(1, 2, 2, 1), + versions(2, 1, 2, 1), + true, + version(2, 1), + }, + { + "all equal", + versions(2, 1, 2, 1), + versions(2, 1, 2, 1), + true, + version(2, 1), + }, + { + "max is smaller than min", + versions(2, 1, 1, 2), + versions(2, 1, 1, 2), + false, + nil, + }, + { + "no overlap, local > peer", + versions(4, 3, 6, 5), + versions(1, 0, 2, 1), + false, + nil, + }, + { + "no overlap, local < peer", + versions(1, 0, 2, 1), + versions(4, 3, 6, 5), + false, + nil, + }, + { + "no overlap, max < min", + versions(6, 5, 4, 3), + versions(2, 1, 1, 0), + false, + nil, + }, + } { + output, maxCommonVersion := checkRPCVersions(tc.local, tc.peer) + if got, want := output, tc.output; got != want { + t.Errorf("%v: checkRPCVersions(%v, %v)=(%v, _), want (%v, _)", tc.desc, tc.local, tc.peer, got, want) + } + if got, want := maxCommonVersion, tc.maxCommonVersion; !proto.Equal(got, want) { + t.Errorf("%v: checkRPCVersions(%v, %v)=(_, %v), want (_, %v)", tc.desc, tc.local, tc.peer, got, want) + } + } +} + +// TestFullHandshake performs a full ALTS handshake between a test client and +// server, where both client and server offload to a local, fake handshaker +// service. +func (s) TestFullHandshake(t *testing.T) { + // Start the fake handshaker service and the server. + var wait sync.WaitGroup + defer wait.Wait() + stopHandshaker, handshakerAddress := startFakeHandshakerService(t, &wait) + defer stopHandshaker() + stopServer, serverAddress := startServer(t, handshakerAddress) + defer stopServer() + + // Ping the server, authenticating with ALTS. + establishAltsConnection(t, handshakerAddress, serverAddress) + + // Close open connections to the fake handshaker service. + if err := service.CloseForTesting(); err != nil { + t.Errorf("service.CloseForTesting() failed: %v", err) + } +} + +// TestHandshakeWithAccessToken performs an ALTS handshake between a test client and +// server, where both client and server offload to a local, fake handshaker +// service, and expects the StartClient request to include a bound access token. +func (s) TestHandshakeWithAccessToken(t *testing.T) { + // Start the fake handshaker service and the server. + var wait sync.WaitGroup + defer wait.Wait() + boundAccessToken := "fake-bound-access-token" + stopHandshaker, handshakerAddress := startFakeHandshakerServiceWithExpectedBoundAccessToken(t, &wait, boundAccessToken) + defer stopHandshaker() + stopServer, serverAddress := startServer(t, handshakerAddress) + defer stopServer() + + // Ping the server, authenticating with ALTS and a bound access token. + establishAltsConnectionWithBoundAccessToken(t, handshakerAddress, serverAddress, boundAccessToken) + + // Close open connections to the fake handshaker service. + if err := service.CloseForTesting(); err != nil { + t.Errorf("service.CloseForTesting() failed: %v", err) + } +} + +// TestConcurrentHandshakes performs a several, concurrent ALTS handshakes +// between a test client and server, where both client and server offload to a +// local, fake handshaker service. +func (s) TestConcurrentHandshakes(t *testing.T) { + // Set the max number of concurrent handshakes to 3, so that we can + // test the handshaker behavior when handshakes are queued by + // performing more than 3 concurrent handshakes (specifically, 10). + handshaker.ResetConcurrentHandshakeSemaphoreForTesting(3) + + // Start the fake handshaker service and the server. + var wait sync.WaitGroup + defer wait.Wait() + stopHandshaker, handshakerAddress := startFakeHandshakerService(t, &wait) + defer stopHandshaker() + stopServer, serverAddress := startServer(t, handshakerAddress) + defer stopServer() + + // Ping the server, authenticating with ALTS. + var waitForConnections sync.WaitGroup + for i := 0; i < 10; i++ { + waitForConnections.Add(1) + go func() { + establishAltsConnection(t, handshakerAddress, serverAddress) + waitForConnections.Done() + }() + } + waitForConnections.Wait() + + // Close open connections to the fake handshaker service. + if err := service.CloseForTesting(); err != nil { + t.Errorf("service.CloseForTesting() failed: %v", err) + } +} + +func version(major, minor uint32) *altspb.RpcProtocolVersions_Version { + return &altspb.RpcProtocolVersions_Version{ + Major: major, + Minor: minor, + } +} + +func versions(minMajor, minMinor, maxMajor, maxMinor uint32) *altspb.RpcProtocolVersions { + return &altspb.RpcProtocolVersions{ + MinRpcVersion: version(minMajor, minMinor), + MaxRpcVersion: version(maxMajor, maxMinor), + } +} + +func establishAltsConnection(t *testing.T, handshakerAddress, serverAddress string) { + establishAltsConnectionWithBoundAccessToken(t, handshakerAddress, serverAddress, "") +} + +func establishAltsConnectionWithBoundAccessToken(t *testing.T, handshakerAddress, serverAddress, boundAccessToken string) { + clientCreds := NewClientCreds(&ClientOptions{HandshakerServiceAddress: handshakerAddress}) + if boundAccessToken != "" { + altsCreds := clientCreds.(*altsTC) + altsCreds.boundAccessToken = boundAccessToken + } + conn, err := grpc.NewClient(serverAddress, grpc.WithTransportCredentials(clientCreds)) + if err != nil { + t.Fatalf("grpc.NewClient(%v) failed: %v", serverAddress, err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestLongTimeout) + defer cancel() + c := testgrpc.NewTestServiceClient(conn) + var peer peer.Peer + success := false + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + _, err = c.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(&peer)) + if err == nil { + success = true + break + } + if code := status.Code(err); code == codes.Unavailable || code == codes.DeadlineExceeded { + // The server is not ready yet or there were too many concurrent handshakes. + // Try again. + continue + } + t.Fatalf("c.UnaryCall() failed: %v", err) + } + if !success { + t.Fatalf("c.UnaryCall() timed out after %v", defaultTestShortTimeout) + } + + // Check that peer.AuthInfo was populated with an ALTS AuthInfo + // instance. As a sanity check, also verify that the AuthType() and + // ApplicationProtocol() have the expected values. + if got, want := peer.AuthInfo.AuthType(), "alts"; got != want { + t.Errorf("authInfo.AuthType() = %s, want = %s", got, want) + } + authInfo, err := AuthInfoFromPeer(&peer) + if err != nil { + t.Errorf("AuthInfoFromPeer failed: %v", err) + } + if got, want := authInfo.ApplicationProtocol(), "grpc"; got != want { + t.Errorf("authInfo.ApplicationProtocol() = %s, want = %s", got, want) + } +} + +func startFakeHandshakerService(t *testing.T, wait *sync.WaitGroup) (stop func(), address string) { + return startFakeHandshakerServiceWithExpectedBoundAccessToken(t, wait, "") +} + +func startFakeHandshakerServiceWithExpectedBoundAccessToken(t *testing.T, wait *sync.WaitGroup, boundAccessToken string) (stop func(), address string) { + listener, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("LocalTCPListener() failed: %v", err) + } + s := grpc.NewServer() + hs := &testutil.FakeHandshaker{} + if boundAccessToken != "" { + hs.ExpectedBoundAccessToken = boundAccessToken + } + altsgrpc.RegisterHandshakerServiceServer(s, hs) + wait.Add(1) + go func() { + defer wait.Done() + if err := s.Serve(listener); err != nil { + t.Errorf("failed to serve: %v", err) + } + }() + return func() { s.Stop() }, listener.Addr().String() +} + +func startServer(t *testing.T, handshakerServiceAddress string) (stop func(), address string) { + listener, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("LocalTCPListener() failed: %v", err) + } + serverOpts := &ServerOptions{HandshakerServiceAddress: handshakerServiceAddress} + creds := NewServerCreds(serverOpts) + stub := &stubserver.StubServer{ + Listener: listener, + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{ + Payload: &testpb.Payload{}, + }, nil + }, + S: grpc.NewServer(grpc.Creds(creds)), + } + stubserver.StartTestService(t, stub) + return func() { stub.S.Stop() }, listener.Addr().String() +} diff --git a/credentials/alts/internal/authinfo/authinfo.go b/credentials/alts/internal/authinfo/authinfo.go new file mode 100644 index 0000000..ebea57d --- /dev/null +++ b/credentials/alts/internal/authinfo/authinfo.go @@ -0,0 +1,95 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package authinfo provide authentication information returned by handshakers. +package authinfo + +import ( + "google.golang.org/grpc/credentials" + altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" +) + +var _ credentials.AuthInfo = (*altsAuthInfo)(nil) + +// altsAuthInfo exposes security information from the ALTS handshake to the +// application. altsAuthInfo is immutable and implements credentials.AuthInfo. +type altsAuthInfo struct { + p *altspb.AltsContext + credentials.CommonAuthInfo +} + +// New returns a new altsAuthInfo object given handshaker results. +func New(result *altspb.HandshakerResult) credentials.AuthInfo { + return newAuthInfo(result) +} + +func newAuthInfo(result *altspb.HandshakerResult) *altsAuthInfo { + return &altsAuthInfo{ + p: &altspb.AltsContext{ + ApplicationProtocol: result.GetApplicationProtocol(), + RecordProtocol: result.GetRecordProtocol(), + // TODO: assign security level from result. + SecurityLevel: altspb.SecurityLevel_INTEGRITY_AND_PRIVACY, + PeerServiceAccount: result.GetPeerIdentity().GetServiceAccount(), + LocalServiceAccount: result.GetLocalIdentity().GetServiceAccount(), + PeerRpcVersions: result.GetPeerRpcVersions(), + PeerAttributes: result.GetPeerIdentity().GetAttributes(), + }, + CommonAuthInfo: credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}, + } +} + +// AuthType identifies the context as providing ALTS authentication information. +func (s *altsAuthInfo) AuthType() string { + return "alts" +} + +// ApplicationProtocol returns the context's application protocol. +func (s *altsAuthInfo) ApplicationProtocol() string { + return s.p.GetApplicationProtocol() +} + +// RecordProtocol returns the context's record protocol. +func (s *altsAuthInfo) RecordProtocol() string { + return s.p.GetRecordProtocol() +} + +// SecurityLevel returns the context's security level. +func (s *altsAuthInfo) SecurityLevel() altspb.SecurityLevel { + return s.p.GetSecurityLevel() +} + +// PeerServiceAccount returns the context's peer service account. +func (s *altsAuthInfo) PeerServiceAccount() string { + return s.p.GetPeerServiceAccount() +} + +// LocalServiceAccount returns the context's local service account. +func (s *altsAuthInfo) LocalServiceAccount() string { + return s.p.GetLocalServiceAccount() +} + +// PeerRPCVersions returns the context's peer RPC versions. +func (s *altsAuthInfo) PeerRPCVersions() *altspb.RpcProtocolVersions { + return s.p.GetPeerRpcVersions() +} + +// PeerAttributes returns the context's peer attributes. +func (s *altsAuthInfo) PeerAttributes() map[string]string { + return s.p.GetPeerAttributes() +} diff --git a/credentials/alts/internal/authinfo/authinfo_test.go b/credentials/alts/internal/authinfo/authinfo_test.go new file mode 100644 index 0000000..722eedd --- /dev/null +++ b/credentials/alts/internal/authinfo/authinfo_test.go @@ -0,0 +1,156 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package authinfo + +import ( + "reflect" + "testing" + + altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + testAppProtocol = "my_app" + testRecordProtocol = "very_secure_protocol" + testPeerAccount = "peer_service_account" + testLocalAccount = "local_service_account" + testPeerHostname = "peer_hostname" + testLocalHostname = "local_hostname" + testLocalPeerAttributeKey = "peer" + testLocalPeerAttributeValue = "attributes" +) + +func (s) TestALTSAuthInfo(t *testing.T) { + testPeerAttributes := make(map[string]string) + testPeerAttributes[testLocalPeerAttributeKey] = testLocalPeerAttributeValue + for _, tc := range []struct { + result *altspb.HandshakerResult + outAppProtocol string + outRecordProtocol string + outSecurityLevel altspb.SecurityLevel + outPeerAccount string + outLocalAccount string + outPeerRPCVersions *altspb.RpcProtocolVersions + outPeerAttributes map[string]string + }{ + { + &altspb.HandshakerResult{ + ApplicationProtocol: testAppProtocol, + RecordProtocol: testRecordProtocol, + PeerIdentity: &altspb.Identity{ + IdentityOneof: &altspb.Identity_ServiceAccount{ + ServiceAccount: testPeerAccount, + }, + Attributes: testPeerAttributes, + }, + LocalIdentity: &altspb.Identity{ + IdentityOneof: &altspb.Identity_ServiceAccount{ + ServiceAccount: testLocalAccount, + }, + }, + }, + testAppProtocol, + testRecordProtocol, + altspb.SecurityLevel_INTEGRITY_AND_PRIVACY, + testPeerAccount, + testLocalAccount, + nil, + testPeerAttributes, + }, + { + &altspb.HandshakerResult{ + ApplicationProtocol: testAppProtocol, + RecordProtocol: testRecordProtocol, + PeerIdentity: &altspb.Identity{ + IdentityOneof: &altspb.Identity_Hostname{ + Hostname: testPeerHostname, + }, + Attributes: testPeerAttributes, + }, + LocalIdentity: &altspb.Identity{ + IdentityOneof: &altspb.Identity_Hostname{ + Hostname: testLocalHostname, + }, + }, + PeerRpcVersions: &altspb.RpcProtocolVersions{ + MaxRpcVersion: &altspb.RpcProtocolVersions_Version{ + Major: 20, + Minor: 21, + }, + MinRpcVersion: &altspb.RpcProtocolVersions_Version{ + Major: 10, + Minor: 11, + }, + }, + }, + testAppProtocol, + testRecordProtocol, + altspb.SecurityLevel_INTEGRITY_AND_PRIVACY, + "", + "", + &altspb.RpcProtocolVersions{ + MaxRpcVersion: &altspb.RpcProtocolVersions_Version{ + Major: 20, + Minor: 21, + }, + MinRpcVersion: &altspb.RpcProtocolVersions_Version{ + Major: 10, + Minor: 11, + }, + }, + testPeerAttributes, + }, + } { + authInfo := newAuthInfo(tc.result) + if got, want := authInfo.AuthType(), "alts"; got != want { + t.Errorf("authInfo.AuthType()=%v, want %v", got, want) + } + if got, want := authInfo.ApplicationProtocol(), tc.outAppProtocol; got != want { + t.Errorf("authInfo.ApplicationProtocol()=%v, want %v", got, want) + } + if got, want := authInfo.RecordProtocol(), tc.outRecordProtocol; got != want { + t.Errorf("authInfo.RecordProtocol()=%v, want %v", got, want) + } + if got, want := authInfo.SecurityLevel(), tc.outSecurityLevel; got != want { + t.Errorf("authInfo.SecurityLevel()=%v, want %v", got, want) + } + if got, want := authInfo.PeerServiceAccount(), tc.outPeerAccount; got != want { + t.Errorf("authInfo.PeerServiceAccount()=%v, want %v", got, want) + } + if got, want := authInfo.LocalServiceAccount(), tc.outLocalAccount; got != want { + t.Errorf("authInfo.LocalServiceAccount()=%v, want %v", got, want) + } + if got, want := authInfo.PeerRPCVersions(), tc.outPeerRPCVersions; !reflect.DeepEqual(got, want) { + t.Errorf("authinfo.PeerRpcVersions()=%v, want %v", got, want) + } + if got, want := authInfo.PeerAttributes(), tc.outPeerAttributes; !reflect.DeepEqual(got, want) { + t.Errorf("authinfo.PeerAttributes()=%v, want %v", got, want) + } + + } +} diff --git a/credentials/alts/internal/common.go b/credentials/alts/internal/common.go new file mode 100644 index 0000000..3896e8c --- /dev/null +++ b/credentials/alts/internal/common.go @@ -0,0 +1,67 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains common core functionality for ALTS. +package internal + +import ( + "context" + "net" + + "google.golang.org/grpc/credentials" +) + +const ( + // ClientSide identifies the client in this communication. + ClientSide Side = iota + // ServerSide identifies the server in this communication. + ServerSide +) + +// PeerNotRespondingError is returned when a peer server is not responding +// after a channel has been established. It is treated as a temporary connection +// error and re-connection to the server should be attempted. +var PeerNotRespondingError = &peerNotRespondingError{} + +// Side identifies the party's role: client or server. +type Side int + +type peerNotRespondingError struct{} + +// Return an error message for the purpose of logging. +func (e *peerNotRespondingError) Error() string { + return "peer server is not responding and re-connection should be attempted." +} + +// Temporary indicates if this connection error is temporary or fatal. +func (e *peerNotRespondingError) Temporary() bool { + return true +} + +// Handshaker defines a ALTS handshaker interface. +type Handshaker interface { + // ClientHandshake starts and completes a client-side handshaking and + // returns a secure connection and corresponding auth information. + ClientHandshake(ctx context.Context) (net.Conn, credentials.AuthInfo, error) + // ServerHandshake starts and completes a server-side handshaking and + // returns a secure connection and corresponding auth information. + ServerHandshake(ctx context.Context) (net.Conn, credentials.AuthInfo, error) + // Close terminates the Handshaker. It should be called when the caller + // obtains the secure connection. + Close() +} diff --git a/credentials/alts/internal/conn/aeadrekey.go b/credentials/alts/internal/conn/aeadrekey.go new file mode 100644 index 0000000..7e4bfee --- /dev/null +++ b/credentials/alts/internal/conn/aeadrekey.go @@ -0,0 +1,131 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package conn + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/sha256" + "encoding/binary" + "fmt" + "strconv" +) + +// rekeyAEAD holds the necessary information for an AEAD based on +// AES-GCM that performs nonce-based key derivation and XORs the +// nonce with a random mask. +type rekeyAEAD struct { + kdfKey []byte + kdfCounter []byte + nonceMask []byte + nonceBuf []byte + gcmAEAD cipher.AEAD +} + +// KeySizeError signals that the given key does not have the correct size. +type KeySizeError int + +func (k KeySizeError) Error() string { + return "alts/conn: invalid key size " + strconv.Itoa(int(k)) +} + +// newRekeyAEAD creates a new instance of aes128gcm with rekeying. +// The key argument should be 44 bytes, the first 32 bytes are used as a key +// for HKDF-expand and the remaining 12 bytes are used as a random mask for +// the counter. +func newRekeyAEAD(key []byte) (*rekeyAEAD, error) { + k := len(key) + if k != kdfKeyLen+nonceLen { + return nil, KeySizeError(k) + } + return &rekeyAEAD{ + kdfKey: key[:kdfKeyLen], + kdfCounter: make([]byte, kdfCounterLen), + nonceMask: key[kdfKeyLen:], + nonceBuf: make([]byte, nonceLen), + gcmAEAD: nil, + }, nil +} + +// Seal rekeys if nonce[2:8] is different than in the last call, masks the nonce, +// and calls Seal for aes128gcm. +func (s *rekeyAEAD) Seal(dst, nonce, plaintext, additionalData []byte) []byte { + if err := s.rekeyIfRequired(nonce); err != nil { + panic(fmt.Sprintf("Rekeying failed with: %s", err.Error())) + } + maskNonce(s.nonceBuf, nonce, s.nonceMask) + return s.gcmAEAD.Seal(dst, s.nonceBuf, plaintext, additionalData) +} + +// Open rekeys if nonce[2:8] is different than in the last call, masks the nonce, +// and calls Open for aes128gcm. +func (s *rekeyAEAD) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) { + if err := s.rekeyIfRequired(nonce); err != nil { + return nil, err + } + maskNonce(s.nonceBuf, nonce, s.nonceMask) + return s.gcmAEAD.Open(dst, s.nonceBuf, ciphertext, additionalData) +} + +// rekeyIfRequired creates a new aes128gcm AEAD if the existing AEAD is nil +// or cannot be used with given nonce. +func (s *rekeyAEAD) rekeyIfRequired(nonce []byte) error { + newKdfCounter := nonce[kdfCounterOffset : kdfCounterOffset+kdfCounterLen] + if s.gcmAEAD != nil && bytes.Equal(newKdfCounter, s.kdfCounter) { + return nil + } + copy(s.kdfCounter, newKdfCounter) + a, err := aes.NewCipher(hkdfExpand(s.kdfKey, s.kdfCounter)) + if err != nil { + return err + } + s.gcmAEAD, err = cipher.NewGCM(a) + return err +} + +// maskNonce XORs the given nonce with the mask and stores the result in dst. +func maskNonce(dst, nonce, mask []byte) { + nonce1 := binary.LittleEndian.Uint64(nonce[:sizeUint64]) + nonce2 := binary.LittleEndian.Uint32(nonce[sizeUint64:]) + mask1 := binary.LittleEndian.Uint64(mask[:sizeUint64]) + mask2 := binary.LittleEndian.Uint32(mask[sizeUint64:]) + binary.LittleEndian.PutUint64(dst[:sizeUint64], nonce1^mask1) + binary.LittleEndian.PutUint32(dst[sizeUint64:], nonce2^mask2) +} + +// NonceSize returns the required nonce size. +func (s *rekeyAEAD) NonceSize() int { + return s.gcmAEAD.NonceSize() +} + +// Overhead returns the ciphertext overhead. +func (s *rekeyAEAD) Overhead() int { + return s.gcmAEAD.Overhead() +} + +// hkdfExpand computes the first 16 bytes of the HKDF-expand function +// defined in RFC5869. +func hkdfExpand(key, info []byte) []byte { + mac := hmac.New(sha256.New, key) + mac.Write(info) + mac.Write([]byte{0x01}[:]) + return mac.Sum(nil)[:aeadKeyLen] +} diff --git a/credentials/alts/internal/conn/aeadrekey_test.go b/credentials/alts/internal/conn/aeadrekey_test.go new file mode 100644 index 0000000..431d6b5 --- /dev/null +++ b/credentials/alts/internal/conn/aeadrekey_test.go @@ -0,0 +1,263 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package conn + +import ( + "bytes" + "encoding/hex" + "testing" +) + +// cryptoTestVector is struct for a rekey test vector +type rekeyAEADTestVector struct { + desc string + key, nonce, plaintext, aad, ciphertext []byte +} + +// Test encrypt and decrypt using (adapted) test vectors for AES-GCM. +func (s) TestAES128GCMRekeyEncrypt(t *testing.T) { + for _, test := range []rekeyAEADTestVector{ + // NIST vectors from: + // http://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/gcm/gcm-revised-spec.pdf + // + // IEEE vectors from: + // http://www.ieee802.org/1/files/public/docs2011/bn-randall-test-vectors-0511-v1.pdf + // + // Key expanded by setting + // expandedKey = (key || + // key ^ {0x01,..,0x01} || + // key ^ {0x02,..,0x02})[0:44]. + { + desc: "Derived from NIST test vector 1", + key: dehex("0000000000000000000000000000000001010101010101010101010101010101020202020202020202020202"), + nonce: dehex("000000000000000000000000"), + aad: dehex(""), + plaintext: dehex(""), + ciphertext: dehex("85e873e002f6ebdc4060954eb8675508"), + }, + { + desc: "Derived from NIST test vector 2", + key: dehex("0000000000000000000000000000000001010101010101010101010101010101020202020202020202020202"), + nonce: dehex("000000000000000000000000"), + aad: dehex(""), + plaintext: dehex("00000000000000000000000000000000"), + ciphertext: dehex("51e9a8cb23ca2512c8256afff8e72d681aca19a1148ac115e83df4888cc00d11"), + }, + { + desc: "Derived from NIST test vector 3", + key: dehex("feffe9928665731c6d6a8f9467308308fffee8938764721d6c6b8e9566318209fcfdeb908467711e6f688d96"), + nonce: dehex("cafebabefacedbaddecaf888"), + aad: dehex(""), + plaintext: dehex("d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b391aafd255"), + ciphertext: dehex("1018ed5a1402a86516d6576d70b2ffccca261b94df88b58f53b64dfba435d18b2f6e3b7869f9353d4ac8cf09afb1663daa7b4017e6fc2c177c0c087c0df1162129952213cee1bc6e9c8495dd705e1f3d"), + }, + { + desc: "Derived from NIST test vector 4", + key: dehex("feffe9928665731c6d6a8f9467308308fffee8938764721d6c6b8e9566318209fcfdeb908467711e6f688d96"), + nonce: dehex("cafebabefacedbaddecaf888"), + aad: dehex("feedfacedeadbeeffeedfacedeadbeefabaddad2"), + plaintext: dehex("d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39"), + ciphertext: dehex("1018ed5a1402a86516d6576d70b2ffccca261b94df88b58f53b64dfba435d18b2f6e3b7869f9353d4ac8cf09afb1663daa7b4017e6fc2c177c0c087c4764565d077e9124001ddb27fc0848c5"), + }, + { + desc: "Derived from adapted NIST test vector 4 for KDF counter boundary (flip nonce bit 15)", + key: dehex("feffe9928665731c6d6a8f9467308308fffee8938764721d6c6b8e9566318209fcfdeb908467711e6f688d96"), + nonce: dehex("ca7ebabefacedbaddecaf888"), + aad: dehex("feedfacedeadbeeffeedfacedeadbeefabaddad2"), + plaintext: dehex("d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39"), + ciphertext: dehex("e650d3c0fb879327f2d03287fa93cd07342b136215adbca00c3bd5099ec41832b1d18e0423ed26bb12c6cd09debb29230a94c0cee15903656f85edb6fc509b1b28216382172ecbcc31e1e9b1"), + }, + { + desc: "Derived from adapted NIST test vector 4 for KDF counter boundary (flip nonce bit 16)", + key: dehex("feffe9928665731c6d6a8f9467308308fffee8938764721d6c6b8e9566318209fcfdeb908467711e6f688d96"), + nonce: dehex("cafebbbefacedbaddecaf888"), + aad: dehex("feedfacedeadbeeffeedfacedeadbeefabaddad2"), + plaintext: dehex("d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39"), + ciphertext: dehex("c0121e6c954d0767f96630c33450999791b2da2ad05c4190169ccad9ac86ff1c721e3d82f2ad22ab463bab4a0754b7dd68ca4de7ea2531b625eda01f89312b2ab957d5c7f8568dd95fcdcd1f"), + }, + { + desc: "Derived from adapted NIST test vector 4 for KDF counter boundary (flip nonce bit 63)", + key: dehex("feffe9928665731c6d6a8f9467308308fffee8938764721d6c6b8e9566318209fcfdeb908467711e6f688d96"), + nonce: dehex("cafebabefacedb2ddecaf888"), + aad: dehex("feedfacedeadbeeffeedfacedeadbeefabaddad2"), + plaintext: dehex("d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39"), + ciphertext: dehex("8af37ea5684a4d81d4fd817261fd9743099e7e6a025eaacf8e54b124fb5743149e05cb89f4a49467fe2e5e5965f29a19f99416b0016b54585d12553783ba59e9f782e82e097c336bf7989f08"), + }, + { + desc: "Derived from adapted NIST test vector 4 for KDF counter boundary (flip nonce bit 64)", + key: dehex("feffe9928665731c6d6a8f9467308308fffee8938764721d6c6b8e9566318209fcfdeb908467711e6f688d96"), + nonce: dehex("cafebabefacedbaddfcaf888"), + aad: dehex("feedfacedeadbeeffeedfacedeadbeefabaddad2"), + plaintext: dehex("d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39"), + ciphertext: dehex("fbd528448d0346bfa878634864d407a35a039de9db2f1feb8e965b3ae9356ce6289441d77f8f0df294891f37ea438b223e3bf2bdc53d4c5a74fb680bb312a8dec6f7252cbcd7f5799750ad78"), + }, + { + desc: "Derived from IEEE 2.1.1 54-byte auth", + key: dehex("ad7a2bd03eac835a6f620fdcb506b345ac7b2ad13fad825b6e630eddb407b244af7829d23cae81586d600dde"), + nonce: dehex("12153524c0895e81b2c28465"), + aad: dehex("d609b1f056637a0d46df998d88e5222ab2c2846512153524c0895e8108000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f30313233340001"), + plaintext: dehex(""), + ciphertext: dehex("3ea0b584f3c85e93f9320ea591699efb"), + }, + { + desc: "Derived from IEEE 2.1.2 54-byte auth", + key: dehex("e3c08a8f06c6e3ad95a70557b23f75483ce33021a9c72b7025666204c69c0b72e1c2888d04c4e1af97a50755"), + nonce: dehex("12153524c0895e81b2c28465"), + aad: dehex("d609b1f056637a0d46df998d88e5222ab2c2846512153524c0895e8108000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f30313233340001"), + plaintext: dehex(""), + ciphertext: dehex("294e028bf1fe6f14c4e8f7305c933eb5"), + }, + { + desc: "Derived from IEEE 2.2.1 60-byte crypt", + key: dehex("ad7a2bd03eac835a6f620fdcb506b345ac7b2ad13fad825b6e630eddb407b244af7829d23cae81586d600dde"), + nonce: dehex("12153524c0895e81b2c28465"), + aad: dehex("d609b1f056637a0d46df998d88e52e00b2c2846512153524c0895e81"), + plaintext: dehex("08000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a0002"), + ciphertext: dehex("db3d25719c6b0a3ca6145c159d5c6ed9aff9c6e0b79f17019ea923b8665ddf52137ad611f0d1bf417a7ca85e45afe106ff9c7569d335d086ae6c03f00987ccd6"), + }, + { + desc: "Derived from IEEE 2.2.2 60-byte crypt", + key: dehex("e3c08a8f06c6e3ad95a70557b23f75483ce33021a9c72b7025666204c69c0b72e1c2888d04c4e1af97a50755"), + nonce: dehex("12153524c0895e81b2c28465"), + aad: dehex("d609b1f056637a0d46df998d88e52e00b2c2846512153524c0895e81"), + plaintext: dehex("08000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a0002"), + ciphertext: dehex("1641f28ec13afcc8f7903389787201051644914933e9202bb9d06aa020c2a67ef51dfe7bc00a856c55b8f8133e77f659132502bad63f5713d57d0c11e0f871ed"), + }, + { + desc: "Derived from IEEE 2.3.1 60-byte auth", + key: dehex("071b113b0ca743fecccf3d051f737382061a103a0da642ffcdce3c041e727283051913390ea541fccecd3f07"), + nonce: dehex("f0761e8dcd3d000176d457ed"), + aad: dehex("e20106d7cd0df0761e8dcd3d88e5400076d457ed08000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a0003"), + plaintext: dehex(""), + ciphertext: dehex("58837a10562b0f1f8edbe58ca55811d3"), + }, + { + desc: "Derived from IEEE 2.3.2 60-byte auth", + key: dehex("691d3ee909d7f54167fd1ca0b5d769081f2bde1aee655fdbab80bd5295ae6be76b1f3ceb0bd5f74365ff1ea2"), + nonce: dehex("f0761e8dcd3d000176d457ed"), + aad: dehex("e20106d7cd0df0761e8dcd3d88e5400076d457ed08000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a0003"), + plaintext: dehex(""), + ciphertext: dehex("c2722ff6ca29a257718a529d1f0c6a3b"), + }, + { + desc: "Derived from IEEE 2.4.1 54-byte crypt", + key: dehex("071b113b0ca743fecccf3d051f737382061a103a0da642ffcdce3c041e727283051913390ea541fccecd3f07"), + nonce: dehex("f0761e8dcd3d000176d457ed"), + aad: dehex("e20106d7cd0df0761e8dcd3d88e54c2a76d457ed"), + plaintext: dehex("08000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f30313233340004"), + ciphertext: dehex("fd96b715b93a13346af51e8acdf792cdc7b2686f8574c70e6b0cbf16291ded427ad73fec48cd298e0528a1f4c644a949fc31dc9279706ddba33f"), + }, + { + desc: "Derived from IEEE 2.4.2 54-byte crypt", + key: dehex("691d3ee909d7f54167fd1ca0b5d769081f2bde1aee655fdbab80bd5295ae6be76b1f3ceb0bd5f74365ff1ea2"), + nonce: dehex("f0761e8dcd3d000176d457ed"), + aad: dehex("e20106d7cd0df0761e8dcd3d88e54c2a76d457ed"), + plaintext: dehex("08000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f30313233340004"), + ciphertext: dehex("b68f6300c2e9ae833bdc070e24021a3477118e78ccf84e11a485d861476c300f175353d5cdf92008a4f878e6cc3577768085c50a0e98fda6cbb8"), + }, + { + desc: "Derived from IEEE 2.5.1 65-byte auth", + key: dehex("013fe00b5f11be7f866d0cbbc55a7a90003ee10a5e10bf7e876c0dbac45b7b91033de2095d13bc7d846f0eb9"), + nonce: dehex("7cfde9f9e33724c68932d612"), + aad: dehex("84c5d513d2aaf6e5bbd2727788e523008932d6127cfde9f9e33724c608000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f0005"), + plaintext: dehex(""), + ciphertext: dehex("cca20eecda6283f09bb3543dd99edb9b"), + }, + { + desc: "Derived from IEEE 2.5.2 65-byte auth", + key: dehex("83c093b58de7ffe1c0da926ac43fb3609ac1c80fee1b624497ef942e2f79a82381c291b78fe5fde3c2d89068"), + nonce: dehex("7cfde9f9e33724c68932d612"), + aad: dehex("84c5d513d2aaf6e5bbd2727788e523008932d6127cfde9f9e33724c608000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f0005"), + plaintext: dehex(""), + ciphertext: dehex("b232cc1da5117bf15003734fa599d271"), + }, + { + desc: "Derived from IEEE 2.6.1 61-byte crypt", + key: dehex("013fe00b5f11be7f866d0cbbc55a7a90003ee10a5e10bf7e876c0dbac45b7b91033de2095d13bc7d846f0eb9"), + nonce: dehex("7cfde9f9e33724c68932d612"), + aad: dehex("84c5d513d2aaf6e5bbd2727788e52f008932d6127cfde9f9e33724c6"), + plaintext: dehex("08000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b0006"), + ciphertext: dehex("ff1910d35ad7e5657890c7c560146fd038707f204b66edbc3d161f8ace244b985921023c436e3a1c3532ecd5d09a056d70be583f0d10829d9387d07d33d872e490"), + }, + { + desc: "Derived from IEEE 2.6.2 61-byte crypt", + key: dehex("83c093b58de7ffe1c0da926ac43fb3609ac1c80fee1b624497ef942e2f79a82381c291b78fe5fde3c2d89068"), + nonce: dehex("7cfde9f9e33724c68932d612"), + aad: dehex("84c5d513d2aaf6e5bbd2727788e52f008932d6127cfde9f9e33724c6"), + plaintext: dehex("08000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b0006"), + ciphertext: dehex("0db4cf956b5f97eca4eab82a6955307f9ae02a32dd7d93f83d66ad04e1cfdc5182ad12abdea5bbb619a1bd5fb9a573590fba908e9c7a46c1f7ba0905d1b55ffda4"), + }, + { + desc: "Derived from IEEE 2.7.1 79-byte crypt", + key: dehex("88ee087fd95da9fbf6725aa9d757b0cd89ef097ed85ca8faf7735ba8d656b1cc8aec0a7ddb5fabf9f47058ab"), + nonce: dehex("7ae8e2ca4ec500012e58495c"), + aad: dehex("68f2e77696ce7ae8e2ca4ec588e541002e58495c08000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d0007"), + plaintext: dehex(""), + ciphertext: dehex("813f0e630f96fb2d030f58d83f5cdfd0"), + }, + { + desc: "Derived from IEEE 2.7.2 79-byte crypt", + key: dehex("4c973dbc7364621674f8b5b89e5c15511fced9216490fb1c1a2caa0ffe0407e54e953fbe7166601476fab7ba"), + nonce: dehex("7ae8e2ca4ec500012e58495c"), + aad: dehex("68f2e77696ce7ae8e2ca4ec588e541002e58495c08000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d0007"), + plaintext: dehex(""), + ciphertext: dehex("77e5a44c21eb07188aacbd74d1980e97"), + }, + { + desc: "Derived from IEEE 2.8.1 61-byte crypt", + key: dehex("88ee087fd95da9fbf6725aa9d757b0cd89ef097ed85ca8faf7735ba8d656b1cc8aec0a7ddb5fabf9f47058ab"), + nonce: dehex("7ae8e2ca4ec500012e58495c"), + aad: dehex("68f2e77696ce7ae8e2ca4ec588e54d002e58495c"), + plaintext: dehex("08000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748490008"), + ciphertext: dehex("958ec3f6d60afeda99efd888f175e5fcd4c87b9bcc5c2f5426253a8b506296c8c43309ab2adb5939462541d95e80811e04e706b1498f2c407c7fb234f8cc01a647550ee6b557b35a7e3945381821f4"), + }, + { + desc: "Derived from IEEE 2.8.2 61-byte crypt", + key: dehex("4c973dbc7364621674f8b5b89e5c15511fced9216490fb1c1a2caa0ffe0407e54e953fbe7166601476fab7ba"), + nonce: dehex("7ae8e2ca4ec500012e58495c"), + aad: dehex("68f2e77696ce7ae8e2ca4ec588e54d002e58495c"), + plaintext: dehex("08000f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748490008"), + ciphertext: dehex("b44d072011cd36d272a9b7a98db9aa90cbc5c67b93ddce67c854503214e2e896ec7e9db649ed4bcf6f850aac0223d0cf92c83db80795c3a17ecc1248bb00591712b1ae71e268164196252162810b00"), + }} { + aead, err := newRekeyAEAD(test.key) + if err != nil { + t.Fatal("unexpected failure in newRekeyAEAD: ", err.Error()) + } + if got := aead.Seal(nil, test.nonce, test.plaintext, test.aad); !bytes.Equal(got, test.ciphertext) { + t.Errorf("Unexpected ciphertext for test vector '%s':\nciphertext=%s\nwant= %s", + test.desc, hex.EncodeToString(got), hex.EncodeToString(test.ciphertext)) + } + if got, err := aead.Open(nil, test.nonce, test.ciphertext, test.aad); err != nil || !bytes.Equal(got, test.plaintext) { + t.Errorf("Unexpected plaintext for test vector '%s':\nplaintext=%s (err=%v)\nwant= %s", + test.desc, hex.EncodeToString(got), err, hex.EncodeToString(test.plaintext)) + } + + } +} + +func dehex(s string) []byte { + if len(s) == 0 { + return make([]byte, 0) + } + b, err := hex.DecodeString(s) + if err != nil { + panic(err) + } + return b +} diff --git a/credentials/alts/internal/conn/aes128gcm.go b/credentials/alts/internal/conn/aes128gcm.go new file mode 100644 index 0000000..04e0adb --- /dev/null +++ b/credentials/alts/internal/conn/aes128gcm.go @@ -0,0 +1,105 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package conn + +import ( + "crypto/aes" + "crypto/cipher" + + core "google.golang.org/grpc/credentials/alts/internal" +) + +const ( + // Overflow length n in bytes, never encrypt more than 2^(n*8) frames (in + // each direction). + overflowLenAES128GCM = 5 +) + +// aes128gcm is the struct that holds necessary information for ALTS record. +// The counter value is NOT included in the payload during the encryption and +// decryption operations. +type aes128gcm struct { + // inCounter is used in ALTS record to check that incoming counters are + // as expected, since ALTS record guarantees that messages are unwrapped + // in the same order that the peer wrapped them. + inCounter Counter + outCounter Counter + aead cipher.AEAD +} + +// NewAES128GCM creates an instance that uses aes128gcm for ALTS record. +func NewAES128GCM(side core.Side, key []byte) (ALTSRecordCrypto, error) { + c, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + a, err := cipher.NewGCM(c) + if err != nil { + return nil, err + } + return &aes128gcm{ + inCounter: NewInCounter(side, overflowLenAES128GCM), + outCounter: NewOutCounter(side, overflowLenAES128GCM), + aead: a, + }, nil +} + +// Encrypt is the encryption function. dst can contain bytes at the beginning of +// the ciphertext that will not be encrypted but will be authenticated. If dst +// has enough capacity to hold these bytes, the ciphertext and the tag, no +// allocation and copy operations will be performed. dst and plaintext do not +// overlap. +func (s *aes128gcm) Encrypt(dst, plaintext []byte) ([]byte, error) { + // If we need to allocate an output buffer, we want to include space for + // GCM tag to avoid forcing ALTS record to reallocate as well. + dlen := len(dst) + dst, out := SliceForAppend(dst, len(plaintext)+GcmTagSize) + seq, err := s.outCounter.Value() + if err != nil { + return nil, err + } + data := out[:len(plaintext)] + copy(data, plaintext) // data may alias plaintext + + // Seal appends the ciphertext and the tag to its first argument and + // returns the updated slice. However, SliceForAppend above ensures that + // dst has enough capacity to avoid a reallocation and copy due to the + // append. + dst = s.aead.Seal(dst[:dlen], seq, data, nil) + s.outCounter.Inc() + return dst, nil +} + +func (s *aes128gcm) EncryptionOverhead() int { + return GcmTagSize +} + +func (s *aes128gcm) Decrypt(dst, ciphertext []byte) ([]byte, error) { + seq, err := s.inCounter.Value() + if err != nil { + return nil, err + } + // If dst is equal to ciphertext[:0], ciphertext storage is reused. + plaintext, err := s.aead.Open(dst, seq, ciphertext, nil) + if err != nil { + return nil, ErrAuth + } + s.inCounter.Inc() + return plaintext, nil +} diff --git a/credentials/alts/internal/conn/aes128gcm_test.go b/credentials/alts/internal/conn/aes128gcm_test.go new file mode 100644 index 0000000..3fd3030 --- /dev/null +++ b/credentials/alts/internal/conn/aes128gcm_test.go @@ -0,0 +1,223 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package conn + +import ( + "bytes" + "testing" + + core "google.golang.org/grpc/credentials/alts/internal" +) + +// cryptoTestVector is struct for a GCM test vector +type cryptoTestVector struct { + key, counter, plaintext, ciphertext, tag []byte + allocateDst bool +} + +// getGCMCryptoPair outputs a client/server pair on aes128gcm. +func getGCMCryptoPair(key []byte, counter []byte, t *testing.T) (ALTSRecordCrypto, ALTSRecordCrypto) { + client, err := NewAES128GCM(core.ClientSide, key) + if err != nil { + t.Fatalf("NewAES128GCM(ClientSide, key) = %v", err) + } + server, err := NewAES128GCM(core.ServerSide, key) + if err != nil { + t.Fatalf("NewAES128GCM(ServerSide, key) = %v", err) + } + // set counter if provided. + if counter != nil { + if CounterSide(counter) == core.ClientSide { + client.(*aes128gcm).outCounter = CounterFromValue(counter, overflowLenAES128GCM) + server.(*aes128gcm).inCounter = CounterFromValue(counter, overflowLenAES128GCM) + } else { + server.(*aes128gcm).outCounter = CounterFromValue(counter, overflowLenAES128GCM) + client.(*aes128gcm).inCounter = CounterFromValue(counter, overflowLenAES128GCM) + } + } + return client, server +} + +func testGCMEncryptionDecryption(sender ALTSRecordCrypto, receiver ALTSRecordCrypto, test *cryptoTestVector, withCounter bool, t *testing.T) { + // Ciphertext is: counter + encrypted text + tag. + ciphertext := []byte(nil) + if withCounter { + ciphertext = append(ciphertext, test.counter...) + } + ciphertext = append(ciphertext, test.ciphertext...) + ciphertext = append(ciphertext, test.tag...) + + // Decrypt. + if got, err := receiver.Decrypt(nil, ciphertext); err != nil || !bytes.Equal(got, test.plaintext) { + t.Errorf("key=%v\ncounter=%v\ntag=%v\nciphertext=%v\nDecrypt = %v, %v\nwant: %v", + test.key, test.counter, test.tag, test.ciphertext, got, err, test.plaintext) + } + + // Encrypt. + var dst []byte + if test.allocateDst { + dst = make([]byte, len(test.plaintext)+sender.EncryptionOverhead()) + } + if got, err := sender.Encrypt(dst[:0], test.plaintext); err != nil || !bytes.Equal(got, ciphertext) { + t.Errorf("key=%v\ncounter=%v\nplaintext=%v\nEncrypt = %v, %v\nwant: %v", + test.key, test.counter, test.plaintext, got, err, ciphertext) + } +} + +// Test encrypt and decrypt using test vectors for aes128gcm. +func (s) TestAES128GCMEncrypt(t *testing.T) { + for _, test := range []cryptoTestVector{ + { + key: dehex("11754cd72aec309bf52f7687212e8957"), + counter: dehex("3c819d9a9bed087615030b65"), + plaintext: nil, + ciphertext: nil, + tag: dehex("250327c674aaf477aef2675748cf6971"), + allocateDst: false, + }, + { + key: dehex("ca47248ac0b6f8372a97ac43508308ed"), + counter: dehex("ffd2b598feabc9019262d2be"), + plaintext: nil, + ciphertext: nil, + tag: dehex("60d20404af527d248d893ae495707d1a"), + allocateDst: false, + }, + { + key: dehex("7fddb57453c241d03efbed3ac44e371c"), + counter: dehex("ee283a3fc75575e33efd4887"), + plaintext: dehex("d5de42b461646c255c87bd2962d3b9a2"), + ciphertext: dehex("2ccda4a5415cb91e135c2a0f78c9b2fd"), + tag: dehex("b36d1df9b9d5e596f83e8b7f52971cb3"), + allocateDst: false, + }, + { + key: dehex("ab72c77b97cb5fe9a382d9fe81ffdbed"), + counter: dehex("54cc7dc2c37ec006bcc6d1da"), + plaintext: dehex("007c5e5b3e59df24a7c355584fc1518d"), + ciphertext: dehex("0e1bde206a07a9c2c1b65300f8c64997"), + tag: dehex("2b4401346697138c7a4891ee59867d0c"), + allocateDst: false, + }, + { + key: dehex("11754cd72aec309bf52f7687212e8957"), + counter: dehex("3c819d9a9bed087615030b65"), + plaintext: nil, + ciphertext: nil, + tag: dehex("250327c674aaf477aef2675748cf6971"), + allocateDst: true, + }, + { + key: dehex("ca47248ac0b6f8372a97ac43508308ed"), + counter: dehex("ffd2b598feabc9019262d2be"), + plaintext: nil, + ciphertext: nil, + tag: dehex("60d20404af527d248d893ae495707d1a"), + allocateDst: true, + }, + { + key: dehex("7fddb57453c241d03efbed3ac44e371c"), + counter: dehex("ee283a3fc75575e33efd4887"), + plaintext: dehex("d5de42b461646c255c87bd2962d3b9a2"), + ciphertext: dehex("2ccda4a5415cb91e135c2a0f78c9b2fd"), + tag: dehex("b36d1df9b9d5e596f83e8b7f52971cb3"), + allocateDst: true, + }, + { + key: dehex("ab72c77b97cb5fe9a382d9fe81ffdbed"), + counter: dehex("54cc7dc2c37ec006bcc6d1da"), + plaintext: dehex("007c5e5b3e59df24a7c355584fc1518d"), + ciphertext: dehex("0e1bde206a07a9c2c1b65300f8c64997"), + tag: dehex("2b4401346697138c7a4891ee59867d0c"), + allocateDst: true, + }, + } { + // Test encryption and decryption for aes128gcm. + client, server := getGCMCryptoPair(test.key, test.counter, t) + if CounterSide(test.counter) == core.ClientSide { + testGCMEncryptionDecryption(client, server, &test, false, t) + } else { + testGCMEncryptionDecryption(server, client, &test, false, t) + } + } +} + +func testGCMEncryptRoundtrip(client ALTSRecordCrypto, server ALTSRecordCrypto, t *testing.T) { + // Encrypt. + const plaintext = "This is plaintext." + var err error + buf := []byte(plaintext) + buf, err = client.Encrypt(buf[:0], buf) + if err != nil { + t.Fatal("Encrypting with client-side context: unexpected error", err, "\n", + "Plaintext:", []byte(plaintext)) + } + + // Encrypt a second message. + const plaintext2 = "This is a second plaintext." + buf2 := []byte(plaintext2) + buf2, err = client.Encrypt(buf2[:0], buf2) + if err != nil { + t.Fatal("Encrypting with client-side context: unexpected error", err, "\n", + "Plaintext:", []byte(plaintext2)) + } + + // Decryption fails: cannot decrypt second message before first. + if got, err := server.Decrypt(nil, buf2); err == nil { + t.Error("Decrypting client-side ciphertext with a client-side context unexpectedly succeeded; want unexpected counter error:\n", + " Original plaintext:", []byte(plaintext2), "\n", + " Ciphertext:", buf2, "\n", + " Decrypted plaintext:", got) + } + + // Decryption fails: wrong counter space. + if got, err := client.Decrypt(nil, buf); err == nil { + t.Error("Decrypting client-side ciphertext with a client-side context unexpectedly succeeded; want counter space error:\n", + " Original plaintext:", []byte(plaintext), "\n", + " Ciphertext:", buf, "\n", + " Decrypted plaintext:", got) + } + + // Decrypt first message. + ciphertext := append([]byte(nil), buf...) + buf, err = server.Decrypt(buf[:0], buf) + if err != nil || string(buf) != plaintext { + t.Fatal("Decrypting client-side ciphertext with a server-side context did not produce original content:\n", + " Original plaintext:", []byte(plaintext), "\n", + " Ciphertext:", ciphertext, "\n", + " Decryption error:", err, "\n", + " Decrypted plaintext:", buf) + } + + // Decryption fails: replay attack. + if got, err := server.Decrypt(nil, buf); err == nil { + t.Error("Decrypting client-side ciphertext with a client-side context unexpectedly succeeded; want unexpected counter error:\n", + " Original plaintext:", []byte(plaintext), "\n", + " Ciphertext:", buf, "\n", + " Decrypted plaintext:", got) + } +} + +// Test encrypt and decrypt on roundtrip messages for aes128gcm. +func (s) TestAES128GCMEncryptRoundtrip(t *testing.T) { + // Test for aes128gcm. + key := make([]byte, 16) + client, server := getGCMCryptoPair(key, nil, t) + testGCMEncryptRoundtrip(client, server, t) +} diff --git a/credentials/alts/internal/conn/aes128gcmrekey.go b/credentials/alts/internal/conn/aes128gcmrekey.go new file mode 100644 index 0000000..b5bbb54 --- /dev/null +++ b/credentials/alts/internal/conn/aes128gcmrekey.go @@ -0,0 +1,116 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package conn + +import ( + "crypto/cipher" + + core "google.golang.org/grpc/credentials/alts/internal" +) + +const ( + // Overflow length n in bytes, never encrypt more than 2^(n*8) frames (in + // each direction). + overflowLenAES128GCMRekey = 8 + nonceLen = 12 + aeadKeyLen = 16 + kdfKeyLen = 32 + kdfCounterOffset = 2 + kdfCounterLen = 6 + sizeUint64 = 8 +) + +// aes128gcmRekey is the struct that holds necessary information for ALTS record. +// The counter value is NOT included in the payload during the encryption and +// decryption operations. +type aes128gcmRekey struct { + // inCounter is used in ALTS record to check that incoming counters are + // as expected, since ALTS record guarantees that messages are unwrapped + // in the same order that the peer wrapped them. + inCounter Counter + outCounter Counter + inAEAD cipher.AEAD + outAEAD cipher.AEAD +} + +// NewAES128GCMRekey creates an instance that uses aes128gcm with rekeying +// for ALTS record. The key argument should be 44 bytes, the first 32 bytes +// are used as a key for HKDF-expand and the remaining 12 bytes are used +// as a random mask for the counter. +func NewAES128GCMRekey(side core.Side, key []byte) (ALTSRecordCrypto, error) { + inCounter := NewInCounter(side, overflowLenAES128GCMRekey) + outCounter := NewOutCounter(side, overflowLenAES128GCMRekey) + inAEAD, err := newRekeyAEAD(key) + if err != nil { + return nil, err + } + outAEAD, err := newRekeyAEAD(key) + if err != nil { + return nil, err + } + return &aes128gcmRekey{ + inCounter, + outCounter, + inAEAD, + outAEAD, + }, nil +} + +// Encrypt is the encryption function. dst can contain bytes at the beginning of +// the ciphertext that will not be encrypted but will be authenticated. If dst +// has enough capacity to hold these bytes, the ciphertext and the tag, no +// allocation and copy operations will be performed. dst and plaintext do not +// overlap. +func (s *aes128gcmRekey) Encrypt(dst, plaintext []byte) ([]byte, error) { + // If we need to allocate an output buffer, we want to include space for + // GCM tag to avoid forcing ALTS record to reallocate as well. + dlen := len(dst) + dst, out := SliceForAppend(dst, len(plaintext)+GcmTagSize) + seq, err := s.outCounter.Value() + if err != nil { + return nil, err + } + data := out[:len(plaintext)] + copy(data, plaintext) // data may alias plaintext + + // Seal appends the ciphertext and the tag to its first argument and + // returns the updated slice. However, SliceForAppend above ensures that + // dst has enough capacity to avoid a reallocation and copy due to the + // append. + dst = s.outAEAD.Seal(dst[:dlen], seq, data, nil) + s.outCounter.Inc() + return dst, nil +} + +func (s *aes128gcmRekey) EncryptionOverhead() int { + return GcmTagSize +} + +func (s *aes128gcmRekey) Decrypt(dst, ciphertext []byte) ([]byte, error) { + seq, err := s.inCounter.Value() + if err != nil { + return nil, err + } + plaintext, err := s.inAEAD.Open(dst, seq, ciphertext, nil) + if err != nil { + return nil, ErrAuth + } + s.inCounter.Inc() + return plaintext, nil +} diff --git a/credentials/alts/internal/conn/aes128gcmrekey_test.go b/credentials/alts/internal/conn/aes128gcmrekey_test.go new file mode 100644 index 0000000..590dfbe --- /dev/null +++ b/credentials/alts/internal/conn/aes128gcmrekey_test.go @@ -0,0 +1,112 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package conn + +import ( + "testing" + + core "google.golang.org/grpc/credentials/alts/internal" +) + +// getGCMCryptoPair outputs a client/server pair on aes128gcmRekey. +func getRekeyCryptoPair(key []byte, counter []byte, t *testing.T) (ALTSRecordCrypto, ALTSRecordCrypto) { + client, err := NewAES128GCMRekey(core.ClientSide, key) + if err != nil { + t.Fatalf("NewAES128GCMRekey(ClientSide, key) = %v", err) + } + server, err := NewAES128GCMRekey(core.ServerSide, key) + if err != nil { + t.Fatalf("NewAES128GCMRekey(ServerSide, key) = %v", err) + } + // set counter if provided. + if counter != nil { + if CounterSide(counter) == core.ClientSide { + client.(*aes128gcmRekey).outCounter = CounterFromValue(counter, overflowLenAES128GCMRekey) + server.(*aes128gcmRekey).inCounter = CounterFromValue(counter, overflowLenAES128GCMRekey) + } else { + server.(*aes128gcmRekey).outCounter = CounterFromValue(counter, overflowLenAES128GCMRekey) + client.(*aes128gcmRekey).inCounter = CounterFromValue(counter, overflowLenAES128GCMRekey) + } + } + return client, server +} + +func testRekeyEncryptRoundtrip(client ALTSRecordCrypto, server ALTSRecordCrypto, t *testing.T) { + // Encrypt. + const plaintext = "This is plaintext." + var err error + buf := []byte(plaintext) + buf, err = client.Encrypt(buf[:0], buf) + if err != nil { + t.Fatal("Encrypting with client-side context: unexpected error", err, "\n", + "Plaintext:", []byte(plaintext)) + } + + // Encrypt a second message. + const plaintext2 = "This is a second plaintext." + buf2 := []byte(plaintext2) + buf2, err = client.Encrypt(buf2[:0], buf2) + if err != nil { + t.Fatal("Encrypting with client-side context: unexpected error", err, "\n", + "Plaintext:", []byte(plaintext2)) + } + + // Decryption fails: cannot decrypt second message before first. + if got, err := server.Decrypt(nil, buf2); err == nil { + t.Error("Decrypting client-side ciphertext with a client-side context unexpectedly succeeded; want unexpected counter error:\n", + " Original plaintext:", []byte(plaintext2), "\n", + " Ciphertext:", buf2, "\n", + " Decrypted plaintext:", got) + } + + // Decryption fails: wrong counter space. + if got, err := client.Decrypt(nil, buf); err == nil { + t.Error("Decrypting client-side ciphertext with a client-side context unexpectedly succeeded; want counter space error:\n", + " Original plaintext:", []byte(plaintext), "\n", + " Ciphertext:", buf, "\n", + " Decrypted plaintext:", got) + } + + // Decrypt first message. + ciphertext := append([]byte(nil), buf...) + buf, err = server.Decrypt(buf[:0], buf) + if err != nil || string(buf) != plaintext { + t.Fatal("Decrypting client-side ciphertext with a server-side context did not produce original content:\n", + " Original plaintext:", []byte(plaintext), "\n", + " Ciphertext:", ciphertext, "\n", + " Decryption error:", err, "\n", + " Decrypted plaintext:", buf) + } + + // Decryption fails: replay attack. + if got, err := server.Decrypt(nil, buf); err == nil { + t.Error("Decrypting client-side ciphertext with a client-side context unexpectedly succeeded; want unexpected counter error:\n", + " Original plaintext:", []byte(plaintext), "\n", + " Ciphertext:", buf, "\n", + " Decrypted plaintext:", got) + } +} + +// Test encrypt and decrypt on roundtrip messages for aes128gcmRekey. +func (s) TestAES128GCMRekeyEncryptRoundtrip(t *testing.T) { + // Test for aes128gcmRekey. + key := make([]byte, 44) + client, server := getRekeyCryptoPair(key, nil, t) + testRekeyEncryptRoundtrip(client, server, t) +} diff --git a/credentials/alts/internal/conn/common.go b/credentials/alts/internal/conn/common.go new file mode 100644 index 0000000..4661713 --- /dev/null +++ b/credentials/alts/internal/conn/common.go @@ -0,0 +1,80 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package conn + +import ( + "encoding/binary" + "errors" + "fmt" +) + +const ( + // GcmTagSize is the GCM tag size is the difference in length between + // plaintext and ciphertext. From crypto/cipher/gcm.go in Go crypto + // library. + GcmTagSize = 16 +) + +// ErrAuth occurs on authentication failure. +var ErrAuth = errors.New("message authentication failed") + +// SliceForAppend takes a slice and a requested number of bytes. It returns a +// slice with the contents of the given slice followed by that many bytes and a +// second slice that aliases into it and contains only the extra bytes. If the +// original slice has sufficient capacity then no allocation is performed. +func SliceForAppend(in []byte, n int) (head, tail []byte) { + if total := len(in) + n; cap(in) >= total { + head = in[:total] + } else { + head = make([]byte, total) + copy(head, in) + } + tail = head[len(in):] + return head, tail +} + +// ParseFramedMsg parse the provided buffer and returns a frame of the format +// msgLength+msg and any remaining bytes in that buffer. +func ParseFramedMsg(b []byte, maxLen uint32) ([]byte, []byte, error) { + // If the size field is not complete, return the provided buffer as + // remaining buffer. + length, sufficientBytes := parseMessageLength(b) + if !sufficientBytes { + return nil, b, nil + } + if length > maxLen { + return nil, nil, fmt.Errorf("received the frame length %d larger than the limit %d", length, maxLen) + } + if len(b) < int(length)+4 { // account for the first 4 msg length bytes. + // Frame is not complete yet. + return nil, b, nil + } + return b[:MsgLenFieldSize+length], b[MsgLenFieldSize+length:], nil +} + +// parseMessageLength returns the message length based on frame header. It also +// returns a boolean indicating if the buffer contains sufficient bytes to parse +// the length header. If there are insufficient bytes, (0, false) is returned. +func parseMessageLength(b []byte) (uint32, bool) { + if len(b) < MsgLenFieldSize { + return 0, false + } + msgLenField := b[:MsgLenFieldSize] + return binary.LittleEndian.Uint32(msgLenField), true +} diff --git a/credentials/alts/internal/conn/counter.go b/credentials/alts/internal/conn/counter.go new file mode 100644 index 0000000..9f00aca --- /dev/null +++ b/credentials/alts/internal/conn/counter.go @@ -0,0 +1,62 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package conn + +import ( + "errors" +) + +const counterLen = 12 + +var ( + errInvalidCounter = errors.New("invalid counter") +) + +// Counter is a 96-bit, little-endian counter. +type Counter struct { + value [counterLen]byte + invalid bool + overflowLen int +} + +// Value returns the current value of the counter as a byte slice. +func (c *Counter) Value() ([]byte, error) { + if c.invalid { + return nil, errInvalidCounter + } + return c.value[:], nil +} + +// Inc increments the counter and checks for overflow. +func (c *Counter) Inc() { + // If the counter is already invalid, there is no need to increase it. + if c.invalid { + return + } + i := 0 + for ; i < c.overflowLen; i++ { + c.value[i]++ + if c.value[i] != 0 { + break + } + } + if i == c.overflowLen { + c.invalid = true + } +} diff --git a/credentials/alts/internal/conn/counter_test.go b/credentials/alts/internal/conn/counter_test.go new file mode 100644 index 0000000..0e752c3 --- /dev/null +++ b/credentials/alts/internal/conn/counter_test.go @@ -0,0 +1,141 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package conn + +import ( + "bytes" + "testing" + + core "google.golang.org/grpc/credentials/alts/internal" +) + +const ( + testOverflowLen = 5 +) + +func (s) TestCounterSides(t *testing.T) { + for _, side := range []core.Side{core.ClientSide, core.ServerSide} { + outCounter := NewOutCounter(side, testOverflowLen) + inCounter := NewInCounter(side, testOverflowLen) + for i := 0; i < 1024; i++ { + value, _ := outCounter.Value() + if g, w := CounterSide(value), side; g != w { + t.Errorf("after %d iterations, CounterSide(outCounter.Value()) = %v, want %v", i, g, w) + break + } + value, _ = inCounter.Value() + if g, w := CounterSide(value), side; g == w { + t.Errorf("after %d iterations, CounterSide(inCounter.Value()) = %v, want %v", i, g, w) + break + } + outCounter.Inc() + inCounter.Inc() + } + } +} + +func (s) TestCounterInc(t *testing.T) { + for _, test := range []struct { + counter []byte + want []byte + }{ + { + counter: []byte{0x00, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + want: []byte{0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + { + counter: []byte{0x00, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80}, + want: []byte{0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80}, + }, + { + counter: []byte{0xff, 0x00, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + want: []byte{0x00, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + { + counter: []byte{0x42, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + want: []byte{0x43, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + { + counter: []byte{0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + want: []byte{0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + }, + { + counter: []byte{0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}, + want: []byte{0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}, + }, + } { + c := CounterFromValue(test.counter, overflowLenAES128GCM) + c.Inc() + value, _ := c.Value() + if g, w := value, test.want; !bytes.Equal(g, w) || c.invalid { + t.Errorf("counter(%v).Inc() =\n%v, want\n%v", test.counter, g, w) + } + } +} + +func (s) TestRolloverCounter(t *testing.T) { + for _, test := range []struct { + desc string + value []byte + overflowLen int + }{ + { + desc: "testing overflow without rekeying 1", + value: []byte{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}, + overflowLen: 5, + }, + { + desc: "testing overflow without rekeying 2", + value: []byte{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + overflowLen: 5, + }, + { + desc: "testing overflow for rekeying mode 1", + value: []byte{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x80}, + overflowLen: 8, + }, + { + desc: "testing overflow for rekeying mode 2", + value: []byte{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00}, + overflowLen: 8, + }, + } { + c := CounterFromValue(test.value, overflowLenAES128GCM) + + // First Inc() + Value() should work. + c.Inc() + _, err := c.Value() + if err != nil { + t.Errorf("%v: first Inc() + Value() unexpectedly failed: %v, want error", test.desc, err) + } + // Second Inc() + Value() should fail. + c.Inc() + _, err = c.Value() + if err != errInvalidCounter { + t.Errorf("%v: second Inc() + Value() unexpectedly succeeded: want %v", test.desc, errInvalidCounter) + } + // Third Inc() + Value() should also fail because the counter is + // already in an invalid state. + c.Inc() + _, err = c.Value() + if err != errInvalidCounter { + t.Errorf("%v: Third Inc() + Value() unexpectedly succeeded: want %v", test.desc, errInvalidCounter) + } + } +} diff --git a/credentials/alts/internal/conn/record.go b/credentials/alts/internal/conn/record.go new file mode 100644 index 0000000..bf33b75 --- /dev/null +++ b/credentials/alts/internal/conn/record.go @@ -0,0 +1,380 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package conn contains an implementation of a secure channel created by gRPC +// handshakers. +package conn + +import ( + "encoding/binary" + "fmt" + "math" + "net" + + core "google.golang.org/grpc/credentials/alts/internal" + imem "google.golang.org/grpc/internal/mem" + "google.golang.org/grpc/internal/transport/readyreader" + "google.golang.org/grpc/mem" +) + +// ALTSRecordCrypto is the interface for gRPC ALTS record protocol. +type ALTSRecordCrypto interface { + // Encrypt encrypts the plaintext, computes the tag (if any) of dst and + // plaintext, and appends the result to dst, returning the updated slice. + // dst and plaintext may fully overlap or not at all. + Encrypt(dst, plaintext []byte) ([]byte, error) + // EncryptionOverhead returns the tag size (if any) in bytes. + EncryptionOverhead() int + // Decrypt decrypts ciphertext and verifies the tag (if any). If successful, + // this function appends the resulting plaintext to dst, returning the + // updated slice. dst and ciphertext may alias exactly or not at all. To + // reuse ciphertext's storage for the decrypted output, use ciphertext[:0] + // as dst. Even if the function fails, the contents of dst, up to its + // capacity, may be overwritten. + Decrypt(dst, ciphertext []byte) ([]byte, error) +} + +// ALTSRecordFunc is a function type for factory functions that create +// ALTSRecordCrypto instances. +type ALTSRecordFunc func(s core.Side, keyData []byte) (ALTSRecordCrypto, error) + +const ( + // MsgLenFieldSize is the byte size of the frame length field of a + // framed message. + MsgLenFieldSize = 4 + // The byte size of the message type field of a framed message. + msgTypeFieldSize = 4 + // The bytes size limit for a ALTS record message. + altsRecordLengthLimit = 1024 * 1024 // 1 MiB + // The default bytes size of a ALTS record message. + altsRecordDefaultLength = 4 * 1024 // 4KiB + // Message type value included in ALTS record framing. + altsRecordMsgType = uint32(0x06) + // The maximum write buffer size, which can hold multiple ALTS frames. + altsWriteBufferMaxSize = 512 * 1024 // 512KiB + // The initial buffer used to read from the network. + // It includes an additional 512 Bytes to hold two 16KiB records plus + // small framing overheads. + altsReadBufferInitialSize = 32*1024 + 512 // 32.5KiB +) + +var ( + protocols = make(map[string]ALTSRecordFunc) + writeBufPool *imem.BinaryTieredBufferPool + // readBufPool pools buffers of at least `altsReadBufferInitialSize` size. + // Since the read buffer size is slightly larger than 32KB, using a regular + // BinaryTieredBufferPool results in allocating buffers of almost double the + // required length. + readBufPool = imem.NewDirtySimplePool() + + // Compile-time check to ensure conn implements ReadyReader. + _ readyreader.Reader = &conn{} +) + +func init() { + pool, err := imem.NewDirtyBinaryTieredBufferPool( + 8, + 12, // Go page size, 4KB + 14, // 16KB (max HTTP/2 frame size used by gRPC) + 15, // 32KB (default buffer size for gRPC) + 16, // 64KB + 17, // 128KB + 19, // 512KB, max write buffer size + ) + if err != nil { + panic(fmt.Sprintf("Failed to create write buffer pool: %v", err)) + } + writeBufPool = pool +} + +// RegisterProtocol register a ALTS record encryption protocol. +func RegisterProtocol(protocol string, f ALTSRecordFunc) error { + if _, ok := protocols[protocol]; ok { + return fmt.Errorf("protocol %v is already registered", protocol) + } + protocols[protocol] = f + return nil +} + +// conn represents a secured connection. It implements the net.Conn interface. +type conn struct { + net.Conn + reader readyreader.Reader + crypto ALTSRecordCrypto + // buf holds data that has been read from the connection and decrypted, + // but has not yet been returned by Read. It is a sub-slice of protected. + buf []byte + payloadLengthLimit int + // protectedHandle buffer holds data read from the network but have not yet + // been decrypted. This data might not compose a complete frame. + // + // The buffer pointer to points to a buffer from the readBufPool. The handle + // should only be returned to the pool once nextFrame and buf are empty. + protectedHandle *[]byte + // nextFrame stores the next frame (in protected buffer) info. + nextFrame []byte + // overhead is the calculated overhead of each frame. + overhead int + constPool constBufferPool // stored as a field to avoid heap allocations. +} + +// NewConn creates a new secure channel instance given the other party role and +// handshaking result. +func NewConn(c net.Conn, side core.Side, recordProtocol string, key []byte, protected []byte) (net.Conn, error) { + return NewConnWithMaxFrameSize(c, side, recordProtocol, key, protected, 0) +} + +// NewConnWithMaxFrameSize creates a new secure channel instance given the +// other party role, handshaking result, and negotiated maximum frame size. +func NewConnWithMaxFrameSize(c net.Conn, side core.Side, recordProtocol string, key []byte, protected []byte, negotiatedMaxFrameSize int) (net.Conn, error) { + newCrypto := protocols[recordProtocol] + if newCrypto == nil { + return nil, fmt.Errorf("negotiated unknown next_protocol %q", recordProtocol) + } + crypto, err := newCrypto(side, key) + if err != nil { + return nil, fmt.Errorf("protocol %q: %v", recordProtocol, err) + } + overhead := MsgLenFieldSize + msgTypeFieldSize + crypto.EncryptionOverhead() + + // Clamp maxRecordLen to be at least altsRecordDefaultLength. + maxRecordLen := max(altsRecordDefaultLength, negotiatedMaxFrameSize) + payloadLengthLimit := maxRecordLen - overhead + // We pre-allocate protected to be of size 32KB during initialization. + // We increase the size of the buffer by the required amount if it can't + // hold a complete encrypted record. + protectedHandle := readBufPool.Get(max(altsReadBufferInitialSize, len(protected))) + protectedBuf := *protectedHandle + // Copy additional data from hanshaker service. + copy(protectedBuf, protected) + protectedBuf = protectedBuf[:len(protected)] + + altsConn := &conn{ + Conn: c, + reader: readyreader.New(c), + crypto: crypto, + payloadLengthLimit: payloadLengthLimit, + protectedHandle: protectedHandle, + nextFrame: protectedBuf, + overhead: overhead, + } + return altsConn, nil +} + +type constBufferPool struct { + buffer []byte +} + +func (p *constBufferPool) Get(int) *[]byte { + return &p.buffer +} + +func (p *constBufferPool) Put(*[]byte) {} + +// Read reads and decrypts a frame from the underlying connection, and copies the +// decrypted payload into b. If the size of the payload is greater than len(b), +// Read retains the remaining bytes in an internal buffer, and subsequent calls +// to Read will read from this buffer until it is exhausted. +func (p *conn) Read(b []byte) (n int, err error) { + p.constPool.buffer = b + _, n, err = p.ReadOnReady(len(b), &p.constPool) + return n, err +} + +func (p *conn) ReadOnReady(bufSize int, pool mem.BufferPool) (*[]byte, int, error) { + if len(p.buf) == 0 { + var framedMsg []byte + var protected []byte + if p.protectedHandle != nil { + protected = *p.protectedHandle + protected = protected[:cap(protected)] + } + var err error + framedMsg, p.nextFrame, err = ParseFramedMsg(p.nextFrame, altsRecordLengthLimit) + if err != nil { + return nil, 0, err + } + // Check whether the next frame to be decrypted has been + // completely received yet. + if len(framedMsg) == 0 { + copy(protected, p.nextFrame) + protected = protected[:len(p.nextFrame)] + // Always copy next incomplete frame to the beginning of + // the protected buffer and reset nextFrame to it. + p.nextFrame = protected + } + // Check whether a complete frame has been received yet. + for len(framedMsg) == 0 { + if p.protectedHandle != nil && len(protected) == cap(protected) { + // We can parse the length header to know exactly how large + // the buffer needs to be to hold the entire frame. + length, didParse := parseMessageLength(protected) + if !didParse { + // The protected buffer is initialized with a capacity of + // larger than 4B. It should always hold the message length + // header. + panic(fmt.Sprintf("protected buffer length shorter than expected: %d vs %d", len(protected), MsgLenFieldSize)) + } + oldProtectedBuf := protected + oldBufHandle := p.protectedHandle + // The new buffer must be able to hold the message length header + // and the entire message. + requiredCapacity := int(length) + MsgLenFieldSize + p.protectedHandle = readBufPool.Get(requiredCapacity) + protected = *p.protectedHandle + // Copy the contents of the old buffer and set the length of the + // new buffer to the number of bytes already read. + copy(protected, oldProtectedBuf) + protected = protected[:len(oldProtectedBuf)] + readBufPool.Put(oldBufHandle) + } + if p.protectedHandle == nil { + // Connection was idle, need to re-allocate the read buffer. + newBuf, nRead, err := p.reader.ReadOnReady(altsReadBufferInitialSize, readBufPool) + if err != nil { + return nil, 0, err + } + p.protectedHandle = newBuf + protected = (*newBuf)[:nRead] + } else { + nRead, err := p.Conn.Read(protected[len(protected):cap(protected)]) + if err != nil { + return nil, 0, err + } + protected = protected[:len(protected)+nRead] + } + framedMsg, p.nextFrame, err = ParseFramedMsg(protected, altsRecordLengthLimit) + if err != nil { + return nil, 0, err + } + } + // Now we have a complete frame, decrypted it. + msg := framedMsg[MsgLenFieldSize:] + if len(msg) < msgTypeFieldSize { + return nil, 0, fmt.Errorf("received frame with size %v which is shorter than message type field size %v", len(msg), msgTypeFieldSize) + } + msgType := binary.LittleEndian.Uint32(msg[:msgTypeFieldSize]) + if msgType&0xff != altsRecordMsgType { + return nil, 0, fmt.Errorf("received frame with incorrect message type %v, expected lower byte %v", + msgType, altsRecordMsgType) + } + ciphertext := msg[msgTypeFieldSize:] + + // Decrypt directly into the buffer, avoiding a copy from p.buf if + // possible. + if bufSize >= len(ciphertext) { + allocatedBuf := pool.Get(bufSize) + dec, err := p.crypto.Decrypt((*allocatedBuf)[:0], ciphertext) + if err != nil { + pool.Put(allocatedBuf) + return nil, 0, err + } + p.dropProtectedIfEmtpy() + return allocatedBuf, len(dec), nil + } + // Decrypt requires that if the dst and ciphertext alias, they + // must alias exactly. Code here used to use msg[:0], but msg + // starts MsgLenFieldSize+msgTypeFieldSize bytes earlier than + // ciphertext, so they alias inexactly. Using ciphertext[:0] + // arranges the appropriate aliasing without needing to copy + // ciphertext or use a separate destination buffer. For more info + // check: https://golang.org/pkg/crypto/cipher/#AEAD. + p.buf, err = p.crypto.Decrypt(ciphertext[:0], ciphertext) + if err != nil { + return nil, 0, err + } + } + + allocatedBuf := pool.Get(bufSize) + n := copy(*allocatedBuf, p.buf) + p.buf = p.buf[n:] + p.dropProtectedIfEmtpy() + return allocatedBuf, n, nil +} + +func (p *conn) dropProtectedIfEmtpy() { + if len(p.buf) > 0 || len(p.nextFrame) > 0 { + return + } + // Potentially idle connection, release the read buffer. + p.nextFrame = nil + p.buf = nil + if p.protectedHandle != nil { + readBufPool.Put(p.protectedHandle) + p.protectedHandle = nil + } +} + +// Write encrypts, frames, and writes bytes from b to the underlying connection. +func (p *conn) Write(b []byte) (n int, err error) { + n = len(b) + // Calculate the output buffer size with framing and encryption overhead. + numOfFrames := int(math.Ceil(float64(len(b)) / float64(p.payloadLengthLimit))) + size := len(b) + numOfFrames*p.overhead + partialBSize := len(b) + if size > altsWriteBufferMaxSize { + size = altsWriteBufferMaxSize + frameLen := p.payloadLengthLimit + p.overhead + numOfFramesInMaxWriteBuf := altsWriteBufferMaxSize / frameLen + partialBSize = numOfFramesInMaxWriteBuf * p.payloadLengthLimit + } + // Get a writeBuf of the required length. + bufHandle := writeBufPool.Get(size) + defer writeBufPool.Put(bufHandle) + writeBuf := *bufHandle + + for partialBStart := 0; partialBStart < len(b); partialBStart += partialBSize { + partialBEnd := min(partialBStart+partialBSize, len(b)) + partialB := b[partialBStart:partialBEnd] + writeBufIndex := 0 + for len(partialB) > 0 { + payloadLen := min(len(partialB), p.payloadLengthLimit) + buf := partialB[:payloadLen] + partialB = partialB[payloadLen:] + + // Write buffer contains: length, type, payload, and tag + // if any. + + // 1. Fill in type field. + msg := writeBuf[writeBufIndex+MsgLenFieldSize:] + binary.LittleEndian.PutUint32(msg, altsRecordMsgType) + + // 2. Encrypt the payload and create a tag if any. + msg, err = p.crypto.Encrypt(msg[:msgTypeFieldSize], buf) + if err != nil { + return n, err + } + + // 3. Fill in the size field. + binary.LittleEndian.PutUint32(writeBuf[writeBufIndex:], uint32(len(msg))) + + // 4. Increase writeBufIndex. + writeBufIndex += len(buf) + p.overhead + } + nn, err := p.Conn.Write(writeBuf[:writeBufIndex]) + if err != nil { + // We need to calculate the actual data size that was + // written. This means we need to remove header, + // encryption overheads, and any partially-written + // frame data. + numOfWrittenFrames := int(math.Floor(float64(nn) / float64(p.payloadLengthLimit+p.overhead))) + return partialBStart + numOfWrittenFrames*p.payloadLengthLimit, err + } + } + return n, nil +} diff --git a/credentials/alts/internal/conn/record_test.go b/credentials/alts/internal/conn/record_test.go new file mode 100644 index 0000000..1938098 --- /dev/null +++ b/credentials/alts/internal/conn/record_test.go @@ -0,0 +1,563 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package conn + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math" + "net" + "reflect" + "strings" + "testing" + + core "google.golang.org/grpc/credentials/alts/internal" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + rekeyRecordProtocol = "ALTSRP_GCM_AES128_REKEY" +) + +var ( + recordProtocols = []string{rekeyRecordProtocol} + altsRecordFuncs = map[string]ALTSRecordFunc{ + // ALTS handshaker protocols. + rekeyRecordProtocol: func(s core.Side, keyData []byte) (ALTSRecordCrypto, error) { + return NewAES128GCM(s, keyData) + }, + } +) + +func init() { + for protocol, f := range altsRecordFuncs { + if err := RegisterProtocol(protocol, f); err != nil { + panic(err) + } + } +} + +// testConn mimics a net.Conn to the peer. +type testConn struct { + net.Conn + in *bytes.Buffer + out *bytes.Buffer +} + +func (c *testConn) Read(b []byte) (n int, err error) { + return c.in.Read(b) +} + +func (c *testConn) Write(b []byte) (n int, err error) { + return c.out.Write(b) +} + +func (c *testConn) Close() error { + return nil +} + +func newTestALTSRecordConn(in, out *bytes.Buffer, side core.Side, rp string, protected []byte) *conn { + key := []byte{ + // 16 arbitrary bytes. + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xd2, 0x4c, 0xce, 0x4f, 0x49} + tc := testConn{ + in: in, + out: out, + } + c, err := NewConn(&tc, side, rp, key, protected) + if err != nil { + panic(fmt.Sprintf("Unexpected error creating test ALTS record connection: %v", err)) + } + return c.(*conn) +} + +func newConnPair(rp string, clientProtected []byte, serverProtected []byte) (client, server *conn) { + clientBuf := new(bytes.Buffer) + serverBuf := new(bytes.Buffer) + clientConn := newTestALTSRecordConn(clientBuf, serverBuf, core.ClientSide, rp, clientProtected) + serverConn := newTestALTSRecordConn(serverBuf, clientBuf, core.ServerSide, rp, serverProtected) + return clientConn, serverConn +} + +func testPingPong(t *testing.T, rp string) { + clientConn, serverConn := newConnPair(rp, nil, nil) + clientMsg := []byte("Client Message") + if n, err := clientConn.Write(clientMsg); n != len(clientMsg) || err != nil { + t.Fatalf("Client Write() = %v, %v; want %v, ", n, err, len(clientMsg)) + } + rcvClientMsg := make([]byte, len(clientMsg)) + if n, err := serverConn.Read(rcvClientMsg); n != len(rcvClientMsg) || err != nil { + t.Fatalf("Server Read() = %v, %v; want %v, ", n, err, len(rcvClientMsg)) + } + if !reflect.DeepEqual(clientMsg, rcvClientMsg) { + t.Fatalf("Client Write()/Server Read() = %v, want %v", rcvClientMsg, clientMsg) + } + + serverMsg := []byte("Server Message") + if n, err := serverConn.Write(serverMsg); n != len(serverMsg) || err != nil { + t.Fatalf("Server Write() = %v, %v; want %v, ", n, err, len(serverMsg)) + } + rcvServerMsg := make([]byte, len(serverMsg)) + if n, err := clientConn.Read(rcvServerMsg); n != len(rcvServerMsg) || err != nil { + t.Fatalf("Client Read() = %v, %v; want %v, ", n, err, len(rcvServerMsg)) + } + if !reflect.DeepEqual(serverMsg, rcvServerMsg) { + t.Fatalf("Server Write()/Client Read() = %v, want %v", rcvServerMsg, serverMsg) + } +} + +func (s) TestPingPong(t *testing.T) { + for _, rp := range recordProtocols { + testPingPong(t, rp) + } +} + +func testSmallReadBuffer(t *testing.T, rp string) { + clientConn, serverConn := newConnPair(rp, nil, nil) + msg := []byte("Very Important Message") + if n, err := clientConn.Write(msg); err != nil { + t.Fatalf("Write() = %v, %v; want %v, ", n, err, len(msg)) + } + rcvMsg := make([]byte, len(msg)) + n := 2 // Arbitrary index to break rcvMsg in two. + rcvMsg1 := rcvMsg[:n] + rcvMsg2 := rcvMsg[n:] + if n, err := serverConn.Read(rcvMsg1); n != len(rcvMsg1) || err != nil { + t.Fatalf("Read() = %v, %v; want %v, ", n, err, len(rcvMsg1)) + } + if n, err := serverConn.Read(rcvMsg2); n != len(rcvMsg2) || err != nil { + t.Fatalf("Read() = %v, %v; want %v, ", n, err, len(rcvMsg2)) + } + if !reflect.DeepEqual(msg, rcvMsg) { + t.Fatalf("Write()/Read() = %v, want %v", rcvMsg, msg) + } +} + +func (s) TestSmallReadBuffer(t *testing.T) { + for _, rp := range recordProtocols { + testSmallReadBuffer(t, rp) + } +} + +func testLargeMsg(t *testing.T, rp string) { + clientConn, serverConn := newConnPair(rp, nil, nil) + // msgLen is such that the length in the framing is larger than the + // default size of one frame. + msgLen := altsRecordDefaultLength - msgTypeFieldSize - clientConn.crypto.EncryptionOverhead() + 1 + msg := make([]byte, msgLen) + if n, err := clientConn.Write(msg); n != len(msg) || err != nil { + t.Fatalf("Write() = %v, %v; want %v, ", n, err, len(msg)) + } + rcvMsg := make([]byte, len(msg)) + if n, err := io.ReadFull(serverConn, rcvMsg); n != len(rcvMsg) || err != nil { + t.Fatalf("Read() = %v, %v; want %v, ", n, err, len(rcvMsg)) + } + if !reflect.DeepEqual(msg, rcvMsg) { + t.Fatalf("Write()/Server Read() = %v, want %v", rcvMsg, msg) + } +} + +func (s) TestLargeMsg(t *testing.T) { + for _, rp := range recordProtocols { + testLargeMsg(t, rp) + } +} + +// TestLargeRecord writes a very large ALTS record and verifies that the server +// receives it correctly. The large ALTS record should cause the reader to +// expand it's read buffer to hold the entire record and store the decrypted +// message until the receiver reads all of the bytes. +func (s) TestLargeRecord(t *testing.T) { + clientConn, serverConn := newConnPair(rekeyRecordProtocol, nil, nil) + msg := []byte(strings.Repeat("a", 2*altsReadBufferInitialSize)) + // Increase the size of ALTS records written by the client. + clientConn.payloadLengthLimit = math.MaxInt32 + if n, err := clientConn.Write(msg); n != len(msg) || err != nil { + t.Fatalf("Write() = %v, %v; want %v, ", n, err, len(msg)) + } + rcvMsg := make([]byte, len(msg)) + if n, err := io.ReadFull(serverConn, rcvMsg); n != len(rcvMsg) || err != nil { + t.Fatalf("Read() = %v, %v; want %v, ", n, err, len(rcvMsg)) + } + if !reflect.DeepEqual(msg, rcvMsg) { + t.Fatalf("Write()/Server Read() = %v, want %v", rcvMsg, msg) + } +} + +// BenchmarkLargeMessage measures the performance of ALTS conns for sending and +// receiving a large message. +func BenchmarkLargeMessage(b *testing.B) { + msgLen := 20 * 1024 * 1024 // 20 MiB + msg := make([]byte, msgLen) + rcvMsg := make([]byte, len(msg)) + b.ResetTimer() + clientConn, serverConn := newConnPair(rekeyRecordProtocol, nil, nil) + for range b.N { + // Write 20 MiB 5 times to transfer a total of 100 MiB. + for range 5 { + if n, err := clientConn.Write(msg); n != len(msg) || err != nil { + b.Fatalf("Write() = %v, %v; want %v, ", n, err, len(msg)) + } + if n, err := io.ReadFull(serverConn, rcvMsg); n != len(rcvMsg) || err != nil { + b.Fatalf("Read() = %v, %v; want %v, ", n, err, len(rcvMsg)) + } + } + } +} + +func testIncorrectMsgType(t *testing.T, rp string) { + // framedMsg is an empty ciphertext with correct framing but wrong + // message type. + framedMsg := make([]byte, MsgLenFieldSize+msgTypeFieldSize) + binary.LittleEndian.PutUint32(framedMsg[:MsgLenFieldSize], msgTypeFieldSize) + wrongMsgType := uint32(0x22) + binary.LittleEndian.PutUint32(framedMsg[MsgLenFieldSize:], wrongMsgType) + + in := bytes.NewBuffer(framedMsg) + c := newTestALTSRecordConn(in, nil, core.ClientSide, rp, nil) + b := make([]byte, 1) + if n, err := c.Read(b); n != 0 || err == nil { + t.Fatalf("Read() = , want %v", fmt.Errorf("received frame with incorrect message type %v", wrongMsgType)) + } +} + +func (s) TestIncorrectMsgType(t *testing.T) { + for _, rp := range recordProtocols { + testIncorrectMsgType(t, rp) + } +} + +func testFrameTooLarge(t *testing.T, rp string) { + buf := new(bytes.Buffer) + clientConn := newTestALTSRecordConn(nil, buf, core.ClientSide, rp, nil) + serverConn := newTestALTSRecordConn(buf, nil, core.ServerSide, rp, nil) + // payloadLen is such that the length in the framing is larger than + // allowed in one frame. + payloadLen := altsRecordLengthLimit - msgTypeFieldSize - clientConn.crypto.EncryptionOverhead() + 1 + payload := make([]byte, payloadLen) + c, err := clientConn.crypto.Encrypt(nil, payload) + if err != nil { + t.Fatalf("Error encrypting message: %v", err) + } + msgLen := msgTypeFieldSize + len(c) + framedMsg := make([]byte, MsgLenFieldSize+msgLen) + binary.LittleEndian.PutUint32(framedMsg[:MsgLenFieldSize], uint32(msgTypeFieldSize+len(c))) + msg := framedMsg[MsgLenFieldSize:] + binary.LittleEndian.PutUint32(msg[:msgTypeFieldSize], altsRecordMsgType) + copy(msg[msgTypeFieldSize:], c) + if _, err = buf.Write(framedMsg); err != nil { + t.Fatalf("Unexpected error writing to buffer: %v", err) + } + b := make([]byte, 1) + if n, err := serverConn.Read(b); n != 0 || err == nil { + t.Fatalf("Read() = , want %v", fmt.Errorf("received the frame length %d larger than the limit %d", altsRecordLengthLimit+1, altsRecordLengthLimit)) + } +} + +func (s) TestFrameTooLarge(t *testing.T) { + for _, rp := range recordProtocols { + testFrameTooLarge(t, rp) + } +} + +func testWriteLargeData(t *testing.T, rp string) { + // Test sending and receiving messages larger than the maximum write + // buffer size. + clientConn, serverConn := newConnPair(rp, nil, nil) + // Message size is intentionally chosen to not be multiple of + // payloadLengthLimit. + msgSize := altsWriteBufferMaxSize + (100 * 1024) + clientMsg := make([]byte, msgSize) + for i := 0; i < msgSize; i++ { + clientMsg[i] = 0xAA + } + if n, err := clientConn.Write(clientMsg); n != len(clientMsg) || err != nil { + t.Fatalf("Client Write() = %v, %v; want %v, ", n, err, len(clientMsg)) + } + // We need to keep reading until the entire message is received. The + // reason we set all bytes of the message to a value other than zero is + // to avoid ambiguous zero-init value of rcvClientMsg buffer and the + // actual received data. + rcvClientMsg := make([]byte, 0, msgSize) + numberOfExpectedFrames := int(math.Ceil(float64(msgSize) / float64(serverConn.payloadLengthLimit))) + for i := 0; i < numberOfExpectedFrames; i++ { + expectedRcvSize := serverConn.payloadLengthLimit + if i == numberOfExpectedFrames-1 { + // Last frame might be smaller. + expectedRcvSize = msgSize % serverConn.payloadLengthLimit + } + tmpBuf := make([]byte, expectedRcvSize) + if n, err := serverConn.Read(tmpBuf); n != len(tmpBuf) || err != nil { + t.Fatalf("Server Read() = %v, %v; want %v, ", n, err, len(tmpBuf)) + } + rcvClientMsg = append(rcvClientMsg, tmpBuf...) + } + if !reflect.DeepEqual(clientMsg, rcvClientMsg) { + t.Fatalf("Client Write()/Server Read() = %v, want %v", rcvClientMsg, clientMsg) + } +} + +func (s) TestWriteLargeData(t *testing.T) { + for _, rp := range recordProtocols { + testWriteLargeData(t, rp) + } +} + +func testProtectedBuffer(t *testing.T, rp string) { + key := []byte{ + // 16 arbitrary bytes. + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xd2, 0x4c, 0xce, 0x4f, 0x49} + + // Encrypt a message to be passed to NewConn as a client-side protected + // buffer. + newCrypto := protocols[rp] + if newCrypto == nil { + t.Fatalf("Unknown record protocol %q", rp) + } + crypto, err := newCrypto(core.ClientSide, key) + if err != nil { + t.Fatalf("Failed to create a crypter for protocol %q: %v", rp, err) + } + msg := []byte("Client Protected Message") + encryptedMsg, err := crypto.Encrypt(nil, msg) + if err != nil { + t.Fatalf("Failed to encrypt the client protected message: %v", err) + } + protectedMsg := make([]byte, 8) // 8 bytes = 4 length + 4 type + binary.LittleEndian.PutUint32(protectedMsg, uint32(len(encryptedMsg))+4) // 4 bytes for the type + binary.LittleEndian.PutUint32(protectedMsg[4:], altsRecordMsgType) + protectedMsg = append(protectedMsg, encryptedMsg...) + + _, serverConn := newConnPair(rp, nil, protectedMsg) + rcvClientMsg := make([]byte, len(msg)) + if n, err := serverConn.Read(rcvClientMsg); n != len(rcvClientMsg) || err != nil { + t.Fatalf("Server Read() = %v, %v; want %v, ", n, err, len(rcvClientMsg)) + } + if !reflect.DeepEqual(msg, rcvClientMsg) { + t.Fatalf("Client protected/Server Read() = %v, want %v", rcvClientMsg, msg) + } +} + +func (s) TestProtectedBuffer(t *testing.T) { + for _, rp := range recordProtocols { + testProtectedBuffer(t, rp) + } +} + +// BenchmarkWriteMemoryUsage measures the allocations per ALTS connection. +// Run this with: go test -bench=BenchmarkWriteMemoryUsage -benchmem +func BenchmarkWriteMemoryUsage(b *testing.B) { + b.ReportAllocs() + data := []byte(strings.Repeat("d", 256)) + key := []byte{ + // 16 arbitrary bytes. + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xd2, 0x4c, 0xce, 0x4f, 0x49} + conn := &noopConn{} + + for b.Loop() { + c, err := NewConn(conn, core.ClientSide, rekeyRecordProtocol, key, nil) + if err != nil { + b.Fatal(err) + } + + if _, err := c.Write(data); err != nil { + b.Fatalf("Write failed: %v", err) + } + } +} + +// BenchmarkReadMemoryUsage measures the allocations per ALTS connection. +// Run this with: go test -bench=BenchmarkReadMemoryUsage -benchmem +func BenchmarkReadMemoryUsage(b *testing.B) { + b.ReportAllocs() + data := []byte(strings.Repeat("d", 257)) + readBuf := make([]byte, 1024) + + for b.Loop() { + clientConn, serverConn := newConnPair(rekeyRecordProtocol, nil, nil) + b.StopTimer() + clientConn.Write(data) + b.StartTimer() + if _, err := serverConn.Read(readBuf); err != nil { + b.Fatalf("Write failed: %v", err) + } + } +} + +type noopConn struct { + net.Conn +} + +func (c *noopConn) Write(b []byte) (n int, err error) { + return len(b), nil +} + +func (c *noopConn) Close() error { + return nil +} + +func (s) TestParseFramedMsgVulnerability(t *testing.T) { + key := []byte{0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xd2, 0x4c, 0xce, 0x4f, 0x49} + tc := testConn{ + in: new(bytes.Buffer), + out: new(bytes.Buffer), + } + // We pass a protected buffer representing a framed message length of 0, + // followed by bytes that will act as a dirty buffer capacity. If the capacity + // bytes happen to start with `0x06` (altsRecordMsgType), the vulnerable + // code will try to slice `msg[4:]` which panics because len(msg) is 0. + malformedProtected := []byte{0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00} + c, err := NewConn(&tc, core.ServerSide, rekeyRecordProtocol, key, malformedProtected) + if err != nil { + t.Fatalf("NewConn failed: %v", err) + } + buf := make([]byte, 1024) + const wantErr = "shorter than message type field size" + if _, err := c.Read(buf); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("c.Read(buf) returned error: %v, want error containing %q", err, wantErr) + } +} + +// countALTSFrames parses raw ALTS wire bytes and returns the number of complete frames. +func countALTSFrames(data []byte) int { + count := 0 + for len(data) >= MsgLenFieldSize { + frameLen := int(binary.LittleEndian.Uint32(data[:MsgLenFieldSize])) + total := MsgLenFieldSize + frameLen + if total > len(data) { + break + } + data = data[total:] + count++ + } + return count +} + +func (s) TestNewConnNegotiatedMaxFrameSize(t *testing.T) { + key := make([]byte, 32) + + for _, tc := range []struct { + name string + negotiatedMaxFrameSize int + wantMaxRecordLen int + }{ + {"NotNegotiated", 0, altsRecordDefaultLength}, + {"SmallerThanDefault_10B", 10, altsRecordDefaultLength}, + {"SmallerThanDefault_1024B", 1024, altsRecordDefaultLength}, + {"Negotiated_4KB", 4 * 1024, 4 * 1024}, + {"Negotiated_16KB", 16 * 1024, 16 * 1024}, + {"Negotiated_64KB", 64 * 1024, 64 * 1024}, + } { + t.Run(tc.name, func(t *testing.T) { + tcConn := testConn{} + c, err := NewConnWithMaxFrameSize(&tcConn, core.ClientSide, rekeyRecordProtocol, key, nil, tc.negotiatedMaxFrameSize) + if err != nil { + t.Fatalf("NewConn() err = %v, want nil", err) + } + altsC := c.(*conn) + wantPayloadLimit := tc.wantMaxRecordLen - altsC.overhead + if altsC.payloadLengthLimit != wantPayloadLimit { + t.Errorf("payloadLengthLimit = %v, want %v", altsC.payloadLengthLimit, wantPayloadLimit) + } + }) + } +} + +// TestWriteHonorsNegotiatedMaxFrameSize verifies that Write uses the negotiated +// max_frame_size so we never send a frame larger than negotiated. +func (s) TestWriteHonorsNegotiatedMaxFrameSize(t *testing.T) { + key := make([]byte, 16) + const negotiatedMaxFrameSize = 16 * 1024 + out := new(bytes.Buffer) + c, err := NewConnWithMaxFrameSize(&testConn{in: new(bytes.Buffer), out: out}, core.ClientSide, rekeyRecordProtocol, key, nil, negotiatedMaxFrameSize) + if err != nil { + t.Fatalf("NewConn failed: %v", err) + } + altsC := c.(*conn) + if got, want := altsC.payloadLengthLimit+altsC.overhead, negotiatedMaxFrameSize; got != want { + t.Fatalf("frame size = %v, want %v", got, want) + } + // Two negotiatedMaxFrameSize-worth of plaintext should split into exactly two frames. + payload := make([]byte, 2*altsC.payloadLengthLimit) + if _, err := c.Write(payload); err != nil { + t.Fatalf("Write failed: %v", err) + } + if got, want := countALTSFrames(out.Bytes()), 2; got != want { + t.Errorf("Write produced %d frames, want %d", got, want) + } +} + +type partialWriteTestConn struct { + net.Conn + maxWriteBytes int + err error + out bytes.Buffer +} + +func (c *partialWriteTestConn) Write(b []byte) (int, error) { + if len(b) > c.maxWriteBytes { + c.out.Write(b[:c.maxWriteBytes]) + return c.maxWriteBytes, c.err + } + c.out.Write(b) + return len(b), nil +} + +// TestWritePartialWriteErrorWithNegotiatedMaxFrameSize verifies that when a partial Write error occurs, +// the number of written bytes returned correctly accounts for the negotiated max_frame_size. +func (s) TestWritePartialWriteErrorWithNegotiatedMaxFrameSize(t *testing.T) { + key := make([]byte, 16) + const negotiatedMaxFrameSize = 16 * 1024 + + // Allow writing 1.5 frames (16384 + 8192 = 24576 bytes) before returning io.ErrUnexpectedEOF. + writeLimit := negotiatedMaxFrameSize + negotiatedMaxFrameSize/2 + errConn := &partialWriteTestConn{ + maxWriteBytes: writeLimit, + err: io.ErrUnexpectedEOF, + } + + c, err := NewConnWithMaxFrameSize(errConn, core.ClientSide, rekeyRecordProtocol, key, nil, negotiatedMaxFrameSize) + if err != nil { + t.Fatalf("NewConn failed: %v", err) + } + altsC := c.(*conn) + + // Send 3 frames worth of payload + payload := make([]byte, 3*altsC.payloadLengthLimit) + n, writeErr := c.Write(payload) + if writeErr != io.ErrUnexpectedEOF { + t.Fatalf("Write err = %v, want %v", writeErr, io.ErrUnexpectedEOF) + } + + // Exactly 1 complete frame was written before the partial write error. + wantWrittenBytes := altsC.payloadLengthLimit + if n != wantWrittenBytes { + t.Errorf("Write() returned n = %d, want %d", n, wantWrittenBytes) + } +} diff --git a/credentials/alts/internal/conn/utils.go b/credentials/alts/internal/conn/utils.go new file mode 100644 index 0000000..84821fa --- /dev/null +++ b/credentials/alts/internal/conn/utils.go @@ -0,0 +1,63 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package conn + +import core "google.golang.org/grpc/credentials/alts/internal" + +// NewOutCounter returns an outgoing counter initialized to the starting sequence +// number for the client/server side of a connection. +func NewOutCounter(s core.Side, overflowLen int) (c Counter) { + c.overflowLen = overflowLen + if s == core.ServerSide { + // Server counters in ALTS record have the little-endian high bit + // set. + c.value[counterLen-1] = 0x80 + } + return +} + +// NewInCounter returns an incoming counter initialized to the starting sequence +// number for the client/server side of a connection. This is used in ALTS record +// to check that incoming counters are as expected, since ALTS record guarantees +// that messages are unwrapped in the same order that the peer wrapped them. +func NewInCounter(s core.Side, overflowLen int) (c Counter) { + c.overflowLen = overflowLen + if s == core.ClientSide { + // Server counters in ALTS record have the little-endian high bit + // set. + c.value[counterLen-1] = 0x80 + } + return +} + +// CounterFromValue creates a new counter given an initial value. +func CounterFromValue(value []byte, overflowLen int) (c Counter) { + c.overflowLen = overflowLen + copy(c.value[:], value) + return +} + +// CounterSide returns the connection side (client/server) a sequence counter is +// associated with. +func CounterSide(c []byte) core.Side { + if c[counterLen-1]&0x80 == 0x80 { + return core.ServerSide + } + return core.ClientSide +} diff --git a/credentials/alts/internal/handshaker/handshaker.go b/credentials/alts/internal/handshaker/handshaker.go new file mode 100644 index 0000000..f2a1e36 --- /dev/null +++ b/credentials/alts/internal/handshaker/handshaker.go @@ -0,0 +1,381 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package handshaker provides ALTS handshaking functionality for GCP. +package handshaker + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "time" + + "golang.org/x/sync/semaphore" + grpc "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + core "google.golang.org/grpc/credentials/alts/internal" + "google.golang.org/grpc/credentials/alts/internal/authinfo" + "google.golang.org/grpc/credentials/alts/internal/conn" + altsgrpc "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" + altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" + "google.golang.org/grpc/internal/envconfig" +) + +const ( + // The maximum byte size of receive frames. + frameLimit = 64 * 1024 // 64 KB + rekeyRecordProtocolName = "ALTSRP_GCM_AES128_REKEY" +) + +var ( + hsProtocol = altspb.HandshakeProtocol_ALTS + appProtocols = []string{"grpc"} + recordProtocols = []string{rekeyRecordProtocolName} + keyLength = map[string]int{ + rekeyRecordProtocolName: 44, + } + altsRecordFuncs = map[string]conn.ALTSRecordFunc{ + // ALTS handshaker protocols. + rekeyRecordProtocolName: func(s core.Side, keyData []byte) (conn.ALTSRecordCrypto, error) { + return conn.NewAES128GCMRekey(s, keyData) + }, + } + // control number of concurrent created (but not closed) handshakes. + clientHandshakes = semaphore.NewWeighted(int64(envconfig.ALTSMaxConcurrentHandshakes)) + serverHandshakes = semaphore.NewWeighted(int64(envconfig.ALTSMaxConcurrentHandshakes)) + // errOutOfBound occurs when the handshake service returns a consumed + // bytes value larger than the buffer that was passed to it originally. + errOutOfBound = errors.New("handshaker service consumed bytes value is out-of-bound") +) + +func init() { + for protocol, f := range altsRecordFuncs { + if err := conn.RegisterProtocol(protocol, f); err != nil { + panic(err) + } + } +} + +// ClientHandshakerOptions contains the client handshaker options that can +// provided by the caller. +type ClientHandshakerOptions struct { + // ClientIdentity is the handshaker client local identity. + ClientIdentity *altspb.Identity + // TargetName is the server service account name for secure name + // checking. + TargetName string + // TargetServiceAccounts contains a list of expected target service + // accounts. One of these accounts should match one of the accounts in + // the handshaker results. Otherwise, the handshake fails. + TargetServiceAccounts []string + // RPCVersions specifies the gRPC versions accepted by the client. + RPCVersions *altspb.RpcProtocolVersions + // BoundAccessToken is a bound access token to be sent to the server for authentication. + BoundAccessToken string +} + +// ServerHandshakerOptions contains the server handshaker options that can +// provided by the caller. +type ServerHandshakerOptions struct { + // RPCVersions specifies the gRPC versions accepted by the server. + RPCVersions *altspb.RpcProtocolVersions +} + +// DefaultClientHandshakerOptions returns the default client handshaker options. +func DefaultClientHandshakerOptions() *ClientHandshakerOptions { + return &ClientHandshakerOptions{} +} + +// DefaultServerHandshakerOptions returns the default client handshaker options. +func DefaultServerHandshakerOptions() *ServerHandshakerOptions { + return &ServerHandshakerOptions{} +} + +// altsHandshaker is used to complete an ALTS handshake between client and +// server. This handshaker talks to the ALTS handshaker service in the metadata +// server. +type altsHandshaker struct { + // RPC stream used to access the ALTS Handshaker service. + stream altsgrpc.HandshakerService_DoHandshakeClient + // the connection to the peer. + conn net.Conn + // a virtual connection to the ALTS handshaker service. + clientConn *grpc.ClientConn + // client handshake options. + clientOpts *ClientHandshakerOptions + // server handshake options. + serverOpts *ServerHandshakerOptions + // defines the side doing the handshake, client or server. + side core.Side +} + +// NewClientHandshaker creates a core.Handshaker that performs a client-side +// ALTS handshake by acting as a proxy between the peer and the ALTS handshaker +// service in the metadata server. +func NewClientHandshaker(_ context.Context, conn *grpc.ClientConn, c net.Conn, opts *ClientHandshakerOptions) (core.Handshaker, error) { + return &altsHandshaker{ + stream: nil, + conn: c, + clientConn: conn, + clientOpts: opts, + side: core.ClientSide, + }, nil +} + +// NewServerHandshaker creates a core.Handshaker that performs a server-side +// ALTS handshake by acting as a proxy between the peer and the ALTS handshaker +// service in the metadata server. +func NewServerHandshaker(_ context.Context, conn *grpc.ClientConn, c net.Conn, opts *ServerHandshakerOptions) (core.Handshaker, error) { + return &altsHandshaker{ + stream: nil, + conn: c, + clientConn: conn, + serverOpts: opts, + side: core.ServerSide, + }, nil +} + +// ClientHandshake starts and completes a client ALTS handshake for GCP. Once +// done, ClientHandshake returns a secure connection. +func (h *altsHandshaker) ClientHandshake(ctx context.Context) (net.Conn, credentials.AuthInfo, error) { + if err := clientHandshakes.Acquire(ctx, 1); err != nil { + return nil, nil, err + } + defer clientHandshakes.Release(1) + + if h.side != core.ClientSide { + return nil, nil, errors.New("only handshakers created using NewClientHandshaker can perform a client handshaker") + } + + // TODO(matthewstevenson88): Change unit tests to use public APIs so + // that h.stream can unconditionally be set based on h.clientConn. + if h.stream == nil { + stream, err := altsgrpc.NewHandshakerServiceClient(h.clientConn).DoHandshake(ctx) + if err != nil { + return nil, nil, fmt.Errorf("failed to establish stream to ALTS handshaker service: %v", err) + } + h.stream = stream + } + + // Create target identities from service account list. + targetIdentities := make([]*altspb.Identity, 0, len(h.clientOpts.TargetServiceAccounts)) + for _, account := range h.clientOpts.TargetServiceAccounts { + targetIdentities = append(targetIdentities, &altspb.Identity{ + IdentityOneof: &altspb.Identity_ServiceAccount{ + ServiceAccount: account, + }, + }) + } + req := &altspb.HandshakerReq{ + ReqOneof: &altspb.HandshakerReq_ClientStart{ + ClientStart: &altspb.StartClientHandshakeReq{ + HandshakeSecurityProtocol: hsProtocol, + ApplicationProtocols: appProtocols, + RecordProtocols: recordProtocols, + TargetIdentities: targetIdentities, + LocalIdentity: h.clientOpts.ClientIdentity, + TargetName: h.clientOpts.TargetName, + RpcVersions: h.clientOpts.RPCVersions, + }, + }, + } + if h.clientOpts.BoundAccessToken != "" { + req.GetClientStart().AccessToken = h.clientOpts.BoundAccessToken + } + conn, result, err := h.doHandshake(req) + if err != nil { + return nil, nil, err + } + authInfo := authinfo.New(result) + return conn, authInfo, nil +} + +// ServerHandshake starts and completes a server ALTS handshake for GCP. Once +// done, ServerHandshake returns a secure connection. +func (h *altsHandshaker) ServerHandshake(ctx context.Context) (net.Conn, credentials.AuthInfo, error) { + if err := serverHandshakes.Acquire(ctx, 1); err != nil { + return nil, nil, err + } + defer serverHandshakes.Release(1) + + if h.side != core.ServerSide { + return nil, nil, errors.New("only handshakers created using NewServerHandshaker can perform a server handshaker") + } + + // TODO(matthewstevenson88): Change unit tests to use public APIs so + // that h.stream can unconditionally be set based on h.clientConn. + if h.stream == nil { + stream, err := altsgrpc.NewHandshakerServiceClient(h.clientConn).DoHandshake(ctx) + if err != nil { + return nil, nil, fmt.Errorf("failed to establish stream to ALTS handshaker service: %v", err) + } + h.stream = stream + } + + p := make([]byte, frameLimit) + n, err := h.conn.Read(p) + if err != nil { + return nil, nil, err + } + + // Prepare server parameters. + params := make(map[int32]*altspb.ServerHandshakeParameters) + params[int32(altspb.HandshakeProtocol_ALTS)] = &altspb.ServerHandshakeParameters{ + RecordProtocols: recordProtocols, + } + req := &altspb.HandshakerReq{ + ReqOneof: &altspb.HandshakerReq_ServerStart{ + ServerStart: &altspb.StartServerHandshakeReq{ + ApplicationProtocols: appProtocols, + HandshakeParameters: params, + InBytes: p[:n], + RpcVersions: h.serverOpts.RPCVersions, + }, + }, + } + + conn, result, err := h.doHandshake(req) + if err != nil { + return nil, nil, err + } + authInfo := authinfo.New(result) + return conn, authInfo, nil +} + +func (h *altsHandshaker) doHandshake(req *altspb.HandshakerReq) (net.Conn, *altspb.HandshakerResult, error) { + resp, err := h.accessHandshakerService(req) + if err != nil { + return nil, nil, err + } + // Check of the returned status is an error. + if resp.GetStatus() != nil { + if got, want := resp.GetStatus().Code, uint32(codes.OK); got != want { + return nil, nil, fmt.Errorf("%v", resp.GetStatus().Details) + } + } + + var extra []byte + if req.GetServerStart() != nil { + if resp.GetBytesConsumed() > uint32(len(req.GetServerStart().GetInBytes())) { + return nil, nil, errOutOfBound + } + extra = req.GetServerStart().GetInBytes()[resp.GetBytesConsumed():] + } + result, extra, err := h.processUntilDone(resp, extra) + if err != nil { + return nil, nil, err + } + // The handshaker returns a 128 bytes key. It should be truncated based + // on the returned record protocol. + keyLen, ok := keyLength[result.RecordProtocol] + if !ok { + return nil, nil, fmt.Errorf("unknown resulted record protocol %v", result.RecordProtocol) + } + maxFrameSize := int(envconfig.ALTSMaxFrameSize) + if peerMax := int(result.GetMaxFrameSize()); peerMax > 0 { + maxFrameSize = min(peerMax, maxFrameSize) + } + sc, err := conn.NewConnWithMaxFrameSize(h.conn, h.side, result.GetRecordProtocol(), result.KeyData[:keyLen], extra, maxFrameSize) + if err != nil { + return nil, nil, err + } + return sc, result, nil +} + +func (h *altsHandshaker) accessHandshakerService(req *altspb.HandshakerReq) (*altspb.HandshakerResp, error) { + if err := h.stream.Send(req); err != nil { + return nil, fmt.Errorf("failed to send ALTS handshaker request: %w", err) + } + resp, err := h.stream.Recv() + if err != nil { + return nil, fmt.Errorf("failed to receive ALTS handshaker response: %w", err) + } + return resp, nil +} + +// processUntilDone processes the handshake until the handshaker service returns +// the results. Handshaker service takes care of frame parsing, so we read +// whatever received from the network and send it to the handshaker service. +func (h *altsHandshaker) processUntilDone(resp *altspb.HandshakerResp, extra []byte) (*altspb.HandshakerResult, []byte, error) { + var lastWriteTime time.Time + buf := make([]byte, frameLimit) + for { + if len(resp.OutFrames) > 0 { + lastWriteTime = time.Now() + if _, err := h.conn.Write(resp.OutFrames); err != nil { + return nil, nil, err + } + } + if resp.Result != nil { + return resp.Result, extra, nil + } + n, err := h.conn.Read(buf) + if err != nil && err != io.EOF { + return nil, nil, err + } + // If there is nothing to send to the handshaker service, and + // nothing is received from the peer, then we are stuck. + // This covers the case when the peer is not responding. Note + // that handshaker service connection issues are caught in + // accessHandshakerService before we even get here. + if len(resp.OutFrames) == 0 && n == 0 { + return nil, nil, core.PeerNotRespondingError + } + // Append extra bytes from the previous interaction with the + // handshaker service with the current buffer read from conn. + p := append(extra, buf[:n]...) + // Compute the time elapsed since the last write to the peer. + timeElapsed := time.Since(lastWriteTime) + timeElapsedMs := uint32(timeElapsed.Milliseconds()) + // From here on, p and extra point to the same slice. + resp, err = h.accessHandshakerService(&altspb.HandshakerReq{ + ReqOneof: &altspb.HandshakerReq_Next{ + Next: &altspb.NextHandshakeMessageReq{ + InBytes: p, + NetworkLatencyMs: timeElapsedMs, + }, + }, + }) + if err != nil { + return nil, nil, err + } + // Set extra based on handshaker service response. + if resp.GetBytesConsumed() > uint32(len(p)) { + return nil, nil, errOutOfBound + } + extra = p[resp.GetBytesConsumed():] + } +} + +// Close terminates the Handshaker. It should be called when the caller obtains +// the secure connection. +func (h *altsHandshaker) Close() { + if h.stream != nil { + h.stream.CloseSend() + } +} + +// ResetConcurrentHandshakeSemaphoreForTesting resets the handshake semaphores +// to allow numberOfAllowedHandshakes concurrent handshakes each. +func ResetConcurrentHandshakeSemaphoreForTesting(numberOfAllowedHandshakes int64) { + clientHandshakes = semaphore.NewWeighted(numberOfAllowedHandshakes) + serverHandshakes = semaphore.NewWeighted(numberOfAllowedHandshakes) +} diff --git a/credentials/alts/internal/handshaker/handshaker_test.go b/credentials/alts/internal/handshaker/handshaker_test.go new file mode 100644 index 0000000..d55532c --- /dev/null +++ b/credentials/alts/internal/handshaker/handshaker_test.go @@ -0,0 +1,437 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package handshaker + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + grpc "google.golang.org/grpc" + core "google.golang.org/grpc/credentials/alts/internal" + altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" + "google.golang.org/grpc/credentials/alts/internal/testutil" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +var ( + testRecordProtocol = rekeyRecordProtocolName + testKey = []byte{ + // 44 arbitrary bytes. + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xd2, 0x4c, 0xce, 0x4f, 0x49, + 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xd2, 0x4c, 0xce, 0x4f, 0x49, 0x1f, 0x8b, + 0xd2, 0x4c, 0xce, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, + } + testServiceAccount = "test_service_account" + testTargetServiceAccounts = []string{testServiceAccount} + testClientIdentity = &altspb.Identity{ + IdentityOneof: &altspb.Identity_Hostname{ + Hostname: "i_am_a_client", + }, + } +) + +const defaultTestTimeout = 10 * time.Second + +// testRPCStream mimics a altspb.HandshakerService_DoHandshakeClient object. +type testRPCStream struct { + grpc.ClientStream + t *testing.T + isClient bool + // The resp expected to be returned by Recv(). Make sure this is set to + // the content the test requires before Recv() is invoked. + recvBuf *altspb.HandshakerResp + // false if it is the first access to Handshaker service on Envelope. + first bool + // useful for testing concurrent calls. + delay time.Duration + // The minimum expected value of the network_latency_ms field in a + // NextHandshakeMessageReq. + minExpectedNetworkLatency time.Duration + // peerMaxFrameSize is the max_frame_size in the HandshakerResult. + peerMaxFrameSize uint32 +} + +func (t *testRPCStream) Recv() (*altspb.HandshakerResp, error) { + resp := t.recvBuf + t.recvBuf = nil + return resp, nil +} + +func (t *testRPCStream) Send(req *altspb.HandshakerReq) error { + var resp *altspb.HandshakerResp + if !t.first { + // Generate the bytes to be returned by Recv() for the initial + // handshaking. + t.first = true + if t.isClient { + resp = &altspb.HandshakerResp{ + OutFrames: testutil.MakeFrame("ClientInit"), + // Simulate consuming ServerInit. + BytesConsumed: 14, + } + } else { + resp = &altspb.HandshakerResp{ + OutFrames: testutil.MakeFrame("ServerInit"), + // Simulate consuming ClientInit. + BytesConsumed: 14, + } + } + } else { + switch req := req.ReqOneof.(type) { + case *altspb.HandshakerReq_Next: + // Compare the network_latency_ms field to the minimum expected network + // latency. + if nl := time.Duration(req.Next.NetworkLatencyMs) * time.Millisecond; nl < t.minExpectedNetworkLatency { + return fmt.Errorf("networkLatency (%v) is smaller than expected min network latency (%v)", nl, t.minExpectedNetworkLatency) + } + default: + return fmt.Errorf("handshake request has unexpected type: %v", req) + } + + // Add delay to test concurrent calls. + cleanup := stat.Update() + defer cleanup() + time.Sleep(t.delay) + + // Generate the response to be returned by Recv() for the + // follow-up handshaking. + result := &altspb.HandshakerResult{ + RecordProtocol: testRecordProtocol, + KeyData: testKey, + MaxFrameSize: t.peerMaxFrameSize, + } + resp = &altspb.HandshakerResp{ + Result: result, + // Simulate consuming ClientFinished or ServerFinished. + BytesConsumed: 18, + } + } + t.recvBuf = resp + return nil +} + +func (t *testRPCStream) CloseSend() error { + return nil +} + +var stat testutil.Stats + +func (s) TestClientHandshake(t *testing.T) { + for _, testCase := range []struct { + delay time.Duration + numberOfHandshakes int + readLatency time.Duration + }{ + {0 * time.Millisecond, 1, time.Duration(0)}, + {0 * time.Millisecond, 1, 2 * time.Millisecond}, + {100 * time.Millisecond, 10 * int(envconfig.ALTSMaxConcurrentHandshakes), time.Duration(0)}, + } { + errc := make(chan error) + stat.Reset() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + for i := 0; i < testCase.numberOfHandshakes; i++ { + stream := &testRPCStream{ + t: t, + isClient: true, + minExpectedNetworkLatency: testCase.readLatency, + } + // Preload the inbound frames. + f1 := testutil.MakeFrame("ServerInit") + f2 := testutil.MakeFrame("ServerFinished") + in := bytes.NewBuffer(f1) + in.Write(f2) + out := new(bytes.Buffer) + tc := testutil.NewTestConnWithReadLatency(in, out, testCase.readLatency) + chs := &altsHandshaker{ + stream: stream, + conn: tc, + clientOpts: &ClientHandshakerOptions{ + TargetServiceAccounts: testTargetServiceAccounts, + ClientIdentity: testClientIdentity, + }, + side: core.ClientSide, + } + go func() { + _, context, err := chs.ClientHandshake(ctx) + if err == nil && context == nil { + errc <- errors.New("expected non-nil ALTS context") + return + } + errc <- err + chs.Close() + }() + } + + // Ensure that there are no errors. + for i := 0; i < testCase.numberOfHandshakes; i++ { + if err := <-errc; err != nil { + t.Errorf("ClientHandshake() = _, %v, want _, ", err) + } + } + + // Ensure that there are no concurrent calls more than the limit. + if stat.MaxConcurrentCalls > int(envconfig.ALTSMaxConcurrentHandshakes) { + t.Errorf("Observed %d concurrent handshakes; want <= %d", stat.MaxConcurrentCalls, envconfig.ALTSMaxConcurrentHandshakes) + } + } +} + +func (s) TestServerHandshake(t *testing.T) { + for _, testCase := range []struct { + delay time.Duration + numberOfHandshakes int + }{ + {0 * time.Millisecond, 1}, + {100 * time.Millisecond, 10 * int(envconfig.ALTSMaxConcurrentHandshakes)}, + } { + errc := make(chan error) + stat.Reset() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + for i := 0; i < testCase.numberOfHandshakes; i++ { + stream := &testRPCStream{ + t: t, + isClient: false, + } + // Preload the inbound frames. + f1 := testutil.MakeFrame("ClientInit") + f2 := testutil.MakeFrame("ClientFinished") + in := bytes.NewBuffer(f1) + in.Write(f2) + out := new(bytes.Buffer) + tc := testutil.NewTestConn(in, out) + shs := &altsHandshaker{ + stream: stream, + conn: tc, + serverOpts: DefaultServerHandshakerOptions(), + side: core.ServerSide, + } + go func() { + _, context, err := shs.ServerHandshake(ctx) + if err == nil && context == nil { + errc <- errors.New("expected non-nil ALTS context") + return + } + errc <- err + shs.Close() + }() + } + + // Ensure that there are no errors. + for i := 0; i < testCase.numberOfHandshakes; i++ { + if err := <-errc; err != nil { + t.Errorf("ServerHandshake() = _, %v, want _, ", err) + } + } + + // Ensure that there are no concurrent calls more than the limit. + if stat.MaxConcurrentCalls > int(envconfig.ALTSMaxConcurrentHandshakes) { + t.Errorf("Observed %d concurrent handshakes; want <= %d", stat.MaxConcurrentCalls, envconfig.ALTSMaxConcurrentHandshakes) + } + } +} + +// testUnresponsiveRPCStream is used for testing the PeerNotResponding case. +type testUnresponsiveRPCStream struct { + grpc.ClientStream +} + +func (t *testUnresponsiveRPCStream) Recv() (*altspb.HandshakerResp, error) { + return &altspb.HandshakerResp{}, nil +} + +func (t *testUnresponsiveRPCStream) Send(*altspb.HandshakerReq) error { + return nil +} + +func (t *testUnresponsiveRPCStream) CloseSend() error { + return nil +} + +func (s) TestPeerNotResponding(t *testing.T) { + stream := &testUnresponsiveRPCStream{} + chs := &altsHandshaker{ + stream: stream, + conn: testutil.NewUnresponsiveTestConn(), + clientOpts: &ClientHandshakerOptions{ + TargetServiceAccounts: testTargetServiceAccounts, + ClientIdentity: testClientIdentity, + }, + side: core.ClientSide, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, context, err := chs.ClientHandshake(ctx) + chs.Close() + if context != nil { + t.Error("expected non-nil ALTS context") + } + if got, want := err, core.PeerNotRespondingError; got != want { + t.Errorf("ClientHandshake() = %v, want %v", got, want) + } +} + +func (s) TestNewClientHandshaker(t *testing.T) { + conn := testutil.NewTestConn(nil, nil) + clientConn := &grpc.ClientConn{} + opts := &ClientHandshakerOptions{} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + hs, err := NewClientHandshaker(ctx, clientConn, conn, opts) + if err != nil { + t.Errorf("NewClientHandshaker returned unexpected error: %v", err) + } + expectedHs := &altsHandshaker{ + stream: nil, + conn: conn, + clientConn: clientConn, + clientOpts: opts, + serverOpts: nil, + side: core.ClientSide, + } + cmpOpts := []cmp.Option{ + cmp.AllowUnexported(altsHandshaker{}), + cmpopts.IgnoreFields(altsHandshaker{}, "conn", "clientConn"), + } + if got, want := hs.(*altsHandshaker), expectedHs; !cmp.Equal(got, want, cmpOpts...) { + t.Errorf("NewClientHandshaker() returned unexpected handshaker: got: %v, want: %v", got, want) + } + if hs.(*altsHandshaker).stream != nil { + t.Errorf("NewClientHandshaker() returned handshaker with non-nil stream") + } + if hs.(*altsHandshaker).clientConn != clientConn { + t.Errorf("NewClientHandshaker() returned handshaker with unexpected clientConn") + } + hs.Close() +} + +func (s) TestNewServerHandshaker(t *testing.T) { + conn := testutil.NewTestConn(nil, nil) + clientConn := &grpc.ClientConn{} + opts := &ServerHandshakerOptions{} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + hs, err := NewServerHandshaker(ctx, clientConn, conn, opts) + if err != nil { + t.Errorf("NewServerHandshaker returned unexpected error: %v", err) + } + expectedHs := &altsHandshaker{ + stream: nil, + conn: conn, + clientConn: clientConn, + clientOpts: nil, + serverOpts: opts, + side: core.ServerSide, + } + cmpOpts := []cmp.Option{ + cmp.AllowUnexported(altsHandshaker{}), + cmpopts.IgnoreFields(altsHandshaker{}, "conn", "clientConn"), + } + if got, want := hs.(*altsHandshaker), expectedHs; !cmp.Equal(got, want, cmpOpts...) { + t.Errorf("NewServerHandshaker() returned unexpected handshaker: got: %v, want: %v", got, want) + } + if hs.(*altsHandshaker).stream != nil { + t.Errorf("NewServerHandshaker() returned handshaker with non-nil stream") + } + if hs.(*altsHandshaker).clientConn != clientConn { + t.Errorf("NewServerHandshaker() returned handshaker with unexpected clientConn") + } + hs.Close() +} + +func (s) TestHandshakeMaxFrameSizeNegotiation(t *testing.T) { + old := envconfig.ALTSMaxFrameSize + envconfig.ALTSMaxFrameSize = 64 * 1024 + defer func() { envconfig.ALTSMaxFrameSize = old }() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + stream := &testRPCStream{ + t: t, + isClient: true, + peerMaxFrameSize: 1024 * 1024, + } + f1 := testutil.MakeFrame("ServerInit") + f2 := testutil.MakeFrame("ServerFinished") + in := bytes.NewBuffer(f1) + in.Write(f2) + out := new(bytes.Buffer) + tc := testutil.NewTestConn(in, out) + chs := &altsHandshaker{ + stream: stream, + conn: tc, + clientOpts: &ClientHandshakerOptions{ + TargetServiceAccounts: testTargetServiceAccounts, + ClientIdentity: testClientIdentity, + }, + side: core.ClientSide, + } + defer chs.Close() + + sc, _, err := chs.ClientHandshake(ctx) + if err != nil { + t.Fatalf("ClientHandshake() failed: %v", err) + } + if sc == nil { + t.Fatal("ClientHandshake() returned nil conn, want non-nil") + } + + // Clear any handshake frames written to out during ClientHandshake. + out.Reset() + + // Write a payload large enough to produce a full ALTS frame. + payload := make([]byte, 128*1024) + if _, err := sc.Write(payload); err != nil { + t.Fatalf("Secure conn Write() failed: %v", err) + } + // In ALTS wire format, the first 4 bytes of a frame contain the little-endian + // frame length (frameLen). Writing a large payload splits it into frames of + // the maximum calculated size. We verify the connection's max frame size by + // asserting that the total wire frame length (4 + frameLen) equals 64 * 1024. + if len(out.Bytes()) < 4 { + t.Fatalf("Output buffer too small: %d bytes", len(out.Bytes())) + } + frameLen := int(binary.LittleEndian.Uint32(out.Bytes()[:4])) + if got, want := 4+frameLen, 64*1024; got != want { + t.Errorf("Wire frame size = %v, want %v", got, want) + } +} diff --git a/credentials/alts/internal/handshaker/service/service.go b/credentials/alts/internal/handshaker/service/service.go new file mode 100644 index 0000000..2580995 --- /dev/null +++ b/credentials/alts/internal/handshaker/service/service.go @@ -0,0 +1,91 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package service manages connections between the VM application and the ALTS +// handshaker service. +package service + +import ( + "sync" + "time" + + grpc "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/keepalive" +) + +var ( + // mu guards hsConnMap and hsDialer. + mu sync.Mutex + // hsConn represents a mapping from a hypervisor handshaker service address + // to a corresponding connection to a hypervisor handshaker service + // instance. + hsConnMap = make(map[string]*grpc.ClientConn) +) + +// Dial dials the handshake service in the hypervisor. If a connection has +// already been established, this function returns it. Otherwise, a new +// connection is created. +func Dial(hsAddress string) (*grpc.ClientConn, error) { + mu.Lock() + defer mu.Unlock() + + hsConn, ok := hsConnMap[hsAddress] + if !ok { + // Create a new connection to the handshaker service. Note that + // this connection stays open until the application is closed. + // Disable the service config to avoid unnecessary TXT record lookups that + // cause timeouts with some versions of systemd-resolved. + var err error + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDisableServiceConfig(), + } + if envconfig.ALTSHandshakerKeepaliveParams { + opts = append(opts, grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Timeout: 10 * time.Second, + Time: 10 * time.Minute, + })) + } + hsConn, err = grpc.NewClient(hsAddress, opts...) + if err != nil { + return nil, err + } + hsConnMap[hsAddress] = hsConn + } + return hsConn, nil +} + +// CloseForTesting closes all open connections to the handshaker service. +// +// For testing purposes only. +func CloseForTesting() error { + for _, hsConn := range hsConnMap { + if hsConn == nil { + continue + } + if err := hsConn.Close(); err != nil { + return err + } + } + + // Reset the connection map. + hsConnMap = make(map[string]*grpc.ClientConn) + return nil +} diff --git a/credentials/alts/internal/handshaker/service/service_test.go b/credentials/alts/internal/handshaker/service/service_test.go new file mode 100644 index 0000000..74bfb7e --- /dev/null +++ b/credentials/alts/internal/handshaker/service/service_test.go @@ -0,0 +1,80 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package service + +import ( + "testing" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + testAddress1 = "some_address_1" + testAddress2 = "some_address_2" +) + +// TestDial verifies the behaviour of alts handshake when there are multiple Dials. +// If a connection has already been established, this function returns it. +// Otherwise, a new connection is created. +func (s) TestDial(t *testing.T) { + // First call to Dial, it should create a connection to the server running + // at the given address. + conn1, err := Dial(testAddress1) + if err != nil { + t.Fatalf("first call to Dial(%v) failed: %v", testAddress1, err) + } + defer conn1.Close() + if got, want := hsConnMap[testAddress1], conn1; got != want { + t.Fatalf("hsConnMap[%v]=%v, want %v", testAddress1, got, want) + } + + // Second call to Dial should return conn1 above. + conn2, err := Dial(testAddress1) + if err != nil { + t.Fatalf("second call to Dial(%v) failed: %v", testAddress1, err) + } + defer conn2.Close() + if got, want := conn2, conn1; got != want { + t.Fatalf("second call to Dial(%v)=(%v, _), want (%v,. _)", testAddress1, got, want) + } + if got, want := hsConnMap[testAddress1], conn1; got != want { + t.Fatalf("hsConnMap[%v]=%v, want %v", testAddress1, got, want) + } + + // Third call to Dial using a different address should create a new connection. + conn3, err := Dial(testAddress2) + if err != nil { + t.Fatalf("third call to Dial(%v) failed: %v", testAddress2, err) + } + defer conn3.Close() + if got, want := hsConnMap[testAddress2], conn3; got != want { + t.Fatalf("hsConnMap[%v]=%v, want %v", testAddress2, got, want) + } + if got, want := conn2 == conn3, false; got != want { + t.Fatalf("(conn2==conn3)=%v, want %v", got, want) + } +} diff --git a/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go b/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go new file mode 100644 index 0000000..bf31168 --- /dev/null +++ b/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go @@ -0,0 +1,211 @@ +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/gcp/altscontext.proto + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/gcp/altscontext.proto + +package grpc_gcp + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type AltsContext struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The application protocol negotiated for this connection. + ApplicationProtocol string `protobuf:"bytes,1,opt,name=application_protocol,json=applicationProtocol,proto3" json:"application_protocol,omitempty"` + // The record protocol negotiated for this connection. + RecordProtocol string `protobuf:"bytes,2,opt,name=record_protocol,json=recordProtocol,proto3" json:"record_protocol,omitempty"` + // The security level of the created secure channel. + SecurityLevel SecurityLevel `protobuf:"varint,3,opt,name=security_level,json=securityLevel,proto3,enum=grpc.gcp.SecurityLevel" json:"security_level,omitempty"` + // The peer service account. + PeerServiceAccount string `protobuf:"bytes,4,opt,name=peer_service_account,json=peerServiceAccount,proto3" json:"peer_service_account,omitempty"` + // The local service account. + LocalServiceAccount string `protobuf:"bytes,5,opt,name=local_service_account,json=localServiceAccount,proto3" json:"local_service_account,omitempty"` + // The RPC protocol versions supported by the peer. + PeerRpcVersions *RpcProtocolVersions `protobuf:"bytes,6,opt,name=peer_rpc_versions,json=peerRpcVersions,proto3" json:"peer_rpc_versions,omitempty"` + // Additional attributes of the peer. + PeerAttributes map[string]string `protobuf:"bytes,7,rep,name=peer_attributes,json=peerAttributes,proto3" json:"peer_attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AltsContext) Reset() { + *x = AltsContext{} + mi := &file_grpc_gcp_altscontext_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AltsContext) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AltsContext) ProtoMessage() {} + +func (x *AltsContext) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_altscontext_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AltsContext.ProtoReflect.Descriptor instead. +func (*AltsContext) Descriptor() ([]byte, []int) { + return file_grpc_gcp_altscontext_proto_rawDescGZIP(), []int{0} +} + +func (x *AltsContext) GetApplicationProtocol() string { + if x != nil { + return x.ApplicationProtocol + } + return "" +} + +func (x *AltsContext) GetRecordProtocol() string { + if x != nil { + return x.RecordProtocol + } + return "" +} + +func (x *AltsContext) GetSecurityLevel() SecurityLevel { + if x != nil { + return x.SecurityLevel + } + return SecurityLevel_SECURITY_NONE +} + +func (x *AltsContext) GetPeerServiceAccount() string { + if x != nil { + return x.PeerServiceAccount + } + return "" +} + +func (x *AltsContext) GetLocalServiceAccount() string { + if x != nil { + return x.LocalServiceAccount + } + return "" +} + +func (x *AltsContext) GetPeerRpcVersions() *RpcProtocolVersions { + if x != nil { + return x.PeerRpcVersions + } + return nil +} + +func (x *AltsContext) GetPeerAttributes() map[string]string { + if x != nil { + return x.PeerAttributes + } + return nil +} + +var File_grpc_gcp_altscontext_proto protoreflect.FileDescriptor + +const file_grpc_gcp_altscontext_proto_rawDesc = "" + + "\n" + + "\x1agrpc/gcp/altscontext.proto\x12\bgrpc.gcp\x1a(grpc/gcp/transport_security_common.proto\"\xf1\x03\n" + + "\vAltsContext\x121\n" + + "\x14application_protocol\x18\x01 \x01(\tR\x13applicationProtocol\x12'\n" + + "\x0frecord_protocol\x18\x02 \x01(\tR\x0erecordProtocol\x12>\n" + + "\x0esecurity_level\x18\x03 \x01(\x0e2\x17.grpc.gcp.SecurityLevelR\rsecurityLevel\x120\n" + + "\x14peer_service_account\x18\x04 \x01(\tR\x12peerServiceAccount\x122\n" + + "\x15local_service_account\x18\x05 \x01(\tR\x13localServiceAccount\x12I\n" + + "\x11peer_rpc_versions\x18\x06 \x01(\v2\x1d.grpc.gcp.RpcProtocolVersionsR\x0fpeerRpcVersions\x12R\n" + + "\x0fpeer_attributes\x18\a \x03(\v2).grpc.gcp.AltsContext.PeerAttributesEntryR\x0epeerAttributes\x1aA\n" + + "\x13PeerAttributesEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01Bl\n" + + "\x15io.grpc.alts.internalB\x10AltsContextProtoP\x01Z?google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcpb\x06proto3" + +var ( + file_grpc_gcp_altscontext_proto_rawDescOnce sync.Once + file_grpc_gcp_altscontext_proto_rawDescData []byte +) + +func file_grpc_gcp_altscontext_proto_rawDescGZIP() []byte { + file_grpc_gcp_altscontext_proto_rawDescOnce.Do(func() { + file_grpc_gcp_altscontext_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_gcp_altscontext_proto_rawDesc), len(file_grpc_gcp_altscontext_proto_rawDesc))) + }) + return file_grpc_gcp_altscontext_proto_rawDescData +} + +var file_grpc_gcp_altscontext_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_grpc_gcp_altscontext_proto_goTypes = []any{ + (*AltsContext)(nil), // 0: grpc.gcp.AltsContext + nil, // 1: grpc.gcp.AltsContext.PeerAttributesEntry + (SecurityLevel)(0), // 2: grpc.gcp.SecurityLevel + (*RpcProtocolVersions)(nil), // 3: grpc.gcp.RpcProtocolVersions +} +var file_grpc_gcp_altscontext_proto_depIdxs = []int32{ + 2, // 0: grpc.gcp.AltsContext.security_level:type_name -> grpc.gcp.SecurityLevel + 3, // 1: grpc.gcp.AltsContext.peer_rpc_versions:type_name -> grpc.gcp.RpcProtocolVersions + 1, // 2: grpc.gcp.AltsContext.peer_attributes:type_name -> grpc.gcp.AltsContext.PeerAttributesEntry + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_grpc_gcp_altscontext_proto_init() } +func file_grpc_gcp_altscontext_proto_init() { + if File_grpc_gcp_altscontext_proto != nil { + return + } + file_grpc_gcp_transport_security_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_gcp_altscontext_proto_rawDesc), len(file_grpc_gcp_altscontext_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_gcp_altscontext_proto_goTypes, + DependencyIndexes: file_grpc_gcp_altscontext_proto_depIdxs, + MessageInfos: file_grpc_gcp_altscontext_proto_msgTypes, + }.Build() + File_grpc_gcp_altscontext_proto = out.File + file_grpc_gcp_altscontext_proto_goTypes = nil + file_grpc_gcp_altscontext_proto_depIdxs = nil +} diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go new file mode 100644 index 0000000..0350eb3 --- /dev/null +++ b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go @@ -0,0 +1,1245 @@ +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/gcp/handshaker.proto + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/gcp/handshaker.proto + +package grpc_gcp + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type HandshakeProtocol int32 + +const ( + // Default value. + HandshakeProtocol_HANDSHAKE_PROTOCOL_UNSPECIFIED HandshakeProtocol = 0 + // TLS handshake protocol. + HandshakeProtocol_TLS HandshakeProtocol = 1 + // Application Layer Transport Security handshake protocol. + HandshakeProtocol_ALTS HandshakeProtocol = 2 +) + +// Enum value maps for HandshakeProtocol. +var ( + HandshakeProtocol_name = map[int32]string{ + 0: "HANDSHAKE_PROTOCOL_UNSPECIFIED", + 1: "TLS", + 2: "ALTS", + } + HandshakeProtocol_value = map[string]int32{ + "HANDSHAKE_PROTOCOL_UNSPECIFIED": 0, + "TLS": 1, + "ALTS": 2, + } +) + +func (x HandshakeProtocol) Enum() *HandshakeProtocol { + p := new(HandshakeProtocol) + *p = x + return p +} + +func (x HandshakeProtocol) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (HandshakeProtocol) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_gcp_handshaker_proto_enumTypes[0].Descriptor() +} + +func (HandshakeProtocol) Type() protoreflect.EnumType { + return &file_grpc_gcp_handshaker_proto_enumTypes[0] +} + +func (x HandshakeProtocol) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use HandshakeProtocol.Descriptor instead. +func (HandshakeProtocol) EnumDescriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{0} +} + +type NetworkProtocol int32 + +const ( + NetworkProtocol_NETWORK_PROTOCOL_UNSPECIFIED NetworkProtocol = 0 + NetworkProtocol_TCP NetworkProtocol = 1 + NetworkProtocol_UDP NetworkProtocol = 2 +) + +// Enum value maps for NetworkProtocol. +var ( + NetworkProtocol_name = map[int32]string{ + 0: "NETWORK_PROTOCOL_UNSPECIFIED", + 1: "TCP", + 2: "UDP", + } + NetworkProtocol_value = map[string]int32{ + "NETWORK_PROTOCOL_UNSPECIFIED": 0, + "TCP": 1, + "UDP": 2, + } +) + +func (x NetworkProtocol) Enum() *NetworkProtocol { + p := new(NetworkProtocol) + *p = x + return p +} + +func (x NetworkProtocol) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (NetworkProtocol) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_gcp_handshaker_proto_enumTypes[1].Descriptor() +} + +func (NetworkProtocol) Type() protoreflect.EnumType { + return &file_grpc_gcp_handshaker_proto_enumTypes[1] +} + +func (x NetworkProtocol) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use NetworkProtocol.Descriptor instead. +func (NetworkProtocol) EnumDescriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{1} +} + +type Endpoint struct { + state protoimpl.MessageState `protogen:"open.v1"` + // IP address. It should contain an IPv4 or IPv6 string literal, e.g. + // "192.168.0.1" or "2001:db8::1". + IpAddress string `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` + // Port number. + Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + // Network protocol (e.g., TCP, UDP) associated with this endpoint. + Protocol NetworkProtocol `protobuf:"varint,3,opt,name=protocol,proto3,enum=grpc.gcp.NetworkProtocol" json:"protocol,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Endpoint) Reset() { + *x = Endpoint{} + mi := &file_grpc_gcp_handshaker_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Endpoint) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Endpoint) ProtoMessage() {} + +func (x *Endpoint) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Endpoint.ProtoReflect.Descriptor instead. +func (*Endpoint) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{0} +} + +func (x *Endpoint) GetIpAddress() string { + if x != nil { + return x.IpAddress + } + return "" +} + +func (x *Endpoint) GetPort() int32 { + if x != nil { + return x.Port + } + return 0 +} + +func (x *Endpoint) GetProtocol() NetworkProtocol { + if x != nil { + return x.Protocol + } + return NetworkProtocol_NETWORK_PROTOCOL_UNSPECIFIED +} + +type Identity struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to IdentityOneof: + // + // *Identity_ServiceAccount + // *Identity_Hostname + IdentityOneof isIdentity_IdentityOneof `protobuf_oneof:"identity_oneof"` + // Additional attributes of the identity. + Attributes map[string]string `protobuf:"bytes,3,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Identity) Reset() { + *x = Identity{} + mi := &file_grpc_gcp_handshaker_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Identity) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Identity) ProtoMessage() {} + +func (x *Identity) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Identity.ProtoReflect.Descriptor instead. +func (*Identity) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{1} +} + +func (x *Identity) GetIdentityOneof() isIdentity_IdentityOneof { + if x != nil { + return x.IdentityOneof + } + return nil +} + +func (x *Identity) GetServiceAccount() string { + if x != nil { + if x, ok := x.IdentityOneof.(*Identity_ServiceAccount); ok { + return x.ServiceAccount + } + } + return "" +} + +func (x *Identity) GetHostname() string { + if x != nil { + if x, ok := x.IdentityOneof.(*Identity_Hostname); ok { + return x.Hostname + } + } + return "" +} + +func (x *Identity) GetAttributes() map[string]string { + if x != nil { + return x.Attributes + } + return nil +} + +type isIdentity_IdentityOneof interface { + isIdentity_IdentityOneof() +} + +type Identity_ServiceAccount struct { + // Service account of a connection endpoint. + ServiceAccount string `protobuf:"bytes,1,opt,name=service_account,json=serviceAccount,proto3,oneof"` +} + +type Identity_Hostname struct { + // Hostname of a connection endpoint. + Hostname string `protobuf:"bytes,2,opt,name=hostname,proto3,oneof"` +} + +func (*Identity_ServiceAccount) isIdentity_IdentityOneof() {} + +func (*Identity_Hostname) isIdentity_IdentityOneof() {} + +type StartClientHandshakeReq struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Handshake security protocol requested by the client. + HandshakeSecurityProtocol HandshakeProtocol `protobuf:"varint,1,opt,name=handshake_security_protocol,json=handshakeSecurityProtocol,proto3,enum=grpc.gcp.HandshakeProtocol" json:"handshake_security_protocol,omitempty"` + // The application protocols supported by the client, e.g., "h2" (for http2), + // "grpc". + ApplicationProtocols []string `protobuf:"bytes,2,rep,name=application_protocols,json=applicationProtocols,proto3" json:"application_protocols,omitempty"` + // The record protocols supported by the client, e.g., + // "ALTSRP_GCM_AES128". + RecordProtocols []string `protobuf:"bytes,3,rep,name=record_protocols,json=recordProtocols,proto3" json:"record_protocols,omitempty"` + // (Optional) Describes which server identities are acceptable by the client. + // If target identities are provided and none of them matches the peer + // identity of the server, handshake will fail. + TargetIdentities []*Identity `protobuf:"bytes,4,rep,name=target_identities,json=targetIdentities,proto3" json:"target_identities,omitempty"` + // (Optional) Application may specify a local identity. Otherwise, the + // handshaker chooses a default local identity. + LocalIdentity *Identity `protobuf:"bytes,5,opt,name=local_identity,json=localIdentity,proto3" json:"local_identity,omitempty"` + // (Optional) Local endpoint information of the connection to the server, + // such as local IP address, port number, and network protocol. + LocalEndpoint *Endpoint `protobuf:"bytes,6,opt,name=local_endpoint,json=localEndpoint,proto3" json:"local_endpoint,omitempty"` + // (Optional) Endpoint information of the remote server, such as IP address, + // port number, and network protocol. + RemoteEndpoint *Endpoint `protobuf:"bytes,7,opt,name=remote_endpoint,json=remoteEndpoint,proto3" json:"remote_endpoint,omitempty"` + // (Optional) If target name is provided, a secure naming check is performed + // to verify that the peer authenticated identity is indeed authorized to run + // the target name. + TargetName string `protobuf:"bytes,8,opt,name=target_name,json=targetName,proto3" json:"target_name,omitempty"` + // (Optional) RPC protocol versions supported by the client. + RpcVersions *RpcProtocolVersions `protobuf:"bytes,9,opt,name=rpc_versions,json=rpcVersions,proto3" json:"rpc_versions,omitempty"` + // (Optional) Maximum frame size supported by the client. + MaxFrameSize uint32 `protobuf:"varint,10,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"` + // (Optional) An access token created by the caller only intended for use in + // ALTS connections. The access token that should be used to authenticate to + // the peer. The access token MUST be strongly bound to the ALTS credentials + // used to establish the connection that the token is sent over. + AccessToken string `protobuf:"bytes,11,opt,name=access_token,json=accessToken,proto3" json:"access_token,omitempty"` + // (Optional) Ordered transport protocol preferences supported by the client. + TransportProtocolPreferences *TransportProtocolPreferences `protobuf:"bytes,12,opt,name=transport_protocol_preferences,json=transportProtocolPreferences,proto3" json:"transport_protocol_preferences,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StartClientHandshakeReq) Reset() { + *x = StartClientHandshakeReq{} + mi := &file_grpc_gcp_handshaker_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StartClientHandshakeReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartClientHandshakeReq) ProtoMessage() {} + +func (x *StartClientHandshakeReq) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartClientHandshakeReq.ProtoReflect.Descriptor instead. +func (*StartClientHandshakeReq) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{2} +} + +func (x *StartClientHandshakeReq) GetHandshakeSecurityProtocol() HandshakeProtocol { + if x != nil { + return x.HandshakeSecurityProtocol + } + return HandshakeProtocol_HANDSHAKE_PROTOCOL_UNSPECIFIED +} + +func (x *StartClientHandshakeReq) GetApplicationProtocols() []string { + if x != nil { + return x.ApplicationProtocols + } + return nil +} + +func (x *StartClientHandshakeReq) GetRecordProtocols() []string { + if x != nil { + return x.RecordProtocols + } + return nil +} + +func (x *StartClientHandshakeReq) GetTargetIdentities() []*Identity { + if x != nil { + return x.TargetIdentities + } + return nil +} + +func (x *StartClientHandshakeReq) GetLocalIdentity() *Identity { + if x != nil { + return x.LocalIdentity + } + return nil +} + +func (x *StartClientHandshakeReq) GetLocalEndpoint() *Endpoint { + if x != nil { + return x.LocalEndpoint + } + return nil +} + +func (x *StartClientHandshakeReq) GetRemoteEndpoint() *Endpoint { + if x != nil { + return x.RemoteEndpoint + } + return nil +} + +func (x *StartClientHandshakeReq) GetTargetName() string { + if x != nil { + return x.TargetName + } + return "" +} + +func (x *StartClientHandshakeReq) GetRpcVersions() *RpcProtocolVersions { + if x != nil { + return x.RpcVersions + } + return nil +} + +func (x *StartClientHandshakeReq) GetMaxFrameSize() uint32 { + if x != nil { + return x.MaxFrameSize + } + return 0 +} + +func (x *StartClientHandshakeReq) GetAccessToken() string { + if x != nil { + return x.AccessToken + } + return "" +} + +func (x *StartClientHandshakeReq) GetTransportProtocolPreferences() *TransportProtocolPreferences { + if x != nil { + return x.TransportProtocolPreferences + } + return nil +} + +type ServerHandshakeParameters struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The record protocols supported by the server, e.g., + // "ALTSRP_GCM_AES128". + RecordProtocols []string `protobuf:"bytes,1,rep,name=record_protocols,json=recordProtocols,proto3" json:"record_protocols,omitempty"` + // (Optional) A list of local identities supported by the server, if + // specified. Otherwise, the handshaker chooses a default local identity. + LocalIdentities []*Identity `protobuf:"bytes,2,rep,name=local_identities,json=localIdentities,proto3" json:"local_identities,omitempty"` + // A token created by the caller only intended for use in + // ALTS connections. The token should be used to authenticate to + // the peer. The token MUST be strongly bound to the ALTS credentials + // used to establish the connection that the token is sent over. + Token *string `protobuf:"bytes,3,opt,name=token,proto3,oneof" json:"token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerHandshakeParameters) Reset() { + *x = ServerHandshakeParameters{} + mi := &file_grpc_gcp_handshaker_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerHandshakeParameters) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerHandshakeParameters) ProtoMessage() {} + +func (x *ServerHandshakeParameters) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerHandshakeParameters.ProtoReflect.Descriptor instead. +func (*ServerHandshakeParameters) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{3} +} + +func (x *ServerHandshakeParameters) GetRecordProtocols() []string { + if x != nil { + return x.RecordProtocols + } + return nil +} + +func (x *ServerHandshakeParameters) GetLocalIdentities() []*Identity { + if x != nil { + return x.LocalIdentities + } + return nil +} + +func (x *ServerHandshakeParameters) GetToken() string { + if x != nil && x.Token != nil { + return *x.Token + } + return "" +} + +type StartServerHandshakeReq struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The application protocols supported by the server, e.g., "h2" (for http2), + // "grpc". + ApplicationProtocols []string `protobuf:"bytes,1,rep,name=application_protocols,json=applicationProtocols,proto3" json:"application_protocols,omitempty"` + // Handshake parameters (record protocols and local identities supported by + // the server) mapped by the handshake protocol. Each handshake security + // protocol (e.g., TLS or ALTS) has its own set of record protocols and local + // identities. Since protobuf does not support enum as key to the map, the key + // to handshake_parameters is the integer value of HandshakeProtocol enum. + HandshakeParameters map[int32]*ServerHandshakeParameters `protobuf:"bytes,2,rep,name=handshake_parameters,json=handshakeParameters,proto3" json:"handshake_parameters,omitempty" protobuf_key:"varint,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // Bytes in out_frames returned from the peer's HandshakerResp. It is possible + // that the peer's out_frames are split into multiple HandshakeReq messages. + InBytes []byte `protobuf:"bytes,3,opt,name=in_bytes,json=inBytes,proto3" json:"in_bytes,omitempty"` + // (Optional) Local endpoint information of the connection to the client, + // such as local IP address, port number, and network protocol. + LocalEndpoint *Endpoint `protobuf:"bytes,4,opt,name=local_endpoint,json=localEndpoint,proto3" json:"local_endpoint,omitempty"` + // (Optional) Endpoint information of the remote client, such as IP address, + // port number, and network protocol. + RemoteEndpoint *Endpoint `protobuf:"bytes,5,opt,name=remote_endpoint,json=remoteEndpoint,proto3" json:"remote_endpoint,omitempty"` + // (Optional) RPC protocol versions supported by the server. + RpcVersions *RpcProtocolVersions `protobuf:"bytes,6,opt,name=rpc_versions,json=rpcVersions,proto3" json:"rpc_versions,omitempty"` + // (Optional) Maximum frame size supported by the server. + MaxFrameSize uint32 `protobuf:"varint,7,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"` + // (Optional) Transport protocol preferences supported by the server. + TransportProtocolPreferences *TransportProtocolPreferences `protobuf:"bytes,8,opt,name=transport_protocol_preferences,json=transportProtocolPreferences,proto3" json:"transport_protocol_preferences,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StartServerHandshakeReq) Reset() { + *x = StartServerHandshakeReq{} + mi := &file_grpc_gcp_handshaker_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StartServerHandshakeReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartServerHandshakeReq) ProtoMessage() {} + +func (x *StartServerHandshakeReq) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartServerHandshakeReq.ProtoReflect.Descriptor instead. +func (*StartServerHandshakeReq) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{4} +} + +func (x *StartServerHandshakeReq) GetApplicationProtocols() []string { + if x != nil { + return x.ApplicationProtocols + } + return nil +} + +func (x *StartServerHandshakeReq) GetHandshakeParameters() map[int32]*ServerHandshakeParameters { + if x != nil { + return x.HandshakeParameters + } + return nil +} + +func (x *StartServerHandshakeReq) GetInBytes() []byte { + if x != nil { + return x.InBytes + } + return nil +} + +func (x *StartServerHandshakeReq) GetLocalEndpoint() *Endpoint { + if x != nil { + return x.LocalEndpoint + } + return nil +} + +func (x *StartServerHandshakeReq) GetRemoteEndpoint() *Endpoint { + if x != nil { + return x.RemoteEndpoint + } + return nil +} + +func (x *StartServerHandshakeReq) GetRpcVersions() *RpcProtocolVersions { + if x != nil { + return x.RpcVersions + } + return nil +} + +func (x *StartServerHandshakeReq) GetMaxFrameSize() uint32 { + if x != nil { + return x.MaxFrameSize + } + return 0 +} + +func (x *StartServerHandshakeReq) GetTransportProtocolPreferences() *TransportProtocolPreferences { + if x != nil { + return x.TransportProtocolPreferences + } + return nil +} + +type NextHandshakeMessageReq struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Bytes in out_frames returned from the peer's HandshakerResp. It is possible + // that the peer's out_frames are split into multiple NextHandshakerMessageReq + // messages. + InBytes []byte `protobuf:"bytes,1,opt,name=in_bytes,json=inBytes,proto3" json:"in_bytes,omitempty"` + // Number of milliseconds between when the application send the last handshake + // message to the peer and when the application received the current handshake + // message (in the in_bytes field) from the peer. + NetworkLatencyMs uint32 `protobuf:"varint,2,opt,name=network_latency_ms,json=networkLatencyMs,proto3" json:"network_latency_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NextHandshakeMessageReq) Reset() { + *x = NextHandshakeMessageReq{} + mi := &file_grpc_gcp_handshaker_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NextHandshakeMessageReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NextHandshakeMessageReq) ProtoMessage() {} + +func (x *NextHandshakeMessageReq) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NextHandshakeMessageReq.ProtoReflect.Descriptor instead. +func (*NextHandshakeMessageReq) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{5} +} + +func (x *NextHandshakeMessageReq) GetInBytes() []byte { + if x != nil { + return x.InBytes + } + return nil +} + +func (x *NextHandshakeMessageReq) GetNetworkLatencyMs() uint32 { + if x != nil { + return x.NetworkLatencyMs + } + return 0 +} + +type HandshakerReq struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to ReqOneof: + // + // *HandshakerReq_ClientStart + // *HandshakerReq_ServerStart + // *HandshakerReq_Next + ReqOneof isHandshakerReq_ReqOneof `protobuf_oneof:"req_oneof"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HandshakerReq) Reset() { + *x = HandshakerReq{} + mi := &file_grpc_gcp_handshaker_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HandshakerReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HandshakerReq) ProtoMessage() {} + +func (x *HandshakerReq) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HandshakerReq.ProtoReflect.Descriptor instead. +func (*HandshakerReq) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{6} +} + +func (x *HandshakerReq) GetReqOneof() isHandshakerReq_ReqOneof { + if x != nil { + return x.ReqOneof + } + return nil +} + +func (x *HandshakerReq) GetClientStart() *StartClientHandshakeReq { + if x != nil { + if x, ok := x.ReqOneof.(*HandshakerReq_ClientStart); ok { + return x.ClientStart + } + } + return nil +} + +func (x *HandshakerReq) GetServerStart() *StartServerHandshakeReq { + if x != nil { + if x, ok := x.ReqOneof.(*HandshakerReq_ServerStart); ok { + return x.ServerStart + } + } + return nil +} + +func (x *HandshakerReq) GetNext() *NextHandshakeMessageReq { + if x != nil { + if x, ok := x.ReqOneof.(*HandshakerReq_Next); ok { + return x.Next + } + } + return nil +} + +type isHandshakerReq_ReqOneof interface { + isHandshakerReq_ReqOneof() +} + +type HandshakerReq_ClientStart struct { + // The start client handshake request message. + ClientStart *StartClientHandshakeReq `protobuf:"bytes,1,opt,name=client_start,json=clientStart,proto3,oneof"` +} + +type HandshakerReq_ServerStart struct { + // The start server handshake request message. + ServerStart *StartServerHandshakeReq `protobuf:"bytes,2,opt,name=server_start,json=serverStart,proto3,oneof"` +} + +type HandshakerReq_Next struct { + // The next handshake request message. + Next *NextHandshakeMessageReq `protobuf:"bytes,3,opt,name=next,proto3,oneof"` +} + +func (*HandshakerReq_ClientStart) isHandshakerReq_ReqOneof() {} + +func (*HandshakerReq_ServerStart) isHandshakerReq_ReqOneof() {} + +func (*HandshakerReq_Next) isHandshakerReq_ReqOneof() {} + +type HandshakerResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The application protocol negotiated for this connection. + ApplicationProtocol string `protobuf:"bytes,1,opt,name=application_protocol,json=applicationProtocol,proto3" json:"application_protocol,omitempty"` + // The record protocol negotiated for this connection. + RecordProtocol string `protobuf:"bytes,2,opt,name=record_protocol,json=recordProtocol,proto3" json:"record_protocol,omitempty"` + // Cryptographic key data. The key data may be more than the key length + // required for the record protocol, thus the client of the handshaker + // service needs to truncate the key data into the right key length. + KeyData []byte `protobuf:"bytes,3,opt,name=key_data,json=keyData,proto3" json:"key_data,omitempty"` + // The authenticated identity of the peer. + PeerIdentity *Identity `protobuf:"bytes,4,opt,name=peer_identity,json=peerIdentity,proto3" json:"peer_identity,omitempty"` + // The local identity used in the handshake. + LocalIdentity *Identity `protobuf:"bytes,5,opt,name=local_identity,json=localIdentity,proto3" json:"local_identity,omitempty"` + // Indicate whether the handshaker service client should keep the channel + // between the handshaker service open, e.g., in order to handle + // post-handshake messages in the future. + KeepChannelOpen bool `protobuf:"varint,6,opt,name=keep_channel_open,json=keepChannelOpen,proto3" json:"keep_channel_open,omitempty"` + // The RPC protocol versions supported by the peer. + PeerRpcVersions *RpcProtocolVersions `protobuf:"bytes,7,opt,name=peer_rpc_versions,json=peerRpcVersions,proto3" json:"peer_rpc_versions,omitempty"` + // The maximum frame size of the peer. + MaxFrameSize uint32 `protobuf:"varint,8,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"` + // (Optional) The transport protocol negotiated for this connection. + TransportProtocol *NegotiatedTransportProtocol `protobuf:"bytes,9,opt,name=transport_protocol,json=transportProtocol,proto3" json:"transport_protocol,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HandshakerResult) Reset() { + *x = HandshakerResult{} + mi := &file_grpc_gcp_handshaker_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HandshakerResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HandshakerResult) ProtoMessage() {} + +func (x *HandshakerResult) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HandshakerResult.ProtoReflect.Descriptor instead. +func (*HandshakerResult) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{7} +} + +func (x *HandshakerResult) GetApplicationProtocol() string { + if x != nil { + return x.ApplicationProtocol + } + return "" +} + +func (x *HandshakerResult) GetRecordProtocol() string { + if x != nil { + return x.RecordProtocol + } + return "" +} + +func (x *HandshakerResult) GetKeyData() []byte { + if x != nil { + return x.KeyData + } + return nil +} + +func (x *HandshakerResult) GetPeerIdentity() *Identity { + if x != nil { + return x.PeerIdentity + } + return nil +} + +func (x *HandshakerResult) GetLocalIdentity() *Identity { + if x != nil { + return x.LocalIdentity + } + return nil +} + +func (x *HandshakerResult) GetKeepChannelOpen() bool { + if x != nil { + return x.KeepChannelOpen + } + return false +} + +func (x *HandshakerResult) GetPeerRpcVersions() *RpcProtocolVersions { + if x != nil { + return x.PeerRpcVersions + } + return nil +} + +func (x *HandshakerResult) GetMaxFrameSize() uint32 { + if x != nil { + return x.MaxFrameSize + } + return 0 +} + +func (x *HandshakerResult) GetTransportProtocol() *NegotiatedTransportProtocol { + if x != nil { + return x.TransportProtocol + } + return nil +} + +type HandshakerStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The status code. This could be the gRPC status code. + Code uint32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` + // The status details. + Details string `protobuf:"bytes,2,opt,name=details,proto3" json:"details,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HandshakerStatus) Reset() { + *x = HandshakerStatus{} + mi := &file_grpc_gcp_handshaker_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HandshakerStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HandshakerStatus) ProtoMessage() {} + +func (x *HandshakerStatus) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HandshakerStatus.ProtoReflect.Descriptor instead. +func (*HandshakerStatus) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{8} +} + +func (x *HandshakerStatus) GetCode() uint32 { + if x != nil { + return x.Code + } + return 0 +} + +func (x *HandshakerStatus) GetDetails() string { + if x != nil { + return x.Details + } + return "" +} + +type HandshakerResp struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Frames to be given to the peer for the NextHandshakeMessageReq. May be + // empty if no out_frames have to be sent to the peer or if in_bytes in the + // HandshakerReq are incomplete. All the non-empty out frames must be sent to + // the peer even if the handshaker status is not OK as these frames may + // contain the alert frames. + OutFrames []byte `protobuf:"bytes,1,opt,name=out_frames,json=outFrames,proto3" json:"out_frames,omitempty"` + // Number of bytes in the in_bytes consumed by the handshaker. It is possible + // that part of in_bytes in HandshakerReq was unrelated to the handshake + // process. + BytesConsumed uint32 `protobuf:"varint,2,opt,name=bytes_consumed,json=bytesConsumed,proto3" json:"bytes_consumed,omitempty"` + // This is set iff the handshake was successful. out_frames may still be set + // to frames that needs to be forwarded to the peer. + Result *HandshakerResult `protobuf:"bytes,3,opt,name=result,proto3" json:"result,omitempty"` + // Status of the handshaker. + Status *HandshakerStatus `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HandshakerResp) Reset() { + *x = HandshakerResp{} + mi := &file_grpc_gcp_handshaker_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HandshakerResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HandshakerResp) ProtoMessage() {} + +func (x *HandshakerResp) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HandshakerResp.ProtoReflect.Descriptor instead. +func (*HandshakerResp) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{9} +} + +func (x *HandshakerResp) GetOutFrames() []byte { + if x != nil { + return x.OutFrames + } + return nil +} + +func (x *HandshakerResp) GetBytesConsumed() uint32 { + if x != nil { + return x.BytesConsumed + } + return 0 +} + +func (x *HandshakerResp) GetResult() *HandshakerResult { + if x != nil { + return x.Result + } + return nil +} + +func (x *HandshakerResp) GetStatus() *HandshakerStatus { + if x != nil { + return x.Status + } + return nil +} + +var File_grpc_gcp_handshaker_proto protoreflect.FileDescriptor + +const file_grpc_gcp_handshaker_proto_rawDesc = "" + + "\n" + + "\x19grpc/gcp/handshaker.proto\x12\bgrpc.gcp\x1a(grpc/gcp/transport_security_common.proto\"t\n" + + "\bEndpoint\x12\x1d\n" + + "\n" + + "ip_address\x18\x01 \x01(\tR\tipAddress\x12\x12\n" + + "\x04port\x18\x02 \x01(\x05R\x04port\x125\n" + + "\bprotocol\x18\x03 \x01(\x0e2\x19.grpc.gcp.NetworkProtocolR\bprotocol\"\xe8\x01\n" + + "\bIdentity\x12)\n" + + "\x0fservice_account\x18\x01 \x01(\tH\x00R\x0eserviceAccount\x12\x1c\n" + + "\bhostname\x18\x02 \x01(\tH\x00R\bhostname\x12B\n" + + "\n" + + "attributes\x18\x03 \x03(\v2\".grpc.gcp.Identity.AttributesEntryR\n" + + "attributes\x1a=\n" + + "\x0fAttributesEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\x10\n" + + "\x0eidentity_oneof\"\xe9\x05\n" + + "\x17StartClientHandshakeReq\x12[\n" + + "\x1bhandshake_security_protocol\x18\x01 \x01(\x0e2\x1b.grpc.gcp.HandshakeProtocolR\x19handshakeSecurityProtocol\x123\n" + + "\x15application_protocols\x18\x02 \x03(\tR\x14applicationProtocols\x12)\n" + + "\x10record_protocols\x18\x03 \x03(\tR\x0frecordProtocols\x12?\n" + + "\x11target_identities\x18\x04 \x03(\v2\x12.grpc.gcp.IdentityR\x10targetIdentities\x129\n" + + "\x0elocal_identity\x18\x05 \x01(\v2\x12.grpc.gcp.IdentityR\rlocalIdentity\x129\n" + + "\x0elocal_endpoint\x18\x06 \x01(\v2\x12.grpc.gcp.EndpointR\rlocalEndpoint\x12;\n" + + "\x0fremote_endpoint\x18\a \x01(\v2\x12.grpc.gcp.EndpointR\x0eremoteEndpoint\x12\x1f\n" + + "\vtarget_name\x18\b \x01(\tR\n" + + "targetName\x12@\n" + + "\frpc_versions\x18\t \x01(\v2\x1d.grpc.gcp.RpcProtocolVersionsR\vrpcVersions\x12$\n" + + "\x0emax_frame_size\x18\n" + + " \x01(\rR\fmaxFrameSize\x12&\n" + + "\faccess_token\x18\v \x01(\tB\x03\x80\x01\x01R\vaccessToken\x12l\n" + + "\x1etransport_protocol_preferences\x18\f \x01(\v2&.grpc.gcp.TransportProtocolPreferencesR\x1ctransportProtocolPreferences\"\xaf\x01\n" + + "\x19ServerHandshakeParameters\x12)\n" + + "\x10record_protocols\x18\x01 \x03(\tR\x0frecordProtocols\x12=\n" + + "\x10local_identities\x18\x02 \x03(\v2\x12.grpc.gcp.IdentityR\x0flocalIdentities\x12\x1e\n" + + "\x05token\x18\x03 \x01(\tB\x03\x80\x01\x01H\x00R\x05token\x88\x01\x01B\b\n" + + "\x06_token\"\x93\x05\n" + + "\x17StartServerHandshakeReq\x123\n" + + "\x15application_protocols\x18\x01 \x03(\tR\x14applicationProtocols\x12m\n" + + "\x14handshake_parameters\x18\x02 \x03(\v2:.grpc.gcp.StartServerHandshakeReq.HandshakeParametersEntryR\x13handshakeParameters\x12\x19\n" + + "\bin_bytes\x18\x03 \x01(\fR\ainBytes\x129\n" + + "\x0elocal_endpoint\x18\x04 \x01(\v2\x12.grpc.gcp.EndpointR\rlocalEndpoint\x12;\n" + + "\x0fremote_endpoint\x18\x05 \x01(\v2\x12.grpc.gcp.EndpointR\x0eremoteEndpoint\x12@\n" + + "\frpc_versions\x18\x06 \x01(\v2\x1d.grpc.gcp.RpcProtocolVersionsR\vrpcVersions\x12$\n" + + "\x0emax_frame_size\x18\a \x01(\rR\fmaxFrameSize\x12l\n" + + "\x1etransport_protocol_preferences\x18\b \x01(\v2&.grpc.gcp.TransportProtocolPreferencesR\x1ctransportProtocolPreferences\x1ak\n" + + "\x18HandshakeParametersEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\x05R\x03key\x129\n" + + "\x05value\x18\x02 \x01(\v2#.grpc.gcp.ServerHandshakeParametersR\x05value:\x028\x01\"b\n" + + "\x17NextHandshakeMessageReq\x12\x19\n" + + "\bin_bytes\x18\x01 \x01(\fR\ainBytes\x12,\n" + + "\x12network_latency_ms\x18\x02 \x01(\rR\x10networkLatencyMs\"\xe5\x01\n" + + "\rHandshakerReq\x12F\n" + + "\fclient_start\x18\x01 \x01(\v2!.grpc.gcp.StartClientHandshakeReqH\x00R\vclientStart\x12F\n" + + "\fserver_start\x18\x02 \x01(\v2!.grpc.gcp.StartServerHandshakeReqH\x00R\vserverStart\x127\n" + + "\x04next\x18\x03 \x01(\v2!.grpc.gcp.NextHandshakeMessageReqH\x00R\x04nextB\v\n" + + "\treq_oneof\"\xf0\x03\n" + + "\x10HandshakerResult\x121\n" + + "\x14application_protocol\x18\x01 \x01(\tR\x13applicationProtocol\x12'\n" + + "\x0frecord_protocol\x18\x02 \x01(\tR\x0erecordProtocol\x12\x19\n" + + "\bkey_data\x18\x03 \x01(\fR\akeyData\x127\n" + + "\rpeer_identity\x18\x04 \x01(\v2\x12.grpc.gcp.IdentityR\fpeerIdentity\x129\n" + + "\x0elocal_identity\x18\x05 \x01(\v2\x12.grpc.gcp.IdentityR\rlocalIdentity\x12*\n" + + "\x11keep_channel_open\x18\x06 \x01(\bR\x0fkeepChannelOpen\x12I\n" + + "\x11peer_rpc_versions\x18\a \x01(\v2\x1d.grpc.gcp.RpcProtocolVersionsR\x0fpeerRpcVersions\x12$\n" + + "\x0emax_frame_size\x18\b \x01(\rR\fmaxFrameSize\x12T\n" + + "\x12transport_protocol\x18\t \x01(\v2%.grpc.gcp.NegotiatedTransportProtocolR\x11transportProtocol\"@\n" + + "\x10HandshakerStatus\x12\x12\n" + + "\x04code\x18\x01 \x01(\rR\x04code\x12\x18\n" + + "\adetails\x18\x02 \x01(\tR\adetails\"\xbe\x01\n" + + "\x0eHandshakerResp\x12\x1d\n" + + "\n" + + "out_frames\x18\x01 \x01(\fR\toutFrames\x12%\n" + + "\x0ebytes_consumed\x18\x02 \x01(\rR\rbytesConsumed\x122\n" + + "\x06result\x18\x03 \x01(\v2\x1a.grpc.gcp.HandshakerResultR\x06result\x122\n" + + "\x06status\x18\x04 \x01(\v2\x1a.grpc.gcp.HandshakerStatusR\x06status*J\n" + + "\x11HandshakeProtocol\x12\"\n" + + "\x1eHANDSHAKE_PROTOCOL_UNSPECIFIED\x10\x00\x12\a\n" + + "\x03TLS\x10\x01\x12\b\n" + + "\x04ALTS\x10\x02*E\n" + + "\x0fNetworkProtocol\x12 \n" + + "\x1cNETWORK_PROTOCOL_UNSPECIFIED\x10\x00\x12\a\n" + + "\x03TCP\x10\x01\x12\a\n" + + "\x03UDP\x10\x022[\n" + + "\x11HandshakerService\x12F\n" + + "\vDoHandshake\x12\x17.grpc.gcp.HandshakerReq\x1a\x18.grpc.gcp.HandshakerResp\"\x00(\x010\x01Bk\n" + + "\x15io.grpc.alts.internalB\x0fHandshakerProtoP\x01Z?google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcpb\x06proto3" + +var ( + file_grpc_gcp_handshaker_proto_rawDescOnce sync.Once + file_grpc_gcp_handshaker_proto_rawDescData []byte +) + +func file_grpc_gcp_handshaker_proto_rawDescGZIP() []byte { + file_grpc_gcp_handshaker_proto_rawDescOnce.Do(func() { + file_grpc_gcp_handshaker_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_gcp_handshaker_proto_rawDesc), len(file_grpc_gcp_handshaker_proto_rawDesc))) + }) + return file_grpc_gcp_handshaker_proto_rawDescData +} + +var file_grpc_gcp_handshaker_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_grpc_gcp_handshaker_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_grpc_gcp_handshaker_proto_goTypes = []any{ + (HandshakeProtocol)(0), // 0: grpc.gcp.HandshakeProtocol + (NetworkProtocol)(0), // 1: grpc.gcp.NetworkProtocol + (*Endpoint)(nil), // 2: grpc.gcp.Endpoint + (*Identity)(nil), // 3: grpc.gcp.Identity + (*StartClientHandshakeReq)(nil), // 4: grpc.gcp.StartClientHandshakeReq + (*ServerHandshakeParameters)(nil), // 5: grpc.gcp.ServerHandshakeParameters + (*StartServerHandshakeReq)(nil), // 6: grpc.gcp.StartServerHandshakeReq + (*NextHandshakeMessageReq)(nil), // 7: grpc.gcp.NextHandshakeMessageReq + (*HandshakerReq)(nil), // 8: grpc.gcp.HandshakerReq + (*HandshakerResult)(nil), // 9: grpc.gcp.HandshakerResult + (*HandshakerStatus)(nil), // 10: grpc.gcp.HandshakerStatus + (*HandshakerResp)(nil), // 11: grpc.gcp.HandshakerResp + nil, // 12: grpc.gcp.Identity.AttributesEntry + nil, // 13: grpc.gcp.StartServerHandshakeReq.HandshakeParametersEntry + (*RpcProtocolVersions)(nil), // 14: grpc.gcp.RpcProtocolVersions + (*TransportProtocolPreferences)(nil), // 15: grpc.gcp.TransportProtocolPreferences + (*NegotiatedTransportProtocol)(nil), // 16: grpc.gcp.NegotiatedTransportProtocol +} +var file_grpc_gcp_handshaker_proto_depIdxs = []int32{ + 1, // 0: grpc.gcp.Endpoint.protocol:type_name -> grpc.gcp.NetworkProtocol + 12, // 1: grpc.gcp.Identity.attributes:type_name -> grpc.gcp.Identity.AttributesEntry + 0, // 2: grpc.gcp.StartClientHandshakeReq.handshake_security_protocol:type_name -> grpc.gcp.HandshakeProtocol + 3, // 3: grpc.gcp.StartClientHandshakeReq.target_identities:type_name -> grpc.gcp.Identity + 3, // 4: grpc.gcp.StartClientHandshakeReq.local_identity:type_name -> grpc.gcp.Identity + 2, // 5: grpc.gcp.StartClientHandshakeReq.local_endpoint:type_name -> grpc.gcp.Endpoint + 2, // 6: grpc.gcp.StartClientHandshakeReq.remote_endpoint:type_name -> grpc.gcp.Endpoint + 14, // 7: grpc.gcp.StartClientHandshakeReq.rpc_versions:type_name -> grpc.gcp.RpcProtocolVersions + 15, // 8: grpc.gcp.StartClientHandshakeReq.transport_protocol_preferences:type_name -> grpc.gcp.TransportProtocolPreferences + 3, // 9: grpc.gcp.ServerHandshakeParameters.local_identities:type_name -> grpc.gcp.Identity + 13, // 10: grpc.gcp.StartServerHandshakeReq.handshake_parameters:type_name -> grpc.gcp.StartServerHandshakeReq.HandshakeParametersEntry + 2, // 11: grpc.gcp.StartServerHandshakeReq.local_endpoint:type_name -> grpc.gcp.Endpoint + 2, // 12: grpc.gcp.StartServerHandshakeReq.remote_endpoint:type_name -> grpc.gcp.Endpoint + 14, // 13: grpc.gcp.StartServerHandshakeReq.rpc_versions:type_name -> grpc.gcp.RpcProtocolVersions + 15, // 14: grpc.gcp.StartServerHandshakeReq.transport_protocol_preferences:type_name -> grpc.gcp.TransportProtocolPreferences + 4, // 15: grpc.gcp.HandshakerReq.client_start:type_name -> grpc.gcp.StartClientHandshakeReq + 6, // 16: grpc.gcp.HandshakerReq.server_start:type_name -> grpc.gcp.StartServerHandshakeReq + 7, // 17: grpc.gcp.HandshakerReq.next:type_name -> grpc.gcp.NextHandshakeMessageReq + 3, // 18: grpc.gcp.HandshakerResult.peer_identity:type_name -> grpc.gcp.Identity + 3, // 19: grpc.gcp.HandshakerResult.local_identity:type_name -> grpc.gcp.Identity + 14, // 20: grpc.gcp.HandshakerResult.peer_rpc_versions:type_name -> grpc.gcp.RpcProtocolVersions + 16, // 21: grpc.gcp.HandshakerResult.transport_protocol:type_name -> grpc.gcp.NegotiatedTransportProtocol + 9, // 22: grpc.gcp.HandshakerResp.result:type_name -> grpc.gcp.HandshakerResult + 10, // 23: grpc.gcp.HandshakerResp.status:type_name -> grpc.gcp.HandshakerStatus + 5, // 24: grpc.gcp.StartServerHandshakeReq.HandshakeParametersEntry.value:type_name -> grpc.gcp.ServerHandshakeParameters + 8, // 25: grpc.gcp.HandshakerService.DoHandshake:input_type -> grpc.gcp.HandshakerReq + 11, // 26: grpc.gcp.HandshakerService.DoHandshake:output_type -> grpc.gcp.HandshakerResp + 26, // [26:27] is the sub-list for method output_type + 25, // [25:26] is the sub-list for method input_type + 25, // [25:25] is the sub-list for extension type_name + 25, // [25:25] is the sub-list for extension extendee + 0, // [0:25] is the sub-list for field type_name +} + +func init() { file_grpc_gcp_handshaker_proto_init() } +func file_grpc_gcp_handshaker_proto_init() { + if File_grpc_gcp_handshaker_proto != nil { + return + } + file_grpc_gcp_transport_security_common_proto_init() + file_grpc_gcp_handshaker_proto_msgTypes[1].OneofWrappers = []any{ + (*Identity_ServiceAccount)(nil), + (*Identity_Hostname)(nil), + } + file_grpc_gcp_handshaker_proto_msgTypes[3].OneofWrappers = []any{} + file_grpc_gcp_handshaker_proto_msgTypes[6].OneofWrappers = []any{ + (*HandshakerReq_ClientStart)(nil), + (*HandshakerReq_ServerStart)(nil), + (*HandshakerReq_Next)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_gcp_handshaker_proto_rawDesc), len(file_grpc_gcp_handshaker_proto_rawDesc)), + NumEnums: 2, + NumMessages: 12, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_gcp_handshaker_proto_goTypes, + DependencyIndexes: file_grpc_gcp_handshaker_proto_depIdxs, + EnumInfos: file_grpc_gcp_handshaker_proto_enumTypes, + MessageInfos: file_grpc_gcp_handshaker_proto_msgTypes, + }.Build() + File_grpc_gcp_handshaker_proto = out.File + file_grpc_gcp_handshaker_proto_goTypes = nil + file_grpc_gcp_handshaker_proto_depIdxs = nil +} diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go new file mode 100644 index 0000000..96a40fc --- /dev/null +++ b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go @@ -0,0 +1,144 @@ +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/gcp/handshaker.proto + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: grpc/gcp/handshaker.proto + +package grpc_gcp + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + HandshakerService_DoHandshake_FullMethodName = "/grpc.gcp.HandshakerService/DoHandshake" +) + +// HandshakerServiceClient is the client API for HandshakerService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type HandshakerServiceClient interface { + // Handshaker service accepts a stream of handshaker request, returning a + // stream of handshaker response. Client is expected to send exactly one + // message with either client_start or server_start followed by one or more + // messages with next. Each time client sends a request, the handshaker + // service expects to respond. Client does not have to wait for service's + // response before sending next request. + DoHandshake(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[HandshakerReq, HandshakerResp], error) +} + +type handshakerServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewHandshakerServiceClient(cc grpc.ClientConnInterface) HandshakerServiceClient { + return &handshakerServiceClient{cc} +} + +func (c *handshakerServiceClient) DoHandshake(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[HandshakerReq, HandshakerResp], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &HandshakerService_ServiceDesc.Streams[0], HandshakerService_DoHandshake_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[HandshakerReq, HandshakerResp]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type HandshakerService_DoHandshakeClient = grpc.BidiStreamingClient[HandshakerReq, HandshakerResp] + +// HandshakerServiceServer is the server API for HandshakerService service. +// All implementations must embed UnimplementedHandshakerServiceServer +// for forward compatibility. +type HandshakerServiceServer interface { + // Handshaker service accepts a stream of handshaker request, returning a + // stream of handshaker response. Client is expected to send exactly one + // message with either client_start or server_start followed by one or more + // messages with next. Each time client sends a request, the handshaker + // service expects to respond. Client does not have to wait for service's + // response before sending next request. + DoHandshake(grpc.BidiStreamingServer[HandshakerReq, HandshakerResp]) error + mustEmbedUnimplementedHandshakerServiceServer() +} + +// UnimplementedHandshakerServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedHandshakerServiceServer struct{} + +func (UnimplementedHandshakerServiceServer) DoHandshake(grpc.BidiStreamingServer[HandshakerReq, HandshakerResp]) error { + return status.Error(codes.Unimplemented, "method DoHandshake not implemented") +} +func (UnimplementedHandshakerServiceServer) mustEmbedUnimplementedHandshakerServiceServer() {} +func (UnimplementedHandshakerServiceServer) testEmbeddedByValue() {} + +// UnsafeHandshakerServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to HandshakerServiceServer will +// result in compilation errors. +type UnsafeHandshakerServiceServer interface { + mustEmbedUnimplementedHandshakerServiceServer() +} + +func RegisterHandshakerServiceServer(s grpc.ServiceRegistrar, srv HandshakerServiceServer) { + // If the following call panics, it indicates UnimplementedHandshakerServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&HandshakerService_ServiceDesc, srv) +} + +func _HandshakerService_DoHandshake_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(HandshakerServiceServer).DoHandshake(&grpc.GenericServerStream[HandshakerReq, HandshakerResp]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type HandshakerService_DoHandshakeServer = grpc.BidiStreamingServer[HandshakerReq, HandshakerResp] + +// HandshakerService_ServiceDesc is the grpc.ServiceDesc for HandshakerService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var HandshakerService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.gcp.HandshakerService", + HandlerType: (*HandshakerServiceServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "DoHandshake", + Handler: _HandshakerService_DoHandshake_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/gcp/handshaker.proto", +} diff --git a/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go b/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go new file mode 100644 index 0000000..b3d29b1 --- /dev/null +++ b/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go @@ -0,0 +1,366 @@ +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/gcp/transport_security_common.proto + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/gcp/transport_security_common.proto + +package grpc_gcp + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// The security level of the created channel. The list is sorted in increasing +// level of security. This order must always be maintained. +type SecurityLevel int32 + +const ( + SecurityLevel_SECURITY_NONE SecurityLevel = 0 + SecurityLevel_INTEGRITY_ONLY SecurityLevel = 1 + SecurityLevel_INTEGRITY_AND_PRIVACY SecurityLevel = 2 +) + +// Enum value maps for SecurityLevel. +var ( + SecurityLevel_name = map[int32]string{ + 0: "SECURITY_NONE", + 1: "INTEGRITY_ONLY", + 2: "INTEGRITY_AND_PRIVACY", + } + SecurityLevel_value = map[string]int32{ + "SECURITY_NONE": 0, + "INTEGRITY_ONLY": 1, + "INTEGRITY_AND_PRIVACY": 2, + } +) + +func (x SecurityLevel) Enum() *SecurityLevel { + p := new(SecurityLevel) + *p = x + return p +} + +func (x SecurityLevel) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SecurityLevel) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_gcp_transport_security_common_proto_enumTypes[0].Descriptor() +} + +func (SecurityLevel) Type() protoreflect.EnumType { + return &file_grpc_gcp_transport_security_common_proto_enumTypes[0] +} + +func (x SecurityLevel) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SecurityLevel.Descriptor instead. +func (SecurityLevel) EnumDescriptor() ([]byte, []int) { + return file_grpc_gcp_transport_security_common_proto_rawDescGZIP(), []int{0} +} + +// Max and min supported RPC protocol versions. +type RpcProtocolVersions struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Maximum supported RPC version. + MaxRpcVersion *RpcProtocolVersions_Version `protobuf:"bytes,1,opt,name=max_rpc_version,json=maxRpcVersion,proto3" json:"max_rpc_version,omitempty"` + // Minimum supported RPC version. + MinRpcVersion *RpcProtocolVersions_Version `protobuf:"bytes,2,opt,name=min_rpc_version,json=minRpcVersion,proto3" json:"min_rpc_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RpcProtocolVersions) Reset() { + *x = RpcProtocolVersions{} + mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RpcProtocolVersions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RpcProtocolVersions) ProtoMessage() {} + +func (x *RpcProtocolVersions) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RpcProtocolVersions.ProtoReflect.Descriptor instead. +func (*RpcProtocolVersions) Descriptor() ([]byte, []int) { + return file_grpc_gcp_transport_security_common_proto_rawDescGZIP(), []int{0} +} + +func (x *RpcProtocolVersions) GetMaxRpcVersion() *RpcProtocolVersions_Version { + if x != nil { + return x.MaxRpcVersion + } + return nil +} + +func (x *RpcProtocolVersions) GetMinRpcVersion() *RpcProtocolVersions_Version { + if x != nil { + return x.MinRpcVersion + } + return nil +} + +// The ordered list of protocols that the client wishes to use, or the set +// that the server supports. +type TransportProtocolPreferences struct { + state protoimpl.MessageState `protogen:"open.v1"` + TransportProtocol []string `protobuf:"bytes,1,rep,name=transport_protocol,json=transportProtocol,proto3" json:"transport_protocol,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TransportProtocolPreferences) Reset() { + *x = TransportProtocolPreferences{} + mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TransportProtocolPreferences) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TransportProtocolPreferences) ProtoMessage() {} + +func (x *TransportProtocolPreferences) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TransportProtocolPreferences.ProtoReflect.Descriptor instead. +func (*TransportProtocolPreferences) Descriptor() ([]byte, []int) { + return file_grpc_gcp_transport_security_common_proto_rawDescGZIP(), []int{1} +} + +func (x *TransportProtocolPreferences) GetTransportProtocol() []string { + if x != nil { + return x.TransportProtocol + } + return nil +} + +// The negotiated transport protocol. +type NegotiatedTransportProtocol struct { + state protoimpl.MessageState `protogen:"open.v1"` + TransportProtocol string `protobuf:"bytes,1,opt,name=transport_protocol,json=transportProtocol,proto3" json:"transport_protocol,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NegotiatedTransportProtocol) Reset() { + *x = NegotiatedTransportProtocol{} + mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NegotiatedTransportProtocol) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NegotiatedTransportProtocol) ProtoMessage() {} + +func (x *NegotiatedTransportProtocol) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NegotiatedTransportProtocol.ProtoReflect.Descriptor instead. +func (*NegotiatedTransportProtocol) Descriptor() ([]byte, []int) { + return file_grpc_gcp_transport_security_common_proto_rawDescGZIP(), []int{2} +} + +func (x *NegotiatedTransportProtocol) GetTransportProtocol() string { + if x != nil { + return x.TransportProtocol + } + return "" +} + +// RPC version contains a major version and a minor version. +type RpcProtocolVersions_Version struct { + state protoimpl.MessageState `protogen:"open.v1"` + Major uint32 `protobuf:"varint,1,opt,name=major,proto3" json:"major,omitempty"` + Minor uint32 `protobuf:"varint,2,opt,name=minor,proto3" json:"minor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RpcProtocolVersions_Version) Reset() { + *x = RpcProtocolVersions_Version{} + mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RpcProtocolVersions_Version) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RpcProtocolVersions_Version) ProtoMessage() {} + +func (x *RpcProtocolVersions_Version) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RpcProtocolVersions_Version.ProtoReflect.Descriptor instead. +func (*RpcProtocolVersions_Version) Descriptor() ([]byte, []int) { + return file_grpc_gcp_transport_security_common_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *RpcProtocolVersions_Version) GetMajor() uint32 { + if x != nil { + return x.Major + } + return 0 +} + +func (x *RpcProtocolVersions_Version) GetMinor() uint32 { + if x != nil { + return x.Minor + } + return 0 +} + +var File_grpc_gcp_transport_security_common_proto protoreflect.FileDescriptor + +const file_grpc_gcp_transport_security_common_proto_rawDesc = "" + + "\n" + + "(grpc/gcp/transport_security_common.proto\x12\bgrpc.gcp\"\xea\x01\n" + + "\x13RpcProtocolVersions\x12M\n" + + "\x0fmax_rpc_version\x18\x01 \x01(\v2%.grpc.gcp.RpcProtocolVersions.VersionR\rmaxRpcVersion\x12M\n" + + "\x0fmin_rpc_version\x18\x02 \x01(\v2%.grpc.gcp.RpcProtocolVersions.VersionR\rminRpcVersion\x1a5\n" + + "\aVersion\x12\x14\n" + + "\x05major\x18\x01 \x01(\rR\x05major\x12\x14\n" + + "\x05minor\x18\x02 \x01(\rR\x05minor\"M\n" + + "\x1cTransportProtocolPreferences\x12-\n" + + "\x12transport_protocol\x18\x01 \x03(\tR\x11transportProtocol\"L\n" + + "\x1bNegotiatedTransportProtocol\x12-\n" + + "\x12transport_protocol\x18\x01 \x01(\tR\x11transportProtocol*Q\n" + + "\rSecurityLevel\x12\x11\n" + + "\rSECURITY_NONE\x10\x00\x12\x12\n" + + "\x0eINTEGRITY_ONLY\x10\x01\x12\x19\n" + + "\x15INTEGRITY_AND_PRIVACY\x10\x02Bx\n" + + "\x15io.grpc.alts.internalB\x1cTransportSecurityCommonProtoP\x01Z?google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcpb\x06proto3" + +var ( + file_grpc_gcp_transport_security_common_proto_rawDescOnce sync.Once + file_grpc_gcp_transport_security_common_proto_rawDescData []byte +) + +func file_grpc_gcp_transport_security_common_proto_rawDescGZIP() []byte { + file_grpc_gcp_transport_security_common_proto_rawDescOnce.Do(func() { + file_grpc_gcp_transport_security_common_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_gcp_transport_security_common_proto_rawDesc), len(file_grpc_gcp_transport_security_common_proto_rawDesc))) + }) + return file_grpc_gcp_transport_security_common_proto_rawDescData +} + +var file_grpc_gcp_transport_security_common_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_grpc_gcp_transport_security_common_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_grpc_gcp_transport_security_common_proto_goTypes = []any{ + (SecurityLevel)(0), // 0: grpc.gcp.SecurityLevel + (*RpcProtocolVersions)(nil), // 1: grpc.gcp.RpcProtocolVersions + (*TransportProtocolPreferences)(nil), // 2: grpc.gcp.TransportProtocolPreferences + (*NegotiatedTransportProtocol)(nil), // 3: grpc.gcp.NegotiatedTransportProtocol + (*RpcProtocolVersions_Version)(nil), // 4: grpc.gcp.RpcProtocolVersions.Version +} +var file_grpc_gcp_transport_security_common_proto_depIdxs = []int32{ + 4, // 0: grpc.gcp.RpcProtocolVersions.max_rpc_version:type_name -> grpc.gcp.RpcProtocolVersions.Version + 4, // 1: grpc.gcp.RpcProtocolVersions.min_rpc_version:type_name -> grpc.gcp.RpcProtocolVersions.Version + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_grpc_gcp_transport_security_common_proto_init() } +func file_grpc_gcp_transport_security_common_proto_init() { + if File_grpc_gcp_transport_security_common_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_gcp_transport_security_common_proto_rawDesc), len(file_grpc_gcp_transport_security_common_proto_rawDesc)), + NumEnums: 1, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_gcp_transport_security_common_proto_goTypes, + DependencyIndexes: file_grpc_gcp_transport_security_common_proto_depIdxs, + EnumInfos: file_grpc_gcp_transport_security_common_proto_enumTypes, + MessageInfos: file_grpc_gcp_transport_security_common_proto_msgTypes, + }.Build() + File_grpc_gcp_transport_security_common_proto = out.File + file_grpc_gcp_transport_security_common_proto_goTypes = nil + file_grpc_gcp_transport_security_common_proto_depIdxs = nil +} diff --git a/credentials/alts/internal/testutil/testutil.go b/credentials/alts/internal/testutil/testutil.go new file mode 100644 index 0000000..1dfccdc --- /dev/null +++ b/credentials/alts/internal/testutil/testutil.go @@ -0,0 +1,323 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package testutil include useful test utilities for the handshaker. +package testutil + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "net" + "sync" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/alts/internal/conn" + altsgrpc "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" + altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" +) + +// Stats is used to collect statistics about concurrent handshake calls. +type Stats struct { + mu sync.Mutex + calls int + MaxConcurrentCalls int +} + +// Update updates the statistics by adding one call. +func (s *Stats) Update() func() { + s.mu.Lock() + s.calls++ + if s.calls > s.MaxConcurrentCalls { + s.MaxConcurrentCalls = s.calls + } + s.mu.Unlock() + + return func() { + s.mu.Lock() + s.calls-- + s.mu.Unlock() + } +} + +// Reset resets the statistics. +func (s *Stats) Reset() { + s.mu.Lock() + defer s.mu.Unlock() + s.calls = 0 + s.MaxConcurrentCalls = 0 +} + +// testConn mimics a net.Conn to the peer. +type testConn struct { + net.Conn + in *bytes.Buffer + out *bytes.Buffer + readLatency time.Duration +} + +// NewTestConn creates a new instance of testConn object. +func NewTestConn(in *bytes.Buffer, out *bytes.Buffer) net.Conn { + return &testConn{ + in: in, + out: out, + readLatency: time.Duration(0), + } +} + +// NewTestConnWithReadLatency creates a new instance of testConn object that +// pauses for readLatency before any call to Read() returns. +func NewTestConnWithReadLatency(in *bytes.Buffer, out *bytes.Buffer, readLatency time.Duration) net.Conn { + return &testConn{ + in: in, + out: out, + readLatency: readLatency, + } +} + +// Read reads from the in buffer. +func (c *testConn) Read(b []byte) (n int, err error) { + time.Sleep(c.readLatency) + return c.in.Read(b) +} + +// Write writes to the out buffer. +func (c *testConn) Write(b []byte) (n int, err error) { + return c.out.Write(b) +} + +// Close closes the testConn object. +func (c *testConn) Close() error { + return nil +} + +// unresponsiveTestConn mimics a net.Conn for an unresponsive peer. It is used +// for testing the PeerNotResponding case. +type unresponsiveTestConn struct { + net.Conn +} + +// NewUnresponsiveTestConn creates a new instance of unresponsiveTestConn object. +func NewUnresponsiveTestConn() net.Conn { + return &unresponsiveTestConn{} +} + +// Read reads from the in buffer. +func (c *unresponsiveTestConn) Read([]byte) (n int, err error) { + return 0, io.EOF +} + +// Write writes to the out buffer. +func (c *unresponsiveTestConn) Write([]byte) (n int, err error) { + return 0, nil +} + +// Close closes the TestConn object. +func (c *unresponsiveTestConn) Close() error { + return nil +} + +// MakeFrame creates a handshake frame. +func MakeFrame(pl string) []byte { + f := make([]byte, len(pl)+conn.MsgLenFieldSize) + binary.LittleEndian.PutUint32(f, uint32(len(pl))) + copy(f[conn.MsgLenFieldSize:], []byte(pl)) + return f +} + +// FakeHandshaker is a fake implementation of the ALTS handshaker service. +type FakeHandshaker struct { + altsgrpc.HandshakerServiceServer + // ExpectedBoundAccessToken is the expected bound access token in the ClientStart request. + ExpectedBoundAccessToken string +} + +// DoHandshake performs a fake ALTS handshake. +func (h *FakeHandshaker) DoHandshake(stream altsgrpc.HandshakerService_DoHandshakeServer) error { + var isAssistingClient bool + var handshakeFramesReceivedSoFar []byte + for { + req, err := stream.Recv() + if err != nil { + if err == io.EOF { + return nil + } + return fmt.Errorf("stream recv failure: %v", err) + } + var resp *altspb.HandshakerResp + switch req := req.ReqOneof.(type) { + case *altspb.HandshakerReq_ClientStart: + isAssistingClient = true + resp, err = h.processStartClient(req.ClientStart) + if err != nil { + return fmt.Errorf("processStartClient failure: %v", err) + } + case *altspb.HandshakerReq_ServerStart: + // If we have received the full ClientInit, send the ServerInit and + // ServerFinished. Otherwise, wait for more bytes to arrive from the client. + isAssistingClient = false + handshakeFramesReceivedSoFar = append(handshakeFramesReceivedSoFar, req.ServerStart.InBytes...) + sendHandshakeFrame := bytes.Equal(handshakeFramesReceivedSoFar, []byte("ClientInit")) + resp, err = h.processServerStart(req.ServerStart, sendHandshakeFrame) + if err != nil { + return fmt.Errorf("processServerStart failure: %v", err) + } + case *altspb.HandshakerReq_Next: + // If we have received all handshake frames, send the handshake result. + // Otherwise, wait for more bytes to arrive from the peer. + oldHandshakesBytes := len(handshakeFramesReceivedSoFar) + handshakeFramesReceivedSoFar = append(handshakeFramesReceivedSoFar, req.Next.InBytes...) + isHandshakeComplete := false + if isAssistingClient { + isHandshakeComplete = bytes.HasPrefix(handshakeFramesReceivedSoFar, []byte("ServerInitServerFinished")) + } else { + isHandshakeComplete = bytes.HasPrefix(handshakeFramesReceivedSoFar, []byte("ClientInitClientFinished")) + } + if !isHandshakeComplete { + resp = &altspb.HandshakerResp{ + BytesConsumed: uint32(len(handshakeFramesReceivedSoFar) - oldHandshakesBytes), + Status: &altspb.HandshakerStatus{ + Code: uint32(codes.OK), + }, + } + break + } + resp, err = h.getHandshakeResult(isAssistingClient) + if err != nil { + return fmt.Errorf("getHandshakeResult failure: %v", err) + } + default: + return fmt.Errorf("handshake request has unexpected type: %v", req) + } + + if err = stream.Send(resp); err != nil { + return fmt.Errorf("stream send failure: %v", err) + } + } +} + +func (h *FakeHandshaker) processStartClient(req *altspb.StartClientHandshakeReq) (*altspb.HandshakerResp, error) { + if req.HandshakeSecurityProtocol != altspb.HandshakeProtocol_ALTS { + return nil, fmt.Errorf("unexpected handshake security protocol: %v", req.HandshakeSecurityProtocol) + } + if len(req.ApplicationProtocols) != 1 || req.ApplicationProtocols[0] != "grpc" { + return nil, fmt.Errorf("unexpected application protocols: %v", req.ApplicationProtocols) + } + if len(req.RecordProtocols) != 1 || req.RecordProtocols[0] != "ALTSRP_GCM_AES128_REKEY" { + return nil, fmt.Errorf("unexpected record protocols: %v", req.RecordProtocols) + } + if h.ExpectedBoundAccessToken != req.GetAccessToken() { + return nil, fmt.Errorf("unexpected access token: %v", req.GetAccessToken()) + } + return &altspb.HandshakerResp{ + OutFrames: []byte("ClientInit"), + BytesConsumed: 0, + Status: &altspb.HandshakerStatus{ + Code: uint32(codes.OK), + }, + }, nil +} + +func (h *FakeHandshaker) processServerStart(req *altspb.StartServerHandshakeReq, sendHandshakeFrame bool) (*altspb.HandshakerResp, error) { + if len(req.ApplicationProtocols) != 1 || req.ApplicationProtocols[0] != "grpc" { + return nil, fmt.Errorf("unexpected application protocols: %v", req.ApplicationProtocols) + } + parameters, ok := req.GetHandshakeParameters()[int32(altspb.HandshakeProtocol_ALTS)] + if !ok { + return nil, fmt.Errorf("missing ALTS handshake parameters") + } + if len(parameters.RecordProtocols) != 1 || parameters.RecordProtocols[0] != "ALTSRP_GCM_AES128_REKEY" { + return nil, fmt.Errorf("unexpected record protocols: %v", parameters.RecordProtocols) + } + if sendHandshakeFrame { + return &altspb.HandshakerResp{ + OutFrames: []byte("ServerInitServerFinished"), + BytesConsumed: uint32(len(req.InBytes)), + Status: &altspb.HandshakerStatus{ + Code: uint32(codes.OK), + }, + }, nil + } + return &altspb.HandshakerResp{ + OutFrames: []byte("ServerInitServerFinished"), + BytesConsumed: 10, + Status: &altspb.HandshakerStatus{ + Code: uint32(codes.OK), + }, + }, nil +} + +func (h *FakeHandshaker) getHandshakeResult(isAssistingClient bool) (*altspb.HandshakerResp, error) { + if isAssistingClient { + return &altspb.HandshakerResp{ + OutFrames: []byte("ClientFinished"), + BytesConsumed: 24, + Result: &altspb.HandshakerResult{ + ApplicationProtocol: "grpc", + RecordProtocol: "ALTSRP_GCM_AES128_REKEY", + KeyData: []byte("negotiated-key-data-for-altsrp-gcm-aes128-rekey"), + PeerIdentity: &altspb.Identity{ + IdentityOneof: &altspb.Identity_ServiceAccount{ + ServiceAccount: "server@bar.com", + }, + }, + PeerRpcVersions: &altspb.RpcProtocolVersions{ + MaxRpcVersion: &altspb.RpcProtocolVersions_Version{ + Minor: 1, + Major: 2, + }, + MinRpcVersion: &altspb.RpcProtocolVersions_Version{ + Minor: 1, + Major: 2, + }, + }, + }, + Status: &altspb.HandshakerStatus{ + Code: uint32(codes.OK), + }, + }, nil + } + return &altspb.HandshakerResp{ + BytesConsumed: 14, + Result: &altspb.HandshakerResult{ + ApplicationProtocol: "grpc", + RecordProtocol: "ALTSRP_GCM_AES128_REKEY", + KeyData: []byte("negotiated-key-data-for-altsrp-gcm-aes128-rekey"), + PeerIdentity: &altspb.Identity{ + IdentityOneof: &altspb.Identity_ServiceAccount{ + ServiceAccount: "client@baz.com", + }, + }, + PeerRpcVersions: &altspb.RpcProtocolVersions{ + MaxRpcVersion: &altspb.RpcProtocolVersions_Version{ + Minor: 1, + Major: 2, + }, + MinRpcVersion: &altspb.RpcProtocolVersions_Version{ + Minor: 1, + Major: 2, + }, + }, + }, + Status: &altspb.HandshakerStatus{ + Code: uint32(codes.OK), + }, + }, nil +} diff --git a/credentials/alts/utils.go b/credentials/alts/utils.go new file mode 100644 index 0000000..cbfd056 --- /dev/null +++ b/credentials/alts/utils.go @@ -0,0 +1,70 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package alts + +import ( + "context" + "errors" + "strings" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +// AuthInfoFromContext extracts the alts.AuthInfo object from the given context, +// if it exists. This API should be used by gRPC server RPC handlers to get +// information about the communicating peer. For client-side, use grpc.Peer() +// CallOption. +func AuthInfoFromContext(ctx context.Context) (AuthInfo, error) { + p, ok := peer.FromContext(ctx) + if !ok { + return nil, errors.New("no Peer found in Context") + } + return AuthInfoFromPeer(p) +} + +// AuthInfoFromPeer extracts the alts.AuthInfo object from the given peer, if it +// exists. This API should be used by gRPC clients after obtaining a peer object +// using the grpc.Peer() CallOption. +func AuthInfoFromPeer(p *peer.Peer) (AuthInfo, error) { + altsAuthInfo, ok := p.AuthInfo.(AuthInfo) + if !ok { + return nil, errors.New("no alts.AuthInfo found in Peer") + } + return altsAuthInfo, nil +} + +// ClientAuthorizationCheck checks whether the client is authorized to access +// the requested resources based on the given expected client service accounts. +// This API should be used by gRPC server RPC handlers. This API should not be +// used by clients. +func ClientAuthorizationCheck(ctx context.Context, expectedServiceAccounts []string) error { + authInfo, err := AuthInfoFromContext(ctx) + if err != nil { + return status.Errorf(codes.PermissionDenied, "The context is not an ALTS-compatible context: %v", err) + } + peer := authInfo.PeerServiceAccount() + for _, sa := range expectedServiceAccounts { + if strings.EqualFold(peer, sa) { + return nil + } + } + return status.Errorf(codes.PermissionDenied, "Client %v is not authorized", peer) +} diff --git a/credentials/alts/utils_test.go b/credentials/alts/utils_test.go new file mode 100644 index 0000000..531cdfc --- /dev/null +++ b/credentials/alts/utils_test.go @@ -0,0 +1,167 @@ +//go:build linux || windows +// +build linux windows + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package alts + +import ( + "context" + "strings" + "testing" + "time" + + "google.golang.org/grpc/codes" + altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +const ( + testServiceAccount1 = "service_account1" + testServiceAccount2 = "service_account2" + testServiceAccount3 = "service_account3" + + defaultTestTimeout = 10 * time.Second +) + +func (s) TestAuthInfoFromContext(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + altsAuthInfo := &fakeALTSAuthInfo{} + p := &peer.Peer{ + AuthInfo: altsAuthInfo, + } + for _, tc := range []struct { + desc string + ctx context.Context + success bool + out AuthInfo + }{ + { + "working case", + peer.NewContext(ctx, p), + true, + altsAuthInfo, + }, + } { + authInfo, err := AuthInfoFromContext(tc.ctx) + if got, want := (err == nil), tc.success; got != want { + t.Errorf("%v: AuthInfoFromContext(_)=(err=nil)=%v, want %v", tc.desc, got, want) + } + if got, want := authInfo, tc.out; got != want { + t.Errorf("%v:, AuthInfoFromContext(_)=(%v, _), want (%v, _)", tc.desc, got, want) + } + } +} + +func (s) TestAuthInfoFromPeer(t *testing.T) { + altsAuthInfo := &fakeALTSAuthInfo{} + p := &peer.Peer{ + AuthInfo: altsAuthInfo, + } + for _, tc := range []struct { + desc string + p *peer.Peer + success bool + out AuthInfo + }{ + { + "working case", + p, + true, + altsAuthInfo, + }, + } { + authInfo, err := AuthInfoFromPeer(tc.p) + if got, want := (err == nil), tc.success; got != want { + t.Errorf("%v: AuthInfoFromPeer(_)=(err=nil)=%v, want %v", tc.desc, got, want) + } + if got, want := authInfo, tc.out; got != want { + t.Errorf("%v:, AuthInfoFromPeer(_)=(%v, _), want (%v, _)", tc.desc, got, want) + } + } +} + +func (s) TestClientAuthorizationCheck(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + altsAuthInfo := &fakeALTSAuthInfo{testServiceAccount1} + p := &peer.Peer{ + AuthInfo: altsAuthInfo, + } + for _, tc := range []struct { + desc string + ctx context.Context + expectedServiceAccounts []string + success bool + code codes.Code + }{ + { + "working case", + peer.NewContext(ctx, p), + []string{testServiceAccount1, testServiceAccount2}, + true, + codes.OK, // err is nil, code is OK. + }, + { + "working case (case ignored)", + peer.NewContext(ctx, p), + []string{strings.ToUpper(testServiceAccount1), testServiceAccount2}, + true, + codes.OK, // err is nil, code is OK. + }, + { + "context does not have AuthInfo", + ctx, + []string{testServiceAccount1, testServiceAccount2}, + false, + codes.PermissionDenied, + }, + { + "unauthorized client", + peer.NewContext(ctx, p), + []string{testServiceAccount2, testServiceAccount3}, + false, + codes.PermissionDenied, + }, + } { + err := ClientAuthorizationCheck(tc.ctx, tc.expectedServiceAccounts) + if got, want := (err == nil), tc.success; got != want { + t.Errorf("%v: ClientAuthorizationCheck(_, %v)=(err=nil)=%v, want %v", tc.desc, tc.expectedServiceAccounts, got, want) + } + if got, want := status.Code(err), tc.code; got != want { + t.Errorf("%v: ClientAuthorizationCheck(_, %v).Code=%v, want %v", tc.desc, tc.expectedServiceAccounts, got, want) + } + } +} + +type fakeALTSAuthInfo struct { + peerServiceAccount string +} + +func (*fakeALTSAuthInfo) AuthType() string { return "" } +func (*fakeALTSAuthInfo) ApplicationProtocol() string { return "" } +func (*fakeALTSAuthInfo) RecordProtocol() string { return "" } +func (*fakeALTSAuthInfo) SecurityLevel() altspb.SecurityLevel { + return altspb.SecurityLevel_SECURITY_NONE +} +func (f *fakeALTSAuthInfo) PeerServiceAccount() string { return f.peerServiceAccount } +func (*fakeALTSAuthInfo) LocalServiceAccount() string { return "" } +func (*fakeALTSAuthInfo) PeerRPCVersions() *altspb.RpcProtocolVersions { return nil } diff --git a/credentials/credentials.go b/credentials/credentials.go new file mode 100644 index 0000000..06f6c6c --- /dev/null +++ b/credentials/credentials.go @@ -0,0 +1,337 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package credentials implements various credentials supported by gRPC library, +// which encapsulate all the state needed by a client to authenticate with a +// server and make various assertions, e.g., about the client's identity, role, +// or whether it is authorized to make a particular call. +package credentials // import "google.golang.org/grpc/credentials" + +import ( + "context" + "errors" + "fmt" + "net" + + "google.golang.org/grpc/attributes" + icredentials "google.golang.org/grpc/internal/credentials" + "google.golang.org/protobuf/proto" +) + +// PerRPCCredentials defines the common interface for the credentials which need to +// attach security information to every RPC (e.g., oauth2). +type PerRPCCredentials interface { + // GetRequestMetadata gets the current request metadata, refreshing tokens + // if required. This should be called by the transport layer on each + // request, and the data should be populated in headers or other + // context. If a status code is returned, it will be used as the status for + // the RPC (restricted to an allowable set of codes as defined by gRFC + // A54). uri is the URI of the entry point for the request. When supported + // by the underlying implementation, ctx can be used for timeout and + // cancellation. Additionally, RequestInfo data will be available via ctx + // to this call. + GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) + // RequireTransportSecurity indicates whether the credentials requires + // transport security. + RequireTransportSecurity() bool +} + +// SecurityLevel defines the protection level on an established connection. +// +// This API is experimental. +type SecurityLevel int + +const ( + // InvalidSecurityLevel indicates an invalid security level. + // The zero SecurityLevel value is invalid for backward compatibility. + InvalidSecurityLevel SecurityLevel = iota + // NoSecurity indicates a connection is insecure. + NoSecurity + // IntegrityOnly indicates a connection only provides integrity protection. + IntegrityOnly + // PrivacyAndIntegrity indicates a connection provides both privacy and integrity protection. + PrivacyAndIntegrity +) + +// String returns SecurityLevel in a string format. +func (s SecurityLevel) String() string { + switch s { + case NoSecurity: + return "NoSecurity" + case IntegrityOnly: + return "IntegrityOnly" + case PrivacyAndIntegrity: + return "PrivacyAndIntegrity" + } + return fmt.Sprintf("invalid SecurityLevel: %v", int(s)) +} + +// CommonAuthInfo contains authenticated information common to AuthInfo implementations. +// It should be embedded in a struct implementing AuthInfo to provide additional information +// about the credentials. +// +// This API is experimental. +type CommonAuthInfo struct { + SecurityLevel SecurityLevel +} + +// GetCommonAuthInfo returns the pointer to CommonAuthInfo struct. +func (c CommonAuthInfo) GetCommonAuthInfo() CommonAuthInfo { + return c +} + +// ProtocolInfo provides static information regarding transport credentials. +type ProtocolInfo struct { + // ProtocolVersion is the gRPC wire protocol version. + // + // Deprecated: this is unused by gRPC. + ProtocolVersion string + // SecurityProtocol is the security protocol in use. + SecurityProtocol string + // SecurityVersion is the security protocol version. It is a static version string from the + // credentials, not a value that reflects per-connection protocol negotiation. To retrieve + // details about the credentials used for a connection, use the Peer's AuthInfo field instead. + // + // Deprecated: please use Peer.AuthInfo. + SecurityVersion string + // ServerName is the user-configured server name. If set, this overrides + // the default :authority header used for all RPCs on the channel using the + // containing credentials, unless grpc.WithAuthority is set on the channel, + // in which case that setting will take precedence. + // + // This must be a valid `:authority` header according to + // [RFC3986](https://datatracker.ietf.org/doc/html/rfc3986#section-3.2). + // + // Deprecated: Users should use grpc.WithAuthority to override the authority + // on a channel instead of configuring the credentials. + ServerName string +} + +// AuthInfo defines the common interface for the auth information the users are interested in. +// A struct that implements AuthInfo should embed CommonAuthInfo by including additional +// information about the credentials in it. +type AuthInfo interface { + AuthType() string +} + +// AuthorityValidator validates the authority used to override the `:authority` +// header. This is an optional interface that implementations of AuthInfo can +// implement if they support per-RPC authority overrides. It is invoked when the +// application attempts to override the HTTP/2 `:authority` header using the +// CallAuthority call option. +type AuthorityValidator interface { + // ValidateAuthority checks the authority value used to override the + // `:authority` header. The authority parameter is the override value + // provided by the application via the CallAuthority option. This value + // typically corresponds to the server hostname or endpoint the RPC is + // targeting. It returns non-nil error if the validation fails. + ValidateAuthority(authority string) error +} + +// ErrConnDispatched indicates that rawConn has been dispatched out of gRPC +// and the caller should not close rawConn. +var ErrConnDispatched = errors.New("credentials: rawConn is dispatched out of gRPC") + +// TransportCredentials defines the common interface for all the live gRPC wire +// protocols and supported transport security protocols (e.g., TLS, SSL). +type TransportCredentials interface { + // ClientHandshake does the authentication handshake specified by the + // corresponding authentication protocol on rawConn for clients. It returns + // the authenticated connection and the corresponding auth information + // about the connection. The auth information should embed CommonAuthInfo + // to return additional information about the credentials. Implementations + // must use the provided context to implement timely cancellation. gRPC + // will try to reconnect if the error returned is a temporary error + // (io.EOF, context.DeadlineExceeded or err.Temporary() == true). If the + // returned error is a wrapper error, implementations should make sure that + // the error implements Temporary() to have the correct retry behaviors. + // Additionally, ClientHandshakeInfo data will be available via the context + // passed to this call. + // + // The second argument to this method is the `:authority` header value used + // while creating new streams on this connection after authentication + // succeeds. Implementations must use this as the server name during the + // authentication handshake. + // + // If the returned net.Conn is closed, it MUST close the net.Conn provided. + ClientHandshake(context.Context, string, net.Conn) (net.Conn, AuthInfo, error) + // ServerHandshake does the authentication handshake for servers. It returns + // the authenticated connection and the corresponding auth information about + // the connection. The auth information should embed CommonAuthInfo to return additional information + // about the credentials. + // + // If the returned net.Conn is closed, it MUST close the net.Conn provided. + ServerHandshake(net.Conn) (net.Conn, AuthInfo, error) + // Info provides the ProtocolInfo of this TransportCredentials. + Info() ProtocolInfo + // Clone makes a copy of this TransportCredentials. + Clone() TransportCredentials + // OverrideServerName specifies the value used for the following: + // + // - verifying the hostname on the returned certificates + // - as SNI in the client's handshake to support virtual hosting + // - as the value for `:authority` header at stream creation time + // + // The provided string should be a valid `:authority` header according to + // [RFC3986](https://datatracker.ietf.org/doc/html/rfc3986#section-3.2). + // + // Deprecated: this method is unused by gRPC. Users should use + // grpc.WithAuthority to override the authority on a channel instead of + // configuring the credentials. + OverrideServerName(string) error +} + +// Bundle is a combination of TransportCredentials and PerRPCCredentials. +// +// It also contains a mode switching method, so it can be used as a combination +// of different credential policies. +// +// Bundle cannot be used together with individual TransportCredentials. +// PerRPCCredentials from Bundle will be appended to other PerRPCCredentials. +// +// This API is experimental. +type Bundle interface { + // TransportCredentials returns the transport credentials from the Bundle. + // + // Implementations must return non-nil transport credentials. If transport + // security is not needed by the Bundle, implementations may choose to + // return insecure.NewCredentials(). + TransportCredentials() TransportCredentials + + // PerRPCCredentials returns the per-RPC credentials from the Bundle. + // + // May be nil if per-RPC credentials are not needed. + PerRPCCredentials() PerRPCCredentials + + // NewWithMode should make a copy of Bundle, and switch mode. Modifying the + // existing Bundle may cause races. + // + // NewWithMode returns nil if the requested mode is not supported. + NewWithMode(mode string) (Bundle, error) +} + +// RequestInfo contains request data attached to the context passed to GetRequestMetadata calls. +// +// This API is experimental. +type RequestInfo struct { + // The method passed to Invoke or NewStream for this RPC. (For proto methods, this has the format "/some.Service/Method") + Method string + // AuthInfo contains the information from a security handshake (TransportCredentials.ClientHandshake, TransportCredentials.ServerHandshake) + AuthInfo AuthInfo +} + +// requestInfoKey is a struct to be used as the key to store RequestInfo in a +// context. +type requestInfoKey struct{} + +// RequestInfoFromContext extracts the RequestInfo from the context if it exists. +// +// This API is experimental. +func RequestInfoFromContext(ctx context.Context) (ri RequestInfo, ok bool) { + ri, ok = ctx.Value(requestInfoKey{}).(RequestInfo) + return ri, ok +} + +// NewContextWithRequestInfo creates a new context from ctx and attaches ri to it. +// +// This RequestInfo will be accessible via RequestInfoFromContext. +// +// Intended to be used from tests for PerRPCCredentials implementations (that +// often need to check connection's SecurityLevel). Should not be used from +// non-test code: the gRPC client already prepares a context with the correct +// RequestInfo attached when calling PerRPCCredentials.GetRequestMetadata. +// +// This API is experimental. +func NewContextWithRequestInfo(ctx context.Context, ri RequestInfo) context.Context { + return context.WithValue(ctx, requestInfoKey{}, ri) +} + +// ClientHandshakeInfo holds data to be passed to ClientHandshake. This makes +// it possible to pass arbitrary data to the handshaker from gRPC, resolver, +// balancer etc. Individual credential implementations control the actual +// format of the data that they are willing to receive. +// +// This API is experimental. +type ClientHandshakeInfo struct { + // Attributes contains the attributes for the address. It could be provided + // by the gRPC, resolver, balancer etc. + Attributes *attributes.Attributes +} + +// ClientHandshakeInfoFromContext returns the ClientHandshakeInfo struct stored +// in ctx. +// +// This API is experimental. +func ClientHandshakeInfoFromContext(ctx context.Context) ClientHandshakeInfo { + chi, _ := icredentials.ClientHandshakeInfoFromContext(ctx).(ClientHandshakeInfo) + return chi +} + +// CheckSecurityLevel checks if a connection's security level is greater than or equal to the specified one. +// It returns success if 1) the condition is satisfied or 2) AuthInfo struct does not implement GetCommonAuthInfo() method +// or 3) CommonAuthInfo.SecurityLevel has an invalid zero value. For 2) and 3), it is for the purpose of backward-compatibility. +// +// This API is experimental. +func CheckSecurityLevel(ai AuthInfo, level SecurityLevel) error { + type internalInfo interface { + GetCommonAuthInfo() CommonAuthInfo + } + if ai == nil { + return errors.New("AuthInfo is nil") + } + if ci, ok := ai.(internalInfo); ok { + // CommonAuthInfo.SecurityLevel has an invalid value. + if ci.GetCommonAuthInfo().SecurityLevel == InvalidSecurityLevel { + return nil + } + if ci.GetCommonAuthInfo().SecurityLevel < level { + return fmt.Errorf("requires SecurityLevel %v; connection has %v", level, ci.GetCommonAuthInfo().SecurityLevel) + } + } + // The condition is satisfied or AuthInfo struct does not implement GetCommonAuthInfo() method. + return nil +} + +// ChannelzSecurityInfo defines the interface that security protocols should implement +// in order to provide security info to channelz. +// +// This API is experimental. +type ChannelzSecurityInfo interface { + GetSecurityValue() ChannelzSecurityValue +} + +// ChannelzSecurityValue defines the interface that GetSecurityValue() return value +// should satisfy. This interface should only be satisfied by *TLSChannelzSecurityValue +// and *OtherChannelzSecurityValue. +// +// This API is experimental. +type ChannelzSecurityValue interface { + isChannelzSecurityValue() +} + +// OtherChannelzSecurityValue defines the struct that non-TLS protocol should return +// from GetSecurityValue(), which contains protocol specific security info. Note +// the Value field will be sent to users of channelz requesting channel info, and +// thus sensitive info should better be avoided. +// +// This API is experimental. +type OtherChannelzSecurityValue struct { + ChannelzSecurityValue + Name string + Value proto.Message +} diff --git a/credentials/credentials_ext_test.go b/credentials/credentials_ext_test.go new file mode 100644 index 0000000..0cc622a --- /dev/null +++ b/credentials/credentials_ext_test.go @@ -0,0 +1,538 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials_test + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "math/big" + "net" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/local" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func authorityChecker(ctx context.Context, wantAuthority string) error { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return status.Error(codes.InvalidArgument, "failed to parse metadata") + } + auths, ok := md[":authority"] + if !ok { + return status.Error(codes.InvalidArgument, "no authority header") + } + if len(auths) != 1 { + return status.Errorf(codes.InvalidArgument, "expected exactly one authority header, got %v", auths) + } + if auths[0] != wantAuthority { + return status.Errorf(codes.InvalidArgument, "invalid authority header %q, want %q", auths[0], wantAuthority) + } + return nil +} + +func loadTLSCreds(t *testing.T) (grpc.ServerOption, grpc.DialOption) { + t.Helper() + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("Failed to load key pair: %v", err) + return nil, nil + } + serverCreds := grpc.Creds(credentials.NewServerTLSFromCert(&cert)) + + clientCreds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") + if err != nil { + t.Fatalf("Failed to create client credentials: %v", err) + } + return serverCreds, grpc.WithTransportCredentials(clientCreds) +} + +// Tests the scenario where the `grpc.CallAuthority` call option is used with +// different transport credentials. The test verifies that the specified +// authority is correctly propagated to the serve when a correct authority is +// used. +func (s) TestCorrectAuthorityWithCreds(t *testing.T) { + const authority = "auth.test.example.com" + const authorityWithPort = "auth.test.example.com:8010" + + tests := []struct { + name string + creds func(t *testing.T) (grpc.ServerOption, grpc.DialOption) + expectedAuth string + }{ + { + name: "Insecure", + creds: func(*testing.T) (grpc.ServerOption, grpc.DialOption) { + c := insecure.NewCredentials() + return grpc.Creds(c), grpc.WithTransportCredentials(c) + }, + expectedAuth: authority, + }, + { + name: "Local", + creds: func(*testing.T) (grpc.ServerOption, grpc.DialOption) { + c := local.NewCredentials() + return grpc.Creds(c), grpc.WithTransportCredentials(c) + }, + expectedAuth: authority, + }, + { + name: "TLS", + creds: func(t *testing.T) (grpc.ServerOption, grpc.DialOption) { + return loadTLSCreds(t) + }, + expectedAuth: authority, + }, + { + name: "TLSAuthorityWithPort", + creds: func(t *testing.T) (grpc.ServerOption, grpc.DialOption) { + return loadTLSCreds(t) + }, + expectedAuth: authorityWithPort, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + if err := authorityChecker(ctx, tt.expectedAuth); err != nil { + return nil, err + } + return &testpb.Empty{}, nil + }, + } + serverOpt, dialOpt := tt.creds(t) + if err := ss.StartServer(serverOpt); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + cc, err := grpc.NewClient(ss.Address, dialOpt) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", ss.Address, err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}, grpc.CallAuthority(tt.expectedAuth)); err != nil { + t.Fatalf("EmptyCall() rpc failed: %v", err) + } + }) + } +} + +// Tests the `grpc.CallAuthority` option with TLS credentials. This test verifies +// that the RPC fails with `UNAVAILABLE` status code and doesn't reach the server +// when an incorrect authority is used. +func (s) TestIncorrectAuthorityWithTLS(t *testing.T) { + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("Failed to load key pair: %s", err) + } + creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") + if err != nil { + t.Fatalf("Failed to create credentials %v", err) + } + + tests := []struct { + name string + authority string + }{ + { + name: "IncorrectAuthority", + authority: "auth.example.com", + }, + { + name: "IncorrectAuthorityWithPort", + authority: "auth.example.com:8443", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + serverCalled := make(chan struct{}) + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + close(serverCalled) + return nil, nil + }, + } + if err := ss.StartServer(grpc.Creds(credentials.NewServerTLSFromCert(&cert))); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + cc, err := grpc.NewClient(ss.Address, grpc.WithTransportCredentials(creds)) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", ss.Address, err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}, grpc.CallAuthority(tt.authority)); status.Code(err) != codes.Unavailable { + t.Fatalf("EmptyCall() returned status %v, want %v", status.Code(err), codes.Unavailable) + } + select { + case <-serverCalled: + t.Fatalf("Server handler should not have been called") + case <-time.After(defaultTestShortTimeout): + } + }) + } +} + +// testAuthInfoNoValidator implements only credentials.AuthInfo and not +// credentials.AuthorityValidator. +type testAuthInfoNoValidator struct{} + +// AuthType returns the authentication type. +func (testAuthInfoNoValidator) AuthType() string { + return "test" +} + +// testAuthInfoWithValidator implements both credentials.AuthInfo and +// credentials.AuthorityValidator. +type testAuthInfoWithValidator struct { + validAuthority string +} + +// AuthType returns the authentication type. +func (testAuthInfoWithValidator) AuthType() string { + return "test" +} + +// ValidateAuthority implements credentials.AuthorityValidator. +func (v testAuthInfoWithValidator) ValidateAuthority(authority string) error { + if authority == v.validAuthority { + return nil + } + return fmt.Errorf("invalid authority %q, want %q", authority, v.validAuthority) +} + +// testCreds is a test TransportCredentials that can optionally support +// authority validation. +type testCreds struct { + authority string +} + +// ClientHandshake performs the client-side handshake. +func (c *testCreds) ClientHandshake(_ context.Context, _ string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + if c.authority != "" { + return rawConn, testAuthInfoWithValidator{validAuthority: c.authority}, nil + } + return rawConn, testAuthInfoNoValidator{}, nil +} + +// ServerHandshake performs the server-side handshake. +func (c *testCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + if c.authority != "" { + return rawConn, testAuthInfoWithValidator{validAuthority: c.authority}, nil + } + return rawConn, testAuthInfoNoValidator{}, nil +} + +// Clone creates a copy of testCreds. +func (c *testCreds) Clone() credentials.TransportCredentials { + return &testCreds{authority: c.authority} +} + +// Info provides protocol information. +func (c *testCreds) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{} +} + +// OverrideServerName overrides the server name used for verification. +func (c *testCreds) OverrideServerName(string) error { + return nil +} + +// TestAuthorityValidationFailureWithCustomCreds tests the `grpc.CallAuthority` +// call option using custom credentials. It covers two failure scenarios: +// - The credentials implement AuthorityValidator but authority used to override +// is not valid. +// - The credentials do not implement AuthorityValidator, but an authority +// override is specified. +// In both cases, the RPC is expected to fail with an `UNAVAILABLE` status code. +func (s) TestAuthorityValidationFailureWithCustomCreds(t *testing.T) { + tests := []struct { + name string + creds credentials.TransportCredentials + authority string + }{ + { + name: "IncorrectAuthorityWithFakeCreds", + authority: "auth.example.com", + creds: &testCreds{authority: "auth.test.example.com"}, + }, + { + name: "FakeCredsWithNoAuthValidator", + creds: &testCreds{}, + authority: "auth.test.example.com", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + serverCalled := make(chan struct{}) + ss := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + close(serverCalled) + return nil, nil + }, + } + if err := ss.StartServer(); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + cc, err := grpc.NewClient(ss.Address, grpc.WithTransportCredentials(tt.creds)) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", ss.Address, err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}, grpc.CallAuthority(tt.authority)); status.Code(err) != codes.Unavailable { + t.Fatalf("EmptyCall() returned status %v, want %v", status.Code(err), codes.Unavailable) + } + select { + case <-serverCalled: + t.Fatalf("Server should not have been called") + case <-time.After(defaultTestShortTimeout): + } + }) + } + +} + +// TestCorrectAuthorityWithCustomCreds tests the `grpc.CallAuthority` call +// option using custom credentials. It verifies that the provided authority is +// correctly propagated to the server when a correct authority is used. +func (s) TestCorrectAuthorityWithCustomCreds(t *testing.T) { + const authority = "auth.test.example.com" + creds := &testCreds{authority: "auth.test.example.com"} + ss := stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + if err := authorityChecker(ctx, authority); err != nil { + return nil, err + } + return &testpb.Empty{}, nil + }, + } + if err := ss.StartServer(); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + cc, err := grpc.NewClient(ss.Address, grpc.WithTransportCredentials(creds)) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", ss.Address, err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}, grpc.CallAuthority(authority)); status.Code(err) != codes.OK { + t.Fatalf("EmptyCall() returned status %v, want %v", status.Code(err), codes.OK) + } +} + +// TestAuthorityOverrideWithCertChain tests that the authority being used to +// override per-RPC authority is validated against the leaf certificate only +// and not against the intermediate certificates. +func (s) TestAuthorityOverrideWithCertChain(t *testing.T) { + rootCert, certChain, leafKey := generateCertChain(t, "root.example.com", "intermediate.example.com", "*.leaf.example.com") + + // Construct server credentials from leaf and intermediate certificates. + serverCert := tls.Certificate{ + Certificate: [][]byte{certChain[0].Raw, certChain[1].Raw}, + PrivateKey: leafKey, + } + serverCreds := credentials.NewServerTLSFromCert(&serverCert) + + // Create client credentials trusting the Root CA. + certPool := x509.NewCertPool() + certPool.AddCert(rootCert) + clientCreds := credentials.NewTLS(&tls.Config{ + RootCAs: certPool, + ServerName: "test1.leaf.example.com", + }) + + tests := []struct { + name string + authority string + wantCode codes.Code + wantErr string + }{ + { + name: "AuthorityMatchesIntermediate", + authority: "intermediate.example.com", + wantCode: codes.Unavailable, + wantErr: "failed to validate authority", + }, + { + name: "AuthorityMatchesLeaf", + authority: "test2.leaf.example.com", + wantCode: codes.OK, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup and start the stub server. + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + if err := authorityChecker(ctx, tt.authority); err != nil { + return nil, err + } + return &testpb.Empty{}, nil + }, + } + if err := ss.StartServer(grpc.Creds(serverCreds)); err != nil { + t.Fatalf("failed to start server: %v", err) + } + defer ss.Stop() + + cc, err := grpc.NewClient(ss.Address, grpc.WithTransportCredentials(clientCreds)) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", ss.Address, err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + _, err = testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}, grpc.CallAuthority(tt.authority)) + if got := status.Code(err); got != tt.wantCode { + t.Fatalf("EmptyCall() with authority %q: got code %v, want %v", tt.authority, got, tt.wantCode) + } + if tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)) { + t.Fatalf("EmptyCall() with authority %q: expected error to contain %q, got %v", tt.authority, tt.wantErr, err) + } + }) + } +} + +// certConfig defines the configuration for generating a certificate. +type certConfig struct { + commonName string + dnsNames []string + isCA bool + serial int64 + parentCert *x509.Certificate + parentKey *rsa.PrivateKey +} + +// createCertificate generates a certificate based on the provided certConfig. +// It creates self-signed certificates if parentCert is nil otherwise it creates +// certificates signed by a parent certificate. +func createCertificate(t *testing.T, cfg certConfig) (*x509.Certificate, *rsa.PrivateKey) { + t.Helper() + + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + + now := time.Now() + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(cfg.serial), + Subject: pkix.Name{CommonName: cfg.commonName}, + DNSNames: cfg.dnsNames, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(time.Hour), + BasicConstraintsValid: true, + IsCA: cfg.isCA, + } + + // If no parent is provided, the certificate is self-signed + signingCert := cfg.parentCert + signingKey := cfg.parentKey + if signingCert == nil { + signingCert = tmpl + signingKey = key + } + + der, err := x509.CreateCertificate(rand.Reader, tmpl, signingCert, key.Public(), signingKey) + if err != nil { + t.Fatal(err) + } + + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + + return cert, key +} + +// generateCertChain creates a 3 certificate chain (Root -> Intermediate -> +// Leaf). It returns the root certificate, a slice containing the leaf and +// intermediate certificates in the order [leaf, intermediate], and the private +// key for the leaf certificate. +func generateCertChain(t *testing.T, rootName, interName, leafName string) (root *x509.Certificate, chain []*x509.Certificate, leafKey *rsa.PrivateKey) { + t.Helper() + + rootCfg := certConfig{ + commonName: rootName, + isCA: true, + } + root, rootKey := createCertificate(t, rootCfg) + + interCfg := certConfig{ + commonName: interName, + dnsNames: []string{interName}, + isCA: true, + serial: 2, + parentCert: root, + parentKey: rootKey, + } + intermediate, interKey := createCertificate(t, interCfg) + + leafCfg := certConfig{ + commonName: leafName, + dnsNames: []string{leafName}, + isCA: false, + serial: 3, + parentCert: intermediate, + parentKey: interKey, + } + leaf, leafKey := createCertificate(t, leafCfg) + + return root, []*x509.Certificate{leaf, intermediate}, leafKey +} diff --git a/credentials/credentials_test.go b/credentials/credentials_test.go new file mode 100644 index 0000000..8dc30ad --- /dev/null +++ b/credentials/credentials_test.go @@ -0,0 +1,322 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials + +import ( + "context" + "crypto/tls" + "net" + "strings" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/testdata" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// A struct that implements AuthInfo interface but does not implement GetCommonAuthInfo() method. +type testAuthInfoNoGetCommonAuthInfoMethod struct{} + +func (ta testAuthInfoNoGetCommonAuthInfoMethod) AuthType() string { + return "testAuthInfoNoGetCommonAuthInfoMethod" +} + +// A struct that implements AuthInfo interface and implements CommonAuthInfo() method. +type testAuthInfo struct { + CommonAuthInfo +} + +func (ta testAuthInfo) AuthType() string { + return "testAuthInfo" +} + +func (s) TestCheckSecurityLevel(t *testing.T) { + testCases := []struct { + authLevel SecurityLevel + testLevel SecurityLevel + want bool + }{ + { + authLevel: PrivacyAndIntegrity, + testLevel: PrivacyAndIntegrity, + want: true, + }, + { + authLevel: IntegrityOnly, + testLevel: PrivacyAndIntegrity, + want: false, + }, + { + authLevel: IntegrityOnly, + testLevel: NoSecurity, + want: true, + }, + { + authLevel: InvalidSecurityLevel, + testLevel: IntegrityOnly, + want: true, + }, + { + authLevel: InvalidSecurityLevel, + testLevel: PrivacyAndIntegrity, + want: true, + }, + } + for _, tc := range testCases { + err := CheckSecurityLevel(testAuthInfo{CommonAuthInfo: CommonAuthInfo{SecurityLevel: tc.authLevel}}, tc.testLevel) + if tc.want && (err != nil) { + t.Fatalf("CheckSeurityLevel(%s, %s) returned failure but want success", tc.authLevel.String(), tc.testLevel.String()) + } else if !tc.want && (err == nil) { + t.Fatalf("CheckSeurityLevel(%s, %s) returned success but want failure", tc.authLevel.String(), tc.testLevel.String()) + + } + } +} + +func (s) TestCheckSecurityLevelNoGetCommonAuthInfoMethod(t *testing.T) { + if err := CheckSecurityLevel(testAuthInfoNoGetCommonAuthInfoMethod{}, PrivacyAndIntegrity); err != nil { + t.Fatalf("CheckSeurityLevel() returned failure but want success") + } +} + +func (s) TestTLSOverrideServerName(t *testing.T) { + expectedServerName := "server.name" + c := NewTLS(nil) + c.OverrideServerName(expectedServerName) + if c.Info().ServerName != expectedServerName { + t.Fatalf("c.Info().ServerName = %v, want %v", c.Info().ServerName, expectedServerName) + } +} + +func (s) TestTLSClone(t *testing.T) { + expectedServerName := "server.name" + c := NewTLS(nil) + c.OverrideServerName(expectedServerName) + cc := c.Clone() + if cc.Info().ServerName != expectedServerName { + t.Fatalf("cc.Info().ServerName = %v, want %v", cc.Info().ServerName, expectedServerName) + } + cc.OverrideServerName("") + if c.Info().ServerName != expectedServerName { + t.Fatalf("Change in clone should not affect the original, c.Info().ServerName = %v, want %v", c.Info().ServerName, expectedServerName) + } + +} + +type serverHandshake func(net.Conn) (AuthInfo, error) + +func (s) TestClientHandshakeReturnsAuthInfo(t *testing.T) { + tcs := []struct { + name string + address string + }{ + { + name: "localhost", + address: "localhost:0", + }, + { + name: "ipv4", + address: "127.0.0.1:0", + }, + { + name: "ipv6", + address: "[::1]:0", + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + done := make(chan AuthInfo, 1) + lis := launchServerOnListenAddress(t, tlsServerHandshake, done, tc.address) + defer lis.Close() + lisAddr := lis.Addr().String() + clientAuthInfo := clientHandle(t, gRPCClientHandshake, lisAddr) + // wait until server sends serverAuthInfo or fails. + serverAuthInfo, ok := <-done + if !ok { + t.Fatalf("Error at server-side") + } + if !compare(clientAuthInfo, serverAuthInfo) { + t.Fatalf("c.ClientHandshake(_, %v, _) = %v, want %v.", lisAddr, clientAuthInfo, serverAuthInfo) + } + }) + } +} + +func (s) TestServerHandshakeReturnsAuthInfo(t *testing.T) { + done := make(chan AuthInfo, 1) + lis := launchServer(t, gRPCServerHandshake, done) + defer lis.Close() + clientAuthInfo := clientHandle(t, tlsClientHandshake, lis.Addr().String()) + // wait until server sends serverAuthInfo or fails. + serverAuthInfo, ok := <-done + if !ok { + t.Fatalf("Error at server-side") + } + if !compare(clientAuthInfo, serverAuthInfo) { + t.Fatalf("ServerHandshake(_) = %v, want %v.", serverAuthInfo, clientAuthInfo) + } +} + +func (s) TestServerAndClientHandshake(t *testing.T) { + done := make(chan AuthInfo, 1) + lis := launchServer(t, gRPCServerHandshake, done) + defer lis.Close() + clientAuthInfo := clientHandle(t, gRPCClientHandshake, lis.Addr().String()) + // wait until server sends serverAuthInfo or fails. + serverAuthInfo, ok := <-done + if !ok { + t.Fatalf("Error at server-side") + } + if !compare(clientAuthInfo, serverAuthInfo) { + t.Fatalf("AuthInfo returned by server: %v and client: %v aren't same", serverAuthInfo, clientAuthInfo) + } +} + +func compare(a1, a2 AuthInfo) bool { + if a1.AuthType() != a2.AuthType() { + return false + } + switch a1.AuthType() { + case "tls": + state1 := a1.(TLSInfo).State + state2 := a2.(TLSInfo).State + if state1.Version == state2.Version && + state1.HandshakeComplete == state2.HandshakeComplete && + state1.CipherSuite == state2.CipherSuite && + state1.NegotiatedProtocol == state2.NegotiatedProtocol { + return true + } + return false + default: + return false + } +} + +func launchServer(t *testing.T, hs serverHandshake, done chan AuthInfo) net.Listener { + return launchServerOnListenAddress(t, hs, done, "localhost:0") +} + +func launchServerOnListenAddress(t *testing.T, hs serverHandshake, done chan AuthInfo, address string) net.Listener { + lis, err := net.Listen("tcp", address) + if err != nil { + if strings.Contains(err.Error(), "bind: cannot assign requested address") || + strings.Contains(err.Error(), "socket: address family not supported by protocol") { + t.Skipf("no support for address %v", address) + } + t.Fatalf("Failed to listen: %v", err) + } + go serverHandle(t, hs, done, lis) + return lis +} + +// Is run in a separate goroutine. +func serverHandle(t *testing.T, hs serverHandshake, done chan AuthInfo, lis net.Listener) { + serverRawConn, err := lis.Accept() + if err != nil { + t.Errorf("Server failed to accept connection: %v", err) + close(done) + return + } + serverAuthInfo, err := hs(serverRawConn) + if err != nil { + t.Errorf("Server failed while handshake. Error: %v", err) + serverRawConn.Close() + close(done) + return + } + done <- serverAuthInfo +} + +func clientHandle(t *testing.T, hs func(net.Conn, string) (AuthInfo, error), lisAddr string) AuthInfo { + conn, err := net.Dial("tcp", lisAddr) + if err != nil { + t.Fatalf("Client failed to connect to %s. Error: %v", lisAddr, err) + } + defer conn.Close() + clientAuthInfo, err := hs(conn, lisAddr) + if err != nil { + t.Fatalf("Error on client while handshake. Error: %v", err) + } + return clientAuthInfo +} + +// Server handshake implementation in gRPC. +func gRPCServerHandshake(conn net.Conn) (AuthInfo, error) { + serverTLS, err := NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + return nil, err + } + _, serverAuthInfo, err := serverTLS.ServerHandshake(conn) + if err != nil { + return nil, err + } + return serverAuthInfo, nil +} + +// Client handshake implementation in gRPC. +func gRPCClientHandshake(conn net.Conn, lisAddr string) (AuthInfo, error) { + clientTLS := NewTLS(&tls.Config{InsecureSkipVerify: true}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, authInfo, err := clientTLS.ClientHandshake(ctx, lisAddr, conn) + if err != nil { + return nil, err + } + return authInfo, nil +} + +func tlsServerHandshake(conn net.Conn) (AuthInfo, error) { + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + return nil, err + } + serverTLSConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + NextProtos: []string{"h2"}, + } + serverConn := tls.Server(conn, serverTLSConfig) + err = serverConn.Handshake() + if err != nil { + return nil, err + } + return TLSInfo{State: serverConn.ConnectionState(), CommonAuthInfo: CommonAuthInfo{SecurityLevel: PrivacyAndIntegrity}}, nil +} + +func tlsClientHandshake(conn net.Conn, _ string) (AuthInfo, error) { + clientTLSConfig := &tls.Config{ + InsecureSkipVerify: true, // NOLINT + NextProtos: []string{"h2"}, + } + clientConn := tls.Client(conn, clientTLSConfig) + if err := clientConn.Handshake(); err != nil { + return nil, err + } + return TLSInfo{State: clientConn.ConnectionState(), CommonAuthInfo: CommonAuthInfo{SecurityLevel: PrivacyAndIntegrity}}, nil +} diff --git a/credentials/google/gcp_service_account_identity_credentials.go b/credentials/google/gcp_service_account_identity_credentials.go new file mode 100644 index 0000000..160ec81 --- /dev/null +++ b/credentials/google/gcp_service_account_identity_credentials.go @@ -0,0 +1,278 @@ +/* +* +* Copyright 2026 gRPC authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package google + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "cloud.google.com/go/auth" + "cloud.google.com/go/auth/credentials/idtoken" + "cloud.google.com/go/compute/metadata" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/google/internal" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/status" +) + +const ( + // preemptiveRefresh is the window before a token's actual expiration during + // which the token is considered stale. Requests using a stale but still + // valid token will trigger a background asynchronous refresh. This avoids + // blocking the current RPC, preventing periodic latency spikes during token + // refresh. + preemptiveRefresh = 1 * time.Minute + + // metadataTimeout is the timeout for the call to the metadata server to + // prevent the background token fetch from hanging indefinitely in case + // of connection issues. + metadataTimeout = 60 * time.Second +) + +type gcpServiceAccountIdentityCallCreds struct { + // The following fields are initialized at creation time and are read-only + // after that. + ctx context.Context + audience string + creds *auth.Credentials + backoff backoff.Strategy + + // The following fields are protected by mu. + mu sync.Mutex + token *auth.Token + tokenExpiry time.Time // timestamp after which the cached token is considered invalid + preemptiveTokenRefresh time.Time // timestamp after which background preemptive refresh is triggered + fetching chan struct{} // used to ensure a single in-progress background token fetch + nextRetryTime time.Time // timestamp after which we can attempt the next token fetch + retryAttempt int // consecutive fetch failure count used to compute backoff delay + lastErr error // cached error returned from the most recent token fetch attempt +} + +func init() { + internal.BackoffStrategy = backoff.DefaultExponential + internal.NewIDTokenCredentials = func(opts *idtoken.Options) (*auth.Credentials, error) { + return idtoken.NewCredentials(opts) + } +} + +// NewServiceAccountIdentityCredentials creates a PerRPCCredentials that +// authenticates using a GCP Service Account Identity JWT token for the given +// audience. +// +// This credential fetches the ID token from the GCE metadata server and is +// only valid for use in environments running on GCP. The ctx and audience +// parameters cannot be empty. +// +// The credentials object starts asynchronous background token fetches to +// refresh expired tokens. The provided context propagates cancellation to +// these background tasks. Users should not pass an RPC-scoped context here, +// but rather a context that is valid for the entire lifetime of the +// credentials and should cancel the context when they are done. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func NewServiceAccountIdentityCredentials(ctx context.Context, audience string) (credentials.PerRPCCredentials, error) { + if ctx == nil { + return nil, fmt.Errorf("credentials: ctx cannot be nil") + } + + if audience == "" { + return nil, fmt.Errorf("credentials: audience cannot be empty") + } + + creds, err := internal.NewIDTokenCredentials(&idtoken.Options{Audience: audience}) + if err != nil { + return nil, fmt.Errorf("credentials: failed to create ID token credentials: %v", err) + } + + return &gcpServiceAccountIdentityCallCreds{ + ctx: ctx, + audience: audience, + creds: creds, + backoff: internal.BackoffStrategy, + }, nil +} + +// GetRequestMetadata gets the current request metadata, refreshing tokens if +// required. This implementation follows the PerRPCCredentials interface. +// +// It guarantees that only one underlying token fetch will be executed +// concurrently. If a valid token is cached, it is returned immediately. If +// a fetch recently failed, the cached error is returned until the backoff +// interval expires. Otherwise, it initiates a new token fetch or blocks +// waiting for an already-in-progress fetch to complete. +func (c *gcpServiceAccountIdentityCallCreds) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) { + ri, _ := credentials.RequestInfoFromContext(ctx) + if err := credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil { + return nil, fmt.Errorf("credentials: cannot send secure credentials on an insecure connection: %v", err) + } + + if md, err := c.cachedRequestMetadata(true); md != nil || err != nil { + return md, err + } + + c.mu.Lock() + // Now that we have the lock, did someone else finish the fetch while we + // were waiting for the lock? + md, err := c.cachedRequestMetadataLocked(false) + if md != nil || err != nil { + c.mu.Unlock() + return md, err + } + + // If no one is fetching, start it. + if c.fetching == nil { + c.fetching = make(chan struct{}) + go c.startFetch() + } + wait := c.fetching + c.mu.Unlock() + + select { + case <-wait: + // Return the new cached result from the most recent fetch. + md, err := c.cachedRequestMetadata(false) + if md == nil && err == nil { + return nil, status.Error(codes.Unavailable, "credentials: fetched token is expired") + } + return md, err + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// cachedRequestMetadata handles its own locking for the "fast path". +func (c *gcpServiceAccountIdentityCallCreds) cachedRequestMetadata(attemptPreemptiveRefresh bool) (map[string]string, error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.cachedRequestMetadataLocked(attemptPreemptiveRefresh) +} + +// cachedRequestMetadataLocked returns the cached token if it is valid, or nil +// if it is not. If attemptPreemptiveRefresh is true, it will also trigger a +// background refresh if the token is stale but still valid. It returns the +// cached error if the last fetch failed and the backoff interval has not +// expired. +// +// Returns (nil, nil) if the token is not valid and a new fetch is required. The +// caller is responsible for initiating the fetch or waiting for an in-progress +// fetch to complete. +// +// It must be called with mu locked. +func (c *gcpServiceAccountIdentityCallCreds) cachedRequestMetadataLocked(attemptPreemptiveRefresh bool) (map[string]string, error) { + if c.token != nil && c.isTokenValidLocked() { + if attemptPreemptiveRefresh && c.isTokenStaleLocked() && c.fetching == nil { + c.fetching = make(chan struct{}) + go c.startFetch() + } + + return map[string]string{"authorization": "Bearer " + c.token.Value}, nil + } + + if c.lastErr != nil && time.Now().Before(c.nextRetryTime) { + return nil, c.lastErr + } + + return nil, nil +} + +// RequireTransportSecurity indicates whether the credentials requires +// transport security. +func (c *gcpServiceAccountIdentityCallCreds) RequireTransportSecurity() bool { + return true +} + +// isTokenStaleLocked checks if the token falls within the +// preemptiveRefresh window. It must be called with mu locked. +func (c *gcpServiceAccountIdentityCallCreds) isTokenStaleLocked() bool { + return c.preemptiveTokenRefresh.Before(time.Now()) +} + +// isTokenValidLocked checks if the token is not expired yet. It must be called +// with mu locked. +func (c *gcpServiceAccountIdentityCallCreds) isTokenValidLocked() bool { + return c.tokenExpiry.After(time.Now()) +} + +// startFetch initiates a token fetch and updates the credential +// state upon completion. +func (c *gcpServiceAccountIdentityCallCreds) startFetch() { + ctx, cancel := context.WithTimeout(c.ctx, metadataTimeout) + defer cancel() + token, err := c.creds.TokenProvider.Token(ctx) + + c.mu.Lock() + defer c.mu.Unlock() + + close(c.fetching) + c.fetching = nil + c.updateStateLocked(token, err) +} + +// updateStateLocked updates the credentials local token cache and +// backoff state based on the outcome of a background fetch attempt. +// +// If the fetch succeeded, the cached token is updated, and the backoff timers +// and error are reset. +// +// If the fetch failed, backoff attempts are calculated and the error is mapped +// to a gRPC status. +// - If the HTTP request fails with a status that maps to gRPC UNAVAILABLE +// according to HTTP to gRPC status code mappings, it returns UNAVAILABLE. +// - All other HTTP error status codes map to UNAUTHENTICATED. +// - Non-HTTP request failures are mapped to UNAVAILABLE. +// +// It must be called with mu locked. +func (c *gcpServiceAccountIdentityCallCreds) updateStateLocked(token *auth.Token, err error) { + if err != nil { + var mappedErr error + var metadataErr *metadata.Error + if errors.As(err, &metadataErr) { + switch transport.HTTPStatusConvTab[metadataErr.Code] { + case codes.Unavailable: + mappedErr = status.Errorf(codes.Unavailable, "credentials: failed to fetch token from metadata server: %v", err) + default: + mappedErr = status.Errorf(codes.Unauthenticated, "credentials: failed to fetch token from metadata server: %v", err) + } + } else { + mappedErr = status.Errorf(codes.Unavailable, "credentials: failed to fetch ID token: %v", err) + } + + c.lastErr = mappedErr + backoffDelay := c.backoff.Backoff(c.retryAttempt) + c.retryAttempt++ + c.nextRetryTime = time.Now().Add(backoffDelay) + return + } + c.lastErr = nil + c.retryAttempt = 0 + c.nextRetryTime = time.Time{} + c.token = token + // Per gRFC A83, the cached token is considered invalid 30 seconds before its + // actual expiration time to accommodate for clock skew. + c.tokenExpiry = token.Expiry.Add(-30 * time.Second) + c.preemptiveTokenRefresh = c.tokenExpiry.Add(-preemptiveRefresh) +} diff --git a/credentials/google/gcp_service_account_identity_credentials_test.go b/credentials/google/gcp_service_account_identity_credentials_test.go new file mode 100644 index 0000000..da0504d --- /dev/null +++ b/credentials/google/gcp_service_account_identity_credentials_test.go @@ -0,0 +1,545 @@ +/* +* +* Copyright 2026 gRPC authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package google_test + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "cloud.google.com/go/auth" + "cloud.google.com/go/auth/credentials/idtoken" + "cloud.google.com/go/compute/metadata" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc/credentials/google/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/status" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// gcpTestAuthInfo implements credentials.AuthInfo for testing. +type gcpTestAuthInfo struct { + credentials.CommonAuthInfo +} + +func (t *gcpTestAuthInfo) AuthType() string { + return "test" +} + +// stubBackoff implements a backoff strategy interface for testing. +type stubBackoff struct { + backoffDelay time.Duration +} + +func (m *stubBackoff) Backoff(int) time.Duration { + return m.backoffDelay +} + +// stubTokenProvider implements auth.TokenProvider for unit testing. +// It supplies mocked token payloads, simulates processing delay, and tracks +// invocation frequency to verify caching and backoff behaviors. +type stubTokenProvider struct { + mu sync.Mutex + err error + token *auth.Token + callCount int // tracks fetch attempts to verify caching and backoff behaviors. +} + +func (c *stubTokenProvider) Token(context.Context) (*auth.Token, error) { + c.mu.Lock() + c.callCount++ + token := c.token + err := c.err + c.mu.Unlock() + return token, err +} + +func (c *stubTokenProvider) setErr(err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.err = err +} + +func (c *stubTokenProvider) setToken(token *auth.Token) { + c.mu.Lock() + defer c.mu.Unlock() + c.token = token +} + +func (c *stubTokenProvider) getCallCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.callCount +} + +// setupStubTokenProvider initializes and returns a stubTokenProvider +// configured with the specified token value, token expiry, error and delay. +func setupStubTokenProvider(token string, err error) *stubTokenProvider { + return &stubTokenProvider{ + err: err, + token: &auth.Token{ + Value: token, + Expiry: time.Now().Add(1 * time.Hour), // idtoken package works with hardcoded 1 hour token expiry. + }, + } +} + +// setupTestGCPServiceAccountIdentityCreds constructs a GCP service account +// identity credentials instance with an injected stub token provider. +// +// It overrides internal.NewIDTokenCredentials to use the stubTokenProvider, +// and registers a cleanup function to restore original hook after the test. +func setupTestGCPServiceAccountIdentityCreds(ctx context.Context, t *testing.T, stubToken *stubTokenProvider) credentials.PerRPCCredentials { + // Override the ID token credentials to use stub token provider. + origNewIDTokenCredentials := internal.NewIDTokenCredentials + internal.NewIDTokenCredentials = func(*idtoken.Options) (*auth.Credentials, error) { + return auth.NewCredentials(&auth.CredentialsOptions{ + TokenProvider: auth.NewCachedTokenProvider(stubToken, &auth.CachedTokenProviderOptions{})}), nil + } + t.Cleanup(func() { internal.NewIDTokenCredentials = origNewIDTokenCredentials }) + + creds, err := google.NewServiceAccountIdentityCredentials(ctx, "audience") + if err != nil { + t.Fatalf("NewServiceAccountIdentityCredentials() failed: %v", err) + } + + return creds +} + +// Test verifies that NewServiceAccountIdentityCredentials returns an error +// when called with empty audience. +func (s) TestNewServiceAccountIdentityCredentials_EmptyAudience(t *testing.T) { + const wantErr = "credentials: audience cannot be empty" + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + if _, err := google.NewServiceAccountIdentityCredentials(ctx, ""); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("NewServiceAccountIdentityCredentials() returned error = %v, want error containing %q", err, wantErr) + } +} + +// Test verifies the successful retrieval of an ID token from the underlying +// token provider. It ensures that a valid token is correctly retrieved and +// formatted into the required "authorization" metadata header with the +// "Bearer " prefix. +func (s) TestGCPServiceAccountIdentityCallCreds_GetRequestMetadata(t *testing.T) { + const token = "token" + stubToken := setupStubTokenProvider(token, nil) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + creds := setupTestGCPServiceAccountIdentityCreds(ctx, t, stubToken) + + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &gcpTestAuthInfo{credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}}, + }) + + md, err := creds.GetRequestMetadata(ctx) + if err != nil { + t.Fatalf("GetRequestMetadata() failed: %v", err) + } + + want := "Bearer " + token + if got := md["authorization"]; got != want { + t.Errorf("Unexpected token from GetRequestMetadata(), got %q, want %q", got, want) + } +} + +// Test verifies that when a backoff delay is set, a token fetch failure caches +// the failure error, and subsequent calls to GetRequestMetadata before the +// backoff window expires return the cached error immediately without +// initiating another fetch request to the provider. +func (s) TestGCPServiceAccountIdentityCallCreds_Backoff(t *testing.T) { + const wantErr = "failed while fetching idToken" + + // Override the backoff strategy with a very large value to guarantee + // it doesn't expire during this test. + origBackoff := internal.BackoffStrategy + internal.BackoffStrategy = &stubBackoff{backoffDelay: 2 * defaultTestTimeout} + t.Cleanup(func() { internal.BackoffStrategy = origBackoff }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stubToken := setupStubTokenProvider("token", errors.New(wantErr)) + creds := setupTestGCPServiceAccountIdentityCreds(ctx, t, stubToken) + + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &gcpTestAuthInfo{credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}}, + }) + + // First call triggers initial fetch, which fails. + if _, err := creds.GetRequestMetadata(ctx); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("GetRequestMetadata() failed with err = %v, want error %q", err, wantErr) + } + + // Verify that calls within the backoff window fail immediately with + // the cached error without attempting a new token fetch. + const numCalls = 3 + for i := 0; i < numCalls; i++ { + if _, err := creds.GetRequestMetadata(ctx); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("GetRequestMetadata() attempt %d failed with err = %v, want error %q", i, err, wantErr) + } + } + + // Verify that no subsequent fetch attempt made. + if got := stubToken.getCallCount(); got != 1 { + t.Fatalf("Unexpected call count to token provider: got %d, want 1", got) + } +} + +// Test verifies that when backoff expired, a subsequent request after an +// initial failure triggers a new fetch attempt and returns the new error. +func (s) TestGCPServiceAccountIdentityCallCreds_BackoffExpired(t *testing.T) { + const ( + wantErr = "failed while fetching idToken" + wantErr2 = "second attempt to fetch token" + ) + + // Override the backoff strategy with a short delay. + const backoffDelay = 10 * time.Millisecond + origBackoff := internal.BackoffStrategy + internal.BackoffStrategy = &stubBackoff{backoffDelay: backoffDelay} + t.Cleanup(func() { internal.BackoffStrategy = origBackoff }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stubToken := setupStubTokenProvider("token", errors.New(wantErr)) + creds := setupTestGCPServiceAccountIdentityCreds(ctx, t, stubToken) + + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &gcpTestAuthInfo{credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}}, + }) + + // First call triggers token fetch and fails. + if _, err := creds.GetRequestMetadata(ctx); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("GetRequestMetadata() failed with err = %v, want error %q", err, wantErr) + } + + if got := stubToken.getCallCount(); got != 1 { + t.Fatalf("Unexpected call count to token provider: got %d, want 1", got) + } + + // Wait for backoff to expire. + select { + case <-time.After(2 * backoffDelay): + case <-ctx.Done(): + t.Fatal("Timed out waiting for backoff to expire") + } + + // Update token provider with second failure error to distinguish the + // new attempt. + stubToken.setErr(errors.New(wantErr2)) + + // Since backoff is 0s (already expired), the next request + // immediately triggers a new fetch attempt. + if _, err := creds.GetRequestMetadata(ctx); err == nil || !strings.Contains(err.Error(), wantErr2) { + t.Fatalf("GetRequestMetadata() failed with err = %v, want error %q", err, wantErr2) + } + + // Verify that a second token fetch actually happened. + if got := stubToken.getCallCount(); got != 2 { + t.Fatalf("Unexpected call count to token provider: got %d, want 2", got) + } +} + +// Test verifies that when backoff expires, a subsequent request after an +// initial failure triggers a new fetch attempt and returns the new token. +func (s) TestGCPServiceAccountIdentityCallCreds_BackoffExpiredRecovery(t *testing.T) { + const ( + wantErr = "failed while fetching idToken" + token = "token" + ) + + // Override the backoff strategy with a short delay. + const backoffDelay = 10 * time.Millisecond + origBackoff := internal.BackoffStrategy + internal.BackoffStrategy = &stubBackoff{backoffDelay: backoffDelay} + t.Cleanup(func() { internal.BackoffStrategy = origBackoff }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stubToken := setupStubTokenProvider(token, errors.New(wantErr)) + creds := setupTestGCPServiceAccountIdentityCreds(ctx, t, stubToken) + + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &gcpTestAuthInfo{credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}}, + }) + + // First call triggers token fetch and fails. + if _, err := creds.GetRequestMetadata(ctx); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("GetRequestMetadata() failed with err = %v, want error %q", err, wantErr) + } + + if got := stubToken.getCallCount(); got != 1 { + t.Fatalf("Unexpected call count to token provider: got %d, want 1", got) + } + + // Wait for backoff to expire. + select { + case <-time.After(2 * backoffDelay): + case <-ctx.Done(): + t.Fatal("Timed out waiting for backoff to expire") + } + + // Update token provider to return a valid token and nil error. + stubToken.setErr(nil) + + md, err := creds.GetRequestMetadata(ctx) + if err != nil { + t.Fatalf("GetRequestMetadata() failed unexpectedly: %v", err) + } + + want := "Bearer " + token + if got := md["authorization"]; got != want { + t.Errorf("GetRequestMetadata() got %q, want %q", got, want) + } + + // Verify that the second fetch happened successfully. + if got := stubToken.getCallCount(); got != 2 { + t.Fatalf("Unexpected call count to token provider: got %d, want 2", got) + } +} + +// Test verifies that when multiple goroutines request a token concurrently, +// only a single fetch is executed and all blocked requests successfully +// receive the same valid token. +func (s) TestGCPServiceAccountIdentityCallCreds_ConcurrentCalls_Success(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stubToken := setupStubTokenProvider("token", nil) + creds := setupTestGCPServiceAccountIdentityCreds(ctx, t, stubToken) + + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &gcpTestAuthInfo{credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}}, + }) + + const numCalls = 5 + errs := make([]error, numCalls) + var wg sync.WaitGroup + for i := range numCalls { + wg.Add(1) + go func(idx int) { + defer wg.Done() + _, errs[idx] = creds.GetRequestMetadata(ctx) + }(i) + } + wg.Wait() + + for _, err := range errs { + if err != nil { + t.Fatalf("Concurrent call to GetRequestMetadata() failed with err = %v", err) + } + } + + if got := stubToken.getCallCount(); got != 1 { + t.Fatalf("Unexpected call count to token provider: got %d, want 1", got) + } +} + +// Test verifies that when multiple goroutines request a token concurrently, +// and the first fetch fails, all blocked requests receives the exact same +// error after the fetch. +func (s) TestGCPServiceAccountIdentityCallCreds_ConcurrentCalls_Failure(t *testing.T) { + const wantErr = "failed while fetching idToken" + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stubToken := setupStubTokenProvider("token", errors.New(wantErr)) + creds := setupTestGCPServiceAccountIdentityCreds(ctx, t, stubToken) + + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &gcpTestAuthInfo{credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}}, + }) + + const numCalls = 5 + errs := make([]error, numCalls) + var wg sync.WaitGroup + for i := range numCalls { + wg.Add(1) + go func(idx int) { + defer wg.Done() + _, errs[idx] = creds.GetRequestMetadata(ctx) + }(i) + } + wg.Wait() + + for _, err := range errs { + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("Concurrent call to GetRequestMetadata() failed with err = %v, want error containing %q", err, wantErr) + } + } + + if got := stubToken.getCallCount(); got != 1 { + t.Fatalf("Unexpected call count to token provider: got %d, want 1", got) + } +} + +// Test verifies that credentials fail to return metadata when the security +// level of the connection is not secure. +func (s) TestGCPServiceAccountIdentityCallCreds_SecurityLevelFailure(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stubToken := setupStubTokenProvider("token", nil) + creds := setupTestGCPServiceAccountIdentityCreds(ctx, t, stubToken) + + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &gcpTestAuthInfo{credentials.CommonAuthInfo{SecurityLevel: credentials.NoSecurity}}, + }) + + const wantErr = "cannot send secure credentials on an insecure connection" + _, err := creds.GetRequestMetadata(ctx) + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Errorf("GetRequestMetadata() failed with error = %v, want error %q", err, wantErr) + } +} + +// Test verifies that when a cached token is within the early expiration window +// (stale but still valid), the second call to GetRequestMetadata immediately +// returns the existing token and triggers an asynchronous background fetch for +// a new token. A subsequent call, after waiting for the background fetch to +// complete, returns the newly acquired token. +func (s) TestGCPServiceAccountIdentityCallCreds_EarlyExpiry(t *testing.T) { + const ( + firstToken = "token-A" + secondToken = "token-B" + ) + stubToken := setupStubTokenProvider(firstToken, nil) + stubToken.setToken( + &auth.Token{ + Value: firstToken, + // Keeping it 1 minute as we subtract 30 seconds from the token expiry. + Expiry: time.Now().Add(1 * time.Minute), + }, + ) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + creds := setupTestGCPServiceAccountIdentityCreds(ctx, t, stubToken) + + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &gcpTestAuthInfo{credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}}, + }) + + md, err := creds.GetRequestMetadata(ctx) + if err != nil { + t.Fatalf("GetRequestMetadata() failed: %v", err) + } + wantfirstToken := "Bearer " + firstToken + if got := md["authorization"]; got != wantfirstToken { + t.Errorf("GetRequestMetadata() returned %q, want %q", got, wantfirstToken) + } + + if got := stubToken.getCallCount(); got != 1 { + t.Fatalf("Unexpected call count to token provider: got %d, want 1", got) + } + + // Update stub token provider to return a new token + stubToken.setToken(&auth.Token{Value: secondToken, Expiry: time.Now().Add(1 * time.Hour)}) + + // The cached token has not expired yet but is stale (the current time is + // past the preemptiveTokenRefresh window). GetRequestMetadata should + // immediately return the cached first token to avoid blocking the RPC, and + // asynchronously trigger a background fetch for the new token. + md, err = creds.GetRequestMetadata(ctx) + if err != nil { + t.Fatalf("GetRequestMetadata() failed: %v", err) + } + if got := md["authorization"]; got != wantfirstToken { + t.Errorf("GetRequestMetadata() returned %q, want %q", got, wantfirstToken) + } + + wantSecondToken := "Bearer " + secondToken + + for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) { + md, err = creds.GetRequestMetadata(ctx) + if err != nil { + t.Fatalf("GetRequestMetadata() failed: %v", err) + } + if md["authorization"] == wantSecondToken { + // Verify that only 2 fetch calls are made. + if got := stubToken.getCallCount(); got != 2 { + t.Fatalf("Unexpected call count to token provider: got %d, want 2", got) + } + break + } + } + if ctx.Err() != nil { + t.Fatal("timed out waiting for background fetch to update token") + } +} + +// Test verifies that different types of errors from the metadata server are +// mapped to the correct gRPC status codes when returned through +// GetRequestMetadata. +func (s) TestGCPServiceAccountIdentityCallCreds_ErrorMapping(t *testing.T) { + tests := []struct { + name string + err error + want codes.Code + }{ + { + name: "429_too_many_requests", + err: &metadata.Error{Code: 429, Message: "too many requests"}, + want: codes.Unavailable, + }, + { + name: "503_service_unavailable", + err: &metadata.Error{Code: 503, Message: "unavailable"}, + want: codes.Unavailable, + }, + { + name: "403_forbidden", + err: &metadata.Error{Code: 403, Message: "forbidden"}, + want: codes.Unauthenticated, + }, + { + name: "generic_protocol_error", + err: fmt.Errorf("generic connection error"), + want: codes.Unavailable, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stubToken := setupStubTokenProvider("token", nil) + creds := setupTestGCPServiceAccountIdentityCreds(ctx, t, stubToken) + stubToken.setErr(tc.err) + + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &gcpTestAuthInfo{credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}}, + }) + + if _, err := creds.GetRequestMetadata(ctx); status.Code(err) != tc.want { + t.Errorf("GetRequestMetadata() failed with gRPC status code %v, want %v for error %v", status.Code(err), tc.want, tc.err) + } + }) + } +} diff --git a/credentials/google/google.go b/credentials/google/google.go new file mode 100644 index 0000000..5a9c946 --- /dev/null +++ b/credentials/google/google.go @@ -0,0 +1,178 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package google defines credentials for google cloud services. +package google + +import ( + "context" + "fmt" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/alts" + "google.golang.org/grpc/credentials/oauth" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" +) + +const defaultCloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform" + +var logger = grpclog.Component("credentials") + +// DefaultCredentialsOptions constructs options to build DefaultCredentials. +type DefaultCredentialsOptions struct { + // PerRPCCreds is a per RPC credentials that is passed to a bundle. + PerRPCCreds credentials.PerRPCCredentials + // ALTSPerRPCCreds is a per RPC credentials that, if specified, will + // supercede PerRPCCreds above for and only for ALTS connections. + ALTSPerRPCCreds credentials.PerRPCCredentials +} + +// NewDefaultCredentialsWithOptions returns a credentials bundle that is +// configured to work with google services. +// +// This API is experimental. +func NewDefaultCredentialsWithOptions(opts DefaultCredentialsOptions) credentials.Bundle { + if opts.PerRPCCreds == nil { + var err error + // If the ADC ends up being Compute Engine Credentials, this context + // won't be used. Otherwise, the context dictates all the subsequent + // token requests via HTTP. So we cannot have any deadline or timeout. + opts.PerRPCCreds, err = newADC(context.TODO()) + if err != nil { + logger.Warningf("NewDefaultCredentialsWithOptions: failed to create application oauth: %v", err) + } + } + if opts.ALTSPerRPCCreds != nil { + opts.PerRPCCreds = &dualPerRPCCreds{ + perRPCCreds: opts.PerRPCCreds, + altsPerRPCCreds: opts.ALTSPerRPCCreds, + } + } + c := &creds{opts: opts} + bundle, err := c.NewWithMode(internal.CredsBundleModeFallback) + if err != nil { + logger.Warningf("NewDefaultCredentialsWithOptions: failed to create new creds: %v", err) + } + return bundle +} + +// NewDefaultCredentials returns a credentials bundle that is configured to work +// with google services. +// +// This API is experimental. +func NewDefaultCredentials() credentials.Bundle { + return NewDefaultCredentialsWithOptions(DefaultCredentialsOptions{}) +} + +// NewComputeEngineCredentials returns a credentials bundle that is configured to work +// with google services. This API must only be used when running on GCE. Authentication configured +// by this API represents the GCE VM's default service account. +// +// This API is experimental. +func NewComputeEngineCredentials() credentials.Bundle { + return NewDefaultCredentialsWithOptions(DefaultCredentialsOptions{ + PerRPCCreds: oauth.NewComputeEngine(), + }) +} + +// creds implements credentials.Bundle. +type creds struct { + opts DefaultCredentialsOptions + + // Supported modes are defined in internal/internal.go. + mode string + // The active transport credentials associated with this bundle. + transportCreds credentials.TransportCredentials + // The active per RPC credentials associated with this bundle. + perRPCCreds credentials.PerRPCCredentials +} + +func (c *creds) TransportCredentials() credentials.TransportCredentials { + return c.transportCreds +} + +func (c *creds) PerRPCCredentials() credentials.PerRPCCredentials { + if c == nil { + return nil + } + return c.perRPCCreds +} + +var ( + newTLS = func() credentials.TransportCredentials { + return credentials.NewTLS(nil) + } + newALTS = func() credentials.TransportCredentials { + return alts.NewClientCreds(alts.DefaultClientOptions()) + } + newADC = func(ctx context.Context) (credentials.PerRPCCredentials, error) { + return oauth.NewApplicationDefault(ctx, defaultCloudPlatformScope) + } +) + +// NewWithMode should make a copy of Bundle, and switch mode. Modifying the +// existing Bundle may cause races. +func (c *creds) NewWithMode(mode string) (credentials.Bundle, error) { + newCreds := &creds{ + opts: c.opts, + mode: mode, + } + + // Create transport credentials. + switch mode { + case internal.CredsBundleModeFallback: + newCreds.transportCreds = newClusterTransportCreds(newTLS(), newALTS()) + case internal.CredsBundleModeBackendFromBalancer, internal.CredsBundleModeBalancer: + // Only the clients can use google default credentials, so we only need + // to create new ALTS client creds here. + newCreds.transportCreds = newALTS() + default: + return nil, fmt.Errorf("unsupported mode: %v", mode) + } + + if mode == internal.CredsBundleModeFallback || mode == internal.CredsBundleModeBackendFromBalancer { + newCreds.perRPCCreds = newCreds.opts.PerRPCCreds + } + + return newCreds, nil +} + +// dualPerRPCCreds implements credentials.PerRPCCredentials by embedding the +// fallback PerRPCCredentials and the ALTS one. It pickes one of them based on +// the channel type. +type dualPerRPCCreds struct { + perRPCCreds credentials.PerRPCCredentials + altsPerRPCCreds credentials.PerRPCCredentials +} + +func (d *dualPerRPCCreds) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + ri, ok := credentials.RequestInfoFromContext(ctx) + if !ok { + return nil, fmt.Errorf("request info not found from context") + } + if authType := ri.AuthInfo.AuthType(); authType == "alts" { + return d.altsPerRPCCreds.GetRequestMetadata(ctx, uri...) + } + // This ensures backward compatibility even if authType is not "tls". + return d.perRPCCreds.GetRequestMetadata(ctx, uri...) +} + +func (d *dualPerRPCCreds) RequireTransportSecurity() bool { + return d.altsPerRPCCreds.RequireTransportSecurity() || d.perRPCCreds.RequireTransportSecurity() +} diff --git a/credentials/google/google_test.go b/credentials/google/google_test.go new file mode 100644 index 0000000..00d9dce --- /dev/null +++ b/credentials/google/google_test.go @@ -0,0 +1,268 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package google + +import ( + "context" + "net" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/credentials" + icredentials "google.golang.org/grpc/internal/credentials" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/resolver" +) + +var defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type testCreds struct { + credentials.TransportCredentials + typ string +} + +func (c *testCreds) ClientHandshake(context.Context, string, net.Conn) (net.Conn, credentials.AuthInfo, error) { + return nil, &testAuthInfo{typ: c.typ}, nil +} + +func (c *testCreds) ServerHandshake(net.Conn) (net.Conn, credentials.AuthInfo, error) { + return nil, &testAuthInfo{typ: c.typ}, nil +} + +type testAuthInfo struct { + typ string +} + +func (t *testAuthInfo) AuthType() string { + return t.typ +} + +type testPerRPCCreds struct { + md map[string]string +} + +func (c *testPerRPCCreds) RequireTransportSecurity() bool { + return true +} + +func (c *testPerRPCCreds) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { + return c.md, nil +} + +var ( + testTLS = &testCreds{typ: "tls"} + testALTS = &testCreds{typ: "alts"} +) + +func overrideNewCredsFuncs() func() { + origNewTLS := newTLS + newTLS = func() credentials.TransportCredentials { + return testTLS + } + origNewALTS := newALTS + newALTS = func() credentials.TransportCredentials { + return testALTS + } + origNewADC := newADC + newADC = func(context.Context) (credentials.PerRPCCredentials, error) { + // We do not use perRPC creds in this test. It is safe to return nil here. + return nil, nil + } + + return func() { + newTLS = origNewTLS + newALTS = origNewALTS + newADC = origNewADC + } +} + +// TestClientHandshakeBasedOnClusterName that by default (without switching +// modes), ClientHandshake does either tls or alts base on the cluster name in +// attributes. +func (s) TestClientHandshakeBasedOnClusterName(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + defer overrideNewCredsFuncs()() + for bundleTyp, tc := range map[string]credentials.Bundle{ + "defaultCredsWithOptions": NewDefaultCredentialsWithOptions(DefaultCredentialsOptions{}), + "defaultCreds": NewDefaultCredentials(), + "computeCreds": NewComputeEngineCredentials(), + } { + tests := []struct { + name string + ctx context.Context + wantTyp string + }{ + { + name: "no cluster name", + ctx: ctx, + wantTyp: "tls", + }, + { + name: "with non-CFE cluster name", + ctx: icredentials.NewClientHandshakeInfoContext(ctx, credentials.ClientHandshakeInfo{ + Attributes: xds.SetXDSHandshakeClusterName(resolver.Address{}, "lalala").Attributes, + }), + // non-CFE backends should use alts. + wantTyp: "alts", + }, + { + name: "with CFE cluster name", + ctx: icredentials.NewClientHandshakeInfoContext(ctx, credentials.ClientHandshakeInfo{ + Attributes: xds.SetXDSHandshakeClusterName(resolver.Address{}, "google_cfe_bigtable.googleapis.com").Attributes, + }), + // CFE should use tls. + wantTyp: "tls", + }, + { + name: "with xdstp CFE cluster name", + ctx: icredentials.NewClientHandshakeInfoContext(ctx, credentials.ClientHandshakeInfo{ + Attributes: xds.SetXDSHandshakeClusterName(resolver.Address{}, "xdstp://traffic-director-c2p.xds.googleapis.com/envoy.config.cluster.v3.Cluster/google_cfe_bigtable.googleapis.com").Attributes, + }), + // CFE should use tls. + wantTyp: "tls", + }, + { + name: "with xdstp non-CFE cluster name", + ctx: icredentials.NewClientHandshakeInfoContext(ctx, credentials.ClientHandshakeInfo{ + Attributes: xds.SetXDSHandshakeClusterName(resolver.Address{}, "xdstp://other.com/envoy.config.cluster.v3.Cluster/google_cfe_bigtable.googleapis.com").Attributes, + }), + // non-CFE should use atls. + wantTyp: "alts", + }, + } + for _, tt := range tests { + t.Run(bundleTyp+" "+tt.name, func(t *testing.T) { + _, info, err := tc.TransportCredentials().ClientHandshake(tt.ctx, "", nil) + if err != nil { + t.Fatalf("ClientHandshake failed: %v", err) + } + if gotType := info.AuthType(); gotType != tt.wantTyp { + t.Fatalf("unexpected authtype: %v, want: %v", gotType, tt.wantTyp) + } + + _, infoServer, err := tc.TransportCredentials().ServerHandshake(nil) + if err != nil { + t.Fatalf("ClientHandshake failed: %v", err) + } + // ServerHandshake should always do TLS. + if gotType := infoServer.AuthType(); gotType != "tls" { + t.Fatalf("unexpected server authtype: %v, want: %v", gotType, "tls") + } + }) + } + } +} + +func TestDefaultCredentialsWithOptions(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + md1 := map[string]string{"foo": "tls"} + md2 := map[string]string{"foo": "alts"} + tests := []struct { + desc string + defaultCredsOpts DefaultCredentialsOptions + authInfo credentials.AuthInfo + wantedMetadata map[string]string + }{ + { + desc: "no ALTSPerRPCCreds with tls channel", + defaultCredsOpts: DefaultCredentialsOptions{ + PerRPCCreds: &testPerRPCCreds{ + md: md1, + }, + }, + authInfo: &testAuthInfo{typ: "tls"}, + wantedMetadata: md1, + }, + { + desc: "no ALTSPerRPCCreds with alts channel", + defaultCredsOpts: DefaultCredentialsOptions{ + PerRPCCreds: &testPerRPCCreds{ + md: md1, + }, + }, + authInfo: &testAuthInfo{typ: "alts"}, + wantedMetadata: md1, + }, + { + desc: "ALTSPerRPCCreds specified with tls channel", + defaultCredsOpts: DefaultCredentialsOptions{ + PerRPCCreds: &testPerRPCCreds{ + md: md1, + }, + ALTSPerRPCCreds: &testPerRPCCreds{ + md: md2, + }, + }, + authInfo: &testAuthInfo{typ: "tls"}, + wantedMetadata: md1, + }, + { + desc: "ALTSPerRPCCreds specified with alts channel", + defaultCredsOpts: DefaultCredentialsOptions{ + PerRPCCreds: &testPerRPCCreds{ + md: md1, + }, + ALTSPerRPCCreds: &testPerRPCCreds{ + md: md2, + }, + }, + authInfo: &testAuthInfo{typ: "alts"}, + wantedMetadata: md2, + }, + { + desc: "ALTSPerRPCCreds specified with unknown channel", + defaultCredsOpts: DefaultCredentialsOptions{ + PerRPCCreds: &testPerRPCCreds{ + md: md1, + }, + ALTSPerRPCCreds: &testPerRPCCreds{ + md: md2, + }, + }, + authInfo: &testAuthInfo{typ: "foo"}, + wantedMetadata: md1, + }, + } + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + bundle := NewDefaultCredentialsWithOptions(tc.defaultCredsOpts) + ri := credentials.RequestInfo{AuthInfo: tc.authInfo} + ctx := credentials.NewContextWithRequestInfo(ctx, ri) + got, err := bundle.PerRPCCredentials().GetRequestMetadata(ctx, "uri") + if err != nil { + t.Fatalf("Bundle's PerRPCCredentials().GetRequestMetadata() unexpected error = %v", err) + } + if diff := cmp.Diff(got, tc.wantedMetadata); diff != "" { + t.Errorf("Unexpected request metadata from bundle's PerRPCCredentials. Diff (-got +want):\n%v", diff) + } + }) + } +} diff --git a/credentials/google/internal/internal.go b/credentials/google/internal/internal.go new file mode 100644 index 0000000..9557d26 --- /dev/null +++ b/credentials/google/internal/internal.go @@ -0,0 +1,35 @@ +/* +* +* Copyright 2026 gRPC authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +// Package internal contains functionality internal to the google package. +package internal + +import ( + "cloud.google.com/go/auth" + "cloud.google.com/go/auth/credentials/idtoken" + "google.golang.org/grpc/internal/backoff" +) + +// The following variables are overridden in tests. +var ( + // BackoffStrategy is the backoff strategy to use when token fetch fails. + BackoffStrategy backoff.Strategy + + // NewIDTokenCredentials builds idtoken credentials using specified options. + NewIDTokenCredentials func(opts *idtoken.Options) (*auth.Credentials, error) +) diff --git a/credentials/google/xds.go b/credentials/google/xds.go new file mode 100644 index 0000000..cccb222 --- /dev/null +++ b/credentials/google/xds.go @@ -0,0 +1,128 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package google + +import ( + "context" + "net" + "net/url" + "strings" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/xds" +) + +const cfeClusterNamePrefix = "google_cfe_" +const cfeClusterResourceNamePrefix = "/envoy.config.cluster.v3.Cluster/google_cfe_" +const cfeClusterAuthorityName = "traffic-director-c2p.xds.googleapis.com" + +// clusterTransportCreds is a combo of TLS + ALTS. +// +// On the client, ClientHandshake picks TLS or ALTS based on address attributes. +// - if attributes has cluster name +// - if cluster name has prefix "google_cfe_", or +// "xdstp://traffic-director-c2p.xds.googleapis.com/envoy.config.cluster.v3.Cluster/google_cfe_", +// use TLS +// - otherwise, use ALTS +// +// - else, do TLS +// +// On the server, ServerHandshake always does TLS. +type clusterTransportCreds struct { + tls credentials.TransportCredentials + alts credentials.TransportCredentials +} + +func newClusterTransportCreds(tls, alts credentials.TransportCredentials) *clusterTransportCreds { + return &clusterTransportCreds{ + tls: tls, + alts: alts, + } +} + +// clusterName returns the xDS cluster name stored in the attributes in the +// context. +func clusterName(ctx context.Context) string { + chi := credentials.ClientHandshakeInfoFromContext(ctx) + if chi.Attributes == nil { + return "" + } + cluster, _ := xds.GetXDSHandshakeClusterName(chi.Attributes) + return cluster +} + +// isDirectPathCluster returns true if the cluster in the context is a +// directpath cluster, meaning ALTS should be used. +func isDirectPathCluster(ctx context.Context) bool { + cluster := clusterName(ctx) + if cluster == "" { + // No cluster; not xDS; use TLS. + return false + } + if strings.HasPrefix(cluster, cfeClusterNamePrefix) { + // xDS cluster prefixed by "google_cfe_"; use TLS. + return false + } + if !strings.HasPrefix(cluster, "xdstp:") { + // Other xDS cluster name; use ALTS. + return true + } + u, err := url.Parse(cluster) + if err != nil { + // Shouldn't happen, but assume ALTS. + return true + } + // If authority AND path match our CFE checks, use TLS; otherwise use ALTS. + return u.Host != cfeClusterAuthorityName || !strings.HasPrefix(u.Path, cfeClusterResourceNamePrefix) +} + +func (c *clusterTransportCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + if isDirectPathCluster(ctx) { + // If attributes have cluster name, and cluster name is not cfe, it's a + // backend address, use ALTS. + return c.alts.ClientHandshake(ctx, authority, rawConn) + } + return c.tls.ClientHandshake(ctx, authority, rawConn) +} + +func (c *clusterTransportCreds) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) { + return c.tls.ServerHandshake(conn) +} + +func (c *clusterTransportCreds) Info() credentials.ProtocolInfo { + // TODO: this always returns tls.Info now, because we don't have a cluster + // name to check when this method is called. This method doesn't affect + // anything important now. We may want to revisit this if it becomes more + // important later. + return c.tls.Info() +} + +func (c *clusterTransportCreds) Clone() credentials.TransportCredentials { + return &clusterTransportCreds{ + tls: c.tls.Clone(), + alts: c.alts.Clone(), + } +} + +func (c *clusterTransportCreds) OverrideServerName(s string) error { + if err := c.tls.OverrideServerName(s); err != nil { + return err + } + return c.alts.OverrideServerName(s) +} diff --git a/credentials/google/xds_test.go b/credentials/google/xds_test.go new file mode 100644 index 0000000..325f527 --- /dev/null +++ b/credentials/google/xds_test.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package google + +import ( + "context" + "testing" + + "google.golang.org/grpc/credentials" + icredentials "google.golang.org/grpc/internal/credentials" + "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/resolver" +) + +func (s) TestIsDirectPathCluster(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + c := func(cluster string) context.Context { + return icredentials.NewClientHandshakeInfoContext(ctx, credentials.ClientHandshakeInfo{ + Attributes: xds.SetXDSHandshakeClusterName(resolver.Address{}, cluster).Attributes, + }) + } + + testCases := []struct { + name string + ctx context.Context + want bool + }{ + {"not an xDS cluster", ctx, false}, + {"cfe", c("google_cfe_bigtable.googleapis.com"), false}, + {"non-cfe", c("google_bigtable.googleapis.com"), true}, + {"starts with xdstp but not cfe format", c("xdstp:google_cfe_bigtable.googleapis.com"), true}, + {"no authority", c("xdstp:///envoy.config.cluster.v3.Cluster/google_cfe_"), true}, + {"wrong authority", c("xdstp://foo.bar/envoy.config.cluster.v3.Cluster/google_cfe_"), true}, + {"xdstp CFE", c("xdstp://traffic-director-c2p.xds.googleapis.com/envoy.config.cluster.v3.Cluster/google_cfe_"), false}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if got := isDirectPathCluster(tc.ctx); got != tc.want { + t.Errorf("isDirectPathCluster(_) = %v; want %v", got, tc.want) + } + }) + } +} diff --git a/credentials/insecure/insecure.go b/credentials/insecure/insecure.go new file mode 100644 index 0000000..93156c0 --- /dev/null +++ b/credentials/insecure/insecure.go @@ -0,0 +1,104 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package insecure provides an implementation of the +// credentials.TransportCredentials interface which disables transport security. +package insecure + +import ( + "context" + "net" + + "google.golang.org/grpc/credentials" +) + +// NewCredentials returns a credentials which disables transport security. +// +// Note that using this credentials with per-RPC credentials which require +// transport security is incompatible and will cause RPCs to fail. +func NewCredentials() credentials.TransportCredentials { + return insecureTC{} +} + +// insecureTC implements the insecure transport credentials. The handshake +// methods simply return the passed in net.Conn and set the security level to +// NoSecurity. +type insecureTC struct{} + +func (insecureTC) ClientHandshake(_ context.Context, _ string, conn net.Conn) (net.Conn, credentials.AuthInfo, error) { + return conn, info{credentials.CommonAuthInfo{SecurityLevel: credentials.NoSecurity}}, nil +} + +func (insecureTC) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) { + return conn, info{credentials.CommonAuthInfo{SecurityLevel: credentials.NoSecurity}}, nil +} + +func (insecureTC) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{SecurityProtocol: "insecure"} +} + +func (insecureTC) Clone() credentials.TransportCredentials { + return insecureTC{} +} + +func (insecureTC) OverrideServerName(string) error { + return nil +} + +// info contains the auth information for an insecure connection. +// It implements the AuthInfo interface. +type info struct { + credentials.CommonAuthInfo +} + +// AuthType returns the type of info as a string. +func (info) AuthType() string { + return "insecure" +} + +// ValidateAuthority allows any value to be overridden for the :authority +// header. +func (info) ValidateAuthority(string) error { + return nil +} + +// insecureBundle implements an insecure bundle. +// An insecure bundle provides a thin wrapper around insecureTC to support +// the credentials.Bundle interface. +type insecureBundle struct{} + +// NewBundle returns a bundle with disabled transport security and no per rpc credential. +func NewBundle() credentials.Bundle { + return insecureBundle{} +} + +// NewWithMode returns a new insecure Bundle. The mode is ignored. +func (insecureBundle) NewWithMode(string) (credentials.Bundle, error) { + return insecureBundle{}, nil +} + +// PerRPCCredentials returns an nil implementation as insecure +// bundle does not support a per rpc credential. +func (insecureBundle) PerRPCCredentials() credentials.PerRPCCredentials { + return nil +} + +// TransportCredentials returns the underlying insecure transport credential. +func (insecureBundle) TransportCredentials() credentials.TransportCredentials { + return NewCredentials() +} diff --git a/credentials/jwt/doc.go b/credentials/jwt/doc.go new file mode 100644 index 0000000..bba687f --- /dev/null +++ b/credentials/jwt/doc.go @@ -0,0 +1,50 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package jwt implements JWT token file-based call credentials. +// +// This package provides support for A97 JWT Call Credentials, allowing gRPC +// clients to authenticate using JWT tokens read from files. While originally +// designed for xDS environments, these credentials are general-purpose. +// +// The credentials can be used directly in gRPC clients or configured via xDS. +// +// # Token Requirements +// +// JWT tokens must: +// - Be valid, well-formed JWT tokens with header, payload, and signature +// - Include an "exp" (expiration) claim +// - Be readable from the specified file path +// +// # Considerations +// +// - Tokens are cached until expiration to avoid excessive file I/O +// - Transport security is required (RequireTransportSecurity returns true) +// - Errors in reading tokens or parsing JWTs will result in RPC UNAVAILALBE or +// UNAUTHENTICATED errors. The errors are cached and retried with exponential +// backoff. +// +// This implementation is originally intended for use in service mesh +// environments like Istio where JWT tokens are provisioned and rotated by the +// infrastructure. +// +// # Experimental +// +// Notice: All APIs in this package are experimental and may be removed in a +// later release. +package jwt diff --git a/credentials/jwt/file_reader.go b/credentials/jwt/file_reader.go new file mode 100644 index 0000000..8337608 --- /dev/null +++ b/credentials/jwt/file_reader.go @@ -0,0 +1,118 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package jwt + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "time" +) + +var ( + errTokenFileAccess = errors.New("token file access error") + errJWTValidation = errors.New("invalid JWT") +) + +// jwtClaims represents the JWT claims structure for extracting expiration time. +type jwtClaims struct { + Exp int64 `json:"exp"` +} + +// jwtFileReader handles reading and parsing JWT tokens from files. +// It is safe to call methods on this type concurrently as no state is stored. +type jwtFileReader struct { + tokenFilePath string +} + +// readToken reads and parses a JWT token from the configured file. +// Returns the token string, expiration time, and any error encountered. +func (r *jwtFileReader) readToken() (string, time.Time, error) { + tokenBytes, err := os.ReadFile(r.tokenFilePath) + if err != nil { + return "", time.Time{}, fmt.Errorf("%v: %w", err, errTokenFileAccess) + } + + token := strings.TrimSpace(string(tokenBytes)) + if token == "" { + return "", time.Time{}, fmt.Errorf("token file %q is empty: %w", r.tokenFilePath, errJWTValidation) + } + + exp, err := r.extractExpiration(token) + if err != nil { + return "", time.Time{}, fmt.Errorf("token file %q: %v: %w", r.tokenFilePath, err, errJWTValidation) + } + + return token, exp, nil +} + +const tokenDelim = "." + +// extractClaimsRaw returns the JWT's claims part as raw string. Even though the +// header and signature are not used, it still expects that the input string to +// be well-formed (ie comprised of exactly three parts, separated by a dot +// character). +func extractClaimsRaw(s string) (string, bool) { + _, s, ok := strings.Cut(s, tokenDelim) + if !ok { // no period found + return "", false + } + claims, s, ok := strings.Cut(s, tokenDelim) + if !ok { // only one period found + return "", false + } + _, _, ok = strings.Cut(s, tokenDelim) + if ok { // three periods found + return "", false + } + return claims, true +} + +// extractExpiration parses the JWT token to extract the expiration time. +func (r *jwtFileReader) extractExpiration(token string) (time.Time, error) { + claimsRaw, ok := extractClaimsRaw(token) + if !ok { + return time.Time{}, fmt.Errorf("expected 3 parts in token") + } + payloadBytes, err := base64.RawURLEncoding.DecodeString(claimsRaw) + if err != nil { + return time.Time{}, fmt.Errorf("decode error: %v", err) + } + + var claims jwtClaims + if err := json.Unmarshal(payloadBytes, &claims); err != nil { + return time.Time{}, fmt.Errorf("unmarshal error: %v", err) + } + + if claims.Exp == 0 { + return time.Time{}, fmt.Errorf("no expiration claims") + } + + expTime := time.Unix(claims.Exp, 0) + + // Check if token is already expired. + if expTime.Before(time.Now()) { + return time.Time{}, fmt.Errorf("expired token") + } + + return expTime, nil +} diff --git a/credentials/jwt/file_reader_test.go b/credentials/jwt/file_reader_test.go new file mode 100644 index 0000000..481f01f --- /dev/null +++ b/credentials/jwt/file_reader_test.go @@ -0,0 +1,172 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package jwt + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" +) + +func (s) TestJWTFileReader_ReadToken_FileErrors(t *testing.T) { + tests := []struct { + name string + create bool + contents string + wantErr error + }{ + { + name: "nonexistent_file", + create: false, + contents: "", + wantErr: errTokenFileAccess, + }, + { + name: "empty_file", + create: true, + contents: "", + wantErr: errJWTValidation, + }, + { + name: "file_with_whitespace_only", + create: true, + contents: " \n\t ", + wantErr: errJWTValidation, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var tokenFile string + if !tt.create { + tokenFile = "/does-not-exist" + } else { + tokenFile = writeTempFile(t, "token", tt.contents) + } + + reader := jwtFileReader{tokenFilePath: tokenFile} + if _, _, err := reader.readToken(); err == nil { + t.Fatal("ReadToken() expected error, got nil") + } else if !errors.Is(err, tt.wantErr) { + t.Fatalf("ReadToken() error = %v, want error %v", err, tt.wantErr) + } + }) + } +} + +func (s) TestJWTFileReader_ReadToken_InvalidJWT(t *testing.T) { + now := time.Now().Truncate(time.Second) + tests := []struct { + name string + tokenContent string + wantErr error + }{ + { + name: "valid_token_without_expiration", + tokenContent: createTestJWT(t, time.Time{}), + wantErr: errJWTValidation, + }, + { + name: "expired_token", + tokenContent: createTestJWT(t, now.Add(-time.Hour)), + wantErr: errJWTValidation, + }, + { + name: "malformed_JWT_not_enough_parts", + tokenContent: "invalid.jwt", + wantErr: errJWTValidation, + }, + { + name: "malformed_JWT_invalid_base64", + tokenContent: "header.invalid_base64!@#.signature", + wantErr: errJWTValidation, + }, + { + name: "malformed_JWT_invalid_JSON", + tokenContent: createInvalidJWT(t), + wantErr: errJWTValidation, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tokenFile := writeTempFile(t, "token", tt.tokenContent) + + reader := jwtFileReader{tokenFilePath: tokenFile} + if _, _, err := reader.readToken(); err == nil { + t.Fatal("ReadToken() expected error, got nil") + } else if !errors.Is(err, tt.wantErr) { + t.Fatalf("ReadToken() error = %v, want error %v", err, tt.wantErr) + } + }) + } +} + +func (s) TestJWTFileReader_ReadToken_ValidToken(t *testing.T) { + now := time.Now().Truncate(time.Second) + tokenExp := now.Add(time.Hour) + token := createTestJWT(t, tokenExp) + tokenFile := writeTempFile(t, "token", token) + + reader := jwtFileReader{tokenFilePath: tokenFile} + readToken, expiry, err := reader.readToken() + if err != nil { + t.Fatalf("ReadToken() unexpected error: %v", err) + } + + if readToken != token { + t.Errorf("ReadToken() token = %q, want %q", readToken, token) + } + + if !expiry.Equal(tokenExp) { + t.Errorf("ReadToken() expiry = %v, want %v", expiry, tokenExp) + } +} + +// createInvalidJWT creates a JWT with invalid JSON in the payload. +func createInvalidJWT(t *testing.T) string { + t.Helper() + + header := map[string]any{ + "typ": "JWT", + "alg": "HS256", + } + + headerBytes, err := json.Marshal(header) + if err != nil { + t.Fatalf("Failed to marshal header: %v", err) + } + + headerB64 := base64.URLEncoding.EncodeToString(headerBytes) + headerB64 = strings.TrimRight(headerB64, "=") + + // Create invalid JSON payload + invalidJSON := "invalid json content" + payloadB64 := base64.URLEncoding.EncodeToString([]byte(invalidJSON)) + payloadB64 = strings.TrimRight(payloadB64, "=") + + signature := base64.URLEncoding.EncodeToString([]byte("fake_signature")) + signature = strings.TrimRight(signature, "=") + + return fmt.Sprintf("%s.%s.%s", headerB64, payloadB64, signature) +} diff --git a/credentials/jwt/token_file_call_creds.go b/credentials/jwt/token_file_call_creds.go new file mode 100644 index 0000000..2005cf4 --- /dev/null +++ b/credentials/jwt/token_file_call_creds.go @@ -0,0 +1,180 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package jwt + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/status" +) + +const preemptiveRefreshThreshold = time.Minute + +// jwtTokenFileCallCreds provides JWT token-based PerRPCCredentials that reads +// tokens from a file. +// This implementation follows the A97 JWT Call Credentials specification. +type jwtTokenFileCallCreds struct { + fileReader *jwtFileReader + backoffStrategy backoff.Strategy + + // cached data protected by mu + mu sync.Mutex + cachedAuthHeader string // "Bearer " + token + cachedExpiry time.Time // Slightly less than actual expiration time + cachedError error // Error from last failed attempt + retryAttempt int // Current retry attempt number + nextRetryTime time.Time // When next retry is allowed + pendingRefresh bool // Whether a refresh is currently in progress +} + +// NewTokenFileCallCredentials creates PerRPCCredentials that reads JWT tokens +// from the specified file path. +func NewTokenFileCallCredentials(tokenFilePath string) (credentials.PerRPCCredentials, error) { + if tokenFilePath == "" { + return nil, fmt.Errorf("tokenFilePath cannot be empty") + } + + creds := &jwtTokenFileCallCreds{ + fileReader: &jwtFileReader{tokenFilePath: tokenFilePath}, + backoffStrategy: backoff.DefaultExponential, + } + + return creds, nil +} + +// GetRequestMetadata gets the current request metadata, refreshing tokens if +// required. This implementation follows the PerRPCCredentials interface. The +// tokens will get automatically refreshed if they are about to expire or if +// they haven't been loaded successfully yet. +// If it's not possible to extract a token from the file, UNAVAILABLE is +// returned. +// If the token is extracted but invalid, then UNAUTHENTICATED is returned. +// If errors are encoutered, a backoff is applied before retrying. +func (c *jwtTokenFileCallCreds) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) { + ri, _ := credentials.RequestInfoFromContext(ctx) + if err := credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil { + return nil, fmt.Errorf("cannot send secure credentials on an insecure connection: %v", err) + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.isTokenValidLocked() { + needsPreemptiveRefresh := time.Until(c.cachedExpiry) < preemptiveRefreshThreshold + if needsPreemptiveRefresh && !c.pendingRefresh { + // Start refresh if not pending (handling the prior RPC may have + // just spawned a goroutine). + c.pendingRefresh = true + go c.refreshToken() + } + return map[string]string{ + "authorization": c.cachedAuthHeader, + }, nil + } + + // If in backoff state, just return the cached error. + if c.cachedError != nil && time.Now().Before(c.nextRetryTime) { + return nil, c.cachedError + } + + // At this point, the token is either invalid or expired and we are no + // longer backing off from any encountered errors. So refresh it. + // NB: We are holding the lock while reading the token from file. This will + // cause other RPCs to block until the read completes (sucecssfully or not) + // and the cache is updated. Subsequent RPCs will end up using the cache. + // This is per A97. + token, expiry, err := c.fileReader.readToken() + c.updateCacheLocked(token, expiry, err) + + if c.cachedError != nil { + return nil, c.cachedError + } + return map[string]string{ + "authorization": c.cachedAuthHeader, + }, nil +} + +// RequireTransportSecurity indicates whether the credentials requires +// transport security. +func (c *jwtTokenFileCallCreds) RequireTransportSecurity() bool { + return true +} + +// isTokenValidLocked checks if the cached token is still valid. +// Caller must hold c.mu lock. +func (c *jwtTokenFileCallCreds) isTokenValidLocked() bool { + if c.cachedAuthHeader == "" { + return false + } + return c.cachedExpiry.After(time.Now()) +} + +// refreshToken reads the token from file and updates the cached data. +func (c *jwtTokenFileCallCreds) refreshToken() { + // Deliberately not locking c.mu here. This way other RPCs can proceed + // while we read the token. This is per gRFC A97. + token, expiry, err := c.fileReader.readToken() + + c.mu.Lock() + defer c.mu.Unlock() + c.updateCacheLocked(token, expiry, err) + c.pendingRefresh = false +} + +// updateCacheLocked updates the cached token, expiry, and error state. +// If an error is provided, it determines whether to set it as an UNAVAILABLE +// or UNAUTHENTICATED error based on the error type. +// NOTE: This method (and its callers) do not queue up a token refresh/retry if +// the expiration is soon / an error was encountered. Instead, this is done when +// handling RPCs. This is as per gRFC A97, which states that it is +// undesirable to retry loading the token if the channel is idle. +// Caller must hold c.mu lock. +func (c *jwtTokenFileCallCreds) updateCacheLocked(token string, expiry time.Time, err error) { + if err != nil { + // Convert to gRPC status codes + if errors.Is(err, errTokenFileAccess) { + c.cachedError = status.Error(codes.Unavailable, err.Error()) + } else if errors.Is(err, errJWTValidation) { + c.cachedError = status.Error(codes.Unauthenticated, err.Error()) + } else { + // Should not happen. Treat unknown errors as UNAUTHENTICATED. + c.cachedError = status.Error(codes.Unauthenticated, err.Error()) + } + c.retryAttempt++ + backoffDelay := c.backoffStrategy.Backoff(c.retryAttempt - 1) + c.nextRetryTime = time.Now().Add(backoffDelay) + return + } + // Success - clear any cached error and update token cache + c.cachedError = nil + c.retryAttempt = 0 + c.nextRetryTime = time.Time{} + + c.cachedAuthHeader = "Bearer " + token + // Per gRFC A97: consider token invalid if it expires within the next 30 + // seconds to accommodate for clock skew and server processing time. + c.cachedExpiry = expiry.Add(-30 * time.Second) +} diff --git a/credentials/jwt/token_file_call_creds_ext_test.go b/credentials/jwt/token_file_call_creds_ext_test.go new file mode 100644 index 0000000..27a8756 --- /dev/null +++ b/credentials/jwt/token_file_call_creds_ext_test.go @@ -0,0 +1,265 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package jwt_test + +import ( + "context" + "crypto/tls" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/jwt" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const defaultTestTimeout = 5 * time.Second + +const wantErr = "cannot send secure credentials on an insecure connection" + +// TestJWTCallCredentials_InsecureTransport_AsCallOption verifies that when JWT +// call credentials are passed as a per-RPC call option over an insecure +// transport, the RPC fails with a meaningful error. +func TestJWTCallCredentials_InsecureTransport_AsCallOption(t *testing.T) { + token := createTestJWT(t, time.Now().Add(time.Hour)) + tokenFile := writeTempTokenFile(t, token) + + jwtCreds, err := jwt.NewTokenFileCallCredentials(tokenFile) + if err != nil { + t.Fatalf("NewTokenFileCallCredentials(%q) failed: %v", tokenFile, err) + } + + ss := &stubserver.StubServer{} + if err := ss.StartServer(grpc.Creds(insecure.NewCredentials())); err != nil { + t.Fatalf("Failed to start server: %v", err) + } + defer ss.Stop() + + cc, err := grpc.NewClient(ss.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", ss.Address, err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + client := testgrpc.NewTestServiceClient(cc) + _, err = client.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(jwtCreds)) + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall() error = %v; want error containing %q", err, wantErr) + } +} + +// TestJWTCallCredentials_InsecureTransport_AsDialOption verifies that when JWT +// call credentials are passed as a dial option over an insecure transport, the +// client creation fails with a meaningful error. +func TestJWTCallCredentials_InsecureTransport_AsDialOption(t *testing.T) { + token := createTestJWT(t, time.Now().Add(time.Hour)) + tokenFile := writeTempTokenFile(t, token) + + jwtCreds, err := jwt.NewTokenFileCallCredentials(tokenFile) + if err != nil { + t.Fatalf("NewTokenFileCallCredentials(%q) failed: %v", tokenFile, err) + } + + ss := &stubserver.StubServer{} + if err := ss.StartServer(grpc.Creds(insecure.NewCredentials())); err != nil { + t.Fatalf("Failed to start server: %v", err) + } + defer ss.Stop() + + _, err = grpc.NewClient(ss.Address, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithPerRPCCredentials(jwtCreds)) + if err == nil || !strings.Contains(err.Error(), "the credentials require transport level security") { + t.Fatalf("grpc.NewClient() error = %v; want error containing %q", err, "the credentials require transport level security") + } +} + +// TestJWTCallCredentials_SecureTransport_AsDialOption verifies that JWT call +// credentials work correctly when passed as a dial option over a secure TLS +// transport. +func TestJWTCallCredentials_SecureTransport_AsDialOption(t *testing.T) { + token := createTestJWT(t, time.Now().Add(time.Hour)) + tokenFile := writeTempTokenFile(t, token) + + jwtCreds, err := jwt.NewTokenFileCallCredentials(tokenFile) + if err != nil { + t.Fatalf("NewTokenFileCallCredentials(%q) failed: %v", tokenFile, err) + } + + wantAuth := "Bearer " + token + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, fmt.Errorf("no metadata received") + } + authHeaders := md.Get("authorization") + if len(authHeaders) != 1 || authHeaders[0] != wantAuth { + return nil, fmt.Errorf("authorization header mismatch: got %v, want %q", authHeaders, wantAuth) + } + return &testpb.Empty{}, nil + }, + } + + serverCert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("Failed to load server cert: %v", err) + } + if err := ss.StartServer(grpc.Creds(credentials.NewServerTLSFromCert(&serverCert))); err != nil { + t.Fatalf("Failed to start server: %v", err) + } + defer ss.Stop() + + clientCreds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") + if err != nil { + t.Fatalf("Failed to create client TLS credentials: %v", err) + } + + cc, err := grpc.NewClient(ss.Address, grpc.WithTransportCredentials(clientCreds), grpc.WithPerRPCCredentials(jwtCreds)) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", ss.Address, err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} + +// TestJWTCallCredentials_SecureTransport_AsCallOption verifies that JWT call +// credentials work correctly when passed as a per-RPC call option over a secure +// TLS transport. +func TestJWTCallCredentials_SecureTransport_AsCallOption(t *testing.T) { + token := createTestJWT(t, time.Now().Add(time.Hour)) + tokenFile := writeTempTokenFile(t, token) + + jwtCreds, err := jwt.NewTokenFileCallCredentials(tokenFile) + if err != nil { + t.Fatalf("NewTokenFileCallCredentials(%q) failed: %v", tokenFile, err) + } + + wantAuth := "Bearer " + token + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, fmt.Errorf("no metadata received") + } + authHeaders := md.Get("authorization") + if len(authHeaders) != 1 || authHeaders[0] != wantAuth { + return nil, fmt.Errorf("authorization header mismatch: got %v, want %q", authHeaders, wantAuth) + } + return &testpb.Empty{}, nil + }, + } + + serverCert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("Failed to load server cert: %v", err) + } + if err := ss.StartServer(grpc.Creds(credentials.NewServerTLSFromCert(&serverCert))); err != nil { + t.Fatalf("Failed to start server: %v", err) + } + defer ss.Stop() + + clientCreds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") + if err != nil { + t.Fatalf("Failed to create client TLS credentials: %v", err) + } + + cc, err := grpc.NewClient(ss.Address, grpc.WithTransportCredentials(clientCreds)) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", ss.Address, err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(jwtCreds)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} + +// createTestJWT creates a test JWT token with the specified expiration. +func createTestJWT(t *testing.T, expiration time.Time) string { + t.Helper() + + claims := map[string]any{} + if !expiration.IsZero() { + claims["exp"] = expiration.Unix() + } + + header := map[string]any{ + "typ": "JWT", + "alg": "HS256", + } + headerBytes, err := json.Marshal(header) + if err != nil { + t.Fatalf("Failed to marshal header: %v", err) + } + + claimsBytes, err := json.Marshal(claims) + if err != nil { + t.Fatalf("Failed to marshal claims: %v", err) + } + + headerB64 := base64.URLEncoding.EncodeToString(headerBytes) + claimsB64 := base64.URLEncoding.EncodeToString(claimsBytes) + + // Remove padding for URL-safe base64. + headerB64 = strings.TrimRight(headerB64, "=") + claimsB64 = strings.TrimRight(claimsB64, "=") + + // For testing, we use a fake signature. + signature := base64.URLEncoding.EncodeToString([]byte("fake_signature")) + signature = strings.TrimRight(signature, "=") + + return fmt.Sprintf("%s.%s.%s", headerB64, claimsB64, signature) +} + +// writeTempTokenFile writes the token to a temporary file and returns the path. +func writeTempTokenFile(t *testing.T, token string) string { + t.Helper() + tempDir := t.TempDir() + filePath := filepath.Join(tempDir, "token") + if err := os.WriteFile(filePath, []byte(token), 0600); err != nil { + t.Fatalf("Failed to write temp token file: %v", err) + } + return filePath +} diff --git a/credentials/jwt/token_file_call_creds_test.go b/credentials/jwt/token_file_call_creds_test.go new file mode 100644 index 0000000..48c1b4f --- /dev/null +++ b/credentials/jwt/token_file_call_creds_test.go @@ -0,0 +1,555 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package jwt + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/status" +) + +const defaultTestTimeout = 5 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestNewTokenFileCallCredentialsValidFilepath(t *testing.T) { + creds, err := NewTokenFileCallCredentials("/path/to/token") + if err != nil { + t.Fatalf("NewTokenFileCallCredentials() unexpected error: %v", err) + } + if creds == nil { + t.Fatal("NewTokenFileCallCredentials() returned nil credentials") + } +} + +func (s) TestNewTokenFileCallCredentialsMissingFilepath(t *testing.T) { + if _, err := NewTokenFileCallCredentials(""); err == nil { + t.Fatalf("NewTokenFileCallCredentials() expected error, got nil") + } +} + +func (s) TestTokenFileCallCreds_RequireTransportSecurity(t *testing.T) { + creds, err := NewTokenFileCallCredentials("/path/to/token") + if err != nil { + t.Fatalf("NewTokenFileCallCredentials() failed: %v", err) + } + + if !creds.RequireTransportSecurity() { + t.Error("RequireTransportSecurity() = false, want true") + } +} + +func (s) TestTokenFileCallCreds_GetRequestMetadata(t *testing.T) { + now := time.Now().Truncate(time.Second) + tests := []struct { + name string + invalidTokenPath bool + tokenContent string + authInfo credentials.AuthInfo + wantCode codes.Code + wantMetadata map[string]string + }{ + { + name: "valid_token_with_future_expiration", + tokenContent: createTestJWT(t, now.Add(time.Hour)), + authInfo: &testAuthInfo{secLevel: credentials.PrivacyAndIntegrity}, + wantCode: codes.OK, + wantMetadata: map[string]string{"authorization": "Bearer " + createTestJWT(t, now.Add(time.Hour))}, + }, + { + name: "insufficient_security_level", + tokenContent: createTestJWT(t, now.Add(time.Hour)), + authInfo: &testAuthInfo{secLevel: credentials.NoSecurity}, + wantCode: codes.Unknown, // http2Client.getCallAuthData actually transforms such errors into into Unauthenticated + }, + { + name: "unreachable_token_file", + invalidTokenPath: true, + authInfo: &testAuthInfo{secLevel: credentials.PrivacyAndIntegrity}, + wantCode: codes.Unavailable, + }, + { + name: "empty_file", + tokenContent: "", + authInfo: &testAuthInfo{secLevel: credentials.PrivacyAndIntegrity}, + wantCode: codes.Unauthenticated, + }, + { + name: "malformed_JWT_token", + tokenContent: "bad contents", + authInfo: &testAuthInfo{secLevel: credentials.PrivacyAndIntegrity}, + wantCode: codes.Unauthenticated, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var tokenFile string + if tt.invalidTokenPath { + tokenFile = "/does-not-exist" + } else { + tokenFile = writeTempFile(t, "token", tt.tokenContent) + } + creds, err := NewTokenFileCallCredentials(tokenFile) + if err != nil { + t.Fatalf("NewTokenFileCallCredentials() failed: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: tt.authInfo, + }) + + metadata, err := creds.GetRequestMetadata(ctx) + if gotCode := status.Code(err); gotCode != tt.wantCode { + t.Fatalf("GetRequestMetadata() = %v, want %v", gotCode, tt.wantCode) + } + + if diff := cmp.Diff(tt.wantMetadata, metadata); diff != "" { + t.Errorf("GetRequestMetadata() returned unexpected metadata (-want +got):\n%s", diff) + } + }) + } +} + +func (s) TestTokenFileCallCreds_TokenCaching(t *testing.T) { + token := createTestJWT(t, time.Now().Add(time.Hour)) + tokenFile := writeTempFile(t, "token", token) + + creds, err := NewTokenFileCallCredentials(tokenFile) + if err != nil { + t.Fatalf("NewTokenFileCallCredentials() failed: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &testAuthInfo{secLevel: credentials.PrivacyAndIntegrity}, + }) + + // First call should read from file. + metadata1, err := creds.GetRequestMetadata(ctx) + if err != nil { + t.Fatalf("First GetRequestMetadata() failed: %v", err) + } + wantMetadata := map[string]string{"authorization": "Bearer " + token} + if diff := cmp.Diff(wantMetadata, metadata1); diff != "" { + t.Errorf("First GetRequestMetadata() returned unexpected metadata (-want +got):\n%s", diff) + } + + // Update the file with a different token. + newToken := createTestJWT(t, time.Now().Add(2*time.Hour)) + if err := os.WriteFile(tokenFile, []byte(newToken), 0600); err != nil { + t.Fatalf("Failed to update token file: %v", err) + } + + // Second call should return cached token (not the updated one). + metadata2, err := creds.GetRequestMetadata(ctx) + if err != nil { + t.Fatalf("Second GetRequestMetadata() failed: %v", err) + } + + if diff := cmp.Diff(metadata1, metadata2); diff != "" { + t.Errorf("Second GetRequestMetadata() returned unexpected metadata (-want +got):\n%s", diff) + } +} + +// testAuthInfo implements credentials.AuthInfo for testing. +type testAuthInfo struct { + secLevel credentials.SecurityLevel +} + +func (t *testAuthInfo) AuthType() string { + return "test" +} + +func (t *testAuthInfo) GetCommonAuthInfo() credentials.CommonAuthInfo { + return credentials.CommonAuthInfo{SecurityLevel: t.secLevel} +} + +// Tests that cached token expiration is set to 30 seconds before actual token +// expiration. +// TODO: Refactor the test to avoid inspecting and mutating internal state. +func (s) TestTokenFileCallCreds_CacheExpirationIsBeforeTokenExpiration(t *testing.T) { + // Create token that expires in 2 hours. + tokenExp := time.Now().Truncate(time.Second).Add(2 * time.Hour) + token := createTestJWT(t, tokenExp) + tokenFile := writeTempFile(t, "token", token) + + creds, err := NewTokenFileCallCredentials(tokenFile) + if err != nil { + t.Fatalf("NewTokenFileCallCredentials() failed: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &testAuthInfo{secLevel: credentials.PrivacyAndIntegrity}, + }) + + // Get token to trigger caching. + if _, err = creds.GetRequestMetadata(ctx); err != nil { + t.Fatalf("GetRequestMetadata() failed: %v", err) + } + + // Verify cached expiration is 30 seconds before actual token expiration. + impl := creds.(*jwtTokenFileCallCreds) + impl.mu.Lock() + cachedExp := impl.cachedExpiry + impl.mu.Unlock() + + wantExp := tokenExp.Add(-30 * time.Second) + if !cachedExp.Equal(wantExp) { + t.Errorf("Cache expiration = %v, want %v", cachedExp, wantExp) + } +} + +// Tests that pre-emptive refresh is triggered within 1 minute of expiration. +// This is tested as follows: +// * A token which expires "soon" is created. +// * On the first call to GetRequestMetadata, the token will get loaded and returned. +// * Another token is created and overwrites the file. +// * On the second call we will still return the (valid) first token but also +// detect that a refresh needs to happen and trigger it. +// * On the third call we confirm the new token has been loaded and returned. +func (s) TestTokenFileCallCreds_PreemptiveRefreshIsTriggered(t *testing.T) { + // Create token that expires in 80 seconds (=> cache expires in ~50s). + // This ensures pre-emptive refresh triggers since 50s < the 1 minute check. + tokenExp := time.Now().Add(80 * time.Second) + expiringToken := createTestJWT(t, tokenExp) + tokenFile := writeTempFile(t, "token", expiringToken) + + creds, err := NewTokenFileCallCredentials(tokenFile) + if err != nil { + t.Fatalf("NewTokenFileCallCredentials() failed: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &testAuthInfo{secLevel: credentials.PrivacyAndIntegrity}, + }) + + // First call should read from file synchronously. + metadata1, err := creds.GetRequestMetadata(ctx) + if err != nil { + t.Fatalf("GetRequestMetadata() failed: %v", err) + } + wantAuth1 := "Bearer " + expiringToken + gotAuth1 := metadata1["authorization"] + if gotAuth1 != wantAuth1 { + t.Fatalf("First call should return original token: got %q, want %q", gotAuth1, wantAuth1) + } + + // Verify token was cached and confirm expectation that refresh should be + // triggered. + impl := creds.(*jwtTokenFileCallCreds) + impl.mu.Lock() + cacheExp := impl.cachedExpiry + tokenCached := impl.cachedAuthHeader != "" + shouldTriggerRefresh := time.Until(cacheExp) < preemptiveRefreshThreshold + impl.mu.Unlock() + + if !tokenCached { + t.Fatal("Token should be cached after successful GetRequestMetadata") + } + + if !shouldTriggerRefresh { + timeUntilExp := time.Until(cacheExp) + t.Fatalf("Cache expires in %v; test precondition requires that this triggers preemptive refresh", timeUntilExp) + } + + // Create new token file with different expiration while refresh is + // happening. + newToken := createTestJWT(t, time.Now().Add(2*time.Hour)) + if err := os.WriteFile(tokenFile, []byte(newToken), 0600); err != nil { + t.Fatalf("Failed to write updated token file: %v", err) + } + + // Get token again - this call should trigger a refresh given that the first + // one was cached but expiring soon. + // However, the function should have returned right away with the current + // cached token because it is still valid and the preemptive refresh is + // meant to happen without blocking the RPC. + metadata2, err := creds.GetRequestMetadata(ctx) + if err != nil { + t.Fatalf("Second GetRequestMetadata() failed: %v", err) + } + wantAuth2 := wantAuth1 + gotAuth2 := metadata2["authorization"] + if gotAuth2 != wantAuth2 { + t.Fatalf("Second call should return the original token: got %q, want %q", gotAuth2, wantAuth2) + } + + // Now should get the new token which was refreshed in the background. + wantAuth3 := "Bearer " + newToken + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &testAuthInfo{secLevel: credentials.PrivacyAndIntegrity}, + }) + for ; ; <-time.After(time.Millisecond) { + if ctx.Err() != nil { + t.Fatal("Context deadline expired before pre-emptive refresh completed") + } + // If the newly returned metadata is different to the old one, verify + // that it matches the token from the updated file. If not, fail the + // test. + metadata3, err := creds.GetRequestMetadata(ctx) + if err != nil { + t.Fatalf("Second GetRequestMetadata() failed: %v", err) + } + // Pre-emptive refresh not completed yet, try again. + gotAuth3 := metadata3["authorization"] + if gotAuth3 == gotAuth2 { + continue + } + if gotAuth3 != wantAuth3 { + t.Fatalf("Third call should return the new token: got %q, want %q", gotAuth3, wantAuth3) + } + break + } +} + +// Tests that backoff behavior handles file read errors correctly. +// It has the following expectations: +// First call to GetRequestMetadata() fails with UNAVAILABLE due to a +// missing file. +// Second call to GetRequestMetadata() fails with UNAVAILABLE due backoff. +// Third call to GetRequestMetadata() fails with UNAVAILABLE due to retry. +// Fourth call to GetRequestMetadata() fails with UNAVAILABLE due to backoff +// even though file exists. +// Fifth call to GetRequestMetadata() succeeds after reading the file and +// backoff has expired. +// TODO: Refactor the test to avoid inspecting and mutating internal state. +func (s) TestTokenFileCallCreds_BackoffBehavior(t *testing.T) { + tempDir := t.TempDir() + nonExistentFile := filepath.Join(tempDir, "nonexistent") + + creds, err := NewTokenFileCallCredentials(nonExistentFile) + if err != nil { + t.Fatalf("NewTokenFileCallCredentials() failed: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &testAuthInfo{secLevel: credentials.PrivacyAndIntegrity}, + }) + + // First call should fail with UNAVAILABLE. + beforeCallRetryTime := time.Now() + _, err = creds.GetRequestMetadata(ctx) + if err == nil { + t.Fatal("Expected error from nonexistent file") + } + if status.Code(err) != codes.Unavailable { + t.Fatalf("GetRequestMetadata() = %v, want UNAVAILABLE", status.Code(err)) + } + + // Verify error is cached internally. + impl := creds.(*jwtTokenFileCallCreds) + impl.mu.Lock() + retryAttempt := impl.retryAttempt + nextRetryTime := impl.nextRetryTime + impl.mu.Unlock() + + if retryAttempt != 1 { + t.Errorf("Expected retry attempt to be 1, got %d", retryAttempt) + } + if !nextRetryTime.After(beforeCallRetryTime) { + t.Error("Next retry time should be set to a time after the first call") + } + + // Second call should still return cached error and not retry. + // Set nextRetryTime far enough in the future to ensure that's the case. + impl.mu.Lock() + impl.nextRetryTime = time.Now().Add(1 * time.Minute) + wantNextRetryTime := impl.nextRetryTime + impl.mu.Unlock() + _, err = creds.GetRequestMetadata(ctx) + if err == nil { + t.Fatalf("creds.GetRequestMetadata() = %v, want non-nil", err) + } + if status.Code(err) != codes.Unavailable { + t.Fatalf("GetRequestMetadata() = %v, want cached UNAVAILABLE", status.Code(err)) + } + + impl.mu.Lock() + retryAttempt2 := impl.retryAttempt + nextRetryTime2 := impl.nextRetryTime + impl.mu.Unlock() + if !nextRetryTime2.Equal(wantNextRetryTime) { + t.Errorf("nextRetryTime should not change due to backoff. Got: %v, Want: %v", nextRetryTime2, wantNextRetryTime) + } + if retryAttempt2 != 1 { + t.Error("Retry attempt should not change due to backoff") + } + + // Third call should retry but still fail with UNAVAILABLE. + // Set the backoff retry time in the past to allow next retry attempt. + impl.mu.Lock() + impl.nextRetryTime = time.Now().Add(-1 * time.Minute) + beforeCallRetryTime = impl.nextRetryTime + impl.mu.Unlock() + _, err = creds.GetRequestMetadata(ctx) + if err == nil { + t.Fatalf("creds.GetRequestMetadata() = %v, want non-nil", err) + } + if status.Code(err) != codes.Unavailable { + t.Fatalf("GetRequestMetadata() = %v, want cached UNAVAILABLE", status.Code(err)) + } + + impl.mu.Lock() + retryAttempt3 := impl.retryAttempt + nextRetryTime3 := impl.nextRetryTime + impl.mu.Unlock() + + if !nextRetryTime3.After(beforeCallRetryTime) { + t.Error("nextRetryTime3 should have been updated after third call") + } + if retryAttempt3 != 2 { + t.Error("Expected retry attempt to increase after retry") + } + + // Create valid token file. + validToken := createTestJWT(t, time.Now().Add(time.Hour)) + if err := os.WriteFile(nonExistentFile, []byte(validToken), 0600); err != nil { + t.Fatalf("Failed to create valid token file: %v", err) + } + + // Fourth call should still fail even though the file now exists due to backoff. + // Set nextRetryTime far enough in the future to ensure that's the case. + _, err = creds.GetRequestMetadata(ctx) + impl.mu.Lock() + impl.nextRetryTime = time.Now().Add(1 * time.Minute) + wantNextRetryTime = impl.nextRetryTime + impl.mu.Unlock() + if err == nil { + t.Fatalf("creds.GetRequestMetadata() = %v, want non-nil", err) + } + if status.Code(err) != codes.Unavailable { + t.Fatalf("GetRequestMetadata() = %v, want cached UNAVAILABLE", status.Code(err)) + } + + impl.mu.Lock() + retryAttempt4 := impl.retryAttempt + nextRetryTime4 := impl.nextRetryTime + impl.mu.Unlock() + + if !nextRetryTime4.Equal(wantNextRetryTime) { + t.Errorf("nextRetryTime should not change due to backoff. Got: %v, Want: %v", nextRetryTime4, wantNextRetryTime) + } + if retryAttempt4 != retryAttempt3 { + t.Error("Retry attempt should not change due to backoff") + } + + // Fifth call should succeed since the file now exists and the backoff has + // expired. + // Set the backoff retry time in the past to allow next retry attempt. + impl.mu.Lock() + impl.nextRetryTime = time.Now().Add(-1 * time.Minute) + impl.mu.Unlock() + _, err = creds.GetRequestMetadata(ctx) + if err != nil { + t.Fatalf("After creating valid token file, backoff should expire and trigger a token reload on the next RPC. GetRequestMetadata() should eventually succeed, but got: %v", err) + } + // If successful, verify error cache and backoff state were cleared. + impl.mu.Lock() + clearedErr := impl.cachedError + retryAttempt = impl.retryAttempt + nextRetryTime = impl.nextRetryTime + impl.mu.Unlock() + + if clearedErr != nil { + t.Errorf("After successful retry, cached error should be cleared, got: %v", clearedErr) + } + if retryAttempt != 0 { + t.Errorf("After successful retry, retry attempt should be reset, got: %d", retryAttempt) + } + if !nextRetryTime.IsZero() { + t.Error("After successful retry, next retry time should be cleared") + } +} + +// createTestJWT creates a test JWT token with the specified expiration. +func createTestJWT(t *testing.T, expiration time.Time) string { + t.Helper() + + claims := map[string]any{} + if !expiration.IsZero() { + claims["exp"] = expiration.Unix() + } + + header := map[string]any{ + "typ": "JWT", + "alg": "HS256", + } + headerBytes, err := json.Marshal(header) + if err != nil { + t.Fatalf("Failed to marshal header: %v", err) + } + + claimsBytes, err := json.Marshal(claims) + if err != nil { + t.Fatalf("Failed to marshal claims: %v", err) + } + + headerB64 := base64.URLEncoding.EncodeToString(headerBytes) + claimsB64 := base64.URLEncoding.EncodeToString(claimsBytes) + + // Remove padding for URL-safe base64 + headerB64 = strings.TrimRight(headerB64, "=") + claimsB64 = strings.TrimRight(claimsB64, "=") + + // For testing, we'll use a fake signature + signature := base64.URLEncoding.EncodeToString([]byte("fake_signature")) + signature = strings.TrimRight(signature, "=") + + return fmt.Sprintf("%s.%s.%s", headerB64, claimsB64, signature) +} + +func writeTempFile(t *testing.T, name, content string) string { + t.Helper() + tempDir := t.TempDir() + filePath := filepath.Join(tempDir, name) + if err := os.WriteFile(filePath, []byte(content), 0600); err != nil { + t.Fatalf("Failed to write temp file: %v", err) + } + return filePath +} diff --git a/credentials/local/local.go b/credentials/local/local.go new file mode 100644 index 0000000..2fe0215 --- /dev/null +++ b/credentials/local/local.go @@ -0,0 +1,121 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package local implements local transport credentials. +// Local credentials reports the security level based on the type +// of connection. If the connection is local TCP, NoSecurity will be +// reported, and if the connection is UDS, PrivacyAndIntegrity will be +// reported. If local credentials is not used in local connections +// (local TCP or UDS), it will fail. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +package local + +import ( + "context" + "fmt" + "net" + "strings" + + "google.golang.org/grpc/credentials" +) + +// info contains the auth information for a local connection. +// It implements the AuthInfo interface. +type info struct { + credentials.CommonAuthInfo +} + +// AuthType returns the type of info as a string. +func (info) AuthType() string { + return "local" +} + +// ValidateAuthority allows any value to be overridden for the :authority +// header. +func (info) ValidateAuthority(string) error { + return nil +} + +// localTC is the credentials required to establish a local connection. +type localTC struct { + info credentials.ProtocolInfo +} + +func (c *localTC) Info() credentials.ProtocolInfo { + return c.info +} + +// getSecurityLevel returns the security level for a local connection. +// It returns an error if a connection is not local. +func getSecurityLevel(network, addr string) (credentials.SecurityLevel, error) { + switch { + // Local TCP connection + case strings.HasPrefix(addr, "127."), strings.HasPrefix(addr, "[::1]:"): + return credentials.NoSecurity, nil + // Windows named pipe connection + case network == "pipe" && strings.HasPrefix(addr, `\\.\pipe\`): + return credentials.NoSecurity, nil + // UDS connection + case network == "unix": + return credentials.PrivacyAndIntegrity, nil + // Not a local connection and should fail + default: + return credentials.InvalidSecurityLevel, fmt.Errorf("local credentials rejected connection to non-local address %q", addr) + } +} + +func (*localTC) ClientHandshake(_ context.Context, _ string, conn net.Conn) (net.Conn, credentials.AuthInfo, error) { + secLevel, err := getSecurityLevel(conn.RemoteAddr().Network(), conn.RemoteAddr().String()) + if err != nil { + return nil, nil, err + } + return conn, info{credentials.CommonAuthInfo{SecurityLevel: secLevel}}, nil +} + +func (*localTC) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) { + secLevel, err := getSecurityLevel(conn.RemoteAddr().Network(), conn.RemoteAddr().String()) + if err != nil { + return nil, nil, err + } + return conn, info{credentials.CommonAuthInfo{SecurityLevel: secLevel}}, nil +} + +// NewCredentials returns a local credential implementing credentials.TransportCredentials. +func NewCredentials() credentials.TransportCredentials { + return &localTC{ + info: credentials.ProtocolInfo{ + SecurityProtocol: "local", + }, + } +} + +// Clone makes a copy of Local credentials. +func (c *localTC) Clone() credentials.TransportCredentials { + return &localTC{info: c.info} +} + +// OverrideServerName overrides the server name used to verify the hostname on the returned certificates from the server. +// Since this feature is specific to TLS (SNI + hostname verification check), it does not take any effect for local credentials. +func (c *localTC) OverrideServerName(serverNameOverride string) error { + c.info.ServerName = serverNameOverride + return nil +} diff --git a/credentials/local/local_test.go b/credentials/local/local_test.go new file mode 100644 index 0000000..47f8dbb --- /dev/null +++ b/credentials/local/local_test.go @@ -0,0 +1,225 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package local + +import ( + "context" + "fmt" + "net" + "runtime" + "strings" + "testing" + "time" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/grpctest" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestGetSecurityLevel(t *testing.T) { + testCases := []struct { + testNetwork string + testAddr string + want credentials.SecurityLevel + }{ + { + testNetwork: "tcp", + testAddr: "127.0.0.1:10000", + want: credentials.NoSecurity, + }, + { + testNetwork: "tcp", + testAddr: "[::1]:10000", + want: credentials.NoSecurity, + }, + { + testNetwork: "unix", + testAddr: "/tmp/grpc_fullstack_test", + want: credentials.PrivacyAndIntegrity, + }, + { + testNetwork: "tcp", + testAddr: "192.168.0.1:10000", + want: credentials.InvalidSecurityLevel, + }, + } + for _, tc := range testCases { + got, _ := getSecurityLevel(tc.testNetwork, tc.testAddr) + if got != tc.want { + t.Fatalf("GetSeurityLevel(%s, %s) returned %s but want %s", tc.testNetwork, tc.testAddr, got.String(), tc.want.String()) + } + } +} + +type serverHandshake func(net.Conn) (credentials.AuthInfo, error) + +func getSecurityLevelFromAuthInfo(ai credentials.AuthInfo) credentials.SecurityLevel { + if c, ok := ai.(interface { + GetCommonAuthInfo() credentials.CommonAuthInfo + }); ok { + return c.GetCommonAuthInfo().SecurityLevel + } + return credentials.InvalidSecurityLevel +} + +// Server local handshake implementation. +func serverLocalHandshake(conn net.Conn) (credentials.AuthInfo, error) { + cred := NewCredentials() + _, authInfo, err := cred.ServerHandshake(conn) + if err != nil { + return nil, err + } + return authInfo, nil +} + +// Client local handshake implementation. +func clientLocalHandshake(conn net.Conn, lisAddr string) (credentials.AuthInfo, error) { + cred := NewCredentials() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + _, authInfo, err := cred.ClientHandshake(ctx, lisAddr, conn) + if err != nil { + return nil, err + } + return authInfo, nil +} + +// Client connects to a server with local credentials. +func clientHandle(hs func(net.Conn, string) (credentials.AuthInfo, error), network, lisAddr string) (credentials.AuthInfo, error) { + conn, _ := net.Dial(network, lisAddr) + defer conn.Close() + clientAuthInfo, err := hs(conn, lisAddr) + if err != nil { + return nil, fmt.Errorf("Error on client while handshake") + } + return clientAuthInfo, nil +} + +type testServerHandleResult struct { + authInfo credentials.AuthInfo + err error +} + +// Server accepts a client's connection with local credentials. +func serverHandle(hs serverHandshake, done chan testServerHandleResult, lis net.Listener) { + serverRawConn, err := lis.Accept() + if err != nil { + done <- testServerHandleResult{authInfo: nil, err: fmt.Errorf("Server failed to accept connection. Error: %v", err)} + return + } + serverAuthInfo, err := hs(serverRawConn) + if err != nil { + serverRawConn.Close() + done <- testServerHandleResult{authInfo: nil, err: fmt.Errorf("Server failed while handshake. Error: %v", err)} + return + } + done <- testServerHandleResult{authInfo: serverAuthInfo, err: nil} +} + +func serverAndClientHandshake(lis net.Listener) (credentials.SecurityLevel, error) { + done := make(chan testServerHandleResult, 1) + const timeout = 5 * time.Second + timer := time.NewTimer(timeout) + defer timer.Stop() + go serverHandle(serverLocalHandshake, done, lis) + defer lis.Close() + clientAuthInfo, err := clientHandle(clientLocalHandshake, lis.Addr().Network(), lis.Addr().String()) + if err != nil { + return credentials.InvalidSecurityLevel, fmt.Errorf("Error at client-side: %v", err) + } + select { + case <-timer.C: + return credentials.InvalidSecurityLevel, fmt.Errorf("Test didn't finish in time") + case serverHandleResult := <-done: + if serverHandleResult.err != nil { + return credentials.InvalidSecurityLevel, fmt.Errorf("Error at server-side: %v", serverHandleResult.err) + } + clientSecLevel := getSecurityLevelFromAuthInfo(clientAuthInfo) + serverSecLevel := getSecurityLevelFromAuthInfo(serverHandleResult.authInfo) + + if clientSecLevel == credentials.InvalidSecurityLevel { + return credentials.InvalidSecurityLevel, fmt.Errorf("Error at client-side: client's AuthInfo does not implement GetCommonAuthInfo()") + } + if serverSecLevel == credentials.InvalidSecurityLevel { + return credentials.InvalidSecurityLevel, fmt.Errorf("Error at server-side: server's AuthInfo does not implement GetCommonAuthInfo()") + } + if clientSecLevel != serverSecLevel { + return credentials.InvalidSecurityLevel, fmt.Errorf("client's AuthInfo contains %s but server's AuthInfo contains %s", clientSecLevel.String(), serverSecLevel.String()) + } + return clientSecLevel, nil + } +} + +func (s) TestServerAndClientHandshake(t *testing.T) { + testCases := []struct { + testNetwork string + testAddr string + want credentials.SecurityLevel + }{ + { + testNetwork: "tcp", + testAddr: "127.0.0.1:0", + want: credentials.NoSecurity, + }, + { + testNetwork: "tcp", + testAddr: "[::1]:0", + want: credentials.NoSecurity, + }, + { + testNetwork: "tcp", + testAddr: "localhost:0", + want: credentials.NoSecurity, + }, + { + testNetwork: "unix", + testAddr: fmt.Sprintf("/tmp/grpc_fullstck_test%d", time.Now().UnixNano()), + want: credentials.PrivacyAndIntegrity, + }, + } + for _, tc := range testCases { + if runtime.GOOS == "windows" && tc.testNetwork == "unix" { + t.Skip("skipping tests for unix connections on Windows") + } + t.Run("serverAndClientHandshakeResult", func(t *testing.T) { + lis, err := net.Listen(tc.testNetwork, tc.testAddr) + if err != nil { + if strings.Contains(err.Error(), "bind: cannot assign requested address") || + strings.Contains(err.Error(), "socket: address family not supported by protocol") { + t.Skipf("no support for address %v", tc.testAddr) + } + t.Fatalf("Failed to listen: %v", err) + } + got, err := serverAndClientHandshake(lis) + if got != tc.want { + t.Fatalf("serverAndClientHandshake(%s, %s) = %v, %v; want %v, nil", tc.testNetwork, tc.testAddr, got, err, tc.want) + } + }) + } +} diff --git a/credentials/oauth/oauth.go b/credentials/oauth/oauth.go new file mode 100644 index 0000000..328b838 --- /dev/null +++ b/credentials/oauth/oauth.go @@ -0,0 +1,244 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package oauth implements gRPC credentials using OAuth. +package oauth + +import ( + "context" + "fmt" + "net/url" + "os" + "sync" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" + "google.golang.org/grpc/credentials" +) + +// TokenSource supplies PerRPCCredentials from an oauth2.TokenSource. +type TokenSource struct { + oauth2.TokenSource +} + +// GetRequestMetadata gets the request metadata as a map from a TokenSource. +func (ts TokenSource) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) { + token, err := ts.Token() + if err != nil { + return nil, err + } + ri, _ := credentials.RequestInfoFromContext(ctx) + if err = credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil { + return nil, fmt.Errorf("unable to transfer TokenSource PerRPCCredentials: %v", err) + } + return map[string]string{ + "authorization": token.Type() + " " + token.AccessToken, + }, nil +} + +// RequireTransportSecurity indicates whether the credentials requires transport security. +func (ts TokenSource) RequireTransportSecurity() bool { + return true +} + +// removeServiceNameFromJWTURI removes RPC service name from URI. +func removeServiceNameFromJWTURI(uri string) (string, error) { + parsed, err := url.Parse(uri) + if err != nil { + return "", err + } + parsed.Path = "/" + return parsed.String(), nil +} + +type jwtAccess struct { + jsonKey []byte +} + +// NewJWTAccessFromFile creates PerRPCCredentials from the given keyFile. +func NewJWTAccessFromFile(keyFile string) (credentials.PerRPCCredentials, error) { + jsonKey, err := os.ReadFile(keyFile) + if err != nil { + return nil, fmt.Errorf("credentials: failed to read the service account key file: %v", err) + } + return NewJWTAccessFromKey(jsonKey) +} + +// NewJWTAccessFromKey creates PerRPCCredentials from the given jsonKey. +func NewJWTAccessFromKey(jsonKey []byte) (credentials.PerRPCCredentials, error) { + return jwtAccess{jsonKey}, nil +} + +func (j jwtAccess) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + // Remove RPC service name from URI that will be used as audience + // in a self-signed JWT token. It follows https://google.aip.dev/auth/4111. + aud, err := removeServiceNameFromJWTURI(uri[0]) + if err != nil { + return nil, err + } + // TODO: the returned TokenSource is reusable. Store it in a sync.Map, with + // uri as the key, to avoid recreating for every RPC. + ts, err := google.JWTAccessTokenSourceFromJSON(j.jsonKey, aud) + if err != nil { + return nil, err + } + token, err := ts.Token() + if err != nil { + return nil, err + } + ri, _ := credentials.RequestInfoFromContext(ctx) + if err = credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil { + return nil, fmt.Errorf("unable to transfer jwtAccess PerRPCCredentials: %v", err) + } + return map[string]string{ + "authorization": token.Type() + " " + token.AccessToken, + }, nil +} + +func (j jwtAccess) RequireTransportSecurity() bool { + return true +} + +// oauthAccess supplies PerRPCCredentials from a given token. +type oauthAccess struct { + token oauth2.Token +} + +// NewOauthAccess constructs the PerRPCCredentials using a given token. +// +// Deprecated: use oauth.TokenSource instead. +func NewOauthAccess(token *oauth2.Token) credentials.PerRPCCredentials { + return oauthAccess{token: *token} +} + +func (oa oauthAccess) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) { + ri, _ := credentials.RequestInfoFromContext(ctx) + if err := credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil { + return nil, fmt.Errorf("unable to transfer oauthAccess PerRPCCredentials: %v", err) + } + return map[string]string{ + "authorization": oa.token.Type() + " " + oa.token.AccessToken, + }, nil +} + +func (oa oauthAccess) RequireTransportSecurity() bool { + return true +} + +// NewComputeEngine constructs the PerRPCCredentials that fetches access tokens from +// Google Compute Engine (GCE)'s metadata server. It is only valid to use this +// if your program is running on a GCE instance. +// TODO(dsymonds): Deprecate and remove this. +func NewComputeEngine() credentials.PerRPCCredentials { + return TokenSource{google.ComputeTokenSource("")} +} + +// serviceAccount represents PerRPCCredentials via JWT signing key. +type serviceAccount struct { + mu sync.Mutex + config *jwt.Config + t *oauth2.Token +} + +func (s *serviceAccount) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.t.Valid() { + var err error + s.t, err = s.config.TokenSource(ctx).Token() + if err != nil { + return nil, err + } + } + ri, _ := credentials.RequestInfoFromContext(ctx) + if err := credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil { + return nil, fmt.Errorf("unable to transfer serviceAccount PerRPCCredentials: %v", err) + } + return map[string]string{ + "authorization": s.t.Type() + " " + s.t.AccessToken, + }, nil +} + +func (s *serviceAccount) RequireTransportSecurity() bool { + return true +} + +// NewServiceAccountFromKey constructs the PerRPCCredentials using the JSON key slice +// from a Google Developers service account. +func NewServiceAccountFromKey(jsonKey []byte, scope ...string) (credentials.PerRPCCredentials, error) { + config, err := google.JWTConfigFromJSON(jsonKey, scope...) + if err != nil { + return nil, err + } + return &serviceAccount{config: config}, nil +} + +// NewServiceAccountFromFile constructs the PerRPCCredentials using the JSON key file +// of a Google Developers service account. +func NewServiceAccountFromFile(keyFile string, scope ...string) (credentials.PerRPCCredentials, error) { + jsonKey, err := os.ReadFile(keyFile) + if err != nil { + return nil, fmt.Errorf("credentials: failed to read the service account key file: %v", err) + } + return NewServiceAccountFromKey(jsonKey, scope...) +} + +// NewApplicationDefault returns "Application Default Credentials". For more +// detail, see https://developers.google.com/accounts/docs/application-default-credentials. +func NewApplicationDefault(ctx context.Context, scope ...string) (credentials.PerRPCCredentials, error) { + creds, err := google.FindDefaultCredentials(ctx, scope...) + if err != nil { + return nil, err + } + + // If JSON is nil, the authentication is provided by the environment and not + // with a credentials file, e.g. when code is running on Google Cloud + // Platform. Use the returned token source. + if creds.JSON == nil { + return TokenSource{creds.TokenSource}, nil + } + + // If auth is provided by env variable or creds file, the behavior will be + // different based on whether scope is set. Because the returned + // creds.TokenSource does oauth with jwt by default, and it requires scope. + // We can only use it if scope is not empty, otherwise it will fail with + // missing scope error. + // + // If scope is set, use it, it should just work. + // + // If scope is not set, we try to use jwt directly without oauth (this only + // works if it's a service account). + + if len(scope) != 0 { + return TokenSource{creds.TokenSource}, nil + } + + // Try to convert JSON to a jwt config without setting the optional scope + // parameter to check if it's a service account (the function errors if it's + // not). This is necessary because the returned config doesn't show the type + // of the account. + if _, err := google.JWTConfigFromJSON(creds.JSON); err != nil { + // If this fails, it's not a service account, return the original + // TokenSource from above. + return TokenSource{creds.TokenSource}, nil + } + + // If it's a service account, create a JWT only access with the key. + return NewJWTAccessFromKey(creds.JSON) +} diff --git a/credentials/oauth/oauth_test.go b/credentials/oauth/oauth_test.go new file mode 100644 index 0000000..7e62ecb --- /dev/null +++ b/credentials/oauth/oauth_test.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package oauth + +import ( + "strings" + "testing" +) + +func checkErrorMsg(err error, msg string) bool { + if err == nil && msg == "" { + return true + } else if err != nil { + return strings.Contains(err.Error(), msg) + } + return false +} + +func TestRemoveServiceNameFromJWTURI(t *testing.T) { + tests := []struct { + name string + uri string + wantedURI string + wantedErrMsg string + }{ + { + name: "invalid URI", + uri: "ht tp://foo.com", + wantedErrMsg: "first path segment in URL cannot contain colon", + }, + { + name: "valid URI", + uri: "https://foo.com/go/", + wantedURI: "https://foo.com/", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := removeServiceNameFromJWTURI(tt.uri); got != tt.wantedURI || !checkErrorMsg(err, tt.wantedErrMsg) { + t.Errorf("RemoveServiceNameFromJWTURI() = %s, %v, want %s, %v", got, err, tt.wantedURI, tt.wantedErrMsg) + } + }) + } +} diff --git a/credentials/sts/sts.go b/credentials/sts/sts.go new file mode 100644 index 0000000..ec29a56 --- /dev/null +++ b/credentials/sts/sts.go @@ -0,0 +1,407 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package sts implements call credentials using STS (Security Token Service) as +// defined in https://tools.ietf.org/html/rfc8693. +// +// # Experimental +// +// Notice: All APIs in this package are experimental and may be changed or +// removed in a later release. +package sts + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "sync" + "time" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" +) + +const ( + // HTTP request timeout set on the http.Client used to make STS requests. + stsRequestTimeout = 5 * time.Second + // If lifetime left in a cached token is lesser than this value, we fetch a + // new one instead of returning the current one. + minCachedTokenLifetime = 300 * time.Second + + tokenExchangeGrantType = "urn:ietf:params:oauth:grant-type:token-exchange" + defaultCloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform" +) + +// For overriding in tests. +var ( + loadSystemCertPool = x509.SystemCertPool + makeHTTPDoer = makeHTTPClient + readSubjectTokenFrom = os.ReadFile + readActorTokenFrom = os.ReadFile + logger = grpclog.Component("credentials") +) + +// Options configures the parameters used for an STS based token exchange. +type Options struct { + // TokenExchangeServiceURI is the address of the server which implements STS + // token exchange functionality. + TokenExchangeServiceURI string // Required. + + // Resource is a URI that indicates the target service or resource where the + // client intends to use the requested security token. + Resource string // Optional. + + // Audience is the logical name of the target service where the client + // intends to use the requested security token + Audience string // Optional. + + // Scope is a list of space-delimited, case-sensitive strings, that allow + // the client to specify the desired scope of the requested security token + // in the context of the service or resource where the token will be used. + // If this field is left unspecified, a default value of + // https://www.googleapis.com/auth/cloud-platform will be used. + Scope string // Optional. + + // RequestedTokenType is an identifier, as described in + // https://tools.ietf.org/html/rfc8693#section-3, that indicates the type of + // the requested security token. + RequestedTokenType string // Optional. + + // SubjectTokenPath is a filesystem path which contains the security token + // that represents the identity of the party on behalf of whom the request + // is being made. + SubjectTokenPath string // Required. + + // SubjectTokenType is an identifier, as described in + // https://tools.ietf.org/html/rfc8693#section-3, that indicates the type of + // the security token in the "subject_token_path" parameter. + SubjectTokenType string // Required. + + // ActorTokenPath is a security token that represents the identity of the + // acting party. + ActorTokenPath string // Optional. + + // ActorTokenType is an identifier, as described in + // https://tools.ietf.org/html/rfc8693#section-3, that indicates the type of + // the security token in the "actor_token_path" parameter. + ActorTokenType string // Optional. +} + +func (o Options) String() string { + return fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s:%s:%s", o.TokenExchangeServiceURI, o.Resource, o.Audience, o.Scope, o.RequestedTokenType, o.SubjectTokenPath, o.SubjectTokenType, o.ActorTokenPath, o.ActorTokenType) +} + +// NewCredentials returns a new PerRPCCredentials implementation, configured +// using opts, which performs token exchange using STS. +func NewCredentials(opts Options) (credentials.PerRPCCredentials, error) { + if err := validateOptions(opts); err != nil { + return nil, err + } + + // Load the system roots to validate the certificate presented by the STS + // endpoint during the TLS handshake. + roots, err := loadSystemCertPool() + if err != nil { + return nil, err + } + + return &callCreds{ + opts: opts, + client: makeHTTPDoer(roots), + }, nil +} + +// callCreds provides the implementation of call credentials based on an STS +// token exchange. +type callCreds struct { + opts Options + client httpDoer + + // Cached accessToken to avoid an STS token exchange for every call to + // GetRequestMetadata. + mu sync.Mutex + tokenMetadata map[string]string + tokenExpiry time.Time +} + +// GetRequestMetadata returns the cached accessToken, if available and valid, or +// fetches a new one by performing an STS token exchange. +func (c *callCreds) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) { + ri, _ := credentials.RequestInfoFromContext(ctx) + if err := credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil { + return nil, fmt.Errorf("unable to transfer STS PerRPCCredentials: %v", err) + } + + // Holding the lock for the whole duration of the STS request and response + // processing ensures that concurrent RPCs don't end up in multiple + // requests being made. + c.mu.Lock() + defer c.mu.Unlock() + + if md := c.cachedMetadata(); md != nil { + return md, nil + } + req, err := constructRequest(ctx, c.opts) + if err != nil { + return nil, err + } + respBody, err := sendRequest(c.client, req) + if err != nil { + return nil, err + } + ti, err := tokenInfoFromResponse(respBody) + if err != nil { + return nil, err + } + c.tokenMetadata = map[string]string{"Authorization": fmt.Sprintf("%s %s", ti.tokenType, ti.token)} + c.tokenExpiry = ti.expiryTime + return c.tokenMetadata, nil +} + +// RequireTransportSecurity indicates whether the credentials requires +// transport security. +func (c *callCreds) RequireTransportSecurity() bool { + return true +} + +// httpDoer wraps the single method on the http.Client type that we use. This +// helps with overriding in unittests. +type httpDoer interface { + Do(req *http.Request) (*http.Response, error) +} + +func makeHTTPClient(roots *x509.CertPool) httpDoer { + return &http.Client{ + // The token exchange request carries the subject token (and an optional + // actor token) in its body. Following a redirect replays that body to + // the redirect target, so refuse to follow redirects and let sendRequest + // surface the 3xx as a non-2xx status instead. + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + Timeout: stsRequestTimeout, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: roots, + }, + }, + } +} + +// validateOptions performs the following validation checks on opts: +// - tokenExchangeServiceURI is not empty +// - tokenExchangeServiceURI is a valid URI with a http(s) scheme +// - subjectTokenPath and subjectTokenType are not empty. +func validateOptions(opts Options) error { + if opts.TokenExchangeServiceURI == "" { + return errors.New("empty token_exchange_service_uri in options") + } + u, err := url.Parse(opts.TokenExchangeServiceURI) + if err != nil { + return err + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("scheme is not supported: %q. Only http(s) is supported", u.Scheme) + } + + if opts.SubjectTokenPath == "" { + return errors.New("required field SubjectTokenPath is not specified") + } + if opts.SubjectTokenType == "" { + return errors.New("required field SubjectTokenType is not specified") + } + return nil +} + +// cachedMetadata returns the cached metadata provided it is not going to +// expire anytime soon. +// +// Caller must hold c.mu. +func (c *callCreds) cachedMetadata() map[string]string { + now := time.Now() + // If the cached token has not expired and the lifetime remaining on that + // token is greater than the minimum value we are willing to accept, go + // ahead and use it. + if c.tokenExpiry.After(now) && c.tokenExpiry.Sub(now) > minCachedTokenLifetime { + return c.tokenMetadata + } + return nil +} + +// constructRequest creates the STS request body in JSON based on the provided +// options. +// - Contents of the subjectToken are read from the file specified in +// options. If we encounter an error here, we bail out. +// - Contents of the actorToken are read from the file specified in options. +// If we encounter an error here, we ignore this field because this is +// optional. +// - Most of the other fields in the request come directly from options. +// +// A new HTTP request is created by calling http.NewRequestWithContext() and +// passing the provided context, thereby enforcing any timeouts specified in +// the latter. +func constructRequest(ctx context.Context, opts Options) (*http.Request, error) { + subToken, err := readSubjectTokenFrom(opts.SubjectTokenPath) + if err != nil { + return nil, err + } + reqScope := opts.Scope + if reqScope == "" { + reqScope = defaultCloudPlatformScope + } + reqParams := &requestParameters{ + GrantType: tokenExchangeGrantType, + Resource: opts.Resource, + Audience: opts.Audience, + Scope: reqScope, + RequestedTokenType: opts.RequestedTokenType, + SubjectToken: string(subToken), + SubjectTokenType: opts.SubjectTokenType, + } + if opts.ActorTokenPath != "" { + actorToken, err := readActorTokenFrom(opts.ActorTokenPath) + if err != nil { + return nil, err + } + reqParams.ActorToken = string(actorToken) + reqParams.ActorTokenType = opts.ActorTokenType + } + jsonBody, err := json.Marshal(reqParams) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, "POST", opts.TokenExchangeServiceURI, bytes.NewBuffer(jsonBody)) + if err != nil { + return nil, fmt.Errorf("failed to create http request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + return req, nil +} + +func sendRequest(client httpDoer, req *http.Request) ([]byte, error) { + // http.Client returns a non-nil error only if it encounters an error + // caused by client policy (such as CheckRedirect), or failure to speak + // HTTP (such as a network connectivity problem). A non-2xx status code + // doesn't cause an error. + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + // When the http.Client returns a non-nil error, it is the + // responsibility of the caller to read the response body till an EOF is + // encountered and to close it. + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusOK { + return body, nil + } + logger.Warningf("http status %d, body: %s", resp.StatusCode, string(body)) + return nil, fmt.Errorf("http status %d, body: %s", resp.StatusCode, string(body)) +} + +func tokenInfoFromResponse(respBody []byte) (*tokenInfo, error) { + respData := &responseParameters{} + if err := json.Unmarshal(respBody, respData); err != nil { + return nil, fmt.Errorf("json.Unmarshal(%v): %v", respBody, err) + } + if respData.AccessToken == "" { + return nil, fmt.Errorf("empty accessToken in response (%v)", string(respBody)) + } + return &tokenInfo{ + tokenType: respData.TokenType, + token: respData.AccessToken, + expiryTime: time.Now().Add(time.Duration(respData.ExpiresIn) * time.Second), + }, nil +} + +// requestParameters stores all STS request attributes defined in +// https://tools.ietf.org/html/rfc8693#section-2.1. +type requestParameters struct { + // REQUIRED. The value "urn:ietf:params:oauth:grant-type:token-exchange" + // indicates that a token exchange is being performed. + GrantType string `json:"grant_type"` + // OPTIONAL. Indicates the location of the target service or resource where + // the client intends to use the requested security token. + Resource string `json:"resource,omitempty"` + // OPTIONAL. The logical name of the target service where the client intends + // to use the requested security token. + Audience string `json:"audience,omitempty"` + // OPTIONAL. A list of space-delimited, case-sensitive strings, that allow + // the client to specify the desired scope of the requested security token + // in the context of the service or Resource where the token will be used. + Scope string `json:"scope,omitempty"` + // OPTIONAL. An identifier, for the type of the requested security token. + RequestedTokenType string `json:"requested_token_type,omitempty"` + // REQUIRED. A security token that represents the identity of the party on + // behalf of whom the request is being made. + SubjectToken string `json:"subject_token"` + // REQUIRED. An identifier, that indicates the type of the security token in + // the "subject_token" parameter. + SubjectTokenType string `json:"subject_token_type"` + // OPTIONAL. A security token that represents the identity of the acting + // party. + ActorToken string `json:"actor_token,omitempty"` + // An identifier, that indicates the type of the security token in the + // "actor_token" parameter. + ActorTokenType string `json:"actor_token_type,omitempty"` +} + +// responseParameters stores all attributes sent as JSON in a successful STS +// response. These attributes are defined in +// https://tools.ietf.org/html/rfc8693#section-2.2.1. +type responseParameters struct { + // REQUIRED. The security token issued by the authorization server + // in response to the token exchange request. + AccessToken string `json:"access_token"` + // REQUIRED. An identifier, representation of the issued security token. + IssuedTokenType string `json:"issued_token_type"` + // REQUIRED. A case-insensitive value specifying the method of using the access + // token issued. It provides the client with information about how to utilize the + // access token to access protected resources. + TokenType string `json:"token_type"` + // RECOMMENDED. The validity lifetime, in seconds, of the token issued by the + // authorization server. + ExpiresIn int64 `json:"expires_in"` + // OPTIONAL, if the Scope of the issued security token is identical to the + // Scope requested by the client; otherwise, REQUIRED. + Scope string `json:"scope"` + // OPTIONAL. A refresh token will typically not be issued when the exchange is + // of one temporary credential (the subject_token) for a different temporary + // credential (the issued token) for use in some other context. + RefreshToken string `json:"refresh_token"` +} + +// tokenInfo wraps the information received in a successful STS response. +type tokenInfo struct { + tokenType string + token string + expiryTime time.Time +} diff --git a/credentials/sts/sts_test.go b/credentials/sts/sts_test.go new file mode 100644 index 0000000..7c29bdd --- /dev/null +++ b/credentials/sts/sts_test.go @@ -0,0 +1,803 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package sts + +import ( + "bytes" + "context" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/http/httputil" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" +) + +const ( + requestedTokenType = "urn:ietf:params:oauth:token-type:access-token" + actorTokenPath = "/var/run/secrets/token.jwt" + actorTokenType = "urn:ietf:params:oauth:token-type:refresh_token" + actorTokenContents = "actorToken.jwt.contents" + accessTokenContents = "access_token" + subjectTokenPath = "/var/run/secrets/token.jwt" + subjectTokenType = "urn:ietf:params:oauth:token-type:id_token" + subjectTokenContents = "subjectToken.jwt.contents" + serviceURI = "http://localhost" + exampleResource = "https://backend.example.com/api" + exampleAudience = "example-backend-service" + testScope = "https://www.googleapis.com/auth/monitoring" + defaultTestTimeout = 1 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +var ( + goodOptions = Options{ + TokenExchangeServiceURI: serviceURI, + Audience: exampleAudience, + RequestedTokenType: requestedTokenType, + SubjectTokenPath: subjectTokenPath, + SubjectTokenType: subjectTokenType, + } + goodRequestParams = &requestParameters{ + GrantType: tokenExchangeGrantType, + Audience: exampleAudience, + Scope: defaultCloudPlatformScope, + RequestedTokenType: requestedTokenType, + SubjectToken: subjectTokenContents, + SubjectTokenType: subjectTokenType, + } + goodMetadata = map[string]string{ + "Authorization": fmt.Sprintf("Bearer %s", accessTokenContents), + } +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// A struct that implements AuthInfo interface and added to the context passed +// to GetRequestMetadata from tests. +type testAuthInfo struct { + credentials.CommonAuthInfo +} + +func (ta testAuthInfo) AuthType() string { + return "testAuthInfo" +} + +func createTestContext(ctx context.Context, s credentials.SecurityLevel) context.Context { + auth := &testAuthInfo{CommonAuthInfo: credentials.CommonAuthInfo{SecurityLevel: s}} + ri := credentials.RequestInfo{ + Method: "testInfo", + AuthInfo: auth, + } + return credentials.NewContextWithRequestInfo(ctx, ri) +} + +// errReader implements the io.Reader interface and returns an error from the +// Read method. +type errReader struct{} + +func (r errReader) Read([]byte) (n int, err error) { + return 0, errors.New("read error") +} + +// We need a function to construct the response instead of simply declaring it +// as a variable since the response body will be consumed by the +// credentials, and therefore we will need a new one everytime. +func makeGoodResponse() *http.Response { + respJSON, _ := json.Marshal(responseParameters{ + AccessToken: accessTokenContents, + IssuedTokenType: "urn:ietf:params:oauth:token-type:access_token", + TokenType: "Bearer", + ExpiresIn: 3600, + }) + respBody := io.NopCloser(bytes.NewReader(respJSON)) + return &http.Response{ + Status: "200 OK", + StatusCode: http.StatusOK, + Body: respBody, + } +} + +// Overrides the http.Client with a fakeClient which sends a good response. +func overrideHTTPClientGood() (*testutils.FakeHTTPClient, func()) { + fc := &testutils.FakeHTTPClient{ + ReqChan: testutils.NewChannel(), + RespChan: testutils.NewChannel(), + } + fc.RespChan.Send(makeGoodResponse()) + + origMakeHTTPDoer := makeHTTPDoer + makeHTTPDoer = func(_ *x509.CertPool) httpDoer { return fc } + return fc, func() { makeHTTPDoer = origMakeHTTPDoer } +} + +// Overrides the http.Client with the provided fakeClient. +func overrideHTTPClient(fc *testutils.FakeHTTPClient) func() { + origMakeHTTPDoer := makeHTTPDoer + makeHTTPDoer = func(_ *x509.CertPool) httpDoer { return fc } + return func() { makeHTTPDoer = origMakeHTTPDoer } +} + +// Overrides the subject token read to return a const which we can compare in +// our tests. +func overrideSubjectTokenGood() func() { + origReadSubjectTokenFrom := readSubjectTokenFrom + readSubjectTokenFrom = func(string) ([]byte, error) { + return []byte(subjectTokenContents), nil + } + return func() { readSubjectTokenFrom = origReadSubjectTokenFrom } +} + +// Overrides the subject token read to always return an error. +func overrideSubjectTokenError() func() { + origReadSubjectTokenFrom := readSubjectTokenFrom + readSubjectTokenFrom = func(string) ([]byte, error) { + return nil, errors.New("error reading subject token") + } + return func() { readSubjectTokenFrom = origReadSubjectTokenFrom } +} + +// Overrides the actor token read to return a const which we can compare in +// our tests. +func overrideActorTokenGood() func() { + origReadActorTokenFrom := readActorTokenFrom + readActorTokenFrom = func(string) ([]byte, error) { + return []byte(actorTokenContents), nil + } + return func() { readActorTokenFrom = origReadActorTokenFrom } +} + +// Overrides the actor token read to always return an error. +func overrideActorTokenError() func() { + origReadActorTokenFrom := readActorTokenFrom + readActorTokenFrom = func(string) ([]byte, error) { + return nil, errors.New("error reading actor token") + } + return func() { readActorTokenFrom = origReadActorTokenFrom } +} + +// compareRequest compares the http.Request received in the test with the +// expected requestParameters specified in wantReqParams. +func compareRequest(gotRequest *http.Request, wantReqParams *requestParameters) error { + jsonBody, err := json.Marshal(wantReqParams) + if err != nil { + return err + } + wantReq, err := http.NewRequest("POST", serviceURI, bytes.NewBuffer(jsonBody)) + if err != nil { + return fmt.Errorf("failed to create http request: %v", err) + } + wantReq.Header.Set("Content-Type", "application/json") + + wantR, err := httputil.DumpRequestOut(wantReq, true) + if err != nil { + return err + } + gotR, err := httputil.DumpRequestOut(gotRequest, true) + if err != nil { + return err + } + if diff := cmp.Diff(string(wantR), string(gotR)); diff != "" { + return fmt.Errorf("sts request diff (-want +got):\n%s", diff) + } + return nil +} + +// receiveAndCompareRequest waits for a request to be sent out by the +// credentials implementation using the fakeHTTPClient and compares it to an +// expected goodRequest. This is expected to be called in a separate goroutine +// by the tests. So, any errors encountered are pushed to an error channel +// which is monitored by the test. +func receiveAndCompareRequest(ReqChan *testutils.Channel, errCh chan error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + val, err := ReqChan.Receive(ctx) + if err != nil { + errCh <- err + return + } + req := val.(*http.Request) + if err := compareRequest(req, goodRequestParams); err != nil { + errCh <- err + return + } + errCh <- nil +} + +// TestGetRequestMetadataSuccess verifies the successful case of sending an +// token exchange request and processing the response. +func (s) TestGetRequestMetadataSuccess(t *testing.T) { + defer overrideSubjectTokenGood()() + fc, cancel := overrideHTTPClientGood() + defer cancel() + + creds, err := NewCredentials(goodOptions) + if err != nil { + t.Fatalf("NewCredentials(%v) = %v", goodOptions, err) + } + + errCh := make(chan error, 1) + go receiveAndCompareRequest(fc.ReqChan, errCh) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + gotMetadata, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), "") + if err != nil { + t.Fatalf("creds.GetRequestMetadata() = %v", err) + } + if !cmp.Equal(gotMetadata, goodMetadata) { + t.Fatalf("creds.GetRequestMetadata() = %v, want %v", gotMetadata, goodMetadata) + } + if err := <-errCh; err != nil { + t.Fatal(err) + } + + // Make another call to get request metadata and this should return contents + // from the cache. This will fail if the credentials tries to send a fresh + // request here since we have not configured our fakeClient to return any + // response on retries. + gotMetadata, err = creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), "") + if err != nil { + t.Fatalf("creds.GetRequestMetadata() = %v", err) + } + if !cmp.Equal(gotMetadata, goodMetadata) { + t.Fatalf("creds.GetRequestMetadata() = %v, want %v", gotMetadata, goodMetadata) + } +} + +// TestGetRequestMetadataBadSecurityLevel verifies the case where the +// securityLevel specified in the context passed to GetRequestMetadata is not +// sufficient. +func (s) TestGetRequestMetadataBadSecurityLevel(t *testing.T) { + defer overrideSubjectTokenGood()() + + creds, err := NewCredentials(goodOptions) + if err != nil { + t.Fatalf("NewCredentials(%v) = %v", goodOptions, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + gotMetadata, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.IntegrityOnly), "") + if err == nil { + t.Fatalf("creds.GetRequestMetadata() succeeded with metadata %v, expected to fail", gotMetadata) + } +} + +// TestGetRequestMetadataCacheExpiry verifies the case where the cached access +// token has expired, and the credentials implementation will have to send a +// fresh token exchange request. +func (s) TestGetRequestMetadataCacheExpiry(t *testing.T) { + const expiresInSecs = 1 + defer overrideSubjectTokenGood()() + fc := &testutils.FakeHTTPClient{ + ReqChan: testutils.NewChannel(), + RespChan: testutils.NewChannel(), + } + defer overrideHTTPClient(fc)() + + creds, err := NewCredentials(goodOptions) + if err != nil { + t.Fatalf("NewCredentials(%v) = %v", goodOptions, err) + } + + // The fakeClient is configured to return an access_token with a one second + // expiry. So, in the second iteration, the credentials will find the cache + // entry, but that would have expired, and therefore we expect it to send + // out a fresh request. + for i := 0; i < 2; i++ { + errCh := make(chan error, 1) + go receiveAndCompareRequest(fc.ReqChan, errCh) + + respJSON, _ := json.Marshal(responseParameters{ + AccessToken: accessTokenContents, + IssuedTokenType: "urn:ietf:params:oauth:token-type:access_token", + TokenType: "Bearer", + ExpiresIn: expiresInSecs, + }) + respBody := io.NopCloser(bytes.NewReader(respJSON)) + resp := &http.Response{ + Status: "200 OK", + StatusCode: http.StatusOK, + Body: respBody, + } + fc.RespChan.Send(resp) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + gotMetadata, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), "") + if err != nil { + t.Fatalf("creds.GetRequestMetadata() = %v", err) + } + if !cmp.Equal(gotMetadata, goodMetadata) { + t.Fatalf("creds.GetRequestMetadata() = %v, want %v", gotMetadata, goodMetadata) + } + if err := <-errCh; err != nil { + t.Fatal(err) + } + time.Sleep(expiresInSecs * time.Second) + } +} + +// TestGetRequestMetadataBadResponses verifies the scenario where the token +// exchange server returns bad responses. +func (s) TestGetRequestMetadataBadResponses(t *testing.T) { + tests := []struct { + name string + response *http.Response + }{ + { + name: "bad JSON", + response: &http.Response{ + Status: "200 OK", + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("not JSON")), + }, + }, + { + name: "no access token", + response: &http.Response{ + Status: "200 OK", + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("{}")), + }, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + defer overrideSubjectTokenGood()() + + fc := &testutils.FakeHTTPClient{ + ReqChan: testutils.NewChannel(), + RespChan: testutils.NewChannel(), + } + defer overrideHTTPClient(fc)() + + creds, err := NewCredentials(goodOptions) + if err != nil { + t.Fatalf("NewCredentials(%v) = %v", goodOptions, err) + } + + errCh := make(chan error, 1) + go receiveAndCompareRequest(fc.ReqChan, errCh) + + fc.RespChan.Send(test.response) + if _, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), ""); err == nil { + t.Fatal("creds.GetRequestMetadata() succeeded when expected to fail") + } + if err := <-errCh; err != nil { + t.Fatal(err) + } + }) + } +} + +// TestGetRequestMetadataBadSubjectTokenRead verifies the scenario where the +// attempt to read the subjectToken fails. +func (s) TestGetRequestMetadataBadSubjectTokenRead(t *testing.T) { + defer overrideSubjectTokenError()() + fc, cancel := overrideHTTPClientGood() + defer cancel() + + creds, err := NewCredentials(goodOptions) + if err != nil { + t.Fatalf("NewCredentials(%v) = %v", goodOptions, err) + } + + errCh := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := fc.ReqChan.Receive(ctx); err != context.DeadlineExceeded { + errCh <- err + return + } + errCh <- nil + }() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), ""); err == nil { + t.Fatal("creds.GetRequestMetadata() succeeded when expected to fail") + } + if err := <-errCh; err != nil { + t.Fatal(err) + } +} + +func (s) TestNewCredentials(t *testing.T) { + tests := []struct { + name string + opts Options + errSystemRoots bool + wantErr bool + }{ + { + name: "invalid options - empty subjectTokenPath", + opts: Options{ + TokenExchangeServiceURI: serviceURI, + }, + wantErr: true, + }, + { + name: "invalid system root certs", + opts: goodOptions, + errSystemRoots: true, + wantErr: true, + }, + { + name: "good case", + opts: goodOptions, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.errSystemRoots { + oldSystemRoots := loadSystemCertPool + loadSystemCertPool = func() (*x509.CertPool, error) { + return nil, errors.New("failed to load system cert pool") + } + defer func() { + loadSystemCertPool = oldSystemRoots + }() + } + + creds, err := NewCredentials(test.opts) + if (err != nil) != test.wantErr { + t.Fatalf("NewCredentials(%v) = %v, want %v", test.opts, err, test.wantErr) + } + if err == nil { + if !creds.RequireTransportSecurity() { + t.Errorf("creds.RequireTransportSecurity() returned false") + } + } + }) + } +} + +func (s) TestValidateOptions(t *testing.T) { + tests := []struct { + name string + opts Options + wantErrPrefix string + }{ + { + name: "empty token exchange service URI", + opts: Options{}, + wantErrPrefix: "empty token_exchange_service_uri in options", + }, + { + name: "invalid URI", + opts: Options{ + TokenExchangeServiceURI: "\tI'm a bad URI\n", + }, + wantErrPrefix: "invalid control character in URL", + }, + { + name: "unsupported scheme", + opts: Options{ + TokenExchangeServiceURI: "unix:///path/to/socket", + }, + wantErrPrefix: "scheme is not supported", + }, + { + name: "empty subjectTokenPath", + opts: Options{ + TokenExchangeServiceURI: serviceURI, + }, + wantErrPrefix: "required field SubjectTokenPath is not specified", + }, + { + name: "empty subjectTokenType", + opts: Options{ + TokenExchangeServiceURI: serviceURI, + SubjectTokenPath: subjectTokenPath, + }, + wantErrPrefix: "required field SubjectTokenType is not specified", + }, + { + name: "good options", + opts: goodOptions, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := validateOptions(test.opts) + if (err != nil) != (test.wantErrPrefix != "") { + t.Errorf("validateOptions(%v) = %v, want %v", test.opts, err, test.wantErrPrefix) + } + if err != nil && !strings.Contains(err.Error(), test.wantErrPrefix) { + t.Errorf("validateOptions(%v) = %v, want %v", test.opts, err, test.wantErrPrefix) + } + }) + } +} + +func (s) TestConstructRequest(t *testing.T) { + tests := []struct { + name string + opts Options + subjectTokenReadErr bool + actorTokenReadErr bool + wantReqParams *requestParameters + wantErr bool + }{ + { + name: "subject token read failure", + subjectTokenReadErr: true, + opts: goodOptions, + wantErr: true, + }, + { + name: "actor token read failure", + actorTokenReadErr: true, + opts: Options{ + TokenExchangeServiceURI: serviceURI, + Audience: exampleAudience, + RequestedTokenType: requestedTokenType, + SubjectTokenPath: subjectTokenPath, + SubjectTokenType: subjectTokenType, + ActorTokenPath: actorTokenPath, + ActorTokenType: actorTokenType, + }, + wantErr: true, + }, + { + name: "default cloud platform scope", + opts: goodOptions, + wantReqParams: goodRequestParams, + }, + { + name: "all good", + opts: Options{ + TokenExchangeServiceURI: serviceURI, + Resource: exampleResource, + Audience: exampleAudience, + Scope: testScope, + RequestedTokenType: requestedTokenType, + SubjectTokenPath: subjectTokenPath, + SubjectTokenType: subjectTokenType, + ActorTokenPath: actorTokenPath, + ActorTokenType: actorTokenType, + }, + wantReqParams: &requestParameters{ + GrantType: tokenExchangeGrantType, + Resource: exampleResource, + Audience: exampleAudience, + Scope: testScope, + RequestedTokenType: requestedTokenType, + SubjectToken: subjectTokenContents, + SubjectTokenType: subjectTokenType, + ActorToken: actorTokenContents, + ActorTokenType: actorTokenType, + }, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.subjectTokenReadErr { + defer overrideSubjectTokenError()() + } else { + defer overrideSubjectTokenGood()() + } + + if test.actorTokenReadErr { + defer overrideActorTokenError()() + } else { + defer overrideActorTokenGood()() + } + + gotRequest, err := constructRequest(ctx, test.opts) + if (err != nil) != test.wantErr { + t.Fatalf("constructRequest(%v) = %v, wantErr: %v", test.opts, err, test.wantErr) + } + if test.wantErr { + return + } + if err := compareRequest(gotRequest, test.wantReqParams); err != nil { + t.Fatal(err) + } + }) + } +} + +func (s) TestSendRequest(t *testing.T) { + defer overrideSubjectTokenGood()() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + req, err := constructRequest(ctx, goodOptions) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + resp *http.Response + respErr error + wantErr bool + }{ + { + name: "client error", + respErr: errors.New("http.Client.Do failed"), + wantErr: true, + }, + { + name: "bad response body", + resp: &http.Response{ + Status: "200 OK", + StatusCode: http.StatusOK, + Body: io.NopCloser(errReader{}), + }, + wantErr: true, + }, + { + name: "nonOK status code", + resp: &http.Response{ + Status: "400 BadRequest", + StatusCode: http.StatusBadRequest, + Body: io.NopCloser(strings.NewReader("")), + }, + wantErr: true, + }, + { + name: "good case", + resp: makeGoodResponse(), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := &testutils.FakeHTTPClient{ + ReqChan: testutils.NewChannel(), + RespChan: testutils.NewChannel(), + Err: test.respErr, + } + client.RespChan.Send(test.resp) + _, err := sendRequest(client, req) + if (err != nil) != test.wantErr { + t.Errorf("sendRequest(%v) = %v, wantErr: %v", req, err, test.wantErr) + } + }) + } +} + +func (s) TestSendRequestDoesNotFollowRedirects(t *testing.T) { + // Stands in for a redirect target that must never receive the token + // exchange request. It records the body of any request it does get. + leaked := make(chan string, 1) + redirectTarget := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + select { + case leaked <- string(body): + default: + } + })) + defer redirectTarget.Close() + + // The configured token exchange endpoint redirects the POST, whose body + // carries the subject token, to another host. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, redirectTarget.URL, http.StatusTemporaryRedirect) + })) + defer server.Close() + + req, err := http.NewRequest(http.MethodPost, server.URL, bytes.NewBufferString(`{"subject_token":"secret"}`)) + if err != nil { + t.Fatalf("http.NewRequest() failed: %v", err) + } + req.Header.Set("Content-Type", "application/json") + + if _, err := sendRequest(makeHTTPClient(nil), req); err == nil { + t.Error("sendRequest() succeeded across a redirect; want a non-nil error") + } + select { + case body := <-leaked: + t.Errorf("Redirect target received request body %q; the subject token must not be replayed to another host", body) + default: + } +} + +func (s) TestTokenInfoFromResponse(t *testing.T) { + noAccessToken, _ := json.Marshal(responseParameters{ + IssuedTokenType: "urn:ietf:params:oauth:token-type:access_token", + TokenType: "Bearer", + ExpiresIn: 3600, + }) + goodResponse, _ := json.Marshal(responseParameters{ + IssuedTokenType: requestedTokenType, + AccessToken: accessTokenContents, + TokenType: "Bearer", + ExpiresIn: 3600, + }) + + tests := []struct { + name string + respBody []byte + wantTokenInfo *tokenInfo + wantErr bool + }{ + { + name: "bad JSON", + respBody: []byte("not JSON"), + wantErr: true, + }, + { + name: "empty response", + respBody: []byte(""), + wantErr: true, + }, + { + name: "non-empty response with no access token", + respBody: noAccessToken, + wantErr: true, + }, + { + name: "good response", + respBody: goodResponse, + wantTokenInfo: &tokenInfo{ + tokenType: "Bearer", + token: accessTokenContents, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotTokenInfo, err := tokenInfoFromResponse(test.respBody) + if (err != nil) != test.wantErr { + t.Fatalf("tokenInfoFromResponse(%+v) = %v, wantErr: %v", test.respBody, err, test.wantErr) + } + if test.wantErr { + return + } + // Can't do a cmp.Equal on the whole struct since the expiryField + // is populated based on time.Now(). + if gotTokenInfo.tokenType != test.wantTokenInfo.tokenType || gotTokenInfo.token != test.wantTokenInfo.token { + t.Errorf("tokenInfoFromResponse(%+v) = %+v, want: %+v", test.respBody, gotTokenInfo, test.wantTokenInfo) + } + }) + } +} diff --git a/credentials/tls.go b/credentials/tls.go new file mode 100644 index 0000000..a6083c3 --- /dev/null +++ b/credentials/tls.go @@ -0,0 +1,322 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "net/url" + "os" + + "google.golang.org/grpc/grpclog" + credinternal "google.golang.org/grpc/internal/credentials" + "google.golang.org/grpc/internal/envconfig" +) + +const alpnFailureHelpMessage = "If you upgraded from a grpc-go version earlier than 1.67, your TLS connections may have stopped working due to ALPN enforcement. For more details, see: https://github.com/grpc/grpc-go/issues/434" + +var logger = grpclog.Component("credentials") + +// TLSInfo contains the auth information for a TLS authenticated connection. +// It implements the AuthInfo interface. +type TLSInfo struct { + State tls.ConnectionState + CommonAuthInfo + // This API is experimental. + SPIFFEID *url.URL +} + +// AuthType returns the type of TLSInfo as a string. +func (t TLSInfo) AuthType() string { + return "tls" +} + +// ValidateAuthority validates the provided authority being used to override the +// :authority header by verifying it against the peer certificate. It returns a +// non-nil error if the validation fails. +func (t TLSInfo) ValidateAuthority(authority string) error { + host, _, err := net.SplitHostPort(authority) + if err != nil { + host = authority + } + + // Verify authority against the leaf certificate. + if len(t.State.PeerCertificates) == 0 { + // This is not expected to happen as the TLS handshake has already + // completed and should have populated PeerCertificates. + return fmt.Errorf("credentials: no peer certificates found to verify authority %q", host) + } + return t.State.PeerCertificates[0].VerifyHostname(host) +} + +// cipherSuiteLookup returns the string version of a TLS cipher suite ID. +func cipherSuiteLookup(cipherSuiteID uint16) string { + for _, s := range tls.CipherSuites() { + if s.ID == cipherSuiteID { + return s.Name + } + } + for _, s := range tls.InsecureCipherSuites() { + if s.ID == cipherSuiteID { + return s.Name + } + } + return fmt.Sprintf("unknown ID: %v", cipherSuiteID) +} + +// GetSecurityValue returns security info requested by channelz. +func (t TLSInfo) GetSecurityValue() ChannelzSecurityValue { + v := &TLSChannelzSecurityValue{ + StandardName: cipherSuiteLookup(t.State.CipherSuite), + } + // Currently there's no way to get LocalCertificate info from tls package. + if len(t.State.PeerCertificates) > 0 { + v.RemoteCertificate = t.State.PeerCertificates[0].Raw + } + return v +} + +// tlsCreds is the credentials required for authenticating a connection using TLS. +type tlsCreds struct { + // TLS configuration + config *tls.Config +} + +func (c tlsCreds) Info() ProtocolInfo { + return ProtocolInfo{ + SecurityProtocol: "tls", + SecurityVersion: "1.2", + ServerName: c.config.ServerName, + } +} + +func (c *tlsCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (_ net.Conn, _ AuthInfo, err error) { + // use local cfg to avoid clobbering ServerName if using multiple endpoints + cfg := credinternal.CloneTLSConfig(c.config) + + serverName, _, err := net.SplitHostPort(authority) + if err != nil { + // If the authority had no host port or if the authority cannot be parsed, use it as-is. + serverName = authority + } + cfg.ServerName = serverName + + conn := tls.Client(rawConn, cfg) + errChannel := make(chan error, 1) + go func() { + errChannel <- conn.Handshake() + close(errChannel) + }() + select { + case err := <-errChannel: + if err != nil { + conn.Close() + return nil, nil, err + } + case <-ctx.Done(): + conn.Close() + return nil, nil, ctx.Err() + } + + // The negotiated protocol can be either of the following: + // 1. h2: When the server supports ALPN. Only HTTP/2 can be negotiated since + // it is the only protocol advertised by the client during the handshake. + // The tls library ensures that the server chooses a protocol advertised + // by the client. + // 2. "" (empty string): If the server doesn't support ALPN. ALPN is a requirement + // for using HTTP/2 over TLS. We can terminate the connection immediately. + np := conn.ConnectionState().NegotiatedProtocol + if np == "" { + if envconfig.EnforceALPNEnabled { + conn.Close() + return nil, nil, fmt.Errorf("credentials: cannot check peer: missing selected ALPN property. %s", alpnFailureHelpMessage) + } + logger.Warningf("Allowing TLS connection to server %q with ALPN disabled. TLS connections to servers with ALPN disabled will be disallowed in future grpc-go releases", cfg.ServerName) + } + tlsInfo := TLSInfo{ + State: conn.ConnectionState(), + CommonAuthInfo: CommonAuthInfo{ + SecurityLevel: PrivacyAndIntegrity, + }, + } + id := credinternal.SPIFFEIDFromState(conn.ConnectionState()) + if id != nil { + tlsInfo.SPIFFEID = id + } + return credinternal.WrapSyscallConn(rawConn, conn), tlsInfo, nil +} + +func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error) { + conn := tls.Server(rawConn, c.config) + if err := conn.Handshake(); err != nil { + conn.Close() + return nil, nil, err + } + cs := conn.ConnectionState() + // The negotiated application protocol can be empty only if the client doesn't + // support ALPN. In such cases, we can close the connection since ALPN is required + // for using HTTP/2 over TLS. + if cs.NegotiatedProtocol == "" { + if envconfig.EnforceALPNEnabled { + conn.Close() + return nil, nil, fmt.Errorf("credentials: cannot check peer: missing selected ALPN property. %s", alpnFailureHelpMessage) + } else if logger.V(2) { + logger.Info("Allowing TLS connection from client with ALPN disabled. TLS connections with ALPN disabled will be disallowed in future grpc-go releases") + } + } + tlsInfo := TLSInfo{ + State: cs, + CommonAuthInfo: CommonAuthInfo{ + SecurityLevel: PrivacyAndIntegrity, + }, + } + id := credinternal.SPIFFEIDFromState(conn.ConnectionState()) + if id != nil { + tlsInfo.SPIFFEID = id + } + return credinternal.WrapSyscallConn(rawConn, conn), tlsInfo, nil +} + +func (c *tlsCreds) Clone() TransportCredentials { + return NewTLS(c.config) +} + +func (c *tlsCreds) OverrideServerName(serverNameOverride string) error { + c.config.ServerName = serverNameOverride + return nil +} + +// The following cipher suites are forbidden for use with HTTP/2 by +// https://datatracker.ietf.org/doc/html/rfc7540#appendix-A +var tls12ForbiddenCipherSuites = map[uint16]struct{}{ + tls.TLS_RSA_WITH_AES_128_CBC_SHA: {}, + tls.TLS_RSA_WITH_AES_256_CBC_SHA: {}, + tls.TLS_RSA_WITH_AES_128_GCM_SHA256: {}, + tls.TLS_RSA_WITH_AES_256_GCM_SHA384: {}, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA: {}, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA: {}, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA: {}, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: {}, +} + +// NewTLS uses c to construct a TransportCredentials based on TLS. +func NewTLS(c *tls.Config) TransportCredentials { + config := applyDefaults(c) + if config.GetConfigForClient != nil { + oldFn := config.GetConfigForClient + config.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + cfgForClient, err := oldFn(hello) + if err != nil || cfgForClient == nil { + return cfgForClient, err + } + return applyDefaults(cfgForClient), nil + } + } + return &tlsCreds{config: config} +} + +func applyDefaults(c *tls.Config) *tls.Config { + config := credinternal.CloneTLSConfig(c) + config.NextProtos = credinternal.AppendH2ToNextProtos(config.NextProtos) + // If the user did not configure a MinVersion and did not configure a + // MaxVersion < 1.2, use MinVersion=1.2, which is required by + // https://datatracker.ietf.org/doc/html/rfc7540#section-9.2 + if config.MinVersion == 0 && (config.MaxVersion == 0 || config.MaxVersion >= tls.VersionTLS12) { + config.MinVersion = tls.VersionTLS12 + } + // If the user did not configure CipherSuites, use all "secure" cipher + // suites reported by the TLS package, but remove some explicitly forbidden + // by https://datatracker.ietf.org/doc/html/rfc7540#appendix-A + if config.CipherSuites == nil { + for _, cs := range tls.CipherSuites() { + if _, ok := tls12ForbiddenCipherSuites[cs.ID]; !ok { + config.CipherSuites = append(config.CipherSuites, cs.ID) + } + } + } + return config +} + +// NewClientTLSFromCert constructs TLS credentials from the provided root +// certificate authority certificate(s) to validate server connections. If +// certificates to establish the identity of the client need to be included in +// the credentials (eg: for mTLS), use NewTLS instead, where a complete +// tls.Config can be specified. +// +// serverNameOverride is for testing only. If set to a non empty string, it will +// override the virtual host name of authority (e.g. :authority header field) in +// requests. Users should use grpc.WithAuthority passed to grpc.NewClient to +// override the authority of the client instead. +func NewClientTLSFromCert(cp *x509.CertPool, serverNameOverride string) TransportCredentials { + return NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}) +} + +// NewClientTLSFromFile constructs TLS credentials from the provided root +// certificate authority certificate file(s) to validate server connections. If +// certificates to establish the identity of the client need to be included in +// the credentials (eg: for mTLS), use NewTLS instead, where a complete +// tls.Config can be specified. +// +// serverNameOverride is for testing only. If set to a non empty string, it will +// override the virtual host name of authority (e.g. :authority header field) in +// requests. Users should use grpc.WithAuthority passed to grpc.NewClient to +// override the authority of the client instead. +func NewClientTLSFromFile(certFile, serverNameOverride string) (TransportCredentials, error) { + b, err := os.ReadFile(certFile) + if err != nil { + return nil, err + } + cp := x509.NewCertPool() + if !cp.AppendCertsFromPEM(b) { + return nil, fmt.Errorf("credentials: failed to append certificates") + } + return NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}), nil +} + +// NewServerTLSFromCert constructs TLS credentials from the input certificate for server. +func NewServerTLSFromCert(cert *tls.Certificate) TransportCredentials { + return NewTLS(&tls.Config{Certificates: []tls.Certificate{*cert}}) +} + +// NewServerTLSFromFile constructs TLS credentials from the input certificate file and key +// file for server. +func NewServerTLSFromFile(certFile, keyFile string) (TransportCredentials, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + return NewTLS(&tls.Config{Certificates: []tls.Certificate{cert}}), nil +} + +// TLSChannelzSecurityValue defines the struct that TLS protocol should return +// from GetSecurityValue(), containing security info like cipher and certificate used. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type TLSChannelzSecurityValue struct { + ChannelzSecurityValue + StandardName string + LocalCertificate []byte + RemoteCertificate []byte +} diff --git a/credentials/tls/certprovider/distributor.go b/credentials/tls/certprovider/distributor.go new file mode 100644 index 0000000..11fae92 --- /dev/null +++ b/credentials/tls/certprovider/distributor.go @@ -0,0 +1,114 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package certprovider + +import ( + "context" + "sync" + + "google.golang.org/grpc/internal/grpcsync" +) + +// Distributor makes it easy for provider implementations to furnish new key +// materials by handling synchronization between the producer and consumers of +// the key material. +// +// Provider implementations which choose to use a Distributor should do the +// following: +// - create a new Distributor using the NewDistributor() function. +// - invoke the Set() method whenever they have new key material or errors to +// report. +// - delegate to the distributor when handing calls to KeyMaterial(). +// - invoke the Stop() method when they are done using the distributor. +type Distributor struct { + // mu protects the underlying key material. + mu sync.Mutex + km *KeyMaterial + pErr error + + // ready channel to unblock KeyMaterial() invocations blocked on + // availability of key material. + ready *grpcsync.Event + // done channel to notify provider implementations and unblock any + // KeyMaterial() calls, once the Distributor is closed. + closed *grpcsync.Event +} + +// NewDistributor returns a new Distributor. +func NewDistributor() *Distributor { + return &Distributor{ + ready: grpcsync.NewEvent(), + closed: grpcsync.NewEvent(), + } +} + +// Set updates the key material in the distributor with km. +// +// Provider implementations which use the distributor must not modify the +// contents of the KeyMaterial struct pointed to by km. +// +// A non-nil err value indicates the error that the provider implementation ran +// into when trying to fetch key material, and makes it possible to surface the +// error to the user. A non-nil error value passed here causes distributor's +// KeyMaterial() method to return nil key material. +func (d *Distributor) Set(km *KeyMaterial, err error) { + d.mu.Lock() + d.km = km + d.pErr = err + if err != nil { + // If a non-nil err is passed, we ignore the key material being passed. + d.km = nil + } + d.ready.Fire() + d.mu.Unlock() +} + +// KeyMaterial returns the most recent key material provided to the Distributor. +// If no key material was provided at the time of this call, it will block until +// the deadline on the context expires or fresh key material arrives. +func (d *Distributor) KeyMaterial(ctx context.Context) (*KeyMaterial, error) { + if d.closed.HasFired() { + return nil, errProviderClosed + } + + if d.ready.HasFired() { + return d.keyMaterial() + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-d.closed.Done(): + return nil, errProviderClosed + case <-d.ready.Done(): + return d.keyMaterial() + } +} + +func (d *Distributor) keyMaterial() (*KeyMaterial, error) { + d.mu.Lock() + defer d.mu.Unlock() + return d.km, d.pErr +} + +// Stop turns down the distributor, releases allocated resources and fails any +// active KeyMaterial() call waiting for new key material. +func (d *Distributor) Stop() { + d.closed.Fire() +} diff --git a/credentials/tls/certprovider/distributor_test.go b/credentials/tls/certprovider/distributor_test.go new file mode 100644 index 0000000..48d5137 --- /dev/null +++ b/credentials/tls/certprovider/distributor_test.go @@ -0,0 +1,110 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package certprovider + +import ( + "context" + "errors" + "testing" + "time" +) + +var errProviderTestInternal = errors.New("provider internal error") + +// TestDistributorEmpty tries to read key material from an empty distributor and +// expects the call to timeout. +func (s) TestDistributorEmpty(t *testing.T) { + dist := NewDistributor() + + // This call to KeyMaterial() should timeout because no key material has + // been set on the distributor as yet. + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := readAndVerifyKeyMaterial(ctx, dist, nil); !errors.Is(err, context.DeadlineExceeded) { + t.Fatal(err) + } +} + +// TestDistributor invokes the different methods on the Distributor type and +// verifies the results. +func (s) TestDistributor(t *testing.T) { + dist := NewDistributor() + + // Read cert/key files from testdata. + km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") + + // Push key material into the distributor and make sure that a call to + // KeyMaterial() returns the expected key material, with both the local + // certs and root certs. + dist.Set(km1, nil) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := readAndVerifyKeyMaterial(ctx, dist, km1); err != nil { + t.Fatal(err) + } + + // Push new key material into the distributor and make sure that a call to + // KeyMaterial() returns the expected key material, with only root certs. + dist.Set(km2, nil) + if err := readAndVerifyKeyMaterial(ctx, dist, km2); err != nil { + t.Fatal(err) + } + + // Push an error into the distributor and make sure that a call to + // KeyMaterial() returns that error and nil keyMaterial. + dist.Set(km2, errProviderTestInternal) + if gotKM, err := dist.KeyMaterial(ctx); gotKM != nil || !errors.Is(err, errProviderTestInternal) { + t.Fatalf("KeyMaterial() = {%v, %v}, want {nil, %v}", gotKM, err, errProviderTestInternal) + } + + // Stop the distributor and KeyMaterial() should return errProviderClosed. + dist.Stop() + if km, err := dist.KeyMaterial(ctx); !errors.Is(err, errProviderClosed) { + t.Fatalf("KeyMaterial() = {%v, %v}, want {nil, %v}", km, err, errProviderClosed) + } +} + +// TestDistributorConcurrency invokes methods on the distributor in parallel. It +// exercises that the scenario where a distributor's KeyMaterial() method is +// blocked waiting for keyMaterial, while the Set() method is called from +// another goroutine. It verifies that the KeyMaterial() method eventually +// returns with expected keyMaterial. +func (s) TestDistributorConcurrency(t *testing.T) { + dist := NewDistributor() + + // Read cert/key files from testdata. + km := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Push key material into the distributor from a goroutine and read from + // here to verify that the distributor returns the expected keyMaterial. + go func() { + // Add a small sleep here to make sure that the call to KeyMaterial() + // happens before the call to Set(), thereby the former is blocked till + // the latter happens. + time.Sleep(100 * time.Microsecond) + dist.Set(km, nil) + }() + if err := readAndVerifyKeyMaterial(ctx, dist, km); err != nil { + t.Fatal(err) + } +} diff --git a/credentials/tls/certprovider/pemfile/builder.go b/credentials/tls/certprovider/pemfile/builder.go new file mode 100644 index 0000000..a7f2f79 --- /dev/null +++ b/credentials/tls/certprovider/pemfile/builder.go @@ -0,0 +1,103 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package pemfile + +import ( + "encoding/json" + "fmt" + "time" + + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/durationpb" +) + +const ( + // PluginName is the name of the PEM file watcher plugin. + PluginName = "file_watcher" + defaultRefreshInterval = 10 * time.Minute +) + +func init() { + certprovider.Register(&pluginBuilder{}) +} + +type pluginBuilder struct{} + +func (p *pluginBuilder) ParseConfig(c any) (*certprovider.BuildableConfig, error) { + data, ok := c.(json.RawMessage) + if !ok { + return nil, fmt.Errorf("meshca: unsupported config type: %T", c) + } + opts, err := pluginConfigFromJSON(data) + if err != nil { + return nil, err + } + return certprovider.NewBuildableConfig(PluginName, opts.canonical(), func(certprovider.BuildOptions) certprovider.Provider { + return newProvider(opts) + }), nil +} + +func (p *pluginBuilder) Name() string { + return PluginName +} + +func pluginConfigFromJSON(jd json.RawMessage) (Options, error) { + // The only difference between this anonymous struct and the Options struct + // is that the refresh_interval is represented here as a duration proto, + // while in the latter a time.Duration is used. + cfg := &struct { + CertificateFile string `json:"certificate_file,omitempty"` + PrivateKeyFile string `json:"private_key_file,omitempty"` + CACertificateFile string `json:"ca_certificate_file,omitempty"` + SPIFFETrustBundleMapFile string `json:"spiffe_trust_bundle_map_file,omitempty"` + RefreshInterval json.RawMessage `json:"refresh_interval,omitempty"` + }{} + if err := json.Unmarshal(jd, cfg); err != nil { + return Options{}, fmt.Errorf("pemfile: json.Unmarshal(%s) failed: %v", string(jd), err) + } + if !envconfig.XDSSPIFFEEnabled { + cfg.SPIFFETrustBundleMapFile = "" + } + + opts := Options{ + CertFile: cfg.CertificateFile, + KeyFile: cfg.PrivateKeyFile, + RootFile: cfg.CACertificateFile, + SPIFFEBundleMapFile: cfg.SPIFFETrustBundleMapFile, + // Refresh interval is the only field in the configuration for which we + // support a default value. We cannot possibly have valid defaults for + // file paths to watch. Also, it is valid to specify an empty path for + // some of those fields if the user does not want to watch them. + RefreshDuration: defaultRefreshInterval, + } + if cfg.RefreshInterval != nil { + dur := &durationpb.Duration{} + if err := protojson.Unmarshal(cfg.RefreshInterval, dur); err != nil { + return Options{}, fmt.Errorf("pemfile: protojson.Unmarshal(%+v) failed: %v", cfg.RefreshInterval, err) + } + opts.RefreshDuration = dur.AsDuration() + } + + if err := opts.validate(); err != nil { + return Options{}, err + } + return opts, nil +} diff --git a/credentials/tls/certprovider/pemfile/builder_test.go b/credentials/tls/certprovider/pemfile/builder_test.go new file mode 100644 index 0000000..0f2ca1d --- /dev/null +++ b/credentials/tls/certprovider/pemfile/builder_test.go @@ -0,0 +1,162 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package pemfile + +import ( + "encoding/json" + "testing" + + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/testutils" +) + +func TestParseConfig(t *testing.T) { + tests := []struct { + desc string + input any + wantOutput string + wantErr bool + enabledSpiffe bool + }{ + { + desc: "non JSON input", + input: new(int), + wantErr: true, + }, + { + desc: "invalid JSON", + input: json.RawMessage(`bad bad json`), + wantErr: true, + }, + { + desc: "JSON input does not match expected", + input: json.RawMessage(`["foo": "bar"]`), + wantErr: true, + }, + { + desc: "no credential files", + input: json.RawMessage(`{}`), + wantErr: true, + }, + { + desc: "only cert file", + input: json.RawMessage(` + { + "certificate_file": "/a/b/cert.pem" + }`), + wantErr: true, + }, + { + desc: "only key file", + input: json.RawMessage(` + { + "private_key_file": "/a/b/key.pem" + }`), + wantErr: true, + }, + { + desc: "cert and key in different directories", + input: json.RawMessage(` + { + "certificate_file": "/b/a/cert.pem", + "private_key_file": "/a/b/key.pem" + }`), + wantErr: true, + }, + { + desc: "bad refresh duration", + input: json.RawMessage(` + { + "certificate_file": "/a/b/cert.pem", + "private_key_file": "/a/b/key.pem", + "ca_certificate_file": "/a/b/ca.pem", + "refresh_interval": "duration" + }`), + wantErr: true, + }, + { + desc: "good config with default refresh interval", + input: json.RawMessage(` + { + "certificate_file": "/a/b/cert.pem", + "private_key_file": "/a/b/key.pem", + "ca_certificate_file": "/a/b/ca.pem" + }`), + wantOutput: "file_watcher:/a/b/cert.pem:/a/b/key.pem:/a/b/ca.pem::10m0s", + }, + { + desc: "good config", + input: json.RawMessage(` + { + "certificate_file": "/a/b/cert.pem", + "private_key_file": "/a/b/key.pem", + "ca_certificate_file": "/a/b/ca.pem", + "refresh_interval": "200s" + }`), + wantOutput: "file_watcher:/a/b/cert.pem:/a/b/key.pem:/a/b/ca.pem::3m20s", + }, + { + desc: "good config with spiffe disabled", + input: json.RawMessage(` + { + "certificate_file": "/a/b/cert.pem", + "private_key_file": "/a/b/key.pem", + "ca_certificate_file": "/a/b/ca.pem", + "spiffe_trust_bundle_map_file": "/a/b/spiffe_bundle.json", + "refresh_interval": "200s" + }`), + wantOutput: "file_watcher:/a/b/cert.pem:/a/b/key.pem:/a/b/ca.pem::3m20s", + }, + { + desc: "good config with spiffe enabled", + input: json.RawMessage(` + { + "certificate_file": "/a/b/cert.pem", + "private_key_file": "/a/b/key.pem", + "ca_certificate_file": "/a/b/ca.pem", + "spiffe_trust_bundle_map_file": "/a/b/spiffe_bundle.json", + "refresh_interval": "200s" + }`), + wantOutput: "file_watcher:/a/b/cert.pem:/a/b/key.pem:/a/b/ca.pem:/a/b/spiffe_bundle.json:3m20s", + enabledSpiffe: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if test.enabledSpiffe { + testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true) + } + builder := &pluginBuilder{} + + bc, err := builder.ParseConfig(test.input) + if (err != nil) != test.wantErr { + t.Fatalf("ParseConfig(%+v) failed: %v", test.input, err) + } + if test.wantErr { + return + } + + gotConfig := bc.String() + if gotConfig != test.wantOutput { + t.Fatalf("ParseConfig(%v) = %s, want %s", test.input, gotConfig, test.wantOutput) + } + }) + } +} diff --git a/credentials/tls/certprovider/pemfile/watcher.go b/credentials/tls/certprovider/pemfile/watcher.go new file mode 100644 index 0000000..bc9c545 --- /dev/null +++ b/credentials/tls/certprovider/pemfile/watcher.go @@ -0,0 +1,297 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package pemfile provides a file watching certificate provider plugin +// implementation which works for files with PEM contents. +// +// # Experimental +// +// Notice: All APIs in this package are experimental and may be removed in a +// later release. +package pemfile + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/credentials/spiffe" +) + +const defaultCertRefreshDuration = 1 * time.Hour + +var ( + // For overriding from unit tests. + newDistributor = func() distributor { return certprovider.NewDistributor() } + + logger = grpclog.Component("pemfile") +) + +// Options configures a certificate provider plugin that watches a specified set +// of files that contain certificates and keys in PEM format. +type Options struct { + // CertFile is the file that holds the identity certificate. + // Optional. If this is set, KeyFile must also be set. + CertFile string + // KeyFile is the file that holds identity private key. + // Optional. If this is set, CertFile must also be set. + KeyFile string + // RootFile is the file that holds trusted root certificate(s). + // Optional. + RootFile string + // SPIFFEBundleMapFile is the file that holds the spiffe bundle map. + // If a given provider configures both the RootFile and the + // SPIFFEBundleMapFile, the SPIFFEBundleMapFile will be preferred. + // Optional. + SPIFFEBundleMapFile string + // RefreshDuration is the amount of time the plugin waits before checking + // for updates in the specified files. + // Optional. If not set, a default value (1 hour) will be used. + RefreshDuration time.Duration +} + +func (o Options) canonical() []byte { + return []byte(fmt.Sprintf("%s:%s:%s:%s:%s", o.CertFile, o.KeyFile, o.RootFile, o.SPIFFEBundleMapFile, o.RefreshDuration)) +} + +func (o Options) validate() error { + if o.CertFile == "" && o.KeyFile == "" && o.RootFile == "" && o.SPIFFEBundleMapFile == "" { + return fmt.Errorf("pemfile: at least one credential file needs to be specified") + } + if keySpecified, certSpecified := o.KeyFile != "", o.CertFile != ""; keySpecified != certSpecified { + return fmt.Errorf("pemfile: private key file and identity cert file should be both specified or not specified") + } + // C-core has a limitation that they cannot verify that a certificate file + // matches a key file. So, the only way to get around this is to make sure + // that both files are in the same directory and that they do an atomic + // read. Even though Java/Go do not have this limitation, we want the + // overall plugin behavior to be consistent across languages. + if certDir, keyDir := filepath.Dir(o.CertFile), filepath.Dir(o.KeyFile); certDir != keyDir { + return errors.New("pemfile: certificate and key file must be in the same directory") + } + return nil +} + +// NewProvider returns a new certificate provider plugin that is configured to +// watch the PEM files specified in the passed in options. +func NewProvider(o Options) (certprovider.Provider, error) { + if err := o.validate(); err != nil { + return nil, err + } + return newProvider(o), nil +} + +// newProvider is used to create a new certificate provider plugin after +// validating the options, and hence does not return an error. +func newProvider(o Options) certprovider.Provider { + if o.RefreshDuration == 0 { + o.RefreshDuration = defaultCertRefreshDuration + } + + provider := &watcher{opts: o} + if o.CertFile != "" && o.KeyFile != "" { + provider.identityDistributor = newDistributor() + } + if o.RootFile != "" || o.SPIFFEBundleMapFile != "" { + provider.rootDistributor = newDistributor() + } + + ctx, cancel := context.WithCancel(context.Background()) + provider.cancel = cancel + go provider.run(ctx) + return provider +} + +// watcher is a certificate provider plugin that implements the +// certprovider.Provider interface. It watches a set of certificate and key +// files and provides the most up-to-date key material for consumption by +// credentials implementation. +type watcher struct { + identityDistributor distributor + rootDistributor distributor + opts Options + certFileContents []byte + keyFileContents []byte + rootFileContents []byte + spiffeBundleMapFileContents []byte + cancel context.CancelFunc +} + +// distributor wraps the methods on certprovider.Distributor which are used by +// the plugin. This is very useful in tests which need to know exactly when the +// plugin updates its key material. +type distributor interface { + KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) + Set(km *certprovider.KeyMaterial, err error) + Stop() +} + +// updateIdentityDistributor checks if the cert/key files that the plugin is +// watching have changed, and if so, reads the new contents and updates the +// identityDistributor with the new key material. +// +// Skips updates when file reading or parsing fails. +// TODO(easwars): Retry with limit (on the number of retries or the amount of +// time) upon failures. +func (w *watcher) updateIdentityDistributor() { + if w.identityDistributor == nil { + return + } + + certFileContents, err := os.ReadFile(w.opts.CertFile) + if err != nil { + logger.Warningf("certFile (%s) read failed: %v", w.opts.CertFile, err) + return + } + keyFileContents, err := os.ReadFile(w.opts.KeyFile) + if err != nil { + logger.Warningf("keyFile (%s) read failed: %v", w.opts.KeyFile, err) + return + } + // If the file contents have not changed, skip updating the distributor. + if bytes.Equal(w.certFileContents, certFileContents) && bytes.Equal(w.keyFileContents, keyFileContents) { + return + } + + cert, err := tls.X509KeyPair(certFileContents, keyFileContents) + if err != nil { + logger.Warningf("tls.X509KeyPair(%q, %q) failed: %v", certFileContents, keyFileContents, err) + return + } + w.certFileContents = certFileContents + w.keyFileContents = keyFileContents + w.identityDistributor.Set(&certprovider.KeyMaterial{Certs: []tls.Certificate{cert}}, nil) +} + +// updateRootDistributor checks if the root cert file that the plugin is +// watching hs changed, and if so, updates the rootDistributor with the new key +// material. +// +// Skips updates when root cert reading or parsing fails. +// TODO(easwars): Retry with limit (on the number of retries or the amount of +// time) upon failures. +func (w *watcher) updateRootDistributor() { + if w.rootDistributor == nil { + return + } + + // If SPIFFEBundleMap is set, use it and DON'T use the RootFile, even if it + // fails + if w.opts.SPIFFEBundleMapFile != "" { + w.maybeUpdateSPIFFEBundleMap() + } else { + w.maybeUpdateRootFile() + } +} + +func (w *watcher) maybeUpdateSPIFFEBundleMap() { + spiffeBundleMapContents, err := os.ReadFile(w.opts.SPIFFEBundleMapFile) + if err != nil { + logger.Warningf("spiffeBundleMapFile (%s) read failed: %v", w.opts.SPIFFEBundleMapFile, err) + return + } + // If the file contents have not changed, skip updating the distributor. + if bytes.Equal(w.spiffeBundleMapFileContents, spiffeBundleMapContents) { + return + } + bundleMap, err := spiffe.BundleMapFromBytes(spiffeBundleMapContents) + if err != nil { + logger.Warning("Failed to parse spiffe bundle map") + return + } + w.spiffeBundleMapFileContents = spiffeBundleMapContents + w.rootDistributor.Set(&certprovider.KeyMaterial{SPIFFEBundleMap: bundleMap}, nil) +} + +func (w *watcher) maybeUpdateRootFile() { + rootFileContents, err := os.ReadFile(w.opts.RootFile) + if err != nil { + logger.Warningf("rootFile (%s) read failed: %v", w.opts.RootFile, err) + return + } + trustPool := x509.NewCertPool() + if !trustPool.AppendCertsFromPEM(rootFileContents) { + logger.Warning("Failed to parse root certificate") + return + } + // If the file contents have not changed, skip updating the distributor. + if bytes.Equal(w.rootFileContents, rootFileContents) { + return + } + + w.rootFileContents = rootFileContents + w.rootDistributor.Set(&certprovider.KeyMaterial{Roots: trustPool}, nil) +} + +// run is a long running goroutine which watches the configured files for +// changes, and pushes new key material into the appropriate distributors which +// is returned from calls to KeyMaterial(). +func (w *watcher) run(ctx context.Context) { + ticker := time.NewTicker(w.opts.RefreshDuration) + for { + w.updateIdentityDistributor() + w.updateRootDistributor() + select { + case <-ctx.Done(): + ticker.Stop() + if w.identityDistributor != nil { + w.identityDistributor.Stop() + } + if w.rootDistributor != nil { + w.rootDistributor.Stop() + } + return + case <-ticker.C: + } + } +} + +// KeyMaterial returns the key material sourced by the watcher. +// Callers are expected to use the returned value as read-only. +func (w *watcher) KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) { + km := &certprovider.KeyMaterial{} + if w.identityDistributor != nil { + identityKM, err := w.identityDistributor.KeyMaterial(ctx) + if err != nil { + return nil, err + } + km.Certs = identityKM.Certs + } + if w.rootDistributor != nil { + rootKM, err := w.rootDistributor.KeyMaterial(ctx) + if err != nil { + return nil, err + } + km.SPIFFEBundleMap = rootKM.SPIFFEBundleMap + km.Roots = rootKM.Roots + } + return km, nil +} + +// Close cleans up resources allocated by the watcher. +func (w *watcher) Close() { + w.cancel() +} diff --git a/credentials/tls/certprovider/pemfile/watcher_test.go b/credentials/tls/certprovider/pemfile/watcher_test.go new file mode 100644 index 0000000..08e67ad --- /dev/null +++ b/credentials/tls/certprovider/pemfile/watcher_test.go @@ -0,0 +1,629 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package pemfile + +import ( + "context" + "fmt" + "os" + "path" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/testdata" +) + +const ( + // These are the names of files inside temporary directories, which the + // plugin is asked to watch. + certFile = "cert.pem" + keyFile = "key.pem" + rootFile = "ca.pem" + spiffeBundleFile = "spiffebundle.json" + + defaultTestRefreshDuration = 100 * time.Millisecond + defaultTestTimeout = 5 * time.Second +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func compareKeyMaterial(got, want *certprovider.KeyMaterial) error { + if len(got.Certs) != len(want.Certs) { + return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want) + } + for i := 0; i < len(got.Certs); i++ { + if !got.Certs[i].Leaf.Equal(want.Certs[i].Leaf) { + return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want) + } + } + + if gotR, wantR := got.Roots, want.Roots; !gotR.Equal(wantR) { + return fmt.Errorf("keyMaterial roots = %v, want %v", gotR, wantR) + } + + if gotBundle, wantBundle := got.SPIFFEBundleMap, want.SPIFFEBundleMap; !cmp.Equal(gotBundle, wantBundle) { + return fmt.Errorf("keyMaterial spiffe bundle map = %v, want %v", gotBundle, wantBundle) + } + + return nil +} + +// TestNewProvider tests the NewProvider() function with different inputs. +func (s) TestNewProvider(t *testing.T) { + tests := []struct { + desc string + options Options + wantError bool + }{ + { + desc: "No credential files specified", + options: Options{}, + wantError: true, + }, + { + desc: "Only identity cert is specified", + options: Options{ + CertFile: testdata.Path("x509/client1_cert.pem"), + }, + wantError: true, + }, + { + desc: "Only identity key is specified", + options: Options{ + KeyFile: testdata.Path("x509/client1_key.pem"), + }, + wantError: true, + }, + { + desc: "Identity cert/key pair is specified", + options: Options{ + KeyFile: testdata.Path("x509/client1_key.pem"), + CertFile: testdata.Path("x509/client1_cert.pem"), + }, + }, + { + desc: "Only root certs are specified", + options: Options{ + RootFile: testdata.Path("x509/client_ca_cert.pem"), + }, + }, + { + desc: "Only spiffe bundle map specified", + options: Options{ + SPIFFEBundleMapFile: testdata.Path("spiffe/spiffebundle.json"), + }, + }, + { + desc: "Everything is specified", + options: Options{ + KeyFile: testdata.Path("x509/client1_key.pem"), + CertFile: testdata.Path("x509/client1_cert.pem"), + RootFile: testdata.Path("x509/client_ca_cert.pem"), + SPIFFEBundleMapFile: testdata.Path("spiffe/spiffebundle.json"), + }, + wantError: false, + }, + } + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + provider, err := NewProvider(test.options) + if (err != nil) != test.wantError { + t.Fatalf("NewProvider(%v) = %v, want %v", test.options, err, test.wantError) + } + if err != nil { + return + } + provider.Close() + }) + } +} + +// wrappedDistributor wraps a distributor and pushes on a channel whenever new +// key material is pushed to the distributor. +type wrappedDistributor struct { + *certprovider.Distributor + distCh *testutils.Channel +} + +func newWrappedDistributor(distCh *testutils.Channel) *wrappedDistributor { + return &wrappedDistributor{ + distCh: distCh, + Distributor: certprovider.NewDistributor(), + } +} + +func (wd *wrappedDistributor) Set(km *certprovider.KeyMaterial, err error) { + wd.Distributor.Set(km, err) + wd.distCh.Send(nil) +} + +func createTmpFile(t *testing.T, src, dst string) { + t.Helper() + + data, err := os.ReadFile(src) + if err != nil { + t.Fatalf("os.ReadFile(%q) failed: %v", src, err) + } + if err := os.WriteFile(dst, data, os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(%q) failed: %v", dst, err) + } + t.Logf("Wrote file at: %s", dst) + t.Logf("%s", string(data)) +} + +func removeTmpFile(t *testing.T, filePath string) { + t.Helper() + if err := os.Remove(filePath); err != nil { + t.Fatalf("os.RemoveFIle(%q) failed: %v", filePath, err) + } + t.Logf("Removed file at: %s", filePath) +} + +// createTempDirWithFiles creates a temporary directory under the system default +// tempDir with the given dirSuffix. It also reads from certSrc, keySrc and +// rootSrc files are creates appropriate files under the newly create tempDir. +// Returns the name of the created tempDir. +func createTmpDirWithFiles(t *testing.T, dirSuffix, certSrc, keySrc, rootSrc, spiffeBundleSrc string) string { + t.Helper() + + // Create a temp directory. Passing an empty string for the first argument + // uses the system temp directory. + dir, err := os.MkdirTemp("", dirSuffix) + if err != nil { + t.Fatalf("os.MkdirTemp() failed: %v", err) + } + t.Logf("Using tmpdir: %s", dir) + + createTmpFile(t, testdata.Path(certSrc), path.Join(dir, certFile)) + createTmpFile(t, testdata.Path(keySrc), path.Join(dir, keyFile)) + createTmpFile(t, testdata.Path(rootSrc), path.Join(dir, rootFile)) + createTmpFile(t, testdata.Path(spiffeBundleSrc), path.Join(dir, spiffeBundleFile)) + return dir +} + +// initializeProvider performs setup steps common to all tests (except the one +// which uses symlinks). +func initializeProvider(t *testing.T, testName string, useSPIFFEBundle bool) (string, certprovider.Provider, *testutils.Channel, func()) { + t.Helper() + + // Override the newDistributor to one which pushes on a channel that we + // can block on. + origDistributorFunc := newDistributor + distCh := testutils.NewChannel() + d := newWrappedDistributor(distCh) + newDistributor = func() distributor { return d } + + // Create a new provider to watch the files in tmpdir. + dir := createTmpDirWithFiles(t, testName+"*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/client_ca_cert.pem", "spiffe/spiffebundle.json") + opts := Options{ + CertFile: path.Join(dir, certFile), + KeyFile: path.Join(dir, keyFile), + RootFile: path.Join(dir, rootFile), + RefreshDuration: defaultTestRefreshDuration, + } + if useSPIFFEBundle { + opts.SPIFFEBundleMapFile = path.Join(dir, spiffeBundleFile) + } + prov, err := NewProvider(opts) + if err != nil { + t.Fatalf("NewProvider(%+v) failed: %v", opts, err) + } + + // Make sure the provider picks up the files and pushes the key material on + // to the distributors. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for i := 0; i < 2; i++ { + // Since we have root and identity certs, we need to make sure the + // update is pushed on both of them. + if _, err := distCh.Receive(ctx); err != nil { + t.Fatalf("Timeout waiting for provider to read files and push key material to distributor: %v", err) + } + } + + return dir, prov, distCh, func() { + newDistributor = origDistributorFunc + prov.Close() + } +} + +// TestProvider_NoUpdate tests the case where a file watcher plugin is created +// successfully, and the underlying files do not change. Verifies that the +// plugin does not push new updates to the distributor in this case. +func (s) TestProvider_NoUpdate(t *testing.T) { + baseName := "no_update" + for _, useSPIFFEBundle := range []bool{true, false} { + testName := baseName + if useSPIFFEBundle { + testName = testName + "_" + "withSPIFFEBundle" + } + t.Run(testName, func(t *testing.T) { + _, prov, distCh, cancel := initializeProvider(t, "no_update", useSPIFFEBundle) + defer cancel() + + // Make sure the provider is healthy and returns key material. + ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cc() + if _, err := prov.KeyMaterial(ctx); err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + + // Files haven't change. Make sure no updates are pushed by the provider. + sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration) + defer sc() + if _, err := distCh.Receive(sCtx); err == nil { + t.Fatal("New key material pushed to distributor when underlying files did not change") + } + }) + } +} + +// TestProvider_UpdateSuccess tests the case where a file watcher plugin is +// created successfully and the underlying files change. Verifies that the +// changes are picked up by the provider. +func (s) TestProvider_UpdateSuccess(t *testing.T) { + baseName := "update_success" + for _, useSPIFFEBundle := range []bool{true, false} { + testName := baseName + if useSPIFFEBundle { + testName = testName + "_" + "withSPIFFEBundle" + } + t.Run(testName, func(t *testing.T) { + dir, prov, distCh, cancel := initializeProvider(t, "update_success", useSPIFFEBundle) + defer cancel() + + // Make sure the provider is healthy and returns key material. + ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cc() + km1, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + + // Change only the root file. + if useSPIFFEBundle { + createTmpFile(t, testdata.Path("spiffe/spiffebundle2.json"), path.Join(dir, spiffeBundleFile)) + } else { + createTmpFile(t, testdata.Path("x509/server_ca_cert.pem"), path.Join(dir, rootFile)) + } + if _, err := distCh.Receive(ctx); err != nil { + t.Fatal("Timeout waiting for new key material to be pushed to the distributor") + } + + // Make sure update is picked up. + km2, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + if err := compareKeyMaterial(km1, km2); err == nil { + t.Fatal("Expected provider to return new key material after update to underlying file") + } + + // Change only cert/key files. + createTmpFile(t, testdata.Path("x509/client2_cert.pem"), path.Join(dir, certFile)) + createTmpFile(t, testdata.Path("x509/client2_key.pem"), path.Join(dir, keyFile)) + if _, err := distCh.Receive(ctx); err != nil { + t.Fatal("Timeout waiting for new key material to be pushed to the distributor") + } + + // Make sure update is picked up. + km3, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + if err := compareKeyMaterial(km2, km3); err == nil { + t.Fatal("Expected provider to return new key material after update to underlying file") + } + }) + } +} + +// TestProvider_UpdateSuccessWithSymlink tests the case where a file watcher +// plugin is created successfully to watch files through a symlink and the +// symlink is updates to point to new files. Verifies that the changes are +// picked up by the provider. +func (s) TestProvider_UpdateSuccessWithSymlink(t *testing.T) { + baseName := "update_with_symlink" + for _, useSPIFFEBundle := range []bool{true, false} { + testName := baseName + if useSPIFFEBundle { + testName = testName + "_" + "withSPIFFEBundle" + } + t.Run(testName, func(t *testing.T) { + // Override the newDistributor to one which pushes on a channel that we + // can block on. + origDistributorFunc := newDistributor + distCh := testutils.NewChannel() + d := newWrappedDistributor(distCh) + newDistributor = func() distributor { return d } + defer func() { newDistributor = origDistributorFunc }() + + // Create two tempDirs with different files. + dir1 := createTmpDirWithFiles(t, "update_with_symlink1_*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/client_ca_cert.pem", "spiffe/spiffebundle.json") + dir2 := createTmpDirWithFiles(t, "update_with_symlink2_*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/server_ca_cert.pem", "spiffe/spiffebundle2.json") + + // Create a symlink under a new tempdir, and make it point to dir1. + tmpdir, err := os.MkdirTemp("", "test_symlink_*") + if err != nil { + t.Fatalf("os.MkdirTemp() failed: %v", err) + } + symLinkName := path.Join(tmpdir, "test_symlink") + if err := os.Symlink(dir1, symLinkName); err != nil { + t.Fatalf("Failed to create symlink to %q: %v", dir1, err) + } + + // Create a provider which watches the files pointed to by the symlink. + opts := Options{ + CertFile: path.Join(symLinkName, certFile), + KeyFile: path.Join(symLinkName, keyFile), + RootFile: path.Join(symLinkName, rootFile), + SPIFFEBundleMapFile: path.Join(symLinkName, spiffeBundleFile), + RefreshDuration: defaultTestRefreshDuration, + } + if useSPIFFEBundle { + opts.SPIFFEBundleMapFile = path.Join(symLinkName, spiffeBundleFile) + } + prov, err := NewProvider(opts) + if err != nil { + t.Fatalf("NewProvider(%+v) failed: %v", opts, err) + } + defer prov.Close() + + // Make sure the provider picks up the files and pushes the key material on + // to the distributors. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for i := 0; i < 2; i++ { + // Since we have root and identity certs, we need to make sure the + // update is pushed on both of them. + if _, err := distCh.Receive(ctx); err != nil { + t.Fatalf("Timeout waiting for provider to read files and push key material to distributor: %v", err) + } + } + km1, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + + // Update the symlink to point to dir2. + symLinkTmpName := path.Join(tmpdir, "test_symlink.tmp") + if err := os.Symlink(dir2, symLinkTmpName); err != nil { + t.Fatalf("Failed to create symlink to %q: %v", dir2, err) + } + if err := os.Rename(symLinkTmpName, symLinkName); err != nil { + t.Fatalf("Failed to update symlink: %v", err) + } + + // Make sure the provider picks up the new files and pushes the key material + // on to the distributors. + for i := 0; i < 2; i++ { + // Since we have root and identity certs, we need to make sure the + // update is pushed on both of them. + if _, err := distCh.Receive(ctx); err != nil { + t.Fatalf("Timeout waiting for provider to read files and push key material to distributor: %v", err) + } + } + km2, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + + if err := compareKeyMaterial(km1, km2); err == nil { + t.Fatal("Expected provider to return new key material after symlink update") + } + }) + } +} + +// TestProvider_UpdateFailure_ThenSuccess tests the case where updating cert/key +// files fail. Verifies that the failed update does not push anything on the +// distributor. Then the update succeeds, and the test verifies that the key +// material is updated. +func (s) TestProvider_UpdateFailure_ThenSuccess(t *testing.T) { + dir, prov, distCh, cancel := initializeProvider(t, "update_failure", false) + defer cancel() + + // Make sure the provider is healthy and returns key material. + ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cc() + km1, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + + // Update only the cert file. The key file is left unchanged. This should + // lead to these two files being not compatible with each other. This + // simulates the case where the watching goroutine might catch the files in + // the midst of an update. + createTmpFile(t, testdata.Path("x509/server1_cert.pem"), path.Join(dir, certFile)) + + // Since the last update left the files in an incompatible state, the update + // should not be picked up by our provider. + sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration) + defer sc() + if _, err := distCh.Receive(sCtx); err == nil { + t.Fatal("New key material pushed to distributor when underlying files did not change") + } + + // The provider should return key material corresponding to the old state. + km2, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + if err := compareKeyMaterial(km1, km2); err != nil { + t.Fatalf("Expected provider to not update key material: %v", err) + } + + // Update the key file to match the cert file. + createTmpFile(t, testdata.Path("x509/server1_key.pem"), path.Join(dir, keyFile)) + + // Make sure update is picked up. + if _, err := distCh.Receive(ctx); err != nil { + t.Fatal("Timeout waiting for new key material to be pushed to the distributor") + } + km3, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + if err := compareKeyMaterial(km2, km3); err == nil { + t.Fatal("Expected provider to return new key material after update to underlying file") + } +} + +// TestProvider_UpdateFailure_ThenSuccess tests the case where updating cert/key +// files fail. Verifies that the failed update does not push anything on the +// distributor. Then the update succeeds, and the test verifies that the key +// material is updated. +func (s) TestProvider_UpdateFailureSPIFFE(t *testing.T) { + tests := []struct { + name string + badFile string + }{ + { + name: "malformed spiffe", + badFile: "spiffe/spiffebundle_malformed.json", + }, + { + name: "invalid bundle", + badFile: "spiffe/spiffebundle_wrong_kty.json", + }, + { + name: "cert in the x5c field is invalid", + badFile: "spiffe/spiffebundle_corrupted_cert.json", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir, prov, distCh, cancel := initializeProvider(t, tc.name, true) + defer cancel() + + // Make sure the provider is healthy and returns key material. + ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cc() + km1, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + + // Update the file with a bad update + createTmpFile(t, testdata.Path(tc.badFile), path.Join(dir, spiffeBundleFile)) + + // Since the last update left the files in an incompatible state, the update + // should not be picked up by our provider. + sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration) + defer sc() + if _, err := distCh.Receive(sCtx); err == nil { + t.Fatal("New key material pushed to distributor when underlying files did not change") + } + + // The provider should return key material corresponding to the old state. + km2, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + if err := compareKeyMaterial(km1, km2); err != nil { + t.Fatalf("Expected provider to not update key material: %v", err) + } + }) + } +} + +// TestProvider_UpdateFailure_ThenSuccess tests the case where updating cert/key +// files fail. Verifies that the failed update does not push anything on the +// distributor. Then the update succeeds, and the test verifies that the key +// material is updated. +func (s) TestProvider_UpdateFailureSPIFFE_MissingFile(t *testing.T) { + dir, prov, distCh, cancel := initializeProvider(t, "Delete spiffe file being read", true) + defer cancel() + + // Make sure the provider is healthy and returns key material. + ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cc() + km1, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + + // Remove the file that we are reading + removeTmpFile(t, path.Join(dir, spiffeBundleFile)) + + // Since the last update left the files in an incompatible state, the update + // should not be picked up by our provider. + sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration) + defer sc() + if _, err := distCh.Receive(sCtx); err == nil { + t.Fatal("new key material pushed to distributor when underlying files did not change") + } + + // The provider should return key material corresponding to the old state. + km2, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + if err := compareKeyMaterial(km1, km2); err != nil { + t.Fatalf("expected provider to not update key material: %v", err) + } +} + +// TestProvider_UpdateFailure_ThenSuccess tests the case where updating cert/key +// files fail. Verifies that the failed update does not push anything on the +// distributor. Then the update succeeds, and the test verifies that the key +// material is updated. +func (s) TestProvider_UpdateFailureRoot_MissingFile(t *testing.T) { + dir, prov, distCh, cancel := initializeProvider(t, "Delete root file being read", false) + defer cancel() + + // Make sure the provider is healthy and returns key material. + ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cc() + km1, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + + // Remove the file that we are reading + removeTmpFile(t, path.Join(dir, rootFile)) + + // Since the last update left the files in an incompatible state, the update + // should not be picked up by our provider. + sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration) + defer sc() + if _, err := distCh.Receive(sCtx); err == nil { + t.Fatal("new key material pushed to distributor when underlying files did not change") + } + + // The provider should return key material corresponding to the old state. + km2, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + if err := compareKeyMaterial(km1, km2); err != nil { + t.Fatalf("expected provider to not update key material: %v", err) + } +} diff --git a/credentials/tls/certprovider/provider.go b/credentials/tls/certprovider/provider.go new file mode 100644 index 0000000..c415b82 --- /dev/null +++ b/credentials/tls/certprovider/provider.go @@ -0,0 +1,115 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package certprovider defines APIs for Certificate Providers in gRPC. +// +// # Experimental +// +// Notice: All APIs in this package are experimental and may be removed in a +// later release. +package certprovider + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + + "github.com/spiffe/go-spiffe/v2/bundle/spiffebundle" + "google.golang.org/grpc/internal" +) + +func init() { + internal.GetCertificateProviderBuilder = getBuilder +} + +var ( + // errProviderClosed is returned by Distributor.KeyMaterial when it is + // closed. + errProviderClosed = errors.New("provider instance is closed") + + // m is a map from name to Provider builder. + m = make(map[string]Builder) +) + +// Register registers the Provider builder, whose name as returned by its Name() +// method will be used as the name registered with this builder. Registered +// Builders are used by the Store to create Providers. +func Register(b Builder) { + m[b.Name()] = b +} + +// getBuilder returns the Provider builder registered with the given name. +// If no builder is registered with the provided name, nil will be returned. +func getBuilder(name string) Builder { + if b, ok := m[name]; ok { + return b + } + return nil +} + +// Builder creates a Provider. +type Builder interface { + // ParseConfig parses the given config, which is in a format specific to individual + // implementations, and returns a BuildableConfig on success. + ParseConfig(any) (*BuildableConfig, error) + + // Name returns the name of providers built by this builder. + Name() string +} + +// Provider makes it possible to keep channel credential implementations up to +// date with secrets that they rely on to secure communications on the +// underlying channel. +// +// Provider implementations are free to rely on local or remote sources to fetch +// the latest secrets, and free to share any state between different +// instantiations as they deem fit. +type Provider interface { + // KeyMaterial returns the key material sourced by the Provider. + // Callers are expected to use the returned value as read-only. + KeyMaterial(ctx context.Context) (*KeyMaterial, error) + + // Close cleans up resources allocated by the Provider. + Close() +} + +// KeyMaterial wraps the certificates and keys returned by a Provider instance. +type KeyMaterial struct { + // Certs contains a slice of cert/key pairs used to prove local identity. + Certs []tls.Certificate + // Roots contains the set of trusted roots to validate the peer's identity. + // This field will only be used if the `SPIFFEBundleMap` field is unset. + Roots *x509.CertPool + // SPIFFEBundleMap is an in-memory representation of a spiffe trust bundle + // map. If this value exists, it will be used to find the roots for a given + // trust domain rather than the Roots in this struct. + SPIFFEBundleMap map[string]*spiffebundle.Bundle +} + +// BuildOptions contains parameters passed to a Provider at build time. +type BuildOptions struct { + // CertName holds the certificate name, whose key material is of interest to + // the caller. + CertName string + // WantRoot indicates if the caller is interested in the root certificate. + WantRoot bool + // WantIdentity indicates if the caller is interested in the identity + // certificate. + WantIdentity bool +} diff --git a/credentials/tls/certprovider/store.go b/credentials/tls/certprovider/store.go new file mode 100644 index 0000000..a4b99e3 --- /dev/null +++ b/credentials/tls/certprovider/store.go @@ -0,0 +1,196 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package certprovider + +import ( + "context" + "fmt" + "sync" + "sync/atomic" +) + +// provStore is the global singleton certificate provider store. +var provStore = &store{ + providers: make(map[storeKey]*wrappedProvider), +} + +// storeKey acts as the key to the map of providers maintained by the store. A +// combination of provider name and configuration is used to uniquely identify +// every provider instance in the store. Go maps need to be indexed by +// comparable types, so the provider configuration is converted from `any` to +// `string` using the ParseConfig method while creating this key. +type storeKey struct { + // name of the certificate provider. + name string + // configuration of the certificate provider in string form. + config string + // opts contains the certificate name and other keyMaterial options. + opts BuildOptions +} + +// wrappedProvider wraps a provider instance with a reference count. +type wrappedProvider struct { + Provider + refCount int + + // A reference to the key and store are also kept here to override the + // Close method on the provider. + storeKey storeKey + store *store +} + +// closedProvider always returns errProviderClosed error. +type closedProvider struct{} + +func (c closedProvider) KeyMaterial(context.Context) (*KeyMaterial, error) { + return nil, errProviderClosed +} + +func (c closedProvider) Close() { +} + +// singleCloseWrappedProvider wraps a provider instance with a reference count +// to properly handle multiple calls to Close. +type singleCloseWrappedProvider struct { + provider atomic.Pointer[Provider] +} + +// store is a collection of provider instances, safe for concurrent access. +type store struct { + mu sync.Mutex + providers map[storeKey]*wrappedProvider +} + +// Close overrides the Close method of the embedded provider. It releases the +// reference held by the caller on the underlying provider and if the +// provider's reference count reaches zero, it is removed from the store, and +// its Close method is also invoked. +func (wp *wrappedProvider) Close() { + ps := wp.store + ps.mu.Lock() + defer ps.mu.Unlock() + + wp.refCount-- + if wp.refCount == 0 { + wp.Provider.Close() + delete(ps.providers, wp.storeKey) + } +} + +// Close overrides the Close method of the embedded provider to avoid release the +// already released reference. +func (w *singleCloseWrappedProvider) Close() { + newProvider := Provider(closedProvider{}) + oldProvider := w.provider.Swap(&newProvider) + (*oldProvider).Close() +} + +// KeyMaterial returns the key material sourced by the Provider. +// Callers are expected to use the returned value as read-only. +func (w *singleCloseWrappedProvider) KeyMaterial(ctx context.Context) (*KeyMaterial, error) { + return (*w.provider.Load()).KeyMaterial(ctx) +} + +// newSingleCloseWrappedProvider create wrapper a provider instance with a reference count +// to properly handle multiple calls to Close. +func newSingleCloseWrappedProvider(provider Provider) *singleCloseWrappedProvider { + w := &singleCloseWrappedProvider{} + w.provider.Store(&provider) + return w +} + +// BuildableConfig wraps parsed provider configuration and functionality to +// instantiate provider instances. +type BuildableConfig struct { + name string + config []byte + starter func(BuildOptions) Provider + pStore *store +} + +// NewBuildableConfig creates a new BuildableConfig with the given arguments. +// Provider implementations are expected to invoke this function after parsing +// the given configuration as part of their ParseConfig() method. +// Equivalent configurations are expected to invoke this function with the same +// config argument. +func NewBuildableConfig(name string, config []byte, starter func(BuildOptions) Provider) *BuildableConfig { + return &BuildableConfig{ + name: name, + config: config, + starter: starter, + pStore: provStore, + } +} + +// Build kicks off a provider instance with the wrapped configuration. Multiple +// invocations of this method with the same opts will result in provider +// instances being reused. +func (bc *BuildableConfig) Build(opts BuildOptions) (Provider, error) { + provStore.mu.Lock() + defer provStore.mu.Unlock() + + sk := storeKey{ + name: bc.name, + config: string(bc.config), + opts: opts, + } + if wp, ok := provStore.providers[sk]; ok { + wp.refCount++ + return newSingleCloseWrappedProvider(wp), nil + } + + provider := bc.starter(opts) + if provider == nil { + return nil, fmt.Errorf("provider(%q, %q).Build(%v) failed", sk.name, sk.config, opts) + } + wp := &wrappedProvider{ + Provider: provider, + refCount: 1, + storeKey: sk, + store: provStore, + } + provStore.providers[sk] = wp + return newSingleCloseWrappedProvider(wp), nil +} + +// String returns the provider name and config as a colon separated string. +func (bc *BuildableConfig) String() string { + return fmt.Sprintf("%s:%s", bc.name, string(bc.config)) +} + +// ParseConfig is a convenience function to create a BuildableConfig given a +// provider name and configuration. Returns an error if there is no registered +// builder for the given name or if the config parsing fails. +func ParseConfig(name string, config any) (*BuildableConfig, error) { + parser := getBuilder(name) + if parser == nil { + return nil, fmt.Errorf("no certificate provider builder found for %q", name) + } + return parser.ParseConfig(config) +} + +// GetProvider is a convenience function to create a provider given the name, +// config and build options. +func GetProvider(name string, config any, opts BuildOptions) (Provider, error) { + bc, err := ParseConfig(name, config) + if err != nil { + return nil, err + } + return bc.Build(opts) +} diff --git a/credentials/tls/certprovider/store_test.go b/credentials/tls/certprovider/store_test.go new file mode 100644 index 0000000..d6f673b --- /dev/null +++ b/credentials/tls/certprovider/store_test.go @@ -0,0 +1,400 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package certprovider + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "os" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/testdata" +) + +const ( + fakeProvider1Name = "fake-certificate-provider-1" + fakeProvider2Name = "fake-certificate-provider-2" + fakeConfig = "my fake config" + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +var fpb1, fpb2 *fakeProviderBuilder + +func init() { + fpb1 = &fakeProviderBuilder{ + name: fakeProvider1Name, + providerChan: testutils.NewChannel(), + } + fpb2 = &fakeProviderBuilder{ + name: fakeProvider2Name, + providerChan: testutils.NewChannel(), + } + Register(fpb1) + Register(fpb2) +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// fakeProviderBuilder builds new instances of fakeProvider and interprets the +// config provided to it as a string. +type fakeProviderBuilder struct { + name string + providerChan *testutils.Channel +} + +func (b *fakeProviderBuilder) ParseConfig(config any) (*BuildableConfig, error) { + s, ok := config.(string) + if !ok { + return nil, fmt.Errorf("providerBuilder %s received config of type %T, want string", b.name, config) + } + return NewBuildableConfig(b.name, []byte(s), func(BuildOptions) Provider { + fp := &fakeProvider{ + Distributor: NewDistributor(), + config: s, + } + b.providerChan.Send(fp) + return fp + }), nil +} + +func (b *fakeProviderBuilder) Name() string { + return b.name +} + +// fakeProvider is an implementation of the Provider interface which provides a +// method for tests to invoke to push new key materials. +type fakeProvider struct { + *Distributor + config string +} + +func (p *fakeProvider) Start(BuildOptions) Provider { + // This is practically a no-op since this provider doesn't do any work which + // needs to be started at this point. + return p +} + +// newKeyMaterial allows tests to push new key material to the fake provider +// which will be made available to users of this provider. +func (p *fakeProvider) newKeyMaterial(km *KeyMaterial, err error) { + p.Distributor.Set(km, err) +} + +// Close helps implement the Provider interface. +func (p *fakeProvider) Close() { + p.Distributor.Stop() +} + +// loadKeyMaterials is a helper to read cert/key files from testdata and convert +// them into a KeyMaterialReader struct. +func loadKeyMaterials(t *testing.T, cert, key, ca string) *KeyMaterial { + t.Helper() + + certs, err := tls.LoadX509KeyPair(testdata.Path(cert), testdata.Path(key)) + if err != nil { + t.Fatalf("Failed to load keyPair: %v", err) + } + + pemData, err := os.ReadFile(testdata.Path(ca)) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(pemData) + return &KeyMaterial{Certs: []tls.Certificate{certs}, Roots: roots} +} + +// kmReader wraps the KeyMaterial method exposed by Provider and Distributor +// implementations. Defining the interface here makes it possible to use the +// same helper from both provider and distributor tests. +type kmReader interface { + KeyMaterial(context.Context) (*KeyMaterial, error) +} + +// readAndVerifyKeyMaterial attempts to read key material from the given +// provider and compares it against the expected key material. +func readAndVerifyKeyMaterial(ctx context.Context, kmr kmReader, wantKM *KeyMaterial) error { + gotKM, err := kmr.KeyMaterial(ctx) + if err != nil { + return fmt.Errorf("KeyMaterial(ctx) failed: %w", err) + } + return compareKeyMaterial(gotKM, wantKM) +} + +func compareKeyMaterial(got, want *KeyMaterial) error { + if len(got.Certs) != len(want.Certs) { + return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want) + } + for i := 0; i < len(got.Certs); i++ { + if !got.Certs[i].Leaf.Equal(want.Certs[i].Leaf) { + return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want) + } + } + + if gotR, wantR := got.Roots, want.Roots; !gotR.Equal(wantR) { + return fmt.Errorf("keyMaterial roots = %v, want %v", gotR, wantR) + } + + return nil +} + +func createProvider(t *testing.T, name, config string, opts BuildOptions) Provider { + t.Helper() + prov, err := GetProvider(name, config, opts) + if err != nil { + t.Fatalf("GetProvider(%s, %s, %v) failed: %v", name, config, opts, err) + } + return prov +} + +// TestStoreSingleProvider creates a single provider through the store and calls +// methods on them. +func (s) TestStoreSingleProvider(t *testing.T) { + prov := createProvider(t, fakeProvider1Name, fakeConfig, BuildOptions{CertName: "default"}) + defer prov.Close() + + // Our fakeProviderBuilder pushes newly created providers on a channel. Grab + // the fake provider from that channel. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + p, err := fpb1.providerChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) + } + fakeProv := p.(*fakeProvider) + + // Attempt to read from key material from the Provider returned by the + // store. This will fail because we have not pushed any key material into + // our fake provider. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if err := readAndVerifyKeyMaterial(sCtx, prov, nil); !errors.Is(err, context.DeadlineExceeded) { + t.Fatal(err) + } + + // Load key material from testdata directory, push it into out fakeProvider + // and attempt to read from the Provider returned by the store. + testKM1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + fakeProv.newKeyMaterial(testKM1, nil) + if err := readAndVerifyKeyMaterial(ctx, prov, testKM1); err != nil { + t.Fatal(err) + } + + // Push new key material and read from the Provider. This should returned + // updated key material. + testKM2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") + fakeProv.newKeyMaterial(testKM2, nil) + if err := readAndVerifyKeyMaterial(ctx, prov, testKM2); err != nil { + t.Fatal(err) + } +} + +// TestStoreSingleProviderSameConfigDifferentOpts creates multiple providers of +// same type, for same configs but different keyMaterial options through the +// store (and expects the store's sharing mechanism to kick in) and calls +// methods on them. +func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) { + // Create three readers on the same fake provider. Two of these readers use + // certName `foo`, while the third one uses certName `bar`. + optsFoo := BuildOptions{CertName: "foo"} + provFoo1 := createProvider(t, fakeProvider1Name, fakeConfig, optsFoo) + provFoo2 := createProvider(t, fakeProvider1Name, fakeConfig, optsFoo) + defer func() { + provFoo1.Close() + provFoo2.Close() + }() + + // Our fakeProviderBuilder pushes newly created providers on a channel. + // Grab the fake provider for optsFoo. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + p, err := fpb1.providerChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) + } + fakeProvFoo := p.(*fakeProvider) + + // Make sure only provider was created by the builder so far. The store + // should be able to share the providers. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := fpb1.providerChan.Receive(sCtx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("A second provider created when expected to be shared by the store") + } + + optsBar := BuildOptions{CertName: "bar"} + provBar1 := createProvider(t, fakeProvider1Name, fakeConfig, optsBar) + defer provBar1.Close() + + // Grab the fake provider for optsBar. + p, err = fpb1.providerChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) + } + fakeProvBar := p.(*fakeProvider) + + // Push key material for optsFoo, and make sure the foo providers return + // appropriate key material and the bar provider times out. + fooKM := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + fakeProvFoo.newKeyMaterial(fooKM, nil) + if err := readAndVerifyKeyMaterial(ctx, provFoo1, fooKM); err != nil { + t.Fatal(err) + } + if err := readAndVerifyKeyMaterial(ctx, provFoo2, fooKM); err != nil { + t.Fatal(err) + } + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if err := readAndVerifyKeyMaterial(sCtx, provBar1, nil); !errors.Is(err, context.DeadlineExceeded) { + t.Fatal(err) + } + + // Push key material for optsBar, and make sure the bar provider returns + // appropriate key material. + barKM := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") + fakeProvBar.newKeyMaterial(barKM, nil) + if err := readAndVerifyKeyMaterial(ctx, provBar1, barKM); err != nil { + t.Fatal(err) + } + + // Make sure the above push of new key material does not affect foo readers. + if err := readAndVerifyKeyMaterial(ctx, provFoo1, fooKM); err != nil { + t.Fatal(err) + } +} + +// TestStoreSingleProviderDifferentConfigs creates multiple instances of the +// same type of provider through the store with different configs. The store +// would end up creating different provider instances for these and no sharing +// would take place. +func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) { + // Create two providers of the same type, but with different configs. + opts := BuildOptions{CertName: "foo"} + cfg1 := fakeConfig + "1111" + cfg2 := fakeConfig + "2222" + + prov1 := createProvider(t, fakeProvider1Name, cfg1, opts) + defer prov1.Close() + // Our fakeProviderBuilder pushes newly created providers on a channel. Grab + // the fake provider from that channel. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + p1, err := fpb1.providerChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) + } + fakeProv1 := p1.(*fakeProvider) + + prov2 := createProvider(t, fakeProvider1Name, cfg2, opts) + defer prov2.Close() + // Grab the second provider from the channel. + p2, err := fpb1.providerChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) + } + fakeProv2 := p2.(*fakeProvider) + + // Push the same key material into both fake providers and verify that the + // providers returned by the store return the appropriate key material. + km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + fakeProv1.newKeyMaterial(km1, nil) + fakeProv2.newKeyMaterial(km1, nil) + if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil { + t.Fatal(err) + } + if err := readAndVerifyKeyMaterial(ctx, prov2, km1); err != nil { + t.Fatal(err) + } + + // Push new key material into only one of the fake providers and verify + // that the providers returned by the store return the appropriate key + // material. + km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") + fakeProv2.newKeyMaterial(km2, nil) + if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil { + t.Fatal(err) + } + if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil { + t.Fatal(err) + } + + // Close one of the providers and verify that the other one is not affected. + prov1.Close() + if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil { + t.Fatal(err) + } +} + +// TestStoreMultipleProviders creates providers of different types and makes +// sure closing of one does not affect the other. +func (s) TestStoreMultipleProviders(t *testing.T) { + opts := BuildOptions{CertName: "foo"} + prov1 := createProvider(t, fakeProvider1Name, fakeConfig, opts) + defer prov1.Close() + // Our fakeProviderBuilder pushes newly created providers on a channel. Grab + // the fake provider from that channel. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + p1, err := fpb1.providerChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) + } + fakeProv1 := p1.(*fakeProvider) + + prov2 := createProvider(t, fakeProvider2Name, fakeConfig, opts) + defer prov2.Close() + // Grab the second provider from the channel. + p2, err := fpb2.providerChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider2Name) + } + fakeProv2 := p2.(*fakeProvider) + + // Push the key material into both providers and verify that the + // readers return the appropriate key material. + km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + fakeProv1.newKeyMaterial(km1, nil) + km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") + fakeProv2.newKeyMaterial(km2, nil) + if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil { + t.Fatal(err) + } + if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil { + t.Fatal(err) + } + + // Close one of the providers and verify that the other one is not affected. + prov1.Close() + if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil { + t.Fatal(err) + } +} diff --git a/credentials/tls_ext_test.go b/credentials/tls_ext_test.go new file mode 100644 index 0000000..ceb810a --- /dev/null +++ b/credentials/tls_ext_test.go @@ -0,0 +1,608 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials_test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "os" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const defaultTestTimeout = 10 * time.Second +const defaultTestShortTimeout = 10 * time.Millisecond + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +var serverCert tls.Certificate +var certPool *x509.CertPool +var serverName = "x.test.example.com" + +func init() { + var err error + serverCert, err = tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + panic(fmt.Sprintf("tls.LoadX509KeyPair(server1.pem, server1.key) failed: %v", err)) + } + + b, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem")) + if err != nil { + panic(fmt.Sprintf("Error reading CA cert file: %v", err)) + } + certPool = x509.NewCertPool() + if !certPool.AppendCertsFromPEM(b) { + panic("Error appending cert from PEM") + } +} + +// Tests that the MinVersion of tls.Config is set to 1.2 if it is not already +// set by the user. +func (s) TestTLS_MinVersion12(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + testCases := []struct { + name string + serverTLS func() *tls.Config + }{ + { + name: "base_case", + serverTLS: func() *tls.Config { + return &tls.Config{ + // MinVersion should be set to 1.2 by gRPC by default. + Certificates: []tls.Certificate{serverCert}, + } + }, + }, + { + name: "fallback_to_base", + serverTLS: func() *tls.Config { + config := &tls.Config{ + // MinVersion should be set to 1.2 by gRPC by default. + Certificates: []tls.Certificate{serverCert}, + } + config.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + return nil, nil + } + return config + }, + }, + { + name: "dynamic_using_get_config_for_client", + serverTLS: func() *tls.Config { + return &tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + return &tls.Config{ + // MinVersion should be set to 1.2 by gRPC by default. + Certificates: []tls.Certificate{serverCert}, + }, nil + }, + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create server creds without a minimum version. + serverCreds := credentials.NewTLS(tc.serverTLS()) + ss := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + // Create client creds that supports V1.0-V1.1. + clientCreds := credentials.NewTLS(&tls.Config{ + ServerName: serverName, + RootCAs: certPool, + MinVersion: tls.VersionTLS10, + MaxVersion: tls.VersionTLS11, + }) + + // Start server and client separately, because Start() blocks on a + // successful connection, which we will not get. + if err := ss.StartServer(grpc.Creds(serverCreds)); err != nil { + t.Fatalf("Error starting server: %v", err) + } + defer ss.Stop() + + cc, err := grpc.NewClient(ss.Address, grpc.WithTransportCredentials(clientCreds)) + if err != nil { + t.Fatalf("grpc.NewClient error: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + const wantStr = "authentication handshake failed" + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(status.Convert(err).Message(), wantStr) { + t.Fatalf("EmptyCall err = %v; want code=%v, message contains %q", err, codes.Unavailable, wantStr) + } + + }) + } +} + +// Tests that the MinVersion of tls.Config is not changed if it is set by the +// user. +func (s) TestTLS_MinVersionOverridable(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + var allCipherSuites []uint16 + for _, cs := range tls.CipherSuites() { + allCipherSuites = append(allCipherSuites, cs.ID) + } + testCases := []struct { + name string + serverTLS func() *tls.Config + }{ + { + name: "base_case", + serverTLS: func() *tls.Config { + return &tls.Config{ + MinVersion: tls.VersionTLS10, + Certificates: []tls.Certificate{serverCert}, + CipherSuites: allCipherSuites, + } + }, + }, + { + name: "fallback_to_base", + serverTLS: func() *tls.Config { + config := &tls.Config{ + MinVersion: tls.VersionTLS10, + Certificates: []tls.Certificate{serverCert}, + CipherSuites: allCipherSuites, + } + config.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + return nil, nil + } + return config + }, + }, + { + name: "dynamic_using_get_config_for_client", + serverTLS: func() *tls.Config { + return &tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + return &tls.Config{ + MinVersion: tls.VersionTLS10, + Certificates: []tls.Certificate{serverCert}, + CipherSuites: allCipherSuites, + }, nil + }, + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create server creds that allow v1.0. + serverCreds := credentials.NewTLS(tc.serverTLS()) + ss := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + // Create client creds that supports V1.0-V1.1. + clientCreds := credentials.NewTLS(&tls.Config{ + ServerName: serverName, + RootCAs: certPool, + CipherSuites: allCipherSuites, + MinVersion: tls.VersionTLS10, + MaxVersion: tls.VersionTLS11, + }) + + if err := ss.Start([]grpc.ServerOption{grpc.Creds(serverCreds)}, grpc.WithTransportCredentials(clientCreds)); err != nil { + t.Fatalf("Error starting stub server: %v", err) + } + defer ss.Stop() + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall err = %v; want ", err) + } + }) + } +} + +// Tests that CipherSuites is set to exclude HTTP/2 forbidden suites by default. +func (s) TestTLS_CipherSuites(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testCases := []struct { + name string + serverTLS func() *tls.Config + }{ + { + name: "base_case", + serverTLS: func() *tls.Config { + return &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + } + }, + }, + { + name: "fallback_to_base", + serverTLS: func() *tls.Config { + config := &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + } + config.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + return nil, nil + } + return config + }, + }, + { + name: "dynamic_using_get_config_for_client", + serverTLS: func() *tls.Config { + return &tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + return &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + }, nil + }, + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create server creds without cipher suites. + serverCreds := credentials.NewTLS(tc.serverTLS()) + ss := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + // Create client creds that use a forbidden suite only. + clientCreds := credentials.NewTLS(&tls.Config{ + ServerName: serverName, + RootCAs: certPool, + CipherSuites: []uint16{tls.TLS_RSA_WITH_AES_128_CBC_SHA}, + MaxVersion: tls.VersionTLS12, // TLS1.3 cipher suites are not configurable, so limit to 1.2. + }) + + // Start server and client separately, because Start() blocks on a + // successful connection, which we will not get. + if err := ss.StartServer(grpc.Creds(serverCreds)); err != nil { + t.Fatalf("Error starting server: %v", err) + } + defer ss.Stop() + + cc, err := grpc.NewClient("dns:"+ss.Address, grpc.WithTransportCredentials(clientCreds)) + if err != nil { + t.Fatalf("grpc.NewClient error: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + const wantStr = "authentication handshake failed" + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(status.Convert(err).Message(), wantStr) { + t.Fatalf("EmptyCall err = %v; want code=%v, message contains %q", err, codes.Unavailable, wantStr) + } + }) + } +} + +// Tests that CipherSuites is not overridden when it is set. +func (s) TestTLS_CipherSuitesOverridable(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + testCases := []struct { + name string + serverTLS func() *tls.Config + }{ + { + name: "base_case", + serverTLS: func() *tls.Config { + return &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + CipherSuites: []uint16{tls.TLS_RSA_WITH_AES_128_CBC_SHA}, + } + }, + }, + { + name: "fallback_to_base", + serverTLS: func() *tls.Config { + config := &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + CipherSuites: []uint16{tls.TLS_RSA_WITH_AES_128_CBC_SHA}, + } + config.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + return nil, nil + } + return config + }, + }, + { + name: "dynamic_using_get_config_for_client", + serverTLS: func() *tls.Config { + return &tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + return &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + CipherSuites: []uint16{tls.TLS_RSA_WITH_AES_128_CBC_SHA}, + }, nil + }, + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create server that allows only a forbidden cipher suite. + serverCreds := credentials.NewTLS(tc.serverTLS()) + ss := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + // Create server that allows only a forbidden cipher suite. + clientCreds := credentials.NewTLS(&tls.Config{ + ServerName: serverName, + RootCAs: certPool, + CipherSuites: []uint16{tls.TLS_RSA_WITH_AES_128_CBC_SHA}, + MaxVersion: tls.VersionTLS12, // TLS1.3 cipher suites are not configurable, so limit to 1.2. + }) + + if err := ss.Start([]grpc.ServerOption{grpc.Creds(serverCreds)}, grpc.WithTransportCredentials(clientCreds)); err != nil { + t.Fatalf("Error starting stub server: %v", err) + } + defer ss.Stop() + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall err = %v; want ", err) + } + }) + } +} + +// TestTLS_ServerConfiguresALPNByDefault verifies that ALPN is configured +// correctly for a server that doesn't specify the NextProtos field and uses +// GetConfigForClient to provide the TLS config during the handshake. +func (s) TestTLS_ServerConfiguresALPNByDefault(t *testing.T) { + initialVal := envconfig.EnforceALPNEnabled + defer func() { + envconfig.EnforceALPNEnabled = initialVal + }() + envconfig.EnforceALPNEnabled = true + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create a server that doesn't set the NextProtos field. + serverCreds := credentials.NewTLS(&tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + return &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + }, nil + }, + }) + + ss := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + clientCreds := credentials.NewTLS(&tls.Config{ + ServerName: serverName, + RootCAs: certPool, + }) + + if err := ss.Start([]grpc.ServerOption{grpc.Creds(serverCreds)}, grpc.WithTransportCredentials(clientCreds)); err != nil { + t.Fatalf("Error starting stub server: %v", err) + } + defer ss.Stop() + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall err = %v; want ", err) + } +} + +// TestTLS_DisabledALPNClient tests the behaviour of TransportCredentials when +// connecting to a server that doesn't support ALPN. +func (s) TestTLS_DisabledALPNClient(t *testing.T) { + initialVal := envconfig.EnforceALPNEnabled + defer func() { + envconfig.EnforceALPNEnabled = initialVal + }() + + tests := []struct { + name string + alpnEnforced bool + wantErr bool + }{ + { + name: "enforced", + alpnEnforced: true, + wantErr: true, + }, + { + name: "not_enforced", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + envconfig.EnforceALPNEnabled = tc.alpnEnforced + + listener, err := tls.Listen("tcp", "localhost:0", &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + NextProtos: []string{}, // Empty list indicates ALPN is disabled. + }) + if err != nil { + t.Fatalf("Error starting TLS server: %v", err) + } + + errCh := make(chan error, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + errCh <- fmt.Errorf("listener.Accept returned error: %v", err) + } else { + // The first write to the TLS listener initiates the TLS handshake. + conn.Write([]byte("Hello, World!")) + conn.Close() + } + close(errCh) + }() + + serverAddr := listener.Addr().String() + conn, err := net.Dial("tcp", serverAddr) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", serverAddr, err) + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + clientCfg := tls.Config{ + ServerName: serverName, + RootCAs: certPool, + NextProtos: []string{"h2"}, + } + _, _, err = credentials.NewTLS(&clientCfg).ClientHandshake(ctx, serverName, conn) + + if gotErr := (err != nil); gotErr != tc.wantErr { + t.Errorf("ClientHandshake returned unexpected error: got=%v, want=%t", err, tc.wantErr) + } + + select { + case err := <-errCh: + if err != nil { + t.Fatalf("Unexpected error received from server: %v", err) + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for error from server") + } + }) + } +} + +// TestTLS_DisabledALPNServer tests the behaviour of TransportCredentials when +// accepting a request from a client that doesn't support ALPN. +func (s) TestTLS_DisabledALPNServer(t *testing.T) { + initialVal := envconfig.EnforceALPNEnabled + defer func() { + envconfig.EnforceALPNEnabled = initialVal + }() + + tests := []struct { + name string + alpnEnforced bool + wantErr bool + }{ + { + name: "enforced", + alpnEnforced: true, + wantErr: true, + }, + { + name: "not_enforced", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + envconfig.EnforceALPNEnabled = tc.alpnEnforced + + listener, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error starting server: %v", err) + } + + errCh := make(chan error, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + errCh <- fmt.Errorf("listener.Accept returned error: %v", err) + return + } + defer conn.Close() + serverCfg := tls.Config{ + Certificates: []tls.Certificate{serverCert}, + NextProtos: []string{"h2"}, + } + _, _, err = credentials.NewTLS(&serverCfg).ServerHandshake(conn) + if gotErr := (err != nil); gotErr != tc.wantErr { + t.Errorf("ServerHandshake returned unexpected error: got=%v, want=%t", err, tc.wantErr) + } + close(errCh) + }() + + serverAddr := listener.Addr().String() + clientCfg := &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + NextProtos: []string{}, // Empty list indicates ALPN is disabled. + RootCAs: certPool, + ServerName: serverName, + } + conn, err := tls.Dial("tcp", serverAddr, clientCfg) + if err != nil { + t.Fatalf("tls.Dial(%s) failed: %v", serverAddr, err) + } + defer conn.Close() + + select { + case <-time.After(defaultTestTimeout): + t.Fatal("Timed out waiting for completion") + case err := <-errCh: + if err != nil { + t.Fatalf("Unexpected server error: %v", err) + } + } + }) + } +} diff --git a/credentials/xds/xds.go b/credentials/xds/xds.go new file mode 100644 index 0000000..553a023 --- /dev/null +++ b/credentials/xds/xds.go @@ -0,0 +1,236 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xds provides a transport credentials implementation where the +// security configuration is pushed by a management server using xDS APIs. +package xds + +import ( + "context" + "crypto/tls" + "errors" + "net" + "time" + + "google.golang.org/grpc/credentials" + credinternal "google.golang.org/grpc/internal/credentials" + xdsinternal "google.golang.org/grpc/internal/credentials/xds" + "google.golang.org/grpc/internal/grpcsync" +) + +// ClientOptions contains parameters to configure a new client-side xDS +// credentials implementation. +type ClientOptions struct { + // FallbackCreds specifies the fallback credentials to be used when either + // the `xds` scheme is not used in the user's dial target or when the + // management server does not return any security configuration. Attempts to + // create client credentials without fallback credentials will fail. + FallbackCreds credentials.TransportCredentials +} + +// NewClientCredentials returns a new client-side transport credentials +// implementation which uses xDS APIs to fetch its security configuration. +func NewClientCredentials(opts ClientOptions) (credentials.TransportCredentials, error) { + if opts.FallbackCreds == nil { + return nil, errors.New("missing fallback credentials") + } + return &credsImpl{ + isClient: true, + fallback: opts.FallbackCreds, + }, nil +} + +// ServerOptions contains parameters to configure a new server-side xDS +// credentials implementation. +type ServerOptions struct { + // FallbackCreds specifies the fallback credentials to be used when the + // management server does not return any security configuration. Attempts to + // create server credentials without fallback credentials will fail. + FallbackCreds credentials.TransportCredentials +} + +// NewServerCredentials returns a new server-side transport credentials +// implementation which uses xDS APIs to fetch its security configuration. +func NewServerCredentials(opts ServerOptions) (credentials.TransportCredentials, error) { + if opts.FallbackCreds == nil { + return nil, errors.New("missing fallback credentials") + } + return &credsImpl{ + isClient: false, + fallback: opts.FallbackCreds, + }, nil +} + +// credsImpl is an implementation of the credentials.TransportCredentials +// interface which uses xDS APIs to fetch its security configuration. +type credsImpl struct { + isClient bool + fallback credentials.TransportCredentials +} + +// ClientHandshake performs the TLS handshake on the client-side. +// +// It looks for the presence of a HandshakeInfo value in the passed in context +// (added using a call to NewContextWithHandshakeInfo()), and retrieves identity +// and root certificates from there. It also retrieves a list of acceptable SANs +// and uses a custom verification function to validate the certificate presented +// by the peer. It uses fallback credentials if no HandshakeInfo is present in +// the passed in context. +func (c *credsImpl) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + if !c.isClient { + return nil, nil, errors.New("ClientHandshake() is not supported for server credentials") + } + + // The clusterimpl balancer constructs a new HandshakeInfo using a call to + // NewHandshakeInfo(), and then adds it to the attributes field of the + // resolver.Address when handling calls to NewSubConn(). The transport layer + // takes care of shipping these attributes in the context to this handshake + // function. We first read the credentials.ClientHandshakeInfo type from the + // context, which contains the attributes added by the clusterimpl balancer. + // We then read the HandshakeInfo from the attributes to get to the actual + // data that we need here for the handshake. + chi := credentials.ClientHandshakeInfoFromContext(ctx) + // If there are no attributes in the received context or the attributes does + // not contain a HandshakeInfo, it could either mean that the user did not + // specify an `xds` scheme in their dial target or that the xDS server did + // not provide any security configuration. In both of these cases, we use + // the fallback credentials specified by the user. + if chi.Attributes == nil { + return c.fallback.ClientHandshake(ctx, authority, rawConn) + } + + hiPtr := xdsinternal.HandshakeInfoFromAttributes(chi.Attributes) + hostname := xdsinternal.Hostname(chi.Attributes) + cfg, useFallback, done, err := xdsinternal.ClientSideTLSConfig(ctx, hiPtr, hostname) + if err != nil { + return nil, nil, err + } + if useFallback { + return c.fallback.ClientHandshake(ctx, authority, rawConn) + } + defer done() + + // Perform the TLS handshake with the tls.Config that we have. We run the + // actual Handshake() function in a goroutine because we need to respect the + // deadline specified on the passed in context, and we need a way to cancel + // the handshake if the context is cancelled. + conn := tls.Client(rawConn, cfg) + errCh := make(chan error, 1) + go func() { + errCh <- conn.Handshake() + close(errCh) + }() + select { + case err := <-errCh: + if err != nil { + conn.Close() + return nil, nil, err + } + case <-ctx.Done(): + conn.Close() + return nil, nil, ctx.Err() + } + info := credentials.TLSInfo{ + State: conn.ConnectionState(), + CommonAuthInfo: credentials.CommonAuthInfo{ + SecurityLevel: credentials.PrivacyAndIntegrity, + }, + SPIFFEID: credinternal.SPIFFEIDFromState(conn.ConnectionState()), + } + return credinternal.WrapSyscallConn(rawConn, conn), info, nil +} + +// ServerHandshake performs the TLS handshake on the server-side. +func (c *credsImpl) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + if c.isClient { + return nil, nil, errors.New("ServerHandshake is not supported for client credentials") + } + + // An xds-enabled gRPC server wraps the underlying raw net.Conn in a type + // that provides a way to retrieve `HandshakeInfo`, which contains the + // certificate providers to be used during the handshake. If the net.Conn + // passed to this function does not implement this interface, or if the + // `HandshakeInfo` does not contain the information we are looking for, we + // delegate the handshake to the fallback credentials. + hiConn, ok := rawConn.(interface { + XDSHandshakeInfo() (*grpcsync.RefCounted[*xdsinternal.HandshakeInfo], error) + }) + if !ok { + return c.fallback.ServerHandshake(rawConn) + } + hi, err := hiConn.XDSHandshakeInfo() + if err != nil { + return nil, nil, err + } + + // An xds-enabled gRPC server is expected to wrap the underlying raw + // net.Conn in a type which provides a way to retrieve the deadline set on + // it. If we cannot retrieve the deadline here, we fail (by setting deadline + // to time.Now()), instead of using a default deadline and possibly taking + // longer to eventually fail. + deadline := time.Now() + if dConn, ok := rawConn.(interface{ GetDeadline() time.Time }); ok { + deadline = dConn.GetDeadline() + } + ctx, cancel := context.WithDeadline(context.Background(), deadline) + defer cancel() + + cfg, useFallback, done, err := xdsinternal.ServerSideTLSConfig(ctx, hi) + if err != nil { + return nil, nil, err + } + if useFallback { + return c.fallback.ServerHandshake(rawConn) + } + defer done() + + conn := tls.Server(rawConn, cfg) + if err := conn.Handshake(); err != nil { + conn.Close() + return nil, nil, err + } + info := credentials.TLSInfo{ + State: conn.ConnectionState(), + CommonAuthInfo: credentials.CommonAuthInfo{ + SecurityLevel: credentials.PrivacyAndIntegrity, + }, + } + info.SPIFFEID = credinternal.SPIFFEIDFromState(conn.ConnectionState()) + return credinternal.WrapSyscallConn(rawConn, conn), info, nil +} + +// Info provides the ProtocolInfo of this TransportCredentials. +func (c *credsImpl) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{SecurityProtocol: "tls"} +} + +// Clone makes a copy of this TransportCredentials. +func (c *credsImpl) Clone() credentials.TransportCredentials { + clone := *c + return &clone +} + +func (c *credsImpl) OverrideServerName(_ string) error { + return errors.New("serverName for peer validation must be configured as a list of acceptable SANs") +} + +// UsesXDS returns true if c uses xDS to fetch security configuration +// used at handshake time, and false otherwise. +func (c *credsImpl) UsesXDS() bool { + return true +} diff --git a/credentials/xds/xds_client_test.go b/credentials/xds/xds_client_test.go new file mode 100644 index 0000000..a56512d --- /dev/null +++ b/credentials/xds/xds_client_test.go @@ -0,0 +1,676 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + icredentials "google.golang.org/grpc/internal/credentials" + xdsinternal "google.golang.org/grpc/internal/credentials/xds" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/testdata" +) + +const ( + defaultTestTimeout = 1 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + defaultTestCertSAN = "abc.test.example.com" + authority = "x.test.example.com" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// Helper function to create a real TLS client credentials which is used as +// fallback credentials from multiple tests. +func makeFallbackClientCreds(t *testing.T) credentials.TransportCredentials { + creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "") + if err != nil { + t.Fatal(err) + } + return creds +} + +// testServer is a no-op server which listens on a local TCP port for incoming +// connections, and performs a manual TLS handshake on the received raw +// connection using a user specified handshake function. It then makes the +// result of the handshake operation available through a channel for tests to +// inspect. Tests should stop the testServer as part of their cleanup. +type testServer struct { + lis net.Listener + address string // Listening address of the test server. + handshakeFunc testHandshakeFunc // Test specified handshake function. + hsResult *testutils.Channel // Channel to deliver handshake results. +} + +// handshakeResult wraps the result of the handshake operation on the test +// server. It consists of TLS connection state and an error, if the handshake +// failed. This result is delivered on the `hsResult` channel on the testServer. +type handshakeResult struct { + connState tls.ConnectionState + err error +} + +// Configurable handshake function for the testServer. Tests can set this to +// simulate different conditions like handshake success, failure, timeout etc. +type testHandshakeFunc func(net.Conn) handshakeResult + +// newTestServerWithHandshakeFunc starts a new testServer which listens for +// connections on a local TCP port, and uses the provided custom handshake +// function to perform TLS handshake. +func newTestServerWithHandshakeFunc(ctx context.Context, f testHandshakeFunc) *testServer { + ts := &testServer{ + handshakeFunc: f, + hsResult: testutils.NewChannel(), + } + ts.start(ctx) + return ts +} + +// starts actually starts listening on a local TCP port, and spawns a goroutine +// to handle new connections. +func (ts *testServer) start(ctx context.Context) error { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + return err + } + ts.lis = lis + ts.address = lis.Addr().String() + go ts.handleConn(ctx) + return nil +} + +// handleConn accepts a new raw connection, and invokes the test provided +// handshake function to perform TLS handshake, and returns the result on the +// `hsResult` channel. +func (ts *testServer) handleConn(ctx context.Context) { + for { + rawConn, err := ts.lis.Accept() + if err != nil { + // Once the listeners closed, Accept() will return with an error. + return + } + hsr := ts.handshakeFunc(rawConn) + ts.hsResult.SendContext(ctx, hsr) + } +} + +// stop closes the associated listener which causes the connection handling +// goroutine to exit. +func (ts *testServer) stop() { + ts.lis.Close() +} + +// A handshake function which simulates a successful handshake without client +// authentication (server does not request for client certificate during the +// handshake here). +func testServerTLSHandshake(rawConn net.Conn) handshakeResult { + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + return handshakeResult{err: err} + } + cfg := &tls.Config{ + Certificates: []tls.Certificate{cert}, + NextProtos: []string{"h2"}, + } + conn := tls.Server(rawConn, cfg) + if err := conn.Handshake(); err != nil { + return handshakeResult{err: err} + } + return handshakeResult{connState: conn.ConnectionState()} +} + +// A handshake function which simulates a successful handshake with mutual +// authentication. +func testServerMutualTLSHandshake(rawConn net.Conn) handshakeResult { + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + return handshakeResult{err: err} + } + pemData, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem")) + if err != nil { + return handshakeResult{err: err} + } + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(pemData) + cfg := &tls.Config{ + Certificates: []tls.Certificate{cert}, + ClientCAs: roots, + } + conn := tls.Server(rawConn, cfg) + if err := conn.Handshake(); err != nil { + return handshakeResult{err: err} + } + return handshakeResult{connState: conn.ConnectionState()} +} + +// fakeProvider is an implementation of the certprovider.Provider interface +// which returns the configured key material and error in calls to +// KeyMaterial(). +type fakeProvider struct { + km *certprovider.KeyMaterial + err error +} + +func (f *fakeProvider) KeyMaterial(context.Context) (*certprovider.KeyMaterial, error) { + return f.km, f.err +} + +func (f *fakeProvider) Close() {} + +// makeIdentityProvider creates a new instance of the fakeProvider returning the +// identity key material specified in the provider file paths. +func makeIdentityProvider(t *testing.T, certPath, keyPath string) certprovider.Provider { + t.Helper() + cert, err := tls.LoadX509KeyPair(testdata.Path(certPath), testdata.Path(keyPath)) + if err != nil { + t.Fatal(err) + } + return &fakeProvider{km: &certprovider.KeyMaterial{Certs: []tls.Certificate{cert}}} +} + +// makeRootProvider creates a new instance of the fakeProvider returning the +// root key material specified in the provider file paths. +func makeRootProvider(t *testing.T, caPath string) *fakeProvider { + pemData, err := os.ReadFile(testdata.Path(caPath)) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(pemData) + return &fakeProvider{km: &certprovider.KeyMaterial{Roots: roots}} +} + +// newTestContextWithHandshakeInfo returns a copy of parent with HandshakeInfo +// context value added to it. +func newTestContextWithHandshakeInfo(parent context.Context, root, identity certprovider.Provider, sanExactMatch, sni string, validateSANUsingSNI bool) context.Context { + // Creating the HandshakeInfo and adding it to the attributes is very + // similar to what the CDS balancer would do when it intercepts calls to + // NewSubConn(). + var sms []matcher.StringMatcher + if sanExactMatch != "" { + sms = []matcher.StringMatcher{matcher.NewExactStringMatcher(sanExactMatch, false)} + } + var hiPtr atomic.Pointer[grpcsync.RefCounted[*xdsinternal.HandshakeInfo]] + info := xdsinternal.NewHandshakeInfo(root, identity, sms, sni, false, validateSANUsingSNI, false) + hiPtr.Store(info) + addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, &hiPtr) + + // Moving the attributes from the resolver.Address to the context passed to + // the handshaker is done in the transport layer. Since we directly call the + // handshaker in these tests, we need to do the same here. + return icredentials.NewClientHandshakeInfoContext(parent, credentials.ClientHandshakeInfo{Attributes: addr.Attributes}) +} + +// compareAuthInfo compares the AuthInfo received on the client side after a +// successful handshake with the authInfo available on the testServer. +func compareAuthInfo(ctx context.Context, ts *testServer, ai credentials.AuthInfo) error { + if ai.AuthType() != "tls" { + return fmt.Errorf("ClientHandshake returned authType %q, want %q", ai.AuthType(), "tls") + } + info, ok := ai.(credentials.TLSInfo) + if !ok { + return fmt.Errorf("ClientHandshake returned authInfo of type %T, want %T", ai, credentials.TLSInfo{}) + } + gotState := info.State + + // Read the handshake result from the testServer which contains the TLS + // connection state and compare it with the one received on the client-side. + val, err := ts.hsResult.Receive(ctx) + if err != nil { + return fmt.Errorf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != nil { + return fmt.Errorf("testServer handshake failure: %v", hsr.err) + } + // AuthInfo contains a variety of information. We only verify a subset here. + // This is the same subset which is verified in TLS credentials tests. + if err := compareConnState(gotState, hsr.connState); err != nil { + return err + } + return nil +} + +func compareConnState(got, want tls.ConnectionState) error { + switch { + case got.Version != want.Version: + return fmt.Errorf("TLS.ConnectionState got Version: %v, want: %v", got.Version, want.Version) + case got.HandshakeComplete != want.HandshakeComplete: + return fmt.Errorf("TLS.ConnectionState got HandshakeComplete: %v, want: %v", got.HandshakeComplete, want.HandshakeComplete) + case got.CipherSuite != want.CipherSuite: + return fmt.Errorf("TLS.ConnectionState got CipherSuite: %v, want: %v", got.CipherSuite, want.CipherSuite) + case got.NegotiatedProtocol != want.NegotiatedProtocol: + return fmt.Errorf("TLS.ConnectionState got NegotiatedProtocol: %v, want: %v", got.NegotiatedProtocol, want.NegotiatedProtocol) + } + return nil +} + +// TestClientCredsWithoutFallback verifies that the call to +// NewClientCredentials() fails when no fallback is specified. +func (s) TestClientCredsWithoutFallback(t *testing.T) { + if _, err := NewClientCredentials(ClientOptions{}); err == nil { + t.Fatal("NewClientCredentials() succeeded without specifying fallback") + } +} + +// TestClientCredsInvalidHandshakeInfo verifies scenarios where the passed in +// HandshakeInfo is invalid because it does not contain the expected certificate +// providers. +func (s) TestClientCredsInvalidHandshakeInfo(t *testing.T) { + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + creds, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + pCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx := newTestContextWithHandshakeInfo(pCtx, nil, &fakeProvider{}, "", "", false) + if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil { + t.Fatal("ClientHandshake succeeded without root certificate provider in HandshakeInfo") + } +} + +// TestClientCredsProviderFailure verifies the cases where an expected +// certificate provider is missing in the HandshakeInfo value in the context. +func (s) TestClientCredsProviderFailure(t *testing.T) { + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + creds, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + tests := []struct { + desc string + rootProvider certprovider.Provider + identityProvider certprovider.Provider + wantErr string + }{ + { + desc: "erroring root provider", + rootProvider: &fakeProvider{err: errors.New("root provider error")}, + wantErr: "root provider error", + }, + { + desc: "erroring identity provider", + rootProvider: &fakeProvider{km: &certprovider.KeyMaterial{}}, + identityProvider: &fakeProvider{err: errors.New("identity provider error")}, + wantErr: "identity provider error", + }, + } + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, test.identityProvider, "", "", false) + if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("ClientHandshake() returned error: %q, wantErr: %q", err, test.wantErr) + } + }) + } +} + +// TestClientCredsSuccess verifies successful client handshake cases. +func (s) TestClientCredsSuccess(t *testing.T) { + tests := []struct { + desc string + handshakeFunc testHandshakeFunc + handshakeInfoCtx func(ctx context.Context) context.Context + enableSNIFlag bool + }{ + { + desc: "fallback", + handshakeFunc: testServerTLSHandshake, + handshakeInfoCtx: func(ctx context.Context) context.Context { + // Since we don't add a HandshakeInfo to the context, the + // ClientHandshake() method will delegate to the fallback. + return ctx + }, + }, + { + desc: "TLS", + handshakeFunc: testServerTLSHandshake, + handshakeInfoCtx: func(ctx context.Context) context.Context { + return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN, "", false) + }, + }, + { + desc: "mTLS", + handshakeFunc: testServerMutualTLSHandshake, + handshakeInfoCtx: func(ctx context.Context) context.Context { + return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"), defaultTestCertSAN, "", false) + }, + }, + { + desc: "mTLS with no acceptedSANs specified", + handshakeFunc: testServerMutualTLSHandshake, + handshakeInfoCtx: func(ctx context.Context) context.Context { + return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"), "", "", false) + }, + }, + { + desc: "TLS with SNI", + handshakeFunc: testServerTLSHandshake, + handshakeInfoCtx: func(ctx context.Context) context.Context { + return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, "bad-match", defaultTestCertSAN, true) + }, + enableSNIFlag: true, + }, + { + desc: "TLS with SNI, env variable disabled, AutoSniSanValidation enabled", + handshakeFunc: testServerTLSHandshake, + handshakeInfoCtx: func(ctx context.Context) context.Context { + return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN, "bad-sni", true) + }, + }, + { + desc: "TLS with SNI, env variable enabled but AutoSniSanValidation disabled", + handshakeFunc: testServerTLSHandshake, + handshakeInfoCtx: func(ctx context.Context) context.Context { + return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN, "bad-sni", false) + }, + enableSNIFlag: true, + }, + { + desc: "TLS with empty SNI, env variable enabled, AutoSniSanValidation enabled", + handshakeFunc: testServerTLSHandshake, + handshakeInfoCtx: func(ctx context.Context) context.Context { + return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN, "", true) + }, + enableSNIFlag: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, test.enableSNIFlag) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ts := newTestServerWithHandshakeFunc(ctx, test.handshakeFunc) + defer ts.stop() + + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + creds, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + conn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer conn.Close() + + _, ai, err := creds.ClientHandshake(test.handshakeInfoCtx(ctx), authority, conn) + if err != nil { + t.Fatalf("ClientHandshake() returned failed: %q", err) + } + if err := compareAuthInfo(ctx, ts, ai); err != nil { + t.Fatal(err) + } + }) + } +} + +func (s) TestClientCredsHandshakeTimeout(t *testing.T) { + clientDone := make(chan struct{}) + ctx, sCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer sCancel() + // A handshake function which simulates a handshake timeout from the + // server-side by simply blocking on the client-side handshake to timeout + // and not writing any handshake data. + hErr := errors.New("server handshake error") + ts := newTestServerWithHandshakeFunc(ctx, func(net.Conn) handshakeResult { + <-clientDone + return handshakeResult{err: hErr} + }) + defer ts.stop() + + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + creds, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + conn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer conn.Close() + + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + ctx = newTestContextWithHandshakeInfo(sCtx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN, "", false) + if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil { + t.Fatal("ClientHandshake() succeeded when expected to timeout") + } + close(clientDone) + + // Read the handshake result from the testServer and make sure the expected + // error is returned. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := ts.hsResult.Receive(ctx) + if err != nil { + t.Fatalf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != hErr { + t.Fatalf("testServer handshake returned error: %v, want: %v", hsr.err, hErr) + } +} + +// TestClientCredsHandshakeFailure verifies different handshake failure cases. +func (s) TestClientCredsHandshakeFailure(t *testing.T) { + tests := []struct { + desc string + handshakeFunc testHandshakeFunc + rootProvider certprovider.Provider + san string + sni string + validateSANUsingSNI bool + enableSNIFlag bool + wantErr string + }{ + { + desc: "cert validation failure", + handshakeFunc: testServerTLSHandshake, + rootProvider: makeRootProvider(t, "x509/client_ca_cert.pem"), + san: defaultTestCertSAN, + wantErr: "x509: certificate signed by unknown authority", + }, + { + desc: "SAN mismatch", + handshakeFunc: testServerTLSHandshake, + rootProvider: makeRootProvider(t, "x509/server_ca_cert.pem"), + san: "bad-san", + wantErr: "do not match any of the accepted SANs", + }, + { + desc: "SNI SAN mismatch", + handshakeFunc: testServerTLSHandshake, + rootProvider: makeRootProvider(t, "x509/server_ca_cert.pem"), + sni: "bad-sni", + validateSANUsingSNI: true, + wantErr: "do not match the SNI", + enableSNIFlag: true, + }, + { + desc: "SNI set, AutoSniSanValidation disabled with SAN mismatch", + handshakeFunc: testServerTLSHandshake, + rootProvider: makeRootProvider(t, "x509/server_ca_cert.pem"), + sni: defaultTestCertSAN, + san: "bad-san", + validateSANUsingSNI: false, + wantErr: "do not match any of the accepted SANs", + enableSNIFlag: true, + }, + { + desc: "SNI set with SAN mismatch and AutoSniSanValidation enabled, environment variable disabled", + handshakeFunc: testServerTLSHandshake, + rootProvider: makeRootProvider(t, "x509/server_ca_cert.pem"), + sni: defaultTestCertSAN, + san: "bad-san", + validateSANUsingSNI: true, + wantErr: "do not match any of the accepted SANs", + }, + { + desc: "SNI empty, AutoSniSanValidation enabled with SAN mismatch", + handshakeFunc: testServerTLSHandshake, + rootProvider: makeRootProvider(t, "x509/server_ca_cert.pem"), + sni: "", + san: "bad-san", + validateSANUsingSNI: true, + wantErr: "do not match any of the accepted SANs", + enableSNIFlag: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, test.enableSNIFlag) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ts := newTestServerWithHandshakeFunc(ctx, test.handshakeFunc) + defer ts.stop() + + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + creds, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + conn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer conn.Close() + + ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, nil, test.san, test.sni, test.validateSANUsingSNI) + if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("ClientHandshake() returned %q, wantErr %q", err, test.wantErr) + } + }) + } +} + +// TestClientCredsProviderSwitch verifies the case where the first attempt of +// ClientHandshake fails because of a handshake failure. Then we update the +// certificate provider and the second attempt succeeds. This is an +// approximation of the flow of events when the control plane specifies new +// security config which results in new certificate providers being used. +func (s) TestClientCredsProviderSwitch(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ts := newTestServerWithHandshakeFunc(ctx, testServerTLSHandshake) + defer ts.stop() + + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + creds, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + conn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer conn.Close() + + // Create a root provider which will fail the handshake because it does not + // use the correct trust roots. + root1 := makeRootProvider(t, "x509/client_ca_cert.pem") + handshakeInfo := xdsinternal.NewHandshakeInfo(root1, nil, []matcher.StringMatcher{matcher.NewExactStringMatcher(defaultTestCertSAN, false)}, "", false, false, false) + // We need to repeat most of what newTestContextWithHandshakeInfo() does + // here because we need access to the underlying HandshakeInfo so that we + // can update it before the next call to ClientHandshake(). + var hiPtr atomic.Pointer[grpcsync.RefCounted[*xdsinternal.HandshakeInfo]] + hiPtr.Store(handshakeInfo) + addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, &hiPtr) + ctx = icredentials.NewClientHandshakeInfoContext(ctx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes}) + if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil { + t.Fatal("ClientHandshake() succeeded when expected to fail") + } + // Drain the result channel on the test server so that we can inspect the + // result for the next handshake. + _, err = ts.hsResult.Receive(ctx) + if err != nil { + t.Errorf("testServer failed to return handshake result: %v", err) + } + + conn, err = net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer conn.Close() + + // Create a new root provider which uses the correct trust roots. And update + // the HandshakeInfo with the new provider. + root2 := makeRootProvider(t, "x509/server_ca_cert.pem") + handshakeInfo = xdsinternal.NewHandshakeInfo(root2, nil, []matcher.StringMatcher{matcher.NewExactStringMatcher(defaultTestCertSAN, false)}, "", false, false, false) + // Update the existing pointer, which address attribute will continue to + // point to. + hiPtr.Store(handshakeInfo) + _, ai, err := creds.ClientHandshake(ctx, authority, conn) + if err != nil { + t.Fatalf("ClientHandshake() returned failed: %q", err) + } + if err := compareAuthInfo(ctx, ts, ai); err != nil { + t.Fatal(err) + } +} + +// TestClientClone verifies the Clone() method on client credentials. +func (s) TestClientClone(t *testing.T) { + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + orig, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + // The credsImpl does not have any exported fields, and it does not make + // sense to use any cmp options to look deep into. So, all we make sure here + // is that the cloned object points to a different location in memory. + if clone := orig.Clone(); clone == orig { + t.Fatal("return value from Clone() doesn't point to new credentials instance") + } +} diff --git a/credentials/xds/xds_server_test.go b/credentials/xds/xds_server_test.go new file mode 100644 index 0000000..3855d63 --- /dev/null +++ b/credentials/xds/xds_server_test.go @@ -0,0 +1,543 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net" + "os" + "strings" + "testing" + "time" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + xdsinternal "google.golang.org/grpc/internal/credentials/xds" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/testdata" +) + +func makeClientTLSConfig(t *testing.T, mTLS bool) *tls.Config { + t.Helper() + + pemData, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem")) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(pemData) + + var certs []tls.Certificate + if mTLS { + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/client1_cert.pem"), testdata.Path("x509/client1_key.pem")) + if err != nil { + t.Fatal(err) + } + certs = append(certs, cert) + } + + return &tls.Config{ + Certificates: certs, + RootCAs: roots, + ServerName: "*.test.example.com", + // Setting this to true completely turns off the certificate validation + // on the client side. So, the client side handshake always seems to + // succeed. But if we want to turn this ON, we will need to generate + // certificates which work with localhost, or supply a custom + // verification function. So, the server credentials tests will rely + // solely on the success/failure of the server-side handshake. + InsecureSkipVerify: true, + NextProtos: []string{"h2"}, + } +} + +// Helper function to create a real TLS server credentials which is used as +// fallback credentials from multiple tests. +func makeFallbackServerCreds(t *testing.T) credentials.TransportCredentials { + t.Helper() + + creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatal(err) + } + return creds +} + +type errorCreds struct { + credentials.TransportCredentials +} + +// TestServerCredsWithoutFallback verifies that the call to +// NewServerCredentials() fails when no fallback is specified. +func (s) TestServerCredsWithoutFallback(t *testing.T) { + if _, err := NewServerCredentials(ServerOptions{}); err == nil { + t.Fatal("NewServerCredentials() succeeded without specifying fallback") + } +} + +type wrapperConn struct { + net.Conn + xdsHI *grpcsync.RefCounted[*xdsinternal.HandshakeInfo] + deadline time.Time + handshakeInfoErr error +} + +func (wc *wrapperConn) XDSHandshakeInfo() (*grpcsync.RefCounted[*xdsinternal.HandshakeInfo], error) { + return wc.xdsHI, wc.handshakeInfoErr +} + +func (wc *wrapperConn) GetDeadline() time.Time { + return wc.deadline +} + +func newWrappedConn(conn net.Conn, xdsHI *grpcsync.RefCounted[*xdsinternal.HandshakeInfo], deadline time.Time) *wrapperConn { + return &wrapperConn{Conn: conn, xdsHI: xdsHI, deadline: deadline} +} + +// TestServerCredsInvalidHandshakeInfo verifies scenarios where the passed in +// HandshakeInfo is invalid because it does not contain the expected certificate +// providers. +func (s) TestServerCredsInvalidHandshakeInfo(t *testing.T) { + opts := ServerOptions{FallbackCreds: &errorCreds{}} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + + info := xdsinternal.NewHandshakeInfo(&fakeProvider{}, nil, nil, "", false, false, false) + conn := newWrappedConn(nil, info, time.Time{}) + if _, _, err := creds.ServerHandshake(conn); err == nil { + t.Fatal("ServerHandshake succeeded without identity certificate provider in HandshakeInfo") + } +} + +// TestServerCredsProviderFailure verifies the cases where an expected +// certificate provider is missing in the HandshakeInfo value in the context. +func (s) TestServerCredsProviderFailure(t *testing.T) { + opts := ServerOptions{FallbackCreds: &errorCreds{}} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + + tests := []struct { + desc string + rootProvider certprovider.Provider + identityProvider certprovider.Provider + wantErr string + }{ + { + desc: "erroring identity provider", + identityProvider: &fakeProvider{err: errors.New("identity provider error")}, + wantErr: "identity provider error", + }, + { + desc: "erroring root provider", + identityProvider: &fakeProvider{km: &certprovider.KeyMaterial{}}, + rootProvider: &fakeProvider{err: errors.New("root provider error")}, + wantErr: "root provider error", + }, + } + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + info := xdsinternal.NewHandshakeInfo(test.rootProvider, test.identityProvider, nil, "", false, false, false) + conn := newWrappedConn(nil, info, time.Time{}) + if _, _, err := creds.ServerHandshake(conn); err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("ServerHandshake() returned error: %q, wantErr: %q", err, test.wantErr) + } + }) + } +} + +// TestServerCredsHandshake_XDSHandshakeInfoError verifies the case where the +// call to XDSHandshakeInfo() from the ServerHandshake() method returns an +// error, and the test verifies that the ServerHandshake() fails with the +// expected error. +func (s) TestServerCredsHandshake_XDSHandshakeInfoError(t *testing.T) { + opts := ServerOptions{FallbackCreds: &errorCreds{}} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create a test server which uses the xDS server credentials created above + // to perform TLS handshake on incoming connections. + ts := newTestServerWithHandshakeFunc(ctx, func(rawConn net.Conn) handshakeResult { + // Create a wrapped conn which returns a nil HandshakeInfo and a non-nil error. + conn := newWrappedConn(rawConn, nil, time.Now().Add(defaultTestTimeout)) + hiErr := errors.New("xdsHandshakeInfo error") + conn.handshakeInfoErr = hiErr + + // Invoke the ServerHandshake() method on the xDS credentials and verify + // that the error returned by the XDSHandshakeInfo() method on the + // wrapped conn is returned here. + _, _, err := creds.ServerHandshake(conn) + if !errors.Is(err, hiErr) { + return handshakeResult{err: fmt.Errorf("ServerHandshake() returned err: %v, wantErr: %v", err, hiErr)} + } + return handshakeResult{} + }) + defer ts.stop() + + // Dial the test server, but don't trigger the TLS handshake. This will + // cause ServerHandshake() to fail. + rawConn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer rawConn.Close() + + // Read handshake result from the testServer which will return an error if + // the handshake succeeded. + val, err := ts.hsResult.Receive(ctx) + if err != nil { + t.Fatalf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != nil { + t.Fatalf("testServer handshake failure: %v", hsr.err) + } +} + +// TestServerCredsHandshakeTimeout verifies the case where the client does not +// send required handshake data before the deadline set on the net.Conn passed +// to ServerHandshake(). +func (s) TestServerCredsHandshakeTimeout(t *testing.T) { + opts := ServerOptions{FallbackCreds: &errorCreds{}} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create a test server which uses the xDS server credentials created above + // to perform TLS handshake on incoming connections. + ts := newTestServerWithHandshakeFunc(ctx, func(rawConn net.Conn) handshakeResult { + hi := xdsinternal.NewHandshakeInfo(makeRootProvider(t, "x509/client_ca_cert.pem"), makeIdentityProvider(t, "x509/server2_cert.pem", "x509/server2_key.pem"), nil, "", true, false, false) + + // Create a wrapped conn which can return the HandshakeInfo created + // above with a very small deadline. + d := time.Now().Add(defaultTestShortTimeout) + rawConn.SetDeadline(d) + conn := newWrappedConn(rawConn, hi, d) + + // ServerHandshake() on the xDS credentials is expected to fail. + if _, _, err := creds.ServerHandshake(conn); err == nil { + return handshakeResult{err: errors.New("ServerHandshake() succeeded when expected to timeout")} + } + return handshakeResult{} + }) + defer ts.stop() + + // Dial the test server, but don't trigger the TLS handshake. This will + // cause ServerHandshake() to fail. + rawConn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer rawConn.Close() + + // Read handshake result from the testServer and expect a failure result. + val, err := ts.hsResult.Receive(ctx) + if err != nil { + t.Fatalf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != nil { + t.Fatalf("testServer handshake failure: %v", hsr.err) + } +} + +// TestServerCredsHandshakeFailure verifies the case where the server-side +// credentials uses a root certificate which does not match the certificate +// presented by the client, and hence the handshake must fail. +func (s) TestServerCredsHandshakeFailure(t *testing.T) { + opts := ServerOptions{FallbackCreds: &errorCreds{}} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create a test server which uses the xDS server credentials created above + // to perform TLS handshake on incoming connections. + ts := newTestServerWithHandshakeFunc(ctx, func(rawConn net.Conn) handshakeResult { + // Create a HandshakeInfo which has a root provider which does not match + // the certificate sent by the client. + hi := xdsinternal.NewHandshakeInfo(makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/client2_cert.pem", "x509/client2_key.pem"), nil, "", true, false, false) + + // Create a wrapped conn which can return the HandshakeInfo and + // configured deadline to the xDS credentials' ServerHandshake() + // method. + conn := newWrappedConn(rawConn, hi, time.Now().Add(defaultTestTimeout)) + + // ServerHandshake() on the xDS credentials is expected to fail. + if _, _, err := creds.ServerHandshake(conn); err == nil { + return handshakeResult{err: errors.New("ServerHandshake() succeeded when expected to fail")} + } + return handshakeResult{} + }) + defer ts.stop() + + // Dial the test server, and trigger the TLS handshake. + rawConn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer rawConn.Close() + tlsConn := tls.Client(rawConn, makeClientTLSConfig(t, true)) + tlsConn.SetDeadline(time.Now().Add(defaultTestTimeout)) + if err := tlsConn.Handshake(); err != nil { + t.Fatal(err) + } + + // Read handshake result from the testServer which will return an error if + // the handshake succeeded. + val, err := ts.hsResult.Receive(ctx) + if err != nil { + t.Fatalf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != nil { + t.Fatalf("testServer handshake failure: %v", hsr.err) + } +} + +// TestServerCredsHandshakeSuccess verifies success handshake cases. +func (s) TestServerCredsHandshakeSuccess(t *testing.T) { + tests := []struct { + desc string + fallbackCreds credentials.TransportCredentials + rootProvider certprovider.Provider + identityProvider certprovider.Provider + requireClientCert bool + }{ + { + desc: "fallback", + fallbackCreds: makeFallbackServerCreds(t), + }, + { + desc: "TLS", + fallbackCreds: &errorCreds{}, + identityProvider: makeIdentityProvider(t, "x509/server2_cert.pem", "x509/server2_key.pem"), + }, + { + desc: "mTLS", + fallbackCreds: &errorCreds{}, + identityProvider: makeIdentityProvider(t, "x509/server2_cert.pem", "x509/server2_key.pem"), + rootProvider: makeRootProvider(t, "x509/client_ca_cert.pem"), + requireClientCert: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // Create an xDS server credentials. + opts := ServerOptions{FallbackCreds: test.fallbackCreds} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create a test server which uses the xDS server credentials + // created above to perform TLS handshake on incoming connections. + ts := newTestServerWithHandshakeFunc(ctx, func(rawConn net.Conn) handshakeResult { + // Create a HandshakeInfo with information from the test table. + hi := xdsinternal.NewHandshakeInfo(test.rootProvider, test.identityProvider, nil, "", test.requireClientCert, false, false) + + // Create a wrapped conn which can return the HandshakeInfo and + // configured deadline to the xDS credentials' ServerHandshake() + // method. + conn := newWrappedConn(rawConn, hi, time.Now().Add(defaultTestTimeout)) + + // Invoke the ServerHandshake() method on the xDS credentials + // and make some sanity checks before pushing the result for + // inspection by the main test body. + _, ai, err := creds.ServerHandshake(conn) + if err != nil { + return handshakeResult{err: fmt.Errorf("ServerHandshake() failed: %v", err)} + } + if ai.AuthType() != "tls" { + return handshakeResult{err: fmt.Errorf("ServerHandshake returned authType %q, want %q", ai.AuthType(), "tls")} + } + info, ok := ai.(credentials.TLSInfo) + if !ok { + return handshakeResult{err: fmt.Errorf("ServerHandshake returned authInfo of type %T, want %T", ai, credentials.TLSInfo{})} + } + return handshakeResult{connState: info.State} + }) + defer ts.stop() + + // Dial the test server, and trigger the TLS handshake. + rawConn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer rawConn.Close() + tlsConn := tls.Client(rawConn, makeClientTLSConfig(t, test.requireClientCert)) + tlsConn.SetDeadline(time.Now().Add(defaultTestTimeout)) + if err := tlsConn.Handshake(); err != nil { + t.Fatal(err) + } + + // Read the handshake result from the testServer which contains the + // TLS connection state on the server-side and compare it with the + // one received on the client-side. + val, err := ts.hsResult.Receive(ctx) + if err != nil { + t.Fatalf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != nil { + t.Fatalf("testServer handshake failure: %v", hsr.err) + } + + // AuthInfo contains a variety of information. We only verify a + // subset here. This is the same subset which is verified in TLS + // credentials tests. + if err := compareConnState(tlsConn.ConnectionState(), hsr.connState); err != nil { + t.Fatal(err) + } + }) + } +} + +func (s) TestServerCredsProviderSwitch(t *testing.T) { + opts := ServerOptions{FallbackCreds: &errorCreds{}} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // The first time the handshake function is invoked, it returns a + // HandshakeInfo which is expected to fail. Further invocations return a + // HandshakeInfo which is expected to succeed. + cnt := 0 + // Create a test server which uses the xDS server credentials created above + // to perform TLS handshake on incoming connections. + ts := newTestServerWithHandshakeFunc(ctx, func(rawConn net.Conn) handshakeResult { + cnt++ + var hi *grpcsync.RefCounted[*xdsinternal.HandshakeInfo] + if cnt == 1 { + // Create a HandshakeInfo which has a root provider which does not match + // the certificate sent by the client. + hi = xdsinternal.NewHandshakeInfo(makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/client2_cert.pem", "x509/client2_key.pem"), nil, "", true, false, false) + + // Create a wrapped conn which can return the HandshakeInfo and + // configured deadline to the xDS credentials' ServerHandshake() + // method. + conn := newWrappedConn(rawConn, hi, time.Now().Add(defaultTestTimeout)) + + // ServerHandshake() on the xDS credentials is expected to fail. + if _, _, err := creds.ServerHandshake(conn); err == nil { + return handshakeResult{err: errors.New("ServerHandshake() succeeded when expected to fail")} + } + return handshakeResult{} + } + + hi = xdsinternal.NewHandshakeInfo(makeRootProvider(t, "x509/client_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"), nil, "", true, false, false) + + // Create a wrapped conn which can return the HandshakeInfo and + // configured deadline to the xDS credentials' ServerHandshake() + // method. + conn := newWrappedConn(rawConn, hi, time.Now().Add(defaultTestTimeout)) + + // Invoke the ServerHandshake() method on the xDS credentials + // and make some sanity checks before pushing the result for + // inspection by the main test body. + _, ai, err := creds.ServerHandshake(conn) + if err != nil { + return handshakeResult{err: fmt.Errorf("ServerHandshake() failed: %v", err)} + } + if ai.AuthType() != "tls" { + return handshakeResult{err: fmt.Errorf("ServerHandshake returned authType %q, want %q", ai.AuthType(), "tls")} + } + info, ok := ai.(credentials.TLSInfo) + if !ok { + return handshakeResult{err: fmt.Errorf("ServerHandshake returned authInfo of type %T, want %T", ai, credentials.TLSInfo{})} + } + return handshakeResult{connState: info.State} + }) + defer ts.stop() + + for i := 0; i < 5; i++ { + // Dial the test server, and trigger the TLS handshake. + rawConn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer rawConn.Close() + tlsConn := tls.Client(rawConn, makeClientTLSConfig(t, true)) + tlsConn.SetDeadline(time.Now().Add(defaultTestTimeout)) + if err := tlsConn.Handshake(); err != nil { + t.Fatal(err) + } + + // Read the handshake result from the testServer which contains the + // TLS connection state on the server-side and compare it with the + // one received on the client-side. + val, err := ts.hsResult.Receive(ctx) + if err != nil { + t.Fatalf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != nil { + t.Fatalf("testServer handshake failure: %v", hsr.err) + } + if i == 0 { + // We expect the first handshake to fail. So, we skip checks which + // compare connection state. + continue + } + // AuthInfo contains a variety of information. We only verify a + // subset here. This is the same subset which is verified in TLS + // credentials tests. + if err := compareConnState(tlsConn.ConnectionState(), hsr.connState); err != nil { + t.Fatal(err) + } + } +} + +// TestServerClone verifies the Clone() method on client credentials. +func (s) TestServerClone(t *testing.T) { + opts := ServerOptions{FallbackCreds: makeFallbackServerCreds(t)} + orig, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + + // The credsImpl does not have any exported fields, and it does not make + // sense to use any cmp options to look deep into. So, all we make sure here + // is that the cloned object points to a different location in memory. + if clone := orig.Clone(); clone == orig { + t.Fatal("return value from Clone() doesn't point to new credentials instance") + } +} diff --git a/default_dial_option_server_option_test.go b/default_dial_option_server_option_test.go new file mode 100644 index 0000000..67ea9e8 --- /dev/null +++ b/default_dial_option_server_option_test.go @@ -0,0 +1,178 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "fmt" + "net/url" + "strings" + "testing" + + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" +) + +func (s) TestAddGlobalDialOptions(t *testing.T) { + // Ensure the NewClient fails without credentials + if _, err := NewClient("fake"); err == nil { + t.Fatalf("NewClient without a credential did not fail") + } else { + if !strings.Contains(err.Error(), "no transport security set") { + t.Fatalf("NewClient failed with unexpected error: %v", err) + } + } + + // Set and check the DialOptions + opts := []DialOption{WithTransportCredentials(insecure.NewCredentials()), WithTransportCredentials(insecure.NewCredentials()), WithTransportCredentials(insecure.NewCredentials())} + internal.AddGlobalDialOptions.(func(opt ...DialOption))(opts...) + defer internal.ClearGlobalDialOptions() + for i, opt := range opts { + if globalDialOptions[i] != opt { + t.Fatalf("Unexpected global dial option at index %d: %v != %v", i, globalDialOptions[i], opt) + } + } + + // Ensure the NewClient passes with the extra dial options + if cc, err := NewClient("fake"); err != nil { + t.Fatalf("NewClient with insecure credential failed: %v", err) + } else { + cc.Close() + } + + internal.ClearGlobalDialOptions() + if len(globalDialOptions) != 0 { + t.Fatalf("Unexpected len of globalDialOptions: %d != 0", len(globalDialOptions)) + } +} + +// TestDisableGlobalOptions tests dialing with the disableGlobalDialOptions dial +// option. Dialing with this set should not pick up global options. +func (s) TestDisableGlobalOptions(t *testing.T) { + // Set transport credentials as a global option. + internal.AddGlobalDialOptions.(func(opt ...DialOption))(WithTransportCredentials(insecure.NewCredentials())) + defer internal.ClearGlobalDialOptions() + // Dial with the disable global options dial option. This dial should fail + // due to the global dial options with credentials not being picked up due + // to global options being disabled. + noTSecStr := "no transport security set" + if _, err := NewClient("fake", internal.DisableGlobalDialOptions.(func() DialOption)()); !strings.Contains(fmt.Sprint(err), noTSecStr) { + t.Fatalf("NewClient received unexpected error: %v, want error containing %q", err, noTSecStr) + } +} + +type testPerTargetDialOption struct{} + +func (do *testPerTargetDialOption) DialOptionForTarget(parsedTarget url.URL) DialOption { + if parsedTarget.Scheme == "passthrough" { + return WithTransportCredentials(insecure.NewCredentials()) // credentials provided, should pass NewClient. + } + return EmptyDialOption{} // no credentials, should fail NewClient +} + +// TestGlobalPerTargetDialOption configures a global per target dial option that +// produces transport credentials for channels using "passthrough" scheme. +// Channels that use the passthrough scheme should be successfully created due +// to picking up transport credentials, whereas other channels should fail at +// creation due to not having transport credentials. +func (s) TestGlobalPerTargetDialOption(t *testing.T) { + internal.AddGlobalPerTargetDialOptions.(func(opt any))(&testPerTargetDialOption{}) + defer internal.ClearGlobalPerTargetDialOptions() + noTSecStr := "no transport security set" + if _, err := NewClient("dns:///fake"); !strings.Contains(fmt.Sprint(err), noTSecStr) { + t.Fatalf("NewClient received unexpected error: %v, want error containing %q", err, noTSecStr) + } + cc, err := NewClient("passthrough:///nice") + if err != nil { + t.Fatalf("NewClient with insecure credentials failed: %v", err) + } + cc.Close() +} + +func (s) TestAddGlobalServerOptions(t *testing.T) { + const maxRecvSize = 998765 + // Set and check the ServerOptions + opts := []ServerOption{Creds(insecure.NewCredentials()), MaxRecvMsgSize(maxRecvSize)} + internal.AddGlobalServerOptions.(func(opt ...ServerOption))(opts...) + defer internal.ClearGlobalServerOptions() + for i, opt := range opts { + if globalServerOptions[i] != opt { + t.Fatalf("Unexpected global server option at index %d: %v != %v", i, globalServerOptions[i], opt) + } + } + + // Ensure the extra server options applies to new servers + s := NewServer() + if s.opts.maxReceiveMessageSize != maxRecvSize { + t.Fatalf("Unexpected s.opts.maxReceiveMessageSize: %d != %d", s.opts.maxReceiveMessageSize, maxRecvSize) + } + + internal.ClearGlobalServerOptions() + if len(globalServerOptions) != 0 { + t.Fatalf("Unexpected len of globalServerOptions: %d != 0", len(globalServerOptions)) + } +} + +// TestJoinDialOption tests the join dial option. It configures a joined dial +// option with three individual dial options, and verifies that all three are +// successfully applied. +func (s) TestJoinDialOption(t *testing.T) { + const maxRecvSize = 998765 + const initialWindowSize = 100 + jdo := newJoinDialOption(WithTransportCredentials(insecure.NewCredentials()), WithReadBufferSize(maxRecvSize), WithInitialWindowSize(initialWindowSize)) + cc, err := NewClient("fake", jdo) + if err != nil { + t.Fatalf("NewClient with insecure credentials failed: %v", err) + } + defer cc.Close() + if cc.dopts.copts.ReadBufferSize != maxRecvSize { + t.Fatalf("Unexpected cc.dopts.copts.ReadBufferSize: %d != %d", cc.dopts.copts.ReadBufferSize, maxRecvSize) + } + if cc.dopts.copts.InitialWindowSize != initialWindowSize { + t.Fatalf("Unexpected cc.dopts.copts.InitialWindowSize: %d != %d", cc.dopts.copts.InitialWindowSize, initialWindowSize) + } +} + +// TestJoinServerOption tests the join server option. It configures a joined +// server option with three individual server options, and verifies that all +// three are successfully applied. +func (s) TestJoinServerOption(t *testing.T) { + const maxRecvSize = 998765 + const initialWindowSize = 100 + jso := newJoinServerOption(Creds(insecure.NewCredentials()), MaxRecvMsgSize(maxRecvSize), InitialWindowSize(initialWindowSize)) + s := NewServer(jso) + if s.opts.maxReceiveMessageSize != maxRecvSize { + t.Fatalf("Unexpected s.opts.maxReceiveMessageSize: %d != %d", s.opts.maxReceiveMessageSize, maxRecvSize) + } + if s.opts.initialWindowSize != initialWindowSize { + t.Fatalf("Unexpected s.opts.initialWindowSize: %d != %d", s.opts.initialWindowSize, initialWindowSize) + } +} + +// funcTestHeaderListSizeDialOptionServerOption tests +func (s) TestHeaderListSizeDialOptionServerOption(t *testing.T) { + const maxHeaderListSize uint32 = 998765 + clientHeaderListSize := WithMaxHeaderListSize(maxHeaderListSize) + if clientHeaderListSize.(MaxHeaderListSizeDialOption).MaxHeaderListSize != maxHeaderListSize { + t.Fatalf("Unexpected s.opts.MaxHeaderListSizeDialOption.MaxHeaderListSize: %d != %d", clientHeaderListSize, maxHeaderListSize) + } + serverHeaderListSize := MaxHeaderListSize(maxHeaderListSize) + if serverHeaderListSize.(MaxHeaderListSizeServerOption).MaxHeaderListSize != maxHeaderListSize { + t.Fatalf("Unexpected s.opts.MaxHeaderListSizeDialOption.MaxHeaderListSize: %d != %d", serverHeaderListSize, maxHeaderListSize) + } +} diff --git a/dial_test.go b/dial_test.go new file mode 100644 index 0000000..7b74aac --- /dev/null +++ b/dial_test.go @@ -0,0 +1,348 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "fmt" + "net" + "strings" + "testing" + "time" + + "golang.org/x/net/http2" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/testdata" +) + +func (s) TestDialWithTimeout(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis.Close() + lisAddr := resolver.Address{Addr: lis.Addr().String()} + lisDone := make(chan struct{}) + dialDone := make(chan struct{}) + // 1st listener accepts the connection and then does nothing + go func() { + defer close(lisDone) + conn, err := lis.Accept() + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings. Err: %v", err) + return + } + <-dialDone // Close conn only after dial returns. + }() + + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{lisAddr}}) + client, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithTimeout(5*time.Second)) + close(dialDone) + if err != nil { + t.Fatalf("Dial failed. Err: %v", err) + } + defer client.Close() + timeout := time.After(1 * time.Second) + select { + case <-timeout: + t.Fatal("timed out waiting for server to finish") + case <-lisDone: + } +} + +func (s) TestDialWaitsForServerSettings(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis.Close() + done := make(chan struct{}) + sent := make(chan struct{}) + dialDone := make(chan struct{}) + go func() { // Launch the server. + defer func() { + close(done) + }() + conn, err := lis.Accept() + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + defer conn.Close() + // Sleep for a little bit to make sure that Dial on client + // side blocks until settings are received. + time.Sleep(100 * time.Millisecond) + framer := http2.NewFramer(conn, conn) + close(sent) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings. Err: %v", err) + return + } + <-dialDone // Close conn only after dial returns. + }() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + client, err := DialContext(ctx, lis.Addr().String(), WithTransportCredentials(insecure.NewCredentials()), WithBlock()) + close(dialDone) + if err != nil { + t.Fatalf("Error while dialing. Err: %v", err) + } + defer client.Close() + select { + case <-sent: + default: + t.Fatalf("Dial returned before server settings were sent") + } + <-done +} + +func (s) TestDialWaitsForServerSettingsAndFails(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + done := make(chan struct{}) + numConns := 0 + go func() { // Launch the server. + defer func() { + close(done) + }() + for { + conn, err := lis.Accept() + if err != nil { + break + } + numConns++ + defer conn.Close() + } + }() + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + client, err := DialContext(ctx, + lis.Addr().String(), + WithTransportCredentials(insecure.NewCredentials()), + WithReturnConnectionError(), + WithConnectParams(ConnectParams{ + Backoff: backoff.Config{}, + MinConnectTimeout: 250 * time.Millisecond, + })) + lis.Close() + if err == nil { + client.Close() + t.Fatalf("Unexpected success (err=nil) while dialing") + } + expectedMsg := "server preface" + if !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) || !strings.Contains(err.Error(), expectedMsg) { + t.Fatalf("DialContext(_) = %v; want a message that includes both %q and %q", err, context.DeadlineExceeded.Error(), expectedMsg) + } + <-done + if numConns < 2 { + t.Fatalf("dial attempts: %v; want > 1", numConns) + } +} + +func (s) TestWithTimeout(t *testing.T) { + conn, err := Dial("passthrough:///Non-Existent.Server:80", + WithTimeout(time.Millisecond), + WithBlock(), + WithTransportCredentials(insecure.NewCredentials())) + if err == nil { + conn.Close() + } + if err != context.DeadlineExceeded { + t.Fatalf("Dial(_, _) = %v, %v, want %v", conn, err, context.DeadlineExceeded) + } +} + +func (s) TestWithTransportCredentialsTLS(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") + if err != nil { + t.Fatalf("Failed to create credentials %v", err) + } + conn, err := DialContext(ctx, "passthrough:///Non-Existent.Server:80", WithTransportCredentials(creds), WithBlock()) + if err == nil { + conn.Close() + } + if err != context.DeadlineExceeded { + t.Fatalf("Dial(_, _) = %v, %v, want %v", conn, err, context.DeadlineExceeded) + } +} + +func (s) TestDialContextCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := DialContext(ctx, "Non-Existent.Server:80", WithBlock(), WithTransportCredentials(insecure.NewCredentials())); err != context.Canceled { + t.Fatalf("DialContext(%v, _) = _, %v, want _, %v", ctx, err, context.Canceled) + } +} + +type failFastError struct{} + +func (failFastError) Error() string { return "failfast" } +func (failFastError) Temporary() bool { return false } + +func (s) TestDialContextFailFast(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + failErr := failFastError{} + dialer := func(string, time.Duration) (net.Conn, error) { + return nil, failErr + } + + _, err := DialContext(ctx, "Non-Existent.Server:80", WithBlock(), WithTransportCredentials(insecure.NewCredentials()), WithDialer(dialer), FailOnNonTempDialError(true)) + if terr, ok := err.(transport.ConnectionError); !ok || terr.Origin() != failErr { + t.Fatalf("DialContext() = _, %v, want _, %v", err, failErr) + } +} + +func (s) TestClientUpdatesParamsAfterGoAway(t *testing.T) { + grpctest.ExpectError("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\"") + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen. Err: %v", err) + } + defer lis.Close() + connected := grpcsync.NewEvent() + defer connected.Fire() + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("error accepting connection: %v", err) + return + } + defer conn.Close() + f := http2.NewFramer(conn, conn) + // Start a goroutine to read from the conn to prevent the client from + // blocking after it writes its preface. + go func() { + for { + if _, err := f.ReadFrame(); err != nil { + return + } + } + }() + if err := f.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("error writing settings: %v", err) + return + } + <-connected.Done() + if err := f.WriteGoAway(0, http2.ErrCodeEnhanceYourCalm, []byte("too_many_pings")); err != nil { + t.Errorf("error writing GOAWAY: %v", err) + return + } + }() + addr := lis.Addr().String() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cc, err := DialContext(ctx, addr, WithBlock(), WithTransportCredentials(insecure.NewCredentials()), WithKeepaliveParams(keepalive.ClientParameters{ + Time: 10 * time.Second, + Timeout: 100 * time.Millisecond, + PermitWithoutStream: true, + })) + if err != nil { + t.Fatalf("DialContext(%s) failed: %v, want: nil", addr, err) + } + defer cc.Close() + connected.Fire() + for { + time.Sleep(10 * time.Millisecond) + cc.mu.RLock() + v := cc.keepaliveParams.Time + cc.mu.RUnlock() + if v == 20*time.Second { + // Success + return + } + if ctx.Err() != nil { + // Timeout + t.Fatalf("cc.dopts.copts.Keepalive.Time = %v , want 20s", v) + } + } +} + +// Test ensures that there is no panic if the attributes within +// resolver.State.Addresses contains a typed-nil value. +func (s) TestResolverAddressesWithTypedNilAttribute(t *testing.T) { + r := manual.NewBuilderWithScheme(t.Name()) + resolver.Register(r) + + addrAttr := attributes.New("typed_nil", (*stringerVal)(nil)) + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: "addr1", Attributes: addrAttr}}}) + + cc, err := Dial(r.Scheme()+":///", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) + if err != nil { + t.Fatalf("Unexpected error dialing: %v", err) + } + defer cc.Close() +} + +type stringerVal struct{ s string } + +func (s stringerVal) String() string { return s.s } + +const errResolverBuilderScheme = "test-resolver-build-failure" + +// errResolverBuilder is a resolver builder that returns an error from its Build +// method. +type errResolverBuilder struct { + err error +} + +func (b *errResolverBuilder) Build(resolver.Target, resolver.ClientConn, resolver.BuildOptions) (resolver.Resolver, error) { + return nil, b.err +} + +func (b *errResolverBuilder) Scheme() string { + return errResolverBuilderScheme +} + +// Tests that Dial returns an error if the resolver builder returns an error +// from its Build method. +func (s) TestDial_ResolverBuilder_Error(t *testing.T) { + resolverErr := fmt.Errorf("resolver builder error") + dopts := []DialOption{ + WithTransportCredentials(insecure.NewCredentials()), + WithResolvers(&errResolverBuilder{err: resolverErr}), + } + _, err := Dial(errResolverBuilderScheme+":///test.server", dopts...) + if err == nil { + t.Fatalf("Dial() succeeded when it should have failed") + } + if !strings.Contains(err.Error(), resolverErr.Error()) { + t.Fatalf("Dial() failed with error %v, want %v", err, resolverErr) + } +} diff --git a/dialoptions.go b/dialoptions.go new file mode 100644 index 0000000..3af08e1 --- /dev/null +++ b/dialoptions.go @@ -0,0 +1,812 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "net" + "net/url" + "time" + + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/channelz" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + internalbackoff "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/binarylog" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/stats" +) + +const ( + // https://github.com/grpc/proposal/blob/master/A6-client-retries.md#limits-on-retries-and-hedges + defaultMaxCallAttempts = 5 +) + +func init() { + internal.AddGlobalDialOptions = func(opt ...DialOption) { + globalDialOptions = append(globalDialOptions, opt...) + } + internal.ClearGlobalDialOptions = func() { + globalDialOptions = nil + } + internal.AddGlobalPerTargetDialOptions = func(opt any) { + if ptdo, ok := opt.(perTargetDialOption); ok { + globalPerTargetDialOptions = append(globalPerTargetDialOptions, ptdo) + } + } + internal.ClearGlobalPerTargetDialOptions = func() { + globalPerTargetDialOptions = nil + } + internal.WithBinaryLogger = withBinaryLogger + internal.JoinDialOptions = newJoinDialOption + internal.DisableGlobalDialOptions = newDisableGlobalDialOptions + internal.WithBufferPool = withBufferPool +} + +// dialOptions configure a Dial call. dialOptions are set by the DialOption +// values passed to Dial. +type dialOptions struct { + unaryInt UnaryClientInterceptor + streamInt StreamClientInterceptor + + chainUnaryInts []UnaryClientInterceptor + chainStreamInts []StreamClientInterceptor + + compressorV0 Compressor + dc Decompressor + bs internalbackoff.Strategy + block bool + returnLastError bool + timeout time.Duration + authority string + binaryLogger binarylog.Logger + copts transport.ConnectOptions + callOptions []CallOption + channelzParent channelz.Identifier + disableServiceConfig bool + disableRetry bool + disableHealthCheck bool + minConnectTimeout func() time.Duration + defaultServiceConfig *ServiceConfig // defaultServiceConfig is parsed from defaultServiceConfigRawJSON. + defaultServiceConfigRawJSON *string + resolvers []resolver.Builder + idleTimeout time.Duration + defaultScheme string + maxCallAttempts int + enableLocalDNSResolution bool // Specifies if target hostnames should be resolved when proxying is enabled. + useProxy bool // Specifies if a server should be connected via proxy. +} + +// DialOption configures how we set up the connection. +type DialOption interface { + apply(*dialOptions) +} + +var globalDialOptions []DialOption + +// perTargetDialOption takes a parsed target and returns a dial option to apply. +// +// This gets called after NewClient() parses the target, and allows per target +// configuration set through a returned DialOption. The DialOption will not take +// effect if specifies a resolver builder, as that Dial Option is factored in +// while parsing target. +type perTargetDialOption interface { + // DialOption returns a Dial Option to apply. + DialOptionForTarget(parsedTarget url.URL) DialOption +} + +var globalPerTargetDialOptions []perTargetDialOption + +// EmptyDialOption does not alter the dial configuration. It can be embedded in +// another structure to build custom dial options. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type EmptyDialOption struct{} + +func (EmptyDialOption) apply(*dialOptions) {} + +type disableGlobalDialOptions struct{} + +func (disableGlobalDialOptions) apply(*dialOptions) {} + +// newDisableGlobalDialOptions returns a DialOption that prevents the ClientConn +// from applying the global DialOptions (set via AddGlobalDialOptions). +func newDisableGlobalDialOptions() DialOption { + return &disableGlobalDialOptions{} +} + +// funcDialOption wraps a function that modifies dialOptions into an +// implementation of the DialOption interface. +type funcDialOption struct { + f func(*dialOptions) +} + +func (fdo *funcDialOption) apply(do *dialOptions) { + fdo.f(do) +} + +func newFuncDialOption(f func(*dialOptions)) *funcDialOption { + return &funcDialOption{ + f: f, + } +} + +type joinDialOption struct { + opts []DialOption +} + +func (jdo *joinDialOption) apply(do *dialOptions) { + for _, opt := range jdo.opts { + opt.apply(do) + } +} + +func newJoinDialOption(opts ...DialOption) DialOption { + return &joinDialOption{opts: opts} +} + +// WithSharedWriteBuffer allows reusing per-connection transport write buffer. +// If this option is set to true every connection will release the buffer after +// flushing the data on the wire. +// +// Deprecated: shared write buffer is enabled by default. WithSharedWriteBuffer +// will be removed in a future release. +func WithSharedWriteBuffer(val bool) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.SharedWriteBuffer = val + }) +} + +// WithWriteBufferSize determines how much data can be batched before doing a +// write on the wire. The default value for this buffer is 32KB. +// +// Zero or negative values will disable the write buffer such that each write +// will be on underlying connection. Note: A Send call may not directly +// translate to a write. +func WithWriteBufferSize(s int) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.WriteBufferSize = s + }) +} + +// WithReadBufferSize lets you set the size of read buffer, this determines how +// much data can be read at most for each read syscall. +// +// The default value for this buffer is 32KB. Zero or negative values will +// disable read buffer for a connection so data framer can access the +// underlying conn directly. +func WithReadBufferSize(s int) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.ReadBufferSize = s + }) +} + +// WithInitialWindowSize returns a DialOption which sets the value for initial +// window size on a stream. The lower bound for window size is 64K and any value +// smaller than that will be ignored. +func WithInitialWindowSize(s int32) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.InitialWindowSize = s + o.copts.StaticWindowSize = true + }) +} + +// WithInitialConnWindowSize returns a DialOption which sets the value for +// initial window size on a connection. The lower bound for window size is 64K +// and any value smaller than that will be ignored. +func WithInitialConnWindowSize(s int32) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.InitialConnWindowSize = s + o.copts.StaticWindowSize = true + }) +} + +// WithStaticStreamWindowSize returns a DialOption which sets the initial +// stream window size to the value provided and disables dynamic flow control. +// +// Note that this also disables dynamic flow control for the connection, +// falling back to a default static connection-level window of 64KB. To +// use a larger connection-level window, you must also use the +// [WithStaticConnWindowSize] DialOption. +// +// Most users should not configure static flow control windows unless +// operating in a memory-constrained environment. +func WithStaticStreamWindowSize(s int32) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.InitialWindowSize = s + o.copts.StaticWindowSize = true + }) +} + +// WithStaticConnWindowSize returns a DialOption which sets the initial +// connection window size to the value provided and disables dynamic flow +// control. +// +// Note that this also disables dynamic flow control for individual streams, +// falling back to a default static connection-level window of 64KB. To +// explicitly configure the stream-level window size, you must also use the +// [WithStaticStreamWindowSize] DialOption. +// +// Most users should not configure static flow control windows unless +// operating in a memory-constrained environment. +func WithStaticConnWindowSize(s int32) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.InitialConnWindowSize = s + o.copts.StaticWindowSize = true + }) +} + +// WithMaxMsgSize returns a DialOption which sets the maximum message size the +// client can receive. +// +// Deprecated: use WithDefaultCallOptions(MaxCallRecvMsgSize(s)) instead. Will +// be supported throughout 1.x. +func WithMaxMsgSize(s int) DialOption { + return WithDefaultCallOptions(MaxCallRecvMsgSize(s)) +} + +// WithDefaultCallOptions returns a DialOption which sets the default +// CallOptions for calls over the connection. +func WithDefaultCallOptions(cos ...CallOption) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.callOptions = append(o.callOptions, cos...) + }) +} + +// WithCodec returns a DialOption which sets a codec for message marshaling and +// unmarshaling. +// +// Deprecated: use WithDefaultCallOptions(ForceCodec(_)) instead. Will be +// supported throughout 1.x. +func WithCodec(c Codec) DialOption { + return WithDefaultCallOptions(CallCustomCodec(c)) +} + +// WithCompressor returns a DialOption which sets a Compressor to use for +// message compression. It has lower priority than the compressor set by the +// UseCompressor CallOption. +// +// Deprecated: use UseCompressor instead. Will be supported throughout 1.x. +func WithCompressor(cp Compressor) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.compressorV0 = cp + }) +} + +// WithDecompressor returns a DialOption which sets a Decompressor to use for +// incoming message decompression. If incoming response messages are encoded +// using the decompressor's Type(), it will be used. Otherwise, the message +// encoding will be used to look up the compressor registered via +// encoding.RegisterCompressor, which will then be used to decompress the +// message. If no compressor is registered for the encoding, an Unimplemented +// status error will be returned. +// +// Deprecated: use encoding.RegisterCompressor instead. Will be supported +// throughout 1.x. +func WithDecompressor(dc Decompressor) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.dc = dc + }) +} + +// WithConnectParams configures the ClientConn to use the provided ConnectParams +// for creating and maintaining connections to servers. +// +// The backoff configuration specified as part of the ConnectParams overrides +// all defaults specified in +// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. Consider +// using the backoff.DefaultConfig as a base, in cases where you want to +// override only a subset of the backoff configuration. +func WithConnectParams(p ConnectParams) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.bs = internalbackoff.Exponential{Config: p.Backoff} + o.minConnectTimeout = func() time.Duration { + return p.MinConnectTimeout + } + }) +} + +// WithBackoffMaxDelay configures the dialer to use the provided maximum delay +// when backing off after failed connection attempts. +// +// Deprecated: use WithConnectParams instead. Will be supported throughout 1.x. +func WithBackoffMaxDelay(md time.Duration) DialOption { + return WithBackoffConfig(BackoffConfig{MaxDelay: md}) +} + +// WithBackoffConfig configures the dialer to use the provided backoff +// parameters after connection failures. +// +// Deprecated: use WithConnectParams instead. Will be supported throughout 1.x. +func WithBackoffConfig(b BackoffConfig) DialOption { + bc := backoff.DefaultConfig + bc.MaxDelay = b.MaxDelay + return withBackoff(internalbackoff.Exponential{Config: bc}) +} + +// withBackoff sets the backoff strategy used for connectRetryNum after a failed +// connection attempt. +// +// This can be exported if arbitrary backoff strategies are allowed by gRPC. +func withBackoff(bs internalbackoff.Strategy) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.bs = bs + }) +} + +// WithBlock returns a DialOption which makes callers of Dial block until the +// underlying connection is up. Without this, Dial returns immediately and +// connecting the server happens in background. +// +// Use of this feature is not recommended. For more information, please see: +// https://github.com/grpc/grpc-go/blob/master/Documentation/anti-patterns.md +// +// Deprecated: this DialOption is not supported by NewClient. +// Will be supported throughout 1.x. +func WithBlock() DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.block = true + }) +} + +// WithReturnConnectionError returns a DialOption which makes the client connection +// return a string containing both the last connection error that occurred and +// the context.DeadlineExceeded error. +// Implies WithBlock() +// +// Use of this feature is not recommended. For more information, please see: +// https://github.com/grpc/grpc-go/blob/master/Documentation/anti-patterns.md +// +// Deprecated: this DialOption is not supported by NewClient. +// Will be supported throughout 1.x. +func WithReturnConnectionError() DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.block = true + o.returnLastError = true + }) +} + +// WithInsecure returns a DialOption which disables transport security for this +// ClientConn. Under the hood, it uses insecure.NewCredentials(). +// +// Note that using this DialOption with per-RPC credentials (through +// WithCredentialsBundle or WithPerRPCCredentials) which require transport +// security is incompatible and will cause RPCs to fail. +// +// Deprecated: use WithTransportCredentials and insecure.NewCredentials() +// instead. Will be supported throughout 1.x. +func WithInsecure() DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.TransportCredentials = insecure.NewCredentials() + }) +} + +// WithNoProxy returns a DialOption which disables the use of proxies for this +// ClientConn. This is ignored if WithDialer or WithContextDialer are used. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func WithNoProxy() DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.useProxy = false + }) +} + +// WithLocalDNSResolution forces local DNS name resolution even when a proxy is +// specified in the environment. By default, the server name is provided +// directly to the proxy as part of the CONNECT handshake. This is ignored if +// WithNoProxy is used. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func WithLocalDNSResolution() DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.enableLocalDNSResolution = true + }) +} + +// WithTransportCredentials returns a DialOption which configures a connection +// level security credentials (e.g., TLS/SSL). This should not be used together +// with WithCredentialsBundle. +func WithTransportCredentials(creds credentials.TransportCredentials) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.TransportCredentials = creds + }) +} + +// WithPerRPCCredentials returns a DialOption which sets credentials and places +// auth state on each outbound RPC. +func WithPerRPCCredentials(creds credentials.PerRPCCredentials) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.PerRPCCredentials = append(o.copts.PerRPCCredentials, creds) + }) +} + +// WithCredentialsBundle returns a DialOption to set a credentials bundle for +// the ClientConn.WithCreds. This should not be used together with +// WithTransportCredentials. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func WithCredentialsBundle(b credentials.Bundle) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.CredsBundle = b + }) +} + +// WithTimeout returns a DialOption that configures a timeout for dialing a +// ClientConn initially. This is valid if and only if WithBlock() is present. +// +// Deprecated: this DialOption is not supported by NewClient. +// Will be supported throughout 1.x. +func WithTimeout(d time.Duration) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.timeout = d + }) +} + +// WithContextDialer returns a DialOption that sets a dialer to create +// connections. If FailOnNonTempDialError() is set to true, and an error is +// returned by f, gRPC checks the error's Temporary() method to decide if it +// should try to reconnect to the network address. +// +// Note that gRPC by default performs name resolution on the target passed to +// NewClient. To bypass name resolution and cause the target string to be +// passed directly to the dialer here instead, use the "passthrough" resolver +// by specifying it in the target string, e.g. "passthrough:target". +// +// Note: All supported releases of Go (as of December 2023) override the OS +// defaults for TCP keepalive time and interval to 15s. To enable TCP keepalive +// with OS defaults for keepalive time and interval, use a net.Dialer that sets +// the KeepAlive field to a negative value, and sets the SO_KEEPALIVE socket +// option to true from the Control field. For a concrete example of how to do +// this, see internal.NetDialerWithTCPKeepalive(). +// +// For more information, please see [issue 23459] in the Go GitHub repo. +// +// [issue 23459]: https://github.com/golang/go/issues/23459 +func WithContextDialer(f func(context.Context, string) (net.Conn, error)) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.Dialer = f + }) +} + +// WithDialer returns a DialOption that specifies a function to use for dialing +// network addresses. If FailOnNonTempDialError() is set to true, and an error +// is returned by f, gRPC checks the error's Temporary() method to decide if it +// should try to reconnect to the network address. +// +// Deprecated: use WithContextDialer instead. Will be supported throughout +// 1.x. +func WithDialer(f func(string, time.Duration) (net.Conn, error)) DialOption { + return WithContextDialer( + func(ctx context.Context, addr string) (net.Conn, error) { + if deadline, ok := ctx.Deadline(); ok { + return f(addr, time.Until(deadline)) + } + return f(addr, 0) + }) +} + +// WithStatsHandler returns a DialOption that specifies the stats handler for +// all the RPCs and underlying network connections in this ClientConn. +func WithStatsHandler(h stats.Handler) DialOption { + return newFuncDialOption(func(o *dialOptions) { + if h == nil { + logger.Error("ignoring nil parameter in grpc.WithStatsHandler ClientOption") + // Do not allow a nil stats handler, which would otherwise cause + // panics. + return + } + o.copts.StatsHandlers = append(o.copts.StatsHandlers, h) + }) +} + +// withBinaryLogger returns a DialOption that specifies the binary logger for +// this ClientConn. +func withBinaryLogger(bl binarylog.Logger) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.binaryLogger = bl + }) +} + +// FailOnNonTempDialError returns a DialOption that specifies if gRPC fails on +// non-temporary dial errors. If f is true, and dialer returns a non-temporary +// error, gRPC will fail the connection to the network address and won't try to +// reconnect. The default value of FailOnNonTempDialError is false. +// +// FailOnNonTempDialError only affects the initial dial, and does not do +// anything useful unless you are also using WithBlock(). +// +// Use of this feature is not recommended. For more information, please see: +// https://github.com/grpc/grpc-go/blob/master/Documentation/anti-patterns.md +// +// Deprecated: this DialOption is not supported by NewClient. +// This API may be changed or removed in a +// later release. +func FailOnNonTempDialError(f bool) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.FailOnNonTempDialError = f + }) +} + +// WithUserAgent returns a DialOption that specifies a user agent string for all +// the RPCs. +func WithUserAgent(s string) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.UserAgent = s + " " + grpcUA + }) +} + +// WithKeepaliveParams returns a DialOption that specifies keepalive parameters +// for the client transport. +// +// Keepalive is disabled by default. +func WithKeepaliveParams(kp keepalive.ClientParameters) DialOption { + if kp.Time < internal.KeepaliveMinPingTime { + logger.Warningf("Adjusting keepalive ping interval to minimum period of %v", internal.KeepaliveMinPingTime) + kp.Time = internal.KeepaliveMinPingTime + } + return newFuncDialOption(func(o *dialOptions) { + o.copts.KeepaliveParams = kp + }) +} + +// WithUnaryInterceptor returns a DialOption that specifies the interceptor for +// unary RPCs. +func WithUnaryInterceptor(f UnaryClientInterceptor) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.unaryInt = f + }) +} + +// WithChainUnaryInterceptor returns a DialOption that specifies the chained +// interceptor for unary RPCs. The first interceptor will be the outer most, +// while the last interceptor will be the inner most wrapper around the real call. +// All interceptors added by this method will be chained, and the interceptor +// defined by WithUnaryInterceptor will always be prepended to the chain. +func WithChainUnaryInterceptor(interceptors ...UnaryClientInterceptor) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.chainUnaryInts = append(o.chainUnaryInts, interceptors...) + }) +} + +// WithStreamInterceptor returns a DialOption that specifies the interceptor for +// streaming RPCs. +func WithStreamInterceptor(f StreamClientInterceptor) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.streamInt = f + }) +} + +// WithChainStreamInterceptor returns a DialOption that specifies the chained +// interceptor for streaming RPCs. The first interceptor will be the outer most, +// while the last interceptor will be the inner most wrapper around the real call. +// All interceptors added by this method will be chained, and the interceptor +// defined by WithStreamInterceptor will always be prepended to the chain. +func WithChainStreamInterceptor(interceptors ...StreamClientInterceptor) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.chainStreamInts = append(o.chainStreamInts, interceptors...) + }) +} + +// WithAuthority returns a DialOption that specifies the value to be used as the +// :authority pseudo-header and as the server name in authentication handshake. +// This overrides all other ways of setting authority on the channel, but can be +// overridden per-call by using grpc.CallAuthority. +func WithAuthority(a string) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.authority = a + }) +} + +// WithChannelzParentID returns a DialOption that specifies the channelz ID of +// current ClientConn's parent. This function is used in nested channel creation +// (e.g. grpclb dial). +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func WithChannelzParentID(c channelz.Identifier) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.channelzParent = c + }) +} + +// WithDisableServiceConfig returns a DialOption that causes gRPC to ignore any +// service config provided by the resolver and provides a hint to the resolver +// to not fetch service configs. +// +// Note that this dial option only disables service config from resolver. If +// default service config is provided, gRPC will use the default service config. +func WithDisableServiceConfig() DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.disableServiceConfig = true + }) +} + +// WithDefaultServiceConfig returns a DialOption that configures the default +// service config, which will be used in cases where: +// +// 1. WithDisableServiceConfig is also used, or +// +// 2. The name resolver does not provide a service config or provides an +// invalid service config. +// +// The parameter s is the JSON representation of the default service config. +// For more information about service configs, see: +// https://github.com/grpc/grpc/blob/master/doc/service_config.md +// For a simple example of usage, see: +// examples/features/load_balancing/client/main.go +func WithDefaultServiceConfig(s string) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.defaultServiceConfigRawJSON = &s + }) +} + +// WithDisableRetry returns a DialOption that disables retries, even if the +// service config enables them. This does not impact transparent retries, which +// will happen automatically if no data is written to the wire or if the RPC is +// unprocessed by the remote server. +func WithDisableRetry() DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.disableRetry = true + }) +} + +// MaxHeaderListSizeDialOption is a DialOption that specifies the maximum +// (uncompressed) size of header list that the client is prepared to accept. +type MaxHeaderListSizeDialOption struct { + MaxHeaderListSize uint32 +} + +func (o MaxHeaderListSizeDialOption) apply(do *dialOptions) { + do.copts.MaxHeaderListSize = &o.MaxHeaderListSize +} + +// WithMaxHeaderListSize returns a DialOption that specifies the maximum +// (uncompressed) size of header list that the client is prepared to accept. +func WithMaxHeaderListSize(s uint32) DialOption { + return MaxHeaderListSizeDialOption{ + MaxHeaderListSize: s, + } +} + +// WithDisableHealthCheck disables the LB channel health checking for all +// SubConns of this ClientConn. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func WithDisableHealthCheck() DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.disableHealthCheck = true + }) +} + +func defaultDialOptions() dialOptions { + return dialOptions{ + copts: transport.ConnectOptions{ + ReadBufferSize: defaultReadBufSize, + WriteBufferSize: defaultWriteBufSize, + SharedWriteBuffer: true, + UserAgent: grpcUA, + BufferPool: mem.DefaultBufferPool(), + }, + bs: internalbackoff.DefaultExponential, + idleTimeout: 30 * time.Minute, + defaultScheme: "dns", + maxCallAttempts: defaultMaxCallAttempts, + useProxy: true, + enableLocalDNSResolution: false, + } +} + +// withMinConnectDeadline specifies the function that clientconn uses to +// get minConnectDeadline. This can be used to make connection attempts happen +// faster/slower. +// +// For testing purpose only. +func withMinConnectDeadline(f func() time.Duration) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.minConnectTimeout = f + }) +} + +// withDefaultScheme is used to allow Dial to use "passthrough" as the default +// name resolver, while NewClient uses "dns" otherwise. +func withDefaultScheme(s string) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.defaultScheme = s + }) +} + +// WithResolvers allows a list of resolver implementations to be registered +// locally with the ClientConn without needing to be globally registered via +// resolver.Register. They will be matched against the scheme used for the +// current Dial only, and will take precedence over the global registry. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func WithResolvers(rs ...resolver.Builder) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.resolvers = append(o.resolvers, rs...) + }) +} + +// WithIdleTimeout returns a DialOption that configures an idle timeout for the +// channel. If the channel is idle for the configured timeout, i.e there are no +// ongoing RPCs and no new RPCs are initiated, the channel will enter idle mode +// and as a result the name resolver and load balancer will be shut down. The +// channel will exit idle mode when the Connect() method is called or when an +// RPC is initiated. +// +// A default timeout of 30 minutes will be used if this dial option is not set +// at dial time and idleness can be disabled by passing a timeout of zero. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func WithIdleTimeout(d time.Duration) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.idleTimeout = d + }) +} + +// WithMaxCallAttempts returns a DialOption that configures the maximum number +// of attempts per call (including retries and hedging) using the channel. +// Service owners may specify a higher value for these parameters, but higher +// values will be treated as equal to the maximum value by the client +// implementation. This mitigates security concerns related to the service +// config being transferred to the client via DNS. +// +// A value of 5 will be used if this dial option is not set or n < 2. +func WithMaxCallAttempts(n int) DialOption { + return newFuncDialOption(func(o *dialOptions) { + if n < 2 { + n = defaultMaxCallAttempts + } + o.maxCallAttempts = n + }) +} + +func withBufferPool(bufferPool mem.BufferPool) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.BufferPool = bufferPool + }) +} diff --git a/doc.go b/doc.go new file mode 100644 index 0000000..e7b532b --- /dev/null +++ b/doc.go @@ -0,0 +1,26 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +//go:generate ./scripts/regenerate.sh + +/* +Package grpc implements an RPC system called gRPC. + +See grpc.io for more information about gRPC. +*/ +package grpc // import "google.golang.org/grpc" diff --git a/encoding/compressor_test.go b/encoding/compressor_test.go new file mode 100644 index 0000000..b41b95f --- /dev/null +++ b/encoding/compressor_test.go @@ -0,0 +1,281 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package encoding_test + +import ( + "bytes" + "context" + "io" + "sync/atomic" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/encoding/internal" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/encoding/gzip" +) + +// wrapCompressor is a wrapper of encoding.Compressor which maintains count of +// Compressor method invokes. +type wrapCompressor struct { + encoding.Compressor + compressInvokes int32 +} + +func (wc *wrapCompressor) Compress(w io.Writer) (io.WriteCloser, error) { + atomic.AddInt32(&wc.compressInvokes, 1) + return wc.Compressor.Compress(w) +} + +func setupGzipWrapCompressor(t *testing.T) *wrapCompressor { + regFn := internal.RegisterCompressorForTesting.(func(encoding.Compressor) func()) + c := &wrapCompressor{Compressor: encoding.GetCompressor("gzip")} + unreg := regFn(c) + t.Cleanup(unreg) + return c +} + +func (s) TestSetSendCompressorSuccess(t *testing.T) { + for _, tt := range []struct { + name string + desc string + payload *testpb.Payload + dialOpts []grpc.DialOption + resCompressor string + wantCompressInvokes int32 + }{ + { + name: "identity_request_and_gzip_response", + desc: "request is uncompressed and response is gzip compressed", + payload: &testpb.Payload{Body: []byte("payload")}, + resCompressor: "gzip", + wantCompressInvokes: 1, + }, + { + name: "identity_request_and_empty_response", + desc: "request is uncompressed and response is gzip compressed", + payload: nil, + resCompressor: "gzip", + wantCompressInvokes: 0, + }, + { + name: "gzip_request_and_identity_response", + desc: "request is gzip compressed and response is uncompressed with identity", + payload: &testpb.Payload{Body: []byte("payload")}, + resCompressor: "identity", + dialOpts: []grpc.DialOption{ + // Use WithCompressor instead of UseCompressor to avoid counting + // the client's compressor usage. + grpc.WithCompressor(grpc.NewGZIPCompressor()), + }, + wantCompressInvokes: 0, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Run("unary", func(t *testing.T) { + testUnarySetSendCompressorSuccess(t, tt.payload, tt.resCompressor, tt.wantCompressInvokes, tt.dialOpts) + }) + + t.Run("stream", func(t *testing.T) { + testStreamSetSendCompressorSuccess(t, tt.payload, tt.resCompressor, tt.wantCompressInvokes, tt.dialOpts) + }) + }) + } +} + +func testUnarySetSendCompressorSuccess(t *testing.T, payload *testpb.Payload, resCompressor string, wantCompressInvokes int32, dialOpts []grpc.DialOption) { + wc := setupGzipWrapCompressor(t) + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + if err := grpc.SetSendCompressor(ctx, resCompressor); err != nil { + return nil, err + } + return &testpb.SimpleResponse{ + Payload: payload, + }, nil + }, + } + if err := ss.Start(nil, dialOpts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected unary call error, got: %v, want: nil", err) + } + + compressInvokes := atomic.LoadInt32(&wc.compressInvokes) + if compressInvokes != wantCompressInvokes { + t.Fatalf("Unexpected compress invokes, got:%d, want: %d", compressInvokes, wantCompressInvokes) + } +} + +func testStreamSetSendCompressorSuccess(t *testing.T, payload *testpb.Payload, resCompressor string, wantCompressInvokes int32, dialOpts []grpc.DialOption) { + wc := setupGzipWrapCompressor(t) + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + + if err := grpc.SetSendCompressor(stream.Context(), resCompressor); err != nil { + return err + } + + return stream.Send(&testpb.StreamingOutputCallResponse{ + Payload: payload, + }) + }, + } + if err := ss.Start(nil, dialOpts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + s, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("Unexpected full duplex call error, got: %v, want: nil", err) + } + + if err := s.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("Unexpected full duplex call send error, got: %v, want: nil", err) + } + + if _, err := s.Recv(); err != nil { + t.Fatalf("Unexpected full duplex recv error, got: %v, want: nil", err) + } + + compressInvokes := atomic.LoadInt32(&wc.compressInvokes) + if compressInvokes != wantCompressInvokes { + t.Fatalf("Unexpected compress invokes, got:%d, want: %d", compressInvokes, wantCompressInvokes) + } +} + +// fakeCompressor returns a messages of a configured size, irrespective of the +// input. +type fakeCompressor struct { + decompressedMessageSize int +} + +func (f *fakeCompressor) Compress(w io.Writer) (io.WriteCloser, error) { + return nopWriteCloser{w}, nil +} + +func (f *fakeCompressor) Decompress(io.Reader) (io.Reader, error) { + return bytes.NewReader(make([]byte, f.decompressedMessageSize)), nil +} + +func (f *fakeCompressor) Name() string { + // Use the name of an existing compressor to avoid interactions with other + // tests since compressors can't be un-registered. + return "fake" +} + +type nopWriteCloser struct { + io.Writer +} + +func (nopWriteCloser) Close() error { + return nil +} + +// TestDecompressionExceedsMaxMessageSize uses a fake compressor that produces +// messages of size 100 bytes on decompression. A server is started with the +// max receive message size restricted to 99 bytes. The test verifies that the +// client receives a ResourceExhausted response from the server. +func (s) TestDecompressionExceedsMaxMessageSize(t *testing.T) { + const messageLen = 100 + regFn := internal.RegisterCompressorForTesting.(func(encoding.Compressor) func()) + compressor := &fakeCompressor{decompressedMessageSize: messageLen} + unreg := regFn(compressor) + defer unreg() + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + } + if err := ss.Start([]grpc.ServerOption{grpc.MaxRecvMsgSize(messageLen - 1)}); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + req := &testpb.SimpleRequest{Payload: &testpb.Payload{}} + _, err := ss.Client.UnaryCall(ctx, req, grpc.UseCompressor(compressor.Name())) + if got, want := status.Code(err), codes.ResourceExhausted; got != want { + t.Errorf("Client.UnaryCall(%+v) returned status %v, want %v", req, got, want) + } +} + +// Test verifies that decompression failures (after stream establishment) are +// handled gracefully without duplicate status writes. +func (s) TestStreamingDecompressionExceedsMaxMessageSize(t *testing.T) { + const messageLen = 100 + regFn := internal.RegisterCompressorForTesting.(func(encoding.Compressor) func()) + compressor := &fakeCompressor{decompressedMessageSize: messageLen} + unreg := regFn(compressor) + defer unreg() + + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + if _, err := stream.Recv(); err != nil { + return err + } + } + }, + } + if err := ss.Start([]grpc.ServerOption{grpc.MaxRecvMsgSize(messageLen - 1)}); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := ss.Client.FullDuplexCall(ctx, grpc.UseCompressor(compressor.Name())) + if err != nil { + t.Fatalf("FullDuplexCall failed: %v", err) + } + + // Send message that will result in decompression failure on the server + // because it exceeds the max message size. + if err := stream.Send(&testpb.StreamingOutputCallRequest{Payload: &testpb.Payload{Body: []byte("payload")}}); err != nil { + t.Fatalf("Send() failed: %v", err) + } + + // We expect the stream to fail with ResourceExhausted on Recv. + if _, err = stream.Recv(); status.Code(err) != codes.ResourceExhausted { + t.Fatalf("stream.Recv() returned status %v, want %v", status.Code(err), codes.ResourceExhausted) + } +} diff --git a/encoding/encoding.go b/encoding/encoding.go new file mode 100644 index 0000000..bfa8b26 --- /dev/null +++ b/encoding/encoding.go @@ -0,0 +1,150 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package encoding defines the interface for the compressor and codec, and +// functions to register and retrieve compressors and codecs. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +package encoding + +import ( + "io" + "slices" + "strings" + + "google.golang.org/grpc/encoding/internal" + "google.golang.org/grpc/internal/grpcutil" +) + +// Identity specifies the optional encoding for uncompressed streams. +// It is intended for grpc internal use only. +const Identity = "identity" + +func init() { + internal.RegisterCompressorForTesting = func(c Compressor) func() { + name := c.Name() + curCompressor, found := registeredCompressor[name] + RegisterCompressor(c) + return func() { + if found { + registeredCompressor[name] = curCompressor + return + } + delete(registeredCompressor, name) + grpcutil.RegisteredCompressorNames = slices.DeleteFunc(grpcutil.RegisteredCompressorNames, func(s string) bool { + return s == name + }) + } + } +} + +// Compressor is used for compressing and decompressing when sending or +// receiving messages. +type Compressor interface { + // Compress writes the data written to wc to w after compressing it. If an + // error occurs while initializing the compressor, that error is returned + // instead. + Compress(w io.Writer) (io.WriteCloser, error) + // Decompress reads data from r, decompresses it, and provides the + // uncompressed data via the returned io.Reader. If an error occurs while + // initializing the decompressor, that error is returned instead. + // + // The returned io.Reader may optionally implement io.ReadCloser, and if it + // does, gRPC will call Close() exactly once. + Decompress(r io.Reader) (io.Reader, error) + // Name is the name of the compression codec and is used to set the content + // coding header. The result must be static; the result cannot change + // between calls. + Name() string +} + +var registeredCompressor = make(map[string]Compressor) + +// RegisterCompressor registers the compressor with gRPC by its name. It can +// be activated when sending an RPC via grpc.UseCompressor(). It will be +// automatically accessed when receiving a message based on the content coding +// header. Servers also use it to send a response with the same encoding as +// the request. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple Compressors are +// registered with the same name, the one registered last will take effect. +func RegisterCompressor(c Compressor) { + registeredCompressor[c.Name()] = c + if !grpcutil.IsCompressorNameRegistered(c.Name()) { + grpcutil.RegisteredCompressorNames = append(grpcutil.RegisteredCompressorNames, c.Name()) + } +} + +// GetCompressor returns Compressor for the given compressor name. +func GetCompressor(name string) Compressor { + return registeredCompressor[name] +} + +// Codec defines the interface gRPC uses to encode and decode messages. Note +// that implementations of this interface must be thread safe; a Codec's +// methods can be called from concurrent goroutines. +type Codec interface { + // Marshal returns the wire format of v. + Marshal(v any) ([]byte, error) + // Unmarshal parses the wire format into v. + Unmarshal(data []byte, v any) error + // Name returns the name of the Codec implementation. The returned string + // will be used as part of content type in transmission. The result must be + // static; the result cannot change between calls. + Name() string +} + +var registeredCodecs = make(map[string]any) + +// RegisterCodec registers the provided Codec for use with all gRPC clients and +// servers. +// +// The Codec will be stored and looked up by result of its Name() method, which +// should match the content-subtype of the encoding handled by the Codec. This +// is case-insensitive, and is stored and looked up as lowercase. If the +// result of calling Name() is an empty string, RegisterCodec will panic. See +// Content-Type on +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple Codecs are +// registered with the same name, the one registered last will take effect. +func RegisterCodec(codec Codec) { + if codec == nil { + panic("cannot register a nil Codec") + } + if codec.Name() == "" { + panic("cannot register Codec with empty string result for Name()") + } + contentSubtype := strings.ToLower(codec.Name()) + registeredCodecs[contentSubtype] = codec +} + +// GetCodec gets a registered Codec by content-subtype, or nil if no Codec is +// registered for the content-subtype. +// +// The content-subtype is expected to be lowercase. +func GetCodec(contentSubtype string) Codec { + c, _ := registeredCodecs[contentSubtype].(Codec) + return c +} diff --git a/encoding/encoding_test.go b/encoding/encoding_test.go new file mode 100644 index 0000000..a446059 --- /dev/null +++ b/encoding/encoding_test.go @@ -0,0 +1,400 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package encoding_test + +import ( + "context" + "errors" + "fmt" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/encoding/proto" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/grpcutil" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type mockNamedCompressor struct { + encoding.Compressor +} + +func (mockNamedCompressor) Name() string { + return "mock-compressor" +} + +// Tests the case where a compressor with the same name is registered multiple +// times. Test verifies the following: +// - the most recent registration is the one which is active +// - grpcutil.RegisteredCompressorNames contains a single instance of the +// previously registered compressor's name +func (s) TestDuplicateCompressorRegister(t *testing.T) { + encoding.RegisterCompressor(&mockNamedCompressor{}) + + // Register another instance of the same compressor. + mc := &mockNamedCompressor{} + encoding.RegisterCompressor(mc) + if got := encoding.GetCompressor("mock-compressor"); got != mc { + t.Fatalf("Unexpected compressor, got: %+v, want:%+v", got, mc) + } + + wantNames := []string{"gzip", "mock-compressor"} + if !cmp.Equal(wantNames, grpcutil.RegisteredCompressorNames) { + t.Fatalf("Unexpected compressor names, got: %+v, want:%+v", grpcutil.RegisteredCompressorNames, wantNames) + } +} + +// errProtoCodec wraps the proto codec and delegates to it if it is configured +// to return a nil error. Else, it returns the configured error. +type errProtoCodec struct { + name string + encodingErr error + decodingErr error +} + +func (c *errProtoCodec) Marshal(v any) (mem.BufferSlice, error) { + if c.encodingErr != nil { + return nil, c.encodingErr + } + return encoding.GetCodecV2(proto.Name).Marshal(v) +} + +func (c *errProtoCodec) Unmarshal(data mem.BufferSlice, v any) error { + if c.decodingErr != nil { + return c.decodingErr + } + return encoding.GetCodecV2(proto.Name).Unmarshal(data, v) +} + +func (c *errProtoCodec) Name() string { + return c.name +} + +// Tests the case where encoding fails on the server. Verifies that there is +// no panic and that the encoding error is propagated to the client. +func (s) TestEncodeDoesntPanicOnServer(t *testing.T) { + // Create a codec that errors when encoding messages. + encodingErr := errors.New("encoding failed") + ec := &errProtoCodec{name: t.Name(), encodingErr: encodingErr} + + // Start a server with the above codec. + backend1 := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := backend1.Start([]grpc.ServerOption{grpc.ForceServerCodecV2(ec)}, grpc.WithTransportCredentials(insecure.NewCredentials())); err != nil { + t.Fatal(err) + } + defer backend1.Stop() + + // Make an RPC and expect it to fail. Since we do not specify any codec + // here, the proto codec will get automatically used. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := backend1.Client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil || !strings.Contains(err.Error(), encodingErr.Error()) { + t.Fatalf("RPC failed with error: %v, want: %v", err, encodingErr) + } + + // Configure the codec on the server to not return errors anymore and expect + // the RPC to succeed. + ec = &errProtoCodec{name: t.Name()} + backend2 := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := backend2.Start([]grpc.ServerOption{grpc.ForceServerCodecV2(ec)}, grpc.WithTransportCredentials(insecure.NewCredentials())); err != nil { + t.Fatal(err) + } + defer backend2.Stop() + + if _, err := backend2.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("RPC failed with error: %v", err) + } +} + +// Tests the case where decoding fails on the server. Verifies that there is +// no panic and that the decoding error is propagated to the client. +func (s) TestDecodeDoesntPanicOnServer(t *testing.T) { + // Create a codec that errors when decoding messages. + decodingErr := errors.New("decoding failed") + ec := &errProtoCodec{name: t.Name(), decodingErr: decodingErr} + + // Start a server with the above codec and a channel to the server. + backend1 := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := backend1.Start([]grpc.ServerOption{grpc.ForceServerCodecV2(ec)}, grpc.WithTransportCredentials(insecure.NewCredentials())); err != nil { + t.Fatal(err) + } + defer backend1.Stop() + + // Make an RPC and expect it to fail. Since we do not specify any codec + // here, the proto codec will get automatically used. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := backend1.Client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil || !strings.Contains(err.Error(), decodingErr.Error()) || !strings.Contains(err.Error(), "grpc: failed to unmarshal the received message") { + t.Fatalf("RPC failed with error: %v, want: %v", err, decodingErr) + } + + // Configure the codec on the server to not return errors anymore and expect + // the RPC to succeed. + ec = &errProtoCodec{name: t.Name()} + backend2 := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := backend2.Start([]grpc.ServerOption{grpc.ForceServerCodecV2(ec)}, grpc.WithTransportCredentials(insecure.NewCredentials())); err != nil { + t.Fatal(err) + } + defer backend2.Stop() + + if _, err := backend2.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("RPC failed with error: %v", err) + } +} + +// Tests the case where encoding fails on the client . Verifies that there is +// no panic and that the encoding error is propagated to the RPC caller. +func (s) TestEncodeDoesntPanicOnClient(t *testing.T) { + // Start a server and since we do not specify any codec here, the proto + // codec will get automatically used. + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + + // Create a codec that errors when encoding messages. + encodingErr := errors.New("encoding failed") + ec := &errProtoCodec{name: t.Name(), encodingErr: encodingErr} + + // Create a channel to the above server. + cc, err := grpc.NewClient(backend.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial test backend at %q: %v", backend.Address, err) + } + defer cc.Close() + + // Make an RPC with the erroring codec and expect it to fail. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + _, err = client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodecV2(ec)) + if err == nil || !strings.Contains(err.Error(), encodingErr.Error()) { + t.Fatalf("RPC failed with error: %v, want: %v", err, encodingErr) + } + + // Configure the codec on the client to not return errors anymore and expect + // the RPC to succeed. + ec.encodingErr = nil + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodecV2(ec)); err != nil { + t.Fatalf("RPC failed with error: %v", err) + } +} + +// Tests the case where decoding fails on the server. Verifies that there is +// no panic and that the decoding error is propagated to the RPC caller. +func (s) TestDecodeDoesntPanicOnClient(t *testing.T) { + // Start a server and since we do not specify any codec here, the proto + // codec will get automatically used. + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + + // Create a codec that errors when decoding messages. + decodingErr := errors.New("decoding failed") + ec := &errProtoCodec{name: t.Name(), decodingErr: decodingErr} + + // Create a channel to the above server. + cc, err := grpc.NewClient(backend.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial test backend at %q: %v", backend.Address, err) + } + defer cc.Close() + + // Make an RPC with the erroring codec and expect it to fail. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + _, err = client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodecV2(ec)) + if err == nil || !strings.Contains(err.Error(), decodingErr.Error()) { + t.Fatalf("RPC failed with error: %v, want: %v", err, decodingErr) + } + + // Configure the codec on the client to not return errors anymore and expect + // the RPC to succeed. + ec.decodingErr = nil + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodecV2(ec)); err != nil { + t.Fatalf("RPC failed with error: %v", err) + } +} + +// countingProtoCodec wraps the proto codec and counts the number of times +// Marshal and Unmarshal are called. +type countingProtoCodec struct { + name string + + // The following fields are accessed atomically. + marshalCount int32 + unmarshalCount int32 +} + +func (p *countingProtoCodec) Marshal(v any) (mem.BufferSlice, error) { + atomic.AddInt32(&p.marshalCount, 1) + return encoding.GetCodecV2(proto.Name).Marshal(v) +} + +func (p *countingProtoCodec) Unmarshal(data mem.BufferSlice, v any) error { + atomic.AddInt32(&p.unmarshalCount, 1) + return encoding.GetCodecV2(proto.Name).Unmarshal(data, v) +} + +func (p *countingProtoCodec) Name() string { + return p.name +} + +// Tests the case where ForceServerCodec option is used on the server. Verifies +// that encoding and decoding happen once per RPC. +func (s) TestForceServerCodec(t *testing.T) { + // Create a server with the counting proto codec. + codec := &countingProtoCodec{name: t.Name()} + backend := stubserver.StartTestService(t, nil, grpc.ForceServerCodecV2(codec)) + defer backend.Stop() + + // Create a channel to the above server. + cc, err := grpc.NewClient(backend.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial test backend at %q: %v", backend.Address, err) + } + defer cc.Close() + + // Make an RPC and expect it to fail. Since we do not specify any codec + // here, the proto codec will get automatically used. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) + } + + unmarshalCount := atomic.LoadInt32(&codec.unmarshalCount) + const wantUnmarshalCount = 1 + if unmarshalCount != wantUnmarshalCount { + t.Fatalf("Unmarshal Count = %d; want %d", unmarshalCount, wantUnmarshalCount) + } + marshalCount := atomic.LoadInt32(&codec.marshalCount) + const wantMarshalCount = 1 + if marshalCount != wantMarshalCount { + t.Fatalf("MarshalCount = %d; want %d", marshalCount, wantMarshalCount) + } +} + +// renameProtoCodec wraps the proto codec and allows customizing the Name(). +type renameProtoCodec struct { + encoding.CodecV2 + name string +} + +func (r *renameProtoCodec) Name() string { return r.name } + +// TestForceCodecName confirms that the ForceCodec call option sets the subtype +// in the content-type header according to the Name() of the codec provided. +// Verifies that the name is converted to lowercase before transmitting. +func (s) TestForceCodecName(t *testing.T) { + wantContentTypeCh := make(chan []string, 1) + defer close(wantContentTypeCh) + + // Create a test service backend that pushes the received content-type on a + // channel for the test to inspect. + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Errorf(codes.Internal, "no metadata in context") + } + if got, want := md["content-type"], <-wantContentTypeCh; !cmp.Equal(got, want) { + return nil, status.Errorf(codes.Internal, "got content-type=%q; want [%q]", got, want) + } + return &testpb.Empty{}, nil + }, + } + // Since we don't specify a codec as a server option, it will end up + // automatically using the proto codec. + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Force the use of the custom codec on the client with the ForceCodec call + // option. Confirm the name is converted to lowercase before transmitting. + codec := &renameProtoCodec{CodecV2: encoding.GetCodecV2(proto.Name), name: t.Name()} + wantContentTypeCh <- []string{fmt.Sprintf("application/grpc+%s", strings.ToLower(t.Name()))} + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodecV2(codec)); err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) + } +} + +// Tests the case where the client uses a codec (one that uses proto encoding +// but uses a different content-subtype name) that the server does not support. +// Verifies that the server falls back to the proto codec and that the client +// can successfully make RPCs. +// +// TODO(https://github.com/grpc/grpc-go/issues/1824): Once we add an environment +// variable to change the behavior on the server to reject unsupported codecs, +// we should modify this test to verify that the RPC fails in that case. +func (s) TestUnsupportedCodecOnServer(t *testing.T) { + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + + cc, err := grpc.NewClient(backend.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial test backend at %q: %v", backend.Address, err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + codec := &renameProtoCodec{CodecV2: encoding.GetCodecV2(proto.Name), name: t.Name()} + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodecV2(codec)); err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) + } +} diff --git a/encoding/encoding_v2.go b/encoding/encoding_v2.go new file mode 100644 index 0000000..074c5e2 --- /dev/null +++ b/encoding/encoding_v2.go @@ -0,0 +1,81 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package encoding + +import ( + "strings" + + "google.golang.org/grpc/mem" +) + +// CodecV2 defines the interface gRPC uses to encode and decode messages. Note +// that implementations of this interface must be thread safe; a CodecV2's +// methods can be called from concurrent goroutines. +type CodecV2 interface { + // Marshal returns the wire format of v. The buffers in the returned + // [mem.BufferSlice] must have at least one reference each, which will be freed + // by gRPC when they are no longer needed. + Marshal(v any) (out mem.BufferSlice, err error) + // Unmarshal parses the wire format into v. Note that data will be freed as soon + // as this function returns. If the codec wishes to guarantee access to the data + // after this function, it must take its own reference that it frees when it is + // no longer needed. + Unmarshal(data mem.BufferSlice, v any) error + // Name returns the name of the Codec implementation. The returned string + // will be used as part of content type in transmission. The result must be + // static; the result cannot change between calls. + Name() string +} + +// RegisterCodecV2 registers the provided CodecV2 for use with all gRPC clients and +// servers. +// +// The CodecV2 will be stored and looked up by result of its Name() method, which +// should match the content-subtype of the encoding handled by the CodecV2. This +// is case-insensitive, and is stored and looked up as lowercase. If the +// result of calling Name() is an empty string, RegisterCodecV2 will panic. See +// Content-Type on +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +// +// If both a Codec and CodecV2 are registered with the same name, the CodecV2 +// will be used. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple Codecs are +// registered with the same name, the one registered last will take effect. +func RegisterCodecV2(codec CodecV2) { + if codec == nil { + panic("cannot register a nil CodecV2") + } + if codec.Name() == "" { + panic("cannot register CodecV2 with empty string result for Name()") + } + contentSubtype := strings.ToLower(codec.Name()) + registeredCodecs[contentSubtype] = codec +} + +// GetCodecV2 gets a registered CodecV2 by content-subtype, or nil if no CodecV2 is +// registered for the content-subtype. +// +// The content-subtype is expected to be lowercase. +func GetCodecV2(contentSubtype string) CodecV2 { + c, _ := registeredCodecs[contentSubtype].(CodecV2) + return c +} diff --git a/encoding/gzip/gzip.go b/encoding/gzip/gzip.go new file mode 100644 index 0000000..65908d9 --- /dev/null +++ b/encoding/gzip/gzip.go @@ -0,0 +1,124 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package gzip implements and registers the gzip compressor +// during the initialization. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +package gzip + +import ( + "compress/gzip" + "fmt" + "io" + "sync" + + "google.golang.org/grpc/encoding" +) + +// Name is the name registered for the gzip compressor. +const Name = "gzip" + +func init() { + c := &compressor{} + c.poolCompressor.New = func() any { + return &writer{Writer: gzip.NewWriter(io.Discard), pool: &c.poolCompressor} + } + encoding.RegisterCompressor(c) +} + +type writer struct { + *gzip.Writer + pool *sync.Pool +} + +// SetLevel updates the registered gzip compressor to use the compression level specified (gzip.HuffmanOnly is not supported). +// NOTE: this function must only be called during initialization time (i.e. in an init() function), +// and is not thread-safe. +// +// The error returned will be nil if the specified level is valid. +func SetLevel(level int) error { + if level < gzip.DefaultCompression || level > gzip.BestCompression { + return fmt.Errorf("grpc: invalid gzip compression level: %d", level) + } + c := encoding.GetCompressor(Name).(*compressor) + c.poolCompressor.New = func() any { + w, err := gzip.NewWriterLevel(io.Discard, level) + if err != nil { + panic(err) + } + return &writer{Writer: w, pool: &c.poolCompressor} + } + return nil +} + +func (c *compressor) Compress(w io.Writer) (io.WriteCloser, error) { + z := c.poolCompressor.Get().(*writer) + z.Writer.Reset(w) + return z, nil +} + +func (z *writer) Close() error { + defer z.pool.Put(z) + return z.Writer.Close() +} + +var _ io.Closer = &reader{} + +type reader struct { + *gzip.Reader + pool *sync.Pool +} + +func (c *compressor) Decompress(r io.Reader) (io.Reader, error) { + z, inPool := c.poolDecompressor.Get().(*reader) + if !inPool { + newZ, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + return &reader{Reader: newZ, pool: &c.poolDecompressor}, nil + } + if err := z.Reset(r); err != nil { + c.poolDecompressor.Put(z) + return nil, err + } + return z, nil +} + +func (r *reader) Read(p []byte) (n int, err error) { + n, err = r.Reader.Read(p) + return n, err +} + +func (r *reader) Close() error { + defer r.pool.Put(r) + return r.Reader.Close() +} + +func (c *compressor) Name() string { + return Name +} + +type compressor struct { + poolCompressor sync.Pool + poolDecompressor sync.Pool +} diff --git a/encoding/internal/internal.go b/encoding/internal/internal.go new file mode 100644 index 0000000..ee9acb4 --- /dev/null +++ b/encoding/internal/internal.go @@ -0,0 +1,28 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains code internal to the encoding package. +package internal + +// RegisterCompressorForTesting registers a compressor in the global compressor +// registry. It returns a cleanup function that should be called at the end +// of the test to unregister the compressor. +// +// This prevents compressors registered in one test from appearing in the +// encoding headers of subsequent tests. +var RegisterCompressorForTesting any // func RegisterCompressor(c Compressor) func() diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go new file mode 100644 index 0000000..1ab874c7 --- /dev/null +++ b/encoding/proto/proto.go @@ -0,0 +1,112 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package proto defines the protobuf codec. Importing this package will +// register the codec. +package proto + +import ( + "fmt" + + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/mem" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/protoadapt" +) + +// Name is the name registered for the proto compressor. +const Name = "proto" + +func init() { + encoding.RegisterCodecV2(&codecV2{}) +} + +// codec is a CodecV2 implementation with protobuf. It is the default codec for +// gRPC. +type codecV2 struct{} + +func (c *codecV2) Marshal(v any) (data mem.BufferSlice, err error) { + vv := messageV2Of(v) + if vv == nil { + return nil, fmt.Errorf("proto: failed to marshal, message is %T, want proto.Message", v) + } + + // Important: if we remove this Size call then we cannot use + // UseCachedSize in MarshalOptions below. + size := proto.Size(vv) + + // MarshalOptions with UseCachedSize allows reusing the result from the + // previous Size call. This is safe here because: + // + // 1. We just computed the size. + // 2. We assume the message is not being mutated concurrently. + // + // Important: If the proto.Size call above is removed, using UseCachedSize + // becomes unsafe and may lead to incorrect marshaling. + // + // For more details, see the doc of UseCachedSize: + // https://pkg.go.dev/google.golang.org/protobuf/proto#MarshalOptions + marshalOptions := proto.MarshalOptions{UseCachedSize: true} + + if mem.IsBelowBufferPoolingThreshold(size) { + buf, err := marshalOptions.Marshal(vv) + if err != nil { + return nil, err + } + data = append(data, mem.SliceBuffer(buf)) + } else { + pool := mem.DefaultBufferPool() + buf := pool.Get(size) + if _, err := marshalOptions.MarshalAppend((*buf)[:0], vv); err != nil { + pool.Put(buf) + return nil, err + } + data = append(data, mem.NewBuffer(buf, pool)) + } + + return data, nil +} + +func (c *codecV2) Unmarshal(data mem.BufferSlice, v any) (err error) { + vv := messageV2Of(v) + if vv == nil { + return fmt.Errorf("failed to unmarshal, message is %T, want proto.Message", v) + } + + buf := data.MaterializeToBuffer(mem.DefaultBufferPool()) + defer buf.Free() + // TODO: Upgrade proto.Unmarshal to support mem.BufferSlice. Right now, it's not + // really possible without a major overhaul of the proto package, but the + // vtprotobuf library may be able to support this. + return proto.Unmarshal(buf.ReadOnlyData(), vv) +} + +func messageV2Of(v any) proto.Message { + switch v := v.(type) { + case protoadapt.MessageV1: + return protoadapt.MessageV2Of(v) + case protoadapt.MessageV2: + return v + } + + return nil +} + +func (c *codecV2) Name() string { + return Name +} diff --git a/encoding/proto/proto_benchmark_test.go b/encoding/proto/proto_benchmark_test.go new file mode 100644 index 0000000..58007dc --- /dev/null +++ b/encoding/proto/proto_benchmark_test.go @@ -0,0 +1,99 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package proto + +import ( + "fmt" + "testing" + + "google.golang.org/grpc/encoding" + "google.golang.org/protobuf/proto" + + pb "google.golang.org/grpc/test/codec_perf" +) + +func setupBenchmarkProtoCodecInputs(payloadBaseSize uint32) []proto.Message { + payloadBase := make([]byte, payloadBaseSize) + // arbitrary byte slices + payloadSuffixes := [][]byte{ + []byte("one"), + []byte("two"), + []byte("three"), + []byte("four"), + []byte("five"), + } + protoStructs := make([]proto.Message, 0) + + for _, p := range payloadSuffixes { + ps := &pb.Buffer{} + ps.Body = append(payloadBase, p...) + protoStructs = append(protoStructs, ps) + } + + return protoStructs +} + +// The possible use of certain protobuf APIs like the proto.Buffer API potentially involves caching +// on our side. This can add checks around memory allocations and possible contention. +// Example run: go test -v -run=^$ -bench=BenchmarkProtoCodec -benchmem +func BenchmarkProtoCodec(b *testing.B) { + // range of message sizes + payloadBaseSizes := make([]uint32, 0) + for i := uint32(0); i <= 12; i += 4 { + payloadBaseSizes = append(payloadBaseSizes, 1<&1 &>/dev/null + if [ $? -eq 0 ]; then + pass "server started" + return + fi + sleep 1 + done + fail "cannot determine if server started" +} + +declare -A EXPECTED_SERVER_OUTPUT=( + ["helloworld"]="Received: world" + ["route_guide"]="" + ["features/authentication"]="server starting on port 50051..." + ["features/authz"]="unary echoing message \"hello world\"" + ["features/cancellation"]="server: error receiving from stream: rpc error: code = Canceled desc = context canceled" + ["features/compression"]="UnaryEcho called with message \"compress\"" + ["features/customloadbalancer"]="serving on localhost:50051" + ["features/deadline"]="" + ["features/dualstack"]="serving on \[::\]:50051" + ["features/encryption/TLS"]="" + ["features/error_details"]="" + ["features/error_handling"]="" + ["features/flow_control"]="Stream ended successfully." + ["features/interceptor"]="unary echoing message \"hello world\"" + ["features/load_balancing"]="serving on :50051" + ["features/metadata"]="message:\"this is examples/metadata\", sending echo" + ["features/metadata_interceptor"]="key1 from metadata: " + ["features/multiplex"]=":50051" + ["features/name_resolving"]="serving on localhost:50051" + ["features/orca"]="Server listening" + ["features/retry"]="request succeeded count: 4" + ["features/unix_abstract"]="serving on @abstract-unix-socket" + ["features/advancedtls"]="" + ["features/gracefulstop"]="Server stopped gracefully." +) + +declare -A EXPECTED_CLIENT_OUTPUT=( + ["helloworld"]="Greeting: Hello world" + ["route_guide"]="Feature: name: \"\", point:(416851321, -742674555)" + ["features/authentication"]="UnaryEcho: hello world" + ["features/authz"]="UnaryEcho: hello world" + ["features/cancellation"]="cancelling context" + ["features/compression"]="UnaryEcho call returned \"compress\", " + ["features/customloadbalancer"]="Successful multiple iterations of 1:2 ratio" + ["features/deadline"]="wanted = DeadlineExceeded, got = DeadlineExceeded" + ["features/dualstack"]="Successful multiple iterations of 1:1:1 ratio" + ["features/encryption/TLS"]="UnaryEcho: hello world" + ["features/error_details"]="Greeting: Hello world" + ["features/error_handling"]="Received error" + ["features/flow_control"]="Stream ended successfully." + ["features/interceptor"]="UnaryEcho: hello world" + ["features/load_balancing"]="calling helloworld.Greeter/SayHello with pick_first" + ["features/metadata"]="this is examples/metadata" + ["features/metadata_interceptor"]="BidiStreaming Echo: hello world" + ["features/multiplex"]="Greeting: Hello multiplex" + ["features/name_resolving"]="calling helloworld.Greeter/SayHello to \"example:///resolver.example.grpc.io\"" + ["features/orca"]="Per-call load report received: map\[db_queries:10\]" + ["features/retry"]="UnaryEcho reply: message:\"Try and Success\"" + ["features/unix_abstract"]="calling echo.Echo/UnaryEcho to unix-abstract:abstract-unix-socket" + ["features/advancedtls"]="" + ["features/gracefulstop"]="Successful unary requests processed by server and made by client are same." +) + +cd ./examples + +for example in ${EXAMPLES[@]}; do + echo "$(tput setaf 4) testing: ${example} $(tput sgr 0)" + + # Build server + if ! go build -o /dev/null ./${example}/*server/*.go; then + fail "failed to build server" + else + pass "successfully built server" + fi + + # Start server + SERVER_LOG="$(mktemp)" + server_args=${SERVER_ARGS[$example]:-${SERVER_ARGS["default"]}} + go run ./$example/*server/*.go $server_args &> $SERVER_LOG & + + wait_for_server $example + + # Build client + if ! go build -o /dev/null ./${example}/*client/*.go; then + fail "failed to build client" + else + pass "successfully built client" + fi + + # Start client + CLIENT_LOG="$(mktemp)" + client_args=${CLIENT_ARGS[$example]:-${CLIENT_ARGS["default"]}} + if ! timeout 20 go run ${example}/*client/*.go $client_args &> $CLIENT_LOG; then + fail "client failed to communicate with server + got server log: + $(cat $SERVER_LOG) + got client log: + $(cat $CLIENT_LOG) + " + else + pass "client successfully communicated with server" + fi + + # Check server log for expected output if expecting an + # output + if [ -n "${EXPECTED_SERVER_OUTPUT[$example]}" ]; then + found=false + + # Poll for up to 10 seconds. + for i in {1..10}; do + if grep -q "${EXPECTED_SERVER_OUTPUT[$example]}" "$SERVER_LOG"; then + found=true + break + fi + sleep 1 + done + + if [ "$found" = "false" ]; then + fail "server log missing output: ${EXPECTED_SERVER_OUTPUT[$example]} + got server log: + $(cat $SERVER_LOG) + got client log: + $(cat $CLIENT_LOG) + " + else + pass "server log contains expected output: ${EXPECTED_SERVER_OUTPUT[$example]}" + fi + fi + + # Check client log for expected output if expecting an + # output + if [ -n "${EXPECTED_CLIENT_OUTPUT[$example]}" ]; then + if ! grep -q "${EXPECTED_CLIENT_OUTPUT[$example]}" $CLIENT_LOG; then + fail "client log missing output: ${EXPECTED_CLIENT_OUTPUT[$example]} + got server log: + $(cat $SERVER_LOG) + got client log: + $(cat $CLIENT_LOG) + " + else + pass "client log contains expected output: ${EXPECTED_CLIENT_OUTPUT[$example]}" + fi + fi + clean + echo "" +done diff --git a/examples/features/advancedtls/README.md b/examples/features/advancedtls/README.md new file mode 100644 index 0000000..441fcaf --- /dev/null +++ b/examples/features/advancedtls/README.md @@ -0,0 +1,28 @@ +# gRPC Advanced Security Examples +This repo contains example code for different security configurations for grpc-go using `advancedtls`. + +The servers run a basic echo server with the following setups: +* Port 8885: A server with a good certificate using certificate providers and crl providers. +* Port 8884: A server with a revoked certificate using certificate providers and crl providers. +* Port 8883: A server running using InsecureCredentials. + +The clients are designed to call these servers with varying configurations of credentials and revocation configurations. +* mTLS with certificate providers and CRLs +* mTLS with custom verification +* mTLS with credentials from credentials.NewTLS (directly using the tls.Config) +* Insecure Credentials + +## Building and Running +``` +# Run the server +$ go run server/main.go -credentials_directory $(pwd)/creds +# Run the clients from the `grpc-go/examples/features/advancedtls` directory +$ go run client/main.go -credentials_directory $(pwd)/creds +``` + +Stop the servers with ctrl-c or by killing the process. + +## Developer Note - Generate the credentials used in the examples +The credentials used for these examples were generated by running the `examples/features/advancedtls/generate.sh` script. + +If the credentials need to be re-generated, run `./generate.sh` from `/path/to/grpc-go/examples/features/advancedtls` to re-create the `creds` directory containing the certificates and CRLs needed for these examples. diff --git a/examples/features/advancedtls/client/main.go b/examples/features/advancedtls/client/main.go new file mode 100644 index 0000000..76688f4 --- /dev/null +++ b/examples/features/advancedtls/client/main.go @@ -0,0 +1,306 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an example client demonstrating use of advancedtls, to set +// up a secure gRPC client connection with various TLS authentication methods. +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "flag" + "fmt" + "os" + "path/filepath" + "time" + + pb "google.golang.org/grpc/examples/features/proto/echo" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/credentials/tls/certprovider/pemfile" + "google.golang.org/grpc/security/advancedtls" +) + +const credRefreshInterval = 1 * time.Minute +const serverAddr = "localhost" +const goodServerPort string = "50051" +const revokedServerPort string = "50053" +const insecurePort string = "50054" +const message string = "Hello" + +// -- TLS -- + +func makeRootProvider(credsDirectory string) certprovider.Provider { + rootOptions := pemfile.Options{ + RootFile: filepath.Join(credsDirectory, "ca_cert.pem"), + RefreshDuration: credRefreshInterval, + } + rootProvider, err := pemfile.NewProvider(rootOptions) + if err != nil { + fmt.Printf("Error %v\n", err) + os.Exit(1) + } + return rootProvider +} + +func makeIdentityProvider(revoked bool, credsDirectory string) certprovider.Provider { + var certFile string + if revoked { + certFile = filepath.Join(credsDirectory, "client_cert_revoked.pem") + } else { + certFile = filepath.Join(credsDirectory, "client_cert.pem") + } + identityOptions := pemfile.Options{ + CertFile: certFile, + KeyFile: filepath.Join(credsDirectory, "client_key.pem"), + RefreshDuration: credRefreshInterval, + } + identityProvider, err := pemfile.NewProvider(identityOptions) + if err != nil { + fmt.Printf("Error %v\n", err) + os.Exit(1) + } + return identityProvider +} + +func runClientWithProviders(rootProvider certprovider.Provider, identityProvider certprovider.Provider, crlProvider advancedtls.CRLProvider, port string, shouldFail bool) { + options := &advancedtls.Options{ + // Setup the certificates to be used + IdentityOptions: advancedtls.IdentityCertificateOptions{ + IdentityProvider: identityProvider, + }, + // Setup the roots to be used + RootOptions: advancedtls.RootCertificateOptions{ + RootProvider: rootProvider, + }, + // Tell the client to verify the server cert + VerificationType: advancedtls.CertVerification, + } + + // Configure revocation and CRLs + options.RevocationOptions = &advancedtls.RevocationOptions{ + CRLProvider: crlProvider, + } + + clientTLSCreds, err := advancedtls.NewClientCreds(options) + + if err != nil { + fmt.Printf("Error %v\n", err) + os.Exit(1) + } + fullServerAddr := serverAddr + ":" + port + runWithCredentials(clientTLSCreds, fullServerAddr, !shouldFail) +} + +func tlsWithCRLsToGoodServer(credsDirectory string) { + rootProvider := makeRootProvider(credsDirectory) + defer rootProvider.Close() + identityProvider := makeIdentityProvider(false, credsDirectory) + defer identityProvider.Close() + crlProvider := makeCRLProvider(credsDirectory) + defer crlProvider.Close() + + runClientWithProviders(rootProvider, identityProvider, crlProvider, goodServerPort, false) +} + +func tlsWithCRLsToRevokedServer(credsDirectory string) { + rootProvider := makeRootProvider(credsDirectory) + defer rootProvider.Close() + identityProvider := makeIdentityProvider(false, credsDirectory) + defer identityProvider.Close() + crlProvider := makeCRLProvider(credsDirectory) + defer crlProvider.Close() + + runClientWithProviders(rootProvider, identityProvider, crlProvider, revokedServerPort, true) +} + +func tlsWithCRLs(credsDirectory string) { + tlsWithCRLsToGoodServer(credsDirectory) + tlsWithCRLsToRevokedServer(credsDirectory) +} + +func makeCRLProvider(crlDirectory string) *advancedtls.FileWatcherCRLProvider { + options := advancedtls.FileWatcherOptions{ + CRLDirectory: crlDirectory, + } + provider, err := advancedtls.NewFileWatcherCRLProvider(options) + if err != nil { + fmt.Printf("Error making CRL Provider: %v\nExiting...", err) + os.Exit(1) + } + return provider +} + +// --- Custom Verification --- +func customVerificationSucceed(info *advancedtls.HandshakeVerificationInfo) (*advancedtls.PostHandshakeVerificationResults, error) { + // Looks at info for what you care about as the custom verification implementer + if info.ServerName != "localhost:50051" { + return nil, fmt.Errorf("expected servername of localhost:50051, got %v", info.ServerName) + } + return &advancedtls.PostHandshakeVerificationResults{}, nil +} + +func customVerificationFail(info *advancedtls.HandshakeVerificationInfo) (*advancedtls.PostHandshakeVerificationResults, error) { + // Looks at info for what you care about as the custom verification implementer + if info.ServerName != "ExampleDesignedToFail" { + return nil, fmt.Errorf("expected servername of ExampleDesignedToFail, got %v", info.ServerName) + } + return &advancedtls.PostHandshakeVerificationResults{}, nil +} + +func customVerification(credsDirectory string) { + runClientWithCustomVerification(credsDirectory, goodServerPort) + +} + +func runClientWithCustomVerification(credsDirectory string, port string) { + rootProvider := makeRootProvider(credsDirectory) + defer rootProvider.Close() + identityProvider := makeIdentityProvider(false, credsDirectory) + defer identityProvider.Close() + fullServerAddr := serverAddr + ":" + port + { + // Run with the custom verification func that will succeed + options := &advancedtls.Options{ + // Setup the certificates to be used + IdentityOptions: advancedtls.IdentityCertificateOptions{ + IdentityProvider: identityProvider, + }, + // Setup the roots to be used + RootOptions: advancedtls.RootCertificateOptions{ + RootProvider: rootProvider, + }, + // Tell the client to verify the server cert + VerificationType: advancedtls.CertVerification, + AdditionalPeerVerification: customVerificationSucceed, + } + + clientTLSCreds, err := advancedtls.NewClientCreds(options) + + if err != nil { + fmt.Printf("Error %v\n", err) + os.Exit(1) + } + runWithCredentials(clientTLSCreds, fullServerAddr, true) + } + { + // Run with the custom verification func that will fail + options := &advancedtls.Options{ + // Setup the certificates to be used + IdentityOptions: advancedtls.IdentityCertificateOptions{ + IdentityProvider: identityProvider, + }, + // Setup the roots to be used + RootOptions: advancedtls.RootCertificateOptions{ + RootProvider: rootProvider, + }, + // Tell the client to verify the server cert + VerificationType: advancedtls.CertVerification, + AdditionalPeerVerification: customVerificationFail, + } + + clientTLSCreds, err := advancedtls.NewClientCreds(options) + + if err != nil { + fmt.Printf("Error %v\n", err) + os.Exit(1) + } + runWithCredentials(clientTLSCreds, fullServerAddr, false) + } +} + +// -- credentials.NewTLS example -- +func credentialsNewTLSExample(credsDirectory string) { + cert, err := tls.LoadX509KeyPair(filepath.Join(credsDirectory, "client_cert.pem"), filepath.Join(credsDirectory, "client_key.pem")) + if err != nil { + os.Exit(1) + } + rootPem, err := os.ReadFile(filepath.Join(credsDirectory, "ca_cert.pem")) + if err != nil { + os.Exit(1) + } + root := x509.NewCertPool() + if !root.AppendCertsFromPEM(rootPem) { + os.Exit(1) + } + + config := &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: root, + } + + // Directly create credentials from a tls.Config. + creds := credentials.NewTLS(config) + port := goodServerPort + fullServerAddr := serverAddr + ":" + port + runWithCredentials(creds, fullServerAddr, true) + +} + +// -- Insecure -- +func insecureCredentialsExample() { + creds := insecure.NewCredentials() + port := insecurePort + fullServerAddr := serverAddr + ":" + port + runWithCredentials(creds, fullServerAddr, true) +} + +// -- Main and Runner -- + +// All of these examples differ in how they configure the +// credentials.TransportCredentials object. Once we have that, actually making +// the calls with gRPC is the same. +func runWithCredentials(creds credentials.TransportCredentials, fullServerAddr string, shouldSucceed bool) { + conn, err := grpc.NewClient(fullServerAddr, grpc.WithTransportCredentials(creds)) + if err != nil { + fmt.Printf("Error during grpc.NewClient %v\n", err) + os.Exit(1) + } + defer conn.Close() + client := pb.NewEchoClient(conn) + req := &pb.EchoRequest{ + Message: message, + } + context, cancel := context.WithTimeout(context.Background(), 10*time.Second) + resp, err := client.UnaryEcho(context, req) + defer cancel() + + if shouldSucceed && err != nil { + fmt.Printf("Error during client.UnaryEcho %v\n", err) + } else if !shouldSucceed && err == nil { + fmt.Printf("Should have failed but didn't, got response: %v\n", resp) + } + +} +func main() { + credsDirectory := flag.String("credentials_directory", "", "Path to the creds directory of this example repo") + flag.Parse() + + if *credsDirectory == "" { + fmt.Println("Must set credentials_directory argument to this repo's creds directory") + os.Exit(1) + } + tlsWithCRLs(*credsDirectory) + customVerification(*credsDirectory) + credentialsNewTLSExample(*credsDirectory) + insecureCredentialsExample() +} diff --git a/examples/features/advancedtls/creds/ca_cert.pem b/examples/features/advancedtls/creds/ca_cert.pem new file mode 100644 index 0000000..060473b --- /dev/null +++ b/examples/features/advancedtls/creds/ca_cert.pem @@ -0,0 +1,35 @@ +-----BEGIN CERTIFICATE----- +MIIGJTCCBA2gAwIBAgIUQIWlFBWaWCYUunTANnlB4XZeFeUwDQYJKoZIhvcNAQEL +BQAwgaExCzAJBgNVBAYTAlVTMRAwDgYDVQQIDAdHZW9yZ2lhMRAwDgYDVQQHDAdB +dGxhbnRhMRAwDgYDVQQKDAdUZXN0IENBMRwwGgYDVQQLDBNUZXN0IENBIE9yZ2Fu +emF0aW9uMR0wGwYDVQQDDBRUZXN0IENBIE9yZ2FuaXphdGlvbjEfMB0GCSqGSIb3 +DQEJARYQdGVzdEBleGFtcGxlLmNvbTAeFw0yNDA4MDgxODA2MjFaFw0zNDA4MDYx +ODA2MjFaMIGhMQswCQYDVQQGEwJVUzEQMA4GA1UECAwHR2VvcmdpYTEQMA4GA1UE +BwwHQXRsYW50YTEQMA4GA1UECgwHVGVzdCBDQTEcMBoGA1UECwwTVGVzdCBDQSBP +cmdhbnphdGlvbjEdMBsGA1UEAwwUVGVzdCBDQSBPcmdhbml6YXRpb24xHzAdBgkq +hkiG9w0BCQEWEHRlc3RAZXhhbXBsZS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4IC +DwAwggIKAoICAQCs+Px6CMv0x3dmmK9PEdIq95J0JQ7Y6NojD93oosZxqi0QLzxU +LiRamNOvoMSBgbUl1GtC8xcQQ/YiaBS0A+tc+7NxZ6SJXIa/i7tbJcebPY5bnbHc +ILXPOt4FLEgcBqyv9UquPstkYytJje4J0N+G/nqfKsh+mo+emnKFSy1QS7NoPr/T +fDKemnf2DBk0HOiBnIr2gh3gqThXqUt/dZlDNJALeJU+7IpLDThOM3sf1QOOkSF9 +O1IM1YJt3B9GeTDwPnqKbXVOKf23eBi51QyvWde1ZscTRh0p9HX4VRCYOGfkQnWw +0d3BpFg/a6rGVNLSPBGE2H6O68L4K1bBDV0CvdTjVD8/vgrLm/7NAOlg/58TKIaq +NxFalXeLmdKr0c5d4JZEbbPgg26O8Fsq769s8Jc1dtnAiFwB2opIOvOLZkNwzPG8 +EjAET9HmjWHHzZ/OmswWamqywPukW8jdLH5f4RsuGpGHsUvs/53fUUeAdAlceJ+1 +KuLNuk7ULRU59TRbppt6m/Ws81bWJLQtw79BdyDNgJ4q7Vyl5tCuC2mZzDqOb/uK +py5Gx6Upoy0klAsMjvUBiw3cpkVCl1/RCSx2HmV85itS20QCiFcT+KeJ3xSbIc+P +ScNvinnbwtRhENQY+fy5MAfy9kvEdlYlsM2yp3l1B+Z4My6w8e2CcQO4RwIDAQAB +o1MwUTAdBgNVHQ4EFgQUUHbDXGsS4ZIPKPjyQ6aAwpzoVtYwHwYDVR0jBBgwFoAU +UHbDXGsS4ZIPKPjyQ6aAwpzoVtYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B +AQsFAAOCAgEAe7/P+MvYYM8gBN0AHQtmG4SaPpiE+Wi8TU4KSU6n+gzM347bPUnH +TbxMs1gYkiQ4IsYnU/uY2L+lCVvBpd66aIM9dJ5WGHS1RyRjRCUwZNEu9UIizemp +JSWu6hql96ib1AFAnXbjC8uNFG8OK+aF/NhChnu1pWKLLAMgBXhG8e5z7wNjQHqB +D/FOOBEn6ljR6MhBsRyPZxz/tqEt5hGflgeQnZXC2dzmQQDRfEWq9jjDgIVGpOjZ +VNYnua0GxdJGmRtExPHCf4bmClGf9uW1GK1ViCnj6Qlsvln0eOgNkI/m/VxjuSvE +NDUF+jWK7z+O0nagDSDTIGUU/enSFpdAHrUQuyqKS1S8WHhf4AIi0DNkUhHVojk6 +40nUPxVHl8R7wPXu3K7sTCfNJFJsqY8+oMhS3lk05voDuPJAgWnvG3wnE5rDWi/Q +R7CLMnnYQ7oIyJ9mE8ZLDWd9Udov+n/y5VkFVh8WFbu9Vidvlpy9xXQKaJVP4EHa +K0nLHGSw1zRrB+zx0Ep7ow/zGDxT8kCcKMQ/Uonv6kRxpi90oBdvNNXzsTkQ+FZ3 +168nBjWf+X6XX/HalbRiKmgww6SqG+hoVXP0cFw3vJwgESeXJHbxCcu1mJdzSbr3 +HzRkGKgTKIBV0z2AMG3cLCW/DO4+45GKi/DYibz0GjvFkXT8cGhN5vM= +-----END CERTIFICATE----- diff --git a/examples/features/advancedtls/creds/ca_key.pem b/examples/features/advancedtls/creds/ca_key.pem new file mode 100644 index 0000000..b42b138 --- /dev/null +++ b/examples/features/advancedtls/creds/ca_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQCs+Px6CMv0x3dm +mK9PEdIq95J0JQ7Y6NojD93oosZxqi0QLzxULiRamNOvoMSBgbUl1GtC8xcQQ/Yi +aBS0A+tc+7NxZ6SJXIa/i7tbJcebPY5bnbHcILXPOt4FLEgcBqyv9UquPstkYytJ +je4J0N+G/nqfKsh+mo+emnKFSy1QS7NoPr/TfDKemnf2DBk0HOiBnIr2gh3gqThX +qUt/dZlDNJALeJU+7IpLDThOM3sf1QOOkSF9O1IM1YJt3B9GeTDwPnqKbXVOKf23 +eBi51QyvWde1ZscTRh0p9HX4VRCYOGfkQnWw0d3BpFg/a6rGVNLSPBGE2H6O68L4 +K1bBDV0CvdTjVD8/vgrLm/7NAOlg/58TKIaqNxFalXeLmdKr0c5d4JZEbbPgg26O +8Fsq769s8Jc1dtnAiFwB2opIOvOLZkNwzPG8EjAET9HmjWHHzZ/OmswWamqywPuk +W8jdLH5f4RsuGpGHsUvs/53fUUeAdAlceJ+1KuLNuk7ULRU59TRbppt6m/Ws81bW +JLQtw79BdyDNgJ4q7Vyl5tCuC2mZzDqOb/uKpy5Gx6Upoy0klAsMjvUBiw3cpkVC +l1/RCSx2HmV85itS20QCiFcT+KeJ3xSbIc+PScNvinnbwtRhENQY+fy5MAfy9kvE +dlYlsM2yp3l1B+Z4My6w8e2CcQO4RwIDAQABAoICAA//iW6KEL8nkcIR/ijsh4lE +061dXhWu17oldgtVvs/1gux7yfMpP2CHwRB96J7nzcbdcjxDeo8dEg9VnBCYSjUT +7KFhCiVQQwBFXsNL573SgC+2EqS++8Haen10/ohlD6TIpasfELXMvEy1zV3oDTyR +nerJzLh0+DKdq1jrvpmuHr5WC2z2kEH+HHlL3irlP5X5UhsBptzIGfd1p49244GF +Q4tkED29J/9QDjSha1Ji48zUXIoWKf0Y5FLf6J6eh+m4haH3BMIBfT9yYqsRavZu +81YKVwBP3FOskhqxV3MUyHsisHr1tjJ6TlUzUpy8bLFYL/CfC3mRkbtdWs1JPKBk +2BFZVBU0JeTS4SB2kSSjxHMDTi5lhCzTgdzNk9z3FvrwPYAYV9eTdEoWnwRlGjQo +IAwde0EQk508JCBG7RXpn+yp7ye0y1WvmxvwTx5mshSf9S90wrquaFryOZzAO+qa +FbQBPhWdtz/NBEqZa3teNo+kvhm90Ey6BcoO75EFVVPJaDlCjZ7jrzSy4XuYi99F +NjgmXUnGTRgYu+aOItBX1ckBNUg6kSVXk4iIVpXD65wANTjNladeK8ZWBq1k1JEd +V+VBdQu5H0JzOi01i4jDzzb/6T7lIj1NFpi5PL7T6q6EelC7QpxJpoKGd3YVQPWr +zvLR1bS2Fsg3hNlkFr6RAoIBAQDYsZjJcnlGREg6aNmjyY+s2jQhk3SU65VmBH+z +IKg8Nk1erW0eVtY/nYdFVcuyH84VxpINDDIfbVzwr7tz6qAWb3+dgEKZUVv1MHv0 +S12snsO3NdZ0UKdrqr16K0d9oa3OyaCSi89zKKtfPhIPMxLRehZipXecyCM+1Rda +AHCAjmJWD9VA+izHtLQB92+d+pl3BUpWi5wxBomOP5VqJ2slOdpd2sD94EhumQbR +Gk0/4kj5PHx3r7nhgSJoAvO2HMV/PvvoiGXf+4Oi3vhgACnBA8N+zEaKvLHBRZNF +nIkoxAgu5erEsnuJJEOEgpnsiLD7ZChbnPQgpD4p9t7Da7jXAoIBAQDMWSb28T4m +DIm50G4Q2Hnhz6EEaGZSEbyq5AejWLrhgvJHpo/mOaFoNOPQuzV3WpBgM6Uvltcq +Kk6uthA6Vr3haiJkxWMXI5rnszHnEd2VMKBfsui60Z2leWOdiEH2CkBroU67ZV0x +9X7P3LlxClJ91yP9iPLz6Cx8QZWZ2WMQabODb86K8aQ5pqLcTgclujCk7NmSM9T3 +eBz8mlVMFBsnudIe1A04e4EYHkVUvWtAltQbILsvbxLVMGRGNe2rKZ+wkke/xbf+ +Agv1LL6LwyGOAdt+71DWzFsdML5UEAkJ5EA3ERiOthhpFllvmV3pr9cMCIS0ivSH +S925tO1rvt4RAoIBAGEWOSFQs7tizoW1AoYawc+tOBwvB9XNM3Ow4lIseJP5tHKN ++0zTlUyNVNUg2pHlJB2niTplU3O3OSPxaGhIIA/NRv0XQT+WL0BMx8ytk7vKql/E +tGAK3ugjaJ97Ep3cOZZjyhi+oWS0PQwAMHE07eKC89Kg1lWdagU1zi+Z8M34fWCX +2XEyZavYb6pN5Wl/pRCpgyQBiyqABlOAc35LSPs1z3urjjpxKaK7100Knr/Xr+BT +VGT/i6XYiMTXRcA7ZdVcL9uAeTyAYPsxMVE54XtEJ2wBND3myzGP7asLtnxYUF5K +zwPv/99zKvkM1tAeckVAG8DoMo0JaXy9yhL+iaMCggEAXFqSfJqM/v89o4fqppxf +gUmoOOjCDadMgGNsfEuWsmLPAsjpUiCLrR/yMhzZziZVB9Vve3GNrtXOF7Ha5bLc +QCsKfkajQQrrcHoRPKBbZ5jBcl7WRdCEkguplMHHJd5+POZ7QcBO/Uw5UtIr0UXc +AFmiP2yMeOVebY3qgcy4s+tBoU5/p1YMZa3E/xIYstlSMMeGkUfxoSJc32EU2bxg +hXS63QnzK6rNrkvIA8NT3K4OEHCbiJWHimhDeWPYFTpLnK6P1MEUJa1hIB5nw5yd +5qM6Q0T/YQSczTWBX1ab7yeESh7k3WK4542dQA2tXvcElsCm0T3Xw+nqvIpjnwV1 +MQKCAQAbK3pgm6uE4x7iC7KuFm+j1ccm1BX9t80XlSqMv3aPTufF2wbv+OIOJgLA +HmY14nobAxt1f1AQ/9NfbZxiw8lzQMo407aXpubzHLSX4+S700quSAGBpexy2cqM +Sft9gHWiblHw7NNC1IWG/H7MUv7UA+b8GQuVhRYvVi/YeErk4eb+tAvp8T3jg465 +PwQBCO4hkXZhUDYS8S0dL04vEeMSo8eh252LrNkjho/iU58ZDmGiyHr9XDq3awDR +mfdkufXVKVigaCan7HmEOJUt2Dt2sIVn1dQ3qFzulG5FDTrUi/mefsH/FaAJoNsa +/9XEh58NyCCGvjV0a6MHrcZRMl3y +-----END PRIVATE KEY----- diff --git a/examples/features/advancedtls/creds/client_cert.pem b/examples/features/advancedtls/creds/client_cert.pem new file mode 100644 index 0000000..77a9d13 --- /dev/null +++ b/examples/features/advancedtls/creds/client_cert.pem @@ -0,0 +1,127 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 3 (0x3) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=Georgia, L=Atlanta, O=Test CA, OU=Test CA Organzation, CN=Test CA Organization/emailAddress=test@example.com + Validity + Not Before: Aug 8 18:06:22 2024 GMT + Not After : Dec 25 18:06:22 2051 GMT + Subject: C=US, ST=Georgia, L=Atlanta, O=Test client, OU=Test client Organzation, CN=Test client Organization + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (4096 bit) + Modulus: + 00:bf:8d:88:b7:20:0e:04:3e:5e:3a:f1:a3:78:2d: + a5:44:f6:68:b3:f3:ec:3c:7e:8f:cd:e2:cd:55:9c: + 2c:a2:a6:a0:31:41:b4:10:cb:3a:a8:8e:9e:ae:b5: + 65:13:18:02:fc:35:38:7c:5e:6d:ba:e0:13:31:f0: + 65:bb:a6:d3:61:7c:7f:86:bd:d6:84:d2:b1:06:92: + fe:47:5d:dd:3e:1f:99:6c:55:6f:67:eb:44:eb:d8: + da:79:70:2e:d7:48:75:6f:1d:cb:bd:e6:59:17:22: + d7:d9:23:26:90:0c:b9:63:85:91:9f:8e:58:92:52: + b6:09:3a:80:b7:40:91:fe:47:b6:e8:3c:4d:44:97: + ef:1c:11:a7:75:e0:19:d2:79:cb:3e:5d:f9:0c:81: + 95:63:6d:df:58:43:e5:03:62:78:52:0b:5b:5a:5c: + c3:d9:8e:39:15:e5:72:37:b0:3a:ce:99:67:c0:72: + ca:9f:65:25:7b:23:bf:87:bf:1f:a9:f5:0f:f2:bf: + a1:ec:43:3b:8a:67:d0:5f:61:d8:03:74:e6:b1:25: + 91:45:70:85:d0:a2:70:65:df:4d:ed:39:6c:4d:c4: + fd:fe:8d:71:92:06:90:ad:19:8e:de:0b:35:e1:50: + 79:30:6f:f6:bb:3d:74:a7:66:dd:0e:7b:d0:63:f2: + 5d:58:dc:17:a1:a2:e4:45:4e:b7:9c:32:b8:bc:56: + 88:31:de:6f:27:f3:56:29:54:45:07:68:f3:76:9d: + b7:63:c0:d7:cf:6b:11:c5:3a:d2:9f:1a:34:96:2a: + df:64:e1:df:fe:be:1d:4a:48:58:33:be:2e:c7:ac: + c7:12:6f:9a:a6:10:e5:ef:a4:ae:0b:8d:c9:56:2c: + 49:60:ff:54:91:2c:41:05:90:74:70:3e:dd:54:58: + b3:83:ae:c4:b4:4e:91:0b:a5:f1:3d:e4:5a:6d:34: + 5c:3b:ee:f6:d7:62:0b:a8:55:8f:5d:8a:ed:56:9a: + 8d:e7:80:16:0f:97:1b:f5:eb:0d:7f:1f:9a:51:e1: + 9b:3e:14:ac:f7:c3:36:42:06:11:7c:e9:ef:75:54: + ae:1b:3b:68:b7:c4:79:fd:67:5c:26:9e:a5:d4:55: + 6c:c7:92:15:51:73:57:99:bc:de:fb:56:ab:70:db: + 98:10:1a:63:71:9c:c3:9f:11:9f:c2:c5:8b:ac:5c: + 52:69:c7:58:a1:b1:26:86:e3:68:85:23:17:68:62: + 30:01:79:1a:51:d7:e9:1b:a4:da:81:b6:46:33:1e: + 9a:2b:9b:f6:20:26:d0:21:10:b0:15:58:91:08:b5: + bd:b7:c0:05:c1:cf:2f:bd:3b:18:40:17:08:92:58: + 6e:bb:bb + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + DE:03:BD:A3:0E:63:F4:97:C2:52:70:63:E8:BE:A9:DF:F1:9A:7B:56 + X509v3 Authority Key Identifier: + 50:76:C3:5C:6B:12:E1:92:0F:28:F8:F2:43:A6:80:C2:9C:E8:56:D6 + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:0.0.0.0 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 92:69:25:55:69:46:6e:3b:c3:a1:9d:00:b2:6e:b5:ae:1b:5a: + 19:2a:77:7f:12:e3:f7:84:72:37:35:26:78:45:5e:90:3d:0b: + 57:6f:1f:42:05:77:ec:4b:0c:29:dd:d7:db:02:cb:b7:2f:7b: + cc:81:4a:cc:71:2f:54:aa:3a:27:e3:8e:cd:87:76:c1:5f:60: + b6:34:0c:16:ef:fc:b6:ae:61:44:6b:b2:e1:db:86:15:e4:24: + db:47:48:f2:29:14:fb:61:0b:10:97:b1:b2:79:c3:69:dc:f3: + 65:e9:15:a6:89:17:34:46:83:b1:a6:89:4f:12:e0:69:27:66: + f8:89:df:36:21:59:a9:a5:e5:6a:8b:10:8c:19:39:cf:6e:61: + a5:43:6f:34:b4:e1:79:7a:0a:f9:1d:2d:06:66:cb:a0:91:9c: + 04:85:4f:0b:3d:c1:54:a8:06:d3:89:2e:16:5c:f2:29:c5:f7: + 6e:d9:4b:ca:81:65:96:3c:ba:66:8e:40:16:a3:20:ca:ed:5a: + ea:72:97:7a:2c:c4:b6:b5:c2:00:83:fb:1b:8a:d0:72:85:49: + 88:ad:81:9e:87:42:31:99:1a:39:ad:b5:ff:24:b5:e0:90:07: + 08:2e:1d:4a:a7:01:ef:97:9a:07:d4:e6:09:f5:c8:36:37:ce: + e3:b2:94:2a:5e:95:e1:6a:06:68:d1:31:24:da:b4:fe:ce:af: + a5:23:87:bc:7e:35:54:dd:c3:77:a5:44:95:43:a0:b1:f5:c4: + f8:98:4d:a3:fc:33:ef:7a:d7:4b:5b:ae:de:2b:1f:7a:a1:3f: + df:85:6b:97:57:4d:fa:b1:1a:79:4b:a7:96:62:09:99:b0:54: + f1:46:65:dd:3a:31:bc:1b:07:97:ff:e7:1b:0a:d4:82:68:62: + cc:66:9c:06:d4:18:70:3b:71:82:2d:76:bf:e7:56:88:4f:d9: + 5e:1b:46:9c:f9:9c:15:bc:73:ca:f5:e5:44:3d:f1:e4:b9:55: + e6:06:80:e2:0d:4f:ba:19:e2:01:29:da:5b:6f:1f:79:6a:6c: + d4:e8:c2:e1:12:c2:13:d0:5a:63:1d:35:f1:36:d4:1b:48:26: + 72:18:df:5f:7e:30:8d:86:42:cf:22:90:db:f8:6c:9d:b0:e7: + 3b:a1:0d:8a:b1:d9:de:a1:d0:4b:de:33:a2:fc:6c:cc:b0:7d: + a6:57:43:fe:db:2a:44:e3:6c:68:ff:c8:82:91:19:68:f0:c5: + 6b:9d:3b:4c:f8:2d:8f:0e:44:04:79:4e:99:ec:08:c6:e6:25: + 90:5b:2d:16:18:94:fe:0b:86:9b:01:f2:40:66:ec:fa:ac:28: + ba:33:fc:58:c1:8e:a2:06 +-----BEGIN CERTIFICATE----- +MIIGIjCCBAqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADCBoTELMAkGA1UEBhMCVVMx +EDAOBgNVBAgMB0dlb3JnaWExEDAOBgNVBAcMB0F0bGFudGExEDAOBgNVBAoMB1Rl +c3QgQ0ExHDAaBgNVBAsME1Rlc3QgQ0EgT3JnYW56YXRpb24xHTAbBgNVBAMMFFRl +c3QgQ0EgT3JnYW5pemF0aW9uMR8wHQYJKoZIhvcNAQkBFhB0ZXN0QGV4YW1wbGUu +Y29tMCAXDTI0MDgwODE4MDYyMloYDzIwNTExMjI1MTgwNjIyWjCBjDELMAkGA1UE +BhMCVVMxEDAOBgNVBAgMB0dlb3JnaWExEDAOBgNVBAcMB0F0bGFudGExFDASBgNV +BAoMC1Rlc3QgY2xpZW50MSAwHgYDVQQLDBdUZXN0IGNsaWVudCBPcmdhbnphdGlv +bjEhMB8GA1UEAwwYVGVzdCBjbGllbnQgT3JnYW5pemF0aW9uMIICIjANBgkqhkiG +9w0BAQEFAAOCAg8AMIICCgKCAgEAv42ItyAOBD5eOvGjeC2lRPZos/PsPH6PzeLN +VZwsoqagMUG0EMs6qI6errVlExgC/DU4fF5tuuATMfBlu6bTYXx/hr3WhNKxBpL+ +R13dPh+ZbFVvZ+tE69jaeXAu10h1bx3LveZZFyLX2SMmkAy5Y4WRn45YklK2CTqA +t0CR/ke26DxNRJfvHBGndeAZ0nnLPl35DIGVY23fWEPlA2J4UgtbWlzD2Y45FeVy +N7A6zplnwHLKn2UleyO/h78fqfUP8r+h7EM7imfQX2HYA3TmsSWRRXCF0KJwZd9N +7TlsTcT9/o1xkgaQrRmO3gs14VB5MG/2uz10p2bdDnvQY/JdWNwXoaLkRU63nDK4 +vFaIMd5vJ/NWKVRFB2jzdp23Y8DXz2sRxTrSnxo0lirfZOHf/r4dSkhYM74ux6zH +Em+aphDl76SuC43JVixJYP9UkSxBBZB0cD7dVFizg67EtE6RC6XxPeRabTRcO+72 +12ILqFWPXYrtVpqN54AWD5cb9esNfx+aUeGbPhSs98M2QgYRfOnvdVSuGztot8R5 +/WdcJp6l1FVsx5IVUXNXmbze+1arcNuYEBpjcZzDnxGfwsWLrFxSacdYobEmhuNo +hSMXaGIwAXkaUdfpG6TagbZGMx6aK5v2ICbQIRCwFViRCLW9t8AFwc8vvTsYQBcI +klhuu7sCAwEAAaN2MHQwHQYDVR0OBBYEFN4DvaMOY/SXwlJwY+i+qd/xmntWMB8G +A1UdIwQYMBaAFFB2w1xrEuGSDyj48kOmgMKc6FbWMAkGA1UdEwQCMAAwCwYDVR0P +BAQDAgWgMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEAAAAADANBgkqhkiG9w0BAQsF +AAOCAgEAkmklVWlGbjvDoZ0Asm61rhtaGSp3fxLj94RyNzUmeEVekD0LV28fQgV3 +7EsMKd3X2wLLty97zIFKzHEvVKo6J+OOzYd2wV9gtjQMFu/8tq5hRGuy4duGFeQk +20dI8ikU+2ELEJexsnnDadzzZekVpokXNEaDsaaJTxLgaSdm+InfNiFZqaXlaosQ +jBk5z25hpUNvNLTheXoK+R0tBmbLoJGcBIVPCz3BVKgG04kuFlzyKcX3btlLyoFl +ljy6Zo5AFqMgyu1a6nKXeizEtrXCAIP7G4rQcoVJiK2BnodCMZkaOa21/yS14JAH +CC4dSqcB75eaB9TmCfXINjfO47KUKl6V4WoGaNExJNq0/s6vpSOHvH41VN3Dd6VE +lUOgsfXE+JhNo/wz73rXS1uu3isfeqE/34Vrl1dN+rEaeUunlmIJmbBU8UZl3Tox +vBsHl//nGwrUgmhizGacBtQYcDtxgi12v+dWiE/ZXhtGnPmcFbxzyvXlRD3x5LlV +5gaA4g1PuhniASnaW28feWps1OjC4RLCE9BaYx018TbUG0gmchjfX34wjYZCzyKQ +2/hsnbDnO6ENirHZ3qHQS94zovxszLB9pldD/tsqRONsaP/IgpEZaPDFa507TPgt +jw5EBHlOmewIxuYlkFstFhiU/guGmwHyQGbs+qwoujP8WMGOogY= +-----END CERTIFICATE----- diff --git a/examples/features/advancedtls/creds/client_cert_revoked.pem b/examples/features/advancedtls/creds/client_cert_revoked.pem new file mode 100644 index 0000000..3812b95 --- /dev/null +++ b/examples/features/advancedtls/creds/client_cert_revoked.pem @@ -0,0 +1,127 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 4 (0x4) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=Georgia, L=Atlanta, O=Test CA, OU=Test CA Organzation, CN=Test CA Organization/emailAddress=test@example.com + Validity + Not Before: Aug 8 18:06:22 2024 GMT + Not After : Dec 25 18:06:22 2051 GMT + Subject: C=US, ST=Georgia, L=Atlanta, O=Test client, OU=Test client Organzation, CN=Test client Organization + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (4096 bit) + Modulus: + 00:bf:8d:88:b7:20:0e:04:3e:5e:3a:f1:a3:78:2d: + a5:44:f6:68:b3:f3:ec:3c:7e:8f:cd:e2:cd:55:9c: + 2c:a2:a6:a0:31:41:b4:10:cb:3a:a8:8e:9e:ae:b5: + 65:13:18:02:fc:35:38:7c:5e:6d:ba:e0:13:31:f0: + 65:bb:a6:d3:61:7c:7f:86:bd:d6:84:d2:b1:06:92: + fe:47:5d:dd:3e:1f:99:6c:55:6f:67:eb:44:eb:d8: + da:79:70:2e:d7:48:75:6f:1d:cb:bd:e6:59:17:22: + d7:d9:23:26:90:0c:b9:63:85:91:9f:8e:58:92:52: + b6:09:3a:80:b7:40:91:fe:47:b6:e8:3c:4d:44:97: + ef:1c:11:a7:75:e0:19:d2:79:cb:3e:5d:f9:0c:81: + 95:63:6d:df:58:43:e5:03:62:78:52:0b:5b:5a:5c: + c3:d9:8e:39:15:e5:72:37:b0:3a:ce:99:67:c0:72: + ca:9f:65:25:7b:23:bf:87:bf:1f:a9:f5:0f:f2:bf: + a1:ec:43:3b:8a:67:d0:5f:61:d8:03:74:e6:b1:25: + 91:45:70:85:d0:a2:70:65:df:4d:ed:39:6c:4d:c4: + fd:fe:8d:71:92:06:90:ad:19:8e:de:0b:35:e1:50: + 79:30:6f:f6:bb:3d:74:a7:66:dd:0e:7b:d0:63:f2: + 5d:58:dc:17:a1:a2:e4:45:4e:b7:9c:32:b8:bc:56: + 88:31:de:6f:27:f3:56:29:54:45:07:68:f3:76:9d: + b7:63:c0:d7:cf:6b:11:c5:3a:d2:9f:1a:34:96:2a: + df:64:e1:df:fe:be:1d:4a:48:58:33:be:2e:c7:ac: + c7:12:6f:9a:a6:10:e5:ef:a4:ae:0b:8d:c9:56:2c: + 49:60:ff:54:91:2c:41:05:90:74:70:3e:dd:54:58: + b3:83:ae:c4:b4:4e:91:0b:a5:f1:3d:e4:5a:6d:34: + 5c:3b:ee:f6:d7:62:0b:a8:55:8f:5d:8a:ed:56:9a: + 8d:e7:80:16:0f:97:1b:f5:eb:0d:7f:1f:9a:51:e1: + 9b:3e:14:ac:f7:c3:36:42:06:11:7c:e9:ef:75:54: + ae:1b:3b:68:b7:c4:79:fd:67:5c:26:9e:a5:d4:55: + 6c:c7:92:15:51:73:57:99:bc:de:fb:56:ab:70:db: + 98:10:1a:63:71:9c:c3:9f:11:9f:c2:c5:8b:ac:5c: + 52:69:c7:58:a1:b1:26:86:e3:68:85:23:17:68:62: + 30:01:79:1a:51:d7:e9:1b:a4:da:81:b6:46:33:1e: + 9a:2b:9b:f6:20:26:d0:21:10:b0:15:58:91:08:b5: + bd:b7:c0:05:c1:cf:2f:bd:3b:18:40:17:08:92:58: + 6e:bb:bb + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + DE:03:BD:A3:0E:63:F4:97:C2:52:70:63:E8:BE:A9:DF:F1:9A:7B:56 + X509v3 Authority Key Identifier: + 50:76:C3:5C:6B:12:E1:92:0F:28:F8:F2:43:A6:80:C2:9C:E8:56:D6 + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:0.0.0.0 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 58:b7:35:45:3b:6b:5e:7d:6b:58:70:be:e6:39:96:14:2e:69: + 17:fc:a4:8e:1b:ae:ca:62:73:ec:12:92:ca:a8:1f:92:b8:1e: + 09:a5:7e:c0:49:d2:a3:29:48:2f:4c:67:ae:a6:fb:ad:7a:1b: + 2a:29:0b:75:6d:11:0f:99:8c:1d:dc:af:1c:a8:e7:cb:7c:66: + 34:de:7e:8f:e6:aa:26:6e:56:17:aa:1f:34:e9:1f:ff:7a:58: + d2:7e:7c:65:62:56:d1:de:04:bd:71:cf:a2:6c:ad:47:cb:10: + e8:72:b0:0a:9e:24:79:e0:1a:b6:e2:61:6f:fd:94:8b:3c:19: + d0:8e:62:4f:a2:3a:fd:3d:97:c2:e7:93:1f:2c:aa:13:f5:c6: + d0:03:4c:ee:90:48:94:3b:03:d9:2c:80:59:97:fb:a2:7f:00: + 23:19:51:0b:89:2a:92:36:57:94:0b:73:8b:f3:ae:5d:f0:68: + 29:ea:a1:f3:eb:83:48:f5:19:d1:42:fe:94:cd:13:37:c9:9a: + c1:65:b3:97:eb:7e:82:f1:e3:98:c8:da:0c:41:c0:6f:4f:42: + 49:38:8b:c4:57:f4:07:cb:7f:f5:70:81:f0:72:3e:c7:e1:69: + e3:38:e5:d0:4a:97:b2:b6:bf:25:c9:fe:91:79:39:d0:eb:04: + a5:5d:b6:ca:4a:83:6e:9a:32:a2:6f:b1:ed:34:71:6f:9e:ee: + ed:e4:c3:1b:07:ec:e1:d2:19:9f:f8:b0:a0:91:e6:dd:92:cf: + 2a:dd:45:b5:29:12:57:1b:6c:f2:04:37:be:4d:20:e8:f4:f4: + 2c:f1:bc:3e:76:ed:85:64:26:0f:81:c5:dc:63:f6:6e:77:fc: + 32:18:0b:a0:e4:8a:b5:af:93:d3:55:26:5d:7f:5d:a1:5d:1d: + 2e:f2:11:66:bd:5a:32:cc:80:6d:cf:c2:45:17:b4:bf:46:c6: + 99:2d:ae:1e:20:b8:21:b0:80:8f:72:25:9d:62:b6:80:71:9e: + 90:80:ef:52:19:a3:68:05:80:f9:8b:dc:f5:89:57:35:5c:1b: + 11:f0:e0:15:4e:ca:19:3c:19:61:86:8f:6b:3c:c3:d1:cf:6f: + c5:28:88:35:7d:c8:ae:1b:98:a1:7c:b8:e8:df:36:a9:9a:9b: + bd:71:48:c2:89:d6:5c:27:31:c9:c3:4c:71:95:67:aa:7a:c4: + 2e:7e:05:6f:d2:53:16:cc:6b:5b:64:43:ff:e5:1a:d5:47:d9: + ff:47:1f:28:91:43:88:5d:34:ca:61:fe:38:b7:8f:35:43:51: + 78:b1:c1:2b:e2:29:2a:a1:69:bb:1f:14:2e:c5:f3:18:9d:81: + ee:bc:d6:fc:e7:52:d6:d6 +-----BEGIN CERTIFICATE----- +MIIGIjCCBAqgAwIBAgIBBDANBgkqhkiG9w0BAQsFADCBoTELMAkGA1UEBhMCVVMx +EDAOBgNVBAgMB0dlb3JnaWExEDAOBgNVBAcMB0F0bGFudGExEDAOBgNVBAoMB1Rl +c3QgQ0ExHDAaBgNVBAsME1Rlc3QgQ0EgT3JnYW56YXRpb24xHTAbBgNVBAMMFFRl +c3QgQ0EgT3JnYW5pemF0aW9uMR8wHQYJKoZIhvcNAQkBFhB0ZXN0QGV4YW1wbGUu +Y29tMCAXDTI0MDgwODE4MDYyMloYDzIwNTExMjI1MTgwNjIyWjCBjDELMAkGA1UE +BhMCVVMxEDAOBgNVBAgMB0dlb3JnaWExEDAOBgNVBAcMB0F0bGFudGExFDASBgNV +BAoMC1Rlc3QgY2xpZW50MSAwHgYDVQQLDBdUZXN0IGNsaWVudCBPcmdhbnphdGlv +bjEhMB8GA1UEAwwYVGVzdCBjbGllbnQgT3JnYW5pemF0aW9uMIICIjANBgkqhkiG +9w0BAQEFAAOCAg8AMIICCgKCAgEAv42ItyAOBD5eOvGjeC2lRPZos/PsPH6PzeLN +VZwsoqagMUG0EMs6qI6errVlExgC/DU4fF5tuuATMfBlu6bTYXx/hr3WhNKxBpL+ +R13dPh+ZbFVvZ+tE69jaeXAu10h1bx3LveZZFyLX2SMmkAy5Y4WRn45YklK2CTqA +t0CR/ke26DxNRJfvHBGndeAZ0nnLPl35DIGVY23fWEPlA2J4UgtbWlzD2Y45FeVy +N7A6zplnwHLKn2UleyO/h78fqfUP8r+h7EM7imfQX2HYA3TmsSWRRXCF0KJwZd9N +7TlsTcT9/o1xkgaQrRmO3gs14VB5MG/2uz10p2bdDnvQY/JdWNwXoaLkRU63nDK4 +vFaIMd5vJ/NWKVRFB2jzdp23Y8DXz2sRxTrSnxo0lirfZOHf/r4dSkhYM74ux6zH +Em+aphDl76SuC43JVixJYP9UkSxBBZB0cD7dVFizg67EtE6RC6XxPeRabTRcO+72 +12ILqFWPXYrtVpqN54AWD5cb9esNfx+aUeGbPhSs98M2QgYRfOnvdVSuGztot8R5 +/WdcJp6l1FVsx5IVUXNXmbze+1arcNuYEBpjcZzDnxGfwsWLrFxSacdYobEmhuNo +hSMXaGIwAXkaUdfpG6TagbZGMx6aK5v2ICbQIRCwFViRCLW9t8AFwc8vvTsYQBcI +klhuu7sCAwEAAaN2MHQwHQYDVR0OBBYEFN4DvaMOY/SXwlJwY+i+qd/xmntWMB8G +A1UdIwQYMBaAFFB2w1xrEuGSDyj48kOmgMKc6FbWMAkGA1UdEwQCMAAwCwYDVR0P +BAQDAgWgMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEAAAAADANBgkqhkiG9w0BAQsF +AAOCAgEAWLc1RTtrXn1rWHC+5jmWFC5pF/ykjhuuymJz7BKSyqgfkrgeCaV+wEnS +oylIL0xnrqb7rXobKikLdW0RD5mMHdyvHKjny3xmNN5+j+aqJm5WF6ofNOkf/3pY +0n58ZWJW0d4EvXHPomytR8sQ6HKwCp4keeAatuJhb/2UizwZ0I5iT6I6/T2XwueT +HyyqE/XG0ANM7pBIlDsD2SyAWZf7on8AIxlRC4kqkjZXlAtzi/OuXfBoKeqh8+uD +SPUZ0UL+lM0TN8mawWWzl+t+gvHjmMjaDEHAb09CSTiLxFf0B8t/9XCB8HI+x+Fp +4zjl0EqXsra/Jcn+kXk50OsEpV22ykqDbpoyom+x7TRxb57u7eTDGwfs4dIZn/iw +oJHm3ZLPKt1FtSkSVxts8gQ3vk0g6PT0LPG8PnbthWQmD4HF3GP2bnf8MhgLoOSK +ta+T01UmXX9doV0dLvIRZr1aMsyAbc/CRRe0v0bGmS2uHiC4IbCAj3IlnWK2gHGe +kIDvUhmjaAWA+Yvc9YlXNVwbEfDgFU7KGTwZYYaPazzD0c9vxSiINX3IrhuYoXy4 +6N82qZqbvXFIwonWXCcxycNMcZVnqnrELn4Fb9JTFsxrW2RD/+Ua1UfZ/0cfKJFD +iF00ymH+OLePNUNReLHBK+IpKqFpux8ULsXzGJ2B7rzW/OdS1tY= +-----END CERTIFICATE----- diff --git a/examples/features/advancedtls/creds/client_key.pem b/examples/features/advancedtls/creds/client_key.pem new file mode 100644 index 0000000..f9ec8c4 --- /dev/null +++ b/examples/features/advancedtls/creds/client_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC/jYi3IA4EPl46 +8aN4LaVE9miz8+w8fo/N4s1VnCyipqAxQbQQyzqojp6utWUTGAL8NTh8Xm264BMx +8GW7ptNhfH+GvdaE0rEGkv5HXd0+H5lsVW9n60Tr2Np5cC7XSHVvHcu95lkXItfZ +IyaQDLljhZGfjliSUrYJOoC3QJH+R7boPE1El+8cEad14BnSecs+XfkMgZVjbd9Y +Q+UDYnhSC1taXMPZjjkV5XI3sDrOmWfAcsqfZSV7I7+Hvx+p9Q/yv6HsQzuKZ9Bf +YdgDdOaxJZFFcIXQonBl303tOWxNxP3+jXGSBpCtGY7eCzXhUHkwb/a7PXSnZt0O +e9Bj8l1Y3BehouRFTrecMri8Vogx3m8n81YpVEUHaPN2nbdjwNfPaxHFOtKfGjSW +Kt9k4d/+vh1KSFgzvi7HrMcSb5qmEOXvpK4LjclWLElg/1SRLEEFkHRwPt1UWLOD +rsS0TpELpfE95FptNFw77vbXYguoVY9diu1Wmo3ngBYPlxv16w1/H5pR4Zs+FKz3 +wzZCBhF86e91VK4bO2i3xHn9Z1wmnqXUVWzHkhVRc1eZvN77Vqtw25gQGmNxnMOf +EZ/CxYusXFJpx1ihsSaG42iFIxdoYjABeRpR1+kbpNqBtkYzHporm/YgJtAhELAV +WJEItb23wAXBzy+9OxhAFwiSWG67uwIDAQABAoICAA9XpZV5UUtrYr5uNULTxnLV +blcfgz091oR4lSwKuWkrrGqq5w0Yogb3KWQ37jO5zsoRnzJtmR9d1tu/SVQJjXu3 +mOVGFkUG38agMRKumiEMLzcwnp7gXAxXOaR+ukRkWYJADN4Vxu6l9qDoglcJAIj6 +nUp+KxTz4ABcObazyshm87z6e3vc1CTXkHtOAEjN7YaoyIuvuLNBN8wzip0bSXoq +I7PYo0KCdDW57iHiESEUGMMezqDhXT+q13/mwdap5Lrzjg4aPSNy+OBy5KDeP6NS +WCY2LYRORm9lE9zw0P5noqpn/NU5MQ7+8Y+nrotF7UxWY2462A7tG76PEdwM1ly7 +GyslA5o+V9RE1CsQJfK0yInD0W2cC9cggdcGlzm5p0jhWKk3Z3QaEXv6KxlkQjct +q9uX7xdQff01WQVnS96VG4I5YlJP8vdTww3KL+7fpsBS1uNn1/pxDZW3LgT1XEHf +lJxRSHKwyViHJtQyVaKzuzS9AOztOO1OWJvIwLAwImdjyXZeoKdBptkVLRfbX4+U +4zCE0fgwkkmuFrvFQSoXvQsul+BXULiUhRTilUaN6VweP1Q99g9dzVqkyNLmm6+2 +vaOR4F3y+DYgu5wa7wQdULnpkzpHwiPUcqulOgg5xLo7tr5Q9nVrwyQy5zvuvYas +N/LsYR2xBrMqRZRbxSWVAoIBAQD5mBydPI2jtw2jw6NBs9lqZasmUXoFkUO1IPuG +oNauIzQdhQ7OCdBmwyWVyI7QnM0ZNB495qOg+lmxMpkg3hoQlN0PFKEBvVZde6Yz +sB4eCL95x5bdp3oblUvq7h/0d4i+1I8OVKZz56zSn6CyoEH5zIxq+dXgGVsdfWOW +tqnLSGM0apZlfKWoHTPc5OtJAiXXwWA0aGBGCa3M438HJ8V6qy4QREOOxR6HapUJ +VODPsj9h6XL5jxFqcbaojjpckpb3SYh7rB3kgoriuSBgPGKM+PLjW1mCGwcLgGYB +cn+2EHxKLvm+ShzZGdwjrpCv0aRuiYIGEb1/Afu4nOGnnJXnAoIBAQDEeBQBez5E +SBiy6W5ZCIUXMS8DHFVei/GYDsAYMH0+vnbwUxBWNDKpUbiVVE6yrTjiRPfNamf0 +0zKSsI2MOMLNnb8nIkEWeVweHfQF5R1xxubemqmQ55Rd11ep59p9Er7O1WtR0968 +KWwT9ZN4OoXSRIFHV0fjY2V2Ns0VDyHl8vWgDk80JpEX/4BWFtRzuPaRCGuxFAJY +3Md5ynCwOPaPKPXMc1by1ZCKGBxpXoeErWM0/hvbt2YUfI8+n80W0ws2p1/YDWE+ +9PhCHUFPxxTvb7+rSYYHJs3tcnS7c3K3jhkxbbR9FS95IVMTopjD24nl3G6nSyPT +UJP+qnE3WgkNAoIBAEKqt5HkF60P+uuwGM423K7HozRj9OTBzUT3H1fxZAY1TvlH +jhHIm0qne0WLwWHFUB8YRa+hCDm6RPTIoBAgYvPk3zrk9rCBQy1LFrSdqR26lnJP +tmNUFZCCizmgCxcASp02J1Pblm5FBmtnycOMfLLdSPBV9SObgjPZRx19gtLSbfUV +N0C6T4Ec87pfxtzEXxlHBIxbCMQMV8jvRwHBRMUkLfSYVzcuPZ5MAKzyZ+3yHW3o +rhYseall4DUbcElDumEo2fS2n3Fm0PQIILazylr/L9k8kCbpUzNmQ1jFnYki1B/4 +diq2nwf6GUvKl8juhS4lOn6mhGgFPpgsBzX+5CcCggEAL/1Epbf81aDmp4ztL0It +gCS7Xv8kuxtjv8iak04EybasRreDXgsR9NnJRHB7aJl3M421Ga/MBLkxuTL24DFd +I+xMLLrpOxwZrCGU4Xu9XXVAH0+X65UlYGahOxcu/y38/XiT5kDiPwO/KoDprIxe +86VYDpz7Kke1GNL59RLlLM3TwWy9W/evqTT3nA+nhTzAvVxZMb+5cws6jj0smV7Q +mtdecroZmucfjxuklPhKEdZoTSFknJ6HiKmEM7/E0LZsHsVzW8qo3j/oA/4xXdM7 +AeFB6AzleAm6cy1p5f+lHcDP1or9czAhkGzbZghpWC3f2Q2m2aY48fzUqXfof6S2 +YQKCAQEAhnYgWar6sj/llQiKGmwQxxw9PkJMHGLAX44H3P9xAPwDIsgkL+hGmVIR +7/ILhEbPtBGCvaoI/bqR5zh8VdMbqnm8ocZqz4xnu4WMpMfmF07TxF8aVC9TBoas +Ad6khQfL4c91YrwTThvfyZ6im3aP/e8CSiZrkg89tF3a6rLvsJRx1qlj0DciLxW5 +/7soumtv9DCa1YmuuBAad14WprxEvAG7OVpH6SJPx6V3Di7LdWZUJQ79xeWn3Coh +jfC/JlCEkRvxmzW8oDPWxHzbIJ194ukXPQot8eFzH+oOWtGkODxjhbdiTBl6ty2f +egtZ+t//dA1KBWPMdPk3MoWZopTVgw== +-----END PRIVATE KEY----- diff --git a/examples/features/advancedtls/creds/crl/client_revoked.crl b/examples/features/advancedtls/creds/crl/client_revoked.crl new file mode 100644 index 0000000..1792d59 --- /dev/null +++ b/examples/features/advancedtls/creds/crl/client_revoked.crl @@ -0,0 +1,20 @@ +-----BEGIN X509 CRL----- +MIIDOzCCASMCAQEwDQYJKoZIhvcNAQELBQAwgaExCzAJBgNVBAYTAlVTMRAwDgYD +VQQIDAdHZW9yZ2lhMRAwDgYDVQQHDAdBdGxhbnRhMRAwDgYDVQQKDAdUZXN0IENB +MRwwGgYDVQQLDBNUZXN0IENBIE9yZ2FuemF0aW9uMR0wGwYDVQQDDBRUZXN0IENB +IE9yZ2FuaXphdGlvbjEfMB0GCSqGSIb3DQEJARYQdGVzdEBleGFtcGxlLmNvbRcN +MjQwODA4MTgwNjIyWhcNMjQwOTA3MTgwNjIyWjAoMBICAQIXDTI0MDgwODE4MDYy +MVowEgIBBBcNMjQwODA4MTgwNjIyWqAjMCEwHwYDVR0jBBgwFoAUUHbDXGsS4ZIP +KPjyQ6aAwpzoVtYwDQYJKoZIhvcNAQELBQADggIBAArFuFeXXCWCCNLy8qk0UG5r +CljVMSWrOPTy3eyQH+pSbzdwA5PYW2i5BOBcr6ULKW5aamFjhYMviqroFXrib7yU +hNhiK8FtH9cl2O7pbdFGBdjqHoGOSOWXG++0LU+Hhh5kTr/iZrgkYvB3RHycofC1 +85nY01t//fGZZJ3e8hBwf8sNdR4L7vQ2WJtbzj8mj6mU4K//UkTiqZv2yGlbDXmh +p0HDdu9/nBFLrLE35N/0m/1R4pW7AXm3R6WBiqxY8KdA4Us9tC9+qvtsWwEe/klN +5E9FLcARMTl9kwJLNJZpVoe6tyt/S4WXs4nh+XEpiD5uZgbMh0N0jwaCMWyz3wo6 +tLkMmg+4mXEViAKQZTGVU2fTVaBH1C6A4ugB7IcFG1gXVw2DnF6I1XQB9+EcPbpb +6ZTBo1msSR0Bzr0sUOdCiKhSc60DTjeNjcLhNT4k06cVvzQcyb2KePG+NnA/Tfbz +yMuDcx62T2BTL1X2aVMUSLY3mwWnqyFdHbEQOoKH084Nrhizq7H2YwdoL992UTuH +PzjyEqJN3hIePthlHl2g9fGh9dIJtxu6didm2M4WoHKeCfpWPH8fc37zhX8QYpqj +U9vDvc2F567lRpAGwyqKZti+2xg2L2K/qBSGvKdtf5hPsOvVlEnWC4mTbjo19aUn +YvLKT6e3D16ao5jVKITj +-----END X509 CRL----- diff --git a/examples/features/advancedtls/creds/localhost-openssl.cnf b/examples/features/advancedtls/creds/localhost-openssl.cnf new file mode 100644 index 0000000..bacdb00 --- /dev/null +++ b/examples/features/advancedtls/creds/localhost-openssl.cnf @@ -0,0 +1,24 @@ +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req + +[req_distinguished_name] +countryName = Country Name (2 letter code) +countryName_default = US +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = Georgia +localityName = Locality Name (eg, city) +localityName_default = Atlanta +organizationName = Organization Name (eg, company) +organizationName_default = Test Department +commonName = Common Name (eg, YOUR name) +commonName_max = 64 + +[v3_req] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names + +[alt_names] +DNS.1 = localhost +IP.1 = 0.0.0.0 diff --git a/examples/features/advancedtls/creds/openssl-ca.cnf b/examples/features/advancedtls/creds/openssl-ca.cnf new file mode 100644 index 0000000..31e0b39 --- /dev/null +++ b/examples/features/advancedtls/creds/openssl-ca.cnf @@ -0,0 +1,94 @@ +base_dir = . +certificate = $base_dir/ca_cert.pem # The CA certificate +private_key = $base_dir/ca_key.pem # The CA private key +new_certs_dir = $base_dir # Location for new certs after signing +database = $base_dir/index.txt # Database index file +serial = $base_dir/serial.txt # The current serial number + +unique_subject = no # Set to 'no' to allow creation of + # several certificates with same subject. + +HOME = . +RANDFILE = $ENV::HOME/.rnd + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +[ CA_default ] + +default_days = 10000 # How long to certify for +default_crl_days = 30 # How long before next CRL +default_md = sha256 # Use public key default MD +preserve = no # Keep passed DN ordering + +x509_extensions = ca_extensions # The extensions to add to the cert +crl_extensions = crl_ext + +email_in_dn = no # Don't concat the email in the DN +copy_extensions = copy # Required to copy SANs from CSR to cert + +#################################################################### +[ req ] +default_bits = 4096 +default_keyfile = ca_key.pem +distinguished_name = ca_distinguished_name +x509_extensions = ca_extensions +string_mask = utf8only + +#################################################################### +[ ca_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = Georgia + +localityName = Locality Name (eg, city) +localityName_default = Atlanta + +organizationName = Organization Name (eg, company) +organizationName_default = Test CA + +organizationalUnitName = Organizational Unit (eg, division) +organizationalUnitName_default = Test CA Organization + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_default = Test CA Organization + +emailAddress = Email Address +emailAddress_default = test@example.com + +#################################################################### +[ ca_extensions ] + +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always, issuer +basicConstraints = critical, CA:true +keyUsage = keyCertSign, cRLSign + + + + +#################################################################### +[ signing_policy ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ signing_req ] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment + +[ crl_ext ] +# CRL extensions. +# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL. +#issuerAltName=issuer:copy +authorityKeyIdentifier=keyid:always diff --git a/examples/features/advancedtls/creds/server_cert.pem b/examples/features/advancedtls/creds/server_cert.pem new file mode 100644 index 0000000..2be81a6 --- /dev/null +++ b/examples/features/advancedtls/creds/server_cert.pem @@ -0,0 +1,127 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 1 (0x1) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=Georgia, L=Atlanta, O=Test CA, OU=Test CA Organzation, CN=Test CA Organization/emailAddress=test@example.com + Validity + Not Before: Aug 8 18:06:21 2024 GMT + Not After : Dec 25 18:06:21 2051 GMT + Subject: C=US, ST=Georgia, L=Atlanta, O=Test Server, OU=Test Server Organzation, CN=Test Server Organization + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (4096 bit) + Modulus: + 00:bd:47:df:b7:d5:38:92:af:b4:69:e7:48:3b:a0: + 7b:9e:6b:83:0e:76:91:06:06:94:a3:80:a3:73:8f: + 50:e5:43:80:f8:f7:fb:65:7b:f0:a3:94:cc:8e:a6: + 7e:fe:59:43:ce:80:68:6d:55:67:8e:33:aa:90:79: + 21:ac:de:6e:f0:03:27:1e:6f:50:31:cf:d2:3e:c3: + 8e:98:f5:bb:f9:e9:44:3f:3f:59:ae:7c:a3:b8:a7: + ae:94:ff:68:70:d0:fb:7b:cb:cc:35:7d:04:81:f5: + 2b:12:78:bf:6e:1b:a3:cd:d1:74:41:41:9f:ee:02: + 1f:b3:42:fd:c9:01:b5:28:43:ee:31:03:3a:5d:60: + d3:df:8f:69:1e:73:4a:c4:83:35:95:00:93:83:6e: + d6:b0:d2:0b:30:31:7f:95:eb:ce:c9:73:83:b9:76: + eb:45:f1:20:8b:75:de:81:a3:32:b0:f7:0f:21:64: + a7:1d:cc:3b:00:82:c8:48:74:c9:3a:0b:f9:cb:6e: + 8c:ab:fc:b0:94:20:bd:60:06:eb:d0:12:15:55:48: + d7:d3:30:ef:59:67:98:df:f6:31:92:6d:63:1c:4a: + 93:7c:97:a8:99:f6:61:e5:78:12:36:a2:24:56:37: + 4b:38:ce:63:00:a2:26:b3:31:05:93:23:3c:c1:ed: + b1:fb:25:7d:fc:54:04:3a:b9:3a:f7:17:a4:58:10: + 4f:e8:6d:90:69:49:b6:1f:1b:81:fb:f5:c7:6c:aa: + b3:e0:4a:b1:38:40:77:83:a2:aa:8c:e2:7c:91:a9: + 3e:cd:43:be:90:c3:e7:b1:23:94:47:f9:68:db:e4: + 2c:df:65:e7:88:b6:64:dc:62:d0:86:33:9b:13:64: + 94:37:aa:0e:56:9f:a3:42:19:67:30:a1:e9:3b:5b: + 4a:e6:e1:81:52:81:21:2a:78:ac:c1:77:77:52:fc: + 4a:95:b9:3f:f7:e6:32:9e:59:5b:46:4c:a9:8a:12: + d3:2c:fc:33:73:3a:28:26:28:22:4c:1c:a9:b1:59: + 96:ab:a5:f6:e9:e7:55:32:a8:2b:a2:33:de:a0:e2: + 5f:77:d8:cd:d1:aa:1f:4f:c6:69:10:66:4e:9d:aa: + 77:83:82:78:96:5a:07:21:12:db:4c:97:51:cd:ba: + ea:00:cd:94:97:40:b8:50:62:90:2b:8c:b0:1b:2c: + aa:a5:63:0c:bb:7d:d5:7d:3f:c1:4a:00:6b:cb:74: + fa:23:35:26:1e:26:1a:30:b2:96:bc:1b:16:2a:62: + 96:1f:51:20:72:95:36:1a:87:20:26:9f:76:d6:84: + 1b:67:2a:32:68:b7:e0:c7:80:75:a3:fa:b7:da:a3: + 03:71:c1 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 4C:57:E2:72:97:CF:DC:C4:B8:4E:DB:D4:C1:C6:3D:AE:EF:D7:0A:19 + X509v3 Authority Key Identifier: + 50:76:C3:5C:6B:12:E1:92:0F:28:F8:F2:43:A6:80:C2:9C:E8:56:D6 + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:0.0.0.0 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 10:9b:66:d5:4b:8f:e2:7b:25:8b:fe:5b:9c:a6:dd:4e:d5:ee: + 27:ad:a9:e5:c4:5d:9b:f9:2c:f1:d6:8d:0e:d6:9b:e6:9f:87: + 0b:14:1b:c9:a3:dc:da:82:d0:1e:e8:c5:f7:f4:ea:99:ea:01: + f1:2e:7c:f0:07:15:28:74:15:b0:36:27:a5:3f:2d:c7:32:fc: + 81:61:44:15:9a:9a:88:20:fb:c6:d9:8a:26:61:df:e2:04:a2: + 54:98:76:90:40:98:80:d3:eb:ff:73:29:d7:2f:3f:79:ca:ba: + c3:1b:34:53:6e:f0:da:06:f8:19:3e:97:de:34:74:d1:4c:90: + e1:ce:6a:36:31:6e:58:d2:22:b1:5a:05:71:d8:0b:d9:c2:03: + 17:0d:98:78:f5:e2:24:7c:0a:7d:7b:49:4f:fe:31:a6:c3:0e: + 11:9e:af:6e:88:83:72:5a:34:a9:34:94:ef:6b:ee:cc:c1:71: + 5c:53:c6:dd:52:7e:a7:4c:9a:48:76:e9:72:b9:c4:26:74:87: + 64:c9:89:34:7d:bc:f2:ff:8a:ac:32:b5:3d:50:19:09:5f:30: + 19:49:6e:86:4e:84:e3:13:cc:9f:4c:a9:4a:20:89:5e:e3:91: + ad:8d:5e:3f:ac:ea:63:f1:48:18:f2:22:e9:b6:c3:6f:dc:b4: + 46:fc:41:71:33:ee:a7:4b:33:79:11:0f:c9:81:4d:10:c3:df: + b6:4d:75:62:74:39:e4:8d:5d:33:37:1b:91:ce:23:a3:47:15: + 58:57:5b:09:ba:4f:d5:1b:0f:4f:7b:03:10:d7:49:76:86:e0: + 69:7f:1a:7e:cb:6c:2a:80:b4:d8:9e:03:66:5c:89:3c:d3:82: + 86:d9:50:65:d9:15:51:e1:0b:3b:2f:e8:c7:44:6d:27:e3:09: + 2d:58:ce:a1:af:f9:d9:2f:0a:fd:fb:65:3d:3b:30:5a:42:b1: + ab:34:28:20:0d:a4:31:dd:84:65:eb:87:d1:59:33:1d:db:b1: + 64:e3:e5:6f:25:1a:15:ae:f1:39:b6:cc:91:d0:82:6e:e6:82: + 9e:f0:fc:c9:41:2b:a4:d7:b5:e7:af:1e:13:46:c0:e6:04:ac: + 98:53:ab:52:f3:85:bb:95:0d:b0:fb:e0:0a:c9:5e:da:99:ec: + 63:6c:7c:78:21:12:8d:21:6b:c3:bf:6c:cb:88:dc:c3:7a:24: + b9:4b:ba:36:63:b3:01:91:b3:07:a9:b0:1f:2c:ab:ae:d4:cd: + a7:a2:46:c0:29:df:1f:c2:29:d4:f9:49:9e:c5:e0:ca:02:f7: + eb:de:b8:b9:6e:1f:18:3a:6d:0f:07:0d:97:d2:16:0d:84:2c: + 81:24:c6:e6:e5:f5:e4:59 +-----BEGIN CERTIFICATE----- +MIIGIjCCBAqgAwIBAgIBATANBgkqhkiG9w0BAQsFADCBoTELMAkGA1UEBhMCVVMx +EDAOBgNVBAgMB0dlb3JnaWExEDAOBgNVBAcMB0F0bGFudGExEDAOBgNVBAoMB1Rl +c3QgQ0ExHDAaBgNVBAsME1Rlc3QgQ0EgT3JnYW56YXRpb24xHTAbBgNVBAMMFFRl +c3QgQ0EgT3JnYW5pemF0aW9uMR8wHQYJKoZIhvcNAQkBFhB0ZXN0QGV4YW1wbGUu +Y29tMCAXDTI0MDgwODE4MDYyMVoYDzIwNTExMjI1MTgwNjIxWjCBjDELMAkGA1UE +BhMCVVMxEDAOBgNVBAgMB0dlb3JnaWExEDAOBgNVBAcMB0F0bGFudGExFDASBgNV +BAoMC1Rlc3QgU2VydmVyMSAwHgYDVQQLDBdUZXN0IFNlcnZlciBPcmdhbnphdGlv +bjEhMB8GA1UEAwwYVGVzdCBTZXJ2ZXIgT3JnYW5pemF0aW9uMIICIjANBgkqhkiG +9w0BAQEFAAOCAg8AMIICCgKCAgEAvUfft9U4kq+0aedIO6B7nmuDDnaRBgaUo4Cj +c49Q5UOA+Pf7ZXvwo5TMjqZ+/llDzoBobVVnjjOqkHkhrN5u8AMnHm9QMc/SPsOO +mPW7+elEPz9ZrnyjuKeulP9ocND7e8vMNX0EgfUrEni/bhujzdF0QUGf7gIfs0L9 +yQG1KEPuMQM6XWDT349pHnNKxIM1lQCTg27WsNILMDF/levOyXODuXbrRfEgi3Xe +gaMysPcPIWSnHcw7AILISHTJOgv5y26Mq/ywlCC9YAbr0BIVVUjX0zDvWWeY3/Yx +km1jHEqTfJeomfZh5XgSNqIkVjdLOM5jAKImszEFkyM8we2x+yV9/FQEOrk69xek +WBBP6G2QaUm2HxuB+/XHbKqz4EqxOEB3g6KqjOJ8kak+zUO+kMPnsSOUR/lo2+Qs +32XniLZk3GLQhjObE2SUN6oOVp+jQhlnMKHpO1tK5uGBUoEhKniswXd3UvxKlbk/ +9+YynllbRkypihLTLPwzczooJigiTBypsVmWq6X26edVMqgrojPeoOJfd9jN0aof +T8ZpEGZOnap3g4J4lloHIRLbTJdRzbrqAM2Ul0C4UGKQK4ywGyyqpWMMu33VfT/B +SgBry3T6IzUmHiYaMLKWvBsWKmKWH1EgcpU2GocgJp921oQbZyoyaLfgx4B1o/q3 +2qMDccECAwEAAaN2MHQwHQYDVR0OBBYEFExX4nKXz9zEuE7b1MHGPa7v1woZMB8G +A1UdIwQYMBaAFFB2w1xrEuGSDyj48kOmgMKc6FbWMAkGA1UdEwQCMAAwCwYDVR0P +BAQDAgWgMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEAAAAADANBgkqhkiG9w0BAQsF +AAOCAgEAEJtm1UuP4nsli/5bnKbdTtXuJ62p5cRdm/ks8daNDtab5p+HCxQbyaPc +2oLQHujF9/TqmeoB8S588AcVKHQVsDYnpT8txzL8gWFEFZqaiCD7xtmKJmHf4gSi +VJh2kECYgNPr/3Mp1y8/ecq6wxs0U27w2gb4GT6X3jR00UyQ4c5qNjFuWNIisVoF +cdgL2cIDFw2YePXiJHwKfXtJT/4xpsMOEZ6vboiDclo0qTSU72vuzMFxXFPG3VJ+ +p0yaSHbpcrnEJnSHZMmJNH288v+KrDK1PVAZCV8wGUluhk6E4xPMn0ypSiCJXuOR +rY1eP6zqY/FIGPIi6bbDb9y0RvxBcTPup0szeREPyYFNEMPftk11YnQ55I1dMzcb +kc4jo0cVWFdbCbpP1RsPT3sDENdJdobgaX8afstsKoC02J4DZlyJPNOChtlQZdkV +UeELOy/ox0RtJ+MJLVjOoa/52S8K/ftlPTswWkKxqzQoIA2kMd2EZeuH0VkzHdux +ZOPlbyUaFa7xObbMkdCCbuaCnvD8yUErpNe1568eE0bA5gSsmFOrUvOFu5UNsPvg +Csle2pnsY2x8eCESjSFrw79sy4jcw3okuUu6NmOzAZGzB6mwHyyrrtTNp6JGwCnf +H8Ip1PlJnsXgygL36964uW4fGDptDwcNl9IWDYQsgSTG5uX15Fk= +-----END CERTIFICATE----- diff --git a/examples/features/advancedtls/creds/server_cert_revoked.pem b/examples/features/advancedtls/creds/server_cert_revoked.pem new file mode 100644 index 0000000..fe0a64d --- /dev/null +++ b/examples/features/advancedtls/creds/server_cert_revoked.pem @@ -0,0 +1,127 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 2 (0x2) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=Georgia, L=Atlanta, O=Test CA, OU=Test CA Organzation, CN=Test CA Organization/emailAddress=test@example.com + Validity + Not Before: Aug 8 18:06:21 2024 GMT + Not After : Dec 25 18:06:21 2051 GMT + Subject: C=US, ST=Georgia, L=Atlanta, O=Test server, OU=Test server Organzation, CN=Test server Organization + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (4096 bit) + Modulus: + 00:bd:47:df:b7:d5:38:92:af:b4:69:e7:48:3b:a0: + 7b:9e:6b:83:0e:76:91:06:06:94:a3:80:a3:73:8f: + 50:e5:43:80:f8:f7:fb:65:7b:f0:a3:94:cc:8e:a6: + 7e:fe:59:43:ce:80:68:6d:55:67:8e:33:aa:90:79: + 21:ac:de:6e:f0:03:27:1e:6f:50:31:cf:d2:3e:c3: + 8e:98:f5:bb:f9:e9:44:3f:3f:59:ae:7c:a3:b8:a7: + ae:94:ff:68:70:d0:fb:7b:cb:cc:35:7d:04:81:f5: + 2b:12:78:bf:6e:1b:a3:cd:d1:74:41:41:9f:ee:02: + 1f:b3:42:fd:c9:01:b5:28:43:ee:31:03:3a:5d:60: + d3:df:8f:69:1e:73:4a:c4:83:35:95:00:93:83:6e: + d6:b0:d2:0b:30:31:7f:95:eb:ce:c9:73:83:b9:76: + eb:45:f1:20:8b:75:de:81:a3:32:b0:f7:0f:21:64: + a7:1d:cc:3b:00:82:c8:48:74:c9:3a:0b:f9:cb:6e: + 8c:ab:fc:b0:94:20:bd:60:06:eb:d0:12:15:55:48: + d7:d3:30:ef:59:67:98:df:f6:31:92:6d:63:1c:4a: + 93:7c:97:a8:99:f6:61:e5:78:12:36:a2:24:56:37: + 4b:38:ce:63:00:a2:26:b3:31:05:93:23:3c:c1:ed: + b1:fb:25:7d:fc:54:04:3a:b9:3a:f7:17:a4:58:10: + 4f:e8:6d:90:69:49:b6:1f:1b:81:fb:f5:c7:6c:aa: + b3:e0:4a:b1:38:40:77:83:a2:aa:8c:e2:7c:91:a9: + 3e:cd:43:be:90:c3:e7:b1:23:94:47:f9:68:db:e4: + 2c:df:65:e7:88:b6:64:dc:62:d0:86:33:9b:13:64: + 94:37:aa:0e:56:9f:a3:42:19:67:30:a1:e9:3b:5b: + 4a:e6:e1:81:52:81:21:2a:78:ac:c1:77:77:52:fc: + 4a:95:b9:3f:f7:e6:32:9e:59:5b:46:4c:a9:8a:12: + d3:2c:fc:33:73:3a:28:26:28:22:4c:1c:a9:b1:59: + 96:ab:a5:f6:e9:e7:55:32:a8:2b:a2:33:de:a0:e2: + 5f:77:d8:cd:d1:aa:1f:4f:c6:69:10:66:4e:9d:aa: + 77:83:82:78:96:5a:07:21:12:db:4c:97:51:cd:ba: + ea:00:cd:94:97:40:b8:50:62:90:2b:8c:b0:1b:2c: + aa:a5:63:0c:bb:7d:d5:7d:3f:c1:4a:00:6b:cb:74: + fa:23:35:26:1e:26:1a:30:b2:96:bc:1b:16:2a:62: + 96:1f:51:20:72:95:36:1a:87:20:26:9f:76:d6:84: + 1b:67:2a:32:68:b7:e0:c7:80:75:a3:fa:b7:da:a3: + 03:71:c1 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 4C:57:E2:72:97:CF:DC:C4:B8:4E:DB:D4:C1:C6:3D:AE:EF:D7:0A:19 + X509v3 Authority Key Identifier: + 50:76:C3:5C:6B:12:E1:92:0F:28:F8:F2:43:A6:80:C2:9C:E8:56:D6 + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:0.0.0.0 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 07:74:84:18:37:74:23:9c:c2:f1:e8:d2:44:49:57:f8:51:fa: + cb:db:0e:42:04:6b:61:5b:60:f0:82:7a:df:1b:af:69:75:a8: + 17:62:89:18:b7:71:3e:8c:40:10:5d:2b:88:35:6a:97:9c:44: + 9f:93:24:f3:b8:d2:56:dd:2f:aa:27:55:96:67:07:fa:b1:8d: + 20:df:ea:f7:96:51:9e:46:e5:35:9a:34:53:d0:e7:60:da:a7: + 02:76:68:c2:12:6d:aa:bc:b6:81:e0:c9:96:67:b6:9e:fa:6d: + 43:63:80:19:70:49:9b:38:78:68:3d:aa:f2:5d:ec:af:45:65: + 4c:75:3c:d6:0b:92:8e:d7:7c:c9:76:55:51:ef:c6:d6:33:68: + 66:58:17:47:21:d7:14:4f:69:d1:59:1e:b2:78:bb:45:f4:24: + 8b:6b:ba:c4:83:6d:e8:11:c1:56:d8:df:84:3c:56:d2:e7:00: + 6c:b6:5c:f5:b8:33:e4:11:27:76:88:16:bd:d3:3d:ba:7b:d9: + 25:68:17:9c:0a:02:2f:d5:d0:57:b4:c9:f3:b1:9d:8e:6b:c9: + f1:6f:8f:39:8a:ad:0b:38:07:29:9b:cb:9a:3b:06:b5:03:1a: + 83:f4:ef:1e:91:a1:4b:eb:cf:fa:89:6f:91:47:5e:f2:bc:cb: + c2:8a:dd:7b:19:54:f4:9f:c7:54:7f:d2:e8:ea:a8:d9:c8:c1: + 6d:17:63:a3:47:30:05:5b:80:90:47:54:81:1f:0a:9b:11:48: + c6:ee:52:80:c3:b9:75:9d:d2:ee:1b:83:43:b2:de:05:aa:52: + d9:01:a3:f1:71:d3:23:90:28:35:25:0a:71:80:1d:ae:1a:6a: + 72:c1:2b:ee:a7:a2:72:54:f0:0e:19:87:97:a4:62:79:1a:ea: + ec:e2:73:b1:79:d5:c7:25:4f:c7:e6:a4:55:ad:be:3d:d7:59: + 8c:fb:ee:c3:2e:75:6d:1f:65:4a:be:46:c9:4e:54:bd:2e:49: + 3e:2f:70:b6:97:eb:8a:41:f4:bb:75:64:84:f4:71:29:e3:f2: + b2:30:75:41:5a:04:ac:a6:d1:d0:9c:4d:52:19:76:7f:0d:c7: + 08:f4:6e:cf:20:c7:3c:a6:d9:6f:72:88:46:16:0c:43:12:28: + 24:a1:d2:63:d3:04:4c:cd:12:67:1c:8f:00:e6:7b:47:0a:03: + 87:18:02:d6:bc:01:59:da:90:c4:c6:b1:72:b1:e6:a4:bc:23: + fd:5c:cf:32:0c:d9:e0:24:83:5b:55:7a:d0:db:3c:d6:b2:9f: + 22:a1:a0:f4:48:96:fb:d6:73:a1:43:f7:46:e0:ef:dd:b1:9a: + 0e:ef:6f:1d:1a:b4:b2:d4 +-----BEGIN CERTIFICATE----- +MIIGIjCCBAqgAwIBAgIBAjANBgkqhkiG9w0BAQsFADCBoTELMAkGA1UEBhMCVVMx +EDAOBgNVBAgMB0dlb3JnaWExEDAOBgNVBAcMB0F0bGFudGExEDAOBgNVBAoMB1Rl +c3QgQ0ExHDAaBgNVBAsME1Rlc3QgQ0EgT3JnYW56YXRpb24xHTAbBgNVBAMMFFRl +c3QgQ0EgT3JnYW5pemF0aW9uMR8wHQYJKoZIhvcNAQkBFhB0ZXN0QGV4YW1wbGUu +Y29tMCAXDTI0MDgwODE4MDYyMVoYDzIwNTExMjI1MTgwNjIxWjCBjDELMAkGA1UE +BhMCVVMxEDAOBgNVBAgMB0dlb3JnaWExEDAOBgNVBAcMB0F0bGFudGExFDASBgNV +BAoMC1Rlc3Qgc2VydmVyMSAwHgYDVQQLDBdUZXN0IHNlcnZlciBPcmdhbnphdGlv +bjEhMB8GA1UEAwwYVGVzdCBzZXJ2ZXIgT3JnYW5pemF0aW9uMIICIjANBgkqhkiG +9w0BAQEFAAOCAg8AMIICCgKCAgEAvUfft9U4kq+0aedIO6B7nmuDDnaRBgaUo4Cj +c49Q5UOA+Pf7ZXvwo5TMjqZ+/llDzoBobVVnjjOqkHkhrN5u8AMnHm9QMc/SPsOO +mPW7+elEPz9ZrnyjuKeulP9ocND7e8vMNX0EgfUrEni/bhujzdF0QUGf7gIfs0L9 +yQG1KEPuMQM6XWDT349pHnNKxIM1lQCTg27WsNILMDF/levOyXODuXbrRfEgi3Xe +gaMysPcPIWSnHcw7AILISHTJOgv5y26Mq/ywlCC9YAbr0BIVVUjX0zDvWWeY3/Yx +km1jHEqTfJeomfZh5XgSNqIkVjdLOM5jAKImszEFkyM8we2x+yV9/FQEOrk69xek +WBBP6G2QaUm2HxuB+/XHbKqz4EqxOEB3g6KqjOJ8kak+zUO+kMPnsSOUR/lo2+Qs +32XniLZk3GLQhjObE2SUN6oOVp+jQhlnMKHpO1tK5uGBUoEhKniswXd3UvxKlbk/ +9+YynllbRkypihLTLPwzczooJigiTBypsVmWq6X26edVMqgrojPeoOJfd9jN0aof +T8ZpEGZOnap3g4J4lloHIRLbTJdRzbrqAM2Ul0C4UGKQK4ywGyyqpWMMu33VfT/B +SgBry3T6IzUmHiYaMLKWvBsWKmKWH1EgcpU2GocgJp921oQbZyoyaLfgx4B1o/q3 +2qMDccECAwEAAaN2MHQwHQYDVR0OBBYEFExX4nKXz9zEuE7b1MHGPa7v1woZMB8G +A1UdIwQYMBaAFFB2w1xrEuGSDyj48kOmgMKc6FbWMAkGA1UdEwQCMAAwCwYDVR0P +BAQDAgWgMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEAAAAADANBgkqhkiG9w0BAQsF +AAOCAgEAB3SEGDd0I5zC8ejSRElX+FH6y9sOQgRrYVtg8IJ63xuvaXWoF2KJGLdx +PoxAEF0riDVql5xEn5Mk87jSVt0vqidVlmcH+rGNIN/q95ZRnkblNZo0U9DnYNqn +AnZowhJtqry2geDJlme2nvptQ2OAGXBJmzh4aD2q8l3sr0VlTHU81guSjtd8yXZV +Ue/G1jNoZlgXRyHXFE9p0Vkesni7RfQki2u6xINt6BHBVtjfhDxW0ucAbLZc9bgz +5BEndogWvdM9unvZJWgXnAoCL9XQV7TJ87GdjmvJ8W+POYqtCzgHKZvLmjsGtQMa +g/TvHpGhS+vP+olvkUde8rzLwordexlU9J/HVH/S6Oqo2cjBbRdjo0cwBVuAkEdU +gR8KmxFIxu5SgMO5dZ3S7huDQ7LeBapS2QGj8XHTI5AoNSUKcYAdrhpqcsEr7qei +clTwDhmHl6RieRrq7OJzsXnVxyVPx+akVa2+PddZjPvuwy51bR9lSr5GyU5UvS5J +Pi9wtpfrikH0u3VkhPRxKePysjB1QVoErKbR0JxNUhl2fw3HCPRuzyDHPKbZb3KI +RhYMQxIoJKHSY9METM0SZxyPAOZ7RwoDhxgC1rwBWdqQxMaxcrHmpLwj/VzPMgzZ +4CSDW1V60Ns81rKfIqGg9EiW+9ZzoUP3RuDv3bGaDu9vHRq0stQ= +-----END CERTIFICATE----- diff --git a/examples/features/advancedtls/creds/server_key.pem b/examples/features/advancedtls/creds/server_key.pem new file mode 100644 index 0000000..6f81dc2 --- /dev/null +++ b/examples/features/advancedtls/creds/server_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC9R9+31TiSr7Rp +50g7oHuea4MOdpEGBpSjgKNzj1DlQ4D49/tle/CjlMyOpn7+WUPOgGhtVWeOM6qQ +eSGs3m7wAyceb1Axz9I+w46Y9bv56UQ/P1mufKO4p66U/2hw0Pt7y8w1fQSB9SsS +eL9uG6PN0XRBQZ/uAh+zQv3JAbUoQ+4xAzpdYNPfj2kec0rEgzWVAJODbtaw0gsw +MX+V687Jc4O5dutF8SCLdd6BozKw9w8hZKcdzDsAgshIdMk6C/nLboyr/LCUIL1g +BuvQEhVVSNfTMO9ZZ5jf9jGSbWMcSpN8l6iZ9mHleBI2oiRWN0s4zmMAoiazMQWT +IzzB7bH7JX38VAQ6uTr3F6RYEE/obZBpSbYfG4H79cdsqrPgSrE4QHeDoqqM4nyR +qT7NQ76Qw+exI5RH+Wjb5CzfZeeItmTcYtCGM5sTZJQ3qg5Wn6NCGWcwoek7W0rm +4YFSgSEqeKzBd3dS/EqVuT/35jKeWVtGTKmKEtMs/DNzOigmKCJMHKmxWZarpfbp +51UyqCuiM96g4l932M3Rqh9PxmkQZk6dqneDgniWWgchEttMl1HNuuoAzZSXQLhQ +YpArjLAbLKqlYwy7fdV9P8FKAGvLdPojNSYeJhowspa8GxYqYpYfUSBylTYahyAm +n3bWhBtnKjJot+DHgHWj+rfaowNxwQIDAQABAoICAAg2RBMvtwKg3jecsdo/E4iY +qtjydUbzpiM/Li2R/DUrgT72qKY12FzgfyIj6xfnO4qMBlEoBr7OqF8YQkkZEBXD +ApbOxttCZEwWI+uoTagsDKqfJFRDqBQXglAzPI7DIlteniomlbl7BOFfnRPj3cQi +NY8B5TRoTIPJ1kTlmWcj0K5jsMvjADjkz0S478zS0dJNx23zsxt8zBYihPc2LISP +nxlperpQGdSzH8eJaGZGccDweFJR+AVaUeItiZrGcOdN5ostArmKf2zZIAX+TYVO +Yb68ksXS5CO4r1yQ+QnTL87qAttF1egkwMrfV1WTlI85tRCOoxXcfJzDnJvf+ia9 ++laYPj5av7ZoXwZQ4tIh3RqpIFHXZMQU1jRCYOibYNhdOO/H9Z6eH+8HyyFQ9ihe +7keGKouLSo/E6dSIJ9D1g+Tr8xELj816a4KL8ShYLTXP+ga43yFVdp94Yd0vFHWK +qjyu/x2wLqZAPpVbYg9PkO6Dr4tmyolhL1ZjruM5IqAI+hALzUZDccYQVXu2swpC +6p6evB24MI5LQ87+28U4rCcbo/xfdQAEY4LSP9XfvSQ32zUyDHYX1gLvF99fZD5Y +1IEX11bGbGFCT0EIwFPXJUuxJlpvMF1bZ+Z+eXVL43yTNdeu+7E7dhpPjz5W3cRa +6SKxxcUAbdxpQP2DtIP5AoIBAQD+AuqckaGK/sZotPWstgcRlG31ptW6d2ZNk5X3 +i99mqaTPDOx+UavkpsBfcUPYmYjo6fEC4yMhkx97XC9tUxi97LBXJrBfwLDnCqQL +vhOVcWs9mDcJTJDVT1P2StBm7a6DmCmiqUcxiG57UY9dgSanLwxg+6K4PgOin11Y +ChECWFNvdwdOmyQhWm+O0y0R2s7iGINKF6s1t2RweCfXw1W2N8iaTVkNei71bblW +sTjPGdEm2CFwlBf031Hkr5S2MIo23RX53tkalMzvyCGfREn+wrWcMfagygHjEjn/ +C4ZcQIOXC5u9tIc4tps1mr1C8nMScxfSrIHRManE1haqXcBLAoIBAQC+wznR5wzR +Az5r0ek1s2sr6gwsmJfbA6tNHwgqrYblsxYMR6BdbGUBkSqwEYDjDw/rreDcQk6D +nMxc6BYh2Dk6t/AJQ3c76tuJohlmZfbuFXL+KAwApVnbwpZuAQGgkNMhVm/8FAgi +PgCvuHuvOISTHmniQBU91kWGpsqFoF826Mcxa7bmbe2c+jgCmFikp0rzN7jA6Ps/ +bIRKVIpPhCtAgH5JsFEpim0HubV8qRNmeBh1oSyAFicntEAeL/VSLmDI44kjJyqO +qmspz+uyANt7/xxYfAZed1Q503K1tLUws1K8Ux+EEdo1zXGxoL6b90OfcjivJC0d +/bv8X5DEyMajAoIBAQCGYSabJBQxQ23V0P4zm60LqNmvXs6tMiOGIPDyoCXU2ySc +gPrQLQbiFTGqjHJXMYqTpcfiPiXEyl+aVH+mt5JcT85OnOIsFfXAlQmKSMl1gyY3 +1MIxAjeREcGah6PPACkV5zcHncRTORkx1kkhL4UyZxqGaDmCfRRRQTwRqmmrMu0Z +CABunnazynNAPQoX6wkN5ef3F6R064uQUJDLfcRnfQV8VDUrgxs6rgyiB2nFbqQO +h8LRGxe9bTOW5yimZfGI6teIdFOo01XD+L2I04jN5VZMxsXx9EyhQ3A5NHCld1/m +VbbT2qC66SgdaLp9o2QrO4Y75xVahYqJ3rTo9mYXAoIBAF6qrWfwPFkBPhntqsj+ +h+HcHTyIYVvL31e/XaMoSDh3fiqL5RZXs2xqqP+FQCvuDp2LxXoo4aPIzVYRyuHy +1rvACjveoi4258nOisJZOYh/VniwUPyFEinP0C05DKCtHkl+BsbW/g5YLKkHaUHU +T15fCnbADIqKaihfX0OfCYFLVYa+CJ8j0HZFakRHbD4R000Nyv7Y385iwOfOOnEp +ivlQittwx2ZRDrh1vY3mrfz8/k5ptJa/56B5gBQ7AohNAbTPzf+G8USpZ9LxHutQ +J5vKRzvWGKcKmt6zg0qPKhfH9fgFXC+DWIG4uYJH3i+yLnnTCjRIRKeMgpzEpCgz +5vcCggEBAIi6qKDHfoh4KaMFgW+/EvjssXslDtozMXlPYAswjST+5fegIKVxi8Nz +c19KRiWNpsIfACRC1VuI2p2tcJpamiHV+C3nd3e/CMGWKTcA6u6zynhO8pDjfg/A +k8Vg85S8bxGkiaVqL9DdmgRJohUbULV365gG2LvncNxps8Jr4VQaUHB4VJYNH+6B +DXwDb3N4iNs5wfmM2GB7MlPpu0pS6qoYSxZvXQexPFQ7sQzZc3mP04/BvHm11kSR +2DOV28IdXE/ewJfL9cr/ywXuoyz+0tD6FmTGUpzDSxhlvq4TcVnsWRbfqandbg8g +znviHVPPYZhKHQW5wGuGa6eMMWa6aog= +-----END PRIVATE KEY----- diff --git a/examples/features/advancedtls/creds/server_revoked.crl b/examples/features/advancedtls/creds/server_revoked.crl new file mode 100644 index 0000000..0cad864 --- /dev/null +++ b/examples/features/advancedtls/creds/server_revoked.crl @@ -0,0 +1,19 @@ +-----BEGIN X509 CRL----- +MIIDJzCCAQ8CAQEwDQYJKoZIhvcNAQELBQAwgaExCzAJBgNVBAYTAlVTMRAwDgYD +VQQIDAdHZW9yZ2lhMRAwDgYDVQQHDAdBdGxhbnRhMRAwDgYDVQQKDAdUZXN0IENB +MRwwGgYDVQQLDBNUZXN0IENBIE9yZ2FuemF0aW9uMR0wGwYDVQQDDBRUZXN0IENB +IE9yZ2FuaXphdGlvbjEfMB0GCSqGSIb3DQEJARYQdGVzdEBleGFtcGxlLmNvbRcN +MjQwODA4MTgwNjIxWhcNMjQwOTA3MTgwNjIxWjAUMBICAQIXDTI0MDgwODE4MDYy +MVqgIzAhMB8GA1UdIwQYMBaAFFB2w1xrEuGSDyj48kOmgMKc6FbWMA0GCSqGSIb3 +DQEBCwUAA4ICAQAvc4fK9H6OnUrtIzBls7ctVnmg4Up8NLWS9BWM9Ncw4MxLEXwM +4iD2Tgg5Yq51iE5pDNet5shae3l/kDNWigPixhpfiT8xwxnrOA3BUtD3affc0Nem +kS9Heq99jqvRdDF2nlEoiJesElxlSrwaljpev2SDzX/qnP5iRsBEuRS7Dr+83rcf +ysYt0Hd84MCaSJ2iITF7Kg7Zg7R+HV0O++k+ZXCuJkDDo8TGa+Kr87WtFsER1+SX +rtIgZEmikF/rEiKOHYV7QSujn8bzSsKzW2S/8/xygQnZ39vk5OPKKokPMUqesMPE +8hUwGjCnDDijj6WGyP54FHKcH61P6R5a1EsjNfTUli9J7BNSHjb7AqPFRMCz6Ihj +GUidGqCz4mpxkUpOwGJZWyefeWlKCdoBqDHlRtf8EjdE9BClOacJeVx2dmXd7k5r +y6grIFfNjHYJmVa8+o2YCV80wY9XFtRUsGpEwHFTrtAjec+y2gtILKpsv5aqyvOa ++nffFdMAR05Dx4MeFJSKYQGk64hPmgrRK+MGc224JPmQDi9uMwDKo+jB1TkCgwvR +ZTF0VpPWBfhkB6x2haYyq2whf6zHfR+jMA0npUA8vHSUIiQrgWbvwsCc3nFGt4nz +WjEZ7Q8Sw9CBfcvXrSV+WQdJF6kHgwaiI56x7DvdEAoDxuDLbmb+D/5gIg== +-----END X509 CRL----- diff --git a/examples/features/advancedtls/generate.sh b/examples/features/advancedtls/generate.sh new file mode 100755 index 0000000..94d7263 --- /dev/null +++ b/examples/features/advancedtls/generate.sh @@ -0,0 +1,91 @@ +#!/bin/bash +if [ -d "creds" ]; then + echo "creds directory already exists. Remove it and re-run this script." + exit 1 +fi +mkdir creds +pushd creds +touch index.txt +echo "01" > serial.txt +cp "../localhost-openssl.cnf" . +cp "../openssl-ca.cnf" . + +# Create the CA private key and certificate +openssl req -x509 -newkey rsa:4096 -keyout ca_key.pem -out ca_cert.pem -nodes -days 3650 -subj "/C=US/ST=Georgia/L=Atlanta/O=Test CA/OU=Test CA Organzation/CN=Test CA Organization/emailAddress=test@example.com" + +#################### Server cert + +# Generate Server private key +openssl genrsa -out server_key.pem 4096 + +# Generate Server Certificate Signing Request (CSR) +openssl req -config "localhost-openssl.cnf" -new -key server_key.pem -out server_csr.pem -subj "/C=US/ST=Georgia/L=Atlanta/O=Test Server/OU=Test Server Organzation/CN=Test Server Organization/emailAddress=testserver@example.com" + +# Use the CA to sign the Server CSR +openssl ca -config "openssl-ca.cnf" -policy signing_policy -extensions signing_req -out server_cert.pem -in server_csr.pem -keyfile ca_key.pem -cert ca_cert.pem -batch + +# Verify the server cert works +openssl verify -verbose -CAfile ca_cert.pem server_cert.pem + +## Generate another server cert to be revoked +openssl req -config "localhost-openssl.cnf" -new -key server_key.pem -out server_csr_revoked.pem -subj "/C=US/ST=Georgia/L=Atlanta/O=Test server/OU=Test server Organzation/CN=Test server Organization/emailAddress=testserver@example.com" + +# Use the CA to sign the server CSR +openssl ca -config "openssl-ca.cnf" -policy signing_policy -extensions signing_req -out server_cert_revoked.pem -in server_csr_revoked.pem -keyfile ca_key.pem -cert ca_cert.pem -batch + +# Verify the server cert works +openssl verify -verbose -CAfile ca_cert.pem server_cert_revoked.pem + +# Revoke the cert +openssl ca -config "openssl-ca.cnf" -revoke server_cert_revoked.pem + +# Generate the CRL +openssl ca -config "openssl-ca.cnf" -gencrl -out server_revoked.crl + +# Make sure the cert is actually revoked +openssl verify -verbose -CAfile ca_cert.pem -CRLfile server_revoked.crl -crl_check_all server_cert_revoked.pem + +#################### Client cert +# Generate client private key +openssl genrsa -out client_key.pem 4096 + +# Generate client Certificate Signing Request (CSR) +openssl req -config "localhost-openssl.cnf" -new -key client_key.pem -out client_csr.pem -subj "/C=US/ST=Georgia/L=Atlanta/O=Test client/OU=Test client Organzation/CN=Test client Organization/emailAddress=testclient@example.com" + +# Use the CA to sign the client CSR +openssl ca -config "openssl-ca.cnf" -policy signing_policy -extensions signing_req -out client_cert.pem -in client_csr.pem -keyfile ca_key.pem -cert ca_cert.pem -batch + +# Verify the client cert works +openssl verify -verbose -CAfile ca_cert.pem client_cert.pem + +## Generate another client cert to be revoked +openssl req -config "localhost-openssl.cnf" -new -key client_key.pem -out client_csr_revoked.pem -subj "/C=US/ST=Georgia/L=Atlanta/O=Test client/OU=Test client Organzation/CN=Test client Organization/emailAddress=testclient@example.com" + +# Use the CA to sign the client CSR +openssl ca -config "openssl-ca.cnf" -policy signing_policy -extensions signing_req -out client_cert_revoked.pem -in client_csr_revoked.pem -keyfile ca_key.pem -cert ca_cert.pem -batch + +# Verify the client cert works +openssl verify -verbose -CAfile ca_cert.pem client_cert_revoked.pem + +# Revoke the cert +openssl ca -config "openssl-ca.cnf" -revoke client_cert_revoked.pem + +# Generate the CRL +openssl ca -config "openssl-ca.cnf" -gencrl -out client_revoked.crl + +# Make sure the cert is actually revoked +openssl verify -verbose -CAfile ca_cert.pem -CRLfile client_revoked.crl -crl_check_all client_cert_revoked.pem + +mkdir crl +mv client_revoked.crl crl/ + +rm 01.pem +rm 02.pem +rm 03.pem +rm 04.pem +rm *csr* +rm *.txt* + + + +popd diff --git a/examples/features/advancedtls/localhost-openssl.cnf b/examples/features/advancedtls/localhost-openssl.cnf new file mode 100644 index 0000000..bacdb00 --- /dev/null +++ b/examples/features/advancedtls/localhost-openssl.cnf @@ -0,0 +1,24 @@ +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req + +[req_distinguished_name] +countryName = Country Name (2 letter code) +countryName_default = US +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = Georgia +localityName = Locality Name (eg, city) +localityName_default = Atlanta +organizationName = Organization Name (eg, company) +organizationName_default = Test Department +commonName = Common Name (eg, YOUR name) +commonName_max = 64 + +[v3_req] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names + +[alt_names] +DNS.1 = localhost +IP.1 = 0.0.0.0 diff --git a/examples/features/advancedtls/openssl-ca.cnf b/examples/features/advancedtls/openssl-ca.cnf new file mode 100644 index 0000000..31e0b39 --- /dev/null +++ b/examples/features/advancedtls/openssl-ca.cnf @@ -0,0 +1,94 @@ +base_dir = . +certificate = $base_dir/ca_cert.pem # The CA certificate +private_key = $base_dir/ca_key.pem # The CA private key +new_certs_dir = $base_dir # Location for new certs after signing +database = $base_dir/index.txt # Database index file +serial = $base_dir/serial.txt # The current serial number + +unique_subject = no # Set to 'no' to allow creation of + # several certificates with same subject. + +HOME = . +RANDFILE = $ENV::HOME/.rnd + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +[ CA_default ] + +default_days = 10000 # How long to certify for +default_crl_days = 30 # How long before next CRL +default_md = sha256 # Use public key default MD +preserve = no # Keep passed DN ordering + +x509_extensions = ca_extensions # The extensions to add to the cert +crl_extensions = crl_ext + +email_in_dn = no # Don't concat the email in the DN +copy_extensions = copy # Required to copy SANs from CSR to cert + +#################################################################### +[ req ] +default_bits = 4096 +default_keyfile = ca_key.pem +distinguished_name = ca_distinguished_name +x509_extensions = ca_extensions +string_mask = utf8only + +#################################################################### +[ ca_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = Georgia + +localityName = Locality Name (eg, city) +localityName_default = Atlanta + +organizationName = Organization Name (eg, company) +organizationName_default = Test CA + +organizationalUnitName = Organizational Unit (eg, division) +organizationalUnitName_default = Test CA Organization + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_default = Test CA Organization + +emailAddress = Email Address +emailAddress_default = test@example.com + +#################################################################### +[ ca_extensions ] + +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always, issuer +basicConstraints = critical, CA:true +keyUsage = keyCertSign, cRLSign + + + + +#################################################################### +[ signing_policy ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ signing_req ] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment + +[ crl_ext ] +# CRL extensions. +# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL. +#issuerAltName=issuer:copy +authorityKeyIdentifier=keyid:always diff --git a/examples/features/advancedtls/server/main.go b/examples/features/advancedtls/server/main.go new file mode 100644 index 0000000..d8b79a1 --- /dev/null +++ b/examples/features/advancedtls/server/main.go @@ -0,0 +1,177 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is an example client demonstrating how to set up a secure gRPC +// server using advancedtls. +package main + +import ( + "context" + "flag" + "fmt" + "net" + "os" + "path/filepath" + "time" + + pb "google.golang.org/grpc/examples/features/proto/echo" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/credentials/tls/certprovider/pemfile" + "google.golang.org/grpc/security/advancedtls" +) + +type server struct { + pb.UnimplementedEchoServer + name string +} + +const credRefreshInterval = 1 * time.Minute +const goodServerWithCRLPort int = 50051 +const revokedServerWithCRLPort int = 50053 +const insecurePort int = 50054 + +func (s *server) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: req.Message}, nil +} + +func insecureServer() { + createAndRunInsecureServer(insecurePort) +} + +func createAndRunInsecureServer(port int) { + creds := insecure.NewCredentials() + s := grpc.NewServer(grpc.Creds(creds)) + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + fmt.Printf("Failed to listen: %v\n", err) + } + pb.RegisterEchoServer(s, &server{name: "Insecure Server"}) + if err := s.Serve(lis); err != nil { + fmt.Printf("Failed to serve: %v\n", err) + os.Exit(1) + } +} + +func createAndRunTLSServer(credsDirectory string, useRevokedCert bool, port int) { + identityProvider := makeIdentityProvider(useRevokedCert, credsDirectory) + defer identityProvider.Close() + + rootProvider := makeRootProvider(credsDirectory) + defer rootProvider.Close() + + crlProvider := makeCRLProvider(filepath.Join(credsDirectory, "crl")) + defer crlProvider.Close() + + options := &advancedtls.Options{ + IdentityOptions: advancedtls.IdentityCertificateOptions{ + IdentityProvider: identityProvider, + }, + RootOptions: advancedtls.RootCertificateOptions{ + RootProvider: rootProvider, + }, + RequireClientCert: true, + VerificationType: advancedtls.CertVerification, + } + + options.RevocationOptions = &advancedtls.RevocationOptions{ + CRLProvider: crlProvider, + } + + serverTLSCreds, err := advancedtls.NewServerCreds(options) + if err != nil { + fmt.Printf("Error %v\n", err) + os.Exit(1) + } + + s := grpc.NewServer(grpc.Creds(serverTLSCreds)) + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + fmt.Printf("Failed to listen: %v\n", err) + } + name := "Good TLS Server" + if useRevokedCert { + name = "Revoked TLS Server" + } + pb.RegisterEchoServer(s, &server{name: name}) + if err := s.Serve(lis); err != nil { + fmt.Printf("Failed to serve: %v\n", err) + os.Exit(1) + } + +} + +func makeRootProvider(credsDirectory string) certprovider.Provider { + rootOptions := pemfile.Options{ + RootFile: filepath.Join(credsDirectory, "/ca_cert.pem"), + RefreshDuration: credRefreshInterval, + } + + rootProvider, err := pemfile.NewProvider(rootOptions) + if err != nil { + fmt.Printf("Error %v\n", err) + os.Exit(1) + } + return rootProvider +} + +func makeIdentityProvider(useRevokedCert bool, credsDirectory string) certprovider.Provider { + certFilePath := "" + if useRevokedCert { + certFilePath = filepath.Join(credsDirectory, "server_cert_revoked.pem") + } else { + certFilePath = filepath.Join(credsDirectory, "server_cert.pem") + } + identityOptions := pemfile.Options{ + CertFile: certFilePath, + KeyFile: filepath.Join(credsDirectory, "server_key.pem"), + RefreshDuration: credRefreshInterval, + } + identityProvider, err := pemfile.NewProvider(identityOptions) + if err != nil { + fmt.Printf("Error %v\n", err) + os.Exit(1) + } + return identityProvider +} + +func makeCRLProvider(crlDirectory string) *advancedtls.FileWatcherCRLProvider { + options := advancedtls.FileWatcherOptions{ + CRLDirectory: crlDirectory, + } + provider, err := advancedtls.NewFileWatcherCRLProvider(options) + if err != nil { + fmt.Printf("Error making CRL Provider: %v\nExiting...", err) + os.Exit(1) + } + return provider +} + +func main() { + credentialsDirectory := flag.String("credentials_directory", "", "Path to the creds directory of this repo") + flag.Parse() + if *credentialsDirectory == "" { + fmt.Println("Must set credentials_directory argument") + os.Exit(1) + } + go createAndRunTLSServer(*credentialsDirectory, false, goodServerWithCRLPort) + go createAndRunTLSServer(*credentialsDirectory, true, revokedServerWithCRLPort) + insecureServer() +} diff --git a/examples/features/authentication/README.md b/examples/features/authentication/README.md new file mode 100644 index 0000000..c0c52eb --- /dev/null +++ b/examples/features/authentication/README.md @@ -0,0 +1,62 @@ +# Authentication + +In grpc, authentication is abstracted as +[`credentials.PerRPCCredentials`](https://godoc.org/google.golang.org/grpc/credentials#PerRPCCredentials). +It usually also encompasses authorization. Users can configure it on a +per-connection basis or a per-call basis. + +The example for authentication currently includes an example for using oauth2 +with grpc. + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +### OAuth2 + +OAuth 2.0 Protocol is a widely used authentication and authorization mechanism +nowadays. And grpc provides convenient APIs to configure OAuth to use with grpc. +Please refer to the godoc: +https://godoc.org/google.golang.org/grpc/credentials/oauth for details. + +#### Client + +On client side, users should first get a valid oauth token, and then initialize a +[`oauth.TokenSource`](https://godoc.org/google.golang.org/grpc/credentials/oauth#TokenSource) +which implements `credentials.PerRPCCredentials`. Next, if user wants to +apply a single OAuth token for all RPC calls on the same connection, then +configure grpc `Dial` with `DialOption` +[`WithPerRPCCredentials`](https://godoc.org/google.golang.org/grpc#WithPerRPCCredentials). +Or, if user wants to apply OAuth token per call, then configure the grpc RPC +call with `CallOption` +[`PerRPCCredentials`](https://godoc.org/google.golang.org/grpc#PerRPCCredentials). + +Note that OAuth requires the underlying transport to be secure (e.g. TLS, etc.) + +Inside grpc, the provided token is prefixed with the token type and a space, and +is then attached to the metadata with the key "authorization". + +### Server + +On server side, users usually get the token and verify it inside an interceptor. +To get the token, call +[`metadata.FromIncomingContext`](https://godoc.org/google.golang.org/grpc/metadata#FromIncomingContext) +on the given context. It returns the metadata map. Next, use the key +"authorization" to get corresponding value, which is a slice of strings. For +OAuth, the slice should only contain one element, which is a string in the +format of ` + " " + `. Users can easily get the token by +parsing the string, and then verify the validity of it. + +If the token is not valid, returns an error with error code +`codes.Unauthenticated`. + +If the token is valid, then invoke the method handler to start processing the +RPC. diff --git a/examples/features/authentication/client/main.go b/examples/features/authentication/client/main.go new file mode 100644 index 0000000..fb5e63b --- /dev/null +++ b/examples/features/authentication/client/main.go @@ -0,0 +1,86 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// The client demonstrates how to supply an OAuth2 token for every RPC. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "golang.org/x/oauth2" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/oauth" + "google.golang.org/grpc/examples/data" + ecpb "google.golang.org/grpc/examples/features/proto/echo" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +func callUnaryEcho(client ecpb.EchoClient, message string) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := client.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message}) + if err != nil { + log.Fatalf("client.UnaryEcho(_) = _, %v: ", err) + } + fmt.Println("UnaryEcho: ", resp.Message) +} + +func main() { + flag.Parse() + + // Set up the credentials for the connection. + perRPC := oauth.TokenSource{TokenSource: oauth2.StaticTokenSource(fetchToken())} + creds, err := credentials.NewClientTLSFromFile(data.Path("x509/ca_cert.pem"), "x.test.example.com") + if err != nil { + log.Fatalf("failed to load credentials: %v", err) + } + opts := []grpc.DialOption{ + // In addition to the following grpc.DialOption, callers may also use + // the grpc.CallOption grpc.PerRPCCredentials with the RPC invocation + // itself. + // See: https://godoc.org/google.golang.org/grpc#PerRPCCredentials + grpc.WithPerRPCCredentials(perRPC), + // oauth.TokenSource requires the configuration of transport + // credentials. + grpc.WithTransportCredentials(creds), + } + + conn, err := grpc.NewClient(*addr, opts...) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + rgc := ecpb.NewEchoClient(conn) + + callUnaryEcho(rgc, "hello world") +} + +// fetchToken simulates a token lookup and omits the details of proper token +// acquisition. For examples of how to acquire an OAuth2 token, see: +// https://godoc.org/golang.org/x/oauth2 +func fetchToken() *oauth2.Token { + return &oauth2.Token{ + AccessToken: "some-secret-token", + } +} diff --git a/examples/features/authentication/server/main.go b/examples/features/authentication/server/main.go new file mode 100644 index 0000000..6c4506c --- /dev/null +++ b/examples/features/authentication/server/main.go @@ -0,0 +1,112 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// The server demonstrates how to consume and validate OAuth2 tokens provided by +// clients for each RPC. +package main + +import ( + "context" + "crypto/tls" + "flag" + "fmt" + "log" + "net" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/examples/data" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var ( + errMissingMetadata = status.Errorf(codes.InvalidArgument, "missing metadata") + errInvalidToken = status.Errorf(codes.Unauthenticated, "invalid token") +) + +var port = flag.Int("port", 50051, "the port to serve on") + +func main() { + flag.Parse() + fmt.Printf("server starting on port %d...\n", *port) + + cert, err := tls.LoadX509KeyPair(data.Path("x509/server_cert.pem"), data.Path("x509/server_key.pem")) + if err != nil { + log.Fatalf("failed to load key pair: %s", err) + } + opts := []grpc.ServerOption{ + // The following grpc.ServerOption adds an interceptor for all unary + // RPCs. To configure an interceptor for streaming RPCs, see: + // https://godoc.org/google.golang.org/grpc#StreamInterceptor + grpc.UnaryInterceptor(ensureValidToken), + // Enable TLS for all incoming connections. + grpc.Creds(credentials.NewServerTLSFromCert(&cert)), + } + s := grpc.NewServer(opts...) + pb.RegisterEchoServer(s, &ecServer{}) + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} + +type ecServer struct { + pb.UnimplementedEchoServer +} + +func (s *ecServer) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: req.Message}, nil +} + +// valid validates the authorization. +func valid(authorization []string) bool { + if len(authorization) < 1 { + return false + } + token := strings.TrimPrefix(authorization[0], "Bearer ") + // Perform the token validation here. For the sake of this example, the code + // here forgoes any of the usual OAuth2 token validation and instead checks + // for a token matching an arbitrary string. + return token == "some-secret-token" +} + +// ensureValidToken ensures a valid token exists within a request's metadata. If +// the token is missing or invalid, the interceptor blocks execution of the +// handler and returns an error. Otherwise, the interceptor invokes the unary +// handler. +func ensureValidToken(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, errMissingMetadata + } + // The keys within metadata.MD are normalized to lowercase. + // See: https://godoc.org/google.golang.org/grpc/metadata#New + if !valid(md["authorization"]) { + return nil, errInvalidToken + } + // Continue execution of handler after ensuring a valid token. + return handler(ctx, req) +} diff --git a/examples/features/authz/README.md b/examples/features/authz/README.md new file mode 100644 index 0000000..b896474 --- /dev/null +++ b/examples/features/authz/README.md @@ -0,0 +1,75 @@ +# RBAC authorization + +This example uses the `StaticInterceptor` and `FileWatcherInterceptor` from the +`google.golang.org/grpc/authz` package. It uses a header based RBAC policy to +match each gRPC method to a required role. For simplicity, the context is +injected with mock metadata which includes the required roles, but this should +be fetched from an appropriate service based on the authenticated context. + +## Try it + +Server is expected to require the following roles on an authenticated user to +authorize usage of these methods: + +- `UnaryEcho` requires the role `UNARY_ECHO:W` +- `BidirectionalStreamingEcho` requires the role `STREAM_ECHO:RW` + +Upon receiving a request, the server first checks that a token was supplied, +decodes it and checks that a secret is correctly set (hardcoded to +`super-secret` for simplicity, this should use a proper ID provider in +production). + +If the above is successful, it uses the username in the token to set appropriate +roles (hardcoded to the 2 required roles above if the username matches +`super-user` for simplicity, these roles should be supplied externally as well). + +### Authorization with static policy + +Start the server with: + +``` +go run server/main.go +``` + +The client implementation shows how using a valid token (setting username and +secret) with each of the endpoints will return successfully. It also exemplifies +how using a bad token will result in `codes.PermissionDenied` being returned +from the service. + +Start the client with: + +``` +go run client/main.go +``` + +### Authorization by watching a policy file + +The server accepts an optional `--authz-option filewatcher` flag to set up +authorization policy by reading a [policy +file](/examples/data/rbac/policy.json), and to look for update on the policy +file every 100 millisecond. Having `GRPC_GO_LOG_SEVERITY_LEVEL` environment +variable set to `info` will log out the reload activity of the policy every time +a file update is detected. + +Start the server with: + +``` +GRPC_GO_LOG_SEVERITY_LEVEL=info go run server/main.go --authz-option filewatcher +``` + +Start the client with: + +``` +go run client/main.go +``` + +The client will first hit `codes.PermissionDenied` error when invoking +`UnaryEcho` although a legitimate username (`super-user`) is associated with the +RPC. This is because the policy file has an intentional glitch (falsely asks for +role `UNARY_ECHO:RW`). + +While the server is still running, edit and save the policy file to replace +`UNARY_ECHO:RW` with the correct role `UNARY_ECHO:W` (policy reload activity +should now be found in server logs). This time when the client is started again +with the command above, it will be able to get responses just as in the +static-policy example. diff --git a/examples/features/authz/client/main.go b/examples/features/authz/client/main.go new file mode 100644 index 0000000..db05ab2 --- /dev/null +++ b/examples/features/authz/client/main.go @@ -0,0 +1,141 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to include authorization credentials in the +// form of metadata in every RPC for server side validation. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "time" + + "golang.org/x/oauth2" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/oauth" + "google.golang.org/grpc/examples/data" + "google.golang.org/grpc/examples/features/authz/token" + ecpb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/status" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +func callUnaryEcho(ctx context.Context, client ecpb.EchoClient, message string, opts ...grpc.CallOption) error { + resp, err := client.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message}, opts...) + if err != nil { + return status.Errorf(status.Code(err), "UnaryEcho RPC failed: %v", err) + } + fmt.Println("UnaryEcho: ", resp.Message) + return nil +} + +func callBidiStreamingEcho(ctx context.Context, client ecpb.EchoClient, opts ...grpc.CallOption) error { + c, err := client.BidirectionalStreamingEcho(ctx, opts...) + if err != nil { + return status.Errorf(status.Code(err), "BidirectionalStreamingEcho RPC failed: %v", err) + } + for i := 0; i < 5; i++ { + err := c.Send(&ecpb.EchoRequest{Message: fmt.Sprintf("Request %d", i+1)}) + if err == io.EOF { + // Bidi streaming RPC errors happen and make Send return io.EOF, + // not the RPC error itself. Call Recv to determine the error. + break + } + if err != nil { + // Some local errors are reported this way, e.g. errors serializing + // the request message. + return status.Errorf(status.Code(err), "sending StreamingEcho message: %v", err) + } + } + c.CloseSend() + for { + resp, err := c.Recv() + if err == io.EOF { + break + } + if err != nil { + return status.Errorf(status.Code(err), "receiving StreamingEcho message: %v", err) + } + fmt.Println("BidiStreaming Echo: ", resp.Message) + } + return nil +} + +func newCredentialsCallOption(t token.Token) grpc.CallOption { + tokenBase64, err := t.Encode() + if err != nil { + log.Fatalf("encoding token: %v", err) + } + oath2Token := oauth2.Token{AccessToken: tokenBase64} + return grpc.PerRPCCredentials(oauth.TokenSource{TokenSource: oauth2.StaticTokenSource(&oath2Token)}) +} + +func main() { + flag.Parse() + + // Create tls based credential. + creds, err := credentials.NewClientTLSFromFile(data.Path("x509/ca_cert.pem"), "x.test.example.com") + if err != nil { + log.Fatalf("failed to load credentials: %v", err) + } + // Set up a connection to the server. + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(creds)) + if err != nil { + log.Fatalf("grpc.NewClient(%q): %v", *addr, err) + } + defer conn.Close() + + // Make an echo client and send RPCs. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + client := ecpb.NewEchoClient(conn) + + // Make RPCs as an authorized user and expect them to succeed. + authorisedUserTokenCallOption := newCredentialsCallOption(token.Token{Username: "super-user", Secret: "super-secret"}) + if err := callUnaryEcho(ctx, client, "hello world", authorisedUserTokenCallOption); err != nil { + log.Fatalf("Unary RPC by authorized user failed: %v", err) + } + if err := callBidiStreamingEcho(ctx, client, authorisedUserTokenCallOption); err != nil { + log.Fatalf("Bidirectional RPC by authorized user failed: %v", err) + } + + // Make RPCs as an unauthorized user and expect them to fail with status code PermissionDenied. + unauthorisedUserTokenCallOption := newCredentialsCallOption(token.Token{Username: "bad-actor", Secret: "super-secret"}) + if err := callUnaryEcho(ctx, client, "hello world", unauthorisedUserTokenCallOption); err != nil { + switch c := status.Code(err); c { + case codes.PermissionDenied: + log.Printf("Unary RPC by unauthorized user failed as expected: %v", err) + default: + log.Fatalf("Unary RPC by unauthorized user failed unexpectedly: %v, %v", c, err) + } + } + if err := callBidiStreamingEcho(ctx, client, unauthorisedUserTokenCallOption); err != nil { + switch c := status.Code(err); c { + case codes.PermissionDenied: + log.Printf("Bidirectional RPC by unauthorized user failed as expected: %v", err) + default: + log.Fatalf("Bidirectional RPC by unauthorized user failed unexpectedly: %v", err) + } + } +} diff --git a/examples/features/authz/server/main.go b/examples/features/authz/server/main.go new file mode 100644 index 0000000..9e699ca --- /dev/null +++ b/examples/features/authz/server/main.go @@ -0,0 +1,238 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to validate authorization credential metadata +// for incoming RPCs. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "log" + "net" + "strings" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/authz" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/examples/data" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + "google.golang.org/grpc/examples/features/authz/token" + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +const ( + unaryEchoWriterRole = "UNARY_ECHO:W" + streamEchoReadWriterRole = "STREAM_ECHO:RW" + authzPolicy = ` + { + "name": "authz", + "allow_rules": [ + { + "name": "allow_UnaryEcho", + "request": { + "paths": ["/grpc.examples.echo.Echo/UnaryEcho"], + "headers": [ + { + "key": "UNARY_ECHO:W", + "values": ["true"] + } + ] + } + }, + { + "name": "allow_BidirectionalStreamingEcho", + "request": { + "paths": ["/grpc.examples.echo.Echo/BidirectionalStreamingEcho"], + "headers": [ + { + "key": "STREAM_ECHO:RW", + "values": ["true"] + } + ] + } + } + ], + "deny_rules": [] + } + ` + authzOptStatic = "static" + authzOptFileWatcher = "filewatcher" +) + +var ( + port = flag.Int("port", 50051, "the port to serve on") + authzOpt = flag.String("authz-option", authzOptStatic, "the authz option (static or filewatcher)") + + errMissingMetadata = status.Errorf(codes.InvalidArgument, "missing metadata") +) + +func newContextWithRoles(ctx context.Context, username string) context.Context { + md := metadata.MD{} + if username == "super-user" { + md.Set(unaryEchoWriterRole, "true") + md.Set(streamEchoReadWriterRole, "true") + } + return metadata.NewIncomingContext(ctx, md) +} + +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(_ context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { + fmt.Printf("unary echoing message %q\n", in.Message) + return &pb.EchoResponse{Message: in.Message}, nil +} + +func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { + for { + in, err := stream.Recv() + if err != nil { + if err == io.EOF { + return nil + } + fmt.Printf("Receiving message from stream: %v\n", err) + return err + } + fmt.Printf("bidi echoing message %q\n", in.Message) + stream.Send(&pb.EchoResponse{Message: in.Message}) + } +} + +// isAuthenticated validates the authorization. +func isAuthenticated(authorization []string) (username string, err error) { + if len(authorization) < 1 { + return "", errors.New("received empty authorization token from client") + } + tokenBase64 := strings.TrimPrefix(authorization[0], "Bearer ") + // Perform the token validation here. For the sake of this example, the code + // here forgoes any of the usual OAuth2 token validation and instead checks + // for a token matching an arbitrary string. + var token token.Token + err = token.Decode(tokenBase64) + if err != nil { + return "", fmt.Errorf("base64 decoding of received token %q: %v", tokenBase64, err) + } + if token.Secret != "super-secret" { + return "", fmt.Errorf("received token %q does not match expected %q", token.Secret, "super-secret") + } + return token.Username, nil +} + +// authUnaryInterceptor looks up the authorization header from the incoming RPC context, +// retrieves the username from it and creates a new context with the username before invoking +// the provided handler. +func authUnaryInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, errMissingMetadata + } + username, err := isAuthenticated(md["authorization"]) + if err != nil { + return nil, status.Error(codes.Unauthenticated, err.Error()) + } + return handler(newContextWithRoles(ctx, username), req) +} + +// wrappedStream wraps a grpc.ServerStream associated with an incoming RPC, and +// a custom context containing the username derived from the authorization header +// specified in the incoming RPC metadata +type wrappedStream struct { + grpc.ServerStream + ctx context.Context +} + +func (w *wrappedStream) Context() context.Context { + return w.ctx +} + +func newWrappedStream(ctx context.Context, s grpc.ServerStream) grpc.ServerStream { + return &wrappedStream{s, ctx} +} + +// authStreamInterceptor looks up the authorization header from the incoming RPC context, +// retrieves the username from it and creates a new context with the username before invoking +// the provided handler. +func authStreamInterceptor(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + md, ok := metadata.FromIncomingContext(ss.Context()) + if !ok { + return errMissingMetadata + } + username, err := isAuthenticated(md["authorization"]) + if err != nil { + return status.Error(codes.Unauthenticated, err.Error()) + } + return handler(srv, newWrappedStream(newContextWithRoles(ss.Context(), username), ss)) +} + +func main() { + flag.Parse() + + if *authzOpt != authzOptStatic && *authzOpt != authzOptFileWatcher { + log.Fatalf("Invalid authz option: %s", *authzOpt) + } + + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("Listening on local port %q: %v", *port, err) + } + + // Create tls based credential. + creds, err := credentials.NewServerTLSFromFile(data.Path("x509/server_cert.pem"), data.Path("x509/server_key.pem")) + if err != nil { + log.Fatalf("Loading credentials: %v", err) + } + + // Create authorization interceptors according to the authz-option command-line flag. + var unaryAuthzInterceptor grpc.UnaryServerInterceptor + var streamAuthzInterceptor grpc.StreamServerInterceptor + if *authzOpt == authzOptStatic { + // Create an authorization interceptor using a static policy. + staticInterceptor, err := authz.NewStatic(authzPolicy) + if err != nil { + log.Fatalf("Creating a static authz interceptor: %v", err) + } + unaryAuthzInterceptor, streamAuthzInterceptor = staticInterceptor.UnaryInterceptor, staticInterceptor.StreamInterceptor + } else if *authzOpt == authzOptFileWatcher { + // Create an authorization interceptor by watching a policy file. + fileWatcherInterceptor, err := authz.NewFileWatcher(data.Path("rbac/policy.json"), 100*time.Millisecond) + if err != nil { + log.Fatalf("Creating a file watcher authz interceptor: %v", err) + } + unaryAuthzInterceptor, streamAuthzInterceptor = fileWatcherInterceptor.UnaryInterceptor, fileWatcherInterceptor.StreamInterceptor + } + + unaryInterceptors := grpc.ChainUnaryInterceptor(authUnaryInterceptor, unaryAuthzInterceptor) + streamInterceptors := grpc.ChainStreamInterceptor(authStreamInterceptor, streamAuthzInterceptor) + s := grpc.NewServer(grpc.Creds(creds), unaryInterceptors, streamInterceptors) + + // Register EchoServer on the server. + pb.RegisterEchoServer(s, &server{}) + + if err := s.Serve(lis); err != nil { + log.Fatalf("Serving Echo service on local port: %v", err) + } +} diff --git a/examples/features/authz/token/token.go b/examples/features/authz/token/token.go new file mode 100644 index 0000000..4899f3b --- /dev/null +++ b/examples/features/authz/token/token.go @@ -0,0 +1,55 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package token implements an example of authorization token encoding/decoding +// that can be used in RPC headers. +package token + +import ( + "encoding/base64" + "encoding/json" +) + +// Token is a mock authorization token sent by the client as part of the RPC headers, +// and used by the server for authorization against a predefined policy. +type Token struct { + // Secret is used by the server to authenticate the user + Secret string `json:"secret"` + // Username is used by the server to assign roles in the metadata for authorization + Username string `json:"username"` +} + +// Encode returns a base64 encoded version of the JSON representation of token. +func (t *Token) Encode() (string, error) { + barr, err := json.Marshal(t) + if err != nil { + return "", err + } + s := base64.StdEncoding.EncodeToString(barr) + return s, nil +} + +// Decode updates the internals of Token using the passed in base64 +// encoded version of the JSON representation of token. +func (t *Token) Decode(s string) error { + barr, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return err + } + return json.Unmarshal(barr, t) +} diff --git a/examples/features/cancellation/README.md b/examples/features/cancellation/README.md new file mode 100644 index 0000000..33e4182 --- /dev/null +++ b/examples/features/cancellation/README.md @@ -0,0 +1,13 @@ +# Cancellation + +This example shows how clients can cancel in-flight RPCs by canceling the +context passed to the RPC call. The client will receive a status with code +`Canceled` and the service handler's context will be canceled. + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` diff --git a/examples/features/cancellation/client/main.go b/examples/features/cancellation/client/main.go new file mode 100644 index 0000000..9ac765b --- /dev/null +++ b/examples/features/cancellation/client/main.go @@ -0,0 +1,96 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to cancel in-flight RPCs by canceling the +// context passed to the RPC. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/status" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +func sendMessage(stream pb.Echo_BidirectionalStreamingEchoClient, msg string) error { + fmt.Printf("sending message %q\n", msg) + return stream.Send(&pb.EchoRequest{Message: msg}) +} + +func recvMessage(stream pb.Echo_BidirectionalStreamingEchoClient, wantErrCode codes.Code) { + res, err := stream.Recv() + if status.Code(err) != wantErrCode { + log.Fatalf("stream.Recv() = %v, %v; want _, status.Code(err)=%v", res, err, wantErrCode) + } + if err != nil { + fmt.Printf("stream.Recv() returned expected error %v\n", err) + return + } + fmt.Printf("received message %q\n", res.GetMessage()) +} + +func main() { + flag.Parse() + + // Set up a connection to the server. + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + c := pb.NewEchoClient(conn) + + // Initiate the stream with a context that supports cancellation. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + stream, err := c.BidirectionalStreamingEcho(ctx) + if err != nil { + log.Fatalf("error creating stream: %v", err) + } + + // Send some test messages. + if err := sendMessage(stream, "hello"); err != nil { + log.Fatalf("error sending on stream: %v", err) + } + if err := sendMessage(stream, "world"); err != nil { + log.Fatalf("error sending on stream: %v", err) + } + + // Ensure the RPC is working. + recvMessage(stream, codes.OK) + recvMessage(stream, codes.OK) + + fmt.Println("cancelling context") + cancel() + + // This Send may or may not return an error, depending on whether the + // monitored context detects cancellation before the call is made. + sendMessage(stream, "closed") + + // This Recv should never succeed. + recvMessage(stream, codes.Canceled) +} diff --git a/examples/features/cancellation/server/main.go b/examples/features/cancellation/server/main.go new file mode 100644 index 0000000..53a3a61 --- /dev/null +++ b/examples/features/cancellation/server/main.go @@ -0,0 +1,67 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to handle canceled contexts when a client +// cancels an in-flight RPC. +package main + +import ( + "flag" + "fmt" + "io" + "log" + "net" + + "google.golang.org/grpc" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { + for { + in, err := stream.Recv() + if err != nil { + fmt.Printf("server: error receiving from stream: %v\n", err) + if err == io.EOF { + return nil + } + return err + } + fmt.Printf("echoing message %q\n", in.Message) + stream.Send(&pb.EchoResponse{Message: in.Message}) + } +} + +func main() { + flag.Parse() + + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + fmt.Printf("server listening at port %v\n", lis.Addr()) + s := grpc.NewServer() + pb.RegisterEchoServer(s, &server{}) + s.Serve(lis) +} diff --git a/examples/features/compression/README.md b/examples/features/compression/README.md new file mode 100644 index 0000000..760b157 --- /dev/null +++ b/examples/features/compression/README.md @@ -0,0 +1,14 @@ +# Compression + +This example shows how clients can specify compression options when performing +RPCs, and how to install support for compressors on the server. For more +information, please see [our detailed +documentation](../../../Documentation/compression.md). + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` diff --git a/examples/features/compression/client/main.go b/examples/features/compression/client/main.go new file mode 100644 index 0000000..482acfe --- /dev/null +++ b/examples/features/compression/client/main.go @@ -0,0 +1,62 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to specify compression options when performing +// RPCs. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/encoding/gzip" // Install the gzip compressor + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +func main() { + flag.Parse() + + // Set up a connection to the server. + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + c := pb.NewEchoClient(conn) + + // Send the RPC compressed. If all RPCs on a client should be sent this + // way, use the DialOption: + // grpc.WithDefaultCallOptions(grpc.UseCompressor(gzip.Name)) + const msg = "compress" + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + res, err := c.UnaryEcho(ctx, &pb.EchoRequest{Message: msg}, grpc.UseCompressor(gzip.Name)) + fmt.Printf("UnaryEcho call returned %q, %v\n", res.GetMessage(), err) + if err != nil || res.GetMessage() != msg { + log.Fatalf("Message=%q, err=%v; want Message=%q, err=", res.GetMessage(), err, msg) + } + +} diff --git a/examples/features/compression/server/main.go b/examples/features/compression/server/main.go new file mode 100644 index 0000000..a741aaa --- /dev/null +++ b/examples/features/compression/server/main.go @@ -0,0 +1,61 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to install and support compressors for +// incoming RPCs. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + // Installing the gzip encoding registers it as an available compressor. + // gRPC will automatically negotiate and use gzip if the client supports it. + _ "google.golang.org/grpc/encoding/gzip" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(_ context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { + fmt.Printf("UnaryEcho called with message %q\n", in.GetMessage()) + return &pb.EchoResponse{Message: in.Message}, nil +} + +func main() { + flag.Parse() + + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + fmt.Printf("server listening at %v\n", lis.Addr()) + + s := grpc.NewServer() + pb.RegisterEchoServer(s, &server{}) + s.Serve(lis) +} diff --git a/examples/features/csm_observability/README.md b/examples/features/csm_observability/README.md new file mode 100644 index 0000000..c7b8c5a --- /dev/null +++ b/examples/features/csm_observability/README.md @@ -0,0 +1,38 @@ +# CSM Observability + +This examples shows how to configure CSM Observability for gRPC client and +server applications (configured once per binary), and shows what type of +telemetry data it can produce for certain RPCs with additional CSM Labels. The +gRPC Client accepts configuration from an xDS Control plane as the default +address that it connects to is "xds:///helloworld:50051", but this can be +overridden with the command line flag --server_addr. This can be plugged into +the steps outlined in the CSM Observability User Guide. + +## Try it (locally if overwritten xDS Address) + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +Curl to the port where Prometheus exporter is outputting metrics data: +``` +curl localhost:9464/metrics +``` + +# Building +From the grpc-go directory: + +Client: +docker build -t -f examples/features/csm_observability/client/Dockerfile . + +Server: +docker build -t -f examples/features/csm_observability/server/Dockerfile . + +Note that this example will not work by default, as the client uses an xDS +Scheme and thus needs xDS Resources to connect to the server. Deploy the built +client and server containers within Cloud Service Mesh in order for this example +to work, or overwrite target to point to :. diff --git a/examples/features/csm_observability/client/Dockerfile b/examples/features/csm_observability/client/Dockerfile new file mode 100644 index 0000000..6d46d07 --- /dev/null +++ b/examples/features/csm_observability/client/Dockerfile @@ -0,0 +1,35 @@ +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for building the example client. To build the image, run the +# following command from grpc-go directory: +# docker build -t -f examples/features/csm_observability/client/Dockerfile . +FROM golang:1.25-alpine as build + +RUN apk --no-cache add curl + +# Make a grpc-go directory and copy the repo into it. +WORKDIR /go/src/grpc-go +COPY . . + +# Build a static binary without cgo so that we can copy just the binary in the +# final image, and can get rid of the Go compiler and gRPC-Go dependencies. +RUN cd examples/features/csm_observability/client && go build -tags osusergo,netgo . + +FROM alpine +RUN apk --no-cache add curl +COPY --from=build /go/src/grpc-go/examples/features/csm_observability/client/client . +ENV GRPC_GO_LOG_VERBOSITY_LEVEL=99 +ENV GRPC_GO_LOG_SEVERITY_LEVEL="info" +ENTRYPOINT ["./client"] diff --git a/examples/features/csm_observability/client/main.go b/examples/features/csm_observability/client/main.go new file mode 100644 index 0000000..c5b81ff --- /dev/null +++ b/examples/features/csm_observability/client/main.go @@ -0,0 +1,88 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is a client for the CSM Observability example. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net/http" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/stats/opentelemetry" + "google.golang.org/grpc/stats/opentelemetry/csm" + _ "google.golang.org/grpc/xds" // To install the xds resolvers and balancers. + + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/sdk/metric" +) + +const defaultName = "world" + +var ( + target = flag.String("target", "xds:///helloworld:50051", "the server address to connect to") + prometheusEndpoint = flag.String("prometheus_endpoint", ":9464", "the Prometheus exporter endpoint") + name = flag.String("name", defaultName, "Name to greet") +) + +func main() { + flag.Parse() + exporter, err := prometheus.New() + if err != nil { + log.Fatalf("Failed to start prometheus exporter: %v", err) + } + provider := metric.NewMeterProvider(metric.WithReader(exporter)) + go http.ListenAndServe(*prometheusEndpoint, promhttp.Handler()) + + cleanup := csm.EnableObservability(context.Background(), opentelemetry.Options{MetricsOptions: opentelemetry.MetricsOptions{MeterProvider: provider}}) + defer cleanup() + + // Set up xds credentials that fall back to insecure as described in: + // https://cloud.google.com/service-mesh/docs/service-routing/security-proxyless-setup#workloads_are_unable_to_communicate_in_the_security_setup. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + log.Fatalf("Failed to create xDS credentials: %v", err) + } + cc, err := grpc.NewClient(*target, grpc.WithTransportCredentials(creds)) + if err != nil { + log.Fatalf("Failed to start NewClient: %v", err) + } + defer cc.Close() + c := pb.NewGreeterClient(cc) + + // Make an RPC every second. This should trigger telemetry to be emitted from + // the client and the server. + for { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + r, err := c.SayHello(ctx, &pb.HelloRequest{Name: *name}) + if err != nil { + log.Fatalf("Could not greet: %v", err) + } + fmt.Println(r) + time.Sleep(time.Second) + cancel() + } +} diff --git a/examples/features/csm_observability/server/Dockerfile b/examples/features/csm_observability/server/Dockerfile new file mode 100644 index 0000000..7ab8e2e --- /dev/null +++ b/examples/features/csm_observability/server/Dockerfile @@ -0,0 +1,34 @@ +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for building the example server. To build the image, run the +# following command from grpc-go directory: +# docker build -t -f examples/features/csm_observability/server/Dockerfile . + +FROM golang:1.25-alpine as build +RUN apk --no-cache add curl +# Make a grpc-go directory and copy the repo into it. +WORKDIR /go/src/grpc-go +COPY . . + +# Build a static binary without cgo so that we can copy just the binary in the +# final image, and can get rid of the Go compiler and gRPC-Go dependencies. +RUN cd examples/features/csm_observability/server && go build -tags osusergo,netgo . + +FROM alpine +RUN apk --no-cache add curl +COPY --from=build /go/src/grpc-go/examples/features/csm_observability/server/server . +ENV GRPC_GO_LOG_VERBOSITY_LEVEL=99 +ENV GRPC_GO_LOG_SEVERITY_LEVEL="info" +ENTRYPOINT ["./server"] diff --git a/examples/features/csm_observability/server/main.go b/examples/features/csm_observability/server/main.go new file mode 100644 index 0000000..3c4c7a3 --- /dev/null +++ b/examples/features/csm_observability/server/main.go @@ -0,0 +1,91 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is a server for the CSM Observability example. +package main + +import ( + "context" + "flag" + "log" + "net" + "net/http" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/stats/opentelemetry" + "google.golang.org/grpc/stats/opentelemetry/csm" + "google.golang.org/grpc/xds" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/sdk/metric" +) + +var ( + port = flag.String("port", "50051", "the server address to connect to") + prometheusEndpoint = flag.String("prometheus_endpoint", ":9464", "the Prometheus exporter endpoint") +) + +// server is used to implement helloworld.GreeterServer. +type server struct { + pb.UnimplementedGreeterServer + addr string +} + +// SayHello implements helloworld.GreeterServer +func (s *server) SayHello(_ context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil +} + +func main() { + flag.Parse() + exporter, err := prometheus.New() + if err != nil { + log.Fatalf("Failed to start prometheus exporter: %v", err) + } + provider := metric.NewMeterProvider(metric.WithReader(exporter)) + go http.ListenAndServe(*prometheusEndpoint, promhttp.Handler()) + + cleanup := csm.EnableObservability(context.Background(), opentelemetry.Options{MetricsOptions: opentelemetry.MetricsOptions{MeterProvider: provider}}) + defer cleanup() + + lis, err := net.Listen("tcp4", "0.0.0.0:"+*port) + if err != nil { + log.Fatalf("Failed to listen: %v", err) + } + // Set up xds credentials that fall back to insecure as described in: + // https://cloud.google.com/service-mesh/docs/service-routing/security-proxyless-setup#workloads_are_unable_to_communicate_in_the_security_setup. + creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + log.Fatalf("Failed to create xDS credentials: %v", err) + } + s, err := xds.NewGRPCServer(grpc.Creds(creds)) + if err != nil { + log.Fatalf("Failed to start xDS Server: %v", err) + } + pb.RegisterGreeterServer(s, &server{addr: ":" + *port}) + + log.Printf("Serving on %s\n", *port) + + if err := s.Serve(lis); err != nil { + log.Fatalf("Failed to serve: %v", err) + } +} diff --git a/examples/features/customloadbalancer/README.md b/examples/features/customloadbalancer/README.md new file mode 100644 index 0000000..db39bf2 --- /dev/null +++ b/examples/features/customloadbalancer/README.md @@ -0,0 +1,52 @@ +# Custom Load Balancer + +This example shows how to deploy a custom load balancer in a `ClientConn`. + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +Two echo servers are serving on "localhost:20000" and "localhost:20001". They +will include their serving address in the response. So the server on +"localhost:20001" will reply to the RPC with `this is +examples/customloadbalancing (from localhost:20001)`. + +A client is created, to connect to both of these servers (they get both server +addresses from the name resolver in two separate endpoints). The client is +configured with the load balancer specified in the service config, which in this +case is custom_round_robin. + +### custom_round_robin + +The client is configured to use `custom_round_robin`. `custom_round_robin` +creates a pick first child for every endpoint it receives. It waits until both +pick first children become ready, then defers to the first pick first child's +picker, choosing the connection to localhost:20000, except every chooseSecond +times, where it defers to second pick first child's picker, choosing the +connection to localhost:20001 (or vice versa). + +`custom_round_robin` is written as a delegating policy wrapping `pick_first` +load balancers, one for every endpoint received. This is the intended way a user +written custom lb should be specified, as pick first will contain a lot of +useful functionality, such as Sticky Transient Failure, Happy Eyeballs, and +Health Checking. + +``` +this is examples/customloadbalancing (from localhost:50050) +this is examples/customloadbalancing (from localhost:50050) +this is examples/customloadbalancing (from localhost:50051) +this is examples/customloadbalancing (from localhost:50050) +this is examples/customloadbalancing (from localhost:50050) +this is examples/customloadbalancing (from localhost:50051) +this is examples/customloadbalancing (from localhost:50050) +this is examples/customloadbalancing (from localhost:50050) +this is examples/customloadbalancing (from localhost:50051) +``` diff --git a/examples/features/customloadbalancer/client/customroundrobin/customroundrobin.go b/examples/features/customloadbalancer/client/customroundrobin/customroundrobin.go new file mode 100644 index 0000000..7fd660d --- /dev/null +++ b/examples/features/customloadbalancer/client/customroundrobin/customroundrobin.go @@ -0,0 +1,148 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package customroundrobin provides an example for the custom roundrobin balancer. +package customroundrobin + +import ( + "encoding/json" + "fmt" + "sync/atomic" + + _ "google.golang.org/grpc" // to register pick_first + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/endpointsharding" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/serviceconfig" +) + +func init() { + balancer.Register(customRoundRobinBuilder{}) +} + +const customRRName = "custom_round_robin" + +type customRRConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + // ChooseSecond represents how often pick iterations choose the second + // SubConn in the list. Defaults to 3. If 0 never choose the second SubConn. + ChooseSecond uint32 `json:"chooseSecond,omitempty"` +} + +type customRoundRobinBuilder struct{} + +func (customRoundRobinBuilder) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + lbConfig := &customRRConfig{ + ChooseSecond: 3, + } + + if err := json.Unmarshal(s, lbConfig); err != nil { + return nil, fmt.Errorf("custom-round-robin: unable to unmarshal customRRConfig: %v", err) + } + return lbConfig, nil +} + +func (customRoundRobinBuilder) Name() string { + return customRRName +} + +func (customRoundRobinBuilder) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + crr := &customRoundRobin{ + ClientConn: cc, + bOpts: bOpts, + } + crr.Balancer = endpointsharding.NewBalancer(crr, bOpts, balancer.Get(pickfirst.Name).Build, endpointsharding.Options{}) + return crr +} + +type customRoundRobin struct { + // All state and operations on this balancer are either initialized at build + // time and read only after, or are only accessed as part of its + // balancer.Balancer API (UpdateState from children only comes in from + // balancer.Balancer calls as well, and children are called one at a time), + // in which calls are guaranteed to come synchronously. Thus, no extra + // synchronization is required in this balancer. + balancer.Balancer + balancer.ClientConn + bOpts balancer.BuildOptions + + cfg atomic.Pointer[customRRConfig] +} + +func (crr *customRoundRobin) UpdateClientConnState(state balancer.ClientConnState) error { + crrCfg, ok := state.BalancerConfig.(*customRRConfig) + if !ok { + return balancer.ErrBadResolverState + } + if el := state.ResolverState.Endpoints; len(el) != 2 { + return fmt.Errorf("UpdateClientConnState wants two endpoints, got: %v", el) + } + crr.cfg.Store(crrCfg) + // A call to UpdateClientConnState should always produce a new Picker. That + // is guaranteed to happen since the aggregator will always call + // UpdateChildState in its UpdateClientConnState. + return crr.Balancer.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: state.ResolverState, + }) +} + +func (crr *customRoundRobin) UpdateState(state balancer.State) { + if state.ConnectivityState == connectivity.Ready { + childStates := endpointsharding.ChildStatesFromPicker(state.Picker) + var readyPickers []balancer.Picker + for _, childState := range childStates { + if childState.State.ConnectivityState == connectivity.Ready { + readyPickers = append(readyPickers, childState.State.Picker) + } + } + // If both children are ready, pick using the custom round robin + // algorithm. + if len(readyPickers) == 2 { + picker := &customRoundRobinPicker{ + pickers: readyPickers, + chooseSecond: crr.cfg.Load().ChooseSecond, + next: 0, + } + crr.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: picker, + }) + return + } + } + // Delegate to default behavior/picker from below. + crr.ClientConn.UpdateState(state) +} + +type customRoundRobinPicker struct { + pickers []balancer.Picker + chooseSecond uint32 + next uint32 +} + +func (crrp *customRoundRobinPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + next := atomic.AddUint32(&crrp.next, 1) + index := 0 + if next != 0 && next%crrp.chooseSecond == 0 { + index = 1 + } + childPicker := crrp.pickers[index%len(crrp.pickers)] + return childPicker.Pick(info) +} diff --git a/examples/features/customloadbalancer/client/main.go b/examples/features/customloadbalancer/client/main.go new file mode 100644 index 0000000..301c134 --- /dev/null +++ b/examples/features/customloadbalancer/client/main.go @@ -0,0 +1,155 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is a client for the custom load balancer example. +package main + +import ( + "context" + "fmt" + "log" + "strings" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + _ "google.golang.org/grpc/examples/features/customloadbalancer/client/customroundrobin" // To register custom_round_robin. + pb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" +) + +var ( + addr1 = "localhost:50050" + addr2 = "localhost:50051" +) + +func main() { + mr := manual.NewBuilderWithScheme("example") + defer mr.Close() + + // You can also plug in your own custom lb policy, which needs to be + // configurable. This n is configurable. Try changing n and see how the + // behavior changes. + json := `{"loadBalancingConfig": [{"custom_round_robin":{"chooseSecond": 3}}]}` + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(json) + mr.InitialState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: addr1}}}, + {Addresses: []resolver.Address{{Addr: addr2}}}, + }, + ServiceConfig: sc, + }) + + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + ec := pb.NewEchoClient(cc) + if err := waitForDistribution(ctx, ec); err != nil { + log.Fatal(err) + } + fmt.Println("Successful multiple iterations of 1:2 ratio") +} + +// waitForDistribution makes RPC's on the echo client until 3 RPC's follow the +// same 1:2 address ratio for the peer. Returns an error if fails to do so +// before context timeout. +func waitForDistribution(ctx context.Context, ec pb.EchoClient) error { + for { + results := make(map[string]uint32) + InnerLoop: + for { + if ctx.Err() != nil { + return fmt.Errorf("timeout waiting for 1:2 distribution between addresses %v and %v", addr1, addr2) + } + + for i := 0; i < 3; i++ { + res := make(map[string]uint32) + for j := 0; j < 3; j++ { + var peer peer.Peer + r, err := ec.UnaryEcho(ctx, &pb.EchoRequest{Message: "this is examples/customloadbalancing"}, grpc.Peer(&peer)) + if err != nil { + return fmt.Errorf("UnaryEcho failed: %v", err) + } + fmt.Println(r) + peerAddr := peer.Addr.String() + if !strings.HasSuffix(peerAddr, "50050") && !strings.HasSuffix(peerAddr, "50051") { + return fmt.Errorf("peer address was not one of %v or %v, got: %v", addr1, addr2, peerAddr) + } + res[peerAddr]++ + time.Sleep(time.Millisecond) + } + // Make sure the addresses come in a 1:2 ratio for this + // iteration. + var seen1, seen2 bool + for addr, count := range res { + if count != 1 && count != 2 { + break InnerLoop + } + if count == 1 { + if seen1 { + break InnerLoop + } + seen1 = true + } + if count == 2 { + if seen2 { + break InnerLoop + } + seen2 = true + } + results[addr] = results[addr] + count + } + if !seen1 || !seen2 { + break InnerLoop + } + } + // Make sure iteration is 3 and 6 for addresses seen. This makes + // sure the distribution is the same 1:2 ratio for each iteration. + var seen3, seen6 bool + for _, count := range results { + if count != 3 && count != 6 { + break InnerLoop + } + if count == 3 { + if seen3 { + break InnerLoop + } + seen3 = true + } + if count == 6 { + if seen6 { + break InnerLoop + } + seen6 = true + } + return nil + } + if !seen3 || !seen6 { + break InnerLoop + } + } + } +} diff --git a/examples/features/customloadbalancer/server/main.go b/examples/features/customloadbalancer/server/main.go new file mode 100644 index 0000000..9f2cb24 --- /dev/null +++ b/examples/features/customloadbalancer/server/main.go @@ -0,0 +1,67 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is a server for the custom load balancer example. +package main + +import ( + "context" + "fmt" + "log" + "net" + "sync" + + "google.golang.org/grpc" + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var ( + addrs = []string{"localhost:50050", "localhost:50051"} +) + +type echoServer struct { + pb.UnimplementedEchoServer + addr string +} + +func (s *echoServer) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: fmt.Sprintf("%s (from %s)", req.Message, s.addr)}, nil +} + +func main() { + var wg sync.WaitGroup + for _, addr := range addrs { + lis, err := net.Listen("tcp", addr) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + s := grpc.NewServer() + pb.RegisterEchoServer(s, &echoServer{ + addr: addr, + }) + log.Printf("serving on %s\n", addr) + wg.Add(1) + go func() { + defer wg.Done() + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } + }() + } + wg.Wait() +} diff --git a/examples/features/deadline/client/main.go b/examples/features/deadline/client/main.go new file mode 100644 index 0000000..4d16a02 --- /dev/null +++ b/examples/features/deadline/client/main.go @@ -0,0 +1,97 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to set deadlines for RPCs and how to handle +// deadline-exceeded errors. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/status" +) + +var addr = flag.String("addr", "localhost:50052", "the address to connect to") + +func unaryCall(c pb.EchoClient, requestID int, message string, want codes.Code) { + // Creates a context with a one second deadline for the RPC. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + req := &pb.EchoRequest{Message: message} + + _, err := c.UnaryEcho(ctx, req) + got := status.Code(err) + fmt.Printf("[%v] wanted = %v, got = %v\n", requestID, want, got) +} + +func streamingCall(c pb.EchoClient, requestID int, message string, want codes.Code) { + // Creates a context with a one second deadline for the RPC. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + stream, err := c.BidirectionalStreamingEcho(ctx) + if err != nil { + log.Printf("Stream err: %v", err) + return + } + + err = stream.Send(&pb.EchoRequest{Message: message}) + if err != nil { + log.Printf("Send error: %v", err) + return + } + + _, err = stream.Recv() + + got := status.Code(err) + fmt.Printf("[%v] wanted = %v, got = %v\n", requestID, want, got) +} + +func main() { + flag.Parse() + + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + c := pb.NewEchoClient(conn) + + // A successful request + unaryCall(c, 1, "world", codes.OK) + // Exceeds deadline + unaryCall(c, 2, "delay", codes.DeadlineExceeded) + // A successful request with propagated deadline + unaryCall(c, 3, "[propagate me]world", codes.OK) + // Exceeds propagated deadline + unaryCall(c, 4, "[propagate me][propagate me]world", codes.DeadlineExceeded) + // Receives a response from the stream successfully. + streamingCall(c, 5, "[propagate me]world", codes.OK) + // Exceeds propagated deadline before receiving a response + streamingCall(c, 6, "[propagate me][propagate me]world", codes.DeadlineExceeded) +} diff --git a/examples/features/deadline/server/main.go b/examples/features/deadline/server/main.go new file mode 100644 index 0000000..a826b39 --- /dev/null +++ b/examples/features/deadline/server/main.go @@ -0,0 +1,124 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to handle RPCs with deadlines and how to +// propagate deadlines in requests. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "net" + "strings" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50052, "port number") + +// server is used to implement EchoServer. +type server struct { + pb.UnimplementedEchoServer + client pb.EchoClient + cc *grpc.ClientConn +} + +func (s *server) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + message := req.Message + if strings.HasPrefix(message, "[propagate me]") { + time.Sleep(800 * time.Millisecond) + message = strings.TrimPrefix(message, "[propagate me]") + return s.client.UnaryEcho(ctx, &pb.EchoRequest{Message: message}) + } + + if message == "delay" { + time.Sleep(1500 * time.Millisecond) + } + + return &pb.EchoResponse{Message: req.Message}, nil +} + +func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { + for { + req, err := stream.Recv() + if err == io.EOF { + return status.Error(codes.InvalidArgument, "request message not received") + } + if err != nil { + return err + } + + message := req.Message + if strings.HasPrefix(message, "[propagate me]") { + time.Sleep(800 * time.Millisecond) + message = strings.TrimPrefix(message, "[propagate me]") + res, err := s.client.UnaryEcho(stream.Context(), &pb.EchoRequest{Message: message}) + if err != nil { + return err + } + stream.Send(res) + } + + if message == "delay" { + time.Sleep(1500 * time.Millisecond) + } + stream.Send(&pb.EchoResponse{Message: message}) + } +} + +func (s *server) Close() { + s.cc.Close() +} + +func newEchoServer() *server { + target := fmt.Sprintf("localhost:%v", *port) + cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + return &server{client: pb.NewEchoClient(cc), cc: cc} +} + +func main() { + flag.Parse() + + address := fmt.Sprintf(":%v", *port) + lis, err := net.Listen("tcp", address) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + echoServer := newEchoServer() + defer echoServer.Close() + + grpcServer := grpc.NewServer() + pb.RegisterEchoServer(grpcServer, echoServer) + + if err := grpcServer.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/debugging/README.md b/examples/features/debugging/README.md new file mode 100644 index 0000000..77c0f98 --- /dev/null +++ b/examples/features/debugging/README.md @@ -0,0 +1,29 @@ +# Debugging + +Currently, grpc provides two major tools to help user debug issues, which are logging and channelz. + +## Logs +gRPC has put substantial logging instruments on critical paths of gRPC to help users debug issues. +The [Log Levels](https://github.com/grpc/grpc-go/blob/master/Documentation/log_levels.md) doc describes +what each log level means in the gRPC context. + +To turn on the logs for debugging, run the code with the following environment variable: +`GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info`. + +## Channelz +We also provide a runtime debugging tool, Channelz, to help users with live debugging. + +See the channelz blog post here ([link](https://grpc.io/blog/a-short-introduction-to-channelz/)) for +details about how to use channelz service to debug live program. + +## Try it +The example is able to showcase how logging and channelz can help with debugging. See the channelz +blog post linked above for full explanation. + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` diff --git a/examples/features/debugging/client/main.go b/examples/features/debugging/client/main.go new file mode 100644 index 0000000..0036aba --- /dev/null +++ b/examples/features/debugging/client/main.go @@ -0,0 +1,93 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to use logging and Channelz for debugging +// gRPC operations. +package main + +import ( + "context" + "flag" + "log" + "net" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/channelz/service" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + + pb "google.golang.org/grpc/examples/helloworld/helloworld" +) + +const ( + defaultName = "world" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") + name = flag.String("name", defaultName, "Name to greet") +) + +func main() { + flag.Parse() + /***** Set up the server serving channelz service. *****/ + lis, err := net.Listen("tcp", *addr) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + defer lis.Close() + s := grpc.NewServer() + service.RegisterChannelzServiceToServer(s) + go s.Serve(lis) + defer s.Stop() + + /***** Initialize manual resolver and Dial *****/ + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: ":10001"}, {Addr: ":10002"}, {Addr: ":10003"}}}) + // Set up a connection to the server. + conn, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r), grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`)) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + // Manually provide resolved addresses for the target. + + c := pb.NewGreeterClient(conn) + + // Contact the server and print out its response. + + /***** Make 100 SayHello RPCs *****/ + for i := 0; i < 100; i++ { + // Setting a 150ms timeout on the RPC. + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + r, err := c.SayHello(ctx, &pb.HelloRequest{Name: *name}) + if err != nil { + log.Printf("could not greet: %v", err) + } else { + log.Printf("Greeting: %s", r.Message) + } + } + + /***** Wait for user exiting the program *****/ + // Unless you exit the program (e.g. CTRL+C), channelz data will be available for querying. + // Users can take time to examine and learn about the info provided by channelz. + select {} +} diff --git a/examples/features/debugging/server/main.go b/examples/features/debugging/server/main.go new file mode 100644 index 0000000..815e896 --- /dev/null +++ b/examples/features/debugging/server/main.go @@ -0,0 +1,92 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to enable logging and Channelz for debugging +// gRPC services. +package main + +import ( + "context" + "log" + rand "math/rand/v2" + "net" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/channelz/service" + + pb "google.golang.org/grpc/examples/helloworld/helloworld" +) + +var ( + ports = []string{":10001", ":10002", ":10003"} +) + +// server is used to implement helloworld.GreeterServer. +type server struct { + pb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer +func (s *server) SayHello(_ context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + return &pb.HelloReply{Message: "Hello " + in.Name}, nil +} + +// slow server is used to simulate a server that has a variable delay in its response. +type slowServer struct { + pb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer +func (s *slowServer) SayHello(_ context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + // Delay 100ms ~ 200ms before replying + time.Sleep(time.Duration(100+rand.IntN(100)) * time.Millisecond) + return &pb.HelloReply{Message: "Hello " + in.Name}, nil +} + +func main() { + /***** Set up the server serving channelz service. *****/ + lis, err := net.Listen("tcp", ":50052") + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + defer lis.Close() + s := grpc.NewServer() + service.RegisterChannelzServiceToServer(s) + go s.Serve(lis) + defer s.Stop() + + /***** Start three GreeterServers(with one of them to be the slowServer). *****/ + for i := 0; i < 3; i++ { + lis, err := net.Listen("tcp", ports[i]) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + defer lis.Close() + s := grpc.NewServer() + if i == 2 { + pb.RegisterGreeterServer(s, &slowServer{}) + } else { + pb.RegisterGreeterServer(s, &server{}) + } + go s.Serve(lis) + } + + /***** Wait for user exiting the program *****/ + select {} +} diff --git a/examples/features/dualstack/README.md b/examples/features/dualstack/README.md new file mode 100644 index 0000000..13a6001 --- /dev/null +++ b/examples/features/dualstack/README.md @@ -0,0 +1,29 @@ +# Dualstack + +The dualstack example uses a custom name resolver that provides both IPv4 and +IPv6 localhost endpoints for each of 3 server instances. The client will first +use the default name resolver and load balancers which will only connect to the +first server. It will then use the custom name resolver with round robin to +connect to each of the servers in turn. The 3 instances of the server will bind +respectively to: both IPv4 and IPv6, IPv4 only, and IPv6 only. + +Three servers are serving on the following loopback addresses: + +1. `[::]:50052`: Listening on both IPv4 and IPv6 loopback addresses. +1. `127.0.0.1:50050`: Listening only on the IPv4 loopback address. +1. `[::1]:50051`: Listening only on the IPv6 loopback address. + +The server response will include its serving port and address type (IPv4, IPv6 +or both). So the server on "127.0.0.1:50050" will reply to the RPC with the +following message: `Greeting:Hello request:1 from server<50052> type: IPv4 +only)`. + +## Try it + +```sh +go run server/main.go +``` + +```sh +go run client/main.go +``` diff --git a/examples/features/dualstack/client/main.go b/examples/features/dualstack/client/main.go new file mode 100644 index 0000000..2a9a526 --- /dev/null +++ b/examples/features/dualstack/client/main.go @@ -0,0 +1,189 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is a client for the dualstack example. +package main + +import ( + "context" + "fmt" + "log" + "slices" + "strings" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + hwpb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" +) + +const ( + port1 = 50051 + port2 = 50052 + port3 = 50053 +) + +func init() { + resolver.Register(&exampleResolver{}) +} + +// exampleResolver implements both, a fake `resolver.Resolver` and +// `resolver.Builder`. This resolver sends a hard-coded list of 3 endpoints each +// with 2 addresses (one IPv4 and one IPv6) to the channel. +type exampleResolver struct{} + +func (*exampleResolver) Close() {} + +func (*exampleResolver) ResolveNow(resolver.ResolveNowOptions) {} + +func (*exampleResolver) Build(_ resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) { + go func() { + err := cc.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{ + {Addr: fmt.Sprintf("[::1]:%d", port1)}, + {Addr: fmt.Sprintf("127.0.0.1:%d", port1)}, + }}, + {Addresses: []resolver.Address{ + {Addr: fmt.Sprintf("[::1]:%d", port2)}, + {Addr: fmt.Sprintf("127.0.0.1:%d", port2)}, + }}, + {Addresses: []resolver.Address{ + {Addr: fmt.Sprintf("[::1]:%d", port3)}, + {Addr: fmt.Sprintf("127.0.0.1:%d", port3)}, + }}, + }, + }) + if err != nil { + log.Fatal("Failed to update resolver state", err) + } + }() + + return &exampleResolver{}, nil +} + +func (*exampleResolver) Scheme() string { + return "example" +} + +func main() { + // First send 5 requests using the default DNS and pickfirst load balancer. + log.Print("**** Use default DNS resolver ****") + target := fmt.Sprintf("localhost:%d", port1) + cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("Failed to create client: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + client := hwpb.NewGreeterClient(cc) + + for i := 0; i < 5; i++ { + resp, err := client.SayHello(ctx, &hwpb.HelloRequest{ + Name: fmt.Sprintf("request:%d", i), + }) + if err != nil { + log.Panicf("RPC failed: %v", err) + } + log.Print("Greeting:", resp.GetMessage()) + } + cc.Close() + + log.Print("**** Change to use example name resolver ****") + dOpts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err = grpc.NewClient("example:///ignored", dOpts...) + if err != nil { + log.Fatalf("Failed to create client: %v", err) + } + client = hwpb.NewGreeterClient(cc) + + // Send 10 requests using the example nameresolver and round robin load + // balancer. These requests are evenly distributed among the 3 servers + // rather than favoring the server listening on both addresses because the + // resolver groups the 3 servers as 3 endpoints each with 2 addresses. + if err := waitForDistribution(ctx, client); err != nil { + log.Panic(err) + } + log.Print("Successful multiple iterations of 1:1:1 ratio") +} + +// waitForDistribution makes RPC's on the greeter client until 3 RPC's follow +// the same 1:1:1 address ratio for the peer. Returns an error if fails to do so +// before context timeout. +func waitForDistribution(ctx context.Context, client hwpb.GreeterClient) error { + wantPeers := []string{ + // Server 1 is listening on both IPv4 and IPv6 loopback addresses. + // Since the IPv6 address comes first in the resolver list, it will be + // given higher priority. + fmt.Sprintf("[::1]:%d", port1), + // Server 2 is listening only on the IPv4 loopback address. + fmt.Sprintf("127.0.0.1:%d", port2), + // Server 3 is listening only on the IPv6 loopback address. + fmt.Sprintf("[::1]:%d", port3), + } + const iterationsToVerify = 3 + const backendCount = 3 + requestCounter := 0 + + for ctx.Err() == nil { + result := make(map[string]int) + badRatioSeen := false + for i := 1; i <= iterationsToVerify && !badRatioSeen; i++ { + for j := 0; j < backendCount; j++ { + var peer peer.Peer + resp, err := client.SayHello(ctx, &hwpb.HelloRequest{ + Name: fmt.Sprintf("request:%d", requestCounter), + }, grpc.Peer(&peer)) + requestCounter++ + if err != nil { + return fmt.Errorf("RPC failed: %v", err) + } + log.Print("Greeting:", resp.GetMessage()) + + peerAddr := peer.Addr.String() + if !slices.Contains(wantPeers, peerAddr) { + return fmt.Errorf("peer address was not one of %q, got: %v", strings.Join(wantPeers, ", "), peerAddr) + } + result[peerAddr]++ + time.Sleep(time.Millisecond) + } + + // Verify the results of this iteration. + for _, count := range result { + if count == i { + continue + } + badRatioSeen = true + break + } + if !badRatioSeen { + log.Print("Got iteration with 1:1:1 distribution between addresses.") + } + } + if !badRatioSeen { + return nil + } + } + return fmt.Errorf("timeout waiting for 1:1:1 distribution between addresses %v", wantPeers) +} diff --git a/examples/features/dualstack/server/main.go b/examples/features/dualstack/server/main.go new file mode 100644 index 0000000..c6e9f7d --- /dev/null +++ b/examples/features/dualstack/server/main.go @@ -0,0 +1,84 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is a server for the dualstack example. +package main + +import ( + "context" + "fmt" + "log" + "net" + "sync" + + "google.golang.org/grpc" + hwpb "google.golang.org/grpc/examples/helloworld/helloworld" +) + +type greeterServer struct { + hwpb.UnimplementedGreeterServer + addressType string + address string + port uint32 +} + +func (s *greeterServer) SayHello(_ context.Context, req *hwpb.HelloRequest) (*hwpb.HelloReply, error) { + return &hwpb.HelloReply{ + Message: fmt.Sprintf("Hello %s from server<%d> type: %s)", req.GetName(), s.port, s.addressType), + }, nil +} + +func main() { + servers := []*greeterServer{ + { + addressType: "both IPv4 and IPv6", + address: "[::]", + port: 50051, + }, + { + addressType: "IPv4 only", + address: "127.0.0.1", + port: 50052, + }, + { + addressType: "IPv6 only", + address: "[::1]", + port: 50053, + }, + } + + var wg sync.WaitGroup + for _, server := range servers { + bindAddr := fmt.Sprintf("%s:%d", server.address, server.port) + lis, err := net.Listen("tcp", bindAddr) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + s := grpc.NewServer() + hwpb.RegisterGreeterServer(s, server) + wg.Add(1) + go func() { + defer wg.Done() + if err := s.Serve(lis); err != nil { + log.Panicf("failed to serve: %v", err) + } + }() + log.Printf("serving on %s\n", bindAddr) + } + wg.Wait() +} diff --git a/examples/features/encryption/ALTS/client/main.go b/examples/features/encryption/ALTS/client/main.go new file mode 100644 index 0000000..5ac1f98 --- /dev/null +++ b/examples/features/encryption/ALTS/client/main.go @@ -0,0 +1,63 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to use ALTS credentials for secure +// gRPC communication. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/alts" + ecpb "google.golang.org/grpc/examples/features/proto/echo" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +func callUnaryEcho(client ecpb.EchoClient, message string) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := client.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message}) + if err != nil { + log.Fatalf("client.UnaryEcho(_) = _, %v: ", err) + } + fmt.Println("UnaryEcho: ", resp.Message) +} + +func main() { + flag.Parse() + + // Create alts based credential. + altsTC := alts.NewClientCreds(alts.DefaultClientOptions()) + + // Set up a connection to the server. + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(altsTC)) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + // Make an echo client and send an RPC. + rgc := ecpb.NewEchoClient(conn) + callUnaryEcho(rgc, "hello world") +} diff --git a/examples/features/encryption/ALTS/server/main.go b/examples/features/encryption/ALTS/server/main.go new file mode 100644 index 0000000..174e188 --- /dev/null +++ b/examples/features/encryption/ALTS/server/main.go @@ -0,0 +1,64 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to use ALTS credentials to secure gRPC +// services. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/alts" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +type ecServer struct { + pb.UnimplementedEchoServer +} + +func (s *ecServer) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: req.Message}, nil +} + +func main() { + flag.Parse() + + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + // Create alts based credential. + altsTC := alts.NewServerCreds(alts.DefaultServerOptions()) + + s := grpc.NewServer(grpc.Creds(altsTC)) + + // Register EchoServer on the server. + pb.RegisterEchoServer(s, &ecServer{}) + + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/encryption/README.md b/examples/features/encryption/README.md new file mode 100644 index 0000000..5bfcc0e --- /dev/null +++ b/examples/features/encryption/README.md @@ -0,0 +1,119 @@ +# Encryption + +The example for encryption includes three individual examples for TLS, ALTS +and mTLS encryption mechanism respectively. + +## Try it + +In each example's subdirectory: + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +### TLS + +TLS is a commonly used cryptographic protocol to provide end-to-end +communication security. In the example, we show how to set up a server +authenticated TLS connection to transmit RPC. + +In our `grpc/credentials` package, we provide several convenience methods to +create grpc +[`credentials.TransportCredentials`](https://godoc.org/google.golang.org/grpc/credentials#TransportCredentials) +base on TLS. Refer to the +[godoc](https://godoc.org/google.golang.org/grpc/credentials) for details. + +In our example, we use the public/private keys created ahead: +* "server_cert.pem" contains the server certificate (public key). +* "server_key.pem" contains the server private key. +* "ca_cert.pem" contains the certificate (certificate authority) +that can verify the server's certificate. + +On server side, we provide the paths to "server_cert.pem" and "server_key.pem" to +configure TLS and create the server credential using +[`credentials.NewServerTLSFromFile`](https://godoc.org/google.golang.org/grpc/credentials#NewServerTLSFromFile). + +On client side, we provide the path to the "ca_cert.pem" to configure TLS and create +the client credential using +[`credentials.NewClientTLSFromFile`](https://godoc.org/google.golang.org/grpc/credentials#NewClientTLSFromFile). +Note that we override the server name with "x.test.example.com", as the server +certificate is valid for *.test.example.com but not localhost. It is solely for +the convenience of making an example. + +Once the credentials have been created at both sides, we can start the server +with the just created server credential (by calling +[`grpc.Creds`](https://godoc.org/google.golang.org/grpc#Creds)) and let client dial +to the server with the created client credential (by calling +[`grpc.WithTransportCredentials`](https://godoc.org/google.golang.org/grpc#WithTransportCredentials)) + +And finally we make an RPC call over the created `grpc.ClientConn` to test the secure +connection based upon TLS is successfully up. + +### ALTS +NOTE: ALTS currently needs special early access permission on GCP. You can ask +about the detailed process in https://groups.google.com/forum/#!forum/grpc-io. + +ALTS is the Google's Application Layer Transport Security, which supports mutual +authentication and transport encryption. Note that ALTS is currently only +supported on Google Cloud Platform, and therefore you can only run the example +successfully in a GCP environment. In our example, we show how to initiate a +secure connection that is based on ALTS. + +Unlike TLS, ALTS makes certificate/key management transparent to user. So it is +easier to set up. + +On server side, first call +[`alts.DefaultServerOptions`](https://godoc.org/google.golang.org/grpc/credentials/alts#DefaultServerOptions) +to get the configuration for alts and then provide the configuration to +[`alts.NewServerCreds`](https://godoc.org/google.golang.org/grpc/credentials/alts#NewServerCreds) +to create the server credential based upon alts. + +On client side, first call +[`alts.DefaultClientOptions`](https://godoc.org/google.golang.org/grpc/credentials/alts#DefaultClientOptions) +to get the configuration for alts and then provide the configuration to +[`alts.NewClientCreds`](https://godoc.org/google.golang.org/grpc/credentials/alts#NewClientCreds) +to create the client credential based upon alts. + +Next, same as TLS, start the server with the server credential and let client +dial to server with the client credential. + +Finally, make an RPC to test the secure connection based upon ALTS is +successfully up. + +### mTLS + +In mutual TLS (mTLS), the client and the server authenticate each other. gRPC +allows users to configure mutual TLS at the connection level. + +In this example, we use the following public/private keys created ahead of time: + +* "server_cert.pem" contains the server's certificate (public key). +* "server_key.pem" contains the server's private key. +* "ca_cert.pem" contains the certificate of the certificate authority that can + verify the server's certificate. +* "client_cert.pem" contains the client's certificate (public key). +* "client_key.pem" contains the client's private key. +* "client_ca_cert.pem" contains the certificate of the certificate authority + that can verify the client's certificate. + +In normal TLS, the server is only concerned with presenting the server +certificate for clients to verify. In mutual TLS, the server also loads in a +list of trusted CA files for verifying the client's presented certificates. +This is done by setting +[`tls.Config.ClientCAs`](https://pkg.go.dev/crypto/tls#Config.ClientCAs) +to the list of trusted CA files, +and setting +[`tls.config.ClientAuth`](https://pkg.go.dev/crypto/tls#Config.ClientAuth) +to +[`tls.RequireAndVerifyClientCert`](https://pkg.go.dev/crypto/tls#RequireAndVerifyClientCert). + +In normal TLS, the client is only concerned with authenticating the server by +using one or more trusted CA file. In mutual TLS, the client also presents its +client certificate to the server for authentication. This is done by setting +[`tls.Config.Certificates`](https://pkg.go.dev/crypto/tls#Config.Certificates). diff --git a/examples/features/encryption/TLS/client/main.go b/examples/features/encryption/TLS/client/main.go new file mode 100644 index 0000000..ff53c7a --- /dev/null +++ b/examples/features/encryption/TLS/client/main.go @@ -0,0 +1,67 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to use TLS credentials for secure +// gRPC communication. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/examples/data" + ecpb "google.golang.org/grpc/examples/features/proto/echo" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +func callUnaryEcho(client ecpb.EchoClient, message string) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := client.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message}) + if err != nil { + log.Fatalf("client.UnaryEcho(_) = _, %v: ", err) + } + fmt.Println("UnaryEcho: ", resp.Message) +} + +func main() { + flag.Parse() + + // Create tls based credential. + creds, err := credentials.NewClientTLSFromFile(data.Path("x509/ca_cert.pem"), "x.test.example.com") + if err != nil { + log.Fatalf("failed to load credentials: %v", err) + } + + // Set up a connection to the server. + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(creds)) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + // Make an echo client and send an RPC. + rgc := ecpb.NewEchoClient(conn) + callUnaryEcho(rgc, "hello world") +} diff --git a/examples/features/encryption/TLS/server/main.go b/examples/features/encryption/TLS/server/main.go new file mode 100644 index 0000000..82162a9 --- /dev/null +++ b/examples/features/encryption/TLS/server/main.go @@ -0,0 +1,68 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to use TLS credentials to secure gRPC services. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/examples/data" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +type ecServer struct { + pb.UnimplementedEchoServer +} + +func (s *ecServer) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: req.Message}, nil +} + +func main() { + flag.Parse() + + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + // Create tls based credential. + creds, err := credentials.NewServerTLSFromFile(data.Path("x509/server_cert.pem"), data.Path("x509/server_key.pem")) + if err != nil { + log.Fatalf("failed to create credentials: %v", err) + } + + s := grpc.NewServer(grpc.Creds(creds)) + + // Register EchoServer on the server. + pb.RegisterEchoServer(s, &ecServer{}) + + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/encryption/mTLS/client/main.go b/examples/features/encryption/mTLS/client/main.go new file mode 100644 index 0000000..a6549b3 --- /dev/null +++ b/examples/features/encryption/mTLS/client/main.go @@ -0,0 +1,81 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an example client which connects to the server using mTLS. +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "flag" + "fmt" + "log" + "os" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/examples/data" + ecpb "google.golang.org/grpc/examples/features/proto/echo" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +func callUnaryEcho(client ecpb.EchoClient, message string) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := client.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message}) + if err != nil { + log.Fatalf("client.UnaryEcho(_) = _, %v: ", err) + } + fmt.Println("UnaryEcho: ", resp.Message) +} + +func main() { + flag.Parse() + + cert, err := tls.LoadX509KeyPair(data.Path("x509/client_cert.pem"), data.Path("x509/client_key.pem")) + if err != nil { + log.Fatalf("failed to load client cert: %v", err) + } + + ca := x509.NewCertPool() + caFilePath := data.Path("x509/ca_cert.pem") + caBytes, err := os.ReadFile(caFilePath) + if err != nil { + log.Fatalf("failed to read ca cert %q: %v", caFilePath, err) + } + if ok := ca.AppendCertsFromPEM(caBytes); !ok { + log.Fatalf("failed to parse %q", caFilePath) + } + + tlsConfig := &tls.Config{ + ServerName: "x.test.example.com", + Certificates: []tls.Certificate{cert}, + RootCAs: ca, + } + + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + callUnaryEcho(ecpb.NewEchoClient(conn), "hello world") +} diff --git a/examples/features/encryption/mTLS/server/main.go b/examples/features/encryption/mTLS/server/main.go new file mode 100644 index 0000000..26dbbfd --- /dev/null +++ b/examples/features/encryption/mTLS/server/main.go @@ -0,0 +1,82 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is an example server which authenticates clients using mTLS. +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "flag" + "fmt" + "log" + "net" + "os" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/examples/data" + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +type ecServer struct { + pb.UnimplementedEchoServer +} + +func (s *ecServer) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: req.Message}, nil +} + +func main() { + flag.Parse() + log.Printf("server starting on port %d...\n", *port) + + cert, err := tls.LoadX509KeyPair(data.Path("x509/server_cert.pem"), data.Path("x509/server_key.pem")) + if err != nil { + log.Fatalf("failed to load key pair: %s", err) + } + + ca := x509.NewCertPool() + caFilePath := data.Path("x509/client_ca_cert.pem") + caBytes, err := os.ReadFile(caFilePath) + if err != nil { + log.Fatalf("failed to read ca cert %q: %v", caFilePath, err) + } + if ok := ca.AppendCertsFromPEM(caBytes); !ok { + log.Fatalf("failed to parse %q", caFilePath) + } + + tlsConfig := &tls.Config{ + ClientAuth: tls.RequireAndVerifyClientCert, + Certificates: []tls.Certificate{cert}, + ClientCAs: ca, + } + + s := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig))) + pb.RegisterEchoServer(s, &ecServer{}) + lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/error_details/README.md b/examples/features/error_details/README.md new file mode 100644 index 0000000..62eee83 --- /dev/null +++ b/examples/features/error_details/README.md @@ -0,0 +1,25 @@ +# Description + +This example demonstrates the use of status details in grpc errors. + +# Run the sample code + +Run the server: + +```sh +$ go run ./server/main.go +``` +Then run the client in another terminal: + +```sh +$ go run ./client/main.go +``` + +It should succeed and print the greeting it received from the server. +Then run the client again: + +```sh +$ go run ./client/main.go +``` + +This time, it should fail by printing error status details that it received from the server. diff --git a/examples/features/error_details/client/main.go b/examples/features/error_details/client/main.go new file mode 100644 index 0000000..d7fc9cc --- /dev/null +++ b/examples/features/error_details/client/main.go @@ -0,0 +1,69 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to handle error messages from a gRPC server. +package main + +import ( + "context" + "flag" + "log" + "os" + "time" + + epb "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/status" +) + +var addr = flag.String("addr", "localhost:50052", "the address to connect to") + +func main() { + flag.Parse() + + // Set up a connection to the server. + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer func() { + if e := conn.Close(); e != nil { + log.Printf("failed to close connection: %s", e) + } + }() + c := pb.NewGreeterClient(conn) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + r, err := c.SayHello(ctx, &pb.HelloRequest{Name: "world"}) + if err != nil { + s := status.Convert(err) + for _, d := range s.Details() { + switch info := d.(type) { + case *epb.QuotaFailure: + log.Printf("Quota failure: %s", info) + default: + log.Printf("Unexpected type: %s", info) + } + } + os.Exit(1) + } + log.Printf("Greeting: %s", r.Message) +} diff --git a/examples/features/error_details/server/main.go b/examples/features/error_details/server/main.go new file mode 100644 index 0000000..798513b --- /dev/null +++ b/examples/features/error_details/server/main.go @@ -0,0 +1,85 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to provide error messages in gRPC responses. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + epb "google.golang.org/genproto/googleapis/rpc/errdetails" + pb "google.golang.org/grpc/examples/helloworld/helloworld" +) + +var port = flag.Int("port", 50052, "port number") + +// server is used to implement helloworld.GreeterServer. +type server struct { + pb.UnimplementedGreeterServer + mu sync.Mutex + count map[string]int +} + +// SayHello implements helloworld.GreeterServer +func (s *server) SayHello(_ context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + s.mu.Lock() + defer s.mu.Unlock() + // Track the number of times the user has been greeted. + s.count[in.Name]++ + if s.count[in.Name] > 1 { + st := status.New(codes.ResourceExhausted, "Request limit exceeded.") + ds, err := st.WithDetails( + &epb.QuotaFailure{ + Violations: []*epb.QuotaFailure_Violation{{ + Subject: fmt.Sprintf("name:%s", in.Name), + Description: "Limit one greeting per person", + }}, + }, + ) + if err != nil { + return nil, st.Err() + } + return nil, ds.Err() + } + return &pb.HelloReply{Message: "Hello " + in.Name}, nil +} + +func main() { + flag.Parse() + + address := fmt.Sprintf(":%v", *port) + lis, err := net.Listen("tcp", address) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + s := grpc.NewServer() + pb.RegisterGreeterServer(s, &server{count: make(map[string]int)}) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/error_handling/README.md b/examples/features/error_handling/README.md new file mode 100644 index 0000000..c6c4ba2 --- /dev/null +++ b/examples/features/error_handling/README.md @@ -0,0 +1,22 @@ +# Description + +This example demonstrates basic RPC error handling in gRPC. + +# Run the sample code + +Run the server, which returns an error if the RPC request's `Name` field is +empty. + +```sh +$ go run ./server/main.go +``` + +Then run the client in another terminal, which does two requests: one with an +empty Name field and one with it populated with the current username provided by +os/user. + +```sh +$ go run ./client/main.go +``` + +It should print the status codes it received from the server. diff --git a/examples/features/error_handling/client/main.go b/examples/features/error_handling/client/main.go new file mode 100644 index 0000000..7eadaad --- /dev/null +++ b/examples/features/error_handling/client/main.go @@ -0,0 +1,70 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to handle errors returned by a gRPC server. +package main + +import ( + "context" + "flag" + "log" + "os/user" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/status" +) + +var addr = flag.String("addr", "localhost:50052", "the address to connect to") + +func main() { + flag.Parse() + + name := "unknown" + if u, err := user.Current(); err == nil && u.Username != "" { + name = u.Username + } + + // Set up a connection to the server. + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("Failed to connect: %v", err) + } + defer conn.Close() + c := pb.NewGreeterClient(conn) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + for _, reqName := range []string{"", name} { + log.Printf("Calling SayHello with Name:%q", reqName) + r, err := c.SayHello(ctx, &pb.HelloRequest{Name: reqName}) + if err != nil { + if status.Code(err) != codes.InvalidArgument { + log.Printf("Received unexpected error: %v", err) + continue + } + log.Printf("Received error: %v", err) + continue + } + log.Printf("Received response: %s", r.Message) + } +} diff --git a/examples/features/error_handling/server/main.go b/examples/features/error_handling/server/main.go new file mode 100644 index 0000000..44c8809 --- /dev/null +++ b/examples/features/error_handling/server/main.go @@ -0,0 +1,66 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to return specific error codes in gRPC +// responses. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + pb "google.golang.org/grpc/examples/helloworld/helloworld" +) + +var port = flag.Int("port", 50052, "port number") + +// server is used to implement helloworld.GreeterServer. +type server struct { + pb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer. +func (s *server) SayHello(_ context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + if in.Name == "" { + return nil, status.Errorf(codes.InvalidArgument, "request missing required field: Name") + } + return &pb.HelloReply{Message: "Hello " + in.Name}, nil +} + +func main() { + flag.Parse() + + address := fmt.Sprintf(":%v", *port) + lis, err := net.Listen("tcp", address) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + s := grpc.NewServer() + pb.RegisterGreeterServer(s, &server{}) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/flow_control/README.md b/examples/features/flow_control/README.md new file mode 100644 index 0000000..0ffb7c1 --- /dev/null +++ b/examples/features/flow_control/README.md @@ -0,0 +1,55 @@ +# Flow Control + +Flow control is a feature in gRPC that prevents senders from writing more data +on a stream than a receiver is capable of handling. This feature behaves the +same for both clients and servers. Because gRPC-Go uses a blocking-style API +for stream operations, flow control pushback is implemented by simply blocking +send operations on a stream when that stream's flow control limits have been +reached. When the receiver has read enough data from the stream, the send +operation will unblock automatically. Flow control is configured automatically +based on a connection's Bandwidth Delay Product (BDP) to ensure the buffer is +the minimum size necessary to allow for maximum throughput on the stream if the +receiver is reading at its maximum speed. + +## Try it + +``` +go run ./server +``` + +``` +go run ./client +``` + +## Example explanation + +The example client and server are written to demonstrate the blocking by +intentionally sending messages while the other side is not receiving. The +bidirectional echo stream in the example begins by having the client send +messages until it detects it has blocked (utilizing another goroutine). The +server sleeps for 2 seconds to allow this to occur. Then the server will read +all of these messages, and the roles of the client and server are swapped so the +server attempts to send continuously while the client sleeps. After the client +sleeps for 2 seconds, it will read again to unblock the server. The server will +detect that it has blocked, and end the stream once it has unblocked. + +### Expected Output + +The client output should look like: +``` +2023/09/19 15:49:49 New stream began. +2023/09/19 15:49:50 Sending is blocked. +2023/09/19 15:49:51 Sent 25 messages. +2023/09/19 15:49:53 Read 25 messages. +2023/09/19 15:49:53 Stream ended successfully. +``` + +while the server should output the following logs: + +``` +2023/09/19 15:49:49 New stream began. +2023/09/19 15:49:51 Read 25 messages. +2023/09/19 15:49:52 Sending is blocked. +2023/09/19 15:49:53 Sent 25 messages. +2023/09/19 15:49:53 Stream ended successfully. +``` diff --git a/examples/features/flow_control/client/main.go b/examples/features/flow_control/client/main.go new file mode 100644 index 0000000..eeba8a2 --- /dev/null +++ b/examples/features/flow_control/client/main.go @@ -0,0 +1,103 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how the gRPC flow control blocks sending when the +// receiver is not ready. +package main + +import ( + "context" + "flag" + "io" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/internal/grpcsync" +) + +var addr = flag.String("addr", "localhost:50052", "the address to connect to") + +var payload = string(make([]byte, 8*1024)) // 8KB + +func main() { + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + c := pb.NewEchoClient(conn) + + stream, err := c.BidirectionalStreamingEcho(ctx) + if err != nil { + log.Fatalf("Error creating stream: %v", err) + } + log.Printf("New stream began.") + + // First we will send data on the stream until we cannot send any more. We + // detect this by not seeing a message sent 1s after the last sent message. + stopSending := grpcsync.NewEvent() + sentOne := make(chan struct{}) + go func() { + i := 0 + for !stopSending.HasFired() { + i++ + if err := stream.Send(&pb.EchoRequest{Message: payload}); err != nil { + log.Fatalf("Error sending data: %v", err) + } + sentOne <- struct{}{} + } + log.Printf("Sent %v messages.", i) + stream.CloseSend() + }() + + for !stopSending.HasFired() { + after := time.NewTimer(time.Second) + select { + case <-sentOne: + after.Stop() + case <-after.C: + log.Printf("Sending is blocked.") + stopSending.Fire() + <-sentOne + } + } + + // Next, we wait 2 seconds before reading from the stream, to give the + // server an opportunity to block while sending its responses. + time.Sleep(2 * time.Second) + + // Finally, read all the data sent by the server to allow it to unblock. + for i := 0; true; i++ { + if _, err := stream.Recv(); err != nil { + log.Printf("Read %v messages.", i) + if err == io.EOF { + log.Printf("Stream ended successfully.") + return + } + log.Fatalf("Error receiving data: %v", err) + } + } +} diff --git a/examples/features/flow_control/server/main.go b/examples/features/flow_control/server/main.go new file mode 100644 index 0000000..af03f0d --- /dev/null +++ b/examples/features/flow_control/server/main.go @@ -0,0 +1,111 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how gRPC flow control block sending when the +// receiver is not ready. +package main + +import ( + "flag" + "fmt" + "io" + "log" + "net" + "time" + + "google.golang.org/grpc" + + pb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/internal/grpcsync" +) + +var port = flag.Int("port", 50052, "port number") + +var payload = string(make([]byte, 8*1024)) // 8KB + +// server is used to implement EchoServer. +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { + log.Printf("New stream began.") + // First, we wait 2 seconds before reading from the stream, to give the + // client an opportunity to block while sending its requests. + time.Sleep(2 * time.Second) + + // Next, read all the data sent by the client to allow it to unblock. + for i := 0; true; i++ { + if _, err := stream.Recv(); err != nil { + log.Printf("Read %v messages.", i) + if err == io.EOF { + break + } + log.Printf("Error receiving data: %v", err) + return err + } + } + + // Finally, send data until we block, then end the stream after we unblock. + stopSending := grpcsync.NewEvent() + sentOne := make(chan struct{}) + go func() { + for !stopSending.HasFired() { + after := time.NewTimer(time.Second) + select { + case <-sentOne: + after.Stop() + case <-after.C: + log.Printf("Sending is blocked.") + stopSending.Fire() + <-sentOne + } + } + }() + + i := 0 + for !stopSending.HasFired() { + i++ + if err := stream.Send(&pb.EchoResponse{Message: payload}); err != nil { + log.Printf("Error sending data: %v", err) + return err + } + sentOne <- struct{}{} + } + log.Printf("Sent %v messages.", i) + + log.Printf("Stream ended successfully.") + return nil +} + +func main() { + flag.Parse() + + address := fmt.Sprintf(":%v", *port) + lis, err := net.Listen("tcp", address) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + grpcServer := grpc.NewServer() + pb.RegisterEchoServer(grpcServer, &server{}) + + if err := grpcServer.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/gracefulstop/README.md b/examples/features/gracefulstop/README.md new file mode 100644 index 0000000..b9a7eff --- /dev/null +++ b/examples/features/gracefulstop/README.md @@ -0,0 +1,45 @@ +# Graceful Stop + +This example demonstrates how to gracefully stop a gRPC server using +`Server.GracefulStop()`. The graceful shutdown process involves two key steps: + +- Initiate `Server.GracefulStop()`. This function blocks until all currently + running RPCs have completed. This ensures that in-flight requests are + allowed to finish processing. + +- It's crucial to call `Server.Stop()` with a timeout before calling + `GracefulStop()`. This acts as a safety net, ensuring that the server + eventually shuts down even if some in-flight RPCs don't complete within a + reasonable timeframe. This prevents indefinite blocking. + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +The server starts with a client streaming and unary request handler. When +client streaming is started, client streaming handler signals the server to +initiate graceful stop and waits for the stream to be closed or aborted. Until +the`Server.GracefulStop()` is initiated, server will continue to accept unary +requests. Once `Server.GracefulStop()` is initiated, server will not accept +new unary requests. + +Client will start the client stream to the server and starts making unary +requests until receiving an error. Error will indicate that the server graceful +shutdown is initiated so client will stop making further unary requests and +closes the client stream. + +Server and client will keep track of number of unary requests processed on +their side. Once the client has successfully closed the stream, server returns +the total number of unary requests processed as response. The number from +stream response should be equal to the number of unary requests tracked by +client. This indicates that server has processed all in-flight requests before +shutting down. + diff --git a/examples/features/gracefulstop/client/main.go b/examples/features/gracefulstop/client/main.go new file mode 100644 index 0000000..772d128 --- /dev/null +++ b/examples/features/gracefulstop/client/main.go @@ -0,0 +1,80 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Binary client demonstrates sending multiple requests to server and observe +// graceful stop. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var addr = flag.String("addr", "localhost:50052", "the address to connect to") + +func main() { + flag.Parse() + + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("Failed to create new client: %v", err) + } + defer conn.Close() + c := pb.NewEchoClient(conn) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + // Start a client stream and keep calling the `c.UnaryEcho` until receiving + // an error. Error will indicate that server graceful stop is initiated and + // it won't accept any new requests. + stream, err := c.ClientStreamingEcho(ctx) + if err != nil { + log.Fatalf("Error starting stream: %v", err) + } + + // Keep track of successful unary requests which can be compared later to + // the successful unary requests reported by the server. + unaryRequests := 0 + for { + r, err := c.UnaryEcho(ctx, &pb.EchoRequest{Message: "Hello"}) + if err != nil { + log.Printf("Error calling `UnaryEcho`. Server graceful stop initiated: %v", err) + break + } + unaryRequests++ + time.Sleep(200 * time.Millisecond) + log.Print(r.Message) + } + log.Printf("Successful unary requests made by client: %d", unaryRequests) + + r, err := stream.CloseAndRecv() + if err != nil { + log.Fatalf("Error closing stream: %v", err) + } + if fmt.Sprintf("%d", unaryRequests) != r.Message { + log.Fatalf("Got %s successful unary requests processed from server, want: %d", r.Message, unaryRequests) + } + log.Printf("Successful unary requests processed by server and made by client are same.") +} diff --git a/examples/features/gracefulstop/server/main.go b/examples/features/gracefulstop/server/main.go new file mode 100644 index 0000000..3abfa07 --- /dev/null +++ b/examples/features/gracefulstop/server/main.go @@ -0,0 +1,117 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Binary server demonstrates how to gracefully stop a gRPC server. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "log" + "net" + "sync/atomic" + "time" + + "google.golang.org/grpc" + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var ( + port = flag.Int("port", 50052, "port number") +) + +type server struct { + pb.UnimplementedEchoServer + + unaryRequests atomic.Int32 // to track number of unary RPCs processed + streamStart chan struct{} // to signal if server streaming started +} + +// ClientStreamingEcho implements the EchoService.ClientStreamingEcho method. +// It signals the server that streaming has started and waits for the stream to +// be done or aborted. If `io.EOF` is received on stream that means client +// has successfully closed the stream using `stream.CloseAndRecv()`, so it +// returns an `EchoResponse` with the total number of unary RPCs processed +// otherwise, it returns the error indicating stream is aborted. +func (s *server) ClientStreamingEcho(stream pb.Echo_ClientStreamingEchoServer) error { + // Signal streaming start to initiate graceful stop which should wait until + // server streaming finishes. + s.streamStart <- struct{}{} + + if err := stream.RecvMsg(&pb.EchoResponse{}); err != nil { + if errors.Is(err, io.EOF) { + stream.SendAndClose(&pb.EchoResponse{Message: fmt.Sprintf("%d", s.unaryRequests.Load())}) + return nil + } + return err + } + + return nil +} + +// UnaryEcho implements the EchoService.UnaryEcho method. It increments +// `s.unaryRequests` on every call and returns it as part of `EchoResponse`. +func (s *server) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + s.unaryRequests.Add(1) + return &pb.EchoResponse{Message: req.Message}, nil +} + +func main() { + flag.Parse() + + address := fmt.Sprintf(":%v", *port) + lis, err := net.Listen("tcp", address) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + s := grpc.NewServer() + ss := &server{streamStart: make(chan struct{})} + pb.RegisterEchoServer(s, ss) + serverStopped := make(chan struct{}, 1) + + go func() { + <-ss.streamStart // wait until server streaming starts + time.Sleep(1 * time.Second) + log.Println("Initiating graceful shutdown...") + timer := time.AfterFunc(10*time.Second, func() { + log.Println("Server couldn't stop gracefully in time. Doing force stop.") + s.Stop() + // Unblock the main function. + select { + case serverStopped <- struct{}{}: + default: + } + }) + defer timer.Stop() + s.GracefulStop() // gracefully stop server after in-flight server streaming rpc finishes + log.Println("Server stopped gracefully.") + // Unblock the main function. + select { + case serverStopped <- struct{}{}: + default: + } + }() + + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } + <-serverStopped +} diff --git a/examples/features/health/README.md b/examples/features/health/README.md new file mode 100644 index 0000000..926b775 --- /dev/null +++ b/examples/features/health/README.md @@ -0,0 +1,74 @@ +# Health + +gRPC provides a health library to communicate a system's health to their +clients. It works by providing a service definition via the +[health/v1](https://github.com/grpc/grpc-proto/blob/master/grpc/health/v1/health.proto) +api. + +Clients use the health library to gracefully avoid servers that encounter +issues. Most languages provide an out-of-box implementation, which makes it +interoperable between systems. + +## Try it + +``` +go run server/main.go -port=50051 -sleep=5s +go run server/main.go -port=50052 -sleep=10s +``` + +``` +go run client/main.go +``` + +## Explanation + +### Client + +Clients have two ways to monitor a server's health. They use `Check()` to probe +a server's health or `Watch()` to observe changes. + +In most cases, clients do not directly check backend servers. Instead, they do +this transparently when you specify a `healthCheckConfig` in the [service +config](https://github.com/grpc/proposal/blob/master/A17-client-side-health-checking.md#service-config-changes) +and import the [health +package](https://pkg.go.dev/google.golang.org/grpc/health). The `serviceName` in +`healthCheckConfig` is used in the health check when connections are +established. An empty string (`""`) typically indicates the overall health of a +server. + +```go +// import grpc/health to enable transparent client side checking +import _ "google.golang.org/grpc/health" + +serviceConfig := grpc.WithDefaultServiceConfig(`{ + "loadBalancingPolicy": "round_robin", + "healthCheckConfig": { + "serviceName": "" + } +}`) + +conn, err := grpc.NewClient(..., serviceConfig) +``` + +See [A17 - Client-Side Health +Checking](https://github.com/grpc/proposal/blob/master/A17-client-side-health-checking.md) +for more details. + +### Server + +Servers control their serving status. They do this by inspecting dependent +systems, then update their status accordingly. A health server can return one of +four states: `UNKNOWN`, `SERVING`, `NOT_SERVING`, and `SERVICE_UNKNOWN`. + +`UNKNOWN` indicates the system does not yet know the current state. Servers +often show this state at startup. + +`SERVING` means that the system is healthy and ready to service requests. +Conversely, `NOT_SERVING` indicates the system is unable to service requests at +the time. + +`SERVICE_UNKNOWN` communicates the `serviceName` requested by the client is not +known by the server. This status is only reported by the `Watch()` call. + +A server may toggle its health using +`healthServer.SetServingStatus("serviceName", servingStatus)`. diff --git a/examples/features/health/client/main.go b/examples/features/health/client/main.go new file mode 100644 index 0000000..f7c97aa --- /dev/null +++ b/examples/features/health/client/main.go @@ -0,0 +1,88 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to check and observe gRPC server health using +// the health library. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/features/proto/echo" + _ "google.golang.org/grpc/health" // REQUIRED to enable health checks + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" +) + +const serviceConfig = `{ + "loadBalancingPolicy": "round_robin", + "healthCheckConfig": { + "serviceName": "" + } +}` + +func callUnaryEcho(c pb.EchoClient) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + r, err := c.UnaryEcho(ctx, &pb.EchoRequest{}) + if err != nil { + fmt.Println("UnaryEcho: _, ", err) + } else { + fmt.Println("UnaryEcho: ", r.GetMessage()) + } +} + +func main() { + flag.Parse() + + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{ + Addresses: []resolver.Address{ + {Addr: "localhost:50051"}, + {Addr: "localhost:50052"}, + }, + }) + + address := fmt.Sprintf("%s:///unused", r.Scheme()) + + options := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + // google.golang.org/grpc/health must also be imported + grpc.WithDefaultServiceConfig(serviceConfig), + } + + conn, err := grpc.NewClient(address, options...) + if err != nil { + log.Fatalf("grpc.NewClient(%q): %v", address, err) + } + defer conn.Close() + + echoClient := pb.NewEchoClient(conn) + + for { + callUnaryEcho(echoClient) + time.Sleep(time.Second) + } +} diff --git a/examples/features/health/server/main.go b/examples/features/health/server/main.go new file mode 100644 index 0000000..ea94c36 --- /dev/null +++ b/examples/features/health/server/main.go @@ -0,0 +1,90 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to manage and report its health status using +// the gRPC health library. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "time" + + "google.golang.org/grpc" + pb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/health" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +var ( + port = flag.Int("port", 50051, "the port to serve on") + sleep = flag.Duration("sleep", time.Second*5, "duration between changes in health") + + system = "" // empty string represents the health of the system +) + +type echoServer struct { + pb.UnimplementedEchoServer +} + +func (e *echoServer) UnaryEcho(context.Context, *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{ + Message: fmt.Sprintf("hello from localhost:%d", *port), + }, nil +} + +var _ pb.EchoServer = &echoServer{} + +func main() { + flag.Parse() + + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + s := grpc.NewServer() + healthcheck := health.NewServer() + healthgrpc.RegisterHealthServer(s, healthcheck) + pb.RegisterEchoServer(s, &echoServer{}) + + go func() { + // asynchronously inspect dependencies and toggle serving status as needed + next := healthpb.HealthCheckResponse_SERVING + + for { + healthcheck.SetServingStatus(system, next) + + if next == healthpb.HealthCheckResponse_SERVING { + next = healthpb.HealthCheckResponse_NOT_SERVING + } else { + next = healthpb.HealthCheckResponse_SERVING + } + + time.Sleep(*sleep) + } + }() + + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/interceptor/README.md b/examples/features/interceptor/README.md new file mode 100644 index 0000000..bb9dff7 --- /dev/null +++ b/examples/features/interceptor/README.md @@ -0,0 +1,130 @@ +# Interceptor + +gRPC provides simple APIs to implement and install interceptors on a per +ClientConn/Server basis. Interceptors act as a layer between the application and +gRPC and can be used to observe or control the behavior of gRPC. Interceptors +can be used for logging, authentication/authorization, metrics collection, and +other functionality that is shared across RPCs. + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +gRPC has separate interceptors for unary RPCs and streaming RPCs. See the +[gRPC docs](https://grpc.io/docs/guides/concepts.html#rpc-life-cycle) for an +explanation about unary and streaming RPCs. Both the client and the server have +their own types of unary and stream interceptors. Thus, there are four different +types of interceptors in total. + +### Client-side + +#### Unary Interceptor + +The type for client-side unary interceptors is +[`UnaryClientInterceptor`](https://godoc.org/google.golang.org/grpc#UnaryClientInterceptor). +It is a function type with signature: + +```golang +func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error +``` + +Unary interceptor implementations can usually be divided into three parts: +pre-processing, invoking the RPC method, and post-processing. + +For pre-processing, users can get info about the current RPC call by examining +the args passed in. The args include the RPC context, method string, request to +be sent, and the CallOptions configured. With this info, users can even modify +the RPC call. For instance, in the example, we examine the list of CallOptions +and check if the call credentials have been configured. If not, the interceptor +configures the RPC call to use oauth2 with a token "some-secret-token" as a +fallback. In our example, we intentionally omit configuring the per RPC +credential to resort to the fallback. + +After pre-processing, users can invoke the RPC call by calling the `invoker`. + +Once the invoker returns, users can post-process the RPC call. This usually +involves dealing with the returned reply and error. In the example, we log the +RPC timing and error info. + +To install a unary interceptor on a ClientConn, configure `Dial` with the +[`WithUnaryInterceptor`](https://godoc.org/google.golang.org/grpc#WithUnaryInterceptor) +`DialOption`. + +#### Stream Interceptor + +The type for client-side stream interceptors is +[`StreamClientInterceptor`](https://godoc.org/google.golang.org/grpc#StreamClientInterceptor). +It is a function type with signature: + +```golang +func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error) +``` + +An implementation of a stream interceptor usually includes pre-processing, and +stream operation interception. + +The pre-processing is similar to unary interceptors. + +However, rather than invoking the RPC method followed by post-processing, stream +interceptors intercept the users' operations on the stream. The interceptor +first calls the passed-in `streamer` to get a `ClientStream`, and then wraps the +`ClientStream` while overriding its methods with the interception logic. +Finally, the interceptor returns the wrapped `ClientStream` to user to operate +on. + +In the example, we define a new struct `wrappedStream`, which embeds a +`ClientStream`. We then implement (override) the `SendMsg` and `RecvMsg` methods +on `wrappedStream` to intercept these two operations on the embedded +`ClientStream`. In the example, we log the message type info and time info for +interception purpose. + +To install a stream interceptor for a ClientConn, configure `Dial` with the +[`WithStreamInterceptor`](https://godoc.org/google.golang.org/grpc#WithStreamInterceptor) +`DialOption`. + +### Server-side + +Server side interceptors are similar to client side interceptors, with slightly +different information provided as args. + +#### Unary Interceptor + +The type for server-side unary interceptors is +[`UnaryServerInterceptor`](https://godoc.org/google.golang.org/grpc#UnaryServerInterceptor). +It is a function type with signature: + +```golang +func(ctx context.Context, req interface{}, info *UnaryServerInfo, handler UnaryHandler) (resp interface{}, err error) +``` + +Refer to the client-side unary interceptor section for a detailed implementation +and explanation. + +To install a unary interceptor on a Server, configure `NewServer` with the +[`UnaryInterceptor`](https://godoc.org/google.golang.org/grpc#UnaryInterceptor) +`ServerOption`. + +#### Stream Interceptor + +The type for server-side stream interceptors is +[`StreamServerInterceptor`](https://godoc.org/google.golang.org/grpc#StreamServerInterceptor). +It is a function type with the signature: + +```golang +func(srv interface{}, ss ServerStream, info *StreamServerInfo, handler StreamHandler) error +``` + +Refer to the client-side stream interceptor section for a detailed +implementation and explanation. + +To install a stream interceptor on a Server, configure `NewServer` with the +[`StreamInterceptor`](https://godoc.org/google.golang.org/grpc#StreamInterceptor) +`ServerOption`. diff --git a/examples/features/interceptor/client/main.go b/examples/features/interceptor/client/main.go new file mode 100644 index 0000000..feb59bf --- /dev/null +++ b/examples/features/interceptor/client/main.go @@ -0,0 +1,167 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to use interceptors to observe or control the +// behavior of gRPC including logging, authentication,metrics collection, etc. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "time" + + "golang.org/x/oauth2" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/oauth" + "google.golang.org/grpc/examples/data" + ecpb "google.golang.org/grpc/examples/features/proto/echo" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +const fallbackToken = "some-secret-token" + +// logger is to mock a sophisticated logging system. To simplify the example, we just print out the content. +func logger(format string, a ...any) { + fmt.Printf("LOG:\t"+format+"\n", a...) +} + +// unaryInterceptor is an example unary interceptor. +func unaryInterceptor(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + var credsConfigured bool + for _, o := range opts { + _, ok := o.(grpc.PerRPCCredsCallOption) + if ok { + credsConfigured = true + break + } + } + if !credsConfigured { + opts = append(opts, grpc.PerRPCCredentials(oauth.TokenSource{ + TokenSource: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: fallbackToken}), + })) + } + start := time.Now() + err := invoker(ctx, method, req, reply, cc, opts...) + end := time.Now() + logger("RPC: %s, start time: %s, end time: %s, err: %v", method, start.Format("Basic"), end.Format(time.RFC3339), err) + return err +} + +// wrappedStream wraps around the embedded grpc.ClientStream, and intercepts the RecvMsg and +// SendMsg method call. +type wrappedStream struct { + grpc.ClientStream +} + +func (w *wrappedStream) RecvMsg(m any) error { + logger("Receive a message (Type: %T) at %v", m, time.Now().Format(time.RFC3339)) + return w.ClientStream.RecvMsg(m) +} + +func (w *wrappedStream) SendMsg(m any) error { + logger("Send a message (Type: %T) at %v", m, time.Now().Format(time.RFC3339)) + return w.ClientStream.SendMsg(m) +} + +func newWrappedStream(s grpc.ClientStream) grpc.ClientStream { + return &wrappedStream{s} +} + +// streamInterceptor is an example stream interceptor. +func streamInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + var credsConfigured bool + for _, o := range opts { + _, ok := o.(*grpc.PerRPCCredsCallOption) + if ok { + credsConfigured = true + break + } + } + if !credsConfigured { + opts = append(opts, grpc.PerRPCCredentials(oauth.TokenSource{ + TokenSource: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: fallbackToken}), + })) + } + s, err := streamer(ctx, desc, cc, method, opts...) + if err != nil { + return nil, err + } + return newWrappedStream(s), nil +} + +func callUnaryEcho(client ecpb.EchoClient, message string) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := client.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message}) + if err != nil { + log.Fatalf("client.UnaryEcho(_) = _, %v: ", err) + } + fmt.Println("UnaryEcho: ", resp.Message) +} + +func callBidiStreamingEcho(client ecpb.EchoClient) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c, err := client.BidirectionalStreamingEcho(ctx) + if err != nil { + return + } + for i := 0; i < 5; i++ { + if err := c.Send(&ecpb.EchoRequest{Message: fmt.Sprintf("Request %d", i+1)}); err != nil { + log.Fatalf("failed to send request due to error: %v", err) + } + } + c.CloseSend() + for { + resp, err := c.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("failed to receive response due to error: %v", err) + } + fmt.Println("BidiStreaming Echo: ", resp.Message) + } +} + +func main() { + flag.Parse() + + // Create tls based credential. + creds, err := credentials.NewClientTLSFromFile(data.Path("x509/ca_cert.pem"), "x.test.example.com") + if err != nil { + log.Fatalf("failed to load credentials: %v", err) + } + + // Set up a connection to the server. + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(creds), grpc.WithUnaryInterceptor(unaryInterceptor), grpc.WithStreamInterceptor(streamInterceptor)) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + // Make an echo client and send RPCs. + rgc := ecpb.NewEchoClient(conn) + callUnaryEcho(rgc, "hello world") + callBidiStreamingEcho(rgc) +} diff --git a/examples/features/interceptor/server/main.go b/examples/features/interceptor/server/main.go new file mode 100644 index 0000000..40919ec --- /dev/null +++ b/examples/features/interceptor/server/main.go @@ -0,0 +1,166 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to use interceptors to observe or control the +// behavior of gRPC including logging, authentication,metrics collection, etc. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "net" + "strings" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/examples/data" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var ( + port = flag.Int("port", 50051, "the port to serve on") + + errMissingMetadata = status.Errorf(codes.InvalidArgument, "missing metadata") + errInvalidToken = status.Errorf(codes.Unauthenticated, "invalid token") +) + +// logger is to mock a sophisticated logging system. To simplify the example, we just print out the content. +func logger(format string, a ...any) { + fmt.Printf("LOG:\t"+format+"\n", a...) +} + +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(_ context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { + fmt.Printf("unary echoing message %q\n", in.Message) + return &pb.EchoResponse{Message: in.Message}, nil +} + +func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { + for { + in, err := stream.Recv() + if err != nil { + if err == io.EOF { + return nil + } + fmt.Printf("server: error receiving from stream: %v\n", err) + return err + } + fmt.Printf("bidi echoing message %q\n", in.Message) + stream.Send(&pb.EchoResponse{Message: in.Message}) + } +} + +// valid validates the authorization. +func valid(authorization []string) bool { + if len(authorization) < 1 { + return false + } + token := strings.TrimPrefix(authorization[0], "Bearer ") + // Perform the token validation here. For the sake of this example, the code + // here forgoes any of the usual OAuth2 token validation and instead checks + // for a token matching an arbitrary string. + return token == "some-secret-token" +} + +func unaryInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + // authentication (token verification) + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, errMissingMetadata + } + if !valid(md["authorization"]) { + return nil, errInvalidToken + } + m, err := handler(ctx, req) + if err != nil { + logger("RPC failed with error: %v", err) + } + return m, err +} + +// wrappedStream wraps around the embedded grpc.ServerStream, and intercepts the RecvMsg and +// SendMsg method call. +type wrappedStream struct { + grpc.ServerStream +} + +func (w *wrappedStream) RecvMsg(m any) error { + logger("Receive a message (Type: %T) at %s", m, time.Now().Format(time.RFC3339)) + return w.ServerStream.RecvMsg(m) +} + +func (w *wrappedStream) SendMsg(m any) error { + logger("Send a message (Type: %T) at %v", m, time.Now().Format(time.RFC3339)) + return w.ServerStream.SendMsg(m) +} + +func newWrappedStream(s grpc.ServerStream) grpc.ServerStream { + return &wrappedStream{s} +} + +func streamInterceptor(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + // authentication (token verification) + md, ok := metadata.FromIncomingContext(ss.Context()) + if !ok { + return errMissingMetadata + } + if !valid(md["authorization"]) { + return errInvalidToken + } + + err := handler(srv, newWrappedStream(ss)) + if err != nil { + logger("RPC failed with error: %v", err) + } + return err +} + +func main() { + flag.Parse() + + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + // Create tls based credential. + creds, err := credentials.NewServerTLSFromFile(data.Path("x509/server_cert.pem"), data.Path("x509/server_key.pem")) + if err != nil { + log.Fatalf("failed to create credentials: %v", err) + } + + s := grpc.NewServer(grpc.Creds(creds), grpc.UnaryInterceptor(unaryInterceptor), grpc.StreamInterceptor(streamInterceptor)) + + // Register EchoServer on the server. + pb.RegisterEchoServer(s, &server{}) + + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/keepalive/README.md b/examples/features/keepalive/README.md new file mode 100644 index 0000000..e97fbc4 --- /dev/null +++ b/examples/features/keepalive/README.md @@ -0,0 +1,15 @@ +# Keepalive + +This example illustrates how to set up client-side keepalive pings and +server-side keepalive ping enforcement and connection idleness settings. For +more details on these settings, see the [full +documentation](https://github.com/grpc/grpc-go/tree/master/Documentation/keepalive.md). + + +``` +go run server/main.go +``` + +``` +GODEBUG=http2debug=2 go run client/main.go +``` diff --git a/examples/features/keepalive/client/main.go b/examples/features/keepalive/client/main.go new file mode 100644 index 0000000..6de0017 --- /dev/null +++ b/examples/features/keepalive/client/main.go @@ -0,0 +1,64 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to configure keepalive pings to maintain +// connectivity and detect stale connections. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/keepalive" +) + +var addr = flag.String("addr", "localhost:50052", "the address to connect to") + +var kacp = keepalive.ClientParameters{ + Time: 10 * time.Second, // send pings every 10 seconds if there is no activity + Timeout: time.Second, // wait 1 second for ping ack before considering the connection dead + PermitWithoutStream: true, // send pings even without active streams +} + +func main() { + flag.Parse() + + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithKeepaliveParams(kacp)) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + c := pb.NewEchoClient(conn) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + fmt.Println("Performing unary request") + res, err := c.UnaryEcho(ctx, &pb.EchoRequest{Message: "keepalive demo"}) + if err != nil { + log.Fatalf("unexpected error from UnaryEcho: %v", err) + } + fmt.Println("RPC response:", res) + select {} // Block forever; run with GODEBUG=http2debug=2 to observe ping frames and GOAWAYs due to idleness. +} diff --git a/examples/features/keepalive/server/main.go b/examples/features/keepalive/server/main.go new file mode 100644 index 0000000..f7ab42e --- /dev/null +++ b/examples/features/keepalive/server/main.go @@ -0,0 +1,76 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to enforce keepalive settings and manage idle +// connections to maintain active client connections. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50052, "port number") + +var kaep = keepalive.EnforcementPolicy{ + MinTime: 5 * time.Second, // If a client pings more than once every 5 seconds, terminate the connection + PermitWithoutStream: true, // Allow pings even when there are no active streams +} + +var kasp = keepalive.ServerParameters{ + MaxConnectionIdle: 15 * time.Second, // If a client is idle for 15 seconds, send a GOAWAY + MaxConnectionAge: 30 * time.Second, // If any connection is alive for more than 30 seconds, send a GOAWAY + MaxConnectionAgeGrace: 5 * time.Second, // Allow 5 seconds for pending RPCs to complete before forcibly closing connections + Time: 5 * time.Second, // Ping the client if it is idle for 5 seconds to ensure the connection is still active + Timeout: 1 * time.Second, // Wait 1 second for the ping ack before assuming the connection is dead +} + +// server implements EchoServer. +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: req.Message}, nil +} + +func main() { + flag.Parse() + + address := fmt.Sprintf(":%v", *port) + lis, err := net.Listen("tcp", address) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp)) + pb.RegisterEchoServer(s, &server{}) + + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/load_balancing/README.md b/examples/features/load_balancing/README.md new file mode 100644 index 0000000..3c9898d --- /dev/null +++ b/examples/features/load_balancing/README.md @@ -0,0 +1,83 @@ +# Load balancing + +This examples shows how `ClientConn` can pick different load balancing policies. + +Note: to show the effect of load balancers, an example resolver is installed in +this example to get the backend addresses. It's suggested to read the name +resolver example before this example. + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +Two echo servers are serving on ":50051" and ":50052". They will include their +serving address in the response. So the server on ":50051" will reply to the RPC +with `this is examples/load_balancing (from :50051)`. + +Two clients are created, to connect to both of these servers (they get both +server addresses from the name resolver). + +Each client picks a different load balancer (using +`grpc.WithDefaultServiceConfig`): `pick_first` or `round_robin`. (These two +policies are supported in gRPC by default. To add a custom balancing policy, +implement the interfaces defined in +https://godoc.org/google.golang.org/grpc/balancer). + +Note that balancers can also be switched using service config, which allows +service owners (instead of client owners) to pick the balancer to use. Service +config doc is available at +https://github.com/grpc/grpc/blob/master/doc/service_config.md. + +### pick_first + +The first client is configured to use `pick_first`. `pick_first` tries to +connect to the first address, uses it for all RPCs if it connects, or try the +next address if it fails (and keep doing that until one connection is +successful). Because of this, all the RPCs will be sent to the same backend. The +responses received all show the same backend address. + +``` +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50051) +``` + +### round_robin + +The second client is configured to use `round_robin`. `round_robin` connects to +all the addresses it sees, and sends an RPC to each backend one at a time in +order. E.g. the first RPC will be sent to backend-1, the second RPC will be +sent to backend-2, and the third RPC will be sent to backend-1 again. + +``` +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50052) +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50052) +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50052) +this is examples/load_balancing (from :50051) +this is examples/load_balancing (from :50052) +this is examples/load_balancing (from :50051) +``` + +Note that it's possible to see two continues RPC sent to the same backend. +That's because `round_robin` only picks the connections ready for RPCs. So if +one of the two connections is not ready for some reason, all RPCs will be sent +to the ready connection. diff --git a/examples/features/load_balancing/client/main.go b/examples/features/load_balancing/client/main.go new file mode 100644 index 0000000..c970b90 --- /dev/null +++ b/examples/features/load_balancing/client/main.go @@ -0,0 +1,127 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to configure load balancing policies to +// distribute RPCs across backend servers. +package main + +import ( + "context" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + ecpb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/resolver" +) + +const ( + exampleScheme = "example" + exampleServiceName = "lb.example.grpc.io" +) + +var addrs = []string{"localhost:50051", "localhost:50052"} + +func callUnaryEcho(c ecpb.EchoClient, message string) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + r, err := c.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message}) + if err != nil { + log.Fatalf("could not greet: %v", err) + } + fmt.Println(r.Message) +} + +func makeRPCs(cc *grpc.ClientConn, n int) { + hwc := ecpb.NewEchoClient(cc) + for i := 0; i < n; i++ { + callUnaryEcho(hwc, "this is examples/load_balancing") + } +} + +func main() { + // "pick_first" is the default, so there's no need to set the load balancing policy. + pickfirstConn, err := grpc.NewClient( + fmt.Sprintf("%s:///%s", exampleScheme, exampleServiceName), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer pickfirstConn.Close() + + fmt.Println("--- calling helloworld.Greeter/SayHello with pick_first ---") + makeRPCs(pickfirstConn, 10) + + fmt.Println() + + // Make another ClientConn with round_robin policy. + roundrobinConn, err := grpc.NewClient( + fmt.Sprintf("%s:///%s", exampleScheme, exampleServiceName), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), // This sets the initial balancing policy. + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer roundrobinConn.Close() + + fmt.Println("--- calling helloworld.Greeter/SayHello with round_robin ---") + makeRPCs(roundrobinConn, 10) +} + +// Following is an example name resolver implementation. Read the name +// resolution example to learn more about it. + +type exampleResolverBuilder struct{} + +func (*exampleResolverBuilder) Build(target resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) { + r := &exampleResolver{ + target: target, + cc: cc, + addrsStore: map[string][]string{ + exampleServiceName: addrs, + }, + } + r.start() + return r, nil +} +func (*exampleResolverBuilder) Scheme() string { return exampleScheme } + +type exampleResolver struct { + target resolver.Target + cc resolver.ClientConn + addrsStore map[string][]string +} + +func (r *exampleResolver) start() { + addrStrs := r.addrsStore[r.target.Endpoint()] + addrs := make([]resolver.Address, len(addrStrs)) + for i, s := range addrStrs { + addrs[i] = resolver.Address{Addr: s} + } + r.cc.UpdateState(resolver.State{Addresses: addrs}) +} +func (*exampleResolver) ResolveNow(resolver.ResolveNowOptions) {} +func (*exampleResolver) Close() {} + +func init() { + resolver.Register(&exampleResolverBuilder{}) +} diff --git a/examples/features/load_balancing/server/main.go b/examples/features/load_balancing/server/main.go new file mode 100644 index 0000000..c88f80c --- /dev/null +++ b/examples/features/load_balancing/server/main.go @@ -0,0 +1,71 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to spin up multiple server backends +// to enable client-side load balancing. +package main + +import ( + "context" + "fmt" + "log" + "net" + "sync" + + "google.golang.org/grpc" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var ( + addrs = []string{":50051", ":50052"} +) + +type ecServer struct { + pb.UnimplementedEchoServer + addr string +} + +func (s *ecServer) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: fmt.Sprintf("%s (from %s)", req.Message, s.addr)}, nil +} + +func startServer(addr string) { + lis, err := net.Listen("tcp", addr) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + s := grpc.NewServer() + pb.RegisterEchoServer(s, &ecServer{addr: addr}) + log.Printf("serving on %s\n", addr) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} + +func main() { + var wg sync.WaitGroup + for _, addr := range addrs { + wg.Add(1) + go func(addr string) { + defer wg.Done() + startServer(addr) + }(addr) + } + wg.Wait() +} diff --git a/examples/features/metadata/README.md b/examples/features/metadata/README.md new file mode 100644 index 0000000..af5ae1c --- /dev/null +++ b/examples/features/metadata/README.md @@ -0,0 +1,18 @@ +# Metadata example + +This example shows how to set and read metadata in RPC headers and trailers. +Please see +[grpc-metadata.md](https://github.com/grpc/grpc-go/blob/master/Documentation/grpc-metadata.md) +for more information. + +## Start the server + +``` +go run server/main.go +``` + +## Run the client + +``` +go run client/main.go +``` diff --git a/examples/features/metadata/client/main.go b/examples/features/metadata/client/main.go new file mode 100644 index 0000000..861e3ee --- /dev/null +++ b/examples/features/metadata/client/main.go @@ -0,0 +1,308 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to send and receive metadata to and from an RPC. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/metadata" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +const ( + timestampFormat = time.StampNano // "Jan _2 15:04:05.000" + streamingCount = 10 +) + +func unaryCallWithMetadata(c pb.EchoClient, message string) { + fmt.Printf("--- unary ---\n") + // Create metadata and context. + md := metadata.Pairs("timestamp", time.Now().Format(timestampFormat)) + ctx := metadata.NewOutgoingContext(context.Background(), md) + + // Make RPC using the context with the metadata. + var header, trailer metadata.MD + r, err := c.UnaryEcho(ctx, &pb.EchoRequest{Message: message}, grpc.Header(&header), grpc.Trailer(&trailer)) + if err != nil { + log.Fatalf("failed to call UnaryEcho: %v", err) + } + + if t, ok := header["timestamp"]; ok { + fmt.Printf("timestamp from header:\n") + for i, e := range t { + fmt.Printf(" %d. %s\n", i, e) + } + } else { + log.Fatal("timestamp expected but doesn't exist in header") + } + if l, ok := header["location"]; ok { + fmt.Printf("location from header:\n") + for i, e := range l { + fmt.Printf(" %d. %s\n", i, e) + } + } else { + log.Fatal("location expected but doesn't exist in header") + } + fmt.Printf("response:\n") + fmt.Printf(" - %s\n", r.Message) + + if t, ok := trailer["timestamp"]; ok { + fmt.Printf("timestamp from trailer:\n") + for i, e := range t { + fmt.Printf(" %d. %s\n", i, e) + } + } else { + log.Fatal("timestamp expected but doesn't exist in trailer") + } +} + +func serverStreamingWithMetadata(c pb.EchoClient, message string) { + fmt.Printf("--- server streaming ---\n") + // Create metadata and context. + md := metadata.Pairs("timestamp", time.Now().Format(timestampFormat)) + ctx := metadata.NewOutgoingContext(context.Background(), md) + + // Make RPC using the context with the metadata. + stream, err := c.ServerStreamingEcho(ctx, &pb.EchoRequest{Message: message}) + if err != nil { + log.Fatalf("failed to call ServerStreamingEcho: %v", err) + } + + // Read the header when the header arrives. + header, err := stream.Header() + if err != nil { + log.Fatalf("failed to get header from stream: %v", err) + } + // Read metadata from server's header. + if t, ok := header["timestamp"]; ok { + fmt.Printf("timestamp from header:\n") + for i, e := range t { + fmt.Printf(" %d. %s\n", i, e) + } + } else { + log.Fatal("timestamp expected but doesn't exist in header") + } + if l, ok := header["location"]; ok { + fmt.Printf("location from header:\n") + for i, e := range l { + fmt.Printf(" %d. %s\n", i, e) + } + } else { + log.Fatal("location expected but doesn't exist in header") + } + + // Read all the responses. + var rpcStatus error + fmt.Printf("response:\n") + for { + r, err := stream.Recv() + if err != nil { + rpcStatus = err + break + } + fmt.Printf(" - %s\n", r.Message) + } + if rpcStatus != io.EOF { + log.Fatalf("failed to finish server streaming: %v", rpcStatus) + } + + // Read the trailer after the RPC is finished. + trailer := stream.Trailer() + // Read metadata from server's trailer. + if t, ok := trailer["timestamp"]; ok { + fmt.Printf("timestamp from trailer:\n") + for i, e := range t { + fmt.Printf(" %d. %s\n", i, e) + } + } else { + log.Fatal("timestamp expected but doesn't exist in trailer") + } +} + +func clientStreamWithMetadata(c pb.EchoClient, message string) { + fmt.Printf("--- client streaming ---\n") + // Create metadata and context. + md := metadata.Pairs("timestamp", time.Now().Format(timestampFormat)) + ctx := metadata.NewOutgoingContext(context.Background(), md) + + // Make RPC using the context with the metadata. + stream, err := c.ClientStreamingEcho(ctx) + if err != nil { + log.Fatalf("failed to call ClientStreamingEcho: %v\n", err) + } + + // Read the header when the header arrives. + header, err := stream.Header() + if err != nil { + log.Fatalf("failed to get header from stream: %v", err) + } + // Read metadata from server's header. + if t, ok := header["timestamp"]; ok { + fmt.Printf("timestamp from header:\n") + for i, e := range t { + fmt.Printf(" %d. %s\n", i, e) + } + } else { + log.Fatal("timestamp expected but doesn't exist in header") + } + if l, ok := header["location"]; ok { + fmt.Printf("location from header:\n") + for i, e := range l { + fmt.Printf(" %d. %s\n", i, e) + } + } else { + log.Fatal("location expected but doesn't exist in header") + } + + // Send all requests to the server. + for i := 0; i < streamingCount; i++ { + if err := stream.Send(&pb.EchoRequest{Message: message}); err != nil { + log.Fatalf("failed to send streaming: %v\n", err) + } + } + + // Read the response. + r, err := stream.CloseAndRecv() + if err != nil { + log.Fatalf("failed to CloseAndRecv: %v\n", err) + } + fmt.Printf("response:\n") + fmt.Printf(" - %s\n\n", r.Message) + + // Read the trailer after the RPC is finished. + trailer := stream.Trailer() + // Read metadata from server's trailer. + if t, ok := trailer["timestamp"]; ok { + fmt.Printf("timestamp from trailer:\n") + for i, e := range t { + fmt.Printf(" %d. %s\n", i, e) + } + } else { + log.Fatal("timestamp expected but doesn't exist in trailer") + } +} + +func bidirectionalWithMetadata(c pb.EchoClient, message string) { + fmt.Printf("--- bidirectional ---\n") + // Create metadata and context. + md := metadata.Pairs("timestamp", time.Now().Format(timestampFormat)) + ctx := metadata.NewOutgoingContext(context.Background(), md) + + // Make RPC using the context with the metadata. + stream, err := c.BidirectionalStreamingEcho(ctx) + if err != nil { + log.Fatalf("failed to call BidirectionalStreamingEcho: %v\n", err) + } + + go func() { + // Read the header when the header arrives. + header, err := stream.Header() + if err != nil { + log.Fatalf("failed to get header from stream: %v", err) + } + // Read metadata from server's header. + if t, ok := header["timestamp"]; ok { + fmt.Printf("timestamp from header:\n") + for i, e := range t { + fmt.Printf(" %d. %s\n", i, e) + } + } else { + log.Fatal("timestamp expected but doesn't exist in header") + } + if l, ok := header["location"]; ok { + fmt.Printf("location from header:\n") + for i, e := range l { + fmt.Printf(" %d. %s\n", i, e) + } + } else { + log.Fatal("location expected but doesn't exist in header") + } + + // Send all requests to the server. + for i := 0; i < streamingCount; i++ { + if err := stream.Send(&pb.EchoRequest{Message: message}); err != nil { + log.Fatalf("failed to send streaming: %v\n", err) + } + } + stream.CloseSend() + }() + + // Read all the responses. + var rpcStatus error + fmt.Printf("response:\n") + for { + r, err := stream.Recv() + if err != nil { + rpcStatus = err + break + } + fmt.Printf(" - %s\n", r.Message) + } + if rpcStatus != io.EOF { + log.Fatalf("failed to finish server streaming: %v", rpcStatus) + } + + // Read the trailer after the RPC is finished. + trailer := stream.Trailer() + // Read metadata from server's trailer. + if t, ok := trailer["timestamp"]; ok { + fmt.Printf("timestamp from trailer:\n") + for i, e := range t { + fmt.Printf(" %d. %s\n", i, e) + } + } else { + log.Fatal("timestamp expected but doesn't exist in trailer") + } + +} + +const message = "this is examples/metadata" + +func main() { + flag.Parse() + // Set up a connection to the server. + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + c := pb.NewEchoClient(conn) + + unaryCallWithMetadata(c, message) + time.Sleep(1 * time.Second) + + serverStreamingWithMetadata(c, message) + time.Sleep(1 * time.Second) + + clientStreamWithMetadata(c, message) + time.Sleep(1 * time.Second) + + bidirectionalWithMetadata(c, message) +} diff --git a/examples/features/metadata/server/main.go b/examples/features/metadata/server/main.go new file mode 100644 index 0000000..6989242 --- /dev/null +++ b/examples/features/metadata/server/main.go @@ -0,0 +1,208 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to send and read metadata to and from RPC. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "net" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +const ( + timestampFormat = time.StampNano + streamingCount = 10 +) + +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { + fmt.Printf("--- UnaryEcho ---\n") + // Create trailer in defer to record function return time. + defer func() { + trailer := metadata.Pairs("timestamp", time.Now().Format(timestampFormat)) + grpc.SetTrailer(ctx, trailer) + }() + + // Read metadata from client. + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Errorf(codes.DataLoss, "UnaryEcho: failed to get metadata") + } + if t, ok := md["timestamp"]; ok { + fmt.Printf("timestamp from metadata:\n") + for i, e := range t { + fmt.Printf(" %d. %s\n", i, e) + } + } + + // Create and send header. + header := metadata.New(map[string]string{"location": "MTV", "timestamp": time.Now().Format(timestampFormat)}) + grpc.SendHeader(ctx, header) + + fmt.Printf("request received: %v, sending echo\n", in) + + return &pb.EchoResponse{Message: in.Message}, nil +} + +func (s *server) ServerStreamingEcho(in *pb.EchoRequest, stream pb.Echo_ServerStreamingEchoServer) error { + fmt.Printf("--- ServerStreamingEcho ---\n") + // Create trailer in defer to record function return time. + defer func() { + trailer := metadata.Pairs("timestamp", time.Now().Format(timestampFormat)) + stream.SetTrailer(trailer) + }() + + // Read metadata from client. + md, ok := metadata.FromIncomingContext(stream.Context()) + if !ok { + return status.Errorf(codes.DataLoss, "ServerStreamingEcho: failed to get metadata") + } + if t, ok := md["timestamp"]; ok { + fmt.Printf("timestamp from metadata:\n") + for i, e := range t { + fmt.Printf(" %d. %s\n", i, e) + } + } + + // Create and send header. + header := metadata.New(map[string]string{"location": "MTV", "timestamp": time.Now().Format(timestampFormat)}) + stream.SendHeader(header) + + fmt.Printf("request received: %v\n", in) + + // Read requests and send responses. + for i := 0; i < streamingCount; i++ { + fmt.Printf("echo message %v\n", in.Message) + err := stream.Send(&pb.EchoResponse{Message: in.Message}) + if err != nil { + return err + } + } + return nil +} + +func (s *server) ClientStreamingEcho(stream pb.Echo_ClientStreamingEchoServer) error { + fmt.Printf("--- ClientStreamingEcho ---\n") + // Create trailer in defer to record function return time. + defer func() { + trailer := metadata.Pairs("timestamp", time.Now().Format(timestampFormat)) + stream.SetTrailer(trailer) + }() + + // Read metadata from client. + md, ok := metadata.FromIncomingContext(stream.Context()) + if !ok { + return status.Errorf(codes.DataLoss, "ClientStreamingEcho: failed to get metadata") + } + if t, ok := md["timestamp"]; ok { + fmt.Printf("timestamp from metadata:\n") + for i, e := range t { + fmt.Printf(" %d. %s\n", i, e) + } + } + + // Create and send header. + header := metadata.New(map[string]string{"location": "MTV", "timestamp": time.Now().Format(timestampFormat)}) + stream.SendHeader(header) + + // Read requests and send responses. + var message string + for { + in, err := stream.Recv() + if err == io.EOF { + fmt.Printf("echo last received message\n") + return stream.SendAndClose(&pb.EchoResponse{Message: message}) + } + message = in.Message + fmt.Printf("request received: %v, building echo\n", in) + if err != nil { + return err + } + } +} + +func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { + fmt.Printf("--- BidirectionalStreamingEcho ---\n") + // Create trailer in defer to record function return time. + defer func() { + trailer := metadata.Pairs("timestamp", time.Now().Format(timestampFormat)) + stream.SetTrailer(trailer) + }() + + // Read metadata from client. + md, ok := metadata.FromIncomingContext(stream.Context()) + if !ok { + return status.Errorf(codes.DataLoss, "BidirectionalStreamingEcho: failed to get metadata") + } + + if t, ok := md["timestamp"]; ok { + fmt.Printf("timestamp from metadata:\n") + for i, e := range t { + fmt.Printf(" %d. %s\n", i, e) + } + } + + // Create and send header. + header := metadata.New(map[string]string{"location": "MTV", "timestamp": time.Now().Format(timestampFormat)}) + stream.SendHeader(header) + + // Read requests and send responses. + for { + in, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + fmt.Printf("request received %v, sending echo\n", in) + if err := stream.Send(&pb.EchoResponse{Message: in.Message}); err != nil { + return err + } + } +} + +func main() { + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + fmt.Printf("server listening at %v\n", lis.Addr()) + + s := grpc.NewServer() + pb.RegisterEchoServer(s, &server{}) + s.Serve(lis) +} diff --git a/examples/features/metadata_interceptor/README.md b/examples/features/metadata_interceptor/README.md new file mode 100644 index 0000000..ae5f824 --- /dev/null +++ b/examples/features/metadata_interceptor/README.md @@ -0,0 +1,70 @@ +# Metadata interceptor example + +This example shows how to update metadata from unary and streaming interceptors on the server. +Please see +[grpc-metadata.md](https://github.com/grpc/grpc-go/blob/master/Documentation/grpc-metadata.md) +for more information. + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +#### Unary interceptor + +The interceptor can read existing metadata from the RPC context passed to it. +Since Go contexts are immutable, the interceptor will have to create a new context +with updated metadata and pass it to the provided handler. + +```go +func SomeInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + // Get the incoming metadata from the RPC context, and add a new + // key-value pair to it. + md, ok := metadata.FromIncomingContext(ctx) + md.Append("key1", "value1") + + // Create a context with the new metadata and pass it to handler. + ctx = metadata.NewIncomingContext(ctx, md) + return handler(ctx, req) +} +``` + +#### Streaming interceptor + +`grpc.ServerStream` does not provide a way to modify its RPC context. The streaming +interceptor therefore needs to implement the `grpc.ServerStream` interface and return +a context with updated metadata. + +The easiest way to do this would be to create a type which embeds the `grpc.ServerStream` +interface and overrides only the `Context()` method to return a context with updated +metadata. The streaming interceptor would then pass this wrapped stream to the provided handler. + +```go +type wrappedStream struct { + grpc.ServerStream + ctx context.Context +} + +func (s *wrappedStream) Context() context.Context { + return s.ctx +} + +func SomeStreamInterceptor(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + // Get the incoming metadata from the RPC context, and add a new + // key-value pair to it. + md, ok := metadata.FromIncomingContext(ctx) + md.Append("key1", "value1") + + // Create a context with the new metadata and pass it to handler. + ctx = metadata.NewIncomingContext(ctx, md) + + return handler(srv, &wrappedStream{ss, ctx}) +} +``` diff --git a/examples/features/metadata_interceptor/client/main.go b/examples/features/metadata_interceptor/client/main.go new file mode 100644 index 0000000..cc8754d --- /dev/null +++ b/examples/features/metadata_interceptor/client/main.go @@ -0,0 +1,114 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to receive metadata in RPC headers +// and trailers. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +func callUnaryEcho(ctx context.Context, client pb.EchoClient) { + var header, trailer metadata.MD + resp, err := client.UnaryEcho(ctx, &pb.EchoRequest{Message: "hello world"}, grpc.Header(&header), grpc.Trailer(&trailer)) + if err != nil { + log.Fatalf("UnaryEcho: %v", err) + } + fmt.Println("UnaryEcho: ", resp.Message) + + fmt.Println("Received headers:") + for k, v := range header { + fmt.Printf("%s: %v\n", k, v) + } + + fmt.Println("Received trailers:") + for k, v := range trailer { + fmt.Printf("%s: %v\n", k, v) + } +} + +func callBidiStreamingEcho(ctx context.Context, client pb.EchoClient) { + c, err := client.BidirectionalStreamingEcho(ctx) + if err != nil { + log.Fatalf("BidiStreamingEcho: %v", err) + } + + if err := c.Send(&pb.EchoRequest{Message: "hello world"}); err != nil { + log.Fatalf("Sending echo request: %v", err) + } + c.CloseSend() + + for { + resp, err := c.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("Receiving echo response: %v", err) + } + fmt.Println("BidiStreaming Echo: ", resp.Message) + } + + header, err := c.Header() + if err != nil { + log.Fatalf("Receiving headers: %v", err) + } + fmt.Println("Received headers:") + for k, v := range header { + fmt.Printf("%s: %v\n", k, v) + } + + trailer := c.Trailer() + fmt.Println("Received tailers:") + for k, v := range trailer { + fmt.Printf("%s: %v\n", k, v) + } +} + +func main() { + flag.Parse() + + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("grpc.NewClient(%q): %v", *addr, err) + } + defer conn.Close() + + ec := pb.NewEchoClient(conn) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + callUnaryEcho(ctx, ec) + + callBidiStreamingEcho(ctx, ec) +} diff --git a/examples/features/metadata_interceptor/server/main.go b/examples/features/metadata_interceptor/server/main.go new file mode 100644 index 0000000..e1525d9 --- /dev/null +++ b/examples/features/metadata_interceptor/server/main.go @@ -0,0 +1,164 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to update metadata from interceptors on server. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +var errMissingMetadata = status.Errorf(codes.InvalidArgument, "no incoming metadata in rpc context") + +type server struct { + pb.UnimplementedEchoServer +} + +func unaryInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, errMissingMetadata + } + + // Create and set metadata from interceptor to server. + md.Append("key1", "value1") + ctx = metadata.NewIncomingContext(ctx, md) + + // Call the handler to complete the normal execution of the RPC. + resp, err := handler(ctx, req) + + // Create and set header metadata from interceptor to client. + header := metadata.Pairs("header-key", "val") + grpc.SetHeader(ctx, header) + + // Create and set trailer metadata from interceptor to client. + trailer := metadata.Pairs("trailer-key", "val") + grpc.SetTrailer(ctx, trailer) + + return resp, err +} + +func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { + fmt.Printf("--- UnaryEcho ---\n") + + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Errorf(codes.Internal, "UnaryEcho: missing incoming metadata in rpc context") + } + + // Read and print metadata added by the interceptor. + if v, ok := md["key1"]; ok { + fmt.Printf("key1 from metadata: \n") + for i, e := range v { + fmt.Printf(" %d. %s\n", i, e) + } + } + + return &pb.EchoResponse{Message: in.Message}, nil +} + +type wrappedStream struct { + grpc.ServerStream + ctx context.Context +} + +func (s *wrappedStream) Context() context.Context { + return s.ctx +} + +func streamInterceptor(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + md, ok := metadata.FromIncomingContext(ss.Context()) + if !ok { + return errMissingMetadata + } + + // Create and set metadata from interceptor to server. + md.Append("key1", "value1") + ctx := metadata.NewIncomingContext(ss.Context(), md) + + // Call the handler to complete the normal execution of the RPC. + err := handler(srv, &wrappedStream{ss, ctx}) + + // Create and set header metadata from interceptor to client. + header := metadata.Pairs("header-key", "val") + ss.SetHeader(header) + + // Create and set trailer metadata from interceptor to client. + trailer := metadata.Pairs("trailer-key", "val") + ss.SetTrailer(trailer) + + return err +} + +func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { + fmt.Printf("--- BidirectionalStreamingEcho ---\n") + + md, ok := metadata.FromIncomingContext(stream.Context()) + if !ok { + return status.Errorf(codes.Internal, "BidirectionalStreamingEcho: missing incoming metadata in rpc context") + } + + // Read and print metadata added by the interceptor. + if v, ok := md["key1"]; ok { + fmt.Printf("key1 from metadata: \n") + for i, e := range v { + fmt.Printf(" %d. %s\n", i, e) + } + } + + // Read requests and send responses. + for { + in, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if err = stream.Send(&pb.EchoResponse{Message: in.Message}); err != nil { + return err + } + } +} + +func main() { + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("net.Listen() failed: %v", err) + } + fmt.Printf("Server listening at %v\n", lis.Addr()) + + s := grpc.NewServer(grpc.UnaryInterceptor(unaryInterceptor), grpc.StreamInterceptor(streamInterceptor)) + pb.RegisterEchoServer(s, &server{}) + s.Serve(lis) +} diff --git a/examples/features/multiplex/README.md b/examples/features/multiplex/README.md new file mode 100644 index 0000000..803a8d7 --- /dev/null +++ b/examples/features/multiplex/README.md @@ -0,0 +1,12 @@ +# Multiplex + +A `grpc.ClientConn` can be shared by two stubs and two services can share a +`grpc.Server`. This example illustrates how to perform both types of sharing. + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` diff --git a/examples/features/multiplex/client/main.go b/examples/features/multiplex/client/main.go new file mode 100644 index 0000000..d5a432e --- /dev/null +++ b/examples/features/multiplex/client/main.go @@ -0,0 +1,79 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to use a single grpc.ClientConn for multiple +// service stubs. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + ecpb "google.golang.org/grpc/examples/features/proto/echo" + hwpb "google.golang.org/grpc/examples/helloworld/helloworld" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +// callSayHello calls SayHello on c with the given name, and prints the +// response. +func callSayHello(c hwpb.GreeterClient, name string) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + r, err := c.SayHello(ctx, &hwpb.HelloRequest{Name: name}) + if err != nil { + log.Fatalf("client.SayHello(_) = _, %v", err) + } + fmt.Println("Greeting: ", r.Message) +} + +func callUnaryEcho(client ecpb.EchoClient, message string) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := client.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message}) + if err != nil { + log.Fatalf("client.UnaryEcho(_) = _, %v: ", err) + } + fmt.Println("UnaryEcho: ", resp.Message) +} + +func main() { + flag.Parse() + // Set up a connection to the server. + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + fmt.Println("--- calling helloworld.Greeter/SayHello ---") + // Make a greeter client and send an RPC. + hwc := hwpb.NewGreeterClient(conn) + callSayHello(hwc, "multiplex") + + fmt.Println() + fmt.Println("--- calling routeguide.RouteGuide/GetFeature ---") + // Make a routeguide client with the same ClientConn. + rgc := ecpb.NewEchoClient(conn) + callUnaryEcho(rgc, "this is examples/multiplex") +} diff --git a/examples/features/multiplex/server/main.go b/examples/features/multiplex/server/main.go new file mode 100644 index 0000000..edd6ede --- /dev/null +++ b/examples/features/multiplex/server/main.go @@ -0,0 +1,75 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to use a single grpc.Server instance to +// register and serve multiple services. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + + ecpb "google.golang.org/grpc/examples/features/proto/echo" + hwpb "google.golang.org/grpc/examples/helloworld/helloworld" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +// hwServer is used to implement helloworld.GreeterServer. +type hwServer struct { + hwpb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer +func (s *hwServer) SayHello(_ context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) { + return &hwpb.HelloReply{Message: "Hello " + in.Name}, nil +} + +type ecServer struct { + ecpb.UnimplementedEchoServer +} + +func (s *ecServer) UnaryEcho(_ context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) { + return &ecpb.EchoResponse{Message: req.Message}, nil +} + +func main() { + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + fmt.Printf("server listening at %v\n", lis.Addr()) + + s := grpc.NewServer() + + // Register Greeter on the server. + hwpb.RegisterGreeterServer(s, &hwServer{}) + + // Register RouteGuide on the same server. + ecpb.RegisterEchoServer(s, &ecServer{}) + + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/name_resolving/README.md b/examples/features/name_resolving/README.md new file mode 100644 index 0000000..407b23f --- /dev/null +++ b/examples/features/name_resolving/README.md @@ -0,0 +1,41 @@ +# Name resolving + +This examples shows how `ClientConn` can pick different name resolvers. + +## What is a name resolver + +A name resolver can be seen as a `map[service-name][]backend-ip`. It takes a +service name, and returns a list of IPs of the backends. A common used name +resolver is DNS. + +In this example, a resolver is created to resolve `resolver.example.grpc.io` to +`localhost:50051`. + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +The echo server is serving on ":50051". Two clients are created, one is dialing +to `passthrough:///localhost:50051`, while the other is dialing to +`example:///resolver.example.grpc.io`. Both of them can connect the server. + +Name resolver is picked based on the `scheme` in the target string. See +https://github.com/grpc/grpc/blob/master/doc/naming.md for the target syntax. + +The first client picks the `passthrough` resolver, which takes the input, and +use it as the backend addresses. + +The second is connecting to service name `resolver.example.grpc.io`. Without a +proper name resolver, this would fail. In the example it picks the `example` +resolver that we installed. The `example` resolver can handle +`resolver.example.grpc.io` correctly by returning the backend address. So even +though the backend IP is not set when ClientConn is created, the connection will +be created to the correct backend. diff --git a/examples/features/name_resolving/client/main.go b/examples/features/name_resolving/client/main.go new file mode 100644 index 0000000..6d50032 --- /dev/null +++ b/examples/features/name_resolving/client/main.go @@ -0,0 +1,137 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to use custom name resolvers to resolve +// server backend addresses. +package main + +import ( + "context" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + ecpb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/resolver" +) + +const ( + exampleScheme = "example" + exampleServiceName = "resolver.example.grpc.io" + + backendAddr = "localhost:50051" +) + +func callUnaryEcho(c ecpb.EchoClient, message string) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + r, err := c.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message}) + if err != nil { + log.Fatalf("could not greet: %v", err) + } + fmt.Println(r.Message) +} + +func makeRPCs(cc *grpc.ClientConn, n int) { + hwc := ecpb.NewEchoClient(cc) + for i := 0; i < n; i++ { + callUnaryEcho(hwc, "this is examples/name_resolving") + } +} + +func main() { + passthroughConn, err := grpc.NewClient( + fmt.Sprintf("passthrough:///%s", backendAddr), // Dial to "passthrough:///localhost:50051" + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer passthroughConn.Close() + + fmt.Printf("--- calling helloworld.Greeter/SayHello to \"passthrough:///%s\"\n", backendAddr) + makeRPCs(passthroughConn, 10) + + fmt.Println() + + exampleConn, err := grpc.NewClient( + fmt.Sprintf("%s:///%s", exampleScheme, exampleServiceName), // Dial to "example:///resolver.example.grpc.io" + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer exampleConn.Close() + + fmt.Printf("--- calling helloworld.Greeter/SayHello to \"%s:///%s\"\n", exampleScheme, exampleServiceName) + makeRPCs(exampleConn, 10) +} + +// Following is an example name resolver. It includes a +// ResolverBuilder(https://godoc.org/google.golang.org/grpc/resolver#Builder) +// and a Resolver(https://godoc.org/google.golang.org/grpc/resolver#Resolver). +// +// A ResolverBuilder is registered for a scheme (in this example, "example" is +// the scheme). When a ClientConn is created for this scheme, the +// ResolverBuilder will be picked to build a Resolver. Note that a new Resolver +// is built for each ClientConn. The Resolver will watch the updates for the +// target, and send updates to the ClientConn. + +// exampleResolverBuilder is a +// ResolverBuilder(https://godoc.org/google.golang.org/grpc/resolver#Builder). +type exampleResolverBuilder struct{} + +func (*exampleResolverBuilder) Build(target resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) { + r := &exampleResolver{ + target: target, + cc: cc, + addrsStore: map[string][]string{ + exampleServiceName: {backendAddr}, + }, + } + r.start() + return r, nil +} +func (*exampleResolverBuilder) Scheme() string { return exampleScheme } + +// exampleResolver is a +// Resolver(https://godoc.org/google.golang.org/grpc/resolver#Resolver). +type exampleResolver struct { + target resolver.Target + cc resolver.ClientConn + addrsStore map[string][]string +} + +func (r *exampleResolver) start() { + addrStrs := r.addrsStore[r.target.Endpoint()] + addrs := make([]resolver.Address, len(addrStrs)) + for i, s := range addrStrs { + addrs[i] = resolver.Address{Addr: s} + } + r.cc.UpdateState(resolver.State{Addresses: addrs}) +} +func (*exampleResolver) ResolveNow(resolver.ResolveNowOptions) {} +func (*exampleResolver) Close() {} + +func init() { + // Register the example ResolverBuilder. This is usually done in a package's + // init() function. + resolver.Register(&exampleResolverBuilder{}) +} diff --git a/examples/features/name_resolving/server/main.go b/examples/features/name_resolving/server/main.go new file mode 100644 index 0000000..6a7297a --- /dev/null +++ b/examples/features/name_resolving/server/main.go @@ -0,0 +1,56 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to set up a gRPC server that listens on a +// specified port for name resolution examples. +package main + +import ( + "context" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +const addr = "localhost:50051" + +type ecServer struct { + pb.UnimplementedEchoServer + addr string +} + +func (s *ecServer) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: fmt.Sprintf("%s (from %s)", req.Message, s.addr)}, nil +} + +func main() { + lis, err := net.Listen("tcp", addr) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + s := grpc.NewServer() + pb.RegisterEchoServer(s, &ecServer{addr: addr}) + log.Printf("serving on %s\n", addr) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/observability/README.md b/examples/features/observability/README.md new file mode 100644 index 0000000..f2aa521 --- /dev/null +++ b/examples/features/observability/README.md @@ -0,0 +1,3 @@ +This example is the Hello World example instrumented for logs, metrics, and tracing. + +Please refer to Microservices Observability user guide for setup. diff --git a/examples/features/observability/client/clientConfig.json b/examples/features/observability/client/clientConfig.json new file mode 100644 index 0000000..b98ae25 --- /dev/null +++ b/examples/features/observability/client/clientConfig.json @@ -0,0 +1,17 @@ +{ + "cloud_monitoring": {}, + "cloud_trace": { + "sampling_rate": 1.0 + }, + "cloud_logging": { + "client_rpc_events": [{ + "methods": ["*"] + }], + "server_rpc_events": [{ + "methods": ["*"] + }] + }, + "labels": { + "environment" : "example-client" + } +} diff --git a/examples/features/observability/client/main.go b/examples/features/observability/client/main.go new file mode 100644 index 0000000..ad08c8a --- /dev/null +++ b/examples/features/observability/client/main.go @@ -0,0 +1,71 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to instrument RPCs with logging, metrics, +// and tracing. +package main + +import ( + "context" + "flag" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/gcp/observability" +) + +const ( + defaultName = "world" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") + name = flag.String("name", defaultName, "Name to greet") +) + +func main() { + // Turn on global telemetry for the whole binary. If a configuration is + // specified, any created gRPC Client Conn's or Servers will emit telemetry + // data according the configuration. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + err := observability.Start(ctx) + if err != nil { + log.Fatalf("observability.Start() failed: %v", err) + } + defer observability.End() + + flag.Parse() + // Set up a connection to the server. + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + c := pb.NewGreeterClient(conn) + + // Contact the server and print out its response. + r, err := c.SayHello(ctx, &pb.HelloRequest{Name: *name}) + if err != nil { + log.Fatalf("could not greet: %v", err) + } + log.Printf("Greeting: %s", r.GetMessage()) +} diff --git a/examples/features/observability/server/main.go b/examples/features/observability/server/main.go new file mode 100644 index 0000000..b8455ef --- /dev/null +++ b/examples/features/observability/server/main.go @@ -0,0 +1,90 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to instrument RPCs for logging, metrics, +// and tracing. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "os" + "os/signal" + "syscall" + "time" + + "google.golang.org/grpc" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/gcp/observability" +) + +var ( + port = flag.Int("port", 50051, "The server port") +) + +// server is used to implement helloworld.GreeterServer. +type server struct { + pb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer +func (s *server) SayHello(_ context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + log.Printf("Received: %v", in.GetName()) + return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil +} + +func main() { + // Turn on global telemetry for the whole binary. If a configuration is + // specified, any created gRPC Client Conn's or Servers will emit telemetry + // data according the configuration. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + err := observability.Start(ctx) + if err != nil { + log.Fatalf("observability.Start() failed: %v", err) + } + defer observability.End() + + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + s := grpc.NewServer() + pb.RegisterGreeterServer(s, &server{}) + log.Printf("server listening at %v", lis.Addr()) + + // This server can potentially be terminated by an external signal from the + // Operating System. The following catches those signals and calls s.Stop(). + // This causes the s.Serve() call to return and run main()'s defers, + // including the observability.End() call that ensures any pending + // observability data is sent to Cloud Operations. + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + s.Stop() + }() + + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/observability/server/serverConfig.json b/examples/features/observability/server/serverConfig.json new file mode 100644 index 0000000..a2bf7f6 --- /dev/null +++ b/examples/features/observability/server/serverConfig.json @@ -0,0 +1,17 @@ +{ + "cloud_monitoring": {}, + "cloud_trace": { + "sampling_rate": 1.0 + }, + "cloud_logging": { + "client_rpc_events": [{ + "methods": ["*"] + }], + "server_rpc_events": [{ + "methods": ["*"] + }] + }, + "labels": { + "environment" : "example-server" + } +} diff --git a/examples/features/opentelemetry/README.md b/examples/features/opentelemetry/README.md new file mode 100644 index 0000000..5e9db48 --- /dev/null +++ b/examples/features/opentelemetry/README.md @@ -0,0 +1,39 @@ +# OpenTelemetry + +This example shows how to configure OpenTelemetry on a client and server, and +shows what type of telemetry data it can produce for certain RPCs. +This example shows how to enable experimental gRPC metrics, which are disabled +by default and must be explicitly configured on the client and/or server. + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +``` +curl localhost:9464/metrics +curl localhost:9465/metrics +``` + +## Explanation + +The client continuously makes RPCs to a server. The client and server both +expose a prometheus exporter to listen and provide metrics. This defaults to +:9464 for the server and :9465 for the client. The client and server are also +configured to output traces directly to their standard output streams using +`stdouttrace`. + +OpenTelemetry is configured on both the client and the server, and exports to +the Prometheus exporter. The exporter exposes metrics on the Prometheus ports +described above. OpenTelemetry exports traces using the `stdouttrace` exporter, +which prints structured trace data to the console output of both the client and +server. Each RPC call produces trace information that captures the execution +flow and timing of operations. + +Curling to the exposed Prometheus ports outputs the metrics recorded on the +client and server. diff --git a/examples/features/opentelemetry/client/main.go b/examples/features/opentelemetry/client/main.go new file mode 100644 index 0000000..6064e93 --- /dev/null +++ b/examples/features/opentelemetry/client/main.go @@ -0,0 +1,100 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is a client for the OpenTelemetry example. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel/exporters/prometheus" + otelstdouttrace "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + otelpropagation "go.opentelemetry.io/otel/propagation" + otelmetric "go.opentelemetry.io/otel/sdk/metric" + otelresource "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.24.0" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/examples/features/proto/echo" + oteltracing "google.golang.org/grpc/experimental/opentelemetry" + "google.golang.org/grpc/stats/opentelemetry" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the server address to connect to") + prometheusEndpoint = flag.String("prometheus_endpoint", ":9465", "the Prometheus exporter endpoint for metrics") +) + +func main() { + exporter, err := prometheus.New() + if err != nil { + log.Fatalf("Failed to start prometheus exporter: %v", err) + } + // Configure meter provider for metrics + meterProvider := otelmetric.NewMeterProvider(otelmetric.WithReader(exporter)) + // Configure exporter for traces + traceExporter, err := otelstdouttrace.New(otelstdouttrace.WithPrettyPrint()) + if err != nil { + log.Fatalf("Failed to create stdouttrace exporter: %v", err) + } + traceProvider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(traceExporter), sdktrace.WithResource(otelresource.NewWithAttributes(semconv.SchemaURL, semconv.ServiceName("grpc-client")))) + // Configure W3C Trace Context Propagator for traces + textMapPropagator := otelpropagation.TraceContext{} + do := opentelemetry.DialOption(opentelemetry.Options{ + MetricsOptions: opentelemetry.MetricsOptions{ + MeterProvider: meterProvider, + // These are example experimental gRPC metrics, which are disabled + // by default and must be explicitly enabled. For the full, + // up-to-date list of metrics, see: + // https://grpc.io/docs/guides/opentelemetry-metrics/#instruments + Metrics: opentelemetry.DefaultMetrics().Add( + "grpc.subchannel.connection_attempts_succeeded", + "grpc.subchannel.connection_attempts_failed", + ), + }, + TraceOptions: oteltracing.TraceOptions{TracerProvider: traceProvider, TextMapPropagator: textMapPropagator}, + }) + + go http.ListenAndServe(*prometheusEndpoint, promhttp.Handler()) + + cc, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials()), do) + if err != nil { + log.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + c := echo.NewEchoClient(cc) + ctx := context.Background() + + // Make an RPC every second. This should trigger telemetry to be emitted from + // the client and the server. + for { + r, err := c.UnaryEcho(ctx, &echo.EchoRequest{Message: "this is examples/opentelemetry"}) + if err != nil { + log.Fatalf("UnaryEcho failed: %v", err) + } + fmt.Println(r) + time.Sleep(time.Second) + } +} diff --git a/examples/features/opentelemetry/server/main.go b/examples/features/opentelemetry/server/main.go new file mode 100644 index 0000000..821c265 --- /dev/null +++ b/examples/features/opentelemetry/server/main.go @@ -0,0 +1,93 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is a server for the OpenTelemetry example. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "net/http" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel/exporters/prometheus" + otelstdouttrace "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + otelpropagation "go.opentelemetry.io/otel/propagation" + otelmetric "go.opentelemetry.io/otel/sdk/metric" + otelresource "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.24.0" + "google.golang.org/grpc" + pb "google.golang.org/grpc/examples/features/proto/echo" + oteltracing "google.golang.org/grpc/experimental/opentelemetry" + "google.golang.org/grpc/stats/opentelemetry" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the server address to connect to") + prometheusEndpoint = flag.String("prometheus_endpoint", ":9464", "the Prometheus exporter endpoint for metrics") +) + +type echoServer struct { + pb.UnimplementedEchoServer + addr string +} + +func (s *echoServer) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: fmt.Sprintf("%s (from %s)", req.Message, s.addr)}, nil +} + +func main() { + exporter, err := prometheus.New() + if err != nil { + log.Fatalf("Failed to start prometheus exporter: %v", err) + } + // Configure meter provider for metrics + meterProvider := otelmetric.NewMeterProvider(otelmetric.WithReader(exporter)) + // Configure exporter for traces + traceExporter, err := otelstdouttrace.New(otelstdouttrace.WithPrettyPrint()) + if err != nil { + log.Fatalf("Failed to create stdouttrace exporter: %v", err) + } + traceProvider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(traceExporter), sdktrace.WithResource(otelresource.NewWithAttributes(semconv.SchemaURL, semconv.ServiceName("grpc-server")))) + // Configure W3C Trace Context Propagator for traces + textMapPropagator := otelpropagation.TraceContext{} + so := opentelemetry.ServerOption(opentelemetry.Options{ + MetricsOptions: opentelemetry.MetricsOptions{ + MeterProvider: meterProvider, + }, + TraceOptions: oteltracing.TraceOptions{TracerProvider: traceProvider, TextMapPropagator: textMapPropagator}}) + + go http.ListenAndServe(*prometheusEndpoint, promhttp.Handler()) + + lis, err := net.Listen("tcp", *addr) + if err != nil { + log.Fatalf("Failed to listen: %v", err) + } + s := grpc.NewServer(so) + pb.RegisterEchoServer(s, &echoServer{addr: *addr}) + + log.Printf("Serving on %s\n", *addr) + + if err := s.Serve(lis); err != nil { + log.Fatalf("Failed to serve: %v", err) + } +} diff --git a/examples/features/orca/README.md b/examples/features/orca/README.md new file mode 100644 index 0000000..ef99aa2 --- /dev/null +++ b/examples/features/orca/README.md @@ -0,0 +1,48 @@ +# ORCA Load Reporting + +ORCA is a protocol for reporting load between servers and clients. This +example shows how to implement this from both the client and server side. For +more details, please see [gRFC +A51](https://github.com/grpc/proposal/blob/master/A51-custom-backend-metrics.md) + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +gRPC ORCA support provides two different ways to report load data to clients +from servers: out-of-band and per-RPC. Out-of-band metrics are reported +regularly at some interval on a stream, while per-RPC metrics are reported +along with the trailers at the end of a call. Both of these mechanisms are +optional and work independently. + +The full ORCA API documentation is available here: +https://pkg.go.dev/google.golang.org/grpc/orca + +### Out-of-band Metrics + +The server registers an ORCA service that is used for out-of-band metrics. It +does this by using `orca.Register()` and then setting metrics on the returned +`orca.Service` using its methods. + +The client receives out-of-band metrics via the LB policy. It receives +callbacks to a listener by registering the listener on a `SubConn` via +`orca.RegisterOOBListener`. + +### Per-RPC Metrics + +The server is set up to report query cost metrics in its RPC handler. For +per-RPC metrics to be reported, the gRPC server must be created with the +`orca.CallMetricsServerOption()` option, and metrics are set by calling methods +on the returned `orca.CallMetricRecorder` from +`orca.CallMetricRecorderFromContext()`. + +The client performs one RPC per second. Per-RPC metrics are available for each +call via the `Done()` callback returned from the LB policy's picker. diff --git a/examples/features/orca/client/main.go b/examples/features/orca/client/main.go new file mode 100644 index 0000000..e21f309 --- /dev/null +++ b/examples/features/orca/client/main.go @@ -0,0 +1,159 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates the use of a custom LB policy that handles ORCA +// per-call and out-of-band metrics for load reporting. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/orca" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") +var test = flag.Bool("test", false, "if set, only 1 RPC is performed before exiting") + +func main() { + flag.Parse() + + // Set up a connection to the server. Configure to use our custom LB + // policy which will receive all the ORCA load reports. + conn, err := grpc.NewClient(*addr, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"orca_example":{}}]}`), + ) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + c := pb.NewEchoClient(conn) + + // Perform RPCs once per second. + ticker := time.NewTicker(time.Second) + for range ticker.C { + func() { + // Use an anonymous function to ensure context cancellation via defer. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := c.UnaryEcho(ctx, &pb.EchoRequest{Message: "test echo message"}); err != nil { + log.Fatalf("Error from UnaryEcho call: %v", err) + } + }() + if *test { + return + } + } + +} + +// Register an ORCA load balancing policy to receive per-call metrics and +// out-of-band metrics. +func init() { + balancer.Register(orcaLBBuilder{}) +} + +type orcaLBBuilder struct{} + +func (orcaLBBuilder) Name() string { return "orca_example" } +func (orcaLBBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { + return &orcaLB{cc: cc} +} + +// orcaLB is an incomplete LB policy designed to show basic ORCA load reporting +// functionality. It collects per-call metrics in the `Done` callback returned +// by its picker, and it collects out-of-band metrics by registering a listener +// when its SubConn is created. It does not follow general LB policy best +// practices and makes assumptions about the simple test environment it is +// designed to run within. +type orcaLB struct { + cc balancer.ClientConn + sc balancer.SubConn +} + +func (o *orcaLB) UpdateClientConnState(ccs balancer.ClientConnState) error { + addrs := ccs.ResolverState.Addresses + // Create one SubConn for the address and connect it. + var sc balancer.SubConn + sc, err := o.cc.NewSubConn(addrs, balancer.NewSubConnOptions{ + StateListener: func(scs balancer.SubConnState) { + if scs.ConnectivityState == connectivity.Ready { + o.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: &picker{sc}}) + } + }, + }) + if err != nil { + return fmt.Errorf("orcaLB: error creating SubConn: %v", err) + } + sc.Connect() + o.sc = sc + + // Register a simple ORCA OOB listener on the SubConn. We request a 1 + // second report interval, but in this example the server indicated the + // minimum interval it will allow is 3 seconds, so reports will only be + // sent that often. + orca.RegisterOOBListener(sc, orcaLis{}, orca.OOBListenerOptions{ReportInterval: time.Second}) + + return nil +} + +func (o *orcaLB) ResolverError(error) {} + +func (o *orcaLB) ExitIdle() { + if o.sc != nil { + o.sc.Connect() + } +} + +// TODO: unused; remove when no longer required. +func (o *orcaLB) UpdateSubConnState(balancer.SubConn, balancer.SubConnState) {} + +func (o *orcaLB) Close() {} + +type picker struct { + sc balancer.SubConn +} + +func (p *picker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + return balancer.PickResult{ + SubConn: p.sc, + Done: func(di balancer.DoneInfo) { + fmt.Println("Per-call load report received:", di.ServerLoad.(*v3orcapb.OrcaLoadReport).GetRequestCost()) + }, + }, nil +} + +// orcaLis is the out-of-band load report listener that we pass to +// orca.RegisterOOBListener to receive periodic load report information. +type orcaLis struct{} + +func (orcaLis) OnLoadReport(lr *v3orcapb.OrcaLoadReport) { + fmt.Println("Out-of-band load report received:", lr) +} diff --git a/examples/features/orca/server/main.go b/examples/features/orca/server/main.go new file mode 100644 index 0000000..65c767a --- /dev/null +++ b/examples/features/orca/server/main.go @@ -0,0 +1,97 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to use ORCA for reporting out-of-band +// and per-RPC load metrics. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/status" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { + // Report a sample cost for this query. + cmr := orca.CallMetricsRecorderFromContext(ctx) + if cmr == nil { + return nil, status.Errorf(codes.Internal, "unable to retrieve call metrics recorder (missing ORCA ServerOption?)") + } + cmr.SetRequestCost("db_queries", 10) + + return &pb.EchoResponse{Message: in.Message}, nil +} + +func main() { + flag.Parse() + + lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", *port)) + if err != nil { + log.Fatalf("Failed to listen: %v", err) + } + fmt.Printf("Server listening at %v\n", lis.Addr()) + + // Create the gRPC server with the orca.CallMetricsServerOption() option, + // which will enable per-call metric recording. No ServerMetricsProvider + // is given here because the out-of-band reporting is enabled separately. + s := grpc.NewServer(orca.CallMetricsServerOption(nil)) + pb.RegisterEchoServer(s, &server{}) + + // Register the orca service for out-of-band metric reporting, and set the + // minimum reporting interval to 3 seconds. Note that, by default, the + // minimum interval must be at least 30 seconds, but 3 seconds is set via + // an internal-only option for illustration purposes only. + smr := orca.NewServerMetricsRecorder() + opts := orca.ServiceOptions{ + MinReportingInterval: 3 * time.Second, + ServerMetricsProvider: smr, + } + internal.ORCAAllowAnyMinReportingInterval.(func(so *orca.ServiceOptions))(&opts) + if err := orca.Register(s, opts); err != nil { + log.Fatalf("Failed to register ORCA service: %v", err) + } + + // Simulate CPU utilization reporting. + go func() { + for { + smr.SetCPUUtilization(.5) + time.Sleep(2 * time.Second) + smr.SetCPUUtilization(.9) + time.Sleep(2 * time.Second) + } + }() + + s.Serve(lis) +} diff --git a/examples/features/proto/echo/echo.pb.go b/examples/features/proto/echo/echo.pb.go new file mode 100644 index 0000000..7733f97 --- /dev/null +++ b/examples/features/proto/echo/echo.pb.go @@ -0,0 +1,201 @@ +// +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: examples/features/proto/echo/echo.proto + +package echo + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// EchoRequest is the request for echo. +type EchoRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EchoRequest) Reset() { + *x = EchoRequest{} + mi := &file_examples_features_proto_echo_echo_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EchoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EchoRequest) ProtoMessage() {} + +func (x *EchoRequest) ProtoReflect() protoreflect.Message { + mi := &file_examples_features_proto_echo_echo_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EchoRequest.ProtoReflect.Descriptor instead. +func (*EchoRequest) Descriptor() ([]byte, []int) { + return file_examples_features_proto_echo_echo_proto_rawDescGZIP(), []int{0} +} + +func (x *EchoRequest) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +// EchoResponse is the response for echo. +type EchoResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EchoResponse) Reset() { + *x = EchoResponse{} + mi := &file_examples_features_proto_echo_echo_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EchoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EchoResponse) ProtoMessage() {} + +func (x *EchoResponse) ProtoReflect() protoreflect.Message { + mi := &file_examples_features_proto_echo_echo_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EchoResponse.ProtoReflect.Descriptor instead. +func (*EchoResponse) Descriptor() ([]byte, []int) { + return file_examples_features_proto_echo_echo_proto_rawDescGZIP(), []int{1} +} + +func (x *EchoResponse) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +var File_examples_features_proto_echo_echo_proto protoreflect.FileDescriptor + +const file_examples_features_proto_echo_echo_proto_rawDesc = "" + + "\n" + + "'examples/features/proto/echo/echo.proto\x12\x12grpc.examples.echo\"'\n" + + "\vEchoRequest\x12\x18\n" + + "\amessage\x18\x01 \x01(\tR\amessage\"(\n" + + "\fEchoResponse\x12\x18\n" + + "\amessage\x18\x01 \x01(\tR\amessage2\xfb\x02\n" + + "\x04Echo\x12P\n" + + "\tUnaryEcho\x12\x1f.grpc.examples.echo.EchoRequest\x1a .grpc.examples.echo.EchoResponse\"\x00\x12\\\n" + + "\x13ServerStreamingEcho\x12\x1f.grpc.examples.echo.EchoRequest\x1a .grpc.examples.echo.EchoResponse\"\x000\x01\x12\\\n" + + "\x13ClientStreamingEcho\x12\x1f.grpc.examples.echo.EchoRequest\x1a .grpc.examples.echo.EchoResponse\"\x00(\x01\x12e\n" + + "\x1aBidirectionalStreamingEcho\x12\x1f.grpc.examples.echo.EchoRequest\x1a .grpc.examples.echo.EchoResponse\"\x00(\x010\x01B5Z3google.golang.org/grpc/examples/features/proto/echob\x06proto3" + +var ( + file_examples_features_proto_echo_echo_proto_rawDescOnce sync.Once + file_examples_features_proto_echo_echo_proto_rawDescData []byte +) + +func file_examples_features_proto_echo_echo_proto_rawDescGZIP() []byte { + file_examples_features_proto_echo_echo_proto_rawDescOnce.Do(func() { + file_examples_features_proto_echo_echo_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_examples_features_proto_echo_echo_proto_rawDesc), len(file_examples_features_proto_echo_echo_proto_rawDesc))) + }) + return file_examples_features_proto_echo_echo_proto_rawDescData +} + +var file_examples_features_proto_echo_echo_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_examples_features_proto_echo_echo_proto_goTypes = []any{ + (*EchoRequest)(nil), // 0: grpc.examples.echo.EchoRequest + (*EchoResponse)(nil), // 1: grpc.examples.echo.EchoResponse +} +var file_examples_features_proto_echo_echo_proto_depIdxs = []int32{ + 0, // 0: grpc.examples.echo.Echo.UnaryEcho:input_type -> grpc.examples.echo.EchoRequest + 0, // 1: grpc.examples.echo.Echo.ServerStreamingEcho:input_type -> grpc.examples.echo.EchoRequest + 0, // 2: grpc.examples.echo.Echo.ClientStreamingEcho:input_type -> grpc.examples.echo.EchoRequest + 0, // 3: grpc.examples.echo.Echo.BidirectionalStreamingEcho:input_type -> grpc.examples.echo.EchoRequest + 1, // 4: grpc.examples.echo.Echo.UnaryEcho:output_type -> grpc.examples.echo.EchoResponse + 1, // 5: grpc.examples.echo.Echo.ServerStreamingEcho:output_type -> grpc.examples.echo.EchoResponse + 1, // 6: grpc.examples.echo.Echo.ClientStreamingEcho:output_type -> grpc.examples.echo.EchoResponse + 1, // 7: grpc.examples.echo.Echo.BidirectionalStreamingEcho:output_type -> grpc.examples.echo.EchoResponse + 4, // [4:8] is the sub-list for method output_type + 0, // [0:4] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_examples_features_proto_echo_echo_proto_init() } +func file_examples_features_proto_echo_echo_proto_init() { + if File_examples_features_proto_echo_echo_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_examples_features_proto_echo_echo_proto_rawDesc), len(file_examples_features_proto_echo_echo_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_examples_features_proto_echo_echo_proto_goTypes, + DependencyIndexes: file_examples_features_proto_echo_echo_proto_depIdxs, + MessageInfos: file_examples_features_proto_echo_echo_proto_msgTypes, + }.Build() + File_examples_features_proto_echo_echo_proto = out.File + file_examples_features_proto_echo_echo_proto_goTypes = nil + file_examples_features_proto_echo_echo_proto_depIdxs = nil +} diff --git a/examples/features/proto/echo/echo.proto b/examples/features/proto/echo/echo.proto new file mode 100644 index 0000000..2dde563 --- /dev/null +++ b/examples/features/proto/echo/echo.proto @@ -0,0 +1,45 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +syntax = "proto3"; + +option go_package = "google.golang.org/grpc/examples/features/proto/echo"; + +package grpc.examples.echo; + +// EchoRequest is the request for echo. +message EchoRequest { + string message = 1; +} + +// EchoResponse is the response for echo. +message EchoResponse { + string message = 1; +} + +// Echo is the echo service. +service Echo { + // UnaryEcho is unary echo. + rpc UnaryEcho(EchoRequest) returns (EchoResponse) {} + // ServerStreamingEcho is server side streaming. + rpc ServerStreamingEcho(EchoRequest) returns (stream EchoResponse) {} + // ClientStreamingEcho is client side streaming. + rpc ClientStreamingEcho(stream EchoRequest) returns (EchoResponse) {} + // BidirectionalStreamingEcho is bidi streaming. + rpc BidirectionalStreamingEcho(stream EchoRequest) returns (stream EchoResponse) {} +} diff --git a/examples/features/proto/echo/echo_grpc.pb.go b/examples/features/proto/echo/echo_grpc.pb.go new file mode 100644 index 0000000..88677bd --- /dev/null +++ b/examples/features/proto/echo/echo_grpc.pb.go @@ -0,0 +1,255 @@ +// +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: examples/features/proto/echo/echo.proto + +package echo + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Echo_UnaryEcho_FullMethodName = "/grpc.examples.echo.Echo/UnaryEcho" + Echo_ServerStreamingEcho_FullMethodName = "/grpc.examples.echo.Echo/ServerStreamingEcho" + Echo_ClientStreamingEcho_FullMethodName = "/grpc.examples.echo.Echo/ClientStreamingEcho" + Echo_BidirectionalStreamingEcho_FullMethodName = "/grpc.examples.echo.Echo/BidirectionalStreamingEcho" +) + +// EchoClient is the client API for Echo service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Echo is the echo service. +type EchoClient interface { + // UnaryEcho is unary echo. + UnaryEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (*EchoResponse, error) + // ServerStreamingEcho is server side streaming. + ServerStreamingEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[EchoResponse], error) + // ClientStreamingEcho is client side streaming. + ClientStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[EchoRequest, EchoResponse], error) + // BidirectionalStreamingEcho is bidi streaming. + BidirectionalStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[EchoRequest, EchoResponse], error) +} + +type echoClient struct { + cc grpc.ClientConnInterface +} + +func NewEchoClient(cc grpc.ClientConnInterface) EchoClient { + return &echoClient{cc} +} + +func (c *echoClient) UnaryEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (*EchoResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(EchoResponse) + err := c.cc.Invoke(ctx, Echo_UnaryEcho_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *echoClient) ServerStreamingEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[EchoResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Echo_ServiceDesc.Streams[0], Echo_ServerStreamingEcho_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[EchoRequest, EchoResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Echo_ServerStreamingEchoClient = grpc.ServerStreamingClient[EchoResponse] + +func (c *echoClient) ClientStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[EchoRequest, EchoResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Echo_ServiceDesc.Streams[1], Echo_ClientStreamingEcho_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[EchoRequest, EchoResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Echo_ClientStreamingEchoClient = grpc.ClientStreamingClient[EchoRequest, EchoResponse] + +func (c *echoClient) BidirectionalStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[EchoRequest, EchoResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Echo_ServiceDesc.Streams[2], Echo_BidirectionalStreamingEcho_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[EchoRequest, EchoResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Echo_BidirectionalStreamingEchoClient = grpc.BidiStreamingClient[EchoRequest, EchoResponse] + +// EchoServer is the server API for Echo service. +// All implementations must embed UnimplementedEchoServer +// for forward compatibility. +// +// Echo is the echo service. +type EchoServer interface { + // UnaryEcho is unary echo. + UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error) + // ServerStreamingEcho is server side streaming. + ServerStreamingEcho(*EchoRequest, grpc.ServerStreamingServer[EchoResponse]) error + // ClientStreamingEcho is client side streaming. + ClientStreamingEcho(grpc.ClientStreamingServer[EchoRequest, EchoResponse]) error + // BidirectionalStreamingEcho is bidi streaming. + BidirectionalStreamingEcho(grpc.BidiStreamingServer[EchoRequest, EchoResponse]) error + mustEmbedUnimplementedEchoServer() +} + +// UnimplementedEchoServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedEchoServer struct{} + +func (UnimplementedEchoServer) UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error) { + return nil, status.Error(codes.Unimplemented, "method UnaryEcho not implemented") +} +func (UnimplementedEchoServer) ServerStreamingEcho(*EchoRequest, grpc.ServerStreamingServer[EchoResponse]) error { + return status.Error(codes.Unimplemented, "method ServerStreamingEcho not implemented") +} +func (UnimplementedEchoServer) ClientStreamingEcho(grpc.ClientStreamingServer[EchoRequest, EchoResponse]) error { + return status.Error(codes.Unimplemented, "method ClientStreamingEcho not implemented") +} +func (UnimplementedEchoServer) BidirectionalStreamingEcho(grpc.BidiStreamingServer[EchoRequest, EchoResponse]) error { + return status.Error(codes.Unimplemented, "method BidirectionalStreamingEcho not implemented") +} +func (UnimplementedEchoServer) mustEmbedUnimplementedEchoServer() {} +func (UnimplementedEchoServer) testEmbeddedByValue() {} + +// UnsafeEchoServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to EchoServer will +// result in compilation errors. +type UnsafeEchoServer interface { + mustEmbedUnimplementedEchoServer() +} + +func RegisterEchoServer(s grpc.ServiceRegistrar, srv EchoServer) { + // If the following call panics, it indicates UnimplementedEchoServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Echo_ServiceDesc, srv) +} + +func _Echo_UnaryEcho_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(EchoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(EchoServer).UnaryEcho(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Echo_UnaryEcho_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(EchoServer).UnaryEcho(ctx, req.(*EchoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Echo_ServerStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(EchoRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(EchoServer).ServerStreamingEcho(m, &grpc.GenericServerStream[EchoRequest, EchoResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Echo_ServerStreamingEchoServer = grpc.ServerStreamingServer[EchoResponse] + +func _Echo_ClientStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(EchoServer).ClientStreamingEcho(&grpc.GenericServerStream[EchoRequest, EchoResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Echo_ClientStreamingEchoServer = grpc.ClientStreamingServer[EchoRequest, EchoResponse] + +func _Echo_BidirectionalStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(EchoServer).BidirectionalStreamingEcho(&grpc.GenericServerStream[EchoRequest, EchoResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Echo_BidirectionalStreamingEchoServer = grpc.BidiStreamingServer[EchoRequest, EchoResponse] + +// Echo_ServiceDesc is the grpc.ServiceDesc for Echo service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Echo_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.examples.echo.Echo", + HandlerType: (*EchoServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "UnaryEcho", + Handler: _Echo_UnaryEcho_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "ServerStreamingEcho", + Handler: _Echo_ServerStreamingEcho_Handler, + ServerStreams: true, + }, + { + StreamName: "ClientStreamingEcho", + Handler: _Echo_ClientStreamingEcho_Handler, + ClientStreams: true, + }, + { + StreamName: "BidirectionalStreamingEcho", + Handler: _Echo_BidirectionalStreamingEcho_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "examples/features/proto/echo/echo.proto", +} diff --git a/examples/features/reflection/README.md b/examples/features/reflection/README.md new file mode 100644 index 0000000..068ef64 --- /dev/null +++ b/examples/features/reflection/README.md @@ -0,0 +1,21 @@ +# Reflection + +This example shows how reflection can be registered on a gRPC server. + +See +https://github.com/grpc/grpc-go/blob/master/Documentation/server-reflection-tutorial.md +for a tutorial. + + +# Try it + +```go +go run server/main.go +``` + +There are multiple existing reflection clients. + +To use `gRPC CLI`, follow +https://github.com/grpc/grpc-go/blob/master/Documentation/server-reflection-tutorial.md#grpc-cli. + +To use `grpcurl`, see https://github.com/fullstorydev/grpcurl. diff --git a/examples/features/reflection/server/main.go b/examples/features/reflection/server/main.go new file mode 100644 index 0000000..a970455 --- /dev/null +++ b/examples/features/reflection/server/main.go @@ -0,0 +1,79 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates how to register multiple services and enable +// client discovery. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/reflection" + + ecpb "google.golang.org/grpc/examples/features/proto/echo" + hwpb "google.golang.org/grpc/examples/helloworld/helloworld" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +// hwServer is used to implement helloworld.GreeterServer. +type hwServer struct { + hwpb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer +func (s *hwServer) SayHello(_ context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) { + return &hwpb.HelloReply{Message: "Hello " + in.Name}, nil +} + +type ecServer struct { + ecpb.UnimplementedEchoServer +} + +func (s *ecServer) UnaryEcho(_ context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) { + return &ecpb.EchoResponse{Message: req.Message}, nil +} + +func main() { + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + fmt.Printf("server listening at %v\n", lis.Addr()) + + s := grpc.NewServer() + + // Register Greeter on the server. + hwpb.RegisterGreeterServer(s, &hwServer{}) + + // Register RouteGuide on the same server. + ecpb.RegisterEchoServer(s, &ecServer{}) + + // Register reflection service on gRPC server. + reflection.Register(s) + + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/retry/README.md b/examples/features/retry/README.md new file mode 100644 index 0000000..1ad3139 --- /dev/null +++ b/examples/features/retry/README.md @@ -0,0 +1,64 @@ +# Retry + +This example shows how to enable and configure retry on gRPC clients. + +## Documentation + +[gRFC for client-side retry support](https://github.com/grpc/proposal/blob/master/A6-client-retries.md) + +## Try it + +This example includes a service implementation that fails requests three times with status +code `Unavailable`, then passes the fourth. The client is configured to make four retry attempts +when receiving an `Unavailable` status code. + +First start the server: + +```bash +go run server/main.go +``` + +Then run the client: + +```bash +go run client/main.go +``` + +## Usage + +### Define your retry policy + +Retry is enabled via the service config, which can be provided by the name resolver or +a DialOption (described below). In the below config, we set retry policy for the +"grpc.example.echo.Echo" method. + +MaxAttempts: how many times to attempt the RPC before failing. +InitialBackoff, MaxBackoff, BackoffMultiplier: configures delay between attempts. +RetryableStatusCodes: Retry only when receiving these status codes. + +```go + var retryPolicy = `{ + "methodConfig": [{ + // config per method or all methods under service + "name": [{"service": "grpc.examples.echo.Echo"}], + + "retryPolicy": { + "MaxAttempts": 4, + "InitialBackoff": ".01s", + "MaxBackoff": ".01s", + "BackoffMultiplier": 1.0, + // this value is grpc code + "RetryableStatusCodes": [ "UNAVAILABLE" ] + } + }] + }` +``` + +### Providing the retry policy as a DialOption + +To use the above service config, pass it with `grpc.WithDefaultServiceConfig` to +`grpc.NewClient`. + +```go +conn, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(retryPolicy)) +``` diff --git a/examples/features/retry/client/main.go b/examples/features/retry/client/main.go new file mode 100644 index 0000000..751a74a --- /dev/null +++ b/examples/features/retry/client/main.go @@ -0,0 +1,77 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client demonstrates how to enable and configure retry policies for +// gRPC requests. +package main + +import ( + "context" + "flag" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var ( + addr = flag.String("addr", "localhost:50052", "the address to connect to") + // see https://github.com/grpc/grpc/blob/master/doc/service_config.md to know more about service config + retryPolicy = `{ + "methodConfig": [{ + "name": [{"service": "grpc.examples.echo.Echo"}], + "retryPolicy": { + "MaxAttempts": 4, + "InitialBackoff": ".01s", + "MaxBackoff": ".01s", + "BackoffMultiplier": 1.0, + "RetryableStatusCodes": [ "UNAVAILABLE" ] + } + }]}` +) + +func main() { + flag.Parse() + + // Set up a connection to the server with service config and create the channel. + // However, the recommended approach is to fetch the retry configuration + // (which is part of the service config) from the name resolver rather than + // defining it on the client side. + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(retryPolicy)) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer func() { + if e := conn.Close(); e != nil { + log.Printf("failed to close connection: %s", e) + } + }() + + c := pb.NewEchoClient(conn) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + reply, err := c.UnaryEcho(ctx, &pb.EchoRequest{Message: "Try and Success"}) + if err != nil { + log.Fatalf("UnaryEcho error: %v", err) + } + log.Printf("UnaryEcho reply: %v", reply) +} diff --git a/examples/features/retry/server/main.go b/examples/features/retry/server/main.go new file mode 100644 index 0000000..657c682 --- /dev/null +++ b/examples/features/retry/server/main.go @@ -0,0 +1,93 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrates to enforce retries on client side. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50052, "port number") + +type failingServer struct { + pb.UnimplementedEchoServer + mu sync.Mutex + + reqCounter uint + reqModulo uint +} + +// this method will fail reqModulo - 1 times RPCs and return status code Unavailable, +// and succeeded RPC on reqModulo times. +func (s *failingServer) maybeFailRequest() error { + s.mu.Lock() + defer s.mu.Unlock() + s.reqCounter++ + if (s.reqModulo > 0) && (s.reqCounter%s.reqModulo == 0) { + return nil + } + + return status.Errorf(codes.Unavailable, "maybeFailRequest: failing it") +} + +func (s *failingServer) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + if err := s.maybeFailRequest(); err != nil { + log.Println("request failed count:", s.reqCounter) + return nil, err + } + + log.Println("request succeeded count:", s.reqCounter) + return &pb.EchoResponse{Message: req.Message}, nil +} + +func main() { + flag.Parse() + + address := fmt.Sprintf(":%v", *port) + lis, err := net.Listen("tcp", address) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + fmt.Println("listen on address", address) + + s := grpc.NewServer() + + // Configure server to pass every fourth RPC; + // client is configured to make four attempts. + failingservice := &failingServer{ + reqCounter: 0, + reqModulo: 4, + } + + pb.RegisterEchoServer(s, failingservice) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/stats_monitoring/README.md b/examples/features/stats_monitoring/README.md new file mode 100644 index 0000000..06b1a82 --- /dev/null +++ b/examples/features/stats_monitoring/README.md @@ -0,0 +1,58 @@ +# Stats Monitoring Handler + +This example demonstrates the use of the [`stats`](https://pkg.go.dev/google.golang.org/grpc/stats) package for reporting various +network and RPC stats. +_Note that all fields are READ-ONLY and the APIs of the `stats` package are +experimental_. + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +gRPC provides a mechanism to hook on to various events (phases) of the +request-response network cycle through the [`stats.Handler`](https://pkg.go.dev/google.golang.org/grpc/stats#Handler) interface. To access +these events, a concrete type that implements `stats.Handler` should be passed to +`grpc.WithStatsHandler()` on the client side and `grpc.StatsHandler()` on the +server side. + +The `HandleRPC(context.Context, RPCStats)` method on `stats.Handler` is called +multiple times during a request-response cycle, and various event stats are +passed to its `RPCStats` parameter (an interface). The concrete types that +implement this interface are: `*stats.Begin`, `*stats.InHeader`, `*stats.InPayload`, +`*stats.InTrailer`, `*stats.OutHeader`, `*stats.OutPayload`, `*stats.OutTrailer`, and +`*stats.End`. The order of these events differs on client and server. + +Similarly, the `HandleConn(context.Context, ConnStats)` method on `stats.Handler` +is called twice, once at the beginning of the connection with `*stats.ConnBegin` +and once at the end with `*stats.ConnEnd`. + +The [`stats.Handler`](https://pkg.go.dev/google.golang.org/grpc/stats#Handler) interface also provides +`TagRPC(context.Context, *RPCTagInfo) context.Context` and +`TagConn(context.Context, *ConnTagInfo) context.Context` methods. These methods +are mainly used to attach network related information to the given context. + +The `TagRPC(context.Context, *RPCTagInfo) context.Context` method returns a +context from which the context used for the rest lifetime of the RPC will be +derived. This behavior is consistent between the gRPC client and server. + +The context returned from +`TagConn(context.Context, *ConnTagInfo) context.Context` has varied lifespan: + +- In the gRPC client: + The context used for the rest lifetime of the RPC will NOT be derived from + this context. Hence the information attached to this context can only be + consumed by `HandleConn(context.Context, ConnStats)` method. +- In the gRPC server: + The context used for the rest lifetime of the RPC will be derived from + this context. + +NOTE: The [stats](https://pkg.go.dev/google.golang.org/grpc/stats) package should only be used for network monitoring purposes, +and not as an alternative to [interceptors](https://github.com/grpc/grpc-go/blob/master/examples/features/interceptor). diff --git a/examples/features/stats_monitoring/client/main.go b/examples/features/stats_monitoring/client/main.go new file mode 100644 index 0000000..49fdeb2 --- /dev/null +++ b/examples/features/stats_monitoring/client/main.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an example client to illustrate the use of the stats handler. +package main + +import ( + "context" + "flag" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + echogrpc "google.golang.org/grpc/examples/features/proto/echo" + echopb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/examples/features/stats_monitoring/statshandler" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +func main() { + flag.Parse() + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithStatsHandler(statshandler.New()), + } + conn, err := grpc.NewClient(*addr, opts...) + if err != nil { + log.Fatalf("failed to connect to server %q: %v", *addr, err) + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + c := echogrpc.NewEchoClient(conn) + + resp, err := c.UnaryEcho(ctx, &echopb.EchoRequest{Message: "stats handler demo"}) + if err != nil { + log.Fatalf("unexpected error from UnaryEcho: %v", err) + } + log.Printf("RPC response: %s", resp.Message) +} diff --git a/examples/features/stats_monitoring/server/main.go b/examples/features/stats_monitoring/server/main.go new file mode 100644 index 0000000..9c69043 --- /dev/null +++ b/examples/features/stats_monitoring/server/main.go @@ -0,0 +1,62 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is an example server to illustrate the use of the stats handler. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "time" + + "google.golang.org/grpc" + + echogrpc "google.golang.org/grpc/examples/features/proto/echo" + echopb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/examples/features/stats_monitoring/statshandler" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +type server struct { + echogrpc.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(_ context.Context, req *echopb.EchoRequest) (*echopb.EchoResponse, error) { + time.Sleep(2 * time.Second) + return &echopb.EchoResponse{Message: req.Message}, nil +} + +func main() { + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen on port %d: %v", *port, err) + } + log.Printf("server listening at %v\n", lis.Addr()) + + s := grpc.NewServer(grpc.StatsHandler(statshandler.New())) + echogrpc.RegisterEchoServer(s, &server{}) + + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/stats_monitoring/statshandler/handler.go b/examples/features/stats_monitoring/statshandler/handler.go new file mode 100644 index 0000000..85688b8 --- /dev/null +++ b/examples/features/stats_monitoring/statshandler/handler.go @@ -0,0 +1,93 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package statshandler is an example pkg to illustrate the use of the stats handler. +package statshandler + +import ( + "context" + "log" + "net" + "path/filepath" + + "google.golang.org/grpc/stats" +) + +// Handler implements [stats.Handler](https://pkg.go.dev/google.golang.org/grpc/stats#Handler) interface. +type Handler struct{} + +type connStatCtxKey struct{} + +// TagConn can attach some information to the given context. +// The context used in HandleConn for this connection will be derived from the context returned. +// In the gRPC client: +// The context used in HandleRPC for RPCs on this connection will be the user's context and NOT derived from the context returned here. +// In the gRPC server: +// The context used in HandleRPC for RPCs on this connection will be derived from the context returned here. +func (st *Handler) TagConn(ctx context.Context, stat *stats.ConnTagInfo) context.Context { + log.Printf("[TagConn] [%T]: %+[1]v", stat) + return context.WithValue(ctx, connStatCtxKey{}, stat) +} + +// HandleConn processes the Conn stats. +func (st *Handler) HandleConn(ctx context.Context, stat stats.ConnStats) { + var rAddr net.Addr + if s, ok := ctx.Value(connStatCtxKey{}).(*stats.ConnTagInfo); ok { + rAddr = s.RemoteAddr + } + + if stat.IsClient() { + log.Printf("[server addr: %s] [HandleConn] [%T]: %+[2]v", rAddr, stat) + } else { + log.Printf("[client addr: %s] [HandleConn] [%T]: %+[2]v", rAddr, stat) + } +} + +type rpcStatCtxKey struct{} + +// TagRPC can attach some information to the given context. +// The context used for the rest lifetime of the RPC will be derived from the returned context. +func (st *Handler) TagRPC(ctx context.Context, stat *stats.RPCTagInfo) context.Context { + log.Printf("[TagRPC] [%T]: %+[1]v", stat) + return context.WithValue(ctx, rpcStatCtxKey{}, stat) +} + +// HandleRPC processes the RPC stats. Note: All stat fields are read-only. +func (st *Handler) HandleRPC(ctx context.Context, stat stats.RPCStats) { + var sMethod string + if s, ok := ctx.Value(rpcStatCtxKey{}).(*stats.RPCTagInfo); ok { + sMethod = filepath.Base(s.FullMethodName) + } + + var cAddr net.Addr + // for gRPC clients, key connStatCtxKey{} will not be present in HandleRPC's context. + if s, ok := ctx.Value(connStatCtxKey{}).(*stats.ConnTagInfo); ok { + cAddr = s.RemoteAddr + } + + if stat.IsClient() { + log.Printf("[server method: %s] [HandleRPC] [%T]: %+[2]v", sMethod, stat) + } else { + log.Printf("[client addr: %s] [HandleRPC] [%T]: %+[2]v", cAddr, stat) + } +} + +// New returns a new implementation of [stats.Handler](https://pkg.go.dev/google.golang.org/grpc/stats#Handler) interface. +func New() *Handler { + return &Handler{} +} diff --git a/examples/features/unix_abstract/README.md b/examples/features/unix_abstract/README.md new file mode 100644 index 0000000..32b3bd5 --- /dev/null +++ b/examples/features/unix_abstract/README.md @@ -0,0 +1,29 @@ +# Unix abstract sockets + +This examples shows how to start a gRPC server listening on a unix abstract +socket and how to get a gRPC client to connect to it. + +## What is a unix abstract socket + +An abstract socket address is distinguished from a regular unix socket by the +fact that the first byte of the address is a null byte ('\0'). The address has +no connection with filesystem path names. + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +The gRPC server in this example listens on an address starting with a null byte +and the network is `unix`. The client uses the `unix-abstract` scheme with the +endpoint set to the abstract unix socket address without the null byte. The +`unix` resolver takes care of adding the null byte on the client. See +https://github.com/grpc/grpc/blob/master/doc/naming.md for the more details. + diff --git a/examples/features/unix_abstract/client/main.go b/examples/features/unix_abstract/client/main.go new file mode 100644 index 0000000..3bbbbc9 --- /dev/null +++ b/examples/features/unix_abstract/client/main.go @@ -0,0 +1,75 @@ +//go:build linux +// +build linux + +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an example client which dials a server on an abstract unix +// socket. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + ecpb "google.golang.org/grpc/examples/features/proto/echo" +) + +var ( + // A dial target of `unix:@abstract-unix-socket` should also work fine for + // this example because of golang conventions (net.Dial behavior). But we do + // not recommend this since we explicitly added the `unix-abstract` scheme + // for cross-language compatibility. + addr = flag.String("addr", "abstract-unix-socket", "The unix abstract socket address") +) + +func callUnaryEcho(c ecpb.EchoClient, message string) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + r, err := c.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message}) + if err != nil { + log.Fatalf("could not greet: %v", err) + } + fmt.Println(r.Message) +} + +func makeRPCs(cc *grpc.ClientConn, n int) { + hwc := ecpb.NewEchoClient(cc) + for i := 0; i < n; i++ { + callUnaryEcho(hwc, "this is examples/unix_abstract") + } +} + +func main() { + flag.Parse() + sockAddr := fmt.Sprintf("unix-abstract:%v", *addr) + cc, err := grpc.NewClient(sockAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("grpc.NewClient(%q) failed: %v", sockAddr, err) + } + defer cc.Close() + + fmt.Printf("--- calling echo.Echo/UnaryEcho to %s\n", sockAddr) + makeRPCs(cc, 10) + fmt.Println() +} diff --git a/examples/features/unix_abstract/server/main.go b/examples/features/unix_abstract/server/main.go new file mode 100644 index 0000000..19b15db --- /dev/null +++ b/examples/features/unix_abstract/server/main.go @@ -0,0 +1,65 @@ +//go:build linux +// +build linux + +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is an example server listening for gRPC connections on an +// abstract unix socket. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var ( + addr = flag.String("addr", "abstract-unix-socket", "The unix abstract socket address") +) + +type ecServer struct { + pb.UnimplementedEchoServer + addr string +} + +func (s *ecServer) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: fmt.Sprintf("%s (from %s)", req.Message, s.addr)}, nil +} + +func main() { + flag.Parse() + netw := "unix" + socketAddr := fmt.Sprintf("@%v", *addr) + lis, err := net.Listen(netw, socketAddr) + if err != nil { + log.Fatalf("net.Listen(%q, %q) failed: %v", netw, socketAddr, err) + } + s := grpc.NewServer() + pb.RegisterEchoServer(s, &ecServer{addr: socketAddr}) + log.Printf("serving on %s\n", lis.Addr().String()) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/wait_for_ready/README.md b/examples/features/wait_for_ready/README.md new file mode 100644 index 0000000..a5bdabf --- /dev/null +++ b/examples/features/wait_for_ready/README.md @@ -0,0 +1,11 @@ +# Wait for ready example + +This example shows how to enable "wait for ready" in RPC calls. + +This code starts a server with a 2 seconds delay. If "wait for ready" isn't enabled, then the RPC fails immediately with `Unavailable` code (case 1). If "wait for ready" is enabled, then the RPC waits for the server. If context dies before the server is available, then it fails with `DeadlineExceeded` (case 3). Otherwise it succeeds (case 2). + +## Run the example + +``` +go run main.go +``` diff --git a/examples/features/wait_for_ready/main.go b/examples/features/wait_for_ready/main.go new file mode 100644 index 0000000..a00b76a --- /dev/null +++ b/examples/features/wait_for_ready/main.go @@ -0,0 +1,117 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary wait_for_ready is an example for "wait for ready". +package main + +import ( + "context" + "fmt" + "log" + "net" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +// server is used to implement EchoServer. +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(_ context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: req.Message}, nil +} + +// serve starts listening with a 2 seconds delay. +func serve() { + lis, err := net.Listen("tcp", ":50053") + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + s := grpc.NewServer() + pb.RegisterEchoServer(s, &server{}) + + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} + +func main() { + conn, err := grpc.NewClient("localhost:50053", grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + c := pb.NewEchoClient(conn) + + var wg sync.WaitGroup + wg.Add(3) + + // "Wait for ready" is not enabled, returns error with code "Unavailable". + go func() { + defer wg.Done() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := c.UnaryEcho(ctx, &pb.EchoRequest{Message: "Hi!"}) + + got := status.Code(err) + fmt.Printf("[1] wanted = %v, got = %v\n", codes.Unavailable, got) + }() + + // "Wait for ready" is enabled, returns nil error. + go func() { + defer wg.Done() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err := c.UnaryEcho(ctx, &pb.EchoRequest{Message: "Hi!"}, grpc.WaitForReady(true)) + + got := status.Code(err) + fmt.Printf("[2] wanted = %v, got = %v\n", codes.OK, got) + }() + + // "Wait for ready" is enabled but exceeds the deadline before server starts listening, + // returns error with code "DeadlineExceeded". + go func() { + defer wg.Done() + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + _, err := c.UnaryEcho(ctx, &pb.EchoRequest{Message: "Hi!"}, grpc.WaitForReady(true)) + + got := status.Code(err) + fmt.Printf("[3] wanted = %v, got = %v\n", codes.DeadlineExceeded, got) + }() + + time.Sleep(2 * time.Second) + go serve() + + wg.Wait() +} diff --git a/examples/features/xds/README.md b/examples/features/xds/README.md new file mode 100644 index 0000000..a6a78b0 --- /dev/null +++ b/examples/features/xds/README.md @@ -0,0 +1,35 @@ +# gRPC xDS example + +xDS is the protocol initially used by Envoy, that is evolving into a universal +data plane API for service mesh. + +The xDS example is a Hello World client/server capable of being configured with +the XDS management protocol. Out-of-the-box it behaves the same as [our other +hello world +example](https://github.com/grpc/grpc-go/tree/master/examples/helloworld). The +server replies with responses including its hostname. + +## xDS environment setup + +This example doesn't include instructions to setup xDS environment. Please refer +to documentation specific for your xDS management server. Examples will be added +later. + +The client also needs a bootstrap file. See [gRFC +A27](https://github.com/grpc/proposal/blob/master/A27-xds-global-load-balancing.md#xdsclient-and-bootstrap-file) +for the bootstrap format. + +## The client + +The client application needs to import the xDS package to install the resolver and balancers: + +```go +_ "google.golang.org/grpc/xds" // To install the xds resolvers and balancers. +``` + +Then, use `xds` target scheme for the ClientConn. + +``` +$ export GRPC_XDS_BOOTSTRAP=/path/to/bootstrap.json +$ go run client/main.go "xDS world" xds:///target_service +``` diff --git a/examples/features/xds/client/main.go b/examples/features/xds/client/main.go new file mode 100644 index 0000000..5f52739 --- /dev/null +++ b/examples/features/xds/client/main.go @@ -0,0 +1,73 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary main implements a client for Greeter service using gRPC's client-side +// support for xDS APIs. +package main + +import ( + "context" + "flag" + "log" + "strings" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + + _ "google.golang.org/grpc/xds" // To install the xds resolvers and balancers. +) + +var ( + target = flag.String("target", "xds:///localhost:50051", "uri of the Greeter Server, e.g. 'xds:///helloworld-service:8080'") + name = flag.String("name", "world", "name you wished to be greeted by the server") + xdsCreds = flag.Bool("xds_creds", false, "whether the server should use xDS APIs to receive security configuration") +) + +func main() { + flag.Parse() + + if !strings.HasPrefix(*target, "xds:///") { + log.Fatalf("-target must use a URI with scheme set to 'xds'") + } + + creds := insecure.NewCredentials() + if *xdsCreds { + log.Println("Using xDS credentials...") + var err error + if creds, err = xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}); err != nil { + log.Fatalf("failed to create client-side xDS credentials: %v", err) + } + } + conn, err := grpc.NewClient(*target, grpc.WithTransportCredentials(creds)) + if err != nil { + log.Fatalf("grpc.NewClient(%s) failed: %v", *target, err) + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c := pb.NewGreeterClient(conn) + r, err := c.SayHello(ctx, &pb.HelloRequest{Name: *name}) + if err != nil { + log.Fatalf("could not greet: %v", err) + } + log.Printf("Greeting: %s", r.GetMessage()) +} diff --git a/examples/features/xds/server/main.go b/examples/features/xds/server/main.go new file mode 100644 index 0000000..c4a2768 --- /dev/null +++ b/examples/features/xds/server/main.go @@ -0,0 +1,107 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server demonstrated gRPC's support for xDS APIs on the server-side. It +// exposes the Greeter service that will response with the hostname. +package main + +import ( + "context" + "flag" + "fmt" + "log" + rand "math/rand/v2" + "net" + "os" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/health" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/xds" +) + +var ( + port = flag.Int("port", 50051, "the port to serve Greeter service requests on. Health service will be served on `port+1`") + xdsCreds = flag.Bool("xds_creds", false, "whether the server should use xDS APIs to receive security configuration") +) + +// server implements helloworld.GreeterServer interface. +type server struct { + pb.UnimplementedGreeterServer + serverName string +} + +// SayHello implements helloworld.GreeterServer interface. +func (s *server) SayHello(_ context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + log.Printf("Received: %v", in.GetName()) + return &pb.HelloReply{Message: "Hello " + in.GetName() + ", from " + s.serverName}, nil +} + +func determineHostname() string { + hostname, err := os.Hostname() + if err != nil { + log.Printf("Failed to get hostname: %v, will generate one", err) + return fmt.Sprintf("generated-%03d", rand.Int()%100) + } + return hostname +} + +func main() { + flag.Parse() + + greeterPort := fmt.Sprintf(":%d", *port) + greeterLis, err := net.Listen("tcp4", greeterPort) + if err != nil { + log.Fatalf("net.Listen(tcp4, %q) failed: %v", greeterPort, err) + } + + creds := insecure.NewCredentials() + if *xdsCreds { + log.Println("Using xDS credentials...") + var err error + if creds, err = xdscreds.NewServerCredentials(xdscreds.ServerOptions{FallbackCreds: insecure.NewCredentials()}); err != nil { + log.Fatalf("failed to create server-side xDS credentials: %v", err) + } + } + + greeterServer, err := xds.NewGRPCServer(grpc.Creds(creds)) + if err != nil { + log.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + pb.RegisterGreeterServer(greeterServer, &server{serverName: determineHostname()}) + + healthPort := fmt.Sprintf(":%d", *port+1) + healthLis, err := net.Listen("tcp4", healthPort) + if err != nil { + log.Fatalf("net.Listen(tcp4, %q) failed: %v", healthPort, err) + } + grpcServer := grpc.NewServer() + healthServer := health.NewServer() + healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING) + healthgrpc.RegisterHealthServer(grpcServer, healthServer) + + log.Printf("Serving GreeterService on %s and HealthService on %s", greeterLis.Addr().String(), healthLis.Addr().String()) + go func() { + greeterServer.Serve(greeterLis) + }() + grpcServer.Serve(healthLis) +} diff --git a/examples/go.mod b/examples/go.mod new file mode 100644 index 0000000..5ea7aa5 --- /dev/null +++ b/examples/go.mod @@ -0,0 +1,92 @@ +module google.golang.org/grpc/examples + +go 1.25.0 + +require ( + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 + github.com/prometheus/client_golang v1.24.1 + go.opentelemetry.io/otel v1.45.0 + go.opentelemetry.io/otel/exporters/prometheus v0.67.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 + go.opentelemetry.io/otel/sdk v1.45.0 + go.opentelemetry.io/otel/sdk/metric v1.45.0 + golang.org/x/oauth2 v0.36.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 + google.golang.org/grpc v1.83.0 + google.golang.org/grpc/gcp/observability v1.0.1 + google.golang.org/grpc/security/advancedtls v1.0.0 + google.golang.org/protobuf v1.36.12 +) + +require ( + cel.dev/expr v0.25.3 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.23.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/logging v1.19.1 // indirect + cloud.google.com/go/longrunning v1.2.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect + cloud.google.com/go/trace v1.16.0 // indirect + contrib.go.opencensus.io/exporter/stackdriver v0.13.15-0.20230702191903-2de6d2748484 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.43.6 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.37 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.36 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.45.6 // indirect + github.com/aws/smithy-go v1.27.8 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.21 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/otlptranslator v1.0.0 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/api v0.293.0 // indirect + google.golang.org/genproto v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/grpc/stats/opencensus v1.0.0 // indirect +) + +replace google.golang.org/grpc => ../ + +replace google.golang.org/grpc/gcp/observability => ../gcp/observability + +replace google.golang.org/grpc/stats/opentelemetry => ../stats/opentelemetry diff --git a/examples/gotutorial.md b/examples/gotutorial.md new file mode 100644 index 0000000..b1b7546 --- /dev/null +++ b/examples/gotutorial.md @@ -0,0 +1,423 @@ +# gRPC Basics: Go + +This tutorial provides a basic Go programmer's introduction to working with gRPC. By walking through this example you'll learn how to: + +- Define a service in a `.proto` file. +- Generate server and client code using the protocol buffer compiler. +- Use the Go gRPC API to write a simple client and server for your service. + +It assumes that you have read the [Getting started](https://github.com/grpc/grpc/tree/master/examples) guide and are familiar with [protocol buffers](https://developers.google.com/protocol-buffers/docs/overview). Note that the example in this tutorial uses the proto3 version of the protocol buffers language, you can find out more in the [proto3 language guide](https://developers.google.com/protocol-buffers/docs/proto3) and see the [release notes](https://github.com/google/protobuf/releases) for the new version in the protocol buffers GitHub repository. + +This isn't a comprehensive guide to using gRPC in Go: more reference documentation is coming soon. + +## Why use gRPC? + +Our example is a simple route mapping application that lets clients get information about features on their route, create a summary of their route, and exchange route information such as traffic updates with the server and other clients. + +With gRPC we can define our service once in a `.proto` file and implement clients and servers in any of gRPC's supported languages, which in turn can be run in environments ranging from servers inside Google to your own tablet - all the complexity of communication between different languages and environments is handled for you by gRPC. We also get all the advantages of working with protocol buffers, including efficient serialization, a simple IDL, and easy interface updating. + +## Example code and setup + +The example code for our tutorial is in [grpc/grpc-go/examples/route_guide](https://github.com/grpc/grpc-go/tree/master/examples/route_guide). To download the example, clone the `grpc-go` repository by running the following command: +```shell +$ go get google.golang.org/grpc +``` + +Then change your current directory to `grpc-go/examples/route_guide`: +```shell +$ cd $GOPATH/src/google.golang.org/grpc/examples/route_guide +``` + +Ensure you have the relevant tools installed to generate the server and client interface code. If you don't, follow the setup instructions in [the Go quick start guide](https://grpc.io/docs/languages/go/quickstart). + + +## Defining the service + +Our first step (as you'll know from the [quick start](https://grpc.io/docs/#quick-start)) is to define the gRPC *service* and the method *request* and *response* types using [protocol buffers](https://developers.google.com/protocol-buffers/docs/overview). You can see the complete `.proto` file in [examples/route_guide/routeguide/route_guide.proto](https://github.com/grpc/grpc-go/tree/master/examples/route_guide/routeguide/route_guide.proto). + +To define a service, you specify a named `service` in your `.proto` file: + +```proto +service RouteGuide { + ... +} +``` + +Then you define `rpc` methods inside your service definition, specifying their request and response types. gRPC lets you define four kinds of service method, all of which are used in the `RouteGuide` service: + +- A *simple RPC* where the client sends a request to the server using the stub and waits for a response to come back, just like a normal function call. +```proto + // Obtains the feature at a given position. + rpc GetFeature(Point) returns (Feature) {} +``` + +- A *server-side streaming RPC* where the client sends a request to the server and gets a stream to read a sequence of messages back. The client reads from the returned stream until there are no more messages. As you can see in our example, you specify a server-side streaming method by placing the `stream` keyword before the *response* type. +```proto + // Obtains the Features available within the given Rectangle. Results are + // streamed rather than returned at once (e.g. in a response message with a + // repeated field), as the rectangle may cover a large area and contain a + // huge number of features. + rpc ListFeatures(Rectangle) returns (stream Feature) {} +``` + +- A *client-side streaming RPC* where the client writes a sequence of messages and sends them to the server, again using a provided stream. Once the client has finished writing the messages, it waits for the server to read them all and return its response. You specify a client-side streaming method by placing the `stream` keyword before the *request* type. +```proto + // Accepts a stream of Points on a route being traversed, returning a + // RouteSummary when traversal is completed. + rpc RecordRoute(stream Point) returns (RouteSummary) {} +``` + +- A *bidirectional streaming RPC* where both sides send a sequence of messages using a read-write stream. The two streams operate independently, so clients and servers can read and write in whatever order they like: for example, the server could wait to receive all the client messages before writing its responses, or it could alternately read a message then write a message, or some other combination of reads and writes. The order of messages in each stream is preserved. You specify this type of method by placing the `stream` keyword before both the request and the response. +```proto + // Accepts a stream of RouteNotes sent while a route is being traversed, + // while receiving other RouteNotes (e.g. from other users). + rpc RouteChat(stream RouteNote) returns (stream RouteNote) {} +``` + +Our `.proto` file also contains protocol buffer message type definitions for all the request and response types used in our service methods - for example, here's the `Point` message type: +```proto +// Points are represented as latitude-longitude pairs in the E7 representation +// (degrees multiplied by 10**7 and rounded to the nearest integer). +// Latitudes should be in the range +/- 90 degrees and longitude should be in +// the range +/- 180 degrees (inclusive). +message Point { + int32 latitude = 1; + int32 longitude = 2; +} +``` + + +## Generating client and server code + +Next we need to generate the gRPC client and server interfaces from our `.proto` service definition. We do this using the protocol buffer compiler `protoc` with a special gRPC Go plugin (if you want to run this by yourself, make sure you've installed protoc and followed the gRPC-Go [installation instructions](https://github.com/grpc/grpc-go/blob/master/README.md) first) and run: + +```shell +protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative "route_guide.proto" +``` + +Running this command generates the following file in your current directory: +- `route_guide.pb.go` + +This contains: +- All the protocol buffer code to populate, serialize, and retrieve our request and response message types +- An interface type (or *stub*) for clients to call with the methods defined in the `RouteGuide` service. +- An interface type for servers to implement, also with the methods defined in the `RouteGuide` service. + + + +## Creating the server + +First let's look at how we create a `RouteGuide` server. If you're only interested in creating gRPC clients, you can skip this section and go straight to [Creating the client](#client) (though you might find it interesting anyway!). + +There are two parts to making our `RouteGuide` service do its job: +- Implementing the service interface generated from our service definition: doing the actual "work" of our service. +- Running a gRPC server to listen for requests from clients and dispatch them to the right service implementation. + +You can find our example `RouteGuide` server in [grpc-go/examples/route_guide/server/server.go](https://github.com/grpc/grpc-go/tree/master/examples/route_guide/server/server.go). Let's take a closer look at how it works. + +### Implementing RouteGuide + +As you can see, our server has a `routeGuideServer` struct type that implements the generated `RouteGuideServer` interface: + +```go +type routeGuideServer struct { + ... +} +... + +func (s *routeGuideServer) GetFeature(ctx context.Context, point *pb.Point) (*pb.Feature, error) { + ... +} +... + +func (s *routeGuideServer) ListFeatures(rect *pb.Rectangle, stream pb.RouteGuide_ListFeaturesServer) error { + ... +} +... + +func (s *routeGuideServer) RecordRoute(stream pb.RouteGuide_RecordRouteServer) error { + ... +} +... + +func (s *routeGuideServer) RouteChat(stream pb.RouteGuide_RouteChatServer) error { + ... +} +... +``` + +#### Simple RPC +`routeGuideServer` implements all our service methods. Let's look at the simplest type first, `GetFeature`, which just gets a `Point` from the client and returns the corresponding feature information from its database in a `Feature`. + +```go +func (s *routeGuideServer) GetFeature(ctx context.Context, point *pb.Point) (*pb.Feature, error) { + for _, feature := range s.savedFeatures { + if proto.Equal(feature.Location, point) { + return feature, nil + } + } + // No feature was found, return an unnamed feature + return &pb.Feature{"", point}, nil +} +``` + +The method is passed a context object for the RPC and the client's `Point` protocol buffer request. It returns a `Feature` protocol buffer object with the response information and an `error`. In the method we populate the `Feature` with the appropriate information, and then `return` it along with an `nil` error to tell gRPC that we've finished dealing with the RPC and that the `Feature` can be returned to the client. + +#### Server-side streaming RPC +Now let's look at one of our streaming RPCs. `ListFeatures` is a server-side streaming RPC, so we need to send back multiple `Feature`s to our client. + +```go +func (s *routeGuideServer) ListFeatures(rect *pb.Rectangle, stream pb.RouteGuide_ListFeaturesServer) error { + for _, feature := range s.savedFeatures { + if inRange(feature.Location, rect) { + if err := stream.Send(feature); err != nil { + return err + } + } + } + return nil +} +``` + +As you can see, instead of getting simple request and response objects in our method parameters, this time we get a request object (the `Rectangle` in which our client wants to find `Feature`s) and a special `RouteGuide_ListFeaturesServer` object to write our responses. + +In the method, we populate as many `Feature` objects as we need to return, writing them to the `RouteGuide_ListFeaturesServer` using its `Send()` method. Finally, as in our simple RPC, we return a `nil` error to tell gRPC that we've finished writing responses. Should any error happen in this call, we return a non-`nil` error; the gRPC layer will translate it into an appropriate RPC status to be sent on the wire. + +#### Client-side streaming RPC +Now let's look at something a little more complicated: the client-side streaming method `RecordRoute`, where we get a stream of `Point`s from the client and return a single `RouteSummary` with information about their trip. As you can see, this time the method doesn't have a request parameter at all. Instead, it gets a `RouteGuide_RecordRouteServer` stream, which the server can use to both read *and* write messages - it can receive client messages using its `Recv()` method and return its single response using its `SendAndClose()` method. + +```go +func (s *routeGuideServer) RecordRoute(stream pb.RouteGuide_RecordRouteServer) error { + var pointCount, featureCount, distance int32 + var lastPoint *pb.Point + startTime := time.Now() + for { + point, err := stream.Recv() + if err == io.EOF { + endTime := time.Now() + return stream.SendAndClose(&pb.RouteSummary{ + PointCount: pointCount, + FeatureCount: featureCount, + Distance: distance, + ElapsedTime: int32(endTime.Sub(startTime).Seconds()), + }) + } + if err != nil { + return err + } + pointCount++ + for _, feature := range s.savedFeatures { + if proto.Equal(feature.Location, point) { + featureCount++ + } + } + if lastPoint != nil { + distance += calcDistance(lastPoint, point) + } + lastPoint = point + } +} +``` + +In the method body we use the `RouteGuide_RecordRouteServer`s `Recv()` method to repeatedly read in our client's requests to a request object (in this case a `Point`) until there are no more messages: the server needs to check the error returned from `Recv()` after each call. If this is `nil`, the stream is still good and it can continue reading; if it's `io.EOF` the message stream has ended and the server can return its `RouteSummary`. If it has any other value, we return the error "as is" so that it'll be translated to an RPC status by the gRPC layer. + +#### Bidirectional streaming RPC +Finally, let's look at our bidirectional streaming RPC `RouteChat()`. + +```go +func (s *routeGuideServer) RouteChat(stream pb.RouteGuide_RouteChatServer) error { + for { + in, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + key := serialize(in.Location) + ... // look for notes to be sent to client + for _, note := range s.routeNotes[key] { + if err := stream.Send(note); err != nil { + return err + } + } + } +} +``` + +This time we get a `RouteGuide_RouteChatServer` stream that, as in our client-side streaming example, can be used to read and write messages. However, this time we return values via our method's stream while the client is still writing messages to *their* message stream. + +The syntax for reading and writing here is very similar to our client-streaming method, except the server uses the stream's `Send()` method rather than `SendAndClose()` because it's writing multiple responses. Although each side will always get the other's messages in the order they were written, both the client and server can read and write in any order — the streams operate completely independently. + +### Starting the server + +Once we've implemented all our methods, we also need to start up a gRPC server so that clients can actually use our service. The following snippet shows how we do this for our `RouteGuide` service: + +```go +flag.Parse() +lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", *port)) +if err != nil { + log.Fatalf("failed to listen: %v", err) +} +grpcServer := grpc.NewServer() +pb.RegisterRouteGuideServer(grpcServer, &routeGuideServer{}) +... // determine whether to use TLS +grpcServer.Serve(lis) +``` +To build and start a server, we: + +1. Specify the port we want to use to listen for client requests using `lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", *port))`. +2. Create an instance of the gRPC server using `grpc.NewServer()`. +3. Register our service implementation with the gRPC server. +4. Call `Serve()` on the server with our port details to do a blocking wait until the process is killed or `Stop()` is called. + + +## Creating the client + +In this section, we'll look at creating a Go client for our `RouteGuide` service. You can see our complete example client code in [grpc-go/examples/route_guide/client/client.go](https://github.com/grpc/grpc-go/tree/master/examples/route_guide/client/client.go). + +### Creating a stub + +To call service methods, we first need to create a gRPC *channel* to communicate with the server. We create this by passing the server address and port number to `grpc.NewClient()` as follows: + +```go +conn, err := grpc.NewClient(*serverAddr) +if err != nil { + ... +} +defer conn.Close() +``` + +You can use `DialOptions` to set the auth credentials (e.g., TLS, GCE credentials, JWT credentials) in `grpc.NewClient` if the service you request requires that - however, we don't need to do this for our `RouteGuide` service. + +Once the gRPC *channel* is setup, we need a client *stub* to perform RPCs. We get this using the `NewRouteGuideClient` method provided in the `pb` package we generated from our `.proto` file. + +```go +client := pb.NewRouteGuideClient(conn) +``` + +### Calling service methods + +Now let's look at how we call our service methods. Note that in gRPC-Go, RPCs operate in a blocking/synchronous mode, which means that the RPC call waits for the server to respond, and will either return a response or an error. + +#### Simple RPC + +Calling the simple RPC `GetFeature` is nearly as straightforward as calling a local method. + +```go +feature, err := client.GetFeature(ctx, &pb.Point{409146138, -746188906}) +if err != nil { + ... +} +``` + +As you can see, we call the method on the stub we got earlier. In our method parameters we create and populate a request protocol buffer object (in our case `Point`). We also pass a `context.Context` object which lets us change our RPC's behaviour if necessary, such as time-out/cancel an RPC in flight. If the call doesn't return an error, then we can read the response information from the server from the first return value. + +```go +log.Println(feature) +``` + +#### Server-side streaming RPC + +Here's where we call the server-side streaming method `ListFeatures`, which returns a stream of geographical `Feature`s. If you've already read [Creating the server](#server) some of this may look very familiar - streaming RPCs are implemented in a similar way on both sides. + +```go +rect := &pb.Rectangle{ ... } // initialize a pb.Rectangle +stream, err := client.ListFeatures(ctx, rect) +if err != nil { + ... +} +for { + feature, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("%v.ListFeatures(_) = _, %v", client, err) + } + log.Println(feature) +} +``` + +As in the simple RPC, we pass the method a context and a request. However, instead of getting a response object back, we get back an instance of `RouteGuide_ListFeaturesClient`. The client can use the `RouteGuide_ListFeaturesClient` stream to read the server's responses. + +We use the `RouteGuide_ListFeaturesClient`'s `Recv()` method to repeatedly read in the server's responses to a response protocol buffer object (in this case a `Feature`) until there are no more messages: the client needs to check the error `err` returned from `Recv()` after each call. If `nil`, the stream is still good and it can continue reading; if it's `io.EOF` then the message stream has ended; otherwise there must be an RPC error, which is passed over through `err`. + +#### Client-side streaming RPC + +The client-side streaming method `RecordRoute` is similar to the server-side method, except that we only pass the method a context and get a `RouteGuide_RecordRouteClient` stream back, which we can use to both write *and* read messages. + +```go +// Create a random number of random points +r := rand.New(rand.NewSource(time.Now().UnixNano())) +pointCount := int(r.Int31n(100)) + 2 // Traverse at least two points +var points []*pb.Point +for i := 0; i < pointCount; i++ { + points = append(points, randomPoint(r)) +} +log.Printf("Traversing %d points.", len(points)) +stream, err := client.RecordRoute(ctx) +if err != nil { + log.Fatalf("%v.RecordRoute(_) = _, %v", client, err) +} +for _, point := range points { + if err := stream.Send(point); err != nil { + log.Fatalf("%v.Send(%v) = %v", stream, point, err) + } +} +reply, err := stream.CloseAndRecv() +if err != nil { + log.Fatalf("%v.CloseAndRecv() got error %v, want %v", stream, err, nil) +} +log.Printf("Route summary: %v", reply) +``` + +The `RouteGuide_RecordRouteClient` has a `Send()` method that we can use to send requests to the server. Once we've finished writing our client's requests to the stream using `Send()`, we need to call `CloseAndRecv()` on the stream to let gRPC know that we've finished writing and are expecting to receive a response. We get our RPC status from the `err` returned from `CloseAndRecv()`. If the status is `nil`, then the first return value from `CloseAndRecv()` will be a valid server response. + +#### Bidirectional streaming RPC + +Finally, let's look at our bidirectional streaming RPC `RouteChat()`. As in the case of `RecordRoute`, we only pass the method a context object and get back a stream that we can use to both write and read messages. However, this time we return values via our method's stream while the server is still writing messages to *their* message stream. + +```go +stream, err := client.RouteChat(ctx) +waitc := make(chan struct{}) +go func() { + for { + in, err := stream.Recv() + if err == io.EOF { + // read done. + close(waitc) + return + } + if err != nil { + log.Fatalf("Failed to receive a note : %v", err) + } + log.Printf("Got message %s at point(%d, %d)", in.Message, in.Location.Latitude, in.Location.Longitude) + } +}() +for _, note := range notes { + if err := stream.Send(note); err != nil { + log.Fatalf("Failed to send a note: %v", err) + } +} +stream.CloseSend() +<-waitc +``` + +The syntax for reading and writing here is very similar to our client-side streaming method, except we use the stream's `CloseSend()` method once we've finished our call. Although each side will always get the other's messages in the order they were written, both the client and server can read and write in any order — the streams operate completely independently. + +## Try it out! + +To compile and run the server, assuming you are in the folder +`$GOPATH/src/google.golang.org/grpc/examples/route_guide`, simply: + +```sh +$ go run server/server.go +``` + +Likewise, to run the client: + +```sh +$ go run client/client.go +``` + diff --git a/examples/helloworld/README.md b/examples/helloworld/README.md new file mode 100644 index 0000000..bb2138f --- /dev/null +++ b/examples/helloworld/README.md @@ -0,0 +1,29 @@ +# gRPC Hello World + +Follow these setup to run the [quick start][] example: + + 1. Get the code: + + ```console + $ go get google.golang.org/grpc/examples/helloworld/greeter_client + $ go get google.golang.org/grpc/examples/helloworld/greeter_server + ``` + + 2. Run the server: + + ```console + $ $(go env GOPATH)/bin/greeter_server & + ``` + + 3. Run the client: + + ```console + $ $(go env GOPATH)/bin/greeter_client + Greeting: Hello world + ``` + +For more details (including instructions for making a small change to the +example code) or if you're having trouble running this example, see [Quick +Start][]. + +[quick start]: https://grpc.io/docs/languages/go/quickstart diff --git a/examples/helloworld/greeter_client/main.go b/examples/helloworld/greeter_client/main.go new file mode 100644 index 0000000..d746840 --- /dev/null +++ b/examples/helloworld/greeter_client/main.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package main implements a client for Greeter service. +package main + +import ( + "context" + "flag" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/helloworld/helloworld" +) + +const ( + defaultName = "world" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") + name = flag.String("name", defaultName, "Name to greet") +) + +func main() { + flag.Parse() + // Set up a connection to the server. + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + c := pb.NewGreeterClient(conn) + + // Contact the server and print out its response. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + r, err := c.SayHello(ctx, &pb.HelloRequest{Name: *name}) + if err != nil { + log.Fatalf("could not greet: %v", err) + } + log.Printf("Greeting: %s", r.GetMessage()) +} diff --git a/examples/helloworld/greeter_server/main.go b/examples/helloworld/greeter_server/main.go new file mode 100644 index 0000000..f7ccf1c --- /dev/null +++ b/examples/helloworld/greeter_server/main.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package main implements a server for Greeter service. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + pb "google.golang.org/grpc/examples/helloworld/helloworld" +) + +var ( + port = flag.Int("port", 50051, "The server port") +) + +// server is used to implement helloworld.GreeterServer. +type server struct { + pb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer +func (s *server) SayHello(_ context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + log.Printf("Received: %v", in.GetName()) + return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil +} + +func main() { + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + s := grpc.NewServer() + pb.RegisterGreeterServer(s, &server{}) + log.Printf("server listening at %v", lis.Addr()) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/helloworld/helloworld/helloworld.pb.go b/examples/helloworld/helloworld/helloworld.pb.go new file mode 100644 index 0000000..48555f9 --- /dev/null +++ b/examples/helloworld/helloworld/helloworld.pb.go @@ -0,0 +1,192 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: examples/helloworld/helloworld/helloworld.proto + +package helloworld + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// The request message containing the user's name. +type HelloRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HelloRequest) Reset() { + *x = HelloRequest{} + mi := &file_examples_helloworld_helloworld_helloworld_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HelloRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HelloRequest) ProtoMessage() {} + +func (x *HelloRequest) ProtoReflect() protoreflect.Message { + mi := &file_examples_helloworld_helloworld_helloworld_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HelloRequest.ProtoReflect.Descriptor instead. +func (*HelloRequest) Descriptor() ([]byte, []int) { + return file_examples_helloworld_helloworld_helloworld_proto_rawDescGZIP(), []int{0} +} + +func (x *HelloRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +// The response message containing the greetings +type HelloReply struct { + state protoimpl.MessageState `protogen:"open.v1"` + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HelloReply) Reset() { + *x = HelloReply{} + mi := &file_examples_helloworld_helloworld_helloworld_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HelloReply) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HelloReply) ProtoMessage() {} + +func (x *HelloReply) ProtoReflect() protoreflect.Message { + mi := &file_examples_helloworld_helloworld_helloworld_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HelloReply.ProtoReflect.Descriptor instead. +func (*HelloReply) Descriptor() ([]byte, []int) { + return file_examples_helloworld_helloworld_helloworld_proto_rawDescGZIP(), []int{1} +} + +func (x *HelloReply) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +var File_examples_helloworld_helloworld_helloworld_proto protoreflect.FileDescriptor + +const file_examples_helloworld_helloworld_helloworld_proto_rawDesc = "" + + "\n" + + "/examples/helloworld/helloworld/helloworld.proto\x12\n" + + "helloworld\"\"\n" + + "\fHelloRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\"&\n" + + "\n" + + "HelloReply\x12\x18\n" + + "\amessage\x18\x01 \x01(\tR\amessage2I\n" + + "\aGreeter\x12>\n" + + "\bSayHello\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00Bg\n" + + "\x1bio.grpc.examples.helloworldB\x0fHelloWorldProtoP\x01Z5google.golang.org/grpc/examples/helloworld/helloworldb\x06proto3" + +var ( + file_examples_helloworld_helloworld_helloworld_proto_rawDescOnce sync.Once + file_examples_helloworld_helloworld_helloworld_proto_rawDescData []byte +) + +func file_examples_helloworld_helloworld_helloworld_proto_rawDescGZIP() []byte { + file_examples_helloworld_helloworld_helloworld_proto_rawDescOnce.Do(func() { + file_examples_helloworld_helloworld_helloworld_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_examples_helloworld_helloworld_helloworld_proto_rawDesc), len(file_examples_helloworld_helloworld_helloworld_proto_rawDesc))) + }) + return file_examples_helloworld_helloworld_helloworld_proto_rawDescData +} + +var file_examples_helloworld_helloworld_helloworld_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_examples_helloworld_helloworld_helloworld_proto_goTypes = []any{ + (*HelloRequest)(nil), // 0: helloworld.HelloRequest + (*HelloReply)(nil), // 1: helloworld.HelloReply +} +var file_examples_helloworld_helloworld_helloworld_proto_depIdxs = []int32{ + 0, // 0: helloworld.Greeter.SayHello:input_type -> helloworld.HelloRequest + 1, // 1: helloworld.Greeter.SayHello:output_type -> helloworld.HelloReply + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_examples_helloworld_helloworld_helloworld_proto_init() } +func file_examples_helloworld_helloworld_helloworld_proto_init() { + if File_examples_helloworld_helloworld_helloworld_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_examples_helloworld_helloworld_helloworld_proto_rawDesc), len(file_examples_helloworld_helloworld_helloworld_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_examples_helloworld_helloworld_helloworld_proto_goTypes, + DependencyIndexes: file_examples_helloworld_helloworld_helloworld_proto_depIdxs, + MessageInfos: file_examples_helloworld_helloworld_helloworld_proto_msgTypes, + }.Build() + File_examples_helloworld_helloworld_helloworld_proto = out.File + file_examples_helloworld_helloworld_helloworld_proto_goTypes = nil + file_examples_helloworld_helloworld_helloworld_proto_depIdxs = nil +} diff --git a/examples/helloworld/helloworld/helloworld.proto b/examples/helloworld/helloworld/helloworld.proto new file mode 100644 index 0000000..692ef9d --- /dev/null +++ b/examples/helloworld/helloworld/helloworld.proto @@ -0,0 +1,38 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +option go_package = "google.golang.org/grpc/examples/helloworld/helloworld"; +option java_multiple_files = true; +option java_package = "io.grpc.examples.helloworld"; +option java_outer_classname = "HelloWorldProto"; + +package helloworld; + +// The greeting service definition. +service Greeter { + // Sends a greeting + rpc SayHello (HelloRequest) returns (HelloReply) {} +} + +// The request message containing the user's name. +message HelloRequest { + string name = 1; +} + +// The response message containing the greetings +message HelloReply { + string message = 1; +} diff --git a/examples/helloworld/helloworld/helloworld_grpc.pb.go b/examples/helloworld/helloworld/helloworld_grpc.pb.go new file mode 100644 index 0000000..bf8263b --- /dev/null +++ b/examples/helloworld/helloworld/helloworld_grpc.pb.go @@ -0,0 +1,141 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: examples/helloworld/helloworld/helloworld.proto + +package helloworld + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Greeter_SayHello_FullMethodName = "/helloworld.Greeter/SayHello" +) + +// GreeterClient is the client API for Greeter service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// The greeting service definition. +type GreeterClient interface { + // Sends a greeting + SayHello(ctx context.Context, in *HelloRequest, opts ...grpc.CallOption) (*HelloReply, error) +} + +type greeterClient struct { + cc grpc.ClientConnInterface +} + +func NewGreeterClient(cc grpc.ClientConnInterface) GreeterClient { + return &greeterClient{cc} +} + +func (c *greeterClient) SayHello(ctx context.Context, in *HelloRequest, opts ...grpc.CallOption) (*HelloReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(HelloReply) + err := c.cc.Invoke(ctx, Greeter_SayHello_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// GreeterServer is the server API for Greeter service. +// All implementations must embed UnimplementedGreeterServer +// for forward compatibility. +// +// The greeting service definition. +type GreeterServer interface { + // Sends a greeting + SayHello(context.Context, *HelloRequest) (*HelloReply, error) + mustEmbedUnimplementedGreeterServer() +} + +// UnimplementedGreeterServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedGreeterServer struct{} + +func (UnimplementedGreeterServer) SayHello(context.Context, *HelloRequest) (*HelloReply, error) { + return nil, status.Error(codes.Unimplemented, "method SayHello not implemented") +} +func (UnimplementedGreeterServer) mustEmbedUnimplementedGreeterServer() {} +func (UnimplementedGreeterServer) testEmbeddedByValue() {} + +// UnsafeGreeterServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to GreeterServer will +// result in compilation errors. +type UnsafeGreeterServer interface { + mustEmbedUnimplementedGreeterServer() +} + +func RegisterGreeterServer(s grpc.ServiceRegistrar, srv GreeterServer) { + // If the following call panics, it indicates UnimplementedGreeterServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Greeter_ServiceDesc, srv) +} + +func _Greeter_SayHello_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(HelloRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GreeterServer).SayHello(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Greeter_SayHello_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GreeterServer).SayHello(ctx, req.(*HelloRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// Greeter_ServiceDesc is the grpc.ServiceDesc for Greeter service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Greeter_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "helloworld.Greeter", + HandlerType: (*GreeterServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "SayHello", + Handler: _Greeter_SayHello_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "examples/helloworld/helloworld/helloworld.proto", +} diff --git a/examples/route_guide/README.md b/examples/route_guide/README.md new file mode 100644 index 0000000..29343b1 --- /dev/null +++ b/examples/route_guide/README.md @@ -0,0 +1,35 @@ +# Description +The route guide server and client demonstrate how to use grpc go libraries to +perform unary, client streaming, server streaming and full duplex RPCs. + +Please refer to [gRPC Basics: Go](https://grpc.io/docs/tutorials/basic/go.html) for more information. + +See the definition of the route guide service in `routeguide/route_guide.proto`. + +# Run the sample code +To compile and run the server, assuming you are in the root of the `route_guide` +folder, i.e., `.../examples/route_guide/`, simply: + +```sh +$ go run server/server.go +``` + +Likewise, to run the client: + +```sh +$ go run client/client.go +``` + +# Optional command line flags +The server and client both take optional command line flags. For example, the +client and server run without TLS by default. To enable TLS: + +```sh +$ go run server/server.go -tls=true +``` + +and + +```sh +$ go run client/client.go -tls=true +``` diff --git a/examples/route_guide/client/client.go b/examples/route_guide/client/client.go new file mode 100644 index 0000000..57f868a --- /dev/null +++ b/examples/route_guide/client/client.go @@ -0,0 +1,194 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package main implements a simple gRPC client that demonstrates how to use gRPC-Go libraries +// to perform unary, client streaming, server streaming and full duplex RPCs. +// +// It interacts with the route guide service whose definition can be found in routeguide/route_guide.proto. +package main + +import ( + "context" + "flag" + "io" + "log" + rand "math/rand/v2" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/examples/data" + pb "google.golang.org/grpc/examples/route_guide/routeguide" +) + +var ( + tls = flag.Bool("tls", false, "Connection uses TLS if true, else plain TCP") + caFile = flag.String("ca_file", "", "The file containing the CA root cert file") + serverAddr = flag.String("addr", "localhost:50051", "The server address in the format of host:port") + serverHostOverride = flag.String("server_host_override", "x.test.example.com", "The server name used to verify the hostname returned by the TLS handshake") +) + +// printFeature gets the feature for the given point. +func printFeature(client pb.RouteGuideClient, point *pb.Point) { + log.Printf("Getting feature for point (%d, %d)", point.Latitude, point.Longitude) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + feature, err := client.GetFeature(ctx, point) + if err != nil { + log.Fatalf("client.GetFeature failed: %v", err) + } + log.Println(feature) +} + +// printFeatures lists all the features within the given bounding Rectangle. +func printFeatures(client pb.RouteGuideClient, rect *pb.Rectangle) { + log.Printf("Looking for features within %v", rect) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + stream, err := client.ListFeatures(ctx, rect) + if err != nil { + log.Fatalf("client.ListFeatures failed: %v", err) + } + for { + feature, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("client.ListFeatures failed: %v", err) + } + log.Printf("Feature: name: %q, point:(%v, %v)", feature.GetName(), + feature.GetLocation().GetLatitude(), feature.GetLocation().GetLongitude()) + } +} + +// runRecordRoute sends a sequence of points to server and expects to get a RouteSummary from server. +func runRecordRoute(client pb.RouteGuideClient) { + // Create a random number of random points + pointCount := int(rand.Int32N(100)) + 2 // Traverse at least two points + var points []*pb.Point + for i := 0; i < pointCount; i++ { + points = append(points, randomPoint()) + } + log.Printf("Traversing %d points.", len(points)) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + stream, err := client.RecordRoute(ctx) + if err != nil { + log.Fatalf("client.RecordRoute failed: %v", err) + } + for _, point := range points { + if err := stream.Send(point); err != nil { + log.Fatalf("client.RecordRoute: stream.Send(%v) failed: %v", point, err) + } + } + reply, err := stream.CloseAndRecv() + if err != nil { + log.Fatalf("client.RecordRoute failed: %v", err) + } + log.Printf("Route summary: %v", reply) +} + +// runRouteChat receives a sequence of route notes, while sending notes for various locations. +func runRouteChat(client pb.RouteGuideClient) { + notes := []*pb.RouteNote{ + {Location: &pb.Point{Latitude: 0, Longitude: 1}, Message: "First message"}, + {Location: &pb.Point{Latitude: 0, Longitude: 2}, Message: "Second message"}, + {Location: &pb.Point{Latitude: 0, Longitude: 3}, Message: "Third message"}, + {Location: &pb.Point{Latitude: 0, Longitude: 1}, Message: "Fourth message"}, + {Location: &pb.Point{Latitude: 0, Longitude: 2}, Message: "Fifth message"}, + {Location: &pb.Point{Latitude: 0, Longitude: 3}, Message: "Sixth message"}, + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + stream, err := client.RouteChat(ctx) + if err != nil { + log.Fatalf("client.RouteChat failed: %v", err) + } + waitc := make(chan struct{}) + go func() { + for { + in, err := stream.Recv() + if err == io.EOF { + // read done. + close(waitc) + return + } + if err != nil { + log.Fatalf("client.RouteChat failed: %v", err) + } + log.Printf("Got message %s at point(%d, %d)", in.Message, in.Location.Latitude, in.Location.Longitude) + } + }() + for _, note := range notes { + if err := stream.Send(note); err != nil { + log.Fatalf("client.RouteChat: stream.Send(%v) failed: %v", note, err) + } + } + stream.CloseSend() + <-waitc +} + +func randomPoint() *pb.Point { + lat := (rand.Int32N(180) - 90) * 1e7 + long := (rand.Int32N(360) - 180) * 1e7 + return &pb.Point{Latitude: lat, Longitude: long} +} + +func main() { + flag.Parse() + var opts []grpc.DialOption + if *tls { + if *caFile == "" { + *caFile = data.Path("x509/ca_cert.pem") + } + creds, err := credentials.NewClientTLSFromFile(*caFile, *serverHostOverride) + if err != nil { + log.Fatalf("Failed to create TLS credentials: %v", err) + } + opts = append(opts, grpc.WithTransportCredentials(creds)) + } else { + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + + conn, err := grpc.NewClient(*serverAddr, opts...) + if err != nil { + log.Fatalf("fail to dial: %v", err) + } + defer conn.Close() + client := pb.NewRouteGuideClient(conn) + + // Looking for a valid feature + printFeature(client, &pb.Point{Latitude: 409146138, Longitude: -746188906}) + + // Feature missing. + printFeature(client, &pb.Point{Latitude: 0, Longitude: 0}) + + // Looking for features between 40, -75 and 42, -73. + printFeatures(client, &pb.Rectangle{ + Lo: &pb.Point{Latitude: 400000000, Longitude: -750000000}, + Hi: &pb.Point{Latitude: 420000000, Longitude: -730000000}, + }) + + // RecordRoute + runRecordRoute(client) + + // RouteChat + runRouteChat(client) +} diff --git a/examples/route_guide/routeguide/route_guide.pb.go b/examples/route_guide/routeguide/route_guide.pb.go new file mode 100644 index 0000000..5c1fed6 --- /dev/null +++ b/examples/route_guide/routeguide/route_guide.pb.go @@ -0,0 +1,434 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: examples/route_guide/routeguide/route_guide.proto + +package routeguide + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Points are represented as latitude-longitude pairs in the E7 representation +// (degrees multiplied by 10**7 and rounded to the nearest integer). +// Latitudes should be in the range +/- 90 degrees and longitude should be in +// the range +/- 180 degrees (inclusive). +type Point struct { + state protoimpl.MessageState `protogen:"open.v1"` + Latitude int32 `protobuf:"varint,1,opt,name=latitude" json:"latitude,omitempty"` + Longitude int32 `protobuf:"varint,2,opt,name=longitude" json:"longitude,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Point) Reset() { + *x = Point{} + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Point) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Point) ProtoMessage() {} + +func (x *Point) ProtoReflect() protoreflect.Message { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Point.ProtoReflect.Descriptor instead. +func (*Point) Descriptor() ([]byte, []int) { + return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{0} +} + +func (x *Point) GetLatitude() int32 { + if x != nil { + return x.Latitude + } + return 0 +} + +func (x *Point) GetLongitude() int32 { + if x != nil { + return x.Longitude + } + return 0 +} + +// A latitude-longitude rectangle, represented as two diagonally opposite +// points "lo" and "hi". +type Rectangle struct { + state protoimpl.MessageState `protogen:"open.v1"` + // One corner of the rectangle. + Lo *Point `protobuf:"bytes,1,opt,name=lo" json:"lo,omitempty"` + // The other corner of the rectangle. + Hi *Point `protobuf:"bytes,2,opt,name=hi" json:"hi,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Rectangle) Reset() { + *x = Rectangle{} + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Rectangle) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Rectangle) ProtoMessage() {} + +func (x *Rectangle) ProtoReflect() protoreflect.Message { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Rectangle.ProtoReflect.Descriptor instead. +func (*Rectangle) Descriptor() ([]byte, []int) { + return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{1} +} + +func (x *Rectangle) GetLo() *Point { + if x != nil { + return x.Lo + } + return nil +} + +func (x *Rectangle) GetHi() *Point { + if x != nil { + return x.Hi + } + return nil +} + +// A feature names something at a given point. +// +// If a feature could not be named, the name is empty. +type Feature struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The name of the feature. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The point where the feature is detected. + Location *Point `protobuf:"bytes,2,opt,name=location" json:"location,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Feature) Reset() { + *x = Feature{} + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Feature) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Feature) ProtoMessage() {} + +func (x *Feature) ProtoReflect() protoreflect.Message { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Feature.ProtoReflect.Descriptor instead. +func (*Feature) Descriptor() ([]byte, []int) { + return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{2} +} + +func (x *Feature) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Feature) GetLocation() *Point { + if x != nil { + return x.Location + } + return nil +} + +// A RouteNote is a message sent while at a given point. +type RouteNote struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The location from which the message is sent. + Location *Point `protobuf:"bytes,1,opt,name=location" json:"location,omitempty"` + // The message to be sent. + Message string `protobuf:"bytes,2,opt,name=message" json:"message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RouteNote) Reset() { + *x = RouteNote{} + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RouteNote) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RouteNote) ProtoMessage() {} + +func (x *RouteNote) ProtoReflect() protoreflect.Message { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RouteNote.ProtoReflect.Descriptor instead. +func (*RouteNote) Descriptor() ([]byte, []int) { + return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{3} +} + +func (x *RouteNote) GetLocation() *Point { + if x != nil { + return x.Location + } + return nil +} + +func (x *RouteNote) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +// A RouteSummary is received in response to a RecordRoute rpc. +// +// It contains the number of individual points received, the number of +// detected features, and the total distance covered as the cumulative sum of +// the distance between each point. +type RouteSummary struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The number of points received. + PointCount int32 `protobuf:"varint,1,opt,name=point_count,json=pointCount" json:"point_count,omitempty"` + // The number of known features passed while traversing the route. + FeatureCount int32 `protobuf:"varint,2,opt,name=feature_count,json=featureCount" json:"feature_count,omitempty"` + // The distance covered in metres. + Distance int32 `protobuf:"varint,3,opt,name=distance" json:"distance,omitempty"` + // The duration of the traversal in seconds. + ElapsedTime int32 `protobuf:"varint,4,opt,name=elapsed_time,json=elapsedTime" json:"elapsed_time,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RouteSummary) Reset() { + *x = RouteSummary{} + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RouteSummary) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RouteSummary) ProtoMessage() {} + +func (x *RouteSummary) ProtoReflect() protoreflect.Message { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RouteSummary.ProtoReflect.Descriptor instead. +func (*RouteSummary) Descriptor() ([]byte, []int) { + return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{4} +} + +func (x *RouteSummary) GetPointCount() int32 { + if x != nil { + return x.PointCount + } + return 0 +} + +func (x *RouteSummary) GetFeatureCount() int32 { + if x != nil { + return x.FeatureCount + } + return 0 +} + +func (x *RouteSummary) GetDistance() int32 { + if x != nil { + return x.Distance + } + return 0 +} + +func (x *RouteSummary) GetElapsedTime() int32 { + if x != nil { + return x.ElapsedTime + } + return 0 +} + +var File_examples_route_guide_routeguide_route_guide_proto protoreflect.FileDescriptor + +const file_examples_route_guide_routeguide_route_guide_proto_rawDesc = "" + + "\n" + + "1examples/route_guide/routeguide/route_guide.proto\x12\n" + + "routeguide\"A\n" + + "\x05Point\x12\x1a\n" + + "\blatitude\x18\x01 \x01(\x05R\blatitude\x12\x1c\n" + + "\tlongitude\x18\x02 \x01(\x05R\tlongitude\"Q\n" + + "\tRectangle\x12!\n" + + "\x02lo\x18\x01 \x01(\v2\x11.routeguide.PointR\x02lo\x12!\n" + + "\x02hi\x18\x02 \x01(\v2\x11.routeguide.PointR\x02hi\"L\n" + + "\aFeature\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12-\n" + + "\blocation\x18\x02 \x01(\v2\x11.routeguide.PointR\blocation\"T\n" + + "\tRouteNote\x12-\n" + + "\blocation\x18\x01 \x01(\v2\x11.routeguide.PointR\blocation\x12\x18\n" + + "\amessage\x18\x02 \x01(\tR\amessage\"\x93\x01\n" + + "\fRouteSummary\x12\x1f\n" + + "\vpoint_count\x18\x01 \x01(\x05R\n" + + "pointCount\x12#\n" + + "\rfeature_count\x18\x02 \x01(\x05R\ffeatureCount\x12\x1a\n" + + "\bdistance\x18\x03 \x01(\x05R\bdistance\x12!\n" + + "\felapsed_time\x18\x04 \x01(\x05R\velapsedTime2\x85\x02\n" + + "\n" + + "RouteGuide\x126\n" + + "\n" + + "GetFeature\x12\x11.routeguide.Point\x1a\x13.routeguide.Feature\"\x00\x12>\n" + + "\fListFeatures\x12\x15.routeguide.Rectangle\x1a\x13.routeguide.Feature\"\x000\x01\x12>\n" + + "\vRecordRoute\x12\x11.routeguide.Point\x1a\x18.routeguide.RouteSummary\"\x00(\x01\x12?\n" + + "\tRouteChat\x12\x15.routeguide.RouteNote\x1a\x15.routeguide.RouteNote\"\x00(\x010\x01Bm\n" + + "\x1bio.grpc.examples.routeguideB\x0fRouteGuideProtoP\x01Z6google.golang.org/grpc/examples/route_guide/routeguide\x92\x03\x02\b\x02b\beditionsp\xe8\a" + +var ( + file_examples_route_guide_routeguide_route_guide_proto_rawDescOnce sync.Once + file_examples_route_guide_routeguide_route_guide_proto_rawDescData []byte +) + +func file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP() []byte { + file_examples_route_guide_routeguide_route_guide_proto_rawDescOnce.Do(func() { + file_examples_route_guide_routeguide_route_guide_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_examples_route_guide_routeguide_route_guide_proto_rawDesc), len(file_examples_route_guide_routeguide_route_guide_proto_rawDesc))) + }) + return file_examples_route_guide_routeguide_route_guide_proto_rawDescData +} + +var file_examples_route_guide_routeguide_route_guide_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_examples_route_guide_routeguide_route_guide_proto_goTypes = []any{ + (*Point)(nil), // 0: routeguide.Point + (*Rectangle)(nil), // 1: routeguide.Rectangle + (*Feature)(nil), // 2: routeguide.Feature + (*RouteNote)(nil), // 3: routeguide.RouteNote + (*RouteSummary)(nil), // 4: routeguide.RouteSummary +} +var file_examples_route_guide_routeguide_route_guide_proto_depIdxs = []int32{ + 0, // 0: routeguide.Rectangle.lo:type_name -> routeguide.Point + 0, // 1: routeguide.Rectangle.hi:type_name -> routeguide.Point + 0, // 2: routeguide.Feature.location:type_name -> routeguide.Point + 0, // 3: routeguide.RouteNote.location:type_name -> routeguide.Point + 0, // 4: routeguide.RouteGuide.GetFeature:input_type -> routeguide.Point + 1, // 5: routeguide.RouteGuide.ListFeatures:input_type -> routeguide.Rectangle + 0, // 6: routeguide.RouteGuide.RecordRoute:input_type -> routeguide.Point + 3, // 7: routeguide.RouteGuide.RouteChat:input_type -> routeguide.RouteNote + 2, // 8: routeguide.RouteGuide.GetFeature:output_type -> routeguide.Feature + 2, // 9: routeguide.RouteGuide.ListFeatures:output_type -> routeguide.Feature + 4, // 10: routeguide.RouteGuide.RecordRoute:output_type -> routeguide.RouteSummary + 3, // 11: routeguide.RouteGuide.RouteChat:output_type -> routeguide.RouteNote + 8, // [8:12] is the sub-list for method output_type + 4, // [4:8] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_examples_route_guide_routeguide_route_guide_proto_init() } +func file_examples_route_guide_routeguide_route_guide_proto_init() { + if File_examples_route_guide_routeguide_route_guide_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_examples_route_guide_routeguide_route_guide_proto_rawDesc), len(file_examples_route_guide_routeguide_route_guide_proto_rawDesc)), + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_examples_route_guide_routeguide_route_guide_proto_goTypes, + DependencyIndexes: file_examples_route_guide_routeguide_route_guide_proto_depIdxs, + MessageInfos: file_examples_route_guide_routeguide_route_guide_proto_msgTypes, + }.Build() + File_examples_route_guide_routeguide_route_guide_proto = out.File + file_examples_route_guide_routeguide_route_guide_proto_goTypes = nil + file_examples_route_guide_routeguide_route_guide_proto_depIdxs = nil +} diff --git a/examples/route_guide/routeguide/route_guide.proto b/examples/route_guide/routeguide/route_guide.proto new file mode 100644 index 0000000..7353f56 --- /dev/null +++ b/examples/route_guide/routeguide/route_guide.proto @@ -0,0 +1,112 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +edition = "2023"; + +option features.field_presence = IMPLICIT; +option go_package = "google.golang.org/grpc/examples/route_guide/routeguide"; +option java_multiple_files = true; +option java_package = "io.grpc.examples.routeguide"; +option java_outer_classname = "RouteGuideProto"; + +package routeguide; + +// Interface exported by the server. +service RouteGuide { + // A simple RPC. + // + // Obtains the feature at a given position. + // + // A feature with an empty name is returned if there's no feature at the given + // position. + rpc GetFeature(Point) returns (Feature) {} + + // A server-to-client streaming RPC. + // + // Obtains the Features available within the given Rectangle. Results are + // streamed rather than returned at once (e.g. in a response message with a + // repeated field), as the rectangle may cover a large area and contain a + // huge number of features. + rpc ListFeatures(Rectangle) returns (stream Feature) {} + + // A client-to-server streaming RPC. + // + // Accepts a stream of Points on a route being traversed, returning a + // RouteSummary when traversal is completed. + rpc RecordRoute(stream Point) returns (RouteSummary) {} + + // A Bidirectional streaming RPC. + // + // Accepts a stream of RouteNotes sent while a route is being traversed, + // while receiving other RouteNotes (e.g. from other users). + rpc RouteChat(stream RouteNote) returns (stream RouteNote) {} +} + +// Points are represented as latitude-longitude pairs in the E7 representation +// (degrees multiplied by 10**7 and rounded to the nearest integer). +// Latitudes should be in the range +/- 90 degrees and longitude should be in +// the range +/- 180 degrees (inclusive). +message Point { + int32 latitude = 1; + int32 longitude = 2; +} + +// A latitude-longitude rectangle, represented as two diagonally opposite +// points "lo" and "hi". +message Rectangle { + // One corner of the rectangle. + Point lo = 1; + + // The other corner of the rectangle. + Point hi = 2; +} + +// A feature names something at a given point. +// +// If a feature could not be named, the name is empty. +message Feature { + // The name of the feature. + string name = 1; + + // The point where the feature is detected. + Point location = 2; +} + +// A RouteNote is a message sent while at a given point. +message RouteNote { + // The location from which the message is sent. + Point location = 1; + + // The message to be sent. + string message = 2; +} + +// A RouteSummary is received in response to a RecordRoute rpc. +// +// It contains the number of individual points received, the number of +// detected features, and the total distance covered as the cumulative sum of +// the distance between each point. +message RouteSummary { + // The number of points received. + int32 point_count = 1; + + // The number of known features passed while traversing the route. + int32 feature_count = 2; + + // The distance covered in metres. + int32 distance = 3; + + // The duration of the traversal in seconds. + int32 elapsed_time = 4; +} diff --git a/examples/route_guide/routeguide/route_guide_grpc.pb.go b/examples/route_guide/routeguide/route_guide_grpc.pb.go new file mode 100644 index 0000000..4a43301 --- /dev/null +++ b/examples/route_guide/routeguide/route_guide_grpc.pb.go @@ -0,0 +1,284 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: examples/route_guide/routeguide/route_guide.proto + +package routeguide + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + RouteGuide_GetFeature_FullMethodName = "/routeguide.RouteGuide/GetFeature" + RouteGuide_ListFeatures_FullMethodName = "/routeguide.RouteGuide/ListFeatures" + RouteGuide_RecordRoute_FullMethodName = "/routeguide.RouteGuide/RecordRoute" + RouteGuide_RouteChat_FullMethodName = "/routeguide.RouteGuide/RouteChat" +) + +// RouteGuideClient is the client API for RouteGuide service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Interface exported by the server. +type RouteGuideClient interface { + // A simple RPC. + // + // Obtains the feature at a given position. + // + // A feature with an empty name is returned if there's no feature at the given + // position. + GetFeature(ctx context.Context, in *Point, opts ...grpc.CallOption) (*Feature, error) + // A server-to-client streaming RPC. + // + // Obtains the Features available within the given Rectangle. Results are + // streamed rather than returned at once (e.g. in a response message with a + // repeated field), as the rectangle may cover a large area and contain a + // huge number of features. + ListFeatures(ctx context.Context, in *Rectangle, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Feature], error) + // A client-to-server streaming RPC. + // + // Accepts a stream of Points on a route being traversed, returning a + // RouteSummary when traversal is completed. + RecordRoute(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[Point, RouteSummary], error) + // A Bidirectional streaming RPC. + // + // Accepts a stream of RouteNotes sent while a route is being traversed, + // while receiving other RouteNotes (e.g. from other users). + RouteChat(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[RouteNote, RouteNote], error) +} + +type routeGuideClient struct { + cc grpc.ClientConnInterface +} + +func NewRouteGuideClient(cc grpc.ClientConnInterface) RouteGuideClient { + return &routeGuideClient{cc} +} + +func (c *routeGuideClient) GetFeature(ctx context.Context, in *Point, opts ...grpc.CallOption) (*Feature, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Feature) + err := c.cc.Invoke(ctx, RouteGuide_GetFeature_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *routeGuideClient) ListFeatures(ctx context.Context, in *Rectangle, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Feature], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &RouteGuide_ServiceDesc.Streams[0], RouteGuide_ListFeatures_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[Rectangle, Feature]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type RouteGuide_ListFeaturesClient = grpc.ServerStreamingClient[Feature] + +func (c *routeGuideClient) RecordRoute(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[Point, RouteSummary], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &RouteGuide_ServiceDesc.Streams[1], RouteGuide_RecordRoute_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[Point, RouteSummary]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type RouteGuide_RecordRouteClient = grpc.ClientStreamingClient[Point, RouteSummary] + +func (c *routeGuideClient) RouteChat(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[RouteNote, RouteNote], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &RouteGuide_ServiceDesc.Streams[2], RouteGuide_RouteChat_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[RouteNote, RouteNote]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type RouteGuide_RouteChatClient = grpc.BidiStreamingClient[RouteNote, RouteNote] + +// RouteGuideServer is the server API for RouteGuide service. +// All implementations must embed UnimplementedRouteGuideServer +// for forward compatibility. +// +// Interface exported by the server. +type RouteGuideServer interface { + // A simple RPC. + // + // Obtains the feature at a given position. + // + // A feature with an empty name is returned if there's no feature at the given + // position. + GetFeature(context.Context, *Point) (*Feature, error) + // A server-to-client streaming RPC. + // + // Obtains the Features available within the given Rectangle. Results are + // streamed rather than returned at once (e.g. in a response message with a + // repeated field), as the rectangle may cover a large area and contain a + // huge number of features. + ListFeatures(*Rectangle, grpc.ServerStreamingServer[Feature]) error + // A client-to-server streaming RPC. + // + // Accepts a stream of Points on a route being traversed, returning a + // RouteSummary when traversal is completed. + RecordRoute(grpc.ClientStreamingServer[Point, RouteSummary]) error + // A Bidirectional streaming RPC. + // + // Accepts a stream of RouteNotes sent while a route is being traversed, + // while receiving other RouteNotes (e.g. from other users). + RouteChat(grpc.BidiStreamingServer[RouteNote, RouteNote]) error + mustEmbedUnimplementedRouteGuideServer() +} + +// UnimplementedRouteGuideServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedRouteGuideServer struct{} + +func (UnimplementedRouteGuideServer) GetFeature(context.Context, *Point) (*Feature, error) { + return nil, status.Error(codes.Unimplemented, "method GetFeature not implemented") +} +func (UnimplementedRouteGuideServer) ListFeatures(*Rectangle, grpc.ServerStreamingServer[Feature]) error { + return status.Error(codes.Unimplemented, "method ListFeatures not implemented") +} +func (UnimplementedRouteGuideServer) RecordRoute(grpc.ClientStreamingServer[Point, RouteSummary]) error { + return status.Error(codes.Unimplemented, "method RecordRoute not implemented") +} +func (UnimplementedRouteGuideServer) RouteChat(grpc.BidiStreamingServer[RouteNote, RouteNote]) error { + return status.Error(codes.Unimplemented, "method RouteChat not implemented") +} +func (UnimplementedRouteGuideServer) mustEmbedUnimplementedRouteGuideServer() {} +func (UnimplementedRouteGuideServer) testEmbeddedByValue() {} + +// UnsafeRouteGuideServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to RouteGuideServer will +// result in compilation errors. +type UnsafeRouteGuideServer interface { + mustEmbedUnimplementedRouteGuideServer() +} + +func RegisterRouteGuideServer(s grpc.ServiceRegistrar, srv RouteGuideServer) { + // If the following call panics, it indicates UnimplementedRouteGuideServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&RouteGuide_ServiceDesc, srv) +} + +func _RouteGuide_GetFeature_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Point) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(RouteGuideServer).GetFeature(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: RouteGuide_GetFeature_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(RouteGuideServer).GetFeature(ctx, req.(*Point)) + } + return interceptor(ctx, in, info, handler) +} + +func _RouteGuide_ListFeatures_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(Rectangle) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(RouteGuideServer).ListFeatures(m, &grpc.GenericServerStream[Rectangle, Feature]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type RouteGuide_ListFeaturesServer = grpc.ServerStreamingServer[Feature] + +func _RouteGuide_RecordRoute_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(RouteGuideServer).RecordRoute(&grpc.GenericServerStream[Point, RouteSummary]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type RouteGuide_RecordRouteServer = grpc.ClientStreamingServer[Point, RouteSummary] + +func _RouteGuide_RouteChat_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(RouteGuideServer).RouteChat(&grpc.GenericServerStream[RouteNote, RouteNote]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type RouteGuide_RouteChatServer = grpc.BidiStreamingServer[RouteNote, RouteNote] + +// RouteGuide_ServiceDesc is the grpc.ServiceDesc for RouteGuide service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var RouteGuide_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "routeguide.RouteGuide", + HandlerType: (*RouteGuideServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetFeature", + Handler: _RouteGuide_GetFeature_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "ListFeatures", + Handler: _RouteGuide_ListFeatures_Handler, + ServerStreams: true, + }, + { + StreamName: "RecordRoute", + Handler: _RouteGuide_RecordRoute_Handler, + ClientStreams: true, + }, + { + StreamName: "RouteChat", + Handler: _RouteGuide_RouteChat_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "examples/route_guide/routeguide/route_guide.proto", +} diff --git a/examples/route_guide/server/server.go b/examples/route_guide/server/server.go new file mode 100644 index 0000000..7680f54 --- /dev/null +++ b/examples/route_guide/server/server.go @@ -0,0 +1,845 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package main implements a simple gRPC server that demonstrates how to use gRPC-Go libraries +// to perform unary, client streaming, server streaming and full duplex RPCs. +// +// It implements the route guide service whose definition can be found in routeguide/route_guide.proto. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "math" + "net" + "os" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/examples/data" + "google.golang.org/protobuf/proto" + + pb "google.golang.org/grpc/examples/route_guide/routeguide" +) + +var ( + tls = flag.Bool("tls", false, "Connection uses TLS if true, else plain TCP") + certFile = flag.String("cert_file", "", "The TLS cert file") + keyFile = flag.String("key_file", "", "The TLS key file") + jsonDBFile = flag.String("json_db_file", "", "A json file containing a list of features") + port = flag.Int("port", 50051, "The server port") +) + +type routeGuideServer struct { + pb.UnimplementedRouteGuideServer + savedFeatures []*pb.Feature // read-only after initialized + + mu sync.Mutex // protects routeNotes + routeNotes map[string][]*pb.RouteNote +} + +// GetFeature returns the feature at the given point. +func (s *routeGuideServer) GetFeature(_ context.Context, point *pb.Point) (*pb.Feature, error) { + for _, feature := range s.savedFeatures { + if proto.Equal(feature.Location, point) { + return feature, nil + } + } + // No feature was found, return an unnamed feature + return &pb.Feature{Location: point}, nil +} + +// ListFeatures lists all features contained within the given bounding Rectangle. +func (s *routeGuideServer) ListFeatures(rect *pb.Rectangle, stream pb.RouteGuide_ListFeaturesServer) error { + for _, feature := range s.savedFeatures { + if inRange(feature.Location, rect) { + if err := stream.Send(feature); err != nil { + return err + } + } + } + return nil +} + +// RecordRoute records a route composited of a sequence of points. +// +// It gets a stream of points, and responds with statistics about the "trip": +// number of points, number of known features visited, total distance traveled, and +// total time spent. +func (s *routeGuideServer) RecordRoute(stream pb.RouteGuide_RecordRouteServer) error { + var pointCount, featureCount, distance int32 + var lastPoint *pb.Point + startTime := time.Now() + for { + point, err := stream.Recv() + if err == io.EOF { + endTime := time.Now() + return stream.SendAndClose(&pb.RouteSummary{ + PointCount: pointCount, + FeatureCount: featureCount, + Distance: distance, + ElapsedTime: int32(endTime.Sub(startTime).Seconds()), + }) + } + if err != nil { + return err + } + pointCount++ + for _, feature := range s.savedFeatures { + if proto.Equal(feature.Location, point) { + featureCount++ + } + } + if lastPoint != nil { + distance += calcDistance(lastPoint, point) + } + lastPoint = point + } +} + +// RouteChat receives a stream of message/location pairs, and responds with a stream of all +// previous messages at each of those locations. +func (s *routeGuideServer) RouteChat(stream pb.RouteGuide_RouteChatServer) error { + for { + in, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + key := serialize(in.Location) + + s.mu.Lock() + s.routeNotes[key] = append(s.routeNotes[key], in) + // Note: this copy prevents blocking other clients while serving this one. + // We don't need to do a deep copy, because elements in the slice are + // insert-only and never modified. + rn := make([]*pb.RouteNote, len(s.routeNotes[key])) + copy(rn, s.routeNotes[key]) + s.mu.Unlock() + + for _, note := range rn { + if err := stream.Send(note); err != nil { + return err + } + } + } +} + +// loadFeatures loads features from a JSON file. +func (s *routeGuideServer) loadFeatures(filePath string) { + var data []byte + if filePath != "" { + var err error + data, err = os.ReadFile(filePath) + if err != nil { + log.Fatalf("Failed to load default features: %v", err) + } + } else { + data = exampleData + } + if err := json.Unmarshal(data, &s.savedFeatures); err != nil { + log.Fatalf("Failed to load default features: %v", err) + } +} + +func toRadians(num float64) float64 { + return num * math.Pi / float64(180) +} + +// calcDistance calculates the distance between two points using the "haversine" formula. +// The formula is based on http://mathforum.org/library/drmath/view/51879.html. +func calcDistance(p1 *pb.Point, p2 *pb.Point) int32 { + const CordFactor float64 = 1e7 + const R = float64(6371000) // earth radius in metres + lat1 := toRadians(float64(p1.Latitude) / CordFactor) + lat2 := toRadians(float64(p2.Latitude) / CordFactor) + lng1 := toRadians(float64(p1.Longitude) / CordFactor) + lng2 := toRadians(float64(p2.Longitude) / CordFactor) + dlat := lat2 - lat1 + dlng := lng2 - lng1 + + a := math.Sin(dlat/2)*math.Sin(dlat/2) + + math.Cos(lat1)*math.Cos(lat2)* + math.Sin(dlng/2)*math.Sin(dlng/2) + c := 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a)) + + distance := R * c + return int32(distance) +} + +func inRange(point *pb.Point, rect *pb.Rectangle) bool { + left := math.Min(float64(rect.Lo.Longitude), float64(rect.Hi.Longitude)) + right := math.Max(float64(rect.Lo.Longitude), float64(rect.Hi.Longitude)) + top := math.Max(float64(rect.Lo.Latitude), float64(rect.Hi.Latitude)) + bottom := math.Min(float64(rect.Lo.Latitude), float64(rect.Hi.Latitude)) + + if float64(point.Longitude) >= left && + float64(point.Longitude) <= right && + float64(point.Latitude) >= bottom && + float64(point.Latitude) <= top { + return true + } + return false +} + +func serialize(point *pb.Point) string { + return fmt.Sprintf("%d %d", point.Latitude, point.Longitude) +} + +func newServer() *routeGuideServer { + s := &routeGuideServer{routeNotes: make(map[string][]*pb.RouteNote)} + s.loadFeatures(*jsonDBFile) + return s +} + +func main() { + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + var opts []grpc.ServerOption + if *tls { + if *certFile == "" { + *certFile = data.Path("x509/server_cert.pem") + } + if *keyFile == "" { + *keyFile = data.Path("x509/server_key.pem") + } + creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile) + if err != nil { + log.Fatalf("Failed to generate credentials: %v", err) + } + opts = []grpc.ServerOption{grpc.Creds(creds)} + } + grpcServer := grpc.NewServer(opts...) + pb.RegisterRouteGuideServer(grpcServer, newServer()) + grpcServer.Serve(lis) +} + +// exampleData is a copy of testdata/route_guide_db.json. It's to avoid +// specifying file path with `go run`. +var exampleData = []byte(`[{ + "location": { + "latitude": 407838351, + "longitude": -746143763 + }, + "name": "Patriots Path, Mendham, NJ 07945, USA" +}, { + "location": { + "latitude": 408122808, + "longitude": -743999179 + }, + "name": "101 New Jersey 10, Whippany, NJ 07981, USA" +}, { + "location": { + "latitude": 413628156, + "longitude": -749015468 + }, + "name": "U.S. 6, Shohola, PA 18458, USA" +}, { + "location": { + "latitude": 419999544, + "longitude": -740371136 + }, + "name": "5 Conners Road, Kingston, NY 12401, USA" +}, { + "location": { + "latitude": 414008389, + "longitude": -743951297 + }, + "name": "Mid Hudson Psychiatric Center, New Hampton, NY 10958, USA" +}, { + "location": { + "latitude": 419611318, + "longitude": -746524769 + }, + "name": "287 Flugertown Road, Livingston Manor, NY 12758, USA" +}, { + "location": { + "latitude": 406109563, + "longitude": -742186778 + }, + "name": "4001 Tremley Point Road, Linden, NJ 07036, USA" +}, { + "location": { + "latitude": 416802456, + "longitude": -742370183 + }, + "name": "352 South Mountain Road, Wallkill, NY 12589, USA" +}, { + "location": { + "latitude": 412950425, + "longitude": -741077389 + }, + "name": "Bailey Turn Road, Harriman, NY 10926, USA" +}, { + "location": { + "latitude": 412144655, + "longitude": -743949739 + }, + "name": "193-199 Wawayanda Road, Hewitt, NJ 07421, USA" +}, { + "location": { + "latitude": 415736605, + "longitude": -742847522 + }, + "name": "406-496 Ward Avenue, Pine Bush, NY 12566, USA" +}, { + "location": { + "latitude": 413843930, + "longitude": -740501726 + }, + "name": "162 Merrill Road, Highland Mills, NY 10930, USA" +}, { + "location": { + "latitude": 410873075, + "longitude": -744459023 + }, + "name": "Clinton Road, West Milford, NJ 07480, USA" +}, { + "location": { + "latitude": 412346009, + "longitude": -744026814 + }, + "name": "16 Old Brook Lane, Warwick, NY 10990, USA" +}, { + "location": { + "latitude": 402948455, + "longitude": -747903913 + }, + "name": "3 Drake Lane, Pennington, NJ 08534, USA" +}, { + "location": { + "latitude": 406337092, + "longitude": -740122226 + }, + "name": "6324 8th Avenue, Brooklyn, NY 11220, USA" +}, { + "location": { + "latitude": 406421967, + "longitude": -747727624 + }, + "name": "1 Merck Access Road, Whitehouse Station, NJ 08889, USA" +}, { + "location": { + "latitude": 416318082, + "longitude": -749677716 + }, + "name": "78-98 Schalck Road, Narrowsburg, NY 12764, USA" +}, { + "location": { + "latitude": 415301720, + "longitude": -748416257 + }, + "name": "282 Lakeview Drive Road, Highland Lake, NY 12743, USA" +}, { + "location": { + "latitude": 402647019, + "longitude": -747071791 + }, + "name": "330 Evelyn Avenue, Hamilton Township, NJ 08619, USA" +}, { + "location": { + "latitude": 412567807, + "longitude": -741058078 + }, + "name": "New York State Reference Route 987E, Southfields, NY 10975, USA" +}, { + "location": { + "latitude": 416855156, + "longitude": -744420597 + }, + "name": "103-271 Tempaloni Road, Ellenville, NY 12428, USA" +}, { + "location": { + "latitude": 404663628, + "longitude": -744820157 + }, + "name": "1300 Airport Road, North Brunswick Township, NJ 08902, USA" +}, { + "location": { + "latitude": 407113723, + "longitude": -749746483 + }, + "name": "" +}, { + "location": { + "latitude": 402133926, + "longitude": -743613249 + }, + "name": "" +}, { + "location": { + "latitude": 400273442, + "longitude": -741220915 + }, + "name": "" +}, { + "location": { + "latitude": 411236786, + "longitude": -744070769 + }, + "name": "" +}, { + "location": { + "latitude": 411633782, + "longitude": -746784970 + }, + "name": "211-225 Plains Road, Augusta, NJ 07822, USA" +}, { + "location": { + "latitude": 415830701, + "longitude": -742952812 + }, + "name": "" +}, { + "location": { + "latitude": 413447164, + "longitude": -748712898 + }, + "name": "165 Pedersen Ridge Road, Milford, PA 18337, USA" +}, { + "location": { + "latitude": 405047245, + "longitude": -749800722 + }, + "name": "100-122 Locktown Road, Frenchtown, NJ 08825, USA" +}, { + "location": { + "latitude": 418858923, + "longitude": -746156790 + }, + "name": "" +}, { + "location": { + "latitude": 417951888, + "longitude": -748484944 + }, + "name": "650-652 Willi Hill Road, Swan Lake, NY 12783, USA" +}, { + "location": { + "latitude": 407033786, + "longitude": -743977337 + }, + "name": "26 East 3rd Street, New Providence, NJ 07974, USA" +}, { + "location": { + "latitude": 417548014, + "longitude": -740075041 + }, + "name": "" +}, { + "location": { + "latitude": 410395868, + "longitude": -744972325 + }, + "name": "" +}, { + "location": { + "latitude": 404615353, + "longitude": -745129803 + }, + "name": "" +}, { + "location": { + "latitude": 406589790, + "longitude": -743560121 + }, + "name": "611 Lawrence Avenue, Westfield, NJ 07090, USA" +}, { + "location": { + "latitude": 414653148, + "longitude": -740477477 + }, + "name": "18 Lannis Avenue, New Windsor, NY 12553, USA" +}, { + "location": { + "latitude": 405957808, + "longitude": -743255336 + }, + "name": "82-104 Amherst Avenue, Colonia, NJ 07067, USA" +}, { + "location": { + "latitude": 411733589, + "longitude": -741648093 + }, + "name": "170 Seven Lakes Drive, Sloatsburg, NY 10974, USA" +}, { + "location": { + "latitude": 412676291, + "longitude": -742606606 + }, + "name": "1270 Lakes Road, Monroe, NY 10950, USA" +}, { + "location": { + "latitude": 409224445, + "longitude": -748286738 + }, + "name": "509-535 Alphano Road, Great Meadows, NJ 07838, USA" +}, { + "location": { + "latitude": 406523420, + "longitude": -742135517 + }, + "name": "652 Garden Street, Elizabeth, NJ 07202, USA" +}, { + "location": { + "latitude": 401827388, + "longitude": -740294537 + }, + "name": "349 Sea Spray Court, Neptune City, NJ 07753, USA" +}, { + "location": { + "latitude": 410564152, + "longitude": -743685054 + }, + "name": "13-17 Stanley Street, West Milford, NJ 07480, USA" +}, { + "location": { + "latitude": 408472324, + "longitude": -740726046 + }, + "name": "47 Industrial Avenue, Teterboro, NJ 07608, USA" +}, { + "location": { + "latitude": 412452168, + "longitude": -740214052 + }, + "name": "5 White Oak Lane, Stony Point, NY 10980, USA" +}, { + "location": { + "latitude": 409146138, + "longitude": -746188906 + }, + "name": "Berkshire Valley Management Area Trail, Jefferson, NJ, USA" +}, { + "location": { + "latitude": 404701380, + "longitude": -744781745 + }, + "name": "1007 Jersey Avenue, New Brunswick, NJ 08901, USA" +}, { + "location": { + "latitude": 409642566, + "longitude": -746017679 + }, + "name": "6 East Emerald Isle Drive, Lake Hopatcong, NJ 07849, USA" +}, { + "location": { + "latitude": 408031728, + "longitude": -748645385 + }, + "name": "1358-1474 New Jersey 57, Port Murray, NJ 07865, USA" +}, { + "location": { + "latitude": 413700272, + "longitude": -742135189 + }, + "name": "367 Prospect Road, Chester, NY 10918, USA" +}, { + "location": { + "latitude": 404310607, + "longitude": -740282632 + }, + "name": "10 Simon Lake Drive, Atlantic Highlands, NJ 07716, USA" +}, { + "location": { + "latitude": 409319800, + "longitude": -746201391 + }, + "name": "11 Ward Street, Mount Arlington, NJ 07856, USA" +}, { + "location": { + "latitude": 406685311, + "longitude": -742108603 + }, + "name": "300-398 Jefferson Avenue, Elizabeth, NJ 07201, USA" +}, { + "location": { + "latitude": 419018117, + "longitude": -749142781 + }, + "name": "43 Dreher Road, Roscoe, NY 12776, USA" +}, { + "location": { + "latitude": 412856162, + "longitude": -745148837 + }, + "name": "Swan Street, Pine Island, NY 10969, USA" +}, { + "location": { + "latitude": 416560744, + "longitude": -746721964 + }, + "name": "66 Pleasantview Avenue, Monticello, NY 12701, USA" +}, { + "location": { + "latitude": 405314270, + "longitude": -749836354 + }, + "name": "" +}, { + "location": { + "latitude": 414219548, + "longitude": -743327440 + }, + "name": "" +}, { + "location": { + "latitude": 415534177, + "longitude": -742900616 + }, + "name": "565 Winding Hills Road, Montgomery, NY 12549, USA" +}, { + "location": { + "latitude": 406898530, + "longitude": -749127080 + }, + "name": "231 Rocky Run Road, Glen Gardner, NJ 08826, USA" +}, { + "location": { + "latitude": 407586880, + "longitude": -741670168 + }, + "name": "100 Mount Pleasant Avenue, Newark, NJ 07104, USA" +}, { + "location": { + "latitude": 400106455, + "longitude": -742870190 + }, + "name": "517-521 Huntington Drive, Manchester Township, NJ 08759, USA" +}, { + "location": { + "latitude": 400066188, + "longitude": -746793294 + }, + "name": "" +}, { + "location": { + "latitude": 418803880, + "longitude": -744102673 + }, + "name": "40 Mountain Road, Napanoch, NY 12458, USA" +}, { + "location": { + "latitude": 414204288, + "longitude": -747895140 + }, + "name": "" +}, { + "location": { + "latitude": 414777405, + "longitude": -740615601 + }, + "name": "" +}, { + "location": { + "latitude": 415464475, + "longitude": -747175374 + }, + "name": "48 North Road, Forestburgh, NY 12777, USA" +}, { + "location": { + "latitude": 404062378, + "longitude": -746376177 + }, + "name": "" +}, { + "location": { + "latitude": 405688272, + "longitude": -749285130 + }, + "name": "" +}, { + "location": { + "latitude": 400342070, + "longitude": -748788996 + }, + "name": "" +}, { + "location": { + "latitude": 401809022, + "longitude": -744157964 + }, + "name": "" +}, { + "location": { + "latitude": 404226644, + "longitude": -740517141 + }, + "name": "9 Thompson Avenue, Leonardo, NJ 07737, USA" +}, { + "location": { + "latitude": 410322033, + "longitude": -747871659 + }, + "name": "" +}, { + "location": { + "latitude": 407100674, + "longitude": -747742727 + }, + "name": "" +}, { + "location": { + "latitude": 418811433, + "longitude": -741718005 + }, + "name": "213 Bush Road, Stone Ridge, NY 12484, USA" +}, { + "location": { + "latitude": 415034302, + "longitude": -743850945 + }, + "name": "" +}, { + "location": { + "latitude": 411349992, + "longitude": -743694161 + }, + "name": "" +}, { + "location": { + "latitude": 404839914, + "longitude": -744759616 + }, + "name": "1-17 Bergen Court, New Brunswick, NJ 08901, USA" +}, { + "location": { + "latitude": 414638017, + "longitude": -745957854 + }, + "name": "35 Oakland Valley Road, Cuddebackville, NY 12729, USA" +}, { + "location": { + "latitude": 412127800, + "longitude": -740173578 + }, + "name": "" +}, { + "location": { + "latitude": 401263460, + "longitude": -747964303 + }, + "name": "" +}, { + "location": { + "latitude": 412843391, + "longitude": -749086026 + }, + "name": "" +}, { + "location": { + "latitude": 418512773, + "longitude": -743067823 + }, + "name": "" +}, { + "location": { + "latitude": 404318328, + "longitude": -740835638 + }, + "name": "42-102 Main Street, Belford, NJ 07718, USA" +}, { + "location": { + "latitude": 419020746, + "longitude": -741172328 + }, + "name": "" +}, { + "location": { + "latitude": 404080723, + "longitude": -746119569 + }, + "name": "" +}, { + "location": { + "latitude": 401012643, + "longitude": -744035134 + }, + "name": "" +}, { + "location": { + "latitude": 404306372, + "longitude": -741079661 + }, + "name": "" +}, { + "location": { + "latitude": 403966326, + "longitude": -748519297 + }, + "name": "" +}, { + "location": { + "latitude": 405002031, + "longitude": -748407866 + }, + "name": "" +}, { + "location": { + "latitude": 409532885, + "longitude": -742200683 + }, + "name": "" +}, { + "location": { + "latitude": 416851321, + "longitude": -742674555 + }, + "name": "" +}, { + "location": { + "latitude": 406411633, + "longitude": -741722051 + }, + "name": "3387 Richmond Terrace, Staten Island, NY 10303, USA" +}, { + "location": { + "latitude": 413069058, + "longitude": -744597778 + }, + "name": "261 Van Sickle Road, Goshen, NY 10924, USA" +}, { + "location": { + "latitude": 418465462, + "longitude": -746859398 + }, + "name": "" +}, { + "location": { + "latitude": 411733222, + "longitude": -744228360 + }, + "name": "" +}, { + "location": { + "latitude": 410248224, + "longitude": -747127767 + }, + "name": "3 Hasta Way, Newton, NJ 07860, USA" +}]`) diff --git a/examples/route_guide/testdata/route_guide_db.json b/examples/route_guide/testdata/route_guide_db.json new file mode 100644 index 0000000..9d6a980 --- /dev/null +++ b/examples/route_guide/testdata/route_guide_db.json @@ -0,0 +1,601 @@ +[{ + "location": { + "latitude": 407838351, + "longitude": -746143763 + }, + "name": "Patriots Path, Mendham, NJ 07945, USA" +}, { + "location": { + "latitude": 408122808, + "longitude": -743999179 + }, + "name": "101 New Jersey 10, Whippany, NJ 07981, USA" +}, { + "location": { + "latitude": 413628156, + "longitude": -749015468 + }, + "name": "U.S. 6, Shohola, PA 18458, USA" +}, { + "location": { + "latitude": 419999544, + "longitude": -740371136 + }, + "name": "5 Conners Road, Kingston, NY 12401, USA" +}, { + "location": { + "latitude": 414008389, + "longitude": -743951297 + }, + "name": "Mid Hudson Psychiatric Center, New Hampton, NY 10958, USA" +}, { + "location": { + "latitude": 419611318, + "longitude": -746524769 + }, + "name": "287 Flugertown Road, Livingston Manor, NY 12758, USA" +}, { + "location": { + "latitude": 406109563, + "longitude": -742186778 + }, + "name": "4001 Tremley Point Road, Linden, NJ 07036, USA" +}, { + "location": { + "latitude": 416802456, + "longitude": -742370183 + }, + "name": "352 South Mountain Road, Wallkill, NY 12589, USA" +}, { + "location": { + "latitude": 412950425, + "longitude": -741077389 + }, + "name": "Bailey Turn Road, Harriman, NY 10926, USA" +}, { + "location": { + "latitude": 412144655, + "longitude": -743949739 + }, + "name": "193-199 Wawayanda Road, Hewitt, NJ 07421, USA" +}, { + "location": { + "latitude": 415736605, + "longitude": -742847522 + }, + "name": "406-496 Ward Avenue, Pine Bush, NY 12566, USA" +}, { + "location": { + "latitude": 413843930, + "longitude": -740501726 + }, + "name": "162 Merrill Road, Highland Mills, NY 10930, USA" +}, { + "location": { + "latitude": 410873075, + "longitude": -744459023 + }, + "name": "Clinton Road, West Milford, NJ 07480, USA" +}, { + "location": { + "latitude": 412346009, + "longitude": -744026814 + }, + "name": "16 Old Brook Lane, Warwick, NY 10990, USA" +}, { + "location": { + "latitude": 402948455, + "longitude": -747903913 + }, + "name": "3 Drake Lane, Pennington, NJ 08534, USA" +}, { + "location": { + "latitude": 406337092, + "longitude": -740122226 + }, + "name": "6324 8th Avenue, Brooklyn, NY 11220, USA" +}, { + "location": { + "latitude": 406421967, + "longitude": -747727624 + }, + "name": "1 Merck Access Road, Whitehouse Station, NJ 08889, USA" +}, { + "location": { + "latitude": 416318082, + "longitude": -749677716 + }, + "name": "78-98 Schalck Road, Narrowsburg, NY 12764, USA" +}, { + "location": { + "latitude": 415301720, + "longitude": -748416257 + }, + "name": "282 Lakeview Drive Road, Highland Lake, NY 12743, USA" +}, { + "location": { + "latitude": 402647019, + "longitude": -747071791 + }, + "name": "330 Evelyn Avenue, Hamilton Township, NJ 08619, USA" +}, { + "location": { + "latitude": 412567807, + "longitude": -741058078 + }, + "name": "New York State Reference Route 987E, Southfields, NY 10975, USA" +}, { + "location": { + "latitude": 416855156, + "longitude": -744420597 + }, + "name": "103-271 Tempaloni Road, Ellenville, NY 12428, USA" +}, { + "location": { + "latitude": 404663628, + "longitude": -744820157 + }, + "name": "1300 Airport Road, North Brunswick Township, NJ 08902, USA" +}, { + "location": { + "latitude": 407113723, + "longitude": -749746483 + }, + "name": "" +}, { + "location": { + "latitude": 402133926, + "longitude": -743613249 + }, + "name": "" +}, { + "location": { + "latitude": 400273442, + "longitude": -741220915 + }, + "name": "" +}, { + "location": { + "latitude": 411236786, + "longitude": -744070769 + }, + "name": "" +}, { + "location": { + "latitude": 411633782, + "longitude": -746784970 + }, + "name": "211-225 Plains Road, Augusta, NJ 07822, USA" +}, { + "location": { + "latitude": 415830701, + "longitude": -742952812 + }, + "name": "" +}, { + "location": { + "latitude": 413447164, + "longitude": -748712898 + }, + "name": "165 Pedersen Ridge Road, Milford, PA 18337, USA" +}, { + "location": { + "latitude": 405047245, + "longitude": -749800722 + }, + "name": "100-122 Locktown Road, Frenchtown, NJ 08825, USA" +}, { + "location": { + "latitude": 418858923, + "longitude": -746156790 + }, + "name": "" +}, { + "location": { + "latitude": 417951888, + "longitude": -748484944 + }, + "name": "650-652 Willi Hill Road, Swan Lake, NY 12783, USA" +}, { + "location": { + "latitude": 407033786, + "longitude": -743977337 + }, + "name": "26 East 3rd Street, New Providence, NJ 07974, USA" +}, { + "location": { + "latitude": 417548014, + "longitude": -740075041 + }, + "name": "" +}, { + "location": { + "latitude": 410395868, + "longitude": -744972325 + }, + "name": "" +}, { + "location": { + "latitude": 404615353, + "longitude": -745129803 + }, + "name": "" +}, { + "location": { + "latitude": 406589790, + "longitude": -743560121 + }, + "name": "611 Lawrence Avenue, Westfield, NJ 07090, USA" +}, { + "location": { + "latitude": 414653148, + "longitude": -740477477 + }, + "name": "18 Lannis Avenue, New Windsor, NY 12553, USA" +}, { + "location": { + "latitude": 405957808, + "longitude": -743255336 + }, + "name": "82-104 Amherst Avenue, Colonia, NJ 07067, USA" +}, { + "location": { + "latitude": 411733589, + "longitude": -741648093 + }, + "name": "170 Seven Lakes Drive, Sloatsburg, NY 10974, USA" +}, { + "location": { + "latitude": 412676291, + "longitude": -742606606 + }, + "name": "1270 Lakes Road, Monroe, NY 10950, USA" +}, { + "location": { + "latitude": 409224445, + "longitude": -748286738 + }, + "name": "509-535 Alphano Road, Great Meadows, NJ 07838, USA" +}, { + "location": { + "latitude": 406523420, + "longitude": -742135517 + }, + "name": "652 Garden Street, Elizabeth, NJ 07202, USA" +}, { + "location": { + "latitude": 401827388, + "longitude": -740294537 + }, + "name": "349 Sea Spray Court, Neptune City, NJ 07753, USA" +}, { + "location": { + "latitude": 410564152, + "longitude": -743685054 + }, + "name": "13-17 Stanley Street, West Milford, NJ 07480, USA" +}, { + "location": { + "latitude": 408472324, + "longitude": -740726046 + }, + "name": "47 Industrial Avenue, Teterboro, NJ 07608, USA" +}, { + "location": { + "latitude": 412452168, + "longitude": -740214052 + }, + "name": "5 White Oak Lane, Stony Point, NY 10980, USA" +}, { + "location": { + "latitude": 409146138, + "longitude": -746188906 + }, + "name": "Berkshire Valley Management Area Trail, Jefferson, NJ, USA" +}, { + "location": { + "latitude": 404701380, + "longitude": -744781745 + }, + "name": "1007 Jersey Avenue, New Brunswick, NJ 08901, USA" +}, { + "location": { + "latitude": 409642566, + "longitude": -746017679 + }, + "name": "6 East Emerald Isle Drive, Lake Hopatcong, NJ 07849, USA" +}, { + "location": { + "latitude": 408031728, + "longitude": -748645385 + }, + "name": "1358-1474 New Jersey 57, Port Murray, NJ 07865, USA" +}, { + "location": { + "latitude": 413700272, + "longitude": -742135189 + }, + "name": "367 Prospect Road, Chester, NY 10918, USA" +}, { + "location": { + "latitude": 404310607, + "longitude": -740282632 + }, + "name": "10 Simon Lake Drive, Atlantic Highlands, NJ 07716, USA" +}, { + "location": { + "latitude": 409319800, + "longitude": -746201391 + }, + "name": "11 Ward Street, Mount Arlington, NJ 07856, USA" +}, { + "location": { + "latitude": 406685311, + "longitude": -742108603 + }, + "name": "300-398 Jefferson Avenue, Elizabeth, NJ 07201, USA" +}, { + "location": { + "latitude": 419018117, + "longitude": -749142781 + }, + "name": "43 Dreher Road, Roscoe, NY 12776, USA" +}, { + "location": { + "latitude": 412856162, + "longitude": -745148837 + }, + "name": "Swan Street, Pine Island, NY 10969, USA" +}, { + "location": { + "latitude": 416560744, + "longitude": -746721964 + }, + "name": "66 Pleasantview Avenue, Monticello, NY 12701, USA" +}, { + "location": { + "latitude": 405314270, + "longitude": -749836354 + }, + "name": "" +}, { + "location": { + "latitude": 414219548, + "longitude": -743327440 + }, + "name": "" +}, { + "location": { + "latitude": 415534177, + "longitude": -742900616 + }, + "name": "565 Winding Hills Road, Montgomery, NY 12549, USA" +}, { + "location": { + "latitude": 406898530, + "longitude": -749127080 + }, + "name": "231 Rocky Run Road, Glen Gardner, NJ 08826, USA" +}, { + "location": { + "latitude": 407586880, + "longitude": -741670168 + }, + "name": "100 Mount Pleasant Avenue, Newark, NJ 07104, USA" +}, { + "location": { + "latitude": 400106455, + "longitude": -742870190 + }, + "name": "517-521 Huntington Drive, Manchester Township, NJ 08759, USA" +}, { + "location": { + "latitude": 400066188, + "longitude": -746793294 + }, + "name": "" +}, { + "location": { + "latitude": 418803880, + "longitude": -744102673 + }, + "name": "40 Mountain Road, Napanoch, NY 12458, USA" +}, { + "location": { + "latitude": 414204288, + "longitude": -747895140 + }, + "name": "" +}, { + "location": { + "latitude": 414777405, + "longitude": -740615601 + }, + "name": "" +}, { + "location": { + "latitude": 415464475, + "longitude": -747175374 + }, + "name": "48 North Road, Forestburgh, NY 12777, USA" +}, { + "location": { + "latitude": 404062378, + "longitude": -746376177 + }, + "name": "" +}, { + "location": { + "latitude": 405688272, + "longitude": -749285130 + }, + "name": "" +}, { + "location": { + "latitude": 400342070, + "longitude": -748788996 + }, + "name": "" +}, { + "location": { + "latitude": 401809022, + "longitude": -744157964 + }, + "name": "" +}, { + "location": { + "latitude": 404226644, + "longitude": -740517141 + }, + "name": "9 Thompson Avenue, Leonardo, NJ 07737, USA" +}, { + "location": { + "latitude": 410322033, + "longitude": -747871659 + }, + "name": "" +}, { + "location": { + "latitude": 407100674, + "longitude": -747742727 + }, + "name": "" +}, { + "location": { + "latitude": 418811433, + "longitude": -741718005 + }, + "name": "213 Bush Road, Stone Ridge, NY 12484, USA" +}, { + "location": { + "latitude": 415034302, + "longitude": -743850945 + }, + "name": "" +}, { + "location": { + "latitude": 411349992, + "longitude": -743694161 + }, + "name": "" +}, { + "location": { + "latitude": 404839914, + "longitude": -744759616 + }, + "name": "1-17 Bergen Court, New Brunswick, NJ 08901, USA" +}, { + "location": { + "latitude": 414638017, + "longitude": -745957854 + }, + "name": "35 Oakland Valley Road, Cuddebackville, NY 12729, USA" +}, { + "location": { + "latitude": 412127800, + "longitude": -740173578 + }, + "name": "" +}, { + "location": { + "latitude": 401263460, + "longitude": -747964303 + }, + "name": "" +}, { + "location": { + "latitude": 412843391, + "longitude": -749086026 + }, + "name": "" +}, { + "location": { + "latitude": 418512773, + "longitude": -743067823 + }, + "name": "" +}, { + "location": { + "latitude": 404318328, + "longitude": -740835638 + }, + "name": "42-102 Main Street, Belford, NJ 07718, USA" +}, { + "location": { + "latitude": 419020746, + "longitude": -741172328 + }, + "name": "" +}, { + "location": { + "latitude": 404080723, + "longitude": -746119569 + }, + "name": "" +}, { + "location": { + "latitude": 401012643, + "longitude": -744035134 + }, + "name": "" +}, { + "location": { + "latitude": 404306372, + "longitude": -741079661 + }, + "name": "" +}, { + "location": { + "latitude": 403966326, + "longitude": -748519297 + }, + "name": "" +}, { + "location": { + "latitude": 405002031, + "longitude": -748407866 + }, + "name": "" +}, { + "location": { + "latitude": 409532885, + "longitude": -742200683 + }, + "name": "" +}, { + "location": { + "latitude": 416851321, + "longitude": -742674555 + }, + "name": "" +}, { + "location": { + "latitude": 406411633, + "longitude": -741722051 + }, + "name": "3387 Richmond Terrace, Staten Island, NY 10303, USA" +}, { + "location": { + "latitude": 413069058, + "longitude": -744597778 + }, + "name": "261 Van Sickle Road, Goshen, NY 10924, USA" +}, { + "location": { + "latitude": 418465462, + "longitude": -746859398 + }, + "name": "" +}, { + "location": { + "latitude": 411733222, + "longitude": -744228360 + }, + "name": "" +}, { + "location": { + "latitude": 410248224, + "longitude": -747127767 + }, + "name": "3 Hasta Way, Newton, NJ 07860, USA" +}] diff --git a/experimental/balancer/hostname/hostname.go b/experimental/balancer/hostname/hostname.go new file mode 100644 index 0000000..f42a7cf --- /dev/null +++ b/experimental/balancer/hostname/hostname.go @@ -0,0 +1,47 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package hostname contains utilities for the endpoint hostname attribute +// (used for per-endpoint :authority / SNI override). +// +// # Experimental +// +// Notice: All APIs in this package are EXPERIMENTAL and may be changed +// or removed in a later release. +package hostname + +import "google.golang.org/grpc/resolver" + +type hostnameKey struct{} + +// Set returns a copy of the given endpoint with the hostname attribute +// set. If hostname is empty the endpoint is returned unmodified. +func Set(endpoint resolver.Endpoint, hostname string) resolver.Endpoint { + if hostname == "" { + return endpoint + } + endpoint.Attributes = endpoint.Attributes.WithValue(hostnameKey{}, hostname) + return endpoint +} + +// FromEndpoint returns the hostname attribute of endpoint. If this +// attribute is not set, it returns the empty string. +func FromEndpoint(endpoint resolver.Endpoint) string { + h, _ := endpoint.Attributes.Value(hostnameKey{}).(string) + return h +} diff --git a/experimental/balancer/hostname/hostname_test.go b/experimental/balancer/hostname/hostname_test.go new file mode 100644 index 0000000..76ce81b --- /dev/null +++ b/experimental/balancer/hostname/hostname_test.go @@ -0,0 +1,75 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package hostname_test + +import ( + "testing" + + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/experimental/balancer/hostname" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/resolver" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestHostnameToAndFromEndpoint(t *testing.T) { + tests := []struct { + desc string + inputHostname string + inputAttributes *attributes.Attributes + wantHostname string + }{ + { + desc: "empty_attributes", + inputHostname: "myservice.example.com", + wantHostname: "myservice.example.com", + }, + { + desc: "non-empty_attributes", + inputHostname: "myservice.example.com", + inputAttributes: attributes.New("foo", "bar"), + wantHostname: "myservice.example.com", + }, + { + desc: "hostname_not_present_in_empty_attributes", + }, + { + desc: "hostname_not_present_in_non-empty_attributes", + inputAttributes: attributes.New("foo", "bar"), + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + endpoint := resolver.Endpoint{Attributes: test.inputAttributes} + endpoint = hostname.Set(endpoint, test.inputHostname) + gotHostname := hostname.FromEndpoint(endpoint) + if gotHostname != test.wantHostname { + t.Errorf("hostname.FromEndpoint(%+v) = %q, want %q", endpoint, gotHostname, test.wantHostname) + } + }) + } +} diff --git a/experimental/balancer/weight/weight.go b/experimental/balancer/weight/weight.go new file mode 100644 index 0000000..beab9e0 --- /dev/null +++ b/experimental/balancer/weight/weight.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package weight contains utilities to manage endpoint weights. +// Weights may be used by LB policies to distribute load across +// multiple endpoints. +// +// # Experimental +// +// Notice: All APIs in this package are EXPERIMENTAL and may be changed +// or removed in a later release. +package weight + +import "google.golang.org/grpc/resolver" + +// attributeKey is the type used as the key to store EndpointInfo in the +// Attributes field of resolver.Endpoint. +type attributeKey struct{} + +// EndpointInfo will be stored in the Attributes field of Endpoints. +type EndpointInfo struct { + Weight uint32 +} + +// Equal allows the values to be compared by Attributes.Equal. +func (a EndpointInfo) Equal(o any) bool { + oa, ok := o.(EndpointInfo) + return ok && oa.Weight == a.Weight +} + +// Set returns a copy of endpoint in which the Attributes field is +// updated with EndpointInfo. +func Set(endpoint resolver.Endpoint, epInfo EndpointInfo) resolver.Endpoint { + endpoint.Attributes = endpoint.Attributes.WithValue(attributeKey{}, epInfo) + return endpoint +} + +// FromEndpoint returns the EndpointInfo stored in the Attributes +// field of an endpoint. It returns an empty EndpointInfo if attribute +// is not found. +func FromEndpoint(endpoint resolver.Endpoint) EndpointInfo { + v := endpoint.Attributes.Value(attributeKey{}) + ei, _ := v.(EndpointInfo) + return ei +} diff --git a/experimental/balancer/weight/weight_test.go b/experimental/balancer/weight/weight_test.go new file mode 100644 index 0000000..531ea85 --- /dev/null +++ b/experimental/balancer/weight/weight_test.go @@ -0,0 +1,83 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weight_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/experimental/balancer/weight" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/resolver" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestEndpointInfoToAndFromAttributes(t *testing.T) { + tests := []struct { + desc string + inputEndpointInfo weight.EndpointInfo + inputAttributes *attributes.Attributes + wantEndpointInfo weight.EndpointInfo + }{ + { + desc: "empty_attributes", + inputEndpointInfo: weight.EndpointInfo{Weight: 100}, + inputAttributes: nil, + wantEndpointInfo: weight.EndpointInfo{Weight: 100}, + }, + { + desc: "non-empty_attributes", + inputEndpointInfo: weight.EndpointInfo{Weight: 100}, + inputAttributes: attributes.New("foo", "bar"), + wantEndpointInfo: weight.EndpointInfo{Weight: 100}, + }, + { + desc: "endpointInfo_not_present_in_empty_attributes", + inputEndpointInfo: weight.EndpointInfo{}, + inputAttributes: nil, + wantEndpointInfo: weight.EndpointInfo{}, + }, + { + desc: "endpointInfo_not_present_in_non-empty_attributes", + inputEndpointInfo: weight.EndpointInfo{}, + inputAttributes: attributes.New("foo", "bar"), + wantEndpointInfo: weight.EndpointInfo{}, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + endpoint := resolver.Endpoint{Attributes: test.inputAttributes} + endpoint = weight.Set(endpoint, test.inputEndpointInfo) + gotEndpointInfo := weight.FromEndpoint(endpoint) + if !cmp.Equal(gotEndpointInfo, test.wantEndpointInfo) { + t.Errorf("gotEndpointInfo: %v, wantEndpointInfo: %v", gotEndpointInfo, test.wantEndpointInfo) + } + + }) + } +} diff --git a/experimental/credentials/credentials_test.go b/experimental/credentials/credentials_test.go new file mode 100644 index 0000000..4b44a52 --- /dev/null +++ b/experimental/credentials/credentials_test.go @@ -0,0 +1,252 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials + +import ( + "context" + "crypto/tls" + "net" + "strings" + "testing" + "time" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/testdata" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestTLSOverrideServerName(t *testing.T) { + expectedServerName := "server.name" + c := NewTLSWithALPNDisabled(nil) + c.OverrideServerName(expectedServerName) + if c.Info().ServerName != expectedServerName { + t.Fatalf("c.Info().ServerName = %v, want %v", c.Info().ServerName, expectedServerName) + } +} + +func (s) TestTLSClone(t *testing.T) { + expectedServerName := "server.name" + c := NewTLSWithALPNDisabled(nil) + c.OverrideServerName(expectedServerName) + cc := c.Clone() + if cc.Info().ServerName != expectedServerName { + t.Fatalf("cc.Info().ServerName = %v, want %v", cc.Info().ServerName, expectedServerName) + } + cc.OverrideServerName("") + if c.Info().ServerName != expectedServerName { + t.Fatalf("Change in clone should not affect the original, c.Info().ServerName = %v, want %v", c.Info().ServerName, expectedServerName) + } + +} + +type serverHandshake func(net.Conn) (credentials.AuthInfo, error) + +func (s) TestClientHandshakeReturnsAuthInfo(t *testing.T) { + tcs := []struct { + name string + address string + }{ + { + name: "localhost", + address: "localhost:0", + }, + { + name: "ipv4", + address: "127.0.0.1:0", + }, + { + name: "ipv6", + address: "[::1]:0", + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + done := make(chan credentials.AuthInfo, 1) + lis := launchServerOnListenAddress(t, tlsServerHandshake, done, tc.address) + defer lis.Close() + lisAddr := lis.Addr().String() + clientAuthInfo := clientHandle(t, gRPCClientHandshake, lisAddr) + // wait until server sends serverAuthInfo or fails. + serverAuthInfo, ok := <-done + if !ok { + t.Fatalf("Error at server-side") + } + if !compare(clientAuthInfo, serverAuthInfo) { + t.Fatalf("c.ClientHandshake(_, %v, _) = %v, want %v.", lisAddr, clientAuthInfo, serverAuthInfo) + } + }) + } +} + +func (s) TestServerHandshakeReturnsAuthInfo(t *testing.T) { + done := make(chan credentials.AuthInfo, 1) + lis := launchServer(t, gRPCServerHandshake, done) + defer lis.Close() + clientAuthInfo := clientHandle(t, tlsClientHandshake, lis.Addr().String()) + // wait until server sends serverAuthInfo or fails. + serverAuthInfo, ok := <-done + if !ok { + t.Fatalf("Error at server-side") + } + if !compare(clientAuthInfo, serverAuthInfo) { + t.Fatalf("ServerHandshake(_) = %v, want %v.", serverAuthInfo, clientAuthInfo) + } +} + +func (s) TestServerAndClientHandshake(t *testing.T) { + done := make(chan credentials.AuthInfo, 1) + lis := launchServer(t, gRPCServerHandshake, done) + defer lis.Close() + clientAuthInfo := clientHandle(t, gRPCClientHandshake, lis.Addr().String()) + // wait until server sends serverAuthInfo or fails. + serverAuthInfo, ok := <-done + if !ok { + t.Fatalf("Error at server-side") + } + if !compare(clientAuthInfo, serverAuthInfo) { + t.Fatalf("AuthInfo returned by server: %v and client: %v aren't same", serverAuthInfo, clientAuthInfo) + } +} + +func compare(a1, a2 credentials.AuthInfo) bool { + if a1.AuthType() != a2.AuthType() { + return false + } + switch a1.AuthType() { + case "tls": + state1 := a1.(credentials.TLSInfo).State + state2 := a2.(credentials.TLSInfo).State + if state1.Version == state2.Version && + state1.HandshakeComplete == state2.HandshakeComplete && + state1.CipherSuite == state2.CipherSuite && + state1.NegotiatedProtocol == state2.NegotiatedProtocol { + return true + } + return false + default: + return false + } +} + +func launchServer(t *testing.T, hs serverHandshake, done chan credentials.AuthInfo) net.Listener { + return launchServerOnListenAddress(t, hs, done, "localhost:0") +} + +func launchServerOnListenAddress(t *testing.T, hs serverHandshake, done chan credentials.AuthInfo, address string) net.Listener { + lis, err := net.Listen("tcp", address) + if err != nil { + if strings.Contains(err.Error(), "bind: cannot assign requested address") || + strings.Contains(err.Error(), "socket: address family not supported by protocol") { + t.Skipf("no support for address %v", address) + } + t.Fatalf("Failed to listen: %v", err) + } + go serverHandle(t, hs, done, lis) + return lis +} + +// Is run in a separate goroutine. +func serverHandle(t *testing.T, hs serverHandshake, done chan credentials.AuthInfo, lis net.Listener) { + serverRawConn, err := lis.Accept() + if err != nil { + t.Errorf("Server failed to accept connection: %v", err) + close(done) + return + } + serverAuthInfo, err := hs(serverRawConn) + if err != nil { + t.Errorf("Server failed while handshake. Error: %v", err) + serverRawConn.Close() + close(done) + return + } + done <- serverAuthInfo +} + +func clientHandle(t *testing.T, hs func(net.Conn, string) (credentials.AuthInfo, error), lisAddr string) credentials.AuthInfo { + conn, err := net.Dial("tcp", lisAddr) + if err != nil { + t.Fatalf("Client failed to connect to %s. Error: %v", lisAddr, err) + } + defer conn.Close() + clientAuthInfo, err := hs(conn, lisAddr) + if err != nil { + t.Fatalf("Error on client while handshake. Error: %v", err) + } + return clientAuthInfo +} + +// Server handshake implementation in gRPC. +func gRPCServerHandshake(conn net.Conn) (credentials.AuthInfo, error) { + serverTLS, err := NewServerTLSFromFileWithALPNDisabled(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + return nil, err + } + _, serverAuthInfo, err := serverTLS.ServerHandshake(conn) + if err != nil { + return nil, err + } + return serverAuthInfo, nil +} + +// Client handshake implementation in gRPC. +func gRPCClientHandshake(conn net.Conn, lisAddr string) (credentials.AuthInfo, error) { + clientTLS := NewTLSWithALPNDisabled(&tls.Config{InsecureSkipVerify: true}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, authInfo, err := clientTLS.ClientHandshake(ctx, lisAddr, conn) + if err != nil { + return nil, err + } + return authInfo, nil +} + +func tlsServerHandshake(conn net.Conn) (credentials.AuthInfo, error) { + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + return nil, err + } + serverTLSConfig := &tls.Config{Certificates: []tls.Certificate{cert}} + serverConn := tls.Server(conn, serverTLSConfig) + err = serverConn.Handshake() + if err != nil { + return nil, err + } + return credentials.TLSInfo{State: serverConn.ConnectionState(), CommonAuthInfo: credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}}, nil +} + +func tlsClientHandshake(conn net.Conn, _ string) (credentials.AuthInfo, error) { + clientTLSConfig := &tls.Config{InsecureSkipVerify: true} + clientConn := tls.Client(conn, clientTLSConfig) + if err := clientConn.Handshake(); err != nil { + return nil, err + } + return credentials.TLSInfo{State: clientConn.ConnectionState(), CommonAuthInfo: credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}}, nil +} diff --git a/experimental/credentials/internal/spiffe.go b/experimental/credentials/internal/spiffe.go new file mode 100644 index 0000000..569419a --- /dev/null +++ b/experimental/credentials/internal/spiffe.go @@ -0,0 +1,75 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal defines APIs for parsing SPIFFE ID. +// +// All APIs in this package are experimental. +package internal + +import ( + "crypto/tls" + "crypto/x509" + "net/url" + + "google.golang.org/grpc/grpclog" +) + +var logger = grpclog.Component("credentials") + +// SPIFFEIDFromState parses the SPIFFE ID from State. If the SPIFFE ID format +// is invalid, return nil with warning. +func SPIFFEIDFromState(state tls.ConnectionState) *url.URL { + if len(state.PeerCertificates) == 0 || len(state.PeerCertificates[0].URIs) == 0 { + return nil + } + return SPIFFEIDFromCert(state.PeerCertificates[0]) +} + +// SPIFFEIDFromCert parses the SPIFFE ID from x509.Certificate. If the SPIFFE +// ID format is invalid, return nil with warning. +func SPIFFEIDFromCert(cert *x509.Certificate) *url.URL { + if cert == nil || cert.URIs == nil { + return nil + } + var spiffeID *url.URL + for _, uri := range cert.URIs { + if uri == nil || uri.Scheme != "spiffe" || uri.Opaque != "" || (uri.User != nil && uri.User.Username() != "") { + continue + } + // From this point, we assume the uri is intended for a SPIFFE ID. + if len(uri.String()) > 2048 { + logger.Warning("invalid SPIFFE ID: total ID length larger than 2048 bytes") + return nil + } + if len(uri.Host) == 0 || len(uri.Path) == 0 { + logger.Warning("invalid SPIFFE ID: domain or workload ID is empty") + return nil + } + if len(uri.Host) > 255 { + logger.Warning("invalid SPIFFE ID: domain length larger than 255 characters") + return nil + } + // A valid SPIFFE certificate can only have exactly one URI SAN field. + if len(cert.URIs) > 1 { + logger.Warning("invalid SPIFFE ID: multiple URI SANs") + return nil + } + spiffeID = uri + } + return spiffeID +} diff --git a/experimental/credentials/internal/spiffe_test.go b/experimental/credentials/internal/spiffe_test.go new file mode 100644 index 0000000..885fe1b --- /dev/null +++ b/experimental/credentials/internal/spiffe_test.go @@ -0,0 +1,233 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package internal + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "net/url" + "os" + "testing" + + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/testdata" +) + +const wantURI = "spiffe://foo.bar.com/client/workload/1" + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestSPIFFEIDFromState(t *testing.T) { + tests := []struct { + name string + urls []*url.URL + // If we expect a SPIFFE ID to be returned. + wantID bool + }{ + { + name: "empty URIs", + urls: []*url.URL{}, + wantID: false, + }, + { + name: "good SPIFFE ID", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + }, + wantID: true, + }, + { + name: "invalid host", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: "", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + }, + wantID: false, + }, + { + name: "invalid path", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: "foo.bar.com", + Path: "", + RawPath: "", + }, + }, + wantID: false, + }, + { + name: "large path", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: "foo.bar.com", + Path: string(make([]byte, 2050)), + RawPath: string(make([]byte, 2050)), + }, + }, + wantID: false, + }, + { + name: "large host", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: string(make([]byte, 256)), + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + }, + wantID: false, + }, + { + name: "multiple URI SANs", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + { + Scheme: "spiffe", + Host: "bar.baz.com", + Path: "workload/wl2", + RawPath: "workload/wl2", + }, + { + Scheme: "https", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + }, + wantID: false, + }, + { + name: "multiple URI SANs without SPIFFE ID", + urls: []*url.URL{ + { + Scheme: "https", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + { + Scheme: "ssh", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + }, + wantID: false, + }, + { + name: "multiple URI SANs with one SPIFFE ID", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + { + Scheme: "https", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + }, + wantID: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + state := tls.ConnectionState{PeerCertificates: []*x509.Certificate{{URIs: tt.urls}}} + id := SPIFFEIDFromState(state) + if got, want := id != nil, tt.wantID; got != want { + t.Errorf("want wantID = %v, but SPIFFE ID is %v", want, id) + } + }) + } +} + +func (s) TestSPIFFEIDFromCert(t *testing.T) { + tests := []struct { + name string + dataPath string + // If we expect a SPIFFE ID to be returned. + wantID bool + }{ + { + name: "good certificate with SPIFFE ID", + dataPath: "x509/spiffe_cert.pem", + wantID: true, + }, + { + name: "bad certificate with SPIFFE ID and another URI", + dataPath: "x509/multiple_uri_cert.pem", + wantID: false, + }, + { + name: "certificate without SPIFFE ID", + dataPath: "x509/client1_cert.pem", + wantID: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := os.ReadFile(testdata.Path(tt.dataPath)) + if err != nil { + t.Fatalf("os.ReadFile(%s) failed: %v", testdata.Path(tt.dataPath), err) + } + block, _ := pem.Decode(data) + if block == nil { + t.Fatalf("Failed to parse the certificate: byte block is nil") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("x509.ParseCertificate(%b) failed: %v", block.Bytes, err) + } + uri := SPIFFEIDFromCert(cert) + if (uri != nil) != tt.wantID { + t.Fatalf("wantID got and want mismatch, got %t, want %t", uri != nil, tt.wantID) + } + if uri != nil && uri.String() != wantURI { + t.Fatalf("SPIFFE ID not expected, got %s, want %s", uri.String(), wantURI) + } + }) + } +} diff --git a/experimental/credentials/internal/syscallconn.go b/experimental/credentials/internal/syscallconn.go new file mode 100644 index 0000000..35d67bd --- /dev/null +++ b/experimental/credentials/internal/syscallconn.go @@ -0,0 +1,58 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package internal + +import ( + "net" + "syscall" +) + +type sysConn = syscall.Conn + +// syscallConn keeps reference of rawConn to support syscall.Conn for channelz. +// SyscallConn() (the method in interface syscall.Conn) is explicitly +// implemented on this type, +// +// Interface syscall.Conn is implemented by most net.Conn implementations (e.g. +// TCPConn, UnixConn), but is not part of net.Conn interface. So wrapper conns +// that embed net.Conn don't implement syscall.Conn. (Side note: tls.Conn +// doesn't embed net.Conn, so even if syscall.Conn is part of net.Conn, it won't +// help here). +type syscallConn struct { + net.Conn + // sysConn is a type alias of syscall.Conn. It's necessary because the name + // `Conn` collides with `net.Conn`. + sysConn +} + +// WrapSyscallConn tries to wrap rawConn and newConn into a net.Conn that +// implements syscall.Conn. rawConn will be used to support syscall, and newConn +// will be used for read/write. +// +// This function returns newConn if rawConn doesn't implement syscall.Conn. +func WrapSyscallConn(rawConn, newConn net.Conn) net.Conn { + sysConn, ok := rawConn.(syscall.Conn) + if !ok { + return newConn + } + return &syscallConn{ + Conn: newConn, + sysConn: sysConn, + } +} diff --git a/experimental/credentials/internal/syscallconn_test.go b/experimental/credentials/internal/syscallconn_test.go new file mode 100644 index 0000000..75d7cfa --- /dev/null +++ b/experimental/credentials/internal/syscallconn_test.go @@ -0,0 +1,56 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package internal + +import ( + "net" + "syscall" + "testing" +) + +func (*syscallConn) SyscallConn() (syscall.RawConn, error) { + return nil, nil +} + +type nonSyscallConn struct { + net.Conn +} + +func (s) TestWrapSyscallConn(t *testing.T) { + sc := &syscallConn{} + nsc := &nonSyscallConn{} + + wrapConn := WrapSyscallConn(sc, nsc) + if _, ok := wrapConn.(syscall.Conn); !ok { + t.Errorf("returned conn (type %T) doesn't implement syscall.Conn, want implement", wrapConn) + } +} + +func (s) TestWrapSyscallConnNoWrap(t *testing.T) { + nscRaw := &nonSyscallConn{} + nsc := &nonSyscallConn{} + + wrapConn := WrapSyscallConn(nscRaw, nsc) + if _, ok := wrapConn.(syscall.Conn); ok { + t.Errorf("returned conn (type %T) implements syscall.Conn, want not implement", wrapConn) + } + if wrapConn != nsc { + t.Errorf("returned conn is %p, want %p (the passed-in newConn)", wrapConn, nsc) + } +} diff --git a/experimental/credentials/tls.go b/experimental/credentials/tls.go new file mode 100644 index 0000000..0e19c86 --- /dev/null +++ b/experimental/credentials/tls.go @@ -0,0 +1,249 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package credentials provides experimental TLS credentials. +// The use of this package is strongly discouraged. These credentials exist +// solely to maintain compatibility for users interacting with clients that +// violate the HTTP/2 specification by not advertising support for "h2" in ALPN. +// This package is slated for removal in upcoming grpc-go releases. Users must +// not rely on this package directly. Instead, they should either vendor a +// specific version of gRPC or copy the relevant credentials into their own +// codebase if absolutely necessary. +package credentials + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "os" + + "golang.org/x/net/http2" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/experimental/credentials/internal" +) + +// tlsCreds is the credentials required for authenticating a connection using TLS. +type tlsCreds struct { + // TLS configuration + config *tls.Config +} + +func (c tlsCreds) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{ + SecurityProtocol: "tls", + SecurityVersion: "1.2", + ServerName: c.config.ServerName, + } +} + +func (c *tlsCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (_ net.Conn, _ credentials.AuthInfo, err error) { + // use local cfg to avoid clobbering ServerName if using multiple endpoints + cfg := cloneTLSConfig(c.config) + + serverName, _, err := net.SplitHostPort(authority) + if err != nil { + // If the authority had no host port or if the authority cannot be parsed, use it as-is. + serverName = authority + } + cfg.ServerName = serverName + + conn := tls.Client(rawConn, cfg) + errChannel := make(chan error, 1) + go func() { + errChannel <- conn.Handshake() + close(errChannel) + }() + select { + case err := <-errChannel: + if err != nil { + conn.Close() + return nil, nil, err + } + case <-ctx.Done(): + conn.Close() + return nil, nil, ctx.Err() + } + + tlsInfo := credentials.TLSInfo{ + State: conn.ConnectionState(), + CommonAuthInfo: credentials.CommonAuthInfo{ + SecurityLevel: credentials.PrivacyAndIntegrity, + }, + } + id := internal.SPIFFEIDFromState(conn.ConnectionState()) + if id != nil { + tlsInfo.SPIFFEID = id + } + return internal.WrapSyscallConn(rawConn, conn), tlsInfo, nil +} + +func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + conn := tls.Server(rawConn, c.config) + if err := conn.Handshake(); err != nil { + conn.Close() + return nil, nil, err + } + cs := conn.ConnectionState() + tlsInfo := credentials.TLSInfo{ + State: cs, + CommonAuthInfo: credentials.CommonAuthInfo{ + SecurityLevel: credentials.PrivacyAndIntegrity, + }, + } + id := internal.SPIFFEIDFromState(conn.ConnectionState()) + if id != nil { + tlsInfo.SPIFFEID = id + } + return internal.WrapSyscallConn(rawConn, conn), tlsInfo, nil +} + +func (c *tlsCreds) Clone() credentials.TransportCredentials { + return NewTLSWithALPNDisabled(c.config) +} + +func (c *tlsCreds) OverrideServerName(serverNameOverride string) error { + c.config.ServerName = serverNameOverride + return nil +} + +// The following cipher suites are forbidden for use with HTTP/2 by +// https://datatracker.ietf.org/doc/html/rfc7540#appendix-A +var tls12ForbiddenCipherSuites = map[uint16]struct{}{ + tls.TLS_RSA_WITH_AES_128_CBC_SHA: {}, + tls.TLS_RSA_WITH_AES_256_CBC_SHA: {}, + tls.TLS_RSA_WITH_AES_128_GCM_SHA256: {}, + tls.TLS_RSA_WITH_AES_256_GCM_SHA384: {}, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA: {}, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA: {}, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA: {}, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: {}, +} + +// NewTLSWithALPNDisabled uses c to construct a TransportCredentials based on +// TLS. ALPN verification is disabled. +func NewTLSWithALPNDisabled(c *tls.Config) credentials.TransportCredentials { + config := applyDefaults(c) + if config.GetConfigForClient != nil { + oldFn := config.GetConfigForClient + config.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + cfgForClient, err := oldFn(hello) + if err != nil || cfgForClient == nil { + return cfgForClient, err + } + return applyDefaults(cfgForClient), nil + } + } + return &tlsCreds{config: config} +} + +func applyDefaults(c *tls.Config) *tls.Config { + config := cloneTLSConfig(c) + config.NextProtos = appendH2ToNextProtos(config.NextProtos) + // If the user did not configure a MinVersion and did not configure a + // MaxVersion < 1.2, use MinVersion=1.2, which is required by + // https://datatracker.ietf.org/doc/html/rfc7540#section-9.2 + if config.MinVersion == 0 && (config.MaxVersion == 0 || config.MaxVersion >= tls.VersionTLS12) { + config.MinVersion = tls.VersionTLS12 + } + // If the user did not configure CipherSuites, use all "secure" cipher + // suites reported by the TLS package, but remove some explicitly forbidden + // by https://datatracker.ietf.org/doc/html/rfc7540#appendix-A + if config.CipherSuites == nil { + for _, cs := range tls.CipherSuites() { + if _, ok := tls12ForbiddenCipherSuites[cs.ID]; !ok { + config.CipherSuites = append(config.CipherSuites, cs.ID) + } + } + } + return config +} + +// NewClientTLSFromCertWithALPNDisabled constructs TLS credentials from the +// provided root certificate authority certificate(s) to validate server +// connections. If certificates to establish the identity of the client need to +// be included in the credentials (eg: for mTLS), use NewTLS instead, where a +// complete tls.Config can be specified. +// serverNameOverride is for testing only. If set to a non empty string, +// it will override the virtual host name of authority (e.g. :authority header +// field) in requests. ALPN verification is disabled. +func NewClientTLSFromCertWithALPNDisabled(cp *x509.CertPool, serverNameOverride string) credentials.TransportCredentials { + return NewTLSWithALPNDisabled(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}) +} + +// NewClientTLSFromFileWithALPNDisabled constructs TLS credentials from the +// provided root certificate authority certificate file(s) to validate server +// connections. If certificates to establish the identity of the client need to +// be included in the credentials (eg: for mTLS), use NewTLS instead, where a +// complete tls.Config can be specified. +// serverNameOverride is for testing only. If set to a non empty string, +// it will override the virtual host name of authority (e.g. :authority header +// field) in requests. ALPN verification is disabled. +func NewClientTLSFromFileWithALPNDisabled(certFile, serverNameOverride string) (credentials.TransportCredentials, error) { + b, err := os.ReadFile(certFile) + if err != nil { + return nil, err + } + cp := x509.NewCertPool() + if !cp.AppendCertsFromPEM(b) { + return nil, fmt.Errorf("credentials: failed to append certificates") + } + return NewTLSWithALPNDisabled(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}), nil +} + +// NewServerTLSFromCertWithALPNDisabled constructs TLS credentials from the +// input certificate for server. ALPN verification is disabled. +func NewServerTLSFromCertWithALPNDisabled(cert *tls.Certificate) credentials.TransportCredentials { + return NewTLSWithALPNDisabled(&tls.Config{Certificates: []tls.Certificate{*cert}}) +} + +// NewServerTLSFromFileWithALPNDisabled constructs TLS credentials from the +// input certificate file and key file for server. ALPN verification is disabled. +func NewServerTLSFromFileWithALPNDisabled(certFile, keyFile string) (credentials.TransportCredentials, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + return NewTLSWithALPNDisabled(&tls.Config{Certificates: []tls.Certificate{cert}}), nil +} + +// cloneTLSConfig returns a shallow clone of the exported +// fields of cfg, ignoring the unexported sync.Once, which +// contains a mutex and must not be copied. +// +// If cfg is nil, a new zero tls.Config is returned. +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + + return cfg.Clone() +} + +// appendH2ToNextProtos appends h2 to next protos. +func appendH2ToNextProtos(ps []string) []string { + for _, p := range ps { + if p == http2.NextProtoTLS { + return ps + } + } + ret := make([]string, 0, len(ps)+1) + ret = append(ret, ps...) + return append(ret, http2.NextProtoTLS) +} diff --git a/experimental/credentials/tls_ext_test.go b/experimental/credentials/tls_ext_test.go new file mode 100644 index 0000000..3d4e473 --- /dev/null +++ b/experimental/credentials/tls_ext_test.go @@ -0,0 +1,604 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials_test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "os" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + credsstable "google.golang.org/grpc/credentials" + "google.golang.org/grpc/experimental/credentials" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +var serverCert tls.Certificate +var certPool *x509.CertPool +var serverName = "x.test.example.com" + +func init() { + var err error + serverCert, err = tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + panic(fmt.Sprintf("tls.LoadX509KeyPair(server1.pem, server1.key) failed: %v", err)) + } + + b, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem")) + if err != nil { + panic(fmt.Sprintf("Error reading CA cert file: %v", err)) + } + certPool = x509.NewCertPool() + if !certPool.AppendCertsFromPEM(b) { + panic("Error appending cert from PEM") + } +} + +// Tests that the MinVersion of tls.Config is set to 1.2 if it is not already +// set by the user. +func (s) TestTLS_MinVersion12(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + testCases := []struct { + name string + serverTLS func() *tls.Config + }{ + { + name: "base_case", + serverTLS: func() *tls.Config { + return &tls.Config{ + // MinVersion should be set to 1.2 by gRPC by default. + Certificates: []tls.Certificate{serverCert}, + } + }, + }, + { + name: "fallback_to_base", + serverTLS: func() *tls.Config { + config := &tls.Config{ + // MinVersion should be set to 1.2 by gRPC by default. + Certificates: []tls.Certificate{serverCert}, + } + config.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + return nil, nil + } + return config + }, + }, + { + name: "dynamic_using_get_config_for_client", + serverTLS: func() *tls.Config { + return &tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + return &tls.Config{ + // MinVersion should be set to 1.2 by gRPC by default. + Certificates: []tls.Certificate{serverCert}, + }, nil + }, + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create server creds without a minimum version. + serverCreds := credentials.NewTLSWithALPNDisabled(tc.serverTLS()) + ss := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + // Create client creds that supports V1.0-V1.1. + clientCreds := credentials.NewTLSWithALPNDisabled(&tls.Config{ + ServerName: serverName, + RootCAs: certPool, + MinVersion: tls.VersionTLS10, + MaxVersion: tls.VersionTLS11, + }) + + // Start server and client separately, because Start() blocks on a + // successful connection, which we will not get. + if err := ss.StartServer(grpc.Creds(serverCreds)); err != nil { + t.Fatalf("Error starting server: %v", err) + } + defer ss.Stop() + + cc, err := grpc.NewClient(ss.Address, grpc.WithTransportCredentials(clientCreds)) + if err != nil { + t.Fatalf("grpc.NewClient error: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + const wantStr = "authentication handshake failed" + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(status.Convert(err).Message(), wantStr) { + t.Fatalf("EmptyCall err = %v; want code=%v, message contains %q", err, codes.Unavailable, wantStr) + } + + }) + } +} + +// Tests that the MinVersion of tls.Config is not changed if it is set by the +// user. +func (s) TestTLS_MinVersionOverridable(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + var allCipherSuites []uint16 + for _, cs := range tls.CipherSuites() { + allCipherSuites = append(allCipherSuites, cs.ID) + } + testCases := []struct { + name string + serverTLS func() *tls.Config + }{ + { + name: "base_case", + serverTLS: func() *tls.Config { + return &tls.Config{ + MinVersion: tls.VersionTLS10, + Certificates: []tls.Certificate{serverCert}, + CipherSuites: allCipherSuites, + } + }, + }, + { + name: "fallback_to_base", + serverTLS: func() *tls.Config { + config := &tls.Config{ + MinVersion: tls.VersionTLS10, + Certificates: []tls.Certificate{serverCert}, + CipherSuites: allCipherSuites, + } + config.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + return nil, nil + } + return config + }, + }, + { + name: "dynamic_using_get_config_for_client", + serverTLS: func() *tls.Config { + return &tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + return &tls.Config{ + MinVersion: tls.VersionTLS10, + Certificates: []tls.Certificate{serverCert}, + CipherSuites: allCipherSuites, + }, nil + }, + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create server creds that allow v1.0. + serverCreds := credentials.NewTLSWithALPNDisabled(tc.serverTLS()) + ss := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + // Create client creds that supports V1.0-V1.1. + clientCreds := credentials.NewTLSWithALPNDisabled(&tls.Config{ + ServerName: serverName, + RootCAs: certPool, + CipherSuites: allCipherSuites, + MinVersion: tls.VersionTLS10, + MaxVersion: tls.VersionTLS11, + }) + + if err := ss.Start([]grpc.ServerOption{grpc.Creds(serverCreds)}, grpc.WithTransportCredentials(clientCreds)); err != nil { + t.Fatalf("Error starting stub server: %v", err) + } + defer ss.Stop() + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall err = %v; want ", err) + } + }) + } +} + +// Tests that CipherSuites is set to exclude HTTP/2 forbidden suites by default. +func (s) TestTLS_CipherSuites(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testCases := []struct { + name string + serverTLS func() *tls.Config + }{ + { + name: "base_case", + serverTLS: func() *tls.Config { + return &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + } + }, + }, + { + name: "fallback_to_base", + serverTLS: func() *tls.Config { + config := &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + } + config.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + return nil, nil + } + return config + }, + }, + { + name: "dynamic_using_get_config_for_client", + serverTLS: func() *tls.Config { + return &tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + return &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + }, nil + }, + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create server creds without cipher suites. + serverCreds := credentials.NewTLSWithALPNDisabled(tc.serverTLS()) + ss := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + // Create client creds that use a forbidden suite only. + clientCreds := credentials.NewTLSWithALPNDisabled(&tls.Config{ + ServerName: serverName, + RootCAs: certPool, + CipherSuites: []uint16{tls.TLS_RSA_WITH_AES_128_CBC_SHA}, + MaxVersion: tls.VersionTLS12, // TLS1.3 cipher suites are not configurable, so limit to 1.2. + }) + + // Start server and client separately, because Start() blocks on a + // successful connection, which we will not get. + if err := ss.StartServer(grpc.Creds(serverCreds)); err != nil { + t.Fatalf("Error starting server: %v", err) + } + defer ss.Stop() + + cc, err := grpc.NewClient("dns:"+ss.Address, grpc.WithTransportCredentials(clientCreds)) + if err != nil { + t.Fatalf("grpc.NewClient error: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + const wantStr = "authentication handshake failed" + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(status.Convert(err).Message(), wantStr) { + t.Fatalf("EmptyCall err = %v; want code=%v, message contains %q", err, codes.Unavailable, wantStr) + } + }) + } +} + +// Tests that CipherSuites is not overridden when it is set. +func (s) TestTLS_CipherSuitesOverridable(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + testCases := []struct { + name string + serverTLS func() *tls.Config + }{ + { + name: "base_case", + serverTLS: func() *tls.Config { + return &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + CipherSuites: []uint16{tls.TLS_RSA_WITH_AES_128_CBC_SHA}, + } + }, + }, + { + name: "fallback_to_base", + serverTLS: func() *tls.Config { + config := &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + CipherSuites: []uint16{tls.TLS_RSA_WITH_AES_128_CBC_SHA}, + } + config.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + return nil, nil + } + return config + }, + }, + { + name: "dynamic_using_get_config_for_client", + serverTLS: func() *tls.Config { + return &tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + return &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + CipherSuites: []uint16{tls.TLS_RSA_WITH_AES_128_CBC_SHA}, + }, nil + }, + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create server that allows only a forbidden cipher suite. + serverCreds := credentials.NewTLSWithALPNDisabled(tc.serverTLS()) + ss := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + // Create server that allows only a forbidden cipher suite. + clientCreds := credentials.NewTLSWithALPNDisabled(&tls.Config{ + ServerName: serverName, + RootCAs: certPool, + CipherSuites: []uint16{tls.TLS_RSA_WITH_AES_128_CBC_SHA}, + MaxVersion: tls.VersionTLS12, // TLS1.3 cipher suites are not configurable, so limit to 1.2. + }) + + if err := ss.Start([]grpc.ServerOption{grpc.Creds(serverCreds)}, grpc.WithTransportCredentials(clientCreds)); err != nil { + t.Fatalf("Error starting stub server: %v", err) + } + defer ss.Stop() + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall err = %v; want ", err) + } + }) + } +} + +// TestTLS_ServerConfiguresALPNByDefault verifies that ALPN is configured +// correctly for a server that doesn't specify the NextProtos field and uses +// GetConfigForClient to provide the TLS config during the handshake. +func (s) TestTLS_ServerConfiguresALPNByDefault(t *testing.T) { + initialVal := envconfig.EnforceALPNEnabled + defer func() { + envconfig.EnforceALPNEnabled = initialVal + }() + envconfig.EnforceALPNEnabled = true + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create a server that doesn't set the NextProtos field. + serverCreds := credentials.NewTLSWithALPNDisabled(&tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + return &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + }, nil + }, + }) + + ss := stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + clientCreds := credsstable.NewTLS(&tls.Config{ + ServerName: serverName, + RootCAs: certPool, + }) + + if err := ss.Start([]grpc.ServerOption{grpc.Creds(serverCreds)}, grpc.WithTransportCredentials(clientCreds)); err != nil { + t.Fatalf("Error starting stub server: %v", err) + } + defer ss.Stop() + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall err = %v; want ", err) + } +} + +// TestTLS_DisabledALPNClient tests the behaviour of TransportCredentials when +// connecting to a server that doesn't support ALPN. +func (s) TestTLS_DisabledALPNClient(t *testing.T) { + initialVal := envconfig.EnforceALPNEnabled + defer func() { + envconfig.EnforceALPNEnabled = initialVal + }() + + tests := []struct { + name string + alpnEnforced bool + wantErr bool + }{ + { + name: "enforced", + }, + { + name: "not_enforced", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + envconfig.EnforceALPNEnabled = tc.alpnEnforced + + listener, err := tls.Listen("tcp", "localhost:0", &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + NextProtos: []string{}, // Empty list indicates ALPN is disabled. + }) + if err != nil { + t.Fatalf("Error starting TLS server: %v", err) + } + + errCh := make(chan error, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + errCh <- fmt.Errorf("listener.Accept returned error: %v", err) + } else { + // The first write to the TLS listener initiates the TLS handshake. + conn.Write([]byte("Hello, World!")) + conn.Close() + } + close(errCh) + }() + + serverAddr := listener.Addr().String() + conn, err := net.Dial("tcp", serverAddr) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", serverAddr, err) + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + clientCfg := tls.Config{ + ServerName: serverName, + RootCAs: certPool, + NextProtos: []string{"h2"}, + } + _, _, err = credentials.NewTLSWithALPNDisabled(&clientCfg).ClientHandshake(ctx, serverName, conn) + + if gotErr := (err != nil); gotErr != tc.wantErr { + t.Errorf("ClientHandshake returned unexpected error: got=%v, want=%t", err, tc.wantErr) + } + + select { + case err := <-errCh: + if err != nil { + t.Fatalf("Unexpected error received from server: %v", err) + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for error from server") + } + }) + } +} + +// TestTLS_DisabledALPNServer tests the behaviour of TransportCredentials when +// accepting a request from a client that doesn't support ALPN. +func (s) TestTLS_DisabledALPNServer(t *testing.T) { + initialVal := envconfig.EnforceALPNEnabled + defer func() { + envconfig.EnforceALPNEnabled = initialVal + }() + + tests := []struct { + name string + alpnEnforced bool + wantErr bool + }{ + { + name: "enforced", + }, + { + name: "not_enforced", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + envconfig.EnforceALPNEnabled = tc.alpnEnforced + + listener, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error starting server: %v", err) + } + + errCh := make(chan error, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + errCh <- fmt.Errorf("listener.Accept returned error: %v", err) + return + } + defer conn.Close() + serverCfg := tls.Config{ + Certificates: []tls.Certificate{serverCert}, + NextProtos: []string{"h2"}, + } + _, _, err = credentials.NewTLSWithALPNDisabled(&serverCfg).ServerHandshake(conn) + if gotErr := (err != nil); gotErr != tc.wantErr { + t.Errorf("ServerHandshake returned unexpected error: got=%v, want=%t", err, tc.wantErr) + } + close(errCh) + }() + + serverAddr := listener.Addr().String() + clientCfg := &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + NextProtos: []string{}, // Empty list indicates ALPN is disabled. + RootCAs: certPool, + ServerName: serverName, + } + conn, err := tls.Dial("tcp", serverAddr, clientCfg) + if err != nil { + t.Fatalf("tls.Dial(%s) failed: %v", serverAddr, err) + } + defer conn.Close() + + select { + case <-time.After(defaultTestTimeout): + t.Fatal("Timed out waiting for completion") + case err := <-errCh: + if err != nil { + t.Fatalf("Unexpected server error: %v", err) + } + } + }) + } +} diff --git a/experimental/experimental.go b/experimental/experimental.go new file mode 100644 index 0000000..a7c008a --- /dev/null +++ b/experimental/experimental.go @@ -0,0 +1,85 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package experimental is a collection of experimental features that might +// have some rough edges to them. Housing experimental features in this package +// results in a user accessing these APIs as `experimental.Foo`, thereby making +// it explicit that the feature is experimental and using them in production +// code is at their own risk. +// +// All APIs in this package are experimental. +package experimental + +import ( + "google.golang.org/grpc" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/mem" +) + +// SetDefaultBufferPool sets the default buffer pool used by all grpc clients +// and servers that do not have a buffer pool configured via WithBufferPool or +// BufferPool. It also changes the buffer pool used by the proto codec, which +// can't be changed otherwise. The provided buffer pool must be non-nil. The +// default value is mem.DefaultBufferPool. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. The last caller wins. +func SetDefaultBufferPool(bufferPool mem.BufferPool) { + internal.SetDefaultBufferPool.(func(mem.BufferPool))(bufferPool) +} + +// WithBufferPool returns a grpc.DialOption that configures the use of bufferPool +// for parsing incoming messages on a grpc.ClientConn, and for temporary buffers +// when marshaling outgoing messages. By default, mem.DefaultBufferPool is used, +// and this option only exists to provide alternative buffer pool implementations +// to the client, such as more optimized size allocations etc. However, the +// default buffer pool is already tuned to account for many different use-cases. +// +// Note: The following options will interfere with the buffer pool because they +// require a fully materialized buffer instead of a sequence of buffers: +// EnableTracing, and binary logging. In such cases, materializing the buffer +// will generate a lot of garbage, reducing the overall benefit from using a +// pool. +func WithBufferPool(bufferPool mem.BufferPool) grpc.DialOption { + return internal.WithBufferPool.(func(mem.BufferPool) grpc.DialOption)(bufferPool) +} + +// BufferPool returns a grpc.ServerOption that configures the server to use the +// provided buffer pool for parsing incoming messages and for temporary buffers +// when marshaling outgoing messages. By default, mem.DefaultBufferPool is used, +// and this option only exists to provide alternative buffer pool implementations +// to the server, such as more optimized size allocations etc. However, the +// default buffer pool is already tuned to account for many different use-cases. +// +// Note: The following options will interfere with the buffer pool because they +// require a fully materialized buffer instead of a sequence of buffers: +// EnableTracing, and binary logging. In such cases, materializing the buffer +// will generate a lot of garbage, reducing the overall benefit from using a +// pool. +func BufferPool(bufferPool mem.BufferPool) grpc.ServerOption { + return internal.BufferPool.(func(mem.BufferPool) grpc.ServerOption)(bufferPool) +} + +// AcceptCompressors returns a CallOption that limits the values +// advertised in the grpc-accept-encoding header for the provided RPC. The +// supplied names must correspond to compressors registered via +// encoding.RegisterCompressor. Passing no names advertises "identity" (no +// compression) only. +func AcceptCompressors(names ...string) grpc.CallOption { + return internal.AcceptCompressors.(func(...string) grpc.CallOption)(names...) +} diff --git a/experimental/opentelemetry/trace_options.go b/experimental/opentelemetry/trace_options.go new file mode 100644 index 0000000..9dc0079 --- /dev/null +++ b/experimental/opentelemetry/trace_options.go @@ -0,0 +1,36 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package opentelemetry is EXPERIMENTAL and will be moved to stats/opentelemetry +// package in a later release. +package opentelemetry + +import ( + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" +) + +// TraceOptions contains the tracing settings for OpenTelemetry instrumentation. +type TraceOptions struct { + // TracerProvider is the OpenTelemetry tracer which is required to + // record traces/trace spans for instrumentation. If unset, tracing + // will not be recorded. + TracerProvider trace.TracerProvider + + // TextMapPropagator propagates span context through text map carrier. + // If unset, tracing will not be recorded. + TextMapPropagator propagation.TextMapPropagator +} diff --git a/experimental/shared_buffer_pool_test.go b/experimental/shared_buffer_pool_test.go new file mode 100644 index 0000000..ce001bd --- /dev/null +++ b/experimental/shared_buffer_pool_test.go @@ -0,0 +1,211 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package experimental_test + +import ( + "bytes" + "context" + "io" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/encoding/gzip" + "google.golang.org/grpc/experimental" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const defaultTestTimeout = 10 * time.Second + +func (s) TestRecvBufferPoolStream(t *testing.T) { + // TODO: How much of this test can be preserved now that buffer reuse happens at + // the codec and HTTP/2 level? + t.SkipNow() + tcs := []struct { + name string + callOpts []grpc.CallOption + }{ + { + name: "default", + }, + { + name: "useCompressor", + callOpts: []grpc.CallOption{ + grpc.UseCompressor(gzip.Name), + }, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + const reqCount = 10 + + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for i := 0; i < reqCount; i++ { + preparedMsg := &grpc.PreparedMsg{} + if err := preparedMsg.Encode(stream, &testpb.StreamingOutputCallResponse{ + Payload: &testpb.Payload{ + Body: []byte{'0' + uint8(i)}, + }, + }); err != nil { + return err + } + stream.SendMsg(preparedMsg) + } + return nil + }, + } + + pool := &checkBufferPool{} + sopts := []grpc.ServerOption{experimental.BufferPool(pool)} + dopts := []grpc.DialOption{experimental.WithBufferPool(pool)} + if err := ss.Start(sopts, dopts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + stream, err := ss.Client.FullDuplexCall(ctx, tc.callOpts...) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %v", err) + } + + var ngot int + var buf bytes.Buffer + for { + reply, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + ngot++ + if buf.Len() > 0 { + buf.WriteByte(',') + } + buf.Write(reply.GetPayload().GetBody()) + } + if want := 10; ngot != want { + t.Fatalf("Got %d replies, want %d", ngot, want) + } + if got, want := buf.String(), "0,1,2,3,4,5,6,7,8,9"; got != want { + t.Fatalf("Got replies %q; want %q", got, want) + } + + if len(pool.puts) != reqCount { + t.Fatalf("Expected 10 buffers to be returned to the pool, got %d", len(pool.puts)) + } + }) + } +} + +func (s) TestRecvBufferPoolUnary(t *testing.T) { + // TODO: See above + t.SkipNow() + tcs := []struct { + name string + callOpts []grpc.CallOption + }{ + { + name: "default", + }, + { + name: "useCompressor", + callOpts: []grpc.CallOption{ + grpc.UseCompressor(gzip.Name), + }, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + const largeSize = 1024 + + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{ + Payload: &testpb.Payload{ + Body: make([]byte, largeSize), + }, + }, nil + }, + } + + pool := &checkBufferPool{} + sopts := []grpc.ServerOption{experimental.BufferPool(pool)} + dopts := []grpc.DialOption{experimental.WithBufferPool(pool)} + if err := ss.Start(sopts, dopts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const reqCount = 10 + for i := 0; i < reqCount; i++ { + if _, err := ss.Client.UnaryCall( + ctx, + &testpb.SimpleRequest{ + Payload: &testpb.Payload{ + Body: make([]byte, largeSize), + }, + }, + tc.callOpts..., + ); err != nil { + t.Fatalf("ss.Client.UnaryCall failed: %v", err) + } + } + + const bufferCount = reqCount * 2 // req + resp + if len(pool.puts) != bufferCount { + t.Fatalf("Expected %d buffers to be returned to the pool, got %d", bufferCount, len(pool.puts)) + } + }) + } +} + +type checkBufferPool struct { + puts [][]byte +} + +func (p *checkBufferPool) Get(size int) *[]byte { + b := make([]byte, size) + return &b +} + +func (p *checkBufferPool) Put(bs *[]byte) { + p.puts = append(p.puts, *bs) +} diff --git a/experimental/stats/metricregistry.go b/experimental/stats/metricregistry.go new file mode 100644 index 0000000..472813f --- /dev/null +++ b/experimental/stats/metricregistry.go @@ -0,0 +1,342 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package stats + +import ( + "maps" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/stats" +) + +func init() { + internal.SnapshotMetricRegistryForTesting = snapshotMetricsRegistryForTesting +} + +var logger = grpclog.Component("metrics-registry") + +// DefaultMetrics are the default metrics registered through global metrics +// registry. This is written to at initialization time only, and is read only +// after initialization. +var DefaultMetrics = stats.NewMetricSet() + +// MetricDescriptor is the data for a registered metric. +type MetricDescriptor struct { + // The name of this metric. This name must be unique across the whole binary + // (including any per call metrics). See + // https://github.com/grpc/proposal/blob/master/A79-non-per-call-metrics-architecture.md#metric-instrument-naming-conventions + // for metric naming conventions. + Name string + // The description of this metric. + Description string + // The unit (e.g. entries, seconds) of this metric. + Unit string + // The required label keys for this metric. These are intended to + // metrics emitted from a stats handler. + Labels []string + // The optional label keys for this metric. These are intended to attached + // to metrics emitted from a stats handler if configured. + OptionalLabels []string + // Whether this metric is on by default. + Default bool + // The type of metric. This is set by the metric registry, and not intended + // to be set by a component registering a metric. + Type MetricType + // Bounds are the bounds of this metric. This only applies to histogram + // metrics. If unset or set with length 0, stats handlers will fall back to + // default bounds. + Bounds []float64 +} + +// MetricType is the type of metric. +type MetricType int + +// Type of metric supported by this instrument registry. +const ( + MetricTypeIntCount MetricType = iota + MetricTypeFloatCount + MetricTypeIntHisto + MetricTypeFloatHisto + MetricTypeIntGauge + MetricTypeIntUpDownCount + MetricTypeIntAsyncGauge +) + +// Int64CountHandle is a typed handle for a int count metric. This handle +// is passed at the recording point in order to know which metric to record +// on. +type Int64CountHandle MetricDescriptor + +// Descriptor returns the int64 count handle typecast to a pointer to a +// MetricDescriptor. +func (h *Int64CountHandle) Descriptor() *MetricDescriptor { + return (*MetricDescriptor)(h) +} + +// Record records the int64 count value on the metrics recorder provided. +func (h *Int64CountHandle) Record(recorder MetricsRecorder, incr int64, labels ...string) { + recorder.RecordInt64Count(h, incr, labels...) +} + +// Int64UpDownCountHandle is a typed handle for an int up-down counter metric. +// This handle is passed at the recording point in order to know which metric +// to record on. +type Int64UpDownCountHandle MetricDescriptor + +// Descriptor returns the int64 up-down counter handle typecast to a pointer to a +// MetricDescriptor. +func (h *Int64UpDownCountHandle) Descriptor() *MetricDescriptor { + return (*MetricDescriptor)(h) +} + +// Record records the int64 up-down counter value on the metrics recorder provided. +// The value 'v' can be positive to increment or negative to decrement. +func (h *Int64UpDownCountHandle) Record(recorder MetricsRecorder, v int64, labels ...string) { + recorder.RecordInt64UpDownCount(h, v, labels...) +} + +// Float64CountHandle is a typed handle for a float count metric. This handle is +// passed at the recording point in order to know which metric to record on. +type Float64CountHandle MetricDescriptor + +// Descriptor returns the float64 count handle typecast to a pointer to a +// MetricDescriptor. +func (h *Float64CountHandle) Descriptor() *MetricDescriptor { + return (*MetricDescriptor)(h) +} + +// Record records the float64 count value on the metrics recorder provided. +func (h *Float64CountHandle) Record(recorder MetricsRecorder, incr float64, labels ...string) { + recorder.RecordFloat64Count(h, incr, labels...) +} + +// Int64HistoHandle is a typed handle for an int histogram metric. This handle +// is passed at the recording point in order to know which metric to record on. +type Int64HistoHandle MetricDescriptor + +// Descriptor returns the int64 histo handle typecast to a pointer to a +// MetricDescriptor. +func (h *Int64HistoHandle) Descriptor() *MetricDescriptor { + return (*MetricDescriptor)(h) +} + +// Record records the int64 histo value on the metrics recorder provided. +func (h *Int64HistoHandle) Record(recorder MetricsRecorder, incr int64, labels ...string) { + recorder.RecordInt64Histo(h, incr, labels...) +} + +// Float64HistoHandle is a typed handle for a float histogram metric. This +// handle is passed at the recording point in order to know which metric to +// record on. +type Float64HistoHandle MetricDescriptor + +// Descriptor returns the float64 histo handle typecast to a pointer to a +// MetricDescriptor. +func (h *Float64HistoHandle) Descriptor() *MetricDescriptor { + return (*MetricDescriptor)(h) +} + +// Record records the float64 histo value on the metrics recorder provided. +func (h *Float64HistoHandle) Record(recorder MetricsRecorder, incr float64, labels ...string) { + recorder.RecordFloat64Histo(h, incr, labels...) +} + +// Int64GaugeHandle is a typed handle for an int gauge metric. This handle is +// passed at the recording point in order to know which metric to record on. +type Int64GaugeHandle MetricDescriptor + +// Descriptor returns the int64 gauge handle typecast to a pointer to a +// MetricDescriptor. +func (h *Int64GaugeHandle) Descriptor() *MetricDescriptor { + return (*MetricDescriptor)(h) +} + +// Record records the int64 histo value on the metrics recorder provided. +func (h *Int64GaugeHandle) Record(recorder MetricsRecorder, incr int64, labels ...string) { + recorder.RecordInt64Gauge(h, incr, labels...) +} + +// AsyncMetric is a marker interface for asynchronous metric types. +type AsyncMetric interface { + isAsync() + Descriptor() *MetricDescriptor +} + +// Int64AsyncGaugeHandle is a typed handle for an int gauge metric. This handle is +// passed at the recording point in order to know which metric to record on. +type Int64AsyncGaugeHandle MetricDescriptor + +// isAsync implements the AsyncMetric interface. +func (h *Int64AsyncGaugeHandle) isAsync() {} + +// Descriptor returns the int64 gauge handle typecast to a pointer to a +// MetricDescriptor. +func (h *Int64AsyncGaugeHandle) Descriptor() *MetricDescriptor { + return (*MetricDescriptor)(h) +} + +// Record records the int64 gauge value on the metrics recorder provided. +func (h *Int64AsyncGaugeHandle) Record(recorder AsyncMetricsRecorder, value int64, labels ...string) { + recorder.RecordInt64AsyncGauge(h, value, labels...) +} + +// registeredMetrics are the registered metric descriptor names. +var registeredMetrics = make(map[string]bool) + +// metricsRegistry contains all of the registered metrics. +// +// This is written to only at init time, and read only after that. +var metricsRegistry = make(map[string]*MetricDescriptor) + +// DescriptorForMetric returns the MetricDescriptor from the global registry. +// +// Returns nil if MetricDescriptor not present. +func DescriptorForMetric(metricName string) *MetricDescriptor { + return metricsRegistry[metricName] +} + +func registerMetric(metricName string, def bool) { + if registeredMetrics[metricName] { + logger.Fatalf("metric %v already registered", metricName) + } + registeredMetrics[metricName] = true + if def { + DefaultMetrics = DefaultMetrics.Add(metricName) + } +} + +// RegisterInt64Count registers the metric description onto the global registry. +// It returns a typed handle to use to recording data. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple metrics are +// registered with the same name, this function will panic. +func RegisterInt64Count(descriptor MetricDescriptor) *Int64CountHandle { + registerMetric(descriptor.Name, descriptor.Default) + descriptor.Type = MetricTypeIntCount + descPtr := &descriptor + metricsRegistry[descriptor.Name] = descPtr + return (*Int64CountHandle)(descPtr) +} + +// RegisterFloat64Count registers the metric description onto the global +// registry. It returns a typed handle to use to recording data. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple metrics are +// registered with the same name, this function will panic. +func RegisterFloat64Count(descriptor MetricDescriptor) *Float64CountHandle { + registerMetric(descriptor.Name, descriptor.Default) + descriptor.Type = MetricTypeFloatCount + descPtr := &descriptor + metricsRegistry[descriptor.Name] = descPtr + return (*Float64CountHandle)(descPtr) +} + +// RegisterInt64Histo registers the metric description onto the global registry. +// It returns a typed handle to use to recording data. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple metrics are +// registered with the same name, this function will panic. +func RegisterInt64Histo(descriptor MetricDescriptor) *Int64HistoHandle { + registerMetric(descriptor.Name, descriptor.Default) + descriptor.Type = MetricTypeIntHisto + descPtr := &descriptor + metricsRegistry[descriptor.Name] = descPtr + return (*Int64HistoHandle)(descPtr) +} + +// RegisterFloat64Histo registers the metric description onto the global +// registry. It returns a typed handle to use to recording data. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple metrics are +// registered with the same name, this function will panic. +func RegisterFloat64Histo(descriptor MetricDescriptor) *Float64HistoHandle { + registerMetric(descriptor.Name, descriptor.Default) + descriptor.Type = MetricTypeFloatHisto + descPtr := &descriptor + metricsRegistry[descriptor.Name] = descPtr + return (*Float64HistoHandle)(descPtr) +} + +// RegisterInt64Gauge registers the metric description onto the global registry. +// It returns a typed handle to use to recording data. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple metrics are +// registered with the same name, this function will panic. +func RegisterInt64Gauge(descriptor MetricDescriptor) *Int64GaugeHandle { + registerMetric(descriptor.Name, descriptor.Default) + descriptor.Type = MetricTypeIntGauge + descPtr := &descriptor + metricsRegistry[descriptor.Name] = descPtr + return (*Int64GaugeHandle)(descPtr) +} + +// RegisterInt64UpDownCount registers the metric description onto the global registry. +// It returns a typed handle to use for recording data. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple metrics are +// registered with the same name, this function will panic. +func RegisterInt64UpDownCount(descriptor MetricDescriptor) *Int64UpDownCountHandle { + registerMetric(descriptor.Name, descriptor.Default) + // Set the specific metric type for the up-down counter + descriptor.Type = MetricTypeIntUpDownCount + descPtr := &descriptor + metricsRegistry[descriptor.Name] = descPtr + return (*Int64UpDownCountHandle)(descPtr) +} + +// RegisterInt64AsyncGauge registers the metric description onto the global registry. +// It returns a typed handle to use for recording data. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple metrics are +// registered with the same name, this function will panic. +func RegisterInt64AsyncGauge(descriptor MetricDescriptor) *Int64AsyncGaugeHandle { + registerMetric(descriptor.Name, descriptor.Default) + descriptor.Type = MetricTypeIntAsyncGauge + descPtr := &descriptor + metricsRegistry[descriptor.Name] = descPtr + return (*Int64AsyncGaugeHandle)(descPtr) +} + +// snapshotMetricsRegistryForTesting snapshots the global data of the metrics +// registry. Returns a cleanup function that sets the metrics registry to its +// original state. +func snapshotMetricsRegistryForTesting() func() { + oldDefaultMetrics := DefaultMetrics + oldRegisteredMetrics := registeredMetrics + oldMetricsRegistry := metricsRegistry + + registeredMetrics = make(map[string]bool) + metricsRegistry = make(map[string]*MetricDescriptor) + maps.Copy(registeredMetrics, registeredMetrics) + maps.Copy(metricsRegistry, metricsRegistry) + + return func() { + DefaultMetrics = oldDefaultMetrics + registeredMetrics = oldRegisteredMetrics + metricsRegistry = oldMetricsRegistry + } +} diff --git a/experimental/stats/metricregistry_test.go b/experimental/stats/metricregistry_test.go new file mode 100644 index 0000000..8797d92 --- /dev/null +++ b/experimental/stats/metricregistry_test.go @@ -0,0 +1,318 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package stats + +import ( + "fmt" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestPanic tests that registering two metrics with the same name across any +// type of metric triggers a panic. +func (s) TestPanic(t *testing.T) { + cleanup := snapshotMetricsRegistryForTesting() + defer cleanup() + + want := "metric simple counter already registered" + defer func() { + if r := recover(); !strings.Contains(fmt.Sprint(r), want) { + t.Errorf("expected panic contains %q, got %q", want, r) + } + }() + desc := MetricDescriptor{ + // Type is not expected to be set from the registerer, but meant to be + // set by the metric registry. + Name: "simple counter", + Description: "number of times recorded on tests", + Unit: "{call}", + } + RegisterInt64Count(desc) + RegisterInt64Gauge(desc) +} + +// TestInstrumentRegistry tests the metric registry. It registers testing only +// metrics using the metric registry, and creates a fake metrics recorder which +// uses these metrics. Using the handles returned from the metric registry, this +// test records stats using the fake metrics recorder. Then, the test verifies +// the persisted metrics data in the metrics recorder is what is expected. Thus, +// this tests the interactions between the metrics recorder and the metrics +// registry. +func (s) TestMetricRegistry(t *testing.T) { + cleanup := snapshotMetricsRegistryForTesting() + defer cleanup() + + intCountHandle1 := RegisterInt64Count(MetricDescriptor{ + Name: "simple counter", + Description: "sum of all emissions from tests", + Unit: "int", + Labels: []string{"int counter label"}, + OptionalLabels: []string{"int counter optional label"}, + Default: false, + }) + floatCountHandle1 := RegisterFloat64Count(MetricDescriptor{ + Name: "float counter", + Description: "sum of all emissions from tests", + Unit: "float", + Labels: []string{"float counter label"}, + OptionalLabels: []string{"float counter optional label"}, + Default: false, + }) + intHistoHandle1 := RegisterInt64Histo(MetricDescriptor{ + Name: "int histo", + Description: "sum of all emissions from tests", + Unit: "int", + Labels: []string{"int histo label"}, + OptionalLabels: []string{"int histo optional label"}, + Default: false, + }) + floatHistoHandle1 := RegisterFloat64Histo(MetricDescriptor{ + Name: "float histo", + Description: "sum of all emissions from tests", + Unit: "float", + Labels: []string{"float histo label"}, + OptionalLabels: []string{"float histo optional label"}, + Default: false, + }) + intGaugeHandle1 := RegisterInt64Gauge(MetricDescriptor{ + Name: "simple gauge", + Description: "the most recent int emitted by test", + Unit: "int", + Labels: []string{"int gauge label"}, + OptionalLabels: []string{"int gauge optional label"}, + Default: false, + }) + intUpDownCountHandle1 := RegisterInt64UpDownCount(MetricDescriptor{ + Name: "simple up down counter", + Description: "current number of emissions from tests", + Unit: "int", + Labels: []string{"int up down counter label"}, + OptionalLabels: []string{"int up down counter optional label"}, + Default: false, + }) + intAsyncGaugeHandle1 := RegisterInt64AsyncGauge(MetricDescriptor{ + Name: "simple async gauge", + Description: "the most recent int emitted by test", + Unit: "int", + Labels: []string{"int async gauge label"}, + OptionalLabels: []string{"int async gauge optional label"}, + Default: false, + }) + + fmr := newFakeMetricsRecorder(t) + + intCountHandle1.Record(fmr, 1, []string{"some label value", "some optional label value"}...) + // The Metric Descriptor in the handle should be able to identify the metric + // information. This is the key passed to metrics recorder to identify + // metric. + if got := fmr.intValues[intCountHandle1.Descriptor()]; got != 1 { + t.Fatalf("fmr.intValues[intCountHandle1.MetricDescriptor] got %v, want: %v", got, 1) + } + + intUpDownCountHandle1.Record(fmr, 2, []string{"some label value", "some optional label value"}...) + // The Metric Descriptor in the handle should be able to identify the metric + // information. This is the key passed to metrics recorder to identify + // metric. + if got := fmr.intValues[intUpDownCountHandle1.Descriptor()]; got != 2 { + t.Fatalf("fmr.intValues[intUpDownCountHandle1.MetricDescriptor] got %v, want: %v", got, 2) + } + + floatCountHandle1.Record(fmr, 1.2, []string{"some label value", "some optional label value"}...) + if got := fmr.floatValues[floatCountHandle1.Descriptor()]; got != 1.2 { + t.Fatalf("fmr.floatValues[floatCountHandle1.MetricDescriptor] got %v, want: %v", got, 1.2) + } + + intHistoHandle1.Record(fmr, 3, []string{"some label value", "some optional label value"}...) + if got := fmr.intValues[intHistoHandle1.Descriptor()]; got != 3 { + t.Fatalf("fmr.intValues[intHistoHandle1.MetricDescriptor] got %v, want: %v", got, 3) + } + + floatHistoHandle1.Record(fmr, 4.3, []string{"some label value", "some optional label value"}...) + if got := fmr.floatValues[floatHistoHandle1.Descriptor()]; got != 4.3 { + t.Fatalf("fmr.floatValues[floatHistoHandle1.MetricDescriptor] got %v, want: %v", got, 4.3) + } + + intGaugeHandle1.Record(fmr, 7, []string{"some label value", "some optional label value"}...) + if got := fmr.intValues[intGaugeHandle1.Descriptor()]; got != 7 { + t.Fatalf("fmr.intValues[intGaugeHandle1.MetricDescriptor] got %v, want: %v", got, 7) + } + intAsyncGaugeHandle1.Record(fmr, 9, []string{"some label value", "some optional label value"}...) + if got := fmr.intValues[intAsyncGaugeHandle1.Descriptor()]; got != 9 { + t.Fatalf("fmr.intValues[intAsyncGaugeHandle1.MetricDescriptor] got %v, want: %v", got, 9) + } +} + +func (s) TestUpDownCounts(t *testing.T) { + cleanup := snapshotMetricsRegistryForTesting() + defer cleanup() + + intUpDownCountHandle1 := RegisterInt64UpDownCount(MetricDescriptor{ + Name: "simple up down counter", + Description: "current number of emissions from tests", + Unit: "int", + Labels: []string{"int up down counter label"}, + OptionalLabels: []string{"int up down counter optional label"}, + Default: false, + }) + + fmr := newFakeMetricsRecorder(t) + intUpDownCountHandle1.Record(fmr, 2, []string{"up down value", "some optional label value"}...) + intUpDownCountHandle1.Record(fmr, -1, []string{"up down value", "some optional label value"}...) + + if got := fmr.intValues[intUpDownCountHandle1.Descriptor()]; got != 1 { + t.Fatalf("fmr.intValues[intUpDownCountHandle1.MetricDescriptor] got %v, want: %v", got, 1) + } +} + +// TestNumerousIntCounts tests numerous int count metrics registered onto the +// metric registry. A component (simulated by test) should be able to record on +// the different registered int count metrics. +func (s) TestNumerousIntCounts(t *testing.T) { + cleanup := snapshotMetricsRegistryForTesting() + defer cleanup() + + intCountHandle1 := RegisterInt64Count(MetricDescriptor{ + Name: "int counter", + Description: "sum of all emissions from tests", + Unit: "int", + Labels: []string{"int counter label"}, + OptionalLabels: []string{"int counter optional label"}, + Default: false, + }) + intCountHandle2 := RegisterInt64Count(MetricDescriptor{ + Name: "int counter 2", + Description: "sum of all emissions from tests", + Unit: "int", + Labels: []string{"int counter label"}, + OptionalLabels: []string{"int counter optional label"}, + Default: false, + }) + intCountHandle3 := RegisterInt64Count(MetricDescriptor{ + Name: "int counter 3", + Description: "sum of all emissions from tests", + Unit: "int", + Labels: []string{"int counter label"}, + OptionalLabels: []string{"int counter optional label"}, + Default: false, + }) + + fmr := newFakeMetricsRecorder(t) + + intCountHandle1.Record(fmr, 1, []string{"some label value", "some optional label value"}...) + got := []int64{fmr.intValues[intCountHandle1.Descriptor()], fmr.intValues[intCountHandle2.Descriptor()], fmr.intValues[intCountHandle3.Descriptor()]} + want := []int64{1, 0, 0} + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("fmr.intValues (-got, +want): %v", diff) + } + + intCountHandle2.Record(fmr, 1, []string{"some label value", "some optional label value"}...) + got = []int64{fmr.intValues[intCountHandle1.Descriptor()], fmr.intValues[intCountHandle2.Descriptor()], fmr.intValues[intCountHandle3.Descriptor()]} + want = []int64{1, 1, 0} + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("fmr.intValues (-got, +want): %v", diff) + } + + intCountHandle3.Record(fmr, 1, []string{"some label value", "some optional label value"}...) + got = []int64{fmr.intValues[intCountHandle1.Descriptor()], fmr.intValues[intCountHandle2.Descriptor()], fmr.intValues[intCountHandle3.Descriptor()]} + want = []int64{1, 1, 1} + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("fmr.intValues (-got, +want): %v", diff) + } + + intCountHandle3.Record(fmr, 1, []string{"some label value", "some optional label value"}...) + got = []int64{fmr.intValues[intCountHandle1.Descriptor()], fmr.intValues[intCountHandle2.Descriptor()], fmr.intValues[intCountHandle3.Descriptor()]} + want = []int64{1, 1, 2} + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("fmr.intValues (-got, +want): %v", diff) + } +} + +type fakeMetricsRecorder struct { + UnimplementedMetricsRecorder + t *testing.T + + intValues map[*MetricDescriptor]int64 + floatValues map[*MetricDescriptor]float64 +} + +// newFakeMetricsRecorder returns a fake metrics recorder based off the current +// state of global metric registry. +func newFakeMetricsRecorder(t *testing.T) *fakeMetricsRecorder { + fmr := &fakeMetricsRecorder{ + t: t, + intValues: make(map[*MetricDescriptor]int64), + floatValues: make(map[*MetricDescriptor]float64), + } + return fmr +} + +// verifyLabels verifies that the labels received are of the expected length. +func verifyLabels(t *testing.T, labelsWant []string, optionalLabelsWant []string, labelsGot []string) { + if len(labelsWant)+len(optionalLabelsWant) != len(labelsGot) { + t.Fatalf("length of optional labels expected did not match got %v, want %v", len(labelsGot), len(labelsWant)+len(optionalLabelsWant)) + } +} + +func (r *fakeMetricsRecorder) RecordInt64Count(handle *Int64CountHandle, incr int64, labels ...string) { + verifyLabels(r.t, handle.Descriptor().Labels, handle.Descriptor().OptionalLabels, labels) + r.intValues[handle.Descriptor()] += incr +} + +func (r *fakeMetricsRecorder) RecordFloat64Count(handle *Float64CountHandle, incr float64, labels ...string) { + verifyLabels(r.t, handle.Descriptor().Labels, handle.Descriptor().OptionalLabels, labels) + r.floatValues[handle.Descriptor()] += incr +} + +func (r *fakeMetricsRecorder) RecordInt64Histo(handle *Int64HistoHandle, incr int64, labels ...string) { + verifyLabels(r.t, handle.Descriptor().Labels, handle.Descriptor().OptionalLabels, labels) + r.intValues[handle.Descriptor()] += incr +} + +func (r *fakeMetricsRecorder) RecordFloat64Histo(handle *Float64HistoHandle, incr float64, labels ...string) { + verifyLabels(r.t, handle.Descriptor().Labels, handle.Descriptor().OptionalLabels, labels) + r.floatValues[handle.Descriptor()] += incr +} + +func (r *fakeMetricsRecorder) RecordInt64Gauge(handle *Int64GaugeHandle, incr int64, labels ...string) { + verifyLabels(r.t, handle.Descriptor().Labels, handle.Descriptor().OptionalLabels, labels) + r.intValues[handle.Descriptor()] += incr +} + +func (r *fakeMetricsRecorder) RecordInt64UpDownCount(handle *Int64UpDownCountHandle, incr int64, labels ...string) { + verifyLabels(r.t, handle.Descriptor().Labels, handle.Descriptor().OptionalLabels, labels) + r.intValues[handle.Descriptor()] += incr +} + +func (r *fakeMetricsRecorder) RecordInt64AsyncGauge(handle *Int64AsyncGaugeHandle, val int64, labels ...string) { + verifyLabels(r.t, handle.Descriptor().Labels, handle.Descriptor().OptionalLabels, labels) + // Async gauges in OTel are "Observer" instruments; they report + // the current state of the world every cycle, they do not accumulate deltas. + r.intValues[handle.Descriptor()] = val +} diff --git a/experimental/stats/metrics.go b/experimental/stats/metrics.go new file mode 100644 index 0000000..8732e53 --- /dev/null +++ b/experimental/stats/metrics.go @@ -0,0 +1,148 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package stats contains experimental metrics/stats API's. +package stats + +import ( + "context" + + "google.golang.org/grpc/internal" + "google.golang.org/grpc/stats" +) + +type customLabelKey struct{} + +// NewContextWithCustomLabel returns a new context with the provided custom label +// attached. The label will be propagated to all metric instruments specified in gRFC A108. +func NewContextWithCustomLabel(ctx context.Context, label string) context.Context { + return context.WithValue(ctx, customLabelKey{}, label) +} + +// CustomLabelFromContext returns the custom label from the context if it exists. +// If the custom label is not present, it returns an empty string. +func CustomLabelFromContext(ctx context.Context) string { + label, _ := ctx.Value(customLabelKey{}).(string) + return label +} + +// MetricsRecorder records on metrics derived from metric registry. +// Implementors must embed UnimplementedMetricsRecorder. +type MetricsRecorder interface { + // RecordInt64Count records the measurement alongside labels on the int + // count associated with the provided handle. + RecordInt64Count(handle *Int64CountHandle, incr int64, labels ...string) + // RecordFloat64Count records the measurement alongside labels on the float + // count associated with the provided handle. + RecordFloat64Count(handle *Float64CountHandle, incr float64, labels ...string) + // RecordInt64Histo records the measurement alongside labels on the int + // histo associated with the provided handle. + RecordInt64Histo(handle *Int64HistoHandle, incr int64, labels ...string) + // RecordFloat64Histo records the measurement alongside labels on the float + // histo associated with the provided handle. + RecordFloat64Histo(handle *Float64HistoHandle, incr float64, labels ...string) + // RecordInt64Gauge records the measurement alongside labels on the int + // gauge associated with the provided handle. + RecordInt64Gauge(handle *Int64GaugeHandle, incr int64, labels ...string) + // RecordInt64UpDownCounter records the measurement alongside labels on the int + // count associated with the provided handle. + RecordInt64UpDownCount(handle *Int64UpDownCountHandle, incr int64, labels ...string) + // RegisterAsyncReporter registers a reporter to produce metric values for + // only the listed descriptors. The returned function must be called when + // the metrics are no longer needed, which will remove the reporter. The + // returned method needs to be idempotent and concurrent safe. + RegisterAsyncReporter(reporter AsyncMetricReporter, descriptors ...AsyncMetric) func() + + // EnforceMetricsRecorderEmbedding is included to force implementers to embed + // another implementation of this interface, allowing gRPC to add methods + // without breaking users. + internal.EnforceMetricsRecorderEmbedding +} + +// AsyncMetricReporter is an interface for types that record metrics asynchronously +// for the set of descriptors they are registered with. The AsyncMetricsRecorder +// parameter is used to record values for these metrics. +// +// Implementations must make unique recordings across all registered +// AsyncMetricReporters. Meaning, they should not report values for a metric with +// the same attributes as another AsyncMetricReporter will report. +// +// Implementations must be concurrent-safe. +type AsyncMetricReporter interface { + // Report records metric values using the provided recorder. + Report(AsyncMetricsRecorder) error +} + +// AsyncMetricReporterFunc is an adapter to allow the use of ordinary functions as +// AsyncMetricReporters. +type AsyncMetricReporterFunc func(AsyncMetricsRecorder) error + +// Report calls f(r). +func (f AsyncMetricReporterFunc) Report(r AsyncMetricsRecorder) error { + return f(r) +} + +// AsyncMetricsRecorder records on asynchronous metrics derived from metric registry. +type AsyncMetricsRecorder interface { + // RecordInt64AsyncGauge records the measurement alongside labels on the int + // count associated with the provided handle asynchronously + RecordInt64AsyncGauge(handle *Int64AsyncGaugeHandle, incr int64, labels ...string) +} + +// Metrics is an experimental legacy alias of the now-stable stats.MetricSet. +// Metrics will be deleted in a future release. +type Metrics = stats.MetricSet + +// Metric was replaced by direct usage of strings. +type Metric = string + +// NewMetrics is an experimental legacy alias of the now-stable +// stats.NewMetricSet. NewMetrics will be deleted in a future release. +func NewMetrics(metrics ...Metric) *Metrics { + return stats.NewMetricSet(metrics...) +} + +// UnimplementedMetricsRecorder must be embedded to have forward compatible implementations. +type UnimplementedMetricsRecorder struct { + internal.EnforceMetricsRecorderEmbedding +} + +// RecordInt64Count provides a no-op implementation. +func (UnimplementedMetricsRecorder) RecordInt64Count(*Int64CountHandle, int64, ...string) {} + +// RecordFloat64Count provides a no-op implementation. +func (UnimplementedMetricsRecorder) RecordFloat64Count(*Float64CountHandle, float64, ...string) {} + +// RecordInt64Histo provides a no-op implementation. +func (UnimplementedMetricsRecorder) RecordInt64Histo(*Int64HistoHandle, int64, ...string) {} + +// RecordFloat64Histo provides a no-op implementation. +func (UnimplementedMetricsRecorder) RecordFloat64Histo(*Float64HistoHandle, float64, ...string) {} + +// RecordInt64Gauge provides a no-op implementation. +func (UnimplementedMetricsRecorder) RecordInt64Gauge(*Int64GaugeHandle, int64, ...string) {} + +// RecordInt64UpDownCount provides a no-op implementation. +func (UnimplementedMetricsRecorder) RecordInt64UpDownCount(*Int64UpDownCountHandle, int64, ...string) { +} + +// RegisterAsyncReporter provides a no-op implementation. +func (UnimplementedMetricsRecorder) RegisterAsyncReporter(AsyncMetricReporter, ...AsyncMetric) func() { + // No-op: Return an empty function to ensure caller doesn't panic on nil function call + return func() {} +} diff --git a/experimental/stats/telemetry/labels.go b/experimental/stats/telemetry/labels.go new file mode 100644 index 0000000..b27485c --- /dev/null +++ b/experimental/stats/telemetry/labels.go @@ -0,0 +1,42 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package telemetry defines stats APIs for interacting with +// telemetry labels. +// +// # Experimental +// +// Notice: All APIs in this package are experimental and may be changed or +// removed in a later release. +package telemetry + +import ( + "context" + + stats "google.golang.org/grpc/internal/stats" +) + +// NewContextWithLabelCallback registers a callback function that is executed +// whenever telemetry labels are updated. This does _not_ require opentelemetry +// instrumentation to be configured on the client or server. +// +// WARNING: Callbacks are executed on the RPC path so users should be mindful +// of the potential performance impact when this is eventually executed. +func NewContextWithLabelCallback(ctx context.Context, callback stats.LabelCallback) context.Context { + return stats.RegisterTelemetryLabelCallback(ctx, callback) +} diff --git a/gcp/observability/config.go b/gcp/observability/config.go new file mode 100644 index 0000000..4ef3487 --- /dev/null +++ b/gcp/observability/config.go @@ -0,0 +1,273 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package observability + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + + gcplogging "cloud.google.com/go/logging" + "golang.org/x/oauth2/google" + "google.golang.org/grpc/internal/envconfig" +) + +const envProjectID = "GOOGLE_CLOUD_PROJECT" + +// fetchDefaultProjectID fetches the default GCP project id from environment. +func fetchDefaultProjectID(ctx context.Context) string { + // Step 1: Check ENV var + if s := os.Getenv(envProjectID); s != "" { + logger.Infof("Found project ID from env %v: %v", envProjectID, s) + return s + } + // Step 2: Check default credential + credentials, err := google.FindDefaultCredentials(ctx, gcplogging.WriteScope) + if err != nil { + logger.Infof("Failed to locate Google Default Credential: %v", err) + return "" + } + if credentials.ProjectID == "" { + logger.Infof("Failed to find project ID in default credential: %v", err) + return "" + } + logger.Infof("Found project ID from Google Default Credential: %v", credentials.ProjectID) + return credentials.ProjectID +} + +// validateMethodString validates whether the string passed in is a valid +// pattern. +func validateMethodString(method string) error { + if strings.HasPrefix(method, "/") { + return errors.New("cannot have a leading slash") + } + serviceMethod := strings.Split(method, "/") + if len(serviceMethod) != 2 { + return errors.New("/ must come in between service and method, only one /") + } + if serviceMethod[1] == "" { + return errors.New("method name must be non empty") + } + if serviceMethod[0] == "*" { + return errors.New("cannot have service wildcard * i.e. (*/m)") + } + return nil +} + +func validateLogEventMethod(methods []string, exclude bool) error { + for _, method := range methods { + if method == "*" { + if exclude { + return errors.New("cannot have exclude and a '*' wildcard") + } + continue + } + if err := validateMethodString(method); err != nil { + return fmt.Errorf("invalid method string: %v, err: %v", method, err) + } + } + return nil +} + +func validateLoggingEvents(config *config) error { + if config.CloudLogging == nil { + return nil + } + for _, clientRPCEvent := range config.CloudLogging.ClientRPCEvents { + if err := validateLogEventMethod(clientRPCEvent.Methods, clientRPCEvent.Exclude); err != nil { + return fmt.Errorf("error in clientRPCEvent method: %v", err) + } + } + for _, serverRPCEvent := range config.CloudLogging.ServerRPCEvents { + if err := validateLogEventMethod(serverRPCEvent.Methods, serverRPCEvent.Exclude); err != nil { + return fmt.Errorf("error in serverRPCEvent method: %v", err) + } + } + return nil +} + +// unmarshalAndVerifyConfig unmarshals a json string representing an +// observability config into its internal go format, and also verifies the +// configuration's fields for validity. +func unmarshalAndVerifyConfig(rawJSON json.RawMessage) (*config, error) { + var config config + if err := json.Unmarshal(rawJSON, &config); err != nil { + return nil, fmt.Errorf("error parsing observability config: %v", err) + } + if err := validateLoggingEvents(&config); err != nil { + return nil, fmt.Errorf("error parsing observability config: %v", err) + } + if config.CloudTrace != nil && (config.CloudTrace.SamplingRate > 1 || config.CloudTrace.SamplingRate < 0) { + return nil, fmt.Errorf("error parsing observability config: invalid cloud trace sampling rate %v", config.CloudTrace.SamplingRate) + } + logger.Infof("Parsed ObservabilityConfig: %+v", &config) + return &config, nil +} + +func parseObservabilityConfig() (*config, error) { + if f := envconfig.ObservabilityConfigFile; f != "" { + if envconfig.ObservabilityConfig != "" { + logger.Warning("Ignoring GRPC_GCP_OBSERVABILITY_CONFIG and using GRPC_GCP_OBSERVABILITY_CONFIG_FILE contents.") + } + content, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("error reading observability configuration file %q: %v", f, err) + } + return unmarshalAndVerifyConfig(content) + } else if envconfig.ObservabilityConfig != "" { + return unmarshalAndVerifyConfig([]byte(envconfig.ObservabilityConfig)) + } + // If the ENV var doesn't exist, do nothing + return nil, nil +} + +func ensureProjectIDInObservabilityConfig(ctx context.Context, config *config) error { + if config.ProjectID == "" { + // Try to fetch the GCP project id + projectID := fetchDefaultProjectID(ctx) + if projectID == "" { + return fmt.Errorf("empty destination project ID") + } + config.ProjectID = projectID + } + return nil +} + +type clientRPCEvents struct { + // Methods is a list of strings which can select a group of methods. By + // default, the list is empty, matching no methods. + // + // The value of the method is in the form of /. + // + // "*" is accepted as a wildcard for: + // 1. The method name. If the value is /*, it matches all + // methods in the specified service. + // 2. The whole value of the field which matches any /. + // It’s not supported when Exclude is true. + // 3. The * wildcard cannot be used on the service name independently, + // */ is not supported. + // + // The service name, when specified, must be the fully qualified service + // name, including the package name. + // + // Examples: + // 1."goo.Foo/Bar" selects only the method "Bar" from service "goo.Foo", + // here “goo” is the package name. + // 2."goo.Foo/*" selects all methods from service "goo.Foo" + // 3. "*" selects all methods from all services. + Methods []string `json:"methods,omitempty"` + // Exclude represents whether the methods denoted by Methods should be + // excluded from logging. The default value is false, meaning the methods + // denoted by Methods are included in the logging. If Exclude is true, the + // wildcard `*` cannot be used as value of an entry in Methods. + Exclude bool `json:"exclude,omitempty"` + // MaxMetadataBytes is the maximum number of bytes of each header to log. If + // the size of the metadata is greater than the defined limit, content past + // the limit will be truncated. The default value is 0. + MaxMetadataBytes int `json:"max_metadata_bytes"` + // MaxMessageBytes is the maximum number of bytes of each message to log. If + // the size of the message is greater than the defined limit, content past + // the limit will be truncated. The default value is 0. + MaxMessageBytes int `json:"max_message_bytes"` +} + +type serverRPCEvents struct { + // Methods is a list of strings which can select a group of methods. By + // default, the list is empty, matching no methods. + // + // The value of the method is in the form of /. + // + // "*" is accepted as a wildcard for: + // 1. The method name. If the value is /*, it matches all + // methods in the specified service. + // 2. The whole value of the field which matches any /. + // It’s not supported when Exclude is true. + // 3. The * wildcard cannot be used on the service name independently, + // */ is not supported. + // + // The service name, when specified, must be the fully qualified service + // name, including the package name. + // + // Examples: + // 1."goo.Foo/Bar" selects only the method "Bar" from service "goo.Foo", + // here “goo” is the package name. + // 2."goo.Foo/*" selects all methods from service "goo.Foo" + // 3. "*" selects all methods from all services. + Methods []string `json:"methods,omitempty"` + // Exclude represents whether the methods denoted by Methods should be + // excluded from logging. The default value is false, meaning the methods + // denoted by Methods are included in the logging. If Exclude is true, the + // wildcard `*` cannot be used as value of an entry in Methods. + Exclude bool `json:"exclude,omitempty"` + // MaxMetadataBytes is the maximum number of bytes of each header to log. If + // the size of the metadata is greater than the defined limit, content past + // the limit will be truncated. The default value is 0. + MaxMetadataBytes int `json:"max_metadata_bytes"` + // MaxMessageBytes is the maximum number of bytes of each message to log. If + // the size of the message is greater than the defined limit, content past + // the limit will be truncated. The default value is 0. + MaxMessageBytes int `json:"max_message_bytes"` +} + +type cloudLogging struct { + // ClientRPCEvents represents the configuration for outgoing RPC's from the + // binary. The client_rpc_events configs are evaluated in text order, the + // first one matched is used. If an RPC doesn't match an entry, it will + // continue on to the next entry in the list. + ClientRPCEvents []clientRPCEvents `json:"client_rpc_events,omitempty"` + + // ServerRPCEvents represents the configuration for incoming RPC's to the + // binary. The server_rpc_events configs are evaluated in text order, the + // first one matched is used. If an RPC doesn't match an entry, it will + // continue on to the next entry in the list. + ServerRPCEvents []serverRPCEvents `json:"server_rpc_events,omitempty"` +} + +type cloudMonitoring struct{} + +type cloudTrace struct { + // SamplingRate is the global setting that controls the probability of an RPC + // being traced. For example, 0.05 means there is a 5% chance for an RPC to + // be traced, 1.0 means trace every call, 0 means don’t start new traces. By + // default, the sampling_rate is 0. + SamplingRate float64 `json:"sampling_rate,omitempty"` +} + +type config struct { + // ProjectID is the destination GCP project identifier for uploading log + // entries. If empty, the gRPC Observability plugin will attempt to fetch + // the project_id from the GCP environment variables, or from the default + // credentials. If not found, the observability init functions will return + // an error. + ProjectID string `json:"project_id,omitempty"` + // CloudLogging defines the logging options. If not present, logging is disabled. + CloudLogging *cloudLogging `json:"cloud_logging,omitempty"` + // CloudMonitoring determines whether or not metrics are enabled based on + // whether it is present or not. If present, monitoring will be enabled, if + // not present, monitoring is disabled. + CloudMonitoring *cloudMonitoring `json:"cloud_monitoring,omitempty"` + // CloudTrace defines the tracing options. When present, tracing is enabled + // with default configurations. When absent, the tracing is disabled. + CloudTrace *cloudTrace `json:"cloud_trace,omitempty"` + // Labels are applied to cloud logging, monitoring, and trace. + Labels map[string]string `json:"labels,omitempty"` +} diff --git a/gcp/observability/exporting.go b/gcp/observability/exporting.go new file mode 100644 index 0000000..3c27b35 --- /dev/null +++ b/gcp/observability/exporting.go @@ -0,0 +1,108 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package observability + +import ( + "context" + "fmt" + "time" + + "google.golang.org/api/option" + "google.golang.org/grpc" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/stats/opencensus" + + gcplogging "cloud.google.com/go/logging" +) + +// cOptsDisableLogTrace are client options for the go client libraries which are +// used to configure connections to GCP exporting backends. These disable global +// dial and server options set by this module, which configure logging, metrics, +// and tracing on all created grpc.ClientConn's and grpc.Server's. These options +// turn on only metrics, and also disable the client libraries behavior of +// plumbing in the older opencensus instrumentation code. +var cOptsDisableLogTrace = []option.ClientOption{ + option.WithTelemetryDisabled(), + option.WithGRPCDialOption(internal.DisableGlobalDialOptions.(func() grpc.DialOption)()), + option.WithGRPCDialOption(opencensus.DialOption(opencensus.TraceOptions{ + DisableTrace: true, + })), +} + +// loggingExporter is the interface of logging exporter for gRPC Observability. +// In future, we might expose this to allow users provide custom exporters. But +// now, it exists for testing purposes. +type loggingExporter interface { + // EmitGrpcLogRecord writes a gRPC LogRecord to cache without blocking. + EmitGcpLoggingEntry(entry gcplogging.Entry) + // Close flushes all pending data and closes the exporter. + Close() error +} + +type cloudLoggingExporter struct { + projectID string + client *gcplogging.Client + logger *gcplogging.Logger +} + +func newCloudLoggingExporter(ctx context.Context, config *config) (loggingExporter, error) { + c, err := gcplogging.NewClient(ctx, fmt.Sprintf("projects/%v", config.ProjectID), cOptsDisableLogTrace...) + if err != nil { + return nil, fmt.Errorf("failed to create cloudLoggingExporter: %v", err) + } + defer logger.Infof("Successfully created cloudLoggingExporter") + if len(config.Labels) != 0 { + logger.Infof("Adding labels: %+v", config.Labels) + } + return &cloudLoggingExporter{ + projectID: config.ProjectID, + client: c, + logger: c.Logger("microservices.googleapis.com/observability/grpc", gcplogging.CommonLabels(config.Labels), gcplogging.BufferedByteLimit(1024*1024*50), gcplogging.DelayThreshold(time.Second*10)), + }, nil +} + +func (cle *cloudLoggingExporter) EmitGcpLoggingEntry(entry gcplogging.Entry) { + cle.logger.Log(entry) + if logger.V(2) { + logger.Infof("Uploading event to CloudLogging: %+v", entry) + } +} + +func (cle *cloudLoggingExporter) Close() error { + var errFlush, errClose error + if cle.logger != nil { + errFlush = cle.logger.Flush() + } + if cle.client != nil { + errClose = cle.client.Close() + } + if errFlush != nil && errClose != nil { + return fmt.Errorf("failed to close exporter. Flush failed: %v; Close failed: %v", errFlush, errClose) + } + if errFlush != nil { + return errFlush + } + if errClose != nil { + return errClose + } + cle.logger = nil + cle.client = nil + logger.Infof("Closed CloudLogging exporter") + return nil +} diff --git a/gcp/observability/go.mod b/gcp/observability/go.mod new file mode 100644 index 0000000..14b7df5 --- /dev/null +++ b/gcp/observability/go.mod @@ -0,0 +1,67 @@ +module google.golang.org/grpc/gcp/observability + +go 1.25.0 + +require ( + cloud.google.com/go/logging v1.19.1 + contrib.go.opencensus.io/exporter/stackdriver v0.13.15-0.20230702191903-2de6d2748484 + github.com/google/go-cmp v0.7.0 + github.com/google/uuid v1.6.0 + go.opencensus.io v0.24.0 + golang.org/x/oauth2 v0.36.0 + google.golang.org/api v0.293.0 + google.golang.org/grpc v1.83.0 + google.golang.org/grpc/stats/opencensus v1.0.0 +) + +require ( + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.23.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/longrunning v1.2.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect + cloud.google.com/go/trace v1.16.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.43.6 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.37 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.36 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.45.6 // indirect + github.com/aws/smithy-go v1.27.8 // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.21 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/genproto v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) + +replace google.golang.org/grpc => ../.. diff --git a/gcp/observability/logging.go b/gcp/observability/logging.go new file mode 100644 index 0000000..6675244 --- /dev/null +++ b/gcp/observability/logging.go @@ -0,0 +1,505 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package observability + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "strings" + "time" + + gcplogging "cloud.google.com/go/logging" + "github.com/google/uuid" + "go.opencensus.io/trace" + + "google.golang.org/grpc" + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal" + iblog "google.golang.org/grpc/internal/binarylog" + "google.golang.org/grpc/internal/grpcutil" + "google.golang.org/grpc/stats/opencensus" +) + +var lExporter loggingExporter + +var newLoggingExporter = newCloudLoggingExporter + +var canonicalString = internal.CanonicalString.(func(codes.Code) string) + +// translateMetadata translates the metadata from Binary Logging format to +// its GrpcLogEntry equivalent. +func translateMetadata(m *binlogpb.Metadata) map[string]string { + metadata := make(map[string]string) + for _, entry := range m.GetEntry() { + entryKey := entry.GetKey() + var newVal string + if strings.HasSuffix(entryKey, "-bin") { // bin header + newVal = base64.StdEncoding.EncodeToString(entry.GetValue()) + } else { // normal header + newVal = string(entry.GetValue()) + } + var oldVal string + var ok bool + if oldVal, ok = metadata[entryKey]; !ok { + metadata[entryKey] = newVal + continue + } + metadata[entryKey] = oldVal + "," + newVal + } + return metadata +} + +func setPeerIfPresent(binlogEntry *binlogpb.GrpcLogEntry, grpcLogEntry *grpcLogEntry) { + if binlogEntry.GetPeer() != nil { + grpcLogEntry.Peer.Type = addrType(binlogEntry.GetPeer().GetType()) + grpcLogEntry.Peer.Address = binlogEntry.GetPeer().GetAddress() + grpcLogEntry.Peer.IPPort = binlogEntry.GetPeer().GetIpPort() + } +} + +var loggerTypeToEventLogger = map[binlogpb.GrpcLogEntry_Logger]loggerType{ + binlogpb.GrpcLogEntry_LOGGER_UNKNOWN: loggerUnknown, + binlogpb.GrpcLogEntry_LOGGER_CLIENT: loggerClient, + binlogpb.GrpcLogEntry_LOGGER_SERVER: loggerServer, +} + +type eventType int + +const ( + // eventTypeUnknown is an unknown event type. + eventTypeUnknown eventType = iota + // eventTypeClientHeader is a header sent from client to server. + eventTypeClientHeader + // eventTypeServerHeader is a header sent from server to client. + eventTypeServerHeader + // eventTypeClientMessage is a message sent from client to server. + eventTypeClientMessage + // eventTypeServerMessage is a message sent from server to client. + eventTypeServerMessage + // eventTypeClientHalfClose is a signal that the loggerClient is done sending. + eventTypeClientHalfClose + // eventTypeServerTrailer indicated the end of a gRPC call. + eventTypeServerTrailer + // eventTypeCancel is a signal that the rpc is canceled. + eventTypeCancel +) + +func (t eventType) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + switch t { + case eventTypeUnknown: + buffer.WriteString("EVENT_TYPE_UNKNOWN") + case eventTypeClientHeader: + buffer.WriteString("CLIENT_HEADER") + case eventTypeServerHeader: + buffer.WriteString("SERVER_HEADER") + case eventTypeClientMessage: + buffer.WriteString("CLIENT_MESSAGE") + case eventTypeServerMessage: + buffer.WriteString("SERVER_MESSAGE") + case eventTypeClientHalfClose: + buffer.WriteString("CLIENT_HALF_CLOSE") + case eventTypeServerTrailer: + buffer.WriteString("SERVER_TRAILER") + case eventTypeCancel: + buffer.WriteString("CANCEL") + } + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +type loggerType int + +const ( + loggerUnknown loggerType = iota + loggerClient + loggerServer +) + +func (t loggerType) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + switch t { + case loggerUnknown: + buffer.WriteString("LOGGER_UNKNOWN") + case loggerClient: + buffer.WriteString("CLIENT") + case loggerServer: + buffer.WriteString("SERVER") + } + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +type payload struct { + Metadata map[string]string `json:"metadata,omitempty"` + // Timeout is the RPC timeout value. + Timeout time.Duration `json:"timeout,omitempty"` + // StatusCode is the gRPC status code. + StatusCode string `json:"statusCode,omitempty"` + // StatusMessage is the gRPC status message. + StatusMessage string `json:"statusMessage,omitempty"` + // StatusDetails is the value of the grpc-status-details-bin metadata key, + // if any. This is always an encoded google.rpc.Status message. + StatusDetails []byte `json:"statusDetails,omitempty"` + // MessageLength is the length of the message. + MessageLength uint32 `json:"messageLength,omitempty"` + // Message is the message of this entry. This is populated in the case of a + // message event. + Message []byte `json:"message,omitempty"` +} + +type addrType int + +const ( + typeUnknown addrType = iota // `json:"TYPE_UNKNOWN"` + ipv4 // `json:"IPV4"` + ipv6 // `json:"IPV6"` + unix // `json:"UNIX"` +) + +func (at addrType) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + switch at { + case typeUnknown: + buffer.WriteString("TYPE_UNKNOWN") + case ipv4: + buffer.WriteString("IPV4") + case ipv6: + buffer.WriteString("IPV6") + case unix: + buffer.WriteString("UNIX") + } + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +type address struct { + // Type is the address type of the address of the peer of the RPC. + Type addrType `json:"type,omitempty"` + // Address is the address of the peer of the RPC. + Address string `json:"address,omitempty"` + // IPPort is the ip and port in string form. It is used only for addrType + // typeIPv4 and typeIPv6. + IPPort uint32 `json:"ipPort,omitempty"` +} + +type grpcLogEntry struct { + // CallID is a uuid which uniquely identifies a call. Each call may have + // several log entries. They will all have the same CallID. Nothing is + // guaranteed about their value other than they are unique across different + // RPCs in the same gRPC process. + CallID string `json:"callId,omitempty"` + // SequenceID is the entry sequence ID for this call. The first message has + // a value of 1, to disambiguate from an unset value. The purpose of this + // field is to detect missing entries in environments where durability or + // ordering is not guaranteed. + SequenceID uint64 `json:"sequenceId,omitempty"` + // Type is the type of binary logging event being logged. + Type eventType `json:"type,omitempty"` + // Logger is the entity that generates the log entry. + Logger loggerType `json:"logger,omitempty"` + // Payload is the payload of this log entry. + Payload payload `json:"payload,omitempty"` + // PayloadTruncated is whether the message or metadata field is either + // truncated or emitted due to options specified in the configuration. + PayloadTruncated bool `json:"payloadTruncated,omitempty"` + // Peer is information about the Peer of the RPC. + Peer address `json:"peer,omitempty"` + // A single process may be used to run multiple virtual servers with + // different identities. + // Authority is the name of such a server identify. It is typically a + // portion of the URI in the form of or :. + Authority string `json:"authority,omitempty"` + // ServiceName is the name of the service. + ServiceName string `json:"serviceName,omitempty"` + // MethodName is the name of the RPC method. + MethodName string `json:"methodName,omitempty"` +} + +type methodLoggerBuilder interface { + Build(iblog.LogEntryConfig) *binlogpb.GrpcLogEntry +} + +type binaryMethodLogger struct { + callID, serviceName, methodName, authority, projectID string + + mlb methodLoggerBuilder + exporter loggingExporter + clientSide bool +} + +// buildGCPLoggingEntry converts the binary log entry into a gcp logging +// entry. +func (bml *binaryMethodLogger) buildGCPLoggingEntry(ctx context.Context, c iblog.LogEntryConfig) gcplogging.Entry { + binLogEntry := bml.mlb.Build(c) + + grpcLogEntry := &grpcLogEntry{ + CallID: bml.callID, + SequenceID: binLogEntry.GetSequenceIdWithinCall(), + Logger: loggerTypeToEventLogger[binLogEntry.Logger], + } + + switch binLogEntry.GetType() { + case binlogpb.GrpcLogEntry_EVENT_TYPE_UNKNOWN: + grpcLogEntry.Type = eventTypeUnknown + case binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER: + grpcLogEntry.Type = eventTypeClientHeader + if binLogEntry.GetClientHeader() != nil { + methodName := binLogEntry.GetClientHeader().MethodName + // Example method name: /grpc.testing.TestService/UnaryCall + if strings.Contains(methodName, "/") { + tokens := strings.Split(methodName, "/") + if len(tokens) == 3 { + // Record service name and method name for all events. + bml.serviceName = tokens[1] + bml.methodName = tokens[2] + } else { + logger.Infof("Malformed method name: %v", methodName) + } + } + bml.authority = binLogEntry.GetClientHeader().GetAuthority() + grpcLogEntry.Payload.Timeout = binLogEntry.GetClientHeader().GetTimeout().AsDuration() + grpcLogEntry.Payload.Metadata = translateMetadata(binLogEntry.GetClientHeader().GetMetadata()) + } + grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated() + setPeerIfPresent(binLogEntry, grpcLogEntry) + case binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER: + grpcLogEntry.Type = eventTypeServerHeader + if binLogEntry.GetServerHeader() != nil { + grpcLogEntry.Payload.Metadata = translateMetadata(binLogEntry.GetServerHeader().GetMetadata()) + } + grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated() + setPeerIfPresent(binLogEntry, grpcLogEntry) + case binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE: + grpcLogEntry.Type = eventTypeClientMessage + grpcLogEntry.Payload.Message = binLogEntry.GetMessage().GetData() + grpcLogEntry.Payload.MessageLength = binLogEntry.GetMessage().GetLength() + grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated() + case binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE: + grpcLogEntry.Type = eventTypeServerMessage + grpcLogEntry.Payload.Message = binLogEntry.GetMessage().GetData() + grpcLogEntry.Payload.MessageLength = binLogEntry.GetMessage().GetLength() + grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated() + case binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE: + grpcLogEntry.Type = eventTypeClientHalfClose + case binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER: + grpcLogEntry.Type = eventTypeServerTrailer + grpcLogEntry.Payload.Metadata = translateMetadata(binLogEntry.GetTrailer().Metadata) + grpcLogEntry.Payload.StatusCode = canonicalString(codes.Code(binLogEntry.GetTrailer().GetStatusCode())) + grpcLogEntry.Payload.StatusMessage = binLogEntry.GetTrailer().GetStatusMessage() + grpcLogEntry.Payload.StatusDetails = binLogEntry.GetTrailer().GetStatusDetails() + grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated() + setPeerIfPresent(binLogEntry, grpcLogEntry) + case binlogpb.GrpcLogEntry_EVENT_TYPE_CANCEL: + grpcLogEntry.Type = eventTypeCancel + } + grpcLogEntry.ServiceName = bml.serviceName + grpcLogEntry.MethodName = bml.methodName + grpcLogEntry.Authority = bml.authority + + var sc trace.SpanContext + var ok bool + if bml.clientSide { + // client side span, populated through opencensus trace package. + if span := trace.FromContext(ctx); span != nil { + sc = span.SpanContext() + ok = true + } + } else { + // server side span, populated through stats/opencensus package. + sc, ok = opencensus.SpanContextFromContext(ctx) + } + gcploggingEntry := gcplogging.Entry{ + Timestamp: binLogEntry.GetTimestamp().AsTime(), + Severity: 100, + Payload: grpcLogEntry, + } + if ok { + gcploggingEntry.Trace = "projects/" + bml.projectID + "/traces/" + sc.TraceID.String() + gcploggingEntry.SpanID = sc.SpanID.String() + gcploggingEntry.TraceSampled = sc.IsSampled() + } + return gcploggingEntry +} + +func (bml *binaryMethodLogger) Log(ctx context.Context, c iblog.LogEntryConfig) { + bml.exporter.EmitGcpLoggingEntry(bml.buildGCPLoggingEntry(ctx, c)) +} + +type eventConfig struct { + // ServiceMethod has /s/m syntax for fast matching. + ServiceMethod map[string]bool + Services map[string]bool + MatchAll bool + + // If true, won't log anything. + Exclude bool + HeaderBytes uint64 + MessageBytes uint64 +} + +type binaryLogger struct { + EventConfigs []eventConfig + projectID string + exporter loggingExporter + clientSide bool +} + +func (bl *binaryLogger) GetMethodLogger(methodName string) iblog.MethodLogger { + s, _, err := grpcutil.ParseMethod(methodName) + if err != nil { + logger.Infof("binarylogging: failed to parse %q: %v", methodName, err) + return nil + } + for _, eventConfig := range bl.EventConfigs { + if eventConfig.MatchAll || eventConfig.ServiceMethod[methodName] || eventConfig.Services[s] { + if eventConfig.Exclude { + return nil + } + + return &binaryMethodLogger{ + exporter: bl.exporter, + mlb: iblog.NewTruncatingMethodLogger(eventConfig.HeaderBytes, eventConfig.MessageBytes), + callID: uuid.NewString(), + projectID: bl.projectID, + clientSide: bl.clientSide, + } + } + } + return nil +} + +// parseMethod splits service and method from the input. It expects format +// "service/method". +func parseMethod(method string) (string, string, error) { + pos := strings.Index(method, "/") + if pos < 0 { + // Shouldn't happen, config already validated. + return "", "", errors.New("invalid method name: no / found") + } + return method[:pos], method[pos+1:], nil +} + +func registerClientRPCEvents(config *config, exporter loggingExporter) { + clientRPCEvents := config.CloudLogging.ClientRPCEvents + if len(clientRPCEvents) == 0 { + return + } + var eventConfigs []eventConfig + for _, clientRPCEvent := range clientRPCEvents { + eventConfig := eventConfig{ + Exclude: clientRPCEvent.Exclude, + HeaderBytes: uint64(clientRPCEvent.MaxMetadataBytes), + MessageBytes: uint64(clientRPCEvent.MaxMessageBytes), + } + for _, method := range clientRPCEvent.Methods { + eventConfig.ServiceMethod = make(map[string]bool) + eventConfig.Services = make(map[string]bool) + if method == "*" { + eventConfig.MatchAll = true + continue + } + s, m, err := parseMethod(method) + if err != nil { + continue + } + if m == "*" { + eventConfig.Services[s] = true + continue + } + eventConfig.ServiceMethod["/"+method] = true + } + eventConfigs = append(eventConfigs, eventConfig) + } + clientSideLogger := &binaryLogger{ + EventConfigs: eventConfigs, + exporter: exporter, + projectID: config.ProjectID, + clientSide: true, + } + internal.AddGlobalDialOptions.(func(opt ...grpc.DialOption))(internal.WithBinaryLogger.(func(bl iblog.Logger) grpc.DialOption)(clientSideLogger)) +} + +func registerServerRPCEvents(config *config, exporter loggingExporter) { + serverRPCEvents := config.CloudLogging.ServerRPCEvents + if len(serverRPCEvents) == 0 { + return + } + var eventConfigs []eventConfig + for _, serverRPCEvent := range serverRPCEvents { + eventConfig := eventConfig{ + Exclude: serverRPCEvent.Exclude, + HeaderBytes: uint64(serverRPCEvent.MaxMetadataBytes), + MessageBytes: uint64(serverRPCEvent.MaxMessageBytes), + } + for _, method := range serverRPCEvent.Methods { + eventConfig.ServiceMethod = make(map[string]bool) + eventConfig.Services = make(map[string]bool) + if method == "*" { + eventConfig.MatchAll = true + continue + } + s, m, err := parseMethod(method) + if err != nil { + continue + } + if m == "*" { + eventConfig.Services[s] = true + continue + } + eventConfig.ServiceMethod["/"+method] = true + } + eventConfigs = append(eventConfigs, eventConfig) + } + serverSideLogger := &binaryLogger{ + EventConfigs: eventConfigs, + exporter: exporter, + projectID: config.ProjectID, + clientSide: false, + } + internal.AddGlobalServerOptions.(func(opt ...grpc.ServerOption))(internal.BinaryLogger.(func(bl iblog.Logger) grpc.ServerOption)(serverSideLogger)) +} + +func startLogging(ctx context.Context, config *config) error { + if config == nil || config.CloudLogging == nil { + return nil + } + var err error + lExporter, err = newLoggingExporter(ctx, config) + if err != nil { + return fmt.Errorf("unable to create CloudLogging exporter: %v", err) + } + + registerClientRPCEvents(config, lExporter) + registerServerRPCEvents(config, lExporter) + return nil +} + +func stopLogging() { + internal.ClearGlobalDialOptions() + internal.ClearGlobalServerOptions() + if lExporter != nil { + // This Close() call handles the flushing of the logging buffer. + lExporter.Close() + } +} diff --git a/gcp/observability/logging_test.go b/gcp/observability/logging_test.go new file mode 100644 index 0000000..6776a3c --- /dev/null +++ b/gcp/observability/logging_test.go @@ -0,0 +1,1347 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package observability + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "strings" + "sync" + "testing" + + gcplogging "cloud.google.com/go/logging" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func cmpLoggingEntryList(got []*grpcLogEntry, want []*grpcLogEntry) error { + if diff := cmp.Diff(got, want, + // For nondeterministic metadata iteration. + cmp.Comparer(func(a map[string]string, b map[string]string) bool { + if len(a) > len(b) { + a, b = b, a + } + if len(a) == 0 && len(a) != len(b) { // No metadata for one and the other comparator wants metadata. + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true + }), + cmpopts.IgnoreFields(grpcLogEntry{}, "CallID", "Peer"), + cmpopts.IgnoreFields(address{}, "IPPort", "Type"), + cmpopts.IgnoreFields(payload{}, "Timeout")); diff != "" { + return fmt.Errorf("got unexpected grpcLogEntry list, diff (-got, +want): %v", diff) + } + return nil +} + +type fakeLoggingExporter struct { + t *testing.T + + mu sync.Mutex + entries []*grpcLogEntry + + idsSeen []*traceAndSpanIDString +} + +func (fle *fakeLoggingExporter) EmitGcpLoggingEntry(entry gcplogging.Entry) { + fle.mu.Lock() + defer fle.mu.Unlock() + if entry.Severity != 100 { + fle.t.Errorf("entry.Severity is not 100, this should be hardcoded") + } + + ids := &traceAndSpanIDString{ + traceID: entry.Trace, + spanID: entry.SpanID, + isSampled: entry.TraceSampled, + } + fle.idsSeen = append(fle.idsSeen, ids) + + grpcLogEntry, ok := entry.Payload.(*grpcLogEntry) + if !ok { + fle.t.Errorf("payload passed in isn't grpcLogEntry") + } + fle.entries = append(fle.entries, grpcLogEntry) +} + +func (fle *fakeLoggingExporter) Close() error { + return nil +} + +// setupObservabilitySystemWithConfig sets up the observability system with the +// specified config, and returns a function which cleans up the observability +// system. +func setupObservabilitySystemWithConfig(cfg *config) (func(), error) { + validConfigJSON, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("failed to convert config to JSON: %v", err) + } + oldObservabilityConfig := envconfig.ObservabilityConfig + envconfig.ObservabilityConfig = string(validConfigJSON) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + err = Start(ctx) + cleanup := func() { + End() + envconfig.ObservabilityConfig = oldObservabilityConfig + } + if err != nil { + return cleanup, fmt.Errorf("error in Start: %v", err) + } + return cleanup, nil +} + +// TestClientRPCEventsLogAll tests the observability system configured with a +// client RPC event that logs every call. It performs a Unary and Bidirectional +// Streaming RPC, and expects certain grpcLogEntries to make it's way to the +// exporter. +func (s) TestClientRPCEventsLogAll(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(context.Context, *config) (loggingExporter, error) { + return fle, nil + } + + clientRPCEventLogAllConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(clientRPCEventLogAllConfig) + if err != nil { + t.Fatalf("error setting up observability: %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil { + return err + } + if _, err := stream.Recv(); err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + + grpcLogEntriesWant := []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeClientMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 2, + Authority: ss.Address, + Payload: payload{ + Message: nil, + }, + }, + { + Type: eventTypeServerHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 4, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 5, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + + fle.entries = nil + fle.mu.Unlock() + + // Make a streaming RPC. This should cause Log calls on the MethodLogger. + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("stream.Send() failed: %v", err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("stream.Recv() failed: %v", err) + } + if err := stream.CloseSend(); err != nil { + t.Fatalf("stream.CloseSend()() failed: %v", err) + } + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + grpcLogEntriesWant = []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeClientMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 2, + Authority: ss.Address, + Payload: payload{ + Message: nil, + }, + }, + { + Type: eventTypeServerHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 4, + Authority: ss.Address, + }, + { + Type: eventTypeClientHalfClose, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 5, + Authority: ss.Address, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + Authority: ss.Address, + SequenceID: 6, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + fle.mu.Unlock() +} + +func (s) TestServerRPCEventsLogAll(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(context.Context, *config) (loggingExporter, error) { + return fle, nil + } + + serverRPCEventLogAllConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ServerRPCEvents: []serverRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(serverRPCEventLogAllConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil { + return err + } + if _, err := stream.Recv(); err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + grpcLogEntriesWant := []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeClientMessage, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 2, + Authority: ss.Address, + }, + { + Type: eventTypeServerHeader, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 4, + Payload: payload{ + Message: nil, + }, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 5, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + fle.entries = nil + fle.mu.Unlock() + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("stream.Send() failed: %v", err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("stream.Recv() failed: %v", err) + } + if err := stream.CloseSend(); err != nil { + t.Fatalf("stream.CloseSend()() failed: %v", err) + } + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + + grpcLogEntriesWant = []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeClientMessage, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 2, + Authority: ss.Address, + }, + { + Type: eventTypeServerHeader, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 4, + Authority: ss.Address, + Payload: payload{ + Message: nil, + }, + }, + { + Type: eventTypeClientHalfClose, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 5, + Authority: ss.Address, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + Authority: ss.Address, + SequenceID: 6, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + fle.mu.Unlock() +} + +// TestBothClientAndServerRPCEvents tests the scenario where you have both +// Client and Server RPC Events configured to log. Both sides should log and +// share the exporter, so the exporter should receive the collective amount of +// calls for both a client stream (corresponding to a Client RPC Event) and a +// server stream (corresponding to a Server RPC Event). The specificity of the +// entries are tested in previous tests. +func (s) TestBothClientAndServerRPCEvents(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(context.Context, *config) (loggingExporter, error) { + return fle, nil + } + + serverRPCEventLogAllConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + ServerRPCEvents: []serverRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + + cleanup, err := setupObservabilitySystemWithConfig(serverRPCEventLogAllConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + UnaryCallF: func(_ context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + // Make a Unary RPC. Both client side and server side streams should log + // entries, which share the same exporter. The exporter should thus receive + // entries from both the client and server streams (the specificity of + // entries is checked in previous tests). + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + fle.mu.Lock() + if len(fle.entries) != 10 { + fle.mu.Unlock() + t.Fatalf("Unexpected length of entries %v, want 10 (collective of client and server)", len(fle.entries)) + } + fle.mu.Unlock() + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + fle.mu.Lock() + if len(fle.entries) != 16 { + fle.mu.Unlock() + t.Fatalf("Unexpected length of entries %v, want 16 (collective of client and server)", len(fle.entries)) + } + fle.mu.Unlock() +} + +// TestClientRPCEventsLogAll tests the observability system configured with a +// client RPC event that logs every call and that truncates headers and +// messages. It performs a Unary RPC, and expects events with truncated payloads +// and payloadTruncated set to true, signifying the system properly truncated +// headers and messages logged. +func (s) TestClientRPCEventsTruncateHeaderAndMetadata(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(_ context.Context, _ *config) (loggingExporter, error) { + return fle, nil + } + + clientRPCEventLogAllConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 10, + MaxMessageBytes: 2, + }, + }, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(clientRPCEventLogAllConfig) + if err != nil { + t.Fatalf("error setting up observability: %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + UnaryCallF: func(_ context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + md := metadata.MD{ + "key1": []string{"value1"}, + "key2": []string{"value2"}, + } + ctx = metadata.NewOutgoingContext(ctx, md) + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: []byte("00000")}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + grpcLogEntriesWant := []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + PayloadTruncated: true, + }, + { + Type: eventTypeClientMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 2, + Authority: ss.Address, + Payload: payload{ + MessageLength: 9, + Message: []uint8{ + 0x1a, + 0x07, + }, + }, + PayloadTruncated: true, + }, + { + Type: eventTypeServerHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 4, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 5, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + // Only one metadata entry should have been present in logging due to + // truncation. + if mdLen := len(fle.entries[0].Payload.Metadata); mdLen != 1 { + t.Fatalf("Metadata should have only 1 entry due to truncation, got %v", mdLen) + } + fle.mu.Unlock() +} + +// TestPrecedenceOrderingInConfiguration tests the scenario where the logging +// part of observability is configured with three client RPC events, the first +// two on specific methods in the service, the last one for any method within +// the service. This test sends three RPC's, one corresponding to each log +// entry. The logging logic dictated by that specific event should be what is +// used for emission. The second event will specify to exclude logging on RPC's, +// which should generate no log entries if an RPC gets to and matches that +// event. +func (s) TestPrecedenceOrderingInConfiguration(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(_ context.Context, _ *config) (loggingExporter, error) { + return fle, nil + } + + threeEventsConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"grpc.testing.TestService/UnaryCall"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + { + Methods: []string{"grpc.testing.TestService/EmptyCall"}, + Exclude: true, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + { + Methods: []string{"grpc.testing.TestService/*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + + cleanup, err := setupObservabilitySystemWithConfig(threeEventsConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + EmptyCallF: func(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + UnaryCallF: func(_ context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + // A Unary RPC should match with first event and logs should correspond + // accordingly. The first event it matches to should be used for the + // configuration, even though it could potentially match to events in the + // future. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + grpcLogEntriesWant := []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeClientMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 2, + Authority: ss.Address, + Payload: payload{ + Message: nil, + }, + }, + { + Type: eventTypeServerHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 4, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 5, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + fle.entries = nil + fle.mu.Unlock() + + // A unary empty RPC should match with the second event, which has the exclude + // flag set. Thus, a unary empty RPC should cause no downstream logs. + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("Unexpected error from EmptyCall: %v", err) + } + // The exporter should have received no new log entries due to this call. + fle.mu.Lock() + if len(fle.entries) != 0 { + fle.mu.Unlock() + t.Fatalf("Unexpected length of entries %v, want 0", len(fle.entries)) + } + fle.mu.Unlock() + + // A third RPC, a full duplex call, which doesn't match with first two and + // matches to last one, due to being a wildcard for every method in the + // service, should log accordingly to the last event's logic. + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + + grpcLogEntriesWant = []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeClientHalfClose, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 2, + Authority: ss.Address, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + Authority: ss.Address, + SequenceID: 3, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + fle.mu.Unlock() +} + +func (s) TestTranslateMetadata(t *testing.T) { + concatBinLogValue := base64.StdEncoding.EncodeToString([]byte("value1")) + "," + base64.StdEncoding.EncodeToString([]byte("value2")) + tests := []struct { + name string + binLogMD *binlogpb.Metadata + wantMD map[string]string + }{ + { + name: "two-entries-different-key", + binLogMD: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + { + Key: "header1", + Value: []byte("value1"), + }, + { + Key: "header2", + Value: []byte("value2"), + }, + }, + }, + wantMD: map[string]string{ + "header1": "value1", + "header2": "value2", + }, + }, + { + name: "two-entries-same-key", + binLogMD: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + { + Key: "header1", + Value: []byte("value1"), + }, + { + Key: "header1", + Value: []byte("value2"), + }, + }, + }, + wantMD: map[string]string{ + "header1": "value1,value2", + }, + }, + { + name: "two-entries-same-key-bin-header", + binLogMD: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + { + Key: "header1-bin", + Value: []byte("value1"), + }, + { + Key: "header1-bin", + Value: []byte("value2"), + }, + }, + }, + wantMD: map[string]string{ + "header1-bin": concatBinLogValue, + }, + }, + { + name: "four-entries-two-keys", + binLogMD: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + { + Key: "header1", + Value: []byte("value1"), + }, + { + Key: "header1", + Value: []byte("value2"), + }, + { + Key: "header1-bin", + Value: []byte("value1"), + }, + { + Key: "header1-bin", + Value: []byte("value2"), + }, + }, + }, + wantMD: map[string]string{ + "header1": "value1,value2", + "header1-bin": concatBinLogValue, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if gotMD := translateMetadata(test.binLogMD); !cmp.Equal(gotMD, test.wantMD) { + t.Fatalf("translateMetadata(%v) = %v, want %v", test.binLogMD, gotMD, test.wantMD) + } + }) + } +} + +func (s) TestMarshalJSON(t *testing.T) { + logEntry := &grpcLogEntry{ + CallID: "300-300-300", + SequenceID: 3, + Type: eventTypeUnknown, + Logger: loggerClient, + Payload: payload{ + Metadata: map[string]string{"header1": "value1"}, + Timeout: 20, + StatusCode: "UNKNOWN", + StatusMessage: "ok", + StatusDetails: []byte("ok"), + MessageLength: 3, + Message: []byte("wow"), + }, + Peer: address{ + Type: ipv4, + Address: "localhost", + IPPort: 16000, + }, + PayloadTruncated: false, + Authority: "server", + ServiceName: "grpc-testing", + MethodName: "UnaryRPC", + } + if _, err := json.Marshal(logEntry); err != nil { + t.Fatalf("json.Marshal(%v) failed with error: %v", logEntry, err) + } +} + +// TestMetadataTruncationAccountsKey tests that the metadata truncation takes +// into account both the key and value of metadata. It configures an +// observability system with a maximum byte length for metadata, which is +// greater than just the byte length of the metadata value but less than the +// byte length of the metadata key + metadata value. Thus, in the ClientHeader +// logging event, no metadata should be logged. +func (s) TestMetadataTruncationAccountsKey(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(_ context.Context, _ *config) (loggingExporter, error) { + return fle, nil + } + + const mdValue = "value" + configMetadataLimit := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: len(mdValue) + 1, + }, + }, + }, + } + + cleanup, err := setupObservabilitySystemWithConfig(configMetadataLimit) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + UnaryCallF: func(_ context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // the set config MaxMetadataBytes is in between len(mdValue) and len("key") + // + len(mdValue), and thus shouldn't log this metadata entry. + md := metadata.MD{ + "key": []string{mdValue}, + } + ctx = metadata.NewOutgoingContext(ctx, md) + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: []byte("00000")}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + + grpcLogEntriesWant := []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + PayloadTruncated: true, + }, + { + Type: eventTypeClientMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 2, + Authority: ss.Address, + Payload: payload{ + MessageLength: 9, + Message: []uint8{}, + }, + PayloadTruncated: true, + }, + { + Type: eventTypeServerHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 4, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 5, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + fle.mu.Unlock() +} + +// TestMethodInConfiguration tests different method names with an expectation on +// whether they should error or not. +func (s) TestMethodInConfiguration(t *testing.T) { + + // To skip creating a stackdriver exporter. + fle := &fakeLoggingExporter{ + t: t, + } + + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(_ context.Context, _ *config) (loggingExporter, error) { + return fle, nil + } + + tests := []struct { + name string + config *config + wantErr string + }{ + { + name: "leading-slash", + config: &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"/service/method"}, + }, + }, + }, + }, + wantErr: "cannot have a leading slash", + }, + { + name: "wildcard service/method", + config: &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*/method"}, + }, + }, + }, + }, + wantErr: "cannot have service wildcard *", + }, + { + name: "/ in service name", + config: &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"ser/vice/method"}, + }, + }, + }, + }, + wantErr: "only one /", + }, + { + name: "empty method name", + config: &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"service/"}, + }, + }, + }, + }, + wantErr: "method name must be non empty", + }, + { + name: "normal", + config: &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"service/method"}, + }, + }, + }, + }, + wantErr: "", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cleanup, gotErr := setupObservabilitySystemWithConfig(test.config) + if cleanup != nil { + defer cleanup() + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("Start(%v) = %v, wantErr %v", test.config, gotErr, test.wantErr) + } + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("Start(%v) = %v, wantErr %v", test.config, gotErr, test.wantErr) + } + }) + } +} diff --git a/gcp/observability/observability.go b/gcp/observability/observability.go new file mode 100644 index 0000000..6b7d4b1 --- /dev/null +++ b/gcp/observability/observability.go @@ -0,0 +1,92 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package observability implements the tracing, metrics, and logging data +// collection, and provides controlling knobs via a config file. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +package observability + +import ( + "context" + "fmt" + + "google.golang.org/grpc/grpclog" +) + +var logger = grpclog.Component("observability") + +// Start is the opt-in API for gRPC Observability plugin. This function should +// be invoked in the main function, and before creating any gRPC clients or +// servers, otherwise, they might not be instrumented. At high-level, this +// module does the following: +// +// - it loads observability config from environment; +// - it registers default exporters if not disabled by the config; +// - it sets up telemetry collectors (binary logging sink or StatsHandlers). +// +// Note: this method should only be invoked once. +// Note: handle the error +func Start(ctx context.Context) error { + config, err := parseObservabilityConfig() + if err != nil { + return err + } + if config == nil { + return fmt.Errorf("no ObservabilityConfig found") + } + + // Set the project ID if it isn't configured manually. + if err = ensureProjectIDInObservabilityConfig(ctx, config); err != nil { + return err + } + + // Cleanup any created resources this function created in case this function + // errors. + defer func() { + if err != nil { + End() + } + }() + + // Enabling tracing and metrics via OpenCensus + if err = startOpenCensus(config); err != nil { + return fmt.Errorf("failed to instrument OpenCensus: %v", err) + } + + if err = startLogging(ctx, config); err != nil { + return fmt.Errorf("failed to start logging: %v", err) + } + + // Logging is controlled by the config at methods level. + return nil +} + +// End is the clean-up API for gRPC Observability plugin. It is expected to be +// invoked in the main function of the application. The suggested usage is +// "defer observability.End()". This function also flushes data to upstream, and +// cleanup resources. +// +// Note: this method should only be invoked once. +func End() { + stopLogging() + stopOpenCensus() +} diff --git a/gcp/observability/observability_test.go b/gcp/observability/observability_test.go new file mode 100644 index 0000000..31d21c7 --- /dev/null +++ b/gcp/observability/observability_test.go @@ -0,0 +1,1088 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package observability + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "go.opencensus.io/stats/view" + "go.opencensus.io/trace" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/leakcheck" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func init() { + // OpenCensus, once included in binary, will spawn a global goroutine + // recorder that is not controllable by application. + // https://github.com/census-instrumentation/opencensus-go/issues/1191 + leakcheck.RegisterIgnoreGoroutine("go.opencensus.io/stats/view.(*worker).start") + // google-cloud-go leaks HTTP client. They are aware of this: + // https://github.com/googleapis/google-cloud-go/issues/1183 + leakcheck.RegisterIgnoreGoroutine("internal/poll.runtime_pollWait") +} + +var ( + defaultTestTimeout = 10 * time.Second + testOkPayload = []byte{72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100} + defaultRequestCount = 24 +) + +const ( + TypeOpenCensusViewDistribution = "distribution" + TypeOpenCensusViewCount = "count" + TypeOpenCensusViewSum = "sum" + TypeOpenCensusViewLastValue = "last_value" +) + +type fakeOpenCensusExporter struct { + // The map of the observed View name and type + SeenViews map[string]string + // Number of spans + SeenSpans int + + idCh *testutils.Channel + + t *testing.T + mu sync.RWMutex +} + +func (fe *fakeOpenCensusExporter) ExportView(vd *view.Data) { + fe.mu.Lock() + defer fe.mu.Unlock() + for _, row := range vd.Rows { + fe.t.Logf("Metrics[%s]", vd.View.Name) + switch row.Data.(type) { + case *view.DistributionData: + fe.SeenViews[vd.View.Name] = TypeOpenCensusViewDistribution + case *view.CountData: + fe.SeenViews[vd.View.Name] = TypeOpenCensusViewCount + case *view.SumData: + fe.SeenViews[vd.View.Name] = TypeOpenCensusViewSum + case *view.LastValueData: + fe.SeenViews[vd.View.Name] = TypeOpenCensusViewLastValue + } + } +} + +type traceAndSpanID struct { + spanName string + traceID trace.TraceID + spanID trace.SpanID + isSampled bool + spanKind int +} + +type traceAndSpanIDString struct { + traceID string + spanID string + isSampled bool + // SpanKind is the type of span. + SpanKind int +} + +// idsToString is a helper that converts from generated trace and span IDs to +// the string version stored in trace message events. +func (tasi *traceAndSpanID) idsToString(projectID string) traceAndSpanIDString { + return traceAndSpanIDString{ + traceID: "projects/" + projectID + "/traces/" + tasi.traceID.String(), + spanID: tasi.spanID.String(), + isSampled: tasi.isSampled, + SpanKind: tasi.spanKind, + } +} + +func (fe *fakeOpenCensusExporter) ExportSpan(vd *trace.SpanData) { + if fe.idCh != nil { + // This is what export span sees representing the trace/span ID which + // will populate different contexts throughout the system, convert in + // caller to string version as the logging code does. + fe.idCh.Send(traceAndSpanID{ + spanName: vd.Name, + traceID: vd.TraceID, + spanID: vd.SpanID, + isSampled: vd.IsSampled(), + spanKind: vd.SpanKind, + }) + } + + fe.mu.Lock() + defer fe.mu.Unlock() + fe.SeenSpans++ + fe.t.Logf("Span[%v]", vd.Name) +} + +func (fe *fakeOpenCensusExporter) Flush() {} + +func (fe *fakeOpenCensusExporter) Close() error { + return nil +} + +func (s) TestRefuseStartWithInvalidPatterns(t *testing.T) { + invalidConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{":-)"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + invalidConfigJSON, err := json.Marshal(invalidConfig) + if err != nil { + t.Fatalf("failed to convert config to JSON: %v", err) + } + oldObservabilityConfig := envconfig.ObservabilityConfig + oldObservabilityConfigFile := envconfig.ObservabilityConfigFile + envconfig.ObservabilityConfig = string(invalidConfigJSON) + envconfig.ObservabilityConfigFile = "" + defer func() { + envconfig.ObservabilityConfig = oldObservabilityConfig + envconfig.ObservabilityConfigFile = oldObservabilityConfigFile + }() + // If there is at least one invalid pattern, which should not be silently tolerated. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := Start(ctx); err == nil { + t.Fatalf("Invalid patterns not triggering error") + } +} + +// TestRefuseStartWithExcludeAndWildCardAll tests the scenario where an +// observability configuration is provided with client RPC event specifying to +// exclude, and which matches on the '*' wildcard (any). This should cause an +// error when trying to start the observability system. +func (s) TestRefuseStartWithExcludeAndWildCardAll(t *testing.T) { + invalidConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + Exclude: true, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + invalidConfigJSON, err := json.Marshal(invalidConfig) + if err != nil { + t.Fatalf("failed to convert config to JSON: %v", err) + } + oldObservabilityConfig := envconfig.ObservabilityConfig + oldObservabilityConfigFile := envconfig.ObservabilityConfigFile + envconfig.ObservabilityConfig = string(invalidConfigJSON) + envconfig.ObservabilityConfigFile = "" + defer func() { + envconfig.ObservabilityConfig = oldObservabilityConfig + envconfig.ObservabilityConfigFile = oldObservabilityConfigFile + }() + // If there is at least one invalid pattern, which should not be silently tolerated. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := Start(ctx); err == nil { + t.Fatalf("Invalid patterns not triggering error") + } +} + +// createTmpConfigInFileSystem creates a random observability config at a random +// place in the temporary portion of the file system dependent on system. It +// also sets the environment variable GRPC_CONFIG_OBSERVABILITY_JSON to point to +// this created config. +func createTmpConfigInFileSystem(rawJSON string) (func(), error) { + configJSONFile, err := os.CreateTemp(os.TempDir(), "configJSON-") + if err != nil { + return nil, fmt.Errorf("cannot create file %v: %v", configJSONFile.Name(), err) + } + _, err = configJSONFile.Write(json.RawMessage(rawJSON)) + if err != nil { + return nil, fmt.Errorf("cannot write marshalled JSON: %v", err) + } + oldObservabilityConfigFile := envconfig.ObservabilityConfigFile + envconfig.ObservabilityConfigFile = configJSONFile.Name() + return func() { + configJSONFile.Close() + envconfig.ObservabilityConfigFile = oldObservabilityConfigFile + }, nil +} + +// TestJSONEnvVarSet tests a valid observability configuration specified by the +// GRPC_CONFIG_OBSERVABILITY_JSON environment variable, whose value represents a +// file path pointing to a JSON encoded config. +func (s) TestJSONEnvVarSet(t *testing.T) { + configJSON := `{ + "project_id": "fake" + }` + cleanup, err := createTmpConfigInFileSystem(configJSON) + defer cleanup() + + if err != nil { + t.Fatalf("failed to create config in file system: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := Start(ctx); err != nil { + t.Fatalf("error starting observability with valid config through file system: %v", err) + } + defer End() +} + +// TestBothConfigEnvVarsSet tests the scenario where both configuration +// environment variables are set. The file system environment variable should +// take precedence, and an error should return in the case of the file system +// configuration being invalid, even if the direct configuration environment +// variable is set and valid. +func (s) TestBothConfigEnvVarsSet(t *testing.T) { + invalidConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{":-)"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + invalidConfigJSON, err := json.Marshal(invalidConfig) + if err != nil { + t.Fatalf("failed to convert config to JSON: %v", err) + } + cleanup, err := createTmpConfigInFileSystem(string(invalidConfigJSON)) + defer cleanup() + if err != nil { + t.Fatalf("failed to create config in file system: %v", err) + } + // This configuration should be ignored, as precedence 2. + validConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + validConfigJSON, err := json.Marshal(validConfig) + if err != nil { + t.Fatalf("failed to convert config to JSON: %v", err) + } + oldObservabilityConfig := envconfig.ObservabilityConfig + envconfig.ObservabilityConfig = string(validConfigJSON) + defer func() { + envconfig.ObservabilityConfig = oldObservabilityConfig + }() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := Start(ctx); err == nil { + t.Fatalf("Invalid patterns not triggering error") + } +} + +// TestErrInFileSystemEnvVar tests the scenario where an observability +// configuration is specified with environment variable that specifies a +// location in the file system for configuration, and this location doesn't have +// a file (or valid configuration). +func (s) TestErrInFileSystemEnvVar(t *testing.T) { + oldObservabilityConfigFile := envconfig.ObservabilityConfigFile + envconfig.ObservabilityConfigFile = "/this-file/does-not-exist" + defer func() { + envconfig.ObservabilityConfigFile = oldObservabilityConfigFile + }() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := Start(ctx); err == nil { + t.Fatalf("Invalid file system path not triggering error") + } +} + +func (s) TestNoEnvSet(t *testing.T) { + oldObservabilityConfig := envconfig.ObservabilityConfig + oldObservabilityConfigFile := envconfig.ObservabilityConfigFile + envconfig.ObservabilityConfig = "" + envconfig.ObservabilityConfigFile = "" + defer func() { + envconfig.ObservabilityConfig = oldObservabilityConfig + envconfig.ObservabilityConfigFile = oldObservabilityConfigFile + }() + // If there is no observability config set at all, the Start should return an error. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := Start(ctx); err == nil { + t.Fatalf("Invalid patterns not triggering error") + } +} + +func (s) TestOpenCensusIntegration(t *testing.T) { + defaultMetricsReportingInterval = time.Millisecond * 100 + fe := &fakeOpenCensusExporter{SeenViews: make(map[string]string), t: t} + + defer func(ne func(config *config) (tracingMetricsExporter, error)) { + newExporter = ne + }(newExporter) + + newExporter = func(*config) (tracingMetricsExporter, error) { + return fe, nil + } + + openCensusOnConfig := &config{ + ProjectID: "fake", + CloudMonitoring: &cloudMonitoring{}, + CloudTrace: &cloudTrace{ + SamplingRate: 1.0, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(openCensusOnConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + _, err := stream.Recv() + if err == io.EOF { + return nil + } + } + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + for i := 0; i < defaultRequestCount; i++ { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: testOkPayload}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + } + t.Logf("unary call passed count=%v", defaultRequestCount) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + + var errs []error + for ctx.Err() == nil { + errs = nil + fe.mu.RLock() + if value := fe.SeenViews["grpc.io/client/api_latency"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/client/api_latency: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if value := fe.SeenViews["grpc.io/client/started_rpcs"]; value != TypeOpenCensusViewCount { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/client/started_rpcs: %s != %s", value, TypeOpenCensusViewCount)) + } + if value := fe.SeenViews["grpc.io/server/started_rpcs"]; value != TypeOpenCensusViewCount { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/server/started_rpcs: %s != %s", value, TypeOpenCensusViewCount)) + } + + if value := fe.SeenViews["grpc.io/client/completed_rpcs"]; value != TypeOpenCensusViewCount { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/client/completed_rpcs: %s != %s", value, TypeOpenCensusViewCount)) + } + if value := fe.SeenViews["grpc.io/server/completed_rpcs"]; value != TypeOpenCensusViewCount { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/server/completed_rpcs: %s != %s", value, TypeOpenCensusViewCount)) + } + if value := fe.SeenViews["grpc.io/client/roundtrip_latency"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/client/completed_rpcs: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if value := fe.SeenViews["grpc.io/server/server_latency"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("grpc.io/server/server_latency: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if value := fe.SeenViews["grpc.io/client/sent_compressed_message_bytes_per_rpc"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/client/sent_compressed_message_bytes_per_rpc: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if value := fe.SeenViews["grpc.io/client/received_compressed_message_bytes_per_rpc"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/client/received_compressed_message_bytes_per_rpc: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if value := fe.SeenViews["grpc.io/server/sent_compressed_message_bytes_per_rpc"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/server/sent_compressed_message_bytes_per_rpc: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if value := fe.SeenViews["grpc.io/server/received_compressed_message_bytes_per_rpc"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/server/received_compressed_message_bytes_per_rpc: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if fe.SeenSpans <= 0 { + errs = append(errs, fmt.Errorf("unexpected number of seen spans: %v <= 0", fe.SeenSpans)) + } + fe.mu.RUnlock() + if len(errs) == 0 { + break + } + time.Sleep(100 * time.Millisecond) + } + if len(errs) != 0 { + t.Fatalf("Invalid OpenCensus export data: %v", errs) + } +} + +// TestCustomTagsTracingMetrics verifies that the custom tags defined in our +// observability configuration and set to two hardcoded values are passed to the +// function to create an exporter. +func (s) TestCustomTagsTracingMetrics(t *testing.T) { + defer func(ne func(config *config) (tracingMetricsExporter, error)) { + newExporter = ne + }(newExporter) + fe := &fakeOpenCensusExporter{SeenViews: make(map[string]string), t: t} + newExporter = func(config *config) (tracingMetricsExporter, error) { + ct := config.Labels + if len(ct) < 1 { + t.Fatalf("less than 2 custom tags sent in") + } + if val, ok := ct["customtag1"]; !ok || val != "wow" { + t.Fatalf("incorrect custom tag: got %v, want %v", val, "wow") + } + if val, ok := ct["customtag2"]; !ok || val != "nice" { + t.Fatalf("incorrect custom tag: got %v, want %v", val, "nice") + } + return fe, nil + } + + // This configuration present in file system and it's defined custom tags should make it + // to the created exporter. + configJSON := `{ + "project_id": "fake", + "cloud_trace": {}, + "cloud_monitoring": {"sampling_rate": 1.0}, + "labels":{"customtag1":"wow","customtag2":"nice"} + }` + + cleanup, err := createTmpConfigInFileSystem(configJSON) + if err != nil { + t.Fatalf("failed to create config in file system: %v", err) + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + err = Start(ctx) + defer End() + if err != nil { + t.Fatalf("Start() failed with err: %v", err) + } +} + +// TestStartErrorsThenEnd tests that an End call after Start errors works +// without problems, as this is a possible codepath in the public observability +// API. +func (s) TestStartErrorsThenEnd(t *testing.T) { + invalidConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{":-)"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + invalidConfigJSON, err := json.Marshal(invalidConfig) + if err != nil { + t.Fatalf("failed to convert config to JSON: %v", err) + } + oldObservabilityConfig := envconfig.ObservabilityConfig + oldObservabilityConfigFile := envconfig.ObservabilityConfigFile + envconfig.ObservabilityConfig = string(invalidConfigJSON) + envconfig.ObservabilityConfigFile = "" + defer func() { + envconfig.ObservabilityConfig = oldObservabilityConfig + envconfig.ObservabilityConfigFile = oldObservabilityConfigFile + }() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := Start(ctx); err == nil { + t.Fatalf("Invalid patterns not triggering error") + } + End() +} + +// TestLoggingLinkedWithTraceClientSide tests that client side logs get the +// trace and span id corresponding to the created Call Level Span for the RPC. +func (s) TestLoggingLinkedWithTraceClientSide(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + oldNewLoggingExporter := newLoggingExporter + defer func() { + newLoggingExporter = oldNewLoggingExporter + }() + + newLoggingExporter = func(context.Context, *config) (loggingExporter, error) { + return fle, nil + } + + idCh := testutils.NewChannel() + + fe := &fakeOpenCensusExporter{ + t: t, + idCh: idCh, + } + oldNewExporter := newExporter + defer func() { + newExporter = oldNewExporter + }() + + newExporter = func(*config) (tracingMetricsExporter, error) { + return fe, nil + } + + const projectID = "project-id" + tracesAndLogsConfig := &config{ + ProjectID: projectID, + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + CloudTrace: &cloudTrace{ + SamplingRate: 1.0, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(tracesAndLogsConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Spawn a goroutine to receive the trace and span ids received by the + // exporter corresponding to a Unary RPC. + readerErrCh := testutils.NewChannel() + unaryDone := grpcsync.NewEvent() + go func() { + var traceAndSpanIDs []traceAndSpanID + val, err := idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok := val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + <-unaryDone.Done() + var tasiSent traceAndSpanIDString + for _, tasi := range traceAndSpanIDs { + if strings.HasPrefix(tasi.spanName, "grpc.") && tasi.spanKind == trace.SpanKindClient { + tasiSent = tasi.idsToString(projectID) + continue + } + } + + fle.mu.Lock() + for _, tasiSeen := range fle.idsSeen { + if diff := cmp.Diff(tasiSeen, &tasiSent, cmp.AllowUnexported(traceAndSpanIDString{}), cmpopts.IgnoreFields(traceAndSpanIDString{}, "SpanKind")); diff != "" { + readerErrCh.Send(fmt.Errorf("got unexpected id, should be a client span (-got, +want): %v", diff)) + } + } + + fle.entries = nil + fle.mu.Unlock() + readerErrCh.Send(nil) + }() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: testOkPayload}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + unaryDone.Fire() + if chErr, err := readerErrCh.Receive(ctx); chErr != nil || err != nil { + if err != nil { + t.Fatalf("Should have received something from error channel: %v", err) + } + if chErr != nil { + t.Fatalf("Should have received a nil error from channel, instead received: %v", chErr) + } + } +} + +// TestLoggingLinkedWithTraceServerSide tests that server side logs get the +// trace and span id corresponding to the created Server Span for the RPC. +func (s) TestLoggingLinkedWithTraceServerSide(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + oldNewLoggingExporter := newLoggingExporter + defer func() { + newLoggingExporter = oldNewLoggingExporter + }() + + newLoggingExporter = func(context.Context, *config) (loggingExporter, error) { + return fle, nil + } + + idCh := testutils.NewChannel() + + fe := &fakeOpenCensusExporter{ + t: t, + idCh: idCh, + } + oldNewExporter := newExporter + defer func() { + newExporter = oldNewExporter + }() + + newExporter = func(*config) (tracingMetricsExporter, error) { + return fe, nil + } + + const projectID = "project-id" + tracesAndLogsConfig := &config{ + ProjectID: projectID, + CloudLogging: &cloudLogging{ + ServerRPCEvents: []serverRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + CloudTrace: &cloudTrace{ + SamplingRate: 1.0, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(tracesAndLogsConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Spawn a goroutine to receive the trace and span ids received by the + // exporter corresponding to a Unary RPC. + readerErrCh := testutils.NewChannel() + unaryDone := grpcsync.NewEvent() + go func() { + var traceAndSpanIDs []traceAndSpanID + val, err := idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok := val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + <-unaryDone.Done() + var tasiServer traceAndSpanIDString + for _, tasi := range traceAndSpanIDs { + if strings.HasPrefix(tasi.spanName, "grpc.") && tasi.spanKind == trace.SpanKindServer { + tasiServer = tasi.idsToString(projectID) + continue + } + } + + fle.mu.Lock() + for _, tasiSeen := range fle.idsSeen { + if diff := cmp.Diff(tasiSeen, &tasiServer, cmp.AllowUnexported(traceAndSpanIDString{}), cmpopts.IgnoreFields(traceAndSpanIDString{}, "SpanKind")); diff != "" { + readerErrCh.Send(fmt.Errorf("got unexpected id, should be a server span (-got, +want): %v", diff)) + } + } + + fle.entries = nil + fle.mu.Unlock() + readerErrCh.Send(nil) + }() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: testOkPayload}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + unaryDone.Fire() + if chErr, err := readerErrCh.Receive(ctx); chErr != nil || err != nil { + if err != nil { + t.Fatalf("Should have received something from error channel: %v", err) + } + if chErr != nil { + t.Fatalf("Should have received a nil error from channel, instead received: %v", chErr) + } + } +} + +// TestLoggingLinkedWithTrace tests that client and server side logs get the +// trace and span id corresponding to either the Call Level Span or Server Span +// (no determinism, so can only assert one or the other), for Unary and +// Streaming RPCs. +func (s) TestLoggingLinkedWithTrace(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + oldNewLoggingExporter := newLoggingExporter + defer func() { + newLoggingExporter = oldNewLoggingExporter + }() + + newLoggingExporter = func(context.Context, *config) (loggingExporter, error) { + return fle, nil + } + + idCh := testutils.NewChannel() + + fe := &fakeOpenCensusExporter{ + t: t, + idCh: idCh, + } + oldNewExporter := newExporter + defer func() { + newExporter = oldNewExporter + }() + + newExporter = func(*config) (tracingMetricsExporter, error) { + return fe, nil + } + + const projectID = "project-id" + tracesAndLogsConfig := &config{ + ProjectID: projectID, + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + ServerRPCEvents: []serverRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + CloudTrace: &cloudTrace{ + SamplingRate: 1.0, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(tracesAndLogsConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Spawn a goroutine to receive the trace and span ids received by the + // exporter corresponding to a Unary RPC. + readerErrCh := testutils.NewChannel() + unaryDone := grpcsync.NewEvent() + go func() { + var traceAndSpanIDs []traceAndSpanID + val, err := idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok := val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + <-unaryDone.Done() + var tasiSent traceAndSpanIDString + var tasiServer traceAndSpanIDString + for _, tasi := range traceAndSpanIDs { + if strings.HasPrefix(tasi.spanName, "grpc.") && tasi.spanKind == trace.SpanKindClient { + tasiSent = tasi.idsToString(projectID) + continue + } + if strings.HasPrefix(tasi.spanName, "grpc.") && tasi.spanKind == trace.SpanKindServer { + tasiServer = tasi.idsToString(projectID) + } + } + + fle.mu.Lock() + for _, tasiSeen := range fle.idsSeen { + if diff := cmp.Diff(tasiSeen, &tasiSent, cmp.AllowUnexported(traceAndSpanIDString{}), cmpopts.IgnoreFields(traceAndSpanIDString{}, "SpanKind")); diff != "" { + if diff2 := cmp.Diff(tasiSeen, &tasiServer, cmp.AllowUnexported(traceAndSpanIDString{}), cmpopts.IgnoreFields(traceAndSpanIDString{}, "SpanKind")); diff2 != "" { + readerErrCh.Send(fmt.Errorf("got unexpected id, should be a client or server span (-got, +want): %v, %v", diff, diff2)) + } + } + } + + fle.entries = nil + fle.mu.Unlock() + readerErrCh.Send(nil) + }() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: testOkPayload}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + unaryDone.Fire() + if chErr, err := readerErrCh.Receive(ctx); chErr != nil || err != nil { + if err != nil { + t.Fatalf("Should have received something from error channel: %v", err) + } + if chErr != nil { + t.Fatalf("Should have received a nil error from channel, instead received: %v", chErr) + } + } + + fle.mu.Lock() + fle.idsSeen = nil + fle.mu.Unlock() + + // Test streaming. Spawn a goroutine to receive the trace and span ids + // received by the exporter corresponding to a streaming RPC. + readerErrCh = testutils.NewChannel() + streamDone := grpcsync.NewEvent() + go func() { + var traceAndSpanIDs []traceAndSpanID + + val, err := idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok := val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + <-streamDone.Done() + var tasiSent traceAndSpanIDString + var tasiServer traceAndSpanIDString + for _, tasi := range traceAndSpanIDs { + if strings.HasPrefix(tasi.spanName, "grpc.") && tasi.spanKind == trace.SpanKindClient { + tasiSent = tasi.idsToString(projectID) + continue + } + if strings.HasPrefix(tasi.spanName, "grpc.") && tasi.spanKind == trace.SpanKindServer { + tasiServer = tasi.idsToString(projectID) + } + } + + fle.mu.Lock() + for _, tasiSeen := range fle.idsSeen { + if diff := cmp.Diff(tasiSeen, &tasiSent, cmp.AllowUnexported(traceAndSpanIDString{}), cmpopts.IgnoreFields(traceAndSpanIDString{}, "SpanKind")); diff != "" { + if diff2 := cmp.Diff(tasiSeen, &tasiServer, cmp.AllowUnexported(traceAndSpanIDString{}), cmpopts.IgnoreFields(traceAndSpanIDString{}, "SpanKind")); diff2 != "" { + readerErrCh.Send(fmt.Errorf("got unexpected id, should be a client or server span (-got, +want): %v, %v", diff, diff2)) + } + } + } + + fle.entries = nil + fle.mu.Unlock() + readerErrCh.Send(nil) + }() + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + streamDone.Fire() + + if chErr, err := readerErrCh.Receive(ctx); chErr != nil || err != nil { + if err != nil { + t.Fatalf("Should have received something from error channel: %v", err) + } + if chErr != nil { + t.Fatalf("Should have received a nil error from channel, instead received: %v", chErr) + } + } +} diff --git a/gcp/observability/opencensus.go b/gcp/observability/opencensus.go new file mode 100644 index 0000000..b0451e4 --- /dev/null +++ b/gcp/observability/opencensus.go @@ -0,0 +1,179 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package observability + +import ( + "fmt" + "os" + "strconv" + "time" + + "contrib.go.opencensus.io/exporter/stackdriver" + "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" + + "go.opencensus.io/stats/view" + "go.opencensus.io/trace" + "google.golang.org/grpc" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/stats/opencensus" +) + +var ( + // It's a variable instead of const to speed up testing + defaultMetricsReportingInterval = time.Second * 30 + defaultViews = []*view.View{ + opencensus.ClientStartedRPCsView, + opencensus.ClientCompletedRPCsView, + opencensus.ClientRoundtripLatencyView, + opencensus.ClientSentCompressedMessageBytesPerRPCView, + opencensus.ClientReceivedCompressedMessageBytesPerRPCView, + opencensus.ClientAPILatencyView, + opencensus.ServerStartedRPCsView, + opencensus.ServerCompletedRPCsView, + opencensus.ServerSentCompressedMessageBytesPerRPCView, + opencensus.ServerReceivedCompressedMessageBytesPerRPCView, + opencensus.ServerLatencyView, + } +) + +func labelsToMonitoringLabels(labels map[string]string) *stackdriver.Labels { + sdLabels := &stackdriver.Labels{} + for k, v := range labels { + sdLabels.Set(k, v, "") + } + return sdLabels +} + +func labelsToTraceAttributes(labels map[string]string) map[string]any { + ta := make(map[string]any, len(labels)) + for k, v := range labels { + ta[k] = v + } + return ta +} + +type tracingMetricsExporter interface { + trace.Exporter + view.Exporter + Flush() + Close() error +} + +var exporter tracingMetricsExporter + +var newExporter = newStackdriverExporter + +func newStackdriverExporter(config *config) (tracingMetricsExporter, error) { + // Create the Stackdriver exporter, which is shared between tracing and stats + mr := monitoredresource.Autodetect() + logger.Infof("Detected MonitoredResource:: %+v", mr) + var err error + // Custom labels completely overwrite any labels generated in the OpenCensus + // library, including their label that uniquely identifies the process. + // Thus, generate a unique process identifier here to uniquely identify + // process for metrics exporting to function correctly. + metricsLabels := make(map[string]string, len(config.Labels)+1) + for k, v := range config.Labels { + metricsLabels[k] = v + } + metricsLabels["opencensus_task"] = generateUniqueProcessIdentifier() + exporter, err := stackdriver.NewExporter(stackdriver.Options{ + ProjectID: config.ProjectID, + MonitoredResource: mr, + DefaultMonitoringLabels: labelsToMonitoringLabels(metricsLabels), + DefaultTraceAttributes: labelsToTraceAttributes(config.Labels), + MonitoringClientOptions: cOptsDisableLogTrace, + TraceClientOptions: cOptsDisableLogTrace, + }) + if err != nil { + return nil, fmt.Errorf("failed to create Stackdriver exporter: %v", err) + } + return exporter, nil +} + +// generateUniqueProcessIdentifier returns a unique process identifier for the +// process this code is running in. This is the same way the OpenCensus library +// generates the unique process identifier, in the format of +// "go-@". +func generateUniqueProcessIdentifier() string { + hostname, err := os.Hostname() + if err != nil { + hostname = "localhost" + } + return "go-" + strconv.Itoa(os.Getpid()) + "@" + hostname +} + +// This method accepts config and exporter; the exporter argument is exposed to +// assist unit testing of the OpenCensus behavior. +func startOpenCensus(config *config) error { + // If both tracing and metrics are disabled, there's no point inject default + // StatsHandler. + if config == nil || (config.CloudTrace == nil && config.CloudMonitoring == nil) { + return nil + } + + var err error + exporter, err = newExporter(config) + if err != nil { + return err + } + + var to opencensus.TraceOptions + if config.CloudTrace != nil { + to.TS = trace.ProbabilitySampler(config.CloudTrace.SamplingRate) + trace.RegisterExporter(exporter.(trace.Exporter)) + logger.Infof("Start collecting and exporting trace spans with global_trace_sampling_rate=%.2f", config.CloudTrace.SamplingRate) + } + + if config.CloudMonitoring != nil { + if err := view.Register(defaultViews...); err != nil { + return fmt.Errorf("failed to register observability views: %v", err) + } + view.SetReportingPeriod(defaultMetricsReportingInterval) + view.RegisterExporter(exporter.(view.Exporter)) + logger.Infof("Start collecting and exporting metrics") + } + + internal.AddGlobalServerOptions.(func(opt ...grpc.ServerOption))(opencensus.ServerOption(to)) + internal.AddGlobalDialOptions.(func(opt ...grpc.DialOption))(opencensus.DialOption(to)) + logger.Infof("Enabled OpenCensus StatsHandlers for clients and servers") + + return nil +} + +// stopOpenCensus flushes the exporter's and cleans up globals across all +// packages if exporter was created. +func stopOpenCensus() { + if exporter != nil { + internal.ClearGlobalDialOptions() + internal.ClearGlobalServerOptions() + // This Unregister call guarantees the data recorded gets sent to + // exporter, synchronising the view package and exporter. Doesn't matter + // if views not registered, will be a noop if not registered. + view.Unregister(defaultViews...) + // Call these unconditionally, doesn't matter if not registered, will be + // a noop if not registered. + trace.UnregisterExporter(exporter) + view.UnregisterExporter(exporter) + + // This Flush call makes sure recorded telemetry get sent to backend. + exporter.Flush() + exporter.Close() + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6ef5829 --- /dev/null +++ b/go.mod @@ -0,0 +1,54 @@ +module google.golang.org/grpc + +go 1.25.0 + +require ( + cloud.google.com/go/auth v0.23.1 + cloud.google.com/go/compute/metadata v0.9.0 + github.com/cespare/xxhash/v2 v2.3.0 + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 + github.com/envoyproxy/go-control-plane v0.14.0 + github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be + github.com/golang/glog v1.2.5 + github.com/golang/protobuf v1.5.4 + github.com/google/go-cmp v0.7.0 + github.com/google/uuid v1.6.0 + github.com/spiffe/go-spiffe/v2 v2.8.1 + go.opentelemetry.io/contrib/detectors/gcp v1.45.0 + go.opentelemetry.io/otel v1.45.0 + go.opentelemetry.io/otel/metric v1.45.0 + go.opentelemetry.io/otel/sdk v1.45.0 + go.opentelemetry.io/otel/sdk/metric v1.45.0 + go.opentelemetry.io/otel/trace v1.45.0 + golang.org/x/net v0.58.0 + golang.org/x/oauth2 v0.36.0 + golang.org/x/sync v0.22.0 + golang.org/x/sys v0.47.0 + gonum.org/v1/gonum v0.17.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 + google.golang.org/protobuf v1.36.12 +) + +require ( + cel.dev/expr v0.25.3 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect + github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.21 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/api v0.293.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1 // indirect +) + +// v1.74.0 was published prematurely with known issues. +retract [v1.74.0, v1.74.1] diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..ba800b6 --- /dev/null +++ b/go.sum @@ -0,0 +1,99 @@ +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go/auth v0.23.1 h1:1tPpBPG02lQHmoiAvs9egyCASqXP0xgobptjZzov/Jg= +cloud.google.com/go/auth v0.23.1/go.mod h1:4DhBRcqvtljQN3dJ57qtqbib5ZGCYE5f2crfiiC2EM0= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 h1:bN1gA3of5bXtbnLsRPrwfmbbe7A5UWFlcTHseujLnpc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0/go.mod h1:Yj5vHEz/aAepZGliRJsA6uvHAVAQyEwajq9ORCHPxzM= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be h1:SWe0x6yfglnxuvOiYgTTnNq7QD/yvqthOh1RBI8Bj8w= +github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be/go.mod h1:PYEOlng9XcrulfyWpm49jECTPV0LT4q8cO7fLW/xwgk= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= +github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.21 h1:OFdQ3tnCX/zaQ0Cedur3D3z7kI6HiLX9g3TiAN4/DFU= +github.com/googleapis/enterprise-certificate-proxy v0.3.21/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0 h1:9jR0ZPRok9ryaOQ2Wx8rg5F7Aon59mxrqbVI60/vlBk= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0/go.mod h1:VSme3o2fvSg5bVg0dRzyHaj4Z5EVhG+g2Fde6LKzmQA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= +go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.293.0 h1:p9XIWOf63U4OgYx120ZwVU8+vl4XTPmWfgVPnmOAS9w= +google.golang.org/api v0.293.0/go.mod h1:6n5tjEB1gzwniZTepZ0g5u+wM7Bof5GeULCx/zh8ZE0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1 h1:lrupDmKL3p5kEX1M92oan027eCKcouzjuPbH6YBK+Rs= +google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:q/3oV3jAi5vwelxsVAprMBC8BcM2zmNe+IjRGd+9/ks= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 h1:utmqiUzZAEINbLmRyAt1QjVkMyLQ+lvIwCpLaB1rZfw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/grpc_test.go b/grpc_test.go new file mode 100644 index 0000000..49ad9e0 --- /dev/null +++ b/grpc_test.go @@ -0,0 +1,33 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "testing" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} diff --git a/grpclog/component.go b/grpclog/component.go new file mode 100644 index 0000000..f1ae080 --- /dev/null +++ b/grpclog/component.go @@ -0,0 +1,115 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpclog + +import ( + "fmt" +) + +// componentData records the settings for a component. +type componentData struct { + name string +} + +var cache = map[string]*componentData{} + +func (c *componentData) InfoDepth(depth int, args ...any) { + args = append([]any{"[" + string(c.name) + "]"}, args...) + InfoDepth(depth+1, args...) +} + +func (c *componentData) WarningDepth(depth int, args ...any) { + args = append([]any{"[" + string(c.name) + "]"}, args...) + WarningDepth(depth+1, args...) +} + +func (c *componentData) ErrorDepth(depth int, args ...any) { + args = append([]any{"[" + string(c.name) + "]"}, args...) + ErrorDepth(depth+1, args...) +} + +func (c *componentData) FatalDepth(depth int, args ...any) { + args = append([]any{"[" + string(c.name) + "]"}, args...) + FatalDepth(depth+1, args...) +} + +func (c *componentData) Info(args ...any) { + c.InfoDepth(1, args...) +} + +func (c *componentData) Warning(args ...any) { + c.WarningDepth(1, args...) +} + +func (c *componentData) Error(args ...any) { + c.ErrorDepth(1, args...) +} + +func (c *componentData) Fatal(args ...any) { + c.FatalDepth(1, args...) +} + +func (c *componentData) Infof(format string, args ...any) { + c.InfoDepth(1, fmt.Sprintf(format, args...)) +} + +func (c *componentData) Warningf(format string, args ...any) { + c.WarningDepth(1, fmt.Sprintf(format, args...)) +} + +func (c *componentData) Errorf(format string, args ...any) { + c.ErrorDepth(1, fmt.Sprintf(format, args...)) +} + +func (c *componentData) Fatalf(format string, args ...any) { + c.FatalDepth(1, fmt.Sprintf(format, args...)) +} + +func (c *componentData) Infoln(args ...any) { + c.InfoDepth(1, args...) +} + +func (c *componentData) Warningln(args ...any) { + c.WarningDepth(1, args...) +} + +func (c *componentData) Errorln(args ...any) { + c.ErrorDepth(1, args...) +} + +func (c *componentData) Fatalln(args ...any) { + c.FatalDepth(1, args...) +} + +func (c *componentData) V(l int) bool { + return V(l) +} + +// Component creates a new component and returns it for logging. If a component +// with the name already exists, nothing will be created and it will be +// returned. SetLoggerV2 will panic if it is called with a logger created by +// Component. +func Component(componentName string) DepthLoggerV2 { + if cData, ok := cache[componentName]; ok { + return cData + } + c := &componentData{componentName} + cache[componentName] = c + return c +} diff --git a/grpclog/glogger/glogger.go b/grpclog/glogger/glogger.go new file mode 100644 index 0000000..fdabb60 --- /dev/null +++ b/grpclog/glogger/glogger.go @@ -0,0 +1,104 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package glogger defines glog-based logging for grpc. +// Importing this package will install glog as the logger used by grpclog. +package glogger + +import ( + "fmt" + + "github.com/golang/glog" + "google.golang.org/grpc/grpclog" +) + +const d = 2 + +func init() { + grpclog.SetLoggerV2(&glogger{}) +} + +type glogger struct{} + +func (g *glogger) Info(args ...any) { + glog.InfoDepth(d, args...) +} + +func (g *glogger) Infoln(args ...any) { + glog.InfoDepth(d, fmt.Sprintln(args...)) +} + +func (g *glogger) Infof(format string, args ...any) { + glog.InfoDepth(d, fmt.Sprintf(format, args...)) +} + +func (g *glogger) InfoDepth(depth int, args ...any) { + glog.InfoDepth(depth+d, args...) +} + +func (g *glogger) Warning(args ...any) { + glog.WarningDepth(d, args...) +} + +func (g *glogger) Warningln(args ...any) { + glog.WarningDepth(d, fmt.Sprintln(args...)) +} + +func (g *glogger) Warningf(format string, args ...any) { + glog.WarningDepth(d, fmt.Sprintf(format, args...)) +} + +func (g *glogger) WarningDepth(depth int, args ...any) { + glog.WarningDepth(depth+d, args...) +} + +func (g *glogger) Error(args ...any) { + glog.ErrorDepth(d, args...) +} + +func (g *glogger) Errorln(args ...any) { + glog.ErrorDepth(d, fmt.Sprintln(args...)) +} + +func (g *glogger) Errorf(format string, args ...any) { + glog.ErrorDepth(d, fmt.Sprintf(format, args...)) +} + +func (g *glogger) ErrorDepth(depth int, args ...any) { + glog.ErrorDepth(depth+d, args...) +} + +func (g *glogger) Fatal(args ...any) { + glog.FatalDepth(d, args...) +} + +func (g *glogger) Fatalln(args ...any) { + glog.FatalDepth(d, fmt.Sprintln(args...)) +} + +func (g *glogger) Fatalf(format string, args ...any) { + glog.FatalDepth(d, fmt.Sprintf(format, args...)) +} + +func (g *glogger) FatalDepth(depth int, args ...any) { + glog.FatalDepth(depth+d, args...) +} + +func (g *glogger) V(l int) bool { + return bool(glog.V(glog.Level(l))) +} diff --git a/grpclog/grpclog.go b/grpclog/grpclog.go new file mode 100644 index 0000000..db32010 --- /dev/null +++ b/grpclog/grpclog.go @@ -0,0 +1,186 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpclog defines logging for grpc. +// +// In the default logger, severity level can be set by environment variable +// GRPC_GO_LOG_SEVERITY_LEVEL, verbosity level can be set by +// GRPC_GO_LOG_VERBOSITY_LEVEL. +package grpclog + +import ( + "os" + + "google.golang.org/grpc/grpclog/internal" +) + +func init() { + SetLoggerV2(newLoggerV2()) +} + +// V reports whether verbosity level l is at least the requested verbose level. +func V(l int) bool { + return internal.LoggerV2Impl.V(l) +} + +// Info logs to the INFO log. +func Info(args ...any) { + internal.LoggerV2Impl.Info(args...) +} + +// Infof logs to the INFO log. Arguments are handled in the manner of fmt.Printf. +func Infof(format string, args ...any) { + internal.LoggerV2Impl.Infof(format, args...) +} + +// Infoln logs to the INFO log. Arguments are handled in the manner of fmt.Println. +func Infoln(args ...any) { + internal.LoggerV2Impl.Infoln(args...) +} + +// Warning logs to the WARNING log. +func Warning(args ...any) { + internal.LoggerV2Impl.Warning(args...) +} + +// Warningf logs to the WARNING log. Arguments are handled in the manner of fmt.Printf. +func Warningf(format string, args ...any) { + internal.LoggerV2Impl.Warningf(format, args...) +} + +// Warningln logs to the WARNING log. Arguments are handled in the manner of fmt.Println. +func Warningln(args ...any) { + internal.LoggerV2Impl.Warningln(args...) +} + +// Error logs to the ERROR log. +func Error(args ...any) { + internal.LoggerV2Impl.Error(args...) +} + +// Errorf logs to the ERROR log. Arguments are handled in the manner of fmt.Printf. +func Errorf(format string, args ...any) { + internal.LoggerV2Impl.Errorf(format, args...) +} + +// Errorln logs to the ERROR log. Arguments are handled in the manner of fmt.Println. +func Errorln(args ...any) { + internal.LoggerV2Impl.Errorln(args...) +} + +// Fatal logs to the FATAL log. Arguments are handled in the manner of fmt.Print. +// It calls os.Exit() with exit code 1. +func Fatal(args ...any) { + internal.LoggerV2Impl.Fatal(args...) + // Make sure fatal logs will exit. + os.Exit(1) +} + +// Fatalf logs to the FATAL log. Arguments are handled in the manner of fmt.Printf. +// It calls os.Exit() with exit code 1. +func Fatalf(format string, args ...any) { + internal.LoggerV2Impl.Fatalf(format, args...) + // Make sure fatal logs will exit. + os.Exit(1) +} + +// Fatalln logs to the FATAL log. Arguments are handled in the manner of fmt.Println. +// It calls os.Exit() with exit code 1. +func Fatalln(args ...any) { + internal.LoggerV2Impl.Fatalln(args...) + // Make sure fatal logs will exit. + os.Exit(1) +} + +// Print prints to the logger. Arguments are handled in the manner of fmt.Print. +// +// Deprecated: use Info. +func Print(args ...any) { + internal.LoggerV2Impl.Info(args...) +} + +// Printf prints to the logger. Arguments are handled in the manner of fmt.Printf. +// +// Deprecated: use Infof. +func Printf(format string, args ...any) { + internal.LoggerV2Impl.Infof(format, args...) +} + +// Println prints to the logger. Arguments are handled in the manner of fmt.Println. +// +// Deprecated: use Infoln. +func Println(args ...any) { + internal.LoggerV2Impl.Infoln(args...) +} + +// InfoDepth logs to the INFO log at the specified depth. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func InfoDepth(depth int, args ...any) { + if internal.DepthLoggerV2Impl != nil { + internal.DepthLoggerV2Impl.InfoDepth(depth, args...) + } else { + internal.LoggerV2Impl.Infoln(args...) + } +} + +// WarningDepth logs to the WARNING log at the specified depth. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func WarningDepth(depth int, args ...any) { + if internal.DepthLoggerV2Impl != nil { + internal.DepthLoggerV2Impl.WarningDepth(depth, args...) + } else { + internal.LoggerV2Impl.Warningln(args...) + } +} + +// ErrorDepth logs to the ERROR log at the specified depth. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func ErrorDepth(depth int, args ...any) { + if internal.DepthLoggerV2Impl != nil { + internal.DepthLoggerV2Impl.ErrorDepth(depth, args...) + } else { + internal.LoggerV2Impl.Errorln(args...) + } +} + +// FatalDepth logs to the FATAL log at the specified depth. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func FatalDepth(depth int, args ...any) { + if internal.DepthLoggerV2Impl != nil { + internal.DepthLoggerV2Impl.FatalDepth(depth, args...) + } else { + internal.LoggerV2Impl.Fatalln(args...) + } + os.Exit(1) +} diff --git a/grpclog/internal/grpclog.go b/grpclog/internal/grpclog.go new file mode 100644 index 0000000..59c03bc --- /dev/null +++ b/grpclog/internal/grpclog.go @@ -0,0 +1,26 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains functionality internal to the grpclog package. +package internal + +// LoggerV2Impl is the logger used for the non-depth log functions. +var LoggerV2Impl LoggerV2 + +// DepthLoggerV2Impl is the logger used for the depth log functions. +var DepthLoggerV2Impl DepthLoggerV2 diff --git a/grpclog/internal/logger.go b/grpclog/internal/logger.go new file mode 100644 index 0000000..e524fdd --- /dev/null +++ b/grpclog/internal/logger.go @@ -0,0 +1,87 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package internal + +// Logger mimics golang's standard Logger as an interface. +// +// Deprecated: use LoggerV2. +type Logger interface { + Fatal(args ...any) + Fatalf(format string, args ...any) + Fatalln(args ...any) + Print(args ...any) + Printf(format string, args ...any) + Println(args ...any) +} + +// LoggerWrapper wraps Logger into a LoggerV2. +type LoggerWrapper struct { + Logger +} + +// Info logs to INFO log. Arguments are handled in the manner of fmt.Print. +func (l *LoggerWrapper) Info(args ...any) { + l.Logger.Print(args...) +} + +// Infoln logs to INFO log. Arguments are handled in the manner of fmt.Println. +func (l *LoggerWrapper) Infoln(args ...any) { + l.Logger.Println(args...) +} + +// Infof logs to INFO log. Arguments are handled in the manner of fmt.Printf. +func (l *LoggerWrapper) Infof(format string, args ...any) { + l.Logger.Printf(format, args...) +} + +// Warning logs to WARNING log. Arguments are handled in the manner of fmt.Print. +func (l *LoggerWrapper) Warning(args ...any) { + l.Logger.Print(args...) +} + +// Warningln logs to WARNING log. Arguments are handled in the manner of fmt.Println. +func (l *LoggerWrapper) Warningln(args ...any) { + l.Logger.Println(args...) +} + +// Warningf logs to WARNING log. Arguments are handled in the manner of fmt.Printf. +func (l *LoggerWrapper) Warningf(format string, args ...any) { + l.Logger.Printf(format, args...) +} + +// Error logs to ERROR log. Arguments are handled in the manner of fmt.Print. +func (l *LoggerWrapper) Error(args ...any) { + l.Logger.Print(args...) +} + +// Errorln logs to ERROR log. Arguments are handled in the manner of fmt.Println. +func (l *LoggerWrapper) Errorln(args ...any) { + l.Logger.Println(args...) +} + +// Errorf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. +func (l *LoggerWrapper) Errorf(format string, args ...any) { + l.Logger.Printf(format, args...) +} + +// V reports whether verbosity level l is at least the requested verbose level. +func (*LoggerWrapper) V(int) bool { + // Returns true for all verbose level. + return true +} diff --git a/grpclog/internal/loggerv2.go b/grpclog/internal/loggerv2.go new file mode 100644 index 0000000..ed90060 --- /dev/null +++ b/grpclog/internal/loggerv2.go @@ -0,0 +1,267 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package internal + +import ( + "encoding/json" + "fmt" + "io" + "log" + "os" +) + +// LoggerV2 does underlying logging work for grpclog. +type LoggerV2 interface { + // Info logs to INFO log. Arguments are handled in the manner of fmt.Print. + Info(args ...any) + // Infoln logs to INFO log. Arguments are handled in the manner of fmt.Println. + Infoln(args ...any) + // Infof logs to INFO log. Arguments are handled in the manner of fmt.Printf. + Infof(format string, args ...any) + // Warning logs to WARNING log. Arguments are handled in the manner of fmt.Print. + Warning(args ...any) + // Warningln logs to WARNING log. Arguments are handled in the manner of fmt.Println. + Warningln(args ...any) + // Warningf logs to WARNING log. Arguments are handled in the manner of fmt.Printf. + Warningf(format string, args ...any) + // Error logs to ERROR log. Arguments are handled in the manner of fmt.Print. + Error(args ...any) + // Errorln logs to ERROR log. Arguments are handled in the manner of fmt.Println. + Errorln(args ...any) + // Errorf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. + Errorf(format string, args ...any) + // Fatal logs to ERROR log. Arguments are handled in the manner of fmt.Print. + // gRPC ensures that all Fatal logs will exit with os.Exit(1). + // Implementations may also call os.Exit() with a non-zero exit code. + Fatal(args ...any) + // Fatalln logs to ERROR log. Arguments are handled in the manner of fmt.Println. + // gRPC ensures that all Fatal logs will exit with os.Exit(1). + // Implementations may also call os.Exit() with a non-zero exit code. + Fatalln(args ...any) + // Fatalf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. + // gRPC ensures that all Fatal logs will exit with os.Exit(1). + // Implementations may also call os.Exit() with a non-zero exit code. + Fatalf(format string, args ...any) + // V reports whether verbosity level l is at least the requested verbose level. + V(l int) bool +} + +// DepthLoggerV2 logs at a specified call frame. If a LoggerV2 also implements +// DepthLoggerV2, the below functions will be called with the appropriate stack +// depth set for trivial functions the logger may ignore. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type DepthLoggerV2 interface { + LoggerV2 + // InfoDepth logs to INFO log at the specified depth. Arguments are handled in the manner of fmt.Println. + InfoDepth(depth int, args ...any) + // WarningDepth logs to WARNING log at the specified depth. Arguments are handled in the manner of fmt.Println. + WarningDepth(depth int, args ...any) + // ErrorDepth logs to ERROR log at the specified depth. Arguments are handled in the manner of fmt.Println. + ErrorDepth(depth int, args ...any) + // FatalDepth logs to FATAL log at the specified depth. Arguments are handled in the manner of fmt.Println. + FatalDepth(depth int, args ...any) +} + +const ( + // infoLog indicates Info severity. + infoLog int = iota + // warningLog indicates Warning severity. + warningLog + // errorLog indicates Error severity. + errorLog + // fatalLog indicates Fatal severity. + fatalLog +) + +// severityName contains the string representation of each severity. +var severityName = []string{ + infoLog: "INFO", + warningLog: "WARNING", + errorLog: "ERROR", + fatalLog: "FATAL", +} + +// sprintf is fmt.Sprintf. +// These vars exist to make it possible to test that expensive format calls aren't made unnecessarily. +var sprintf = fmt.Sprintf + +// sprint is fmt.Sprint. +// These vars exist to make it possible to test that expensive format calls aren't made unnecessarily. +var sprint = fmt.Sprint + +// sprintln is fmt.Sprintln. +// These vars exist to make it possible to test that expensive format calls aren't made unnecessarily. +var sprintln = fmt.Sprintln + +// exit is os.Exit. +// This var exists to make it possible to test functions calling os.Exit. +var exit = os.Exit + +// loggerT is the default logger used by grpclog. +type loggerT struct { + m []*log.Logger + v int + jsonFormat bool +} + +func (g *loggerT) output(severity int, s string) { + sevStr := severityName[severity] + if !g.jsonFormat { + g.m[severity].Output(2, sevStr+": "+s) + return + } + // TODO: we can also include the logging component, but that needs more + // (API) changes. + b, _ := json.Marshal(map[string]string{ + "severity": sevStr, + "message": s, + }) + g.m[severity].Output(2, string(b)) +} + +func (g *loggerT) printf(severity int, format string, args ...any) { + // Note the discard check is duplicated in each print func, rather than in + // output, to avoid the expensive Sprint calls. + // De-duplicating this by moving to output would be a significant performance regression! + if lg := g.m[severity]; lg.Writer() == io.Discard { + return + } + g.output(severity, sprintf(format, args...)) +} + +func (g *loggerT) print(severity int, v ...any) { + if lg := g.m[severity]; lg.Writer() == io.Discard { + return + } + g.output(severity, sprint(v...)) +} + +func (g *loggerT) println(severity int, v ...any) { + if lg := g.m[severity]; lg.Writer() == io.Discard { + return + } + g.output(severity, sprintln(v...)) +} + +func (g *loggerT) Info(args ...any) { + g.print(infoLog, args...) +} + +func (g *loggerT) Infoln(args ...any) { + g.println(infoLog, args...) +} + +func (g *loggerT) Infof(format string, args ...any) { + g.printf(infoLog, format, args...) +} + +func (g *loggerT) Warning(args ...any) { + g.print(warningLog, args...) +} + +func (g *loggerT) Warningln(args ...any) { + g.println(warningLog, args...) +} + +func (g *loggerT) Warningf(format string, args ...any) { + g.printf(warningLog, format, args...) +} + +func (g *loggerT) Error(args ...any) { + g.print(errorLog, args...) +} + +func (g *loggerT) Errorln(args ...any) { + g.println(errorLog, args...) +} + +func (g *loggerT) Errorf(format string, args ...any) { + g.printf(errorLog, format, args...) +} + +func (g *loggerT) Fatal(args ...any) { + g.print(fatalLog, args...) + exit(1) +} + +func (g *loggerT) Fatalln(args ...any) { + g.println(fatalLog, args...) + exit(1) +} + +func (g *loggerT) Fatalf(format string, args ...any) { + g.printf(fatalLog, format, args...) + exit(1) +} + +func (g *loggerT) V(l int) bool { + return l <= g.v +} + +// LoggerV2Config configures the LoggerV2 implementation. +type LoggerV2Config struct { + // Verbosity sets the verbosity level of the logger. + Verbosity int + // FormatJSON controls whether the logger should output logs in JSON format. + FormatJSON bool +} + +// combineLoggers returns a combined logger for both higher & lower severity logs, +// or only one if the other is io.Discard. +// +// This uses io.Discard instead of io.MultiWriter when all loggers +// are set to io.Discard. Both this package and the standard log package have +// significant optimizations for io.Discard, which io.MultiWriter lacks (as of +// this writing). +func combineLoggers(lower, higher io.Writer) io.Writer { + if lower == io.Discard { + return higher + } + if higher == io.Discard { + return lower + } + return io.MultiWriter(lower, higher) +} + +// NewLoggerV2 creates a new LoggerV2 instance with the provided configuration. +// The infoW, warningW, and errorW writers are used to write log messages of +// different severity levels. +func NewLoggerV2(infoW, warningW, errorW io.Writer, c LoggerV2Config) LoggerV2 { + flag := log.LstdFlags + if c.FormatJSON { + flag = 0 + } + + warningW = combineLoggers(infoW, warningW) + errorW = combineLoggers(errorW, warningW) + + fatalW := errorW + + m := []*log.Logger{ + log.New(infoW, "", flag), + log.New(warningW, "", flag), + log.New(errorW, "", flag), + log.New(fatalW, "", flag), + } + return &loggerT{m: m, v: c.Verbosity, jsonFormat: c.FormatJSON} +} diff --git a/grpclog/internal/loggerv2_test.go b/grpclog/internal/loggerv2_test.go new file mode 100644 index 0000000..3369448 --- /dev/null +++ b/grpclog/internal/loggerv2_test.go @@ -0,0 +1,350 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package internal + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "reflect" + "regexp" + "strings" + "testing" +) + +// logFuncStr is a string used via testCheckLogContainsFuncStr to test the +// logger output. +const logFuncStr = "called-func" + +func makeSprintfErr(t *testing.T) func(format string, a ...any) string { + return func(string, ...any) string { + t.Errorf("got: sprintf called on io.Discard logger, want: expensive sprintf to not be called for io.Discard") + return "" + } +} + +func makeSprintErr(t *testing.T) func(a ...any) string { + return func(...any) string { + t.Errorf("got: sprint called on io.Discard logger, want: expensive sprint to not be called for io.Discard") + return "" + } +} + +// checkLogContainsFuncStr checks that the logger buffer logBuf contains +// logFuncStr. +func checkLogContainsFuncStr(t *testing.T, logBuf []byte) { + if !bytes.Contains(logBuf, []byte(logFuncStr)) { + t.Errorf("got '%v', want logger func to be called and print '%v'", string(logBuf), logFuncStr) + } +} + +// checkBufferWasWrittenAsExpected checks that the log buffer buf was written as expected, +// per the discard, logTYpe, msg, and isJSON arguments. +func checkBufferWasWrittenAsExpected(t *testing.T, buf *bytes.Buffer, discard bool, logType string, msg string, isJSON bool) { + bts, err := buf.ReadBytes('\n') + if discard { + if err == nil { + t.Fatalf("got '%v', want discard %v to not write", string(bts), logType) + } else if err != io.EOF { + t.Fatalf("got '%v', want discard %v buffer to be EOF", err, logType) + } + } else { + if err != nil { + t.Fatalf("got '%v', want non-discard %v to not error", err, logType) + } else if !bytes.Contains(bts, []byte(msg)) { + t.Fatalf("got '%v', want non-discard %v buffer contain message '%v'", string(bts), logType, msg) + } + if isJSON { + obj := map[string]string{} + if err := json.Unmarshal(bts, &obj); err != nil { + t.Fatalf("got '%v', want non-discard json %v to unmarshal", err, logType) + } else if _, ok := obj["severity"]; !ok { + t.Fatalf("got '%v', want non-discard json %v to have severity field", "missing severity", logType) + + } else if jsonMsg, ok := obj["message"]; !ok { + t.Fatalf("got '%v', want non-discard json %v to have message field", "missing message", logType) + + } else if !strings.Contains(jsonMsg, msg) { + t.Fatalf("got '%v', want non-discard json %v buffer contain message '%v'", string(bts), logType, msg) + } + } + } +} + +// check if b is in the format of: +// +// 2017/04/07 14:55:42 WARNING: WARNING +func checkLogForSeverity(s int, b []byte) error { + expected := regexp.MustCompile(fmt.Sprintf(`^[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} %s: %s\n$`, severityName[s], severityName[s])) + if m := expected.Match(b); !m { + return fmt.Errorf("got: %v, want string in format of: %v", string(b), severityName[s]+": 2016/10/05 17:09:26 "+severityName[s]) + } + return nil +} + +func TestLoggerV2Severity(t *testing.T) { + buffers := []*bytes.Buffer{new(bytes.Buffer), new(bytes.Buffer), new(bytes.Buffer)} + l := NewLoggerV2(buffers[infoLog], buffers[warningLog], buffers[errorLog], LoggerV2Config{}) + + l.Info(severityName[infoLog]) + l.Warning(severityName[warningLog]) + l.Error(severityName[errorLog]) + + for i := 0; i < fatalLog; i++ { + buf := buffers[i] + // The content of info buffer should be something like: + // INFO: 2017/04/07 14:55:42 INFO + // WARNING: 2017/04/07 14:55:42 WARNING + // ERROR: 2017/04/07 14:55:42 ERROR + for j := i; j < fatalLog; j++ { + b, err := buf.ReadBytes('\n') + if err != nil { + t.Fatalf("level %d: %v", j, err) + } + if err := checkLogForSeverity(j, b); err != nil { + t.Fatal(err) + } + } + } +} + +// TestLoggerV2PrintFuncDiscardOnlyInfo ensures that logs at the INFO level are +// discarded when set to io.Discard, while logs at other levels (WARN, ERROR) +// are still printed. It does this by using a custom error function that raises +// an error if the logger attempts to print at the INFO level, ensuring early +// return when io.Discard is used. +func TestLoggerV2PrintFuncDiscardOnlyInfo(t *testing.T) { + buffers := []*bytes.Buffer{nil, new(bytes.Buffer), new(bytes.Buffer)} + logger := NewLoggerV2(io.Discard, buffers[warningLog], buffers[errorLog], LoggerV2Config{}) + loggerTp := logger.(*loggerT) + + // test that output doesn't call expensive printf funcs on an io.Discard logger + sprintf = makeSprintfErr(t) + sprint = makeSprintErr(t) + sprintln = makeSprintErr(t) + + loggerTp.output(infoLog, "something") + + sprintf = fmt.Sprintf + sprint = fmt.Sprint + sprintln = fmt.Sprintln + + loggerTp.output(errorLog, logFuncStr) + warnB, err := buffers[warningLog].ReadBytes('\n') + if err != nil { + t.Fatalf("level %v: %v", warningLog, err) + } + checkLogContainsFuncStr(t, warnB) + + errB, err := buffers[errorLog].ReadBytes('\n') + if err != nil { + t.Fatalf("level %v: %v", errorLog, err) + } + checkLogContainsFuncStr(t, errB) +} + +func TestLoggerV2PrintFuncNoDiscard(t *testing.T) { + buffers := []*bytes.Buffer{new(bytes.Buffer), new(bytes.Buffer), new(bytes.Buffer)} + logger := NewLoggerV2(buffers[infoLog], buffers[warningLog], buffers[errorLog], LoggerV2Config{}) + loggerTp := logger.(*loggerT) + + loggerTp.output(errorLog, logFuncStr) + + infoB, err := buffers[infoLog].ReadBytes('\n') + if err != nil { + t.Fatalf("level %v: %v", infoLog, err) + } + checkLogContainsFuncStr(t, infoB) + + warnB, err := buffers[warningLog].ReadBytes('\n') + if err != nil { + t.Fatalf("level %v: %v", warningLog, err) + } + checkLogContainsFuncStr(t, warnB) + + errB, err := buffers[errorLog].ReadBytes('\n') + if err != nil { + t.Fatalf("level %v: %v", errorLog, err) + } + checkLogContainsFuncStr(t, errB) +} + +// TestLoggerV2PrintFuncAllDiscard tests that discard loggers don't log. +func TestLoggerV2PrintFuncAllDiscard(t *testing.T) { + logger := NewLoggerV2(io.Discard, io.Discard, io.Discard, LoggerV2Config{}) + loggerTp := logger.(*loggerT) + + sprintf = makeSprintfErr(t) + sprint = makeSprintErr(t) + sprintln = makeSprintErr(t) + + // test that printFunc doesn't call the log func on discard loggers + // makeLogFuncErr will fail the test if it's called + loggerTp.output(infoLog, logFuncStr) + loggerTp.output(warningLog, logFuncStr) + loggerTp.output(errorLog, logFuncStr) + + sprintf = fmt.Sprintf + sprint = fmt.Sprint + sprintln = fmt.Sprintln +} + +func TestLoggerV2PrintFuncAllCombinations(t *testing.T) { + const ( + print int = iota + printf + println + ) + + type testDiscard struct { + discardInf bool + discardWarn bool + discardErr bool + + printType int + formatJSON bool + } + + discardName := func(td testDiscard) string { + strs := []string{} + if td.discardInf { + strs = append(strs, "discardInfo") + } + if td.discardWarn { + strs = append(strs, "discardWarn") + } + if td.discardErr { + strs = append(strs, "discardErr") + } + if len(strs) == 0 { + strs = append(strs, "noDiscard") + } + return strings.Join(strs, " ") + } + var printName = []string{ + print: "print", + printf: "printf", + println: "println", + } + var jsonName = map[bool]string{ + true: "json", + false: "noJson", + } + + discardTests := []testDiscard{} + for _, di := range []bool{true, false} { + for _, dw := range []bool{true, false} { + for _, de := range []bool{true, false} { + for _, pt := range []int{print, printf, println} { + for _, fj := range []bool{true, false} { + discardTests = append(discardTests, testDiscard{discardInf: di, discardWarn: dw, discardErr: de, printType: pt, formatJSON: fj}) + } + } + } + } + } + + for _, discardTest := range discardTests { + testName := fmt.Sprintf("%v %v %v", jsonName[discardTest.formatJSON], printName[discardTest.printType], discardName(discardTest)) + t.Run(testName, func(t *testing.T) { + cfg := LoggerV2Config{FormatJSON: discardTest.formatJSON} + + buffers := []*bytes.Buffer{new(bytes.Buffer), new(bytes.Buffer), new(bytes.Buffer)} + writers := []io.Writer{buffers[infoLog], buffers[warningLog], buffers[errorLog]} + if discardTest.discardInf { + writers[infoLog] = io.Discard + } + if discardTest.discardWarn { + writers[warningLog] = io.Discard + } + if discardTest.discardErr { + writers[errorLog] = io.Discard + } + logger := NewLoggerV2(writers[infoLog], writers[warningLog], writers[errorLog], cfg) + + msgInf := "someinfo" + msgWarn := "somewarn" + msgErr := "someerr" + if discardTest.printType == print { + logger.Info(msgInf) + logger.Warning(msgWarn) + logger.Error(msgErr) + } else if discardTest.printType == printf { + logger.Infof("%v", msgInf) + logger.Warningf("%v", msgWarn) + logger.Errorf("%v", msgErr) + } else if discardTest.printType == println { + logger.Infoln(msgInf) + logger.Warningln(msgWarn) + logger.Errorln(msgErr) + } + + // verify the test Discard, log type, message, and json arguments were logged as-expected + + checkBufferWasWrittenAsExpected(t, buffers[infoLog], discardTest.discardInf, "info", msgInf, cfg.FormatJSON) + checkBufferWasWrittenAsExpected(t, buffers[warningLog], discardTest.discardWarn, "warning", msgWarn, cfg.FormatJSON) + checkBufferWasWrittenAsExpected(t, buffers[errorLog], discardTest.discardErr, "error", msgErr, cfg.FormatJSON) + }) + } +} + +func TestLoggerV2Fatal(t *testing.T) { + const ( + print = "print" + println = "println" + printf = "printf" + ) + printFuncs := []string{print, println, printf} + + exitCalls := []int{} + + if reflect.ValueOf(exit).Pointer() != reflect.ValueOf(os.Exit).Pointer() { + t.Error("got: exit isn't os.Exit, want exit var to be os.Exit") + } + + exit = func(code int) { + exitCalls = append(exitCalls, code) + } + defer func() { + exit = os.Exit + }() + + for _, printFunc := range printFuncs { + buffers := []*bytes.Buffer{new(bytes.Buffer), new(bytes.Buffer), new(bytes.Buffer)} + logger := NewLoggerV2(buffers[infoLog], buffers[warningLog], buffers[errorLog], LoggerV2Config{}) + switch printFunc { + case print: + logger.Fatal("fatal") + case println: + logger.Fatalln("fatalln") + case printf: + logger.Fatalf("fatalf %d", 42) + default: + t.Errorf("unknown print type '%v'", printFunc) + } + + checkBufferWasWrittenAsExpected(t, buffers[errorLog], false, "fatal", "fatal", false) + if len(exitCalls) == 0 { + t.Error("got: no exit call, want fatal log to call exit") + } + exitCalls = []int{} + } +} diff --git a/grpclog/logger.go b/grpclog/logger.go new file mode 100644 index 0000000..4b20358 --- /dev/null +++ b/grpclog/logger.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpclog + +import "google.golang.org/grpc/grpclog/internal" + +// Logger mimics golang's standard Logger as an interface. +// +// Deprecated: use LoggerV2. +type Logger internal.Logger + +// SetLogger sets the logger that is used in grpc. Call only from +// init() functions. +// +// Deprecated: use SetLoggerV2. +func SetLogger(l Logger) { + internal.LoggerV2Impl = &internal.LoggerWrapper{Logger: l} +} diff --git a/grpclog/loggerv2.go b/grpclog/loggerv2.go new file mode 100644 index 0000000..892dc13 --- /dev/null +++ b/grpclog/loggerv2.go @@ -0,0 +1,97 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpclog + +import ( + "io" + "os" + "strconv" + "strings" + + "google.golang.org/grpc/grpclog/internal" +) + +// LoggerV2 does underlying logging work for grpclog. +type LoggerV2 internal.LoggerV2 + +// SetLoggerV2 sets logger that is used in grpc to a V2 logger. +// Not mutex-protected, should be called before any gRPC functions. +func SetLoggerV2(l LoggerV2) { + if _, ok := l.(*componentData); ok { + panic("cannot use component logger as grpclog logger") + } + internal.LoggerV2Impl = l + internal.DepthLoggerV2Impl, _ = l.(internal.DepthLoggerV2) +} + +// NewLoggerV2 creates a loggerV2 with the provided writers. +// Fatal logs will be written to errorW, warningW, infoW, followed by exit(1). +// Error logs will be written to errorW, warningW and infoW. +// Warning logs will be written to warningW and infoW. +// Info logs will be written to infoW. +func NewLoggerV2(infoW, warningW, errorW io.Writer) LoggerV2 { + return internal.NewLoggerV2(infoW, warningW, errorW, internal.LoggerV2Config{}) +} + +// NewLoggerV2WithVerbosity creates a loggerV2 with the provided writers and +// verbosity level. +func NewLoggerV2WithVerbosity(infoW, warningW, errorW io.Writer, v int) LoggerV2 { + return internal.NewLoggerV2(infoW, warningW, errorW, internal.LoggerV2Config{Verbosity: v}) +} + +// newLoggerV2 creates a loggerV2 to be used as default logger. +// All logs are written to stderr. +func newLoggerV2() LoggerV2 { + errorW := io.Discard + warningW := io.Discard + infoW := io.Discard + + logLevel := os.Getenv("GRPC_GO_LOG_SEVERITY_LEVEL") + switch logLevel { + case "", "ERROR", "error": // If env is unset, set level to ERROR. + errorW = os.Stderr + case "WARNING", "warning": + warningW = os.Stderr + case "INFO", "info": + infoW = os.Stderr + } + + var v int + vLevel := os.Getenv("GRPC_GO_LOG_VERBOSITY_LEVEL") + if vl, err := strconv.Atoi(vLevel); err == nil { + v = vl + } + + jsonFormat := strings.EqualFold(os.Getenv("GRPC_GO_LOG_FORMATTER"), "json") + + return internal.NewLoggerV2(infoW, warningW, errorW, internal.LoggerV2Config{ + Verbosity: v, + FormatJSON: jsonFormat, + }) +} + +// DepthLoggerV2 logs at a specified call frame. If a LoggerV2 also implements +// DepthLoggerV2, the below functions will be called with the appropriate stack +// depth set for trivial functions the logger may ignore. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type DepthLoggerV2 internal.DepthLoggerV2 diff --git a/health/client.go b/health/client.go new file mode 100644 index 0000000..740745c --- /dev/null +++ b/health/client.go @@ -0,0 +1,117 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import ( + "context" + "fmt" + "io" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/status" +) + +var ( + backoffStrategy = backoff.DefaultExponential + backoffFunc = func(ctx context.Context, retries int) bool { + d := backoffStrategy.Backoff(retries) + timer := time.NewTimer(d) + select { + case <-timer.C: + return true + case <-ctx.Done(): + timer.Stop() + return false + } + } +) + +func init() { + internal.HealthCheckFunc = clientHealthCheck +} + +const healthCheckMethod = "/grpc.health.v1.Health/Watch" + +// This function implements the protocol defined at: +// https://github.com/grpc/grpc/blob/master/doc/health-checking.md +func clientHealthCheck(ctx context.Context, newStream func(string) (any, error), setConnectivityState func(connectivity.State, error), service string) error { + tryCnt := 0 + +retryConnection: + for { + // Backs off if the connection has failed in some way without receiving a message in the previous retry. + if tryCnt > 0 && !backoffFunc(ctx, tryCnt-1) { + return nil + } + tryCnt++ + + if ctx.Err() != nil { + return nil + } + setConnectivityState(connectivity.Connecting, nil) + rawS, err := newStream(healthCheckMethod) + if err != nil { + continue retryConnection + } + + s, ok := rawS.(grpc.ClientStream) + // Ideally, this should never happen. But if it happens, the server is marked as healthy for LBing purposes. + if !ok { + setConnectivityState(connectivity.Ready, nil) + return fmt.Errorf("newStream returned %v (type %T); want grpc.ClientStream", rawS, rawS) + } + + if err = s.SendMsg(&healthpb.HealthCheckRequest{Service: service}); err != nil && err != io.EOF { + // Stream should have been closed, so we can safely continue to create a new stream. + continue retryConnection + } + s.CloseSend() + + resp := new(healthpb.HealthCheckResponse) + for { + err = s.RecvMsg(resp) + + // Reports healthy for the LBing purposes if health check is not implemented in the server. + if status.Code(err) == codes.Unimplemented { + setConnectivityState(connectivity.Ready, nil) + return err + } + + // Reports unhealthy if server's Watch method gives an error other than UNIMPLEMENTED. + if err != nil { + setConnectivityState(connectivity.TransientFailure, fmt.Errorf("connection active but received health check RPC error: %v", err)) + continue retryConnection + } + + // As a message has been received, removes the need for backoff for the next retry by resetting the try count. + tryCnt = 0 + if resp.Status == healthpb.HealthCheckResponse_SERVING { + setConnectivityState(connectivity.Ready, nil) + } else { + setConnectivityState(connectivity.TransientFailure, fmt.Errorf("connection active but health check failed. status=%s", resp.Status)) + } + } + } +} diff --git a/health/client_test.go b/health/client_test.go new file mode 100644 index 0000000..4fab9f4 --- /dev/null +++ b/health/client_test.go @@ -0,0 +1,63 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + + "google.golang.org/grpc/connectivity" +) + +const defaultTestTimeout = 10 * time.Second + +func (s) TestClientHealthCheckBackoff(t *testing.T) { + const maxRetries = 5 + + var want []time.Duration + for i := 0; i < maxRetries; i++ { + want = append(want, time.Duration(i+1)*time.Second) + } + + var got []time.Duration + newStream := func(string) (any, error) { + if len(got) < maxRetries { + return nil, errors.New("backoff") + } + return nil, nil + } + + oldBackoffFunc := backoffFunc + backoffFunc = func(_ context.Context, retries int) bool { + got = append(got, time.Duration(retries+1)*time.Second) + return true + } + defer func() { backoffFunc = oldBackoffFunc }() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + clientHealthCheck(ctx, newStream, func(connectivity.State, error) {}, "test") + + if !reflect.DeepEqual(got, want) { + t.Fatalf("Backoff durations for %v retries are %v. (expected: %v)", maxRetries, got, want) + } +} diff --git a/health/grpc_health_v1/health.pb.go b/health/grpc_health_v1/health.pb.go new file mode 100644 index 0000000..35d4c83 --- /dev/null +++ b/health/grpc_health_v1/health.pb.go @@ -0,0 +1,350 @@ +// Copyright 2015 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/health/v1/health.proto + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/health/v1/health.proto + +package grpc_health_v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type HealthCheckResponse_ServingStatus int32 + +const ( + HealthCheckResponse_UNKNOWN HealthCheckResponse_ServingStatus = 0 + HealthCheckResponse_SERVING HealthCheckResponse_ServingStatus = 1 + HealthCheckResponse_NOT_SERVING HealthCheckResponse_ServingStatus = 2 + HealthCheckResponse_SERVICE_UNKNOWN HealthCheckResponse_ServingStatus = 3 // Used only by the Watch method. +) + +// Enum value maps for HealthCheckResponse_ServingStatus. +var ( + HealthCheckResponse_ServingStatus_name = map[int32]string{ + 0: "UNKNOWN", + 1: "SERVING", + 2: "NOT_SERVING", + 3: "SERVICE_UNKNOWN", + } + HealthCheckResponse_ServingStatus_value = map[string]int32{ + "UNKNOWN": 0, + "SERVING": 1, + "NOT_SERVING": 2, + "SERVICE_UNKNOWN": 3, + } +) + +func (x HealthCheckResponse_ServingStatus) Enum() *HealthCheckResponse_ServingStatus { + p := new(HealthCheckResponse_ServingStatus) + *p = x + return p +} + +func (x HealthCheckResponse_ServingStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (HealthCheckResponse_ServingStatus) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_health_v1_health_proto_enumTypes[0].Descriptor() +} + +func (HealthCheckResponse_ServingStatus) Type() protoreflect.EnumType { + return &file_grpc_health_v1_health_proto_enumTypes[0] +} + +func (x HealthCheckResponse_ServingStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use HealthCheckResponse_ServingStatus.Descriptor instead. +func (HealthCheckResponse_ServingStatus) EnumDescriptor() ([]byte, []int) { + return file_grpc_health_v1_health_proto_rawDescGZIP(), []int{1, 0} +} + +type HealthCheckRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HealthCheckRequest) Reset() { + *x = HealthCheckRequest{} + mi := &file_grpc_health_v1_health_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HealthCheckRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HealthCheckRequest) ProtoMessage() {} + +func (x *HealthCheckRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_health_v1_health_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HealthCheckRequest.ProtoReflect.Descriptor instead. +func (*HealthCheckRequest) Descriptor() ([]byte, []int) { + return file_grpc_health_v1_health_proto_rawDescGZIP(), []int{0} +} + +func (x *HealthCheckRequest) GetService() string { + if x != nil { + return x.Service + } + return "" +} + +type HealthCheckResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Status HealthCheckResponse_ServingStatus `protobuf:"varint,1,opt,name=status,proto3,enum=grpc.health.v1.HealthCheckResponse_ServingStatus" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HealthCheckResponse) Reset() { + *x = HealthCheckResponse{} + mi := &file_grpc_health_v1_health_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HealthCheckResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HealthCheckResponse) ProtoMessage() {} + +func (x *HealthCheckResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_health_v1_health_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HealthCheckResponse.ProtoReflect.Descriptor instead. +func (*HealthCheckResponse) Descriptor() ([]byte, []int) { + return file_grpc_health_v1_health_proto_rawDescGZIP(), []int{1} +} + +func (x *HealthCheckResponse) GetStatus() HealthCheckResponse_ServingStatus { + if x != nil { + return x.Status + } + return HealthCheckResponse_UNKNOWN +} + +type HealthListRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HealthListRequest) Reset() { + *x = HealthListRequest{} + mi := &file_grpc_health_v1_health_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HealthListRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HealthListRequest) ProtoMessage() {} + +func (x *HealthListRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_health_v1_health_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HealthListRequest.ProtoReflect.Descriptor instead. +func (*HealthListRequest) Descriptor() ([]byte, []int) { + return file_grpc_health_v1_health_proto_rawDescGZIP(), []int{2} +} + +type HealthListResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // statuses contains all the services and their respective status. + Statuses map[string]*HealthCheckResponse `protobuf:"bytes,1,rep,name=statuses,proto3" json:"statuses,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HealthListResponse) Reset() { + *x = HealthListResponse{} + mi := &file_grpc_health_v1_health_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HealthListResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HealthListResponse) ProtoMessage() {} + +func (x *HealthListResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_health_v1_health_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HealthListResponse.ProtoReflect.Descriptor instead. +func (*HealthListResponse) Descriptor() ([]byte, []int) { + return file_grpc_health_v1_health_proto_rawDescGZIP(), []int{3} +} + +func (x *HealthListResponse) GetStatuses() map[string]*HealthCheckResponse { + if x != nil { + return x.Statuses + } + return nil +} + +var File_grpc_health_v1_health_proto protoreflect.FileDescriptor + +const file_grpc_health_v1_health_proto_rawDesc = "" + + "\n" + + "\x1bgrpc/health/v1/health.proto\x12\x0egrpc.health.v1\".\n" + + "\x12HealthCheckRequest\x12\x18\n" + + "\aservice\x18\x01 \x01(\tR\aservice\"\xb1\x01\n" + + "\x13HealthCheckResponse\x12I\n" + + "\x06status\x18\x01 \x01(\x0e21.grpc.health.v1.HealthCheckResponse.ServingStatusR\x06status\"O\n" + + "\rServingStatus\x12\v\n" + + "\aUNKNOWN\x10\x00\x12\v\n" + + "\aSERVING\x10\x01\x12\x0f\n" + + "\vNOT_SERVING\x10\x02\x12\x13\n" + + "\x0fSERVICE_UNKNOWN\x10\x03\"\x13\n" + + "\x11HealthListRequest\"\xc4\x01\n" + + "\x12HealthListResponse\x12L\n" + + "\bstatuses\x18\x01 \x03(\v20.grpc.health.v1.HealthListResponse.StatusesEntryR\bstatuses\x1a`\n" + + "\rStatusesEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x129\n" + + "\x05value\x18\x02 \x01(\v2#.grpc.health.v1.HealthCheckResponseR\x05value:\x028\x012\xfd\x01\n" + + "\x06Health\x12P\n" + + "\x05Check\x12\".grpc.health.v1.HealthCheckRequest\x1a#.grpc.health.v1.HealthCheckResponse\x12M\n" + + "\x04List\x12!.grpc.health.v1.HealthListRequest\x1a\".grpc.health.v1.HealthListResponse\x12R\n" + + "\x05Watch\x12\".grpc.health.v1.HealthCheckRequest\x1a#.grpc.health.v1.HealthCheckResponse0\x01Bp\n" + + "\x11io.grpc.health.v1B\vHealthProtoP\x01Z,google.golang.org/grpc/health/grpc_health_v1\xa2\x02\fGrpcHealthV1\xaa\x02\x0eGrpc.Health.V1b\x06proto3" + +var ( + file_grpc_health_v1_health_proto_rawDescOnce sync.Once + file_grpc_health_v1_health_proto_rawDescData []byte +) + +func file_grpc_health_v1_health_proto_rawDescGZIP() []byte { + file_grpc_health_v1_health_proto_rawDescOnce.Do(func() { + file_grpc_health_v1_health_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_health_v1_health_proto_rawDesc), len(file_grpc_health_v1_health_proto_rawDesc))) + }) + return file_grpc_health_v1_health_proto_rawDescData +} + +var file_grpc_health_v1_health_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_grpc_health_v1_health_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_grpc_health_v1_health_proto_goTypes = []any{ + (HealthCheckResponse_ServingStatus)(0), // 0: grpc.health.v1.HealthCheckResponse.ServingStatus + (*HealthCheckRequest)(nil), // 1: grpc.health.v1.HealthCheckRequest + (*HealthCheckResponse)(nil), // 2: grpc.health.v1.HealthCheckResponse + (*HealthListRequest)(nil), // 3: grpc.health.v1.HealthListRequest + (*HealthListResponse)(nil), // 4: grpc.health.v1.HealthListResponse + nil, // 5: grpc.health.v1.HealthListResponse.StatusesEntry +} +var file_grpc_health_v1_health_proto_depIdxs = []int32{ + 0, // 0: grpc.health.v1.HealthCheckResponse.status:type_name -> grpc.health.v1.HealthCheckResponse.ServingStatus + 5, // 1: grpc.health.v1.HealthListResponse.statuses:type_name -> grpc.health.v1.HealthListResponse.StatusesEntry + 2, // 2: grpc.health.v1.HealthListResponse.StatusesEntry.value:type_name -> grpc.health.v1.HealthCheckResponse + 1, // 3: grpc.health.v1.Health.Check:input_type -> grpc.health.v1.HealthCheckRequest + 3, // 4: grpc.health.v1.Health.List:input_type -> grpc.health.v1.HealthListRequest + 1, // 5: grpc.health.v1.Health.Watch:input_type -> grpc.health.v1.HealthCheckRequest + 2, // 6: grpc.health.v1.Health.Check:output_type -> grpc.health.v1.HealthCheckResponse + 4, // 7: grpc.health.v1.Health.List:output_type -> grpc.health.v1.HealthListResponse + 2, // 8: grpc.health.v1.Health.Watch:output_type -> grpc.health.v1.HealthCheckResponse + 6, // [6:9] is the sub-list for method output_type + 3, // [3:6] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_grpc_health_v1_health_proto_init() } +func file_grpc_health_v1_health_proto_init() { + if File_grpc_health_v1_health_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_health_v1_health_proto_rawDesc), len(file_grpc_health_v1_health_proto_rawDesc)), + NumEnums: 1, + NumMessages: 5, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_health_v1_health_proto_goTypes, + DependencyIndexes: file_grpc_health_v1_health_proto_depIdxs, + EnumInfos: file_grpc_health_v1_health_proto_enumTypes, + MessageInfos: file_grpc_health_v1_health_proto_msgTypes, + }.Build() + File_grpc_health_v1_health_proto = out.File + file_grpc_health_v1_health_proto_goTypes = nil + file_grpc_health_v1_health_proto_depIdxs = nil +} diff --git a/health/grpc_health_v1/health_grpc.pb.go b/health/grpc_health_v1/health_grpc.pb.go new file mode 100644 index 0000000..537ba05 --- /dev/null +++ b/health/grpc_health_v1/health_grpc.pb.go @@ -0,0 +1,290 @@ +// Copyright 2015 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/health/v1/health.proto + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: grpc/health/v1/health.proto + +package grpc_health_v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Health_Check_FullMethodName = "/grpc.health.v1.Health/Check" + Health_List_FullMethodName = "/grpc.health.v1.Health/List" + Health_Watch_FullMethodName = "/grpc.health.v1.Health/Watch" +) + +// HealthClient is the client API for Health service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Health is gRPC's mechanism for checking whether a server is able to handle +// RPCs. Its semantics are documented in +// https://github.com/grpc/grpc/blob/master/doc/health-checking.md. +type HealthClient interface { + // Check gets the health of the specified service. If the requested service + // is unknown, the call will fail with status NOT_FOUND. If the caller does + // not specify a service name, the server should respond with its overall + // health status. + // + // Clients should set a deadline when calling Check, and can declare the + // server unhealthy if they do not receive a timely response. + Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) + // List provides a non-atomic snapshot of the health of all the available + // services. + // + // The server may respond with a RESOURCE_EXHAUSTED error if too many services + // exist. + // + // Clients should set a deadline when calling List, and can declare the server + // unhealthy if they do not receive a timely response. + // + // Clients should keep in mind that the list of health services exposed by an + // application can change over the lifetime of the process. + List(ctx context.Context, in *HealthListRequest, opts ...grpc.CallOption) (*HealthListResponse, error) + // Performs a watch for the serving status of the requested service. + // The server will immediately send back a message indicating the current + // serving status. It will then subsequently send a new message whenever + // the service's serving status changes. + // + // If the requested service is unknown when the call is received, the + // server will send a message setting the serving status to + // SERVICE_UNKNOWN but will *not* terminate the call. If at some + // future point, the serving status of the service becomes known, the + // server will send a new message with the service's serving status. + // + // If the call terminates with status UNIMPLEMENTED, then clients + // should assume this method is not supported and should not retry the + // call. If the call terminates with any other status (including OK), + // clients should retry the call with appropriate exponential backoff. + Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[HealthCheckResponse], error) +} + +type healthClient struct { + cc grpc.ClientConnInterface +} + +func NewHealthClient(cc grpc.ClientConnInterface) HealthClient { + return &healthClient{cc} +} + +func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(HealthCheckResponse) + err := c.cc.Invoke(ctx, Health_Check_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *healthClient) List(ctx context.Context, in *HealthListRequest, opts ...grpc.CallOption) (*HealthListResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(HealthListResponse) + err := c.cc.Invoke(ctx, Health_List_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[HealthCheckResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Health_ServiceDesc.Streams[0], Health_Watch_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[HealthCheckRequest, HealthCheckResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Health_WatchClient = grpc.ServerStreamingClient[HealthCheckResponse] + +// HealthServer is the server API for Health service. +// All implementations should embed UnimplementedHealthServer +// for forward compatibility. +// +// Health is gRPC's mechanism for checking whether a server is able to handle +// RPCs. Its semantics are documented in +// https://github.com/grpc/grpc/blob/master/doc/health-checking.md. +type HealthServer interface { + // Check gets the health of the specified service. If the requested service + // is unknown, the call will fail with status NOT_FOUND. If the caller does + // not specify a service name, the server should respond with its overall + // health status. + // + // Clients should set a deadline when calling Check, and can declare the + // server unhealthy if they do not receive a timely response. + Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) + // List provides a non-atomic snapshot of the health of all the available + // services. + // + // The server may respond with a RESOURCE_EXHAUSTED error if too many services + // exist. + // + // Clients should set a deadline when calling List, and can declare the server + // unhealthy if they do not receive a timely response. + // + // Clients should keep in mind that the list of health services exposed by an + // application can change over the lifetime of the process. + List(context.Context, *HealthListRequest) (*HealthListResponse, error) + // Performs a watch for the serving status of the requested service. + // The server will immediately send back a message indicating the current + // serving status. It will then subsequently send a new message whenever + // the service's serving status changes. + // + // If the requested service is unknown when the call is received, the + // server will send a message setting the serving status to + // SERVICE_UNKNOWN but will *not* terminate the call. If at some + // future point, the serving status of the service becomes known, the + // server will send a new message with the service's serving status. + // + // If the call terminates with status UNIMPLEMENTED, then clients + // should assume this method is not supported and should not retry the + // call. If the call terminates with any other status (including OK), + // clients should retry the call with appropriate exponential backoff. + Watch(*HealthCheckRequest, grpc.ServerStreamingServer[HealthCheckResponse]) error +} + +// UnimplementedHealthServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedHealthServer struct{} + +func (UnimplementedHealthServer) Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Check not implemented") +} +func (UnimplementedHealthServer) List(context.Context, *HealthListRequest) (*HealthListResponse, error) { + return nil, status.Error(codes.Unimplemented, "method List not implemented") +} +func (UnimplementedHealthServer) Watch(*HealthCheckRequest, grpc.ServerStreamingServer[HealthCheckResponse]) error { + return status.Error(codes.Unimplemented, "method Watch not implemented") +} +func (UnimplementedHealthServer) testEmbeddedByValue() {} + +// UnsafeHealthServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to HealthServer will +// result in compilation errors. +type UnsafeHealthServer interface { + mustEmbedUnimplementedHealthServer() +} + +func RegisterHealthServer(s grpc.ServiceRegistrar, srv HealthServer) { + // If the following call panics, it indicates UnimplementedHealthServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Health_ServiceDesc, srv) +} + +func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(HealthCheckRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HealthServer).Check(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Health_Check_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Health_List_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(HealthListRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HealthServer).List(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Health_List_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HealthServer).List(ctx, req.(*HealthListRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Health_Watch_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(HealthCheckRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(HealthServer).Watch(m, &grpc.GenericServerStream[HealthCheckRequest, HealthCheckResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Health_WatchServer = grpc.ServerStreamingServer[HealthCheckResponse] + +// Health_ServiceDesc is the grpc.ServiceDesc for Health service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Health_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.health.v1.Health", + HandlerType: (*HealthServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Check", + Handler: _Health_Check_Handler, + }, + { + MethodName: "List", + Handler: _Health_List_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Watch", + Handler: _Health_Watch_Handler, + ServerStreams: true, + }, + }, + Metadata: "grpc/health/v1/health.proto", +} diff --git a/health/logging.go b/health/logging.go new file mode 100644 index 0000000..83c6acf --- /dev/null +++ b/health/logging.go @@ -0,0 +1,23 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import "google.golang.org/grpc/grpclog" + +var logger = grpclog.Component("health_service") diff --git a/health/producer.go b/health/producer.go new file mode 100644 index 0000000..f938e57 --- /dev/null +++ b/health/producer.go @@ -0,0 +1,106 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import ( + "context" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/status" +) + +func init() { + producerBuilderSingleton = &producerBuilder{} + internal.RegisterClientHealthCheckListener = registerClientSideHealthCheckListener +} + +type producerBuilder struct{} + +var producerBuilderSingleton *producerBuilder + +// Build constructs and returns a producer and its cleanup function. +func (*producerBuilder) Build(cci any) (balancer.Producer, func()) { + p := &healthServiceProducer{ + cc: cci.(grpc.ClientConnInterface), + cancel: func() {}, + } + return p, func() { + p.mu.Lock() + defer p.mu.Unlock() + p.cancel() + } +} + +type healthServiceProducer struct { + // The following fields are initialized at build time and read-only after + // that and therefore do not need to be guarded by a mutex. + cc grpc.ClientConnInterface + + mu sync.Mutex + cancel func() +} + +// registerClientSideHealthCheckListener accepts a listener to provide server +// health state via the health service. +func registerClientSideHealthCheckListener(ctx context.Context, sc balancer.SubConn, serviceName string, listener func(balancer.SubConnState)) func() { + pr, closeFn := sc.GetOrBuildProducer(producerBuilderSingleton) + p := pr.(*healthServiceProducer) + p.mu.Lock() + defer p.mu.Unlock() + p.cancel() + if listener == nil { + return closeFn + } + + ctx, cancel := context.WithCancel(ctx) + p.cancel = cancel + + go p.startHealthCheck(ctx, sc, serviceName, listener) + return closeFn +} + +func (p *healthServiceProducer) startHealthCheck(ctx context.Context, sc balancer.SubConn, serviceName string, listener func(balancer.SubConnState)) { + newStream := func(method string) (any, error) { + return p.cc.NewStream(ctx, &grpc.StreamDesc{ServerStreams: true}, method) + } + + setConnectivityState := func(state connectivity.State, err error) { + listener(balancer.SubConnState{ + ConnectivityState: state, + ConnectionError: err, + }) + } + + // Call the function through the internal variable as tests use it for + // mocking. + err := internal.HealthCheckFunc(ctx, newStream, setConnectivityState, serviceName) + if err == nil { + return + } + if status.Code(err) == codes.Unimplemented { + logger.Errorf("Subchannel health check is unimplemented at server side, thus health check is disabled for SubConn %p", sc) + } else { + logger.Errorf("Health checking failed for SubConn %p: %v", sc, err) + } +} diff --git a/health/server.go b/health/server.go new file mode 100644 index 0000000..d8eebe9 --- /dev/null +++ b/health/server.go @@ -0,0 +1,187 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package health provides a service that exposes server's health and it must be +// imported to enable support for client-side health checks. +package health + +import ( + "context" + "sync" + + "google.golang.org/grpc/codes" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +const ( + // maxAllowedServices defines the maximum number of resources a List + // operation can return. An error is returned if the number of services + // exceeds this limit. + maxAllowedServices = 100 +) + +// Server implements `service Health`. +type Server struct { + healthgrpc.UnimplementedHealthServer + mu sync.RWMutex + // If shutdown is true, it's expected all serving status is NOT_SERVING, and + // will stay in NOT_SERVING. + shutdown bool + // statusMap stores the serving status of the services this Server monitors. + statusMap map[string]healthpb.HealthCheckResponse_ServingStatus + updates map[string]map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus +} + +// NewServer returns a new Server. +func NewServer() *Server { + return &Server{ + statusMap: map[string]healthpb.HealthCheckResponse_ServingStatus{"": healthpb.HealthCheckResponse_SERVING}, + updates: make(map[string]map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus), + } +} + +// Check implements `service Health`. +func (s *Server) Check(_ context.Context, in *healthpb.HealthCheckRequest) (*healthpb.HealthCheckResponse, error) { + s.mu.RLock() + defer s.mu.RUnlock() + if servingStatus, ok := s.statusMap[in.Service]; ok { + return &healthpb.HealthCheckResponse{ + Status: servingStatus, + }, nil + } + return nil, status.Error(codes.NotFound, "unknown service") +} + +// List implements `service Health`. +func (s *Server) List(_ context.Context, _ *healthpb.HealthListRequest) (*healthpb.HealthListResponse, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if len(s.statusMap) > maxAllowedServices { + return nil, status.Errorf(codes.ResourceExhausted, "server health list exceeds maximum capacity: %d", maxAllowedServices) + } + + statusMap := make(map[string]*healthpb.HealthCheckResponse, len(s.statusMap)) + for k, v := range s.statusMap { + statusMap[k] = &healthpb.HealthCheckResponse{Status: v} + } + + return &healthpb.HealthListResponse{Statuses: statusMap}, nil +} + +// Watch implements `service Health`. +func (s *Server) Watch(in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { + service := in.Service + // update channel is used for getting service status updates. + update := make(chan healthpb.HealthCheckResponse_ServingStatus, 1) + s.mu.Lock() + // Puts the initial status to the channel. + if servingStatus, ok := s.statusMap[service]; ok { + update <- servingStatus + } else { + update <- healthpb.HealthCheckResponse_SERVICE_UNKNOWN + } + + // Registers the update channel to the correct place in the updates map. + if _, ok := s.updates[service]; !ok { + s.updates[service] = make(map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus) + } + s.updates[service][stream] = update + defer func() { + s.mu.Lock() + delete(s.updates[service], stream) + s.mu.Unlock() + }() + s.mu.Unlock() + + var lastSentStatus healthpb.HealthCheckResponse_ServingStatus = -1 + for { + select { + // Status updated. Sends the up-to-date status to the client. + case servingStatus := <-update: + if lastSentStatus == servingStatus { + continue + } + lastSentStatus = servingStatus + err := stream.Send(&healthpb.HealthCheckResponse{Status: servingStatus}) + if err != nil { + return status.Error(codes.Canceled, "Stream has ended.") + } + // Context done. Removes the update channel from the updates map. + case <-stream.Context().Done(): + return status.Error(codes.Canceled, "Stream has ended.") + } + } +} + +// SetServingStatus is called when need to reset the serving status of a service +// or insert a new service entry into the statusMap. +func (s *Server) SetServingStatus(service string, servingStatus healthpb.HealthCheckResponse_ServingStatus) { + s.mu.Lock() + defer s.mu.Unlock() + if s.shutdown { + logger.Infof("health: status changing for %s to %v is ignored because health service is shutdown", service, servingStatus) + return + } + + s.setServingStatusLocked(service, servingStatus) +} + +func (s *Server) setServingStatusLocked(service string, servingStatus healthpb.HealthCheckResponse_ServingStatus) { + s.statusMap[service] = servingStatus + for _, update := range s.updates[service] { + // Clears previous updates, that are not sent to the client, from the channel. + // This can happen if the client is not reading and the server gets flow control limited. + select { + case <-update: + default: + } + // Puts the most recent update to the channel. + update <- servingStatus + } +} + +// Shutdown sets all serving status to NOT_SERVING, and configures the server to +// ignore all future status changes. +// +// This changes serving status for all services. To set status for a particular +// services, call SetServingStatus(). +func (s *Server) Shutdown() { + s.mu.Lock() + defer s.mu.Unlock() + s.shutdown = true + for service := range s.statusMap { + s.setServingStatusLocked(service, healthpb.HealthCheckResponse_NOT_SERVING) + } +} + +// Resume sets all serving status to SERVING, and configures the server to +// accept all future status changes. +// +// This changes serving status for all services. To set status for a particular +// services, call SetServingStatus(). +func (s *Server) Resume() { + s.mu.Lock() + defer s.mu.Unlock() + s.shutdown = false + for service := range s.statusMap { + s.setServingStatusLocked(service, healthpb.HealthCheckResponse_SERVING) + } +} diff --git a/health/server_internal_test.go b/health/server_internal_test.go new file mode 100644 index 0000000..d31c917 --- /dev/null +++ b/health/server_internal_test.go @@ -0,0 +1,153 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/testing/protocmp" + + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestShutdown(t *testing.T) { + const testService = "tteesstt" + s := NewServer() + s.SetServingStatus(testService, healthpb.HealthCheckResponse_SERVING) + + status := s.statusMap[testService] + if status != healthpb.HealthCheckResponse_SERVING { + t.Fatalf("status for %s is %v, want %v", testService, status, healthpb.HealthCheckResponse_SERVING) + } + + var wg sync.WaitGroup + wg.Add(2) + // Run SetServingStatus and Shutdown in parallel. + go func() { + for i := 0; i < 1000; i++ { + s.SetServingStatus(testService, healthpb.HealthCheckResponse_SERVING) + time.Sleep(time.Microsecond) + } + wg.Done() + }() + go func() { + time.Sleep(300 * time.Microsecond) + s.Shutdown() + wg.Done() + }() + wg.Wait() + + s.mu.Lock() + status = s.statusMap[testService] + s.mu.Unlock() + if status != healthpb.HealthCheckResponse_NOT_SERVING { + t.Fatalf("status for %s is %v, want %v", testService, status, healthpb.HealthCheckResponse_NOT_SERVING) + } + + s.Resume() + status = s.statusMap[testService] + if status != healthpb.HealthCheckResponse_SERVING { + t.Fatalf("status for %s is %v, want %v", testService, status, healthpb.HealthCheckResponse_SERVING) + } + + s.SetServingStatus(testService, healthpb.HealthCheckResponse_NOT_SERVING) + status = s.statusMap[testService] + if status != healthpb.HealthCheckResponse_NOT_SERVING { + t.Fatalf("status for %s is %v, want %v", testService, status, healthpb.HealthCheckResponse_NOT_SERVING) + } +} + +// TestList verifies that List() returns the health status of all the services if no. of services are within +// maxAllowedLimits. +func (s) TestList(t *testing.T) { + s := NewServer() + + // Fill out status map with information + const length = 3 + for i := 0; i < length; i++ { + s.SetServingStatus(fmt.Sprintf("%d", i), + healthpb.HealthCheckResponse_ServingStatus(i)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var in healthpb.HealthListRequest + got, err := s.List(ctx, &in) + + if err != nil { + t.Fatalf("s.List(ctx, &in) returned err %v, want nil", err) + } + if len(got.GetStatuses()) != length+1 { + t.Fatalf("len(out.GetStatuses()) = %d, want %d", + len(got.GetStatuses()), length+1) + } + want := &healthpb.HealthListResponse{ + Statuses: map[string]*healthpb.HealthCheckResponse{ + "": {Status: healthpb.HealthCheckResponse_SERVING}, + "0": {Status: healthpb.HealthCheckResponse_UNKNOWN}, + "1": {Status: healthpb.HealthCheckResponse_SERVING}, + "2": {Status: healthpb.HealthCheckResponse_NOT_SERVING}, + }, + } + if diff := cmp.Diff(got, want, protocmp.Transform()); diff != "" { + t.Fatalf("Health response did not match expectation. Diff (-got, +want): %s", diff) + } +} + +// TestListResourceExhausted verifies that List() +// returns a ResourceExhausted error if no. of services are more than +// maxAllowedServices. +func (s) TestListResourceExhausted(t *testing.T) { + s := NewServer() + + // Fill out status map with service information, + // 101 (100 + 1 existing) elements will trigger an error. + for i := 1; i <= maxAllowedServices; i++ { + s.SetServingStatus(fmt.Sprintf("%d", i), + healthpb.HealthCheckResponse_SERVING) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var in healthpb.HealthListRequest + _, err := s.List(ctx, &in) + + want := status.Errorf(codes.ResourceExhausted, + "server health list exceeds maximum capacity: %d", maxAllowedServices) + if !errors.Is(err, want) { + t.Fatalf("s.List(ctx, &in) returned %v, want %v", err, want) + } +} diff --git a/health/server_test.go b/health/server_test.go new file mode 100644 index 0000000..b3307ae --- /dev/null +++ b/health/server_test.go @@ -0,0 +1,43 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health_test + +import ( + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/health" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// Make sure the service implementation complies with the proto definition. +func (s) TestRegister(*testing.T) { + s := grpc.NewServer() + healthgrpc.RegisterHealthServer(s, health.NewServer()) + s.Stop() +} diff --git a/interceptor.go b/interceptor.go new file mode 100644 index 0000000..099e3d0 --- /dev/null +++ b/interceptor.go @@ -0,0 +1,108 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" +) + +// UnaryInvoker is called by UnaryClientInterceptor to complete RPCs. +type UnaryInvoker func(ctx context.Context, method string, req, reply any, cc *ClientConn, opts ...CallOption) error + +// UnaryClientInterceptor intercepts the execution of a unary RPC on the client. +// Unary interceptors can be specified as a DialOption, using +// WithUnaryInterceptor() or WithChainUnaryInterceptor(), when creating a +// ClientConn. When a unary interceptor(s) is set on a ClientConn, gRPC +// delegates all unary RPC invocations to the interceptor, and it is the +// responsibility of the interceptor to call invoker to complete the processing +// of the RPC. +// +// method is the RPC name. req and reply are the corresponding request and +// response messages. cc is the ClientConn on which the RPC was invoked. invoker +// is the handler to complete the RPC and it is the responsibility of the +// interceptor to call it. opts contain all applicable call options, including +// defaults from the ClientConn as well as per-call options. +// +// The returned error must be compatible with the status package. +type UnaryClientInterceptor func(ctx context.Context, method string, req, reply any, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error + +// Streamer is called by StreamClientInterceptor to create a ClientStream. +type Streamer func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) + +// StreamClientInterceptor intercepts the creation of a ClientStream. Stream +// interceptors can be specified as a DialOption, using WithStreamInterceptor() +// or WithChainStreamInterceptor(), when creating a ClientConn. When a stream +// interceptor(s) is set on the ClientConn, gRPC delegates all stream creations +// to the interceptor, and it is the responsibility of the interceptor to call +// streamer. +// +// desc contains a description of the stream. cc is the ClientConn on which the +// RPC was invoked. streamer is the handler to create a ClientStream and it is +// the responsibility of the interceptor to call it. opts contain all applicable +// call options, including defaults from the ClientConn as well as per-call +// options. +// +// StreamClientInterceptor may return a custom ClientStream to intercept all I/O +// operations. The returned error must be compatible with the status package. +type StreamClientInterceptor func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error) + +// UnaryServerInfo consists of various information about a unary RPC on +// server side. All per-rpc information may be mutated by the interceptor. +type UnaryServerInfo struct { + // Server is the service implementation the user provides. This is read-only. + Server any + // FullMethod is the full RPC method string, i.e., /package.service/method. + FullMethod string +} + +// UnaryHandler defines the handler invoked by UnaryServerInterceptor to complete the normal +// execution of a unary RPC. +// +// If a UnaryHandler returns an error, it should either be produced by the +// status package, or be one of the context errors. Otherwise, gRPC will use +// codes.Unknown as the status code and err.Error() as the status message of the +// RPC. +type UnaryHandler func(ctx context.Context, req any) (any, error) + +// UnaryServerInterceptor provides a hook to intercept the execution of a unary RPC on the server. info +// contains all the information of this RPC the interceptor can operate on. And handler is the wrapper +// of the service method implementation. It is the responsibility of the interceptor to invoke handler +// to complete the RPC. +type UnaryServerInterceptor func(ctx context.Context, req any, info *UnaryServerInfo, handler UnaryHandler) (resp any, err error) + +// StreamServerInfo consists of various information about a streaming RPC on +// server side. All per-rpc information may be mutated by the interceptor. +type StreamServerInfo struct { + // FullMethod is the full RPC method string, i.e., /package.service/method. + FullMethod string + // IsClientStream indicates whether the RPC is a client streaming RPC. + IsClientStream bool + // IsServerStream indicates whether the RPC is a server streaming RPC. + IsServerStream bool +} + +// StreamServerInterceptor provides a hook to intercept the execution of a +// streaming RPC on the server. +// +// srv is the service implementation on which the RPC was invoked, and needs to +// be passed to handler, and not used otherwise. ss is the server side of the +// stream. info contains all the information of this RPC the interceptor can +// operate on. And handler is the service method implementation. It is the +// responsibility of the interceptor to invoke handler to complete the RPC. +type StreamServerInterceptor func(srv any, ss ServerStream, info *StreamServerInfo, handler StreamHandler) error diff --git a/internal/admin/admin.go b/internal/admin/admin.go new file mode 100644 index 0000000..a9285ee --- /dev/null +++ b/internal/admin/admin.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package admin contains internal implementation for admin service. +package admin + +import "google.golang.org/grpc" + +// services is a map from name to service register functions. +var services []func(grpc.ServiceRegistrar) (func(), error) + +// AddService adds a service to the list of admin services. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. +// +// If multiple services with the same service name are added (e.g. two services +// for `grpc.channelz.v1.Channelz`), the server will panic on `Register()`. +func AddService(f func(grpc.ServiceRegistrar) (func(), error)) { + services = append(services, f) +} + +// Register registers the set of admin services to the given server. +func Register(s grpc.ServiceRegistrar) (cleanup func(), _ error) { + var cleanups []func() + for _, f := range services { + cleanup, err := f(s) + if err != nil { + callFuncs(cleanups) + return nil, err + } + if cleanup != nil { + cleanups = append(cleanups, cleanup) + } + } + return func() { + callFuncs(cleanups) + }, nil +} + +func callFuncs(fs []func()) { + for _, f := range fs { + f() + } +} diff --git a/internal/backoff/backoff.go b/internal/backoff/backoff.go new file mode 100644 index 0000000..b6ae7f2 --- /dev/null +++ b/internal/backoff/backoff.go @@ -0,0 +1,109 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package backoff implement the backoff strategy for gRPC. +// +// This is kept in internal until the gRPC project decides whether or not to +// allow alternative backoff strategies. +package backoff + +import ( + "context" + "errors" + rand "math/rand/v2" + "time" + + grpcbackoff "google.golang.org/grpc/backoff" +) + +// Strategy defines the methodology for backing off after a grpc connection +// failure. +type Strategy interface { + // Backoff returns the amount of time to wait before the next retry given + // the number of consecutive failures. + Backoff(retries int) time.Duration +} + +// DefaultExponential is an exponential backoff implementation using the +// default values for all the configurable knobs defined in +// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. +var DefaultExponential = Exponential{Config: grpcbackoff.DefaultConfig} + +// Exponential implements exponential backoff algorithm as defined in +// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. +type Exponential struct { + // Config contains all options to configure the backoff algorithm. + Config grpcbackoff.Config +} + +// Backoff returns the amount of time to wait before the next retry given the +// number of retries. +func (bc Exponential) Backoff(retries int) time.Duration { + if retries == 0 { + return bc.Config.BaseDelay + } + backoff, max := float64(bc.Config.BaseDelay), float64(bc.Config.MaxDelay) + for backoff < max && retries > 0 { + backoff *= bc.Config.Multiplier + retries-- + } + if backoff > max { + backoff = max + } + // Randomize backoff delays so that if a cluster of requests start at + // the same time, they won't operate in lockstep. + backoff *= 1 + bc.Config.Jitter*(rand.Float64()*2-1) + if backoff < 0 { + return 0 + } + return time.Duration(backoff) +} + +// ErrResetBackoff is the error to be returned by the function executed by RunF, +// to instruct the latter to reset its backoff state. +var ErrResetBackoff = errors.New("reset backoff state") + +// RunF provides a convenient way to run a function f repeatedly until the +// context expires or f returns a non-nil error that is not ErrResetBackoff. +// When f returns ErrResetBackoff, RunF continues to run f, but resets its +// backoff state before doing so. backoff accepts an integer representing the +// number of retries, and returns the amount of time to backoff. +func RunF(ctx context.Context, f func() error, backoff func(int) time.Duration) { + attempt := 0 + timer := time.NewTimer(0) + for ctx.Err() == nil { + select { + case <-timer.C: + case <-ctx.Done(): + timer.Stop() + return + } + + err := f() + if errors.Is(err, ErrResetBackoff) { + timer.Reset(0) + attempt = 0 + continue + } + if err != nil { + return + } + timer.Reset(backoff(attempt)) + attempt++ + } +} diff --git a/internal/balancer/gracefulswitch/config.go b/internal/balancer/gracefulswitch/config.go new file mode 100644 index 0000000..85540f8 --- /dev/null +++ b/internal/balancer/gracefulswitch/config.go @@ -0,0 +1,84 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package gracefulswitch + +import ( + "encoding/json" + "fmt" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/serviceconfig" +) + +type lbConfig struct { + serviceconfig.LoadBalancingConfig + + childBuilder balancer.Builder + childConfig serviceconfig.LoadBalancingConfig +} + +// ChildName returns the name of the child balancer of the gracefulswitch +// Balancer. +func ChildName(l serviceconfig.LoadBalancingConfig) string { + return l.(*lbConfig).childBuilder.Name() +} + +// ParseConfig parses a child config list and returns a LB config for the +// gracefulswitch Balancer. +// +// cfg is expected to be a json.RawMessage containing a JSON array of LB policy +// names + configs as the format of the "loadBalancingConfig" field in +// ServiceConfig. It returns a type that should be passed to +// UpdateClientConnState in the BalancerConfig field. +func ParseConfig(cfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + var lbCfg []map[string]json.RawMessage + if err := json.Unmarshal(cfg, &lbCfg); err != nil { + return nil, err + } + for i, e := range lbCfg { + if len(e) != 1 { + return nil, fmt.Errorf("expected a JSON struct with one entry; received entry %v at index %d", e, i) + } + + var name string + var jsonCfg json.RawMessage + for name, jsonCfg = range e { + } + + builder := balancer.Get(name) + if builder == nil { + // Skip unregistered balancer names. + continue + } + + parser, ok := builder.(balancer.ConfigParser) + if !ok { + // This is a valid child with no config. + return &lbConfig{childBuilder: builder}, nil + } + + cfg, err := parser.ParseConfig(jsonCfg) + if err != nil { + return nil, fmt.Errorf("error parsing config for policy %q: %v", name, err) + } + return &lbConfig{childBuilder: builder, childConfig: cfg}, nil + } + + return nil, fmt.Errorf("no supported policies found in config: %v", string(cfg)) +} diff --git a/internal/balancer/gracefulswitch/gracefulswitch.go b/internal/balancer/gracefulswitch/gracefulswitch.go new file mode 100644 index 0000000..f38de74 --- /dev/null +++ b/internal/balancer/gracefulswitch/gracefulswitch.go @@ -0,0 +1,421 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package gracefulswitch implements a graceful switch load balancer. +package gracefulswitch + +import ( + "errors" + "fmt" + "sync" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" +) + +var errBalancerClosed = errors.New("gracefulSwitchBalancer is closed") +var _ balancer.Balancer = (*Balancer)(nil) + +// NewBalancer returns a graceful switch Balancer. +func NewBalancer(cc balancer.ClientConn, opts balancer.BuildOptions) *Balancer { + return &Balancer{ + cc: cc, + bOpts: opts, + } +} + +// Balancer is a utility to gracefully switch from one balancer to +// a new balancer. It implements the balancer.Balancer interface. +type Balancer struct { + bOpts balancer.BuildOptions + cc balancer.ClientConn + + // mu protects the following fields and all fields within balancerCurrent + // and balancerPending. mu does not need to be held when calling into the + // child balancers, as all calls into these children happen only as a direct + // result of a call into the gracefulSwitchBalancer, which are also + // guaranteed to be synchronous. There is one exception: an UpdateState call + // from a child balancer when current and pending are populated can lead to + // calling Close() on the current. To prevent that racing with an + // UpdateSubConnState from the channel, we hold currentMu during Close and + // UpdateSubConnState calls. + mu sync.Mutex + balancerCurrent *balancerWrapper + balancerPending *balancerWrapper + closed bool // set to true when this balancer is closed + + // currentMu must be locked before mu. This mutex guards against this + // sequence of events: UpdateSubConnState() called, finds the + // balancerCurrent, gives up lock, updateState comes in, causes Close() on + // balancerCurrent before the UpdateSubConnState is called on the + // balancerCurrent. + currentMu sync.Mutex + + // activeGoroutines tracks all the goroutines that this balancer has started + // and that should be waited on when the balancer closes. + activeGoroutines sync.WaitGroup +} + +// swap swaps out the current lb with the pending lb and updates the ClientConn. +// The caller must hold gsb.mu. +func (gsb *Balancer) swap() { + gsb.cc.UpdateState(gsb.balancerPending.lastState) + cur := gsb.balancerCurrent + gsb.balancerCurrent = gsb.balancerPending + gsb.balancerPending = nil + gsb.activeGoroutines.Add(1) + go func() { + defer gsb.activeGoroutines.Done() + gsb.currentMu.Lock() + defer gsb.currentMu.Unlock() + cur.Close() + }() +} + +// Helper function that checks if the balancer passed in is current or pending. +// The caller must hold gsb.mu. +func (gsb *Balancer) balancerCurrentOrPending(bw *balancerWrapper) bool { + return bw == gsb.balancerCurrent || bw == gsb.balancerPending +} + +// SwitchTo initializes the graceful switch process, which completes based on +// connectivity state changes on the current/pending balancer. Thus, the switch +// process is not complete when this method returns. This method must be called +// synchronously alongside the rest of the balancer.Balancer methods this +// Graceful Switch Balancer implements. +// +// Deprecated: use ParseConfig and pass a parsed config to UpdateClientConnState +// to cause the Balancer to automatically change to the new child when necessary. +func (gsb *Balancer) SwitchTo(builder balancer.Builder) error { + _, err := gsb.switchTo(builder) + return err +} + +func (gsb *Balancer) switchTo(builder balancer.Builder) (*balancerWrapper, error) { + gsb.mu.Lock() + if gsb.closed { + gsb.mu.Unlock() + return nil, errBalancerClosed + } + bw := &balancerWrapper{ + ClientConn: gsb.cc, + builder: builder, + gsb: gsb, + lastState: balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + }, + subconns: make(map[balancer.SubConn]bool), + } + balToClose := gsb.balancerPending // nil if there is no pending balancer + if gsb.balancerCurrent == nil { + gsb.balancerCurrent = bw + } else { + gsb.balancerPending = bw + } + gsb.mu.Unlock() + balToClose.Close() + // This function takes a builder instead of a balancer because builder.Build + // can call back inline, and this utility needs to handle the callbacks. + newBalancer := builder.Build(bw, gsb.bOpts) + if newBalancer == nil { + // This is illegal and should never happen; we clear the balancerWrapper + // we were constructing if it happens to avoid a potential panic. + gsb.mu.Lock() + if gsb.balancerPending != nil { + gsb.balancerPending = nil + } else { + gsb.balancerCurrent = nil + } + gsb.mu.Unlock() + return nil, balancer.ErrBadResolverState + } + + // This write doesn't need to take gsb.mu because this field never gets read + // or written to on any calls from the current or pending. Calls from grpc + // to this balancer are guaranteed to be called synchronously, so this + // bw.Balancer field will never be forwarded to until this SwitchTo() + // function returns. + bw.Balancer = newBalancer + return bw, nil +} + +// Returns nil if the graceful switch balancer is closed. +func (gsb *Balancer) latestBalancer() *balancerWrapper { + gsb.mu.Lock() + defer gsb.mu.Unlock() + if gsb.balancerPending != nil { + return gsb.balancerPending + } + return gsb.balancerCurrent +} + +// UpdateClientConnState forwards the update to the latest balancer created. +// +// If the state's BalancerConfig is the config returned by a call to +// gracefulswitch.ParseConfig, then this function will automatically SwitchTo +// the balancer indicated by the config before forwarding its config to it, if +// necessary. +func (gsb *Balancer) UpdateClientConnState(state balancer.ClientConnState) error { + // The resolver data is only relevant to the most recent LB Policy. + balToUpdate := gsb.latestBalancer() + gsbCfg, ok := state.BalancerConfig.(*lbConfig) + if ok { + // Switch to the child in the config unless it is already active. + if balToUpdate == nil || gsbCfg.childBuilder.Name() != balToUpdate.builder.Name() { + var err error + balToUpdate, err = gsb.switchTo(gsbCfg.childBuilder) + if err != nil { + return fmt.Errorf("could not switch to new child balancer: %w", err) + } + } + // Unwrap the child balancer's config. + state.BalancerConfig = gsbCfg.childConfig + } + + if balToUpdate == nil { + return errBalancerClosed + } + + // Perform this call without gsb.mu to prevent deadlocks if the child calls + // back into the channel. The latest balancer can never be closed during a + // call from the channel, even without gsb.mu held. + return balToUpdate.UpdateClientConnState(state) +} + +// ResolverError forwards the error to the latest balancer created. +func (gsb *Balancer) ResolverError(err error) { + // The resolver data is only relevant to the most recent LB Policy. + balToUpdate := gsb.latestBalancer() + if balToUpdate == nil { + gsb.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(err), + }) + return + } + // Perform this call without gsb.mu to prevent deadlocks if the child calls + // back into the channel. The latest balancer can never be closed during a + // call from the channel, even without gsb.mu held. + balToUpdate.ResolverError(err) +} + +// ExitIdle forwards the call to the latest balancer created. +// +// If the latest balancer does not support ExitIdle, the subConns are +// re-connected to manually. +func (gsb *Balancer) ExitIdle() { + balToUpdate := gsb.latestBalancer() + if balToUpdate == nil { + return + } + // There is no need to protect this read with a mutex, as the write to the + // Balancer field happens in SwitchTo, which completes before this can be + // called. + balToUpdate.ExitIdle() +} + +// updateSubConnState forwards the update to the appropriate child. +func (gsb *Balancer) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState, cb func(balancer.SubConnState)) { + gsb.currentMu.Lock() + defer gsb.currentMu.Unlock() + gsb.mu.Lock() + // Forward update to the appropriate child. Even if there is a pending + // balancer, the current balancer should continue to get SubConn updates to + // maintain the proper state while the pending is still connecting. + var balToUpdate *balancerWrapper + if gsb.balancerCurrent != nil && gsb.balancerCurrent.subconns[sc] { + balToUpdate = gsb.balancerCurrent + } else if gsb.balancerPending != nil && gsb.balancerPending.subconns[sc] { + balToUpdate = gsb.balancerPending + } + if balToUpdate == nil { + // SubConn belonged to a stale lb policy that has not yet fully closed, + // or the balancer was already closed. + gsb.mu.Unlock() + return + } + if state.ConnectivityState == connectivity.Shutdown { + delete(balToUpdate.subconns, sc) + } + gsb.mu.Unlock() + if cb != nil { + cb(state) + } else { + balToUpdate.UpdateSubConnState(sc, state) + } +} + +// UpdateSubConnState forwards the update to the appropriate child. +func (gsb *Balancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + gsb.updateSubConnState(sc, state, nil) +} + +// Close closes any active child balancers. +func (gsb *Balancer) Close() { + gsb.mu.Lock() + gsb.closed = true + currentBalancerToClose := gsb.balancerCurrent + gsb.balancerCurrent = nil + pendingBalancerToClose := gsb.balancerPending + gsb.balancerPending = nil + gsb.mu.Unlock() + + currentBalancerToClose.Close() + pendingBalancerToClose.Close() + gsb.activeGoroutines.Wait() +} + +// balancerWrapper wraps a balancer.Balancer, and overrides some Balancer +// methods to help cleanup SubConns created by the wrapped balancer. +// +// It implements the balancer.ClientConn interface and is passed down in that +// capacity to the wrapped balancer. It maintains a set of subConns created by +// the wrapped balancer and calls from the latter to create/update/shutdown +// SubConns update this set before being forwarded to the parent ClientConn. +// State updates from the wrapped balancer can result in invocation of the +// graceful switch logic. +type balancerWrapper struct { + balancer.ClientConn + balancer.Balancer + gsb *Balancer + builder balancer.Builder + + lastState balancer.State + subconns map[balancer.SubConn]bool // subconns created by this balancer +} + +// Close closes the underlying LB policy and shuts down the subconns it +// created. bw must not be referenced via balancerCurrent or balancerPending in +// gsb when called. gsb.mu must not be held. Does not panic with a nil +// receiver. +func (bw *balancerWrapper) Close() { + // before Close is called. + if bw == nil { + return + } + // There is no need to protect this read with a mutex, as Close() is + // impossible to be called concurrently with the write in SwitchTo(). The + // callsites of Close() for this balancer in Graceful Switch Balancer will + // never be called until SwitchTo() returns. + bw.Balancer.Close() + bw.gsb.mu.Lock() + for sc := range bw.subconns { + sc.Shutdown() + } + bw.gsb.mu.Unlock() +} + +func (bw *balancerWrapper) UpdateState(state balancer.State) { + // Hold the mutex for this entire call to ensure it cannot occur + // concurrently with other updateState() calls. This causes updates to + // lastState and calls to cc.UpdateState to happen atomically. + bw.gsb.mu.Lock() + defer bw.gsb.mu.Unlock() + bw.lastState = state + + // If Close() acquires the mutex before UpdateState(), the balancer + // will already have been removed from the current or pending state when + // reaching this point. + if !bw.gsb.balancerCurrentOrPending(bw) { + // Returning here ensures that (*Balancer).swap() is not invoked after + // (*Balancer).Close() and therefore prevents "use after close". + return + } + + if bw == bw.gsb.balancerCurrent { + // In the case that the current balancer exits READY, and there is a pending + // balancer, you can forward the pending balancer's cached State up to + // ClientConn and swap the pending into the current. This is because there + // is no reason to gracefully switch from and keep using the old policy as + // the ClientConn is not connected to any backends. + if state.ConnectivityState != connectivity.Ready && bw.gsb.balancerPending != nil { + bw.gsb.swap() + return + } + // Even if there is a pending balancer waiting to be gracefully switched to, + // continue to forward current balancer updates to the Client Conn. Ignoring + // state + picker from the current would cause undefined behavior/cause the + // system to behave incorrectly from the current LB policies perspective. + // Also, the current LB is still being used by grpc to choose SubConns per + // RPC, and thus should use the most updated form of the current balancer. + bw.gsb.cc.UpdateState(state) + return + } + // This method is now dealing with a state update from the pending balancer. + // If the current balancer is currently in a state other than READY, the new + // policy can be swapped into place immediately. This is because there is no + // reason to gracefully switch from and keep using the old policy as the + // ClientConn is not connected to any backends. + if state.ConnectivityState != connectivity.Connecting || bw.gsb.balancerCurrent.lastState.ConnectivityState != connectivity.Ready { + bw.gsb.swap() + } +} + +func (bw *balancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + bw.gsb.mu.Lock() + if !bw.gsb.balancerCurrentOrPending(bw) { + bw.gsb.mu.Unlock() + return nil, fmt.Errorf("%T at address %p that called NewSubConn is deleted", bw, bw) + } + bw.gsb.mu.Unlock() + + var sc balancer.SubConn + oldListener := opts.StateListener + opts.StateListener = func(state balancer.SubConnState) { bw.gsb.updateSubConnState(sc, state, oldListener) } + sc, err := bw.gsb.cc.NewSubConn(addrs, opts) + if err != nil { + return nil, err + } + bw.gsb.mu.Lock() + if !bw.gsb.balancerCurrentOrPending(bw) { // balancer was closed during this call + sc.Shutdown() + bw.gsb.mu.Unlock() + return nil, fmt.Errorf("%T at address %p that called NewSubConn is deleted", bw, bw) + } + bw.subconns[sc] = true + bw.gsb.mu.Unlock() + return sc, nil +} + +func (bw *balancerWrapper) ResolveNow(opts resolver.ResolveNowOptions) { + // Ignore ResolveNow requests from anything other than the most recent + // balancer, because older balancers were already removed from the config. + if bw != bw.gsb.latestBalancer() { + return + } + bw.gsb.cc.ResolveNow(opts) +} + +func (bw *balancerWrapper) RemoveSubConn(sc balancer.SubConn) { + // Note: existing third party balancers may call this, so it must remain + // until RemoveSubConn is fully removed. + sc.Shutdown() +} + +func (bw *balancerWrapper) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + bw.gsb.mu.Lock() + if !bw.gsb.balancerCurrentOrPending(bw) { + bw.gsb.mu.Unlock() + return + } + bw.gsb.mu.Unlock() + bw.gsb.cc.UpdateAddresses(sc, addrs) +} diff --git a/internal/balancer/gracefulswitch/gracefulswitch_test.go b/internal/balancer/gracefulswitch/gracefulswitch_test.go new file mode 100644 index 0000000..9f536e8 --- /dev/null +++ b/internal/balancer/gracefulswitch/gracefulswitch_test.go @@ -0,0 +1,1089 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package gracefulswitch + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func setup(t *testing.T) (*testutils.BalancerClientConn, *Balancer) { + tcc := testutils.NewBalancerClientConn(t) + return tcc, NewBalancer(tcc, balancer.BuildOptions{}) +} + +// TestSuccessfulFirstUpdate tests a basic scenario for the graceful switch load +// balancer, where it is setup with a balancer which should populate the current +// load balancer. Any ClientConn updates should then be forwarded to this +// current load balancer. +func (s) TestSuccessfulFirstUpdate(t *testing.T) { + _, gsb := setup(t) + if err := gsb.SwitchTo(mockBalancerBuilder1{}); err != nil { + t.Fatalf("Balancer.SwitchTo failed with error: %v", err) + } + if gsb.balancerCurrent == nil { + t.Fatal("current balancer not populated after a successful call to SwitchTo()") + } + // This will be used to update the graceful switch balancer. This update + // should simply be forwarded down to the current load balancing policy. + ccs := balancer.ClientConnState{ + BalancerConfig: mockBalancerConfig{}, + } + + // Updating ClientConnState should forward the update exactly as is to the + // current balancer. + if err := gsb.UpdateClientConnState(ccs); err != nil { + t.Fatalf("Balancer.UpdateClientConnState(%v) failed: %v", ccs, err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := gsb.balancerCurrent.Balancer.(*mockBalancer).waitForClientConnUpdate(ctx, ccs); err != nil { + t.Fatal(err) + } +} + +// TestTwoBalancersSameType tests the scenario where there is a graceful switch +// load balancer setup with a current and pending load balancer of the same +// type. Any ClientConn update should be forwarded to the current lb if there is +// a current lb and no pending lb, and only the pending lb if the graceful +// switch balancer contains both a current lb and a pending lb. The pending load +// balancer should also swap into current whenever it updates with a +// connectivity state other than CONNECTING. +func (s) TestTwoBalancersSameType(t *testing.T) { + tcc, gsb := setup(t) + // This will be used to update the graceful switch balancer. This update + // should simply be forwarded down to either the current or pending load + // balancing policy. + ccs := balancer.ClientConnState{ + BalancerConfig: mockBalancerConfig{}, + } + + gsb.SwitchTo(mockBalancerBuilder1{}) + gsb.UpdateClientConnState(ccs) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := gsb.balancerCurrent.Balancer.(*mockBalancer).waitForClientConnUpdate(ctx, ccs); err != nil { + t.Fatal(err) + } + + // The current balancer reporting READY should cause this state + // to be forwarded to the ClientConn. + gsb.balancerCurrent.Balancer.(*mockBalancer).updateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &neverErrPicker{}, + }) + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case state := <-tcc.NewStateCh: + if state != connectivity.Ready { + t.Fatalf("current balancer reports connectivity state %v, want %v", state, connectivity.Ready) + } + } + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case picker := <-tcc.NewPickerCh: + // Should receive a never err picker. + if _, err := picker.Pick(balancer.PickInfo{}); err != nil { + t.Fatalf("ClientConn should have received a never err picker from an UpdateState call") + } + } + + // An explicit call to switchTo, even if the same type, should cause the + // balancer to build a new balancer for pending. + gsb.SwitchTo(mockBalancerBuilder1{}) + if gsb.balancerPending == nil { + t.Fatal("pending balancer not populated after another call to SwitchTo()") + } + + // A ClientConn update received should be forwarded to the new pending LB + // policy, and not the current one. + gsb.UpdateClientConnState(ccs) + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if err := gsb.balancerCurrent.Balancer.(*mockBalancer).waitForClientConnUpdate(sCtx, ccs); err == nil { + t.Fatal("current balancer received a ClientConn update when there is a pending balancer") + } + if err := gsb.balancerPending.Balancer.(*mockBalancer).waitForClientConnUpdate(ctx, ccs); err != nil { + t.Fatal(err) + } + + // If the pending load balancer reports that is CONNECTING, no update should + // be sent to the ClientConn. + gsb.balancerPending.Balancer.(*mockBalancer).updateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + }) + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + select { + case <-tcc.NewStateCh: + t.Fatal("balancerPending reporting CONNECTING should not forward up to the ClientConn") + case <-sCtx.Done(): + } + + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + // If the pending load balancer reports a state other than CONNECTING, the + // pending load balancer is logically warmed up, and the ClientConn should + // be updated with the State and Picker to start using the new policy. The + // pending load balancing policy should also be switched into the current + // load balancer. + gsb.balancerPending.Balancer.(*mockBalancer).updateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &neverErrPicker{}, + }) + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case state := <-tcc.NewStateCh: + if state != connectivity.Ready { + t.Fatalf("pending balancer reports connectivity state %v, want %v", state, connectivity.Ready) + } + } + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case picker := <-tcc.NewPickerCh: + // This picker should be the recent one sent from UpdateState(), a never + // err picker, not the nil picker from two updateState() calls previous. + if picker == nil { + t.Fatalf("ClientConn should have received a never err picker, which is the most recent picker, from an UpdateState call") + } + if _, err := picker.Pick(balancer.PickInfo{}); err != nil { + t.Fatalf("ClientConn should have received a never err picker, which is the most recent picker, from an UpdateState call") + } + } + // The current balancer should be closed as a result of the swap. + if err := currBal.waitForClose(ctx); err != nil { + t.Fatal(err) + } +} + +// TestCurrentNotReadyPendingUpdate tests the scenario where there is a current +// and pending load balancer setup in the graceful switch load balancer, and the +// current LB is not in the connectivity state READY. Any update from the +// pending load balancer should cause the graceful switch load balancer to swap +// the pending into current, and update the ClientConn with the pending load +// balancers state. +func (s) TestCurrentNotReadyPendingUpdate(t *testing.T) { + tcc, gsb := setup(t) + gsb.SwitchTo(mockBalancerBuilder1{}) + gsb.SwitchTo(mockBalancerBuilder1{}) + if gsb.balancerPending == nil { + t.Fatal("pending balancer not populated after another call to SwitchTo()") + } + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + // Due to the current load balancer not being in state READY, any update + // from the pending load balancer should cause that update to be forwarded + // to the ClientConn and also cause the pending load balancer to swap into + // the current one. + gsb.balancerPending.Balancer.(*mockBalancer).updateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &neverErrPicker{}, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout waiting for an UpdateState call on the ClientConn") + case state := <-tcc.NewStateCh: + if state != connectivity.Connecting { + t.Fatalf("ClientConn received connectivity state %v, want %v (from pending)", state, connectivity.Connecting) + } + } + select { + case <-ctx.Done(): + t.Fatalf("timeout waiting for an UpdateState call on the ClientConn") + case picker := <-tcc.NewPickerCh: + // Should receive a never err picker. + if _, err := picker.Pick(balancer.PickInfo{}); err != nil { + t.Fatalf("ClientConn should have received a never err picker from an UpdateState call") + } + } + + // The current balancer should be closed as a result of the swap. + if err := currBal.waitForClose(ctx); err != nil { + t.Fatal(err) + } +} + +// TestCurrentLeavingReady tests the scenario where there is a current and +// pending load balancer setup in the graceful switch load balancer, with the +// current load balancer being in the state READY, and the current load balancer +// then transitions into a state other than READY. This should cause the pending +// load balancer to swap into the current load balancer, and the ClientConn to +// be updated with the cached pending load balancing state. Also, once the +// current is cleared from the graceful switch load balancer, any updates sent +// should be intercepted and not forwarded to the ClientConn, as the balancer +// has already been cleared. +func (s) TestCurrentLeavingReady(t *testing.T) { + tcc, gsb := setup(t) + gsb.SwitchTo(mockBalancerBuilder1{}) + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + currBal.updateState(balancer.State{ + ConnectivityState: connectivity.Ready, + }) + + gsb.SwitchTo(mockBalancerBuilder2{}) + // Sends CONNECTING, shouldn't make it's way to ClientConn. + gsb.balancerPending.Balancer.(*mockBalancer).updateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &neverErrPicker{}, + }) + + // The current balancer leaving READY should cause the pending balancer to + // swap to the current balancer. This swap from current to pending should + // also update the ClientConn with the pending balancers cached state and + // picker. + currBal.updateState(balancer.State{ + ConnectivityState: connectivity.Idle, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case state := <-tcc.NewStateCh: + if state != connectivity.Connecting { + t.Fatalf("current balancer reports connectivity state %v, want %v", state, connectivity.Connecting) + } + } + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case picker := <-tcc.NewPickerCh: + // Should receive a never err picker cached from pending LB's updateState() call, which + // was cached. + if _, err := picker.Pick(balancer.PickInfo{}); err != nil { + t.Fatalf("ClientConn should have received a never err picker, the cached picker, from an UpdateState call") + } + } + + // The current balancer should be closed as a result of the swap. + if err := currBal.waitForClose(ctx); err != nil { + t.Fatal(err) + } + + // The current balancer is now cleared from the graceful switch load + // balancer. Thus, any update from the old current should be intercepted by + // the graceful switch load balancer and not forward up to the ClientConn. + currBal.updateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &neverErrPicker{}, + }) + + // This update should not be forwarded to the ClientConn. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-tcc.NewStateCh: + t.Fatal("UpdateState() from a cleared balancer should not make it's way to ClientConn") + } + + if _, err := currBal.newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}); err == nil { + t.Fatal("newSubConn() from a cleared balancer should have returned an error") + } + + // This newSubConn call should also not reach the ClientConn. + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-tcc.NewSubConnCh: + t.Fatal("newSubConn() from a cleared balancer should not make it's way to ClientConn") + } +} + +// TestBalancerSubconns tests the SubConn functionality of the graceful switch +// load balancer. This tests the SubConn update flow in both directions, and +// make sure updates end up at the correct component. +func (s) TestBalancerSubconns(t *testing.T) { + tcc, gsb := setup(t) + gsb.SwitchTo(mockBalancerBuilder1{}) + gsb.SwitchTo(mockBalancerBuilder2{}) + + // A child balancer creating a new SubConn should eventually be forwarded to + // the ClientConn held by the graceful switch load balancer. + sc1, err := gsb.balancerCurrent.Balancer.(*mockBalancer).newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error constructing newSubConn in gsb: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an NewSubConn call on the ClientConn") + case sc := <-tcc.NewSubConnCh: + if sc != sc1 { + t.Fatalf("NewSubConn, want %v, got %v", sc1, sc) + } + } + + // The other child balancer creating a new SubConn should also be eventually + // be forwarded to the ClientConn held by the graceful switch load balancer. + sc2, err := gsb.balancerPending.Balancer.(*mockBalancer).newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error constructing newSubConn in gsb: %v", err) + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an NewSubConn call on the ClientConn") + case sc := <-tcc.NewSubConnCh: + if sc != sc2 { + t.Fatalf("NewSubConn, want %v, got %v", sc2, sc) + } + } + scState := balancer.SubConnState{ConnectivityState: connectivity.Ready} + // Updating the SubConnState for sc1 should cause the graceful switch + // balancer to forward the Update to balancerCurrent for sc1, as that is the + // balancer that created this SubConn. + sc1.(*testutils.TestSubConn).UpdateState(scState) + + // Updating the SubConnState for sc2 should cause the graceful switch + // balancer to forward the Update to balancerPending for sc2, as that is the + // balancer that created this SubConn. + sc2.(*testutils.TestSubConn).UpdateState(scState) + + // Updating the addresses for both SubConns and removing both SubConns + // should get forwarded to the ClientConn. + + // Updating the addresses for sc1 should get forwarded to the ClientConn. + gsb.balancerCurrent.Balancer.(*mockBalancer).updateAddresses(sc1, []resolver.Address{}) + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses call on the ClientConn") + case <-tcc.UpdateAddressesAddrsCh: + } + + // Updating the addresses for sc2 should also get forwarded to the ClientConn. + gsb.balancerPending.Balancer.(*mockBalancer).updateAddresses(sc2, []resolver.Address{}) + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses call on the ClientConn") + case <-tcc.UpdateAddressesAddrsCh: + } + + // balancerCurrent removing sc1 should get forwarded to the ClientConn. + sc1.Shutdown() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses call on the ClientConn") + case sc := <-tcc.ShutdownSubConnCh: + if sc != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, sc) + } + } + // balancerPending removing sc2 should get forwarded to the ClientConn. + sc2.Shutdown() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses call on the ClientConn") + case sc := <-tcc.ShutdownSubConnCh: + if sc != sc2 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc2, sc) + } + } +} + +// TestBalancerClose tests the graceful switch balancer's Close() +// functionality. From the Close() call, the graceful switch balancer should +// shut down any created Subconns and Close() the current and pending load +// balancers. This Close() call should also cause any other events (calls to +// entrance functions) to be no-ops. +func (s) TestBalancerClose(t *testing.T) { + // Setup gsb balancer with current, pending, and one created SubConn on both + // current and pending. + tcc, gsb := setup(t) + gsb.SwitchTo(mockBalancerBuilder1{}) + gsb.SwitchTo(mockBalancerBuilder2{}) + + sc1, err := gsb.balancerCurrent.Balancer.(*mockBalancer).newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + // Will eventually get back a SubConn with an identifying property id 1 + if err != nil { + t.Fatalf("error constructing newSubConn in gsb: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an NewSubConn call on the ClientConn") + case <-tcc.NewSubConnCh: + } + + sc2, err := gsb.balancerPending.Balancer.(*mockBalancer).newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + // Will eventually get back a SubConn with an identifying property id 2 + if err != nil { + t.Fatalf("error constructing newSubConn in gsb: %v", err) + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an NewSubConn call on the ClientConn") + case <-tcc.NewSubConnCh: + } + + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + pendBal := gsb.balancerPending.Balancer.(*mockBalancer) + + // Closing the graceful switch load balancer should lead to removing any + // created SubConns, and closing both the current and pending load balancer. + gsb.Close() + + // The order of SubConns the graceful switch load balancer tells the Client + // Conn to shut down is non deterministic, as it is stored in a + // map. However, the first SubConn shut down should be either sc1 or sc2. + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses call on the ClientConn") + case sc := <-tcc.ShutdownSubConnCh: + if sc != sc1 && sc != sc2 { + t.Fatalf("ShutdownSubConn, want either %v or %v, got %v", sc1, sc2, sc) + } + } + + // The graceful switch load balancer should then tell the ClientConn to + // shut down the other SubConn. + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses call on the ClientConn") + case sc := <-tcc.ShutdownSubConnCh: + if sc != sc1 && sc != sc2 { + t.Fatalf("ShutdownSubConn, want either %v or %v, got %v", sc1, sc2, sc) + } + } + + // The current balancer should get closed as a result of the graceful switch balancer being closed. + if err := currBal.waitForClose(ctx); err != nil { + t.Fatal(err) + } + // The pending balancer should also get closed as a result of the graceful switch balancer being closed. + if err := pendBal.waitForClose(ctx); err != nil { + t.Fatal(err) + } + + // Once the graceful switch load balancer has been closed, any entrance + // function should be a no-op and return errBalancerClosed if the function + // returns an error. + + // SwitchTo() should return an error due to the graceful switch load + // balancer having been closed already. + if err := gsb.SwitchTo(mockBalancerBuilder1{}); err != errBalancerClosed { + t.Fatalf("gsb.SwitchTo(%v) returned error %v, want %v", mockBalancerBuilder1{}, err, errBalancerClosed) + } + + // UpdateClientConnState() should return an error due to the graceful switch + // load balancer having been closed already. + ccs := balancer.ClientConnState{ + BalancerConfig: mockBalancerConfig{}, + } + if err := gsb.UpdateClientConnState(ccs); err != errBalancerClosed { + t.Fatalf("gsb.UpdateClientConnState(%v) returned error %v, want %v", ccs, err, errBalancerClosed) + } + + // After the graceful switch load balancer has been closed, any resolver error + // shouldn't forward to either balancer, as the resolver error is a no-op + // and also even if not, the balancers should have been cleared from the + // graceful switch load balancer. + gsb.ResolverError(balancer.ErrBadResolverState) + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if err := currBal.waitForResolverError(sCtx, balancer.ErrBadResolverState); !strings.Contains(err.Error(), sCtx.Err().Error()) { + t.Fatal("the current balancer should not have received the resolver error after close") + } + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if err := pendBal.waitForResolverError(sCtx, balancer.ErrBadResolverState); !strings.Contains(err.Error(), sCtx.Err().Error()) { + t.Fatal("the pending balancer should not have received the resolver error after close") + } +} + +// TestResolverError tests the functionality of a Resolver Error. If there is a +// current balancer, but no pending, the error should be forwarded to the +// current balancer. If there is both a current and pending balancer, the error +// should be forwarded to only the pending balancer. +func (s) TestResolverError(t *testing.T) { + _, gsb := setup(t) + gsb.SwitchTo(mockBalancerBuilder1{}) + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + // If there is only a current balancer present, the resolver error should be + // forwarded to the current balancer. + gsb.ResolverError(balancer.ErrBadResolverState) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := currBal.waitForResolverError(ctx, balancer.ErrBadResolverState); err != nil { + t.Fatal(err) + } + + gsb.SwitchTo(mockBalancerBuilder1{}) + + // If there is a pending balancer present, then a resolver error should be + // forwarded to only the pending balancer, not the current. + pendBal := gsb.balancerPending.Balancer.(*mockBalancer) + gsb.ResolverError(balancer.ErrBadResolverState) + + // The Resolver Error should not be forwarded to the current load balancer. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if err := currBal.waitForResolverError(sCtx, balancer.ErrBadResolverState); !strings.Contains(err.Error(), sCtx.Err().Error()) { + t.Fatal("the current balancer should not have received the resolver error after close") + } + + // The Resolver Error should be forwarded to the pending load balancer. + if err := pendBal.waitForResolverError(ctx, balancer.ErrBadResolverState); err != nil { + t.Fatal(err) + } +} + +// TestPendingReplacedByAnotherPending tests the scenario where a graceful +// switch balancer has a current and pending load balancer, and receives a +// SwitchTo() call, which then replaces the pending. This should cause the +// graceful switch balancer to clear pending state, close old pending SubConns, +// and Close() the pending balancer being replaced. +func (s) TestPendingReplacedByAnotherPending(t *testing.T) { + tcc, gsb := setup(t) + gsb.SwitchTo(mockBalancerBuilder1{}) + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + currBal.updateState(balancer.State{ + ConnectivityState: connectivity.Ready, + }) + + // Populate pending with a SwitchTo() call. + gsb.SwitchTo(mockBalancerBuilder2{}) + + pendBal := gsb.balancerPending.Balancer.(*mockBalancer) + sc1, err := pendBal.newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error constructing newSubConn in gsb: %v", err) + } + // This picker never returns an error, which can help this test verify + // whether this cached state will get cleared on a new pending balancer + // (will replace it with a picker that always errors). + pendBal.updateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &neverErrPicker{}, + }) + + // Replace pending with a SwitchTo() call. + gsb.SwitchTo(mockBalancerBuilder2{}) + // The pending balancer being replaced should cause the graceful switch + // balancer to Shutdown() any created SubConns for the old pending balancer + // and also Close() the old pending balancer. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a SubConn.Shutdown") + case sc := <-tcc.ShutdownSubConnCh: + if sc != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, sc) + } + } + + if err := pendBal.waitForClose(ctx); err != nil { + t.Fatal(err) + } + + // Switching the current out of READY should cause the pending LB to swap + // into current, causing the graceful switch balancer to update the + // ClientConn with the cached pending state. Since the new pending hasn't + // sent an Update, the default state with connectivity state CONNECTING and + // an errPicker should be sent to the ClientConn. + currBal.updateState(balancer.State{ + ConnectivityState: connectivity.Idle, + }) + + // The update should contain a default connectivity state CONNECTING for the + // state of the new pending LB policy. + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateState() call on the ClientConn") + case state := <-tcc.NewStateCh: + if state != connectivity.Connecting { + t.Fatalf("UpdateState(), want connectivity state %v, got %v", connectivity.Connecting, state) + } + } + // The update should contain a default picker ErrPicker in the picker sent + // for the state of the new pending LB policy. + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateState() call on the ClientConn") + case picker := <-tcc.NewPickerCh: + if _, err := picker.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("ClientConn should have received a never err picker from an UpdateState call") + } + } +} + +// Picker which never errors here for test purposes (can fill up tests further up with this) +type neverErrPicker struct{} + +func (p *neverErrPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + return balancer.PickResult{}, nil +} + +// TestUpdateSubConnStateRace tests the race condition when the graceful switch +// load balancer receives a SubConnUpdate concurrently with an UpdateState() +// call, which can cause the balancer to forward the update to be closed and +// cleared. The balancer API guarantees to never call any method the balancer +// after a Close() call, and the test verifies that doesn't happen within the +// graceful switch load balancer. +func (s) TestUpdateSubConnStateRace(t *testing.T) { + tcc, gsb := setup(t) + gsb.SwitchTo(verifyBalancerBuilder{}) + gsb.SwitchTo(mockBalancerBuilder1{}) + currBal := gsb.balancerCurrent.Balancer.(*verifyBalancer) + currBal.t = t + pendBal := gsb.balancerPending.Balancer.(*mockBalancer) + sc, err := currBal.newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error constructing newSubConn in gsb: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an NewSubConn call on the ClientConn") + case <-tcc.NewSubConnCh: + } + // Spawn a goroutine that constantly calls UpdateSubConn for the current + // balancer, which will get deleted in this testing goroutine. + finished := make(chan struct{}) + go func() { + for { + select { + case <-finished: + return + default: + } + sc.(*testutils.TestSubConn).UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.Ready, + }) + } + }() + time.Sleep(time.Millisecond) + // This UpdateState call causes current to be closed/cleared. + pendBal.updateState(balancer.State{ + ConnectivityState: connectivity.Ready, + }) + // From this, either one of two things happen. Either the graceful switch + // load balancer doesn't Close() the current balancer before it forwards the + // SubConn update to the child, and the call gets forwarded down to the + // current balancer, or it can Close() the current balancer in between + // reading the balancer pointer and writing to it, and in that case the old + // current balancer should not be updated, as the balancer has already been + // closed and the balancer API guarantees it. + close(finished) +} + +// TestInlineCallbackInBuild tests the scenario where a balancer calls back into +// the balancer.ClientConn API inline from its build function. +func (s) TestInlineCallbackInBuild(t *testing.T) { + tcc, gsb := setup(t) + // This build call should cause all of the inline updates to forward to the + // ClientConn. + gsb.SwitchTo(buildCallbackBalancerBuilder{}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateState() call on the ClientConn") + case <-tcc.NewStateCh: + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a NewSubConn() call on the ClientConn") + case <-tcc.NewSubConnCh: + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses() call on the ClientConn") + case <-tcc.UpdateAddressesAddrsCh: + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a Shutdown() call on the SubConn") + case <-tcc.ShutdownSubConnCh: + } + oldCurrent := gsb.balancerCurrent.Balancer.(*buildCallbackBal) + + // Since the callback reports a state READY, this new inline balancer should + // be swapped to the current. + gsb.SwitchTo(buildCallbackBalancerBuilder{}) + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateState() call on the ClientConn") + case <-tcc.NewStateCh: + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a NewSubConn() call on the ClientConn") + case <-tcc.NewSubConnCh: + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses() call on the ClientConn") + case <-tcc.UpdateAddressesAddrsCh: + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a Shutdown() call on the SubConn") + case <-tcc.ShutdownSubConnCh: + } + + // The current balancer should be closed as a result of the swap. + if err := oldCurrent.waitForClose(ctx); err != nil { + t.Fatalf("error waiting for balancer close: %v", err) + } + + // The old balancer should be deprecated and any calls from it should be a no-op. + oldCurrent.newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + select { + case <-tcc.NewSubConnCh: + t.Fatal("Deprecated LB calling NewSubConn() should not forward up to the ClientConn") + case <-sCtx.Done(): + } +} + +// TestExitIdle tests the ExitIdle operation on the Graceful Switch Balancer. +func (s) TestExitIdle(t *testing.T) { + _, gsb := setup(t) + // switch to a balancer that implements ExitIdle{} (will populate current). + gsb.SwitchTo(mockBalancerBuilder1{}) + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // exitIdle on the Graceful Switch Balancer should get forwarded to the + // current child as it implements exitIdle. + gsb.ExitIdle() + if err := currBal.waitForExitIdle(ctx); err != nil { + t.Fatal(err) + } +} + +const balancerName1 = "mock_balancer_1" +const balancerName2 = "mock_balancer_2" +const verifyBalName = "verifyNoSubConnUpdateAfterCloseBalancer" +const buildCallbackBalName = "callbackInBuildBalancer" + +type mockBalancerBuilder1 struct{} + +func (mockBalancerBuilder1) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { + return &mockBalancer{ + ccsCh: testutils.NewChannel(), + scStateCh: testutils.NewChannel(), + resolverErrCh: testutils.NewChannel(), + closeCh: testutils.NewChannel(), + exitIdleCh: testutils.NewChannel(), + cc: cc, + } +} + +func (mockBalancerBuilder1) Name() string { + return balancerName1 +} + +type mockBalancerConfig struct { + serviceconfig.LoadBalancingConfig +} + +// mockBalancer is a fake balancer used to verify different actions from +// the gracefulswitch. It contains a bunch of channels to signal different events +// to the test. +type mockBalancer struct { + // ccsCh is a channel used to signal the receipt of a ClientConn update. + ccsCh *testutils.Channel + // scStateCh is a channel used to signal the receipt of a SubConn update. + scStateCh *testutils.Channel + // resolverErrCh is a channel used to signal a resolver error. + resolverErrCh *testutils.Channel + // closeCh is a channel used to signal the closing of this balancer. + closeCh *testutils.Channel + // exitIdleCh is a channel used to signal the receipt of an ExitIdle call. + exitIdleCh *testutils.Channel + // Hold onto ClientConn wrapper to communicate with it + cc balancer.ClientConn +} + +type subConnWithState struct { + sc balancer.SubConn + state balancer.SubConnState +} + +func (mb1 *mockBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { + // Need to verify this call...use a channel?...all of these will need verification + mb1.ccsCh.Send(ccs) + return nil +} + +func (mb1 *mockBalancer) ResolverError(err error) { + mb1.resolverErrCh.Send(err) +} + +func (mb1 *mockBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + panic(fmt.Sprintf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)) +} + +func (mb1 *mockBalancer) Close() { + mb1.closeCh.Send(struct{}{}) +} + +func (mb1 *mockBalancer) ExitIdle() { + mb1.exitIdleCh.Send(struct{}{}) +} + +// waitForClientConnUpdate verifies if the mockBalancer receives the +// provided ClientConnState within a reasonable amount of time. +func (mb1 *mockBalancer) waitForClientConnUpdate(ctx context.Context, wantCCS balancer.ClientConnState) error { + ccs, err := mb1.ccsCh.Receive(ctx) + if err != nil { + return fmt.Errorf("error waiting for ClientConnUpdate: %v", err) + } + gotCCS := ccs.(balancer.ClientConnState) + if diff := cmp.Diff(gotCCS, wantCCS, cmpopts.IgnoreFields(resolver.State{}, "Attributes")); diff != "" { + return fmt.Errorf("error in ClientConnUpdate: received unexpected ClientConnState, diff (-got +want): %v", diff) + } + return nil +} + +// waitForResolverError verifies if the mockBalancer receives the provided +// resolver error before the context expires. +func (mb1 *mockBalancer) waitForResolverError(ctx context.Context, wantErr error) error { + gotErr, err := mb1.resolverErrCh.Receive(ctx) + if err != nil { + return fmt.Errorf("error waiting for resolver error: %v", err) + } + if gotErr != wantErr { + return fmt.Errorf("received resolver error: %v, want %v", gotErr, wantErr) + } + return nil +} + +// waitForClose verifies that the mockBalancer is closed before the context +// expires. +func (mb1 *mockBalancer) waitForClose(ctx context.Context) error { + if _, err := mb1.closeCh.Receive(ctx); err != nil { + return fmt.Errorf("error waiting for Close(): %v", err) + } + return nil +} + +// waitForExitIdle verifies that ExitIdle gets called on the mockBalancer before +// the context expires. +func (mb1 *mockBalancer) waitForExitIdle(ctx context.Context) error { + if _, err := mb1.exitIdleCh.Receive(ctx); err != nil { + return fmt.Errorf("error waiting for ExitIdle(): %v", err) + } + return nil +} + +func (mb1 *mockBalancer) updateState(state balancer.State) { + mb1.cc.UpdateState(state) +} + +func (mb1 *mockBalancer) newSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (sc balancer.SubConn, err error) { + if opts.StateListener == nil { + opts.StateListener = func(state balancer.SubConnState) { + mb1.scStateCh.Send(subConnWithState{sc: sc, state: state}) + } + } + defer func() { + if sc != nil { + sc.Connect() + } + }() + return mb1.cc.NewSubConn(addrs, opts) +} + +func (mb1 *mockBalancer) updateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + mb1.cc.UpdateAddresses(sc, addrs) +} + +type mockBalancerBuilder2 struct{} + +func (mockBalancerBuilder2) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { + return &mockBalancer{ + ccsCh: testutils.NewChannel(), + scStateCh: testutils.NewChannel(), + resolverErrCh: testutils.NewChannel(), + closeCh: testutils.NewChannel(), + cc: cc, + } +} + +func (mockBalancerBuilder2) Name() string { + return balancerName2 +} + +type verifyBalancerBuilder struct{} + +func (verifyBalancerBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { + return &verifyBalancer{ + closed: grpcsync.NewEvent(), + cc: cc, + } +} + +func (verifyBalancerBuilder) Name() string { + return verifyBalName +} + +// verifyBalancer is a balancer that verifies that after a Close() call, a +// StateListener() call never happens. +type verifyBalancer struct { + closed *grpcsync.Event + // Hold onto the ClientConn wrapper to communicate with it. + cc balancer.ClientConn + // To fail the test if StateListener gets called after Close(). + t *testing.T +} + +func (vb *verifyBalancer) UpdateClientConnState(balancer.ClientConnState) error { + return nil +} + +func (vb *verifyBalancer) ExitIdle() {} + +func (vb *verifyBalancer) ResolverError(error) {} + +func (vb *verifyBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + panic(fmt.Sprintf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)) +} + +func (vb *verifyBalancer) Close() { + vb.closed.Fire() +} + +func (vb *verifyBalancer) newSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (sc balancer.SubConn, err error) { + if opts.StateListener == nil { + opts.StateListener = func(state balancer.SubConnState) { + if vb.closed.HasFired() { + vb.t.Fatalf("StateListener(%+v) was called after Close(), which breaks the balancer API", state) + } + } + } + defer func() { sc.Connect() }() + return vb.cc.NewSubConn(addrs, opts) +} + +type buildCallbackBalancerBuilder struct{} + +func (buildCallbackBalancerBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { + b := &buildCallbackBal{ + cc: cc, + closeCh: testutils.NewChannel(), + } + b.updateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + }) + sc, err := b.newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + if err != nil { + return nil + } + b.updateAddresses(sc, []resolver.Address{}) + sc.Shutdown() + return b +} + +func (buildCallbackBalancerBuilder) Name() string { + return buildCallbackBalName +} + +type buildCallbackBal struct { + // Hold onto the ClientConn wrapper to communicate with it. + cc balancer.ClientConn + // closeCh is a channel used to signal the closing of this balancer. + closeCh *testutils.Channel +} + +func (bcb *buildCallbackBal) UpdateClientConnState(balancer.ClientConnState) error { + return nil +} + +func (bcb *buildCallbackBal) ResolverError(error) {} + +func (bcb *buildCallbackBal) ExitIdle() {} + +func (bcb *buildCallbackBal) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + panic(fmt.Sprintf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)) +} + +func (bcb *buildCallbackBal) Close() { + bcb.closeCh.Send(struct{}{}) +} + +func (bcb *buildCallbackBal) updateState(state balancer.State) { + bcb.cc.UpdateState(state) +} + +func (bcb *buildCallbackBal) newSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (sc balancer.SubConn, err error) { + defer func() { + if sc != nil { + sc.Connect() + } + }() + return bcb.cc.NewSubConn(addrs, opts) +} + +func (bcb *buildCallbackBal) updateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + bcb.cc.UpdateAddresses(sc, addrs) +} + +// waitForClose verifies that the mockBalancer is closed before the context +// expires. +func (bcb *buildCallbackBal) waitForClose(ctx context.Context) error { + if _, err := bcb.closeCh.Receive(ctx); err != nil { + return err + } + return nil +} diff --git a/internal/balancer/nop/nop.go b/internal/balancer/nop/nop.go new file mode 100644 index 0000000..8c5bf96 --- /dev/null +++ b/internal/balancer/nop/nop.go @@ -0,0 +1,65 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package nop implements a balancer with all of its balancer operations as +// no-ops, other than returning a Transient Failure Picker on a Client Conn +// update. +package nop + +import ( + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" +) + +// bal is a balancer with all of its balancer operations as no-ops, other than +// returning a Transient Failure Picker on a Client Conn update. +type bal struct { + cc balancer.ClientConn + err error +} + +// NewBalancer returns a no-op balancer. +func NewBalancer(cc balancer.ClientConn, err error) balancer.Balancer { + return &bal{ + cc: cc, + err: err, + } +} + +// UpdateClientConnState updates the bal's Client Conn with an Error Picker +// and a Connectivity State of TRANSIENT_FAILURE. +func (b *bal) UpdateClientConnState(_ balancer.ClientConnState) error { + b.cc.UpdateState(balancer.State{ + Picker: base.NewErrPicker(b.err), + ConnectivityState: connectivity.TransientFailure, + }) + return nil +} + +// ResolverError is a no-op. +func (b *bal) ResolverError(_ error) {} + +// UpdateSubConnState is a no-op. +func (b *bal) UpdateSubConnState(_ balancer.SubConn, _ balancer.SubConnState) {} + +// Close is a no-op. +func (b *bal) Close() {} + +// ExitIdle is a no-op. +func (b *bal) ExitIdle() {} diff --git a/internal/balancer/stub/stub.go b/internal/balancer/stub/stub.go new file mode 100644 index 0000000..aec5551 --- /dev/null +++ b/internal/balancer/stub/stub.go @@ -0,0 +1,116 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package stub implements a balancer for testing purposes. +package stub + +import ( + "encoding/json" + "fmt" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/serviceconfig" +) + +// BalancerFuncs contains all balancer.Balancer functions with a preceding +// *BalancerData parameter for passing additional instance information. Any +// nil functions will never be called. +type BalancerFuncs struct { + // Init is called after ClientConn and BuildOptions are set in + // BalancerData. It may be used to initialize BalancerData.Data. + Init func(*BalancerData) + // ParseConfig is used for parsing LB configs, if specified. + ParseConfig func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) + + UpdateClientConnState func(*BalancerData, balancer.ClientConnState) error + ResolverError func(*BalancerData, error) + Close func(*BalancerData) + ExitIdle func(*BalancerData) +} + +// BalancerData contains data relevant to a stub balancer. +type BalancerData struct { + // ClientConn is set by the builder. + ClientConn balancer.ClientConn + // BuildOptions is set by the builder. + BuildOptions balancer.BuildOptions + // ChildBalancer holds a child balancer. + ChildBalancer balancer.Balancer +} + +type bal struct { + bf BalancerFuncs + bd *BalancerData +} + +func (b *bal) UpdateClientConnState(c balancer.ClientConnState) error { + if b.bf.UpdateClientConnState != nil { + return b.bf.UpdateClientConnState(b.bd, c) + } + return nil +} + +func (b *bal) ResolverError(e error) { + if b.bf.ResolverError != nil { + b.bf.ResolverError(b.bd, e) + } +} + +func (b *bal) UpdateSubConnState(sc balancer.SubConn, scs balancer.SubConnState) { + panic(fmt.Sprintf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, scs)) +} + +func (b *bal) Close() { + if b.bf.Close != nil { + b.bf.Close(b.bd) + } +} + +func (b *bal) ExitIdle() { + if b.bf.ExitIdle != nil { + b.bf.ExitIdle(b.bd) + } +} + +type bb struct { + name string + bf BalancerFuncs +} + +func (bb bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + b := &bal{bf: bb.bf, bd: &BalancerData{ClientConn: cc, BuildOptions: opts}} + if b.bf.Init != nil { + b.bf.Init(b.bd) + } + return b +} + +func (bb bb) Name() string { return bb.name } + +func (bb bb) ParseConfig(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + if bb.bf.ParseConfig != nil { + return bb.bf.ParseConfig(lbCfg) + } + return nil, nil +} + +// Register registers a stub balancer builder which will call the provided +// functions. The name used should be unique. +func Register(name string, bf BalancerFuncs) { + balancer.Register(bb{name: name, bf: bf}) +} diff --git a/internal/balancergroup/balancergroup.go b/internal/balancergroup/balancergroup.go new file mode 100644 index 0000000..8acb942 --- /dev/null +++ b/internal/balancergroup/balancergroup.go @@ -0,0 +1,607 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package balancergroup implements a utility struct to bind multiple balancers +// into one balancer. +package balancergroup + +import ( + "encoding/json" + "fmt" + "sync" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/gracefulswitch" + "google.golang.org/grpc/internal/cache" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// subBalancerWrapper is used to keep the configurations that will be used to start +// the underlying balancer. It can be called to start/stop the underlying +// balancer. +// +// When the config changes, it will pass the update to the underlying balancer +// if it exists. +// +// TODO: move to a separate file? +type subBalancerWrapper struct { + // subBalancerWrapper is passed to the sub-balancer as a ClientConn + // wrapper, only to keep the state and picker. When sub-balancer is + // restarted while in cache, the picker needs to be resent. + // + // It also contains the sub-balancer ID, so the parent balancer group can + // keep track of SubConn/pickers and the sub-balancers they belong to. Some + // of the actions are forwarded to the parent ClientConn with no change. + // Some are forward to balancer group with the sub-balancer ID. + balancer.ClientConn + id string + group *BalancerGroup + + mu sync.Mutex + state balancer.State + + // The static part of sub-balancer. Keeps balancerBuilders and addresses. + // To be used when restarting sub-balancer. + builder balancer.Builder + // Options to be passed to sub-balancer at the time of creation. + buildOpts balancer.BuildOptions + // ccState is a cache of the addresses/balancer config, so when the balancer + // is restarted after close, it will get the previous update. It's a pointer + // and is set to nil at init, so when the balancer is built for the first + // time (not a restart), it won't receive an empty update. Note that this + // isn't reset to nil when the underlying balancer is closed. + ccState *balancer.ClientConnState + // The dynamic part of sub-balancer. Only used when balancer group is + // started. Gets cleared when sub-balancer is closed. + balancer *gracefulswitch.Balancer +} + +// UpdateState overrides balancer.ClientConn, to keep state and picker. +func (sbc *subBalancerWrapper) UpdateState(state balancer.State) { + sbc.mu.Lock() + sbc.state = state + sbc.group.updateBalancerState(sbc.id, state) + sbc.mu.Unlock() +} + +// NewSubConn overrides balancer.ClientConn, so balancer group can keep track of +// the relation between subconns and sub-balancers. +func (sbc *subBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + return sbc.group.newSubConn(sbc, addrs, opts) +} + +func (sbc *subBalancerWrapper) updateBalancerStateWithCachedPicker() { + sbc.mu.Lock() + if sbc.state.Picker != nil { + sbc.group.updateBalancerState(sbc.id, sbc.state) + } + sbc.mu.Unlock() +} + +func (sbc *subBalancerWrapper) startBalancer() { + if sbc.balancer == nil { + sbc.balancer = gracefulswitch.NewBalancer(sbc, sbc.buildOpts) + } + sbc.group.logger.Infof("Creating child policy of type %q for child %q", sbc.builder.Name(), sbc.id) + sbc.balancer.SwitchTo(sbc.builder) + if sbc.ccState != nil { + sbc.balancer.UpdateClientConnState(*sbc.ccState) + } +} + +// exitIdle invokes the ExitIdle method on the sub-balancer, a gracefulswitch +// balancer. +func (sbc *subBalancerWrapper) exitIdle() { + b := sbc.balancer + if b == nil { + return + } + b.ExitIdle() +} + +func (sbc *subBalancerWrapper) updateClientConnState(s balancer.ClientConnState) error { + sbc.ccState = &s + b := sbc.balancer + if b == nil { + // A sub-balancer is closed when it is removed from the group or the + // group is closed as a whole, and is not expected to receive updates + // after that. But when used with the priority LB policy a sub-balancer + // (and the whole balancer group) could be closed because it's the lower + // priority, but it can still get address updates. + return nil + } + return b.UpdateClientConnState(s) +} + +func (sbc *subBalancerWrapper) resolverError(err error) { + b := sbc.balancer + if b == nil { + // A sub-balancer is closed when it is removed from the group or the + // group is closed as a whole, and is not expected to receive updates + // after that. But when used with the priority LB policy a sub-balancer + // (and the whole balancer group) could be closed because it's the lower + // priority, but it can still get address updates. + return + } + b.ResolverError(err) +} + +func (sbc *subBalancerWrapper) stopBalancer() { + if sbc.balancer == nil { + return + } + sbc.balancer.Close() + sbc.balancer = nil +} + +// BalancerGroup takes a list of balancers, each behind a gracefulswitch +// balancer, and make them into one balancer. +// +// Note that this struct doesn't implement balancer.Balancer, because it's not +// intended to be used directly as a balancer. It's expected to be used as a +// sub-balancer manager by a high level balancer. +// +// Updates from ClientConn are forwarded to sub-balancers +// - service config update +// - address update +// - subConn state change +// - find the corresponding balancer and forward +// +// Actions from sub-balances are forwarded to parent ClientConn +// - new/remove SubConn +// - picker update and health states change +// - sub-pickers are sent to an aggregator provided by the parent, which +// will group them into a group-picker. The aggregated connectivity state is +// also handled by the aggregator. +// - resolveNow +// +// Sub-balancers are only built when the balancer group is started. If the +// balancer group is closed, the sub-balancers are also closed. And it's +// guaranteed that no updates will be sent to parent ClientConn from a closed +// balancer group. +type BalancerGroup struct { + cc balancer.ClientConn + buildOpts balancer.BuildOptions + logger *grpclog.PrefixLogger + + // stateAggregator is where the state/picker updates will be sent to. It's + // provided by the parent balancer, to build a picker with all the + // sub-pickers. + stateAggregator BalancerStateAggregator + + // outgoingMu guards all operations in the direction: + // ClientConn-->Sub-balancer. Including start, stop, resolver updates and + // SubConn state changes. + // + // The corresponding boolean outgoingStarted is used to stop further updates + // to sub-balancers after they are closed. + outgoingMu sync.Mutex + outgoingClosed bool + idToBalancerConfig map[string]*subBalancerWrapper + // Cache for sub-balancers when they are removed. This is `nil` if caching + // is disabled by passing `0` for Options.SubBalancerCloseTimeout`. + deletedBalancerCache *cache.TimeoutCache + + // incomingMu is to make sure this balancer group doesn't send updates to cc + // after it's closed. + // + // We don't share the mutex to avoid deadlocks (e.g. a call to sub-balancer + // may call back to balancer group inline. It causes deadlock if they + // require the same mutex). + // + // We should never need to hold multiple locks at the same time in this + // struct. The case where two locks are held can only happen when the + // underlying balancer calls back into balancer group inline. So there's an + // implicit lock acquisition order that outgoingMu is locked before + // incomingMu. + + // incomingMu guards all operations in the direction: + // Sub-balancer-->ClientConn. Including NewSubConn, RemoveSubConn. It also + // guards the map from SubConn to balancer ID, so updateSubConnState needs + // to hold it shortly to potentially delete from the map. + // + // UpdateState is called by the balancer state aggregator, and it will + // decide when and whether to call. + // + // The corresponding boolean incomingStarted is used to stop further updates + // from sub-balancers after they are closed. + incomingMu sync.Mutex + incomingClosed bool // This boolean only guards calls back to ClientConn. + scToSubBalancer map[balancer.SubConn]*subBalancerWrapper +} + +// Options wraps the arguments to be passed to the BalancerGroup ctor. +type Options struct { + // CC is a reference to the parent balancer.ClientConn. + CC balancer.ClientConn + // BuildOpts contains build options to be used when creating sub-balancers. + BuildOpts balancer.BuildOptions + // StateAggregator is an implementation of the BalancerStateAggregator + // interface to aggregate picker and connectivity states from sub-balancers. + StateAggregator BalancerStateAggregator + // Logger is a group specific prefix logger. + Logger *grpclog.PrefixLogger + // SubBalancerCloseTimeout is the amount of time deleted sub-balancers spend + // in the idle cache. A value of zero here disables caching of deleted + // sub-balancers. + SubBalancerCloseTimeout time.Duration +} + +// New creates a new BalancerGroup. Note that the BalancerGroup +// needs to be started to work. +func New(opts Options) *BalancerGroup { + var bc *cache.TimeoutCache + if opts.SubBalancerCloseTimeout != time.Duration(0) { + bc = cache.NewTimeoutCache(opts.SubBalancerCloseTimeout) + } + + return &BalancerGroup{ + cc: opts.CC, + buildOpts: opts.BuildOpts, + stateAggregator: opts.StateAggregator, + logger: opts.Logger, + + deletedBalancerCache: bc, + idToBalancerConfig: make(map[string]*subBalancerWrapper), + scToSubBalancer: make(map[balancer.SubConn]*subBalancerWrapper), + } +} + +// AddWithClientConn adds a balancer with the given id to the group. The +// balancer is built with a balancer builder registered with balancerName. The +// given ClientConn is passed to the newly built balancer instead of the +// one passed to balancergroup.New(). +// +// TODO: Get rid of the existing Add() API and replace it with this. +func (bg *BalancerGroup) AddWithClientConn(id, balancerName string, cc balancer.ClientConn) error { + bg.logger.Infof("Adding child policy of type %q for child %q", balancerName, id) + builder := balancer.Get(balancerName) + if builder == nil { + return fmt.Errorf("balancergroup: unregistered balancer name %q", balancerName) + } + + // Store data in static map, and then check to see if bg is started. + bg.outgoingMu.Lock() + defer bg.outgoingMu.Unlock() + if bg.outgoingClosed { + return fmt.Errorf("balancergroup: already closed") + } + var sbc *subBalancerWrapper + // Skip searching the cache if disabled. + if bg.deletedBalancerCache != nil { + if old, ok := bg.deletedBalancerCache.Remove(id); ok { + if bg.logger.V(2) { + bg.logger.Infof("Removing and reusing child policy of type %q for child %q from the balancer cache", balancerName, id) + bg.logger.Infof("Number of items remaining in the balancer cache: %d", bg.deletedBalancerCache.Len()) + } + + sbc, _ = old.(*subBalancerWrapper) + if sbc != nil && sbc.builder != builder { + // If the sub-balancer in cache was built with a different + // balancer builder, don't use it, cleanup this old-balancer, + // and behave as sub-balancer is not found in cache. + // + // NOTE that this will also drop the cached addresses for this + // sub-balancer, which seems to be reasonable. + sbc.stopBalancer() + // cleanupSubConns must be done before the new balancer starts, + // otherwise new SubConns created by the new balancer might be + // removed by mistake. + bg.cleanupSubConns(sbc) + sbc = nil + } + } + } + if sbc == nil { + sbc = &subBalancerWrapper{ + ClientConn: cc, + id: id, + group: bg, + builder: builder, + buildOpts: bg.buildOpts, + } + sbc.startBalancer() + } else { + // When brining back a sub-balancer from cache, re-send the cached + // picker and state. + sbc.updateBalancerStateWithCachedPicker() + } + bg.idToBalancerConfig[id] = sbc + return nil +} + +// Add adds a balancer built by builder to the group, with given id. +func (bg *BalancerGroup) Add(id string, builder balancer.Builder) { + bg.AddWithClientConn(id, builder.Name(), bg.cc) +} + +// Remove removes the balancer with id from the group. +// +// But doesn't close the balancer. The balancer is kept in a cache, and will be +// closed after timeout. Cleanup work (closing sub-balancer and removing +// subconns) will be done after timeout. +func (bg *BalancerGroup) Remove(id string) { + bg.removeInternal(id, true) +} + +// RemoveImmediately removes and closes the balancer with id from the group +// immediately. +func (bg *BalancerGroup) RemoveImmediately(id string) { + bg.removeInternal(id, false) +} + +func (bg *BalancerGroup) removeInternal(id string, withCaching bool) { + bg.logger.Infof("Removing child policy for child %q", id) + + bg.outgoingMu.Lock() + if bg.outgoingClosed { + bg.outgoingMu.Unlock() + return + } + + sbToRemove, ok := bg.idToBalancerConfig[id] + if !ok { + bg.logger.Errorf("Child policy for child %q does not exist in the balancer group", id) + bg.outgoingMu.Unlock() + return + } + + // Unconditionally remove the sub-balancer config from the map. + delete(bg.idToBalancerConfig, id) + + if withCaching { + if bg.deletedBalancerCache != nil { + if bg.logger.V(2) { + bg.logger.Infof("Adding child policy for child %q to the balancer cache", id) + } + bg.deletedBalancerCache.Add(id, sbToRemove, func() { + if bg.logger.V(2) { + bg.logger.Infof("Removing child policy for child %q from the balancer cache after timeout", id) + bg.logger.Infof("Number of items remaining in the balancer cache: %d", bg.deletedBalancerCache.Len()) + } + + // A sub-balancer evicted from the timeout cache needs to closed + // and its subConns need to removed, unconditionally. There is a + // possibility that a sub-balancer might be removed (thereby + // moving it to the cache) around the same time that the + // balancergroup is closed, and by the time we get here the + // balancergroup might be closed. Check for `outgoingStarted == + // true` at that point can lead to a leaked sub-balancer. + bg.outgoingMu.Lock() + sbToRemove.stopBalancer() + bg.outgoingMu.Unlock() + bg.cleanupSubConns(sbToRemove) + }) + if bg.logger.V(2) { + bg.logger.Infof("Number of items remaining in the balancer cache: %d", bg.deletedBalancerCache.Len()) + } + bg.outgoingMu.Unlock() + return + } + + // Fall through to remove the sub-balancer with immediate effect if we are not caching. + if bg.logger.V(2) { + bg.logger.Infof("Child policy for child %q was requested to be cached before eventual removal. No such cache exists. Removing right away.", id) + } + } + + // Remove the sub-balancer with immediate effect if we are not caching. + sbToRemove.stopBalancer() + bg.outgoingMu.Unlock() + bg.cleanupSubConns(sbToRemove) +} + +// bg.remove(id) doesn't do cleanup for the sub-balancer. This function does +// cleanup after the timeout. +func (bg *BalancerGroup) cleanupSubConns(config *subBalancerWrapper) { + bg.incomingMu.Lock() + defer bg.incomingMu.Unlock() + // Remove SubConns. This is only done after the balancer is + // actually closed. + // + // NOTE: if NewSubConn is called by this (closed) balancer later, the + // SubConn will be leaked. This shouldn't happen if the balancer + // implementation is correct. To make sure this never happens, we need to + // add another layer (balancer manager) between balancer group and the + // sub-balancers. + for sc, b := range bg.scToSubBalancer { + if b == config { + delete(bg.scToSubBalancer, sc) + } + } +} + +// Following are actions from the parent grpc.ClientConn, forward to sub-balancers. + +// updateSubConnState forwards the update to cb and updates scToSubBalancer if +// needed. +func (bg *BalancerGroup) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState, cb func(balancer.SubConnState)) { + bg.incomingMu.Lock() + if bg.incomingClosed { + bg.incomingMu.Unlock() + return + } + if _, ok := bg.scToSubBalancer[sc]; !ok { + bg.incomingMu.Unlock() + return + } + if state.ConnectivityState == connectivity.Shutdown { + // Only delete sc from the map when state changed to Shutdown. + delete(bg.scToSubBalancer, sc) + } + bg.incomingMu.Unlock() + + bg.outgoingMu.Lock() + defer bg.outgoingMu.Unlock() + if bg.outgoingClosed { + return + } + if cb != nil { + cb(state) + } +} + +// UpdateSubConnState handles the state for the subconn. It finds the +// corresponding balancer and forwards the update. +func (bg *BalancerGroup) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + bg.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +// UpdateClientConnState handles ClientState (including balancer config and +// addresses) from resolver. It finds the balancer and forwards the update. +func (bg *BalancerGroup) UpdateClientConnState(id string, s balancer.ClientConnState) error { + bg.outgoingMu.Lock() + defer bg.outgoingMu.Unlock() + if bg.outgoingClosed { + return nil + } + if config, ok := bg.idToBalancerConfig[id]; ok { + return config.updateClientConnState(s) + } + return nil +} + +// ResolverError forwards resolver errors to all sub-balancers. +func (bg *BalancerGroup) ResolverError(err error) { + bg.outgoingMu.Lock() + defer bg.outgoingMu.Unlock() + if bg.outgoingClosed { + return + } + for _, config := range bg.idToBalancerConfig { + config.resolverError(err) + } +} + +// Following are actions from sub-balancers, forward to ClientConn. + +// newSubConn: forward to ClientConn, and also create a map from sc to balancer, +// so state update will find the right balancer. +// +// One note about removing SubConn: only forward to ClientConn, but not delete +// from map. Delete sc from the map only when state changes to Shutdown. Since +// it's just forwarding the action, there's no need for a removeSubConn() +// wrapper function. +func (bg *BalancerGroup) newSubConn(sbw *subBalancerWrapper, addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + // NOTE: if balancer with id was already removed, this should also return + // error. But since we call balancer.stopBalancer when removing the balancer, this + // shouldn't happen. + bg.incomingMu.Lock() + if bg.incomingClosed { + bg.incomingMu.Unlock() + return nil, fmt.Errorf("balancergroup: NewSubConn is called after balancer group is closed") + } + var sc balancer.SubConn + oldListener := opts.StateListener + opts.StateListener = func(state balancer.SubConnState) { bg.updateSubConnState(sc, state, oldListener) } + sc, err := sbw.ClientConn.NewSubConn(addrs, opts) + if err != nil { + bg.incomingMu.Unlock() + return nil, err + } + bg.scToSubBalancer[sc] = sbw + bg.incomingMu.Unlock() + return sc, nil +} + +// updateBalancerState: forward the new state to balancer state aggregator. The +// aggregator will create an aggregated picker and an aggregated connectivity +// state, then forward to ClientConn. +func (bg *BalancerGroup) updateBalancerState(id string, state balancer.State) { + bg.logger.Infof("Balancer state update from child %v, new state: %+v", id, state) + + // Send new state to the aggregator, without holding the incomingMu. + // incomingMu is to protect all calls to the parent ClientConn, this update + // doesn't necessary trigger a call to ClientConn, and should already be + // protected by aggregator's mutex if necessary. + if bg.stateAggregator != nil { + bg.stateAggregator.UpdateState(id, state) + } +} + +// Close closes the balancer. It stops sub-balancers, and removes the subconns. +// When a BalancerGroup is closed, it can not receive further address updates. +func (bg *BalancerGroup) Close() { + bg.incomingMu.Lock() + bg.incomingClosed = true + // Also remove all SubConns. + for sc := range bg.scToSubBalancer { + sc.Shutdown() + delete(bg.scToSubBalancer, sc) + } + bg.incomingMu.Unlock() + + bg.outgoingMu.Lock() + // Setting `outgoingClosed` ensures that no entries are added to + // `deletedBalancerCache` after this point. + bg.outgoingClosed = true + bg.outgoingMu.Unlock() + + // Clear(true) runs clear function to close sub-balancers in cache. It + // must be called out of outgoing mutex. + if bg.deletedBalancerCache != nil { + bg.deletedBalancerCache.Clear(true) + } + + bg.outgoingMu.Lock() + for id, config := range bg.idToBalancerConfig { + config.stopBalancer() + delete(bg.idToBalancerConfig, id) + } + bg.outgoingMu.Unlock() +} + +// ExitIdle should be invoked when the parent LB policy's ExitIdle is invoked. +// It will trigger this on all sub-balancers, or reconnect their subconns if +// not supported. +func (bg *BalancerGroup) ExitIdle() { + bg.outgoingMu.Lock() + defer bg.outgoingMu.Unlock() + if bg.outgoingClosed { + return + } + for _, config := range bg.idToBalancerConfig { + config.exitIdle() + } +} + +// ExitIdleOne instructs the sub-balancer `id` to exit IDLE state, if +// appropriate and possible. +func (bg *BalancerGroup) ExitIdleOne(id string) { + bg.outgoingMu.Lock() + defer bg.outgoingMu.Unlock() + if bg.outgoingClosed { + return + } + if config := bg.idToBalancerConfig[id]; config != nil { + config.exitIdle() + } +} + +// ParseConfig parses a child config list and returns a LB config for the +// gracefulswitch Balancer. +// +// cfg is expected to be a json.RawMessage containing a JSON array of LB policy +// names + configs as the format of the "loadBalancingConfig" field in +// ServiceConfig. It returns a type that should be passed to +// UpdateClientConnState in the BalancerConfig field. +func ParseConfig(cfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return gracefulswitch.ParseConfig(cfg) +} diff --git a/internal/balancergroup/balancergroup_test.go b/internal/balancergroup/balancergroup_test.go new file mode 100644 index 0000000..da65828 --- /dev/null +++ b/internal/balancergroup/balancergroup_test.go @@ -0,0 +1,930 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package balancergroup_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/balancer/weightedtarget/weightedaggregator" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/balancergroup" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 100 * time.Millisecond +) + +var ( + rrBuilder = balancer.Get(roundrobin.Name) + testBalancerIDs = []string{"b1", "b2", "b3"} + testBackendAddrs []resolver.Address + testBackendEndpoints []resolver.Endpoint +) + +const testBackendAddrsCount = 12 + +func init() { + for i := 0; i < testBackendAddrsCount; i++ { + addr := resolver.Address{Addr: fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)} + testBackendAddrs = append(testBackendAddrs, addr) + testBackendEndpoints = append(testBackendEndpoints, resolver.Endpoint{Addresses: []resolver.Address{addr}}) + } +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// Create a new balancer group, add balancer and backends. +// - b1, weight 2, backends [0,1] +// - b2, weight 1, backends [2,3] +// Start the balancer group and check behavior. +// +// Close the balancer group. +func (s) TestBalancerGroup_start_close(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR) + gator.Start() + bg := balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: balancer.BuildOptions{}, + StateAggregator: gator, + Logger: nil, + SubBalancerCloseTimeout: time.Duration(0), + }) + + // Add two balancers to group and send two resolved addresses to both + // balancers. + gator.Add(testBalancerIDs[0], 2) + bg.Add(testBalancerIDs[0], rrBuilder) + bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Endpoints: testBackendEndpoints[0:2]}}) + gator.Add(testBalancerIDs[1], 1) + bg.Add(testBalancerIDs[1], rrBuilder) + bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Endpoints: testBackendEndpoints[2:4]}}) + + m1 := make(map[string]balancer.SubConn) + for i := 0; i < 4; i++ { + addrs := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + m1[addrs[0].Addr] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + // Test roundrobin on the last picker. + p1 := <-cc.NewPickerCh + want := []balancer.SubConn{ + m1[testBackendAddrs[0].Addr], m1[testBackendAddrs[0].Addr], + m1[testBackendAddrs[1].Addr], m1[testBackendAddrs[1].Addr], + m1[testBackendAddrs[2].Addr], m1[testBackendAddrs[3].Addr], + } + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p1)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + gator.Stop() + bg.Close() + for i := 0; i < 4; i++ { + (<-cc.ShutdownSubConnCh).UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) + } +} + +// Test that balancer group start() doesn't deadlock if the balancer calls back +// into balancer group inline when it gets an update. +// +// The potential deadlock can happen if we +// - hold a lock and send updates to balancer (e.g. update resolved addresses) +// - the balancer calls back (NewSubConn or update picker) in line +// +// The callback will try to hold the same lock again, which will cause a +// deadlock. +// +// This test starts the balancer group with a test balancer, will update picker +// whenever it gets an address update. It's expected that start() doesn't block +// because of deadlock. +func (s) TestBalancerGroup_start_close_deadlock(t *testing.T) { + const balancerName = "stub-TestBalancerGroup_start_close_deadlock" + stub.Register(balancerName, stub.BalancerFuncs{}) + builder := balancer.Get(balancerName) + + cc := testutils.NewBalancerClientConn(t) + gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR) + gator.Start() + bg := balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: balancer.BuildOptions{}, + StateAggregator: gator, + Logger: nil, + SubBalancerCloseTimeout: time.Duration(0), + }) + + gator.Add(testBalancerIDs[0], 2) + bg.Add(testBalancerIDs[0], builder) + bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) + gator.Add(testBalancerIDs[1], 1) + bg.Add(testBalancerIDs[1], builder) + bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}}) +} + +// initBalancerGroupForCachingTest creates a balancer group, and initialize it +// to be ready for caching tests. +// +// Two rr balancers are added to bg, each with 2 ready subConns. A sub-balancer +// is removed later, so the balancer group returned has one sub-balancer in its +// own map, and one sub-balancer in cache. +func initBalancerGroupForCachingTest(t *testing.T, idleCacheTimeout time.Duration) (*weightedaggregator.Aggregator, *balancergroup.BalancerGroup, *testutils.BalancerClientConn, map[string]*testutils.TestSubConn) { + cc := testutils.NewBalancerClientConn(t) + gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR) + gator.Start() + bg := balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: balancer.BuildOptions{}, + StateAggregator: gator, + Logger: nil, + SubBalancerCloseTimeout: idleCacheTimeout, + }) + + // Add two balancers to group and send two resolved addresses to both + // balancers. + gator.Add(testBalancerIDs[0], 2) + bg.Add(testBalancerIDs[0], rrBuilder) + bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Endpoints: testBackendEndpoints[0:2]}}) + gator.Add(testBalancerIDs[1], 1) + bg.Add(testBalancerIDs[1], rrBuilder) + bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Endpoints: testBackendEndpoints[2:4]}}) + + m1 := make(map[string]*testutils.TestSubConn) + for i := 0; i < 4; i++ { + addrs := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + m1[addrs[0].Addr] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + // Test roundrobin on the last picker. + p1 := <-cc.NewPickerCh + want := []balancer.SubConn{ + m1[testBackendAddrs[0].Addr], m1[testBackendAddrs[0].Addr], + m1[testBackendAddrs[1].Addr], m1[testBackendAddrs[1].Addr], + m1[testBackendAddrs[2].Addr], m1[testBackendAddrs[3].Addr], + } + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p1)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + gator.Remove(testBalancerIDs[1]) + bg.Remove(testBalancerIDs[1]) + // Don't wait for SubConns to be removed after close, because they are only + // removed after close timeout. + for i := 0; i < 10; i++ { + select { + case sc := <-cc.ShutdownSubConnCh: + t.Fatalf("Got request to shut down subconn %v, want no shut down subconn (because subconns were still in cache)", sc) + default: + } + time.Sleep(time.Millisecond) + } + // Test roundrobin on the with only sub-balancer0. + p2 := <-cc.NewPickerCh + want = []balancer.SubConn{ + m1[testBackendAddrs[0].Addr], m1[testBackendAddrs[1].Addr], + } + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p2)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + return gator, bg, cc, m1 +} + +// Test that if a sub-balancer is removed, and re-added within close timeout, +// the subConns won't be re-created. +func (s) TestBalancerGroup_locality_caching(t *testing.T) { + gator, bg, cc, addrToSC := initBalancerGroupForCachingTest(t, defaultTestTimeout) + + // Turn down subconn for addr2, shouldn't get picker update because + // sub-balancer1 was removed. + addrToSC[testBackendAddrs[2].Addr].UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: errors.New("test error"), + }) + for i := 0; i < 10; i++ { + select { + case <-cc.NewPickerCh: + t.Fatalf("Got new picker, want no new picker (because the sub-balancer was removed)") + default: + } + time.Sleep(defaultTestShortTimeout) + } + + // Re-add sub-balancer-1, because subconns were in cache, no new subconns + // should be created. But a new picker will still be generated, with subconn + // states update to date. + gator.Add(testBalancerIDs[1], 1) + bg.Add(testBalancerIDs[1], rrBuilder) + + p3 := <-cc.NewPickerCh + want := []balancer.SubConn{ + addrToSC[testBackendAddrs[0].Addr], addrToSC[testBackendAddrs[0].Addr], + addrToSC[testBackendAddrs[1].Addr], addrToSC[testBackendAddrs[1].Addr], + // addr2 is down, b2 only has addr3 in READY state. + addrToSC[testBackendAddrs[3].Addr], addrToSC[testBackendAddrs[3].Addr], + } + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p3)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + for i := 0; i < 10; i++ { + select { + case <-cc.NewSubConnAddrsCh: + t.Fatalf("Got new subconn, want no new subconn (because subconns were still in cache)") + default: + } + time.Sleep(defaultTestShortTimeout) + } +} + +// Sub-balancers are put in cache when they are shut down. If balancer group is +// closed within close timeout, all subconns should still be removed +// immediately. +func (s) TestBalancerGroup_locality_caching_close_group(t *testing.T) { + _, bg, cc, addrToSC := initBalancerGroupForCachingTest(t, defaultTestTimeout) + + bg.Close() + // The balancer group is closed. The subconns should be shutdown immediately. + shutdownTimeout := time.After(time.Millisecond * 500) + scToShutdown := map[balancer.SubConn]int{ + addrToSC[testBackendAddrs[0].Addr]: 1, + addrToSC[testBackendAddrs[1].Addr]: 1, + addrToSC[testBackendAddrs[2].Addr]: 1, + addrToSC[testBackendAddrs[3].Addr]: 1, + } + for i := 0; i < len(scToShutdown); i++ { + select { + case sc := <-cc.ShutdownSubConnCh: + c := scToShutdown[sc] + if c == 0 { + t.Fatalf("Got Shutdown for %v when there's %d shutdown expected", sc, c) + } + scToShutdown[sc] = c - 1 + case <-shutdownTimeout: + t.Fatalf("timeout waiting for subConns (from balancer in cache) to be shut down") + } + } +} + +// Sub-balancers in cache will be closed if not re-added within timeout, and +// subConns will be shut down. +func (s) TestBalancerGroup_locality_caching_not_read_within_timeout(t *testing.T) { + _, _, cc, addrToSC := initBalancerGroupForCachingTest(t, time.Second) + + // The sub-balancer is not re-added within timeout. The subconns should be + // shut down. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + scToShutdown := map[balancer.SubConn]int{ + addrToSC[testBackendAddrs[2].Addr]: 1, + addrToSC[testBackendAddrs[3].Addr]: 1, + } + for i := 0; i < len(scToShutdown); i++ { + select { + case sc := <-cc.ShutdownSubConnCh: + c := scToShutdown[sc] + if c == 0 { + t.Fatalf("Got Shutdown for %v when there's %d shutdown expected", sc, c) + } + scToShutdown[sc] = c - 1 + case <-ctx.Done(): + t.Fatalf("timeout waiting for subConns (from balancer in cache) to be shut down") + } + } +} + +// Wrap the rr builder, so it behaves the same, but has a different name. +type noopBalancerBuilderWrapper struct { + balancer.Builder +} + +func init() { + balancer.Register(&noopBalancerBuilderWrapper{Builder: rrBuilder}) +} + +func (*noopBalancerBuilderWrapper) Name() string { + return "noopBalancerBuilderWrapper" +} + +// After removing a sub-balancer, re-add with same ID, but different balancer +// builder. Old subconns should be shut down, and new subconns should be created. +func (s) TestBalancerGroup_locality_caching_read_with_different_builder(t *testing.T) { + gator, bg, cc, addrToSC := initBalancerGroupForCachingTest(t, defaultTestTimeout) + + // Re-add sub-balancer-1, but with a different balancer builder. The + // sub-balancer was still in cache, but can't be reused. This should cause + // old sub-balancer's subconns to be shut down immediately, and new + // subconns to be created. + gator.Add(testBalancerIDs[1], 1) + bg.Add(testBalancerIDs[1], &noopBalancerBuilderWrapper{rrBuilder}) + + // The cached sub-balancer should be closed, and the subconns should be + // shut down immediately. + shutdownTimeout := time.After(time.Millisecond * 500) + scToShutdown := map[balancer.SubConn]int{ + addrToSC[testBackendAddrs[2].Addr]: 1, + addrToSC[testBackendAddrs[3].Addr]: 1, + } + for i := 0; i < len(scToShutdown); i++ { + select { + case sc := <-cc.ShutdownSubConnCh: + c := scToShutdown[sc] + if c == 0 { + t.Fatalf("Got Shutdown for %v when there's %d shutdown expected", sc, c) + } + scToShutdown[sc] = c - 1 + case <-shutdownTimeout: + t.Fatalf("timeout waiting for subConns (from balancer in cache) to be shut down") + } + } + + bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Endpoints: testBackendEndpoints[4:6]}}) + + newSCTimeout := time.After(time.Millisecond * 500) + scToAdd := map[string]int{ + testBackendAddrs[4].Addr: 1, + testBackendAddrs[5].Addr: 1, + } + for i := 0; i < len(scToAdd); i++ { + select { + case addr := <-cc.NewSubConnAddrsCh: + c := scToAdd[addr[0].Addr] + if c == 0 { + t.Fatalf("Got newSubConn for %v when there's %d new expected", addr, c) + } + scToAdd[addr[0].Addr] = c - 1 + sc := <-cc.NewSubConnCh + addrToSC[addr[0].Addr] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + case <-newSCTimeout: + t.Fatalf("timeout waiting for subConns (from new sub-balancer) to be newed") + } + } + + // Test roundrobin on the new picker. + p3 := <-cc.NewPickerCh + want := []balancer.SubConn{ + addrToSC[testBackendAddrs[0].Addr], addrToSC[testBackendAddrs[0].Addr], + addrToSC[testBackendAddrs[1].Addr], addrToSC[testBackendAddrs[1].Addr], + addrToSC[testBackendAddrs[4].Addr], addrToSC[testBackendAddrs[5].Addr], + } + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p3)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } +} + +// After removing a sub-balancer, it will be kept in cache. Make sure that this +// sub-balancer's Close is called when the balancer group is closed. +func (s) TestBalancerGroup_CloseStopsBalancerInCache(t *testing.T) { + const balancerName = "stub-TestBalancerGroup_check_close" + closed := make(chan struct{}) + stub.Register(balancerName, stub.BalancerFuncs{Close: func(_ *stub.BalancerData) { + close(closed) + }}) + builder := balancer.Get(balancerName) + + gator, bg, _, _ := initBalancerGroupForCachingTest(t, time.Second) + + // Add balancer, and remove + gator.Add(testBalancerIDs[2], 1) + bg.Add(testBalancerIDs[2], builder) + gator.Remove(testBalancerIDs[2]) + bg.Remove(testBalancerIDs[2]) + + // Immediately close balancergroup, before the cache timeout. + bg.Close() + + // Make sure the removed child balancer is closed eventually. + select { + case <-closed: + case <-time.After(time.Second * 2): + t.Fatalf("timeout waiting for the child balancer in cache to be closed") + } +} + +// TestBalancerGroupBuildOptions verifies that the balancer.BuildOptions passed +// to the balancergroup at creation time is passed to child policies. +func (s) TestBalancerGroupBuildOptions(t *testing.T) { + const ( + balancerName = "stubBalancer-TestBalancerGroupBuildOptions" + userAgent = "ua" + ) + + // Setup the stub balancer such that we can read the build options passed to + // it in the UpdateClientConnState method. + bOpts := balancer.BuildOptions{ + DialCreds: insecure.NewCredentials(), + ChannelzParent: channelz.RegisterChannel(nil, "test channel"), + CustomUserAgent: userAgent, + } + stub.Register(balancerName, stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + if bd.BuildOptions.DialCreds != bOpts.DialCreds || bd.BuildOptions.ChannelzParent != bOpts.ChannelzParent || bd.BuildOptions.CustomUserAgent != bOpts.CustomUserAgent { + return fmt.Errorf("buildOptions in child balancer: %v, want %v", bd, bOpts) + } + return nil + }, + }) + cc := testutils.NewBalancerClientConn(t) + bg := balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: bOpts, + StateAggregator: nil, + Logger: nil, + }) + + // Add the stub balancer build above as a child policy. + balancerBuilder := balancer.Get(balancerName) + bg.Add(testBalancerIDs[0], balancerBuilder) + + // Send an empty clientConn state change. This should trigger the + // verification of the buildOptions being passed to the child policy. + if err := bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{}); err != nil { + t.Fatal(err) + } +} + +func (s) TestBalancerGroup_UpdateClientConnState_AfterClose(t *testing.T) { + balancerName := t.Name() + clientConnStateCh := make(chan struct{}, 1) + + stub.Register(balancerName, stub.BalancerFuncs{ + UpdateClientConnState: func(_ *stub.BalancerData, _ balancer.ClientConnState) error { + clientConnStateCh <- struct{}{} + return nil + }, + }) + + bg := balancergroup.New(balancergroup.Options{ + CC: testutils.NewBalancerClientConn(t), + BuildOpts: balancer.BuildOptions{}, + StateAggregator: nil, + Logger: nil, + }) + + bg.Add(testBalancerIDs[0], balancer.Get(balancerName)) + bg.Close() + + if err := bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{}); err != nil { + t.Fatalf("Expected nil error, got %v", err) + } + + select { + case <-clientConnStateCh: + t.Fatalf("UpdateClientConnState was called after BalancerGroup was closed") + case <-time.After(defaultTestShortTimeout): + } +} + +func (s) TestBalancerGroup_ResolverError_AfterClose(t *testing.T) { + balancerName := t.Name() + resolveErrorCh := make(chan struct{}, 1) + + stub.Register(balancerName, stub.BalancerFuncs{ + ResolverError: func(_ *stub.BalancerData, _ error) { + resolveErrorCh <- struct{}{} + }, + }) + + bg := balancergroup.New(balancergroup.Options{ + CC: testutils.NewBalancerClientConn(t), + BuildOpts: balancer.BuildOptions{}, + StateAggregator: nil, + Logger: nil, + }) + + bg.Add(testBalancerIDs[0], balancer.Get(balancerName)) + bg.Close() + + bg.ResolverError(errors.New("test error")) + + select { + case <-resolveErrorCh: + t.Fatalf("ResolverError was called on sub-balancer after BalancerGroup was closed") + case <-time.After(defaultTestShortTimeout): + } +} + +func (s) TestBalancerExitIdleOne(t *testing.T) { + const balancerName = "stub-balancer-test-balancergroup-exit-idle-one" + exitIdleCh := make(chan struct{}, 1) + stub.Register(balancerName, stub.BalancerFuncs{ + ExitIdle: func(*stub.BalancerData) { + exitIdleCh <- struct{}{} + }, + }) + cc := testutils.NewBalancerClientConn(t) + bg := balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: balancer.BuildOptions{}, + StateAggregator: nil, + Logger: nil, + }) + defer bg.Close() + + // Add the stub balancer build above as a child policy. + builder := balancer.Get(balancerName) + bg.Add(testBalancerIDs[0], builder) + + // Call ExitIdleOne on the child policy. + bg.ExitIdleOne(testBalancerIDs[0]) + select { + case <-time.After(time.Second): + t.Fatal("Timeout when waiting for ExitIdle to be invoked on child policy") + case <-exitIdleCh: + } +} + +func (s) TestBalancerGroup_ExitIdleOne_AfterClose(t *testing.T) { + balancerName := t.Name() + exitIdleCh := make(chan struct{}) + + stub.Register(balancerName, stub.BalancerFuncs{ + ExitIdle: func(_ *stub.BalancerData) { + close(exitIdleCh) + }, + }) + + bg := balancergroup.New(balancergroup.Options{ + CC: testutils.NewBalancerClientConn(t), + BuildOpts: balancer.BuildOptions{}, + StateAggregator: nil, + Logger: nil, + }) + + bg.Add(testBalancerIDs[0], balancer.Get(balancerName)) + bg.Close() + bg.ExitIdleOne(testBalancerIDs[0]) + + select { + case <-time.After(defaultTestShortTimeout): + case <-exitIdleCh: + t.Fatalf("ExitIdleOne called ExitIdle on sub-balancer after BalancerGroup was closed") + } +} + +func (s) TestBalancerGroup_ExitIdleOne_NonExistentID(t *testing.T) { + balancerName := t.Name() + exitIdleCh := make(chan struct{}, 1) + + stub.Register(balancerName, stub.BalancerFuncs{ + ExitIdle: func(_ *stub.BalancerData) { + exitIdleCh <- struct{}{} + }, + }) + + bg := balancergroup.New(balancergroup.Options{ + CC: testutils.NewBalancerClientConn(t), + BuildOpts: balancer.BuildOptions{}, + StateAggregator: nil, + Logger: nil, + }) + defer bg.Close() + + bg.Add(testBalancerIDs[0], balancer.Get(balancerName)) + bg.ExitIdleOne("non-existent-id") + + select { + case <-time.After(defaultTestShortTimeout): + case <-exitIdleCh: + t.Fatalf("ExitIdleOne called ExitIdle on wrong sub-balancer ID") + } +} + +// TestBalancerGracefulSwitch tests the graceful switch functionality for a +// child of the balancer group. At first, the child is configured as a round +// robin load balancer, and thus should behave accordingly. The test then +// gracefully switches this child to a custom type which only creates a SubConn +// for the second passed in address and also only picks that created SubConn. +// The new aggregated picker should reflect this change for the child. +func (s) TestBalancerGracefulSwitch(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR) + gator.Start() + bg := balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: balancer.BuildOptions{}, + StateAggregator: gator, + Logger: nil, + }) + gator.Add(testBalancerIDs[0], 1) + bg.Add(testBalancerIDs[0], rrBuilder) + bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Endpoints: testBackendEndpoints[0:2]}}) + + defer bg.Close() + + m1 := make(map[string]balancer.SubConn) + scs := make(map[balancer.SubConn]bool) + for i := 0; i < 2; i++ { + addrs := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + m1[addrs[0].Addr] = sc + scs[sc] = true + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + p1 := <-cc.NewPickerCh + want := []balancer.SubConn{ + m1[testBackendAddrs[0].Addr], m1[testBackendAddrs[1].Addr], + } + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p1)); err != nil { + t.Fatal(err) + } + + // The balancer type for testBalancersIDs[0] is currently Round Robin. Now, + // change it to a balancer that has separate behavior logically (creating + // SubConn for second address in address list and always picking that + // SubConn), and see if the downstream behavior reflects that change. + childPolicyName := t.Name() + stub.Register(childPolicyName, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = balancer.Get(pickfirst.Name).Build(bd.ClientConn, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + ccs.ResolverState.Endpoints = ccs.ResolverState.Endpoints[1:] + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + }) + cfgJSON := json.RawMessage(fmt.Sprintf(`[{%q: {}}]`, t.Name())) + lbCfg, err := balancergroup.ParseConfig(cfgJSON) + if err != nil { + t.Fatalf("ParseConfig(%s) failed: %v", string(cfgJSON), err) + } + if err := bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: testBackendEndpoints[2:4]}, + BalancerConfig: lbCfg, + }); err != nil { + t.Fatalf("error updating ClientConn state: %v", err) + } + + addrs := <-cc.NewSubConnAddrsCh + if addrs[0].Addr != testBackendAddrs[3].Addr { + // Verifies forwarded to new created balancer, as the wrapped pick first + // balancer will delete first address. + t.Fatalf("newSubConn called with wrong address, want: %v, got : %v", testBackendAddrs[3].Addr, addrs[0].Addr) + } + sc := <-cc.NewSubConnCh + + // Update the pick first balancers SubConn as CONNECTING. This will cause + // the pick first balancer to UpdateState() with CONNECTING, which shouldn't send + // a Picker update back, as the Graceful Switch process is not complete. + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + select { + case <-cc.NewPickerCh: + t.Fatalf("No new picker should have been sent due to the Graceful Switch process not completing") + case <-ctx.Done(): + } + + // Update the pick first balancers SubConn as READY. This will cause + // the pick first balancer to UpdateState() with READY, which should send a + // Picker update back, as the Graceful Switch process is complete. This + // Picker should always pick the pick first's created SubConn which + // corresponds to address 3. + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p2 := <-cc.NewPickerCh + pr, err := p2.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("error picking: %v", err) + } + if pr.SubConn != sc { + t.Fatalf("picker.Pick(), want %v, got %v", sc, pr.SubConn) + } + + // The Graceful Switch process completing for the child should cause the + // SubConns for the balancer being gracefully switched from to get deleted. + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for i := 0; i < 2; i++ { + select { + case <-ctx.Done(): + t.Fatalf("error waiting for Shutdown()") + case sc := <-cc.ShutdownSubConnCh: + // The SubConn shut down should have been one of the two created + // SubConns, and both should be deleted. + if ok := scs[sc]; ok { + delete(scs, sc) + continue + } else { + t.Fatalf("Shutdown called for wrong SubConn %v, want in %v", sc, scs) + } + } + } +} + +func (s) TestBalancerExitIdle_All(t *testing.T) { + balancer1 := t.Name() + "-1" + balancer2 := t.Name() + "-2" + + testID1, testID2 := testBalancerIDs[0], testBalancerIDs[1] + + exitIdleCh1, exitIdleCh2 := make(chan struct{}, 1), make(chan struct{}, 1) + + stub.Register(balancer1, stub.BalancerFuncs{ + ExitIdle: func(_ *stub.BalancerData) { + exitIdleCh1 <- struct{}{} + }, + }) + + stub.Register(balancer2, stub.BalancerFuncs{ + ExitIdle: func(_ *stub.BalancerData) { + exitIdleCh2 <- struct{}{} + }, + }) + + cc := testutils.NewBalancerClientConn(t) + bg := balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: balancer.BuildOptions{}, + StateAggregator: nil, + Logger: nil, + }) + defer bg.Close() + + bg.Add(testID1, balancer.Get(balancer1)) + bg.Add(testID2, balancer.Get(balancer2)) + + bg.ExitIdle() + + errCh := make(chan error, 2) + + go func() { + select { + case <-exitIdleCh1: + errCh <- nil + case <-time.After(defaultTestTimeout): + errCh <- fmt.Errorf("timeout waiting for ExitIdle on balancer1") + } + }() + + go func() { + select { + case <-exitIdleCh2: + errCh <- nil + case <-time.After(defaultTestTimeout): + errCh <- fmt.Errorf("timeout waiting for ExitIdle on balancer2") + } + }() + + for i := 0; i < 2; i++ { + if err := <-errCh; err != nil { + t.Fatal(err) + } + } +} + +func (s) TestBalancerGroup_ExitIdle_AfterClose(t *testing.T) { + balancerName := t.Name() + exitIdleCh := make(chan struct{}, 1) + + stub.Register(balancerName, stub.BalancerFuncs{ + ExitIdle: func(_ *stub.BalancerData) { + exitIdleCh <- struct{}{} + }, + }) + + bg := balancergroup.New(balancergroup.Options{ + CC: testutils.NewBalancerClientConn(t), + BuildOpts: balancer.BuildOptions{}, + StateAggregator: nil, + Logger: nil, + }) + + bg.Add(testBalancerIDs[0], balancer.Get(balancerName)) + bg.Close() + bg.ExitIdle() + + select { + case <-exitIdleCh: + t.Fatalf("ExitIdle was called on sub-balancer even after BalancerGroup was closed") + case <-time.After(defaultTestShortTimeout): + } +} + +// Tests verifies that the RemoveImmediately method of balancergroup closes the +// child balancer immediately, while the Remove method does not close the child +// balancer immediately, when the cache is enabled. +func (s) TestBalancerGroup_RemoveImmediately(t *testing.T) { + // Channels to track child balancer creation and closure. + childLBCreated := make(chan string, 1) + childLBClosed := make(chan string, 1) + + childLBName1 := strings.ToLower(t.Name()) + "-child-1" + t.Logf("Registering a child balancer with name %q", childLBName1) + stub.Register(childLBName1, stub.BalancerFuncs{ + Init: func(*stub.BalancerData) { + childLBCreated <- childLBName1 + }, + Close: func(*stub.BalancerData) { + childLBClosed <- childLBName1 + }, + }) + + childLBName2 := strings.ToLower(t.Name()) + "-child-2" + t.Logf("Registering a child balancer with name %q", childLBName2) + stub.Register(childLBName2, stub.BalancerFuncs{ + Init: func(*stub.BalancerData) { + childLBCreated <- childLBName2 + }, + Close: func(*stub.BalancerData) { + childLBClosed <- childLBName2 + }, + }) + + t.Logf("Creating a balancergroup with cache enabled") + tcc := testutils.NewBalancerClientConn(t) + bg := balancergroup.New(balancergroup.Options{ + CC: tcc, + BuildOpts: balancer.BuildOptions{}, + SubBalancerCloseTimeout: defaultTestTimeout, + }) + + t.Logf("Adding a child balancer with name %q to the group", "child-1") + bg.AddWithClientConn("child-1", childLBName1, tcc) + select { + case <-childLBCreated: + case <-time.After(defaultTestTimeout): + t.Fatalf("Timeout when waiting for child LB to be created") + } + + t.Logf("Adding a child balancer with name %q to the group", "child-2") + bg.AddWithClientConn("child-2", childLBName2, tcc) + select { + case <-childLBCreated: + case <-time.After(defaultTestTimeout): + t.Fatalf("Timeout when waiting for child LB to be created") + } + + t.Logf("Removing the child balancer with name %q from the group with immediate effect", "child-1") + bg.RemoveImmediately("child-1") + select { + case <-childLBClosed: + case <-time.After(defaultTestTimeout): + t.Fatalf("Timeout when waiting for child LB to be closed") + } + + t.Logf("Removing the child balancer with name %q from the group with caching", "child-2") + bg.Remove("child-2") + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + select { + case <-childLBClosed: + t.Fatalf("Child LB closed when expected to be cached") + case <-sCtx.Done(): + } + + t.Logf("Closing the balancergroup, which should close the second child balancer") + bg.Close() + select { + case <-childLBClosed: + case <-time.After(defaultTestTimeout): + t.Fatalf("Timeout when waiting for child LB to be closed") + } +} diff --git a/internal/balancergroup/balancerstateaggregator.go b/internal/balancergroup/balancerstateaggregator.go new file mode 100644 index 0000000..8168695 --- /dev/null +++ b/internal/balancergroup/balancerstateaggregator.go @@ -0,0 +1,37 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package balancergroup + +import ( + "google.golang.org/grpc/balancer" +) + +// BalancerStateAggregator aggregates sub-picker and connectivity states into a +// state. +// +// It takes care of merging sub-picker into one picker. The picking config is +// passed directly from the parent to the aggregator implementation (instead +// via balancer group). +type BalancerStateAggregator interface { + // UpdateState updates the state of the id. + // + // It's up to the implementation whether this will trigger an update to the + // parent ClientConn. + UpdateState(id string, state balancer.State) +} diff --git a/internal/balancerload/load.go b/internal/balancerload/load.go new file mode 100644 index 0000000..94a08d6 --- /dev/null +++ b/internal/balancerload/load.go @@ -0,0 +1,46 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package balancerload defines APIs to parse server loads in trailers. The +// parsed loads are sent to balancers in DoneInfo. +package balancerload + +import ( + "google.golang.org/grpc/metadata" +) + +// Parser converts loads from metadata into a concrete type. +type Parser interface { + // Parse parses loads from metadata. + Parse(md metadata.MD) any +} + +var parser Parser + +// SetParser sets the load parser. +// +// Not mutex-protected, should be called before any gRPC functions. +func SetParser(lr Parser) { + parser = lr +} + +// Parse calls parser.Read(). +func Parse(md metadata.MD) any { + if parser == nil { + return nil + } + return parser.Parse(md) +} diff --git a/internal/binarylog/binarylog.go b/internal/binarylog/binarylog.go new file mode 100644 index 0000000..755fdeb --- /dev/null +++ b/internal/binarylog/binarylog.go @@ -0,0 +1,192 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package binarylog implementation binary logging as defined in +// https://github.com/grpc/proposal/blob/master/A16-binary-logging.md. +package binarylog + +import ( + "fmt" + "os" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/grpcutil" +) + +var grpclogLogger = grpclog.Component("binarylog") + +// Logger specifies MethodLoggers for method names with a Log call that +// takes a context. +// +// This is used in the 1.0 release of gcp/observability, and thus must not be +// deleted or changed. +type Logger interface { + GetMethodLogger(methodName string) MethodLogger +} + +// binLogger is the global binary logger for the binary. One of this should be +// built at init time from the configuration (environment variable or flags). +// +// It is used to get a MethodLogger for each individual method. +var binLogger Logger + +// SetLogger sets the binary logger. +// +// Only call this at init time. +func SetLogger(l Logger) { + binLogger = l +} + +// GetLogger gets the binary logger. +// +// Only call this at init time. +func GetLogger() Logger { + return binLogger +} + +// GetMethodLogger returns the MethodLogger for the given methodName. +// +// methodName should be in the format of "/service/method". +// +// Each MethodLogger returned by this method is a new instance. This is to +// generate sequence id within the call. +func GetMethodLogger(methodName string) MethodLogger { + if binLogger == nil { + return nil + } + return binLogger.GetMethodLogger(methodName) +} + +func init() { + const envStr = "GRPC_BINARY_LOG_FILTER" + configStr := os.Getenv(envStr) + binLogger = NewLoggerFromConfigString(configStr) +} + +// MethodLoggerConfig contains the setting for logging behavior of a method +// logger. Currently, it contains the max length of header and message. +type MethodLoggerConfig struct { + // Max length of header and message. + Header, Message uint64 +} + +// LoggerConfig contains the config for loggers to create method loggers. +type LoggerConfig struct { + All *MethodLoggerConfig + Services map[string]*MethodLoggerConfig + Methods map[string]*MethodLoggerConfig + + Blacklist map[string]struct{} +} + +type logger struct { + config LoggerConfig +} + +// NewLoggerFromConfig builds a logger with the given LoggerConfig. +func NewLoggerFromConfig(config LoggerConfig) Logger { + return &logger{config: config} +} + +// newEmptyLogger creates an empty logger. The map fields need to be filled in +// using the set* functions. +func newEmptyLogger() *logger { + return &logger{} +} + +// Set method logger for "*". +func (l *logger) setDefaultMethodLogger(ml *MethodLoggerConfig) error { + if l.config.All != nil { + return fmt.Errorf("conflicting global rules found") + } + l.config.All = ml + return nil +} + +// Set method logger for "service/*". +// +// New MethodLogger with same service overrides the old one. +func (l *logger) setServiceMethodLogger(service string, ml *MethodLoggerConfig) error { + if _, ok := l.config.Services[service]; ok { + return fmt.Errorf("conflicting service rules for service %v found", service) + } + if l.config.Services == nil { + l.config.Services = make(map[string]*MethodLoggerConfig) + } + l.config.Services[service] = ml + return nil +} + +// Set method logger for "service/method". +// +// New MethodLogger with same method overrides the old one. +func (l *logger) setMethodMethodLogger(method string, ml *MethodLoggerConfig) error { + if _, ok := l.config.Blacklist[method]; ok { + return fmt.Errorf("conflicting blacklist rules for method %v found", method) + } + if _, ok := l.config.Methods[method]; ok { + return fmt.Errorf("conflicting method rules for method %v found", method) + } + if l.config.Methods == nil { + l.config.Methods = make(map[string]*MethodLoggerConfig) + } + l.config.Methods[method] = ml + return nil +} + +// Set blacklist method for "-service/method". +func (l *logger) setBlacklist(method string) error { + if _, ok := l.config.Blacklist[method]; ok { + return fmt.Errorf("conflicting blacklist rules for method %v found", method) + } + if _, ok := l.config.Methods[method]; ok { + return fmt.Errorf("conflicting method rules for method %v found", method) + } + if l.config.Blacklist == nil { + l.config.Blacklist = make(map[string]struct{}) + } + l.config.Blacklist[method] = struct{}{} + return nil +} + +// getMethodLogger returns the MethodLogger for the given methodName. +// +// methodName should be in the format of "/service/method". +// +// Each MethodLogger returned by this method is a new instance. This is to +// generate sequence id within the call. +func (l *logger) GetMethodLogger(methodName string) MethodLogger { + s, m, err := grpcutil.ParseMethod(methodName) + if err != nil { + grpclogLogger.Infof("binarylogging: failed to parse %q: %v", methodName, err) + return nil + } + if ml, ok := l.config.Methods[s+"/"+m]; ok { + return NewTruncatingMethodLogger(ml.Header, ml.Message) + } + if _, ok := l.config.Blacklist[s+"/"+m]; ok { + return nil + } + if ml, ok := l.config.Services[s]; ok { + return NewTruncatingMethodLogger(ml.Header, ml.Message) + } + if l.config.All == nil { + return nil + } + return NewTruncatingMethodLogger(l.config.All.Header, l.config.All.Message) +} diff --git a/internal/binarylog/binarylog_test.go b/internal/binarylog/binarylog_test.go new file mode 100644 index 0000000..47f6a54 --- /dev/null +++ b/internal/binarylog/binarylog_test.go @@ -0,0 +1,156 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog + +import ( + "testing" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// Test that get method logger returns the one with the most exact match. +func (s) TestGetMethodLogger(t *testing.T) { + testCases := []struct { + in string + method string + hdr, msg uint64 + }{ + // Global. + { + in: "*{h:12;m:23}", + method: "/s/m", + hdr: 12, msg: 23, + }, + // service/*. + { + in: "*,s/*{h:12;m:23}", + method: "/s/m", + hdr: 12, msg: 23, + }, + // Service/method. + { + in: "*{h;m},s/m{h:12;m:23}", + method: "/s/m", + hdr: 12, msg: 23, + }, + { + in: "*{h;m},s/*{h:314;m},s/m{h:12;m:23}", + method: "/s/m", + hdr: 12, msg: 23, + }, + { + in: "*{h;m},s/*{h:12;m:23},s/m", + method: "/s/m", + hdr: maxUInt, msg: maxUInt, + }, + + // service/*. + { + in: "*{h;m},s/*{h:12;m:23},s/m1", + method: "/s/m", + hdr: 12, msg: 23, + }, + { + in: "*{h;m},s1/*,s/m{h:12;m:23}", + method: "/s/m", + hdr: 12, msg: 23, + }, + + // With black list. + { + in: "*{h:12;m:23},-s/m1", + method: "/s/m", + hdr: 12, msg: 23, + }, + } + for _, tc := range testCases { + l := NewLoggerFromConfigString(tc.in) + if l == nil { + t.Errorf("in: %q, failed to create logger from config string", tc.in) + continue + } + ml := l.GetMethodLogger(tc.method).(*TruncatingMethodLogger) + if ml == nil { + t.Errorf("in: %q, method logger is nil, want non-nil", tc.in) + continue + } + if ml.headerMaxLen != tc.hdr || ml.messageMaxLen != tc.msg { + t.Errorf("in: %q, want header: %v, message: %v, got header: %v, message: %v", tc.in, tc.hdr, tc.msg, ml.headerMaxLen, ml.messageMaxLen) + } + } +} + +// expect method logger to be nil +func (s) TestGetMethodLoggerOff(t *testing.T) { + testCases := []struct { + in string + method string + }{ + // method not specified. + { + in: "s1/m", + method: "/s/m", + }, + { + in: "s/m1", + method: "/s/m", + }, + { + in: "s1/*", + method: "/s/m", + }, + { + in: "s1/*,s/m1", + method: "/s/m", + }, + + // blacklisted. + { + in: "*,-s/m", + method: "/s/m", + }, + { + in: "s/*,-s/m", + method: "/s/m", + }, + { + in: "-s/m,s/*", + method: "/s/m", + }, + } + for _, tc := range testCases { + l := NewLoggerFromConfigString(tc.in) + if l == nil { + t.Errorf("in: %q, failed to create logger from config string", tc.in) + continue + } + ml := l.GetMethodLogger(tc.method) + if ml != nil { + t.Errorf("in: %q, method logger is non-nil, want nil", tc.in) + } + } +} diff --git a/internal/binarylog/binarylog_testutil.go b/internal/binarylog/binarylog_testutil.go new file mode 100644 index 0000000..1ee00a3 --- /dev/null +++ b/internal/binarylog/binarylog_testutil.go @@ -0,0 +1,42 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// This file contains exported variables/functions that are exported for testing +// only. +// +// An ideal way for this would be to put those in a *_test.go but in binarylog +// package. But this doesn't work with staticcheck with go module. Error was: +// "MdToMetadataProto not declared by package binarylog". This could be caused +// by the way staticcheck looks for files for a certain package, which doesn't +// support *_test.go files. +// +// Move those to binary_test.go when staticcheck is fixed. + +package binarylog + +var ( + // AllLogger is a logger that logs all headers/messages for all RPCs. It's + // for testing only. + AllLogger = NewLoggerFromConfigString("*") + // MdToMetadataProto converts metadata to a binary logging proto message. + // It's for testing only. + MdToMetadataProto = mdToMetadataProto + // AddrToProto converts an address to a binary logging proto message. It's + // for testing only. + AddrToProto = addrToProto +) diff --git a/internal/binarylog/env_config.go b/internal/binarylog/env_config.go new file mode 100644 index 0000000..f9e80e2 --- /dev/null +++ b/internal/binarylog/env_config.go @@ -0,0 +1,208 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" +) + +// NewLoggerFromConfigString reads the string and build a logger. It can be used +// to build a new logger and assign it to binarylog.Logger. +// +// Example filter config strings: +// - "" Nothing will be logged +// - "*" All headers and messages will be fully logged. +// - "*{h}" Only headers will be logged. +// - "*{m:256}" Only the first 256 bytes of each message will be logged. +// - "Foo/*" Logs every method in service Foo +// - "Foo/*,-Foo/Bar" Logs every method in service Foo except method /Foo/Bar +// - "Foo/*,Foo/Bar{m:256}" Logs the first 256 bytes of each message in method +// /Foo/Bar, logs all headers and messages in every other method in service +// Foo. +// +// If two configs exist for one certain method or service, the one specified +// later overrides the previous config. +func NewLoggerFromConfigString(s string) Logger { + if s == "" { + return nil + } + l := newEmptyLogger() + methods := strings.Split(s, ",") + for _, method := range methods { + if err := l.fillMethodLoggerWithConfigString(method); err != nil { + grpclogLogger.Warningf("failed to parse binary log config: %v", err) + return nil + } + } + return l +} + +// fillMethodLoggerWithConfigString parses config, creates TruncatingMethodLogger and adds +// it to the right map in the logger. +func (l *logger) fillMethodLoggerWithConfigString(config string) error { + // "" is invalid. + if config == "" { + return errors.New("empty string is not a valid method binary logging config") + } + + // "-service/method", blacklist, no * or {} allowed. + if config[0] == '-' { + s, m, suffix, err := parseMethodConfigAndSuffix(config[1:]) + if err != nil { + return fmt.Errorf("invalid config: %q, %v", config, err) + } + if m == "*" { + return fmt.Errorf("invalid config: %q, %v", config, "* not allowed in blacklist config") + } + if suffix != "" { + return fmt.Errorf("invalid config: %q, %v", config, "header/message limit not allowed in blacklist config") + } + if err := l.setBlacklist(s + "/" + m); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + return nil + } + + // "*{h:256;m:256}" + if config[0] == '*' { + hdr, msg, err := parseHeaderMessageLengthConfig(config[1:]) + if err != nil { + return fmt.Errorf("invalid config: %q, %v", config, err) + } + if err := l.setDefaultMethodLogger(&MethodLoggerConfig{Header: hdr, Message: msg}); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + return nil + } + + s, m, suffix, err := parseMethodConfigAndSuffix(config) + if err != nil { + return fmt.Errorf("invalid config: %q, %v", config, err) + } + hdr, msg, err := parseHeaderMessageLengthConfig(suffix) + if err != nil { + return fmt.Errorf("invalid header/message length config: %q, %v", suffix, err) + } + if m == "*" { + if err := l.setServiceMethodLogger(s, &MethodLoggerConfig{Header: hdr, Message: msg}); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + } else { + if err := l.setMethodMethodLogger(s+"/"+m, &MethodLoggerConfig{Header: hdr, Message: msg}); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + } + return nil +} + +const ( + // TODO: this const is only used by env_config now. But could be useful for + // other config. Move to binarylog.go if necessary. + maxUInt = ^uint64(0) + + // For "p.s/m" plus any suffix. Suffix will be parsed again. See test for + // expected output. + longMethodConfigRegexpStr = `^([\w./]+)/((?:\w+)|[*])(.+)?$` + + // For suffix from above, "{h:123,m:123}". See test for expected output. + optionalLengthRegexpStr = `(?::(\d+))?` // Optional ":123". + headerConfigRegexpStr = `^{h` + optionalLengthRegexpStr + `}$` + messageConfigRegexpStr = `^{m` + optionalLengthRegexpStr + `}$` + headerMessageConfigRegexpStr = `^{h` + optionalLengthRegexpStr + `;m` + optionalLengthRegexpStr + `}$` +) + +var ( + longMethodConfigRegexp = regexp.MustCompile(longMethodConfigRegexpStr) + headerConfigRegexp = regexp.MustCompile(headerConfigRegexpStr) + messageConfigRegexp = regexp.MustCompile(messageConfigRegexpStr) + headerMessageConfigRegexp = regexp.MustCompile(headerMessageConfigRegexpStr) +) + +// Turn "service/method{h;m}" into "service", "method", "{h;m}". +func parseMethodConfigAndSuffix(c string) (service, method, suffix string, _ error) { + // Regexp result: + // + // in: "p.s/m{h:123,m:123}", + // out: []string{"p.s/m{h:123,m:123}", "p.s", "m", "{h:123,m:123}"}, + match := longMethodConfigRegexp.FindStringSubmatch(c) + if match == nil { + return "", "", "", fmt.Errorf("%q contains invalid substring", c) + } + service = match[1] + method = match[2] + suffix = match[3] + return +} + +// Turn "{h:123;m:345}" into 123, 345. +// +// Return maxUInt if length is unspecified. +func parseHeaderMessageLengthConfig(c string) (hdrLenStr, msgLenStr uint64, err error) { + if c == "" { + return maxUInt, maxUInt, nil + } + // Header config only. + if match := headerConfigRegexp.FindStringSubmatch(c); match != nil { + if s := match[1]; s != "" { + hdrLenStr, err = strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("failed to convert %q to uint", s) + } + return hdrLenStr, 0, nil + } + return maxUInt, 0, nil + } + + // Message config only. + if match := messageConfigRegexp.FindStringSubmatch(c); match != nil { + if s := match[1]; s != "" { + msgLenStr, err = strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("failed to convert %q to uint", s) + } + return 0, msgLenStr, nil + } + return 0, maxUInt, nil + } + + // Header and message config both. + if match := headerMessageConfigRegexp.FindStringSubmatch(c); match != nil { + // Both hdr and msg are specified, but one or two of them might be empty. + hdrLenStr = maxUInt + msgLenStr = maxUInt + if s := match[1]; s != "" { + hdrLenStr, err = strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("failed to convert %q to uint", s) + } + } + if s := match[2]; s != "" { + msgLenStr, err = strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("failed to convert %q to uint", s) + } + } + return hdrLenStr, msgLenStr, nil + } + return 0, 0, fmt.Errorf("%q contains invalid substring", c) +} diff --git a/internal/binarylog/env_config_test.go b/internal/binarylog/env_config_test.go new file mode 100644 index 0000000..9f888ad --- /dev/null +++ b/internal/binarylog/env_config_test.go @@ -0,0 +1,478 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog + +import ( + "fmt" + "testing" +) + +// This tests that when multiple configs are specified, all methods loggers will +// be set correctly. Correctness of each logger is covered by other unit tests. +func (s) TestNewLoggerFromConfigString(t *testing.T) { + const ( + s1 = "s1" + m1 = "m1" + m2 = "m2" + fullM1 = s1 + "/" + m1 + fullM2 = s1 + "/" + m2 + ) + c := fmt.Sprintf("*{h:1;m:2},%s{h},%s{m},%s{h;m}", s1+"/*", fullM1, fullM2) + l := NewLoggerFromConfigString(c).(*logger) + + if l.config.All.Header != 1 || l.config.All.Message != 2 { + t.Errorf("l.config.All = %#v, want headerLen: 1, messageLen: 2", l.config.All) + } + + if ml, ok := l.config.Services[s1]; ok { + if ml.Header != maxUInt || ml.Message != 0 { + t.Errorf("want maxUInt header, 0 message, got header: %v, message: %v", ml.Header, ml.Message) + } + } else { + t.Errorf("service/* is not set") + } + + if ml, ok := l.config.Methods[fullM1]; ok { + if ml.Header != 0 || ml.Message != maxUInt { + t.Errorf("want 0 header, maxUInt message, got header: %v, message: %v", ml.Header, ml.Message) + } + } else { + t.Errorf("service/method{h} is not set") + } + + if ml, ok := l.config.Methods[fullM2]; ok { + if ml.Header != maxUInt || ml.Message != maxUInt { + t.Errorf("want maxUInt header, maxUInt message, got header: %v, message: %v", ml.Header, ml.Message) + } + } else { + t.Errorf("service/method{h;m} is not set") + } +} + +func (s) TestNewLoggerFromConfigStringInvalid(t *testing.T) { + testCases := []string{ + "", + "*{}", + "s/m,*{}", + "s/m,s/m{a}", + + // Duplicate rules. + "s/m,-s/m", + "-s/m,s/m", + "s/m,s/m", + "s/m,s/m{h:1;m:1}", + "s/m{h:1;m:1},s/m", + "-s/m,-s/m", + "s/*,s/*{h:1;m:1}", + "*,*{h:1;m:1}", + } + for _, tc := range testCases { + l := NewLoggerFromConfigString(tc) + if l != nil { + t.Errorf("With config %q, want logger %v, got %v", tc, nil, l) + } + } +} + +func (s) TestParseMethodConfigAndSuffix(t *testing.T) { + testCases := []struct { + in, service, method, suffix string + }{ + { + in: "p.s/m", + service: "p.s", method: "m", suffix: "", + }, + { + in: "p.s/m{h,m}", + service: "p.s", method: "m", suffix: "{h,m}", + }, + { + in: "p.s/*", + service: "p.s", method: "*", suffix: "", + }, + { + in: "p.s/*{h,m}", + service: "p.s", method: "*", suffix: "{h,m}", + }, + + // invalid suffix will be detected by another function. + { + in: "p.s/m{invalidsuffix}", + service: "p.s", method: "m", suffix: "{invalidsuffix}", + }, + { + in: "p.s/*{invalidsuffix}", + service: "p.s", method: "*", suffix: "{invalidsuffix}", + }, + { + in: "s/m*", + service: "s", method: "m", suffix: "*", + }, + { + in: "s/*m", + service: "s", method: "*", suffix: "m", + }, + { + in: "s/**", + service: "s", method: "*", suffix: "*", + }, + } + for _, tc := range testCases { + t.Logf("testing parseMethodConfigAndSuffix(%q)", tc.in) + s, m, suffix, err := parseMethodConfigAndSuffix(tc.in) + if err != nil { + t.Errorf("returned error %v, want nil", err) + continue + } + if s != tc.service { + t.Errorf("service = %q, want %q", s, tc.service) + } + if m != tc.method { + t.Errorf("method = %q, want %q", m, tc.method) + } + if suffix != tc.suffix { + t.Errorf("suffix = %q, want %q", suffix, tc.suffix) + } + } +} + +func (s) TestParseMethodConfigAndSuffixInvalid(t *testing.T) { + testCases := []string{ + "*/m", + "*/m{}", + } + for _, tc := range testCases { + s, m, suffix, err := parseMethodConfigAndSuffix(tc) + if err == nil { + t.Errorf("Parsing %q got nil error with %q, %q, %q, want non-nil error", tc, s, m, suffix) + } + } +} + +func (s) TestParseHeaderMessageLengthConfig(t *testing.T) { + testCases := []struct { + in string + hdr, msg uint64 + }{ + { + in: "", + hdr: maxUInt, msg: maxUInt, + }, + { + in: "{h}", + hdr: maxUInt, msg: 0, + }, + { + in: "{h:314}", + hdr: 314, msg: 0, + }, + { + in: "{m}", + hdr: 0, msg: maxUInt, + }, + { + in: "{m:213}", + hdr: 0, msg: 213, + }, + { + in: "{h;m}", + hdr: maxUInt, msg: maxUInt, + }, + { + in: "{h:314;m}", + hdr: 314, msg: maxUInt, + }, + { + in: "{h;m:213}", + hdr: maxUInt, msg: 213, + }, + { + in: "{h:314;m:213}", + hdr: 314, msg: 213, + }, + } + for _, tc := range testCases { + t.Logf("testing parseHeaderMessageLengthConfig(%q)", tc.in) + hdr, msg, err := parseHeaderMessageLengthConfig(tc.in) + if err != nil { + t.Errorf("returned error %v, want nil", err) + continue + } + if hdr != tc.hdr { + t.Errorf("header length = %v, want %v", hdr, tc.hdr) + } + if msg != tc.msg { + t.Errorf("message length = %v, want %v", msg, tc.msg) + } + } +} +func (s) TestParseHeaderMessageLengthConfigInvalid(t *testing.T) { + testCases := []string{ + "{}", + "{h;a}", + "{h;m;b}", + } + for _, tc := range testCases { + _, _, err := parseHeaderMessageLengthConfig(tc) + if err == nil { + t.Errorf("Parsing %q got nil error, want non-nil error", tc) + } + } +} + +func (s) TestFillMethodLoggerWithConfigStringBlacklist(t *testing.T) { + testCases := []string{ + "p.s/m", + "service/method", + } + for _, tc := range testCases { + c := "-" + tc + t.Logf("testing fillMethodLoggerWithConfigString(%q)", c) + l := newEmptyLogger() + if err := l.fillMethodLoggerWithConfigString(c); err != nil { + t.Errorf("returned err %v, want nil", err) + continue + } + _, ok := l.config.Blacklist[tc] + if !ok { + t.Errorf("blacklist[%q] is not set", tc) + } + } +} + +func (s) TestFillMethodLoggerWithConfigStringGlobal(t *testing.T) { + testCases := []struct { + in string + hdr, msg uint64 + }{ + { + in: "", + hdr: maxUInt, msg: maxUInt, + }, + { + in: "{h}", + hdr: maxUInt, msg: 0, + }, + { + in: "{h:314}", + hdr: 314, msg: 0, + }, + { + in: "{m}", + hdr: 0, msg: maxUInt, + }, + { + in: "{m:213}", + hdr: 0, msg: 213, + }, + { + in: "{h;m}", + hdr: maxUInt, msg: maxUInt, + }, + { + in: "{h:314;m}", + hdr: 314, msg: maxUInt, + }, + { + in: "{h;m:213}", + hdr: maxUInt, msg: 213, + }, + { + in: "{h:314;m:213}", + hdr: 314, msg: 213, + }, + } + for _, tc := range testCases { + c := "*" + tc.in + t.Logf("testing fillMethodLoggerWithConfigString(%q)", c) + l := newEmptyLogger() + if err := l.fillMethodLoggerWithConfigString(c); err != nil { + t.Errorf("returned err %v, want nil", err) + continue + } + if l.config.All == nil { + t.Errorf("l.config.All is not set") + continue + } + if hdr := l.config.All.Header; hdr != tc.hdr { + t.Errorf("header length = %v, want %v", hdr, tc.hdr) + + } + if msg := l.config.All.Message; msg != tc.msg { + t.Errorf("message length = %v, want %v", msg, tc.msg) + } + } +} + +func (s) TestFillMethodLoggerWithConfigStringPerService(t *testing.T) { + testCases := []struct { + in string + hdr, msg uint64 + }{ + { + in: "", + hdr: maxUInt, msg: maxUInt, + }, + { + in: "{h}", + hdr: maxUInt, msg: 0, + }, + { + in: "{h:314}", + hdr: 314, msg: 0, + }, + { + in: "{m}", + hdr: 0, msg: maxUInt, + }, + { + in: "{m:213}", + hdr: 0, msg: 213, + }, + { + in: "{h;m}", + hdr: maxUInt, msg: maxUInt, + }, + { + in: "{h:314;m}", + hdr: 314, msg: maxUInt, + }, + { + in: "{h;m:213}", + hdr: maxUInt, msg: 213, + }, + { + in: "{h:314;m:213}", + hdr: 314, msg: 213, + }, + } + const serviceName = "service" + for _, tc := range testCases { + c := serviceName + "/*" + tc.in + t.Logf("testing fillMethodLoggerWithConfigString(%q)", c) + l := newEmptyLogger() + if err := l.fillMethodLoggerWithConfigString(c); err != nil { + t.Errorf("returned err %v, want nil", err) + continue + } + ml, ok := l.config.Services[serviceName] + if !ok { + t.Errorf("l.service[%q] is not set", serviceName) + continue + } + if hdr := ml.Header; hdr != tc.hdr { + t.Errorf("header length = %v, want %v", hdr, tc.hdr) + + } + if msg := ml.Message; msg != tc.msg { + t.Errorf("message length = %v, want %v", msg, tc.msg) + } + } +} + +func (s) TestFillMethodLoggerWithConfigStringPerMethod(t *testing.T) { + testCases := []struct { + in string + hdr, msg uint64 + }{ + { + in: "", + hdr: maxUInt, msg: maxUInt, + }, + { + in: "{h}", + hdr: maxUInt, msg: 0, + }, + { + in: "{h:314}", + hdr: 314, msg: 0, + }, + { + in: "{m}", + hdr: 0, msg: maxUInt, + }, + { + in: "{m:213}", + hdr: 0, msg: 213, + }, + { + in: "{h;m}", + hdr: maxUInt, msg: maxUInt, + }, + { + in: "{h:314;m}", + hdr: 314, msg: maxUInt, + }, + { + in: "{h;m:213}", + hdr: maxUInt, msg: 213, + }, + { + in: "{h:314;m:213}", + hdr: 314, msg: 213, + }, + } + const ( + serviceName = "service" + methodName = "method" + fullMethodName = serviceName + "/" + methodName + ) + for _, tc := range testCases { + c := fullMethodName + tc.in + t.Logf("testing fillMethodLoggerWithConfigString(%q)", c) + l := newEmptyLogger() + if err := l.fillMethodLoggerWithConfigString(c); err != nil { + t.Errorf("returned err %v, want nil", err) + continue + } + ml, ok := l.config.Methods[fullMethodName] + if !ok { + t.Errorf("l.config.Methods[%q] is not set", fullMethodName) + continue + } + if hdr := ml.Header; hdr != tc.hdr { + t.Errorf("header length = %v, want %v", hdr, tc.hdr) + + } + if msg := ml.Message; msg != tc.msg { + t.Errorf("message length = %v, want %v", msg, tc.msg) + } + } +} + +func (s) TestFillMethodLoggerWithConfigStringInvalid(t *testing.T) { + testCases := []string{ + "", + "{}", + "p.s/m{}", + "p.s/m{a}", + "p.s/m*", + "p.s/**", + "*/m", + + "-p.s/*", + "-p.s/m{h}", + } + l := &logger{} + for _, tc := range testCases { + if err := l.fillMethodLoggerWithConfigString(tc); err == nil { + t.Errorf("fillMethodLoggerWithConfigString(%q) returned nil error, want non-nil", tc) + } + } +} diff --git a/internal/binarylog/method_logger.go b/internal/binarylog/method_logger.go new file mode 100644 index 0000000..ca0e586 --- /dev/null +++ b/internal/binarylog/method_logger.go @@ -0,0 +1,446 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog + +import ( + "context" + "net" + "strings" + "sync/atomic" + "time" + + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type callIDGenerator struct { + id uint64 +} + +func (g *callIDGenerator) next() uint64 { + id := atomic.AddUint64(&g.id, 1) + return id +} + +// reset is for testing only, and doesn't need to be thread safe. +func (g *callIDGenerator) reset() { + g.id = 0 +} + +var idGen callIDGenerator + +// MethodLogger is the sub-logger for each method. +// +// This is used in the 1.0 release of gcp/observability, and thus must not be +// deleted or changed. +type MethodLogger interface { + Log(context.Context, LogEntryConfig) +} + +// TruncatingMethodLogger is a method logger that truncates headers and messages +// based on configured fields. +type TruncatingMethodLogger struct { + headerMaxLen, messageMaxLen uint64 + + callID uint64 + idWithinCallGen *callIDGenerator + + sink Sink // TODO(blog): make this pluggable. +} + +// NewTruncatingMethodLogger returns a new truncating method logger. +// +// This is used in the 1.0 release of gcp/observability, and thus must not be +// deleted or changed. +func NewTruncatingMethodLogger(h, m uint64) *TruncatingMethodLogger { + return &TruncatingMethodLogger{ + headerMaxLen: h, + messageMaxLen: m, + + callID: idGen.next(), + idWithinCallGen: &callIDGenerator{}, + + sink: DefaultSink, // TODO(blog): make it pluggable. + } +} + +// Build is an internal only method for building the proto message out of the +// input event. It's made public to enable other library to reuse as much logic +// in TruncatingMethodLogger as possible. +func (ml *TruncatingMethodLogger) Build(c LogEntryConfig) *binlogpb.GrpcLogEntry { + m := c.toProto() + timestamp := timestamppb.Now() + m.Timestamp = timestamp + m.CallId = ml.callID + m.SequenceIdWithinCall = ml.idWithinCallGen.next() + + switch pay := m.Payload.(type) { + case *binlogpb.GrpcLogEntry_ClientHeader: + m.PayloadTruncated = ml.truncateMetadata(pay.ClientHeader.GetMetadata()) + case *binlogpb.GrpcLogEntry_ServerHeader: + m.PayloadTruncated = ml.truncateMetadata(pay.ServerHeader.GetMetadata()) + case *binlogpb.GrpcLogEntry_Message: + m.PayloadTruncated = ml.truncateMessage(pay.Message) + } + return m +} + +// Log creates a proto binary log entry, and logs it to the sink. +func (ml *TruncatingMethodLogger) Log(_ context.Context, c LogEntryConfig) { + ml.sink.Write(ml.Build(c)) +} + +func (ml *TruncatingMethodLogger) truncateMetadata(mdPb *binlogpb.Metadata) (truncated bool) { + if ml.headerMaxLen == maxUInt { + return false + } + var ( + bytesLimit = ml.headerMaxLen + index int + ) + // At the end of the loop, index will be the first entry where the total + // size is greater than the limit: + // + // len(entry[:index]) <= ml.hdr && len(entry[:index+1]) > ml.hdr. + for ; index < len(mdPb.Entry); index++ { + entry := mdPb.Entry[index] + if entry.Key == "grpc-trace-bin" { + // "grpc-trace-bin" is a special key. It's kept in the log entry, + // but not counted towards the size limit. + continue + } + currentEntryLen := uint64(len(entry.GetKey())) + uint64(len(entry.GetValue())) + if currentEntryLen > bytesLimit { + break + } + bytesLimit -= currentEntryLen + } + truncated = index < len(mdPb.Entry) + mdPb.Entry = mdPb.Entry[:index] + return truncated +} + +func (ml *TruncatingMethodLogger) truncateMessage(msgPb *binlogpb.Message) (truncated bool) { + if ml.messageMaxLen == maxUInt { + return false + } + if ml.messageMaxLen >= uint64(len(msgPb.Data)) { + return false + } + msgPb.Data = msgPb.Data[:ml.messageMaxLen] + return true +} + +// LogEntryConfig represents the configuration for binary log entry. +// +// This is used in the 1.0 release of gcp/observability, and thus must not be +// deleted or changed. +type LogEntryConfig interface { + toProto() *binlogpb.GrpcLogEntry +} + +// ClientHeader configs the binary log entry to be a ClientHeader entry. +type ClientHeader struct { + OnClientSide bool + Header metadata.MD + MethodName string + Authority string + Timeout time.Duration + // PeerAddr is required only when it's on server side. + PeerAddr net.Addr +} + +func (c *ClientHeader) toProto() *binlogpb.GrpcLogEntry { + // This function doesn't need to set all the fields (e.g. seq ID). The Log + // function will set the fields when necessary. + clientHeader := &binlogpb.ClientHeader{ + Metadata: mdToMetadataProto(c.Header), + MethodName: c.MethodName, + Authority: c.Authority, + } + if c.Timeout > 0 { + clientHeader.Timeout = durationpb.New(c.Timeout) + } + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, + Payload: &binlogpb.GrpcLogEntry_ClientHeader{ + ClientHeader: clientHeader, + }, + } + if c.OnClientSide { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER + } + if c.PeerAddr != nil { + ret.Peer = addrToProto(c.PeerAddr) + } + return ret +} + +// ServerHeader configs the binary log entry to be a ServerHeader entry. +type ServerHeader struct { + OnClientSide bool + Header metadata.MD + // PeerAddr is required only when it's on client side. + PeerAddr net.Addr +} + +func (c *ServerHeader) toProto() *binlogpb.GrpcLogEntry { + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, + Payload: &binlogpb.GrpcLogEntry_ServerHeader{ + ServerHeader: &binlogpb.ServerHeader{ + Metadata: mdToMetadataProto(c.Header), + }, + }, + } + if c.OnClientSide { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER + } + if c.PeerAddr != nil { + ret.Peer = addrToProto(c.PeerAddr) + } + return ret +} + +// ClientMessage configs the binary log entry to be a ClientMessage entry. +type ClientMessage struct { + OnClientSide bool + // Message can be a proto.Message or []byte. Other messages formats are not + // supported. + Message any +} + +func (c *ClientMessage) toProto() *binlogpb.GrpcLogEntry { + var ( + data []byte + err error + ) + if m, ok := c.Message.(proto.Message); ok { + data, err = proto.Marshal(m) + if err != nil { + grpclogLogger.Infof("binarylogging: failed to marshal proto message: %v", err) + } + } else if b, ok := c.Message.([]byte); ok { + data = b + } else { + grpclogLogger.Infof("binarylogging: message to log is neither proto.message nor []byte") + } + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE, + Payload: &binlogpb.GrpcLogEntry_Message{ + Message: &binlogpb.Message{ + Length: uint32(len(data)), + Data: data, + }, + }, + } + if c.OnClientSide { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER + } + return ret +} + +// ServerMessage configs the binary log entry to be a ServerMessage entry. +type ServerMessage struct { + OnClientSide bool + // Message can be a proto.Message or []byte. Other messages formats are not + // supported. + Message any +} + +func (c *ServerMessage) toProto() *binlogpb.GrpcLogEntry { + var ( + data []byte + err error + ) + if m, ok := c.Message.(proto.Message); ok { + data, err = proto.Marshal(m) + if err != nil { + grpclogLogger.Infof("binarylogging: failed to marshal proto message: %v", err) + } + } else if b, ok := c.Message.([]byte); ok { + data = b + } else { + grpclogLogger.Infof("binarylogging: message to log is neither proto.message nor []byte") + } + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE, + Payload: &binlogpb.GrpcLogEntry_Message{ + Message: &binlogpb.Message{ + Length: uint32(len(data)), + Data: data, + }, + }, + } + if c.OnClientSide { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER + } + return ret +} + +// ClientHalfClose configs the binary log entry to be a ClientHalfClose entry. +type ClientHalfClose struct { + OnClientSide bool +} + +func (c *ClientHalfClose) toProto() *binlogpb.GrpcLogEntry { + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE, + Payload: nil, // No payload here. + } + if c.OnClientSide { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER + } + return ret +} + +// ServerTrailer configs the binary log entry to be a ServerTrailer entry. +type ServerTrailer struct { + OnClientSide bool + Trailer metadata.MD + // Err is the status error. + Err error + // PeerAddr is required only when it's on client side and the RPC is trailer + // only. + PeerAddr net.Addr +} + +func (c *ServerTrailer) toProto() *binlogpb.GrpcLogEntry { + st, ok := status.FromError(c.Err) + if !ok { + grpclogLogger.Info("binarylogging: error in trailer is not a status error") + } + var ( + detailsBytes []byte + err error + ) + stProto := st.Proto() + if stProto != nil && len(stProto.Details) != 0 { + detailsBytes, err = proto.Marshal(stProto) + if err != nil { + grpclogLogger.Infof("binarylogging: failed to marshal status proto: %v", err) + } + } + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, + Payload: &binlogpb.GrpcLogEntry_Trailer{ + Trailer: &binlogpb.Trailer{ + Metadata: mdToMetadataProto(c.Trailer), + StatusCode: uint32(st.Code()), + StatusMessage: st.Message(), + StatusDetails: detailsBytes, + }, + }, + } + if c.OnClientSide { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER + } + if c.PeerAddr != nil { + ret.Peer = addrToProto(c.PeerAddr) + } + return ret +} + +// Cancel configs the binary log entry to be a Cancel entry. +type Cancel struct { + OnClientSide bool +} + +func (c *Cancel) toProto() *binlogpb.GrpcLogEntry { + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CANCEL, + Payload: nil, + } + if c.OnClientSide { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER + } + return ret +} + +// metadataKeyOmit returns whether the metadata entry with this key should be +// omitted. +func metadataKeyOmit(key string) bool { + switch key { + case "lb-token", ":path", ":authority", "content-encoding", "content-type", "user-agent", "te", "authorization": + return true + case "grpc-trace-bin": // grpc-trace-bin is special because it's visible to users. + return false + } + return strings.HasPrefix(key, "grpc-") +} + +func mdToMetadataProto(md metadata.MD) *binlogpb.Metadata { + ret := &binlogpb.Metadata{} + for k, vv := range md { + if metadataKeyOmit(k) { + continue + } + for _, v := range vv { + ret.Entry = append(ret.Entry, + &binlogpb.MetadataEntry{ + Key: k, + Value: []byte(v), + }, + ) + } + } + return ret +} + +func addrToProto(addr net.Addr) *binlogpb.Address { + ret := &binlogpb.Address{} + switch a := addr.(type) { + case *net.TCPAddr: + if a.IP.To4() != nil { + ret.Type = binlogpb.Address_TYPE_IPV4 + } else if a.IP.To16() != nil { + ret.Type = binlogpb.Address_TYPE_IPV6 + } else { + ret.Type = binlogpb.Address_TYPE_UNKNOWN + // Do not set address and port fields. + break + } + ret.Address = a.IP.String() + ret.IpPort = uint32(a.Port) + case *net.UnixAddr: + ret.Type = binlogpb.Address_TYPE_UNIX + ret.Address = a.String() + default: + ret.Type = binlogpb.Address_TYPE_UNKNOWN + } + return ret +} diff --git a/internal/binarylog/method_logger_test.go b/internal/binarylog/method_logger_test.go new file mode 100644 index 0000000..60f1198 --- /dev/null +++ b/internal/binarylog/method_logger_test.go @@ -0,0 +1,562 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog + +import ( + "bytes" + "context" + "fmt" + "net" + "testing" + "time" + + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" +) + +const defaultTestTimeout = 10 * time.Second + +func (s) TestLog(t *testing.T) { + idGen.reset() + ml := NewTruncatingMethodLogger(10, 10) + // Set sink to testing buffer. + buf := bytes.NewBuffer(nil) + ml.sink = newWriterSink(buf) + + addr := "1.2.3.4" + port := 790 + tcpAddr, _ := net.ResolveTCPAddr("tcp", fmt.Sprintf("%v:%d", addr, port)) + addr6 := "2001:1db8:85a3::8a2e:1370:7334" + port6 := 796 + tcpAddr6, _ := net.ResolveTCPAddr("tcp", fmt.Sprintf("[%v]:%d", addr6, port6)) + + testProtoMsg := &binlogpb.Message{ + Length: 1, + Data: []byte{'a'}, + } + testProtoBytes, _ := proto.Marshal(testProtoMsg) + + testCases := []struct { + config LogEntryConfig + want *binlogpb.GrpcLogEntry + }{ + { + config: &ClientHeader{ + OnClientSide: false, + Header: map[string][]string{ + "a": {"b", "bb"}, + }, + MethodName: "testservice/testmethod", + Authority: "test.service.io", + Timeout: 2*time.Second + 3*time.Nanosecond, + PeerAddr: tcpAddr, + }, + want: &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: 1, + SequenceIdWithinCall: 0, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, + Logger: binlogpb.GrpcLogEntry_LOGGER_SERVER, + Payload: &binlogpb.GrpcLogEntry_ClientHeader{ + ClientHeader: &binlogpb.ClientHeader{ + Metadata: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "a", Value: []byte{'b'}}, + {Key: "a", Value: []byte{'b', 'b'}}, + }, + }, + MethodName: "testservice/testmethod", + Authority: "test.service.io", + Timeout: &durationpb.Duration{ + Seconds: 2, + Nanos: 3, + }, + }, + }, + PayloadTruncated: false, + Peer: &binlogpb.Address{ + Type: binlogpb.Address_TYPE_IPV4, + Address: addr, + IpPort: uint32(port), + }, + }, + }, + { + config: &ClientHeader{ + OnClientSide: false, + MethodName: "testservice/testmethod", + Authority: "test.service.io", + }, + want: &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: 1, + SequenceIdWithinCall: 0, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, + Logger: binlogpb.GrpcLogEntry_LOGGER_SERVER, + Payload: &binlogpb.GrpcLogEntry_ClientHeader{ + ClientHeader: &binlogpb.ClientHeader{ + Metadata: &binlogpb.Metadata{}, + MethodName: "testservice/testmethod", + Authority: "test.service.io", + }, + }, + PayloadTruncated: false, + }, + }, + { + config: &ServerHeader{ + OnClientSide: true, + Header: map[string][]string{ + "a": {"b", "bb"}, + }, + PeerAddr: tcpAddr6, + }, + want: &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: 1, + SequenceIdWithinCall: 0, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, + Payload: &binlogpb.GrpcLogEntry_ServerHeader{ + ServerHeader: &binlogpb.ServerHeader{ + Metadata: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "a", Value: []byte{'b'}}, + {Key: "a", Value: []byte{'b', 'b'}}, + }, + }, + }, + }, + PayloadTruncated: false, + Peer: &binlogpb.Address{ + Type: binlogpb.Address_TYPE_IPV6, + Address: addr6, + IpPort: uint32(port6), + }, + }, + }, + { + config: &ClientMessage{ + OnClientSide: true, + Message: testProtoMsg, + }, + want: &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: 1, + SequenceIdWithinCall: 0, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, + Payload: &binlogpb.GrpcLogEntry_Message{ + Message: &binlogpb.Message{ + Length: uint32(len(testProtoBytes)), + Data: testProtoBytes, + }, + }, + PayloadTruncated: false, + Peer: nil, + }, + }, + { + config: &ServerMessage{ + OnClientSide: false, + Message: testProtoMsg, + }, + want: &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: 1, + SequenceIdWithinCall: 0, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE, + Logger: binlogpb.GrpcLogEntry_LOGGER_SERVER, + Payload: &binlogpb.GrpcLogEntry_Message{ + Message: &binlogpb.Message{ + Length: uint32(len(testProtoBytes)), + Data: testProtoBytes, + }, + }, + PayloadTruncated: false, + Peer: nil, + }, + }, + { + config: &ClientHalfClose{ + OnClientSide: false, + }, + want: &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: 1, + SequenceIdWithinCall: 0, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE, + Logger: binlogpb.GrpcLogEntry_LOGGER_SERVER, + Payload: nil, + PayloadTruncated: false, + Peer: nil, + }, + }, + { + config: &ServerTrailer{ + OnClientSide: true, + Err: status.Errorf(codes.Unavailable, "test"), + PeerAddr: tcpAddr, + }, + want: &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: 1, + SequenceIdWithinCall: 0, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, + Payload: &binlogpb.GrpcLogEntry_Trailer{ + Trailer: &binlogpb.Trailer{ + Metadata: &binlogpb.Metadata{}, + StatusCode: uint32(codes.Unavailable), + StatusMessage: "test", + StatusDetails: nil, + }, + }, + PayloadTruncated: false, + Peer: &binlogpb.Address{ + Type: binlogpb.Address_TYPE_IPV4, + Address: addr, + IpPort: uint32(port), + }, + }, + }, + { // Err is nil, Log OK status. + config: &ServerTrailer{ + OnClientSide: true, + }, + want: &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: 1, + SequenceIdWithinCall: 0, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, + Payload: &binlogpb.GrpcLogEntry_Trailer{ + Trailer: &binlogpb.Trailer{ + Metadata: &binlogpb.Metadata{}, + StatusCode: uint32(codes.OK), + StatusMessage: "", + StatusDetails: nil, + }, + }, + PayloadTruncated: false, + Peer: nil, + }, + }, + { + config: &Cancel{ + OnClientSide: true, + }, + want: &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: 1, + SequenceIdWithinCall: 0, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CANCEL, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, + Payload: nil, + PayloadTruncated: false, + Peer: nil, + }, + }, + // gRPC headers should be omitted. + { + config: &ClientHeader{ + OnClientSide: false, + Header: map[string][]string{ + "lb-token": {"to be omitted"}, + ":path": {"to be omitted"}, + ":authority": {"to be omitted"}, + "content-encoding": {"to be omitted"}, + "content-type": {"to be omitted"}, + "user-agent": {"to be omitted"}, + "te": {"to be omitted"}, + "authorization": {"Bearer to-be-omitted"}, + "grpc-reserved": {"to be omitted"}, + + "a": {"b", "bb"}, + }, + }, + want: &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: 1, + SequenceIdWithinCall: 0, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, + Logger: binlogpb.GrpcLogEntry_LOGGER_SERVER, + Payload: &binlogpb.GrpcLogEntry_ClientHeader{ + ClientHeader: &binlogpb.ClientHeader{ + Metadata: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "a", Value: []byte{'b'}}, + {Key: "a", Value: []byte{'b', 'b'}}, + }, + }, + }, + }, + PayloadTruncated: false, + }, + }, + { + config: &ServerHeader{ + OnClientSide: true, + Header: map[string][]string{ + "lb-token": {"to be omitted"}, + ":path": {"to be omitted"}, + ":authority": {"to be omitted"}, + "content-encoding": {"to be omitted"}, + "content-type": {"to be omitted"}, + "user-agent": {"to be omitted"}, + "te": {"to be omitted"}, + "authorization": {"Bearer to-be-omitted"}, + "grpc-reserved": {"to be omitted"}, + + "a": {"b", "bb"}, + }, + }, + want: &binlogpb.GrpcLogEntry{ + Timestamp: nil, + CallId: 1, + SequenceIdWithinCall: 0, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, + Payload: &binlogpb.GrpcLogEntry_ServerHeader{ + ServerHeader: &binlogpb.ServerHeader{ + Metadata: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "a", Value: []byte{'b'}}, + {Key: "a", Value: []byte{'b', 'b'}}, + }, + }, + }, + }, + PayloadTruncated: false, + }, + }, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for i, tc := range testCases { + buf.Reset() + tc.want.SequenceIdWithinCall = uint64(i + 1) + ml.Log(ctx, tc.config) + inSink := new(binlogpb.GrpcLogEntry) + if err := proto.Unmarshal(buf.Bytes()[4:], inSink); err != nil { + t.Errorf("failed to unmarshal bytes in sink to proto: %v", err) + continue + } + inSink.Timestamp = nil // Strip timestamp before comparing. + if !proto.Equal(inSink, tc.want) { + t.Errorf("Log(%+v), in sink: %+v, want %+v", tc.config, inSink, tc.want) + } + } +} + +func (s) TestTruncateMetadataNotTruncated(t *testing.T) { + testCases := []struct { + ml *TruncatingMethodLogger + mpPb *binlogpb.Metadata + }{ + { + ml: NewTruncatingMethodLogger(maxUInt, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "", Value: []byte{1}}, + }, + }, + }, + { + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "", Value: []byte{1}}, + }, + }, + }, + { + ml: NewTruncatingMethodLogger(1, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "", Value: nil}, + }, + }, + }, + { + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "", Value: []byte{1, 1}}, + }, + }, + }, + { + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "", Value: []byte{1}}, + {Key: "", Value: []byte{1}}, + }, + }, + }, + // "grpc-trace-bin" is kept in log but not counted towards the size + // limit. + { + ml: NewTruncatingMethodLogger(1, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "", Value: []byte{1}}, + {Key: "grpc-trace-bin", Value: []byte("some.trace.key")}, + }, + }, + }, + } + + for i, tc := range testCases { + truncated := tc.ml.truncateMetadata(tc.mpPb) + if truncated { + t.Errorf("test case %v, returned truncated, want not truncated", i) + } + } +} + +func (s) TestTruncateMetadataTruncated(t *testing.T) { + testCases := []struct { + ml *TruncatingMethodLogger + mpPb *binlogpb.Metadata + + entryLen int + }{ + { + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "", Value: []byte{1, 1, 1}}, + }, + }, + entryLen: 0, + }, + { + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "", Value: []byte{1}}, + {Key: "", Value: []byte{1}}, + {Key: "", Value: []byte{1}}, + }, + }, + entryLen: 2, + }, + { + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "", Value: []byte{1, 1}}, + {Key: "", Value: []byte{1}}, + }, + }, + entryLen: 1, + }, + { + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + {Key: "", Value: []byte{1}}, + {Key: "", Value: []byte{1, 1}}, + }, + }, + entryLen: 1, + }, + } + + for i, tc := range testCases { + truncated := tc.ml.truncateMetadata(tc.mpPb) + if !truncated { + t.Errorf("test case %v, returned not truncated, want truncated", i) + continue + } + if len(tc.mpPb.Entry) != tc.entryLen { + t.Errorf("test case %v, entry length: %v, want: %v", i, len(tc.mpPb.Entry), tc.entryLen) + } + } +} + +func (s) TestTruncateMessageNotTruncated(t *testing.T) { + testCases := []struct { + ml *TruncatingMethodLogger + msgPb *binlogpb.Message + }{ + { + ml: NewTruncatingMethodLogger(maxUInt, maxUInt), + msgPb: &binlogpb.Message{ + Data: []byte{1}, + }, + }, + { + ml: NewTruncatingMethodLogger(maxUInt, 3), + msgPb: &binlogpb.Message{ + Data: []byte{1, 1}, + }, + }, + { + ml: NewTruncatingMethodLogger(maxUInt, 2), + msgPb: &binlogpb.Message{ + Data: []byte{1, 1}, + }, + }, + } + + for i, tc := range testCases { + truncated := tc.ml.truncateMessage(tc.msgPb) + if truncated { + t.Errorf("test case %v, returned truncated, want not truncated", i) + } + } +} + +func (s) TestTruncateMessageTruncated(t *testing.T) { + testCases := []struct { + ml *TruncatingMethodLogger + msgPb *binlogpb.Message + + oldLength uint32 + }{ + { + ml: NewTruncatingMethodLogger(maxUInt, 2), + msgPb: &binlogpb.Message{ + Length: 3, + Data: []byte{1, 1, 1}, + }, + oldLength: 3, + }, + } + + for i, tc := range testCases { + truncated := tc.ml.truncateMessage(tc.msgPb) + if !truncated { + t.Errorf("test case %v, returned not truncated, want truncated", i) + continue + } + if len(tc.msgPb.Data) != int(tc.ml.messageMaxLen) { + t.Errorf("test case %v, message length: %v, want: %v", i, len(tc.msgPb.Data), tc.ml.messageMaxLen) + } + if tc.msgPb.Length != tc.oldLength { + t.Errorf("test case %v, message.Length field: %v, want: %v", i, tc.msgPb.Length, tc.oldLength) + } + } +} diff --git a/internal/binarylog/regexp_test.go b/internal/binarylog/regexp_test.go new file mode 100644 index 0000000..b4b3694 --- /dev/null +++ b/internal/binarylog/regexp_test.go @@ -0,0 +1,179 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog + +import ( + "reflect" + "testing" +) + +func (s) TestLongMethodConfigRegexp(t *testing.T) { + testCases := []struct { + in string + out []string + }{ + {in: "", out: nil}, + {in: "*/m", out: nil}, + + { + in: "p.s/m{}", + out: []string{"p.s/m{}", "p.s", "m", "{}"}, + }, + + { + in: "p.s/m", + out: []string{"p.s/m", "p.s", "m", ""}, + }, + { + in: "p.s/m{h}", + out: []string{"p.s/m{h}", "p.s", "m", "{h}"}, + }, + { + in: "p.s/m{m}", + out: []string{"p.s/m{m}", "p.s", "m", "{m}"}, + }, + { + in: "p.s/m{h:123}", + out: []string{"p.s/m{h:123}", "p.s", "m", "{h:123}"}, + }, + { + in: "p.s/m{m:123}", + out: []string{"p.s/m{m:123}", "p.s", "m", "{m:123}"}, + }, + { + in: "p.s/m{h:123,m:123}", + out: []string{"p.s/m{h:123,m:123}", "p.s", "m", "{h:123,m:123}"}, + }, + + { + in: "p.s/*", + out: []string{"p.s/*", "p.s", "*", ""}, + }, + { + in: "p.s/*{h}", + out: []string{"p.s/*{h}", "p.s", "*", "{h}"}, + }, + + { + in: "s/m*", + out: []string{"s/m*", "s", "m", "*"}, + }, + { + in: "s/**", + out: []string{"s/**", "s", "*", "*"}, + }, + } + for _, tc := range testCases { + match := longMethodConfigRegexp.FindStringSubmatch(tc.in) + if !reflect.DeepEqual(match, tc.out) { + t.Errorf("in: %q, out: %q, want: %q", tc.in, match, tc.out) + } + } +} + +func (s) TestHeaderConfigRegexp(t *testing.T) { + testCases := []struct { + in string + out []string + }{ + {in: "{}", out: nil}, + {in: "{a:b}", out: nil}, + {in: "{m:123}", out: nil}, + {in: "{h:123;m:123}", out: nil}, + + { + in: "{h}", + out: []string{"{h}", ""}, + }, + { + in: "{h:123}", + out: []string{"{h:123}", "123"}, + }, + } + for _, tc := range testCases { + match := headerConfigRegexp.FindStringSubmatch(tc.in) + if !reflect.DeepEqual(match, tc.out) { + t.Errorf("in: %q, out: %q, want: %q", tc.in, match, tc.out) + } + } +} + +func (s) TestMessageConfigRegexp(t *testing.T) { + testCases := []struct { + in string + out []string + }{ + {in: "{}", out: nil}, + {in: "{a:b}", out: nil}, + {in: "{h:123}", out: nil}, + {in: "{h:123;m:123}", out: nil}, + + { + in: "{m}", + out: []string{"{m}", ""}, + }, + { + in: "{m:123}", + out: []string{"{m:123}", "123"}, + }, + } + for _, tc := range testCases { + match := messageConfigRegexp.FindStringSubmatch(tc.in) + if !reflect.DeepEqual(match, tc.out) { + t.Errorf("in: %q, out: %q, want: %q", tc.in, match, tc.out) + } + } +} + +func (s) TestHeaderMessageConfigRegexp(t *testing.T) { + testCases := []struct { + in string + out []string + }{ + {in: "{}", out: nil}, + {in: "{a:b}", out: nil}, + {in: "{h}", out: nil}, + {in: "{h:123}", out: nil}, + {in: "{m}", out: nil}, + {in: "{m:123}", out: nil}, + + { + in: "{h;m}", + out: []string{"{h;m}", "", ""}, + }, + { + in: "{h:123;m}", + out: []string{"{h:123;m}", "123", ""}, + }, + { + in: "{h;m:123}", + out: []string{"{h;m:123}", "", "123"}, + }, + { + in: "{h:123;m:123}", + out: []string{"{h:123;m:123}", "123", "123"}, + }, + } + for _, tc := range testCases { + match := headerMessageConfigRegexp.FindStringSubmatch(tc.in) + if !reflect.DeepEqual(match, tc.out) { + t.Errorf("in: %q, out: %q, want: %q", tc.in, match, tc.out) + } + } +} diff --git a/internal/binarylog/sink.go b/internal/binarylog/sink.go new file mode 100644 index 0000000..9ea598b --- /dev/null +++ b/internal/binarylog/sink.go @@ -0,0 +1,170 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog + +import ( + "bufio" + "encoding/binary" + "io" + "sync" + "time" + + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + "google.golang.org/protobuf/proto" +) + +var ( + // DefaultSink is the sink where the logs will be written to. It's exported + // for the binarylog package to update. + DefaultSink Sink = &noopSink{} // TODO(blog): change this default (file in /tmp). +) + +// Sink writes log entry into the binary log sink. +// +// sink is a copy of the exported binarylog.Sink, to avoid circular dependency. +type Sink interface { + // Write will be called to write the log entry into the sink. + // + // It should be thread-safe so it can be called in parallel. + Write(*binlogpb.GrpcLogEntry) error + // Close will be called when the Sink is replaced by a new Sink. + Close() error +} + +type noopSink struct{} + +func (ns *noopSink) Write(*binlogpb.GrpcLogEntry) error { return nil } +func (ns *noopSink) Close() error { return nil } + +// newWriterSink creates a binary log sink with the given writer. +// +// Write() marshals the proto message and writes it to the given writer. Each +// message is prefixed with a 4 byte big endian unsigned integer as the length. +// +// No buffer is done, Close() doesn't try to close the writer. +func newWriterSink(w io.Writer) Sink { + return &writerSink{out: w} +} + +type writerSink struct { + out io.Writer +} + +func (ws *writerSink) Write(e *binlogpb.GrpcLogEntry) error { + b, err := proto.Marshal(e) + if err != nil { + grpclogLogger.Errorf("binary logging: failed to marshal proto message: %v", err) + return err + } + hdr := make([]byte, 4) + binary.BigEndian.PutUint32(hdr, uint32(len(b))) + if _, err := ws.out.Write(hdr); err != nil { + return err + } + if _, err := ws.out.Write(b); err != nil { + return err + } + return nil +} + +func (ws *writerSink) Close() error { return nil } + +type bufferedSink struct { + mu sync.Mutex + closer io.Closer + out Sink // out is built on buf. + buf *bufio.Writer // buf is kept for flush. + flusherStarted bool + + writeTicker *time.Ticker + done chan struct{} +} + +func (fs *bufferedSink) Write(e *binlogpb.GrpcLogEntry) error { + fs.mu.Lock() + defer fs.mu.Unlock() + if !fs.flusherStarted { + // Start the write loop when Write is called. + fs.startFlushGoroutine() + fs.flusherStarted = true + } + if err := fs.out.Write(e); err != nil { + return err + } + return nil +} + +const ( + bufFlushDuration = 60 * time.Second +) + +func (fs *bufferedSink) startFlushGoroutine() { + fs.writeTicker = time.NewTicker(bufFlushDuration) + go func() { + for { + select { + case <-fs.done: + return + case <-fs.writeTicker.C: + } + fs.mu.Lock() + if err := fs.buf.Flush(); err != nil { + grpclogLogger.Warningf("failed to flush to Sink: %v", err) + } + fs.mu.Unlock() + } + }() +} + +func (fs *bufferedSink) Close() error { + fs.mu.Lock() + defer fs.mu.Unlock() + if fs.writeTicker != nil { + fs.writeTicker.Stop() + } + close(fs.done) + if err := fs.buf.Flush(); err != nil { + grpclogLogger.Warningf("failed to flush to Sink: %v", err) + } + if err := fs.closer.Close(); err != nil { + grpclogLogger.Warningf("failed to close the underlying WriterCloser: %v", err) + } + if err := fs.out.Close(); err != nil { + grpclogLogger.Warningf("failed to close the Sink: %v", err) + } + return nil +} + +// NewBufferedSink creates a binary log sink with the given WriteCloser. +// +// Write() marshals the proto message and writes it to the given writer. Each +// message is prefixed with a 4 byte big endian unsigned integer as the length. +// +// Content is kept in a buffer, and is flushed every 60 seconds. +// +// Close closes the WriteCloser. +func NewBufferedSink(o io.WriteCloser) Sink { + bufW := bufio.NewWriter(o) + return &bufferedSink{ + closer: o, + out: newWriterSink(bufW), + buf: bufW, + done: make(chan struct{}), + } +} diff --git a/internal/buffer/unbounded.go b/internal/buffer/unbounded.go new file mode 100644 index 0000000..d1f4b63 --- /dev/null +++ b/internal/buffer/unbounded.go @@ -0,0 +1,115 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package buffer provides an implementation of an unbounded buffer. +package buffer + +import ( + "errors" + "sync" +) + +// Unbounded is an implementation of an unbounded buffer which does not use +// extra goroutines. This is typically used for passing updates from one entity +// to another within gRPC. +// +// To avoid extra memory allocations and type assertions, using any on +// performance-critical code paths is discouraged. Use concrete types wherever +// possible when instantiating Unbounded. +// +// All methods on this type are thread-safe and don't block on anything except +// the underlying mutex used for synchronization. +type Unbounded[T any] struct { + c chan T + closed bool + closing bool + mu sync.Mutex + backlog []T +} + +// NewUnbounded returns a new instance of Unbounded. +func NewUnbounded[T any]() *Unbounded[T] { + return &Unbounded[T]{c: make(chan T, 1)} +} + +var errBufferClosed = errors.New("Put called on closed buffer.Unbounded") + +// Put adds t to the unbounded buffer. +func (b *Unbounded[T]) Put(t T) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.closing { + return errBufferClosed + } + if len(b.backlog) == 0 { + select { + case b.c <- t: + return nil + default: + } + } + b.backlog = append(b.backlog, t) + return nil +} + +// Load sends the earliest buffered data, if any, onto the read channel returned +// by Get(). Users are expected to call this every time they successfully read a +// value from the read channel. +func (b *Unbounded[T]) Load() { + b.mu.Lock() + defer b.mu.Unlock() + if len(b.backlog) > 0 { + select { + case b.c <- b.backlog[0]: + var zero T + b.backlog[0] = zero + b.backlog = b.backlog[1:] + default: + } + } else if b.closing && !b.closed { + b.closed = true + close(b.c) + } +} + +// Get returns a read channel on which values added to the buffer, via Put(), +// are sent on. +// +// Upon reading a value from this channel, users are expected to call Load() to +// send the next buffered value onto the channel if there is any. +// +// If the unbounded buffer is closed, the read channel returned by this method +// is closed after all data is drained. +func (b *Unbounded[T]) Get() <-chan T { + return b.c +} + +// Close closes the unbounded buffer. No subsequent data may be Put(), and the +// channel returned from Get() will be closed after all the data is read and +// Load() is called for the final time. +func (b *Unbounded[T]) Close() { + b.mu.Lock() + defer b.mu.Unlock() + if b.closing { + return + } + b.closing = true + if len(b.backlog) == 0 { + b.closed = true + close(b.c) + } +} diff --git a/internal/buffer/unbounded_test.go b/internal/buffer/unbounded_test.go new file mode 100644 index 0000000..80d8717 --- /dev/null +++ b/internal/buffer/unbounded_test.go @@ -0,0 +1,148 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package buffer + +import ( + "reflect" + "sort" + "sync" + "testing" + + "google.golang.org/grpc/internal/grpctest" +) + +const ( + numWriters = 10 + numWrites = 10 +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// wantReads contains the set of values expected to be read by the reader +// goroutine in the tests. +var wantReads []int + +func init() { + for i := 0; i < numWriters; i++ { + for j := 0; j < numWrites; j++ { + wantReads = append(wantReads, i) + } + } +} + +// TestSingleWriter starts one reader and one writer goroutine and makes sure +// that the reader gets all the values added to the buffer by the writer. +func (s) TestSingleWriter(t *testing.T) { + ub := NewUnbounded[int]() + reads := []int{} + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + ch := ub.Get() + for i := 0; i < numWriters*numWrites; i++ { + r := <-ch + reads = append(reads, r) + ub.Load() + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < numWriters; i++ { + for j := 0; j < numWrites; j++ { + ub.Put(i) + } + } + }() + + wg.Wait() + if !reflect.DeepEqual(reads, wantReads) { + t.Errorf("reads: %#v, wantReads: %#v", reads, wantReads) + } +} + +// TestMultipleWriters starts multiple writers and one reader goroutine and +// makes sure that the reader gets all the data written by all writers. +func (s) TestMultipleWriters(t *testing.T) { + ub := NewUnbounded[int]() + reads := []int{} + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + ch := ub.Get() + for i := 0; i < numWriters*numWrites; i++ { + r := <-ch + reads = append(reads, r) + ub.Load() + } + }() + + wg.Add(numWriters) + for i := 0; i < numWriters; i++ { + go func(index int) { + defer wg.Done() + for j := 0; j < numWrites; j++ { + ub.Put(index) + } + }(i) + } + + wg.Wait() + sort.Ints(reads) + if !reflect.DeepEqual(reads, wantReads) { + t.Errorf("reads: %#v, wantReads: %#v", reads, wantReads) + } +} + +// TestClose closes the buffer and makes sure that nothing is sent after the +// buffer is closed. +func (s) TestClose(t *testing.T) { + ub := NewUnbounded[int]() + if err := ub.Put(1); err != nil { + t.Fatalf("Unbounded.Put() = %v; want nil", err) + } + ub.Close() + if err := ub.Put(1); err == nil { + t.Fatalf("Unbounded.Put() = ; want non-nil error") + } + if v, ok := <-ub.Get(); !ok { + t.Errorf("Unbounded.Get() = %v, %v, want %v, %v", v, ok, 1, true) + } + if err := ub.Put(1); err == nil { + t.Fatalf("Unbounded.Put() = ; want non-nil error") + } + ub.Load() + if v, ok := <-ub.Get(); ok { + t.Errorf("Unbounded.Get() = %v, want closed channel", v) + } + if err := ub.Put(1); err == nil { + t.Fatalf("Unbounded.Put() = ; want non-nil error") + } + ub.Close() // ignored +} diff --git a/internal/cache/timeoutCache.go b/internal/cache/timeoutCache.go new file mode 100644 index 0000000..2fa4870 --- /dev/null +++ b/internal/cache/timeoutCache.go @@ -0,0 +1,151 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package cache implements caches to be used in gRPC. +package cache + +import ( + "sync" + "time" +) + +type cacheEntry struct { + item any + // Note that to avoid deadlocks (potentially caused by lock ordering), + // callback can only be called without holding cache's mutex. + callback func() + timer *time.Timer + // deleted is set to true in Remove() when the call to timer.Stop() fails. + // This can happen when the timer in the cache entry fires around the same + // time that timer.stop() is called in Remove(). + deleted bool +} + +// TimeoutCache is a cache with items to be deleted after a timeout. +type TimeoutCache struct { + mu sync.Mutex + timeout time.Duration + cache map[any]*cacheEntry +} + +// NewTimeoutCache creates a TimeoutCache with the given timeout. +func NewTimeoutCache(timeout time.Duration) *TimeoutCache { + return &TimeoutCache{ + timeout: timeout, + cache: make(map[any]*cacheEntry), + } +} + +// Add adds an item to the cache, with the specified callback to be called when +// the item is removed from the cache upon timeout. If the item is removed from +// the cache using a call to Remove before the timeout expires, the callback +// will not be called. +// +// If the Add was successful, it returns (newly added item, true). If there is +// an existing entry for the specified key, the cache entry is not be updated +// with the specified item and it returns (existing item, false). +func (c *TimeoutCache) Add(key, item any, callback func()) (any, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if e, ok := c.cache[key]; ok { + return e.item, false + } + + entry := &cacheEntry{ + item: item, + callback: callback, + } + entry.timer = time.AfterFunc(c.timeout, func() { + c.mu.Lock() + if entry.deleted { + c.mu.Unlock() + // Abort the delete since this has been taken care of in Remove(). + return + } + delete(c.cache, key) + c.mu.Unlock() + entry.callback() + }) + c.cache[key] = entry + return item, true +} + +// Remove the item with the key from the cache. +// +// If the specified key exists in the cache, it returns (item associated with +// key, true) and the callback associated with the item is guaranteed to be not +// called. If the given key is not found in the cache, it returns (nil, false) +func (c *TimeoutCache) Remove(key any) (item any, ok bool) { + c.mu.Lock() + defer c.mu.Unlock() + entry, ok := c.removeInternal(key) + if !ok { + return nil, false + } + return entry.item, true +} + +// removeInternal removes and returns the item with key. +// +// caller must hold c.mu. +func (c *TimeoutCache) removeInternal(key any) (*cacheEntry, bool) { + entry, ok := c.cache[key] + if !ok { + return nil, false + } + delete(c.cache, key) + if !entry.timer.Stop() { + // If stop was not successful, the timer has fired (this can only happen + // in a race). But the deleting function is blocked on c.mu because the + // mutex was held by the caller of this function. + // + // Set deleted to true to abort the deleting function. When the lock is + // released, the delete function will acquire the lock, check the value + // of deleted and return. + entry.deleted = true + } + return entry, true +} + +// Clear removes all entries, and runs the callbacks if runCallback is true. +func (c *TimeoutCache) Clear(runCallback bool) { + var entries []*cacheEntry + c.mu.Lock() + for key := range c.cache { + if e, ok := c.removeInternal(key); ok { + entries = append(entries, e) + } + } + c.mu.Unlock() + + if !runCallback { + return + } + + // removeInternal removes entries from cache, and also stops the timer, so + // the callback is guaranteed to be not called. If runCallback is true, + // manual execute all callbacks. + for _, entry := range entries { + entry.callback() + } +} + +// Len returns the number of entries in the cache. +func (c *TimeoutCache) Len() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.cache) +} diff --git a/internal/cache/timeoutCache_test.go b/internal/cache/timeoutCache_test.go new file mode 100644 index 0000000..0a6ab39 --- /dev/null +++ b/internal/cache/timeoutCache_test.go @@ -0,0 +1,254 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cache + +import ( + "strconv" + "sync" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" +) + +const ( + testCacheTimeout = 100 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (c *TimeoutCache) getForTesting(key any) (*cacheEntry, bool) { + c.mu.Lock() + defer c.mu.Unlock() + r, ok := c.cache[key] + return r, ok +} + +// TestCacheExpire attempts to add an entry to the cache and verifies that it +// was added successfully. It then makes sure that on timeout, it's removed and +// the associated callback is called. +func (s) TestCacheExpire(t *testing.T) { + const k, v = 1, "1" + c := NewTimeoutCache(testCacheTimeout) + + callbackChan := make(chan struct{}) + c.Add(k, v, func() { close(callbackChan) }) + + if gotV, ok := c.getForTesting(k); !ok || gotV.item != v { + t.Fatalf("After Add(), before timeout, from cache got: %v, %v, want %v, %v", gotV.item, ok, v, true) + } + if l := c.Len(); l != 1 { + t.Fatalf("%d number of items in the cache, want 1", l) + } + + select { + case <-callbackChan: + case <-time.After(testCacheTimeout * 2): + t.Fatalf("timeout waiting for callback") + } + + if _, ok := c.getForTesting(k); ok { + t.Fatalf("After Add(), after timeout, from cache got: _, %v, want _, %v", ok, false) + } + if l := c.Len(); l != 0 { + t.Fatalf("%d number of items in the cache, want 0", l) + } +} + +// TestCacheRemove attempts to remove an existing entry from the cache and +// verifies that the entry is removed and the associated callback is not +// invoked. +func (s) TestCacheRemove(t *testing.T) { + const k, v = 1, "1" + c := NewTimeoutCache(testCacheTimeout) + + callbackChan := make(chan struct{}) + c.Add(k, v, func() { close(callbackChan) }) + + if got, ok := c.getForTesting(k); !ok || got.item != v { + t.Fatalf("After Add(), before timeout, from cache got: %v, %v, want %v, %v", got.item, ok, v, true) + } + if l := c.Len(); l != 1 { + t.Fatalf("%d number of items in the cache, want 1", l) + } + + time.Sleep(testCacheTimeout / 2) + + gotV, gotOK := c.Remove(k) + if !gotOK || gotV != v { + t.Fatalf("After Add(), before timeout, Remove() got: %v, %v, want %v, %v", gotV, gotOK, v, true) + } + + if _, ok := c.getForTesting(k); ok { + t.Fatalf("After Add(), before timeout, after Remove(), from cache got: _, %v, want _, %v", ok, false) + } + if l := c.Len(); l != 0 { + t.Fatalf("%d number of items in the cache, want 0", l) + } + + select { + case <-callbackChan: + t.Fatalf("unexpected callback after retrieve") + case <-time.After(testCacheTimeout * 2): + } +} + +// TestCacheClearWithoutCallback attempts to clear all entries from the cache +// and verifies that the associated callbacks are not invoked. +func (s) TestCacheClearWithoutCallback(t *testing.T) { + var values []string + const itemCount = 3 + for i := 0; i < itemCount; i++ { + values = append(values, strconv.Itoa(i)) + } + c := NewTimeoutCache(testCacheTimeout) + + done := make(chan struct{}) + defer close(done) + callbackChan := make(chan struct{}, itemCount) + + for i, v := range values { + callbackChanTemp := make(chan struct{}) + c.Add(i, v, func() { close(callbackChanTemp) }) + go func() { + select { + case <-callbackChanTemp: + callbackChan <- struct{}{} + case <-done: + } + }() + } + + for i, v := range values { + if got, ok := c.getForTesting(i); !ok || got.item != v { + t.Fatalf("After Add(), before timeout, from cache got: %v, %v, want %v, %v", got.item, ok, v, true) + } + } + if l := c.Len(); l != itemCount { + t.Fatalf("%d number of items in the cache, want %d", l, itemCount) + } + + time.Sleep(testCacheTimeout / 2) + c.Clear(false) + + for i := range values { + if _, ok := c.getForTesting(i); ok { + t.Fatalf("After Add(), before timeout, after Remove(), from cache got: _, %v, want _, %v", ok, false) + } + } + if l := c.Len(); l != 0 { + t.Fatalf("%d number of items in the cache, want 0", l) + } + + select { + case <-callbackChan: + t.Fatalf("unexpected callback after Clear") + case <-time.After(testCacheTimeout * 2): + } +} + +// TestCacheClearWithCallback attempts to clear all entries from the cache and +// verifies that the associated callbacks are invoked. +func (s) TestCacheClearWithCallback(t *testing.T) { + var values []string + const itemCount = 3 + for i := 0; i < itemCount; i++ { + values = append(values, strconv.Itoa(i)) + } + c := NewTimeoutCache(time.Hour) + + testDone := make(chan struct{}) + defer close(testDone) + + var wg sync.WaitGroup + wg.Add(itemCount) + for i, v := range values { + callbackChanTemp := make(chan struct{}) + c.Add(i, v, func() { close(callbackChanTemp) }) + go func() { + defer wg.Done() + select { + case <-callbackChanTemp: + case <-testDone: + } + }() + } + + allGoroutineDone := make(chan struct{}, itemCount) + go func() { + wg.Wait() + close(allGoroutineDone) + }() + + for i, v := range values { + if got, ok := c.getForTesting(i); !ok || got.item != v { + t.Fatalf("After Add(), before timeout, from cache got: %v, %v, want %v, %v", got.item, ok, v, true) + } + } + if l := c.Len(); l != itemCount { + t.Fatalf("%d number of items in the cache, want %d", l, itemCount) + } + + time.Sleep(testCacheTimeout / 2) + c.Clear(true) + + for i := range values { + if _, ok := c.getForTesting(i); ok { + t.Fatalf("After Add(), before timeout, after Remove(), from cache got: _, %v, want _, %v", ok, false) + } + } + if l := c.Len(); l != 0 { + t.Fatalf("%d number of items in the cache, want 0", l) + } + + select { + case <-allGoroutineDone: + case <-time.After(testCacheTimeout * 2): + t.Fatalf("timeout waiting for all callbacks") + } +} + +// TestCacheRetrieveTimeoutRace simulates the case where an entry's timer fires +// around the same time that Remove() is called for it. It verifies that there +// is no deadlock. +func (s) TestCacheRetrieveTimeoutRace(t *testing.T) { + c := NewTimeoutCache(time.Nanosecond) + + done := make(chan struct{}) + go func() { + for i := 0; i < 1000; i++ { + // Add starts a timer with 1 ns timeout, then remove will race + // with the timer. + c.Add(i, strconv.Itoa(i), func() {}) + c.Remove(i) + } + close(done) + }() + + select { + case <-time.After(time.Second): + t.Fatalf("Test didn't finish within 1 second. Deadlock") + case <-done: + } +} diff --git a/internal/channelz/channel.go b/internal/channelz/channel.go new file mode 100644 index 0000000..3ec6627 --- /dev/null +++ b/internal/channelz/channel.go @@ -0,0 +1,270 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz + +import ( + "fmt" + "sync/atomic" + + "google.golang.org/grpc/connectivity" +) + +// Channel represents a channel within channelz, which includes metrics and +// internal channelz data, such as channelz id, child list, etc. +type Channel struct { + Entity + // ID is the channelz id of this channel. + ID int64 + // RefName is the human readable reference string of this channel. + RefName string + + closeCalled bool + nestedChans map[int64]string + subChans map[int64]string + Parent *Channel + trace *ChannelTrace + // traceRefCount is the number of trace events that reference this channel. + // Non-zero traceRefCount means the trace of this channel cannot be deleted. + traceRefCount int32 + + // ChannelMetrics holds connectivity state, target and call metrics for the + // channel within channelz. + ChannelMetrics ChannelMetrics +} + +// Implemented to make Channel implement the Identifier interface used for +// nesting. +func (c *Channel) channelzIdentifier() {} + +// String returns a string representation of the Channel, including its parent +// entity and ID. +func (c *Channel) String() string { + if c.Parent == nil { + return fmt.Sprintf("Channel #%d", c.ID) + } + return fmt.Sprintf("%s Channel #%d", c.Parent, c.ID) +} + +func (c *Channel) id() int64 { + return c.ID +} + +// SubChans returns a copy of the map of sub-channels associated with the +// Channel. +func (c *Channel) SubChans() map[int64]string { + db.mu.RLock() + defer db.mu.RUnlock() + return copyMap(c.subChans) +} + +// NestedChans returns a copy of the map of nested channels associated with the +// Channel. +func (c *Channel) NestedChans() map[int64]string { + db.mu.RLock() + defer db.mu.RUnlock() + return copyMap(c.nestedChans) +} + +// Trace returns a copy of the Channel's trace data. +func (c *Channel) Trace() *ChannelTrace { + db.mu.RLock() + defer db.mu.RUnlock() + return c.trace.copy() +} + +// ChannelMetrics holds connectivity state, target and call metrics for the +// channel within channelz. +type ChannelMetrics struct { + // The current connectivity state of the channel. + State atomic.Pointer[connectivity.State] + // The target this channel originally tried to connect to. May be absent + Target atomic.Pointer[string] + // The number of calls started on the channel. + CallsStarted atomic.Int64 + // The number of calls that have completed with an OK status. + CallsSucceeded atomic.Int64 + // The number of calls that have a completed with a non-OK status. + CallsFailed atomic.Int64 + // The last time a call was started on the channel. + LastCallStartedTimestamp atomic.Int64 +} + +// CopyFrom copies the metrics in o to c. For testing only. +func (c *ChannelMetrics) CopyFrom(o *ChannelMetrics) { + c.State.Store(o.State.Load()) + c.Target.Store(o.Target.Load()) + c.CallsStarted.Store(o.CallsStarted.Load()) + c.CallsSucceeded.Store(o.CallsSucceeded.Load()) + c.CallsFailed.Store(o.CallsFailed.Load()) + c.LastCallStartedTimestamp.Store(o.LastCallStartedTimestamp.Load()) +} + +// Equal returns true iff the metrics of c are the same as the metrics of o. +// For testing only. +func (c *ChannelMetrics) Equal(o any) bool { + oc, ok := o.(*ChannelMetrics) + if !ok { + return false + } + if (c.State.Load() == nil) != (oc.State.Load() == nil) { + return false + } + if c.State.Load() != nil && *c.State.Load() != *oc.State.Load() { + return false + } + if (c.Target.Load() == nil) != (oc.Target.Load() == nil) { + return false + } + if c.Target.Load() != nil && *c.Target.Load() != *oc.Target.Load() { + return false + } + return c.CallsStarted.Load() == oc.CallsStarted.Load() && + c.CallsFailed.Load() == oc.CallsFailed.Load() && + c.CallsSucceeded.Load() == oc.CallsSucceeded.Load() && + c.LastCallStartedTimestamp.Load() == oc.LastCallStartedTimestamp.Load() +} + +func strFromPointer(s *string) string { + if s == nil { + return "" + } + return *s +} + +// String returns a string representation of the ChannelMetrics, including its +// state, target, and call metrics. +func (c *ChannelMetrics) String() string { + return fmt.Sprintf("State: %v, Target: %s, CallsStarted: %v, CallsSucceeded: %v, CallsFailed: %v, LastCallStartedTimestamp: %v", + c.State.Load(), strFromPointer(c.Target.Load()), c.CallsStarted.Load(), c.CallsSucceeded.Load(), c.CallsFailed.Load(), c.LastCallStartedTimestamp.Load(), + ) +} + +// NewChannelMetricForTesting creates a new instance of ChannelMetrics with +// specified initial values for testing purposes. +func NewChannelMetricForTesting(state connectivity.State, target string, started, succeeded, failed, timestamp int64) *ChannelMetrics { + c := &ChannelMetrics{} + c.State.Store(&state) + c.Target.Store(&target) + c.CallsStarted.Store(started) + c.CallsSucceeded.Store(succeeded) + c.CallsFailed.Store(failed) + c.LastCallStartedTimestamp.Store(timestamp) + return c +} + +func (c *Channel) addChild(id int64, e entry) { + switch v := e.(type) { + case *SubChannel: + c.subChans[id] = v.RefName + case *Channel: + c.nestedChans[id] = v.RefName + default: + logger.Errorf("cannot add a child (id = %d) of type %T to a channel", id, e) + } +} + +func (c *Channel) deleteChild(id int64) { + delete(c.subChans, id) + delete(c.nestedChans, id) + c.deleteSelfIfReady() +} + +func (c *Channel) triggerDelete() { + c.closeCalled = true + c.deleteSelfIfReady() +} + +func (c *Channel) getParentID() int64 { + if c.Parent == nil { + return -1 + } + return c.Parent.ID +} + +// deleteSelfFromTree tries to delete the channel from the channelz entry relation tree, which means +// deleting the channel reference from its parent's child list. +// +// In order for a channel to be deleted from the tree, it must meet the criteria that, removal of the +// corresponding grpc object has been invoked, and the channel does not have any children left. +// +// The returned boolean value indicates whether the channel has been successfully deleted from tree. +func (c *Channel) deleteSelfFromTree() (deleted bool) { + if !c.closeCalled || len(c.subChans)+len(c.nestedChans) != 0 { + return false + } + // not top channel + if c.Parent != nil { + c.Parent.deleteChild(c.ID) + } + return true +} + +// deleteSelfFromMap checks whether it is valid to delete the channel from the map, which means +// deleting the channel from channelz's tracking entirely. Users can no longer use id to query the +// channel, and its memory will be garbage collected. +// +// The trace reference count of the channel must be 0 in order to be deleted from the map. This is +// specified in the channel tracing gRFC that as long as some other trace has reference to an entity, +// the trace of the referenced entity must not be deleted. In order to release the resource allocated +// by grpc, the reference to the grpc object is reset to a dummy object. +// +// deleteSelfFromMap must be called after deleteSelfFromTree returns true. +// +// It returns a bool to indicate whether the channel can be safely deleted from map. +func (c *Channel) deleteSelfFromMap() (delete bool) { + return c.getTraceRefCount() == 0 +} + +// deleteSelfIfReady tries to delete the channel itself from the channelz database. +// The delete process includes two steps: +// 1. delete the channel from the entry relation tree, i.e. delete the channel reference from its +// parent's child list. +// 2. delete the channel from the map, i.e. delete the channel entirely from channelz. Lookup by id +// will return entry not found error. +func (c *Channel) deleteSelfIfReady() { + if !c.deleteSelfFromTree() { + return + } + if !c.deleteSelfFromMap() { + return + } + db.deleteEntry(c.ID) + c.trace.clear() +} + +func (c *Channel) getChannelTrace() *ChannelTrace { + return c.trace +} + +func (c *Channel) incrTraceRefCount() { + atomic.AddInt32(&c.traceRefCount, 1) +} + +func (c *Channel) decrTraceRefCount() { + atomic.AddInt32(&c.traceRefCount, -1) +} + +func (c *Channel) getTraceRefCount() int { + i := atomic.LoadInt32(&c.traceRefCount) + return int(i) +} + +func (c *Channel) getRefName() string { + return c.RefName +} diff --git a/internal/channelz/channelmap.go b/internal/channelz/channelmap.go new file mode 100644 index 0000000..64c7919 --- /dev/null +++ b/internal/channelz/channelmap.go @@ -0,0 +1,395 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz + +import ( + "fmt" + "sort" + "sync" + "time" +) + +// entry represents a node in the channelz database. +type entry interface { + // addChild adds a child e, whose channelz id is id to child list + addChild(id int64, e entry) + // deleteChild deletes a child with channelz id to be id from child list + deleteChild(id int64) + // triggerDelete tries to delete self from channelz database. However, if + // child list is not empty, then deletion from the database is on hold until + // the last child is deleted from database. + triggerDelete() + // deleteSelfIfReady check whether triggerDelete() has been called before, + // and whether child list is now empty. If both conditions are met, then + // delete self from database. + deleteSelfIfReady() + // getParentID returns parent ID of the entry. 0 value parent ID means no parent. + getParentID() int64 + Entity +} + +// channelMap is the storage data structure for channelz. +// +// Methods of channelMap can be divided into two categories with respect to +// locking. +// +// 1. Methods acquire the global lock. +// 2. Methods that can only be called when global lock is held. +// +// A second type of method need always to be called inside a first type of method. +type channelMap struct { + mu sync.RWMutex + topLevelChannels map[int64]struct{} + channels map[int64]*Channel + subChannels map[int64]*SubChannel + sockets map[int64]*Socket + servers map[int64]*Server +} + +func newChannelMap() *channelMap { + return &channelMap{ + topLevelChannels: make(map[int64]struct{}), + channels: make(map[int64]*Channel), + subChannels: make(map[int64]*SubChannel), + sockets: make(map[int64]*Socket), + servers: make(map[int64]*Server), + } +} + +func (c *channelMap) addServer(id int64, s *Server) { + c.mu.Lock() + defer c.mu.Unlock() + s.cm = c + c.servers[id] = s +} + +func (c *channelMap) addChannel(id int64, cn *Channel, isTopChannel bool, pid int64) { + c.mu.Lock() + defer c.mu.Unlock() + cn.trace.cm = c + c.channels[id] = cn + if isTopChannel { + c.topLevelChannels[id] = struct{}{} + } else if p := c.channels[pid]; p != nil { + p.addChild(id, cn) + } else { + logger.Infof("channel %d references invalid parent ID %d", id, pid) + } +} + +func (c *channelMap) addSubChannel(id int64, sc *SubChannel, pid int64) { + c.mu.Lock() + defer c.mu.Unlock() + sc.trace.cm = c + c.subChannels[id] = sc + if p := c.channels[pid]; p != nil { + p.addChild(id, sc) + } else { + logger.Infof("subchannel %d references invalid parent ID %d", id, pid) + } +} + +func (c *channelMap) addSocket(s *Socket) { + c.mu.Lock() + defer c.mu.Unlock() + s.cm = c + c.sockets[s.ID] = s + if s.Parent == nil { + logger.Infof("normal socket %d has no parent", s.ID) + } + s.Parent.(entry).addChild(s.ID, s) +} + +// removeEntry triggers the removal of an entry, which may not indeed delete the +// entry, if it has to wait on the deletion of its children and until no other +// entity's channel trace references it. It may lead to a chain of entry +// deletion. For example, deleting the last socket of a gracefully shutting down +// server will lead to the server being also deleted. +func (c *channelMap) removeEntry(id int64) { + c.mu.Lock() + defer c.mu.Unlock() + c.findEntry(id).triggerDelete() +} + +// tracedChannel represents tracing operations which are present on both +// channels and subChannels. +type tracedChannel interface { + getChannelTrace() *ChannelTrace + incrTraceRefCount() + decrTraceRefCount() + getRefName() string +} + +// c.mu must be held by the caller +func (c *channelMap) decrTraceRefCount(id int64) { + e := c.findEntry(id) + if v, ok := e.(tracedChannel); ok { + v.decrTraceRefCount() + e.deleteSelfIfReady() + } +} + +// c.mu must be held by the caller. +func (c *channelMap) findEntry(id int64) entry { + if v, ok := c.channels[id]; ok { + return v + } + if v, ok := c.subChannels[id]; ok { + return v + } + if v, ok := c.servers[id]; ok { + return v + } + if v, ok := c.sockets[id]; ok { + return v + } + return &dummyEntry{idNotFound: id} +} + +// c.mu must be held by the caller +// +// deleteEntry deletes an entry from the channelMap. Before calling this method, +// caller must check this entry is ready to be deleted, i.e removeEntry() has +// been called on it, and no children still exist. +func (c *channelMap) deleteEntry(id int64) entry { + if v, ok := c.sockets[id]; ok { + delete(c.sockets, id) + return v + } + if v, ok := c.subChannels[id]; ok { + delete(c.subChannels, id) + return v + } + if v, ok := c.channels[id]; ok { + delete(c.channels, id) + delete(c.topLevelChannels, id) + return v + } + if v, ok := c.servers[id]; ok { + delete(c.servers, id) + return v + } + return &dummyEntry{idNotFound: id} +} + +func (c *channelMap) traceEvent(id int64, desc *TraceEvent) { + c.mu.Lock() + defer c.mu.Unlock() + child := c.findEntry(id) + childTC, ok := child.(tracedChannel) + if !ok { + return + } + childTC.getChannelTrace().append(&traceEvent{Desc: desc.Desc, Severity: desc.Severity, Timestamp: time.Now()}) + if desc.Parent != nil { + parent := c.findEntry(child.getParentID()) + var chanType RefChannelType + switch child.(type) { + case *Channel: + chanType = RefChannel + case *SubChannel: + chanType = RefSubChannel + } + if parentTC, ok := parent.(tracedChannel); ok { + parentTC.getChannelTrace().append(&traceEvent{ + Desc: desc.Parent.Desc, + Severity: desc.Parent.Severity, + Timestamp: time.Now(), + RefID: id, + RefName: childTC.getRefName(), + RefType: chanType, + }) + childTC.incrTraceRefCount() + } + } +} + +type int64Slice []int64 + +func (s int64Slice) Len() int { return len(s) } +func (s int64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s int64Slice) Less(i, j int) bool { return s[i] < s[j] } + +func copyMap(m map[int64]string) map[int64]string { + n := make(map[int64]string) + for k, v := range m { + n[k] = v + } + return n +} + +func (c *channelMap) getTopChannels(id int64, maxResults int) ([]*Channel, bool) { + if maxResults <= 0 { + maxResults = EntriesPerPage + } + c.mu.RLock() + defer c.mu.RUnlock() + l := int64(len(c.topLevelChannels)) + ids := make([]int64, 0, l) + + for k := range c.topLevelChannels { + ids = append(ids, k) + } + sort.Sort(int64Slice(ids)) + idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= id }) + end := true + var t []*Channel + for _, v := range ids[idx:] { + if len(t) == maxResults { + end = false + break + } + if cn, ok := c.channels[v]; ok { + t = append(t, cn) + } + } + return t, end +} + +func (c *channelMap) getServers(id int64, maxResults int) ([]*Server, bool) { + if maxResults <= 0 { + maxResults = EntriesPerPage + } + c.mu.RLock() + defer c.mu.RUnlock() + ids := make([]int64, 0, len(c.servers)) + for k := range c.servers { + ids = append(ids, k) + } + sort.Sort(int64Slice(ids)) + idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= id }) + end := true + var s []*Server + for _, v := range ids[idx:] { + if len(s) == maxResults { + end = false + break + } + if svr, ok := c.servers[v]; ok { + s = append(s, svr) + } + } + return s, end +} + +func (c *channelMap) getServerSockets(id int64, startID int64, maxResults int) ([]*Socket, bool) { + if maxResults <= 0 { + maxResults = EntriesPerPage + } + c.mu.RLock() + defer c.mu.RUnlock() + svr, ok := c.servers[id] + if !ok { + // server with id doesn't exist. + return nil, true + } + svrskts := svr.sockets + ids := make([]int64, 0, len(svrskts)) + sks := make([]*Socket, 0, min(len(svrskts), maxResults)) + for k := range svrskts { + ids = append(ids, k) + } + sort.Sort(int64Slice(ids)) + idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= startID }) + end := true + for _, v := range ids[idx:] { + if len(sks) == maxResults { + end = false + break + } + if ns, ok := c.sockets[v]; ok { + sks = append(sks, ns) + } + } + return sks, end +} + +func (c *channelMap) getChannel(id int64) *Channel { + c.mu.RLock() + defer c.mu.RUnlock() + return c.channels[id] +} + +func (c *channelMap) getSubChannel(id int64) *SubChannel { + c.mu.RLock() + defer c.mu.RUnlock() + return c.subChannels[id] +} + +func (c *channelMap) getSocket(id int64) *Socket { + c.mu.RLock() + defer c.mu.RUnlock() + return c.sockets[id] +} + +func (c *channelMap) getServer(id int64) *Server { + c.mu.RLock() + defer c.mu.RUnlock() + return c.servers[id] +} + +type dummyEntry struct { + // dummyEntry is a fake entry to handle entry not found case. + idNotFound int64 + Entity +} + +func (d *dummyEntry) String() string { + return fmt.Sprintf("non-existent entity #%d", d.idNotFound) +} + +func (d *dummyEntry) ID() int64 { return d.idNotFound } + +func (d *dummyEntry) addChild(id int64, e entry) { + // Note: It is possible for a normal program to reach here under race + // condition. For example, there could be a race between ClientConn.Close() + // info being propagated to addrConn and http2Client. ClientConn.Close() + // cancel the context and result in http2Client to error. The error info is + // then caught by transport monitor and before addrConn.tearDown() is called + // in side ClientConn.Close(). Therefore, the addrConn will create a new + // transport. And when registering the new transport in channelz, its parent + // addrConn could have already been torn down and deleted from channelz + // tracking, and thus reach the code here. + logger.Infof("attempt to add child of type %T with id %d to a parent (id=%d) that doesn't currently exist", e, id, d.idNotFound) +} + +func (d *dummyEntry) deleteChild(id int64) { + // It is possible for a normal program to reach here under race condition. + // Refer to the example described in addChild(). + logger.Infof("attempt to delete child with id %d from a parent (id=%d) that doesn't currently exist", id, d.idNotFound) +} + +func (d *dummyEntry) triggerDelete() { + logger.Warningf("attempt to delete an entry (id=%d) that doesn't currently exist", d.idNotFound) +} + +func (*dummyEntry) deleteSelfIfReady() { + // code should not reach here. deleteSelfIfReady is always called on an existing entry. +} + +func (*dummyEntry) getParentID() int64 { + return 0 +} + +// Entity is implemented by all channelz types. +type Entity interface { + isEntity() + fmt.Stringer + id() int64 +} diff --git a/internal/channelz/funcs.go b/internal/channelz/funcs.go new file mode 100644 index 0000000..078bb81 --- /dev/null +++ b/internal/channelz/funcs.go @@ -0,0 +1,230 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package channelz defines internal APIs for enabling channelz service, entry +// registration/deletion, and accessing channelz data. It also defines channelz +// metric struct formats. +package channelz + +import ( + "sync/atomic" + "time" + + "google.golang.org/grpc/internal" +) + +var ( + // IDGen is the global channelz entity ID generator. It should not be used + // outside this package except by tests. + IDGen IDGenerator + + db = newChannelMap() + // EntriesPerPage defines the number of channelz entries to be shown on a web page. + EntriesPerPage = 50 + curState int32 +) + +// TurnOn turns on channelz data collection. +func TurnOn() { + atomic.StoreInt32(&curState, 1) +} + +func init() { + internal.ChannelzTurnOffForTesting = func() { + atomic.StoreInt32(&curState, 0) + } +} + +// IsOn returns whether channelz data collection is on. +func IsOn() bool { + return atomic.LoadInt32(&curState) == 1 +} + +// GetTopChannels returns a slice of top channel's ChannelMetric, along with a +// boolean indicating whether there's more top channels to be queried for. +// +// The arg id specifies that only top channel with id at or above it will be +// included in the result. The returned slice is up to a length of the arg +// maxResults or EntriesPerPage if maxResults is zero, and is sorted in ascending +// id order. +func GetTopChannels(id int64, maxResults int) ([]*Channel, bool) { + return db.getTopChannels(id, maxResults) +} + +// GetServers returns a slice of server's ServerMetric, along with a +// boolean indicating whether there's more servers to be queried for. +// +// The arg id specifies that only server with id at or above it will be included +// in the result. The returned slice is up to a length of the arg maxResults or +// EntriesPerPage if maxResults is zero, and is sorted in ascending id order. +func GetServers(id int64, maxResults int) ([]*Server, bool) { + return db.getServers(id, maxResults) +} + +// GetServerSockets returns a slice of server's (identified by id) normal socket's +// SocketMetrics, along with a boolean indicating whether there's more sockets to +// be queried for. +// +// The arg startID specifies that only sockets with id at or above it will be +// included in the result. The returned slice is up to a length of the arg maxResults +// or EntriesPerPage if maxResults is zero, and is sorted in ascending id order. +func GetServerSockets(id int64, startID int64, maxResults int) ([]*Socket, bool) { + return db.getServerSockets(id, startID, maxResults) +} + +// GetChannel returns the Channel for the channel (identified by id). +func GetChannel(id int64) *Channel { + return db.getChannel(id) +} + +// GetSubChannel returns the SubChannel for the subchannel (identified by id). +func GetSubChannel(id int64) *SubChannel { + return db.getSubChannel(id) +} + +// GetSocket returns the Socket for the socket (identified by id). +func GetSocket(id int64) *Socket { + return db.getSocket(id) +} + +// GetServer returns the ServerMetric for the server (identified by id). +func GetServer(id int64) *Server { + return db.getServer(id) +} + +// RegisterChannel registers the given channel c in the channelz database with +// target as its target and reference name, and adds it to the child list of its +// parent. parent == nil means no parent. +// +// Returns a unique channelz identifier assigned to this channel. +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterChannel(parent *Channel, target string) *Channel { + id := IDGen.genID() + + if !IsOn() { + return &Channel{ID: id} + } + + isTopChannel := parent == nil + + cn := &Channel{ + ID: id, + RefName: target, + nestedChans: make(map[int64]string), + subChans: make(map[int64]string), + Parent: parent, + trace: &ChannelTrace{CreationTime: time.Now(), Events: make([]*traceEvent, 0, getMaxTraceEntry())}, + } + cn.ChannelMetrics.Target.Store(&target) + db.addChannel(id, cn, isTopChannel, cn.getParentID()) + return cn +} + +// RegisterSubChannel registers the given subChannel c in the channelz database +// with ref as its reference name, and adds it to the child list of its parent +// (identified by pid). +// +// Returns a unique channelz identifier assigned to this subChannel. +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterSubChannel(parent *Channel, ref string) *SubChannel { + id := IDGen.genID() + sc := &SubChannel{ + ID: id, + RefName: ref, + parent: parent, + } + + if !IsOn() { + return sc + } + + sc.sockets = make(map[int64]string) + sc.trace = &ChannelTrace{CreationTime: time.Now(), Events: make([]*traceEvent, 0, getMaxTraceEntry())} + db.addSubChannel(id, sc, parent.ID) + return sc +} + +// RegisterServer registers the given server s in channelz database. It returns +// the unique channelz tracking id assigned to this server. +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterServer(ref string) *Server { + id := IDGen.genID() + if !IsOn() { + return &Server{ID: id} + } + + svr := &Server{ + RefName: ref, + sockets: make(map[int64]string), + listenSockets: make(map[int64]string), + ID: id, + } + db.addServer(id, svr) + return svr +} + +// RegisterSocket registers the given normal socket s in channelz database +// with ref as its reference name, and adds it to the child list of its parent +// (identified by skt.Parent, which must be set). It returns the unique channelz +// tracking id assigned to this normal socket. +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterSocket(skt *Socket) *Socket { + skt.ID = IDGen.genID() + if IsOn() { + db.addSocket(skt) + } + return skt +} + +// RemoveEntry removes an entry with unique channelz tracking id to be id from +// channelz database. +// +// If channelz is not turned ON, this function is a no-op. +func RemoveEntry(id int64) { + if !IsOn() { + return + } + db.removeEntry(id) +} + +// IDGenerator is an incrementing atomic that tracks IDs for channelz entities. +type IDGenerator struct { + id int64 +} + +// Reset resets the generated ID back to zero. Should only be used at +// initialization or by tests sensitive to the ID number. +func (i *IDGenerator) Reset() { + atomic.StoreInt64(&i.id, 0) +} + +func (i *IDGenerator) genID() int64 { + return atomic.AddInt64(&i.id, 1) +} + +// Identifier is an opaque channelz identifier used to expose channelz symbols +// outside of grpc. Currently only implemented by Channel since no other +// types require exposure outside grpc. +type Identifier interface { + Entity + channelzIdentifier() +} diff --git a/internal/channelz/logging.go b/internal/channelz/logging.go new file mode 100644 index 0000000..ee4d721 --- /dev/null +++ b/internal/channelz/logging.go @@ -0,0 +1,75 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" +) + +var logger = grpclog.Component("channelz") + +// Info logs and adds a trace event if channelz is on. +func Info(l grpclog.DepthLoggerV2, e Entity, args ...any) { + AddTraceEvent(l, e, 1, &TraceEvent{ + Desc: fmt.Sprint(args...), + Severity: CtInfo, + }) +} + +// Infof logs and adds a trace event if channelz is on. +func Infof(l grpclog.DepthLoggerV2, e Entity, format string, args ...any) { + AddTraceEvent(l, e, 1, &TraceEvent{ + Desc: fmt.Sprintf(format, args...), + Severity: CtInfo, + }) +} + +// Warning logs and adds a trace event if channelz is on. +func Warning(l grpclog.DepthLoggerV2, e Entity, args ...any) { + AddTraceEvent(l, e, 1, &TraceEvent{ + Desc: fmt.Sprint(args...), + Severity: CtWarning, + }) +} + +// Warningf logs and adds a trace event if channelz is on. +func Warningf(l grpclog.DepthLoggerV2, e Entity, format string, args ...any) { + AddTraceEvent(l, e, 1, &TraceEvent{ + Desc: fmt.Sprintf(format, args...), + Severity: CtWarning, + }) +} + +// Error logs and adds a trace event if channelz is on. +func Error(l grpclog.DepthLoggerV2, e Entity, args ...any) { + AddTraceEvent(l, e, 1, &TraceEvent{ + Desc: fmt.Sprint(args...), + Severity: CtError, + }) +} + +// Errorf logs and adds a trace event if channelz is on. +func Errorf(l grpclog.DepthLoggerV2, e Entity, format string, args ...any) { + AddTraceEvent(l, e, 1, &TraceEvent{ + Desc: fmt.Sprintf(format, args...), + Severity: CtError, + }) +} diff --git a/internal/channelz/server.go b/internal/channelz/server.go new file mode 100644 index 0000000..b5a8249 --- /dev/null +++ b/internal/channelz/server.go @@ -0,0 +1,121 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz + +import ( + "fmt" + "sync/atomic" +) + +// Server is the channelz representation of a server. +type Server struct { + Entity + ID int64 + RefName string + + ServerMetrics ServerMetrics + + closeCalled bool + sockets map[int64]string + listenSockets map[int64]string + cm *channelMap +} + +// ServerMetrics defines a struct containing metrics for servers. +type ServerMetrics struct { + // The number of incoming calls started on the server. + CallsStarted atomic.Int64 + // The number of incoming calls that have completed with an OK status. + CallsSucceeded atomic.Int64 + // The number of incoming calls that have a completed with a non-OK status. + CallsFailed atomic.Int64 + // The last time a call was started on the server. + LastCallStartedTimestamp atomic.Int64 +} + +// NewServerMetricsForTesting returns an initialized ServerMetrics. +func NewServerMetricsForTesting(started, succeeded, failed, timestamp int64) *ServerMetrics { + sm := &ServerMetrics{} + sm.CallsStarted.Store(started) + sm.CallsSucceeded.Store(succeeded) + sm.CallsFailed.Store(failed) + sm.LastCallStartedTimestamp.Store(timestamp) + return sm +} + +// CopyFrom copies the metrics data from the provided ServerMetrics +// instance into the current instance. +func (sm *ServerMetrics) CopyFrom(o *ServerMetrics) { + sm.CallsStarted.Store(o.CallsStarted.Load()) + sm.CallsSucceeded.Store(o.CallsSucceeded.Load()) + sm.CallsFailed.Store(o.CallsFailed.Load()) + sm.LastCallStartedTimestamp.Store(o.LastCallStartedTimestamp.Load()) +} + +// ListenSockets returns the listening sockets for s. +func (s *Server) ListenSockets() map[int64]string { + db.mu.RLock() + defer db.mu.RUnlock() + return copyMap(s.listenSockets) +} + +// String returns a printable description of s. +func (s *Server) String() string { + return fmt.Sprintf("Server #%d", s.ID) +} + +func (s *Server) id() int64 { + return s.ID +} + +func (s *Server) addChild(id int64, e entry) { + switch v := e.(type) { + case *Socket: + switch v.SocketType { + case SocketTypeNormal: + s.sockets[id] = v.RefName + case SocketTypeListen: + s.listenSockets[id] = v.RefName + } + default: + logger.Errorf("cannot add a child (id = %d) of type %T to a server", id, e) + } +} + +func (s *Server) deleteChild(id int64) { + delete(s.sockets, id) + delete(s.listenSockets, id) + s.deleteSelfIfReady() +} + +func (s *Server) triggerDelete() { + s.closeCalled = true + s.deleteSelfIfReady() +} + +func (s *Server) deleteSelfIfReady() { + if !s.closeCalled || len(s.sockets)+len(s.listenSockets) != 0 { + return + } + s.cm.deleteEntry(s.ID) +} + +func (s *Server) getParentID() int64 { + return 0 +} diff --git a/internal/channelz/socket.go b/internal/channelz/socket.go new file mode 100644 index 0000000..9010384 --- /dev/null +++ b/internal/channelz/socket.go @@ -0,0 +1,137 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz + +import ( + "fmt" + "net" + "sync/atomic" + + "google.golang.org/grpc/credentials" +) + +// SocketMetrics defines the struct that the implementor of Socket interface +// should return from ChannelzMetric(). +type SocketMetrics struct { + // The number of streams that have been started. + StreamsStarted atomic.Int64 + // The number of streams that have ended successfully: + // On client side, receiving frame with eos bit set. + // On server side, sending frame with eos bit set. + StreamsSucceeded atomic.Int64 + // The number of streams that have ended unsuccessfully: + // On client side, termination without receiving frame with eos bit set. + // On server side, termination without sending frame with eos bit set. + StreamsFailed atomic.Int64 + // The number of messages successfully sent on this socket. + MessagesSent atomic.Int64 + MessagesReceived atomic.Int64 + // The number of keep alives sent. This is typically implemented with HTTP/2 + // ping messages. + KeepAlivesSent atomic.Int64 + // The last time a stream was created by this endpoint. Usually unset for + // servers. + LastLocalStreamCreatedTimestamp atomic.Int64 + // The last time a stream was created by the remote endpoint. Usually unset + // for clients. + LastRemoteStreamCreatedTimestamp atomic.Int64 + // The last time a message was sent by this endpoint. + LastMessageSentTimestamp atomic.Int64 + // The last time a message was received by this endpoint. + LastMessageReceivedTimestamp atomic.Int64 +} + +// EphemeralSocketMetrics are metrics that change rapidly and are tracked +// outside of channelz. +type EphemeralSocketMetrics struct { + // The amount of window, granted to the local endpoint by the remote endpoint. + // This may be slightly out of date due to network latency. This does NOT + // include stream level or TCP level flow control info. + LocalFlowControlWindow int64 + // The amount of window, granted to the remote endpoint by the local endpoint. + // This may be slightly out of date due to network latency. This does NOT + // include stream level or TCP level flow control info. + RemoteFlowControlWindow int64 +} + +// SocketType represents the type of socket. +type SocketType string + +// SocketType can be one of these. +const ( + SocketTypeNormal = "NormalSocket" + SocketTypeListen = "ListenSocket" +) + +// Socket represents a socket within channelz which includes socket +// metrics and data related to socket activity and provides methods +// for managing and interacting with sockets. +type Socket struct { + Entity + SocketType SocketType + ID int64 + Parent Entity + cm *channelMap + SocketMetrics SocketMetrics + EphemeralMetrics func() *EphemeralSocketMetrics + + RefName string + // The locally bound address. Immutable. + LocalAddr net.Addr + // The remote bound address. May be absent. Immutable. + RemoteAddr net.Addr + // Optional, represents the name of the remote endpoint, if different than + // the original target name. Immutable. + RemoteName string + // Immutable. + SocketOptions *SocketOptionData + // Immutable. + Security credentials.ChannelzSecurityValue +} + +// String returns a string representation of the Socket, including its parent +// entity, socket type, and ID. +func (ls *Socket) String() string { + return fmt.Sprintf("%s %s #%d", ls.Parent, ls.SocketType, ls.ID) +} + +func (ls *Socket) id() int64 { + return ls.ID +} + +func (ls *Socket) addChild(id int64, e entry) { + logger.Errorf("cannot add a child (id = %d) of type %T to a listen socket", id, e) +} + +func (ls *Socket) deleteChild(id int64) { + logger.Errorf("cannot delete a child (id = %d) from a listen socket", id) +} + +func (ls *Socket) triggerDelete() { + ls.cm.deleteEntry(ls.ID) + ls.Parent.(entry).deleteChild(ls.ID) +} + +func (ls *Socket) deleteSelfIfReady() { + logger.Errorf("cannot call deleteSelfIfReady on a listen socket") +} + +func (ls *Socket) getParentID() int64 { + return ls.Parent.id() +} diff --git a/internal/channelz/subchannel.go b/internal/channelz/subchannel.go new file mode 100644 index 0000000..b20802e --- /dev/null +++ b/internal/channelz/subchannel.go @@ -0,0 +1,153 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz + +import ( + "fmt" + "sync/atomic" +) + +// SubChannel is the channelz representation of a subchannel. +type SubChannel struct { + Entity + // ID is the channelz id of this subchannel. + ID int64 + // RefName is the human readable reference string of this subchannel. + RefName string + closeCalled bool + sockets map[int64]string + parent *Channel + trace *ChannelTrace + traceRefCount int32 + + ChannelMetrics ChannelMetrics +} + +func (sc *SubChannel) String() string { + return fmt.Sprintf("%s SubChannel #%d", sc.parent, sc.ID) +} + +func (sc *SubChannel) id() int64 { + return sc.ID +} + +// Sockets returns a copy of the sockets map associated with the SubChannel. +func (sc *SubChannel) Sockets() map[int64]string { + db.mu.RLock() + defer db.mu.RUnlock() + return copyMap(sc.sockets) +} + +// Trace returns a copy of the ChannelTrace associated with the SubChannel. +func (sc *SubChannel) Trace() *ChannelTrace { + db.mu.RLock() + defer db.mu.RUnlock() + return sc.trace.copy() +} + +func (sc *SubChannel) addChild(id int64, e entry) { + if v, ok := e.(*Socket); ok && v.SocketType == SocketTypeNormal { + sc.sockets[id] = v.RefName + } else { + logger.Errorf("cannot add a child (id = %d) of type %T to a subChannel", id, e) + } +} + +func (sc *SubChannel) deleteChild(id int64) { + delete(sc.sockets, id) + sc.deleteSelfIfReady() +} + +func (sc *SubChannel) triggerDelete() { + sc.closeCalled = true + sc.deleteSelfIfReady() +} + +func (sc *SubChannel) getParentID() int64 { + return sc.parent.ID +} + +// deleteSelfFromTree tries to delete the subchannel from the channelz entry relation tree, which +// means deleting the subchannel reference from its parent's child list. +// +// In order for a subchannel to be deleted from the tree, it must meet the criteria that, removal of +// the corresponding grpc object has been invoked, and the subchannel does not have any children left. +// +// The returned boolean value indicates whether the channel has been successfully deleted from tree. +func (sc *SubChannel) deleteSelfFromTree() (deleted bool) { + if !sc.closeCalled || len(sc.sockets) != 0 { + return false + } + sc.parent.deleteChild(sc.ID) + return true +} + +// deleteSelfFromMap checks whether it is valid to delete the subchannel from the map, which means +// deleting the subchannel from channelz's tracking entirely. Users can no longer use id to query +// the subchannel, and its memory will be garbage collected. +// +// The trace reference count of the subchannel must be 0 in order to be deleted from the map. This is +// specified in the channel tracing gRFC that as long as some other trace has reference to an entity, +// the trace of the referenced entity must not be deleted. In order to release the resource allocated +// by grpc, the reference to the grpc object is reset to a dummy object. +// +// deleteSelfFromMap must be called after deleteSelfFromTree returns true. +// +// It returns a bool to indicate whether the channel can be safely deleted from map. +func (sc *SubChannel) deleteSelfFromMap() (delete bool) { + return sc.getTraceRefCount() == 0 +} + +// deleteSelfIfReady tries to delete the subchannel itself from the channelz database. +// The delete process includes two steps: +// 1. delete the subchannel from the entry relation tree, i.e. delete the subchannel reference from +// its parent's child list. +// 2. delete the subchannel from the map, i.e. delete the subchannel entirely from channelz. Lookup +// by id will return entry not found error. +func (sc *SubChannel) deleteSelfIfReady() { + if !sc.deleteSelfFromTree() { + return + } + if !sc.deleteSelfFromMap() { + return + } + db.deleteEntry(sc.ID) + sc.trace.clear() +} + +func (sc *SubChannel) getChannelTrace() *ChannelTrace { + return sc.trace +} + +func (sc *SubChannel) incrTraceRefCount() { + atomic.AddInt32(&sc.traceRefCount, 1) +} + +func (sc *SubChannel) decrTraceRefCount() { + atomic.AddInt32(&sc.traceRefCount, -1) +} + +func (sc *SubChannel) getTraceRefCount() int { + i := atomic.LoadInt32(&sc.traceRefCount) + return int(i) +} + +func (sc *SubChannel) getRefName() string { + return sc.RefName +} diff --git a/internal/channelz/syscall_linux.go b/internal/channelz/syscall_linux.go new file mode 100644 index 0000000..5ac73ff --- /dev/null +++ b/internal/channelz/syscall_linux.go @@ -0,0 +1,65 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +// SocketOptionData defines the struct to hold socket option data, and related +// getter function to obtain info from fd. +type SocketOptionData struct { + Linger *unix.Linger + RecvTimeout *unix.Timeval + SendTimeout *unix.Timeval + TCPInfo *unix.TCPInfo +} + +// Getsockopt defines the function to get socket options requested by channelz. +// It is to be passed to syscall.RawConn.Control(). +func (s *SocketOptionData) Getsockopt(fd uintptr) { + if v, err := unix.GetsockoptLinger(int(fd), syscall.SOL_SOCKET, syscall.SO_LINGER); err == nil { + s.Linger = v + } + if v, err := unix.GetsockoptTimeval(int(fd), syscall.SOL_SOCKET, syscall.SO_RCVTIMEO); err == nil { + s.RecvTimeout = v + } + if v, err := unix.GetsockoptTimeval(int(fd), syscall.SOL_SOCKET, syscall.SO_SNDTIMEO); err == nil { + s.SendTimeout = v + } + if v, err := unix.GetsockoptTCPInfo(int(fd), syscall.SOL_TCP, syscall.TCP_INFO); err == nil { + s.TCPInfo = v + } +} + +// GetSocketOption gets the socket option info of the conn. +func GetSocketOption(socket any) *SocketOptionData { + c, ok := socket.(syscall.Conn) + if !ok { + return nil + } + data := &SocketOptionData{} + if rawConn, err := c.SyscallConn(); err == nil { + rawConn.Control(data.Getsockopt) + return data + } + return nil +} diff --git a/internal/channelz/syscall_nonlinux.go b/internal/channelz/syscall_nonlinux.go new file mode 100644 index 0000000..0e6e18e --- /dev/null +++ b/internal/channelz/syscall_nonlinux.go @@ -0,0 +1,47 @@ +//go:build !linux + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz + +import ( + "sync" +) + +var once sync.Once + +// SocketOptionData defines the struct to hold socket option data, and related +// getter function to obtain info from fd. +// Windows OS doesn't support Socket Option +type SocketOptionData struct { +} + +// Getsockopt defines the function to get socket options requested by channelz. +// It is to be passed to syscall.RawConn.Control(). +// Windows OS doesn't support Socket Option +func (s *SocketOptionData) Getsockopt(uintptr) { + once.Do(func() { + logger.Warning("Channelz: socket options are not supported on non-linux environments") + }) +} + +// GetSocketOption gets the socket option info of the conn. +func GetSocketOption(any) *SocketOptionData { + return nil +} diff --git a/internal/channelz/syscall_test.go b/internal/channelz/syscall_test.go new file mode 100644 index 0000000..ae67c4d --- /dev/null +++ b/internal/channelz/syscall_test.go @@ -0,0 +1,95 @@ +//go:build linux + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz_test + +import ( + "net" + "reflect" + "syscall" + "testing" + + "golang.org/x/sys/unix" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestGetSocketOpt(t *testing.T) { + network, addr := "tcp", ":0" + ln, err := net.Listen(network, addr) + if err != nil { + t.Fatalf("net.Listen(%s,%s) failed with err: %v", network, addr, err) + } + defer ln.Close() + go func() { + ln.Accept() + }() + conn, _ := net.Dial(network, ln.Addr().String()) + defer conn.Close() + tcpc := conn.(*net.TCPConn) + raw, err := tcpc.SyscallConn() + if err != nil { + t.Fatalf("SyscallConn() failed due to %v", err) + } + + l := &unix.Linger{Onoff: 1, Linger: 5} + recvTimeout := &unix.Timeval{Sec: 100} + sendTimeout := &unix.Timeval{Sec: 8888} + raw.Control(func(fd uintptr) { + err := unix.SetsockoptLinger(int(fd), syscall.SOL_SOCKET, syscall.SO_LINGER, l) + if err != nil { + t.Fatalf("failed to SetsockoptLinger(%v,%v,%v,%v) due to %v", int(fd), syscall.SOL_SOCKET, syscall.SO_LINGER, l, err) + } + err = unix.SetsockoptTimeval(int(fd), syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, recvTimeout) + if err != nil { + t.Fatalf("failed to SetsockoptTimeval(%v,%v,%v,%v) due to %v", int(fd), syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, recvTimeout, err) + } + err = unix.SetsockoptTimeval(int(fd), syscall.SOL_SOCKET, syscall.SO_SNDTIMEO, sendTimeout) + if err != nil { + t.Fatalf("failed to SetsockoptTimeval(%v,%v,%v,%v) due to %v", int(fd), syscall.SOL_SOCKET, syscall.SO_SNDTIMEO, sendTimeout, err) + } + }) + sktopt := channelz.GetSocketOption(conn) + if !reflect.DeepEqual(sktopt.Linger, l) { + t.Fatalf("get socket option linger, want: %v, got %v", l, sktopt.Linger) + } + if !reflect.DeepEqual(sktopt.RecvTimeout, recvTimeout) { + t.Logf("get socket option recv timeout, want: %v, got %v, may be caused by system allowing non or partial setting of this value", recvTimeout, sktopt.RecvTimeout) + } + if !reflect.DeepEqual(sktopt.SendTimeout, sendTimeout) { + t.Logf("get socket option send timeout, want: %v, got %v, may be caused by system allowing non or partial setting of this value", sendTimeout, sktopt.SendTimeout) + } + if sktopt == nil || sktopt.TCPInfo != nil && sktopt.TCPInfo.State != 1 { + t.Fatalf("TCPInfo.State want 1 (TCP_ESTABLISHED), got %v", sktopt) + } + + sktopt = channelz.GetSocketOption(ln) + if sktopt == nil || sktopt.TCPInfo == nil || sktopt.TCPInfo.State != 10 { + t.Fatalf("TCPInfo.State want 10 (TCP_LISTEN), got %v", sktopt) + } +} diff --git a/internal/channelz/trace.go b/internal/channelz/trace.go new file mode 100644 index 0000000..3b7ba59 --- /dev/null +++ b/internal/channelz/trace.go @@ -0,0 +1,213 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz + +import ( + "fmt" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/grpclog" +) + +const ( + defaultMaxTraceEntry int32 = 30 +) + +var maxTraceEntry = defaultMaxTraceEntry + +// SetMaxTraceEntry sets maximum number of trace entries per entity (i.e. +// channel/subchannel). Setting it to 0 will disable channel tracing. +func SetMaxTraceEntry(i int32) { + atomic.StoreInt32(&maxTraceEntry, i) +} + +// ResetMaxTraceEntryToDefault resets the maximum number of trace entries per +// entity to default. +func ResetMaxTraceEntryToDefault() { + atomic.StoreInt32(&maxTraceEntry, defaultMaxTraceEntry) +} + +func getMaxTraceEntry() int { + i := atomic.LoadInt32(&maxTraceEntry) + return int(i) +} + +// traceEvent is an internal representation of a single trace event +type traceEvent struct { + // Desc is a simple description of the trace event. + Desc string + // Severity states the severity of this trace event. + Severity Severity + // Timestamp is the event time. + Timestamp time.Time + // RefID is the id of the entity that gets referenced in the event. RefID is 0 if no other entity is + // involved in this event. + // e.g. SubChannel (id: 4[]) Created. --> RefID = 4, RefName = "" (inside []) + RefID int64 + // RefName is the reference name for the entity that gets referenced in the event. + RefName string + // RefType indicates the referenced entity type, i.e Channel or SubChannel. + RefType RefChannelType +} + +// TraceEvent is what the caller of AddTraceEvent should provide to describe the +// event to be added to the channel trace. +// +// The Parent field is optional. It is used for an event that will be recorded +// in the entity's parent trace. +type TraceEvent struct { + Desc string + Severity Severity + Parent *TraceEvent +} + +// ChannelTrace provides tracing information for a channel. +// It tracks various events and metadata related to the channel's lifecycle +// and operations. +type ChannelTrace struct { + cm *channelMap + clearCalled bool + // The time when the trace was created. + CreationTime time.Time + // A counter for the number of events recorded in the + // trace. + EventNum int64 + mu sync.Mutex + // A slice of traceEvent pointers representing the events recorded for + // this channel. + Events []*traceEvent +} + +func (c *ChannelTrace) copy() *ChannelTrace { + return &ChannelTrace{ + CreationTime: c.CreationTime, + EventNum: c.EventNum, + Events: append(([]*traceEvent)(nil), c.Events...), + } +} + +func (c *ChannelTrace) append(e *traceEvent) { + c.mu.Lock() + if len(c.Events) == getMaxTraceEntry() { + del := c.Events[0] + c.Events = c.Events[1:] + if del.RefID != 0 { + // start recursive cleanup in a goroutine to not block the call originated from grpc. + go func() { + // need to acquire c.cm.mu lock to call the unlocked attemptCleanup func. + c.cm.mu.Lock() + c.cm.decrTraceRefCount(del.RefID) + c.cm.mu.Unlock() + }() + } + } + e.Timestamp = time.Now() + c.Events = append(c.Events, e) + c.EventNum++ + c.mu.Unlock() +} + +func (c *ChannelTrace) clear() { + if c.clearCalled { + return + } + c.clearCalled = true + c.mu.Lock() + for _, e := range c.Events { + if e.RefID != 0 { + // caller should have already held the c.cm.mu lock. + c.cm.decrTraceRefCount(e.RefID) + } + } + c.mu.Unlock() +} + +// Severity is the severity level of a trace event. +// The canonical enumeration of all valid values is here: +// https://github.com/grpc/grpc-proto/blob/9b13d199cc0d4703c7ea26c9c330ba695866eb23/grpc/channelz/v1/channelz.proto#L126. +type Severity int + +const ( + // CtUnknown indicates unknown severity of a trace event. + CtUnknown Severity = iota + // CtInfo indicates info level severity of a trace event. + CtInfo + // CtWarning indicates warning level severity of a trace event. + CtWarning + // CtError indicates error level severity of a trace event. + CtError +) + +// RefChannelType is the type of the entity being referenced in a trace event. +type RefChannelType int + +const ( + // RefUnknown indicates an unknown entity type, the zero value for this type. + RefUnknown RefChannelType = iota + // RefChannel indicates the referenced entity is a Channel. + RefChannel + // RefSubChannel indicates the referenced entity is a SubChannel. + RefSubChannel + // RefServer indicates the referenced entity is a Server. + RefServer + // RefListenSocket indicates the referenced entity is a ListenSocket. + RefListenSocket + // RefNormalSocket indicates the referenced entity is a NormalSocket. + RefNormalSocket +) + +var refChannelTypeToString = map[RefChannelType]string{ + RefUnknown: "Unknown", + RefChannel: "Channel", + RefSubChannel: "SubChannel", + RefServer: "Server", + RefListenSocket: "ListenSocket", + RefNormalSocket: "NormalSocket", +} + +// String returns a string representation of the RefChannelType +func (r RefChannelType) String() string { + return refChannelTypeToString[r] +} + +// AddTraceEvent adds trace related to the entity with specified id, using the +// provided TraceEventDesc. +// +// If channelz is not turned ON, this will simply log the event descriptions. +func AddTraceEvent(l grpclog.DepthLoggerV2, e Entity, depth int, desc *TraceEvent) { + // Log only the trace description associated with the bottom most entity. + d := fmt.Sprintf("[%s] %s", e, desc.Desc) + switch desc.Severity { + case CtUnknown, CtInfo: + l.InfoDepth(depth+1, d) + case CtWarning: + l.WarningDepth(depth+1, d) + case CtError: + l.ErrorDepth(depth+1, d) + } + + if getMaxTraceEntry() == 0 { + return + } + if IsOn() { + db.traceEvent(e.id(), desc) + } +} diff --git a/internal/credentials/credentials.go b/internal/credentials/credentials.go new file mode 100644 index 0000000..48b22d9 --- /dev/null +++ b/internal/credentials/credentials.go @@ -0,0 +1,35 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "context" +) + +// clientHandshakeInfoKey is a struct used as the key to store +// ClientHandshakeInfo in a context. +type clientHandshakeInfoKey struct{} + +// ClientHandshakeInfoFromContext extracts the ClientHandshakeInfo from ctx. +func ClientHandshakeInfoFromContext(ctx context.Context) any { + return ctx.Value(clientHandshakeInfoKey{}) +} + +// NewClientHandshakeInfoContext creates a context with chi. +func NewClientHandshakeInfoContext(ctx context.Context, chi any) context.Context { + return context.WithValue(ctx, clientHandshakeInfoKey{}, chi) +} diff --git a/internal/credentials/spiffe.go b/internal/credentials/spiffe.go new file mode 100644 index 0000000..25ade62 --- /dev/null +++ b/internal/credentials/spiffe.go @@ -0,0 +1,75 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package credentials defines APIs for parsing SPIFFE ID. +// +// All APIs in this package are experimental. +package credentials + +import ( + "crypto/tls" + "crypto/x509" + "net/url" + + "google.golang.org/grpc/grpclog" +) + +var logger = grpclog.Component("credentials") + +// SPIFFEIDFromState parses the SPIFFE ID from State. If the SPIFFE ID format +// is invalid, return nil with warning. +func SPIFFEIDFromState(state tls.ConnectionState) *url.URL { + if len(state.PeerCertificates) == 0 || len(state.PeerCertificates[0].URIs) == 0 { + return nil + } + return SPIFFEIDFromCert(state.PeerCertificates[0]) +} + +// SPIFFEIDFromCert parses the SPIFFE ID from x509.Certificate. If the SPIFFE +// ID format is invalid, return nil with warning. +func SPIFFEIDFromCert(cert *x509.Certificate) *url.URL { + if cert == nil || cert.URIs == nil { + return nil + } + var spiffeID *url.URL + for _, uri := range cert.URIs { + if uri == nil || uri.Scheme != "spiffe" || uri.Opaque != "" || (uri.User != nil && uri.User.Username() != "") { + continue + } + // From this point, we assume the uri is intended for a SPIFFE ID. + if len(uri.String()) > 2048 { + logger.Warning("invalid SPIFFE ID: total ID length larger than 2048 bytes") + return nil + } + if len(uri.Host) == 0 || len(uri.Path) == 0 { + logger.Warning("invalid SPIFFE ID: domain or workload ID is empty") + return nil + } + if len(uri.Host) > 255 { + logger.Warning("invalid SPIFFE ID: domain length larger than 255 characters") + return nil + } + // A valid SPIFFE certificate can only have exactly one URI SAN field. + if len(cert.URIs) > 1 { + logger.Warning("invalid SPIFFE ID: multiple URI SANs") + return nil + } + spiffeID = uri + } + return spiffeID +} diff --git a/internal/credentials/spiffe/spiffe.go b/internal/credentials/spiffe/spiffe.go new file mode 100644 index 0000000..7be008a --- /dev/null +++ b/internal/credentials/spiffe/spiffe.go @@ -0,0 +1,107 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package spiffe defines APIs for working with SPIFFE Bundle Maps. +// +// All APIs in this package are experimental. +package spiffe + +import ( + "crypto/x509" + "encoding/json" + "fmt" + + "github.com/spiffe/go-spiffe/v2/bundle/spiffebundle" + "github.com/spiffe/go-spiffe/v2/spiffeid" +) + +type partialParsedSPIFFEBundleMap struct { + Bundles map[string]json.RawMessage `json:"trust_domains"` +} + +// BundleMapFromBytes parses bytes into a SPIFFE Bundle Map. See the +// SPIFFE Bundle Map spec for more detail - +// https://github.com/spiffe/spiffe/blob/main/standards/SPIFFE_Trust_Domain_and_Bundle.md#4-spiffe-bundle-format +// If duplicate keys are encountered in the JSON parsing, Go's default unmarshal +// behavior occurs which causes the last processed entry to be the entry in the +// parsed map. +func BundleMapFromBytes(bundleMapBytes []byte) (map[string]*spiffebundle.Bundle, error) { + var result partialParsedSPIFFEBundleMap + if err := json.Unmarshal(bundleMapBytes, &result); err != nil { + return nil, err + } + if result.Bundles == nil { + return nil, fmt.Errorf("spiffe: BundleMapFromBytes() no bundles parsed from spiffe bundle map bytes") + } + bundleMap := map[string]*spiffebundle.Bundle{} + for td, jsonBundle := range result.Bundles { + trustDomain, err := spiffeid.TrustDomainFromString(td) + if err != nil { + return nil, fmt.Errorf("spiffe: BundleMapFromBytes() invalid trust domain %q found when parsing SPIFFE Bundle Map: %v", td, err) + } + bundle, err := spiffebundle.Parse(trustDomain, jsonBundle) + if err != nil { + return nil, fmt.Errorf("spiffe: BundleMapFromBytes() failed to parse bundle for trust domain %q: %v", td, err) + } + bundleMap[td] = bundle + } + return bundleMap, nil +} + +// GetRootsFromSPIFFEBundleMap returns the root trust certificates from the +// SPIFFE bundle map for the given trust domain from the leaf certificate. +func GetRootsFromSPIFFEBundleMap(bundleMap map[string]*spiffebundle.Bundle, leafCert *x509.Certificate) (*x509.CertPool, error) { + // 1. Upon receiving a peer certificate, verify that it is a well-formed SPIFFE + // leaf certificate. In particular, it must have a single URI SAN containing + // a well-formed SPIFFE ID ([SPIFFE ID format]). + spiffeID, err := idFromCert(leafCert) + if err != nil { + return nil, fmt.Errorf("spiffe: could not get spiffe ID from peer leaf cert but verification with spiffe trust map was configured: %v", err) + } + + // 2. Use the trust domain in the peer certificate's SPIFFE ID to lookup + // the SPIFFE trust bundle. If the trust domain is not contained in the + // configured trust map, reject the certificate. + spiffeBundle, ok := bundleMap[spiffeID.TrustDomain().Name()] + if !ok { + return nil, fmt.Errorf("spiffe: no bundle found for peer certificates trust domain %q but verification with a SPIFFE trust map was configured", spiffeID.TrustDomain().Name()) + } + roots := spiffeBundle.X509Authorities() + rootPool := x509.NewCertPool() + for _, root := range roots { + rootPool.AddCert(root) + } + return rootPool, nil +} + +// idFromCert parses the SPIFFE ID from the x509.Certificate. If the certificate +// does not have a valid SPIFFE ID, returns an error. +func idFromCert(cert *x509.Certificate) (*spiffeid.ID, error) { + if cert == nil { + return nil, fmt.Errorf("input cert is nil") + } + // A valid SPIFFE Certificate should have exactly one URI. + if len(cert.URIs) != 1 { + return nil, fmt.Errorf("input cert has %v URIs but should have 1", len(cert.URIs)) + } + id, err := spiffeid.FromURI(cert.URIs[0]) + if err != nil { + return nil, fmt.Errorf("invalid spiffeid: %v", err) + } + return &id, nil +} diff --git a/internal/credentials/spiffe/spiffe_test.go b/internal/credentials/spiffe/spiffe_test.go new file mode 100644 index 0000000..fe1c3ae --- /dev/null +++ b/internal/credentials/spiffe/spiffe_test.go @@ -0,0 +1,246 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package spiffe + +import ( + "crypto/x509" + "encoding/pem" + "io" + "net/url" + "os" + "strings" + "testing" + + "google.golang.org/grpc/testdata" +) + +const wantURI = "spiffe://foo.bar.com/client/workload/1" + +func loadFileBytes(t *testing.T, filePath string) []byte { + bytes, err := os.ReadFile(filePath) + if err != nil { + t.Fatalf("Error reading file: %v", err) + } + return bytes +} + +func TestKnownSPIFFEBundle(t *testing.T) { + spiffeBundleFile := testdata.Path("spiffe/spiffebundle.json") + spiffeBundleBytes := loadFileBytes(t, spiffeBundleFile) + bundles, err := BundleMapFromBytes(spiffeBundleBytes) + if err != nil { + t.Fatalf("BundleMapFromBytes(%v) error during parsing: %v", spiffeBundleFile, err) + } + const wantBundleSize = 2 + if len(bundles) != wantBundleSize { + t.Fatalf("BundleMapFromBytes(%v) did not parse correct bundle length. got %v want %v", spiffeBundleFile, len(bundles), wantBundleSize) + } + if bundles["example.com"] == nil { + t.Fatalf("BundleMapFromBytes(%v) got no bundle for example.com", spiffeBundleFile) + } + if bundles["test.example.com"] == nil { + t.Fatalf("BundleMapFromBytes(%v) got no bundle for test.example.com", spiffeBundleFile) + } + + wantExampleComCert := loadX509Cert(t, testdata.Path("spiffe/spiffe_cert.pem")) + wantTestExampleComCert := loadX509Cert(t, testdata.Path("spiffe/server1_spiffe.pem")) + if !bundles["example.com"].X509Authorities()[0].Equal(wantExampleComCert) { + t.Errorf("BundleMapFromBytes(%v) parsed wrong cert for example.com.", spiffeBundleFile) + } + if !bundles["test.example.com"].X509Authorities()[0].Equal(wantTestExampleComCert) { + t.Errorf("BundleMapFromBytes(%v) parsed wrong cert for test.example.com", spiffeBundleFile) + } + +} + +func loadX509Cert(t *testing.T, filePath string) *x509.Certificate { + t.Helper() + certFile, _ := os.Open(filePath) + certRaw, _ := io.ReadAll(certFile) + block, _ := pem.Decode([]byte(certRaw)) + if block == nil { + t.Fatalf("pem.Decode(%v) = nil. Want a value.", certRaw) + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("x509.ParseCertificate(%v) failed %v", block.Bytes, err.Error()) + } + return cert +} + +func TestSPIFFEBundleMapFailures(t *testing.T) { + filePaths := []string{ + testdata.Path("spiffe/spiffebundle_corrupted_cert.json"), + testdata.Path("spiffe/spiffebundle_malformed.json"), + testdata.Path("spiffe/spiffebundle_wrong_kid.json"), + testdata.Path("spiffe/spiffebundle_wrong_kty.json"), + testdata.Path("spiffe/spiffebundle_wrong_multi_certs.json"), + testdata.Path("spiffe/spiffebundle_wrong_root.json"), + testdata.Path("spiffe/spiffebundle_wrong_seq_type.json"), + testdata.Path("spiffe/spiffebundle_invalid_trustdomain.json"), + testdata.Path("spiffe/spiffebundle_empty_string_key.json"), + testdata.Path("spiffe/spiffebundle_empty_keys.json"), + } + for _, path := range filePaths { + t.Run(path, func(t *testing.T) { + bundleBytes := loadFileBytes(t, path) + if _, err := BundleMapFromBytes(bundleBytes); err == nil { + t.Fatalf("BundleMapFromBytes(%v) did not fail but should have", path) + } + }) + } +} + +func TestSPIFFEBundleMapX509Failures(t *testing.T) { + // SPIFFE Bundles only support a use of x509-svid and jwt-svid. If a + // use other than this is specified, the parser does not fail, it + // just doesn't add an x509 authority or jwt authority to the bundle + filePath := testdata.Path("spiffe/spiffebundle_wrong_use.json") + bundleBytes := loadFileBytes(t, filePath) + bundle, err := BundleMapFromBytes(bundleBytes) + if err != nil { + t.Fatalf("BundleMapFromBytes(%v) failed with error: %v", filePath, err) + } + if len(bundle["example.com"].X509Authorities()) != 0 { + t.Fatalf("BundleMapFromBytes(%v) did not have empty bundle but should have", filePath) + } +} + +func TestGetRootsFromSPIFFEBundleMapSuccess(t *testing.T) { + bundleMapFile := testdata.Path("spiffe/spiffebundle_match_client_spiffe.json") + bundleBytes := loadFileBytes(t, bundleMapFile) + bundle, err := BundleMapFromBytes(bundleBytes) + if err != nil { + t.Fatalf("BundleMapFromBytes(%v) failed with error: %v", bundleMapFile, err) + } + + cert := loadX509Cert(t, testdata.Path("spiffe/client_spiffe.pem")) + gotRoots, err := GetRootsFromSPIFFEBundleMap(bundle, cert) + if err != nil { + t.Fatalf("GetRootsFromSPIFFEBundleMap() failed with err %v", err) + } + wantRoot := loadX509Cert(t, testdata.Path("spiffe/spiffe_cert.pem")) + wantRoots := x509.NewCertPool() + wantRoots.AddCert(wantRoot) + if !gotRoots.Equal(wantRoots) { + t.Fatalf("GetRootsFromSPIFFEBundleMap() got %v want %v", gotRoots, wantRoots) + } +} + +func TestGetRootsFromSPIFFEBundleMapFailures(t *testing.T) { + bundleMapFile := testdata.Path("spiffe/spiffebundle.json") + bundleBytes := loadFileBytes(t, bundleMapFile) + bundle, err := BundleMapFromBytes(bundleBytes) + certWithTwoURIs := loadX509Cert(t, testdata.Path("spiffe/client_spiffe.pem")) + certWithTwoURIs.URIs = append(certWithTwoURIs.URIs, certWithTwoURIs.URIs[0]) + certWithNoURIs := loadX509Cert(t, testdata.Path("spiffe/client_spiffe.pem")) + certWithNoURIs.URIs = nil + if err != nil { + t.Fatalf("BundleMapFromBytes(%v) failed with error: %v", bundleMapFile, err) + } + tests := []struct { + name string + bundleMapFile string + leafCert *x509.Certificate + wantErr string + }{ + { + name: "no bundle for peer cert spiffeID", + leafCert: loadX509Cert(t, testdata.Path("spiffe/client_spiffe.pem")), + wantErr: "no bundle found for peer certificates", + }, + { + name: "cert has invalid SPIFFE id", + leafCert: loadX509Cert(t, testdata.Path("ca.pem")), + wantErr: "could not get spiffe ID from peer leaf cert", + }, + { + name: "nil cert", + leafCert: nil, + wantErr: "input cert is nil", + }, + { + name: "cert has multiple URIs", + leafCert: certWithTwoURIs, + wantErr: "input cert has 2 URIs but should have 1", + }, + { + name: "cert has no URIs", + leafCert: certWithNoURIs, + wantErr: "input cert has 0 URIs but should have 1", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err = GetRootsFromSPIFFEBundleMap(bundle, tc.leafCert) + if err == nil { + t.Fatalf("GetRootsFromSPIFFEBundleMap() got no error but want error containing %v.", tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("GetRootsFromSPIFFEBundleMap() got error: %v. want error to contain %v", err, tc.wantErr) + } + }) + } +} + +func TestIDFromCert(t *testing.T) { + cert := loadX509Cert(t, testdata.Path("x509/spiffe_cert.pem")) + uri, err := idFromCert(cert) + if err != nil { + t.Fatalf("idFromCert() failed with err: %v", err) + } + if uri != nil && uri.String() != wantURI { + t.Fatalf("ID not expected, got %s, want %s", uri.String(), wantURI) + } +} + +func TestIDFromCertFileFailures(t *testing.T) { + certWithNoURIs := loadX509Cert(t, testdata.Path("spiffe/client_spiffe.pem")) + certWithNoURIs.URIs = nil + certWithInvalidSPIFFEID := loadX509Cert(t, testdata.Path("spiffe/client_spiffe.pem")) + certWithInvalidSPIFFEID.URIs = []*url.URL{{Path: "non-spiffe.bad"}} + tests := []struct { + name string + cert *x509.Certificate + }{ + { + name: "certificate with multiple URIs", + cert: loadX509Cert(t, testdata.Path("x509/multiple_uri_cert.pem")), + }, + { + name: "certificate with invalidSPIFFE ID", + cert: certWithInvalidSPIFFEID, + }, + { + name: "nil cert", + cert: nil, + }, + { + name: "cert with no URIs", + cert: certWithNoURIs, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := idFromCert(tt.cert); err == nil { + t.Fatalf("idFromCert() succeeded but want error") + } + }) + } +} diff --git a/internal/credentials/spiffe_test.go b/internal/credentials/spiffe_test.go new file mode 100644 index 0000000..0011ed0 --- /dev/null +++ b/internal/credentials/spiffe_test.go @@ -0,0 +1,233 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "net/url" + "os" + "testing" + + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/testdata" +) + +const wantURI = "spiffe://foo.bar.com/client/workload/1" + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestSPIFFEIDFromState(t *testing.T) { + tests := []struct { + name string + urls []*url.URL + // If we expect a SPIFFE ID to be returned. + wantID bool + }{ + { + name: "empty URIs", + urls: []*url.URL{}, + wantID: false, + }, + { + name: "good SPIFFE ID", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + }, + wantID: true, + }, + { + name: "invalid host", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: "", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + }, + wantID: false, + }, + { + name: "invalid path", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: "foo.bar.com", + Path: "", + RawPath: "", + }, + }, + wantID: false, + }, + { + name: "large path", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: "foo.bar.com", + Path: string(make([]byte, 2050)), + RawPath: string(make([]byte, 2050)), + }, + }, + wantID: false, + }, + { + name: "large host", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: string(make([]byte, 256)), + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + }, + wantID: false, + }, + { + name: "multiple URI SANs", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + { + Scheme: "spiffe", + Host: "bar.baz.com", + Path: "workload/wl2", + RawPath: "workload/wl2", + }, + { + Scheme: "https", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + }, + wantID: false, + }, + { + name: "multiple URI SANs without SPIFFE ID", + urls: []*url.URL{ + { + Scheme: "https", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + { + Scheme: "ssh", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + }, + wantID: false, + }, + { + name: "multiple URI SANs with one SPIFFE ID", + urls: []*url.URL{ + { + Scheme: "spiffe", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + { + Scheme: "https", + Host: "foo.bar.com", + Path: "workload/wl1", + RawPath: "workload/wl1", + }, + }, + wantID: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + state := tls.ConnectionState{PeerCertificates: []*x509.Certificate{{URIs: tt.urls}}} + id := SPIFFEIDFromState(state) + if got, want := id != nil, tt.wantID; got != want { + t.Errorf("want wantID = %v, but SPIFFE ID is %v", want, id) + } + }) + } +} + +func (s) TestSPIFFEIDFromCert(t *testing.T) { + tests := []struct { + name string + dataPath string + // If we expect a SPIFFE ID to be returned. + wantID bool + }{ + { + name: "good certificate with SPIFFE ID", + dataPath: "x509/spiffe_cert.pem", + wantID: true, + }, + { + name: "bad certificate with SPIFFE ID and another URI", + dataPath: "x509/multiple_uri_cert.pem", + wantID: false, + }, + { + name: "certificate without SPIFFE ID", + dataPath: "x509/client1_cert.pem", + wantID: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := os.ReadFile(testdata.Path(tt.dataPath)) + if err != nil { + t.Fatalf("os.ReadFile(%s) failed: %v", testdata.Path(tt.dataPath), err) + } + block, _ := pem.Decode(data) + if block == nil { + t.Fatalf("Failed to parse the certificate: byte block is nil") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("x509.ParseCertificate(%b) failed: %v", block.Bytes, err) + } + uri := SPIFFEIDFromCert(cert) + if (uri != nil) != tt.wantID { + t.Fatalf("wantID got and want mismatch, got %t, want %t", uri != nil, tt.wantID) + } + if uri != nil && uri.String() != wantURI { + t.Fatalf("SPIFFE ID not expected, got %s, want %s", uri.String(), wantURI) + } + }) + } +} diff --git a/internal/credentials/syscallconn.go b/internal/credentials/syscallconn.go new file mode 100644 index 0000000..2919632 --- /dev/null +++ b/internal/credentials/syscallconn.go @@ -0,0 +1,58 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials + +import ( + "net" + "syscall" +) + +type sysConn = syscall.Conn + +// syscallConn keeps reference of rawConn to support syscall.Conn for channelz. +// SyscallConn() (the method in interface syscall.Conn) is explicitly +// implemented on this type, +// +// Interface syscall.Conn is implemented by most net.Conn implementations (e.g. +// TCPConn, UnixConn), but is not part of net.Conn interface. So wrapper conns +// that embed net.Conn don't implement syscall.Conn. (Side note: tls.Conn +// doesn't embed net.Conn, so even if syscall.Conn is part of net.Conn, it won't +// help here). +type syscallConn struct { + net.Conn + // sysConn is a type alias of syscall.Conn. It's necessary because the name + // `Conn` collides with `net.Conn`. + sysConn +} + +// WrapSyscallConn tries to wrap rawConn and newConn into a net.Conn that +// implements syscall.Conn. rawConn will be used to support syscall, and newConn +// will be used for read/write. +// +// This function returns newConn if rawConn doesn't implement syscall.Conn. +func WrapSyscallConn(rawConn, newConn net.Conn) net.Conn { + sysConn, ok := rawConn.(syscall.Conn) + if !ok { + return newConn + } + return &syscallConn{ + Conn: newConn, + sysConn: sysConn, + } +} diff --git a/internal/credentials/syscallconn_test.go b/internal/credentials/syscallconn_test.go new file mode 100644 index 0000000..b229a47 --- /dev/null +++ b/internal/credentials/syscallconn_test.go @@ -0,0 +1,56 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials + +import ( + "net" + "syscall" + "testing" +) + +func (*syscallConn) SyscallConn() (syscall.RawConn, error) { + return nil, nil +} + +type nonSyscallConn struct { + net.Conn +} + +func (s) TestWrapSyscallConn(t *testing.T) { + sc := &syscallConn{} + nsc := &nonSyscallConn{} + + wrapConn := WrapSyscallConn(sc, nsc) + if _, ok := wrapConn.(syscall.Conn); !ok { + t.Errorf("returned conn (type %T) doesn't implement syscall.Conn, want implement", wrapConn) + } +} + +func (s) TestWrapSyscallConnNoWrap(t *testing.T) { + nscRaw := &nonSyscallConn{} + nsc := &nonSyscallConn{} + + wrapConn := WrapSyscallConn(nscRaw, nsc) + if _, ok := wrapConn.(syscall.Conn); ok { + t.Errorf("returned conn (type %T) implements syscall.Conn, want not implement", wrapConn) + } + if wrapConn != nsc { + t.Errorf("returned conn is %p, want %p (the passed-in newConn)", wrapConn, nsc) + } +} diff --git a/internal/credentials/util.go b/internal/credentials/util.go new file mode 100644 index 0000000..f792fd2 --- /dev/null +++ b/internal/credentials/util.go @@ -0,0 +1,52 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials + +import ( + "crypto/tls" +) + +const alpnProtoStrH2 = "h2" + +// AppendH2ToNextProtos appends h2 to next protos. +func AppendH2ToNextProtos(ps []string) []string { + for _, p := range ps { + if p == alpnProtoStrH2 { + return ps + } + } + ret := make([]string, 0, len(ps)+1) + ret = append(ret, ps...) + return append(ret, alpnProtoStrH2) +} + +// CloneTLSConfig returns a shallow clone of the exported +// fields of cfg, ignoring the unexported sync.Once, which +// contains a mutex and must not be copied. +// +// If cfg is nil, a new zero tls.Config is returned. +// +// TODO: inline this function if possible. +func CloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + + return cfg.Clone() +} diff --git a/internal/credentials/util_test.go b/internal/credentials/util_test.go new file mode 100644 index 0000000..69e01ee --- /dev/null +++ b/internal/credentials/util_test.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials + +import ( + "reflect" + "testing" +) + +func (s) TestAppendH2ToNextProtos(t *testing.T) { + tests := []struct { + name string + ps []string + want []string + }{ + { + name: "empty", + ps: nil, + want: []string{"h2"}, + }, + { + name: "only h2", + ps: []string{"h2"}, + want: []string{"h2"}, + }, + { + name: "with h2", + ps: []string{"alpn", "h2"}, + want: []string{"alpn", "h2"}, + }, + { + name: "no h2", + ps: []string{"alpn"}, + want: []string{"alpn", "h2"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := AppendH2ToNextProtos(tt.ps); !reflect.DeepEqual(got, tt.want) { + t.Errorf("AppendH2ToNextProtos() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/credentials/xds/handshake_info.go b/internal/credentials/xds/handshake_info.go new file mode 100644 index 0000000..dd4cdb9 --- /dev/null +++ b/internal/credentials/xds/handshake_info.go @@ -0,0 +1,469 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xds contains non-user facing functionality of the xds credentials. +package xds + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "strings" + "sync/atomic" + + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/credentials/spiffe" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/resolver" +) + +func init() { + internal.GetXDSHandshakeInfoForTesting = HandshakeInfoFromAttributes +} + +// handshakeAttrKey is the type used as the key to store HandshakeInfo in +// the Attributes field of resolver.Address. +type handshakeAttrKey struct{} + +// hostnameKey is the type used as the key to store the hostname in the +// Attributes field of resolver.Address. +type hostnameKey struct{} + +// SetAddressHostname returns a copy of addr in which the Attributes field is +// updated with the provided hostname. +func SetAddressHostname(addr resolver.Address, hostname string) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(hostnameKey{}, hostname) + return addr +} + +// Hostname returns the endpoint hostname stored in attr. +func Hostname(attr *attributes.Attributes) string { + if attr == nil { + return "" + } + v := attr.Value(hostnameKey{}) + hn, _ := v.(string) + return hn +} + +// SetHandshakeInfo returns a copy of addr in which the Attributes field is +// updated with hiPtr. +func SetHandshakeInfo(addr resolver.Address, hiPtr *atomic.Pointer[grpcsync.RefCounted[*HandshakeInfo]]) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(handshakeAttrKey{}, hiPtr) + return addr +} + +// HandshakeInfoFromAttributes returns the atomic pointer to the +// reference-counted HandshakeInfo stored in attr. +func HandshakeInfoFromAttributes(attr *attributes.Attributes) *atomic.Pointer[grpcsync.RefCounted[*HandshakeInfo]] { + v := attr.Value(handshakeAttrKey{}) + hi, _ := v.(*atomic.Pointer[grpcsync.RefCounted[*HandshakeInfo]]) + return hi +} + +// HandshakeInfo wraps all the security configuration required by client and +// server handshake methods in xds credentials. The xDS implementation will be +// responsible for populating these fields. +type HandshakeInfo struct { + // The fields below are initialized when the HandshakeInfo is created, and are + // not changed thereafter. + sanMatchers []matcher.StringMatcher // Only on the client side. + requireClientCert bool // Only on server side. + sni string // Only on client side, used for Server Name Indication in TLS handshake. + validateSANUsingSNI bool // Only on client side, indicates whether to perform validation of SANs based on SNI value. + useAutoHostSNI bool // Only on client side, indicates whether to use endpoint hostname as SNI. + + rootProvider certprovider.Provider + identityProvider certprovider.Provider +} + +// NewHandshakeInfo returns a new reference counted HandshakeInfo configured +// with the provided options. +func NewHandshakeInfo(rootProvider, identityProvider certprovider.Provider, sanMatchers []matcher.StringMatcher, sni string, requireClientCert, validateSANUsingSNI, useAutoHostSNI bool) *grpcsync.RefCounted[*HandshakeInfo] { + hi := &HandshakeInfo{ + rootProvider: rootProvider, + identityProvider: identityProvider, + sanMatchers: sanMatchers, + sni: sni, + requireClientCert: requireClientCert, + validateSANUsingSNI: validateSANUsingSNI, + useAutoHostSNI: useAutoHostSNI, + } + + return grpcsync.NewRefCounted(hi, hi.close) +} + +func (hi *HandshakeInfo) close() { + if hi.rootProvider != nil { + hi.rootProvider.Close() + } + if hi.identityProvider != nil { + hi.identityProvider.Close() + } +} + +// ClientSideTLSConfig loads the HandshakeInfo from hiPtr, marks it as in-use, +// and returns the tls.Config along with a done callback that MUST be invoked +// when the handshake completes. If no HandshakeInfo is stored in hiPtr or if +// fallback credentials should be used, useFallback returns true. +func ClientSideTLSConfig(ctx context.Context, hiPtr *atomic.Pointer[grpcsync.RefCounted[*HandshakeInfo]], hostname string) (cfg *tls.Config, useFallback bool, done func(), err error) { + if hiPtr == nil { + return nil, true, func() {}, nil + } + for { + hiRC := hiPtr.Load() + if hiRC == nil { + return nil, false, func() {}, errors.New("xds: connection closed or HandshakeInfo dead") + } + if !hiRC.TryIncrement() { + if hiPtr.Load() != hiRC { + continue + } + return nil, false, func() {}, errors.New("xds: connection closed or HandshakeInfo dead") + } + + hi := hiRC.Value() + if hi == nil || hi.UseFallbackCreds() { + hiRC.Decrement() + return nil, true, func() {}, nil + } + cfg, err := hi.clientSideTLSConfigInternal(ctx, hostname) + if err != nil { + hiRC.Decrement() + return nil, false, func() {}, err + } + return cfg, false, hiRC.Decrement, nil + } +} + +// ServerSideTLSConfig checks if hi is configured and marks it as in-use, +// returning the tls.Config along with a done callback that MUST be invoked when +// the handshake completes. If hi is nil or fallback credentials should be used, +// useFallback returns true. +func ServerSideTLSConfig(ctx context.Context, hiRC *grpcsync.RefCounted[*HandshakeInfo]) (cfg *tls.Config, useFallback bool, done func(), err error) { + if hiRC == nil { + return nil, true, func() {}, nil + } + hi := hiRC.Value() + if hi == nil || hi.UseFallbackCreds() { + return nil, true, func() {}, nil + } + if !hiRC.TryIncrement() { + return nil, false, func() {}, errors.New("xds: connection closed or HandshakeInfo dead") + } + cfg, err = hi.serverSideTLSConfigInternal(ctx) + if err != nil { + hiRC.Decrement() + return nil, false, func() {}, err + } + return cfg, false, hiRC.Decrement, nil +} + +// UseFallbackCreds returns true when fallback credentials are to be used based +// on the contents of the HandshakeInfo. +func (hi *HandshakeInfo) UseFallbackCreds() bool { + if hi == nil { + return true + } + return hi.identityProvider == nil && hi.rootProvider == nil +} + +// GetSANMatchersForTesting returns the SAN matchers stored in HandshakeInfo. +// To be used only for testing purposes. +func (hi *HandshakeInfo) GetSANMatchersForTesting() []matcher.StringMatcher { + return append([]matcher.StringMatcher{}, hi.sanMatchers...) +} + +// clientSideTLSConfigInternal constructs a tls.Config to be used in a +// client-side handshake based on the contents of the HandshakeInfo. +// +// hostname is passed as a parameter here instead of being part of the +// HandshakeInfo because HandshakeInfo contains cluster-level security +// configuration that applies to all endpoints in the cluster, while hostname is +// specific to each endpoint. This allows sharing a single HandshakeInfo +// instance across multiple endpoints in the same cluster. +func (hi *HandshakeInfo) clientSideTLSConfigInternal(ctx context.Context, hostname string) (*tls.Config, error) { + // On the client side, rootProvider is mandatory. IdentityProvider is + // optional based on whether the client is doing TLS or mTLS. + if hi.rootProvider == nil { + return nil, errors.New("xds: CertificateProvider to fetch trusted roots is missing, cannot perform TLS handshake. Please check configuration on the management server") + } + + // InsecureSkipVerify needs to be set to true because we need to perform + // custom verification to check the SAN on the received certificate. + // Currently the Go stdlib does complete verification of the cert (which + // includes hostname verification) or none. We are forced to go with the + // latter and perform the normal cert validation ourselves. + cfg := &tls.Config{ + InsecureSkipVerify: true, + NextProtos: []string{"h2"}, + } + + km, err := hi.rootProvider.KeyMaterial(ctx) + if err != nil { + return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err) + } + cfg.RootCAs = km.Roots + + // If AutoHostSNI is true, and the endpoint hostname is present, we use the + // endpoint hostname as the SNI value and also for SAN validation. + // Otherwise, we use the SNI value from HandshakeInfo (which is configured + // by the control plane) and validating SANs based on that. + sni := hi.sni + if hi.useAutoHostSNI && hostname != "" { + sni = hostname + } + + cfg.VerifyPeerCertificate = hi.buildVerifyFunc(km, true, sni) + + if hi.identityProvider != nil { + km, err := hi.identityProvider.KeyMaterial(ctx) + if err != nil { + return nil, fmt.Errorf("xds: fetching identity certificates from CertificateProvider failed: %v", err) + } + cfg.Certificates = km.Certs + } + + if envconfig.XDSSNIEnabled && sni != "" { + cfg.ServerName = sni + } + return cfg, nil +} + +func (hi *HandshakeInfo) buildVerifyFunc(km *certprovider.KeyMaterial, isClient bool, sni string) func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + return func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return fmt.Errorf("xds: no peer certificates presented") + } + // Parse all raw certificates presented by the peer. + var certs []*x509.Certificate + for _, rc := range rawCerts { + cert, err := x509.ParseCertificate(rc) + if err != nil { + return err + } + certs = append(certs, cert) + } + + // Build the intermediates list and verify that the leaf certificate is + // signed by one of the root certificates. If a SPIFFE Bundle Map is + // configured, it is used to get the root certs. Otherwise, the + // configured roots in the root provider are used. + intermediates := x509.NewCertPool() + for _, cert := range certs[1:] { + intermediates.AddCert(cert) + } + roots := km.Roots + // If a SPIFFE Bundle Map is configured, find the roots for the trust + // domain of the leaf certificate. + if km.SPIFFEBundleMap != nil { + var err error + roots, err = spiffe.GetRootsFromSPIFFEBundleMap(km.SPIFFEBundleMap, certs[0]) + if err != nil { + return err + } + } + opts := x509.VerifyOptions{ + Roots: roots, + Intermediates: intermediates, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + if isClient { + opts.KeyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} + } else { + opts.KeyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + } + if _, err := certs[0].Verify(opts); err != nil { + return err + } + + // If XDSSNIEnabled and AutoSNISANValidation are both true and the SNI is + // non-empty, validate only DNS SANs against the SNI. Otherwise, fallback to + // validating all received SANs against the control plane provided SAN + // matchers. + if envconfig.XDSSNIEnabled && hi.validateSANUsingSNI && sni != "" { + // Verify SAN of leaf certificate with SNI using exact DNS matcher. + for _, san := range certs[0].DNSNames { + if dnsMatch(sni, san) { + return nil + } + } + return fmt.Errorf("xds: received DNS SANs: %v do not match the SNI: %s", certs[0].DNSNames, sni) + } + // The SANs sent by the xDS control plane are encoded as SPIFFE IDs. We need to + // only look at the SANs on the leaf cert. + if cert := certs[0]; !hi.MatchingSANExists(cert) { + // TODO: Print the complete certificate once the x509 package + // supports a String() method on the Certificate type. + return fmt.Errorf("xds: received SANs {DNSNames: %v, EmailAddresses: %v, IPAddresses: %v, URIs: %v} do not match any of the accepted SANs", cert.DNSNames, cert.EmailAddresses, cert.IPAddresses, cert.URIs) + } + return nil + } +} + +// serverSideTLSConfigInternal constructs a tls.Config to be used in a +// server-side handshake based on the contents of the HandshakeInfo. +func (hi *HandshakeInfo) serverSideTLSConfigInternal(ctx context.Context) (*tls.Config, error) { + cfg := &tls.Config{ + ClientAuth: tls.NoClientCert, + NextProtos: []string{"h2"}, + } + // On the server side, identityProvider is mandatory. RootProvider is + // optional based on whether the server is doing TLS or mTLS. + if hi.identityProvider == nil { + return nil, errors.New("xds: CertificateProvider to fetch identity certificate is missing, cannot perform TLS handshake. Please check configuration on the management server") + } + if hi.requireClientCert { + cfg.ClientAuth = tls.RequireAndVerifyClientCert + } + + // identityProvider is mandatory on the server side. + km, err := hi.identityProvider.KeyMaterial(ctx) + if err != nil { + return nil, fmt.Errorf("xds: fetching identity certificates from CertificateProvider failed: %v", err) + } + cfg.Certificates = km.Certs + + if hi.rootProvider != nil { + km, err := hi.rootProvider.KeyMaterial(ctx) + if err != nil { + return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err) + } + if km.SPIFFEBundleMap != nil && hi.requireClientCert { + // ClientAuth, if set greater than tls.RequireAnyClientCert, must be + // dropped to tls.RequireAnyClientCert so that custom verification + // to use SPIFFE Bundles is done. + cfg.ClientAuth = tls.RequireAnyClientCert + cfg.VerifyPeerCertificate = hi.buildVerifyFunc(km, false, "") + } else { + cfg.ClientCAs = km.Roots + } + } + return cfg, nil +} + +// MatchingSANExists returns true if the SANs contained in cert match the +// criteria enforced by the list of SAN matchers in HandshakeInfo. +// +// If the list of SAN matchers in the HandshakeInfo is empty, this function +// returns true for all input certificates. +func (hi *HandshakeInfo) MatchingSANExists(cert *x509.Certificate) bool { + if len(hi.sanMatchers) == 0 { + return true + } + + // SANs can be specified in any of these four fields on the parsed cert. + for _, san := range cert.DNSNames { + if hi.matchSAN(san, true) { + return true + } + } + for _, san := range cert.EmailAddresses { + if hi.matchSAN(san, false) { + return true + } + } + for _, san := range cert.IPAddresses { + if hi.matchSAN(san.String(), false) { + return true + } + } + for _, san := range cert.URIs { + if hi.matchSAN(san.String(), false) { + return true + } + } + return false +} + +// Caller must hold mu. +func (hi *HandshakeInfo) matchSAN(san string, isDNS bool) bool { + for _, matcher := range hi.sanMatchers { + if em := matcher.ExactMatch(); em != "" && isDNS { + // This is a special case which is documented in the xDS protos. + // If the DNS SAN is a wildcard entry, and the match criteria is + // `exact`, then we need to perform DNS wildcard matching + // instead of regular string comparison. + if dnsMatch(em, san) { + return true + } + continue + } + if matcher.Match(san) { + return true + } + } + return false +} + +// dnsMatch implements a DNS wildcard matching algorithm based on RFC2828 and +// grpc-java's implementation in `OkHostnameVerifier` class. +// +// NOTE: Here the `host` argument is the one from the set of string matchers in +// the xDS proto and the `san` argument is a DNS SAN from the certificate, and +// this is the one which can potentially contain a wildcard pattern. +func dnsMatch(host, san string) bool { + // Add trailing "." and turn them into absolute domain names. + if !strings.HasSuffix(host, ".") { + host += "." + } + if !strings.HasSuffix(san, ".") { + san += "." + } + // Domain names are case-insensitive. + host = strings.ToLower(host) + san = strings.ToLower(san) + + // If san does not contain a wildcard, do exact match. + if !strings.Contains(san, "*") { + return host == san + } + + // Wildcard dns matching rules + // - '*' is only permitted in the left-most label and must be the only + // character in that label. For example, *.example.com is permitted, while + // *a.example.com, a*.example.com, a*b.example.com, a.*.example.com are + // not permitted. + // - '*' matches a single domain name component. For example, *.example.com + // matches test.example.com but does not match sub.test.example.com. + // - Wildcard patterns for single-label domain names are not permitted. + if san == "*." || !strings.HasPrefix(san, "*.") || strings.Contains(san[1:], "*") { + return false + } + // Optimization: at this point, we know that the san contains a '*' and + // is the first domain component of san. So, the host name must be at + // least as long as the san to be able to match. + if len(host) < len(san) { + return false + } + // Hostname must end with the non-wildcard portion of san. + if !strings.HasSuffix(host, san[1:]) { + return false + } + // At this point we know that the hostName and san share the same suffix + // (the non-wildcard portion of san). Now, we just need to make sure + // that the '*' does not match across domain components. + hostPrefix := strings.TrimSuffix(host, san[1:]) + return !strings.Contains(hostPrefix, ".") +} diff --git a/internal/credentials/xds/handshake_info_test.go b/internal/credentials/xds/handshake_info_test.go new file mode 100644 index 0000000..a514b9e --- /dev/null +++ b/internal/credentials/xds/handshake_info_test.go @@ -0,0 +1,668 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "net/netip" + "net/url" + "os" + "regexp" + "strings" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc/testdata" + + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal/credentials/spiffe" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/matcher" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestDNSMatch(t *testing.T) { + tests := []struct { + desc string + host string + pattern string + wantMatch bool + }{ + { + desc: "invalid wildcard 1", + host: "aa.example.com", + pattern: "*a.example.com", + wantMatch: false, + }, + { + desc: "invalid wildcard 2", + host: "aa.example.com", + pattern: "a*.example.com", + wantMatch: false, + }, + { + desc: "invalid wildcard 3", + host: "abc.example.com", + pattern: "a*c.example.com", + wantMatch: false, + }, + { + desc: "wildcard in one of the middle components", + host: "abc.test.example.com", + pattern: "abc.*.example.com", + wantMatch: false, + }, + { + desc: "single component wildcard", + host: "a.example.com", + pattern: "*", + wantMatch: false, + }, + { + desc: "short host name", + host: "a.com", + pattern: "*.example.com", + wantMatch: false, + }, + { + desc: "suffix mismatch", + host: "a.notexample.com", + pattern: "*.example.com", + wantMatch: false, + }, + { + desc: "wildcard match across components", + host: "sub.test.example.com", + pattern: "*.example.com.", + wantMatch: false, + }, + { + desc: "host doesn't end in period", + host: "test.example.com", + pattern: "test.example.com.", + wantMatch: true, + }, + { + desc: "pattern doesn't end in period", + host: "test.example.com.", + pattern: "test.example.com", + wantMatch: true, + }, + { + desc: "case insensitive", + host: "TEST.EXAMPLE.COM.", + pattern: "test.example.com.", + wantMatch: true, + }, + { + desc: "simple match", + host: "test.example.com", + pattern: "test.example.com", + wantMatch: true, + }, + { + desc: "good wildcard", + host: "a.example.com", + pattern: "*.example.com", + wantMatch: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + gotMatch := dnsMatch(test.host, test.pattern) + if gotMatch != test.wantMatch { + t.Fatalf("dnsMatch(%s, %s) = %v, want %v", test.host, test.pattern, gotMatch, test.wantMatch) + } + }) + } +} + +func (s) TestMatchingSANExists_FailureCases(t *testing.T) { + url1, err := url.Parse("http://golang.org") + if err != nil { + t.Fatalf("url.Parse() failed: %v", err) + } + url2, err := url.Parse("https://github.com/grpc/grpc-go") + if err != nil { + t.Fatalf("url.Parse() failed: %v", err) + } + inputCert := &x509.Certificate{ + DNSNames: []string{"foo.bar.example.com", "bar.baz.test.com", "*.example.com"}, + EmailAddresses: []string{"foobar@example.com", "barbaz@test.com"}, + IPAddresses: []net.IP{ + netip.MustParseAddr("192.0.0.1").AsSlice(), + netip.MustParseAddr("2001:db8::68").AsSlice(), + }, + URIs: []*url.URL{url1, url2}, + } + + tests := []struct { + desc string + sanMatchers []matcher.StringMatcher + }{ + { + desc: "exact match", + sanMatchers: []matcher.StringMatcher{ + matcher.NewExactStringMatcher("abcd.test.com", false), + matcher.NewExactStringMatcher("http://golang", false), + matcher.NewExactStringMatcher("HTTP://GOLANG.ORG", false), + }, + }, + { + desc: "prefix match", + sanMatchers: []matcher.StringMatcher{ + matcher.NewPrefixStringMatcher("i-aint-the-one", false), + matcher.NewPrefixStringMatcher("192.168.1.1", false), + matcher.NewPrefixStringMatcher("FOO.BAR", false), + }, + }, + { + desc: "suffix match", + sanMatchers: []matcher.StringMatcher{ + matcher.NewSuffixStringMatcher("i-aint-the-one", false), + matcher.NewSuffixStringMatcher("1::68", false), + matcher.NewSuffixStringMatcher(".COM", false), + }, + }, + { + desc: "regex match", + sanMatchers: []matcher.StringMatcher{ + matcher.NewRegexStringMatcher(regexp.MustCompile(`.*\.examples\.com`)), + matcher.NewRegexStringMatcher(regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`)), + }, + }, + { + desc: "contains match", + sanMatchers: []matcher.StringMatcher{ + matcher.NewContainsStringMatcher("i-aint-the-one", false), + matcher.NewContainsStringMatcher("2001:db8:1:1::68", false), + matcher.NewContainsStringMatcher("GRPC", false), + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + hi := NewHandshakeInfo(nil, nil, test.sanMatchers, "", false, false, false) + defer hi.Decrement() + + if hi.Value().MatchingSANExists(inputCert) { + t.Fatalf("hi.MatchingSANExists(%+v) with SAN matchers +%v succeeded when expected to fail", inputCert, test.sanMatchers) + } + }) + } +} + +func (s) TestMatchingSANExists_Success(t *testing.T) { + url1, err := url.Parse("http://golang.org") + if err != nil { + t.Fatalf("url.Parse() failed: %v", err) + } + url2, err := url.Parse("https://github.com/grpc/grpc-go") + if err != nil { + t.Fatalf("url.Parse() failed: %v", err) + } + inputCert := &x509.Certificate{ + DNSNames: []string{"baz.test.com", "*.example.com"}, + EmailAddresses: []string{"foobar@example.com", "barbaz@test.com"}, + IPAddresses: []net.IP{ + netip.MustParseAddr("192.0.0.1").AsSlice(), + netip.MustParseAddr("2001:db8::68").AsSlice(), + }, + URIs: []*url.URL{url1, url2}, + } + + tests := []struct { + desc string + sanMatchers []matcher.StringMatcher + }{ + { + desc: "no san matchers", + }, + { + desc: "exact match dns wildcard", + sanMatchers: []matcher.StringMatcher{ + matcher.NewPrefixStringMatcher("192.168.1.1", false), + matcher.NewExactStringMatcher("https://github.com/grpc/grpc-java", false), + matcher.NewExactStringMatcher("abc.example.com", false), + }, + }, + { + desc: "exact match ignore case", + sanMatchers: []matcher.StringMatcher{ + matcher.NewExactStringMatcher("FOOBAR@EXAMPLE.COM", true), + }, + }, + { + desc: "prefix match", + sanMatchers: []matcher.StringMatcher{ + matcher.NewSuffixStringMatcher(".co.in", false), + matcher.NewPrefixStringMatcher("192.168.1.1", false), + matcher.NewPrefixStringMatcher("baz.test", false), + }, + }, + { + desc: "prefix match ignore case", + sanMatchers: []matcher.StringMatcher{ + matcher.NewPrefixStringMatcher("BAZ.test", true), + }, + }, + { + desc: "suffix match", + sanMatchers: []matcher.StringMatcher{ + matcher.NewRegexStringMatcher(regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`)), + matcher.NewSuffixStringMatcher("192.168.1.1", false), + matcher.NewSuffixStringMatcher("@test.com", false), + }, + }, + { + desc: "suffix match ignore case", + sanMatchers: []matcher.StringMatcher{ + matcher.NewSuffixStringMatcher("@test.COM", true), + }, + }, + { + desc: "regex match", + sanMatchers: []matcher.StringMatcher{ + matcher.NewContainsStringMatcher("https://github.com/grpc/grpc-java", false), + matcher.NewRegexStringMatcher(regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`)), + matcher.NewRegexStringMatcher(regexp.MustCompile(`.*\.test\.com`)), + }, + }, + { + desc: "contains match", + sanMatchers: []matcher.StringMatcher{ + matcher.NewExactStringMatcher("https://github.com/grpc/grpc-java", false), + matcher.NewContainsStringMatcher("2001:68::db8", false), + matcher.NewContainsStringMatcher("192.0.0", false), + }, + }, + { + desc: "contains match ignore case", + sanMatchers: []matcher.StringMatcher{ + matcher.NewContainsStringMatcher("GRPC", true), + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + hi := NewHandshakeInfo(nil, nil, test.sanMatchers, "", false, false, false) + defer hi.Decrement() + + if !hi.Value().MatchingSANExists(inputCert) { + t.Fatalf("hi.MatchingSANExists(%+v) with SAN matchers +%v failed when expected to succeed", inputCert, test.sanMatchers) + } + }) + } +} + +// Equal reports whether the handshake info structs are identical. +func (hi *HandshakeInfo) Equal(other *HandshakeInfo) bool { + if hi == nil && other == nil { + return true + } + if hi == nil || other == nil { + return false + } + + r1, i1 := hi.rootProvider, hi.identityProvider + r2, i2 := other.rootProvider, other.identityProvider + + switch { + case r1 != r2 || i1 != i2: + return false + case hi.requireClientCert != other.requireClientCert: + return false + case hi.sni != other.sni: + return false + case hi.validateSANUsingSNI != other.validateSANUsingSNI: + return false + case hi.useAutoHostSNI != other.useAutoHostSNI: + return false + case len(hi.sanMatchers) != len(other.sanMatchers): + return false + } + for i := range hi.sanMatchers { + if !hi.sanMatchers[i].Equal(other.sanMatchers[i]) { + return false + } + } + return true +} + +type testCertProvider struct { + certprovider.Provider +} + +func (s) TestEqual(t *testing.T) { + tests := []struct { + desc string + hi1 *grpcsync.RefCounted[*HandshakeInfo] + hi2 *grpcsync.RefCounted[*HandshakeInfo] + wantMatch bool + }{ + { + desc: "both HandshakeInfo are nil", + hi1: func() *grpcsync.RefCounted[*HandshakeInfo] { + return grpcsync.NewRefCounted((*HandshakeInfo)(nil), func() {}) + }(), + hi2: func() *grpcsync.RefCounted[*HandshakeInfo] { + return grpcsync.NewRefCounted((*HandshakeInfo)(nil), func() {}) + }(), + wantMatch: true, + }, + { + desc: "one HandshakeInfo is nil", + hi1: func() *grpcsync.RefCounted[*HandshakeInfo] { + return grpcsync.NewRefCounted((*HandshakeInfo)(nil), func() {}) + }(), + hi2: NewHandshakeInfo(&testCertProvider{}, nil, nil, "", false, false, false), + wantMatch: false, + }, + { + desc: "different root providers", + hi1: NewHandshakeInfo(&testCertProvider{}, nil, nil, "", false, false, false), + hi2: NewHandshakeInfo(&testCertProvider{}, nil, nil, "", false, false, false), + wantMatch: false, + }, + { + desc: "same providers, same SAN matchers", + hi1: NewHandshakeInfo(testCertProvider{}, testCertProvider{}, []matcher.StringMatcher{ + matcher.NewExactStringMatcher("foo.com", false), + }, "", false, false, false), + hi2: NewHandshakeInfo(testCertProvider{}, testCertProvider{}, []matcher.StringMatcher{ + matcher.NewExactStringMatcher("foo.com", false), + }, "", false, false, false), + wantMatch: true, + }, + { + desc: "same providers, different SAN matchers", + hi1: NewHandshakeInfo(testCertProvider{}, testCertProvider{}, []matcher.StringMatcher{ + matcher.NewExactStringMatcher("foo.com", false), + }, "", false, false, false), + hi2: NewHandshakeInfo(testCertProvider{}, testCertProvider{}, []matcher.StringMatcher{ + matcher.NewExactStringMatcher("bar.com", false), + }, "", false, false, false), + wantMatch: false, + }, + { + desc: "same SAN matchers with different content", + hi1: NewHandshakeInfo(testCertProvider{}, testCertProvider{}, []matcher.StringMatcher{ + matcher.NewExactStringMatcher("foo.com", false), + }, "", false, false, false), + hi2: NewHandshakeInfo(testCertProvider{}, testCertProvider{}, []matcher.StringMatcher{ + matcher.NewExactStringMatcher("foo.com", false), + matcher.NewExactStringMatcher("bar.com", false), + }, "", false, false, false), + wantMatch: false, + }, + { + desc: "different requireClientCert flags", + hi1: NewHandshakeInfo(testCertProvider{}, testCertProvider{}, nil, "", true, false, false), + hi2: NewHandshakeInfo(testCertProvider{}, testCertProvider{}, nil, "", false, false, false), + wantMatch: false, + }, + { + desc: "same identity provider, different root provider", + hi1: NewHandshakeInfo(&testCertProvider{}, testCertProvider{}, nil, "", false, false, false), + hi2: NewHandshakeInfo(&testCertProvider{}, testCertProvider{}, nil, "", false, false, false), + wantMatch: false, + }, + { + desc: "different identity provider, same root provider", + hi1: NewHandshakeInfo(testCertProvider{}, &testCertProvider{}, nil, "", false, false, false), + hi2: NewHandshakeInfo(testCertProvider{}, &testCertProvider{}, nil, "", false, false, false), + wantMatch: false, + }, + { + desc: "same sni and validateSANUsingSNI", + hi1: NewHandshakeInfo(nil, nil, nil, "sni", false, true, false), + hi2: NewHandshakeInfo(nil, nil, nil, "sni", false, true, false), + wantMatch: true, + }, + { + desc: "different sni", + hi1: NewHandshakeInfo(nil, nil, nil, "sni1", false, false, false), + hi2: NewHandshakeInfo(nil, nil, nil, "sni2", false, false, false), + wantMatch: false, + }, + { + desc: "same sni, different validateSANUsingSNI", + hi1: NewHandshakeInfo(nil, nil, nil, "sni", false, false, false), + hi2: NewHandshakeInfo(nil, nil, nil, "sni", false, true, false), + wantMatch: false, + }, + { + desc: "different autoHostSNI", + hi1: NewHandshakeInfo(nil, nil, nil, "", false, false, true), + hi2: NewHandshakeInfo(nil, nil, nil, "", false, false, false), + wantMatch: false, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if gotMatch := test.hi1.Value().Equal(test.hi2.Value()); gotMatch != test.wantMatch { + t.Errorf("hi1.Equal(hi2) = %v; wantMatch %v", gotMatch, test.wantMatch) + } + }) + } +} + +type testCertProviderWithKeyMaterial struct{} + +func (p *testCertProviderWithKeyMaterial) KeyMaterial(_ context.Context) (*certprovider.KeyMaterial, error) { + km := &certprovider.KeyMaterial{} + spiffeBundleMapContents, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffebundle.json")) + if err != nil { + return nil, err + } + bundleMap, err := spiffe.BundleMapFromBytes(spiffeBundleMapContents) + if err != nil { + return nil, err + } + km.SPIFFEBundleMap = bundleMap + rootFileContents, err := os.ReadFile(testdata.Path("spiffe_end2end/ca.pem")) + if err != nil { + return nil, err + } + trustPool := x509.NewCertPool() + if !trustPool.AppendCertsFromPEM(rootFileContents) { + return nil, fmt.Errorf("Failed to parse root certificate") + } + km.Roots = trustPool + + certFileContents, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffe.pem")) + if err != nil { + return nil, err + } + keyFileContents, err := os.ReadFile(testdata.Path("spiffe_end2end/client.key")) + if err != nil { + return nil, err + } + cert, err := tls.X509KeyPair(certFileContents, keyFileContents) + if err != nil { + return nil, err + } + km.Certs = []tls.Certificate{cert} + return km, nil +} + +func (p *testCertProviderWithKeyMaterial) Close() {} + +func (s) TestBuildVerifyFuncFailures(t *testing.T) { + tests := []struct { + desc string + peerCertChain [][]byte + wantErr string + }{ + { + desc: "invalid x509", + peerCertChain: [][]byte{[]byte("NOT_A_CERT")}, + wantErr: "x509: malformed certificate", + }, + { + desc: "invalid SPIFFE ID in peer cert", + // server1.pem doesn't have a valid SPIFFE ID, so attempted to get a + // root from the SPIFFE Bundle Map will fail + peerCertChain: loadCert(t, testdata.Path("server1.pem"), testdata.Path("server1.key")), + wantErr: "spiffe: could not get spiffe ID from peer leaf cert but verification with spiffe trust map was configure", + }, + } + testProvider := testCertProviderWithKeyMaterial{} + hi := NewHandshakeInfo(&testProvider, &testProvider, nil, "", true, false, false) + defer hi.Decrement() + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + hiPtr := new(atomic.Pointer[grpcsync.RefCounted[*HandshakeInfo]]) + hiPtr.Store(hi) + cfg, _, done, err := ClientSideTLSConfig(ctx, hiPtr, "") + if err != nil { + t.Fatalf("GetClientSideTLSConfig() failed with err %v", err) + } + defer done() + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + err = cfg.VerifyPeerCertificate(tc.peerCertChain, nil) + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("VerifyPeerCertificate got err %v, want: %v", err, tc.wantErr) + } + }) + } +} + +func loadCert(t *testing.T, certPath, keyPath string) [][]byte { + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + t.Fatalf("LoadX509KeyPair(%s, %s) failed: %v", certPath, keyPath, err) + } + return cert.Certificate + +} + +type testProviderWithRoots struct { + roots *x509.CertPool +} + +func (m *testProviderWithRoots) KeyMaterial(context.Context) (*certprovider.KeyMaterial, error) { + return &certprovider.KeyMaterial{ + Roots: m.roots, + }, nil +} + +func (m *testProviderWithRoots) Close() {} + +// Tests the scenario where the SNI provided by the xDS server is only present +// in the URI SAN of the certificate presented by the server, and not in the DNS +// SAN. It verifies that if the SNI-based SAN validation is turned on, the TLS +// handshake fails due to SNI mismatch. +func (s) TestAutoHostSNI_DNS_SANValidation_Failures(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true) + + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("rsa.GenerateKey() failed: %v", err) + } + + testURL, err := url.Parse("http://wrong.sni.domain") + if err != nil { + t.Fatalf("url.Parse() failed: %v", err) + } + + template := &x509.Certificate{ + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour), + DNSNames: []string{"unrelated.example.com"}, + URIs: []*url.URL{testURL}, // URI matching SNI + } + + derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv) + if err != nil { + t.Fatalf("x509.CreateCertificate() failed: %v", err) + } + + cert, err := x509.ParseCertificate(derBytes) + if err != nil { + t.Fatalf("x509.ParseCertificate() failed: %v", err) + } + + roots := x509.NewCertPool() + roots.AddCert(cert) + + provider := &testProviderWithRoots{roots: roots} + hi := NewHandshakeInfo(provider, nil, nil, "wrong.sni.domain", true, true, false) + defer hi.Decrement() + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + hiPtr := new(atomic.Pointer[grpcsync.RefCounted[*HandshakeInfo]]) + hiPtr.Store(hi) + cfg, _, done, err := ClientSideTLSConfig(ctx, hiPtr, "") + if err != nil { + t.Fatalf("GetClientSideTLSConfig() failed: %v", err) + } + defer done() + + err = cfg.VerifyPeerCertificate([][]byte{derBytes}, nil) + if err == nil { + t.Fatalf("VerifyPeerCertificate() unexpectedly succeeded when SNI was only in URI SAN") + } + if !strings.Contains(err.Error(), "received DNS SANs: [unrelated.example.com] do not match the SNI: wrong.sni.domain") { + t.Fatalf("VerifyPeerCertificate() failed with %v, want error containing 'do not match the SNI'", err) + } +} + +func (s) TestVerifyPeerCertificateZeroCerts(t *testing.T) { + provider := &testProviderWithRoots{roots: x509.NewCertPool()} + hi := NewHandshakeInfo(provider, nil, nil, "", true, false, false) + defer hi.Decrement() + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + hiPtr := new(atomic.Pointer[grpcsync.RefCounted[*HandshakeInfo]]) + hiPtr.Store(hi) + cfg, _, done, err := ClientSideTLSConfig(ctx, hiPtr, "") + if err != nil { + t.Fatalf("GetClientSideTLSConfig() failed: %v", err) + } + defer done() + + const wantErr = "no peer certificates presented" + if err := cfg.VerifyPeerCertificate(nil, nil); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Errorf("VerifyPeerCertificate(nil, nil) = %v, want error containing %q", err, wantErr) + } +} diff --git a/internal/envconfig/envconfig.go b/internal/envconfig/envconfig.go new file mode 100644 index 0000000..dc6fc0c --- /dev/null +++ b/internal/envconfig/envconfig.go @@ -0,0 +1,244 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package envconfig contains grpc settings configured by environment variables. +package envconfig + +import ( + "os" + "strconv" + "strings" +) + +var ( + // EnableTXTServiceConfig is set if the DNS resolver should perform TXT + // lookups for service config ("GRPC_ENABLE_TXT_SERVICE_CONFIG" is not + // "false"). + EnableTXTServiceConfig = boolFromEnv("GRPC_ENABLE_TXT_SERVICE_CONFIG", true) + + // TXTErrIgnore is set if TXT errors should be ignored + // ("GRPC_GO_IGNORE_TXT_ERRORS" is not "false"). + TXTErrIgnore = boolFromEnv("GRPC_GO_IGNORE_TXT_ERRORS", true) + + // RingHashCap indicates the maximum ring size which defaults to 4096 + // entries but may be overridden by setting the environment variable + // "GRPC_RING_HASH_CAP". This does not override the default bounds + // checking which NACKs configs specifying ring sizes > 8*1024*1024 (~8M). + RingHashCap = uint64FromEnv("GRPC_RING_HASH_CAP", 4096, 1, 8*1024*1024) + + // ALTSMaxConcurrentHandshakes is the maximum number of concurrent ALTS + // handshakes that can be performed. + ALTSMaxConcurrentHandshakes = uint64FromEnv("GRPC_ALTS_MAX_CONCURRENT_HANDSHAKES", 100, 1, 100) + + // EnforceALPNEnabled is set if TLS connections to servers with ALPN disabled + // should be rejected. The HTTP/2 protocol requires ALPN to be enabled, this + // option is present for backward compatibility. This option may be overridden + // by setting the environment variable "GRPC_ENFORCE_ALPN_ENABLED" to "true" + // or "false". + EnforceALPNEnabled = boolFromEnv("GRPC_ENFORCE_ALPN_ENABLED", true) + + // XDSEndpointHashKeyBackwardCompat controls the parsing of the endpoint hash + // key from EDS LbEndpoint metadata. Endpoint hash keys can be disabled by + // setting "GRPC_XDS_ENDPOINT_HASH_KEY_BACKWARD_COMPAT" to "true". A future + // release will remove this environment variable, enabling the new behavior + // unconditionally. + XDSEndpointHashKeyBackwardCompat = boolFromEnv("GRPC_XDS_ENDPOINT_HASH_KEY_BACKWARD_COMPAT", false) + + // LabelServerGoroutines controls setting [runtime/pprof.Labels] on the + // goroutines spawned by [grpc.Server] type. + // For now, this is limited to the goroutines spawned to handle incoming + // requests on the server. + // Set "GRPC_GO_SERVER_GOROUTINE_LABELS" to "grpc.method=true" to + // enable this grpc.method label, or "all" to enable all valid labels. + // This variable is a bit-field. + LabelServerGoroutines = goroutineLabelsFromEnv("GRPC_GO_SERVER_GOROUTINE_LABELS", 0) + + // RingHashSetRequestHashKey is set if the ring hash balancer can get the + // request hash header by setting the "requestHashHeader" field, according + // to gRFC A76. It can be disabled by setting the environment variable + // "GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY" to "false". + RingHashSetRequestHashKey = boolFromEnv("GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY", true) + + // ALTSHandshakerKeepaliveParams is set if we should add the + // KeepaliveParams when dial the ALTS handshaker service. + ALTSHandshakerKeepaliveParams = boolFromEnv("GRPC_EXPERIMENTAL_ALTS_HANDSHAKER_KEEPALIVE_PARAMS", false) + + // EnableDefaultPortForProxyTarget controls whether the resolver adds a default port 443 + // to a target address that lacks one. This flag only has an effect when all of + // the following conditions are met: + // - A connect proxy is being used. + // - Target resolution is disabled. + // - The DNS resolver is being used. + EnableDefaultPortForProxyTarget = boolFromEnv("GRPC_EXPERIMENTAL_ENABLE_DEFAULT_PORT_FOR_PROXY_TARGET", true) + + // CaseSensitiveBalancerRegistries is set if the balancer registry should be + // case-sensitive. This is enabled by default, but can be disabled by setting + // the env variable "GRPC_GO_EXPERIMENTAL_CASE_SENSITIVE_BALANCER_REGISTRIES" + // to "false". + // + // This env varible will be removed in release v1.82.0. + CaseSensitiveBalancerRegistries = boolFromEnv("GRPC_GO_EXPERIMENTAL_CASE_SENSITIVE_BALANCER_REGISTRIES", true) + + // XDSAuthorityRewrite indicates whether xDS authority rewriting is enabled. + // This feature is defined in gRFC A81 and is enabled by setting the + // environment variable GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE to "true". + XDSAuthorityRewrite = boolFromEnv("GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE", false) + + // PickFirstWeightedShuffling indicates whether weighted endpoint shuffling + // is enabled in the pick_first LB policy, as defined in gRFC A113. This + // feature can be disabled by setting the environment variable + // GRPC_EXPERIMENTAL_PF_WEIGHTED_SHUFFLING to "false". + PickFirstWeightedShuffling = boolFromEnv("GRPC_EXPERIMENTAL_PF_WEIGHTED_SHUFFLING", true) + + // XDSRecoverPanicInResourceParsing indicates whether the xdsclient should + // recover from panics while parsing xDS resources. + // + // This feature can be disabled (e.g. for fuzz testing) by setting the + // environment variable "GRPC_GO_EXPERIMENTAL_XDS_RESOURCE_PANIC_RECOVERY" + // to "false". + XDSRecoverPanicInResourceParsing = boolFromEnv("GRPC_GO_EXPERIMENTAL_XDS_RESOURCE_PANIC_RECOVERY", true) + + // EnablePriorityLBChildPolicyCache controls whether the priority balancer + // should cache child balancers that are removed from the LB policy config, + // for a period of 15 minutes. This is disabled by default, but can be + // enabled by setting the env variable + // GRPC_EXPERIMENTAL_ENABLE_PRIORITY_LB_CHILD_POLICY_CACHE to true. + EnablePriorityLBChildPolicyCache = boolFromEnv("GRPC_EXPERIMENTAL_ENABLE_PRIORITY_LB_CHILD_POLICY_CACHE", false) + + // Enable8KBDefaultHeaderListSize indicates that default maximum header list + // size is restricted to 8KB. This is disabled by default, but can be enabled + // by setting the environment variable + // "GRPC_GO_EXPERIMENTAL_ENABLE_8KB_DEFAULT_HEADER_LIST_SIZE" to "true". + // When disabled, the default maximum header list size of 16MB is used. + // + // When enabled, RPCs with a total size of headers exceeding 8KB will fail + // unless explicitly configured otherwise by the user. + // + // TODO: In release v1.82.0, env var will be enabled by default. + Enable8KBDefaultHeaderListSize = boolFromEnv("GRPC_GO_EXPERIMENTAL_ENABLE_8KB_DEFAULT_HEADER_LIST_SIZE", false) + + // EnableHTTPFramerReadBufferPooling enables the use of the + // readyreader.Reader interface to perform non-memory-pinning reads, + // provided the underlying net.Conn supports it. This reduces memory usage + // when subchannels are idle. + // + // This environment variable serves as an escape hatch to disable the + // feature if unforeseen issues arise, and it will be removed in a future + // release. + EnableHTTPFramerReadBufferPooling = boolFromEnv("GRPC_GO_EXPERIMENTAL_HTTP_FRAMER_READ_BUFFER_POOLING", true) + + // ControlBufferThrottleLimit is the maximum number of control frames that can + // be queued in the control buffer before throttling is applied. The value + // must be between 1 and 10,000, and is set to 100 by default. + // + // This environment variable serves as an escape hatch to increase the + // throttling limit if unforeseen issues arise, and it will be removed in a + // future release. + // + // TODO: Remove this env var once v1.83.0 is released. + ControlBufferThrottleLimit = uint64FromEnv("GRPC_GO_EXPERIMENTAL_CONTROL_BUFFER_THROTTLE_LIMIT", 100, 1, 10000) + + // ALTSMaxFrameSize is the maximum frame size for ALTS in bytes. + // This can be overridden by setting the environment variable + // "GRPC_GO_EXPERIMENTAL_ALTS_MAX_FRAME_SIZE" (min: 4096, max: 512*1024, capped at + // 512KiB to match altsWriteBufferMaxSize in + // credentials/alts/internal/conn/record.go). + ALTSMaxFrameSize = uint64FromEnv("GRPC_GO_EXPERIMENTAL_ALTS_MAX_FRAME_SIZE", 4096, 4096, 512*1024) + + // EnableReceiveBufferCompaction enables the compaction of data buffers + // to reduce the number of buffers in the receive buffer. + // + // This environment variable serves as an escape hatch to disable the + // feature if unforeseen issues arise, and it will be removed in a future + // release. + // + // TODO: Remove this env var once v1.85.0 is released. + EnableReceiveBufferCompaction = boolFromEnv("GRPC_GO_EXPERIMENTAL_ENABLE_RECEIVE_BUFFER_COMPACTION", true) +) + +func boolFromEnv(envVar string, def bool) bool { + if def { + // The default is true; return true unless the variable is "false". + return !strings.EqualFold(os.Getenv(envVar), "false") + } + // The default is false; return false unless the variable is "true". + return strings.EqualFold(os.Getenv(envVar), "true") +} + +func uint64FromEnv(envVar string, def, min, max uint64) uint64 { + v, err := strconv.ParseUint(os.Getenv(envVar), 10, 64) + if err != nil { + return def + } + if v < min { + return min + } + if v > max { + return max + } + return v +} + +// GoroutineLabels is a bitfield indicating which goroutine labels are enabled. +type GoroutineLabels uint16 + +func goroutineLabelsFromEnv(envVar string, def GoroutineLabels) GoroutineLabels { + val := def + v := os.Getenv(envVar) + if strings.EqualFold(v, "all") { + return AllGoroutineLabels + } else if strings.EqualFold(v, "none") { + return 0 + } + for s := range strings.SplitSeq(v, ",") { + s = strings.TrimSpace(s) + if len(s) == 0 { + continue + } + pre, post, ok := strings.Cut(s, "=") + if !ok { + // no equals sign + continue + } + post = strings.TrimSpace(post) + pre = strings.TrimSpace(pre) + bitDesignator := GoroutineLabels(0) + switch { + case strings.EqualFold(pre, "grpc.method"): + bitDesignator = GoroutineLabelServerMethod + default: + continue + } + if strings.EqualFold(post, "true") { + val |= bitDesignator + } else if strings.EqualFold(post, "false") { + val &^= bitDesignator + } + } + return val +} + +const ( + // GoroutineLabelServerMethod sets the grpc.method label on new + // server-side gRPC streams. + GoroutineLabelServerMethod GoroutineLabels = 1 << iota +) + +// AllGoroutineLabels is an or'd together bitfield of all valid GoroutineLabels +// constant values (above). +const AllGoroutineLabels = GoroutineLabelServerMethod diff --git a/internal/envconfig/envconfig_test.go b/internal/envconfig/envconfig_test.go new file mode 100644 index 0000000..4382860 --- /dev/null +++ b/internal/envconfig/envconfig_test.go @@ -0,0 +1,218 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package envconfig + +import ( + "os" + "testing" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestUint64FromEnv(t *testing.T) { + var testCases = []struct { + name string + val string + def, min, max uint64 + want uint64 + }{ + { + name: "error parsing", + val: "asdf", def: 5, want: 5, + }, { + name: "unset", + val: "", def: 5, want: 5, + }, { + name: "too low", + val: "5", min: 10, want: 10, + }, { + name: "too high", + val: "5", max: 2, want: 2, + }, { + name: "in range", + val: "17391", def: 13000, min: 12000, max: 18000, want: 17391, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + const testVar = "testvar" + if tc.val == "" { + os.Unsetenv(testVar) + } else { + os.Setenv(testVar, tc.val) + } + if got := uint64FromEnv(testVar, tc.def, tc.min, tc.max); got != tc.want { + t.Errorf("uint64FromEnv(%q(=%q), %v, %v, %v) = %v; want %v", testVar, tc.val, tc.def, tc.min, tc.max, got, tc.want) + } + }) + } +} + +func (s) TestBoolFromEnv(t *testing.T) { + var testCases = []struct { + val string + def bool + want bool + }{ + {val: "", def: true, want: true}, + {val: "", def: false, want: false}, + {val: "true", def: true, want: true}, + {val: "true", def: false, want: true}, + {val: "false", def: true, want: false}, + {val: "false", def: false, want: false}, + {val: "asdf", def: true, want: true}, + {val: "asdf", def: false, want: false}, + } + for _, tc := range testCases { + t.Run("", func(t *testing.T) { + const testVar = "testvar" + if tc.val == "" { + os.Unsetenv(testVar) + } else { + os.Setenv(testVar, tc.val) + } + if got := boolFromEnv(testVar, tc.def); got != tc.want { + t.Errorf("boolFromEnv(%q(=%q), %v) = %v; want %v", testVar, tc.val, tc.def, got, tc.want) + } + }) + } +} + +func (s) TestGoroutineLabelsFromEnv(t *testing.T) { + var testCases = []struct { + name string + val string + def GoroutineLabels + want GoroutineLabels + }{ + { + name: "unset_env_non-zero_default", + val: "", + def: GoroutineLabelServerMethod, + want: GoroutineLabelServerMethod, + }, { + name: "unset_env_zero_default", + val: "", + def: 0, + want: 0, + }, { + name: "force-enable_zero_default", + val: "grpc.method=true", + def: 0, + want: GoroutineLabelServerMethod, + }, { + name: "force-enable_zero_default_all_caps", + val: "grpc.method=TRUE", + def: 0, + want: GoroutineLabelServerMethod, + }, { + name: "force-enable_zero_default_with_whitespace", + val: " grpc.method\t= true", + def: 0, + want: GoroutineLabelServerMethod, + }, { + name: "force-enable_zero_default_with_other_garbage", + val: "grpc.method=true,foobar", + def: 0, + want: GoroutineLabelServerMethod, + }, { + name: "force-enable_mixed_case_zero_default_with_other_garbage", + val: "grpc.method=tRuE,foobar", + def: 0, + want: GoroutineLabelServerMethod, + }, { + name: "force-disable_zero_default", + val: "grpc.method=false", + def: 0, + want: 0, + }, { + name: "force-disable_non-zero_default", + val: "grpc.method=false", + def: GoroutineLabelServerMethod, + want: 0, + }, { + name: "force-disable_non-zero_default_all_caps", + val: "grpc.method=FALSE", + def: GoroutineLabelServerMethod, + want: 0, + }, { + name: "force-disable_non-zero_default_mixed_case", + val: "grpc.method=fAlSe", + def: GoroutineLabelServerMethod, + want: 0, + }, { + name: "unknown_val_no_equal", + val: "grpc.unknown.garbage", + def: GoroutineLabelServerMethod, + want: GoroutineLabelServerMethod, + }, { + name: "unknown_val", + val: "grpc.unknown.garbage=fooble", + def: GoroutineLabelServerMethod, + want: GoroutineLabelServerMethod, + }, { + name: "all_with_empty_default", + val: "all", + def: 0, + want: AllGoroutineLabels, + }, { + name: "all_with_server_method_default", + val: "all", + def: GoroutineLabelServerMethod, + want: AllGoroutineLabels, + }, { + name: "none_with_empty_default", + val: "none", + def: 0, + want: 0, + }, { + name: "none_with_server_method_default", + val: "none", + def: GoroutineLabelServerMethod, + want: 0, + }, { + name: "unparseable_rhs", + val: "grpc.method=quux", + def: GoroutineLabelServerMethod, + want: GoroutineLabelServerMethod, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + const testVar = "testvar" + if tc.val == "" { + os.Unsetenv(testVar) + } else { + os.Setenv(testVar, tc.val) + } + if got := goroutineLabelsFromEnv(testVar, tc.def); got != tc.want { + t.Errorf("goroutineLabelsFromEnv(%q(=%q), %v) = %v; want %v", testVar, tc.val, tc.def, got, tc.want) + } + }) + } + +} diff --git a/internal/envconfig/observability.go b/internal/envconfig/observability.go new file mode 100644 index 0000000..dd314cf --- /dev/null +++ b/internal/envconfig/observability.go @@ -0,0 +1,42 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package envconfig + +import "os" + +const ( + envObservabilityConfig = "GRPC_GCP_OBSERVABILITY_CONFIG" + envObservabilityConfigFile = "GRPC_GCP_OBSERVABILITY_CONFIG_FILE" +) + +var ( + // ObservabilityConfig is the json configuration for the gcp/observability + // package specified directly in the envObservabilityConfig env var. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + ObservabilityConfig = os.Getenv(envObservabilityConfig) + // ObservabilityConfigFile is the json configuration for the + // gcp/observability specified in a file with the location specified in + // envObservabilityConfigFile env var. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + ObservabilityConfigFile = os.Getenv(envObservabilityConfigFile) +) diff --git a/internal/envconfig/xds.go b/internal/envconfig/xds.go new file mode 100644 index 0000000..5ec39bd --- /dev/null +++ b/internal/envconfig/xds.go @@ -0,0 +1,106 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package envconfig + +import ( + "os" +) + +const ( + // XDSBootstrapFileNameEnv is the env variable to set bootstrap file name. + // Do not use this and read from env directly. Its value is read and kept in + // variable XDSBootstrapFileName. + // + // When both bootstrap FileName and FileContent are set, FileName is used. + XDSBootstrapFileNameEnv = "GRPC_XDS_BOOTSTRAP" + // XDSBootstrapFileContentEnv is the env variable to set bootstrap file + // content. Do not use this and read from env directly. Its value is read + // and kept in variable XDSBootstrapFileContent. + // + // When both bootstrap FileName and FileContent are set, FileName is used. + XDSBootstrapFileContentEnv = "GRPC_XDS_BOOTSTRAP_CONFIG" +) + +var ( + // XDSBootstrapFileName holds the name of the file which contains xDS + // bootstrap configuration. Users can specify the location of the bootstrap + // file by setting the environment variable "GRPC_XDS_BOOTSTRAP". + // + // When both bootstrap FileName and FileContent are set, FileName is used. + XDSBootstrapFileName = os.Getenv(XDSBootstrapFileNameEnv) + // XDSBootstrapFileContent holds the content of the xDS bootstrap + // configuration. Users can specify the bootstrap config by setting the + // environment variable "GRPC_XDS_BOOTSTRAP_CONFIG". + // + // When both bootstrap FileName and FileContent are set, FileName is used. + XDSBootstrapFileContent = os.Getenv(XDSBootstrapFileContentEnv) + + // C2PResolverTestOnlyTrafficDirectorURI is the TD URI for testing. + C2PResolverTestOnlyTrafficDirectorURI = os.Getenv("GRPC_TEST_ONLY_GOOGLE_C2P_RESOLVER_TRAFFIC_DIRECTOR_URI") + + // XDSDualstackEndpointsEnabled is true if gRPC should read the + // "additional addresses" in the xDS endpoint resource. + XDSDualstackEndpointsEnabled = boolFromEnv("GRPC_EXPERIMENTAL_XDS_DUALSTACK_ENDPOINTS", true) + + // XDSSystemRootCertsEnabled is true when xDS enabled gRPC clients can use + // the system's default root certificates for TLS certificate validation. + // For more details, see: + // https://github.com/grpc/proposal/blob/master/A82-xds-system-root-certs.md. + XDSSystemRootCertsEnabled = boolFromEnv("GRPC_EXPERIMENTAL_XDS_SYSTEM_ROOT_CERTS", false) + + // XDSSPIFFEEnabled controls if SPIFFE Bundle Maps can be used as roots of + // trust. For more details, see: + // https://github.com/grpc/proposal/blob/master/A87-mtls-spiffe-support.md + XDSSPIFFEEnabled = boolFromEnv("GRPC_EXPERIMENTAL_XDS_MTLS_SPIFFE", false) + + // XDSHTTPConnectEnabled controls support for dynamic HTTP CONNECT proxying + // configured via the xDS control plane. For more details, see: + // https://github.com/grpc/proposal/blob/master/A86-xds-http-connect.md + XDSHTTPConnectEnabled = boolFromEnv("GRPC_EXPERIMENTAL_XDS_HTTP_CONNECT", false) + + // XDSBootstrapCallCredsEnabled controls if call credentials can be used in + // xDS bootstrap configuration via the `call_creds` field. For more details, + // see: https://github.com/grpc/proposal/blob/master/A97-xds-jwt-call-creds.md + XDSBootstrapCallCredsEnabled = boolFromEnv("GRPC_EXPERIMENTAL_XDS_BOOTSTRAP_CALL_CREDS", false) + + // XDSSNIEnabled controls if gRPC should send SNI information in xDS + // configured TLS handshakes. For more details, see: + // https://github.com/grpc/proposal/blob/master/A101-SNI-setting-and-SNI-SAN-validation.md + XDSSNIEnabled = boolFromEnv("GRPC_EXPERIMENTAL_XDS_SNI", false) + + // XDSORCAToLRSPropEnabled controls whether ORCA metrics are explicitly + // filtered and prefix-propagated to the LRS server. For more details, see: + // https://github.com/grpc/proposal/blob/master/A85-lrs-custom-metrics-changes.md + XDSORCAToLRSPropEnabled = boolFromEnv("GRPC_EXPERIMENTAL_XDS_ORCA_LRS_PROPAGATION", true) + + // XDSClientExtProcEnabled indicates whether ExtProc filter is enabled on + // the client side. For more details, see: + // https://github.com/grpc/proposal/blob/master/A93-xds-ext-proc.md + XDSClientExtProcEnabled = boolFromEnv("GRPC_EXPERIMENTAL_XDS_EXT_PROC_ON_CLIENT", false) + + // GCPAuthenticationFilterEnabled enables the xDS GCP Authentication + // filter. For more details, see: + // https://github.com/grpc/proposal/blob/master/A83-xds-gcp-authn-filter.md + GCPAuthenticationFilterEnabled = boolFromEnv("GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER", false) + + // XDSClientExtAuthzEnabled indicates whether the external authorization + // filter is enabled on the client side. For more details, see: + // https://github.com/grpc/proposal/blob/master/A92-xds-ext-authz.md + XDSClientExtAuthzEnabled = boolFromEnv("GRPC_EXPERIMENTAL_XDS_EXT_AUTHZ_ON_CLIENT", false) +) diff --git a/internal/experimental.go b/internal/experimental.go new file mode 100644 index 0000000..8a99991 --- /dev/null +++ b/internal/experimental.go @@ -0,0 +1,35 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package internal + +var ( + // WithBufferPool is implemented by the grpc package and returns a dial + // option to configure a shared buffer pool for a grpc.ClientConn. + WithBufferPool any // func (grpc.SharedBufferPool) grpc.DialOption + + // BufferPool is implemented by the grpc package and returns a server + // option to configure a shared buffer pool for a grpc.Server. + BufferPool any // func (grpc.SharedBufferPool) grpc.ServerOption + + // SetDefaultBufferPool updates the default buffer pool. + SetDefaultBufferPool any // func(mem.BufferPool) + + // AcceptCompressors is implemented by the grpc package and returns + // a call option that restricts the grpc-accept-encoding header for a call. + AcceptCompressors any // func(...string) grpc.CallOption +) diff --git a/internal/googlecloud/googlecloud.go b/internal/googlecloud/googlecloud.go new file mode 100644 index 0000000..43423d8 --- /dev/null +++ b/internal/googlecloud/googlecloud.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package googlecloud contains internal helpful functions for google cloud. +package googlecloud + +import ( + "runtime" + "strings" + "sync" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const logPrefix = "[googlecloud]" + +var ( + vmOnGCEOnce sync.Once + vmOnGCE bool + + logger = internalgrpclog.NewPrefixLogger(grpclog.Component("googlecloud"), logPrefix) +) + +// OnGCE returns whether the client is running on GCE. +// +// It provides similar functionality as metadata.OnGCE from the cloud library +// package. We keep this to avoid depending on the cloud library module. +func OnGCE() bool { + vmOnGCEOnce.Do(func() { + mf, err := manufacturer() + if err != nil { + logger.Infof("failed to read manufacturer, setting onGCE=false: %v") + return + } + vmOnGCE = isRunningOnGCE(mf, runtime.GOOS) + }) + return vmOnGCE +} + +// isRunningOnGCE checks whether the local system, without doing a network request, is +// running on GCP. +func isRunningOnGCE(manufacturer []byte, goos string) bool { + name := string(manufacturer) + switch goos { + case "linux": + name = strings.TrimSpace(name) + return name == "Google" || name == "Google Compute Engine" + case "windows": + name = strings.ReplaceAll(name, " ", "") + name = strings.ReplaceAll(name, "\n", "") + name = strings.ReplaceAll(name, "\r", "") + return name == "Google" + default: + return false + } +} diff --git a/internal/googlecloud/googlecloud_test.go b/internal/googlecloud/googlecloud_test.go new file mode 100644 index 0000000..69ab2fd --- /dev/null +++ b/internal/googlecloud/googlecloud_test.go @@ -0,0 +1,46 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package googlecloud + +import ( + "testing" +) + +func TestIsRunningOnGCE(t *testing.T) { + for _, tc := range []struct { + description string + testOS string + testManufacturer string + out bool + }{ + // Linux tests. + {"linux: not a GCP platform", "linux", "not GCP", false}, + {"Linux: GCP platform (Google)", "linux", "Google", true}, + {"Linux: GCP platform (Google Compute Engine)", "linux", "Google Compute Engine", true}, + {"Linux: GCP platform (Google Compute Engine) with extra spaces", "linux", " Google Compute Engine ", true}, + // Windows tests. + {"windows: not a GCP platform", "windows", "not GCP", false}, + {"windows: GCP platform (Google)", "windows", "Google", true}, + {"windows: GCP platform (Google) with extra spaces", "windows", " Google ", true}, + } { + if got, want := isRunningOnGCE([]byte(tc.testManufacturer), tc.testOS), tc.out; got != want { + t.Errorf("%v: isRunningOnGCE()=%v, want %v", tc.description, got, want) + } + } +} diff --git a/internal/googlecloud/manufacturer.go b/internal/googlecloud/manufacturer.go new file mode 100644 index 0000000..ffa0f1d --- /dev/null +++ b/internal/googlecloud/manufacturer.go @@ -0,0 +1,26 @@ +//go:build !(linux || windows) +// +build !linux,!windows + +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package googlecloud + +func manufacturer() ([]byte, error) { + return nil, nil +} diff --git a/internal/googlecloud/manufacturer_linux.go b/internal/googlecloud/manufacturer_linux.go new file mode 100644 index 0000000..6e455fb --- /dev/null +++ b/internal/googlecloud/manufacturer_linux.go @@ -0,0 +1,27 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package googlecloud + +import "os" + +const linuxProductNameFile = "/sys/class/dmi/id/product_name" + +func manufacturer() ([]byte, error) { + return os.ReadFile(linuxProductNameFile) +} diff --git a/internal/googlecloud/manufacturer_windows.go b/internal/googlecloud/manufacturer_windows.go new file mode 100644 index 0000000..2d7aaaa --- /dev/null +++ b/internal/googlecloud/manufacturer_windows.go @@ -0,0 +1,50 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package googlecloud + +import ( + "errors" + "os/exec" + "regexp" + "strings" +) + +const ( + windowsCheckCommand = "powershell.exe" + windowsCheckCommandArgs = "Get-WmiObject -Class Win32_BIOS" + powershellOutputFilter = "Manufacturer" + windowsManufacturerRegex = ":(.*)" +) + +func manufacturer() ([]byte, error) { + cmd := exec.Command(windowsCheckCommand, windowsCheckCommandArgs) + out, err := cmd.Output() + if err != nil { + return nil, err + } + for _, line := range strings.Split(strings.TrimSuffix(string(out), "\n"), "\n") { + if strings.HasPrefix(line, powershellOutputFilter) { + re := regexp.MustCompile(windowsManufacturerRegex) + name := re.FindString(line) + name = strings.TrimLeft(name, ":") + return []byte(name), nil + } + } + return nil, errors.New("cannot determine the machine's manufacturer") +} diff --git a/internal/grpclog/prefix_logger.go b/internal/grpclog/prefix_logger.go new file mode 100644 index 0000000..092ad18 --- /dev/null +++ b/internal/grpclog/prefix_logger.go @@ -0,0 +1,79 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpclog provides logging functionality for internal gRPC packages, +// outside of the functionality provided by the external `grpclog` package. +package grpclog + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" +) + +// PrefixLogger does logging with a prefix. +// +// Logging method on a nil logs without any prefix. +type PrefixLogger struct { + logger grpclog.DepthLoggerV2 + prefix string +} + +// Infof does info logging. +func (pl *PrefixLogger) Infof(format string, args ...any) { + if pl != nil { + // Handle nil, so the tests can pass in a nil logger. + format = pl.prefix + format + pl.logger.InfoDepth(1, fmt.Sprintf(format, args...)) + return + } + grpclog.InfoDepth(1, fmt.Sprintf(format, args...)) +} + +// Warningf does warning logging. +func (pl *PrefixLogger) Warningf(format string, args ...any) { + if pl != nil { + format = pl.prefix + format + pl.logger.WarningDepth(1, fmt.Sprintf(format, args...)) + return + } + grpclog.WarningDepth(1, fmt.Sprintf(format, args...)) +} + +// Errorf does error logging. +func (pl *PrefixLogger) Errorf(format string, args ...any) { + if pl != nil { + format = pl.prefix + format + pl.logger.ErrorDepth(1, fmt.Sprintf(format, args...)) + return + } + grpclog.ErrorDepth(1, fmt.Sprintf(format, args...)) +} + +// V reports whether verbosity level l is at least the requested verbose level. +func (pl *PrefixLogger) V(l int) bool { + if pl != nil { + return pl.logger.V(l) + } + return true +} + +// NewPrefixLogger creates a prefix logger with the given prefix. +func NewPrefixLogger(logger grpclog.DepthLoggerV2, prefix string) *PrefixLogger { + return &PrefixLogger{logger: logger, prefix: prefix} +} diff --git a/internal/grpcsync/callback_serializer.go b/internal/grpcsync/callback_serializer.go new file mode 100644 index 0000000..caa9dd6 --- /dev/null +++ b/internal/grpcsync/callback_serializer.go @@ -0,0 +1,124 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcsync + +import ( + "context" + "errors" + + "google.golang.org/grpc/internal/buffer" +) + +// ErrSerializerClosed is returned by ScheduleAndWait if the CallbackSerializer +// was closed before the callback could be scheduled. +var ErrSerializerClosed = errors.New("callback serializer is closed") + +// CallbackSerializer provides a mechanism to schedule callbacks in a +// synchronized manner. It provides a FIFO guarantee on the order of execution +// of scheduled callbacks. New callbacks can be scheduled by invoking the +// Schedule() method. +// +// This type is safe for concurrent access. +type CallbackSerializer struct { + // done is closed once the serializer is shut down completely, i.e all + // scheduled callbacks are executed and the serializer has deallocated all + // its resources. + done chan struct{} + + callbacks *buffer.Unbounded[func(context.Context)] +} + +// NewCallbackSerializer returns a new CallbackSerializer instance. The provided +// context will be passed to the scheduled callbacks. Users should cancel the +// provided context to shutdown the CallbackSerializer. It is guaranteed that no +// callbacks will be added once this context is canceled, and any pending un-run +// callbacks will be executed before the serializer is shut down. +func NewCallbackSerializer(ctx context.Context) *CallbackSerializer { + cs := &CallbackSerializer{ + done: make(chan struct{}), + callbacks: buffer.NewUnbounded[func(context.Context)](), + } + go cs.run(ctx) + return cs +} + +// TrySchedule tries to schedule the provided callback function f to be +// executed in the order it was added. This is a best-effort operation. If the +// context passed to NewCallbackSerializer was canceled before this method is +// called, the callback will not be scheduled. +// +// Callbacks are expected to honor the context when performing any blocking +// operations, and should return early when the context is canceled. +func (cs *CallbackSerializer) TrySchedule(f func(ctx context.Context)) { + cs.callbacks.Put(f) +} + +// ScheduleOr schedules the provided callback function f to be executed in the +// order it was added. If the context passed to NewCallbackSerializer has been +// canceled before this method is called, the onFailure callback will be +// executed inline instead. +// +// Callbacks are expected to honor the context when performing any blocking +// operations, and should return early when the context is canceled. +func (cs *CallbackSerializer) ScheduleOr(f func(ctx context.Context), onFailure func()) { + if cs.callbacks.Put(f) != nil { + onFailure() + } +} + +// ScheduleAndWait schedules the provided callback function f to be executed in +// the order it was added and blocks until f has run. If the context passed to +// NewCallbackSerializer was canceled before this method is called, f is not run +// and ScheduleAndWait returns ErrSerializerClosed. +// +// Callbacks are expected to honor the context when performing any blocking +// operations, and should return early when the context is canceled. +func (cs *CallbackSerializer) ScheduleAndWait(f func(ctx context.Context)) error { + done := make(chan struct{}) + var err error + cs.ScheduleOr(func(ctx context.Context) { + f(ctx) + close(done) + }, func() { + err = ErrSerializerClosed + close(done) + }) + <-done + return err +} + +func (cs *CallbackSerializer) run(ctx context.Context) { + defer close(cs.done) + + // Close the buffer when the context is canceled + // to prevent new callbacks from being added. + context.AfterFunc(ctx, cs.callbacks.Close) + + // Run all callbacks. + for cb := range cs.callbacks.Get() { + cs.callbacks.Load() + cb(ctx) + } +} + +// Done returns a channel that is closed after the context passed to +// NewCallbackSerializer is canceled and all callbacks have been executed. +func (cs *CallbackSerializer) Done() <-chan struct{} { + return cs.done +} diff --git a/internal/grpcsync/callback_serializer_test.go b/internal/grpcsync/callback_serializer_test.go new file mode 100644 index 0000000..2b9f650 --- /dev/null +++ b/internal/grpcsync/callback_serializer_test.go @@ -0,0 +1,247 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcsync + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) + +// TestCallbackSerializer_Schedule_FIFO verifies that callbacks are executed in +// the same order in which they were scheduled. +func (s) TestCallbackSerializer_Schedule_FIFO(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + cs := NewCallbackSerializer(ctx) + defer cancel() + + // We have two channels, one to record the order of scheduling, and the + // other to record the order of execution. We spawn a bunch of goroutines + // which record the order of scheduling and call the actual Schedule() + // method as well. The callbacks record the order of execution. + // + // We need to grab a lock to record order of scheduling to guarantee that + // the act of recording and the act of calling Schedule() happen atomically. + const numCallbacks = 100 + var mu sync.Mutex + scheduleOrderCh := make(chan int, numCallbacks) + executionOrderCh := make(chan int, numCallbacks) + for i := 0; i < numCallbacks; i++ { + go func(id int) { + mu.Lock() + defer mu.Unlock() + scheduleOrderCh <- id + cs.TrySchedule(func(ctx context.Context) { + select { + case <-ctx.Done(): + return + case executionOrderCh <- id: + } + }) + }(i) + } + + // Spawn a couple of goroutines to capture the order or scheduling and the + // order of execution. + scheduleOrder := make([]int, numCallbacks) + executionOrder := make([]int, numCallbacks) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; i < numCallbacks; i++ { + select { + case <-ctx.Done(): + return + case id := <-scheduleOrderCh: + scheduleOrder[i] = id + } + } + }() + go func() { + defer wg.Done() + for i := 0; i < numCallbacks; i++ { + select { + case <-ctx.Done(): + return + case id := <-executionOrderCh: + executionOrder[i] = id + } + } + }() + wg.Wait() + + if diff := cmp.Diff(executionOrder, scheduleOrder); diff != "" { + t.Fatalf("Callbacks are not executed in scheduled order. diff(-want, +got):\n%s", diff) + } +} + +// TestCallbackSerializer_Schedule_Concurrent verifies that all concurrently +// scheduled callbacks get executed. +func (s) TestCallbackSerializer_Schedule_Concurrent(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + cs := NewCallbackSerializer(ctx) + defer cancel() + + // Schedule callbacks concurrently by calling Schedule() from goroutines. + // The execution of the callbacks call Done() on the waitgroup, which + // eventually unblocks the test and allows it to complete. + const numCallbacks = 100 + var wg sync.WaitGroup + wg.Add(numCallbacks) + for i := 0; i < numCallbacks; i++ { + go func() { + cs.TrySchedule(func(context.Context) { + wg.Done() + }) + }() + } + + // We call Wait() on the waitgroup from a goroutine so that we can select on + // the Wait() being unblocked and the overall test deadline expiring. + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for all scheduled callbacks to be executed") + case <-done: + } +} + +// TestCallbackSerializer_Schedule_Close verifies that callbacks in the queue +// are not executed once Close() returns. +func (s) TestCallbackSerializer_Schedule_Close(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + serializerCtx, serializerCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + cs := NewCallbackSerializer(serializerCtx) + + // Schedule a callback which blocks until the context passed to it is + // canceled. It also closes a channel to signal that it has started. + firstCallbackStartedCh := make(chan struct{}) + cs.TrySchedule(func(ctx context.Context) { + close(firstCallbackStartedCh) + <-ctx.Done() + }) + + // Schedule a bunch of callbacks. These should be executed since they are + // scheduled before the serializer is closed. + const numCallbacks = 10 + callbackCh := make(chan int, numCallbacks) + for i := 0; i < numCallbacks; i++ { + num := i + callback := func(context.Context) { callbackCh <- num } + onFailure := func() { t.Fatal("Schedule failed to accept a callback when the serializer is yet to be closed") } + cs.ScheduleOr(callback, onFailure) + } + + // Ensure that none of the newer callbacks are executed at this point. + select { + case <-time.After(defaultTestShortTimeout): + case <-callbackCh: + t.Fatal("Newer callback executed when older one is still executing") + } + + // Wait for the first callback to start before closing the scheduler. + <-firstCallbackStartedCh + + // Cancel the context which will unblock the first callback. All of the + // other callbacks (which have not started executing at this point) should + // be executed after this. + serializerCancel() + + // Ensure that the newer callbacks are executed. + for i := 0; i < numCallbacks; i++ { + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for callback scheduled before close to be executed") + case num := <-callbackCh: + if num != i { + t.Fatalf("Executing callback %d, want %d", num, i) + } + } + } + <-cs.Done() + + // Ensure that a callback cannot be scheduled after the serializer is + // closed. + done := make(chan struct{}) + callback := func(context.Context) { t.Fatal("Scheduled a callback after closing the serializer") } + onFailure := func() { close(done) } + cs.ScheduleOr(callback, onFailure) + select { + case <-time.After(defaultTestTimeout): + t.Fatal("Successfully scheduled callback after serializer is closed") + case <-done: + } +} + +// TestCallbackSerializer_ScheduleAndWait verifies that ScheduleAndWait runs the +// provided callback and blocks until it has finished executing, returning a nil +// error. +func (s) TestCallbackSerializer_ScheduleAndWait(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cs := NewCallbackSerializer(ctx) + + executed := false + if err := cs.ScheduleAndWait(func(context.Context) { + executed = true + }); err != nil { + t.Fatalf("ScheduleAndWait() returned error: %v, want nil", err) + } + + // The callback must have run by the time ScheduleAndWait returns. No + // synchronization is needed here because ScheduleAndWait is expected to + // block until the callback completes. + if !executed { + t.Fatal("ScheduleAndWait() returned before the callback was executed") + } +} + +// TestCallbackSerializer_ScheduleAndWait_Closed verifies that ScheduleAndWait +// does not run the callback and returns ErrSerializerClosed when the serializer +// has already been closed. +func (s) TestCallbackSerializer_ScheduleAndWait_Closed(t *testing.T) { + serializerCtx, serializerCancel := context.WithCancel(context.Background()) + cs := NewCallbackSerializer(serializerCtx) + + // Close the serializer and wait for it to finish shutting down. + serializerCancel() + <-cs.Done() + + if err := cs.ScheduleAndWait(func(context.Context) { + t.Fatal("Callback executed on a closed serializer") + }); err != ErrSerializerClosed { + t.Fatalf("ScheduleAndWait() returned error: %v, want %v", err, ErrSerializerClosed) + } +} diff --git a/internal/grpcsync/event.go b/internal/grpcsync/event.go new file mode 100644 index 0000000..d788c24 --- /dev/null +++ b/internal/grpcsync/event.go @@ -0,0 +1,58 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpcsync implements additional synchronization primitives built upon +// the sync package. +package grpcsync + +import ( + "sync/atomic" +) + +// Event represents a one-time event that may occur in the future. +type Event struct { + fired atomic.Bool + c chan struct{} +} + +// Fire causes e to complete. It is safe to call multiple times, and +// concurrently. It returns true iff this call to Fire caused the signaling +// channel returned by Done to close. If Fire returns false, it is possible +// the Done channel has not been closed yet. +func (e *Event) Fire() bool { + if e.fired.CompareAndSwap(false, true) { + close(e.c) + return true + } + return false +} + +// Done returns a channel that will be closed when Fire is called. +func (e *Event) Done() <-chan struct{} { + return e.c +} + +// HasFired returns true if Fire has been called. +func (e *Event) HasFired() bool { + return e.fired.Load() +} + +// NewEvent returns a new, ready-to-use Event. +func NewEvent() *Event { + return &Event{c: make(chan struct{})} +} diff --git a/internal/grpcsync/event_test.go b/internal/grpcsync/event_test.go new file mode 100644 index 0000000..bb448a4 --- /dev/null +++ b/internal/grpcsync/event_test.go @@ -0,0 +1,81 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcsync + +import ( + "testing" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestEventHasFired(t *testing.T) { + e := NewEvent() + if e.HasFired() { + t.Fatal("e.HasFired() = true; want false") + } + if !e.Fire() { + t.Fatal("e.Fire() = false; want true") + } + if !e.HasFired() { + t.Fatal("e.HasFired() = false; want true") + } +} + +func (s) TestEventDoneChannel(t *testing.T) { + e := NewEvent() + select { + case <-e.Done(): + t.Fatal("e.HasFired() = true; want false") + default: + } + if !e.Fire() { + t.Fatal("e.Fire() = false; want true") + } + select { + case <-e.Done(): + default: + t.Fatal("e.HasFired() = false; want true") + } +} + +func (s) TestEventMultipleFires(t *testing.T) { + e := NewEvent() + if e.HasFired() { + t.Fatal("e.HasFired() = true; want false") + } + if !e.Fire() { + t.Fatal("e.Fire() = false; want true") + } + for i := 0; i < 3; i++ { + if !e.HasFired() { + t.Fatal("e.HasFired() = false; want true") + } + if e.Fire() { + t.Fatal("e.Fire() = true; want false") + } + } +} diff --git a/internal/grpcsync/pubsub.go b/internal/grpcsync/pubsub.go new file mode 100644 index 0000000..6d8c2f5 --- /dev/null +++ b/internal/grpcsync/pubsub.go @@ -0,0 +1,121 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcsync + +import ( + "context" + "sync" +) + +// Subscriber represents an entity that is subscribed to messages published on +// a PubSub. It wraps the callback to be invoked by the PubSub when a new +// message is published. +type Subscriber interface { + // OnMessage is invoked when a new message is published. Implementations + // must not block in this method. + OnMessage(msg any) +} + +// PubSub is a simple one-to-many publish-subscribe system that supports +// messages of arbitrary type. It guarantees that messages are delivered in +// the same order in which they were published. +// +// Publisher invokes the Publish() method to publish new messages, while +// subscribers interested in receiving these messages register a callback +// via the Subscribe() method. +// +// Once a PubSub is stopped, no more messages can be published, but any pending +// published messages will be delivered to the subscribers. Done may be used +// to determine when all published messages have been delivered. +type PubSub struct { + cs *CallbackSerializer + + // Access to the below fields are guarded by this mutex. + mu sync.Mutex + msg any + subscribers map[Subscriber]bool +} + +// NewPubSub returns a new PubSub instance. Users should cancel the +// provided context to shutdown the PubSub. +func NewPubSub(ctx context.Context) *PubSub { + return &PubSub{ + cs: NewCallbackSerializer(ctx), + subscribers: map[Subscriber]bool{}, + } +} + +// Subscribe registers the provided Subscriber to the PubSub. +// +// If the PubSub contains a previously published message, the Subscriber's +// OnMessage() callback will be invoked asynchronously with the existing +// message to begin with, and subsequently for every newly published message. +// +// The caller is responsible for invoking the returned cancel function to +// unsubscribe itself from the PubSub. +func (ps *PubSub) Subscribe(sub Subscriber) (cancel func()) { + ps.mu.Lock() + defer ps.mu.Unlock() + + ps.subscribers[sub] = true + + if ps.msg != nil { + msg := ps.msg + ps.cs.TrySchedule(func(context.Context) { + ps.mu.Lock() + defer ps.mu.Unlock() + if !ps.subscribers[sub] { + return + } + sub.OnMessage(msg) + }) + } + + return func() { + ps.mu.Lock() + defer ps.mu.Unlock() + delete(ps.subscribers, sub) + } +} + +// Publish publishes the provided message to the PubSub, and invokes +// callbacks registered by subscribers asynchronously. +func (ps *PubSub) Publish(msg any) { + ps.mu.Lock() + defer ps.mu.Unlock() + + ps.msg = msg + for sub := range ps.subscribers { + s := sub + ps.cs.TrySchedule(func(context.Context) { + ps.mu.Lock() + defer ps.mu.Unlock() + if !ps.subscribers[s] { + return + } + s.OnMessage(msg) + }) + } +} + +// Done returns a channel that is closed after the context passed to NewPubSub +// is canceled and all updates have been sent to subscribers. +func (ps *PubSub) Done() <-chan struct{} { + return ps.cs.Done() +} diff --git a/internal/grpcsync/pubsub_test.go b/internal/grpcsync/pubsub_test.go new file mode 100644 index 0000000..c9f9889 --- /dev/null +++ b/internal/grpcsync/pubsub_test.go @@ -0,0 +1,196 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcsync + +import ( + "context" + "sync" + "testing" + "time" +) + +type testSubscriber struct { + onMsgCh chan int +} + +func newTestSubscriber(chSize int) *testSubscriber { + return &testSubscriber{onMsgCh: make(chan int, chSize)} +} + +func (ts *testSubscriber) OnMessage(msg any) { + select { + case ts.onMsgCh <- msg.(int): + default: + } +} + +func (s) TestPubSub_PublishNoMsg(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + pubsub := NewPubSub(ctx) + + ts := newTestSubscriber(1) + pubsub.Subscribe(ts) + + select { + case <-ts.onMsgCh: + t.Fatal("Subscriber callback invoked when no message was published") + case <-time.After(defaultTestShortTimeout): + } +} + +func (s) TestPubSub_PublishMsgs_RegisterSubs_And_Stop(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + pubsub := NewPubSub(ctx) + + const numPublished = 10 + + ts1 := newTestSubscriber(numPublished) + pubsub.Subscribe(ts1) + + var wg sync.WaitGroup + wg.Add(2) + // Publish ten messages on the pubsub and ensure that they are received in order by the subscriber. + go func() { + for i := 0; i < numPublished; i++ { + pubsub.Publish(i) + } + wg.Done() + }() + + go func() { + defer wg.Done() + for i := 0; i < numPublished; i++ { + select { + case m := <-ts1.onMsgCh: + if m != i { + t.Errorf("Received unexpected message: %q; want: %q", m, i) + return + } + case <-time.After(defaultTestTimeout): + t.Error("Timeout when expecting the onMessage() callback to be invoked") + return + } + } + }() + wg.Wait() + if t.Failed() { + t.FailNow() + } + + // Register another subscriber and ensure that it receives the last published message. + ts2 := newTestSubscriber(numPublished) + pubsub.Subscribe(ts2) + + select { + case m := <-ts2.onMsgCh: + if m != numPublished-1 { + t.Fatalf("Received unexpected message: %q; want: %q", m, numPublished-1) + } + case <-time.After(defaultTestShortTimeout): + t.Fatal("Timeout when expecting the onMessage() callback to be invoked") + } + + wg.Add(3) + // Publish ten messages on the pubsub and ensure that they are received in order by the subscribers. + go func() { + for i := 0; i < numPublished; i++ { + pubsub.Publish(i) + } + wg.Done() + }() + go func() { + defer wg.Done() + for i := 0; i < numPublished; i++ { + select { + case m := <-ts1.onMsgCh: + if m != i { + t.Errorf("Received unexpected message: %q; want: %q", m, i) + return + } + case <-time.After(defaultTestTimeout): + t.Error("Timeout when expecting the onMessage() callback to be invoked") + return + } + } + + }() + go func() { + defer wg.Done() + for i := 0; i < numPublished; i++ { + select { + case m := <-ts2.onMsgCh: + if m != i { + t.Errorf("Received unexpected message: %q; want: %q", m, i) + return + } + case <-time.After(defaultTestTimeout): + t.Error("Timeout when expecting the onMessage() callback to be invoked") + return + } + } + }() + wg.Wait() + if t.Failed() { + t.FailNow() + } + + cancel() + <-pubsub.Done() + + go func() { + pubsub.Publish(99) + }() + // Ensure that the subscriber callback is not invoked as instantiated + // pubsub has already closed. + select { + case <-ts1.onMsgCh: + t.Fatal("The callback was invoked after pubsub being stopped") + case <-ts2.onMsgCh: + t.Fatal("The callback was invoked after pubsub being stopped") + case <-time.After(defaultTestShortTimeout): + } +} + +func (s) TestPubSub_PublishMsgs_BeforeRegisterSub(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + pubsub := NewPubSub(ctx) + + const numPublished = 3 + for i := 0; i < numPublished; i++ { + pubsub.Publish(i) + } + + ts := newTestSubscriber(numPublished) + pubsub.Subscribe(ts) + + // Ensure that the subscriber callback is invoked with a previously + // published message. + select { + case d := <-ts.onMsgCh: + if d != numPublished-1 { + t.Fatalf("Unexpected message received: %q; %q", d, numPublished-1) + } + + case <-time.After(defaultTestShortTimeout): + t.Fatal("Timeout when expecting the onMessage() callback to be invoked") + } +} diff --git a/internal/grpcsync/refcounted.go b/internal/grpcsync/refcounted.go new file mode 100644 index 0000000..bca9b83 --- /dev/null +++ b/internal/grpcsync/refcounted.go @@ -0,0 +1,109 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcsync + +import ( + "sync/atomic" + + "google.golang.org/grpc/grpclog" +) + +var logger = grpclog.Component("grpcsync") + +// RefCounted is a reference counted wrapper of type T. It tracks the number of +// active references and runs a cleanup when the last reference is released. +type RefCounted[T any] struct { + val T + refCount atomic.Int32 + onZero func() +} + +// NewRefCounted creates a new RefCounted instance wrapping the given value with +// initial refcount of one. +// +// The value should typically be a pointer, interface, or handle rather than a +// plain value type (such as a struct or primitive value). +// +// The provided onZero callback must not be nil, and is executed exactly once +// when the reference count drops to zero. Panics if onZero is nil. +// +// WARNING: onZero runs synchronously inside Decrement; it must not acquire +// locks held by Decrement callers. +func NewRefCounted[T any](val T, onZero func()) *RefCounted[T] { + if onZero == nil { + panic("grpcsync: onZero callback cannot be nil") + } + rc := &RefCounted[T]{ + val: val, + onZero: onZero, + } + rc.refCount.Store(1) + return rc +} + +// Value returns the encapsulated resource. +func (rc *RefCounted[T]) Value() T { + return rc.val +} + +// TryIncrement attempts to increment the reference count, returning true if +// successful. It returns false if the count has already reached 0, indicating +// the resource has been cleaned up and cannot be resurrected. +// +// WARNING: Avoid calling TryIncrement on hot paths when an active reference is +// already guaranteed by the caller; use Increment instead to bypass the +// CompareAndSwap loop overhead. TryIncrement should be reserved for speculative +// lookups (such as fetching from a cache or map) where the resource might be +// dead. +func (rc *RefCounted[T]) TryIncrement() bool { + // Utilize a CompareAndSwap loop to prevent race conditions where a concurrent + // decrement could drop the count to zero between the read and the increment + // operation, which would otherwise inadvertently resurrect a closed resource. + for { + count := rc.refCount.Load() + if count <= 0 { + return false // Already dead or dying + } + if rc.refCount.CompareAndSwap(count, count+1) { + return true + } + } +} + +// Increment increments the reference count. +// +// WARNING: Call Increment only when there is a guarantee that an active +// reference is already present, ensuring the resource is not dead. If there is +// a possibility that the resource is dead or its reference count might have +// reached zero, call TryIncrement instead. +func (rc *RefCounted[T]) Increment() { + if rc.refCount.Add(1) <= 1 { + logger.Errorf("Resource already closed or dead") + } +} + +// Decrement decrements the reference count. If it drops to zero, the onZero +// callback is executed synchronously before this method returns. +func (rc *RefCounted[T]) Decrement() { + if v := rc.refCount.Add(-1); v < 0 { + logger.Errorf("Refcount cannot be negative") + } else if v == 0 { + rc.onZero() + } +} diff --git a/internal/grpcsync/refcounted_test.go b/internal/grpcsync/refcounted_test.go new file mode 100644 index 0000000..88ba6eb --- /dev/null +++ b/internal/grpcsync/refcounted_test.go @@ -0,0 +1,155 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcsync + +import ( + "sync" + "sync/atomic" + "testing" + + "google.golang.org/grpc/internal/grpctest" +) + +// Test verifies the scenario where a RefCounted instance is created and its +// reference count is decremented. It verifies that the encapsulated value is +// correctly retrieved and that the onZero callback is invoked when the +// reference count drops to zero. +func (s) TestRefCounted(t *testing.T) { + val := 666 + var onZeroCalled atomic.Bool + rc := NewRefCounted(&val, func() { onZeroCalled.Store(true) }) + if got := rc.Value(); *got != val { + t.Fatalf("Value() = %v, want %v", *got, val) + } + + if got := onZeroCalled.Load(); got { + t.Fatalf("Before decrement, onZeroCalled = %v, want false", got) + } + + rc.Decrement() + + if got := onZeroCalled.Load(); !got { + t.Fatalf("After decrementing count to zero, onZeroCalled = %v, want true", got) + } +} + +// Test verifies the scenario where the reference count of a resource is +// explicitly incremented and decremented multiple times. It verifies that the +// onZero callback is only executed when the count drops to zero, and not +// beforehand. +func (s) TestRefCounted_IncrementDecrement(t *testing.T) { + val := 666 + var onZeroCount atomic.Int32 + rc := NewRefCounted(&val, func() { onZeroCount.Add(1) }) + rc.Increment() + rc.Decrement() + + if got := onZeroCount.Load(); got != 0 { + t.Fatal("OnZero called unexpectedly after decrementing refcount from 2 to 1") + } + + rc.Decrement() + + if got := onZeroCount.Load(); got != 1 { + t.Fatalf("After decrementing count from 1 to 0, onZero called = %v times, want 1", got) + } +} + +// Test verifies the scenario where TryIncrement is called on both an active and +// a dead resource. It verifies that TryIncrement successfully increments active +// resources but fails on resources whose reference count has already dropped to +// zero. +func (s) TestRefCounted_TryIncrement(t *testing.T) { + val := 666 + var onZeroCount atomic.Int32 + rc := NewRefCounted(&val, func() { onZeroCount.Add(1) }) + if got := rc.TryIncrement(); !got { + t.Fatalf("TryIncrement() on active resource = %v, want true", got) + } + + rc.Decrement() + rc.Decrement() + + if got := onZeroCount.Load(); got != 1 { + t.Fatalf("After decrementing count to zero, onZero called = %v times, want 1", got) + } + + if got := rc.TryIncrement(); got { + t.Fatalf("TryIncrement() on dead resource = %v, want false", got) + } +} + +// Test verifies the scenario where multiple goroutines concurrently increment +// and decrement the reference count. It verifies that the reference counting is +// thread-safe and the onZero callback is invoked exactly once. +func (s) TestRefCounted_Concurrent(t *testing.T) { + val := 666 + var onZeroCount atomic.Int32 + rc := NewRefCounted(&val, func() { onZeroCount.Add(1) }) + + var wg sync.WaitGroup + const numGoroutines = 100 + for i := 0; i < numGoroutines; i++ { + wg.Go(func() { + if rc.TryIncrement() { + rc.Decrement() + } + }) + } + rc.Decrement() + wg.Wait() + if got := onZeroCount.Load(); got != 1 { + t.Fatalf("After concurrent increments/decrements and final decrement, onZeroCount = %v, want 1", got) + } +} + +// Test verifies that Decrementing a resource whose reference count has already +// dropped to zero results in a negative refcount log error. +func (s) TestRefCounted_DecrementNegative(_ *testing.T) { + val := 666 + rc := NewRefCounted(&val, func() {}) + rc.Decrement() + + grpctest.ExpectError("Refcount cannot be negative") + rc.Decrement() +} + +// Test verifies that Incrementing a dead resource results in a closed resource +// log error. +func (s) TestRefCounted_IncrementDead(_ *testing.T) { + val := 666 + rc := NewRefCounted(&val, func() {}) + rc.Decrement() + + grpctest.ExpectError("Resource already closed or dead") + rc.Increment() +} + +// Test verifies that NewRefCounted returns an error if the provided onZero +// callback is nil. +func (s) TestNilOnZero(t *testing.T) { + const wantErr = "grpcsync: onZero callback cannot be nil" + defer func() { + if r := recover(); r == nil || r != wantErr { + t.Fatalf("NewRefCounted(_, nil) panic = %v, want: %q", r, wantErr) + } + }() + val := 666 + NewRefCounted(&val, nil) +} diff --git a/internal/grpctest/example_test.go b/internal/grpctest/example_test.go new file mode 100644 index 0000000..d0a62bc --- /dev/null +++ b/internal/grpctest/example_test.go @@ -0,0 +1,61 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpctest_test + +import ( + "testing" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + i int +} + +func (s *s) Setup(t *testing.T) { + t.Log("Per-test setup code") + s.i = 5 +} + +func (s *s) TestSomething(t *testing.T) { + t.Log("TestSomething") + if s.i != 5 { + t.Errorf("s.i = %v; want 5", s.i) + } + s.i = 3 +} + +func (s *s) TestSomethingElse(t *testing.T) { + t.Log("TestSomethingElse") + if got, want := s.i%4, 1; got != want { + t.Errorf("s.i %% 4 = %v; want %v", got, want) + } + s.i = 3 +} + +func (s *s) Teardown(t *testing.T) { + t.Log("Per-test teardown code") + if s.i != 3 { + t.Fatalf("s.i = %v; want 3", s.i) + } +} + +func TestExample(t *testing.T) { + grpctest.RunSubTests(t, &s{}) +} diff --git a/internal/grpctest/grpctest.go b/internal/grpctest/grpctest.go new file mode 100644 index 0000000..1e93fe7 --- /dev/null +++ b/internal/grpctest/grpctest.go @@ -0,0 +1,135 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpctest implements testing helpers. +package grpctest + +import ( + "context" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc/internal/leakcheck" +) + +var lcFailed uint32 + +type logger struct { + t *testing.T +} + +func (e logger) Logf(format string, args ...any) { + e.t.Logf(format, args...) +} + +func (e logger) Errorf(format string, args ...any) { + atomic.StoreUint32(&lcFailed, 1) + e.t.Errorf(format, args...) +} + +// Tester is an implementation of the x interface parameter to +// grpctest.RunSubTests with default Setup and Teardown behavior. Setup updates +// the tlogger and Teardown performs a leak check. Embed in a struct with tests +// defined to use. +type Tester struct{} + +// Setup updates the tlogger. +func (Tester) Setup(t *testing.T) { + tLogr.update(t) + // TODO: There is one final leak around closing connections without completely + // draining the recvBuffer that has yet to be resolved. All other leaks have been + // completely addressed, and this can be turned back on as soon as this issue is + // fixed. + leakcheck.SetTrackingBufferPool(logger{t: t}) + leakcheck.TrackTimers() + leakcheck.TrackAsyncReporters() +} + +// Teardown performs a leak check. +func (Tester) Teardown(t *testing.T) { + leakcheck.CheckTrackingBufferPool() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + leakcheck.CheckTimers(ctx, logger{t: t}) + if atomic.LoadUint32(&lcFailed) == 1 { + return + } + leakcheck.CheckGoroutines(ctx, logger{t: t}) + if atomic.LoadUint32(&lcFailed) == 1 { + t.Log("Goroutine leak check disabled for future tests") + } + leakcheck.CheckAsyncReporters(logger{t: t}) + if atomic.LoadUint32(&lcFailed) == 1 { + return + } + tLogr.endTest(t) +} + +// Interface defines Tester's methods for use in this package. +type Interface interface { + Setup(*testing.T) + Teardown(*testing.T) +} + +func getTestFunc(t *testing.T, xv reflect.Value, name string) func(*testing.T) { + if m := xv.MethodByName(name); m.IsValid() { + if f, ok := m.Interface().(func(*testing.T)); ok { + return f + } + // Method exists but has the wrong type signature. + t.Fatalf("grpctest: function %v has unexpected signature (%T)", name, m.Interface()) + } + return func(*testing.T) {} +} + +// RunSubTests runs all "Test___" functions that are methods of x as subtests +// of the current test. Setup is run before the test function and Teardown is +// run after. +// +// For example usage, see example_test.go. Run it using: +// +// $ go test -v -run TestExample . +// +// To run a specific test/subtest: +// +// $ go test -v -run 'TestExample/^Something$' . +func RunSubTests(t *testing.T, x Interface) { + xt := reflect.TypeOf(x) + xv := reflect.ValueOf(x) + + for i := 0; i < xt.NumMethod(); i++ { + methodName := xt.Method(i).Name + if !strings.HasPrefix(methodName, "Test") { + continue + } + tfunc := getTestFunc(t, xv, methodName) + t.Run(strings.TrimPrefix(methodName, "Test"), func(t *testing.T) { + // Run leakcheck in t.Cleanup() to guarantee it is run even if tfunc + // or setup uses t.Fatal(). + // + // Note that a defer would run before t.Cleanup, so if a goroutine + // is closed by a test's t.Cleanup, a deferred leakcheck would fail. + t.Cleanup(func() { x.Teardown(t) }) + x.Setup(t) + tfunc(t) + }) + } +} diff --git a/internal/grpctest/grpctest_test.go b/internal/grpctest/grpctest_test.go new file mode 100644 index 0000000..7bc704c --- /dev/null +++ b/internal/grpctest/grpctest_test.go @@ -0,0 +1,46 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpctest + +import ( + "reflect" + "testing" +) + +type tRunST struct { + setup, test, teardown bool +} + +func (t *tRunST) Setup(*testing.T) { + t.setup = true +} +func (t *tRunST) TestSubTest(*testing.T) { + t.test = true +} +func (t *tRunST) Teardown(*testing.T) { + t.teardown = true +} + +func TestRunSubTests(t *testing.T) { + x := &tRunST{} + RunSubTests(t, x) + if want := (&tRunST{setup: true, test: true, teardown: true}); !reflect.DeepEqual(x, want) { + t.Fatalf("x = %v; want all fields true", x) + } +} diff --git a/internal/grpctest/tlogger.go b/internal/grpctest/tlogger.go new file mode 100644 index 0000000..746aa9b --- /dev/null +++ b/internal/grpctest/tlogger.go @@ -0,0 +1,300 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpctest + +import ( + "errors" + "fmt" + "os" + "path" + "regexp" + "runtime" + "strconv" + "sync" + "testing" + "time" + + "google.golang.org/grpc/grpclog" +) + +// tLogr serves as the grpclog logger and is the interface through which +// expected errors are declared in tests. +var tLogr *tLogger + +const callingFrame = 4 + +type logType int + +func (l logType) String() string { + switch l { + case infoLog: + return "INFO" + case warningLog: + return "WARNING" + case errorLog: + return "ERROR" + case fatalLog: + return "FATAL" + } + return "UNKNOWN" +} + +const ( + infoLog logType = iota + warningLog + errorLog + fatalLog +) + +type tLogger struct { + v int + initialized bool + + mu sync.Mutex + t *testing.T + start time.Time + logs map[logType]map[*regexp.Regexp]int +} + +func init() { + vLevel := 0 // Default verbosity level + + if vLevelEnv, found := os.LookupEnv("GRPC_GO_LOG_VERBOSITY_LEVEL"); found { + // If found, attempt to convert. If conversion is successful, update vLevel. + // If conversion fails, log a warning, but vLevel remains its default of 0. + if val, err := strconv.Atoi(vLevelEnv); err == nil { + vLevel = val + } else { + // Log the error if the environment variable is not a valid integer. + fmt.Printf("Warning: GRPC_GO_LOG_VERBOSITY_LEVEL environment variable '%s' is not a valid integer. "+ + "Using default verbosity level 0. Error: %v\n", vLevelEnv, err) + } + } + // Initialize tLogr with the determined verbosity level. + logsMap := map[logType]map[*regexp.Regexp]int{ + errorLog: {}, + warningLog: {}, + } + tLogr = &tLogger{logs: logsMap, v: vLevel} +} + +// getCallingPrefix returns the at the given depth from the stack. +func getCallingPrefix(depth int) (string, error) { + _, file, line, ok := runtime.Caller(depth) + if !ok { + return "", errors.New("frame request out-of-bounds") + } + return fmt.Sprintf("%s:%d", path.Base(file), line), nil +} + +// log logs the message with the specified parameters to the tLogger. +func (tl *tLogger) log(ltype logType, depth int, format string, args ...any) { + tl.mu.Lock() + defer tl.mu.Unlock() + prefix, err := getCallingPrefix(callingFrame + depth) + if err != nil { + tl.t.Error(err) + return + } + args = append([]any{ltype.String() + " " + prefix}, args...) + args = append(args, fmt.Sprintf(" (t=+%s)", time.Since(tl.start))) + + if format == "" { + switch ltype { + case errorLog: + // fmt.Sprintln is used rather than fmt.Sprint because tl.Log uses fmt.Sprintln behavior. + if tl.expected(fmt.Sprintln(args...), errorLog) { + tl.t.Log(args...) + } else { + tl.t.Error(args...) + } + case warningLog: + tl.expected(fmt.Sprintln(args...), warningLog) + tl.t.Log(args...) + case fatalLog: + panic(fmt.Sprint(args...)) + default: + tl.t.Log(args...) + } + } else { + // Add formatting directives for the callingPrefix and timeSuffix. + format = "%v " + format + "%s" + switch ltype { + case errorLog: + if tl.expected(fmt.Sprintf(format, args...), errorLog) { + tl.t.Logf(format, args...) + } else { + tl.t.Errorf(format, args...) + } + case warningLog: + tl.expected(fmt.Sprintln(args...), warningLog) + tl.t.Log(args...) + case fatalLog: + panic(fmt.Sprintf(format, args...)) + default: + tl.t.Logf(format, args...) + } + } +} + +// update updates the testing.T that the testing logger logs to. Should be done +// before every test. It also initializes the tLogger if it has not already. +func (tl *tLogger) update(t *testing.T) { + tl.mu.Lock() + defer tl.mu.Unlock() + if !tl.initialized { + grpclog.SetLoggerV2(tl) + tl.initialized = true + } + tl.t = t + tl.start = time.Now() + tl.logs[errorLog] = map[*regexp.Regexp]int{} + tl.logs[warningLog] = map[*regexp.Regexp]int{} +} + +// ExpectError declares an error to be expected. For the next test, the first +// error log matching the expression (using FindString) will not cause the test +// to fail. "For the next test" includes all the time until the next call to +// Update(). Note that if an expected error is not encountered, this will cause +// the test to fail. +func ExpectError(expr string) { + expectLogsN(expr, 1, errorLog) +} + +// ExpectErrorN declares an error to be expected n times. +func ExpectErrorN(expr string, n int) { + expectLogsN(expr, n, errorLog) +} + +// ExpectWarning declares a warning to be expected. +func ExpectWarning(expr string) { + expectLogsN(expr, 1, warningLog) +} + +func expectLogsN(expr string, n int, logType logType) { + tLogr.mu.Lock() + defer tLogr.mu.Unlock() + re, err := regexp.Compile(expr) + if err != nil { + tLogr.t.Error(err) + return + } + tLogr.logs[logType][re] += n +} + +// endTest checks if expected errors were not encountered. +func (tl *tLogger) endTest(t *testing.T) { + tl.mu.Lock() + defer tl.mu.Unlock() + for re, count := range tl.logs[errorLog] { + if count > 0 { + t.Errorf("Expected error '%v' not encountered", re.String()) + } + } + for re, count := range tl.logs[warningLog] { + if count > 0 { + t.Errorf("Expected warning '%v' not encountered", re.String()) + } + } + tl.logs[errorLog] = map[*regexp.Regexp]int{} + tl.logs[warningLog] = map[*regexp.Regexp]int{} +} + +// expected determines if the log string of the particular type is protected or +// not. +func (tl *tLogger) expected(s string, logType logType) bool { + for re, count := range tl.logs[logType] { + if re.FindStringIndex(s) != nil { + tl.logs[logType][re]-- + if count <= 1 { + delete(tl.logs[logType], re) + } + return true + } + } + return false +} + +func (tl *tLogger) Info(args ...any) { + tl.log(infoLog, 0, "", args...) +} + +func (tl *tLogger) Infoln(args ...any) { + tl.log(infoLog, 0, "", args...) +} + +func (tl *tLogger) Infof(format string, args ...any) { + tl.log(infoLog, 0, format, args...) +} + +func (tl *tLogger) InfoDepth(depth int, args ...any) { + tl.log(infoLog, depth, "", args...) +} + +func (tl *tLogger) Warning(args ...any) { + tl.log(warningLog, 0, "", args...) +} + +func (tl *tLogger) Warningln(args ...any) { + tl.log(warningLog, 0, "", args...) +} + +func (tl *tLogger) Warningf(format string, args ...any) { + tl.log(warningLog, 0, format, args...) +} + +func (tl *tLogger) WarningDepth(depth int, args ...any) { + tl.log(warningLog, depth, "", args...) +} + +func (tl *tLogger) Error(args ...any) { + tl.log(errorLog, 0, "", args...) +} + +func (tl *tLogger) Errorln(args ...any) { + tl.log(errorLog, 0, "", args...) +} + +func (tl *tLogger) Errorf(format string, args ...any) { + tl.log(errorLog, 0, format, args...) +} + +func (tl *tLogger) ErrorDepth(depth int, args ...any) { + tl.log(errorLog, depth, "", args...) +} + +func (tl *tLogger) Fatal(args ...any) { + tl.log(fatalLog, 0, "", args...) +} + +func (tl *tLogger) Fatalln(args ...any) { + tl.log(fatalLog, 0, "", args...) +} + +func (tl *tLogger) Fatalf(format string, args ...any) { + tl.log(fatalLog, 0, format, args...) +} + +func (tl *tLogger) FatalDepth(depth int, args ...any) { + tl.log(fatalLog, depth, "", args...) +} + +func (tl *tLogger) V(l int) bool { + return l <= tl.v +} diff --git a/internal/grpctest/tlogger_test.go b/internal/grpctest/tlogger_test.go new file mode 100644 index 0000000..3bb99a8 --- /dev/null +++ b/internal/grpctest/tlogger_test.go @@ -0,0 +1,79 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpctest + +import ( + "testing" + + "google.golang.org/grpc/grpclog" +) + +type s struct { + Tester +} + +func Test(t *testing.T) { + RunSubTests(t, s{}) +} + +func (s) TestInfo(*testing.T) { + grpclog.Info("Info", "message.") +} + +func (s) TestInfoln(*testing.T) { + grpclog.Infoln("Info", "message.") +} + +func (s) TestInfof(*testing.T) { + grpclog.Infof("%v %v.", "Info", "message") +} + +func (s) TestInfoDepth(*testing.T) { + grpclog.InfoDepth(0, "Info", "depth", "message.") +} + +func (s) TestWarning(*testing.T) { + grpclog.Warning("Warning", "message.") +} + +func (s) TestWarningln(*testing.T) { + grpclog.Warningln("Warning", "message.") +} + +func (s) TestWarningf(*testing.T) { + grpclog.Warningf("%v %v.", "Warning", "message") +} + +func (s) TestWarningDepth(*testing.T) { + grpclog.WarningDepth(0, "Warning", "depth", "message.") +} + +func (s) TestError(*testing.T) { + const numErrors = 10 + ExpectError("Expected error") + ExpectError("Expected ln error") + ExpectError("Expected formatted error") + ExpectErrorN("Expected repeated error", numErrors) + grpclog.Error("Expected", "error") + grpclog.Errorln("Expected", "ln", "error") + grpclog.Errorf("%v %v %v", "Expected", "formatted", "error") + for i := 0; i < numErrors; i++ { + grpclog.Error("Expected repeated error") + } +} diff --git a/internal/grpcutil/compressor.go b/internal/grpcutil/compressor.go new file mode 100644 index 0000000..e8d8669 --- /dev/null +++ b/internal/grpcutil/compressor.go @@ -0,0 +1,42 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "strings" +) + +// RegisteredCompressorNames holds names of the registered compressors. +var RegisteredCompressorNames []string + +// IsCompressorNameRegistered returns true when name is available in registry. +func IsCompressorNameRegistered(name string) bool { + for _, compressor := range RegisteredCompressorNames { + if compressor == name { + return true + } + } + return false +} + +// RegisteredCompressors returns a string of registered compressor names +// separated by comma. +func RegisteredCompressors() string { + return strings.Join(RegisteredCompressorNames, ",") +} diff --git a/internal/grpcutil/compressor_test.go b/internal/grpcutil/compressor_test.go new file mode 100644 index 0000000..57857ed --- /dev/null +++ b/internal/grpcutil/compressor_test.go @@ -0,0 +1,32 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "testing" +) + +func TestRegisteredCompressors(t *testing.T) { + defer func(c []string) { RegisteredCompressorNames = c }(RegisteredCompressorNames) + RegisteredCompressorNames = []string{"gzip", "snappy"} + if got, want := RegisteredCompressors(), "gzip,snappy"; got != want { + t.Fatalf("Unexpected compressors got:%s, want:%s", got, want) + + } +} diff --git a/internal/grpcutil/encode_duration.go b/internal/grpcutil/encode_duration.go new file mode 100644 index 0000000..1cc43fc --- /dev/null +++ b/internal/grpcutil/encode_duration.go @@ -0,0 +1,62 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "strconv" + "time" +) + +const maxTimeoutValue int64 = 100000000 - 1 + +// div does integer division and round-up the result. Note that this is +// equivalent to (d+r-1)/r but has less chance to overflow. +func div(d, r time.Duration) int64 { + if d%r > 0 { + return int64(d/r + 1) + } + return int64(d / r) +} + +// EncodeDuration encodes the duration to the format grpc-timeout header +// accepts. +// +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests +func EncodeDuration(t time.Duration) string { + if t <= 0 { + return "0n" + } + if d := div(t, time.Nanosecond); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "n" + } + if d := div(t, time.Microsecond); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "u" + } + if d := div(t, time.Millisecond); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "m" + } + if d := div(t, time.Second); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "S" + } + if d := div(t, time.Minute); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "M" + } + // Note that maxTimeoutValue * time.Hour > MaxInt64. + return strconv.FormatInt(div(t, time.Hour), 10) + "H" +} diff --git a/internal/grpcutil/encode_duration_test.go b/internal/grpcutil/encode_duration_test.go new file mode 100644 index 0000000..eea49e2 --- /dev/null +++ b/internal/grpcutil/encode_duration_test.go @@ -0,0 +1,51 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "testing" + "time" +) + +func TestEncodeDuration(t *testing.T) { + for _, test := range []struct { + in string + out string + }{ + {"12345678ns", "12345678n"}, + {"123456789ns", "123457u"}, + {"12345678us", "12345678u"}, + {"123456789us", "123457m"}, + {"12345678ms", "12345678m"}, + {"123456789ms", "123457S"}, + {"12345678s", "12345678S"}, + {"123456789s", "2057614M"}, + {"12345678m", "12345678M"}, + {"123456789m", "2057614H"}, + } { + d, err := time.ParseDuration(test.in) + if err != nil { + t.Fatalf("failed to parse duration string %s: %v", test.in, err) + } + out := EncodeDuration(d) + if out != test.out { + t.Fatalf("timeoutEncode(%s) = %s, want %s", test.in, out, test.out) + } + } +} diff --git a/internal/grpcutil/grpcutil.go b/internal/grpcutil/grpcutil.go new file mode 100644 index 0000000..e2f948e --- /dev/null +++ b/internal/grpcutil/grpcutil.go @@ -0,0 +1,20 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpcutil provides utility functions used across the gRPC codebase. +package grpcutil diff --git a/internal/grpcutil/metadata.go b/internal/grpcutil/metadata.go new file mode 100644 index 0000000..6f22bd8 --- /dev/null +++ b/internal/grpcutil/metadata.go @@ -0,0 +1,40 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "context" + + "google.golang.org/grpc/metadata" +) + +type mdExtraKey struct{} + +// WithExtraMetadata creates a new context with incoming md attached. +func WithExtraMetadata(ctx context.Context, md metadata.MD) context.Context { + return context.WithValue(ctx, mdExtraKey{}, md) +} + +// ExtraMetadata returns the incoming metadata in ctx if it exists. The +// returned MD should not be modified. Writing to it may cause races. +// Modification should be made to copies of the returned MD. +func ExtraMetadata(ctx context.Context) (md metadata.MD, ok bool) { + md, ok = ctx.Value(mdExtraKey{}).(metadata.MD) + return +} diff --git a/internal/grpcutil/method.go b/internal/grpcutil/method.go new file mode 100644 index 0000000..683d195 --- /dev/null +++ b/internal/grpcutil/method.go @@ -0,0 +1,88 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "errors" + "strings" +) + +// ParseMethod splits service and method from the input. It expects format +// "/service/method". +func ParseMethod(methodName string) (service, method string, _ error) { + if !strings.HasPrefix(methodName, "/") { + return "", "", errors.New("invalid method name: should start with /") + } + methodName = methodName[1:] + + pos := strings.LastIndex(methodName, "/") + if pos < 0 { + return "", "", errors.New("invalid method name: suffix /method is missing") + } + return methodName[:pos], methodName[pos+1:], nil +} + +// baseContentType is the base content-type for gRPC. This is a valid +// content-type on its own, but can also include a content-subtype such as +// "proto" as a suffix after "+" or ";". See +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests +// for more details. +const baseContentType = "application/grpc" + +// ContentSubtype returns the content-subtype for the given content-type. The +// given content-type must be a valid content-type that starts with +// "application/grpc". A content-subtype will follow "application/grpc" after a +// "+" or ";". See +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +// +// If contentType is not a valid content-type for gRPC, the boolean +// will be false, otherwise true. If content-type == "application/grpc", +// "application/grpc+", or "application/grpc;", the boolean will be true, +// but no content-subtype will be returned. +// +// contentType is assumed to be lowercase already. +func ContentSubtype(contentType string) (string, bool) { + if contentType == baseContentType { + return "", true + } + if !strings.HasPrefix(contentType, baseContentType) { + return "", false + } + // guaranteed since != baseContentType and has baseContentType prefix + switch contentType[len(baseContentType)] { + case '+', ';': + // this will return true for "application/grpc+" or "application/grpc;" + // which the previous validContentType function tested to be valid, so we + // just say that no content-subtype is specified in this case + return contentType[len(baseContentType)+1:], true + default: + return "", false + } +} + +// ContentType builds full content type with the given sub-type. +// +// contentSubtype is assumed to be lowercase +func ContentType(contentSubtype string) string { + if contentSubtype == "" { + return baseContentType + } + return baseContentType + "+" + contentSubtype +} diff --git a/internal/grpcutil/method_test.go b/internal/grpcutil/method_test.go new file mode 100644 index 0000000..36c786c --- /dev/null +++ b/internal/grpcutil/method_test.go @@ -0,0 +1,69 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "testing" +) + +func TestParseMethod(t *testing.T) { + testCases := []struct { + methodName string + wantService string + wantMethod string + wantError bool + }{ + {methodName: "/s/m", wantService: "s", wantMethod: "m", wantError: false}, + {methodName: "/p.s/m", wantService: "p.s", wantMethod: "m", wantError: false}, + {methodName: "/p/s/m", wantService: "p/s", wantMethod: "m", wantError: false}, + {methodName: "/", wantError: true}, + {methodName: "/sm", wantError: true}, + {methodName: "", wantError: true}, + {methodName: "sm", wantError: true}, + } + for _, tc := range testCases { + s, m, err := ParseMethod(tc.methodName) + if (err != nil) != tc.wantError || s != tc.wantService || m != tc.wantMethod { + t.Errorf("ParseMethod(%s) = (%s, %s, %v), want (%s, %s, %v)", tc.methodName, s, m, err, tc.wantService, tc.wantMethod, tc.wantError) + } + } +} + +func TestContentSubtype(t *testing.T) { + tests := []struct { + contentType string + want string + wantValid bool + }{ + {"application/grpc", "", true}, + {"application/grpc+", "", true}, + {"application/grpc+blah", "blah", true}, + {"application/grpc;", "", true}, + {"application/grpc;blah", "blah", true}, + {"application/grpcd", "", false}, + {"application/grpd", "", false}, + {"application/grp", "", false}, + } + for _, tt := range tests { + got, gotValid := ContentSubtype(tt.contentType) + if got != tt.want || gotValid != tt.wantValid { + t.Errorf("contentSubtype(%q) = (%v, %v); want (%v, %v)", tt.contentType, got, gotValid, tt.want, tt.wantValid) + } + } +} diff --git a/internal/hierarchy/hierarchy.go b/internal/hierarchy/hierarchy.go new file mode 100644 index 0000000..979d4cd --- /dev/null +++ b/internal/hierarchy/hierarchy.go @@ -0,0 +1,108 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package hierarchy contains functions to set and get hierarchy string from +// addresses. +// +// This package is experimental. +package hierarchy + +import ( + "google.golang.org/grpc/resolver" +) + +type pathKeyType string + +const pathKey = pathKeyType("grpc.internal.address.hierarchical_path") + +type pathValue []string + +func (p pathValue) Equal(o any) bool { + op, ok := o.(pathValue) + if !ok { + return false + } + if len(op) != len(p) { + return false + } + for i, v := range p { + if v != op[i] { + return false + } + } + return true +} + +// FromEndpoint returns the hierarchical path of endpoint. +func FromEndpoint(endpoint resolver.Endpoint) []string { + path, _ := endpoint.Attributes.Value(pathKey).(pathValue) + return path +} + +// SetInEndpoint overrides the hierarchical path in endpoint with path. +func SetInEndpoint(endpoint resolver.Endpoint, path []string) resolver.Endpoint { + endpoint.Attributes = endpoint.Attributes.WithValue(pathKey, pathValue(path)) + return endpoint +} + +// Group splits a slice of endpoints into groups based on +// the first hierarchy path. The first hierarchy path will be removed from the +// result. +// +// Input: +// [ +// +// {endpoint0, path: [p0, wt0]} +// {endpoint1, path: [p0, wt1]} +// {endpoint2, path: [p1, wt2]} +// {endpoint3, path: [p1, wt3]} +// +// ] +// +// Endpoints will be split into p0/p1, and the p0/p1 will be removed from the +// path. +// +// Output: +// +// { +// p0: [ +// {endpoint0, path: [wt0]}, +// {endpoint1, path: [wt1]}, +// ], +// p1: [ +// {endpoint2, path: [wt2]}, +// {endpoint3, path: [wt3]}, +// ], +// } +// +// If hierarchical path is not set, or has no path in it, the endpoint is +// dropped. +func Group(endpoints []resolver.Endpoint) map[string][]resolver.Endpoint { + ret := make(map[string][]resolver.Endpoint) + for _, endpoint := range endpoints { + oldPath := FromEndpoint(endpoint) + if len(oldPath) == 0 { + continue + } + curPath := oldPath[0] + newPath := oldPath[1:] + newEndpoint := SetInEndpoint(endpoint, newPath) + ret[curPath] = append(ret[curPath], newEndpoint) + } + return ret +} diff --git a/internal/hierarchy/hierarchy_ext_test.go b/internal/hierarchy/hierarchy_ext_test.go new file mode 100644 index 0000000..a75f419 --- /dev/null +++ b/internal/hierarchy/hierarchy_ext_test.go @@ -0,0 +1,174 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package hierarchy_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/internal/hierarchy" + "google.golang.org/grpc/resolver" +) + +func TestFromEndpoint(t *testing.T) { + tests := []struct { + name string + ep resolver.Endpoint + want []string + }{ + { + name: "not set", + ep: resolver.Endpoint{}, + want: nil, + }, + { + name: "set", + ep: hierarchy.SetInEndpoint(resolver.Endpoint{}, []string{"a", "b"}), + want: []string{"a", "b"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := hierarchy.FromEndpoint(tt.ep); !cmp.Equal(got, tt.want) { + t.Errorf("FromEndpoint() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSetInEndpoint(t *testing.T) { + tests := []struct { + name string + ep resolver.Endpoint + path []string + }{ + { + name: "before is not set", + ep: resolver.Endpoint{}, + path: []string{"a", "b"}, + }, + { + name: "before is set", + ep: hierarchy.SetInEndpoint(resolver.Endpoint{}, []string{"before", "a", "b"}), + path: []string{"a", "b"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + newEP := hierarchy.SetInEndpoint(tt.ep, tt.path) + newPath := hierarchy.FromEndpoint(newEP) + if !cmp.Equal(newPath, tt.path) { + t.Errorf("path after SetInEndpoint() = %v, want %v", newPath, tt.path) + } + }) + } +} + +func TestGroup(t *testing.T) { + tests := []struct { + name string + eps []resolver.Endpoint + want map[string][]resolver.Endpoint + }{ + { + name: "all with hierarchy", + eps: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "a0"}}}, []string{"a"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "a1"}}}, []string{"a"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "b0"}}}, []string{"b"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "b1"}}}, []string{"b"}), + }, + want: map[string][]resolver.Endpoint{ + "a": { + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "a0"}}}, nil), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "a1"}}}, nil), + }, + "b": { + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "b0"}}}, nil), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "b1"}}}, nil), + }, + }, + }, + { + // Endpoints without hierarchy are ignored. + name: "without hierarchy", + eps: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "a0"}}}, []string{"a"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "a1"}}}, []string{"a"}), + {Addresses: []resolver.Address{{Addr: "b0"}}, Attributes: nil}, + {Addresses: []resolver.Address{{Addr: "b1"}}, Attributes: nil}, + }, + want: map[string][]resolver.Endpoint{ + "a": { + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "a0"}}}, nil), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "a1"}}}, nil), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := hierarchy.Group(tt.eps); !cmp.Equal(got, tt.want, cmp.AllowUnexported(attributes.Attributes{})) { + t.Errorf("Group() = %v, want %v", got, tt.want) + t.Errorf("diff: %v", cmp.Diff(got, tt.want, cmp.AllowUnexported(attributes.Attributes{}))) + } + }) + } +} + +func TestGroupE2E(t *testing.T) { + testHierarchy := map[string]map[string][]string{ + "p0": { + "wt0": {"addr0", "addr1"}, + "wt1": {"addr2", "addr3"}, + }, + "p1": { + "wt10": {"addr10", "addr11"}, + "wt11": {"addr12", "addr13"}, + }, + } + + var epsWithHierarchy []resolver.Endpoint + for p, wts := range testHierarchy { + path1 := []string{p} + for wt, addrs := range wts { + path2 := append([]string(nil), path1...) + path2 = append(path2, wt) + for _, addr := range addrs { + a := hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: addr}}}, path2) + epsWithHierarchy = append(epsWithHierarchy, a) + } + } + } + + gotHierarchy := make(map[string]map[string][]string) + for p1, wts := range hierarchy.Group(epsWithHierarchy) { + gotHierarchy[p1] = make(map[string][]string) + for p2, eps := range hierarchy.Group(wts) { + for _, ep := range eps { + gotHierarchy[p1][p2] = append(gotHierarchy[p1][p2], ep.Addresses[0].Addr) + } + } + } + + if !cmp.Equal(gotHierarchy, testHierarchy) { + t.Errorf("diff: %v", cmp.Diff(gotHierarchy, testHierarchy)) + } +} diff --git a/internal/idle/idle.go b/internal/idle/idle.go new file mode 100644 index 0000000..d3cd24f --- /dev/null +++ b/internal/idle/idle.go @@ -0,0 +1,289 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package idle contains a component for managing idleness (entering and exiting) +// based on RPC activity. +package idle + +import ( + "math" + "sync" + "sync/atomic" + "time" +) + +// For overriding in unit tests. +var timeAfterFunc = func(d time.Duration, f func()) *time.Timer { + return time.AfterFunc(d, f) +} + +// ClientConn is the functionality provided by grpc.ClientConn to enter and exit +// from idle mode. +type ClientConn interface { + ExitIdleMode() + EnterIdleMode() +} + +// Manager implements idleness detection and calls the ClientConn to enter/exit +// idle mode when appropriate. Must be created by NewManager. +type Manager struct { + // State accessed atomically. + lastCallEndTime int64 // Unix timestamp in nanos; time when the most recent RPC completed. + activeCallsCount int32 // Count of active RPCs; -math.MaxInt32 means channel is idle or is trying to get there. + activeSinceLastTimerCheck int32 // Boolean; True if there was an RPC since the last timer callback. + closed int32 // Boolean; True when the manager is closed. + + // Can be accessed without atomics or mutex since these are set at creation + // time and read-only after that. + cc ClientConn // Functionality provided by grpc.ClientConn. + timeout time.Duration + + // idleMu is used to guarantee mutual exclusion in two scenarios: + // - Opposing intentions: + // - a: Idle timeout has fired and handleIdleTimeout() is trying to put + // the channel in idle mode because the channel has been inactive. + // - b: At the same time an RPC is made on the channel, and OnCallBegin() + // is trying to prevent the channel from going idle. + // - Competing intentions: + // - The channel is in idle mode and there are multiple RPCs starting at + // the same time, all trying to move the channel out of idle. Only one + // of them should succeed in doing so, while the other RPCs should + // piggyback on the first one and be successfully handled. + idleMu sync.RWMutex + actuallyIdle bool + timer *time.Timer +} + +// NewManager creates a new idleness manager implementation for the +// given idle timeout. It begins in idle mode. +func NewManager(cc ClientConn, timeout time.Duration) *Manager { + return &Manager{ + cc: cc, + timeout: timeout, + actuallyIdle: true, + activeCallsCount: -math.MaxInt32, + } +} + +// resetIdleTimerLocked resets the idle timer to the given duration. Called +// when exiting idle mode or when the timer fires and we need to reset it. +func (m *Manager) resetIdleTimerLocked(d time.Duration) { + if m.isClosed() || m.timeout == 0 || m.actuallyIdle { + return + } + + // It is safe to ignore the return value from Reset() because this method is + // only ever called from the timer callback or when exiting idle mode. + if m.timer != nil { + m.timer.Stop() + } + m.timer = timeAfterFunc(d, m.handleIdleTimeout) +} + +func (m *Manager) resetIdleTimer(d time.Duration) { + m.idleMu.Lock() + defer m.idleMu.Unlock() + m.resetIdleTimerLocked(d) +} + +// handleIdleTimeout is the timer callback that is invoked upon expiry of the +// configured idle timeout. The channel is considered inactive if there are no +// ongoing calls and no RPC activity since the last time the timer fired. +func (m *Manager) handleIdleTimeout() { + if m.isClosed() { + return + } + + if atomic.LoadInt32(&m.activeCallsCount) > 0 { + m.resetIdleTimer(m.timeout) + return + } + + // There has been activity on the channel since we last got here. Reset the + // timer and return. + if atomic.LoadInt32(&m.activeSinceLastTimerCheck) == 1 { + // Set the timer to fire after a duration of idle timeout, calculated + // from the time the most recent RPC completed. + atomic.StoreInt32(&m.activeSinceLastTimerCheck, 0) + m.resetIdleTimer(time.Duration(atomic.LoadInt64(&m.lastCallEndTime)-time.Now().UnixNano()) + m.timeout) + return + } + + // Now that we've checked that there has been no activity, attempt to enter + // idle mode, which is very likely to succeed. + if m.tryEnterIdleMode(true) { + // Successfully entered idle mode. No timer needed until we exit idle. + return + } + + // Failed to enter idle mode due to a concurrent RPC that kept the channel + // active, or because of an error from the channel. Undo the attempt to + // enter idle, and reset the timer to try again later. + m.resetIdleTimer(m.timeout) +} + +// tryEnterIdleMode instructs the channel to enter idle mode. But before +// that, it performs a last minute check to ensure that no new RPC has come in, +// making the channel active. +// +// checkActivity controls if a check for RPC activity, since the last time the +// idle_timeout fired, is made. + +// Return value indicates whether or not the channel moved to idle mode. +// +// Holds idleMu which ensures mutual exclusion with exitIdleMode. +func (m *Manager) tryEnterIdleMode(checkActivity bool) bool { + // Setting the activeCallsCount to -math.MaxInt32 indicates to OnCallBegin() + // that the channel is either in idle mode or is trying to get there. + if !atomic.CompareAndSwapInt32(&m.activeCallsCount, 0, -math.MaxInt32) { + // This CAS operation can fail if an RPC started after we checked for + // activity in the timer handler, or one was ongoing from before the + // last time the timer fired, or if a test is attempting to enter idle + // mode without checking. In all cases, abort going into idle mode. + return false + } + // N.B. if we fail to enter idle mode after this, we must re-add + // math.MaxInt32 to m.activeCallsCount. + + m.idleMu.Lock() + defer m.idleMu.Unlock() + + if atomic.LoadInt32(&m.activeCallsCount) != -math.MaxInt32 { + // We raced and lost to a new RPC. Very rare, but stop entering idle. + atomic.AddInt32(&m.activeCallsCount, math.MaxInt32) + return false + } + if checkActivity && atomic.LoadInt32(&m.activeSinceLastTimerCheck) == 1 { + // A very short RPC could have come in (and also finished) after we + // checked for calls count and activity in handleIdleTimeout(), but + // before the CAS operation. So, we need to check for activity again. + atomic.AddInt32(&m.activeCallsCount, math.MaxInt32) + return false + } + + // No new RPCs have come in since we set the active calls count value to + // -math.MaxInt32. And since we have the lock, it is safe to enter idle mode + // unconditionally now. + m.cc.EnterIdleMode() + m.actuallyIdle = true + return true +} + +// EnterIdleModeForTesting instructs the channel to enter idle mode. +func (m *Manager) EnterIdleModeForTesting() { + m.tryEnterIdleMode(false) +} + +// OnCallBegin is invoked at the start of every RPC. +func (m *Manager) OnCallBegin() { + if m.isClosed() { + return + } + + if atomic.AddInt32(&m.activeCallsCount, 1) > 0 { + // Channel is not idle now. Set the activity bit and allow the call. + atomic.StoreInt32(&m.activeSinceLastTimerCheck, 1) + return + } + + // Channel is either in idle mode or is in the process of moving to idle + // mode. Attempt to exit idle mode to allow this RPC. + m.ExitIdleMode() + atomic.StoreInt32(&m.activeSinceLastTimerCheck, 1) +} + +// ExitIdleMode instructs m to call the ClientConn's ExitIdleMode and update its +// internal state. +func (m *Manager) ExitIdleMode() { + // Holds idleMu which ensures mutual exclusion with tryEnterIdleMode. + m.idleMu.Lock() + defer m.idleMu.Unlock() + + if m.isClosed() || !m.actuallyIdle { + // This can happen in three scenarios: + // - handleIdleTimeout() set the calls count to -math.MaxInt32 and called + // tryEnterIdleMode(). But before the latter could grab the lock, an RPC + // came in and OnCallBegin() noticed that the calls count is negative. + // - Channel is in idle mode, and multiple new RPCs come in at the same + // time, all of them notice a negative calls count in OnCallBegin and get + // here. The first one to get the lock would get the channel to exit idle. + // - Channel is not in idle mode, and the user calls Connect which calls + // m.ExitIdleMode. + // + // In any case, there is nothing to do here. + return + } + + m.cc.ExitIdleMode() + + // Undo the idle entry process. This also respects any new RPC attempts. + atomic.AddInt32(&m.activeCallsCount, math.MaxInt32) + m.actuallyIdle = false + + // Start a new timer to fire after the configured idle timeout. + m.resetIdleTimerLocked(m.timeout) +} + +// UnsafeSetNotIdle instructs the Manager to update its internal state to +// reflect the reality that the channel is no longer in IDLE mode. +// +// N.B. This method is intended only for internal use by the gRPC client +// when it exits IDLE mode **manually** from `Dial`. The callsite must ensure: +// - The channel was **actually in IDLE mode** immediately prior to the call. +// - There is **no concurrent activity** that could cause the channel to exit +// IDLE mode *naturally* at the same time. +func (m *Manager) UnsafeSetNotIdle() { + m.idleMu.Lock() + defer m.idleMu.Unlock() + + atomic.AddInt32(&m.activeCallsCount, math.MaxInt32) + m.actuallyIdle = false + m.resetIdleTimerLocked(m.timeout) +} + +// OnCallEnd is invoked at the end of every RPC. +func (m *Manager) OnCallEnd() { + if m.isClosed() { + return + } + + // Record the time at which the most recent call finished. + atomic.StoreInt64(&m.lastCallEndTime, time.Now().UnixNano()) + + // Decrement the active calls count. This count can temporarily go negative + // when the timer callback is in the process of moving the channel to idle + // mode, but one or more RPCs come in and complete before the timer callback + // can get done with the process of moving to idle mode. + atomic.AddInt32(&m.activeCallsCount, -1) +} + +func (m *Manager) isClosed() bool { + return atomic.LoadInt32(&m.closed) == 1 +} + +// Close stops the timer associated with the Manager, if it exists. +func (m *Manager) Close() { + atomic.StoreInt32(&m.closed, 1) + + m.idleMu.Lock() + if m.timer != nil { + m.timer.Stop() + m.timer = nil + } + m.idleMu.Unlock() +} diff --git a/internal/idle/idle_e2e_test.go b/internal/idle/idle_e2e_test.go new file mode 100644 index 0000000..3d9b93a --- /dev/null +++ b/internal/idle/idle_e2e_test.go @@ -0,0 +1,589 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package idle_test + +import ( + "context" + "fmt" + "io" + "strings" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func init() { + channelz.TurnOn() +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 100 * time.Millisecond + defaultTestShortIdleTimeout = 500 * time.Millisecond +) + +// channelzTraceEventFound looks up the top-channels in channelz (expects a +// single one), and checks if there is a trace event on the channel matching the +// provided description string. +func channelzTraceEventFound(ctx context.Context, wantDesc string) error { + for ctx.Err() == nil { + tcs, _ := channelz.GetTopChannels(0, 0) + if l := len(tcs); l != 1 { + return fmt.Errorf("when looking for channelz trace event with description %q, found %d top-level channels, want 1", wantDesc, l) + } + trace := tcs[0].Trace() + if trace == nil { + return fmt.Errorf("when looking for channelz trace event with description %q, no trace events found for top-level channel", wantDesc) + } + + for _, e := range trace.Events { + if strings.Contains(e.Desc, wantDesc) { + return nil + } + } + } + return fmt.Errorf("when looking for channelz trace event with description %q, %w", wantDesc, ctx.Err()) +} + +// Registers a wrapped round_robin LB policy for the duration of this test that +// retains all the functionality of the round_robin LB policy and makes the +// balancer close event available for inspection by the test. +// +// Returns a channel that gets pinged when the balancer is closed. +func registerWrappedRoundRobinPolicy(t *testing.T) chan struct{} { + rrBuilder := balancer.Get(roundrobin.Name) + closeCh := make(chan struct{}, 1) + stub.Register(roundrobin.Name, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = rrBuilder.Build(bd.ClientConn, bd.BuildOptions) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + select { + case closeCh <- struct{}{}: + default: + } + bd.ChildBalancer.Close() + }, + }) + t.Cleanup(func() { balancer.Register(rrBuilder) }) + + return closeCh +} + +// Tests the case where channel idleness is disabled by passing an idle_timeout +// of 0. Verifies that a READY channel with no RPCs does not move to IDLE. +func (s) TestChannelIdleness_Disabled_NoActivity(t *testing.T) { + closeCh := registerWrappedRoundRobinPolicy(t) + + // Create a ClientConn with idle_timeout set to 0. + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(0), // Disable idleness. + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + + // Start a test backend and push an address update via the resolver. + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Verify that the ClientConn moves to READY. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Verify that the ClientConn stays in READY. + sCtx, sCancel := context.WithTimeout(ctx, 3*defaultTestShortIdleTimeout) + defer sCancel() + testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready) + + // Verify that the LB policy is not closed which is expected to happen when + // the channel enters IDLE. + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortIdleTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-closeCh: + t.Fatal("LB policy closed when expected not to") + } +} + +// Tests the case where channel idleness is enabled by passing a small value for +// idle_timeout. Verifies that a READY channel with no RPCs moves to IDLE, and +// the connection to the backend is closed. +func (s) TestChannelIdleness_Enabled_NoActivity(t *testing.T) { + closeCh := registerWrappedRoundRobinPolicy(t) + + // Create a ClientConn with a short idle_timeout. + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(defaultTestShortIdleTimeout), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + cc.Connect() + // Start a test backend and push an address update via the resolver. + lis := testutils.NewListenerWrapper(t, nil) + backend := stubserver.StartTestService(t, &stubserver.StubServer{Listener: lis}) + defer backend.Stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Verify that the ClientConn moves to READY. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Retrieve the wrapped conn from the listener. + v, err := lis.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to retrieve conn from test listener: %v", err) + } + conn := v.(*testutils.ConnWrapper) + + // Verify that the ClientConn moves to IDLE as there is no activity. + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Verify idleness related channelz events. + if err := channelzTraceEventFound(ctx, "entering idle mode"); err != nil { + t.Fatal(err) + } + + // Verify that the previously open connection is closed. + if _, err := conn.CloseCh.Receive(ctx); err != nil { + t.Fatalf("Failed when waiting for connection to be closed after channel entered IDLE: %v", err) + } + + // Verify that the LB policy is closed. + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for LB policy to be closed after the channel enters IDLE") + case <-closeCh: + } +} + +// Tests the case where channel idleness is enabled by passing a small value for +// idle_timeout. Verifies that a READY channel with an ongoing RPC stays READY. +func (s) TestChannelIdleness_Enabled_OngoingCall(t *testing.T) { + tests := []struct { + name string + makeRPC func(ctx context.Context, client testgrpc.TestServiceClient) error + }{ + { + name: "unary", + makeRPC: func(ctx context.Context, client testgrpc.TestServiceClient) error { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return fmt.Errorf("EmptyCall RPC failed: %v", err) + } + return nil + }, + }, + { + name: "streaming", + makeRPC: func(ctx context.Context, client testgrpc.TestServiceClient) error { + stream, err := client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall RPC failed: %v", err) + } + if _, err := stream.Recv(); err != nil && err != io.EOF { + t.Fatalf("stream.Recv() failed: %v", err) + } + return nil + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + closeCh := registerWrappedRoundRobinPolicy(t) + + // Create a ClientConn with a short idle_timeout. + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(defaultTestShortIdleTimeout), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + // Start a test backend that keeps the RPC call active by blocking + // on a channel that is closed by the test later on. + blockCh := make(chan struct{}) + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + <-blockCh + return &testpb.Empty{}, nil + }, + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + <-blockCh + return nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + defer backend.Stop() + + // Push an address update containing the address of the above + // backend via the manual resolver. + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Verify that the ClientConn moves to READY. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Spawn a goroutine to check for expected behavior while a blocking + // RPC all is made from the main test goroutine. + errCh := make(chan error, 1) + go func() { + defer close(blockCh) + + // Verify that the ClientConn stays in READY. + sCtx, sCancel := context.WithTimeout(ctx, 3*defaultTestShortIdleTimeout) + defer sCancel() + if cc.WaitForStateChange(sCtx, connectivity.Ready) { + errCh <- fmt.Errorf("state changed from %q to %q when no state change was expected", connectivity.Ready, cc.GetState()) + return + } + + // Verify that the LB policy is not closed which is expected to happen when + // the channel enters IDLE. + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortIdleTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-closeCh: + errCh <- fmt.Errorf("LB policy closed when expected not to") + } + errCh <- nil + }() + + if err := test.makeRPC(ctx, testgrpc.NewTestServiceClient(cc)); err != nil { + t.Fatalf("%s rpc failed: %v", test.name, err) + } + + select { + case err := <-errCh: + if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatalf("Timeout when trying to verify that an active RPC keeps channel from moving to IDLE") + } + }) + } +} + +// Tests the case where channel idleness is enabled by passing a small value for +// idle_timeout. Verifies that activity on a READY channel (frequent and short +// RPCs) keeps it from moving to IDLE. +func (s) TestChannelIdleness_Enabled_ActiveSinceLastCheck(t *testing.T) { + closeCh := registerWrappedRoundRobinPolicy(t) + + // Create a ClientConn with a short idle_timeout. + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(defaultTestShortIdleTimeout), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + // Start a test backend and push an address update via the resolver. + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Verify that the ClientConn moves to READY. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // For a duration of three times the configured idle timeout, making RPCs + // every now and then and ensure that the channel does not move out of + // READY. + sCtx, sCancel := context.WithTimeout(ctx, 3*defaultTestShortIdleTimeout) + defer sCancel() + go func() { + for ; sCtx.Err() == nil; <-time.After(defaultTestShortIdleTimeout / 4) { + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); err != nil { + // While iterating through this for loop, at some point in time, + // the context deadline will expire. It is safe to ignore that + // error code. + if status.Code(err) != codes.DeadlineExceeded { + t.Errorf("EmptyCall RPC failed: %v", err) + return + } + } + } + }() + + // Verify that the ClientConn stays in READY. + testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready) + + // Verify that the LB policy is not closed which is expected to happen when + // the channel enters IDLE. + select { + case <-sCtx.Done(): + case <-closeCh: + t.Fatal("LB policy closed when expected not to") + } +} + +// Tests the case where channel idleness is enabled by passing a small value for +// idle_timeout. Verifies that a READY channel with no RPCs moves to IDLE. Also +// verifies that a subsequent RPC on the IDLE channel kicks it out of IDLE. +func (s) TestChannelIdleness_Enabled_ExitIdleOnRPC(t *testing.T) { + closeCh := registerWrappedRoundRobinPolicy(t) + + // Start a test backend and set the bootstrap state of the resolver to + // include this address. This will ensure that when the resolver is + // restarted when exiting idle, it will push the same address to grpc again. + r := manual.NewBuilderWithScheme("whatever") + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Create a ClientConn with a short idle_timeout. + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(defaultTestShortIdleTimeout), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + // Verify that the ClientConn moves to READY. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Verify that the ClientConn moves to IDLE as there is no activity. + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Verify idleness related channelz events. + if err := channelzTraceEventFound(ctx, "entering idle mode"); err != nil { + t.Fatal(err) + } + + // Verify that the LB policy is closed. + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for LB policy to be closed after the channel enters IDLE") + case <-closeCh: + } + + // Make an RPC and ensure that it succeeds and moves the channel back to + // READY. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall RPC failed: %v", err) + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + if err := channelzTraceEventFound(ctx, "exiting idle mode"); err != nil { + t.Fatal(err) + } +} + +// Tests the case where channel idleness is enabled by passing a small value for +// idle_timeout. Simulates a race between the idle timer firing and RPCs being +// initiated, after a period of inactivity on the channel. +// +// After a period of inactivity (for the configured idle timeout duration), when +// RPCs are started, there are two possibilities: +// - the idle timer wins the race and puts the channel in idle. The RPCs then +// kick it out of idle. +// - the RPCs win the race, and therefore the channel never moves to idle. +// +// In either of these cases, all RPCs must succeed. +func (s) TestChannelIdleness_Enabled_IdleTimeoutRacesWithRPCs(t *testing.T) { + // Start a test backend and set the bootstrap state of the resolver to + // include this address. This will ensure that when the resolver is + // restarted when exiting idle, it will push the same address to grpc again. + r := manual.NewBuilderWithScheme("whatever") + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Create a ClientConn with a short idle_timeout. + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(defaultTestShortTimeout), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + // Verify that the ClientConn moves to READY. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("EmptyCall RPC failed: %v", err) + } + + // Make an RPC every defaultTestShortTimeout duration so as to race with the + // idle timeout. Whether the idle timeout wins the race or the RPC wins the + // race, RPCs must succeed. + for i := 0; i < 20; i++ { + <-time.After(defaultTestShortTimeout) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall RPC failed: %v", err) + } + t.Logf("Iteration %d succeeded", i) + } +} + +// Tests the case where the channel is IDLE and we call cc.Connect. +func (s) TestChannelIdleness_Connect(t *testing.T) { + // Start a test backend and set the bootstrap state of the resolver to + // include this address. This will ensure that when the resolver is + // restarted when exiting idle, it will push the same address to grpc again. + r := manual.NewBuilderWithScheme("whatever") + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Create a ClientConn with a short idle_timeout. + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(defaultTestShortIdleTimeout), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + // Verify that the ClientConn moves to IDLE. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Connect should exit channel idleness. + cc.Connect() + + // Verify that the ClientConn moves back to READY. + testutils.AwaitState(ctx, t, cc, connectivity.Ready) +} + +// TestChannelIdleness_RPCFailingBeforeAttempt_FinishesOnce verifies that an +// RPC which fails during stream creation, before any attempt is started, is +// reported as finished exactly once. +func (s) TestChannelIdleness_RPCFailingBeforeAttempt_FinishesOnce(t *testing.T) { + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + + cc, err := grpc.NewClient(fmt.Sprintf("dns:///%s", backend.Address), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + client := testgrpc.NewTestServiceClient(cc) + + // Warm-up RPC: moves the channel to READY and fires the resolver's + // first-resolution event, so that the next RPC does not block waiting for + // addresses and instead proceeds to fail during stream creation. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Issue an RPC with an already-canceled context so that it fails during + // stream creation, before any attempt is started, and count how many times + // gRPC reports it as finished. The contract is exactly once. + var finishCount atomic.Int32 + cancel() + _, err = client.EmptyCall(ctx, &testpb.Empty{}, grpc.OnFinish(func(error) { finishCount.Add(1) })) + if got, want := status.Code(err), codes.Canceled; got != want { + t.Fatalf("EmptyCall() with canceled context returned code %v, want %v (err: %v)", got, want, err) + } + if got := finishCount.Load(); got != 1 { + t.Fatalf("OnFinish callback invoked %d times, want exactly 1", got) + } +} diff --git a/internal/idle/idle_test.go b/internal/idle/idle_test.go new file mode 100644 index 0000000..acc95ba --- /dev/null +++ b/internal/idle/idle_test.go @@ -0,0 +1,378 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package idle + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" +) + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestIdleTimeout = 500 * time.Millisecond // A short idle_timeout for tests. + defaultTestShortTimeout = 10 * time.Millisecond // A small deadline to wait for events expected to not happen. +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type testEnforcer struct { + exitIdleCh chan struct{} + enterIdleCh chan struct{} +} + +func (ti *testEnforcer) ExitIdleMode() { + ti.exitIdleCh <- struct{}{} +} + +func (ti *testEnforcer) EnterIdleMode() { + ti.enterIdleCh <- struct{}{} +} + +func newTestEnforcer() *testEnforcer { + return &testEnforcer{ + exitIdleCh: make(chan struct{}, 1), + enterIdleCh: make(chan struct{}, 1), + } +} + +// overrideNewTimer overrides the new timer creation function by ensuring that a +// message is pushed on the returned channel everytime the timer fires. +func overrideNewTimer(t *testing.T) <-chan struct{} { + t.Helper() + + ch := make(chan struct{}, 1) + origTimeAfterFunc := timeAfterFunc + timeAfterFunc = func(d time.Duration, callback func()) *time.Timer { + return time.AfterFunc(d, func() { + select { + case ch <- struct{}{}: + default: + } + callback() + }) + } + t.Cleanup(func() { timeAfterFunc = origTimeAfterFunc }) + return ch +} + +// TestManager_Disabled tests the case where the idleness manager is +// disabled by passing an idle_timeout of 0. Verifies the following things: +// - timer callback does not fire +// - an RPC triggers a call to ExitIdleMode on the ClientConn +// - more calls to RPC termination (as compared to RPC initiation) does not +// result in an error log +func (s) TestManager_Disabled(t *testing.T) { + callbackCh := overrideNewTimer(t) + + // Create an idleness manager that is disabled because of idleTimeout being + // set to `0`. + enforcer := newTestEnforcer() + mgr := NewManager(enforcer, time.Duration(0)) + + // Ensure that the timer callback does not fire within a short deadline. + select { + case <-callbackCh: + t.Fatal("Idle timer callback fired when manager is disabled") + case <-time.After(defaultTestShortTimeout): + } + + // The first invocation of OnCallBegin() should lead to a call to + // ExitIdleMode() on the enforcer. + go mgr.OnCallBegin() + select { + case <-enforcer.exitIdleCh: + case <-time.After(defaultTestShortTimeout): + t.Fatal("Timeout waiting for channel to move out of idle mode") + } + + // If the number of calls to OnCallEnd() exceeds the number of calls to + // OnCallBegin(), the idleness manager is expected to throw an error log + // (which will cause our TestLogger to fail the test). But since the manager + // is disabled, this should not happen. + mgr.OnCallEnd() + mgr.OnCallEnd() + + // The idleness manager is explicitly not closed here. But since the manager + // is disabled, it will not start the run goroutine, and hence we expect the + // leak checker to not find any leaked goroutines. +} + +// TestManager_Enabled_TimerFires tests the case where the idle manager +// is enabled. Ensures that when there are no RPCs, the timer callback is +// invoked and the EnterIdleMode() method is invoked on the enforcer. +func (s) TestManager_Enabled_TimerFires(t *testing.T) { + callbackCh := overrideNewTimer(t) + + enforcer := newTestEnforcer() + mgr := NewManager(enforcer, time.Duration(defaultTestIdleTimeout)) + defer mgr.Close() + mgr.ExitIdleMode() + + // Ensure that the timer callback fires within an appropriate amount of time. + select { + case <-callbackCh: + case <-time.After(2 * defaultTestIdleTimeout): + t.Fatal("Timeout waiting for idle timer callback to fire") + } + + // Ensure that the channel moves to idle mode eventually. + select { + case <-enforcer.enterIdleCh: + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout waiting for channel to move to idle") + } +} + +// TestManager_Enabled_OngoingCall tests the case where the idle manager +// is enabled. Ensures that when there is an ongoing RPC, the channel does not +// enter idle mode. +func (s) TestManager_Enabled_OngoingCall(t *testing.T) { + callbackCh := overrideNewTimer(t) + + enforcer := newTestEnforcer() + mgr := NewManager(enforcer, time.Duration(defaultTestIdleTimeout)) + defer mgr.Close() + mgr.ExitIdleMode() + + // Fire up a goroutine that simulates an ongoing RPC that is terminated + // after the timer callback fires for the first time. + timerFired := make(chan struct{}) + go func() { + mgr.OnCallBegin() + <-timerFired + mgr.OnCallEnd() + }() + + // Ensure that the timer callback fires and unblock the above goroutine. + select { + case <-callbackCh: + close(timerFired) + case <-time.After(2 * defaultTestIdleTimeout): + t.Fatal("Timeout waiting for idle timer callback to fire") + } + + // The invocation of the timer callback should not put the channel in idle + // mode since we had an ongoing RPC. + select { + case <-enforcer.enterIdleCh: + t.Fatalf("EnterIdleMode() called on enforcer when active RPC exists") + case <-time.After(defaultTestShortTimeout): + } + + // Since we terminated the ongoing RPC and we have no other active RPCs, the + // channel must move to idle eventually. + select { + case <-enforcer.enterIdleCh: + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout waiting for channel to move to idle") + } +} + +// TestManager_Enabled_ActiveSinceLastCheck tests the case where the +// idle manager is enabled. Ensures that when there are active RPCs in the last +// period (even though there is no active call when the timer fires), the +// channel does not enter idle mode. +func (s) TestManager_Enabled_ActiveSinceLastCheck(t *testing.T) { + callbackCh := overrideNewTimer(t) + + enforcer := newTestEnforcer() + mgr := NewManager(enforcer, time.Duration(defaultTestIdleTimeout)) + defer mgr.Close() + mgr.ExitIdleMode() + + // Fire up a goroutine that simulates unary RPCs until the timer callback + // fires. + timerFired := make(chan struct{}) + go func() { + for ; ; <-time.After(defaultTestShortTimeout) { + mgr.OnCallBegin() + mgr.OnCallEnd() + + select { + case <-timerFired: + return + default: + } + } + }() + + // Ensure that the timer callback fires, and that we don't enter idle as + // part of this invocation of the timer callback, since we had some RPCs in + // this period. + select { + case <-callbackCh: + close(timerFired) + case <-time.After(2 * defaultTestIdleTimeout): + close(timerFired) + t.Fatal("Timeout waiting for idle timer callback to fire") + } + select { + case <-enforcer.enterIdleCh: + t.Fatalf("EnterIdleMode() called on enforcer when one RPC completed in the last period") + case <-time.After(defaultTestShortTimeout): + } + + // Since the unary RPC terminated and we have no other active RPCs, the + // channel must move to idle eventually. + select { + case <-enforcer.enterIdleCh: + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout waiting for channel to move to idle") + } +} + +// TestManager_Enabled_ExitIdleOnRPC tests the case where the idle +// manager is enabled. Ensures that the channel moves out of idle when an RPC is +// initiated. +func (s) TestManager_Enabled_ExitIdleOnRPC(t *testing.T) { + overrideNewTimer(t) + + enforcer := newTestEnforcer() + mgr := NewManager(enforcer, time.Duration(defaultTestIdleTimeout)) + defer mgr.Close() + + mgr.ExitIdleMode() + <-enforcer.exitIdleCh + // Ensure that the channel moves to idle since there are no RPCs. + select { + case <-enforcer.enterIdleCh: + case <-time.After(2 * defaultTestIdleTimeout): + t.Fatal("Timeout waiting for channel to move to idle mode") + } + + for i := 0; i < 100; i++ { + // A call to OnCallBegin and OnCallEnd simulates an RPC. + go func() { + mgr.OnCallBegin() + mgr.OnCallEnd() + }() + } + + // Ensure that the channel moves out of idle as a result of the above RPC. + select { + case <-enforcer.exitIdleCh: + case <-time.After(2 * defaultTestIdleTimeout): + t.Fatal("Timeout waiting for channel to move out of idle mode") + } + + // Ensure that only one call to exit idle mode is made to the CC. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + select { + case <-enforcer.exitIdleCh: + t.Fatal("More than one call to exit idle mode on the ClientConn; only one expected") + case <-sCtx.Done(): + } +} + +type racyState int32 + +const ( + stateInitial racyState = iota + stateEnteredIdle + stateExitedIdle + stateActiveRPCs +) + +// racyEnforcer is a test idleness enforcer used specifically to test the +// race between idle timeout and incoming RPCs. +type racyEnforcer struct { + t *testing.T + state *racyState // Accessed atomically. + started bool +} + +// ExitIdleMode sets the internal state to stateExitedIdle. We should only ever +// exit idle when we are currently in idle. +func (ri *racyEnforcer) ExitIdleMode() { + // Set only on the initial ExitIdleMode + if ri.started == false { + if !atomic.CompareAndSwapInt32((*int32)(ri.state), int32(stateInitial), int32(stateInitial)) { + ri.t.Errorf("idleness enforcer's first ExitIdleMode after EnterIdleMode") + return + } + ri.started = true + return + } + if !atomic.CompareAndSwapInt32((*int32)(ri.state), int32(stateEnteredIdle), int32(stateExitedIdle)) { + ri.t.Errorf("idleness enforcer asked to exit idle when it did not enter idle earlier") + return + } +} + +// EnterIdleMode attempts to set the internal state to stateEnteredIdle. We should only ever enter idle before RPCs start. +func (ri *racyEnforcer) EnterIdleMode() { + if !atomic.CompareAndSwapInt32((*int32)(ri.state), int32(stateInitial), int32(stateEnteredIdle)) { + ri.t.Errorf("idleness enforcer asked to enter idle after rpcs started") + } +} + +// TestManager_IdleTimeoutRacesWithOnCallBegin tests the case where firing of +// the idle timeout races with an incoming RPC. The test verifies that if the +// timer callback wins the race and puts the channel in idle, the RPCs can kick +// it out of idle. And if the RPCs win the race and keep the channel active, +// then the timer callback should not attempt to put the channel in idle mode. +func (s) TestManager_IdleTimeoutRacesWithOnCallBegin(t *testing.T) { + // Run multiple iterations to simulate different possibilities. + for i := 0; i < 20; i++ { + t.Run(fmt.Sprintf("iteration=%d", i), func(t *testing.T) { + var idlenessState racyState + enforcer := &racyEnforcer{t: t, state: &idlenessState} + + // Configure a large idle timeout so that we can control the + // race between the timer callback and RPCs. + mgr := NewManager(enforcer, time.Duration(10*time.Minute)) + defer mgr.Close() + mgr.ExitIdleMode() + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + <-time.After(defaultTestIdleTimeout / 50) + mgr.handleIdleTimeout() + }() + for j := 0; j < 5; j++ { + wg.Add(1) + go func() { + defer wg.Done() + // Wait for the configured idle timeout and simulate an RPC to + // race with the idle timeout timer callback. + <-time.After(defaultTestIdleTimeout / 50) + mgr.OnCallBegin() + atomic.StoreInt32((*int32)(&idlenessState), int32(stateActiveRPCs)) + mgr.OnCallEnd() + }() + } + wg.Wait() + }) + } +} diff --git a/internal/internal.go b/internal/internal.go new file mode 100644 index 0000000..2b7fa6b --- /dev/null +++ b/internal/internal.go @@ -0,0 +1,304 @@ +/* + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains gRPC-internal code, to avoid polluting +// the godoc of the top-level grpc package. It must not import any grpc +// symbols to avoid circular dependencies. +package internal + +import ( + "context" + "time" + + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/serviceconfig" +) + +var ( + // HealthCheckFunc is used to provide client-side LB channel health checking + HealthCheckFunc HealthChecker + // RegisterClientHealthCheckListener is used to provide a listener for + // updates from the client-side health checking service. It returns a + // function that can be called to stop the health producer. + RegisterClientHealthCheckListener any // func(ctx context.Context, sc balancer.SubConn, serviceName string, listener func(balancer.SubConnState)) func() + // BalancerUnregister is exported by package balancer to unregister a balancer. + BalancerUnregister func(name string) + // KeepaliveMinPingTime is the minimum ping interval. This must be 10s by + // default, but tests may wish to set it lower for convenience. + KeepaliveMinPingTime = 10 * time.Second + // KeepaliveMinServerPingTime is the minimum ping interval for servers. + // This must be 1s by default, but tests may wish to set it lower for + // convenience. + KeepaliveMinServerPingTime = time.Second + // ParseServiceConfig parses a JSON representation of the service config. + ParseServiceConfig any // func(string) *serviceconfig.ParseResult + // EqualServiceConfigForTesting is for testing service config generation and + // parsing. Both a and b should be returned by ParseServiceConfig. + // This function compares the config without rawJSON stripped, in case the + // there's difference in white space. + EqualServiceConfigForTesting func(a, b serviceconfig.Config) bool + // GetCertificateProviderBuilder returns the registered builder for the + // given name. This is set by package certprovider for use from xDS + // bootstrap code while parsing certificate provider configs in the + // bootstrap file. + GetCertificateProviderBuilder any // func(string) certprovider.Builder + // GetXDSHandshakeInfoForTesting returns a pointer to the xds.HandshakeInfo + // stored in the passed in attributes. This is set by + // credentials/xds/xds.go. + GetXDSHandshakeInfoForTesting any // func (*attributes.Attributes) *unsafe.Pointer + // GetServerCredentials returns the transport credentials configured on a + // gRPC server. An xDS-enabled server needs to know what type of credentials + // is configured on the underlying gRPC server. This is set by server.go. + GetServerCredentials any // func (*grpc.Server) credentials.TransportCredentials + // MetricsRecorderForServer returns the MetricsRecorderList derived from a + // server's stats handlers. + MetricsRecorderForServer any // func (*grpc.Server) estats.MetricsRecorder + // CanonicalString returns the canonical string of the code defined here: + // https://github.com/grpc/grpc/blob/master/doc/statuscodes.md. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + CanonicalString any // func (codes.Code) string + // IsRegisteredMethod returns whether the passed in method is registered as + // a method on the server. + IsRegisteredMethod any // func(*grpc.Server, string) bool + // ServerFromContext returns the server from the context. + ServerFromContext any // func(context.Context) *grpc.Server + // AddGlobalServerOptions adds an array of ServerOption that will be + // effective globally for newly created servers. The priority will be: 1. + // user-provided; 2. this method; 3. default values. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + AddGlobalServerOptions any // func(opt ...ServerOption) + // ClearGlobalServerOptions clears the array of extra ServerOption. This + // method is useful in testing and benchmarking. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + ClearGlobalServerOptions func() + // AddGlobalDialOptions adds an array of DialOption that will be effective + // globally for newly created client channels. The priority will be: 1. + // user-provided; 2. this method; 3. default values. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + AddGlobalDialOptions any // func(opt ...DialOption) + // DisableGlobalDialOptions returns a DialOption that prevents the + // ClientConn from applying the global DialOptions (set via + // AddGlobalDialOptions). + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + DisableGlobalDialOptions any // func() grpc.DialOption + // ClearGlobalDialOptions clears the array of extra DialOption. This + // method is useful in testing and benchmarking. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + ClearGlobalDialOptions func() + + // AddGlobalPerTargetDialOptions adds a PerTargetDialOption that will be + // configured for newly created ClientConns. + AddGlobalPerTargetDialOptions any // func (opt any) + // ClearGlobalPerTargetDialOptions clears the slice of global late apply + // dial options. + ClearGlobalPerTargetDialOptions func() + + // JoinDialOptions combines the dial options passed as arguments into a + // single dial option. + JoinDialOptions any // func(...grpc.DialOption) grpc.DialOption + // JoinServerOptions combines the server options passed as arguments into a + // single server option. + JoinServerOptions any // func(...grpc.ServerOption) grpc.ServerOption + + // WithBinaryLogger returns a DialOption that specifies the binary logger + // for a ClientConn. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + WithBinaryLogger any // func(binarylog.Logger) grpc.DialOption + // BinaryLogger returns a ServerOption that can set the binary logger for a + // server. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + BinaryLogger any // func(binarylog.Logger) grpc.ServerOption + + // SubscribeToConnectivityStateChanges adds a grpcsync.Subscriber to a + // provided grpc.ClientConn. + SubscribeToConnectivityStateChanges any // func(*grpc.ClientConn, grpcsync.Subscriber) + + // NewXDSResolverWithConfigForTesting creates a new xds resolver builder using + // the provided xds bootstrap config instead of the global configuration from + // the supported environment variables. The resolver.Builder is meant to be + // used in conjunction with the grpc.WithResolvers DialOption. + // + // Testing Only + // + // This function should ONLY be used for testing and may not work with some + // other features, including the CSDS service. + NewXDSResolverWithConfigForTesting any // func([]byte) (resolver.Builder, error) + + // NewXDSResolverWithPoolForTesting creates a new xDS resolver builder + // using the provided xDS pool instead of creating a new one using the + // bootstrap configuration specified by the supported environment variables. + // The resolver.Builder is meant to be used in conjunction with the + // grpc.WithResolvers DialOption. The resolver.Builder does not take + // ownership of the provided xDS client and it is the responsibility of the + // caller to close the client when no longer required. + // + // Testing Only + // + // This function should ONLY be used for testing and may not work with some + // other features, including the CSDS service. + NewXDSResolverWithPoolForTesting any // func(*xdsclient.Pool) (resolver.Builder, error) + + // NewXDSResolverWithClientForTesting creates a new xDS resolver builder + // using the provided xDS client instead of creating a new one using the + // bootstrap configuration specified by the supported environment variables. + // The resolver.Builder is meant to be used in conjunction with the + // grpc.WithResolvers DialOption. The resolver.Builder does not take + // ownership of the provided xDS client and it is the responsibility of the + // caller to close the client when no longer required. + // + // Testing Only + // + // This function should ONLY be used for testing and may not work with some + // other features, including the CSDS service. + NewXDSResolverWithClientForTesting any // func(xdsclient.XDSClient) (resolver.Builder, error) + + // ORCAAllowAnyMinReportingInterval is for examples/orca use ONLY. + ORCAAllowAnyMinReportingInterval any // func(so *orca.ServiceOptions) + + // GRPCResolverSchemeExtraMetadata determines when gRPC will add extra + // metadata to RPCs. + GRPCResolverSchemeExtraMetadata = "xds" + + // EnterIdleModeForTesting gets the ClientConn to enter IDLE mode. + EnterIdleModeForTesting any // func(*grpc.ClientConn) + + // ExitIdleModeForTesting gets the ClientConn to exit IDLE mode. + ExitIdleModeForTesting any // func(*grpc.ClientConn) error + + // ChannelzTurnOffForTesting disables the Channelz service for testing + // purposes. + ChannelzTurnOffForTesting func() + + // TriggerXDSResourceNotFoundForTesting causes the provided xDS Client to + // invoke resource-not-found error for the given resource type and name. + TriggerXDSResourceNotFoundForTesting any // func(xdsclient.XDSClient, xdsresource.Type, string) error + + // FromOutgoingContextRaw returns the un-merged, intermediary contents of + // metadata.rawMD. + FromOutgoingContextRaw any // func(context.Context) (metadata.MD, [][]string, bool) + + // UserSetDefaultScheme is set to true if the user has overridden the + // default resolver scheme. + UserSetDefaultScheme = false + + // SnapshotMetricRegistryForTesting snapshots the global data of the metric + // registry. Returns a cleanup function that sets the metric registry to its + // original state. Only called in testing functions. + SnapshotMetricRegistryForTesting func() func() + + // SetBufferPoolingThresholdForTesting updates the buffer pooling threshold, for + // testing purposes. + SetBufferPoolingThresholdForTesting any // func(int) + + // TimeAfterFunc is used to create timers. During tests the function is + // replaced to track allocated timers and fail the test if a timer isn't + // cancelled. + TimeAfterFunc = func(d time.Duration, f func()) Timer { + return time.AfterFunc(d, f) + } + + // NewStreamWaitingForResolver is a test hook that is triggered when a + // new stream blocks while waiting for name resolution. This can be + // used in tests to synchronize resolver updates and avoid race conditions. + // When set, the function will be called before the stream enters + // the blocking state. + NewStreamWaitingForResolver = func() {} + + // AddressToTelemetryLabels is an xDS-provided function to extract telemetry + // labels from a resolver.Address. Callers must assert its type before calling. + AddressToTelemetryLabels any // func(addr resolver.Address) map[string]string + + // AsyncReporterCleanupDelegate is initialized to a pass-through function by + // default (production behavior), allowing tests to swap it with an + // implementation which tracks registration of async reporter and its + // corresponding cleanup. + AsyncReporterCleanupDelegate = func(cleanup func()) func() { + return cleanup + } + + // XDSFilterWrapperOption returns a ServerOption that sets the internal + // stream wrapper used for server-side xDS HTTP filters. + XDSFilterWrapperOption any // func(func(grpc.ServerStream) (grpc.ServerStream, error)) grpc.ServerOption +) + +// HealthChecker defines the signature of the client-side LB channel health +// checking function. +// +// The implementation is expected to create a health checking RPC stream by +// calling newStream(), watch for the health status of serviceName, and report +// its health back by calling setConnectivityState(). +// +// The health checking protocol is defined at: +// https://github.com/grpc/grpc/blob/master/doc/health-checking.md +type HealthChecker func(ctx context.Context, newStream func(string) (any, error), setConnectivityState func(connectivity.State, error), serviceName string) error + +const ( + // CredsBundleModeFallback switches GoogleDefaultCreds to fallback mode. + CredsBundleModeFallback = "fallback" + // CredsBundleModeBalancer switches GoogleDefaultCreds to grpclb balancer + // mode. + CredsBundleModeBalancer = "balancer" + // CredsBundleModeBackendFromBalancer switches GoogleDefaultCreds to mode + // that supports backend returned by grpclb balancer. + CredsBundleModeBackendFromBalancer = "backend-from-balancer" +) + +// RLSLoadBalancingPolicyName is the name of the RLS LB policy. +// +// It currently has an experimental suffix which would be removed once +// end-to-end testing of the policy is completed. +const RLSLoadBalancingPolicyName = "rls_experimental" + +// EnforceSubConnEmbedding is used to enforce proper SubConn implementation +// embedding. +type EnforceSubConnEmbedding interface { + enforceSubConnEmbedding() +} + +// EnforceClientConnEmbedding is used to enforce proper ClientConn implementation +// embedding. +type EnforceClientConnEmbedding interface { + enforceClientConnEmbedding() +} + +// Timer is an interface to allow injecting different time.Timer implementations +// during tests. +type Timer interface { + Stop() bool +} + +// EnforceMetricsRecorderEmbedding is used to enforce proper MetricsRecorder +// implementation embedding. +type EnforceMetricsRecorderEmbedding interface { + enforceMetricsRecorderEmbedding() +} diff --git a/internal/leakcheck/leakcheck.go b/internal/leakcheck/leakcheck.go new file mode 100644 index 0000000..fe4efd5 --- /dev/null +++ b/internal/leakcheck/leakcheck.go @@ -0,0 +1,489 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package leakcheck contains functions to check leaked goroutines and buffers. +// +// Call the following at the beginning of test: +// +// defer leakcheck.NewLeakChecker(t).Check() +package leakcheck + +import ( + "context" + "fmt" + "runtime" + "runtime/debug" + "slices" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/internal" + "google.golang.org/grpc/mem" +) + +// failTestsOnLeakedBuffers is a special flag that will cause tests to fail if +// leaked buffers are detected, instead of simply logging them as an +// informational failure. This can be enabled with the "checkbuffers" compile +// flag, e.g.: +// +// go test -tags=checkbuffers +var failTestsOnLeakedBuffers = false + +func init() { + defaultPool := mem.DefaultBufferPool() + globalPool.Store(&defaultPool) + internal.SetDefaultBufferPool.(func(mem.BufferPool))(&globalPool) +} + +var globalPool swappableBufferPool +var globalTimerTracker *timerFactory + +type swappableBufferPool struct { + atomic.Pointer[mem.BufferPool] +} + +func (b *swappableBufferPool) Get(length int) *[]byte { + return (*b.Load()).Get(length) +} + +func (b *swappableBufferPool) Put(buf *[]byte) { + (*b.Load()).Put(buf) +} + +// SetTrackingBufferPool replaces the default buffer pool in the mem package to +// one that tracks where buffers are allocated. CheckTrackingBufferPool should +// then be invoked at the end of the test to validate that all buffers pulled +// from the pool were returned. +func SetTrackingBufferPool(logger Logger) { + newPool := mem.BufferPool(&trackingBufferPool{ + pool: *globalPool.Load(), + logger: logger, + allocatedBuffers: make(map[*[]byte][]uintptr), + }) + globalPool.Store(&newPool) +} + +// CheckTrackingBufferPool undoes the effects of SetTrackingBufferPool, and fails +// unit tests if not all buffers were returned. It is invalid to invoke this +// function without previously having invoked SetTrackingBufferPool. +func CheckTrackingBufferPool() { + p := (*globalPool.Load()).(*trackingBufferPool) + p.lock.Lock() + defer p.lock.Unlock() + + globalPool.Store(&p.pool) + + type uniqueTrace struct { + stack []uintptr + count int + } + + var totalLeakedBuffers int + var uniqueTraces []uniqueTrace + for _, stack := range p.allocatedBuffers { + idx, ok := slices.BinarySearchFunc(uniqueTraces, stack, func(trace uniqueTrace, stack []uintptr) int { + return slices.Compare(trace.stack, stack) + }) + if !ok { + uniqueTraces = slices.Insert(uniqueTraces, idx, uniqueTrace{stack: stack}) + } + uniqueTraces[idx].count++ + totalLeakedBuffers++ + } + + for _, ut := range uniqueTraces { + frames := runtime.CallersFrames(ut.stack) + var trace strings.Builder + for { + f, ok := frames.Next() + if !ok { + break + } + trace.WriteString(f.Function) + trace.WriteString("\n\t") + trace.WriteString(f.File) + trace.WriteString(":") + trace.WriteString(strconv.Itoa(f.Line)) + trace.WriteString("\n") + } + format := "%d allocated buffers never freed:\n%s" + args := []any{ut.count, trace.String()} + if failTestsOnLeakedBuffers { + p.logger.Errorf(format, args...) + } else { + p.logger.Logf("WARNING "+format, args...) + } + } + + if totalLeakedBuffers > 0 { + p.logger.Logf("%g%% of buffers never freed", float64(totalLeakedBuffers)/float64(p.bufferCount)) + } +} + +type trackingBufferPool struct { + pool mem.BufferPool + logger Logger + + lock sync.Mutex + bufferCount int + allocatedBuffers map[*[]byte][]uintptr +} + +func (p *trackingBufferPool) Get(length int) *[]byte { + p.lock.Lock() + defer p.lock.Unlock() + p.bufferCount++ + buf := p.pool.Get(length) + p.allocatedBuffers[buf] = currentStack(2) + return buf +} + +func (p *trackingBufferPool) Put(buf *[]byte) { + p.lock.Lock() + defer p.lock.Unlock() + + if _, ok := p.allocatedBuffers[buf]; !ok { + p.logger.Errorf("Unknown buffer freed:\n%s", string(debug.Stack())) + } else { + delete(p.allocatedBuffers, buf) + } + p.pool.Put(buf) +} + +var goroutinesToIgnore = []string{ + "testing.Main(", + "testing.tRunner(", + "testing.(*M).", + "runtime.goexit", + "created by runtime.gc", + "created by runtime/trace.Start", + "interestingGoroutines", + "runtime.MHeap_Scavenger", + "signal.signal_recv", + "sigterm.handler", + "runtime_mcall", + "(*loggingT).flushDaemon", + "goroutine in C code", + // Ignore the http read/write goroutines. gce metadata.OnGCE() was leaking + // these, root cause unknown. + // + // https://github.com/grpc/grpc-go/issues/5171 + // https://github.com/grpc/grpc-go/issues/5173 + "created by net/http.(*Transport).dialConn", +} + +// RegisterIgnoreGoroutine appends s into the ignore goroutine list. The +// goroutines whose stack trace contains s will not be identified as leaked +// goroutines. Not thread-safe, only call this function in init(). +func RegisterIgnoreGoroutine(s string) { + goroutinesToIgnore = append(goroutinesToIgnore, s) +} + +func ignore(g string) bool { + sl := strings.SplitN(g, "\n", 2) + if len(sl) != 2 { + return true + } + stack := strings.TrimSpace(sl[1]) + if strings.HasPrefix(stack, "testing.RunTests") { + return true + } + + if stack == "" { + return true + } + + for _, s := range goroutinesToIgnore { + if strings.Contains(stack, s) { + return true + } + } + + return false +} + +// interestingGoroutines returns all goroutines we care about for the purpose of +// leak checking. It excludes testing or runtime ones. +func interestingGoroutines() (gs []string) { + buf := make([]byte, 2<<20) + buf = buf[:runtime.Stack(buf, true)] + for _, g := range strings.Split(string(buf), "\n\n") { + if !ignore(g) { + gs = append(gs, g) + } + } + sort.Strings(gs) + return +} + +// Logger is the interface that wraps the Logf and Errorf method. It's a subset +// of testing.TB to make it easy to use this package. +type Logger interface { + Logf(format string, args ...any) + Errorf(format string, args ...any) +} + +// CheckGoroutines looks at the currently-running goroutines and checks if there +// are any interesting (created by gRPC) goroutines leaked. It waits up to 10 +// seconds in the error cases. +func CheckGoroutines(ctx context.Context, logger Logger) { + // Loop, waiting for goroutines to shut down. + // Wait up to timeout, but finish as quickly as possible. + var leaked []string + for ctx.Err() == nil { + if leaked = interestingGoroutines(); len(leaked) == 0 { + return + } + time.Sleep(50 * time.Millisecond) + } + for _, g := range leaked { + logger.Errorf("Leaked goroutine: %v", g) + } +} + +// LeakChecker captures a Logger and is returned by NewLeakChecker as a +// convenient method to set up leak check tests in a unit test. +type LeakChecker struct { + logger Logger +} + +// NewLeakChecker offers a convenient way to set up the leak checks for a +// specific unit test. It can be used as follows, at the beginning of tests: +// +// defer leakcheck.NewLeakChecker(t).Check() +// +// It initially invokes SetTrackingBufferPool to set up buffer tracking, then the +// deferred LeakChecker.Check call will invoke CheckTrackingBufferPool and +// CheckGoroutines with a default timeout of 10 seconds. +func NewLeakChecker(logger Logger) *LeakChecker { + SetTrackingBufferPool(logger) + return &LeakChecker{logger: logger} +} + +type timerFactory struct { + mu sync.Mutex + allocatedTimers map[internal.Timer][]uintptr +} + +func (tf *timerFactory) timeAfterFunc(d time.Duration, f func()) internal.Timer { + tf.mu.Lock() + defer tf.mu.Unlock() + ch := make(chan internal.Timer, 1) + timer := time.AfterFunc(d, func() { + f() + tf.remove(<-ch) + }) + ch <- timer + tf.allocatedTimers[timer] = currentStack(2) + return &trackingTimer{ + Timer: timer, + parent: tf, + } +} + +func (tf *timerFactory) remove(timer internal.Timer) { + tf.mu.Lock() + defer tf.mu.Unlock() + delete(tf.allocatedTimers, timer) +} + +func (tf *timerFactory) pendingTimers() []string { + tf.mu.Lock() + defer tf.mu.Unlock() + leaked := []string{} + for _, stack := range tf.allocatedTimers { + leaked = append(leaked, fmt.Sprintf("Allocated timer never cancelled:\n%s", traceToString(stack))) + } + return leaked +} + +type trackingTimer struct { + internal.Timer + parent *timerFactory +} + +func (t *trackingTimer) Stop() bool { + t.parent.remove(t.Timer) + return t.Timer.Stop() +} + +// TrackTimers replaces internal.TimerAfterFunc with one that tracks timer +// creations, stoppages and expirations. CheckTimers should then be invoked at +// the end of the test to validate that all timers created have either executed +// or are cancelled. +func TrackTimers() { + globalTimerTracker = &timerFactory{ + allocatedTimers: make(map[internal.Timer][]uintptr), + } + internal.TimeAfterFunc = globalTimerTracker.timeAfterFunc +} + +// CheckTimers undoes the effects of TrackTimers, and fails unit tests if not +// all timers were cancelled or executed. It is invalid to invoke this function +// without previously having invoked TrackTimers. +func CheckTimers(ctx context.Context, logger Logger) { + tt := globalTimerTracker + + // Loop, waiting for timers to be cancelled. + // Wait up to timeout, but finish as quickly as possible. + var leaked []string + for ctx.Err() == nil { + if leaked = tt.pendingTimers(); len(leaked) == 0 { + return + } + time.Sleep(50 * time.Millisecond) + } + for _, g := range leaked { + logger.Errorf("Leaked timers: %v", g) + } + + // Reset the internal function. + internal.TimeAfterFunc = func(d time.Duration, f func()) internal.Timer { + return time.AfterFunc(d, f) + } +} + +func currentStack(skip int) []uintptr { + var stackBuf [16]uintptr + var stack []uintptr + skip++ + for { + n := runtime.Callers(skip, stackBuf[:]) + stack = append(stack, stackBuf[:n]...) + if n < len(stackBuf) { + break + } + skip += len(stackBuf) + } + return stack +} + +func traceToString(stack []uintptr) string { + frames := runtime.CallersFrames(stack) + var trace strings.Builder + for { + f, ok := frames.Next() + if !ok { + break + } + trace.WriteString(f.Function) + trace.WriteString("\n\t") + trace.WriteString(f.File) + trace.WriteString(":") + trace.WriteString(strconv.Itoa(f.Line)) + trace.WriteString("\n") + } + return trace.String() +} + +// Async Reporter Leak Checking + +var asyncReporterTracker *reporterTracker + +type reporterTracker struct { + mu sync.Mutex + allocations map[*int][]uintptr +} + +func newReporterTracker() *reporterTracker { + return &reporterTracker{ + allocations: make(map[*int][]uintptr), + } +} + +// register records the stack trace. +func (rt *reporterTracker) register() *int { + rt.mu.Lock() + defer rt.mu.Unlock() + + id := new(int) + // Skip 4 frames: register -> internal.Delegate -> stats.RegisterAsyncReporter -> Caller + rt.allocations[id] = currentStack(4) + return id +} + +// unregister removes the ID. +func (rt *reporterTracker) unregister(id *int) { + rt.mu.Lock() + defer rt.mu.Unlock() + delete(rt.allocations, id) +} + +// leakedStackTraces returns formatted stack traces for all currently registered +// reporters. +func (rt *reporterTracker) leakedStackTraces() []string { + rt.mu.Lock() + defer rt.mu.Unlock() + + var traces []string + for _, pcs := range rt.allocations { + msg := "\n--- Leaked Async Reporter Registration ---\n" + traceToString(pcs) + traces = append(traces, msg) + } + return traces +} + +// TrackAsyncReporters installs the tracking delegate. +func TrackAsyncReporters() { + asyncReporterTracker = newReporterTracker() + + // Swap the delegate: Replace the default pass-through with tracking logic. + internal.AsyncReporterCleanupDelegate = func(originalCleanup func()) func() { + // 1. Capture Stack Trace (happens during Registration) + token := asyncReporterTracker.register() + + // 2. Return Wrapped Cleanup + return func() { + // Defer unregister to ensure we stop tracking even if the original cleanup panics. + defer asyncReporterTracker.unregister(token) + + if originalCleanup != nil { + originalCleanup() + } + } + } +} + +// CheckAsyncReporters verifies that no leaks exist and restores the default delegate. +func CheckAsyncReporters(logger Logger) { + // Restore the delegate: Reset to the default pass-through behavior. + internal.AsyncReporterCleanupDelegate = func(cleanup func()) func() { + return cleanup + } + + if asyncReporterTracker == nil { + return + } + + leaks := asyncReporterTracker.leakedStackTraces() + if len(leaks) > 0 { + // Join all stack traces into one message + allTraces := "" + for _, trace := range leaks { + allTraces += trace + } + logger.Errorf("Found %d leaked async reporters:%s", len(leaks), allTraces) + } + + // Clean up global state + asyncReporterTracker = nil +} diff --git a/internal/leakcheck/leakcheck_enabled.go b/internal/leakcheck/leakcheck_enabled.go new file mode 100644 index 0000000..8f10617 --- /dev/null +++ b/internal/leakcheck/leakcheck_enabled.go @@ -0,0 +1,25 @@ +//go:build checkbuffers + +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package leakcheck + +func init() { + failTestsOnLeakedBuffers = true +} diff --git a/internal/leakcheck/leakcheck_test.go b/internal/leakcheck/leakcheck_test.go new file mode 100644 index 0000000..c61cc43 --- /dev/null +++ b/internal/leakcheck/leakcheck_test.go @@ -0,0 +1,173 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package leakcheck + +import ( + "context" + "fmt" + "strings" + "sync" + "testing" + "time" + + "google.golang.org/grpc/internal" +) + +type testLogger struct { + errorCount int + errors []string +} + +func (e *testLogger) Logf(string, ...any) { +} + +func (e *testLogger) Errorf(format string, args ...any) { + e.errors = append(e.errors, fmt.Sprintf(format, args...)) + e.errorCount++ +} + +func TestCheck(t *testing.T) { + const leakCount = 3 + ch := make(chan struct{}) + for i := 0; i < leakCount; i++ { + go func() { <-ch }() + } + if leaked := interestingGoroutines(); len(leaked) != leakCount { + t.Errorf("interestingGoroutines() = %d, want length %d", len(leaked), leakCount) + } + e := &testLogger{} + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if CheckGoroutines(ctx, e); e.errorCount < leakCount { + t.Errorf("CheckGoroutines() = %d, want count %d", e.errorCount, leakCount) + t.Logf("leaked goroutines:\n%v", strings.Join(e.errors, "\n")) + } + close(ch) + ctx, cancel = context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + CheckGoroutines(ctx, t) +} + +func ignoredTestingLeak(d time.Duration) { + time.Sleep(d) +} + +func TestCheckRegisterIgnore(t *testing.T) { + RegisterIgnoreGoroutine("ignoredTestingLeak") + go ignoredTestingLeak(3 * time.Second) + const leakCount = 3 + ch := make(chan struct{}) + for i := 0; i < leakCount; i++ { + go func() { <-ch }() + } + if leaked := interestingGoroutines(); len(leaked) != leakCount { + t.Errorf("interestingGoroutines() = %d, want length %d", len(leaked), leakCount) + } + e := &testLogger{} + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if CheckGoroutines(ctx, e); e.errorCount < leakCount { + t.Errorf("CheckGoroutines() = %d, want count %d", e.errorCount, leakCount) + t.Logf("leaked goroutines:\n%v", strings.Join(e.errors, "\n")) + } + close(ch) + ctx, cancel = context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + CheckGoroutines(ctx, t) +} + +// TestTrackTimers verifies that only leaked timers are reported and expired, +// stopped timers are ignored. +func TestTrackTimers(t *testing.T) { + TrackTimers() + const leakCount = 3 + for i := 0; i < leakCount; i++ { + internal.TimeAfterFunc(2*time.Second, func() { + t.Logf("Timer %d fired.", i) + }) + } + wg := sync.WaitGroup{} + // Let a couple of timers expire. + for i := 0; i < 2; i++ { + wg.Add(1) + internal.TimeAfterFunc(time.Millisecond, func() { + wg.Done() + }) + } + wg.Wait() + + // Stop a couple of timers. + for i := 0; i < leakCount; i++ { + t := internal.TimeAfterFunc(time.Hour, func() { + t.Error("Timer fired before test ended.") + }) + t.Stop() + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + e := &testLogger{} + CheckTimers(ctx, e) + if e.errorCount != leakCount { + t.Errorf("CheckTimers found %v leaks, want %v leaks", e.errorCount, leakCount) + t.Logf("leaked timers:\n%v", strings.Join(e.errors, "\n")) + } + ctx, cancel = context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + CheckTimers(ctx, t) +} + +func TestLeakChecker_DetectsLeak(t *testing.T) { + TrackAsyncReporters() + // Safety defer: ensure we restore the default delegate even if the test crashes + // before CheckAsyncReporters is called. + defer func() { + internal.AsyncReporterCleanupDelegate = func(f func()) func() { return f } + }() + + // We utilize the internal delegate directly to simulate stats.RegisterAsyncReporter behavior. + noOpCleanup := func() {} + wrappedCleanup := internal.AsyncReporterCleanupDelegate(noOpCleanup) + + // Create a leak: We discard 'wrappedCleanup' without calling it. + _ = wrappedCleanup + tl := &testLogger{} + CheckAsyncReporters(tl) + if tl.errorCount == 0 { + t.Error("Expected leak checker to report a leak, but it succeeded silently.") + } + if asyncReporterTracker != nil { + t.Error("Expected CheckAsyncReporters to cleanup global tracker, but it was not nil.") + } +} + +func TestLeakChecker_PassesOnCleanup(t *testing.T) { + TrackAsyncReporters() + defer func() { + internal.AsyncReporterCleanupDelegate = func(f func()) func() { return f } + }() + noOpCleanup := func() {} + wrappedCleanup := internal.AsyncReporterCleanupDelegate(noOpCleanup) + wrappedCleanup() + tl := &testLogger{} + CheckAsyncReporters(tl) + if tl.errorCount > 0 { + t.Errorf("Expected no leaks, but got errors: %v", tl.errors) + } +} diff --git a/internal/mem/buffer_pool.go b/internal/mem/buffer_pool.go new file mode 100644 index 0000000..00aeca4 --- /dev/null +++ b/internal/mem/buffer_pool.go @@ -0,0 +1,363 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package mem provides utilities that facilitate memory reuse in byte slices +// that are used as buffers. +package mem + +import ( + "fmt" + "math/bits" + "slices" + "sort" + "sync" + + "google.golang.org/grpc/internal" +) + +const ( + goPageSize = 4 * 1024 // 4KiB. N.B. this must be a power of 2. +) + +var ( + // BufferPoolingThreshold is the minimum size of a buffer that can be pooled. + // This is used to determine whether to pool buffers or allocate them directly. + BufferPoolingThreshold = 1 << 10 +) + +func init() { + internal.SetBufferPoolingThresholdForTesting = func(threshold int) { + BufferPoolingThreshold = threshold + } +} + +var uintSize = bits.UintSize // use a variable for mocking during tests. + +// bufferPool is a copy of the public bufferPool interface used to avoid +// circular dependencies. +type bufferPool interface { + // Get returns a buffer with specified length from the pool. + Get(length int) *[]byte + + // Put returns a buffer to the pool. + // + // The provided pointer must hold a prefix of the buffer obtained via + // BufferPool.Get to ensure the buffer's entire capacity can be re-used. + Put(*[]byte) +} + +// BinaryTieredBufferPool is a buffer pool that uses multiple sub-pools with +// power-of-two sizes. +type BinaryTieredBufferPool struct { + // exponentToNextLargestPoolMap maps a power-of-two exponent (e.g., 12 for + // 4KB) to the index of the next largest sizedBufferPool. This is used by + // Get() to find the smallest pool that can satisfy a request for a given + // size. + exponentToNextLargestPoolMap []int + // exponentToPreviousLargestPoolMap maps a power-of-two exponent to the + // index of the previous largest sizedBufferPool. This is used by Put() + // to return a buffer to the most appropriate pool based on its capacity. + exponentToPreviousLargestPoolMap []int + sizedPools []bufferPool + fallbackPool bufferPool + maxPoolCap int // Optimization: Cache max capacity +} + +// NewBinaryTieredBufferPool returns a BufferPool backed by multiple sub-pools. +// This structure enables O(1) lookup time for Get and Put operations. +// +// The arguments provided are the exponents for the buffer capacities (powers +// of 2), not the raw byte sizes. For example, to create a pool of 16KB buffers +// (2^14 bytes), pass 14 as the argument. +func NewBinaryTieredBufferPool(powerOfTwoExponents ...uint8) (*BinaryTieredBufferPool, error) { + return newBinaryTiered(func(size int) bufferPool { + return newSizedBufferPool(size, true) + }, &SimpleBufferPool{shouldZero: true}, powerOfTwoExponents...) +} + +// NewDirtyBinaryTieredBufferPool returns a BufferPool backed by multiple +// sub-pools. It is similar to NewBinaryTieredBufferPool but it does not +// initialize the buffers before returning them. +func NewDirtyBinaryTieredBufferPool(powerOfTwoExponents ...uint8) (*BinaryTieredBufferPool, error) { + return newBinaryTiered(func(size int) bufferPool { + return newSizedBufferPool(size, false) + }, NewDirtySimplePool(), powerOfTwoExponents...) +} + +func newBinaryTiered(sizedPoolFactory func(int) bufferPool, fallbackPool bufferPool, powerOfTwoExponents ...uint8) (*BinaryTieredBufferPool, error) { + slices.Sort(powerOfTwoExponents) + powerOfTwoExponents = slices.Compact(powerOfTwoExponents) + + // Determine the maximum exponent we need to support. This depends on the + // word size (32-bit vs 64-bit). + maxExponent := uintSize - 2 + indexOfNextLargestBit := slices.Repeat([]int{-1}, maxExponent+1) + indexOfPreviousLargestBit := slices.Repeat([]int{-1}, maxExponent+1) + + maxTier := 0 + pools := make([]bufferPool, 0, len(powerOfTwoExponents)) + + for i, exp := range powerOfTwoExponents { + // Allocating slices of size > 2^maxExponent isn't possible on + // maxExponent-bit machines. + if int(exp) > maxExponent { + return nil, fmt.Errorf("mem: allocating slice of size 2^%d is not possible", exp) + } + tierSize := 1 << exp + pools = append(pools, sizedPoolFactory(tierSize)) + maxTier = max(maxTier, tierSize) + + // Map the exact power of 2 to this pool index. + indexOfNextLargestBit[exp] = i + indexOfPreviousLargestBit[exp] = i + } + + // Fill gaps for Get() (Next Largest) + // We iterate backwards. If current is empty, take the value from the right (larger). + for i := maxExponent - 1; i >= 0; i-- { + if indexOfNextLargestBit[i] == -1 { + indexOfNextLargestBit[i] = indexOfNextLargestBit[i+1] + } + } + + // Fill gaps for Put() (Previous Largest) + // We iterate forwards. If current is empty, take the value from the left (smaller). + for i := 1; i <= maxExponent; i++ { + if indexOfPreviousLargestBit[i] == -1 { + indexOfPreviousLargestBit[i] = indexOfPreviousLargestBit[i-1] + } + } + + return &BinaryTieredBufferPool{ + exponentToNextLargestPoolMap: indexOfNextLargestBit, + exponentToPreviousLargestPoolMap: indexOfPreviousLargestBit, + sizedPools: pools, + maxPoolCap: maxTier, + fallbackPool: fallbackPool, + }, nil +} + +// Get returns a buffer with specified length from the pool. +func (b *BinaryTieredBufferPool) Get(size int) *[]byte { + return b.poolForGet(size).Get(size) +} + +func (b *BinaryTieredBufferPool) poolForGet(size int) bufferPool { + if size == 0 || size > b.maxPoolCap { + return b.fallbackPool + } + + // Calculate the exponent of the smallest power of 2 >= size. + // We subtract 1 from size to handle exact powers of 2 correctly. + // + // Examples: + // size=16 (0b10000) -> size-1=15 (0b01111) -> bits.Len=4 -> Pool for 2^4 + // size=17 (0b10001) -> size-1=16 (0b10000) -> bits.Len=5 -> Pool for 2^5 + querySize := uint(size - 1) + poolIdx := b.exponentToNextLargestPoolMap[bits.Len(querySize)] + + return b.sizedPools[poolIdx] +} + +// Put returns a buffer to the pool. +func (b *BinaryTieredBufferPool) Put(buf *[]byte) { + // We pass the capacity of the buffer, and not the size of the buffer here. + // If we did the latter, all buffers would eventually move to the smallest + // pool. + b.poolForPut(cap(*buf)).Put(buf) +} + +func (b *BinaryTieredBufferPool) poolForPut(bCap int) bufferPool { + if bCap == 0 { + return NopBufferPool{} + } + if bCap > b.maxPoolCap { + return b.fallbackPool + } + // Find the pool with the largest capacity <= bCap. + // + // We calculate the exponent of the largest power of 2 <= bCap. + // bits.Len(x) returns the minimum number of bits required to represent x; + // i.e. the number of bits up to and including the most significant bit. + // Subtracting 1 gives the 0-based index of the most significant bit, + // which is the exponent of the largest power of 2 <= bCap. + // + // Examples: + // cap=16 (0b10000) -> Len=5 -> 5-1=4 -> 2^4 + // cap=15 (0b01111) -> Len=4 -> 4-1=3 -> 2^3 + largestPowerOfTwo := bits.Len(uint(bCap)) - 1 + poolIdx := b.exponentToPreviousLargestPoolMap[largestPowerOfTwo] + // The buffer is smaller than the smallest power of 2, discard it. + if poolIdx == -1 { + // Buffer is smaller than our smallest pool bucket. + return NopBufferPool{} + } + return b.sizedPools[poolIdx] +} + +// NopBufferPool is a buffer pool that returns new buffers without pooling. +type NopBufferPool struct{} + +// Get returns a buffer with specified length from the pool. +func (NopBufferPool) Get(length int) *[]byte { + b := make([]byte, length) + return &b +} + +// Put returns a buffer to the pool. +func (NopBufferPool) Put(*[]byte) { +} + +// sizedBufferPool is a BufferPool implementation that is optimized for specific +// buffer sizes. For example, HTTP/2 frames within gRPC have a default max size +// of 16kb and a sizedBufferPool can be configured to only return buffers with a +// capacity of 16kb. Note that however it does not support returning larger +// buffers and in fact panics if such a buffer is requested. Because of this, +// this BufferPool implementation is not meant to be used on its own and rather +// is intended to be embedded in a TieredBufferPool such that Get is only +// invoked when the required size is smaller than or equal to defaultSize. +type sizedBufferPool struct { + pool sync.Pool + defaultSize int + shouldZero bool +} + +func (p *sizedBufferPool) Get(size int) *[]byte { + buf, ok := p.pool.Get().(*[]byte) + if !ok { + buf := make([]byte, size, p.defaultSize) + return &buf + } + b := *buf + if p.shouldZero { + clear(b[:cap(b)]) + } + *buf = b[:size] + return buf +} + +func (p *sizedBufferPool) Put(buf *[]byte) { + if cap(*buf) < p.defaultSize { + // Ignore buffers that are too small to fit in the pool. Otherwise, when + // Get is called it will panic as it tries to index outside the bounds + // of the buffer. + return + } + p.pool.Put(buf) +} + +func newSizedBufferPool(size int, zero bool) *sizedBufferPool { + return &sizedBufferPool{ + defaultSize: size, + shouldZero: zero, + } +} + +// TieredBufferPool implements the BufferPool interface with multiple tiers of +// buffer pools for different sizes of buffers. +type TieredBufferPool struct { + sizedPools []*sizedBufferPool + fallbackPool SimpleBufferPool +} + +// NewTieredBufferPool returns a BufferPool implementation that uses multiple +// underlying pools of the given pool sizes. +func NewTieredBufferPool(poolSizes ...int) *TieredBufferPool { + sort.Ints(poolSizes) + pools := make([]*sizedBufferPool, len(poolSizes)) + for i, s := range poolSizes { + pools[i] = newSizedBufferPool(s, true) + } + return &TieredBufferPool{ + sizedPools: pools, + fallbackPool: SimpleBufferPool{shouldZero: true}, + } +} + +// Get returns a buffer with specified length from the pool. +func (p *TieredBufferPool) Get(size int) *[]byte { + return p.getPool(size).Get(size) +} + +// Put returns a buffer to the pool. +func (p *TieredBufferPool) Put(buf *[]byte) { + p.getPool(cap(*buf)).Put(buf) +} + +func (p *TieredBufferPool) getPool(size int) bufferPool { + poolIdx := sort.Search(len(p.sizedPools), func(i int) bool { + return p.sizedPools[i].defaultSize >= size + }) + + if poolIdx == len(p.sizedPools) { + return &p.fallbackPool + } + + return p.sizedPools[poolIdx] +} + +// SimpleBufferPool is an implementation of the mem.BufferPool interface that +// attempts to pool buffers with a sync.Pool. When Get is invoked, it tries to +// acquire a buffer from the pool but if that buffer is too small, it returns it +// to the pool and creates a new one. +type SimpleBufferPool struct { + pool sync.Pool + shouldZero bool +} + +// NewDirtySimplePool constructs a [SimpleBufferPool]. It does not initialize +// the buffers before returning them. Callers must ensure they don't read the +// buffers before writing data to them. +func NewDirtySimplePool() *SimpleBufferPool { + return &SimpleBufferPool{ + shouldZero: false, + } +} + +// Get returns a buffer with specified length from the pool. +func (p *SimpleBufferPool) Get(size int) *[]byte { + bs, ok := p.pool.Get().(*[]byte) + if ok && cap(*bs) >= size { + if p.shouldZero { + clear((*bs)[:cap(*bs)]) + } + *bs = (*bs)[:size] + return bs + } + + // A buffer was pulled from the pool, but it is too small. Put it back in + // the pool and create one large enough. + if ok { + p.pool.Put(bs) + } + + // If we're going to allocate, round up to the nearest page. This way if + // requests frequently arrive with small variation we don't allocate + // repeatedly if we get unlucky and they increase over time. By default we + // only allocate here if size > 1MiB. Because goPageSize is a power of 2, we + // can round up efficiently. + allocSize := (size + goPageSize - 1) & ^(goPageSize - 1) + + b := make([]byte, size, allocSize) + return &b +} + +// Put returns a buffer to the pool. +func (p *SimpleBufferPool) Put(buf *[]byte) { + p.pool.Put(buf) +} diff --git a/internal/mem/buffer_pool_ext_test.go b/internal/mem/buffer_pool_ext_test.go new file mode 100644 index 0000000..f857ecd --- /dev/null +++ b/internal/mem/buffer_pool_ext_test.go @@ -0,0 +1,118 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package mem_test + +import ( + "testing" + "unsafe" + + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/mem" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestBufferPool_Clears(t *testing.T) { + poolConfigs := []struct { + name string + factory func() (*mem.BinaryTieredBufferPool, error) + wantCleared bool + bufferSize int + }{ + { + name: "regular_sized", + factory: func() (*mem.BinaryTieredBufferPool, error) { + return mem.NewBinaryTieredBufferPool(3) // 8 bytes + }, + bufferSize: 8, + wantCleared: true, + }, + { + name: "regular_fallback", + factory: func() (*mem.BinaryTieredBufferPool, error) { + return mem.NewBinaryTieredBufferPool(3) + }, + bufferSize: 10, + wantCleared: true, + }, + { + name: "dirty_sized", + factory: func() (*mem.BinaryTieredBufferPool, error) { + return mem.NewDirtyBinaryTieredBufferPool(3) + }, + bufferSize: 8, + wantCleared: false, + }, + { + name: "dirty_fallback", + factory: func() (*mem.BinaryTieredBufferPool, error) { + return mem.NewDirtyBinaryTieredBufferPool(3) + }, + bufferSize: 10, + wantCleared: false, + }, + } + + for _, tc := range poolConfigs { + t.Run(tc.name, func(t *testing.T) { + pool, err := tc.factory() + if err != nil { + t.Fatalf("Failed to create pool: %v", err) + } + + for { + buf1 := pool.Get(tc.bufferSize) + // Mark the buffer with data. + for i := range *buf1 { + (*buf1)[i] = 0xAA + } + pool.Put(buf1) + + buf2 := pool.Get(tc.bufferSize) + // Check if we got the same underlying array. + if unsafe.SliceData(*buf1) != unsafe.SliceData(*buf2) { + pool.Put(buf2) + continue + } + + // We have a reused buffer. Check if it's cleared. + gotCleared := true + for _, b := range *buf2 { + if b != 0 { + gotCleared = false + break + } + } + + if tc.wantCleared != gotCleared { + t.Fatalf("buffer cleared state mismatch: want %t, got %v", tc.wantCleared, gotCleared) + } + + pool.Put(buf2) + break + } + }) + } +} diff --git a/internal/mem/buffer_pool_test.go b/internal/mem/buffer_pool_test.go new file mode 100644 index 0000000..98d647c --- /dev/null +++ b/internal/mem/buffer_pool_test.go @@ -0,0 +1,130 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package mem + +import "testing" + +var defaultBufferPoolSizeExponents = []uint8{ + 8, + 12, + 14, // 16KB (max HTTP/2 frame size used by gRPC) + 15, // 32KB (default buffer size for io.Copy) + 20, // 1MB +} + +func TestNewBinaryTieredBufferPool_WordSize(t *testing.T) { + origUintSize := uintSize + defer func() { uintSize = origUintSize }() + + tests := []struct { + name string + wordSize int + exponents []uint8 + wantErr bool + }{ + { + name: "32-bit_valid_exponent", + wordSize: 32, + exponents: []uint8{30}, + wantErr: false, + }, + { + name: "32-bit_invalid_exponent", + wordSize: 32, + exponents: []uint8{31}, + wantErr: true, + }, + { + name: "64-bit_valid_exponent", + wordSize: 64, + exponents: []uint8{62}, + wantErr: false, + }, + { + name: "64-bit_invalid_exponent", + wordSize: 64, + exponents: []uint8{63}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + uintSize = tt.wordSize + pool, err := NewBinaryTieredBufferPool(tt.exponents...) + if (err != nil) != tt.wantErr { + t.Fatalf("NewBinaryTieredBufferPool() error = %t, wantErr %t", err, tt.wantErr) + } + if err != nil { + return + } + if len(pool.exponentToNextLargestPoolMap) != tt.wordSize-1 { + t.Errorf("exponentToNextLargestPoolMap length = %d, want %d", len(pool.exponentToNextLargestPoolMap), tt.wordSize) + } + if len(pool.exponentToPreviousLargestPoolMap) != tt.wordSize-1 { + t.Errorf("exponentToPreviousLargestPoolMap length = %d, want %d", len(pool.exponentToPreviousLargestPoolMap), tt.wordSize) + } + }) + } +} + +// BenchmarkTieredPool benchmarks the performance of the tiered buffer pool +// implementations, specifically focusing on the overhead of selecting the +// correct bucket for a given size. +func BenchmarkTieredPool(b *testing.B) { + defaultBufferPoolSizes := make([]int, len(defaultBufferPoolSizeExponents)) + for i, exp := range defaultBufferPoolSizeExponents { + defaultBufferPoolSizes[i] = 1 << exp + } + + b.Run("pool=Tiered", func(b *testing.B) { + p := NewTieredBufferPool(defaultBufferPoolSizes...) + for b.Loop() { + for size := range 1 << 19 { + // One for get, one for put. + _ = p.getPool(size) + _ = p.getPool(size) + } + } + }) + + b.Run("pool=BinaryTiered", func(b *testing.B) { + pool, err := NewBinaryTieredBufferPool(defaultBufferPoolSizeExponents...) + if err != nil { + b.Fatalf("Failed to create buffer pool: %v", err) + } + for b.Loop() { + for size := range 1 << 19 { + _ = pool.poolForGet(size) + _ = pool.poolForPut(size) + } + } + }) +} + +func TestNewBinaryTieredBufferPool_Duplicates(t *testing.T) { + exponents := []uint8{1, 2, 3, 4, 5, 6, 6, 5, 4, 3, 2, 1} + pool, err := NewBinaryTieredBufferPool(exponents...) + if err != nil { + t.Fatalf("NewBinaryTieredBufferPool() error = %v", err) + } + if len(pool.sizedPools) != 6 { + t.Errorf("sized buffer pool count = %d, want %d", len(pool.sizedPools), 6) + } +} diff --git a/internal/metadata/metadata.go b/internal/metadata/metadata.go new file mode 100644 index 0000000..c4055bc --- /dev/null +++ b/internal/metadata/metadata.go @@ -0,0 +1,144 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package metadata contains functions to set and get metadata from addresses. +// +// This package is experimental. +package metadata + +import ( + "fmt" + "strings" + + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" +) + +type mdKeyType string + +const mdKey = mdKeyType("grpc.internal.address.metadata") + +type mdValue metadata.MD + +func (m mdValue) Equal(o any) bool { + om, ok := o.(mdValue) + if !ok { + return false + } + if len(m) != len(om) { + return false + } + for k, v := range m { + ov := om[k] + if len(ov) != len(v) { + return false + } + for i, ve := range v { + if ov[i] != ve { + return false + } + } + } + return true +} + +// Get returns the metadata of addr. +func Get(addr resolver.Address) metadata.MD { + attrs := addr.Attributes + if attrs == nil { + return nil + } + md, _ := attrs.Value(mdKey).(mdValue) + return metadata.MD(md) +} + +// Set sets (overrides) the metadata in addr. +// +// When a SubConn is created with this address, the RPCs sent on it will all +// have this metadata. +func Set(addr resolver.Address, md metadata.MD) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(mdKey, mdValue(md)) + return addr +} + +// Validate validates every pair in md with ValidatePair. +func Validate(md metadata.MD) error { + for k, vals := range md { + if err := ValidatePair(k, vals...); err != nil { + return err + } + } + return nil +} + +// hasNotPrintable return true if msg contains any characters which are not in %x20-%x7E +func hasNotPrintable(msg string) bool { + // for i that saving a conversion if not using for range + for i := 0; i < len(msg); i++ { + if msg[i] < 0x20 || msg[i] > 0x7E { + return true + } + } + return false +} + +// ValidateKey validates a key with the following rules (pseudo-headers are +// skipped): +// - the key must contain one or more characters. +// - the characters in the key must be in [0-9 a-z _ - .]. +func ValidateKey(key string) error { + // key should not be empty + if key == "" { + return fmt.Errorf("there is an empty key in the header") + } + // pseudo-header will be ignored + if key[0] == ':' { + return nil + } + // check key, for i that saving a conversion if not using for range + for i := 0; i < len(key); i++ { + r := key[i] + if !(r >= 'a' && r <= 'z') && !(r >= '0' && r <= '9') && r != '.' && r != '-' && r != '_' { + return fmt.Errorf("header key %q contains illegal characters not in [0-9a-z-_.]", key) + } + } + return nil +} + +// ValidatePair validates a key-value pair with the following rules +// (pseudo-header are skipped): +// - the key must contain one or more characters. +// - the characters in the key must be in [0-9 a-z _ - .]. +// - if the key ends with a "-bin" suffix, no validation of the corresponding +// value is performed. +// - the characters in every value must be printable (in [%x20-%x7E]). +func ValidatePair(key string, vals ...string) error { + if err := ValidateKey(key); err != nil { + return err + } + if strings.HasSuffix(key, "-bin") { + return nil + } + // check value + for _, val := range vals { + if hasNotPrintable(val) { + return fmt.Errorf("header key %q contains value with non-printable ASCII characters", key) + } + } + return nil +} diff --git a/internal/metadata/metadata_test.go b/internal/metadata/metadata_test.go new file mode 100644 index 0000000..8f0e430 --- /dev/null +++ b/internal/metadata/metadata_test.go @@ -0,0 +1,117 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package metadata + +import ( + "errors" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" +) + +func TestGet(t *testing.T) { + tests := []struct { + name string + addr resolver.Address + want metadata.MD + }{ + { + name: "not set", + addr: resolver.Address{}, + want: nil, + }, + { + name: "not set", + addr: resolver.Address{ + Attributes: attributes.New(mdKey, mdValue(metadata.Pairs("k", "v"))), + }, + want: metadata.Pairs("k", "v"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Get(tt.addr); !cmp.Equal(got, tt.want) { + t.Errorf("Get() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSet(t *testing.T) { + tests := []struct { + name string + addr resolver.Address + md metadata.MD + }{ + { + name: "unset before", + addr: resolver.Address{}, + md: metadata.Pairs("k", "v"), + }, + { + name: "set before", + addr: resolver.Address{ + Attributes: attributes.New(mdKey, mdValue(metadata.Pairs("bef", "ore"))), + }, + md: metadata.Pairs("k", "v"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + newAddr := Set(tt.addr, tt.md) + newMD := Get(newAddr) + if !cmp.Equal(newMD, tt.md) { + t.Errorf("md after Set() = %v, want %v", newMD, tt.md) + } + }) + } +} + +func TestValidate(t *testing.T) { + for _, test := range []struct { + md metadata.MD + want error + }{ + { + md: map[string][]string{string(rune(0x19)): {"testVal"}}, + want: errors.New("header key \"\\x19\" contains illegal characters not in [0-9a-z-_.]"), + }, + { + md: map[string][]string{"test": {string(rune(0x19))}}, + want: errors.New("header key \"test\" contains value with non-printable ASCII characters"), + }, + { + md: map[string][]string{"": {"valid"}}, + want: errors.New("there is an empty key in the header"), + }, + { + md: map[string][]string{"test-bin": {string(rune(0x19))}}, + want: nil, + }, + } { + err := Validate(test.md) + if !reflect.DeepEqual(err, test.want) { + t.Errorf("validating metadata which is %v got err :%v, want err :%v", test.md, err, test.want) + } + } +} diff --git a/internal/optional/optional.go b/internal/optional/optional.go new file mode 100644 index 0000000..218794a --- /dev/null +++ b/internal/optional/optional.go @@ -0,0 +1,47 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package optional implements a generic optional type. +package optional + +// Optional represents an optional value of type T. The zero value is usable and +// indicates that no value is set. This type is not safe for concurrent access. +type Optional[T any] struct { + val T + set bool +} + +// New creates a new Optional type with the provided value. +func New[T any](value T) Optional[T] { + return Optional[T]{ + val: value, + set: true, + } +} + +// Value returns the underlying value and a boolean indicating if the value is +// set. If the value is not set, it returns the zero value of T and false. +func (o *Optional[T]) Value() (T, bool) { + return o.val, o.set +} + +// SetValue updates or adds the value to Optional. +func (o *Optional[T]) SetValue(v T) { + o.val = v + o.set = true +} diff --git a/internal/optional/optional_test.go b/internal/optional/optional_test.go new file mode 100644 index 0000000..8df4190 --- /dev/null +++ b/internal/optional/optional_test.go @@ -0,0 +1,130 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package optional_test + +import ( + "slices" + "testing" + + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/optional" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestOptionalInt tests the scenario of using integer optional values and +// verifies that zero value, constructors, and mutation methods return correct +// outputs. +func (s) TestOptionalInt(t *testing.T) { + var opt optional.Optional[int] + + // Test unset value. + if v, set := opt.Value(); set || v != 0 { + t.Fatalf("Zero-value Option[int] = (%v, %v); want (0, false)", v, set) + } + + opt = optional.New(42) + if v, set := opt.Value(); !set || v != 42 { + t.Fatalf("NewValue(42) = (%v, %v); want (42, true)", v, set) + } + + opt.SetValue(100) + if v, set := opt.Value(); !set || v != 100 { + t.Fatalf("Set(100) = (%v, %v); want (100, true)", v, set) + } +} + +// TestOptionalString tests the scenario of using string optional values and +// verifies that zero value, constructors, and mutation methods return correct +// outputs. +func (s) TestOptionalString(t *testing.T) { + var opt optional.Optional[string] + + // Test unset value. + if v, set := opt.Value(); set || v != "" { + t.Fatalf("Zero-value Option[string] = (%q, %v); want (%q, false)", v, set, "") + } + + wantString := "test-string" + opt = optional.New(wantString) + if v, set := opt.Value(); !set || v != wantString { + t.Fatalf("NewValue(%q) = (%q, %v); want (%q, true)", wantString, v, set, wantString) + } + + wantStringNew := "world" + opt.SetValue(wantStringNew) + if v, set := opt.Value(); !set || v != wantStringNew { + t.Fatalf("Set(%q) = (%q, %v); want (%q, true)", wantStringNew, v, set, wantStringNew) + } +} + +// TestOptionalStruct tests the scenario of using a custom struct optional value +// and verifies that custom struct field values are preserved, modified, and +// cleared correctly. +func (s) TestOptionalStruct(t *testing.T) { + type testStruct struct { + name string + age int + } + + var opt optional.Optional[testStruct] + if v, set := opt.Value(); set || v != (testStruct{}) { + t.Fatalf("Zero-value Option[struct] = (%v, %v); want (empty, false)", v, set) + } + + want := testStruct{name: "Alice", age: 30} + opt = optional.New(want) + if v, set := opt.Value(); !set || v != want { + t.Fatalf("NewValue(%v) = (%v, %v); want (%v, true)", want, v, set, want) + } + + want2 := testStruct{name: "Bob", age: 40} + opt.SetValue(want2) + if v, set := opt.Value(); !set || v != want2 { + t.Fatalf("Set(%v) = (%v, %v); want (%v, true)", want2, v, set, want2) + } +} + +// TestOptionalSlice tests the scenario of using a slice optional value and +// verifies that zero value, constructors, and mutation methods return correct +// outputs. +func (s) TestOptionalSlice(t *testing.T) { + var opt optional.Optional[[]int] + if v, set := opt.Value(); set || v != nil { + t.Fatalf("Zero-value Option[[]int] = (%v, %v); want (nil, false)", v, set) + } + + want := []int{1, 2, 3} + opt = optional.New(want) + if v, set := opt.Value(); !set || !slices.Equal(v, want) { + t.Fatalf("NewValue(%v) = (%v, %v); want (%v, true)", want, v, set, want) + } + + want2 := []int{4, 5, 6} + opt.SetValue(want2) + if v, set := opt.Value(); !set || !slices.Equal(v, want2) { + t.Fatalf("Set(%v) = (%v, %v); want (%v, true)", &want2, v, set, &want2) + } +} diff --git a/internal/pretty/pretty.go b/internal/pretty/pretty.go new file mode 100644 index 0000000..dbee7a6 --- /dev/null +++ b/internal/pretty/pretty.go @@ -0,0 +1,73 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package pretty defines helper functions to pretty-print structs for logging. +package pretty + +import ( + "bytes" + "encoding/json" + "fmt" + + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/protoadapt" +) + +const jsonIndent = " " + +// ToJSON marshals the input into a json string. +// +// If marshal fails, it falls back to fmt.Sprintf("%+v"). +func ToJSON(e any) string { + if ee, ok := e.(protoadapt.MessageV1); ok { + e = protoadapt.MessageV2Of(ee) + } + + if ee, ok := e.(protoadapt.MessageV2); ok { + mm := protojson.MarshalOptions{ + Indent: jsonIndent, + Multiline: true, + } + ret, err := mm.Marshal(ee) + if err != nil { + // This may fail for proto.Anys, e.g. for xDS v2, LDS, the v2 + // messages are not imported, and this will fail because the message + // is not found. + return fmt.Sprintf("%+v", ee) + } + return string(ret) + } + + ret, err := json.MarshalIndent(e, "", jsonIndent) + if err != nil { + return fmt.Sprintf("%+v", e) + } + return string(ret) +} + +// FormatJSON formats the input json bytes with indentation. +// +// If Indent fails, it returns the unchanged input as string. +func FormatJSON(b []byte) string { + var out bytes.Buffer + err := json.Indent(&out, b, "", jsonIndent) + if err != nil { + return string(b) + } + return out.String() +} diff --git a/internal/profiling/buffer/buffer.go b/internal/profiling/buffer/buffer.go new file mode 100644 index 0000000..22c8629 --- /dev/null +++ b/internal/profiling/buffer/buffer.go @@ -0,0 +1,272 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package buffer provides a high-performant lock free implementation of a +// circular buffer used by the profiling code. +package buffer + +import ( + "errors" + "math/bits" + "runtime" + "sync" + "sync/atomic" + "unsafe" +) + +type queue struct { + // An array of pointers as references to the items stored in this queue. + arr []unsafe.Pointer + // The maximum number of elements this queue may store before it wraps around + // and overwrites older values. Must be an exponent of 2. + size uint32 + // Always size - 1. A bitwise AND is performed with this mask in place of a + // modulo operation by the Push operation. + mask uint32 + // Each Push operation into this queue increments the acquired counter before + // proceeding forwarding with the actual write to arr. This counter is also + // used by the Drain operation's drainWait subroutine to wait for all pushes + // to complete. + acquired uint32 // Accessed atomically. + // After the completion of a Push operation, the written counter is + // incremented. Also used by drainWait to wait for all pushes to complete. + written uint32 +} + +// Allocates and returns a new *queue. size needs to be an exponent of two. +func newQueue(size uint32) *queue { + return &queue{ + arr: make([]unsafe.Pointer, size), + size: size, + mask: size - 1, + } +} + +// drainWait blocks the caller until all Pushes on this queue are complete. +func (q *queue) drainWait() { + for atomic.LoadUint32(&q.acquired) != atomic.LoadUint32(&q.written) { + runtime.Gosched() + } +} + +// A queuePair has two queues. At any given time, Pushes go into the queue +// referenced by queuePair.q. The active queue gets switched when there's a +// drain operation on the circular buffer. +type queuePair struct { + q0 unsafe.Pointer + q1 unsafe.Pointer + q unsafe.Pointer +} + +// Allocates and returns a new *queuePair with its internal queues allocated. +func newQueuePair(size uint32) *queuePair { + qp := &queuePair{} + qp.q0 = unsafe.Pointer(newQueue(size)) + qp.q1 = unsafe.Pointer(newQueue(size)) + qp.q = qp.q0 + return qp +} + +// Switches the current queue for future Pushes to proceed to the other queue +// so that there's no blocking in Push. Returns a pointer to the old queue that +// was in place before the switch. +func (qp *queuePair) switchQueues() *queue { + // Even though we have mutual exclusion across drainers (thanks to mu.Lock in + // drain), Push operations may access qp.q whilst we're writing to it. + if atomic.CompareAndSwapPointer(&qp.q, qp.q0, qp.q1) { + return (*queue)(qp.q0) + } + + atomic.CompareAndSwapPointer(&qp.q, qp.q1, qp.q0) + return (*queue)(qp.q1) +} + +// In order to not have expensive modulo operations, we require the maximum +// number of elements in the circular buffer (N) to be an exponent of two to +// use a bitwise AND mask. Since a CircularBuffer is a collection of queuePairs +// (see below), we need to divide N; since exponents of two are only divisible +// by other exponents of two, we use floorCPUCount number of queuePairs within +// each CircularBuffer. +// +// Floor of the number of CPUs (and not the ceiling) was found to be the +// optimal number through experiments. +func floorCPUCount() uint32 { + floorExponent := bits.Len32(uint32(runtime.NumCPU())) - 1 + if floorExponent < 0 { + floorExponent = 0 + } + return 1 << uint32(floorExponent) +} + +var numCircularBufferPairs = floorCPUCount() + +// CircularBuffer is a lock-free data structure that supports Push and Drain +// operations. +// +// Note that CircularBuffer is built for performance more than reliability. +// That is, some Push operations may fail without retries in some situations +// (such as during a Drain operation). Order of pushes is not maintained +// either; that is, if A was pushed before B, the Drain operation may return an +// array with B before A. These restrictions are acceptable within gRPC's +// profiling, but if your use-case does not permit these relaxed constraints +// or if performance is not a primary concern, you should probably use a +// lock-based data structure such as internal/buffer.UnboundedBuffer. +type CircularBuffer struct { + drainMutex sync.Mutex + qp []*queuePair + // qpn is a monotonically incrementing counter that's used to determine + // which queuePair a Push operation should write to. This approach's + // performance was found to be better than writing to a random queue. + qpn uint32 + qpMask uint32 +} + +var errInvalidCircularBufferSize = errors.New("buffer size is not an exponent of two") + +// NewCircularBuffer allocates a circular buffer of size size and returns a +// reference to the struct. Only circular buffers of size 2^k are allowed +// (saves us from having to do expensive modulo operations). +func NewCircularBuffer(size uint32) (*CircularBuffer, error) { + if size&(size-1) != 0 { + return nil, errInvalidCircularBufferSize + } + + n := numCircularBufferPairs + if size/numCircularBufferPairs < 8 { + // If each circular buffer is going to hold less than a very small number + // of items (let's say 8), using multiple circular buffers is very likely + // wasteful. Instead, fallback to one circular buffer holding everything. + n = 1 + } + + cb := &CircularBuffer{ + qp: make([]*queuePair, n), + qpMask: n - 1, + } + + for i := uint32(0); i < n; i++ { + cb.qp[i] = newQueuePair(size / n) + } + + return cb, nil +} + +// Push pushes an element in to the circular buffer. Guaranteed to complete in +// a finite number of steps (also lock-free). Does not guarantee that push +// order will be retained. Does not guarantee that the operation will succeed +// if a Drain operation concurrently begins execution. +func (cb *CircularBuffer) Push(x any) { + n := atomic.AddUint32(&cb.qpn, 1) & cb.qpMask + qptr := atomic.LoadPointer(&cb.qp[n].q) + q := (*queue)(qptr) + + acquired := atomic.AddUint32(&q.acquired, 1) - 1 + + // If true, it means that we have incremented acquired before any queuePair + // was switched, and therefore before any drainWait completion. Therefore, it + // is safe to proceed with the Push operation on this queue. Otherwise, it + // means that a Drain operation has begun execution, but we don't know how + // far along the process it is. If it is past the drainWait check, it is not + // safe to proceed with the Push operation. We choose to drop this sample + // entirely instead of retrying, as retrying may potentially send the Push + // operation into a spin loop (we want to guarantee completion of the Push + // operation within a finite time). Before exiting, we increment written so + // that any existing drainWaits can proceed. + if atomic.LoadPointer(&cb.qp[n].q) != qptr { + atomic.AddUint32(&q.written, 1) + return + } + + // At this point, we're definitely writing to the right queue. That is, one + // of the following is true: + // 1. No drainer is in execution on this queue. + // 2. A drainer is in execution on this queue and it is waiting at the + // acquired == written barrier. + // + // Let's say two Pushes A and B happen on the same queue. Say A and B are + // q.size apart; i.e. they get the same index. That is, + // + // index_A = index_B + // acquired_A + q.size = acquired_B + // + // We say "B has wrapped around A" when this happens. In this case, since A + // occurred before B, B's Push should be the final value. However, we + // accommodate A being the final value because wrap-arounds are extremely + // rare and accounting for them requires an additional counter and a + // significant performance penalty. Note that the below approach never leads + // to any data corruption. + index := acquired & q.mask + atomic.StorePointer(&q.arr[index], unsafe.Pointer(&x)) + + // Allows any drainWait checks to proceed. + atomic.AddUint32(&q.written, 1) +} + +// Dereferences non-nil pointers from arr into result. Range of elements from +// arr that are copied is [from, to). Assumes that the result slice is already +// allocated and is large enough to hold all the elements that might be copied. +// Also assumes mutual exclusion on the array of pointers. +func dereferenceAppend(result []any, arr []unsafe.Pointer, from, to uint32) []any { + for i := from; i < to; i++ { + // We have mutual exclusion on arr, there's no need for atomics. + x := (*any)(arr[i]) + if x != nil { + result = append(result, *x) + } + } + return result +} + +// Drain allocates and returns an array of things Pushed in to the circular +// buffer. Push order is not maintained; that is, if B was Pushed after A, +// drain may return B at a lower index than A in the returned array. +func (cb *CircularBuffer) Drain() []any { + cb.drainMutex.Lock() + + qs := make([]*queue, len(cb.qp)) + for i := 0; i < len(cb.qp); i++ { + qs[i] = cb.qp[i].switchQueues() + } + + var wg sync.WaitGroup + wg.Add(len(qs)) + for i := 0; i < len(qs); i++ { + go func(qi int) { + qs[qi].drainWait() + wg.Done() + }(i) + } + wg.Wait() + + result := make([]any, 0) + for i := 0; i < len(qs); i++ { + if acquired := atomic.LoadUint32(&qs[i].acquired); acquired < qs[i].size { + result = dereferenceAppend(result, qs[i].arr, 0, acquired) + } else { + result = dereferenceAppend(result, qs[i].arr, 0, qs[i].size) + } + } + + for i := 0; i < len(qs); i++ { + atomic.StoreUint32(&qs[i].acquired, 0) + atomic.StoreUint32(&qs[i].written, 0) + } + + cb.drainMutex.Unlock() + return result +} diff --git a/internal/profiling/buffer/buffer_test.go b/internal/profiling/buffer/buffer_test.go new file mode 100644 index 0000000..c816a87 --- /dev/null +++ b/internal/profiling/buffer/buffer_test.go @@ -0,0 +1,186 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package buffer + +import ( + "fmt" + "sync" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestCircularBufferSerial(t *testing.T) { + var size, i uint32 + var result []any + + size = 1 << 15 + cb, err := NewCircularBuffer(size) + if err != nil { + t.Fatalf("error allocating CircularBuffer: %v", err) + } + + for i = 0; i < size/2; i++ { + cb.Push(i) + } + + result = cb.Drain() + if uint32(len(result)) != size/2 { + t.Fatalf("len(result) = %d; want %d", len(result), size/2) + } + + // The returned result isn't necessarily sorted. + seen := make(map[uint32]bool) + for _, r := range result { + seen[r.(uint32)] = true + } + + for i = 0; i < uint32(len(result)); i++ { + if !seen[i] { + t.Fatalf("seen[%d] = false; want true", i) + } + } + + for i = 0; i < size; i++ { + cb.Push(i) + } + + result = cb.Drain() + if uint32(len(result)) != size { + t.Fatalf("len(result) = %d; want %d", len(result), size/2) + } +} + +func (s) TestCircularBufferOverflow(t *testing.T) { + var size, i uint32 + var result []any + + size = 1 << 10 + cb, err := NewCircularBuffer(size) + if err != nil { + t.Fatalf("error allocating CircularBuffer: %v", err) + } + + for i = 0; i < 10*size; i++ { + cb.Push(i) + } + + result = cb.Drain() + + if uint32(len(result)) != size { + t.Fatalf("len(result) = %d; want %d", len(result), size) + } + + for idx, x := range result { + if x.(uint32) < size { + t.Fatalf("result[%d] = %d; want it to be >= %d", idx, x, size) + } + } +} + +func (s) TestCircularBufferConcurrent(t *testing.T) { + for tn := 0; tn < 2; tn++ { + var size uint32 + var result []any + + size = 1 << 6 + cb, err := NewCircularBuffer(size) + if err != nil { + t.Fatalf("error allocating CircularBuffer: %v", err) + } + + type item struct { + R uint32 + N uint32 + T time.Time + } + + var wg sync.WaitGroup + for r := uint32(0); r < 1024; r++ { + wg.Add(1) + go func(r uint32) { + for n := uint32(0); n < size; n++ { + cb.Push(item{R: r, N: n, T: time.Now()}) + } + wg.Done() + }(r) + } + + // Wait for all goroutines to finish only in one test. Draining + // concurrently while Pushes are still happening will test for races in the + // Draining lock. + if tn == 0 { + wg.Wait() + } + + result = cb.Drain() + + // Can't expect the buffer to be full if the Pushes aren't necessarily done. + if tn == 0 { + if uint32(len(result)) != size { + t.Fatalf("len(result) = %d; want %d", len(result), size) + } + } + + // There can be absolutely no expectation on the order of the data returned + // by Drain because: (a) everything is happening concurrently (b) a + // round-robin is used to write to different queues (and therefore + // different cachelines) for less write contention. + + // Wait for all goroutines to complete before moving on to other tests. If + // the benchmarks run after this, it might affect performance unfairly. + wg.Wait() + } +} + +func BenchmarkCircularBuffer(b *testing.B) { + x := 1 + for size := 1 << 16; size <= 1<<20; size <<= 1 { + for routines := 1; routines <= 1<<8; routines <<= 1 { + b.Run(fmt.Sprintf("goroutines:%d/size:%d", routines, size), func(b *testing.B) { + cb, err := NewCircularBuffer(uint32(size)) + if err != nil { + b.Fatalf("error allocating CircularBuffer: %v", err) + } + + perRoutine := b.N / routines + var wg sync.WaitGroup + for r := 0; r < routines; r++ { + wg.Add(1) + go func() { + for i := 0; i < perRoutine; i++ { + cb.Push(&x) + } + wg.Done() + }() + } + wg.Wait() + }) + } + } +} diff --git a/internal/profiling/goid_modified.go b/internal/profiling/goid_modified.go new file mode 100644 index 0000000..f2bae99 --- /dev/null +++ b/internal/profiling/goid_modified.go @@ -0,0 +1,82 @@ +//go:build grpcgoid +// +build grpcgoid + +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package profiling + +import ( + "runtime" +) + +// This stubbed function usually returns zero (see goid_regular.go); however, +// if grpc is built with `-tags 'grpcgoid'`, a runtime.Goid function, which +// does not exist in the Go standard library, is expected. While not necessary, +// sometimes, visualising grpc profiling data in trace-viewer is much nicer +// with goroutines separated from each other. +// +// Several other approaches were considered before arriving at this: +// +// 1. Using a CGO module: CGO usually has access to some things that regular +// Go does not. Till go1.4, CGO used to have access to the goroutine struct +// because the Go runtime was written in C. However, 1.5+ uses a native Go +// runtime; as a result, CGO does not have access to the goroutine structure +// anymore in modern Go. Besides, CGO interop wasn't fast enough (estimated +// to be ~170ns/op). This would also make building grpc require a C +// compiler, which isn't a requirement currently, breaking a lot of stuff. +// +// 2. Using runtime.Stack stacktrace: While this would remove the need for a +// modified Go runtime, this is ridiculously slow, thanks to the all the +// string processing shenanigans required to extract the goroutine ID (about +// ~2000ns/op). +// +// 3. Using Go version-specific build tags: For any given Go version, the +// goroutine struct has a fixed structure. As a result, the goroutine ID +// could be extracted if we know the offset using some assembly. This would +// be faster then #1 and #2, but is harder to maintain. This would require +// special Go code that's both architecture-specific and go version-specific +// (a quadratic number of variants to maintain). +// +// 4. This approach, which requires a simple modification [1] to the Go runtime +// to expose the current goroutine's ID. This is the chosen approach and it +// takes about ~2 ns/op, which is negligible in the face of the tens of +// microseconds that grpc takes to complete a RPC request. +// +// [1] To make the goroutine ID visible to Go programs apply the following +// change to the runtime2.go file in your Go runtime installation: +// +// diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go +// --- a/src/runtime/runtime2.go +// +++ b/src/runtime/runtime2.go +// @@ -392,6 +392,10 @@ type stack struct { +// hi uintptr +// } +// +// +func Goid() int64 { +// + return getg().goid +// +} +// + +// type g struct { +// // Stack parameters. +// // stack describes the actual stack memory: [stack.lo, stack.hi). +// +// The exposed runtime.Goid() function will return a int64 goroutine ID. +func goid() int64 { + return runtime.Goid() +} diff --git a/internal/profiling/goid_regular.go b/internal/profiling/goid_regular.go new file mode 100644 index 0000000..0429332 --- /dev/null +++ b/internal/profiling/goid_regular.go @@ -0,0 +1,30 @@ +//go:build !grpcgoid +// +build !grpcgoid + +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package profiling + +// This dummy function always returns 0. In some modified dev environments, +// this may be replaced with a call to a function in a modified Go runtime that +// retrieves the goroutine ID efficiently. See goid_modified.go for a different +// version of goId that requires a grpcgoid build tag to compile. +func goid() int64 { + return 0 +} diff --git a/internal/profiling/profiling.go b/internal/profiling/profiling.go new file mode 100644 index 0000000..f8ae14f --- /dev/null +++ b/internal/profiling/profiling.go @@ -0,0 +1,223 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package profiling contains two logical components: buffer.go and +// profiling.go. The former implements a circular buffer (a.k.a. ring buffer) +// in a lock-free manner using atomics. This ring buffer is used by +// profiling.go to store various statistics. For example, StreamStats is a +// circular buffer of Stat objects, each of which is comprised of Timers. +// +// This abstraction is designed to accommodate more stats in the future; for +// example, if one wants to profile the load balancing layer, which is +// independent of RPC queries, a separate CircularBuffer can be used. +// +// Note that the circular buffer simply takes any type. In the future, more +// types of measurements (such as the number of memory allocations) could be +// measured, which might require a different type of object being pushed into +// the circular buffer. +package profiling + +import ( + "errors" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/internal/profiling/buffer" +) + +// 0 or 1 representing profiling off and on, respectively. Use IsEnabled and +// Enable to get and set this in a safe manner. +var profilingEnabled uint32 + +// IsEnabled returns whether or not profiling is enabled. +func IsEnabled() bool { + return atomic.LoadUint32(&profilingEnabled) > 0 +} + +// Enable turns profiling on and off. +// +// Note that it is impossible to enable profiling for one server and leave it +// turned off for another. This is intentional and by design -- if the status +// of profiling was server-specific, clients wouldn't be able to profile +// themselves. As a result, Enable turns profiling on and off for all servers +// and clients in the binary. Each stat will be, however, tagged with whether +// it's a client stat or a server stat; so you should be able to filter for the +// right type of stats in post-processing. +func Enable(enabled bool) { + if enabled { + atomic.StoreUint32(&profilingEnabled, 1) + } else { + atomic.StoreUint32(&profilingEnabled, 0) + } +} + +// A Timer represents the wall-clock beginning and ending of a logical +// operation. +type Timer struct { + // Tags is a comma-separated list of strings (usually forward-slash-separated + // hierarchical strings) used to categorize a Timer. + Tags string + // Begin marks the beginning of this timer. The timezone is unspecified, but + // must use the same timezone as End; this is so shave off the small, but + // non-zero time required to convert to a standard timezone such as UTC. + Begin time.Time + // End marks the end of a timer. + End time.Time + // Each Timer must be started and ended within the same goroutine; GoID + // captures this goroutine ID. The Go runtime does not typically expose this + // information, so this is set to zero in the typical case. However, a + // trivial patch to the runtime package can make this field useful. See + // goid_modified.go in this package for more details. + GoID int64 +} + +// NewTimer creates and returns a new Timer object. This is useful when you +// don't already have a Stat object to associate this Timer with; for example, +// before the context of a new RPC query is created, a Timer may be needed to +// measure transport-related operations. +// +// Use AppendTimer to append the returned Timer to a Stat. +func NewTimer(tags string) *Timer { + return &Timer{ + Tags: tags, + Begin: time.Now(), + GoID: goid(), + } +} + +// Egress sets the End field of a timer to the current time. +func (timer *Timer) Egress() { + if timer == nil { + return + } + + timer.End = time.Now() +} + +// A Stat is a collection of Timers that represent timing information for +// different components within this Stat. For example, a Stat may be used to +// reference the entire lifetime of an RPC request, with Timers within it +// representing different components such as encoding, compression, and +// transport. +// +// The user is expected to use the included helper functions to do operations +// on the Stat such as creating or appending a new timer. Direct operations on +// the Stat's exported fields (which are exported for encoding reasons) may +// lead to data races. +type Stat struct { + // Tags is a comma-separated list of strings used to categorize a Stat. + Tags string + // Stats may also need to store other unstructured information specific to + // this stat. For example, a StreamStat will use these bytes to encode the + // connection ID and stream ID for each RPC to uniquely identify it. The + // encoding that must be used is unspecified. + Metadata []byte + // A collection of *Timers and a mutex for append operations on the slice. + mu sync.Mutex + Timers []*Timer +} + +// A power of two that's large enough to hold all timers within an average RPC +// request (defined to be a unary request) without any reallocation. A typical +// unary RPC creates 80-100 timers for various things. While this number is +// purely anecdotal and may change in the future as the resolution of profiling +// increases or decreases, it serves as a good estimate for what the initial +// allocation size should be. +const defaultStatAllocatedTimers int32 = 128 + +// NewStat creates and returns a new Stat object. +func NewStat(tags string) *Stat { + return &Stat{ + Tags: tags, + Timers: make([]*Timer, 0, defaultStatAllocatedTimers), + } +} + +// NewTimer creates a Timer object within the given stat if stat is non-nil. +// The value passed in tags will be attached to the newly created Timer. +// NewTimer also automatically sets the Begin value of the Timer to the current +// time. The user is expected to call stat.Egress with the returned index as +// argument to mark the end. +func (stat *Stat) NewTimer(tags string) *Timer { + if stat == nil { + return nil + } + + timer := &Timer{ + Tags: tags, + GoID: goid(), + Begin: time.Now(), + } + stat.mu.Lock() + stat.Timers = append(stat.Timers, timer) + stat.mu.Unlock() + return timer +} + +// AppendTimer appends a given Timer object to the internal slice of timers. A +// deep copy of the timer is made (i.e. no reference is retained to this +// pointer) and the user is expected to lose their reference to the timer to +// allow the Timer object to be garbage collected. +func (stat *Stat) AppendTimer(timer *Timer) { + if stat == nil || timer == nil { + return + } + + stat.mu.Lock() + stat.Timers = append(stat.Timers, timer) + stat.mu.Unlock() +} + +// statsInitialized is 0 before InitStats has been called. Changed to 1 by +// exactly one call to InitStats. +var statsInitialized int32 + +// Stats for the last defaultStreamStatsBufsize RPCs will be stored in memory. +// This can be configured by the registering server at profiling service +// initialization with google.golang.org/grpc/profiling/service.ProfilingConfig +const defaultStreamStatsSize uint32 = 16 << 10 + +// StreamStats is a CircularBuffer containing data from the last N RPC calls +// served, where N is set by the user. This will contain both server stats and +// client stats (but each stat will be tagged with whether it's a server or a +// client in its Tags). +var StreamStats *buffer.CircularBuffer + +var errAlreadyInitialized = errors.New("profiling may be initialized at most once") + +// InitStats initializes all the relevant Stat objects. Must be called exactly +// once per lifetime of a process; calls after the first one will return an +// error. +func InitStats(streamStatsSize uint32) error { + var err error + if !atomic.CompareAndSwapInt32(&statsInitialized, 0, 1) { + return errAlreadyInitialized + } + + if streamStatsSize == 0 { + streamStatsSize = defaultStreamStatsSize + } + + StreamStats, err = buffer.NewCircularBuffer(streamStatsSize) + if err != nil { + return err + } + + return nil +} diff --git a/internal/profiling/profiling_test.go b/internal/profiling/profiling_test.go new file mode 100644 index 0000000..a6184a1 --- /dev/null +++ b/internal/profiling/profiling_test.go @@ -0,0 +1,154 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package profiling + +import ( + "fmt" + "strconv" + "sync" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/profiling/buffer" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestProfiling(t *testing.T) { + cb, err := buffer.NewCircularBuffer(128) + if err != nil { + t.Fatalf("error creating circular buffer: %v", err) + } + + stat := NewStat("foo") + cb.Push(stat) + bar := func(n int) { + if n%2 == 0 { + defer stat.NewTimer(strconv.Itoa(n)).Egress() + } else { + timer := NewTimer(strconv.Itoa(n)) + stat.AppendTimer(timer) + defer timer.Egress() + } + time.Sleep(1 * time.Microsecond) + } + + numTimers := int(8 * defaultStatAllocatedTimers) + for i := 0; i < numTimers; i++ { + bar(i) + } + + results := cb.Drain() + if len(results) != 1 { + t.Fatalf("len(results) = %d; want 1", len(results)) + } + + statReturned := results[0].(*Stat) + if stat.Tags != "foo" { + t.Fatalf("stat.Tags = %s; want foo", stat.Tags) + } + + if len(stat.Timers) != numTimers { + t.Fatalf("len(stat.Timers) = %d; want %d", len(stat.Timers), numTimers) + } + + lastIdx := 0 + for i, timer := range statReturned.Timers { + // Check that they're in the order of append. + if n, err := strconv.Atoi(timer.Tags); err != nil && n != lastIdx { + t.Fatalf("stat.Timers[%d].Tags = %s; wanted %d", i, timer.Tags, lastIdx) + } + + // Check that the timestamps are consistent. + if diff := timer.End.Sub(timer.Begin); diff.Nanoseconds() < 1000 { + t.Fatalf("stat.Timers[%d].End - stat.Timers[%d].Begin = %v; want >= 1000ns", i, i, diff) + } + + lastIdx++ + } +} + +func (s) TestProfilingRace(t *testing.T) { + stat := NewStat("foo") + + var wg sync.WaitGroup + numTimers := int(8 * defaultStatAllocatedTimers) // also tests the slice growth code path + wg.Add(numTimers) + for i := 0; i < numTimers; i++ { + go func(n int) { + defer wg.Done() + if n%2 == 0 { + defer stat.NewTimer(strconv.Itoa(n)).Egress() + } else { + timer := NewTimer(strconv.Itoa(n)) + stat.AppendTimer(timer) + defer timer.Egress() + } + }(i) + } + wg.Wait() + + if len(stat.Timers) != numTimers { + t.Fatalf("len(stat.Timers) = %d; want %d", len(stat.Timers), numTimers) + } + + // The timers need not be ordered, so we can't expect them to be consecutive + // like above. + seen := make(map[int]bool) + for i, timer := range stat.Timers { + n, err := strconv.Atoi(timer.Tags) + if err != nil { + t.Fatalf("stat.Timers[%d].Tags = %s; wanted integer", i, timer.Tags) + } + seen[n] = true + } + + for i := 0; i < numTimers; i++ { + if _, ok := seen[i]; !ok { + t.Fatalf("seen[%d] = false or does not exist; want it to be true", i) + } + } +} + +func BenchmarkProfiling(b *testing.B) { + for routines := 1; routines <= 1<<8; routines <<= 1 { + b.Run(fmt.Sprintf("goroutines:%d", routines), func(b *testing.B) { + perRoutine := b.N / routines + stat := NewStat("foo") + var wg sync.WaitGroup + wg.Add(routines) + for r := 0; r < routines; r++ { + go func() { + for i := 0; i < perRoutine; i++ { + stat.NewTimer("bar").Egress() + } + wg.Done() + }() + } + wg.Wait() + }) + } +} diff --git a/internal/proto/grpc_lookup_v1/rls.pb.go b/internal/proto/grpc_lookup_v1/rls.pb.go new file mode 100644 index 0000000..8569578 --- /dev/null +++ b/internal/proto/grpc_lookup_v1/rls.pb.go @@ -0,0 +1,328 @@ +// Copyright 2020 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/lookup/v1/rls.proto + +package grpc_lookup_v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + anypb "google.golang.org/protobuf/types/known/anypb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Possible reasons for making a request. +type RouteLookupRequest_Reason int32 + +const ( + RouteLookupRequest_REASON_UNKNOWN RouteLookupRequest_Reason = 0 // Unused + RouteLookupRequest_REASON_MISS RouteLookupRequest_Reason = 1 // No data available in local cache + RouteLookupRequest_REASON_STALE RouteLookupRequest_Reason = 2 // Data in local cache is stale +) + +// Enum value maps for RouteLookupRequest_Reason. +var ( + RouteLookupRequest_Reason_name = map[int32]string{ + 0: "REASON_UNKNOWN", + 1: "REASON_MISS", + 2: "REASON_STALE", + } + RouteLookupRequest_Reason_value = map[string]int32{ + "REASON_UNKNOWN": 0, + "REASON_MISS": 1, + "REASON_STALE": 2, + } +) + +func (x RouteLookupRequest_Reason) Enum() *RouteLookupRequest_Reason { + p := new(RouteLookupRequest_Reason) + *p = x + return p +} + +func (x RouteLookupRequest_Reason) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (RouteLookupRequest_Reason) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_lookup_v1_rls_proto_enumTypes[0].Descriptor() +} + +func (RouteLookupRequest_Reason) Type() protoreflect.EnumType { + return &file_grpc_lookup_v1_rls_proto_enumTypes[0] +} + +func (x RouteLookupRequest_Reason) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use RouteLookupRequest_Reason.Descriptor instead. +func (RouteLookupRequest_Reason) EnumDescriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_proto_rawDescGZIP(), []int{0, 0} +} + +type RouteLookupRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Target type allows the client to specify what kind of target format it + // would like from RLS to allow it to find the regional server, e.g. "grpc". + TargetType string `protobuf:"bytes,3,opt,name=target_type,json=targetType,proto3" json:"target_type,omitempty"` + // Reason for making this request. + Reason RouteLookupRequest_Reason `protobuf:"varint,5,opt,name=reason,proto3,enum=grpc.lookup.v1.RouteLookupRequest_Reason" json:"reason,omitempty"` + // For REASON_STALE, the header_data from the stale response, if any. + StaleHeaderData string `protobuf:"bytes,6,opt,name=stale_header_data,json=staleHeaderData,proto3" json:"stale_header_data,omitempty"` + // Map of key values extracted via key builders for the gRPC or HTTP request. + KeyMap map[string]string `protobuf:"bytes,4,rep,name=key_map,json=keyMap,proto3" json:"key_map,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // Application-specific optional extensions. + Extensions []*anypb.Any `protobuf:"bytes,7,rep,name=extensions,proto3" json:"extensions,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RouteLookupRequest) Reset() { + *x = RouteLookupRequest{} + mi := &file_grpc_lookup_v1_rls_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RouteLookupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RouteLookupRequest) ProtoMessage() {} + +func (x *RouteLookupRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RouteLookupRequest.ProtoReflect.Descriptor instead. +func (*RouteLookupRequest) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_proto_rawDescGZIP(), []int{0} +} + +func (x *RouteLookupRequest) GetTargetType() string { + if x != nil { + return x.TargetType + } + return "" +} + +func (x *RouteLookupRequest) GetReason() RouteLookupRequest_Reason { + if x != nil { + return x.Reason + } + return RouteLookupRequest_REASON_UNKNOWN +} + +func (x *RouteLookupRequest) GetStaleHeaderData() string { + if x != nil { + return x.StaleHeaderData + } + return "" +} + +func (x *RouteLookupRequest) GetKeyMap() map[string]string { + if x != nil { + return x.KeyMap + } + return nil +} + +func (x *RouteLookupRequest) GetExtensions() []*anypb.Any { + if x != nil { + return x.Extensions + } + return nil +} + +type RouteLookupResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Prioritized list (best one first) of addressable entities to use + // for routing, using syntax requested by the request target_type. + // The targets will be tried in order until a healthy one is found. + Targets []string `protobuf:"bytes,3,rep,name=targets,proto3" json:"targets,omitempty"` + // Optional header value to pass along to AFE in the X-Google-RLS-Data header. + // Cached with "target" and sent with all requests that match the request key. + // Allows the RLS to pass its work product to the eventual target. + HeaderData string `protobuf:"bytes,2,opt,name=header_data,json=headerData,proto3" json:"header_data,omitempty"` + // Application-specific optional extensions. + Extensions []*anypb.Any `protobuf:"bytes,4,rep,name=extensions,proto3" json:"extensions,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RouteLookupResponse) Reset() { + *x = RouteLookupResponse{} + mi := &file_grpc_lookup_v1_rls_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RouteLookupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RouteLookupResponse) ProtoMessage() {} + +func (x *RouteLookupResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RouteLookupResponse.ProtoReflect.Descriptor instead. +func (*RouteLookupResponse) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_proto_rawDescGZIP(), []int{1} +} + +func (x *RouteLookupResponse) GetTargets() []string { + if x != nil { + return x.Targets + } + return nil +} + +func (x *RouteLookupResponse) GetHeaderData() string { + if x != nil { + return x.HeaderData + } + return "" +} + +func (x *RouteLookupResponse) GetExtensions() []*anypb.Any { + if x != nil { + return x.Extensions + } + return nil +} + +var File_grpc_lookup_v1_rls_proto protoreflect.FileDescriptor + +const file_grpc_lookup_v1_rls_proto_rawDesc = "" + + "\n" + + "\x18grpc/lookup/v1/rls.proto\x12\x0egrpc.lookup.v1\x1a\x19google/protobuf/any.proto\"\xb9\x03\n" + + "\x12RouteLookupRequest\x12\x1f\n" + + "\vtarget_type\x18\x03 \x01(\tR\n" + + "targetType\x12A\n" + + "\x06reason\x18\x05 \x01(\x0e2).grpc.lookup.v1.RouteLookupRequest.ReasonR\x06reason\x12*\n" + + "\x11stale_header_data\x18\x06 \x01(\tR\x0fstaleHeaderData\x12G\n" + + "\akey_map\x18\x04 \x03(\v2..grpc.lookup.v1.RouteLookupRequest.KeyMapEntryR\x06keyMap\x124\n" + + "\n" + + "extensions\x18\a \x03(\v2\x14.google.protobuf.AnyR\n" + + "extensions\x1a9\n" + + "\vKeyMapEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"?\n" + + "\x06Reason\x12\x12\n" + + "\x0eREASON_UNKNOWN\x10\x00\x12\x0f\n" + + "\vREASON_MISS\x10\x01\x12\x10\n" + + "\fREASON_STALE\x10\x02J\x04\b\x01\x10\x02J\x04\b\x02\x10\x03R\x06serverR\x04path\"\x94\x01\n" + + "\x13RouteLookupResponse\x12\x18\n" + + "\atargets\x18\x03 \x03(\tR\atargets\x12\x1f\n" + + "\vheader_data\x18\x02 \x01(\tR\n" + + "headerData\x124\n" + + "\n" + + "extensions\x18\x04 \x03(\v2\x14.google.protobuf.AnyR\n" + + "extensionsJ\x04\b\x01\x10\x02R\x06target2n\n" + + "\x12RouteLookupService\x12X\n" + + "\vRouteLookup\x12\".grpc.lookup.v1.RouteLookupRequest\x1a#.grpc.lookup.v1.RouteLookupResponse\"\x00BM\n" + + "\x11io.grpc.lookup.v1B\bRlsProtoP\x01Z,google.golang.org/grpc/lookup/grpc_lookup_v1b\x06proto3" + +var ( + file_grpc_lookup_v1_rls_proto_rawDescOnce sync.Once + file_grpc_lookup_v1_rls_proto_rawDescData []byte +) + +func file_grpc_lookup_v1_rls_proto_rawDescGZIP() []byte { + file_grpc_lookup_v1_rls_proto_rawDescOnce.Do(func() { + file_grpc_lookup_v1_rls_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_lookup_v1_rls_proto_rawDesc), len(file_grpc_lookup_v1_rls_proto_rawDesc))) + }) + return file_grpc_lookup_v1_rls_proto_rawDescData +} + +var file_grpc_lookup_v1_rls_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_grpc_lookup_v1_rls_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_grpc_lookup_v1_rls_proto_goTypes = []any{ + (RouteLookupRequest_Reason)(0), // 0: grpc.lookup.v1.RouteLookupRequest.Reason + (*RouteLookupRequest)(nil), // 1: grpc.lookup.v1.RouteLookupRequest + (*RouteLookupResponse)(nil), // 2: grpc.lookup.v1.RouteLookupResponse + nil, // 3: grpc.lookup.v1.RouteLookupRequest.KeyMapEntry + (*anypb.Any)(nil), // 4: google.protobuf.Any +} +var file_grpc_lookup_v1_rls_proto_depIdxs = []int32{ + 0, // 0: grpc.lookup.v1.RouteLookupRequest.reason:type_name -> grpc.lookup.v1.RouteLookupRequest.Reason + 3, // 1: grpc.lookup.v1.RouteLookupRequest.key_map:type_name -> grpc.lookup.v1.RouteLookupRequest.KeyMapEntry + 4, // 2: grpc.lookup.v1.RouteLookupRequest.extensions:type_name -> google.protobuf.Any + 4, // 3: grpc.lookup.v1.RouteLookupResponse.extensions:type_name -> google.protobuf.Any + 1, // 4: grpc.lookup.v1.RouteLookupService.RouteLookup:input_type -> grpc.lookup.v1.RouteLookupRequest + 2, // 5: grpc.lookup.v1.RouteLookupService.RouteLookup:output_type -> grpc.lookup.v1.RouteLookupResponse + 5, // [5:6] is the sub-list for method output_type + 4, // [4:5] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_grpc_lookup_v1_rls_proto_init() } +func file_grpc_lookup_v1_rls_proto_init() { + if File_grpc_lookup_v1_rls_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_lookup_v1_rls_proto_rawDesc), len(file_grpc_lookup_v1_rls_proto_rawDesc)), + NumEnums: 1, + NumMessages: 3, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_lookup_v1_rls_proto_goTypes, + DependencyIndexes: file_grpc_lookup_v1_rls_proto_depIdxs, + EnumInfos: file_grpc_lookup_v1_rls_proto_enumTypes, + MessageInfos: file_grpc_lookup_v1_rls_proto_msgTypes, + }.Build() + File_grpc_lookup_v1_rls_proto = out.File + file_grpc_lookup_v1_rls_proto_goTypes = nil + file_grpc_lookup_v1_rls_proto_depIdxs = nil +} diff --git a/internal/proto/grpc_lookup_v1/rls_config.pb.go b/internal/proto/grpc_lookup_v1/rls_config.pb.go new file mode 100644 index 0000000..9a2dd41 --- /dev/null +++ b/internal/proto/grpc_lookup_v1/rls_config.pb.go @@ -0,0 +1,774 @@ +// Copyright 2020 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/lookup/v1/rls_config.proto + +package grpc_lookup_v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Extract a key based on a given name (e.g. header name or query parameter +// name). The name must match one of the names listed in the "name" field. If +// the "required_match" field is true, one of the specified names must be +// present for the keybuilder to match. +type NameMatcher struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The name that will be used in the RLS key_map to refer to this value. + // If required_match is true, you may omit this field or set it to an empty + // string, in which case the matcher will require a match, but won't update + // the key_map. + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // Ordered list of names (headers or query parameter names) that can supply + // this value; the first one with a non-empty value is used. + Names []string `protobuf:"bytes,2,rep,name=names,proto3" json:"names,omitempty"` + // If true, make this extraction required; the key builder will not match + // if no value is found. + RequiredMatch bool `protobuf:"varint,3,opt,name=required_match,json=requiredMatch,proto3" json:"required_match,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NameMatcher) Reset() { + *x = NameMatcher{} + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NameMatcher) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NameMatcher) ProtoMessage() {} + +func (x *NameMatcher) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NameMatcher.ProtoReflect.Descriptor instead. +func (*NameMatcher) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{0} +} + +func (x *NameMatcher) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *NameMatcher) GetNames() []string { + if x != nil { + return x.Names + } + return nil +} + +func (x *NameMatcher) GetRequiredMatch() bool { + if x != nil { + return x.RequiredMatch + } + return false +} + +// A GrpcKeyBuilder applies to a given gRPC service, name, and headers. +type GrpcKeyBuilder struct { + state protoimpl.MessageState `protogen:"open.v1"` + Names []*GrpcKeyBuilder_Name `protobuf:"bytes,1,rep,name=names,proto3" json:"names,omitempty"` + ExtraKeys *GrpcKeyBuilder_ExtraKeys `protobuf:"bytes,3,opt,name=extra_keys,json=extraKeys,proto3" json:"extra_keys,omitempty"` + // Extract keys from all listed headers. + // For gRPC, it is an error to specify "required_match" on the NameMatcher + // protos. + Headers []*NameMatcher `protobuf:"bytes,2,rep,name=headers,proto3" json:"headers,omitempty"` + // You can optionally set one or more specific key/value pairs to be added to + // the key_map. This can be useful to identify which builder built the key, + // for example if you are suppressing the actual method, but need to + // separately cache and request all the matched methods. + ConstantKeys map[string]string `protobuf:"bytes,4,rep,name=constant_keys,json=constantKeys,proto3" json:"constant_keys,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GrpcKeyBuilder) Reset() { + *x = GrpcKeyBuilder{} + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GrpcKeyBuilder) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GrpcKeyBuilder) ProtoMessage() {} + +func (x *GrpcKeyBuilder) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GrpcKeyBuilder.ProtoReflect.Descriptor instead. +func (*GrpcKeyBuilder) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{1} +} + +func (x *GrpcKeyBuilder) GetNames() []*GrpcKeyBuilder_Name { + if x != nil { + return x.Names + } + return nil +} + +func (x *GrpcKeyBuilder) GetExtraKeys() *GrpcKeyBuilder_ExtraKeys { + if x != nil { + return x.ExtraKeys + } + return nil +} + +func (x *GrpcKeyBuilder) GetHeaders() []*NameMatcher { + if x != nil { + return x.Headers + } + return nil +} + +func (x *GrpcKeyBuilder) GetConstantKeys() map[string]string { + if x != nil { + return x.ConstantKeys + } + return nil +} + +// An HttpKeyBuilder applies to a given HTTP URL and headers. +// +// Path and host patterns use the matching syntax from gRPC transcoding to +// extract named key/value pairs from the path and host components of the URL: +// https://github.com/googleapis/googleapis/blob/master/google/api/http.proto +// +// It is invalid to specify the same key name in multiple places in a pattern. +// +// For a service where the project id can be expressed either as a subdomain or +// in the path, separate HttpKeyBuilders must be used: +// +// host_pattern: 'example.com' path_pattern: '/{id}/{object}/**' +// host_pattern: '{id}.example.com' path_pattern: '/{object}/**' +// +// If the host is exactly 'example.com', the first path segment will be used as +// the id and the second segment as the object. If the host has a subdomain, the +// subdomain will be used as the id and the first segment as the object. If +// neither pattern matches, no keys will be extracted. +type HttpKeyBuilder struct { + state protoimpl.MessageState `protogen:"open.v1"` + // host_pattern is an ordered list of host template patterns for the desired + // value. If any host_pattern values are specified, then at least one must + // match, and the last one wins and sets any specified variables. A host + // consists of labels separated by dots. Each label is matched against the + // label in the pattern as follows: + // - "*": Matches any single label. + // - "**": Matches zero or more labels (first or last part of host only). + // - "{=...}": One or more label capture, where "..." can be any + // template that does not include a capture. + // - "{}": A single label capture. Identical to {=*}. + // + // Examples: + // - "example.com": Only applies to the exact host example.com. + // - "*.example.com": Matches subdomains of example.com. + // - "**.example.com": matches example.com, and all levels of subdomains. + // - "{project}.example.com": Extracts the third level subdomain. + // - "{project=**}.example.com": Extracts the third level+ subdomains. + // - "{project=**}": Extracts the entire host. + HostPatterns []string `protobuf:"bytes,1,rep,name=host_patterns,json=hostPatterns,proto3" json:"host_patterns,omitempty"` + // path_pattern is an ordered list of path template patterns for the desired + // value. If any path_pattern values are specified, then at least one must + // match, and the last one wins and sets any specified variables. A path + // consists of segments separated by slashes. Each segment is matched against + // the segment in the pattern as follows: + // - "*": Matches any single segment. + // - "**": Matches zero or more segments (first or last part of path only). + // - "{=...}": One or more segment capture, where "..." can be any + // template that does not include a capture. + // - "{}": A single segment capture. Identical to {=*}. + // + // A custom method may also be specified by appending ":" and the custom + // method name or "*" to indicate any custom method (including no custom + // method). For example, "/*/projects/{project_id}/**:*" extracts + // `{project_id}` for any version, resource and custom method that includes + // it. By default, any custom method will be matched. + // + // Examples: + // - "/v1/{name=messages/*}": extracts a name like "messages/12345". + // - "/v1/messages/{message_id}": extracts a message_id like "12345". + // - "/v1/users/{user_id}/messages/{message_id}": extracts two key values. + PathPatterns []string `protobuf:"bytes,2,rep,name=path_patterns,json=pathPatterns,proto3" json:"path_patterns,omitempty"` + // List of query parameter names to try to match. + // For example: ["parent", "name", "resource.name"] + // We extract all the specified query_parameters (case-sensitively). If any + // are marked as "required_match" and are not present, this keybuilder fails + // to match. If a given parameter appears multiple times (?foo=a&foo=b) we + // will report it as a comma-separated string (foo=a,b). + QueryParameters []*NameMatcher `protobuf:"bytes,3,rep,name=query_parameters,json=queryParameters,proto3" json:"query_parameters,omitempty"` + // List of headers to try to match. + // We extract all the specified header values (case-insensitively). If any + // are marked as "required_match" and are not present, this keybuilder fails + // to match. If a given header appears multiple times in the request we will + // report it as a comma-separated string, in standard HTTP fashion. + Headers []*NameMatcher `protobuf:"bytes,4,rep,name=headers,proto3" json:"headers,omitempty"` + // You can optionally set one or more specific key/value pairs to be added to + // the key_map. This can be useful to identify which builder built the key, + // for example if you are suppressing a lot of information from the URL, but + // need to separately cache and request URLs with that content. + ConstantKeys map[string]string `protobuf:"bytes,5,rep,name=constant_keys,json=constantKeys,proto3" json:"constant_keys,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // If specified, the HTTP method/verb will be extracted under this key name. + Method string `protobuf:"bytes,6,opt,name=method,proto3" json:"method,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HttpKeyBuilder) Reset() { + *x = HttpKeyBuilder{} + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HttpKeyBuilder) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HttpKeyBuilder) ProtoMessage() {} + +func (x *HttpKeyBuilder) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HttpKeyBuilder.ProtoReflect.Descriptor instead. +func (*HttpKeyBuilder) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{2} +} + +func (x *HttpKeyBuilder) GetHostPatterns() []string { + if x != nil { + return x.HostPatterns + } + return nil +} + +func (x *HttpKeyBuilder) GetPathPatterns() []string { + if x != nil { + return x.PathPatterns + } + return nil +} + +func (x *HttpKeyBuilder) GetQueryParameters() []*NameMatcher { + if x != nil { + return x.QueryParameters + } + return nil +} + +func (x *HttpKeyBuilder) GetHeaders() []*NameMatcher { + if x != nil { + return x.Headers + } + return nil +} + +func (x *HttpKeyBuilder) GetConstantKeys() map[string]string { + if x != nil { + return x.ConstantKeys + } + return nil +} + +func (x *HttpKeyBuilder) GetMethod() string { + if x != nil { + return x.Method + } + return "" +} + +type RouteLookupConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Ordered specifications for constructing keys for HTTP requests. Last + // match wins. If no HttpKeyBuilder matches, an empty key_map will be sent to + // the lookup service; it should likely reply with a global default route + // and raise an alert. + HttpKeybuilders []*HttpKeyBuilder `protobuf:"bytes,1,rep,name=http_keybuilders,json=httpKeybuilders,proto3" json:"http_keybuilders,omitempty"` + // Unordered specifications for constructing keys for gRPC requests. All + // GrpcKeyBuilders on this list must have unique "name" fields so that the + // client is free to prebuild a hash map keyed by name. If no GrpcKeyBuilder + // matches, an empty key_map will be sent to the lookup service; it should + // likely reply with a global default route and raise an alert. + GrpcKeybuilders []*GrpcKeyBuilder `protobuf:"bytes,2,rep,name=grpc_keybuilders,json=grpcKeybuilders,proto3" json:"grpc_keybuilders,omitempty"` + // The name of the lookup service as a gRPC URI. Typically, this will be + // a subdomain of the target, such as "lookup.datastore.googleapis.com". + LookupService string `protobuf:"bytes,3,opt,name=lookup_service,json=lookupService,proto3" json:"lookup_service,omitempty"` + // Configure a timeout value for lookup service requests. + // Defaults to 10 seconds if not specified. + LookupServiceTimeout *durationpb.Duration `protobuf:"bytes,4,opt,name=lookup_service_timeout,json=lookupServiceTimeout,proto3" json:"lookup_service_timeout,omitempty"` + // How long are responses valid for (like HTTP Cache-Control). + // If omitted or zero, the longest valid cache time is used. + // This value is clamped to 5 minutes to avoid unflushable bad responses, + // unless stale_age is specified. + MaxAge *durationpb.Duration `protobuf:"bytes,5,opt,name=max_age,json=maxAge,proto3" json:"max_age,omitempty"` + // After a response has been in the client cache for this amount of time + // and is re-requested, start an asynchronous RPC to re-validate it. + // This value should be less than max_age by at least the length of a + // typical RTT to the Route Lookup Service to fully mask the RTT latency. + // If omitted, keys are only re-requested after they have expired. + // This value is clamped to 5 minutes. + StaleAge *durationpb.Duration `protobuf:"bytes,6,opt,name=stale_age,json=staleAge,proto3" json:"stale_age,omitempty"` + // Rough indicator of amount of memory to use for the client cache. Some of + // the data structure overhead is not accounted for, so actual memory consumed + // will be somewhat greater than this value. If this field is omitted or set + // to zero, a client default will be used. The value may be capped to a lower + // amount based on client configuration. + CacheSizeBytes int64 `protobuf:"varint,7,opt,name=cache_size_bytes,json=cacheSizeBytes,proto3" json:"cache_size_bytes,omitempty"` + // This is a list of all the possible targets that can be returned by the + // lookup service. If a target not on this list is returned, it will be + // treated the same as an unhealthy target. + ValidTargets []string `protobuf:"bytes,8,rep,name=valid_targets,json=validTargets,proto3" json:"valid_targets,omitempty"` + // This value provides a default target to use if needed. If set, it will be + // used if RLS returns an error, times out, or returns an invalid response. + // Note that requests can be routed only to a subdomain of the original + // target, e.g. "us_east_1.cloudbigtable.googleapis.com". + DefaultTarget string `protobuf:"bytes,9,opt,name=default_target,json=defaultTarget,proto3" json:"default_target,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RouteLookupConfig) Reset() { + *x = RouteLookupConfig{} + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RouteLookupConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RouteLookupConfig) ProtoMessage() {} + +func (x *RouteLookupConfig) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RouteLookupConfig.ProtoReflect.Descriptor instead. +func (*RouteLookupConfig) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{3} +} + +func (x *RouteLookupConfig) GetHttpKeybuilders() []*HttpKeyBuilder { + if x != nil { + return x.HttpKeybuilders + } + return nil +} + +func (x *RouteLookupConfig) GetGrpcKeybuilders() []*GrpcKeyBuilder { + if x != nil { + return x.GrpcKeybuilders + } + return nil +} + +func (x *RouteLookupConfig) GetLookupService() string { + if x != nil { + return x.LookupService + } + return "" +} + +func (x *RouteLookupConfig) GetLookupServiceTimeout() *durationpb.Duration { + if x != nil { + return x.LookupServiceTimeout + } + return nil +} + +func (x *RouteLookupConfig) GetMaxAge() *durationpb.Duration { + if x != nil { + return x.MaxAge + } + return nil +} + +func (x *RouteLookupConfig) GetStaleAge() *durationpb.Duration { + if x != nil { + return x.StaleAge + } + return nil +} + +func (x *RouteLookupConfig) GetCacheSizeBytes() int64 { + if x != nil { + return x.CacheSizeBytes + } + return 0 +} + +func (x *RouteLookupConfig) GetValidTargets() []string { + if x != nil { + return x.ValidTargets + } + return nil +} + +func (x *RouteLookupConfig) GetDefaultTarget() string { + if x != nil { + return x.DefaultTarget + } + return "" +} + +// RouteLookupClusterSpecifier is used in xDS to represent a cluster specifier +// plugin for RLS. +type RouteLookupClusterSpecifier struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The RLS config for this cluster specifier plugin instance. + RouteLookupConfig *RouteLookupConfig `protobuf:"bytes,1,opt,name=route_lookup_config,json=routeLookupConfig,proto3" json:"route_lookup_config,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RouteLookupClusterSpecifier) Reset() { + *x = RouteLookupClusterSpecifier{} + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RouteLookupClusterSpecifier) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RouteLookupClusterSpecifier) ProtoMessage() {} + +func (x *RouteLookupClusterSpecifier) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RouteLookupClusterSpecifier.ProtoReflect.Descriptor instead. +func (*RouteLookupClusterSpecifier) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{4} +} + +func (x *RouteLookupClusterSpecifier) GetRouteLookupConfig() *RouteLookupConfig { + if x != nil { + return x.RouteLookupConfig + } + return nil +} + +// To match, one of the given Name fields must match; the service and method +// fields are specified as fixed strings. The service name is required and +// includes the proto package name. The method name may be omitted, in +// which case any method on the given service is matched. +type GrpcKeyBuilder_Name struct { + state protoimpl.MessageState `protogen:"open.v1"` + Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"` + Method string `protobuf:"bytes,2,opt,name=method,proto3" json:"method,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GrpcKeyBuilder_Name) Reset() { + *x = GrpcKeyBuilder_Name{} + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GrpcKeyBuilder_Name) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GrpcKeyBuilder_Name) ProtoMessage() {} + +func (x *GrpcKeyBuilder_Name) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GrpcKeyBuilder_Name.ProtoReflect.Descriptor instead. +func (*GrpcKeyBuilder_Name) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{1, 0} +} + +func (x *GrpcKeyBuilder_Name) GetService() string { + if x != nil { + return x.Service + } + return "" +} + +func (x *GrpcKeyBuilder_Name) GetMethod() string { + if x != nil { + return x.Method + } + return "" +} + +// If you wish to include the host, service, or method names as keys in the +// generated RouteLookupRequest, specify key names to use in the extra_keys +// submessage. If a key name is empty, no key will be set for that value. +// If this submessage is specified, the normal host/path fields will be left +// unset in the RouteLookupRequest. We are deprecating host/path in the +// RouteLookupRequest, so services should migrate to the ExtraKeys approach. +type GrpcKeyBuilder_ExtraKeys struct { + state protoimpl.MessageState `protogen:"open.v1"` + Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` + Service string `protobuf:"bytes,2,opt,name=service,proto3" json:"service,omitempty"` + Method string `protobuf:"bytes,3,opt,name=method,proto3" json:"method,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GrpcKeyBuilder_ExtraKeys) Reset() { + *x = GrpcKeyBuilder_ExtraKeys{} + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GrpcKeyBuilder_ExtraKeys) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GrpcKeyBuilder_ExtraKeys) ProtoMessage() {} + +func (x *GrpcKeyBuilder_ExtraKeys) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GrpcKeyBuilder_ExtraKeys.ProtoReflect.Descriptor instead. +func (*GrpcKeyBuilder_ExtraKeys) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{1, 1} +} + +func (x *GrpcKeyBuilder_ExtraKeys) GetHost() string { + if x != nil { + return x.Host + } + return "" +} + +func (x *GrpcKeyBuilder_ExtraKeys) GetService() string { + if x != nil { + return x.Service + } + return "" +} + +func (x *GrpcKeyBuilder_ExtraKeys) GetMethod() string { + if x != nil { + return x.Method + } + return "" +} + +var File_grpc_lookup_v1_rls_config_proto protoreflect.FileDescriptor + +const file_grpc_lookup_v1_rls_config_proto_rawDesc = "" + + "\n" + + "\x1fgrpc/lookup/v1/rls_config.proto\x12\x0egrpc.lookup.v1\x1a\x1egoogle/protobuf/duration.proto\"\\\n" + + "\vNameMatcher\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05names\x18\x02 \x03(\tR\x05names\x12%\n" + + "\x0erequired_match\x18\x03 \x01(\bR\rrequiredMatch\"\xf0\x03\n" + + "\x0eGrpcKeyBuilder\x129\n" + + "\x05names\x18\x01 \x03(\v2#.grpc.lookup.v1.GrpcKeyBuilder.NameR\x05names\x12G\n" + + "\n" + + "extra_keys\x18\x03 \x01(\v2(.grpc.lookup.v1.GrpcKeyBuilder.ExtraKeysR\textraKeys\x125\n" + + "\aheaders\x18\x02 \x03(\v2\x1b.grpc.lookup.v1.NameMatcherR\aheaders\x12U\n" + + "\rconstant_keys\x18\x04 \x03(\v20.grpc.lookup.v1.GrpcKeyBuilder.ConstantKeysEntryR\fconstantKeys\x1a8\n" + + "\x04Name\x12\x18\n" + + "\aservice\x18\x01 \x01(\tR\aservice\x12\x16\n" + + "\x06method\x18\x02 \x01(\tR\x06method\x1aQ\n" + + "\tExtraKeys\x12\x12\n" + + "\x04host\x18\x01 \x01(\tR\x04host\x12\x18\n" + + "\aservice\x18\x02 \x01(\tR\aservice\x12\x16\n" + + "\x06method\x18\x03 \x01(\tR\x06method\x1a?\n" + + "\x11ConstantKeysEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x89\x03\n" + + "\x0eHttpKeyBuilder\x12#\n" + + "\rhost_patterns\x18\x01 \x03(\tR\fhostPatterns\x12#\n" + + "\rpath_patterns\x18\x02 \x03(\tR\fpathPatterns\x12F\n" + + "\x10query_parameters\x18\x03 \x03(\v2\x1b.grpc.lookup.v1.NameMatcherR\x0fqueryParameters\x125\n" + + "\aheaders\x18\x04 \x03(\v2\x1b.grpc.lookup.v1.NameMatcherR\aheaders\x12U\n" + + "\rconstant_keys\x18\x05 \x03(\v20.grpc.lookup.v1.HttpKeyBuilder.ConstantKeysEntryR\fconstantKeys\x12\x16\n" + + "\x06method\x18\x06 \x01(\tR\x06method\x1a?\n" + + "\x11ConstantKeysEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xa6\x04\n" + + "\x11RouteLookupConfig\x12I\n" + + "\x10http_keybuilders\x18\x01 \x03(\v2\x1e.grpc.lookup.v1.HttpKeyBuilderR\x0fhttpKeybuilders\x12I\n" + + "\x10grpc_keybuilders\x18\x02 \x03(\v2\x1e.grpc.lookup.v1.GrpcKeyBuilderR\x0fgrpcKeybuilders\x12%\n" + + "\x0elookup_service\x18\x03 \x01(\tR\rlookupService\x12O\n" + + "\x16lookup_service_timeout\x18\x04 \x01(\v2\x19.google.protobuf.DurationR\x14lookupServiceTimeout\x122\n" + + "\amax_age\x18\x05 \x01(\v2\x19.google.protobuf.DurationR\x06maxAge\x126\n" + + "\tstale_age\x18\x06 \x01(\v2\x19.google.protobuf.DurationR\bstaleAge\x12(\n" + + "\x10cache_size_bytes\x18\a \x01(\x03R\x0ecacheSizeBytes\x12#\n" + + "\rvalid_targets\x18\b \x03(\tR\fvalidTargets\x12%\n" + + "\x0edefault_target\x18\t \x01(\tR\rdefaultTargetJ\x04\b\n" + + "\x10\vR\x1brequest_processing_strategy\"p\n" + + "\x1bRouteLookupClusterSpecifier\x12Q\n" + + "\x13route_lookup_config\x18\x01 \x01(\v2!.grpc.lookup.v1.RouteLookupConfigR\x11routeLookupConfigBS\n" + + "\x11io.grpc.lookup.v1B\x0eRlsConfigProtoP\x01Z,google.golang.org/grpc/lookup/grpc_lookup_v1b\x06proto3" + +var ( + file_grpc_lookup_v1_rls_config_proto_rawDescOnce sync.Once + file_grpc_lookup_v1_rls_config_proto_rawDescData []byte +) + +func file_grpc_lookup_v1_rls_config_proto_rawDescGZIP() []byte { + file_grpc_lookup_v1_rls_config_proto_rawDescOnce.Do(func() { + file_grpc_lookup_v1_rls_config_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_lookup_v1_rls_config_proto_rawDesc), len(file_grpc_lookup_v1_rls_config_proto_rawDesc))) + }) + return file_grpc_lookup_v1_rls_config_proto_rawDescData +} + +var file_grpc_lookup_v1_rls_config_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_grpc_lookup_v1_rls_config_proto_goTypes = []any{ + (*NameMatcher)(nil), // 0: grpc.lookup.v1.NameMatcher + (*GrpcKeyBuilder)(nil), // 1: grpc.lookup.v1.GrpcKeyBuilder + (*HttpKeyBuilder)(nil), // 2: grpc.lookup.v1.HttpKeyBuilder + (*RouteLookupConfig)(nil), // 3: grpc.lookup.v1.RouteLookupConfig + (*RouteLookupClusterSpecifier)(nil), // 4: grpc.lookup.v1.RouteLookupClusterSpecifier + (*GrpcKeyBuilder_Name)(nil), // 5: grpc.lookup.v1.GrpcKeyBuilder.Name + (*GrpcKeyBuilder_ExtraKeys)(nil), // 6: grpc.lookup.v1.GrpcKeyBuilder.ExtraKeys + nil, // 7: grpc.lookup.v1.GrpcKeyBuilder.ConstantKeysEntry + nil, // 8: grpc.lookup.v1.HttpKeyBuilder.ConstantKeysEntry + (*durationpb.Duration)(nil), // 9: google.protobuf.Duration +} +var file_grpc_lookup_v1_rls_config_proto_depIdxs = []int32{ + 5, // 0: grpc.lookup.v1.GrpcKeyBuilder.names:type_name -> grpc.lookup.v1.GrpcKeyBuilder.Name + 6, // 1: grpc.lookup.v1.GrpcKeyBuilder.extra_keys:type_name -> grpc.lookup.v1.GrpcKeyBuilder.ExtraKeys + 0, // 2: grpc.lookup.v1.GrpcKeyBuilder.headers:type_name -> grpc.lookup.v1.NameMatcher + 7, // 3: grpc.lookup.v1.GrpcKeyBuilder.constant_keys:type_name -> grpc.lookup.v1.GrpcKeyBuilder.ConstantKeysEntry + 0, // 4: grpc.lookup.v1.HttpKeyBuilder.query_parameters:type_name -> grpc.lookup.v1.NameMatcher + 0, // 5: grpc.lookup.v1.HttpKeyBuilder.headers:type_name -> grpc.lookup.v1.NameMatcher + 8, // 6: grpc.lookup.v1.HttpKeyBuilder.constant_keys:type_name -> grpc.lookup.v1.HttpKeyBuilder.ConstantKeysEntry + 2, // 7: grpc.lookup.v1.RouteLookupConfig.http_keybuilders:type_name -> grpc.lookup.v1.HttpKeyBuilder + 1, // 8: grpc.lookup.v1.RouteLookupConfig.grpc_keybuilders:type_name -> grpc.lookup.v1.GrpcKeyBuilder + 9, // 9: grpc.lookup.v1.RouteLookupConfig.lookup_service_timeout:type_name -> google.protobuf.Duration + 9, // 10: grpc.lookup.v1.RouteLookupConfig.max_age:type_name -> google.protobuf.Duration + 9, // 11: grpc.lookup.v1.RouteLookupConfig.stale_age:type_name -> google.protobuf.Duration + 3, // 12: grpc.lookup.v1.RouteLookupClusterSpecifier.route_lookup_config:type_name -> grpc.lookup.v1.RouteLookupConfig + 13, // [13:13] is the sub-list for method output_type + 13, // [13:13] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name +} + +func init() { file_grpc_lookup_v1_rls_config_proto_init() } +func file_grpc_lookup_v1_rls_config_proto_init() { + if File_grpc_lookup_v1_rls_config_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_lookup_v1_rls_config_proto_rawDesc), len(file_grpc_lookup_v1_rls_config_proto_rawDesc)), + NumEnums: 0, + NumMessages: 9, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_lookup_v1_rls_config_proto_goTypes, + DependencyIndexes: file_grpc_lookup_v1_rls_config_proto_depIdxs, + MessageInfos: file_grpc_lookup_v1_rls_config_proto_msgTypes, + }.Build() + File_grpc_lookup_v1_rls_config_proto = out.File + file_grpc_lookup_v1_rls_config_proto_goTypes = nil + file_grpc_lookup_v1_rls_config_proto_depIdxs = nil +} diff --git a/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/internal/proto/grpc_lookup_v1/rls_grpc.pb.go new file mode 100644 index 0000000..0bf8477 --- /dev/null +++ b/internal/proto/grpc_lookup_v1/rls_grpc.pb.go @@ -0,0 +1,137 @@ +// Copyright 2020 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: grpc/lookup/v1/rls.proto + +package grpc_lookup_v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + RouteLookupService_RouteLookup_FullMethodName = "/grpc.lookup.v1.RouteLookupService/RouteLookup" +) + +// RouteLookupServiceClient is the client API for RouteLookupService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type RouteLookupServiceClient interface { + // Lookup returns a target for a single key. + RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error) +} + +type routeLookupServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewRouteLookupServiceClient(cc grpc.ClientConnInterface) RouteLookupServiceClient { + return &routeLookupServiceClient{cc} +} + +func (c *routeLookupServiceClient) RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RouteLookupResponse) + err := c.cc.Invoke(ctx, RouteLookupService_RouteLookup_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// RouteLookupServiceServer is the server API for RouteLookupService service. +// All implementations must embed UnimplementedRouteLookupServiceServer +// for forward compatibility. +type RouteLookupServiceServer interface { + // Lookup returns a target for a single key. + RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) + mustEmbedUnimplementedRouteLookupServiceServer() +} + +// UnimplementedRouteLookupServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedRouteLookupServiceServer struct{} + +func (UnimplementedRouteLookupServiceServer) RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) { + return nil, status.Error(codes.Unimplemented, "method RouteLookup not implemented") +} +func (UnimplementedRouteLookupServiceServer) mustEmbedUnimplementedRouteLookupServiceServer() {} +func (UnimplementedRouteLookupServiceServer) testEmbeddedByValue() {} + +// UnsafeRouteLookupServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to RouteLookupServiceServer will +// result in compilation errors. +type UnsafeRouteLookupServiceServer interface { + mustEmbedUnimplementedRouteLookupServiceServer() +} + +func RegisterRouteLookupServiceServer(s grpc.ServiceRegistrar, srv RouteLookupServiceServer) { + // If the following call panics, it indicates UnimplementedRouteLookupServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&RouteLookupService_ServiceDesc, srv) +} + +func _RouteLookupService_RouteLookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RouteLookupRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(RouteLookupServiceServer).RouteLookup(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: RouteLookupService_RouteLookup_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(RouteLookupServiceServer).RouteLookup(ctx, req.(*RouteLookupRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// RouteLookupService_ServiceDesc is the grpc.ServiceDesc for RouteLookupService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var RouteLookupService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.lookup.v1.RouteLookupService", + HandlerType: (*RouteLookupServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "RouteLookup", + Handler: _RouteLookupService_RouteLookup_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/lookup/v1/rls.proto", +} diff --git a/internal/proxyattributes/proxyattributes.go b/internal/proxyattributes/proxyattributes.go new file mode 100644 index 0000000..1f61f1a --- /dev/null +++ b/internal/proxyattributes/proxyattributes.go @@ -0,0 +1,54 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package proxyattributes contains functions for getting and setting proxy +// attributes like the CONNECT address and user info. +package proxyattributes + +import ( + "net/url" + + "google.golang.org/grpc/resolver" +) + +type keyType string + +const proxyOptionsKey = keyType("grpc.resolver.delegatingresolver.proxyOptions") + +// Options holds the proxy connection details needed during the CONNECT +// handshake. +type Options struct { + User *url.Userinfo + ConnectAddr string +} + +// Set returns a copy of addr with opts set in its attributes. +func Set(addr resolver.Address, opts Options) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(proxyOptionsKey, opts) + return addr +} + +// Get returns the Options for the proxy [resolver.Address] and a boolean +// value representing if the attribute is present or not. The returned data +// should not be mutated. +func Get(addr resolver.Address) (Options, bool) { + if a := addr.Attributes.Value(proxyOptionsKey); a != nil { + return a.(Options), true + } + return Options{}, false +} diff --git a/internal/proxyattributes/proxyattributes_test.go b/internal/proxyattributes/proxyattributes_test.go new file mode 100644 index 0000000..2c938c3 --- /dev/null +++ b/internal/proxyattributes/proxyattributes_test.go @@ -0,0 +1,116 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package proxyattributes + +import ( + "net/url" + "testing" + + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/resolver" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// Tests that Get returns a valid proxy attribute. +func (s) TestGet(t *testing.T) { + user := url.UserPassword("username", "password") + tests := []struct { + name string + addr resolver.Address + wantConnectAddr string + wantUser *url.Userinfo + wantAttrPresent bool + }{ + { + name: "connect_address_in_attribute", + addr: resolver.Address{ + Addr: "test-address", + Attributes: attributes.New(proxyOptionsKey, Options{ + ConnectAddr: "proxy-address", + }), + }, + wantConnectAddr: "proxy-address", + wantAttrPresent: true, + }, + { + name: "user_in_attribute", + addr: resolver.Address{ + Addr: "test-address", + Attributes: attributes.New(proxyOptionsKey, Options{ + User: user, + }), + }, + wantUser: user, + wantAttrPresent: true, + }, + { + name: "no_attribute", + addr: resolver.Address{Addr: "test-address"}, + wantAttrPresent: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotOption, attrPresent := Get(tt.addr) + if attrPresent != tt.wantAttrPresent { + t.Errorf("Get(%v) = %v, want %v", tt.addr, attrPresent, tt.wantAttrPresent) + } + + if gotOption.ConnectAddr != tt.wantConnectAddr { + t.Errorf("ConnectAddr(%v) = %v, want %v", tt.addr, gotOption.ConnectAddr, tt.wantConnectAddr) + } + + if gotOption.User != tt.wantUser { + t.Errorf("User(%v) = %v, want %v", tt.addr, gotOption.User, tt.wantUser) + } + }) + } +} + +// Tests that Set returns a copy of addr with attributes containing correct +// user and connect address. +func (s) TestSet(t *testing.T) { + addr := resolver.Address{Addr: "test-address"} + pOpts := Options{ + User: url.UserPassword("username", "password"), + ConnectAddr: "proxy-address", + } + + // Call Set and validate attributes + populatedAddr := Set(addr, pOpts) + gotOption, attrPresent := Get(populatedAddr) + if !attrPresent { + t.Errorf("Get(%v) = %v, want %v ", populatedAddr, attrPresent, true) + } + if got, want := gotOption.ConnectAddr, pOpts.ConnectAddr; got != want { + t.Errorf("unexpected ConnectAddr proxy atrribute = %v, want %v", got, want) + } + if got, want := gotOption.User, pOpts.User; got != want { + t.Errorf("unexpected User proxy attribute = %v, want %v", got, want) + } +} diff --git a/internal/resolver/config_selector.go b/internal/resolver/config_selector.go new file mode 100644 index 0000000..ead6dda --- /dev/null +++ b/internal/resolver/config_selector.go @@ -0,0 +1,100 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package resolver provides internal resolver-related functionality. +package resolver + +import ( + "context" + "sync" + + "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/resolver" +) + +// ConfigSelector controls what configuration to use for every RPC. +type ConfigSelector interface { + // Selects the configuration for the RPC, or terminates it using the error. + // This error will be converted by the gRPC library to a status error with + // code UNKNOWN if it is not returned as a status error. + SelectConfig(RPCInfo) (*RPCConfig, error) +} + +// RPCInfo contains RPC information needed by a ConfigSelector. +type RPCInfo struct { + // Context is the user's context for the RPC and contains headers and + // application timeout. It is passed for interception purposes and for + // efficiency reasons. SelectConfig should not be blocking. + Context context.Context + Method string // i.e. "/Service/Method" + // Authority is the target authority (host) name of the RPC. This is required + // for HTTP filters (such as external processing) to populate request + // attributes. + Authority string +} + +// RPCConfig describes the configuration to use for each RPC. +type RPCConfig struct { + // The context to use for the remainder of the RPC; can pass info to LB + // policy or affect timeout or metadata. + Context context.Context + MethodConfig serviceconfig.MethodConfig // configuration to use for this RPC + OnCommitted func() // Called when the RPC has been committed (retries no longer possible) + Interceptor any +} + +type csKeyType string + +const csKey = csKeyType("grpc.internal.resolver.configSelector") + +// SetConfigSelector sets the config selector in state and returns the new +// state. +func SetConfigSelector(state resolver.State, cs ConfigSelector) resolver.State { + state.Attributes = state.Attributes.WithValue(csKey, cs) + return state +} + +// GetConfigSelector retrieves the config selector from state, if present, and +// returns it or nil if absent. +func GetConfigSelector(state resolver.State) ConfigSelector { + cs, _ := state.Attributes.Value(csKey).(ConfigSelector) + return cs +} + +// SafeConfigSelector allows for safe switching of ConfigSelector +// implementations such that previous values are guaranteed to not be in use +// when UpdateConfigSelector returns. +type SafeConfigSelector struct { + mu sync.RWMutex + cs ConfigSelector +} + +// UpdateConfigSelector swaps to the provided ConfigSelector and blocks until +// all uses of the previous ConfigSelector have completed. +func (scs *SafeConfigSelector) UpdateConfigSelector(cs ConfigSelector) { + scs.mu.Lock() + defer scs.mu.Unlock() + scs.cs = cs +} + +// SelectConfig defers to the current ConfigSelector in scs. +func (scs *SafeConfigSelector) SelectConfig(r RPCInfo) (*RPCConfig, error) { + scs.mu.RLock() + defer scs.mu.RUnlock() + return scs.cs.SelectConfig(r) +} diff --git a/internal/resolver/config_selector_test.go b/internal/resolver/config_selector_test.go new file mode 100644 index 0000000..7a8a5db --- /dev/null +++ b/internal/resolver/config_selector_test.go @@ -0,0 +1,156 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/serviceconfig" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type fakeConfigSelector struct { + selectConfig func(RPCInfo) (*RPCConfig, error) +} + +func (f *fakeConfigSelector) SelectConfig(r RPCInfo) (*RPCConfig, error) { + return f.selectConfig(r) +} + +func (s) TestSafeConfigSelector(t *testing.T) { + testRPCInfo := RPCInfo{Method: "test method"} + + retChan1 := make(chan *RPCConfig) + retChan2 := make(chan *RPCConfig) + defer close(retChan1) + defer close(retChan2) + + one := 1 + two := 2 + + resp1 := &RPCConfig{MethodConfig: serviceconfig.MethodConfig{MaxReqSize: &one}} + resp2 := &RPCConfig{MethodConfig: serviceconfig.MethodConfig{MaxReqSize: &two}} + + cs1Called := make(chan struct{}, 1) + cs2Called := make(chan struct{}, 1) + + cs1 := &fakeConfigSelector{ + selectConfig: func(r RPCInfo) (*RPCConfig, error) { + cs1Called <- struct{}{} + if diff := cmp.Diff(r, testRPCInfo); diff != "" { + t.Errorf("SelectConfig(%v) called; want %v\n Diffs:\n%s", r, testRPCInfo, diff) + } + return <-retChan1, nil + }, + } + cs2 := &fakeConfigSelector{ + selectConfig: func(r RPCInfo) (*RPCConfig, error) { + cs2Called <- struct{}{} + if diff := cmp.Diff(r, testRPCInfo); diff != "" { + t.Errorf("SelectConfig(%v) called; want %v\n Diffs:\n%s", r, testRPCInfo, diff) + } + return <-retChan2, nil + }, + } + + scs := &SafeConfigSelector{} + scs.UpdateConfigSelector(cs1) + + cs1Returned := make(chan struct{}) + go func() { + got, err := scs.SelectConfig(testRPCInfo) // blocks until send to retChan1 + if err != nil || got != resp1 { + t.Errorf("SelectConfig(%v) = %v, %v; want %v, nil", testRPCInfo, got, err, resp1) + } + close(cs1Returned) + }() + + // cs1 is blocked but should be called + select { + case <-time.After(500 * time.Millisecond): + t.Fatalf("timed out waiting for cs1 to be called") + case <-cs1Called: + } + + // swap in cs2 now that cs1 is called + csSwapped := make(chan struct{}) + go func() { + // wait awhile first to ensure cs1 could be called below. + time.Sleep(50 * time.Millisecond) + scs.UpdateConfigSelector(cs2) // Blocks until cs1 done + close(csSwapped) + }() + + // Allow cs1 to return and cs2 to eventually be swapped in. + retChan1 <- resp1 + + cs1Done := false // set when cs2 is first called + for dl := time.Now().Add(150 * time.Millisecond); !time.Now().After(dl); { + gotConfigChan := make(chan *RPCConfig, 1) + go func() { + cfg, _ := scs.SelectConfig(testRPCInfo) + gotConfigChan <- cfg + }() + select { + case <-time.After(500 * time.Millisecond): + t.Fatalf("timed out waiting for cs1 or cs2 to be called") + case <-cs1Called: + // Initially, before swapping to cs2, cs1 should be called + retChan1 <- resp1 + go func() { <-gotConfigChan }() + if cs1Done { + t.Fatalf("cs1 called after cs2") + } + case <-cs2Called: + // Success! the new config selector is being called + if !cs1Done { + select { + case <-csSwapped: + case <-time.After(50 * time.Millisecond): + t.Fatalf("timed out waiting for UpdateConfigSelector to return") + } + select { + case <-cs1Returned: + case <-time.After(50 * time.Millisecond): + t.Fatalf("timed out waiting for cs1 to return") + } + cs1Done = true + } + retChan2 <- resp2 + got := <-gotConfigChan + if diff := cmp.Diff(got, resp2); diff != "" { + t.Fatalf("SelectConfig(%v) = %v; want %v\n Diffs:\n%s", testRPCInfo, got, resp2, diff) + } + } + time.Sleep(10 * time.Millisecond) + } + if !cs1Done { + t.Fatalf("timed out waiting for cs2 to be called") + } +} diff --git a/internal/resolver/delegatingresolver/delegatingresolver.go b/internal/resolver/delegatingresolver/delegatingresolver.go new file mode 100644 index 0000000..5bfa67b --- /dev/null +++ b/internal/resolver/delegatingresolver/delegatingresolver.go @@ -0,0 +1,477 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package delegatingresolver implements a resolver capable of resolving both +// target URIs and proxy addresses. +package delegatingresolver + +import ( + "fmt" + "net" + "net/http" + "net/url" + "sync" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/proxyattributes" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/internal/transport/networktype" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +var ( + logger = grpclog.Component("delegating-resolver") + // HTTPSProxyFromEnvironment will be overwritten in the tests + HTTPSProxyFromEnvironment = http.ProxyFromEnvironment +) + +const defaultPort = "443" + +// delegatingResolver manages both target URI and proxy address resolution by +// delegating these tasks to separate child resolvers. Essentially, it acts as +// an intermediary between the gRPC ClientConn and the child resolvers. +// +// It implements the [resolver.Resolver] interface. +type delegatingResolver struct { + target resolver.Target // parsed target URI to be resolved + cc resolver.ClientConn // gRPC ClientConn + proxyURL *url.URL // proxy URL, derived from proxy environment and target + + // We do not hold both mu and childMu in the same goroutine. Avoid holding + // both locks when calling into the child, as the child resolver may + // synchronously callback into the channel. + mu sync.Mutex // protects all the fields below + targetResolverState *resolver.State // state of the target resolver + proxyAddrs []resolver.Address // resolved proxy addresses; empty if no proxy is configured + + // childMu serializes calls into child resolvers. It also protects access to + // the following fields. + childMu sync.Mutex + targetResolver resolver.Resolver // resolver for the target URI, based on its scheme + proxyResolver resolver.Resolver // resolver for the proxy URI; nil if no proxy is configured +} + +// nopResolver is a resolver that does nothing. +type nopResolver struct{} + +func (nopResolver) ResolveNow(resolver.ResolveNowOptions) {} + +func (nopResolver) Close() {} + +// proxyURLForTarget determines the proxy URL for the given address based on the +// environment. It can return the following: +// - nil URL, nil error: No proxy is configured or the address is excluded +// using the `NO_PROXY` environment variable or if req.URL.Host is +// "localhost" (with or without // a port number) +// - nil URL, non-nil error: An error occurred while retrieving the proxy URL. +// - non-nil URL, nil error: A proxy is configured, and the proxy URL was +// retrieved successfully without any errors. +func proxyURLForTarget(address string) (*url.URL, error) { + req := &http.Request{URL: &url.URL{ + Scheme: "https", + Host: address, + }} + return HTTPSProxyFromEnvironment(req) +} + +// New creates a new delegating resolver that can create up to two child +// resolvers: +// - one to resolve the proxy address specified using the supported +// environment variables. This uses the registered resolver for the "dns" +// scheme. It is lazily built when a target resolver update contains at least +// one TCP address. +// - one to resolve the target URI using the resolver specified by the scheme +// in the target URI or specified by the user using the WithResolvers dial +// option. As a special case, if the target URI's scheme is "dns" and a +// proxy is specified using the supported environment variables, the target +// URI's path portion is used as the resolved address unless target +// resolution is enabled using the dial option. +func New(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions, targetResolverBuilder resolver.Builder, targetResolutionEnabled bool) (resolver.Resolver, error) { + r := &delegatingResolver{ + target: target, + cc: cc, + proxyResolver: nopResolver{}, + targetResolver: nopResolver{}, + } + + addr := target.Endpoint() + var err error + if target.URL.Scheme == "dns" && !targetResolutionEnabled && envconfig.EnableDefaultPortForProxyTarget { + addr, err = parseTarget(addr) + if err != nil { + return nil, fmt.Errorf("delegating_resolver: invalid target address %q: %v", target.Endpoint(), err) + } + } + + r.proxyURL, err = proxyURLForTarget(addr) + if err != nil { + return nil, fmt.Errorf("delegating_resolver: failed to determine proxy URL for target %q: %v", target, err) + } + + // proxy is not configured or proxy address excluded using `NO_PROXY` env + // var, so only target resolver is used. + if r.proxyURL == nil { + return targetResolverBuilder.Build(target, cc, opts) + } + + if logger.V(2) { + logger.Infof("Proxy URL detected : %s", r.proxyURL) + } + + // Resolver updates from one child may trigger calls into the other. Block + // updates until the children are initialized. + r.childMu.Lock() + defer r.childMu.Unlock() + // When the scheme is 'dns' and target resolution on client is not enabled, + // resolution should be handled by the proxy, not the client. Therefore, we + // bypass the target resolver and store the unresolved target address. + if target.URL.Scheme == "dns" && !targetResolutionEnabled { + r.targetResolverState = &resolver.State{ + Addresses: []resolver.Address{{Addr: addr}}, + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: addr}}}}, + } + r.updateTargetResolverState(*r.targetResolverState) + return r, nil + } + wcc := &wrappingClientConn{ + stateListener: r.updateTargetResolverState, + parent: r, + } + if r.targetResolver, err = targetResolverBuilder.Build(target, wcc, opts); err != nil { + return nil, fmt.Errorf("delegating_resolver: unable to build the resolver for target %s: %v", target, err) + } + return r, nil +} + +// proxyURIResolver creates a resolver for resolving proxy URIs using the "dns" +// scheme. It adjusts the proxyURL to conform to the "dns:///" format and builds +// a resolver with a wrappingClientConn to capture resolved addresses. +func (r *delegatingResolver) proxyURIResolver(opts resolver.BuildOptions) (resolver.Resolver, error) { + proxyBuilder := resolver.Get("dns") + if proxyBuilder == nil { + panic("delegating_resolver: resolver for proxy not found for scheme dns") + } + url := *r.proxyURL + url.Scheme = "dns" + url.Path = "/" + r.proxyURL.Host + url.Host = "" // Clear the Host field to conform to the "dns:///" format + + proxyTarget := resolver.Target{URL: url} + wcc := &wrappingClientConn{ + stateListener: r.updateProxyResolverState, + parent: r, + } + return proxyBuilder.Build(proxyTarget, wcc, opts) +} + +func (r *delegatingResolver) ResolveNow(o resolver.ResolveNowOptions) { + r.childMu.Lock() + defer r.childMu.Unlock() + r.targetResolver.ResolveNow(o) + r.proxyResolver.ResolveNow(o) +} + +func (r *delegatingResolver) Close() { + r.childMu.Lock() + defer r.childMu.Unlock() + r.targetResolver.Close() + r.targetResolver = nil + + r.proxyResolver.Close() + r.proxyResolver = nil +} + +func needsProxyResolver(state *resolver.State) bool { + for _, addr := range state.Addresses { + if !skipProxy(addr) { + return true + } + } + for _, endpoint := range state.Endpoints { + for _, addr := range endpoint.Addresses { + if !skipProxy(addr) { + return true + } + } + } + return false +} + +// parseTarget takes a target string and ensures it is a valid "host:port" target. +// +// It does the following: +// 1. If the target already has a port (e.g., "host:port", "[ipv6]:port"), +// it is returned as is. +// 2. If the host part is empty (e.g., ":80"), it defaults to "localhost", +// returning "localhost:80". +// 3. If the target is missing a port (e.g., "host", "ipv6"), the defaultPort +// is added. +// +// An error is returned for empty targets or targets with a trailing colon +// but no port (e.g., "host:"). +func parseTarget(target string) (string, error) { + if target == "" { + return "", fmt.Errorf("missing address") + } + + host, port, err := net.SplitHostPort(target) + if err != nil { + // If SplitHostPort fails, it's likely because the port is missing. + // We append the default port and return the result. + return net.JoinHostPort(target, defaultPort), nil + } + + // If SplitHostPort succeeds, we check for edge cases. + if port == "" { + // A success with an empty port means the target had a trailing colon, + // e.g., "host:", which is an error. + return "", fmt.Errorf("missing port after port-separator colon") + } + if host == "" { + // A success with an empty host means the target was like ":80". + // We default the host to "localhost". + host = "localhost" + } + return net.JoinHostPort(host, port), nil +} + +func skipProxy(address resolver.Address) bool { + // Avoid proxy when network is not tcp. + networkType, ok := networktype.Get(address) + if !ok { + networkType, _ = transport.ParseDialTarget(address.Addr) + } + if networkType != "tcp" { + return true + } + + req := &http.Request{URL: &url.URL{ + Scheme: "https", + Host: address.Addr, + }} + // Avoid proxy when address included in `NO_PROXY` environment variable or + // fails to get the proxy address. + url, err := HTTPSProxyFromEnvironment(req) + if err != nil || url == nil { + return true + } + return false +} + +// updateClientConnStateLocked constructs a combined list of addresses by +// pairing each proxy address with every target address of type TCP. For each +// pair, it creates a new [resolver.Address] using the proxy address and +// attaches the corresponding target address and user info as attributes. Target +// addresses that are not of type TCP are appended to the list as-is. The +// function returns nil if either resolver has not yet provided an update, and +// returns the result of ClientConn.UpdateState once both resolvers have +// provided at least one update. +func (r *delegatingResolver) updateClientConnStateLocked() error { + if r.targetResolverState == nil || r.proxyAddrs == nil { + return nil + } + + // If multiple resolved proxy addresses are present, we send only the + // unresolved proxy host and let net.Dial handle the proxy host name + // resolution when creating the transport. Sending all resolved addresses + // would increase the number of addresses passed to the ClientConn and + // subsequently to load balancing (LB) policies like Round Robin, leading + // to additional TCP connections. However, if there's only one resolved + // proxy address, we send it directly, as it doesn't affect the address + // count returned by the target resolver and the address count sent to the + // ClientConn. + var proxyAddr resolver.Address + if len(r.proxyAddrs) == 1 { + proxyAddr = r.proxyAddrs[0] + } else { + proxyAddr = resolver.Address{Addr: r.proxyURL.Host} + } + var addresses []resolver.Address + for _, targetAddr := range (*r.targetResolverState).Addresses { + if skipProxy(targetAddr) { + addresses = append(addresses, targetAddr) + continue + } + addresses = append(addresses, proxyattributes.Set(proxyAddr, proxyattributes.Options{ + User: r.proxyURL.User, + ConnectAddr: targetAddr.Addr, + })) + } + + // For each target endpoint, construct a new [resolver.Endpoint] that + // includes all addresses from all proxy endpoints and the addresses from + // that target endpoint, preserving the number of target endpoints. + var endpoints []resolver.Endpoint + for _, endpt := range (*r.targetResolverState).Endpoints { + var addrs []resolver.Address + for _, targetAddr := range endpt.Addresses { + // Avoid proxy when network is not tcp. + if skipProxy(targetAddr) { + addrs = append(addrs, targetAddr) + continue + } + for _, proxyAddr := range r.proxyAddrs { + addrs = append(addrs, proxyattributes.Set(proxyAddr, proxyattributes.Options{ + User: r.proxyURL.User, + ConnectAddr: targetAddr.Addr, + })) + } + } + endpoints = append(endpoints, resolver.Endpoint{Addresses: addrs}) + } + // Use the targetResolverState for its service config and attributes + // contents. The state update is only sent after both the target and proxy + // resolvers have sent their updates, and curState has been updated with the + // combined addresses. + curState := *r.targetResolverState + curState.Addresses = addresses + curState.Endpoints = endpoints + return r.cc.UpdateState(curState) +} + +// updateProxyResolverState updates the proxy resolver state by storing proxy +// addresses and endpoints, marking the resolver as ready, and triggering a +// state update if both proxy and target resolvers are ready. If the ClientConn +// returns a non-nil error, it calls `ResolveNow()` on the target resolver. It +// is a StateListener function of wrappingClientConn passed to the proxy +// resolver. +func (r *delegatingResolver) updateProxyResolverState(state resolver.State) error { + r.mu.Lock() + defer r.mu.Unlock() + if logger.V(2) { + logger.Infof("Addresses received from proxy resolver: %s", state.Addresses) + } + if len(state.Endpoints) > 0 { + // We expect exactly one address per endpoint because the proxy resolver + // uses "dns" resolution. + r.proxyAddrs = make([]resolver.Address, 0, len(state.Endpoints)) + for _, endpoint := range state.Endpoints { + r.proxyAddrs = append(r.proxyAddrs, endpoint.Addresses...) + } + } else if state.Addresses != nil { + r.proxyAddrs = state.Addresses + } else { + r.proxyAddrs = []resolver.Address{} // ensure proxyAddrs is non-nil to indicate an update has been received + } + err := r.updateClientConnStateLocked() + // Another possible approach was to block until updates are received from + // both resolvers. But this is not used because calling `New()` triggers + // `Build()` for the first resolver, which calls `UpdateState()`. And the + // second resolver hasn't sent an update yet, so it would cause `New()` to + // block indefinitely. + if err != nil { + go func() { + r.childMu.Lock() + defer r.childMu.Unlock() + if r.targetResolver != nil { + r.targetResolver.ResolveNow(resolver.ResolveNowOptions{}) + } + }() + } + return err +} + +// updateTargetResolverState is the StateListener function provided to the +// target resolver via wrappingClientConn. It updates the resolver state and +// marks the target resolver as ready. If the update includes at least one TCP +// address and the proxy resolver has not yet been constructed, it initializes +// the proxy resolver. A combined state update is triggered once both resolvers +// are ready. If all addresses are non-TCP, it proceeds without waiting for the +// proxy resolver. If ClientConn.UpdateState returns a non-nil error, +// ResolveNow() is called on the proxy resolver. +func (r *delegatingResolver) updateTargetResolverState(state resolver.State) error { + r.mu.Lock() + defer r.mu.Unlock() + + if logger.V(2) { + logger.Infof("Addresses received from target resolver: %v", state.Addresses) + } + r.targetResolverState = &state + // If all addresses returned by the target resolver have a non-TCP network + // type, or are listed in the `NO_PROXY` environment variable, do not wait + // for proxy update. + if !needsProxyResolver(r.targetResolverState) { + return r.cc.UpdateState(*r.targetResolverState) + } + + // The proxy resolver may be rebuilt multiple times, specifically each time + // the target resolver sends an update, even if the target resolver is built + // successfully but building the proxy resolver fails. + if len(r.proxyAddrs) == 0 { + go func() { + r.childMu.Lock() + defer r.childMu.Unlock() + if _, ok := r.proxyResolver.(nopResolver); !ok { + return + } + proxyResolver, err := r.proxyURIResolver(resolver.BuildOptions{}) + if err != nil { + r.cc.ReportError(fmt.Errorf("delegating_resolver: unable to build the proxy resolver: %v", err)) + return + } + r.proxyResolver = proxyResolver + }() + } + + err := r.updateClientConnStateLocked() + if err != nil { + go func() { + r.childMu.Lock() + defer r.childMu.Unlock() + if r.proxyResolver != nil { + r.proxyResolver.ResolveNow(resolver.ResolveNowOptions{}) + } + }() + } + return nil +} + +// wrappingClientConn serves as an intermediary between the parent ClientConn +// and the child resolvers created here. It implements the resolver.ClientConn +// interface and is passed in that capacity to the child resolvers. +type wrappingClientConn struct { + // Callback to deliver resolver state updates + stateListener func(state resolver.State) error + parent *delegatingResolver +} + +// UpdateState receives resolver state updates and forwards them to the +// appropriate listener function (either for the proxy or target resolver). +func (wcc *wrappingClientConn) UpdateState(state resolver.State) error { + return wcc.stateListener(state) +} + +// ReportError intercepts errors from the child resolvers and passes them to +// ClientConn. +func (wcc *wrappingClientConn) ReportError(err error) { + wcc.parent.cc.ReportError(err) +} + +// NewAddress intercepts the new resolved address from the child resolvers and +// passes them to ClientConn. +func (wcc *wrappingClientConn) NewAddress(addrs []resolver.Address) { + wcc.UpdateState(resolver.State{Addresses: addrs}) +} + +// ParseServiceConfig parses the provided service config and returns an object +// that provides the parsed config. +func (wcc *wrappingClientConn) ParseServiceConfig(serviceConfigJSON string) *serviceconfig.ParseResult { + return wcc.parent.cc.ParseServiceConfig(serviceConfigJSON) +} diff --git a/internal/resolver/delegatingresolver/delegatingresolver_ext_test.go b/internal/resolver/delegatingresolver/delegatingresolver_ext_test.go new file mode 100644 index 0000000..7b02d43 --- /dev/null +++ b/internal/resolver/delegatingresolver/delegatingresolver_ext_test.go @@ -0,0 +1,1025 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package delegatingresolver_test + +import ( + "context" + "errors" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/proxyattributes" + "google.golang.org/grpc/internal/resolver/delegatingresolver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/transport/networktype" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + + _ "google.golang.org/grpc/resolver/dns" // To register dns resolver. +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +// createTestResolverClientConn initializes a [testutils.ResolverClientConn] and +// returns it along with channels for resolver state updates and errors. +func createTestResolverClientConn(t *testing.T) (*testutils.ResolverClientConn, chan resolver.State, chan error) { + t.Helper() + stateCh := make(chan resolver.State, 1) + errCh := make(chan error, 1) + + tcc := &testutils.ResolverClientConn{ + Logger: t, + UpdateStateF: func(s resolver.State) error { stateCh <- s; return nil }, + ReportErrorF: func(err error) { errCh <- err }, + } + return tcc, stateCh, errCh +} + +// Tests the scenario where no proxy environment variables are set or proxying +// is disabled by the `NO_PROXY` environment variable. The test verifies that +// the delegating resolver creates only a target resolver and that the +// addresses returned by the delegating resolver exactly match those returned +// by the target resolver. +func (s) TestDelegatingResolverNoProxyEnvVarsSet(t *testing.T) { + hpfe := func(*http.Request) (*url.URL, error) { return nil, nil } + originalhpfe := delegatingresolver.HTTPSProxyFromEnvironment + delegatingresolver.HTTPSProxyFromEnvironment = hpfe + defer func() { + delegatingresolver.HTTPSProxyFromEnvironment = originalhpfe + }() + + const ( + targetTestAddr = "test.com" + resolvedTargetTestAddr1 = "1.1.1.1:8080" + resolvedTargetTestAddr2 = "2.2.2.2:8080" + ) + + // Set up a manual resolver to control the address resolution. + targetResolver := manual.NewBuilderWithScheme("test") + target := targetResolver.Scheme() + ":///" + targetTestAddr + + // Create a delegating resolver with no proxy configuration + tcc, stateCh, _ := createTestResolverClientConn(t) + if _, err := delegatingresolver.New(resolver.Target{URL: *testutils.MustParseURL(target)}, tcc, resolver.BuildOptions{}, targetResolver, false); err != nil { + t.Fatalf("Failed to create delegating resolver: %v", err) + } + + // Update the manual resolver with a test address. + targetResolver.UpdateState(resolver.State{ + Addresses: []resolver.Address{ + {Addr: resolvedTargetTestAddr1}, + {Addr: resolvedTargetTestAddr2}, + }, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + + // Verify that the delegating resolver outputs the same addresses, as returned + // by the target resolver. + wantState := resolver.State{ + Addresses: []resolver.Address{ + {Addr: resolvedTargetTestAddr1}, + {Addr: resolvedTargetTestAddr2}, + }, + ServiceConfig: &serviceconfig.ParseResult{}, + } + + var gotState resolver.State + select { + case gotState = <-stateCh: + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout when waiting for a state update from the delegating resolver") + } + + if diff := cmp.Diff(gotState, wantState); diff != "" { + t.Fatalf("Unexpected state from delegating resolver. Diff (-got +want):\n%v", diff) + } +} + +// setupDNS registers a new manual resolver for the DNS scheme, effectively +// overwriting the previously registered DNS resolver. This allows the test to +// mock the DNS resolution for the proxy resolver. It also registers the +// original DNS resolver after the test is done. +func setupDNS(t *testing.T) (*manual.Resolver, chan struct{}) { + t.Helper() + mr := manual.NewBuilderWithScheme("dns") + + dnsResolverBuilder := resolver.Get("dns") + resolver.Register(mr) + + resolverBuilt := make(chan struct{}) + mr.BuildCallback = func(resolver.Target, resolver.ClientConn, resolver.BuildOptions) { + close(resolverBuilt) + } + + t.Cleanup(func() { resolver.Register(dnsResolverBuilder) }) + return mr, resolverBuilt +} + +func mustBuildResolver(ctx context.Context, t *testing.T, buildCh chan struct{}) { + t.Helper() + select { + case <-buildCh: + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for resolver to be built.") + } +} + +// proxyAddressWithTargetAttribute creates a resolver.Address for the proxy, +// adding the target address as an attribute. +func proxyAddressWithTargetAttribute(proxyAddr string, targetAddr string) resolver.Address { + addr := resolver.Address{Addr: proxyAddr} + addr = proxyattributes.Set(addr, proxyattributes.Options{ConnectAddr: targetAddr}) + return addr +} + +func overrideTestHTTPSProxy(t *testing.T, proxyAddr string) { + t.Helper() + hpfe := func(*http.Request) (*url.URL, error) { + return &url.URL{ + Scheme: "https", + Host: proxyAddr, + }, nil + } + originalhpfe := delegatingresolver.HTTPSProxyFromEnvironment + delegatingresolver.HTTPSProxyFromEnvironment = hpfe + t.Cleanup(func() { delegatingresolver.HTTPSProxyFromEnvironment = originalhpfe }) +} + +// Tests the scenario where proxy is configured and the target URI contains the +// "dns" scheme and target resolution is enabled. The test verifies that the +// addresses returned by the delegating resolver combines the addresses +// returned by the proxy resolver and the target resolver. +func (s) TestDelegatingResolverwithDNSAndProxyWithTargetResolution(t *testing.T) { + const ( + targetTestAddr = "test.com" + resolvedTargetTestAddr1 = "1.1.1.1:8080" + resolvedTargetTestAddr2 = "2.2.2.2:8080" + envProxyAddr = "proxytest.com" + resolvedProxyTestAddr1 = "11.11.11.11:7687" + ) + overrideTestHTTPSProxy(t, envProxyAddr) + + // Manual resolver to control the target resolution. + targetResolver := manual.NewBuilderWithScheme("dns") + target := targetResolver.Scheme() + ":///" + targetTestAddr + // Set up a manual DNS resolver to control the proxy address resolution. + proxyResolver, proxyResolverBuilt := setupDNS(t) + + tcc, stateCh, _ := createTestResolverClientConn(t) + if _, err := delegatingresolver.New(resolver.Target{URL: *testutils.MustParseURL(target)}, tcc, resolver.BuildOptions{}, targetResolver, true); err != nil { + t.Fatalf("Failed to create delegating resolver: %v", err) + } + + targetResolver.UpdateState(resolver.State{ + Addresses: []resolver.Address{ + {Addr: resolvedTargetTestAddr1}, + {Addr: resolvedTargetTestAddr2}, + }, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + + select { + case <-stateCh: + t.Fatalf("Delegating resolver invoked UpdateState before both the proxy and target resolvers had updated their states.") + case <-time.After(defaultTestShortTimeout): + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Wait for the proxy resolver to be built before calling UpdateState. + mustBuildResolver(ctx, t, proxyResolverBuilt) + proxyResolver.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: resolvedProxyTestAddr1}}, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + + // Verify that the delegating resolver outputs the expected address. + wantState := resolver.State{ + Addresses: []resolver.Address{ + proxyAddressWithTargetAttribute(resolvedProxyTestAddr1, resolvedTargetTestAddr1), + proxyAddressWithTargetAttribute(resolvedProxyTestAddr1, resolvedTargetTestAddr2), + }, + ServiceConfig: &serviceconfig.ParseResult{}, + } + var gotState resolver.State + select { + case gotState = <-stateCh: + case <-ctx.Done(): + t.Fatal("Context timeed out when waiting for a state update from the delegating resolver") + } + + if diff := cmp.Diff(gotState, wantState); diff != "" { + t.Fatalf("Unexpected state from delegating resolver. Diff (-got +want):\n%v", diff) + } +} + +// Tests the creation of a delegating resolver when a proxy is configured. It +// verifies successful creation for valid targets and ensures the final address +// is from the proxy resolver and contains the original, correctly-formatted +// target address as an attribute. +func (s) TestDelegatingResolverwithDNSAndProxyWithNoTargetResolutionHappyPaths(t *testing.T) { + const ( + envProxyAddr = "proxytest.com" + resolvedProxyTestAddr1 = "11.11.11.11:7687" + ) + tests := []struct { + name string + target string + wantConnectAddress string + defaultPortEnvVar bool + }{ + { + name: "no_port ", + target: "test.com", + wantConnectAddress: "test.com:443", + defaultPortEnvVar: true, + }, + { + name: "complete_target", + target: "test.com:8080", + wantConnectAddress: "test.com:8080", + defaultPortEnvVar: true, + }, + { + name: "no_port_with_default_port_env_variable_disabled", + target: "test.com", + wantConnectAddress: "test.com", + defaultPortEnvVar: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnableDefaultPortForProxyTarget, test.defaultPortEnvVar) + overrideTestHTTPSProxy(t, envProxyAddr) + + targetResolver := manual.NewBuilderWithScheme("dns") + target := targetResolver.Scheme() + ":///" + test.target + // Set up a manual DNS resolver to control the proxy address resolution. + proxyResolver, proxyResolverBuilt := setupDNS(t) + + tcc, stateCh, _ := createTestResolverClientConn(t) + if _, err := delegatingresolver.New(resolver.Target{URL: *testutils.MustParseURL(target)}, tcc, resolver.BuildOptions{}, targetResolver, false); err != nil { + t.Fatalf("Delegating resolver creation failed unexpectedly with error: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Wait for the proxy resolver to be built before calling UpdateState. + mustBuildResolver(ctx, t, proxyResolverBuilt) + proxyResolver.UpdateState(resolver.State{ + Addresses: []resolver.Address{ + {Addr: resolvedProxyTestAddr1}, + }, + }) + + wantState := resolver.State{ + Addresses: []resolver.Address{proxyAddressWithTargetAttribute(resolvedProxyTestAddr1, test.wantConnectAddress)}, + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{proxyAddressWithTargetAttribute(resolvedProxyTestAddr1, test.wantConnectAddress)}}}, + } + + var gotState resolver.State + select { + case gotState = <-stateCh: + case <-ctx.Done(): + t.Fatal("Context timed out when waiting for a state update from the delegating resolver") + } + + if diff := cmp.Diff(gotState, wantState); diff != "" { + t.Fatalf("Unexpected state from delegating resolver. Diff (-got +want):\n%v", diff) + } + }) + } +} + +// Tests the creation of a delegating resolver when a proxy is configured. It +// verifies correct error handling for invalid targets. +func (s) TestDelegatingResolverwithUnresolvedErrorTargetWithProxy(t *testing.T) { + const ( + envProxyAddr = "proxytest.com" + resolvedProxyTestAddr1 = "11.11.11.11:7687" + ) + tests := []struct { + name string + target string + wantErrorSubstring string + defaultPortEnvVar bool + }{ + + { + name: "colon_after_host_but_no_port", + target: "test.com:", + wantErrorSubstring: "missing port after port-separator colon", + defaultPortEnvVar: true, + }, + { + name: "empty_target", + target: "", + wantErrorSubstring: "missing address", + defaultPortEnvVar: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnableDefaultPortForProxyTarget, test.defaultPortEnvVar) + overrideTestHTTPSProxy(t, envProxyAddr) + + targetResolver := manual.NewBuilderWithScheme("dns") + target := targetResolver.Scheme() + ":///" + test.target + + tcc, _, _ := createTestResolverClientConn(t) + _, err := delegatingresolver.New(resolver.Target{URL: *testutils.MustParseURL(target)}, tcc, resolver.BuildOptions{}, targetResolver, false) + if err == nil { + t.Fatalf("Delegating resolver created, want error containing %q", test.wantErrorSubstring) + } + if !strings.Contains(err.Error(), test.wantErrorSubstring) { + t.Fatalf("Delegating resolver failed with error %v, want error containing %v", err.Error(), test.wantErrorSubstring) + } + }) + } +} + +// Tests the scenario where a proxy is configured, and the target URI scheme is +// not "dns". The test verifies that the addresses returned by the delegating +// resolver include the resolved proxy address and the custom resolved target +// address as attributes of the proxy address. +func (s) TestDelegatingResolverwithCustomResolverAndProxy(t *testing.T) { + const ( + targetTestAddr = "test.com" + resolvedTargetTestAddr1 = "1.1.1.1:8080" + resolvedTargetTestAddr2 = "2.2.2.2:8080" + envProxyAddr = "proxytest.com" + resolvedProxyTestAddr1 = "11.11.11.11:7687" + ) + overrideTestHTTPSProxy(t, envProxyAddr) + + // Manual resolver to control the target resolution. + targetResolver := manual.NewBuilderWithScheme("test") + target := targetResolver.Scheme() + ":///" + targetTestAddr + // Set up a manual DNS resolver to control the proxy address resolution. + proxyResolver, proxyResolverBuilt := setupDNS(t) + + tcc, stateCh, _ := createTestResolverClientConn(t) + if _, err := delegatingresolver.New(resolver.Target{URL: *testutils.MustParseURL(target)}, tcc, resolver.BuildOptions{}, targetResolver, false); err != nil { + t.Fatalf("Failed to create delegating resolver: %v", err) + } + + targetResolver.UpdateState(resolver.State{ + Addresses: []resolver.Address{ + {Addr: resolvedTargetTestAddr1}, + {Addr: resolvedTargetTestAddr2}, + }, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + + select { + case <-stateCh: + t.Fatalf("Delegating resolver invoked UpdateState before both the proxy and target resolvers had updated their states.") + case <-time.After(defaultTestShortTimeout): + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Wait for the proxy resolver to be built before calling UpdateState. + mustBuildResolver(ctx, t, proxyResolverBuilt) + proxyResolver.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: resolvedProxyTestAddr1}}, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + + wantState := resolver.State{ + Addresses: []resolver.Address{ + proxyAddressWithTargetAttribute(resolvedProxyTestAddr1, resolvedTargetTestAddr1), + proxyAddressWithTargetAttribute(resolvedProxyTestAddr1, resolvedTargetTestAddr2), + }, + ServiceConfig: &serviceconfig.ParseResult{}, + } + var gotState resolver.State + select { + case gotState = <-stateCh: + case <-ctx.Done(): + t.Fatal("Context timed out when waiting for a state update from the delegating resolver") + } + + if diff := cmp.Diff(gotState, wantState); diff != "" { + t.Fatalf("Unexpected state from delegating resolver. Diff (-got +want):\n%v", diff) + } +} + +// Tests the scenario where a proxy is configured, the target URI scheme is not +// "dns," and both the proxy and target resolvers return endpoints. The test +// verifies that the delegating resolver combines resolved proxy and target +// addresses correctly, returning endpoints with the proxy address populated +// and the target address included as an attribute of the proxy address for +// each combination of proxy and target endpoints. +func (s) TestDelegatingResolverForEndpointsWithProxy(t *testing.T) { + const ( + targetTestAddr = "test.com" + resolvedTargetTestAddr1 = "1.1.1.1:8080" + resolvedTargetTestAddr2 = "2.2.2.2:8080" + resolvedTargetTestAddr3 = "3.3.3.3:8080" + resolvedTargetTestAddr4 = "4.4.4.4:8080" + envProxyAddr = "proxytest.com" + resolvedProxyTestAddr1 = "11.11.11.11:7687" + resolvedProxyTestAddr2 = "22.22.22.22:7687" + ) + overrideTestHTTPSProxy(t, envProxyAddr) + + // Manual resolver to control the target resolution. + targetResolver := manual.NewBuilderWithScheme("test") + target := targetResolver.Scheme() + ":///" + targetTestAddr + // Set up a manual DNS resolver to control the proxy address resolution. + proxyResolver, proxyResolverBuilt := setupDNS(t) + + tcc, stateCh, _ := createTestResolverClientConn(t) + if _, err := delegatingresolver.New(resolver.Target{URL: *testutils.MustParseURL(target)}, tcc, resolver.BuildOptions{}, targetResolver, false); err != nil { + t.Fatalf("Failed to create delegating resolver: %v", err) + } + + targetResolver.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{ + { + Addresses: []resolver.Address{ + {Addr: resolvedTargetTestAddr1}, + {Addr: resolvedTargetTestAddr2}}, + }, + { + Addresses: []resolver.Address{ + {Addr: resolvedTargetTestAddr3}, + {Addr: resolvedTargetTestAddr4}}, + }, + }, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + select { + case <-stateCh: + t.Fatalf("Delegating resolver invoked UpdateState before both the proxy and target resolvers had updated their states.") + case <-time.After(defaultTestShortTimeout): + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Wait for the proxy resolver to be built before calling UpdateState. + mustBuildResolver(ctx, t, proxyResolverBuilt) + proxyResolver.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: resolvedProxyTestAddr1}}}, + {Addresses: []resolver.Address{{Addr: resolvedProxyTestAddr2}}}, + }, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + wantState := resolver.State{ + Endpoints: []resolver.Endpoint{ + { + Addresses: []resolver.Address{ + proxyAddressWithTargetAttribute(resolvedProxyTestAddr1, resolvedTargetTestAddr1), + proxyAddressWithTargetAttribute(resolvedProxyTestAddr2, resolvedTargetTestAddr1), + proxyAddressWithTargetAttribute(resolvedProxyTestAddr1, resolvedTargetTestAddr2), + proxyAddressWithTargetAttribute(resolvedProxyTestAddr2, resolvedTargetTestAddr2), + }, + }, + { + Addresses: []resolver.Address{ + proxyAddressWithTargetAttribute(resolvedProxyTestAddr1, resolvedTargetTestAddr3), + proxyAddressWithTargetAttribute(resolvedProxyTestAddr2, resolvedTargetTestAddr3), + proxyAddressWithTargetAttribute(resolvedProxyTestAddr1, resolvedTargetTestAddr4), + proxyAddressWithTargetAttribute(resolvedProxyTestAddr2, resolvedTargetTestAddr4), + }, + }, + }, + ServiceConfig: &serviceconfig.ParseResult{}, + } + var gotState resolver.State + select { + case gotState = <-stateCh: + case <-ctx.Done(): + t.Fatal("Contex timed out when waiting for a state update from the delegating resolver") + } + + if diff := cmp.Diff(gotState, wantState); diff != "" { + t.Fatalf("Unexpected state from delegating resolver. Diff (-got +want):\n%v", diff) + } +} + +// Tests the scenario where a proxy is configured, the target URI scheme is not +// "dns," and both the proxy and target resolvers return multiple addresses. +// The test verifies that the delegating resolver combines unresolved proxy +// host and target addresses correctly, returning addresses with the proxy host +// populated and the target address included as an attribute. +func (s) TestDelegatingResolverForMultipleProxyAddress(t *testing.T) { + const ( + targetTestAddr = "test.com" + resolvedTargetTestAddr1 = "1.1.1.1:8080" + resolvedTargetTestAddr2 = "2.2.2.2:8080" + envProxyAddr = "proxytest.com" + resolvedProxyTestAddr1 = "11.11.11.11:7687" + resolvedProxyTestAddr2 = "22.22.22.22:7687" + ) + overrideTestHTTPSProxy(t, envProxyAddr) + + // Manual resolver to control the target resolution. + targetResolver := manual.NewBuilderWithScheme("test") + target := targetResolver.Scheme() + ":///" + targetTestAddr + // Set up a manual DNS resolver to control the proxy address resolution. + proxyResolver, proxyResolverBuilt := setupDNS(t) + tcc, stateCh, _ := createTestResolverClientConn(t) + if _, err := delegatingresolver.New(resolver.Target{URL: *testutils.MustParseURL(target)}, tcc, resolver.BuildOptions{}, targetResolver, false); err != nil { + t.Fatalf("Failed to create delegating resolver: %v", err) + } + + targetResolver.UpdateState(resolver.State{ + Addresses: []resolver.Address{ + {Addr: resolvedTargetTestAddr1}, + {Addr: resolvedTargetTestAddr2}, + }, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + + select { + case <-stateCh: + t.Fatalf("Delegating resolver invoked UpdateState before both the proxy and target resolvers had updated their states.") + case <-time.After(defaultTestShortTimeout): + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Wait for the proxy resolver to be built before calling UpdateState. + mustBuildResolver(ctx, t, proxyResolverBuilt) + proxyResolver.UpdateState(resolver.State{ + Addresses: []resolver.Address{ + {Addr: resolvedProxyTestAddr1}, + {Addr: resolvedProxyTestAddr2}, + }, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + + wantState := resolver.State{ + Addresses: []resolver.Address{ + proxyAddressWithTargetAttribute(envProxyAddr, resolvedTargetTestAddr1), + proxyAddressWithTargetAttribute(envProxyAddr, resolvedTargetTestAddr2), + }, + ServiceConfig: &serviceconfig.ParseResult{}, + } + var gotState resolver.State + select { + case gotState = <-stateCh: + case <-ctx.Done(): + t.Fatal("Context timed out when waiting for a state update from the delegating resolver") + } + + if diff := cmp.Diff(gotState, wantState); diff != "" { + t.Fatalf("Unexpected state from delegating resolver. Diff (-got +want):\n%v", diff) + } +} + +// Tests that delegatingresolver doesn't panic when the channel closes the +// resolver while it's handling an update from it's child. The test closes the +// delegating resolver, verifies the target resolver is closed and blocks the +// proxy resolver from being closed. The test sends an update from the proxy +// resolver and verifies that the target resolver's ResolveNow method is not +// called after the channels returns an error. +func (s) TestDelegatingResolverUpdateStateDuringClose(t *testing.T) { + const envProxyAddr = "proxytest.com" + + overrideTestHTTPSProxy(t, envProxyAddr) + + // Manual resolver to control the target resolution. + targetResolver := manual.NewBuilderWithScheme("test") + targetResolverCalled := make(chan struct{}) + targetResolver.ResolveNowCallback = func(resolver.ResolveNowOptions) { + close(targetResolverCalled) + } + targetResolverCloseCalled := make(chan struct{}) + targetResolver.CloseCallback = func() { + close(targetResolverCloseCalled) + t.Log("Target resolver is closed.") + } + + target := targetResolver.Scheme() + ":///" + "ignored" + // Set up a manual DNS resolver to control the proxy address resolution. + proxyResolver, proxyResolverBuilt := setupDNS(t) + + unblockProxyResolverClose := make(chan struct{}, 1) + proxyResolver.CloseCallback = func() { + <-unblockProxyResolverClose + t.Log("Proxy resolver is closed.") + } + + tcc, _, _ := createTestResolverClientConn(t) + tcc.UpdateStateF = func(resolver.State) error { + return errors.New("test error") + } + dr, err := delegatingresolver.New(resolver.Target{URL: *testutils.MustParseURL(target)}, tcc, resolver.BuildOptions{}, targetResolver, false) + if err != nil { + t.Fatalf("Failed to create delegating resolver: %v", err) + } + + targetResolver.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "1.1.1.1"}}}}, + }) + + // Wait for the proxy resolver to be built before calling Close. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + mustBuildResolver(ctx, t, proxyResolverBuilt) + // Closing the delegating resolver will block until the test writes to the + // unblockProxyResolverClose channel. + go dr.Close() + select { + case <-targetResolverCloseCalled: + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for target resolver's Close method to be called.") + } + + // Updating the channel will result in an error being returned. Since the + // target resolver's Close method is already called, the delegating resolver + // must not call "ResolveNow" on it. + proxyUpdateCh := make(chan struct{}) + go func() { + proxyResolver.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "1.1.1.1"}}}}, + }) + close(proxyUpdateCh) + }() + unblockProxyResolverClose <- struct{}{} + + select { + case <-targetResolverCalled: + t.Fatalf("targetResolver.ResolveNow() called unexpectedly.") + case <-time.After(defaultTestShortTimeout): + } + // Wait for the proxy update to complete before returning from the test and + // before the deferred reassignment of + // delegatingresolver.HTTPSProxyFromEnvironment. This ensures that we read + // from the function before it is reassigned, preventing a race condition. + select { + case <-proxyUpdateCh: + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for proxyResolver.UpdateState() to be called.") + } +} + +// Tests that calling cc.UpdateState in a blocking manner from a child resolver +// while handling a ResolveNow call doesn't result in a deadlock. The test uses +// a fake ClientConn that returns an error when calling cc.UpdateState. The test +// makes the proxy resolver update the resolver state. The test verifies that +// the delegating resolver calls ResolveNow on the target resolver when the +// ClientConn returns an error. +func (s) TestDelegatingResolverUpdateStateFromResolveNow(t *testing.T) { + const envProxyAddr = "proxytest.com" + + overrideTestHTTPSProxy(t, envProxyAddr) + + // Manual resolver to control the target resolution. + targetResolver := manual.NewBuilderWithScheme("test") + targetResolverCalled := make(chan struct{}) + targetResolver.ResolveNowCallback = func(resolver.ResolveNowOptions) { + // Updating the resolver state should not deadlock. + targetResolver.CC().UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "1.1.1.1"}}}}, + }) + close(targetResolverCalled) + } + + target := targetResolver.Scheme() + ":///" + "ignored" + // Set up a manual DNS resolver to control the proxy address resolution. + proxyResolver, proxyResolverBuilt := setupDNS(t) + + tcc, _, _ := createTestResolverClientConn(t) + tcc.UpdateStateF = func(resolver.State) error { + return errors.New("test error") + } + _, err := delegatingresolver.New(resolver.Target{URL: *testutils.MustParseURL(target)}, tcc, resolver.BuildOptions{}, targetResolver, false) + if err != nil { + t.Fatalf("Failed to create delegating resolver: %v", err) + } + + targetResolver.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "1.1.1.1"}}}}, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Wait for the proxy resolver to be built before calling UpdateState. + mustBuildResolver(ctx, t, proxyResolverBuilt) + + // Updating the channel will result in an error being returned. The + // delegating resolver should call call "ResolveNow" on the target resolver. + proxyResolver.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "1.1.1.1"}}}}, + }) + + select { + case <-targetResolverCalled: + case <-ctx.Done(): + t.Fatalf("context timed out waiting for targetResolver.ResolveNow() to be called.") + } +} + +// Tests that calling cc.UpdateState in a blocking manner from child resolvers +// doesn't result in deadlocks. +func (s) TestDelegatingResolverResolveNow(t *testing.T) { + const envProxyAddr = "proxytest.com" + + overrideTestHTTPSProxy(t, envProxyAddr) + + // Manual resolver to control the target resolution. + targetResolver := manual.NewBuilderWithScheme("test") + targetResolverCalled := make(chan struct{}, 1) + targetResolver.ResolveNowCallback = func(resolver.ResolveNowOptions) { + // Updating the resolver state should not deadlock. + targetResolver.CC().UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "1.1.1.1"}}}}, + }) + targetResolverCalled <- struct{}{} + } + + target := targetResolver.Scheme() + ":///" + "ignored" + // Set up a manual DNS resolver to control the proxy address resolution. + proxyResolver, proxyResolverBuilt := setupDNS(t) + + proxyResolverCalled := make(chan struct{}) + proxyResolver.ResolveNowCallback = func(resolver.ResolveNowOptions) { + // Updating the resolver state should not deadlock. + proxyResolver.CC().UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "1.1.1.1"}}}}, + }) + close(proxyResolverCalled) + } + + tcc, _, _ := createTestResolverClientConn(t) + dr, err := delegatingresolver.New(resolver.Target{URL: *testutils.MustParseURL(target)}, tcc, resolver.BuildOptions{}, targetResolver, false) + if err != nil { + t.Fatalf("Failed to create delegating resolver: %v", err) + } + + // ResolveNow of manual proxy resolver will not be called. Proxy resolver is + // only built when we get the first update from target resolver. Therefore + // in the first ResolveNow, proxy resolver will be a no-op resolver and only + // target resolver's ResolveNow will be called. + dr.ResolveNow(resolver.ResolveNowOptions{}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-targetResolverCalled: + case <-ctx.Done(): + t.Fatalf("context timed out waiting for targetResolver.ResolveNow() to be called.") + } + + mustBuildResolver(ctx, t, proxyResolverBuilt) + + dr.ResolveNow(resolver.ResolveNowOptions{}) + + select { + case <-targetResolverCalled: + case <-ctx.Done(): + t.Fatalf("context timed out waiting for targetResolver.ResolveNow() to be called.") + } + select { + case <-proxyResolverCalled: + case <-ctx.Done(): + t.Fatalf("context timed out waiting for proxyResolver.ResolveNow() to be called.") + } +} + +// Tests the scenario where a proxy is configured, and the resolver returns a +// network type other than tcp for all addresses. The test verifies that the +// delegating resolver avoids the proxy build and directly sends the update +// from target resolver to clientconn. +func (s) TestDelegatingResolverForNonTCPTarget(t *testing.T) { + const ( + targetTestAddr = "test.target" + resolvedTargetTestAddr1 = "1.1.1.1:8080" + envProxyAddr = "proxytest.com" + ) + overrideTestHTTPSProxy(t, envProxyAddr) + + // Manual resolver to control the target resolution. + targetResolver := manual.NewBuilderWithScheme("test") + target := targetResolver.Scheme() + ":///" + targetTestAddr + // Set up a manual DNS resolver to control the proxy address resolution. + _, proxyResolverBuilt := setupDNS(t) + + tcc, stateCh, _ := createTestResolverClientConn(t) + if _, err := delegatingresolver.New(resolver.Target{URL: *testutils.MustParseURL(target)}, tcc, resolver.BuildOptions{}, targetResolver, false); err != nil { + t.Fatalf("Failed to create delegating resolver: %v", err) + } + + // Set network to anything other than tcp. + nonTCPAddr := networktype.Set(resolver.Address{Addr: resolvedTargetTestAddr1}, "unix") + targetResolver.UpdateState(resolver.State{ + Addresses: []resolver.Address{nonTCPAddr}, + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{nonTCPAddr}}}, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + + var gotState resolver.State + select { + case gotState = <-stateCh: + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout when waiting for a state update from the delegating resolver") + } + + // Verify that the delegating resolver doesn't call proxy resolver's + // UpdateState since we have no tcp address + select { + case <-proxyResolverBuilt: + t.Fatal("Unexpected call to proxy resolver update state") + case <-time.After(defaultTestShortTimeout): + } + + wantState := resolver.State{ + Addresses: []resolver.Address{nonTCPAddr}, + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{nonTCPAddr}}}, + ServiceConfig: &serviceconfig.ParseResult{}, + } + + // Verify that the state clientconn receives is same as updated by target + // resolver, since we want to avoid proxy for any network type apart from + // tcp. + if diff := cmp.Diff(gotState, wantState); diff != "" { + t.Fatalf("Unexpected state from delegating resolver. Diff (-got +want):\n%s", diff) + } +} + +// Tests the scenario where a proxy is configured, and the resolver returns tcp +// and non-tcp addresses. The test verifies that the delegating resolver doesn't +// add proxyatrribute to adresses with network type other than tcp, but adds +// the proxyattribute to addresses with network type tcp. +func (s) TestDelegatingResolverForMixNetworkType(t *testing.T) { + const ( + targetTestAddr = "test.target" + resolvedTargetTestAddr1 = "1.1.1.1:8080" + resolvedTargetTestAddr2 = "2.2.2.2:8080" + envProxyAddr = "proxytest.com" + ) + overrideTestHTTPSProxy(t, envProxyAddr) + + // Manual resolver to control the target resolution. + targetResolver := manual.NewBuilderWithScheme("test") + target := targetResolver.Scheme() + ":///" + targetTestAddr + // Set up a manual DNS resolver to control the proxy address resolution. + proxyResolver, proxyResolverBuilt := setupDNS(t) + + tcc, stateCh, _ := createTestResolverClientConn(t) + if _, err := delegatingresolver.New(resolver.Target{URL: *testutils.MustParseURL(target)}, tcc, resolver.BuildOptions{}, targetResolver, false); err != nil { + t.Fatalf("Failed to create delegating resolver: %v", err) + } + // Set network to anything other than tcp. + nonTCPAddr := networktype.Set(resolver.Address{Addr: resolvedTargetTestAddr1}, "unix") + targetResolver.UpdateState(resolver.State{ + Addresses: []resolver.Address{nonTCPAddr, {Addr: resolvedTargetTestAddr2}}, + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{nonTCPAddr, {Addr: resolvedTargetTestAddr2}}}}, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + + select { + case <-stateCh: + t.Fatalf("Delegating resolver invoked UpdateState before both the proxy and target resolvers had updated their states.") + case <-time.After(defaultTestShortTimeout): + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Wait for the proxy resolver to be built before calling UpdateState. + mustBuildResolver(ctx, t, proxyResolverBuilt) + proxyResolver.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: envProxyAddr}}, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + + var gotState resolver.State + select { + case gotState = <-stateCh: + case <-ctx.Done(): + t.Fatal("Context timed out when waiting for a state update from the delegating resolver") + } + wantState := resolver.State{ + Addresses: []resolver.Address{nonTCPAddr, proxyAddressWithTargetAttribute(envProxyAddr, resolvedTargetTestAddr2)}, + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{nonTCPAddr, proxyAddressWithTargetAttribute(envProxyAddr, resolvedTargetTestAddr2)}}}, + ServiceConfig: &serviceconfig.ParseResult{}, + } + + if diff := cmp.Diff(gotState, wantState); diff != "" { + t.Fatalf("Unexpected state from delegating resolver. Diff (-got +want):\n%v", diff) + } +} + +// Tests the scenario where a proxy is configured but some addresses are +// excluded (by using the NO_PROXY environment variable). The test verifies that +// the delegating resolver doesn't add proxyatrribute to adresses excluded, but +// adds the proxyattribute to all other addresses. +func (s) TestDelegatingResolverWithNoProxyEnvUsed(t *testing.T) { + const ( + targetTestAddr = "test.target" + noproxyresolvedTargetAddr = "1.1.1.1:8080" + resolvedTargetTestAddr = "2.2.2.2:8080" + envProxyAddr = "proxytest.com" + ) + hpfe := func(req *http.Request) (*url.URL, error) { + // return nil to mimick the scenario where the address is excluded using + // `NO_PROXY` env variable. + if req.URL.Host == noproxyresolvedTargetAddr { + return nil, nil + } + return &url.URL{ + Scheme: "https", + Host: envProxyAddr, + }, nil + } + originalhpfe := delegatingresolver.HTTPSProxyFromEnvironment + delegatingresolver.HTTPSProxyFromEnvironment = hpfe + defer func() { + delegatingresolver.HTTPSProxyFromEnvironment = originalhpfe + }() + + // Manual resolver to control the target resolution. + targetResolver := manual.NewBuilderWithScheme("test") + target := targetResolver.Scheme() + ":///" + targetTestAddr + // Set up a manual DNS resolver to control the proxy address resolution. + proxyResolver, proxyResolverBuilt := setupDNS(t) + + tcc, stateCh, _ := createTestResolverClientConn(t) + if _, err := delegatingresolver.New(resolver.Target{URL: *testutils.MustParseURL(target)}, tcc, resolver.BuildOptions{}, targetResolver, false); err != nil { + t.Fatalf("Failed to create delegating resolver: %v", err) + } + + targetResolver.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: noproxyresolvedTargetAddr}, {Addr: resolvedTargetTestAddr}}, + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: noproxyresolvedTargetAddr}, {Addr: resolvedTargetTestAddr}}}}, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + + select { + case <-stateCh: + t.Fatalf("Delegating resolver invoked UpdateState before both the proxy and target resolvers had updated their states.") + case <-time.After(defaultTestShortTimeout): + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Wait for the proxy resolver to be built before calling UpdateState. + mustBuildResolver(ctx, t, proxyResolverBuilt) + proxyResolver.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: envProxyAddr}}, + ServiceConfig: &serviceconfig.ParseResult{}, + }) + + var gotState resolver.State + select { + case gotState = <-stateCh: + case <-ctx.Done(): + t.Fatal("Context timed out when waiting for a state update from the delegating resolver") + } + wantState := resolver.State{ + Addresses: []resolver.Address{{Addr: noproxyresolvedTargetAddr}, proxyAddressWithTargetAttribute(envProxyAddr, resolvedTargetTestAddr)}, + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: noproxyresolvedTargetAddr}, proxyAddressWithTargetAttribute(envProxyAddr, resolvedTargetTestAddr)}}}, + ServiceConfig: &serviceconfig.ParseResult{}, + } + + if diff := cmp.Diff(gotState, wantState); diff != "" { + t.Fatalf("Unexpected state from delegating resolver. Diff (-got +want):\n%v", diff) + } +} diff --git a/internal/resolver/delegatingresolver/delegatingresolver_test.go b/internal/resolver/delegatingresolver/delegatingresolver_test.go new file mode 100644 index 0000000..154d719 --- /dev/null +++ b/internal/resolver/delegatingresolver/delegatingresolver_test.go @@ -0,0 +1,107 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package delegatingresolver + +import ( + "errors" + "net/http" + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const targetTestAddr = "test.com" + +// overrideHTTPSProxyFromEnvironment function overwrites HTTPSProxyFromEnvironment and +// returns a function to restore the default values. +func overrideHTTPSProxyFromEnvironment(hpfe func(req *http.Request) (*url.URL, error)) func() { + HTTPSProxyFromEnvironment = hpfe + return func() { + HTTPSProxyFromEnvironment = nil + } +} + +// Tests that the proxyURLForTarget function correctly resolves the proxy URL +// for a given target address. Tests all the possible output cases. +func (s) TestproxyURLForTargetEnv(t *testing.T) { + err := errors.New("invalid proxy url") + tests := []struct { + name string + hpfeFunc func(req *http.Request) (*url.URL, error) + wantURL *url.URL + wantErr error + }{ + { + name: "valid_proxy_url_and_nil_error", + hpfeFunc: func(_ *http.Request) (*url.URL, error) { + return &url.URL{ + Scheme: "https", + Host: "proxy.example.com", + }, nil + }, + wantURL: &url.URL{ + Scheme: "https", + Host: "proxy.example.com", + }, + }, + { + name: "invalid_proxy_url_and_non-nil_error", + hpfeFunc: func(_ *http.Request) (*url.URL, error) { + return &url.URL{ + Scheme: "https", + Host: "notproxy.example.com", + }, err + }, + wantURL: &url.URL{ + Scheme: "https", + Host: "notproxy.example.com", + }, + wantErr: err, + }, + { + name: "nil_proxy_url_and_nil_error", + hpfeFunc: func(_ *http.Request) (*url.URL, error) { + return nil, nil + }, + wantURL: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer overrideHTTPSProxyFromEnvironment(tt.hpfeFunc)() + got, err := proxyURLForTarget(targetTestAddr) + if err != tt.wantErr { + t.Errorf("parsedProxyURLForProxy(%v) failed with error :%v, want %v\n", targetTestAddr, err, tt.wantErr) + } + if !cmp.Equal(got, tt.wantURL) { + t.Fatalf("parsedProxyURLForProxy(%v) = %v, want %v\n", targetTestAddr, got, tt.wantURL) + } + }) + } +} diff --git a/internal/resolver/dns/dns_resolver.go b/internal/resolver/dns/dns_resolver.go new file mode 100644 index 0000000..70b89e4 --- /dev/null +++ b/internal/resolver/dns/dns_resolver.go @@ -0,0 +1,472 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package dns implements a dns resolver to be installed as the default resolver +// in grpc. +package dns + +import ( + "context" + "encoding/json" + "fmt" + rand "math/rand/v2" + "net" + "net/netip" + "os" + "strconv" + "strings" + "sync" + "time" + + grpclbstate "google.golang.org/grpc/balancer/grpclb/state" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/resolver/dns/internal" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +var ( + // EnableSRVLookups controls whether the DNS resolver attempts to fetch gRPCLB + // addresses from SRV records. Must not be changed after init time. + EnableSRVLookups = false + + // MinResolutionInterval is the minimum interval at which re-resolutions are + // allowed. This helps to prevent excessive re-resolution. + MinResolutionInterval = 30 * time.Second + + // ResolvingTimeout specifies the maximum duration for a DNS resolution request. + // If the timeout expires before a response is received, the request will be canceled. + // + // It is recommended to set this value at application startup. Avoid modifying this variable + // after initialization as it's not thread-safe for concurrent modification. + ResolvingTimeout = 30 * time.Second + + logger = grpclog.Component("dns") +) + +func init() { + resolver.Register(NewBuilder()) + internal.TimeAfterFunc = time.After + internal.TimeNowFunc = time.Now + internal.TimeUntilFunc = time.Until + internal.NewNetResolver = newNetResolver + internal.AddressDialer = addressDialer +} + +const ( + defaultPort = "443" + defaultDNSSvrPort = "53" + golang = "GO" + // txtPrefix is the prefix string to be prepended to the host name for txt + // record lookup. + txtPrefix = "_grpc_config." + // In DNS, service config is encoded in a TXT record via the mechanism + // described in RFC-1464 using the attribute name grpc_config. + txtAttribute = "grpc_config=" +) + +var addressDialer = func(address string) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, _ string) (net.Conn, error) { + var dialer net.Dialer + return dialer.DialContext(ctx, network, address) + } +} + +var newNetResolver = func(authority string) (internal.NetResolver, error) { + if authority == "" { + return net.DefaultResolver, nil + } + + host, port, err := parseTarget(authority, defaultDNSSvrPort) + if err != nil { + return nil, err + } + + authorityWithPort := net.JoinHostPort(host, port) + + return &net.Resolver{ + PreferGo: true, + Dial: internal.AddressDialer(authorityWithPort), + }, nil +} + +// NewBuilder creates a dnsBuilder which is used to factory DNS resolvers. +func NewBuilder() resolver.Builder { + return &dnsBuilder{} +} + +type dnsBuilder struct{} + +// Build creates and starts a DNS resolver that watches the name resolution of +// the target. +func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { + host, port, err := parseTarget(target.Endpoint(), defaultPort) + if err != nil { + return nil, err + } + + // IP address. + if ipAddr, err := formatIP(host); err == nil { + addr := []resolver.Address{{Addr: ipAddr + ":" + port}} + cc.UpdateState(resolver.State{ + Addresses: addr, + Endpoints: []resolver.Endpoint{{Addresses: addr}}, + }) + return deadResolver{}, nil + } + + // DNS address (non-IP). + ctx, cancel := context.WithCancel(context.Background()) + d := &dnsResolver{ + host: host, + port: port, + ctx: ctx, + cancel: cancel, + cc: cc, + rn: make(chan struct{}, 1), + enableServiceConfig: envconfig.EnableTXTServiceConfig && !opts.DisableServiceConfig, + } + + d.resolver, err = internal.NewNetResolver(target.URL.Host) + if err != nil { + return nil, err + } + + d.wg.Add(1) + go d.watcher() + return d, nil +} + +// Scheme returns the naming scheme of this resolver builder, which is "dns". +func (b *dnsBuilder) Scheme() string { + return "dns" +} + +// deadResolver is a resolver that does nothing. +type deadResolver struct{} + +func (deadResolver) ResolveNow(resolver.ResolveNowOptions) {} + +func (deadResolver) Close() {} + +// dnsResolver watches for the name resolution update for a non-IP target. +type dnsResolver struct { + host string + port string + resolver internal.NetResolver + ctx context.Context + cancel context.CancelFunc + cc resolver.ClientConn + // rn channel is used by ResolveNow() to force an immediate resolution of the + // target. + rn chan struct{} + // wg is used to enforce Close() to return after the watcher() goroutine has + // finished. Otherwise, data race will be possible. [Race Example] in + // dns_resolver_test we replace the real lookup functions with mocked ones to + // facilitate testing. If Close() doesn't wait for watcher() goroutine + // finishes, race detector sometimes will warn lookup (READ the lookup + // function pointers) inside watcher() goroutine has data race with + // replaceNetFunc (WRITE the lookup function pointers). + wg sync.WaitGroup + enableServiceConfig bool +} + +// ResolveNow invoke an immediate resolution of the target that this +// dnsResolver watches. +func (d *dnsResolver) ResolveNow(resolver.ResolveNowOptions) { + select { + case d.rn <- struct{}{}: + default: + } +} + +// Close closes the dnsResolver. +func (d *dnsResolver) Close() { + d.cancel() + d.wg.Wait() +} + +func (d *dnsResolver) watcher() { + defer d.wg.Done() + backoffIndex := 1 + for { + state, err := d.lookup() + if err != nil { + // Report error to the underlying grpc.ClientConn. + d.cc.ReportError(err) + } else { + err = d.cc.UpdateState(*state) + } + + var nextResolutionTime time.Time + if err == nil { + // Success resolving, wait for the next ResolveNow. However, also wait 30 + // seconds at the very least to prevent constantly re-resolving. + backoffIndex = 1 + nextResolutionTime = internal.TimeNowFunc().Add(MinResolutionInterval) + select { + case <-d.ctx.Done(): + return + case <-d.rn: + } + } else { + // Poll on an error found in DNS Resolver or an error received from + // ClientConn. + nextResolutionTime = internal.TimeNowFunc().Add(backoff.DefaultExponential.Backoff(backoffIndex)) + backoffIndex++ + } + select { + case <-d.ctx.Done(): + return + case <-internal.TimeAfterFunc(internal.TimeUntilFunc(nextResolutionTime)): + } + } +} + +func (d *dnsResolver) lookupSRV(ctx context.Context) ([]resolver.Address, error) { + // Skip this particular host to avoid timeouts with some versions of + // systemd-resolved. + if !EnableSRVLookups || d.host == "metadata.google.internal." { + return nil, nil + } + var newAddrs []resolver.Address + _, srvs, err := d.resolver.LookupSRV(ctx, "grpclb", "tcp", d.host) + if err != nil { + err = handleDNSError(err, "SRV") // may become nil + return nil, err + } + for _, s := range srvs { + lbAddrs, err := d.resolver.LookupHost(ctx, s.Target) + if err != nil { + err = handleDNSError(err, "A") // may become nil + if err == nil { + // If there are other SRV records, look them up and ignore this + // one that does not exist. + continue + } + return nil, err + } + for _, a := range lbAddrs { + ip, err := formatIP(a) + if err != nil { + return nil, fmt.Errorf("dns: error parsing A record IP address %v: %v", a, err) + } + addr := ip + ":" + strconv.Itoa(int(s.Port)) + newAddrs = append(newAddrs, resolver.Address{Addr: addr, ServerName: s.Target}) + } + } + return newAddrs, nil +} + +func handleDNSError(err error, lookupType string) error { + dnsErr, ok := err.(*net.DNSError) + if ok && !dnsErr.IsTimeout && !dnsErr.IsTemporary { + // Timeouts and temporary errors should be communicated to gRPC to + // attempt another DNS query (with backoff). Other errors should be + // suppressed (they may represent the absence of a TXT record). + return nil + } + if err != nil { + err = fmt.Errorf("dns: %v record lookup error: %v", lookupType, err) + logger.Info(err) + } + return err +} + +func (d *dnsResolver) lookupTXT(ctx context.Context) *serviceconfig.ParseResult { + ss, err := d.resolver.LookupTXT(ctx, txtPrefix+d.host) + if err != nil { + if envconfig.TXTErrIgnore { + return nil + } + if err = handleDNSError(err, "TXT"); err != nil { + return &serviceconfig.ParseResult{Err: err} + } + return nil + } + var res string + for _, s := range ss { + res += s + } + + // TXT record must have "grpc_config=" attribute in order to be used as + // service config. + if !strings.HasPrefix(res, txtAttribute) { + logger.Warningf("dns: TXT record %v missing %v attribute", res, txtAttribute) + // This is not an error; it is the equivalent of not having a service + // config. + return nil + } + sc := canaryingSC(strings.TrimPrefix(res, txtAttribute)) + return d.cc.ParseServiceConfig(sc) +} + +func (d *dnsResolver) lookupHost(ctx context.Context) ([]resolver.Address, error) { + addrs, err := d.resolver.LookupHost(ctx, d.host) + if err != nil { + err = handleDNSError(err, "A") + return nil, err + } + newAddrs := make([]resolver.Address, 0, len(addrs)) + for _, a := range addrs { + ip, err := formatIP(a) + if err != nil { + return nil, fmt.Errorf("dns: error parsing A record IP address %v: %v", a, err) + } + addr := ip + ":" + d.port + newAddrs = append(newAddrs, resolver.Address{Addr: addr}) + } + return newAddrs, nil +} + +func (d *dnsResolver) lookup() (*resolver.State, error) { + ctx, cancel := context.WithTimeout(d.ctx, ResolvingTimeout) + defer cancel() + srv, srvErr := d.lookupSRV(ctx) + addrs, hostErr := d.lookupHost(ctx) + if hostErr != nil && (srvErr != nil || len(srv) == 0) { + return nil, hostErr + } + + eps := make([]resolver.Endpoint, 0, len(addrs)) + for _, addr := range addrs { + eps = append(eps, resolver.Endpoint{Addresses: []resolver.Address{addr}}) + } + + state := resolver.State{ + Addresses: addrs, + Endpoints: eps, + } + if len(srv) > 0 { + state = grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: srv}) + } + if d.enableServiceConfig { + state.ServiceConfig = d.lookupTXT(ctx) + } + return &state, nil +} + +// formatIP returns an error if addr is not a valid textual representation of +// an IP address. If addr is an IPv4 address, return the addr and error = nil. +// If addr is an IPv6 address, return the addr enclosed in square brackets and +// error = nil. +func formatIP(addr string) (string, error) { + ip, err := netip.ParseAddr(addr) + if err != nil { + return "", err + } + if ip.Is4() { + return addr, nil + } + return "[" + addr + "]", nil +} + +// parseTarget takes the user input target string and default port, returns +// formatted host and port info. If target doesn't specify a port, set the port +// to be the defaultPort. If target is in IPv6 format and host-name is enclosed +// in square brackets, brackets are stripped when setting the host. +// examples: +// target: "www.google.com" defaultPort: "443" returns host: "www.google.com", port: "443" +// target: "ipv4-host:80" defaultPort: "443" returns host: "ipv4-host", port: "80" +// target: "[ipv6-host]" defaultPort: "443" returns host: "ipv6-host", port: "443" +// target: ":80" defaultPort: "443" returns host: "localhost", port: "80" +func parseTarget(target, defaultPort string) (host, port string, err error) { + if target == "" { + return "", "", internal.ErrMissingAddr + } + if _, err := netip.ParseAddr(target); err == nil { + // target is an IPv4 or IPv6(without brackets) address + return target, defaultPort, nil + } + if host, port, err = net.SplitHostPort(target); err == nil { + if port == "" { + // If the port field is empty (target ends with colon), e.g. "[::1]:", + // this is an error. + return "", "", internal.ErrEndsWithColon + } + // target has port, i.e ipv4-host:port, [ipv6-host]:port, host-name:port + if host == "" { + // Keep consistent with net.Dial(): If the host is empty, as in ":80", + // the local system is assumed. + host = "localhost" + } + return host, port, nil + } + if host, port, err = net.SplitHostPort(target + ":" + defaultPort); err == nil { + // target doesn't have port + return host, port, nil + } + return "", "", fmt.Errorf("invalid target address %v, error info: %v", target, err) +} + +type rawChoice struct { + ClientLanguage *[]string `json:"clientLanguage,omitempty"` + Percentage *int `json:"percentage,omitempty"` + ClientHostName *[]string `json:"clientHostName,omitempty"` + ServiceConfig *json.RawMessage `json:"serviceConfig,omitempty"` +} + +func containsString(a *[]string, b string) bool { + if a == nil { + return true + } + for _, c := range *a { + if c == b { + return true + } + } + return false +} + +func chosenByPercentage(a *int) bool { + if a == nil { + return true + } + return rand.IntN(100)+1 <= *a +} + +func canaryingSC(js string) string { + if js == "" { + return "" + } + var rcs []rawChoice + err := json.Unmarshal([]byte(js), &rcs) + if err != nil { + logger.Warningf("dns: error parsing service config json: %v", err) + return "" + } + cliHostname, err := os.Hostname() + if err != nil { + logger.Warningf("dns: error getting client hostname: %v", err) + return "" + } + var sc string + for _, c := range rcs { + if !containsString(c.ClientLanguage, golang) || + !chosenByPercentage(c.Percentage) || + !containsString(c.ClientHostName, cliHostname) || + c.ServiceConfig == nil { + continue + } + sc = string(*c.ServiceConfig) + break + } + return sc +} diff --git a/internal/resolver/dns/dns_resolver_test.go b/internal/resolver/dns/dns_resolver_test.go new file mode 100644 index 0000000..ce04cda --- /dev/null +++ b/internal/resolver/dns/dns_resolver_test.go @@ -0,0 +1,1511 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package dns_test + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + grpclbstate "google.golang.org/grpc/balancer/grpclb/state" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/resolver/dns" + dnsinternal "google.golang.org/grpc/internal/resolver/dns/internal" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + dnspublic "google.golang.org/grpc/resolver/dns" + "google.golang.org/grpc/serviceconfig" + + _ "google.golang.org/grpc" // To initialize internal.ParseServiceConfig +) + +const ( + txtBytesLimit = 255 + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + + colonDefaultPort = ":443" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// Override the default net.Resolver with a test resolver. +func overrideNetResolver(t *testing.T, r *testNetResolver) { + origNetResolver := dnsinternal.NewNetResolver + dnsinternal.NewNetResolver = func(string) (dnsinternal.NetResolver, error) { return r, nil } + t.Cleanup(func() { dnsinternal.NewNetResolver = origNetResolver }) +} + +// Override the DNS minimum resolution interval used by the resolver. +func overrideResolutionInterval(t *testing.T, d time.Duration) { + origMinResInterval := dns.MinResolutionInterval + dnspublic.SetMinResolutionInterval(d) + t.Cleanup(func() { dnspublic.SetMinResolutionInterval(origMinResInterval) }) +} + +// Override the timer used by the DNS resolver to fire after a duration of d. +func overrideTimeAfterFunc(t *testing.T, d time.Duration) { + origTimeAfter := dnsinternal.TimeAfterFunc + dnsinternal.TimeAfterFunc = func(time.Duration) <-chan time.Time { + return time.After(d) + } + t.Cleanup(func() { dnsinternal.TimeAfterFunc = origTimeAfter }) +} + +// Override the timer used by the DNS resolver as follows: +// - use the durChan to read the duration that the resolver wants to wait for +// - use the timerChan to unblock the wait on the timer +func overrideTimeAfterFuncWithChannel(t *testing.T) (durChan chan time.Duration, timeChan chan time.Time) { + origTimeAfter := dnsinternal.TimeAfterFunc + durChan = make(chan time.Duration, 1) + timeChan = make(chan time.Time) + dnsinternal.TimeAfterFunc = func(d time.Duration) <-chan time.Time { + select { + case durChan <- d: + default: + } + return timeChan + } + t.Cleanup(func() { dnsinternal.TimeAfterFunc = origTimeAfter }) + return durChan, timeChan +} + +// Override the current time used by the DNS resolver. +func overrideTimeNowFunc(t *testing.T, now time.Time) { + origTimeNowFunc := dnsinternal.TimeNowFunc + dnsinternal.TimeNowFunc = func() time.Time { return now } + t.Cleanup(func() { dnsinternal.TimeNowFunc = origTimeNowFunc }) +} + +// Override the remaining wait time to allow re-resolution by DNS resolver. +// Use the timeChan to read the time until resolver needs to wait for +// and return 0 wait time. +func overrideTimeUntilFuncWithChannel(t *testing.T) (timeChan chan time.Time) { + timeCh := make(chan time.Time, 1) + origTimeUntil := dnsinternal.TimeUntilFunc + dnsinternal.TimeUntilFunc = func(t time.Time) time.Duration { + timeCh <- t + return 0 + } + t.Cleanup(func() { dnsinternal.TimeUntilFunc = origTimeUntil }) + return timeCh +} + +func enableSRVLookups(t *testing.T) { + origEnableSRVLookups := dns.EnableSRVLookups + dns.EnableSRVLookups = true + t.Cleanup(func() { dns.EnableSRVLookups = origEnableSRVLookups }) +} + +// Builds a DNS resolver for target and returns a couple of channels to read the +// state and error pushed by the resolver respectively. +func buildResolverWithTestClientConn(t *testing.T, target string) (resolver.Resolver, chan resolver.State, chan error) { + t.Helper() + + b := resolver.Get("dns") + if b == nil { + t.Fatalf("Resolver for dns:/// scheme not registered") + } + + stateCh := make(chan resolver.State, 1) + updateStateF := func(s resolver.State) error { + select { + case stateCh <- s: + default: + } + return nil + } + + errCh := make(chan error, 1) + reportErrorF := func(err error) { + select { + case errCh <- err: + default: + } + } + + tcc := &testutils.ResolverClientConn{Logger: t, UpdateStateF: updateStateF, ReportErrorF: reportErrorF} + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("dns:///%s", target))}, tcc, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("Failed to build DNS resolver for target %q: %v\n", target, err) + } + t.Cleanup(func() { r.Close() }) + + return r, stateCh, errCh +} + +// resolverUpdate holds required components of a resolver state for validation. +type resolverUpdate struct { + addrs []resolver.Address + endpoints []resolver.Endpoint + balancerAddrs []resolver.Address // list of grpclb addresses + serviceConfig serviceconfig.Config +} + +// Waits for a state update from the DNS resolver. +func verifyUpdateFromResolver(ctx context.Context, t *testing.T, stateCh chan resolver.State, want resolverUpdate) { + t.Helper() + + var state resolver.State + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for a state update from the resolver") + case state = <-stateCh: + } + + wantResolverState := resolver.State{ + Addresses: want.addrs, + Endpoints: want.endpoints, + ServiceConfig: &serviceconfig.ParseResult{ + Config: want.serviceConfig, + }, + } + if len(want.balancerAddrs) > 0 { + wantResolverState = grpclbstate.Set(wantResolverState, &grpclbstate.State{ + BalancerAddresses: want.balancerAddrs, + }) + } + + scComparer := cmp.Comparer(func(a, b *serviceconfig.ParseResult) bool { + var scA serviceconfig.Config + var scB serviceconfig.Config + if a != nil { + scA = a.Config + } + if b != nil { + scB = b.Config + } + return internal.EqualServiceConfigForTesting(scA, scB) + }) + attrComparer := cmp.Comparer(func(a, b *attributes.Attributes) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + gA := grpclbstate.Get(resolver.State{Attributes: a}) + gB := grpclbstate.Get(resolver.State{Attributes: b}) + return cmp.Equal(gA.BalancerAddresses, gB.BalancerAddresses) + }) + if diff := cmp.Diff(wantResolverState, state, cmpopts.EquateEmpty(), scComparer, attrComparer); diff != "" { + t.Fatalf("Got unexpected resolver.State (-want +got):\n%s", diff) + } +} + +// This is the service config used by the fake net.Resolver in its TXT record. +// - it contains an array of 5 entries +// - the first three will be dropped by the DNS resolver as part of its +// canarying rule matching functionality: +// - the client language does not match in the first entry +// - the percentage is set to 0 in the second entry +// - the client host name does not match in the third entry +// - the fourth and fifth entries will match the canarying rules, and therefore +// the fourth entry will be used as it will be the first matching entry. +const txtRecordGood = ` +[ + { + "clientLanguage": [ + "CPP", + "JAVA" + ], + "serviceConfig": { + "loadBalancingPolicy": "grpclb", + "methodConfig": [ + { + "name": [ + { + "service": "all" + } + ], + "timeout": "1s" + } + ] + } + }, + { + "percentage": 0, + "serviceConfig": { + "loadBalancingPolicy": "grpclb", + "methodConfig": [ + { + "name": [ + { + "service": "all" + } + ], + "timeout": "1s" + } + ] + } + }, + { + "clientHostName": [ + "localhost" + ], + "serviceConfig": { + "loadBalancingPolicy": "grpclb", + "methodConfig": [ + { + "name": [ + { + "service": "all" + } + ], + "timeout": "1s" + } + ] + } + }, + { + "clientLanguage": [ + "GO" + ], + "percentage": 100, + "serviceConfig": { + "loadBalancingPolicy": "round_robin", + "methodConfig": [ + { + "name": [ + { + "service": "foo" + } + ], + "waitForReady": true, + "timeout": "1s" + }, + { + "name": [ + { + "service": "bar" + } + ], + "waitForReady": false + } + ] + } + }, + { + "serviceConfig": { + "loadBalancingPolicy": "round_robin", + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "bar" + } + ], + "waitForReady": true + } + ] + } + } +]` + +// This is the matched portion of the above TXT record entry. +const scJSON = ` +{ + "loadBalancingPolicy": "round_robin", + "methodConfig": [ + { + "name": [ + { + "service": "foo" + } + ], + "waitForReady": true, + "timeout": "1s" + }, + { + "name": [ + { + "service": "bar" + } + ], + "waitForReady": false + } + ] +}` + +// This service config contains three entries, but none of the match the DNS +// resolver's canarying rules and hence the resulting service config pushed by +// the DNS resolver will be an empty one. +const txtRecordNonMatching = ` +[ + { + "clientLanguage": [ + "CPP", + "JAVA" + ], + "serviceConfig": { + "loadBalancingPolicy": "grpclb", + "methodConfig": [ + { + "name": [ + { + "service": "all" + } + ], + "timeout": "1s" + } + ] + } + }, + { + "percentage": 0, + "serviceConfig": { + "loadBalancingPolicy": "grpclb", + "methodConfig": [ + { + "name": [ + { + "service": "all" + } + ], + "timeout": "1s" + } + ] + } + }, + { + "clientHostName": [ + "localhost" + ], + "serviceConfig": { + "loadBalancingPolicy": "grpclb", + "methodConfig": [ + { + "name": [ + { + "service": "all" + } + ], + "timeout": "1s" + } + ] + } + } +]` + +// Tests the scenario where a name resolves to a list of addresses, possibly +// some grpclb addresses as well, and a service config. The test verifies that +// the expected update is pushed to the channel. +func (s) TestDNSResolver_Basic(t *testing.T) { + tests := []struct { + name string + target string + hostLookupTable map[string][]string + srvLookupTable map[string][]*net.SRV + txtLookupTable map[string][]string + want resolverUpdate + }{ + { + name: "default_port", + target: "foo.bar.com", + hostLookupTable: map[string][]string{ + "foo.bar.com": {"1.2.3.4", "5.6.7.8"}, + }, + txtLookupTable: map[string][]string{ + "_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood), + }, + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}}, + endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}}}, + {Addresses: []resolver.Address{{Addr: "5.6.7.8" + colonDefaultPort}}}, + }, + serviceConfig: mustParseServiceConfig(t, scJSON), + }, + }, + { + name: "specified_port", + target: "foo.bar.com:1234", + hostLookupTable: map[string][]string{ + "foo.bar.com": {"1.2.3.4", "5.6.7.8"}, + }, + txtLookupTable: map[string][]string{ + "_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood), + }, + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "1.2.3.4:1234"}, {Addr: "5.6.7.8:1234"}}, + endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.2.3.4:1234"}}}, + {Addresses: []resolver.Address{{Addr: "5.6.7.8:1234"}}}, + }, + serviceConfig: mustParseServiceConfig(t, scJSON), + }, + }, + { + name: "ipv4_with_SRV_and_single_grpclb_address", + target: "srv.ipv4.single.fake", + hostLookupTable: map[string][]string{ + "srv.ipv4.single.fake": {"2.4.6.8"}, + "ipv4.single.fake": {"1.2.3.4"}, + }, + srvLookupTable: map[string][]*net.SRV{ + "_grpclb._tcp.srv.ipv4.single.fake": {&net.SRV{Target: "ipv4.single.fake", Port: 1234}}, + }, + txtLookupTable: map[string][]string{ + "_grpc_config.srv.ipv4.single.fake": txtRecordServiceConfig(txtRecordGood), + }, + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "2.4.6.8" + colonDefaultPort}}, + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "2.4.6.8" + colonDefaultPort}}}}, + balancerAddrs: []resolver.Address{{Addr: "1.2.3.4:1234", ServerName: "ipv4.single.fake"}}, + serviceConfig: mustParseServiceConfig(t, scJSON), + }, + }, + { + name: "ipv4_with_SRV_and_multiple_grpclb_address", + target: "srv.ipv4.multi.fake", + hostLookupTable: map[string][]string{ + "ipv4.multi.fake": {"1.2.3.4", "5.6.7.8", "9.10.11.12"}, + }, + srvLookupTable: map[string][]*net.SRV{ + "_grpclb._tcp.srv.ipv4.multi.fake": {&net.SRV{Target: "ipv4.multi.fake", Port: 1234}}, + }, + txtLookupTable: map[string][]string{ + "_grpc_config.srv.ipv4.multi.fake": txtRecordServiceConfig(txtRecordGood), + }, + want: resolverUpdate{ + balancerAddrs: []resolver.Address{ + {Addr: "1.2.3.4:1234", ServerName: "ipv4.multi.fake"}, + {Addr: "5.6.7.8:1234", ServerName: "ipv4.multi.fake"}, + {Addr: "9.10.11.12:1234", ServerName: "ipv4.multi.fake"}, + }, + serviceConfig: mustParseServiceConfig(t, scJSON), + }, + }, + { + name: "ipv6_with_SRV_and_single_grpclb_address", + target: "srv.ipv6.single.fake", + hostLookupTable: map[string][]string{ + "srv.ipv6.single.fake": nil, + "ipv6.single.fake": {"2607:f8b0:400a:801::1001"}, + }, + srvLookupTable: map[string][]*net.SRV{ + "_grpclb._tcp.srv.ipv6.single.fake": {&net.SRV{Target: "ipv6.single.fake", Port: 1234}}, + }, + txtLookupTable: map[string][]string{ + "_grpc_config.srv.ipv6.single.fake": txtRecordServiceConfig(txtRecordNonMatching), + }, + want: resolverUpdate{ + balancerAddrs: []resolver.Address{{Addr: "[2607:f8b0:400a:801::1001]:1234", ServerName: "ipv6.single.fake"}}, + }, + }, + { + name: "ipv6_with_SRV_and_multiple_grpclb_address", + target: "srv.ipv6.multi.fake", + hostLookupTable: map[string][]string{ + "srv.ipv6.multi.fake": nil, + "ipv6.multi.fake": {"2607:f8b0:400a:801::1001", "2607:f8b0:400a:801::1002", "2607:f8b0:400a:801::1003"}, + }, + srvLookupTable: map[string][]*net.SRV{ + "_grpclb._tcp.srv.ipv6.multi.fake": {&net.SRV{Target: "ipv6.multi.fake", Port: 1234}}, + }, + txtLookupTable: map[string][]string{ + "_grpc_config.srv.ipv6.multi.fake": txtRecordServiceConfig(txtRecordNonMatching), + }, + want: resolverUpdate{ + balancerAddrs: []resolver.Address{ + {Addr: "[2607:f8b0:400a:801::1001]:1234", ServerName: "ipv6.multi.fake"}, + {Addr: "[2607:f8b0:400a:801::1002]:1234", ServerName: "ipv6.multi.fake"}, + {Addr: "[2607:f8b0:400a:801::1003]:1234", ServerName: "ipv6.multi.fake"}, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + overrideTimeAfterFunc(t, 2*defaultTestTimeout) + overrideNetResolver(t, &testNetResolver{ + hostLookupTable: test.hostLookupTable, + srvLookupTable: test.srvLookupTable, + txtLookupTable: test.txtLookupTable, + }) + enableSRVLookups(t) + _, stateCh, _ := buildResolverWithTestClientConn(t, test.target) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + verifyUpdateFromResolver(ctx, t, stateCh, test.want) + }) + } +} + +// Tests the case where the channel returns an error for the update pushed by +// the DNS resolver. Verifies that the DNS resolver backs off before trying to +// resolve. Once the channel returns a nil error, the test verifies that the DNS +// resolver does not backoff anymore. +func (s) TestDNSResolver_ExponentialBackoff(t *testing.T) { + tests := []struct { + name string + target string + hostLookupTable map[string][]string + txtLookupTable map[string][]string + wantAddrs []resolver.Address + wantSC string + }{ + { + name: "happy case default port", + target: "foo.bar.com", + hostLookupTable: map[string][]string{"foo.bar.com": {"1.2.3.4", "5.6.7.8"}}, + txtLookupTable: map[string][]string{ + "_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood), + }, + wantAddrs: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}}, + wantSC: scJSON, + }, + { + name: "happy case specified port", + target: "foo.bar.com:1234", + hostLookupTable: map[string][]string{"foo.bar.com": {"1.2.3.4", "5.6.7.8"}}, + txtLookupTable: map[string][]string{ + "_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood), + }, + wantAddrs: []resolver.Address{{Addr: "1.2.3.4:1234"}, {Addr: "5.6.7.8:1234"}}, + wantSC: scJSON, + }, + { + name: "happy case another default port", + target: "srv.ipv4.single.fake", + hostLookupTable: map[string][]string{ + "srv.ipv4.single.fake": {"2.4.6.8"}, + "ipv4.single.fake": {"1.2.3.4"}, + }, + txtLookupTable: map[string][]string{ + "_grpc_config.srv.ipv4.single.fake": txtRecordServiceConfig(txtRecordGood), + }, + wantAddrs: []resolver.Address{{Addr: "2.4.6.8" + colonDefaultPort}}, + wantSC: scJSON, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + durChan, timeChan := overrideTimeAfterFuncWithChannel(t) + overrideNetResolver(t, &testNetResolver{ + hostLookupTable: test.hostLookupTable, + txtLookupTable: test.txtLookupTable, + }) + + // Set the test clientconn to return error back to the resolver when + // it pushes an update on the channel. + var returnNilErr atomic.Bool + updateStateF := func(resolver.State) error { + if returnNilErr.Load() { + return nil + } + return balancer.ErrBadResolverState + } + tcc := &testutils.ResolverClientConn{Logger: t, UpdateStateF: updateStateF} + + b := resolver.Get("dns") + if b == nil { + t.Fatalf("Resolver for dns:/// scheme not registered") + } + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("dns:///%s", test.target))}, tcc, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("Failed to build DNS resolver for target %q: %v\n", test.target, err) + } + defer r.Close() + + // Expect the DNS resolver to backoff and attempt to re-resolve. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + const retries = 10 + var prevDur time.Duration + for i := 0; i < retries; i++ { + select { + case <-ctx.Done(): + t.Fatalf("(Iteration: %d): Timeout when waiting for DNS resolver to backoff", i) + case dur := <-durChan: + if dur <= prevDur { + t.Fatalf("(Iteration: %d): Unexpected decrease in amount of time to backoff", i) + } + } + + if i == retries-1 { + // Update resolver.ClientConn to not return an error + // anymore before last resolution retry to ensure that + // last resolution attempt doesn't back off. + returnNilErr.Store(true) + } + + // Unblock the DNS resolver's backoff by pushing the current time. + timeChan <- time.Now() + } + + // Verify that the DNS resolver does not backoff anymore. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-durChan: + t.Fatal("Unexpected DNS resolver backoff") + case <-sCtx.Done(): + } + }) + } +} + +// Tests the case where the DNS resolver is asked to re-resolve by invoking the +// ResolveNow method. +func (s) TestDNSResolver_ResolveNow(t *testing.T) { + const target = "foo.bar.com" + + overrideResolutionInterval(t, 0) + overrideTimeAfterFunc(t, 0) + tr := &testNetResolver{ + hostLookupTable: map[string][]string{ + "foo.bar.com": {"1.2.3.4", "5.6.7.8"}, + }, + txtLookupTable: map[string][]string{ + "_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood), + }, + } + overrideNetResolver(t, tr) + + r, stateCh, _ := buildResolverWithTestClientConn(t, target) + + // Verify that the first update pushed by the resolver matches expectations. + wantAddrs := []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}} + wantSC := mustParseServiceConfig(t, scJSON) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + verifyUpdateFromResolver(ctx, t, stateCh, resolverUpdate{ + addrs: wantAddrs, + endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}}}, + {Addresses: []resolver.Address{{Addr: "5.6.7.8" + colonDefaultPort}}}, + }, + serviceConfig: wantSC, + }) + + // Update state in the fake net.Resolver to return only one address and a + // new service config. + tr.UpdateHostLookupTable(map[string][]string{target: {"1.2.3.4"}}) + tr.UpdateTXTLookupTable(map[string][]string{ + "_grpc_config.foo.bar.com": txtRecordServiceConfig(`[{"serviceConfig":{"loadBalancingPolicy": "grpclb"}}]`), + }) + + // Ask the resolver to re-resolve and verify that the new update matches + // expectations. + r.ResolveNow(resolver.ResolveNowOptions{}) + wantAddrs = []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}} + wantSC = mustParseServiceConfig(t, `{"loadBalancingPolicy": "grpclb"}`) + verifyUpdateFromResolver(ctx, t, stateCh, resolverUpdate{ + addrs: wantAddrs, + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}}}}, + serviceConfig: wantSC, + }) + + // Update state in the fake resolver to return no addresses and the same + // service config as before. + tr.UpdateHostLookupTable(map[string][]string{target: nil}) + + // Ask the resolver to re-resolve and verify that the new update matches + // expectations. + r.ResolveNow(resolver.ResolveNowOptions{}) + verifyUpdateFromResolver(ctx, t, stateCh, resolverUpdate{serviceConfig: wantSC}) +} + +// Tests the case where the given name is an IP address and verifies that the +// update pushed by the DNS resolver meets expectations. +func (s) TestIPResolver(t *testing.T) { + tests := []struct { + name string + target string + want resolverUpdate + }{ + { + name: "localhost ipv4 default port", + target: "127.0.0.1", + want: resolverUpdate{addrs: []resolver.Address{{Addr: "127.0.0.1:443"}}, endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "127.0.0.1:443"}}}}}, + }, + { + name: "localhost ipv4 non-default port", + target: "127.0.0.1:12345", + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "127.0.0.1:12345"}}, + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "127.0.0.1:12345"}}}}, + }, + }, + { + name: "localhost ipv6 default port no brackets", + target: "::1", + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "[::1]:443"}}, + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "[::1]:443"}}}}, + }, + }, + { + name: "localhost ipv6 default port with brackets", + target: "[::1]", + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "[::1]:443"}}, + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "[::1]:443"}}}}, + }, + }, + { + name: "localhost ipv6 non-default port", + target: "[::1]:12345", + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "[::1]:12345"}}, + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "[::1]:12345"}}}}, + }, + }, + { + name: "ipv6 default port no brackets", + target: "2001:db8:85a3::8a2e:370:7334", + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "[2001:db8:85a3::8a2e:370:7334]:443"}}, + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "[2001:db8:85a3::8a2e:370:7334]:443"}}}}, + }, + }, + { + name: "ipv6 default port with brackets", + target: "[2001:db8:85a3::8a2e:370:7334]", + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "[2001:db8:85a3::8a2e:370:7334]:443"}}, + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "[2001:db8:85a3::8a2e:370:7334]:443"}}}}, + }, + }, + { + name: "ipv6 non-default port with brackets", + target: "[2001:db8:85a3::8a2e:370:7334]:12345", + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "[2001:db8:85a3::8a2e:370:7334]:12345"}}, + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "[2001:db8:85a3::8a2e:370:7334]:12345"}}}}, + }, + }, + { + name: "abbreviated ipv6 address", + target: "[2001:db8::1]:http", + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "[2001:db8::1]:http"}}, + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "[2001:db8::1]:http"}}}}, + }, + }, + { + name: "ipv6 with zone and port", + target: "[fe80::1%25eth0]:1234", + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "[fe80::1%eth0]:1234"}}, + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "[fe80::1%eth0]:1234"}}}}, + }, + }, + { + name: "ipv6 with zone and default port", + target: "fe80::1%25eth0", + want: resolverUpdate{addrs: []resolver.Address{{Addr: "[fe80::1%eth0]:443"}}, + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "[fe80::1%eth0]:443"}}}}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + overrideResolutionInterval(t, 0) + overrideTimeAfterFunc(t, 2*defaultTestTimeout) + r, stateCh, _ := buildResolverWithTestClientConn(t, test.target) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + verifyUpdateFromResolver(ctx, t, stateCh, test.want) + + // Attempt to re-resolve should not result in a state update. + r.ResolveNow(resolver.ResolveNowOptions{}) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case s := <-stateCh: + t.Fatalf("Unexpected state update from the resolver: %+v", s) + } + }) + } +} + +// Tests the DNS resolver builder with different target names. +func (s) TestResolverBuild(t *testing.T) { + tests := []struct { + name string + target string + wantErr string + }{ + { + name: "valid url", + target: "www.google.com", + }, + { + name: "host port", + target: "foo.bar:12345", + }, + { + name: "ipv4 address with default port", + target: "127.0.0.1", + }, + { + name: "ipv6 address without brackets and default port", + target: "::", + }, + { + name: "ipv4 address with non-default port", + target: "127.0.0.1:12345", + }, + { + name: "localhost ipv6 with brackets", + target: "[::1]:80", + }, + { + name: "ipv6 address with brackets", + target: "[2001:db8:a0b:12f0::1]:21", + }, + { + name: "empty host with port", + target: ":80", + }, + { + name: "ipv6 address with zone", + target: "[fe80::1%25lo0]:80", + }, + { + name: "url with port", + target: "golang.org:http", + }, + { + name: "ipv6 address with non integer port", + target: "[2001:db8::1]:http", + }, + { + name: "address ends with colon", + target: "[2001:db8::1]:", + wantErr: dnsinternal.ErrEndsWithColon.Error(), + }, + { + name: "address contains only a colon", + target: ":", + wantErr: dnsinternal.ErrEndsWithColon.Error(), + }, + { + name: "empty address", + target: "", + wantErr: dnsinternal.ErrMissingAddr.Error(), + }, + { + name: "invalid address", + target: "[2001:db8:a0b:12f0::1", + wantErr: "invalid target address", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + overrideTimeAfterFunc(t, 2*defaultTestTimeout) + + b := resolver.Get("dns") + if b == nil { + t.Fatalf("Resolver for dns:/// scheme not registered") + } + + tcc := &testutils.ResolverClientConn{Logger: t} + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("dns:///%s", test.target))}, tcc, resolver.BuildOptions{}) + if err != nil { + if test.wantErr == "" { + t.Fatalf("DNS resolver build for target %q failed with error: %v", test.target, err) + } + if !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("DNS resolver build for target %q failed with error: %v, wantErr: %s", test.target, err, test.wantErr) + } + return + } + if test.wantErr != "" { + t.Fatalf("DNS resolver build for target %q succeeded when expected to fail with error: %s", test.target, test.wantErr) + } + r.Close() + }) + } +} + +// Tests scenarios where fetching of service config is enabled or disabled, and +// verifies that the expected update is pushed by the DNS resolver. +func (s) TestDisableServiceConfig(t *testing.T) { + tests := []struct { + name string + target string + hostLookupTable map[string][]string + txtLookupTable map[string][]string + txtLookupsEnabled bool + disableServiceConfig bool + want resolverUpdate + }{ + { + name: "not disabled in BuildOptions; enabled globally", + target: "foo.bar.com", + hostLookupTable: map[string][]string{"foo.bar.com": {"1.2.3.4", "5.6.7.8"}}, + txtLookupTable: map[string][]string{ + "_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood), + }, + txtLookupsEnabled: true, + disableServiceConfig: false, + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}}, + endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}}}, + {Addresses: []resolver.Address{{Addr: "5.6.7.8" + colonDefaultPort}}}, + }, + serviceConfig: mustParseServiceConfig(t, scJSON), + }, + }, + { + name: "disabled in BuildOptions; enabled globally", + target: "foo.bar.com", + hostLookupTable: map[string][]string{"foo.bar.com": {"1.2.3.4", "5.6.7.8"}}, + txtLookupTable: map[string][]string{ + "_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood), + }, + txtLookupsEnabled: true, + disableServiceConfig: true, + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}}, + endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}}}, + {Addresses: []resolver.Address{{Addr: "5.6.7.8" + colonDefaultPort}}}, + }, + }, + }, + { + name: "not disabled in BuildOptions; disabled globally", + target: "foo.bar.com", + hostLookupTable: map[string][]string{"foo.bar.com": {"1.2.3.4", "5.6.7.8"}}, + txtLookupTable: map[string][]string{ + "_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood), + }, + txtLookupsEnabled: false, + disableServiceConfig: false, + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}}, + endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}}}, + {Addresses: []resolver.Address{{Addr: "5.6.7.8" + colonDefaultPort}}}, + }, + }, + }, + { + name: "disabled in BuildOptions; disabled globally", + target: "foo.bar.com", + hostLookupTable: map[string][]string{"foo.bar.com": {"1.2.3.4", "5.6.7.8"}}, + txtLookupTable: map[string][]string{ + "_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood), + }, + txtLookupsEnabled: false, + disableServiceConfig: true, + want: resolverUpdate{ + addrs: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}}, + endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}}}, + {Addresses: []resolver.Address{{Addr: "5.6.7.8" + colonDefaultPort}}}, + }, + }, + }, + } + + oldEnableTXTServiceConfig := envconfig.EnableTXTServiceConfig + defer func() { + envconfig.EnableTXTServiceConfig = oldEnableTXTServiceConfig + }() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + envconfig.EnableTXTServiceConfig = test.txtLookupsEnabled + overrideTimeAfterFunc(t, 2*defaultTestTimeout) + overrideNetResolver(t, &testNetResolver{ + hostLookupTable: test.hostLookupTable, + txtLookupTable: test.txtLookupTable, + }) + + b := resolver.Get("dns") + if b == nil { + t.Fatalf("Resolver for dns:/// scheme not registered") + } + + stateCh := make(chan resolver.State, 1) + updateStateF := func(s resolver.State) error { + stateCh <- s + return nil + } + tcc := &testutils.ResolverClientConn{Logger: t, UpdateStateF: updateStateF} + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("dns:///%s", test.target))}, tcc, resolver.BuildOptions{DisableServiceConfig: test.disableServiceConfig}) + if err != nil { + t.Fatalf("Failed to build DNS resolver for target %q: %v\n", test.target, err) + } + defer r.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + verifyUpdateFromResolver(ctx, t, stateCh, test.want) + }) + } +} + +// Tests the case where a TXT lookup is expected to return an error. Verifies +// that errors are ignored with the corresponding env var is set. +func (s) TestTXTError(t *testing.T) { + for _, ignore := range []bool{false, true} { + t.Run(fmt.Sprintf("%v", ignore), func(t *testing.T) { + overrideTimeAfterFunc(t, 2*defaultTestTimeout) + overrideNetResolver(t, &testNetResolver{hostLookupTable: map[string][]string{"ipv4.single.fake": {"1.2.3.4"}}}) + + origTXTIgnore := envconfig.TXTErrIgnore + envconfig.TXTErrIgnore = ignore + defer func() { envconfig.TXTErrIgnore = origTXTIgnore }() + + // There is no entry for "ipv4.single.fake" in the txtLookupTbl + // maintained by the fake net.Resolver. So, a TXT lookup for this + // name will return an error. + _, stateCh, _ := buildResolverWithTestClientConn(t, "ipv4.single.fake") + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + var state resolver.State + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for a state update from the resolver") + case state = <-stateCh: + } + + if ignore { + if state.ServiceConfig != nil { + t.Fatalf("Received non-nil service config: %+v; want nil", state.ServiceConfig) + } + } else { + if state.ServiceConfig == nil || state.ServiceConfig.Err == nil { + t.Fatalf("Received service config %+v; want non-nil error", state.ServiceConfig) + } + } + }) + } +} + +// Tests different cases for a user's dial target that specifies a non-empty +// authority (or Host field of the URL). +func (s) TestCustomAuthority(t *testing.T) { + tests := []struct { + name string + authority string + wantAuthority string + wantBuildErr bool + }{ + { + name: "authority with default DNS port", + authority: "4.3.2.1:53", + wantAuthority: "4.3.2.1:53", + }, + { + name: "authority with non-default DNS port", + authority: "4.3.2.1:123", + wantAuthority: "4.3.2.1:123", + }, + { + name: "authority with no port", + authority: "4.3.2.1", + wantAuthority: "4.3.2.1:53", + }, + { + name: "ipv6 authority with brackets and no port", + authority: "[::1]", + wantAuthority: "[::1]:53", + }, + { + name: "ipv6 authority with brackets and non-default DNS port", + authority: "[::1]:123", + wantAuthority: "[::1]:123", + }, + { + name: "host name with no port", + authority: "dnsserver.com", + wantAuthority: "dnsserver.com:53", + }, + { + name: "no host port and non-default port", + authority: ":123", + wantAuthority: "localhost:123", + }, + { + name: "only colon", + authority: ":", + wantAuthority: "", + wantBuildErr: true, + }, + { + name: "ipv6 name ending in colon", + authority: "[::1]:", + wantAuthority: "", + wantBuildErr: true, + }, + { + name: "host name ending in colon", + authority: "dnsserver.com:", + wantAuthority: "", + wantBuildErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + overrideTimeAfterFunc(t, 2*defaultTestTimeout) + + // Override the address dialer to verify the authority being passed. + origAddressDialer := dnsinternal.AddressDialer + errChan := make(chan error, 1) + dnsinternal.AddressDialer = func(authority string) func(ctx context.Context, network, address string) (net.Conn, error) { + if authority != test.wantAuthority { + errChan <- fmt.Errorf("wrong custom authority passed to resolver. target: %s got authority: %s want authority: %s", test.authority, authority, test.wantAuthority) + } else { + errChan <- nil + } + return func(context.Context, string, string) (net.Conn, error) { + return nil, errors.New("no need to dial") + } + } + defer func() { dnsinternal.AddressDialer = origAddressDialer }() + + b := resolver.Get("dns") + if b == nil { + t.Fatalf("Resolver for dns:/// scheme not registered") + } + + tcc := &testutils.ResolverClientConn{Logger: t} + endpoint := "foo.bar.com" + target := resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("dns://%s/%s", test.authority, endpoint))} + r, err := b.Build(target, tcc, resolver.BuildOptions{}) + if (err != nil) != test.wantBuildErr { + t.Fatalf("DNS resolver build for target %+v returned error %v: wantErr: %v\n", target, err, test.wantBuildErr) + } + if err != nil { + return + } + defer r.Close() + + if err := <-errChan; err != nil { + t.Fatal(err) + } + }) + } +} + +// TestRateLimitedResolve exercises the rate limit enforced on re-resolution +// requests. It sets the re-resolution rate to a small value and repeatedly +// calls ResolveNow() and ensures only the expected number of resolution +// requests are made. +func (s) TestRateLimitedResolve(t *testing.T) { + const target = "foo.bar.com" + _, timeChan := overrideTimeAfterFuncWithChannel(t) + tr := &testNetResolver{ + lookupHostCh: testutils.NewChannel(), + hostLookupTable: map[string][]string{target: {"1.2.3.4", "5.6.7.8"}}, + } + overrideNetResolver(t, tr) + + r, stateCh, _ := buildResolverWithTestClientConn(t, target) + + // Wait for the first resolution request to be done. This happens as part + // of the first iteration of the for loop in watcher(). + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tr.lookupHostCh.Receive(ctx); err != nil { + t.Fatalf("Timed out waiting for lookup() call.") + } + + // Call Resolve Now 100 times, shouldn't continue onto next iteration of + // watcher, thus shouldn't lookup again. + for i := 0; i <= 100; i++ { + r.ResolveNow(resolver.ResolveNowOptions{}) + } + + continueCtx, continueCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer continueCancel() + if _, err := tr.lookupHostCh.Receive(continueCtx); err == nil { + t.Fatalf("Should not have looked up again as DNS Min Res Rate timer has not gone off.") + } + + // Make the DNSMinResRate timer fire immediately, by sending the current + // time on it. This will unblock the resolver which is currently blocked on + // the DNS Min Res Rate timer going off, which will allow it to continue to + // the next iteration of the watcher loop. + select { + case timeChan <- time.Now(): + case <-ctx.Done(): + t.Fatal("Timed out waiting for the DNS resolver to block on DNS Min Res Rate to elapse") + } + + // Now that DNS Min Res Rate timer has gone off, it should lookup again. + if _, err := tr.lookupHostCh.Receive(ctx); err != nil { + t.Fatalf("Timed out waiting for lookup() call.") + } + + // Resolve Now 1000 more times, shouldn't lookup again as DNS Min Res Rate + // timer has not gone off. + for i := 0; i < 1000; i++ { + r.ResolveNow(resolver.ResolveNowOptions{}) + } + continueCtx, continueCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer continueCancel() + if _, err := tr.lookupHostCh.Receive(continueCtx); err == nil { + t.Fatalf("Should not have looked up again as DNS Min Res Rate timer has not gone off.") + } + + // Make the DNSMinResRate timer fire immediately again. + select { + case timeChan <- time.Now(): + case <-ctx.Done(): + t.Fatal("Timed out waiting for the DNS resolver to block on DNS Min Res Rate to elapse") + } + + // Now that DNS Min Res Rate timer has gone off, it should lookup again. + if _, err := tr.lookupHostCh.Receive(ctx); err != nil { + t.Fatalf("Timed out waiting for lookup() call.") + } + + wantAddrs := []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}} + var state resolver.State + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for a state update from the resolver") + case state = <-stateCh: + } + if !cmp.Equal(state.Addresses, wantAddrs, cmpopts.EquateEmpty()) { + t.Fatalf("Got addresses: %+v, want: %+v", state.Addresses, wantAddrs) + } +} + +// Test verifies that when the DNS resolver gets an error from the underlying +// net.Resolver, it reports the error to the channel and backs off and retries. +func (s) TestReportError(t *testing.T) { + durChan, timeChan := overrideTimeAfterFuncWithChannel(t) + overrideNetResolver(t, &testNetResolver{}) + + const target = "notfoundaddress" + _, _, errorCh := buildResolverWithTestClientConn(t, target) + + // Should receive first error. + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for an error from the resolver") + case err := <-errorCh: + if !strings.Contains(err.Error(), "hostLookup error") { + t.Fatalf(`ReportError(err=%v) called; want err contains "hostLookupError"`, err) + } + } + + // Expect the DNS resolver to backoff and attempt to re-resolve. Every time, + // the DNS resolver will receive the same error from the net.Resolver and is + // expected to push it to the channel. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + const retries = 10 + var prevDur time.Duration + for i := 0; i < retries; i++ { + select { + case <-ctx.Done(): + t.Fatalf("(Iteration: %d): Timeout when waiting for DNS resolver to backoff", i) + case dur := <-durChan: + if dur <= prevDur { + t.Fatalf("(Iteration: %d): Unexpected decrease in amount of time to backoff", i) + } + } + + // Unblock the DNS resolver's backoff by pushing the current time. + timeChan <- time.Now() + + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for an error from the resolver") + case err := <-errorCh: + if !strings.Contains(err.Error(), "hostLookup error") { + t.Fatalf(`ReportError(err=%v) called; want err contains "hostLookupError"`, err) + } + } + } +} + +// Override the default dns.ResolvingTimeout with a test duration. +func overrideResolveTimeoutDuration(t *testing.T, dur time.Duration) { + t.Helper() + + origDur := dns.ResolvingTimeout + dnspublic.SetResolvingTimeout(dur) + + t.Cleanup(func() { dnspublic.SetResolvingTimeout(origDur) }) +} + +// Test verifies that the DNS resolver gets timeout error when net.Resolver +// takes too long to resolve a target. +func (s) TestResolveTimeout(t *testing.T) { + // Set DNS resolving timeout duration to 7ms + timeoutDur := 7 * time.Millisecond + overrideResolveTimeoutDuration(t, timeoutDur) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // We are trying to resolve hostname which takes infinity time to resolve. + const target = "infinity" + + // Define a testNetResolver with lookupHostCh, an unbuffered channel, + // so we can block the resolver until reaching timeout. + tr := &testNetResolver{ + lookupHostCh: testutils.NewChannelWithSize(0), + hostLookupTable: map[string][]string{target: {"1.2.3.4"}}, + } + overrideNetResolver(t, tr) + + _, _, errCh := buildResolverWithTestClientConn(t, target) + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for the DNS resolver to timeout") + case err := <-errCh: + if err == nil || !strings.Contains(err.Error(), "context deadline exceeded") { + t.Fatalf(`Expected to see Timeout error; got: %v`, err) + } + } +} + +// Test verifies that changing [MinResolutionInterval] variable correctly effects +// the resolution behaviour +func (s) TestMinResolutionInterval(t *testing.T) { + const target = "foo.bar.com" + + overrideResolutionInterval(t, 1*time.Millisecond) + tr := &testNetResolver{ + hostLookupTable: map[string][]string{ + "foo.bar.com": {"1.2.3.4", "5.6.7.8"}, + }, + txtLookupTable: map[string][]string{ + "_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood), + }, + } + overrideNetResolver(t, tr) + + r, stateCh, _ := buildResolverWithTestClientConn(t, target) + + wantAddrs := []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}} + wantSC := mustParseServiceConfig(t, scJSON) + + for i := 0; i < 5; i++ { + // set context timeout slightly higher than the min resolution interval to make sure resolutions + // happen successfully + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + verifyUpdateFromResolver(ctx, t, stateCh, resolverUpdate{ + addrs: wantAddrs, + endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}}}, + {Addresses: []resolver.Address{{Addr: "5.6.7.8" + colonDefaultPort}}}, + }, + serviceConfig: wantSC}) + r.ResolveNow(resolver.ResolveNowOptions{}) + } +} + +// TestMinResolutionInterval_NoExtraDelay verifies that there is no extra delay +// between two resolution requests apart from [MinResolutionInterval]. +func (s) TestMinResolutionInterval_NoExtraDelay(t *testing.T) { + tr := &testNetResolver{ + hostLookupTable: map[string][]string{ + "foo.bar.com": {"1.2.3.4", "5.6.7.8"}, + }, + txtLookupTable: map[string][]string{ + "_grpc_config.foo.bar.com": txtRecordServiceConfig(txtRecordGood), + }, + } + overrideNetResolver(t, tr) + // Override time.Now() to return a zero value for time. This will allow us + // to verify that the call to time.Until is made with the exact + // [MinResolutionInterval] that we expect. + overrideTimeNowFunc(t, time.Time{}) + // Override time.Until() to read the time passed to it + // and return immediately without any delay + timeCh := overrideTimeUntilFuncWithChannel(t) + + r, stateCh, errorCh := buildResolverWithTestClientConn(t, "foo.bar.com") + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Ensure that the first resolution happens. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for DNS resolver") + case err := <-errorCh: + t.Fatalf("Unexpected error from resolver, %v", err) + case <-stateCh: + } + + // Request re-resolution and verify that the resolver waits for + // [MinResolutionInterval]. + r.ResolveNow(resolver.ResolveNowOptions{}) + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for DNS resolver") + case gotTime := <-timeCh: + wantTime := time.Time{}.Add(dns.MinResolutionInterval) + if !gotTime.Equal(wantTime) { + t.Fatalf("DNS resolver waits for %v time before re-resolution, want %v", gotTime, wantTime) + } + } + + // Ensure that the re-resolution request actually happens. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for an error from the resolver") + case err := <-errorCh: + t.Fatalf("Unexpected error from resolver, %v", err) + case <-stateCh: + } +} + +func mustParseServiceConfig(t *testing.T, scJSON string) serviceconfig.Config { + t.Helper() + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + if sc.Err != nil { + t.Fatalf("Failed to parse service config %q: %v", scJSON, sc.Err) + } + return sc.Config +} diff --git a/internal/resolver/dns/fake_net_resolver_test.go b/internal/resolver/dns/fake_net_resolver_test.go new file mode 100644 index 0000000..3a58a75 --- /dev/null +++ b/internal/resolver/dns/fake_net_resolver_test.go @@ -0,0 +1,126 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package dns_test + +import ( + "context" + "net" + "sync" + + "google.golang.org/grpc/internal/testutils" +) + +// A fake implementation of the internal.NetResolver interface for use in tests. +type testNetResolver struct { + // A write to this channel is made when this resolver receives a resolution + // request. Tests can rely on reading from this channel to be notified about + // resolution requests instead of sleeping for a predefined period of time. + lookupHostCh *testutils.Channel + + mu sync.Mutex + hostLookupTable map[string][]string // Name --> list of addresses + srvLookupTable map[string][]*net.SRV // Name --> list of SRV records + txtLookupTable map[string][]string // Name --> service config for TXT record +} + +func (tr *testNetResolver) LookupHost(ctx context.Context, host string) ([]string, error) { + if tr.lookupHostCh != nil { + if err := tr.lookupHostCh.SendContext(ctx, nil); err != nil { + return nil, err + } + } + + tr.mu.Lock() + defer tr.mu.Unlock() + + if addrs, ok := tr.hostLookupTable[host]; ok { + return addrs, nil + } + + return nil, &net.DNSError{ + Err: "hostLookup error", + Name: host, + Server: "fake", + IsTemporary: true, + } +} + +func (tr *testNetResolver) UpdateHostLookupTable(table map[string][]string) { + tr.mu.Lock() + tr.hostLookupTable = table + tr.mu.Unlock() +} + +func (tr *testNetResolver) LookupSRV(_ context.Context, service, proto, name string) (string, []*net.SRV, error) { + tr.mu.Lock() + defer tr.mu.Unlock() + + cname := "_" + service + "._" + proto + "." + name + if srvs, ok := tr.srvLookupTable[cname]; ok { + return cname, srvs, nil + } + return "", nil, &net.DNSError{ + Err: "srvLookup error", + Name: cname, + Server: "fake", + IsTemporary: true, + } +} + +func (tr *testNetResolver) LookupTXT(_ context.Context, host string) ([]string, error) { + tr.mu.Lock() + defer tr.mu.Unlock() + + if sc, ok := tr.txtLookupTable[host]; ok { + return sc, nil + } + return nil, &net.DNSError{ + Err: "txtLookup error", + Name: host, + Server: "fake", + IsTemporary: true, + } +} + +func (tr *testNetResolver) UpdateTXTLookupTable(table map[string][]string) { + tr.mu.Lock() + tr.txtLookupTable = table + tr.mu.Unlock() +} + +// txtRecordServiceConfig generates a slice of strings (aggregately representing +// a single service config file) for the input config string, that represents +// the result from a real DNS TXT record lookup. +func txtRecordServiceConfig(cfg string) []string { + // In DNS, service config is encoded in a TXT record via the mechanism + // described in RFC-1464 using the attribute name grpc_config. + b := append([]byte("grpc_config="), []byte(cfg)...) + + // Split b into multiple strings, each with a max of 255 bytes, which is + // the DNS TXT record limit. + var r []string + for i := 0; i < len(b); i += txtBytesLimit { + if i+txtBytesLimit > len(b) { + r = append(r, string(b[i:])) + } else { + r = append(r, string(b[i:i+txtBytesLimit])) + } + } + return r +} diff --git a/internal/resolver/dns/internal/internal.go b/internal/resolver/dns/internal/internal.go new file mode 100644 index 0000000..c0eae4f --- /dev/null +++ b/internal/resolver/dns/internal/internal.go @@ -0,0 +1,77 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains functionality internal to the dns resolver package. +package internal + +import ( + "context" + "errors" + "net" + "time" +) + +// NetResolver groups the methods on net.Resolver that are used by the DNS +// resolver implementation. This allows the default net.Resolver instance to be +// overridden from tests. +type NetResolver interface { + LookupHost(ctx context.Context, host string) (addrs []string, err error) + LookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*net.SRV, err error) + LookupTXT(ctx context.Context, name string) (txts []string, err error) +} + +var ( + // ErrMissingAddr is the error returned when building a DNS resolver when + // the provided target name is empty. + ErrMissingAddr = errors.New("dns resolver: missing address") + + // ErrEndsWithColon is the error returned when building a DNS resolver when + // the provided target name ends with a colon that is supposed to be the + // separator between host and port. E.g. "::" is a valid address as it is + // an IPv6 address (host only) and "[::]:" is invalid as it ends with a + // colon as the host and port separator + ErrEndsWithColon = errors.New("dns resolver: missing port after port-separator colon") +) + +// The following vars are overridden from tests. +var ( + // TimeAfterFunc is used by the DNS resolver to wait for the given duration + // to elapse. In non-test code, this is implemented by time.After. In test + // code, this can be used to control the amount of time the resolver is + // blocked waiting for the duration to elapse. + TimeAfterFunc func(time.Duration) <-chan time.Time + + // TimeNowFunc is used by the DNS resolver to get the current time. + // In non-test code, this is implemented by time.Now. In test code, + // this can be used to control the current time for the resolver. + TimeNowFunc func() time.Time + + // TimeUntilFunc is used by the DNS resolver to calculate the remaining + // wait time for re-resolution. In non-test code, this is implemented by + // time.Until. In test code, this can be used to control the remaining + // time for resolver to wait for re-resolution. + TimeUntilFunc func(time.Time) time.Duration + + // NewNetResolver returns the net.Resolver instance for the given target. + NewNetResolver func(string) (NetResolver, error) + + // AddressDialer is the dialer used to dial the DNS server. It accepts the + // Host portion of the URL corresponding to the user's dial target and + // returns a dial function. + AddressDialer func(address string) func(context.Context, string, string) (net.Conn, error) +) diff --git a/internal/resolver/passthrough/passthrough.go b/internal/resolver/passthrough/passthrough.go new file mode 100644 index 0000000..b901c7b --- /dev/null +++ b/internal/resolver/passthrough/passthrough.go @@ -0,0 +1,64 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package passthrough implements a pass-through resolver. It sends the target +// name without scheme back to gRPC as resolved address. +package passthrough + +import ( + "errors" + + "google.golang.org/grpc/resolver" +) + +const scheme = "passthrough" + +type passthroughBuilder struct{} + +func (*passthroughBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { + if target.Endpoint() == "" && opts.Dialer == nil { + return nil, errors.New("passthrough: received empty target in Build()") + } + r := &passthroughResolver{ + target: target, + cc: cc, + } + r.start() + return r, nil +} + +func (*passthroughBuilder) Scheme() string { + return scheme +} + +type passthroughResolver struct { + target resolver.Target + cc resolver.ClientConn +} + +func (r *passthroughResolver) start() { + r.cc.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: r.target.Endpoint()}}}) +} + +func (*passthroughResolver) ResolveNow(resolver.ResolveNowOptions) {} + +func (*passthroughResolver) Close() {} + +func init() { + resolver.Register(&passthroughBuilder{}) +} diff --git a/internal/resolver/unix/unix.go b/internal/resolver/unix/unix.go new file mode 100644 index 0000000..27cd81a --- /dev/null +++ b/internal/resolver/unix/unix.go @@ -0,0 +1,78 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package unix implements a resolver for unix targets. +package unix + +import ( + "fmt" + + "google.golang.org/grpc/internal/transport/networktype" + "google.golang.org/grpc/resolver" +) + +const unixScheme = "unix" +const unixAbstractScheme = "unix-abstract" + +type builder struct { + scheme string +} + +func (b *builder) Build(target resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) { + if target.URL.Host != "" { + return nil, fmt.Errorf("invalid (non-empty) authority: %v", target.URL.Host) + } + + // gRPC was parsing the dial target manually before PR #4817, and we + // switched to using url.Parse() in that PR. To avoid breaking existing + // resolver implementations we ended up stripping the leading "/" from the + // endpoint. This obviously does not work for the "unix" scheme. Hence we + // end up using the parsed URL instead. + endpoint := target.URL.Path + if endpoint == "" { + endpoint = target.URL.Opaque + } + addr := resolver.Address{Addr: endpoint} + if b.scheme == unixAbstractScheme { + // We can not prepend \0 as c++ gRPC does, as in Golang '@' is used to signify we do + // not want trailing \0 in address. + addr.Addr = "@" + addr.Addr + } + cc.UpdateState(resolver.State{Addresses: []resolver.Address{networktype.Set(addr, "unix")}}) + return &nopResolver{}, nil +} + +func (b *builder) Scheme() string { + return b.scheme +} + +func (b *builder) OverrideAuthority(resolver.Target) string { + return "localhost" +} + +type nopResolver struct { +} + +func (*nopResolver) ResolveNow(resolver.ResolveNowOptions) {} + +func (*nopResolver) Close() {} + +func init() { + resolver.Register(&builder{scheme: unixScheme}) + resolver.Register(&builder{scheme: unixAbstractScheme}) +} diff --git a/internal/ringhash/ringhash.go b/internal/ringhash/ringhash.go new file mode 100644 index 0000000..c75ac1c --- /dev/null +++ b/internal/ringhash/ringhash.go @@ -0,0 +1,58 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package ringhash (internal) contains functions and types that need to be +// shared by the ring hash balancer and other gRPC code (such as xDS) +// without being exported. +package ringhash + +import ( + "context" + + "google.golang.org/grpc/serviceconfig" +) + +// LBConfig is the balancer config for ring_hash balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + MinRingSize uint64 `json:"minRingSize,omitempty"` + MaxRingSize uint64 `json:"maxRingSize,omitempty"` + RequestHashHeader string `json:"requestHashHeader,omitempty"` +} + +// xdsHashKey is the type used as the key to store request hash in the context +// used when combining the Ring Hash load balancing policy with xDS. +type xdsHashKey struct{} + +// XDSRequestHash returns the request hash in the context and true if it was set +// from the xDS config selector. If the xDS config selector has not set the hash, +// it returns 0 and false. +func XDSRequestHash(ctx context.Context) (uint64, bool) { + requestHash := ctx.Value(xdsHashKey{}) + if requestHash == nil { + return 0, false + } + return requestHash.(uint64), true +} + +// SetXDSRequestHash adds the request hash to the context for use in Ring Hash +// Load Balancing using xDS route hash_policy. +func SetXDSRequestHash(ctx context.Context, requestHash uint64) context.Context { + return context.WithValue(ctx, xdsHashKey{}, requestHash) +} diff --git a/internal/serviceconfig/duration.go b/internal/serviceconfig/duration.go new file mode 100644 index 0000000..11d82af --- /dev/null +++ b/internal/serviceconfig/duration.go @@ -0,0 +1,130 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package serviceconfig + +import ( + "encoding/json" + "fmt" + "math" + "strconv" + "strings" + "time" +) + +// Duration defines JSON marshal and unmarshal methods to conform to the +// protobuf JSON spec defined [here]. +// +// [here]: https://protobuf.dev/reference/protobuf/google.protobuf/#duration +type Duration time.Duration + +func (d Duration) String() string { + return fmt.Sprint(time.Duration(d)) +} + +// MarshalJSON converts from d to a JSON string output. +func (d Duration) MarshalJSON() ([]byte, error) { + ns := time.Duration(d).Nanoseconds() + sec := ns / int64(time.Second) + ns = ns % int64(time.Second) + + var sign string + if sec < 0 || ns < 0 { + sign, sec, ns = "-", -1*sec, -1*ns + } + + // Generated output always contains 0, 3, 6, or 9 fractional digits, + // depending on required precision. + str := fmt.Sprintf("%s%d.%09d", sign, sec, ns) + str = strings.TrimSuffix(str, "000") + str = strings.TrimSuffix(str, "000") + str = strings.TrimSuffix(str, ".000") + return []byte(fmt.Sprintf("\"%ss\"", str)), nil +} + +// UnmarshalJSON unmarshals b as a duration JSON string into d. +func (d *Duration) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + if !strings.HasSuffix(s, "s") { + return fmt.Errorf("malformed duration %q: missing seconds unit", s) + } + neg := false + if s[0] == '-' { + neg = true + s = s[1:] + } + ss := strings.SplitN(s[:len(s)-1], ".", 3) + if len(ss) > 2 { + return fmt.Errorf("malformed duration %q: too many decimals", s) + } + // hasDigits is set if either the whole or fractional part of the number is + // present, since both are optional but one is required. + hasDigits := false + var sec, ns int64 + if len(ss[0]) > 0 { + var err error + if sec, err = strconv.ParseInt(ss[0], 10, 64); err != nil { + return fmt.Errorf("malformed duration %q: %v", s, err) + } + // Maximum seconds value per the durationpb spec. + const maxProtoSeconds = 315_576_000_000 + if sec > maxProtoSeconds { + return fmt.Errorf("out of range: %q", s) + } + hasDigits = true + } + if len(ss) == 2 && len(ss[1]) > 0 { + if len(ss[1]) > 9 { + return fmt.Errorf("malformed duration %q: too many digits after decimal", s) + } + var err error + if ns, err = strconv.ParseInt(ss[1], 10, 64); err != nil { + return fmt.Errorf("malformed duration %q: %v", s, err) + } + for i := 9; i > len(ss[1]); i-- { + ns *= 10 + } + hasDigits = true + } + if !hasDigits { + return fmt.Errorf("malformed duration %q: contains no numbers", s) + } + + if neg { + sec *= -1 + ns *= -1 + } + + // Maximum/minimum seconds/nanoseconds representable by Go's time.Duration. + const maxSeconds = math.MaxInt64 / int64(time.Second) + const maxNanosAtMaxSeconds = math.MaxInt64 % int64(time.Second) + const minSeconds = math.MinInt64 / int64(time.Second) + const minNanosAtMinSeconds = math.MinInt64 % int64(time.Second) + + if sec > maxSeconds || (sec == maxSeconds && ns >= maxNanosAtMaxSeconds) { + *d = Duration(math.MaxInt64) + } else if sec < minSeconds || (sec == minSeconds && ns <= minNanosAtMinSeconds) { + *d = Duration(math.MinInt64) + } else { + *d = Duration(sec*int64(time.Second) + ns) + } + return nil +} diff --git a/internal/serviceconfig/duration_test.go b/internal/serviceconfig/duration_test.go new file mode 100644 index 0000000..82c0dcd --- /dev/null +++ b/internal/serviceconfig/duration_test.go @@ -0,0 +1,86 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package serviceconfig + +import ( + "fmt" + "math" + rand "math/rand/v2" + "strings" + "testing" + "time" +) + +// Tests both marshalling and unmarshalling of Durations. +func TestDuration_MarshalUnmarshal(t *testing.T) { + testCases := []struct { + json string + td time.Duration + unmarshalErr error + noMarshal bool + }{ + // Basic values. + {json: `"1s"`, td: time.Second}, + {json: `"-100.700s"`, td: -100*time.Second - 700*time.Millisecond}, + {json: `".050s"`, td: 50 * time.Millisecond, noMarshal: true}, + {json: `"-.001s"`, td: -1 * time.Millisecond, noMarshal: true}, + {json: `"-0.200s"`, td: -200 * time.Millisecond}, + // Positive near / out of bounds. + {json: `"9223372036s"`, td: 9223372036 * time.Second}, + {json: `"9223372037s"`, td: math.MaxInt64, noMarshal: true}, + {json: `"9223372036.854775807s"`, td: math.MaxInt64}, + {json: `"9223372036.854775808s"`, td: math.MaxInt64, noMarshal: true}, + {json: `"315576000000s"`, td: math.MaxInt64, noMarshal: true}, + {json: `"315576000001s"`, unmarshalErr: fmt.Errorf("out of range")}, + // Negative near / out of bounds. + {json: `"-9223372036s"`, td: -9223372036 * time.Second}, + {json: `"-9223372037s"`, td: math.MinInt64, noMarshal: true}, + {json: `"-9223372036.854775808s"`, td: math.MinInt64}, + {json: `"-9223372036.854775809s"`, td: math.MinInt64, noMarshal: true}, + {json: `"-315576000000s"`, td: math.MinInt64, noMarshal: true}, + {json: `"-315576000001s"`, unmarshalErr: fmt.Errorf("out of range")}, + // Parse errors. + {json: `123s`, unmarshalErr: fmt.Errorf("invalid character")}, + {json: `"5m"`, unmarshalErr: fmt.Errorf("malformed duration")}, + {json: `"5.3.2s"`, unmarshalErr: fmt.Errorf("malformed duration")}, + {json: `"x.3s"`, unmarshalErr: fmt.Errorf("malformed duration")}, + {json: `"3.xs"`, unmarshalErr: fmt.Errorf("malformed duration")}, + {json: `"3.1234567890s"`, unmarshalErr: fmt.Errorf("malformed duration")}, + {json: `".s"`, unmarshalErr: fmt.Errorf("malformed duration")}, + {json: `"s"`, unmarshalErr: fmt.Errorf("malformed duration")}, + } + for _, tc := range testCases { + // Seed `got` with a random value to ensure we properly reset it in all + // non-error cases. + got := Duration(rand.Uint64()) + err := got.UnmarshalJSON([]byte(tc.json)) + if (err == nil && time.Duration(got) != tc.td) || + (err != nil) != (tc.unmarshalErr != nil) || !strings.Contains(fmt.Sprint(err), fmt.Sprint(tc.unmarshalErr)) { + t.Errorf("UnmarshalJSON of %v = %v, %v; want %v, %v", tc.json, time.Duration(got), err, tc.td, tc.unmarshalErr) + } + + if tc.unmarshalErr == nil && !tc.noMarshal { + d := Duration(tc.td) + got, err := d.MarshalJSON() + if string(got) != tc.json || err != nil { + t.Errorf("MarshalJSON of %v = %v, %v; want %v, nil", d, string(got), err, tc.json) + } + } + } +} diff --git a/internal/serviceconfig/serviceconfig.go b/internal/serviceconfig/serviceconfig.go new file mode 100644 index 0000000..51e733e --- /dev/null +++ b/internal/serviceconfig/serviceconfig.go @@ -0,0 +1,180 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package serviceconfig contains utility functions to parse service config. +package serviceconfig + +import ( + "encoding/json" + "fmt" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + externalserviceconfig "google.golang.org/grpc/serviceconfig" +) + +var logger = grpclog.Component("core") + +// BalancerConfig wraps the name and config associated with one load balancing +// policy. It corresponds to a single entry of the loadBalancingConfig field +// from ServiceConfig. +// +// It implements the json.Unmarshaler interface. +// +// https://github.com/grpc/grpc-proto/blob/54713b1e8bc6ed2d4f25fb4dff527842150b91b2/grpc/service_config/service_config.proto#L247 +type BalancerConfig struct { + Name string + Config externalserviceconfig.LoadBalancingConfig +} + +type intermediateBalancerConfig []map[string]json.RawMessage + +// MarshalJSON implements the json.Marshaler interface. +// +// It marshals the balancer and config into a length-1 slice +// ([]map[string]config). +func (bc *BalancerConfig) MarshalJSON() ([]byte, error) { + if bc.Config == nil { + // If config is nil, return empty config `{}`. + return []byte(fmt.Sprintf(`[{%q: %v}]`, bc.Name, "{}")), nil + } + c, err := json.Marshal(bc.Config) + if err != nil { + return nil, err + } + return []byte(fmt.Sprintf(`[{%q: %s}]`, bc.Name, c)), nil +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +// +// ServiceConfig contains a list of loadBalancingConfigs, each with a name and +// config. This method iterates through that list in order, and stops at the +// first policy that is supported. +// - If the config for the first supported policy is invalid, the whole service +// config is invalid. +// - If the list doesn't contain any supported policy, the whole service config +// is invalid. +func (bc *BalancerConfig) UnmarshalJSON(b []byte) error { + var ir intermediateBalancerConfig + err := json.Unmarshal(b, &ir) + if err != nil { + return err + } + + var names []string + for i, lbcfg := range ir { + if len(lbcfg) != 1 { + return fmt.Errorf("invalid loadBalancingConfig: entry %v does not contain exactly 1 policy/config pair: %q", i, lbcfg) + } + + var ( + name string + jsonCfg json.RawMessage + ) + // Get the key:value pair from the map. We have already made sure that + // the map contains a single entry. + for name, jsonCfg = range lbcfg { + } + + names = append(names, name) + builder := balancer.Get(name) + if builder == nil { + // If the balancer is not registered, move on to the next config. + // This is not an error. + continue + } + bc.Name = name + + parser, ok := builder.(balancer.ConfigParser) + if !ok { + if string(jsonCfg) != "{}" { + logger.Warningf("non-empty balancer configuration %q, but balancer does not implement ParseConfig", string(jsonCfg)) + } + // Stop at this, though the builder doesn't support parsing config. + return nil + } + + cfg, err := parser.ParseConfig(jsonCfg) + if err != nil { + return fmt.Errorf("error parsing loadBalancingConfig for policy %q: %v", name, err) + } + bc.Config = cfg + return nil + } + // This is reached when the for loop iterates over all entries, but didn't + // return. This means we had a loadBalancingConfig slice but did not + // encounter a registered policy. The config is considered invalid in this + // case. + return fmt.Errorf("invalid loadBalancingConfig: no supported policies found in %v", names) +} + +// MethodConfig defines the configuration recommended by the service providers for a +// particular method. +type MethodConfig struct { + // WaitForReady indicates whether RPCs sent to this method should wait until + // the connection is ready by default (!failfast). The value specified via the + // gRPC client API will override the value set here. + WaitForReady *bool + // Timeout is the default timeout for RPCs sent to this method. The actual + // deadline used will be the minimum of the value specified here and the value + // set by the application via the gRPC client API. If either one is not set, + // then the other will be used. If neither is set, then the RPC has no deadline. + Timeout *time.Duration + // MaxReqSize is the maximum allowed payload size for an individual request in a + // stream (client->server) in bytes. The size which is measured is the serialized + // payload after per-message compression (but before stream compression) in bytes. + // The actual value used is the minimum of the value specified here and the value set + // by the application via the gRPC client API. If either one is not set, then the other + // will be used. If neither is set, then the built-in default is used. + MaxReqSize *int + // MaxRespSize is the maximum allowed payload size for an individual response in a + // stream (server->client) in bytes. + MaxRespSize *int + // RetryPolicy configures retry options for the method. + RetryPolicy *RetryPolicy +} + +// RetryPolicy defines the go-native version of the retry policy defined by the +// service config here: +// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#integration-with-service-config +type RetryPolicy struct { + // MaxAttempts is the maximum number of attempts, including the original RPC. + // + // This field is required and must be two or greater. + MaxAttempts int + + // Exponential backoff parameters. The initial retry attempt will occur at + // random(0, initialBackoff). In general, the nth attempt will occur at + // random(0, + // min(initialBackoff*backoffMultiplier**(n-1), maxBackoff)). + // + // These fields are required and must be greater than zero. + InitialBackoff time.Duration + MaxBackoff time.Duration + BackoffMultiplier float64 + + // The set of status codes which may be retried. + // + // Status codes are specified as strings, e.g., "UNAVAILABLE". + // + // This field is required and must be non-empty. + // Note: a set is used to store this for easy lookup. + RetryableStatusCodes map[codes.Code]bool +} diff --git a/internal/serviceconfig/serviceconfig_test.go b/internal/serviceconfig/serviceconfig_test.go new file mode 100644 index 0000000..3a72568 --- /dev/null +++ b/internal/serviceconfig/serviceconfig_test.go @@ -0,0 +1,182 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package serviceconfig + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + externalserviceconfig "google.golang.org/grpc/serviceconfig" +) + +type testBalancerConfigType struct { + externalserviceconfig.LoadBalancingConfig `json:"-"` + + Check bool `json:"check"` +} + +var testBalancerConfig = testBalancerConfigType{Check: true} + +const ( + testBalancerBuilderName = "test-bb" + testBalancerBuilderNotParserName = "test-bb-not-parser" + + testBalancerConfigJSON = `{"check":true}` +) + +type testBalancerBuilder struct { + balancer.Builder +} + +func (testBalancerBuilder) ParseConfig(js json.RawMessage) (externalserviceconfig.LoadBalancingConfig, error) { + if string(js) != testBalancerConfigJSON { + return nil, fmt.Errorf("unexpected config json") + } + return testBalancerConfig, nil +} + +func (testBalancerBuilder) Name() string { + return testBalancerBuilderName +} + +type testBalancerBuilderNotParser struct { + balancer.Builder +} + +func (testBalancerBuilderNotParser) Name() string { + return testBalancerBuilderNotParserName +} + +func init() { + balancer.Register(testBalancerBuilder{}) + balancer.Register(testBalancerBuilderNotParser{}) +} + +func TestBalancerConfigUnmarshalJSON(t *testing.T) { + tests := []struct { + name string + json string + want BalancerConfig + wantErr bool + }{ + { + name: "empty json", + json: "", + wantErr: true, + }, + { + // The config should be a slice of maps, but each map should have + // exactly one entry. + name: "more than one entry for a map", + json: `[{"balancer1":"1","balancer2":"2"}]`, + wantErr: true, + }, + { + name: "no balancer registered", + json: `[{"balancer1":"1"},{"balancer2":"2"}]`, + wantErr: true, + }, + { + name: "OK", + json: fmt.Sprintf("[{%q: %v}]", testBalancerBuilderName, testBalancerConfigJSON), + want: BalancerConfig{ + Name: testBalancerBuilderName, + Config: testBalancerConfig, + }, + wantErr: false, + }, + { + name: "first balancer not registered", + json: fmt.Sprintf(`[{"balancer1":"1"},{%q: %v}]`, testBalancerBuilderName, testBalancerConfigJSON), + want: BalancerConfig{ + Name: testBalancerBuilderName, + Config: testBalancerConfig, + }, + wantErr: false, + }, + { + name: "balancer registered but builder not parser", + json: fmt.Sprintf("[{%q: %v}]", testBalancerBuilderNotParserName, testBalancerConfigJSON), + want: BalancerConfig{ + Name: testBalancerBuilderNotParserName, + Config: nil, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var bc BalancerConfig + if err := bc.UnmarshalJSON([]byte(tt.json)); (err != nil) != tt.wantErr { + t.Errorf("UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + } + if !cmp.Equal(bc, tt.want) { + t.Errorf("diff: %v", cmp.Diff(bc, tt.want)) + } + }) + } +} + +func TestBalancerConfigMarshalJSON(t *testing.T) { + tests := []struct { + name string + bc BalancerConfig + wantJSON string + }{ + { + name: "OK", + bc: BalancerConfig{ + Name: testBalancerBuilderName, + Config: testBalancerConfig, + }, + wantJSON: `[{"test-bb": {"check":true}}]`, + }, + { + name: "OK config is nil", + bc: BalancerConfig{ + Name: testBalancerBuilderNotParserName, + Config: nil, // nil should be marshalled to an empty config "{}". + }, + wantJSON: `[{"test-bb-not-parser": {}}]`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, err := tt.bc.MarshalJSON() + if err != nil { + t.Fatalf("failed to marshal: %v", err) + } + + if str := string(b); str != tt.wantJSON { + t.Fatalf("got str %q, want %q", str, tt.wantJSON) + } + + var bc BalancerConfig + if err := bc.UnmarshalJSON(b); err != nil { + t.Errorf("failed to unmarshal: %v", err) + } + if !cmp.Equal(bc, tt.bc) { + t.Errorf("diff: %v", cmp.Diff(bc, tt.bc)) + } + }) + } +} diff --git a/internal/stats/labels.go b/internal/stats/labels.go new file mode 100644 index 0000000..5ea898c --- /dev/null +++ b/internal/stats/labels.go @@ -0,0 +1,74 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package stats provides internal stats related functionality. +package stats + +import ( + "context" + "maps" +) + +// LabelCallback is a function that is executed when telemetry +// label keys are updated. +type LabelCallback func(map[string]string) +type telemetryLabelCallbackKey struct{} + +// UpdateLabels executes registered telemetry callbacks with the update labels. Labels +// are copied before being processed by any callbacks to ensure mutations are not +// shared among derived contexts. +// +// It is the responsibility of the registrant to handle conflicts or label resets. +func UpdateLabels(ctx context.Context, update map[string]string) { + executeTelemetryLabelCallbacks(ctx, update) +} + +// RegisterTelemetryLabelCallback registers a callback function that is executed whenever +// telemetry labels are updated. +func RegisterTelemetryLabelCallback(ctx context.Context, callback LabelCallback) context.Context { + if callback == nil { + return ctx + } + + callbacks, ok := ctx.Value(telemetryLabelCallbackKey{}).([]LabelCallback) + if !ok { + return context.WithValue(ctx, telemetryLabelCallbackKey{}, []LabelCallback{callback}) + } + return context.WithValue(ctx, telemetryLabelCallbackKey{}, append(append([]LabelCallback(nil), callbacks...), callback)) + +} + +// executeTelemetryLabelCallback runs the registered callbacks in the order they were +// registered on the context with the provided labels. If no callbacks are registered +// it does nothing. +// +// To ensure callbacks do not mutate the state of the provided label map it is copied +// before execution. +func executeTelemetryLabelCallbacks(ctx context.Context, labels map[string]string) { + callbacks, ok := ctx.Value(telemetryLabelCallbackKey{}).([]LabelCallback) + if !ok { + return + } + + labelsCopy := map[string]string{} + maps.Copy(labelsCopy, labels) + for _, callback := range callbacks { + callback(labelsCopy) + } + +} diff --git a/internal/stats/labels_test.go b/internal/stats/labels_test.go new file mode 100644 index 0000000..bb06761 --- /dev/null +++ b/internal/stats/labels_test.go @@ -0,0 +1,216 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package stats_test + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + + "google.golang.org/grpc/internal/stats" +) + +// TestTelemetryLabelCallbacks tests registering a callback function with the context and +// the effects of executing the callback on a local label state tracker. Each test +// case constructs a new context with the provided callback registered. +func (s) TestTelemetryLabels(t *testing.T) { + commonLabelValues := map[string]string{"grpc.lb.backend_service": "grpc.lb.backend_service_val", "grpc.lb.locality": "grpc.lb.locality_val"} + + tests := map[string]struct { + callbacks func(tracker map[string]string) []func(map[string]string) + additionalLabels map[string]string + wantLabels map[string]string + }{ + "NilCallback": { + callbacks: func(map[string]string) []func(map[string]string) { + return []func(map[string]string){nil} + }, + wantLabels: map[string]string{}, + }, + "NoOPCallback": { + callbacks: func(map[string]string) []func(map[string]string) { + return []func(map[string]string){ + func(map[string]string) {}, + } + }, + wantLabels: map[string]string{}, + }, + "TrackingCallback": { + callbacks: func(tracker map[string]string) []func(map[string]string) { + return []func(map[string]string){ + func(u map[string]string) { + for key, value := range u { + tracker[key] = value + } + }, + } + }, + wantLabels: map[string]string{ + "grpc.lb.backend_service": "grpc.lb.backend_service_val", + "grpc.lb.locality": "grpc.lb.locality_val", + }, + }, + "MultipleInvocations": { + callbacks: func(tracker map[string]string) []func(map[string]string) { + return []func(map[string]string){ + func(u map[string]string) { + for key, value := range u { + tracker[key] = value + } + }, + } + }, + additionalLabels: map[string]string{"grpc.lb.other_label": "grpc.lb.other_val"}, + wantLabels: map[string]string{ + "grpc.lb.backend_service": "grpc.lb.backend_service_val", + "grpc.lb.locality": "grpc.lb.locality_val", + "grpc.lb.other_label": "grpc.lb.other_val", + }, + }, + "MultipleInvocationsWithSameLabel": { + callbacks: func(tracker map[string]string) []func(map[string]string) { + return []func(map[string]string){ + func(u map[string]string) { + for key, value := range u { + tracker[key] = value + } + }, + } + }, + additionalLabels: map[string]string{"grpc.lb.backend_service": "grpc.lb.backend_service_other_val"}, + wantLabels: map[string]string{ + "grpc.lb.backend_service": "grpc.lb.backend_service_other_val", + "grpc.lb.locality": "grpc.lb.locality_val", + }, + }, + "MultipleCallbacks": { + callbacks: func(tracker map[string]string) []func(map[string]string) { + return []func(map[string]string){ + func(u map[string]string) { + for key, value := range u { + k, v := key+"_first", value+"_first" + tracker[k] = v + } + }, + func(u map[string]string) { + for key, value := range u { + k, v := key+"_second", value+"_second" + tracker[k] = v + } + }, + } + }, + wantLabels: map[string]string{ + "grpc.lb.backend_service_first": "grpc.lb.backend_service_val_first", + "grpc.lb.locality_first": "grpc.lb.locality_val_first", + "grpc.lb.backend_service_second": "grpc.lb.backend_service_val_second", + "grpc.lb.locality_second": "grpc.lb.locality_val_second", + }, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + tracker := map[string]string{} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + for _, c := range test.callbacks(tracker) { + ctx = stats.RegisterTelemetryLabelCallback(ctx, c) + } + + stats.UpdateLabels(ctx, commonLabelValues) + stats.UpdateLabels(ctx, test.additionalLabels) + + if diff := cmp.Diff(tracker, test.wantLabels); diff != "" { + t.Fatalf("tracked labels did not match expected values (-got, +want): %v", diff) + } + }) + } +} + +// TestTelemetryLabelsMutation verifies that callbacks mutating labels do not +// affect the original map passed to UpdateLabels. +func (s) TestTelemetryLabelsMutation(t *testing.T) { + originalLabels := map[string]string{"key1": "val1", "key2": "val2"} + // Create a copy to compare against after calling UpdateLabels. + wantLabels := map[string]string{"key1": "val1", "key2": "val2"} + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + ctx = stats.RegisterTelemetryLabelCallback(ctx, func(u map[string]string) { + u["key1"] = "mutated1" + u["newkey"] = "newval" + }) + + stats.UpdateLabels(ctx, originalLabels) + + if diff := cmp.Diff(originalLabels, wantLabels); diff != "" { + t.Fatalf("original labels were mutated by callback (-got, +want): %v", diff) + } +} + +// TestTelemetryLabelsDerivedContexts verifies that creating derived contexts +// with additional callbacks does not override or affect sibling contexts. +func (s) TestTelemetryLabelsDerivedContexts(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + var baseExecuted, derived1Executed, derived2Executed uint + + ctxBase := stats.RegisterTelemetryLabelCallback(ctx, func(map[string]string) { + baseExecuted++ + }) + + ctxDerived1 := stats.RegisterTelemetryLabelCallback(ctxBase, func(map[string]string) { + derived1Executed++ + }) + + ctxDerived2 := stats.RegisterTelemetryLabelCallback(ctxBase, func(map[string]string) { + derived2Executed++ + }) + + labels := map[string]string{"k": "v"} + + // Run UpdateLabels to execute the telemetry callbacks on ctxDerived1 which should only increment + // the base and first derived context counters + stats.UpdateLabels(ctxDerived1, labels) + + if baseExecuted != 1 || derived1Executed != 1 || derived2Executed != 0 { + t.Errorf("UpdateLabels(ctxDerived1) executed: base=%v, derived1=%v, derived2=%v; want 1, 1, 0", baseExecuted, derived1Executed, derived2Executed) + } + + // Run UpdateLabels to execute the telemetry callbacks on ctxDerived2 which should only increment + // the base and second derived context counters + stats.UpdateLabels(ctxDerived2, labels) + + if baseExecuted != 2 || derived1Executed != 1 || derived2Executed != 1 { + t.Errorf("UpdateLabels(ctxDerived2) executed: base=%v, derived1=%v, derived2=%v; want 2, 1, 1", baseExecuted, derived1Executed, derived2Executed) + } + + // Run UpdateLabels to execute the telemetry callbacks on ctxBase which should only increment + // the base context counters + stats.UpdateLabels(ctxBase, labels) + + if baseExecuted != 3 || derived1Executed != 1 || derived2Executed != 1 { + t.Errorf("UpdateLabels(ctxDerived2) executed: base=%v, derived1=%v, derived2=%v; want 3, 1, 1", baseExecuted, derived1Executed, derived2Executed) + } +} diff --git a/internal/stats/metrics_recorder_list.go b/internal/stats/metrics_recorder_list.go new file mode 100644 index 0000000..1c8c2ab --- /dev/null +++ b/internal/stats/metrics_recorder_list.go @@ -0,0 +1,175 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stats + +import ( + "fmt" + + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/stats" +) + +// MetricsRecorderList forwards Record calls to all of its metricsRecorders. +// +// It eats any record calls where the label values provided do not match the +// number of label keys. +type MetricsRecorderList struct { + internal.EnforceMetricsRecorderEmbedding + // metricsRecorders are the metrics recorders this list will forward to. + metricsRecorders []estats.MetricsRecorder +} + +// NewMetricsRecorderList creates a new metric recorder list with all the stats +// handlers provided which implement the MetricsRecorder interface. +// If no stats handlers provided implement the MetricsRecorder interface, +// the MetricsRecorder list returned is a no-op. +func NewMetricsRecorderList(shs []stats.Handler) *MetricsRecorderList { + var mrs []estats.MetricsRecorder + for _, sh := range shs { + if mr, ok := sh.(estats.MetricsRecorder); ok { + mrs = append(mrs, mr) + } + } + return &MetricsRecorderList{ + metricsRecorders: mrs, + } +} + +func verifyLabels(desc *estats.MetricDescriptor, labelsRecv ...string) { + if got, want := len(labelsRecv), len(desc.Labels)+len(desc.OptionalLabels); got != want { + panic(fmt.Sprintf("Received %d labels in call to record metric %q, but expected %d.", got, desc.Name, want)) + } +} + +// RecordInt64Count records the measurement alongside labels on the int +// count associated with the provided handle. +func (l *MetricsRecorderList) RecordInt64Count(handle *estats.Int64CountHandle, incr int64, labels ...string) { + verifyLabels(handle.Descriptor(), labels...) + + for _, metricRecorder := range l.metricsRecorders { + metricRecorder.RecordInt64Count(handle, incr, labels...) + } +} + +// RecordInt64UpDownCount records the measurement alongside labels on the int +// count associated with the provided handle. +func (l *MetricsRecorderList) RecordInt64UpDownCount(handle *estats.Int64UpDownCountHandle, incr int64, labels ...string) { + verifyLabels(handle.Descriptor(), labels...) + + for _, metricRecorder := range l.metricsRecorders { + metricRecorder.RecordInt64UpDownCount(handle, incr, labels...) + } +} + +// RecordFloat64Count records the measurement alongside labels on the float +// count associated with the provided handle. +func (l *MetricsRecorderList) RecordFloat64Count(handle *estats.Float64CountHandle, incr float64, labels ...string) { + verifyLabels(handle.Descriptor(), labels...) + + for _, metricRecorder := range l.metricsRecorders { + metricRecorder.RecordFloat64Count(handle, incr, labels...) + } +} + +// RecordInt64Histo records the measurement alongside labels on the int +// histo associated with the provided handle. +func (l *MetricsRecorderList) RecordInt64Histo(handle *estats.Int64HistoHandle, incr int64, labels ...string) { + verifyLabels(handle.Descriptor(), labels...) + + for _, metricRecorder := range l.metricsRecorders { + metricRecorder.RecordInt64Histo(handle, incr, labels...) + } +} + +// RecordFloat64Histo records the measurement alongside labels on the float +// histo associated with the provided handle. +func (l *MetricsRecorderList) RecordFloat64Histo(handle *estats.Float64HistoHandle, incr float64, labels ...string) { + verifyLabels(handle.Descriptor(), labels...) + + for _, metricRecorder := range l.metricsRecorders { + metricRecorder.RecordFloat64Histo(handle, incr, labels...) + } +} + +// RecordInt64Gauge records the measurement alongside labels on the int +// gauge associated with the provided handle. +func (l *MetricsRecorderList) RecordInt64Gauge(handle *estats.Int64GaugeHandle, incr int64, labels ...string) { + verifyLabels(handle.Descriptor(), labels...) + + for _, metricRecorder := range l.metricsRecorders { + metricRecorder.RecordInt64Gauge(handle, incr, labels...) + } +} + +// RegisterAsyncReporter forwards the registration to all underlying metrics +// recorders. +// +// It returns a cleanup function that, when called, invokes the cleanup function +// returned by each underlying recorder, ensuring the reporter is unregistered +// from all of them. +func (l *MetricsRecorderList) RegisterAsyncReporter(reporter estats.AsyncMetricReporter, metrics ...estats.AsyncMetric) func() { + descriptorsMap := make(map[*estats.MetricDescriptor]bool, len(metrics)) + for _, m := range metrics { + descriptorsMap[m.Descriptor()] = true + } + unregisterFns := make([]func(), 0, len(l.metricsRecorders)) + for _, mr := range l.metricsRecorders { + // Wrap the AsyncMetricsRecorder to intercept calls to RecordInt64Gauge + // and validate the labels. + wrappedCallback := func(recorder estats.AsyncMetricsRecorder) error { + wrappedRecorder := &asyncRecorderWrapper{ + delegate: recorder, + descriptors: descriptorsMap, + } + return reporter.Report(wrappedRecorder) + } + unregisterFns = append(unregisterFns, mr.RegisterAsyncReporter(estats.AsyncMetricReporterFunc(wrappedCallback), metrics...)) + } + + // Wrap the cleanup function using the internal delegate. + // In production, this returns realCleanup as-is. + // In tests, the leak checker can swap this to track the registration lifetime. + return internal.AsyncReporterCleanupDelegate(defaultCleanUp(unregisterFns)) +} + +func defaultCleanUp(unregisterFns []func()) func() { + return func() { + for _, unregister := range unregisterFns { + unregister() + } + } +} + +type asyncRecorderWrapper struct { + delegate estats.AsyncMetricsRecorder + descriptors map[*estats.MetricDescriptor]bool +} + +// RecordIntAsync64Gauge records the measurement alongside labels on the int +// gauge associated with the provided handle. +func (w *asyncRecorderWrapper) RecordInt64AsyncGauge(handle *estats.Int64AsyncGaugeHandle, value int64, labels ...string) { + // Ensure only metrics for descriptors passed during callback registration + // are emitted. + d := handle.Descriptor() + if _, ok := w.descriptors[d]; !ok { + return + } + // Validate labels and delegate. + verifyLabels(d, labels...) + w.delegate.RecordInt64AsyncGauge(handle, value, labels...) +} diff --git a/internal/stats/metrics_recorder_list_test.go b/internal/stats/metrics_recorder_list_test.go new file mode 100644 index 0000000..614f5fc --- /dev/null +++ b/internal/stats/metrics_recorder_list_test.go @@ -0,0 +1,336 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package stats_test implements an e2e test for the Metrics Recorder List +// component of the Client Conn. +package stats_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/credentials/insecure" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + istats "google.golang.org/grpc/internal/stats" + "google.golang.org/grpc/internal/testutils/stats" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + gstats "google.golang.org/grpc/stats" +) + +var defaultTestTimeout = 5 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +var ( + intCountHandle = estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "simple counter", + Description: "sum of all emissions from tests", + Unit: "int", + Labels: []string{"int counter label"}, + OptionalLabels: []string{"int counter optional label"}, + Default: false, + }) + floatCountHandle = estats.RegisterFloat64Count(estats.MetricDescriptor{ + Name: "float counter", + Description: "sum of all emissions from tests", + Unit: "float", + Labels: []string{"float counter label"}, + OptionalLabels: []string{"float counter optional label"}, + Default: false, + }) + intHistoHandle = estats.RegisterInt64Histo(estats.MetricDescriptor{ + Name: "int histo", + Description: "sum of all emissions from tests", + Unit: "int", + Labels: []string{"int histo label"}, + OptionalLabels: []string{"int histo optional label"}, + Default: false, + }) + floatHistoHandle = estats.RegisterFloat64Histo(estats.MetricDescriptor{ + Name: "float histo", + Description: "sum of all emissions from tests", + Unit: "float", + Labels: []string{"float histo label"}, + OptionalLabels: []string{"float histo optional label"}, + Default: false, + }) + intGaugeHandle = estats.RegisterInt64Gauge(estats.MetricDescriptor{ + Name: "simple gauge", + Description: "the most recent int emitted by test", + Unit: "int", + Labels: []string{"int gauge label"}, + OptionalLabels: []string{"int gauge optional label"}, + Default: false, + }) +) + +func init() { + balancer.Register(recordingLoadBalancerBuilder{}) +} + +const recordingLoadBalancerName = "recording_load_balancer" + +type recordingLoadBalancerBuilder struct{} + +func (recordingLoadBalancerBuilder) Name() string { + return recordingLoadBalancerName +} + +func (recordingLoadBalancerBuilder) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + intCountHandle.Record(cc.MetricsRecorder(), 1, "int counter label val", "int counter optional label val") + floatCountHandle.Record(cc.MetricsRecorder(), 2, "float counter label val", "float counter optional label val") + intHistoHandle.Record(cc.MetricsRecorder(), 3, "int histo label val", "int histo optional label val") + floatHistoHandle.Record(cc.MetricsRecorder(), 4, "float histo label val", "float histo optional label val") + intGaugeHandle.Record(cc.MetricsRecorder(), 5, "int gauge label val", "int gauge optional label val") + return &recordingLoadBalancer{ + Balancer: balancer.Get(pickfirst.Name).Build(cc, bOpts), + } +} + +type recordingLoadBalancer struct { + balancer.Balancer +} + +// TestMetricsRecorderList tests the metrics recorder list functionality of the +// ClientConn. It configures a global and local stats handler Dial Option. These +// stats handlers implement the MetricsRecorder interface. It also configures a +// balancer which registers metrics and records on metrics at build time. This +// test then asserts that the recorded metrics show up on both configured stats +// handlers. +func (s) TestMetricsRecorderList(t *testing.T) { + cleanup := internal.SnapshotMetricRegistryForTesting() + defer cleanup() + + mr := manual.NewBuilderWithScheme("test-metrics-recorder-list") + defer mr.Close() + + json := `{"loadBalancingConfig": [{"recording_load_balancer":{}}]}` + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(json) + mr.InitialState(resolver.State{ + ServiceConfig: sc, + }) + + // Create two stats.Handlers which also implement MetricsRecorder, configure + // one as a global dial option and one as a local dial option. + mr1 := stats.NewTestMetricsRecorder() + mr2 := stats.NewTestMetricsRecorder() + + defer internal.ClearGlobalDialOptions() + internal.AddGlobalDialOptions.(func(opt ...grpc.DialOption))(grpc.WithStatsHandler(mr1)) + + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithStatsHandler(mr2)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + tsc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Trigger the recording_load_balancer to build, which will trigger metrics + // to record. + tsc.UnaryCall(ctx, &testpb.SimpleRequest{}) + + mdWant := stats.MetricsData{ + Handle: intCountHandle.Descriptor(), + IntIncr: 1, + LabelKeys: []string{"int counter label", "int counter optional label"}, + LabelVals: []string{"int counter label val", "int counter optional label val"}, + } + if err := mr1.WaitForInt64Count(ctx, mdWant); err != nil { + t.Fatal(err.Error()) + } + if err := mr2.WaitForInt64Count(ctx, mdWant); err != nil { + t.Fatal(err.Error()) + } + + mdWant = stats.MetricsData{ + Handle: floatCountHandle.Descriptor(), + FloatIncr: 2, + LabelKeys: []string{"float counter label", "float counter optional label"}, + LabelVals: []string{"float counter label val", "float counter optional label val"}, + } + if err := mr1.WaitForFloat64Count(ctx, mdWant); err != nil { + t.Fatal(err.Error()) + } + if err := mr2.WaitForFloat64Count(ctx, mdWant); err != nil { + t.Fatal(err.Error()) + } + + mdWant = stats.MetricsData{ + Handle: intHistoHandle.Descriptor(), + IntIncr: 3, + LabelKeys: []string{"int histo label", "int histo optional label"}, + LabelVals: []string{"int histo label val", "int histo optional label val"}, + } + if err := mr1.WaitForInt64Histo(ctx, mdWant); err != nil { + t.Fatal(err.Error()) + } + if err := mr2.WaitForInt64Histo(ctx, mdWant); err != nil { + t.Fatal(err.Error()) + } + + mdWant = stats.MetricsData{ + Handle: floatHistoHandle.Descriptor(), + FloatIncr: 4, + LabelKeys: []string{"float histo label", "float histo optional label"}, + LabelVals: []string{"float histo label val", "float histo optional label val"}, + } + if err := mr1.WaitForFloat64Histo(ctx, mdWant); err != nil { + t.Fatal(err.Error()) + } + if err := mr2.WaitForFloat64Histo(ctx, mdWant); err != nil { + t.Fatal(err.Error()) + } + mdWant = stats.MetricsData{ + Handle: intGaugeHandle.Descriptor(), + IntIncr: 5, + LabelKeys: []string{"int gauge label", "int gauge optional label"}, + LabelVals: []string{"int gauge label val", "int gauge optional label val"}, + } + if err := mr1.WaitForInt64Gauge(ctx, mdWant); err != nil { + t.Fatal(err.Error()) + } + if err := mr2.WaitForInt64Gauge(ctx, mdWant); err != nil { + t.Fatal(err.Error()) + } +} + +// TestMetricRecorderListPanic tests that the metrics recorder list panics if +// received the wrong number of labels for a particular metric. +func (s) TestMetricRecorderListPanic(t *testing.T) { + cleanup := internal.SnapshotMetricRegistryForTesting() + defer cleanup() + + intCountHandle := estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "simple counter", + Description: "sum of all emissions from tests", + Unit: "int", + Labels: []string{"int counter label"}, + OptionalLabels: []string{"int counter optional label"}, + Default: false, + }) + mrl := istats.NewMetricsRecorderList(nil) + + want := `Received 1 labels in call to record metric "simple counter", but expected 2.` + defer func() { + if r := recover(); !strings.Contains(fmt.Sprint(r), want) { + t.Errorf("expected panic contains %q, got %q", want, r) + } + }() + + intCountHandle.Record(mrl, 1, "only one label") +} + +// TestMetricsRecorderList_RegisterAsyncReporter verifies that the list implementation +// correctly fans out registration calls to all underlying recorders and +// aggregates the cleanup calls. +func (s) TestMetricsRecorderList_RegisterAsyncReporter(t *testing.T) { + spy1 := &spyMetricsRecorder{name: "spy1"} + spy2 := &spyMetricsRecorder{name: "spy2"} + spy3 := &spyMetricsRecorder{name: "spy3"} + + list := istats.NewMetricsRecorderList([]gstats.Handler{spy1, spy2, spy3}) + + desc := &estats.MetricDescriptor{Name: "test_metric", Description: "test"} + mockMetric := &mockAsyncMetric{d: desc} + + dummyReporter := estats.AsyncMetricReporterFunc(func(estats.AsyncMetricsRecorder) error { + return nil + }) + cleanup := list.RegisterAsyncReporter(dummyReporter, mockMetric) + + // Check that RegisterAsyncReporter was called exactly once on ALL spies + if spy1.registerCalledCount != 1 { + t.Errorf("spy1 register called %d times, want 1", spy1.registerCalledCount) + } + if spy2.registerCalledCount != 1 { + t.Errorf("spy2 register called %d times, want 1", spy2.registerCalledCount) + } + if spy3.registerCalledCount != 1 { + t.Errorf("spy3 register called %d times, want 1", spy3.registerCalledCount) + } + + // Verify that cleanup has NOT been called yet + if spy1.cleanupCalledCount != 0 { + t.Error("spy1 cleanup called prematurely") + } + + cleanup() + + // Check that the cleanup function returned by the list actually triggers + // the cleanup logic on ALL underlying spies. + if spy1.cleanupCalledCount != 1 { + t.Errorf("spy1 cleanup called %d times, want 1", spy1.cleanupCalledCount) + } + if spy2.cleanupCalledCount != 1 { + t.Errorf("spy2 cleanup called %d times, want 1", spy2.cleanupCalledCount) + } + if spy3.cleanupCalledCount != 1 { + t.Errorf("spy3 cleanup called %d times, want 1", spy3.cleanupCalledCount) + } +} + +// --- Helpers & Spies --- + +// mockAsyncMetric implements estats.AsyncMetric +type mockAsyncMetric struct { + estats.AsyncMetric + d *estats.MetricDescriptor +} + +func (m *mockAsyncMetric) Descriptor() *estats.MetricDescriptor { + return m.d +} + +// spyMetricsRecorder implements estats.MetricsRecorder +type spyMetricsRecorder struct { + stats.TestMetricsRecorder + name string + registerCalledCount int + cleanupCalledCount int +} + +// RegisterAsyncReporter implements the interface and tracks calls. +func (s *spyMetricsRecorder) RegisterAsyncReporter(estats.AsyncMetricReporter, ...estats.AsyncMetric) func() { + s.registerCalledCount++ + + // Return a cleanup function that tracks if it was called + return func() { + s.cleanupCalledCount++ + } +} diff --git a/internal/stats/stats.go b/internal/stats/stats.go new file mode 100644 index 0000000..49019b8 --- /dev/null +++ b/internal/stats/stats.go @@ -0,0 +1,70 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package stats + +import ( + "context" + + "google.golang.org/grpc/stats" +) + +type combinedHandler struct { + handlers []stats.Handler +} + +// NewCombinedHandler combines multiple stats.Handlers into a single handler. +// +// It returns nil if no handlers are provided. If only one handler is +// provided, it is returned directly without wrapping. +func NewCombinedHandler(handlers ...stats.Handler) stats.Handler { + switch len(handlers) { + case 0: + return nil + case 1: + return handlers[0] + default: + return &combinedHandler{handlers: handlers} + } +} + +func (ch *combinedHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + for _, h := range ch.handlers { + ctx = h.TagRPC(ctx, info) + } + return ctx +} + +func (ch *combinedHandler) HandleRPC(ctx context.Context, stats stats.RPCStats) { + for _, h := range ch.handlers { + h.HandleRPC(ctx, stats) + } +} + +func (ch *combinedHandler) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context { + for _, h := range ch.handlers { + ctx = h.TagConn(ctx, info) + } + return ctx +} + +func (ch *combinedHandler) HandleConn(ctx context.Context, stats stats.ConnStats) { + for _, h := range ch.handlers { + h.HandleConn(ctx, stats) + } +} diff --git a/internal/status/status.go b/internal/status/status.go new file mode 100644 index 0000000..aad171c --- /dev/null +++ b/internal/status/status.go @@ -0,0 +1,246 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package status implements errors returned by gRPC. These errors are +// serialized and transmitted on the wire between server and client, and allow +// for additional data to be transmitted via the Details field in the status +// proto. gRPC service handlers should return an error created by this +// package, and gRPC clients should expect a corresponding error to be +// returned from the RPC call. +// +// This package upholds the invariants that a non-nil error may not +// contain an OK code, and an OK code must result in a nil error. +package status + +import ( + "errors" + "fmt" + + spb "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/protoadapt" + "google.golang.org/protobuf/types/known/anypb" +) + +// Status represents an RPC status code, message, and details. It is immutable +// and should be created with New, Newf, or FromProto. +type Status struct { + s *spb.Status +} + +// NewWithProto returns a new status including details from statusProto. This +// is meant to be used by the gRPC library only. +func NewWithProto(code codes.Code, message string, statusProto []string) *Status { + if len(statusProto) != 1 { + // No grpc-status-details bin header, or multiple; just ignore. + return &Status{s: &spb.Status{Code: int32(code), Message: message}} + } + st := &spb.Status{} + if err := proto.Unmarshal([]byte(statusProto[0]), st); err != nil { + // Probably not a google.rpc.Status proto; do not provide details. + return &Status{s: &spb.Status{Code: int32(code), Message: message}} + } + if st.Code == int32(code) { + // The codes match between the grpc-status header and the + // grpc-status-details-bin header; use the full details proto. + return &Status{s: st} + } + return &Status{ + s: &spb.Status{ + Code: int32(codes.Internal), + Message: fmt.Sprintf( + "grpc-status-details-bin mismatch: grpc-status=%v, grpc-message=%q, grpc-status-details-bin=%+v", + code, message, st, + ), + }, + } +} + +// New returns a Status representing c and msg. +func New(c codes.Code, msg string) *Status { + return &Status{s: &spb.Status{Code: int32(c), Message: msg}} +} + +// Newf returns New(c, fmt.Sprintf(format, a...)). +func Newf(c codes.Code, format string, a ...any) *Status { + return New(c, fmt.Sprintf(format, a...)) +} + +// FromProto returns a Status representing s. +func FromProto(s *spb.Status) *Status { + return &Status{s: proto.Clone(s).(*spb.Status)} +} + +// Err returns an error representing c and msg. If c is OK, returns nil. +func Err(c codes.Code, msg string) error { + return New(c, msg).Err() +} + +// Errorf returns Error(c, fmt.Sprintf(format, a...)). +func Errorf(c codes.Code, format string, a ...any) error { + return Err(c, fmt.Sprintf(format, a...)) +} + +// Code returns the status code contained in s. +func (s *Status) Code() codes.Code { + if s == nil || s.s == nil { + return codes.OK + } + return codes.Code(s.s.Code) +} + +// Message returns the message contained in s. +func (s *Status) Message() string { + if s == nil || s.s == nil { + return "" + } + return s.s.Message +} + +// Proto returns s's status as an spb.Status proto message. +func (s *Status) Proto() *spb.Status { + if s == nil { + return nil + } + return proto.Clone(s.s).(*spb.Status) +} + +// Err returns an immutable error representing s; returns nil if s.Code() is OK. +func (s *Status) Err() error { + if s.Code() == codes.OK { + return nil + } + return &Error{s: s} +} + +// WithDetails returns a new status with the provided details messages appended to the status. +// If any errors are encountered, it returns nil and the first error encountered. +func (s *Status) WithDetails(details ...protoadapt.MessageV1) (*Status, error) { + if s.Code() == codes.OK { + return nil, errors.New("no error details for status with code OK") + } + // s.Code() != OK implies that s.Proto() != nil. + p := s.Proto() + for _, detail := range details { + m, err := anypb.New(protoadapt.MessageV2Of(detail)) + if err != nil { + return nil, err + } + p.Details = append(p.Details, m) + } + return &Status{s: p}, nil +} + +// Details returns a slice of details messages attached to the status. +// If a detail cannot be decoded, the error is returned in place of the detail. +// If the detail can be decoded, the proto message returned is of the same +// type that was given to WithDetails(). +func (s *Status) Details() []any { + if s == nil || s.s == nil { + return nil + } + details := make([]any, 0, len(s.s.Details)) + for _, any := range s.s.Details { + detail, err := any.UnmarshalNew() + if err != nil { + details = append(details, err) + continue + } + // The call to MessageV1Of is required to unwrap the proto message if + // it implemented only the MessageV1 API. The proto message would have + // been wrapped in a V2 wrapper in Status.WithDetails. V2 messages are + // added to a global registry used by any.UnmarshalNew(). + // MessageV1Of has the following behaviour: + // 1. If the given message is a wrapped MessageV1, it returns the + // unwrapped value. + // 2. If the given message already implements MessageV1, it returns it + // as is. + // 3. Else, it wraps the MessageV2 in a MessageV1 wrapper. + // + // Since the Status.WithDetails() API only accepts MessageV1, calling + // MessageV1Of ensures we return the same type that was given to + // WithDetails: + // * If the give type implemented only MessageV1, the unwrapping from + // point 1 above will restore the type. + // * If the given type implemented both MessageV1 and MessageV2, point 2 + // above will ensure no wrapping is performed. + // * If the given type implemented only MessageV2 and was wrapped using + // MessageV1Of before passing to WithDetails(), it would be unwrapped + // in WithDetails by calling MessageV2Of(). Point 3 above will ensure + // that the type is wrapped in a MessageV1 wrapper again before + // returning. Note that protoc-gen-go doesn't generate code which + // implements ONLY MessageV2 at the time of writing. + // + // NOTE: Status details can also be added using the FromProto method. + // This could theoretically allow passing a Detail message that only + // implements the V2 API. In such a case the message will be wrapped in + // a MessageV1 wrapper when fetched using Details(). + // Since protoc-gen-go generates only code that implements both V1 and + // V2 APIs for backward compatibility, this is not a concern. + details = append(details, protoadapt.MessageV1Of(detail)) + } + return details +} + +func (s *Status) String() string { + return fmt.Sprintf("rpc error: code = %s desc = %s", s.Code(), s.Message()) +} + +// Error wraps a pointer of a status proto. It implements error and Status, +// and a nil *Error should never be returned by this package. +type Error struct { + s *Status +} + +func (e *Error) Error() string { + return e.s.String() +} + +// GRPCStatus returns the Status represented by se. +func (e *Error) GRPCStatus() *Status { + return e.s +} + +// Is implements future error.Is functionality. +// A Error is equivalent if the code and message are identical. +func (e *Error) Is(target error) bool { + tse, ok := target.(*Error) + if !ok { + return false + } + return proto.Equal(e.s.s, tse.s.s) +} + +// IsRestrictedControlPlaneCode returns whether the status includes a code +// restricted for control plane usage as defined by gRFC A54. +func IsRestrictedControlPlaneCode(s *Status) bool { + switch s.Code() { + case codes.InvalidArgument, codes.NotFound, codes.AlreadyExists, codes.FailedPrecondition, codes.Aborted, codes.OutOfRange, codes.DataLoss: + return true + } + return false +} + +// RawStatusProto returns the internal protobuf message for use by gRPC itself. +func RawStatusProto(s *Status) *spb.Status { + if s == nil { + return nil + } + return s.s +} diff --git a/internal/status/status_test.go b/internal/status/status_test.go new file mode 100644 index 0000000..1d3fc4c --- /dev/null +++ b/internal/status/status_test.go @@ -0,0 +1,40 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package status_test + +import ( + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/status" +) + +func TestRawStatusProto(t *testing.T) { + spb := status.RawStatusProto(nil) + if spb != nil { + t.Errorf("RawStatusProto(nil) = %v; must return nil", spb) + } + s := status.New(codes.Internal, "test internal error") + spb1 := status.RawStatusProto(s) + spb2 := status.RawStatusProto(s) + // spb1 and spb2 should be the same pointer: no copies + if spb1 != spb2 { + t.Errorf("RawStatusProto(s)=%p then %p; must return the same pointer", spb1, spb2) + } +} diff --git a/internal/stubserver/stubserver.go b/internal/stubserver/stubserver.go new file mode 100644 index 0000000..9974d21 --- /dev/null +++ b/internal/stubserver/stubserver.go @@ -0,0 +1,302 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package stubserver is a stubbable implementation of +// google.golang.org/grpc/interop/grpc_testing for testing purposes. +package stubserver + +import ( + "context" + "fmt" + "net" + "net/http" + "testing" + "time" + + "golang.org/x/net/http2" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// GRPCServer is an interface that groups methods implemented by a grpc.Server +// or an xds.GRPCServer that are used by the StubServer. +type GRPCServer interface { + grpc.ServiceRegistrar + Stop() + GracefulStop() + Serve(net.Listener) error +} + +// StubServer is a server that is easy to customize within individual test +// cases. +type StubServer struct { + // Guarantees we satisfy this interface; panics if unimplemented methods are called. + testgrpc.TestServiceServer + + // Customizable implementations of server handlers. + EmptyCallF func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) + UnaryCallF func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) + FullDuplexCallF func(stream testgrpc.TestService_FullDuplexCallServer) error + StreamingInputCallF func(stream testgrpc.TestService_StreamingInputCallServer) error + StreamingOutputCallF func(req *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error + + // A client connected to this service the test may use. Created in Start(). + Client testgrpc.TestServiceClient + CC *grpc.ClientConn + + // Server to serve this service from. + // + // If nil, a new grpc.Server is created, listening on the provided Network + // and Address fields, or listening using the provided Listener. + S GRPCServer + + // Parameters for Listen and Dial. Defaults will be used if these are empty + // before Start. + Network string + Address string + Target string + + // Custom listener to use for serving. If unspecified, a new listener is + // created on a local port. + Listener net.Listener + + cleanups []func() // Lambdas executed in Stop(); populated by Start(). + + // Set automatically if Target == "" + R *manual.Resolver +} + +// EmptyCall is the handler for testpb.EmptyCall +func (ss *StubServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return ss.EmptyCallF(ctx, in) +} + +// UnaryCall is the handler for testpb.UnaryCall +func (ss *StubServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return ss.UnaryCallF(ctx, in) +} + +// FullDuplexCall is the handler for testpb.FullDuplexCall +func (ss *StubServer) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error { + return ss.FullDuplexCallF(stream) +} + +// StreamingInputCall is the handler for testpb.StreamingInputCall +func (ss *StubServer) StreamingInputCall(stream testgrpc.TestService_StreamingInputCallServer) error { + return ss.StreamingInputCallF(stream) +} + +// StreamingOutputCall is the handler for testpb.StreamingOutputCall +func (ss *StubServer) StreamingOutputCall(req *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error { + return ss.StreamingOutputCallF(req, stream) +} + +// Start starts the server and creates a client connected to it. +func (ss *StubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) error { + if err := ss.StartServer(sopts...); err != nil { + return err + } + if err := ss.StartClient(dopts...); err != nil { + ss.Stop() + return err + } + return nil +} + +type registerServiceServerOption struct { + grpc.EmptyServerOption + f func(grpc.ServiceRegistrar) +} + +// RegisterServiceServerOption returns a ServerOption that will run f() in +// Start or StartServer with the grpc.Server created before serving. This +// allows other services to be registered on the test server (e.g. ORCA, +// health, or reflection). +func RegisterServiceServerOption(f func(grpc.ServiceRegistrar)) grpc.ServerOption { + return ®isterServiceServerOption{f: f} +} + +func (ss *StubServer) setupServer(sopts ...grpc.ServerOption) (net.Listener, error) { + if ss.Network == "" { + ss.Network = "tcp" + } + if ss.Address == "" { + ss.Address = "localhost:0" + } + if ss.Target == "" { + ss.R = manual.NewBuilderWithScheme("whatever") + } + + lis := ss.Listener + if lis == nil { + var err error + lis, err = net.Listen(ss.Network, ss.Address) + if err != nil { + return nil, fmt.Errorf("net.Listen(%q, %q) = %v", ss.Network, ss.Address, err) + } + } + ss.Address = lis.Addr().String() + + if ss.S == nil { + ss.S = grpc.NewServer(sopts...) + } + for _, so := range sopts { + if x, ok := so.(*registerServiceServerOption); ok { + x.f(ss.S) + } + } + + testgrpc.RegisterTestServiceServer(ss.S, ss) + ss.cleanups = append(ss.cleanups, ss.S.Stop) + return lis, nil +} + +// StartHandlerServer only starts an HTTP server with a gRPC server as the +// handler. It does not create a client to it. Cannot be used in a StubServer +// that also used StartServer. +func (ss *StubServer) StartHandlerServer(sopts ...grpc.ServerOption) error { + lis, err := ss.setupServer(sopts...) + if err != nil { + return err + } + + handler, ok := ss.S.(interface{ http.Handler }) + if !ok { + panic(fmt.Sprintf("server of type %T does not implement http.Handler", ss.S)) + } + + go func() { + hs := &http2.Server{} + opts := &http2.ServeConnOpts{Handler: handler} + for { + conn, err := lis.Accept() + if err != nil { + return + } + hs.ServeConn(conn, opts) + } + }() + ss.cleanups = append(ss.cleanups, func() { lis.Close() }) + + return nil +} + +// StartServer only starts the server. It does not create a client to it. +// Cannot be used in a StubServer that also used StartHandlerServer. +func (ss *StubServer) StartServer(sopts ...grpc.ServerOption) error { + lis, err := ss.setupServer(sopts...) + if err != nil { + return err + } + + go ss.S.Serve(lis) + + return nil +} + +// StartClient creates a client connected to this service that the test may use. +// The newly created client will be available in the Client field of StubServer. +func (ss *StubServer) StartClient(dopts ...grpc.DialOption) error { + opts := append([]grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}, dopts...) + if ss.R != nil { + ss.Target = ss.R.Scheme() + ":///" + ss.Address + opts = append(opts, grpc.WithResolvers(ss.R)) + } + + cc, err := grpc.NewClient(ss.Target, opts...) + if err != nil { + return fmt.Errorf("grpc.NewClient(%q) = %v", ss.Target, err) + } + cc.Connect() + ss.CC = cc + if ss.R != nil { + ss.R.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}}) + } + if err := waitForReady(cc); err != nil { + cc.Close() + return err + } + + ss.cleanups = append(ss.cleanups, func() { cc.Close() }) + + ss.Client = testgrpc.NewTestServiceClient(cc) + return nil +} + +// NewServiceConfig applies sc to ss.Client using the resolver (if present). +func (ss *StubServer) NewServiceConfig(sc string) { + if ss.R != nil { + ss.R.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}, ServiceConfig: parseCfg(ss.R, sc)}) + } +} + +func waitForReady(cc *grpc.ClientConn) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for { + s := cc.GetState() + if s == connectivity.Ready { + return nil + } + if !cc.WaitForStateChange(ctx, s) { + // ctx got timeout or canceled. + return ctx.Err() + } + } +} + +// Stop stops ss and cleans up all resources it consumed. +func (ss *StubServer) Stop() { + for i := len(ss.cleanups) - 1; i >= 0; i-- { + ss.cleanups[i]() + } + ss.cleanups = nil +} + +func parseCfg(r *manual.Resolver, s string) *serviceconfig.ParseResult { + g := r.CC().ParseServiceConfig(s) + if g.Err != nil { + panic(fmt.Sprintf("Error parsing config %q: %v", s, g.Err)) + } + return g +} + +// StartTestService spins up a stub server exposing the TestService on a local +// port. If the passed in server is nil, a stub server that implements only the +// EmptyCall and UnaryCall RPCs is started. +func StartTestService(t *testing.T, server *StubServer, sopts ...grpc.ServerOption) *StubServer { + if server == nil { + server = &StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + } + } + server.StartServer(sopts...) + + t.Logf("Started test service backend at %q", server.Address) + return server +} diff --git a/internal/syscall/syscall_linux.go b/internal/syscall/syscall_linux.go new file mode 100644 index 0000000..b3a7227 --- /dev/null +++ b/internal/syscall/syscall_linux.go @@ -0,0 +1,112 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package syscall provides functionalities that grpc uses to get low-level operating system +// stats/info. +package syscall + +import ( + "fmt" + "net" + "syscall" + "time" + + "golang.org/x/sys/unix" + "google.golang.org/grpc/grpclog" +) + +var logger = grpclog.Component("core") + +// GetCPUTime returns the how much CPU time has passed since the start of this process. +func GetCPUTime() int64 { + var ts unix.Timespec + if err := unix.ClockGettime(unix.CLOCK_PROCESS_CPUTIME_ID, &ts); err != nil { + logger.Fatal(err) + } + return ts.Nano() +} + +// Rusage is an alias for syscall.Rusage under linux environment. +type Rusage = syscall.Rusage + +// GetRusage returns the resource usage of current process. +func GetRusage() *Rusage { + rusage := new(Rusage) + syscall.Getrusage(syscall.RUSAGE_SELF, rusage) + return rusage +} + +// CPUTimeDiff returns the differences of user CPU time and system CPU time used +// between two Rusage structs. +func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) { + var ( + utimeDiffs = latest.Utime.Sec - first.Utime.Sec + utimeDiffus = latest.Utime.Usec - first.Utime.Usec + stimeDiffs = latest.Stime.Sec - first.Stime.Sec + stimeDiffus = latest.Stime.Usec - first.Stime.Usec + ) + + uTimeElapsed := float64(utimeDiffs) + float64(utimeDiffus)*1.0e-6 + sTimeElapsed := float64(stimeDiffs) + float64(stimeDiffus)*1.0e-6 + + return uTimeElapsed, sTimeElapsed +} + +// SetTCPUserTimeout sets the TCP user timeout on a connection's socket +func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error { + tcpconn, ok := conn.(*net.TCPConn) + if !ok { + // not a TCP connection. exit early + return nil + } + rawConn, err := tcpconn.SyscallConn() + if err != nil { + return fmt.Errorf("error getting raw connection: %v", err) + } + err = rawConn.Control(func(fd uintptr) { + err = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, int(timeout/time.Millisecond)) + }) + if err != nil { + return fmt.Errorf("error setting option on socket: %v", err) + } + + return nil +} + +// GetTCPUserTimeout gets the TCP user timeout on a connection's socket +func GetTCPUserTimeout(conn net.Conn) (opt int, err error) { + tcpconn, ok := conn.(*net.TCPConn) + if !ok { + err = fmt.Errorf("conn is not *net.TCPConn. got %T", conn) + return + } + rawConn, err := tcpconn.SyscallConn() + if err != nil { + err = fmt.Errorf("error getting raw connection: %v", err) + return + } + err = rawConn.Control(func(fd uintptr) { + opt, err = syscall.GetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT) + }) + if err != nil { + err = fmt.Errorf("error getting option on socket: %v", err) + return + } + + return +} diff --git a/internal/syscall/syscall_nonlinux.go b/internal/syscall/syscall_nonlinux.go new file mode 100644 index 0000000..54c24c2 --- /dev/null +++ b/internal/syscall/syscall_nonlinux.go @@ -0,0 +1,77 @@ +//go:build !linux +// +build !linux + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package syscall provides functionalities that grpc uses to get low-level +// operating system stats/info. +package syscall + +import ( + "net" + "sync" + "time" + + "google.golang.org/grpc/grpclog" +) + +var once sync.Once +var logger = grpclog.Component("core") + +func log() { + once.Do(func() { + logger.Info("CPU time info is unavailable on non-linux environments.") + }) +} + +// GetCPUTime returns the how much CPU time has passed since the start of this +// process. It always returns 0 under non-linux environments. +func GetCPUTime() int64 { + log() + return 0 +} + +// Rusage is an empty struct under non-linux environments. +type Rusage struct{} + +// GetRusage is a no-op function under non-linux environments. +func GetRusage() *Rusage { + log() + return nil +} + +// CPUTimeDiff returns the differences of user CPU time and system CPU time used +// between two Rusage structs. It a no-op function for non-linux environments. +func CPUTimeDiff(*Rusage, *Rusage) (float64, float64) { + log() + return 0, 0 +} + +// SetTCPUserTimeout is a no-op function under non-linux environments. +func SetTCPUserTimeout(net.Conn, time.Duration) error { + log() + return nil +} + +// GetTCPUserTimeout is a no-op function under non-linux environments. +// A negative return value indicates the operation is not supported +func GetTCPUserTimeout(net.Conn) (int, error) { + log() + return -1, nil +} diff --git a/internal/tcp_keepalive_others.go b/internal/tcp_keepalive_others.go new file mode 100644 index 0000000..4f347ed --- /dev/null +++ b/internal/tcp_keepalive_others.go @@ -0,0 +1,29 @@ +//go:build !unix && !windows + +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package internal + +import ( + "net" +) + +// NetDialerWithTCPKeepalive returns a vanilla net.Dialer on non-unix platforms. +func NetDialerWithTCPKeepalive() *net.Dialer { + return &net.Dialer{} +} diff --git a/internal/tcp_keepalive_unix.go b/internal/tcp_keepalive_unix.go new file mode 100644 index 0000000..7e7aaa5 --- /dev/null +++ b/internal/tcp_keepalive_unix.go @@ -0,0 +1,54 @@ +//go:build unix + +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package internal + +import ( + "net" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +// NetDialerWithTCPKeepalive returns a net.Dialer that enables TCP keepalives on +// the underlying connection with OS default values for keepalive parameters. +// +// TODO: Once https://github.com/golang/go/issues/62254 lands, and the +// appropriate Go version becomes less than our least supported Go version, we +// should look into using the new API to make things more straightforward. +func NetDialerWithTCPKeepalive() *net.Dialer { + return &net.Dialer{ + // Setting a negative value here prevents the Go stdlib from overriding + // the values of TCP keepalive time and interval. It also prevents the + // Go stdlib from enabling TCP keepalives by default. + KeepAlive: time.Duration(-1), + // This method is called after the underlying network socket is created, + // but before dialing the socket (or calling its connect() method). The + // combination of unconditionally enabling TCP keepalives here, and + // disabling the overriding of TCP keepalive parameters by setting the + // KeepAlive field to a negative value above, results in OS defaults for + // the TCP keepalive interval and time parameters. + Control: func(_, _ string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) { + unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_KEEPALIVE, 1) + }) + }, + } +} diff --git a/internal/tcp_keepalive_windows.go b/internal/tcp_keepalive_windows.go new file mode 100644 index 0000000..d5c1085 --- /dev/null +++ b/internal/tcp_keepalive_windows.go @@ -0,0 +1,54 @@ +//go:build windows + +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package internal + +import ( + "net" + "syscall" + "time" + + "golang.org/x/sys/windows" +) + +// NetDialerWithTCPKeepalive returns a net.Dialer that enables TCP keepalives on +// the underlying connection with OS default values for keepalive parameters. +// +// TODO: Once https://github.com/golang/go/issues/62254 lands, and the +// appropriate Go version becomes less than our least supported Go version, we +// should look into using the new API to make things more straightforward. +func NetDialerWithTCPKeepalive() *net.Dialer { + return &net.Dialer{ + // Setting a negative value here prevents the Go stdlib from overriding + // the values of TCP keepalive time and interval. It also prevents the + // Go stdlib from enabling TCP keepalives by default. + KeepAlive: time.Duration(-1), + // This method is called after the underlying network socket is created, + // but before dialing the socket (or calling its connect() method). The + // combination of unconditionally enabling TCP keepalives here, and + // disabling the overriding of TCP keepalive parameters by setting the + // KeepAlive field to a negative value above, results in OS defaults for + // the TCP keepalive interval and time parameters. + Control: func(_, _ string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) { + windows.SetsockoptInt(windows.Handle(fd), windows.SOL_SOCKET, windows.SO_KEEPALIVE, 1) + }) + }, + } +} diff --git a/internal/testutils/balancer.go b/internal/testutils/balancer.go new file mode 100644 index 0000000..5931923 --- /dev/null +++ b/internal/testutils/balancer.go @@ -0,0 +1,413 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "context" + "errors" + "fmt" + "testing" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/resolver" + + istats "google.golang.org/grpc/internal/stats" +) + +// TestSubConn implements the SubConn interface, to be used in tests. +type TestSubConn struct { + balancer.SubConn + tcc *BalancerClientConn // the CC that owns this SubConn + id string + ConnectCh chan struct{} + stateListener func(balancer.SubConnState) + healthListener func(balancer.SubConnState) + connectCalled *grpcsync.Event + Addresses []resolver.Address +} + +// NewTestSubConn returns a newly initialized SubConn. Typically, subconns +// should be created via TestClientConn.NewSubConn instead, but can be useful +// for some tests. +func NewTestSubConn(id string) *TestSubConn { + return &TestSubConn{ + ConnectCh: make(chan struct{}, 1), + connectCalled: grpcsync.NewEvent(), + id: id, + } +} + +// UpdateAddresses is a no-op. +func (tsc *TestSubConn) UpdateAddresses([]resolver.Address) {} + +// Connect is a no-op. +func (tsc *TestSubConn) Connect() { + tsc.connectCalled.Fire() + select { + case tsc.ConnectCh <- struct{}{}: + default: + } +} + +// GetOrBuildProducer is a no-op. +func (tsc *TestSubConn) GetOrBuildProducer(balancer.ProducerBuilder) (balancer.Producer, func()) { + return nil, nil +} + +// UpdateState pushes the state to the listener, if one is registered. +func (tsc *TestSubConn) UpdateState(state balancer.SubConnState) { + <-tsc.connectCalled.Done() + if tsc.stateListener != nil { + tsc.stateListener(state) + } + // pickfirst registers a health listener synchronously while handing updates + // to READY. It updates the balancing state only after receiving the health + // update. We update the health state here so callers of tsc.UpdateState + // can verify picker updates as soon as UpdateState returns. + if state.ConnectivityState == connectivity.Ready && tsc.healthListener != nil { + tsc.healthListener(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } +} + +// Shutdown pushes the SubConn to the ShutdownSubConn channel in the parent +// TestClientConn. +func (tsc *TestSubConn) Shutdown() { + tsc.tcc.logger.Logf("SubConn %s: Shutdown", tsc) + select { + case tsc.tcc.ShutdownSubConnCh <- tsc: + default: + } +} + +// String implements stringer to print human friendly error message. +func (tsc *TestSubConn) String() string { + return tsc.id +} + +// RegisterHealthListener sends a READY update to mock a situation when no +// health checking mechanisms are configured. +func (tsc *TestSubConn) RegisterHealthListener(lis func(balancer.SubConnState)) { + tsc.healthListener = lis +} + +// BalancerClientConn is a mock balancer.ClientConn used in tests. +type BalancerClientConn struct { + internal.EnforceClientConnEmbedding + logger Logger + + NewSubConnAddrsCh chan []resolver.Address // the last 10 []Address to create subconn. + NewSubConnCh chan *TestSubConn // the last 10 subconn created. + ShutdownSubConnCh chan *TestSubConn // the last 10 subconn removed. + UpdateAddressesAddrsCh chan []resolver.Address // last updated address via UpdateAddresses(). + + NewPickerCh chan balancer.Picker // the last picker updated. + NewStateCh chan connectivity.State // the last state. + ResolveNowCh chan resolver.ResolveNowOptions // the last ResolveNow(). + + subConnIdx int +} + +// NewBalancerClientConn creates a BalancerClientConn. +func NewBalancerClientConn(t *testing.T) *BalancerClientConn { + return &BalancerClientConn{ + logger: t, + + NewSubConnAddrsCh: make(chan []resolver.Address, 10), + NewSubConnCh: make(chan *TestSubConn, 10), + ShutdownSubConnCh: make(chan *TestSubConn, 10), + UpdateAddressesAddrsCh: make(chan []resolver.Address, 1), + + NewPickerCh: make(chan balancer.Picker, 1), + NewStateCh: make(chan connectivity.State, 1), + ResolveNowCh: make(chan resolver.ResolveNowOptions, 1), + } +} + +// NewSubConn creates a new SubConn. +func (tcc *BalancerClientConn) NewSubConn(a []resolver.Address, o balancer.NewSubConnOptions) (balancer.SubConn, error) { + sc := &TestSubConn{ + tcc: tcc, + id: fmt.Sprintf("sc%d", tcc.subConnIdx), + ConnectCh: make(chan struct{}, 1), + stateListener: o.StateListener, + connectCalled: grpcsync.NewEvent(), + Addresses: a, + } + tcc.subConnIdx++ + tcc.logger.Logf("testClientConn: NewSubConn(%v, %+v) => %s", a, o, sc) + select { + case tcc.NewSubConnAddrsCh <- a: + default: + } + + select { + case tcc.NewSubConnCh <- sc: + default: + } + + return sc, nil +} + +// MetricsRecorder returns an empty MetricsRecorderList. +func (*BalancerClientConn) MetricsRecorder() stats.MetricsRecorder { + return istats.NewMetricsRecorderList(nil) +} + +// RemoveSubConn is a nop; tests should all be updated to use sc.Shutdown() +// instead. +func (tcc *BalancerClientConn) RemoveSubConn(sc balancer.SubConn) { + tcc.logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc) +} + +// UpdateAddresses updates the addresses on the SubConn. +func (tcc *BalancerClientConn) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + tcc.logger.Logf("testutils.BalancerClientConn: UpdateAddresses(%v, %+v)", sc, addrs) + select { + case tcc.UpdateAddressesAddrsCh <- addrs: + default: + } +} + +// UpdateState updates connectivity state and picker. +func (tcc *BalancerClientConn) UpdateState(bs balancer.State) { + tcc.logger.Logf("testutils.BalancerClientConn: UpdateState(%v)", bs) + select { + case <-tcc.NewStateCh: + default: + } + tcc.NewStateCh <- bs.ConnectivityState + + select { + case <-tcc.NewPickerCh: + default: + } + tcc.NewPickerCh <- bs.Picker +} + +// ResolveNow panics. +func (tcc *BalancerClientConn) ResolveNow(o resolver.ResolveNowOptions) { + select { + case <-tcc.ResolveNowCh: + default: + } + tcc.ResolveNowCh <- o +} + +// Target panics. +func (tcc *BalancerClientConn) Target() string { + panic("not implemented") +} + +// WaitForErrPicker waits until an error picker is pushed to this ClientConn. +// Returns error if the provided context expires or a non-error picker is pushed +// to the ClientConn. +func (tcc *BalancerClientConn) WaitForErrPicker(ctx context.Context) error { + select { + case <-ctx.Done(): + return errors.New("timeout when waiting for an error picker") + case picker := <-tcc.NewPickerCh: + if _, perr := picker.Pick(balancer.PickInfo{Ctx: ctx}); perr == nil { + return fmt.Errorf("balancer returned a picker which is not an error picker") + } + } + return nil +} + +// WaitForPickerWithErr waits until an error picker is pushed to this +// ClientConn with the error matching the wanted error. Returns an error if +// the provided context expires, including the last received picker error (if +// any). +func (tcc *BalancerClientConn) WaitForPickerWithErr(ctx context.Context, want error) error { + lastErr := errors.New("received no picker") + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout when waiting for an error picker with %v; last picker error: %v", want, lastErr) + case picker := <-tcc.NewPickerCh: + if _, lastErr = picker.Pick(balancer.PickInfo{Ctx: ctx}); lastErr != nil && lastErr.Error() == want.Error() { + return nil + } + } + } +} + +// WaitForConnectivityState waits until the state pushed to this ClientConn +// matches the wanted state. Returns an error if the provided context expires, +// including the last received state (if any). +func (tcc *BalancerClientConn) WaitForConnectivityState(ctx context.Context, want connectivity.State) error { + var lastState connectivity.State = -1 + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout when waiting for state to be %s; last state: %s", want, lastState) + case s := <-tcc.NewStateCh: + if s == want { + return nil + } + lastState = s + } + } +} + +// WaitForRoundRobinPicker waits for a picker that passes IsRoundRobin. Also +// drains the matching state channel and requires it to be READY (if an entry +// is pending) to be considered. Returns an error if the provided context +// expires, including the last received error from IsRoundRobin or the picker +// (if any). +func (tcc *BalancerClientConn) WaitForRoundRobinPicker(ctx context.Context, want ...balancer.SubConn) error { + lastErr := errors.New("received no picker") + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout when waiting for round robin picker with %v; last error: %v", want, lastErr) + case p := <-tcc.NewPickerCh: + s := connectivity.Ready + select { + case s = <-tcc.NewStateCh: + default: + } + if s != connectivity.Ready { + lastErr = fmt.Errorf("received state %v instead of ready", s) + break + } + var pickerErr error + if err := IsRoundRobin(want, func() balancer.SubConn { + sc, err := p.Pick(balancer.PickInfo{Ctx: ctx}) + if err != nil { + pickerErr = err + } else if sc.Done != nil { + sc.Done(balancer.DoneInfo{}) + } + return sc.SubConn + }); pickerErr != nil { + lastErr = pickerErr + continue + } else if err != nil { + lastErr = err + continue + } + return nil + } + } +} + +// WaitForPicker waits for a picker that results in f returning nil. If the +// context expires, returns the last error returned by f (if any). +func (tcc *BalancerClientConn) WaitForPicker(ctx context.Context, f func(balancer.Picker) error) error { + lastErr := errors.New("received no picker") + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout when waiting for picker; last error: %v", lastErr) + case p := <-tcc.NewPickerCh: + if err := f(p); err != nil { + lastErr = err + continue + } + return nil + } + } +} + +// IsRoundRobin checks whether f's return value is roundrobin of elements from +// want. But it doesn't check for the order. Note that want can contain +// duplicate items, which makes it weight-round-robin. +// +// Step 1. the return values of f should form a permutation of all elements in +// want, but not necessary in the same order. E.g. if want is {a,a,b}, the check +// fails if f returns: +// - {a,a,a}: third a is returned before b +// - {a,b,b}: second b is returned before the second a +// +// If error is found in this step, the returned error contains only the first +// iteration until where it goes wrong. +// +// Step 2. the return values of f should be repetitions of the same permutation. +// E.g. if want is {a,a,b}, the check fails if f returns: +// - {a,b,a,b,a,a}: though it satisfies step 1, the second iteration is not +// repeating the first iteration. +// +// If error is found in this step, the returned error contains the first +// iteration + the second iteration until where it goes wrong. +func IsRoundRobin(want []balancer.SubConn, f func() balancer.SubConn) error { + wantSet := make(map[balancer.SubConn]int) // SubConn -> count, for weighted RR. + for _, sc := range want { + wantSet[sc]++ + } + + // The first iteration: makes sure f's return values form a permutation of + // elements in want. + // + // Also keep the returns values in a slice, so we can compare the order in + // the second iteration. + gotSliceFirstIteration := make([]balancer.SubConn, 0, len(want)) + for range want { + got := f() + gotSliceFirstIteration = append(gotSliceFirstIteration, got) + wantSet[got]-- + if wantSet[got] < 0 { + return fmt.Errorf("non-roundrobin want: %v, result: %v", want, gotSliceFirstIteration) + } + } + + // The second iteration should repeat the first iteration. + var gotSliceSecondIteration []balancer.SubConn + for i := 0; i < 2; i++ { + for _, w := range gotSliceFirstIteration { + g := f() + gotSliceSecondIteration = append(gotSliceSecondIteration, g) + if w != g { + return fmt.Errorf("non-roundrobin, first iter: %v, second iter: %v", gotSliceFirstIteration, gotSliceSecondIteration) + } + } + } + + return nil +} + +// SubConnFromPicker returns a function which returns a SubConn by calling the +// Pick() method of the provided picker. There is no caching of SubConns here. +// Every invocation of the returned function results in a new pick. +func SubConnFromPicker(p balancer.Picker) func() balancer.SubConn { + return func() balancer.SubConn { + scst, _ := p.Pick(balancer.PickInfo{}) + return scst.SubConn + } +} + +// ErrTestConstPicker is error returned by test const picker. +var ErrTestConstPicker = fmt.Errorf("const picker error") + +// TestConstPicker is a const picker for tests. +type TestConstPicker struct { + Err error + SC balancer.SubConn +} + +// Pick returns the const SubConn or the error. +func (tcp *TestConstPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + if tcp.Err != nil { + return balancer.PickResult{}, tcp.Err + } + return balancer.PickResult{SubConn: tcp.SC}, nil +} diff --git a/internal/testutils/blocking_context_dialer.go b/internal/testutils/blocking_context_dialer.go new file mode 100644 index 0000000..f99a1a8 --- /dev/null +++ b/internal/testutils/blocking_context_dialer.go @@ -0,0 +1,139 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "context" + "net" + "sync" + + "google.golang.org/grpc/grpclog" +) + +var logger = grpclog.Component("testutils") + +// BlockingDialer is a dialer that waits for Resume() to be called before +// dialing. +type BlockingDialer struct { + // mu protects holds. + mu sync.Mutex + // holds maps network addresses to a list of holds for that address. + holds map[string][]*Hold +} + +// NewBlockingDialer returns a dialer that waits for Resume() to be called +// before dialing. +func NewBlockingDialer() *BlockingDialer { + return &BlockingDialer{ + holds: make(map[string][]*Hold), + } +} + +// DialContext implements a context dialer for use with grpc.WithContextDialer +// dial option for a BlockingDialer. +func (d *BlockingDialer) DialContext(ctx context.Context, addr string) (net.Conn, error) { + d.mu.Lock() + holds := d.holds[addr] + if len(holds) == 0 { + // No hold for this addr. + d.mu.Unlock() + return (&net.Dialer{}).DialContext(ctx, "tcp", addr) + } + hold := holds[0] + d.holds[addr] = holds[1:] + d.mu.Unlock() + + logger.Infof("Hold %p: Intercepted connection attempt to addr %q", hold, addr) + close(hold.waitCh) + select { + case err := <-hold.blockCh: + if err != nil { + return nil, err + } + return (&net.Dialer{}).DialContext(ctx, "tcp", addr) + case <-ctx.Done(): + logger.Infof("Hold %p: Connection attempt to addr %q timed out", hold, addr) + return nil, ctx.Err() + } +} + +// Hold is a handle to a single connection attempt. It can be used to block, +// fail and succeed connection attempts. +type Hold struct { + // dialer is the dialer that created this hold. + dialer *BlockingDialer + // waitCh is closed when a connection attempt is received. + waitCh chan struct{} + // blockCh receives the value to return from DialContext for this connection + // attempt (nil on resume, an error on fail). It receives at most 1 value. + blockCh chan error + // addr is the address that this hold is for. + addr string +} + +// Hold blocks the dialer when a connection attempt is made to the given addr. +// A hold is valid for exactly one connection attempt. Multiple holds for an +// addr can be added, and they will apply in the order that the connections are +// attempted. +func (d *BlockingDialer) Hold(addr string) *Hold { + d.mu.Lock() + defer d.mu.Unlock() + + h := Hold{dialer: d, blockCh: make(chan error, 1), waitCh: make(chan struct{}), addr: addr} + d.holds[addr] = append(d.holds[addr], &h) + return &h +} + +// Wait blocks until there is a connection attempt on this Hold, or the context +// expires. Return false if the context has expired, true otherwise. +func (h *Hold) Wait(ctx context.Context) bool { + logger.Infof("Hold %p: Waiting for a connection attempt to addr %q", h, h.addr) + select { + case <-ctx.Done(): + return false + case <-h.waitCh: + return true + } +} + +// Resume unblocks the dialer for the given addr. Either Resume or Fail must be +// called at most once on a hold. Otherwise, Resume panics. +func (h *Hold) Resume() { + logger.Infof("Hold %p: Resuming connection attempt to addr %q", h, h.addr) + h.blockCh <- nil + close(h.blockCh) +} + +// Fail fails the connection attempt. Either Resume or Fail must be +// called at most once on a hold. Otherwise, Resume panics. +func (h *Hold) Fail(err error) { + logger.Infof("Hold %p: Failing connection attempt to addr %q", h, h.addr) + h.blockCh <- err + close(h.blockCh) +} + +// IsStarted returns true if this hold has received a connection attempt. +func (h *Hold) IsStarted() bool { + select { + case <-h.waitCh: + return true + default: + return false + } +} diff --git a/internal/testutils/blocking_context_dialer_test.go b/internal/testutils/blocking_context_dialer_test.go new file mode 100644 index 0000000..b826c5a --- /dev/null +++ b/internal/testutils/blocking_context_dialer_test.go @@ -0,0 +1,214 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "context" + "errors" + "testing" + "time" +) + +const ( + testTimeout = 5 * time.Second + testShortTimeout = 10 * time.Millisecond +) + +func (s) TestBlockingDialer_NoHold(t *testing.T) { + lis, err := LocalTCPListener() + if err != nil { + t.Fatalf("Failed to listen: %v", err) + } + defer lis.Close() + + d := NewBlockingDialer() + + // This should not block. + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + conn, err := d.DialContext(ctx, lis.Addr().String()) + if err != nil { + t.Fatalf("Failed to dial: %v", err) + } + conn.Close() +} + +func (s) TestBlockingDialer_HoldWaitResume(t *testing.T) { + lis, err := LocalTCPListener() + if err != nil { + t.Fatalf("Failed to listen: %v", err) + } + defer lis.Close() + + d := NewBlockingDialer() + h := d.Hold(lis.Addr().String()) + + if h.IsStarted() { + t.Fatalf("hold.IsStarted() = true, want false") + } + + done := make(chan struct{}) + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + go func() { + defer close(done) + conn, err := d.DialContext(ctx, lis.Addr().String()) + if err != nil { + t.Errorf("BlockingDialer.DialContext() got error: %v, want success", err) + return + } + + if !h.IsStarted() { + t.Errorf("hold.IsStarted() = false, want true") + } + conn.Close() + }() + + // This should block until the goroutine above is scheduled. + if !h.Wait(ctx) { + t.Fatalf("Timeout while waiting for a connection attempt to %q", h.addr) + } + + if !h.IsStarted() { + t.Errorf("hold.IsStarted() = false, want true") + } + + select { + case <-done: + t.Fatalf("Expected dialer to be blocked.") + case <-time.After(testShortTimeout): + } + + h.Resume() // Unblock the above goroutine. + + select { + case <-done: + case <-ctx.Done(): + t.Errorf("Timeout waiting for connection attempt to resume.") + } +} + +func (s) TestBlockingDialer_HoldWaitFail(t *testing.T) { + lis, err := LocalTCPListener() + if err != nil { + t.Fatalf("Failed to listen: %v", err) + } + defer lis.Close() + + d := NewBlockingDialer() + h := d.Hold(lis.Addr().String()) + + wantErr := errors.New("test error") + + dialError := make(chan error) + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + go func() { + _, err := d.DialContext(ctx, lis.Addr().String()) + dialError <- err + }() + + if !h.Wait(ctx) { + t.Fatal("Timeout while waiting for a connection attempt to " + h.addr) + } + select { + case err = <-dialError: + t.Errorf("DialContext got unblocked with err %v. Want DialContext to still be blocked after Wait()", err) + case <-time.After(testShortTimeout): + } + + h.Fail(wantErr) + + select { + case err = <-dialError: + if !errors.Is(err, wantErr) { + t.Errorf("BlockingDialer.DialContext() after Fail(): got error %v, want %v", err, wantErr) + } + case <-ctx.Done(): + t.Errorf("Timeout waiting for connection attempt to fail.") + } +} + +func (s) TestBlockingDialer_ContextCanceled(t *testing.T) { + lis, err := LocalTCPListener() + if err != nil { + t.Fatalf("Failed to listen: %v", err) + } + defer lis.Close() + + d := NewBlockingDialer() + h := d.Hold(lis.Addr().String()) + + dialErr := make(chan error) + testCtx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + ctx, cancel := context.WithCancel(testCtx) + defer cancel() + go func() { + _, err := d.DialContext(ctx, lis.Addr().String()) + dialErr <- err + }() + if !h.Wait(testCtx) { + t.Errorf("Timeout while waiting for a connection attempt to %q", h.addr) + } + + cancel() + + select { + case err = <-dialErr: + if !errors.Is(err, context.Canceled) { + t.Errorf("BlockingDialer.DialContext() after context cancel: got error %v, want %v", err, context.Canceled) + } + case <-testCtx.Done(): + t.Errorf("Timeout while waiting for Wait to return.") + } + + h.Resume() // noop, just make sure nothing bad happen. +} + +func (s) TestBlockingDialer_CancelWait(t *testing.T) { + lis, err := LocalTCPListener() + if err != nil { + t.Fatalf("Failed to listen: %v", err) + } + defer lis.Close() + + d := NewBlockingDialer() + h := d.Hold(lis.Addr().String()) + + testCtx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + ctx, cancel := context.WithCancel(testCtx) + cancel() + done := make(chan struct{}) + go func() { + if h.Wait(ctx) { + t.Errorf("Expected cancel to return false when context expires") + } + done <- struct{}{} + }() + + select { + case <-done: + case <-testCtx.Done(): + t.Errorf("Timeout while waiting for Wait to return.") + } +} diff --git a/internal/testutils/channel.go b/internal/testutils/channel.go new file mode 100644 index 0000000..d1bb045 --- /dev/null +++ b/internal/testutils/channel.go @@ -0,0 +1,118 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package testutils + +import ( + "context" +) + +// DefaultChanBufferSize is the default buffer size of the underlying channel. +const DefaultChanBufferSize = 1 + +// Channel wraps a generic channel and provides a timed receive operation. +type Channel struct { + // C is the underlying channel on which values sent using the SendXxx() + // methods are delivered. Tests which cannot use ReceiveXxx() for whatever + // reasons can use C to read the values. + C chan any +} + +// Send sends value on the underlying channel. +func (c *Channel) Send(value any) { + c.C <- value +} + +// SendContext sends value on the underlying channel, or returns an error if +// the context expires. +func (c *Channel) SendContext(ctx context.Context, value any) error { + select { + case c.C <- value: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// SendOrFail attempts to send value on the underlying channel. Returns true +// if successful or false if the channel was full. +func (c *Channel) SendOrFail(value any) bool { + select { + case c.C <- value: + return true + default: + return false + } +} + +// ReceiveOrFail returns the value on the underlying channel and true, or nil +// and false if the channel was empty. +func (c *Channel) ReceiveOrFail() (any, bool) { + select { + case got := <-c.C: + return got, true + default: + return nil, false + } +} + +// Drain drains the channel by repeatedly reading from it until it is empty. +func (c *Channel) Drain() { + for { + select { + case <-c.C: + default: + return + } + } +} + +// Receive returns the value received on the underlying channel, or the error +// returned by ctx if it is closed or cancelled. +func (c *Channel) Receive(ctx context.Context) (any, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case got := <-c.C: + return got, nil + } +} + +// Replace clears the value on the underlying channel, and sends the new value. +// +// It's expected to be used with a size-1 channel, to only keep the most +// up-to-date item. This method is inherently racy when invoked concurrently +// from multiple goroutines. +func (c *Channel) Replace(value any) { + for { + select { + case c.C <- value: + return + case <-c.C: + } + } +} + +// NewChannel returns a new Channel. +func NewChannel() *Channel { + return NewChannelWithSize(DefaultChanBufferSize) +} + +// NewChannelWithSize returns a new Channel with a buffer of bufSize. +func NewChannelWithSize(bufSize int) *Channel { + return &Channel{C: make(chan any, bufSize)} +} diff --git a/internal/testutils/envconfig.go b/internal/testutils/envconfig.go new file mode 100644 index 0000000..3b25aac --- /dev/null +++ b/internal/testutils/envconfig.go @@ -0,0 +1,33 @@ +package testutils + +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ( + "testing" +) + +// SetEnvConfig sets the value of the given variable to the specified value, +// taking care of restoring the original value after the test completes. +func SetEnvConfig[T any](t *testing.T, variable *T, value T) { + t.Helper() + old := *variable + t.Cleanup(func() { + *variable = old + }) + *variable = value +} diff --git a/internal/testutils/fakegrpclb/server.go b/internal/testutils/fakegrpclb/server.go new file mode 100644 index 0000000..f930a1e --- /dev/null +++ b/internal/testutils/fakegrpclb/server.go @@ -0,0 +1,250 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package fakegrpclb provides a fake implementation of the grpclb server. +package fakegrpclb + +import ( + "errors" + "fmt" + "io" + "net" + "net/netip" + "strconv" + "sync" + "time" + + "google.golang.org/grpc" + lbgrpc "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" + lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/status" +) + +var logger = grpclog.Component("fake_grpclb") + +// ServerParams wraps options passed while creating a Server. +type ServerParams struct { + ListenPort int // Listening port for the balancer server. + ServerOptions []grpc.ServerOption // gRPC options for the balancer server. + + LoadBalancedServiceName string // Service name being load balanced for. + LoadBalancedServicePort int // Service port being load balanced for. + BackendAddresses []string // Service backends to balance load across. + ShortStream bool // End balancer stream after sending server list. +} + +// Server is a fake implementation of the grpclb LoadBalancer service. It does +// not support stats reporting from clients, and always sends back a static list +// of backends to the client to balance load across. +// +// It is safe for concurrent access. +type Server struct { + lbgrpc.UnimplementedLoadBalancerServer + + // Options copied over from ServerParams passed to NewServer. + sOpts []grpc.ServerOption // gRPC server options. + serviceName string // Service name being load balanced for. + servicePort int // Service port being load balanced for. + shortStream bool // End balancer stream after sending server list. + + // Values initialized using ServerParams passed to NewServer. + backends []*lbpb.Server // Service backends to balance load across. + lis net.Listener // Listener for grpc connections to the LoadBalancer service. + + // mu guards access to below fields. + mu sync.Mutex + grpcServer *grpc.Server // Underlying grpc server. + address string // Actual listening address. + + stopped chan struct{} // Closed when Stop() is called. +} + +// NewServer creates a new Server with passed in params. Returns a non-nil error +// if the params are invalid. +func NewServer(params ServerParams) (*Server, error) { + var servers []*lbpb.Server + for _, addr := range params.BackendAddresses { + ipStr, portStr, err := net.SplitHostPort(addr) + if err != nil { + return nil, fmt.Errorf("failed to parse list of backend address %q: %v", addr, err) + } + ip, err := netip.ParseAddr(ipStr) + if err != nil { + return nil, fmt.Errorf("failed to parse ip %q: %v", ipStr, err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + return nil, fmt.Errorf("failed to convert port %q to int", portStr) + } + logger.Infof("Adding backend ip: %q, port: %d to server list", ip.String(), port) + servers = append(servers, &lbpb.Server{ + IpAddress: ip.AsSlice(), + Port: int32(port), + }) + } + + lis, err := net.Listen("tcp", "localhost:"+strconv.Itoa(params.ListenPort)) + if err != nil { + return nil, fmt.Errorf("failed to listen on port %q: %v", params.ListenPort, err) + } + + return &Server{ + sOpts: params.ServerOptions, + serviceName: params.LoadBalancedServiceName, + servicePort: params.LoadBalancedServicePort, + shortStream: params.ShortStream, + backends: servers, + lis: lis, + address: lis.Addr().String(), + stopped: make(chan struct{}), + }, nil +} + +// Serve starts serving the LoadBalancer service on a gRPC server. +// +// It returns early with a non-nil error if it is unable to start serving. +// Otherwise, it blocks until Stop() is called, at which point it returns the +// error returned by the underlying grpc.Server's Serve() method. +func (s *Server) Serve() error { + s.mu.Lock() + if s.grpcServer != nil { + s.mu.Unlock() + return errors.New("Serve() called multiple times") + } + + server := grpc.NewServer(s.sOpts...) + s.grpcServer = server + s.mu.Unlock() + + logger.Infof("Begin listening on %s", s.lis.Addr().String()) + lbgrpc.RegisterLoadBalancerServer(server, s) + return server.Serve(s.lis) // This call will block. +} + +// Stop stops serving the LoadBalancer service and unblocks the preceding call +// to Serve(). +func (s *Server) Stop() { + defer close(s.stopped) + s.mu.Lock() + if s.grpcServer != nil { + s.grpcServer.Stop() + s.grpcServer = nil + } + s.mu.Unlock() +} + +// Address returns the host:port on which the LoadBalancer service is serving. +func (s *Server) Address() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.address +} + +// BalanceLoad provides a fake implementation of the LoadBalancer service. +func (s *Server) BalanceLoad(stream lbgrpc.LoadBalancer_BalanceLoadServer) error { + logger.Info("New BalancerLoad stream started") + + req, err := stream.Recv() + if err == io.EOF { + logger.Warning("Received EOF when reading from the stream") + return nil + } + if err != nil { + logger.Warningf("Failed to read LoadBalanceRequest from stream: %v", err) + return err + } + logger.Infof("Received LoadBalancerRequest:\n%s", pretty.ToJSON(req)) + + // Initial request contains the service being load balanced for. + initialReq := req.GetInitialRequest() + if initialReq == nil { + logger.Info("First message on the stream does not contain an InitialLoadBalanceRequest") + return status.Error(codes.Unknown, "First request not an InitialLoadBalanceRequest") + } + + // Basic validation of the service name and port from the incoming request. + // + // Clients targeting service:port can sometimes include the ":port" suffix in + // their requested names; handle this case. + serviceName, port, err := net.SplitHostPort(initialReq.Name) + if err != nil { + // Requested name did not contain a port. So, use the name as is. + serviceName = initialReq.Name + } else { + p, err := strconv.Atoi(port) + if err != nil { + logger.Infof("Failed to parse requested service port %q to integer", port) + return status.Error(codes.Unknown, "Bad requested service port number") + } + if p != s.servicePort { + logger.Infof("Requested service port number %d does not match expected %d", p, s.servicePort) + return status.Error(codes.Unknown, "Bad requested service port number") + } + } + if serviceName != s.serviceName { + logger.Infof("Requested service name %q does not match expected %q", serviceName, s.serviceName) + return status.Error(codes.NotFound, "Bad requested service name") + } + + // Empty initial response disables stats reporting from the client. Stats + // reporting from the client is used to determine backend load and is not + // required for the purposes of this fake. + initResp := &lbpb.LoadBalanceResponse{ + LoadBalanceResponseType: &lbpb.LoadBalanceResponse_InitialResponse{ + InitialResponse: &lbpb.InitialLoadBalanceResponse{}, + }, + } + if err := stream.Send(initResp); err != nil { + logger.Warningf("Failed to send InitialLoadBalanceResponse on the stream: %v", err) + return err + } + + resp := &lbpb.LoadBalanceResponse{ + LoadBalanceResponseType: &lbpb.LoadBalanceResponse_ServerList{ + ServerList: &lbpb.ServerList{Servers: s.backends}, + }, + } + logger.Infof("Sending response with server list: %s", pretty.ToJSON(resp)) + if err := stream.Send(resp); err != nil { + logger.Warningf("Failed to send InitialLoadBalanceResponse on the stream: %v", err) + return err + } + + if s.shortStream { + logger.Info("Ending stream early as the short stream option was set") + return nil + } + + for { + select { + case <-stream.Context().Done(): + return nil + case <-s.stopped: + return nil + case <-time.After(10 * time.Second): + logger.Infof("Sending response with server list: %s", pretty.ToJSON(resp)) + if err := stream.Send(resp); err != nil { + logger.Warningf("Failed to send InitialLoadBalanceResponse on the stream: %v", err) + return err + } + } + } +} diff --git a/internal/testutils/http_client.go b/internal/testutils/http_client.go new file mode 100644 index 0000000..9832bf3 --- /dev/null +++ b/internal/testutils/http_client.go @@ -0,0 +1,63 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package testutils + +import ( + "context" + "net/http" + "time" +) + +// DefaultHTTPRequestTimeout is the default timeout value for the amount of time +// this client waits for a response to be pushed on RespChan before it fails the +// Do() call. +const DefaultHTTPRequestTimeout = 1 * time.Second + +// FakeHTTPClient helps mock out HTTP calls made by the code under test. It +// makes HTTP requests made by the code under test available through a channel, +// and makes it possible to inject various responses. +type FakeHTTPClient struct { + // ReqChan exposes the HTTP.Request made by the code under test. + ReqChan *Channel + // RespChan is a channel on which this fake client accepts responses to be + // sent to the code under test. + RespChan *Channel + // Err, if set, is returned by Do(). + Err error + // RecvTimeout is the amount of the time this client waits for a response to + // be pushed on RespChan before it fails the Do() call. If this field is + // left unspecified, DefaultHTTPRequestTimeout is used. + RecvTimeout time.Duration +} + +// Do pushes req on ReqChan and returns the response available on RespChan. +func (fc *FakeHTTPClient) Do(req *http.Request) (*http.Response, error) { + fc.ReqChan.Send(req) + + timeout := fc.RecvTimeout + if timeout == 0 { + timeout = DefaultHTTPRequestTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + val, err := fc.RespChan.Receive(ctx) + if err != nil { + return nil, err + } + return val.(*http.Response), fc.Err +} diff --git a/internal/testutils/local_listener.go b/internal/testutils/local_listener.go new file mode 100644 index 0000000..f831b95 --- /dev/null +++ b/internal/testutils/local_listener.go @@ -0,0 +1,26 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import "net" + +// LocalTCPListener returns a net.Listener listening on local address and port. +func LocalTCPListener() (net.Listener, error) { + return net.Listen("tcp", "localhost:0") +} diff --git a/internal/testutils/marshal_any.go b/internal/testutils/marshal_any.go new file mode 100644 index 0000000..12ce832 --- /dev/null +++ b/internal/testutils/marshal_any.go @@ -0,0 +1,37 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package testutils + +import ( + "testing" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +// MarshalAny is a convenience function to marshal protobuf messages into any +// protos. function will fail the test with a fatal error if the marshaling fails. +func MarshalAny(t *testing.T, m proto.Message) *anypb.Any { + t.Helper() + + a, err := anypb.New(m) + if err != nil { + t.Fatalf("Failed to marshal proto %+v into an Any: %v", m, err) + } + return a +} diff --git a/internal/testutils/parse_port.go b/internal/testutils/parse_port.go new file mode 100644 index 0000000..c633af0 --- /dev/null +++ b/internal/testutils/parse_port.go @@ -0,0 +1,39 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package testutils + +import ( + "net" + "strconv" + "testing" +) + +// ParsePort returns the port from the given address string, as a unit32. +func ParsePort(t *testing.T, addr string) uint32 { + t.Helper() + + _, p, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("Invalid serving address: %v", err) + } + port, err := strconv.ParseUint(p, 10, 32) + if err != nil { + t.Fatalf("Invalid serving port: %v", err) + } + return uint32(port) +} diff --git a/internal/testutils/parse_url.go b/internal/testutils/parse_url.go new file mode 100644 index 0000000..ff276e4 --- /dev/null +++ b/internal/testutils/parse_url.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "fmt" + "net/url" +) + +// MustParseURL attempts to parse the provided target using url.Parse() +// and panics if parsing fails. +func MustParseURL(target string) *url.URL { + u, err := url.Parse(target) + if err != nil { + panic(fmt.Sprintf("Error parsing target(%s): %v", target, err)) + } + return u +} diff --git a/internal/testutils/pickfirst/pickfirst.go b/internal/testutils/pickfirst/pickfirst.go new file mode 100644 index 0000000..aa90ffc --- /dev/null +++ b/internal/testutils/pickfirst/pickfirst.go @@ -0,0 +1,82 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package pickfirst contains helper functions to check for pickfirst load +// balancing of RPCs in tests. +package pickfirst + +import ( + "context" + "fmt" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// CheckRPCsToBackend makes a bunch of RPCs on the given ClientConn and verifies +// if the RPCs are routed to a peer matching wantAddr. +// +// Returns a non-nil error if context deadline expires before all RPCs begin to +// be routed to the peer matching wantAddr, or if the backend returns RPC errors. +func CheckRPCsToBackend(ctx context.Context, cc *grpc.ClientConn, wantAddr resolver.Address) error { + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + // Make sure that 20 RPCs in a row reach the expected backend. Some + // tests switch from round_robin back to pick_first and call this + // function. None of our tests spin up more than 10 backends. So, + // waiting for 20 RPCs to reach a single backend would a decent + // indicator of having switched to pick_first. + count := 0 + for { + time.Sleep(time.Millisecond) + if ctx.Err() != nil { + return fmt.Errorf("timeout waiting for RPC to be routed to %s", wantAddr.Addr) + } + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + // Some tests remove backends and check if pick_first is happening across + // the remaining backends. In such cases, RPCs can initially fail on the + // connection using the removed backend. Just keep retrying and eventually + // the connection using the removed backend will shutdown and will be + // removed. + continue + } + if peer.Addr.String() != wantAddr.Addr { + count = 0 + continue + } + count++ + if count > 20 { + break + } + } + // Make sure subsequent RPCs are all routed to the same backend. + for i := 0; i < 10; i++ { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + return fmt.Errorf("EmptyCall() = %v, want ", err) + } + if gotAddr := peer.Addr.String(); gotAddr != wantAddr.Addr { + return fmt.Errorf("rpc sent to peer %q, want peer %q", gotAddr, wantAddr) + } + } + return nil +} diff --git a/internal/testutils/pipe_listener.go b/internal/testutils/pipe_listener.go new file mode 100644 index 0000000..6bd3bc0 --- /dev/null +++ b/internal/testutils/pipe_listener.go @@ -0,0 +1,96 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package testutils contains testing helpers. +package testutils + +import ( + "errors" + "net" + "time" +) + +var errClosed = errors.New("closed") + +type pipeAddr struct{} + +func (p pipeAddr) Network() string { return "pipe" } +func (p pipeAddr) String() string { return "pipe" } + +// PipeListener is a listener with an unbuffered pipe. Each write will complete only once the other side reads. It +// should only be created using NewPipeListener. +type PipeListener struct { + c chan chan<- net.Conn + done chan struct{} +} + +// NewPipeListener creates a new pipe listener. +func NewPipeListener() *PipeListener { + return &PipeListener{ + c: make(chan chan<- net.Conn), + done: make(chan struct{}), + } +} + +// Accept accepts a connection. +func (p *PipeListener) Accept() (net.Conn, error) { + var connChan chan<- net.Conn + select { + case <-p.done: + return nil, errClosed + case connChan = <-p.c: + select { + case <-p.done: + close(connChan) + return nil, errClosed + default: + } + } + c1, c2 := net.Pipe() + connChan <- c1 + close(connChan) + return c2, nil +} + +// Close closes the listener. +func (p *PipeListener) Close() error { + close(p.done) + return nil +} + +// Addr returns a pipe addr. +func (p *PipeListener) Addr() net.Addr { + return pipeAddr{} +} + +// Dialer dials a connection. +func (p *PipeListener) Dialer() func(string, time.Duration) (net.Conn, error) { + return func(string, time.Duration) (net.Conn, error) { + connChan := make(chan net.Conn) + select { + case p.c <- connChan: + case <-p.done: + return nil, errClosed + } + conn, ok := <-connChan + if !ok { + return nil, errClosed + } + return conn, nil + } +} diff --git a/internal/testutils/pipe_listener_test.go b/internal/testutils/pipe_listener_test.go new file mode 100644 index 0000000..45cc27e --- /dev/null +++ b/internal/testutils/pipe_listener_test.go @@ -0,0 +1,172 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils_test + +import ( + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestPipeListener(t *testing.T) { + pl := testutils.NewPipeListener() + recvdBytes := make(chan []byte, 1) + const want = "hello world" + + go func() { + c, err := pl.Accept() + if err != nil { + t.Error(err) + } + + read := make([]byte, len(want)) + _, err = c.Read(read) + if err != nil { + t.Error(err) + } + recvdBytes <- read + }() + + dl := pl.Dialer() + conn, err := dl("", time.Duration(0)) + if err != nil { + t.Fatal(err) + } + + _, err = conn.Write([]byte(want)) + if err != nil { + t.Fatal(err) + } + + select { + case gotBytes := <-recvdBytes: + got := string(gotBytes) + if got != want { + t.Fatalf("expected to get %s, got %s", got, want) + } + case <-time.After(100 * time.Millisecond): + t.Fatal("timed out waiting for server to receive bytes") + } +} + +func (s) TestUnblocking(t *testing.T) { + for _, test := range []struct { + desc string + blockFuncShouldError bool + blockFunc func(*testutils.PipeListener, chan struct{}) error + unblockFunc func(*testutils.PipeListener) error + }{ + { + desc: "Accept unblocks Dial", + blockFunc: func(pl *testutils.PipeListener, done chan struct{}) error { + dl := pl.Dialer() + _, err := dl("", time.Duration(0)) + close(done) + return err + }, + unblockFunc: func(pl *testutils.PipeListener) error { + _, err := pl.Accept() + return err + }, + }, + { + desc: "Close unblocks Dial", + blockFuncShouldError: true, // because pl.Close will be called + blockFunc: func(pl *testutils.PipeListener, done chan struct{}) error { + dl := pl.Dialer() + _, err := dl("", time.Duration(0)) + close(done) + return err + }, + unblockFunc: func(pl *testutils.PipeListener) error { + return pl.Close() + }, + }, + { + desc: "Dial unblocks Accept", + blockFunc: func(pl *testutils.PipeListener, done chan struct{}) error { + _, err := pl.Accept() + close(done) + return err + }, + unblockFunc: func(pl *testutils.PipeListener) error { + dl := pl.Dialer() + _, err := dl("", time.Duration(0)) + return err + }, + }, + { + desc: "Close unblocks Accept", + blockFuncShouldError: true, // because pl.Close will be called + blockFunc: func(pl *testutils.PipeListener, done chan struct{}) error { + _, err := pl.Accept() + close(done) + return err + }, + unblockFunc: func(pl *testutils.PipeListener) error { + return pl.Close() + }, + }, + } { + t.Log(test.desc) + testUnblocking(t, test.blockFunc, test.unblockFunc, test.blockFuncShouldError) + } +} + +func testUnblocking(t *testing.T, blockFunc func(*testutils.PipeListener, chan struct{}) error, unblockFunc func(*testutils.PipeListener) error, blockFuncShouldError bool) { + pl := testutils.NewPipeListener() + dialFinished := make(chan struct{}) + + go func() { + err := blockFunc(pl, dialFinished) + if blockFuncShouldError && err == nil { + t.Error("expected blocking func to return error because pl.Close was called, but got nil") + } + + if !blockFuncShouldError && err != nil { + t.Error(err) + } + }() + + select { + case <-dialFinished: + t.Fatal("expected Dial to block until pl.Close or pl.Accept") + default: + } + + if err := unblockFunc(pl); err != nil { + t.Fatal(err) + } + + select { + case <-dialFinished: + case <-time.After(100 * time.Millisecond): + t.Fatal("expected Accept to unblock after pl.Accept was called") + } +} diff --git a/internal/testutils/proxyserver/proxyserver.go b/internal/testutils/proxyserver/proxyserver.go new file mode 100644 index 0000000..576ab11 --- /dev/null +++ b/internal/testutils/proxyserver/proxyserver.go @@ -0,0 +1,134 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package proxyserver provides an implementation of a proxy server for testing purposes. +// The server supports only a single incoming connection at a time and is not concurrent. +// It handles only HTTP CONNECT requests; other HTTP methods are not supported. +package proxyserver + +import ( + "bufio" + "bytes" + "io" + "net" + "net/http" + "testing" + "time" + + "google.golang.org/grpc/internal/testutils" +) + +// ProxyServer represents a test proxy server. +type ProxyServer struct { + lis net.Listener + in net.Conn // Connection from the client to the proxy. + out net.Conn // Connection from the proxy to the backend. + onRequest func(*http.Request) // Function to check the request sent to proxy. + Addr string // Address of the proxy +} + +const defaultTestTimeout = 10 * time.Second + +// Stop closes the ProxyServer and its connections to client and server. +func (p *ProxyServer) stop() { + p.lis.Close() + if p.in != nil { + p.in.Close() + } + if p.out != nil { + p.out.Close() + } +} + +func (p *ProxyServer) handleRequest(t *testing.T, in net.Conn, waitForServerHello bool) { + req, err := http.ReadRequest(bufio.NewReader(in)) + if err != nil { + t.Errorf("failed to read CONNECT req: %v", err) + return + } + if req.Method != http.MethodConnect { + t.Errorf("unexpected Method %q, want %q", req.Method, http.MethodConnect) + } + p.onRequest(req) + + t.Logf("Dialing to %s", req.URL.Host) + out, err := net.Dial("tcp", req.URL.Host) + if err != nil { + in.Close() + t.Logf("failed to dial to server: %v", err) + return + } + out.SetDeadline(time.Now().Add(defaultTestTimeout)) + resp := http.Response{StatusCode: http.StatusOK, Proto: "HTTP/1.0"} + var buf bytes.Buffer + resp.Write(&buf) + + if waitForServerHello { + // Batch the first message from the server with the http connect + // response. This is done to test the cases in which the grpc client has + // the response to the connect request and proxied packets from the + // destination server when it reads the transport. + b := make([]byte, 50) + bytesRead, err := out.Read(b) + if err != nil { + t.Errorf("Got error while reading server hello: %v", err) + in.Close() + out.Close() + return + } + buf.Write(b[0:bytesRead]) + } + p.in = in + p.in.Write(buf.Bytes()) + p.out = out + + go io.Copy(p.in, p.out) + go io.Copy(p.out, p.in) +} + +// New initializes and starts a proxy server, registers a cleanup to +// stop it, and returns a ProxyServer. +func New(t *testing.T, reqCheck func(*http.Request), waitForServerHello bool) *ProxyServer { + t.Helper() + pLis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + + p := &ProxyServer{ + lis: pLis, + onRequest: reqCheck, + Addr: pLis.Addr().String(), + } + + // Start the proxy server. + go func() { + for { + in, err := p.lis.Accept() + if err != nil { + return + } + // p.handleRequest is not invoked in a goroutine because the test + // proxy currently supports handling only one connection at a time. + p.handleRequest(t, in, waitForServerHello) + } + }() + t.Logf("Started proxy at: %q", pLis.Addr().String()) + t.Cleanup(p.stop) + return p +} diff --git a/internal/testutils/resolver.go b/internal/testutils/resolver.go new file mode 100644 index 0000000..8a4debe --- /dev/null +++ b/internal/testutils/resolver.go @@ -0,0 +1,70 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Logger wraps the logging methods from testing.T. +type Logger interface { + Log(args ...any) + Logf(format string, args ...any) + Errorf(format string, args ...any) +} + +// ResolverClientConn is a fake implementation of the resolver.ClientConn +// interface to be used in tests. +type ResolverClientConn struct { + resolver.ClientConn // Embedding the interface to avoid implementing deprecated methods. + + Logger Logger // Tests should pass testing.T for this. + UpdateStateF func(resolver.State) error // Invoked when resolver pushes a state update. + ReportErrorF func(err error) // Invoked when resolver pushes an error. +} + +// UpdateState invokes the test specified callback with the update received from +// the resolver. If the callback returns a non-nil error, the same will be +// propagated to the resolver. +func (t *ResolverClientConn) UpdateState(s resolver.State) error { + t.Logger.Logf("testutils.ResolverClientConn: UpdateState(%s)", pretty.ToJSON(s)) + + if t.UpdateStateF != nil { + return t.UpdateStateF(s) + } + return nil +} + +// ReportError pushes the error received from the resolver on to ErrorCh. +func (t *ResolverClientConn) ReportError(err error) { + t.Logger.Logf("testutils.ResolverClientConn: ReportError(%v)", err) + + if t.ReportErrorF != nil { + t.ReportErrorF(err) + } +} + +// ParseServiceConfig parses the provided service by delegating the work to the +// implementation in the grpc package. +func (t *ResolverClientConn) ParseServiceConfig(jsonSC string) *serviceconfig.ParseResult { + return internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) +} diff --git a/internal/testutils/restartable_listener.go b/internal/testutils/restartable_listener.go new file mode 100644 index 0000000..1849b9c --- /dev/null +++ b/internal/testutils/restartable_listener.go @@ -0,0 +1,102 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "net" + "sync" +) + +type tempError struct{} + +func (*tempError) Error() string { + return "restartable listener temporary error" +} +func (*tempError) Temporary() bool { + return true +} + +// RestartableListener wraps a net.Listener and supports stopping and restarting +// the latter. +type RestartableListener struct { + lis net.Listener + + mu sync.Mutex + stopped bool + conns []net.Conn +} + +// NewRestartableListener returns a new RestartableListener wrapping l. +func NewRestartableListener(l net.Listener) *RestartableListener { + return &RestartableListener{lis: l} +} + +// Accept waits for and returns the next connection to the listener. +// +// If the listener is currently not accepting new connections, because `Stop` +// was called on it, the connection is immediately closed after accepting +// without any bytes being sent on it. +func (l *RestartableListener) Accept() (net.Conn, error) { + conn, err := l.lis.Accept() + if err != nil { + return nil, err + } + + l.mu.Lock() + defer l.mu.Unlock() + if l.stopped { + conn.Close() + return nil, &tempError{} + } + l.conns = append(l.conns, conn) + return conn, nil +} + +// Close closes the listener. +func (l *RestartableListener) Close() error { + return l.lis.Close() +} + +// Addr returns the listener's network address. +func (l *RestartableListener) Addr() net.Addr { + return l.lis.Addr() +} + +// Stop closes existing connections on the listener and prevents new connections +// from being accepted. +func (l *RestartableListener) Stop() { + logger.Infof("Stopping restartable listener %q", l.Addr()) + + l.mu.Lock() + l.stopped = true + for _, conn := range l.conns { + conn.Close() + } + l.conns = nil + l.mu.Unlock() +} + +// Restart gets a previously stopped listener to start accepting connections. +func (l *RestartableListener) Restart() { + logger.Infof("Restarting listener %q", l.Addr()) + + l.mu.Lock() + l.stopped = false + l.mu.Unlock() +} diff --git a/internal/testutils/rls/fake_rls_server.go b/internal/testutils/rls/fake_rls_server.go new file mode 100644 index 0000000..789a208 --- /dev/null +++ b/internal/testutils/rls/fake_rls_server.go @@ -0,0 +1,134 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package rls contains utilities for RouteLookupService e2e tests. +package rls + +import ( + "context" + "net" + "sync" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + rlsgrpc "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/status" +) + +// RouteLookupResponse wraps an RLS response and the associated error to be sent +// to a client when the RouteLookup RPC is invoked. +type RouteLookupResponse struct { + Resp *rlspb.RouteLookupResponse + Err error +} + +// SetupFakeRLSServer starts and returns a fake RouteLookupService server +// listening on the given listener or on a random local port. Also returns a +// channel for tests to get notified whenever the RouteLookup RPC is invoked on +// the fake server. +// +// This function sets up the fake server to respond with an empty response for +// the RouteLookup RPCs. Tests can override this by calling the +// SetResponseCallback() method on the returned fake server. +func SetupFakeRLSServer(t *testing.T, lis net.Listener, opts ...grpc.ServerOption) (*FakeRouteLookupServer, chan struct{}) { + s, cancel := StartFakeRouteLookupServer(t, lis, opts...) + t.Logf("Started fake RLS server at %q", s.Address) + + ch := make(chan struct{}, 1) + s.SetRequestCallback(func(*rlspb.RouteLookupRequest) { + select { + case ch <- struct{}{}: + default: + } + }) + t.Cleanup(cancel) + return s, ch +} + +// FakeRouteLookupServer is a fake implementation of the RouteLookupService. +// +// It is safe for concurrent use. +type FakeRouteLookupServer struct { + rlsgrpc.UnimplementedRouteLookupServiceServer + Address string + + mu sync.Mutex + respCb func(context.Context, *rlspb.RouteLookupRequest) *RouteLookupResponse + reqCb func(*rlspb.RouteLookupRequest) +} + +// StartFakeRouteLookupServer starts a fake RLS server listening for requests on +// lis. If lis is nil, it creates a new listener on a random local port. The +// returned cancel function should be invoked by the caller upon completion of +// the test. +func StartFakeRouteLookupServer(t *testing.T, lis net.Listener, opts ...grpc.ServerOption) (*FakeRouteLookupServer, func()) { + t.Helper() + + if lis == nil { + var err error + lis, err = testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + } + + s := &FakeRouteLookupServer{Address: lis.Addr().String()} + server := grpc.NewServer(opts...) + rlsgrpc.RegisterRouteLookupServiceServer(server, s) + go server.Serve(lis) + return s, func() { server.Stop() } +} + +// RouteLookup implements the RouteLookupService. +func (s *FakeRouteLookupServer) RouteLookup(ctx context.Context, req *rlspb.RouteLookupRequest) (*rlspb.RouteLookupResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.reqCb != nil { + s.reqCb(req) + } + if err := ctx.Err(); err != nil { + return nil, status.Error(codes.DeadlineExceeded, err.Error()) + } + if s.respCb == nil { + return &rlspb.RouteLookupResponse{}, nil + } + resp := s.respCb(ctx, req) + return resp.Resp, resp.Err +} + +// SetResponseCallback sets a callback to be invoked on every RLS request. If +// this callback is set, the response returned by the fake server depends on the +// value returned by the callback. If this callback is not set, the fake server +// responds with an empty response. +func (s *FakeRouteLookupServer) SetResponseCallback(f func(context.Context, *rlspb.RouteLookupRequest) *RouteLookupResponse) { + s.mu.Lock() + s.respCb = f + s.mu.Unlock() +} + +// SetRequestCallback sets a callback to be invoked on every RLS request. The +// callback is given the incoming request, and tests can use this to verify that +// the request matches its expectations. +func (s *FakeRouteLookupServer) SetRequestCallback(f func(*rlspb.RouteLookupRequest)) { + s.mu.Lock() + s.reqCb = f + s.mu.Unlock() +} diff --git a/internal/testutils/roundrobin/roundrobin.go b/internal/testutils/roundrobin/roundrobin.go new file mode 100644 index 0000000..f5b3d59 --- /dev/null +++ b/internal/testutils/roundrobin/roundrobin.go @@ -0,0 +1,286 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package roundrobin contains helper functions to check for roundrobin and +// weighted-roundrobin load balancing of RPCs in tests. +package roundrobin + +import ( + "context" + "fmt" + "math" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "gonum.org/v1/gonum/stat/distuv" + "google.golang.org/grpc" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var logger = grpclog.Component("testutils-roundrobin") + +// waitForTrafficToReachBackends repeatedly makes RPCs using the provided +// TestServiceClient until RPCs reach all backends specified in addrs, or the +// context expires, in which case a non-nil error is returned. +func waitForTrafficToReachBackends(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error { + // Make sure connections to all backends are up. We need to do this two + // times (to be sure that round_robin has kicked in) because the channel + // could have been configured with a different LB policy before the switch + // to round_robin. And the previous LB policy could be sharing backends with + // round_robin, and therefore in the first iteration of this loop, RPCs + // could land on backends owned by the previous LB policy. + for j := 0; j < 2; j++ { + for i := 0; i < len(addrs); i++ { + for { + time.Sleep(time.Millisecond) + if ctx.Err() != nil { + return fmt.Errorf("timeout waiting for connection to %q to be up", addrs[i].Addr) + } + var peer peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + // Some tests remove backends and check if round robin is + // happening across the remaining backends. In such cases, + // RPCs can initially fail on the connection using the + // removed backend. Just keep retrying and eventually the + // connection using the removed backend will shutdown and + // will be removed. + continue + } + if peer.Addr.String() == addrs[i].Addr { + break + } + } + } + } + return nil +} + +// CheckRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn, +// connected to a server exposing the test.grpc_testing.TestService, are +// roundrobined across the given backend addresses. +// +// Returns a non-nil error if context deadline expires before RPCs start to get +// roundrobined across the given backends. +func CheckRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error { + if err := waitForTrafficToReachBackends(ctx, client, addrs); err != nil { + return err + } + + // At this point, RPCs are getting successfully executed at the backends + // that we care about. To support duplicate addresses (in addrs) and + // backends being removed from the list of addresses passed to the + // roundrobin LB, we do the following: + // 1. Determine the count of RPCs that we expect each of our backends to + // receive per iteration. + // 2. Wait until the same pattern repeats a few times, or the context + // deadline expires. + wantAddrCount := make(map[string]int) + for _, addr := range addrs { + wantAddrCount[addr.Addr]++ + } + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + // Perform 3 more iterations. + var iterations [][]string + for i := 0; i < 3; i++ { + iteration := make([]string, len(addrs)) + for c := 0; c < len(addrs); c++ { + var peer peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + return fmt.Errorf("EmptyCall() = %v, want ", err) + } + iteration[c] = peer.Addr.String() + } + iterations = append(iterations, iteration) + } + // Ensure the first iteration contains all addresses in addrs. + gotAddrCount := make(map[string]int) + for _, addr := range iterations[0] { + gotAddrCount[addr]++ + } + if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" { + logger.Infof("non-roundrobin, got address count in one iteration: %v, want: %v, Diff: %s", gotAddrCount, wantAddrCount, diff) + continue + } + // Ensure all three iterations contain the same addresses. + if !cmp.Equal(iterations[0], iterations[1]) || !cmp.Equal(iterations[0], iterations[2]) { + logger.Infof("non-roundrobin, first iter: %v, second iter: %v, third iter: %v", iterations[0], iterations[1], iterations[2]) + continue + } + return nil + } + return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v", addrs) +} + +// CheckWeightedRoundRobinRPCs verifies that EmptyCall RPCs on the given +// ClientConn, connected to a server exposing the test.grpc_testing.TestService, +// are weighted roundrobined (with randomness) across the given backend +// addresses. +// +// Returns a non-nil error if context deadline expires before RPCs start to get +// roundrobined across the given backends. +func CheckWeightedRoundRobinRPCs(ctx context.Context, t *testing.T, client testgrpc.TestServiceClient, addrs []resolver.Address) error { + if err := waitForTrafficToReachBackends(ctx, client, addrs); err != nil { + return err + } + + // At this point, RPCs are getting successfully executed at the backends + // that we care about. To take the randomness of the WRR into account, we + // look for approximate distribution instead of exact. + wantAddrCount := make(map[string]int) + for _, addr := range addrs { + wantAddrCount[addr.Addr]++ + } + attemptCount := attemptCounts(wantAddrCount) + + expectedCount := make(map[string]float64) + for addr, count := range wantAddrCount { + expectedCount[addr] = float64(count) / float64(len(addrs)) * float64(attemptCount) + } + + // There is a small possibility that RPCs are reaching backends that we + // don't expect them to reach here. The can happen because: + // - at time T0, the list of backends [A, B, C, D]. + // - at time T1, the test updates the list of backends to [A, B, C], and + // immediately starts attempting to check the distribution of RPCs to the + // new backends. + // - there is no way for the test to wait for a new picker to be pushed on + // to the channel (which contains the updated list of backends) before + // starting to attempt the RPC distribution checks. + // - This is usually a transitory state and will eventually fix itself when + // the new picker is pushed on the channel, and RPCs will start getting + // routed to only backends that we care about. + // + // We work around this situation by using two loops. The inner loop contains + // the meat of the calculations, and includes the logic which factors out + // the randomness in weighted roundrobin. If we ever see an RPCs getting + // routed to a backend that we don't expect it to get routed to, we break + // from the inner loop thereby resetting all state and start afresh. + for { + observedCount := make(map[string]float64) + InnerLoop: + for { + if ctx.Err() != nil { + return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v", addrs) + } + for i := 0; i < attemptCount; i++ { + var peer peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + return fmt.Errorf("EmptyCall() = %v, want ", err) + } + if addr := peer.Addr.String(); wantAddrCount[addr] == 0 { + break InnerLoop + } + observedCount[peer.Addr.String()]++ + } + + return pearsonsChiSquareTest(t, observedCount, expectedCount) + } + <-time.After(time.Millisecond) + } +} + +// attemptCounts returns the number of attempts needed to verify the expected +// distribution of RPCs. Having more attempts per category will give the test +// a greater ability to detect a small but real deviation from the expected +// distribution. +func attemptCounts(wantAddrWeights map[string]int) int { + if len(wantAddrWeights) == 0 { + return 0 + } + + totalWeight := 0 + minWeight := -1 + + for _, weight := range wantAddrWeights { + totalWeight += weight + if minWeight == -1 || weight < minWeight { + minWeight = weight + } + } + + minRatio := float64(minWeight) / float64(totalWeight) + + // We want the expected count for the smallest category to be at least 500. + // ExpectedCount = TotalAttempts * minRatio + // So, 500 <= TotalAttempts * minRatio + // which means TotalAttempts >= 500 / minRatio + const minExpectedPerCategory = 500.0 + requiredAttempts := minExpectedPerCategory / minRatio + + return int(math.Ceil(requiredAttempts)) +} + +// pearsonsChiSquareTest checks if the observed counts match the expected +// counts. +// Pearson's Chi-Squared Test Formula: +// +// χ² = ∑ (Oᵢ - Eᵢ)² / Eᵢ +// +// Where: +// - χ² is the chi-square statistic +// - Oᵢ is the observed count for category i +// - Eᵢ is the expected count for category i +// - The sum is over all categories (i = 1 to k) +// +// This tests how well the observed distribution matches the expected one. +// Larger χ² values indicate a greater difference between observed and expected +// counts. The p-value is computed as P(χ² ≥ computed value) under the +// chi-square distribution with degrees of freedom: +// df = number of categories - 1 +// See https://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test for more +// details. +func pearsonsChiSquareTest(t *testing.T, observedCounts, expectedCounts map[string]float64) error { + chiSquaredStat := 0.0 + for addr, want := range expectedCounts { + got := observedCounts[addr] + chiSquaredStat += (got - want) * (got - want) / want + } + degreesOfFreedom := len(expectedCounts) - 1 + const alpha = 1e-6 + chiSquareDist := distuv.ChiSquared{K: float64(degreesOfFreedom)} + pValue := chiSquareDist.Survival(chiSquaredStat) + t.Logf("Observed ratio: %v", observedCounts) + t.Logf("Expected ratio: %v", expectedCounts) + t.Logf("χ² statistic: %.4f", chiSquaredStat) + t.Logf("Degrees of freedom: %d\n", degreesOfFreedom) + t.Logf("p-value: %.10f\n", pValue) + // Alpha (α) is the threshold we use to decide if the test "fails". + // It controls how sensitive the chi-square test is. + // + // A smaller alpha means we require stronger evidence to say the load + // balancing is wrong. A larger alpha makes the test more likely to fail, + // even for small random fluctuations. + // + // For CI, we set alpha = 1e-6 to avoid flaky test failures. + // That means: + // - There's only a 1-in-a-million chance the test fails due to random + // variation, assuming the load balancer is working correctly. + // - If the test *does* fail, it's very likely there's a real bug. + // + // TL;DR: smaller alpha = stricter test, fewer false alarms. + if pValue > alpha { + return nil + } + return fmt.Errorf("observed distribution significantly differs from expectations, observeredCounts: %v, expectedCounts: %v", observedCounts, expectedCounts) +} diff --git a/internal/testutils/state.go b/internal/testutils/state.go new file mode 100644 index 0000000..79510b7 --- /dev/null +++ b/internal/testutils/state.go @@ -0,0 +1,85 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "context" + "testing" + + "google.golang.org/grpc/connectivity" +) + +// A StateChanger reports state changes, e.g. a grpc.ClientConn. +type StateChanger interface { + // Connect begins connecting the StateChanger. + Connect() + // GetState returns the current state of the StateChanger. + GetState() connectivity.State + // WaitForStateChange returns true when the state becomes s, or returns + // false if ctx is canceled first. + WaitForStateChange(ctx context.Context, s connectivity.State) bool +} + +// StayConnected makes sc stay connected by repeatedly calling sc.Connect() +// until the state becomes Shutdown or until the context expires. +func StayConnected(ctx context.Context, sc StateChanger) { + for { + state := sc.GetState() + switch state { + case connectivity.Idle: + sc.Connect() + case connectivity.Shutdown: + return + } + if !sc.WaitForStateChange(ctx, state) { + return + } + } +} + +// AwaitState waits for sc to enter stateWant or fatal errors if it doesn't +// happen before ctx expires. +func AwaitState(ctx context.Context, t *testing.T, sc StateChanger, stateWant connectivity.State) { + t.Helper() + for state := sc.GetState(); state != stateWant; state = sc.GetState() { + if !sc.WaitForStateChange(ctx, state) { + t.Fatalf("Timed out waiting for state change. got %v; want %v", state, stateWant) + } + } +} + +// AwaitNotState waits for sc to leave stateDoNotWant or fatal errors if it +// doesn't happen before ctx expires. +func AwaitNotState(ctx context.Context, t *testing.T, sc StateChanger, stateDoNotWant connectivity.State) { + t.Helper() + for state := sc.GetState(); state == stateDoNotWant; state = sc.GetState() { + if !sc.WaitForStateChange(ctx, state) { + t.Fatalf("Timed out waiting for state change. got %v; want NOT %v", state, stateDoNotWant) + } + } +} + +// AwaitNoStateChange expects ctx to be canceled before sc's state leaves +// currState, and fatal errors otherwise. +func AwaitNoStateChange(ctx context.Context, t *testing.T, sc StateChanger, currState connectivity.State) { + t.Helper() + if sc.WaitForStateChange(ctx, currState) { + t.Fatalf("State changed from %q to %q when no state change was expected", currState, sc.GetState()) + } +} diff --git a/internal/testutils/stats/test_metrics_recorder.go b/internal/testutils/stats/test_metrics_recorder.go new file mode 100644 index 0000000..7fc4925 --- /dev/null +++ b/internal/testutils/stats/test_metrics_recorder.go @@ -0,0 +1,323 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package stats implements a TestMetricsRecorder utility. +package stats + +import ( + "context" + "fmt" + "sync" + + "github.com/google/go-cmp/cmp" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/stats" +) + +// TestMetricsRecorder is a MetricsRecorder to be used in tests. It sends +// recording events on channels and provides helpers to check if certain events +// have taken place. It also persists metrics data keyed on the metrics +// descriptor. +type TestMetricsRecorder struct { + estats.UnimplementedMetricsRecorder + intCountCh *testutils.Channel + floatCountCh *testutils.Channel + intHistoCh *testutils.Channel + floatHistoCh *testutils.Channel + intGaugeCh *testutils.Channel + intUpDownCountCh *testutils.Channel + + // mu protects data. + mu sync.Mutex + // data is the most recent update for each metric name. + data map[string]float64 +} + +// NewTestMetricsRecorder returns a new TestMetricsRecorder. +func NewTestMetricsRecorder() *TestMetricsRecorder { + return &TestMetricsRecorder{ + intCountCh: testutils.NewChannelWithSize(10), + floatCountCh: testutils.NewChannelWithSize(10), + intHistoCh: testutils.NewChannelWithSize(10), + floatHistoCh: testutils.NewChannelWithSize(10), + intGaugeCh: testutils.NewChannelWithSize(10), + intUpDownCountCh: testutils.NewChannelWithSize(10), + + data: make(map[string]float64), + } +} + +// Metric returns the most recent data for a metric, and whether this recorder +// has received data for a metric. +func (r *TestMetricsRecorder) Metric(name string) (float64, bool) { + r.mu.Lock() + defer r.mu.Unlock() + data, ok := r.data[name] + return data, ok +} + +// ClearMetrics clears the metrics data store of the test metrics recorder. +func (r *TestMetricsRecorder) ClearMetrics() { + r.mu.Lock() + defer r.mu.Unlock() + r.data = make(map[string]float64) +} + +// MetricsData represents data associated with a metric. +type MetricsData struct { + Handle *estats.MetricDescriptor + + // Only set based on the type of metric. So only one of IntIncr or FloatIncr + // is set. + IntIncr int64 + FloatIncr float64 + + LabelKeys []string + LabelVals []string +} + +// WaitForInt64Count waits for an int64 count metric to be recorded and verifies +// that the recorded metrics data matches the expected metricsDataWant. Returns +// an error if failed to wait or received wrong data. +func (r *TestMetricsRecorder) WaitForInt64Count(ctx context.Context, metricsDataWant MetricsData) error { + got, err := r.intCountCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout waiting for int64Count") + } + metricsDataGot := got.(MetricsData) + if diff := cmp.Diff(metricsDataGot, metricsDataWant); diff != "" { + return fmt.Errorf("int64count metricsData received unexpected value (-got, +want): %v", diff) + } + return nil +} + +// WaitForInt64CountIncr waits for an int64 count metric to be recorded and +// verifies that the recorded metrics data incr matches the expected incr. +// Returns an error if failed to wait or received wrong data. +func (r *TestMetricsRecorder) WaitForInt64CountIncr(ctx context.Context, incrWant int64) error { + got, err := r.intCountCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout waiting for int64Count") + } + metricsDataGot := got.(MetricsData) + if diff := cmp.Diff(metricsDataGot.IntIncr, incrWant); diff != "" { + return fmt.Errorf("int64count metricsData received unexpected value (-got, +want): %v", diff) + } + return nil +} + +// RecordInt64Count sends the metrics data to the intCountCh channel and updates +// the internal data map with the recorded value. +func (r *TestMetricsRecorder) RecordInt64Count(handle *estats.Int64CountHandle, incr int64, labels ...string) { + r.intCountCh.ReceiveOrFail() + r.intCountCh.Send(MetricsData{ + Handle: handle.Descriptor(), + IntIncr: incr, + LabelKeys: append(handle.Labels, handle.OptionalLabels...), + LabelVals: labels, + }) + + r.mu.Lock() + defer r.mu.Unlock() + r.data[handle.Name] = float64(incr) +} + +// RecordInt64UpDownCount sends the metrics data to the intUpDownCountCh channel and updates +// the internal data map with the recorded value. +func (r *TestMetricsRecorder) RecordInt64UpDownCount(handle *estats.Int64UpDownCountHandle, incr int64, labels ...string) { + r.intUpDownCountCh.ReceiveOrFail() + r.intUpDownCountCh.Send(MetricsData{ + Handle: handle.Descriptor(), + IntIncr: incr, + LabelKeys: append(handle.Labels, handle.OptionalLabels...), + LabelVals: labels, + }) + + r.mu.Lock() + defer r.mu.Unlock() + r.data[handle.Name] = float64(incr) +} + +// WaitForFloat64Count waits for a float count metric to be recorded and +// verifies that the recorded metrics data matches the expected metricsDataWant. +// Returns an error if failed to wait or received wrong data. +func (r *TestMetricsRecorder) WaitForFloat64Count(ctx context.Context, metricsDataWant MetricsData) error { + got, err := r.floatCountCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout waiting for float64Count") + } + metricsDataGot := got.(MetricsData) + if diff := cmp.Diff(metricsDataGot, metricsDataWant); diff != "" { + return fmt.Errorf("float64count metricsData received unexpected value (-got, +want): %v", diff) + } + return nil +} + +// RecordFloat64Count sends the metrics data to the floatCountCh channel and +// updates the internal data map with the recorded value. +func (r *TestMetricsRecorder) RecordFloat64Count(handle *estats.Float64CountHandle, incr float64, labels ...string) { + r.floatCountCh.ReceiveOrFail() + r.floatCountCh.Send(MetricsData{ + Handle: handle.Descriptor(), + FloatIncr: incr, + LabelKeys: append(handle.Labels, handle.OptionalLabels...), + LabelVals: labels, + }) + + r.mu.Lock() + defer r.mu.Unlock() + r.data[handle.Name] = incr +} + +// WaitForInt64Histo waits for an int histo metric to be recorded and verifies +// that the recorded metrics data matches the expected metricsDataWant. Returns +// an error if failed to wait or received wrong data. +func (r *TestMetricsRecorder) WaitForInt64Histo(ctx context.Context, metricsDataWant MetricsData) error { + got, err := r.intHistoCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout waiting for int64Histo") + } + metricsDataGot := got.(MetricsData) + if diff := cmp.Diff(metricsDataGot, metricsDataWant); diff != "" { + return fmt.Errorf("int64Histo metricsData received unexpected value (-got, +want): %v", diff) + } + return nil +} + +// RecordInt64Histo sends the metrics data to the intHistoCh channel and updates +// the internal data map with the recorded value. +func (r *TestMetricsRecorder) RecordInt64Histo(handle *estats.Int64HistoHandle, incr int64, labels ...string) { + r.intHistoCh.ReceiveOrFail() + r.intHistoCh.Send(MetricsData{ + Handle: handle.Descriptor(), + IntIncr: incr, + LabelKeys: append(handle.Labels, handle.OptionalLabels...), + LabelVals: labels, + }) + + r.mu.Lock() + defer r.mu.Unlock() + r.data[handle.Name] = float64(incr) +} + +// WaitForFloat64Histo waits for a float histo metric to be recorded and +// verifies that the recorded metrics data matches the expected metricsDataWant. +// Returns an error if failed to wait or received wrong data. +func (r *TestMetricsRecorder) WaitForFloat64Histo(ctx context.Context, metricsDataWant MetricsData) error { + got, err := r.floatHistoCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout waiting for float64Histo") + } + metricsDataGot := got.(MetricsData) + if diff := cmp.Diff(metricsDataGot, metricsDataWant); diff != "" { + return fmt.Errorf("float64Histo metricsData received unexpected value (-got, +want): %v", diff) + } + return nil +} + +// RecordFloat64Histo sends the metrics data to the floatHistoCh channel and +// updates the internal data map with the recorded value. +func (r *TestMetricsRecorder) RecordFloat64Histo(handle *estats.Float64HistoHandle, incr float64, labels ...string) { + r.floatHistoCh.ReceiveOrFail() + r.floatHistoCh.Send(MetricsData{ + Handle: handle.Descriptor(), + FloatIncr: incr, + LabelKeys: append(handle.Labels, handle.OptionalLabels...), + LabelVals: labels, + }) + + r.mu.Lock() + defer r.mu.Unlock() + r.data[handle.Name] = incr +} + +// WaitForInt64Gauge waits for a int gauge metric to be recorded and verifies +// that the recorded metrics data matches the expected metricsDataWant. +func (r *TestMetricsRecorder) WaitForInt64Gauge(ctx context.Context, metricsDataWant MetricsData) error { + got, err := r.intGaugeCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout waiting for int64Gauge") + } + metricsDataGot := got.(MetricsData) + if diff := cmp.Diff(metricsDataGot, metricsDataWant); diff != "" { + return fmt.Errorf("int64Gauge metricsData received unexpected value (-got, +want): %v", diff) + } + return nil +} + +// RecordInt64Gauge sends the metrics data to the intGaugeCh channel and updates +// the internal data map with the recorded value. +func (r *TestMetricsRecorder) RecordInt64Gauge(handle *estats.Int64GaugeHandle, incr int64, labels ...string) { + r.intGaugeCh.ReceiveOrFail() + r.intGaugeCh.Send(MetricsData{ + Handle: handle.Descriptor(), + IntIncr: incr, + LabelKeys: append(handle.Labels, handle.OptionalLabels...), + LabelVals: labels, + }) + + r.mu.Lock() + defer r.mu.Unlock() + r.data[handle.Name] = float64(incr) +} + +// To implement a estats.AsyncMetricsRecorder, which allows it to be used in async metrics: + +// RecordInt64AsyncGauge sends the metrics data to the intGaugeCh channel and updates +// the internal data map with the recorded value. +func (r *TestMetricsRecorder) RecordInt64AsyncGauge(handle *estats.Int64AsyncGaugeHandle, incr int64, labels ...string) { + r.intGaugeCh.ReceiveOrFail() + r.intGaugeCh.Send(MetricsData{ + Handle: handle.Descriptor(), + IntIncr: incr, + LabelKeys: append(handle.Labels, handle.OptionalLabels...), + LabelVals: labels, + }) + + r.mu.Lock() + defer r.mu.Unlock() + r.data[handle.Name] = float64(incr) +} + +// ReadFloat64Histo waits for a float64 histogram metric to be recorded and returns its data. +func (r *TestMetricsRecorder) ReadFloat64Histo(ctx context.Context) (MetricsData, error) { + got, err := r.floatHistoCh.Receive(ctx) + if err != nil { + return MetricsData{}, err + } + return got.(MetricsData), nil +} + +// To implement a stats.Handler, which allows it to be set as a dial option: + +// TagRPC is TestMetricsRecorder's implementation of TagRPC. +func (r *TestMetricsRecorder) TagRPC(ctx context.Context, _ *stats.RPCTagInfo) context.Context { + return ctx +} + +// HandleRPC is TestMetricsRecorder's implementation of HandleRPC. +func (r *TestMetricsRecorder) HandleRPC(context.Context, stats.RPCStats) {} + +// TagConn is TestMetricsRecorder's implementation of TagConn. +func (r *TestMetricsRecorder) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +// HandleConn is TestMetricsRecorder's implementation of HandleConn. +func (r *TestMetricsRecorder) HandleConn(context.Context, stats.ConnStats) {} diff --git a/internal/testutils/status_equal.go b/internal/testutils/status_equal.go new file mode 100644 index 0000000..961e5a3 --- /dev/null +++ b/internal/testutils/status_equal.go @@ -0,0 +1,38 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +// StatusErrEqual returns true iff both err1 and err2 wrap status.Status errors +// and their underlying status protos are equal. +func StatusErrEqual(err1, err2 error) bool { + status1, ok := status.FromError(err1) + if !ok { + return false + } + status2, ok := status.FromError(err2) + if !ok { + return false + } + return proto.Equal(status1.Proto(), status2.Proto()) +} diff --git a/internal/testutils/status_equal_test.go b/internal/testutils/status_equal_test.go new file mode 100644 index 0000000..0c1db3c --- /dev/null +++ b/internal/testutils/status_equal_test.go @@ -0,0 +1,66 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "testing" + + spb "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +var statusErr = status.ErrorProto(&spb.Status{ + Code: int32(codes.DataLoss), + Message: "error for testing", + Details: []*anypb.Any{{ + TypeUrl: "url", + Value: []byte{6, 0, 0, 6, 1, 3}, + }}, +}) + +func (s) TestStatusErrEqual(t *testing.T) { + tests := []struct { + name string + err1 error + err2 error + wantEqual bool + }{ + {"nil errors", nil, nil, true}, + {"equal OK status", status.New(codes.OK, "").Err(), status.New(codes.OK, "").Err(), true}, + {"equal status errors", statusErr, statusErr, true}, + {"different status errors", statusErr, status.New(codes.OK, "").Err(), false}, + } + + for _, test := range tests { + if gotEqual := StatusErrEqual(test.err1, test.err2); gotEqual != test.wantEqual { + t.Errorf("%v: StatusErrEqual(%v, %v) = %v, want %v", test.name, test.err1, test.err2, gotEqual, test.wantEqual) + } + } +} diff --git a/internal/testutils/stubstatshandler.go b/internal/testutils/stubstatshandler.go new file mode 100644 index 0000000..3dd4c01 --- /dev/null +++ b/internal/testutils/stubstatshandler.go @@ -0,0 +1,65 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "context" + + "google.golang.org/grpc/stats" +) + +// StubStatsHandler is a stats handler that is easy to customize within +// individual test cases. It is a stubbable implementation of +// google.golang.org/grpc/stats.Handler for testing purposes. +type StubStatsHandler struct { + TagRPCF func(ctx context.Context, info *stats.RPCTagInfo) context.Context + HandleRPCF func(ctx context.Context, info stats.RPCStats) + TagConnF func(ctx context.Context, info *stats.ConnTagInfo) context.Context + HandleConnF func(ctx context.Context, info stats.ConnStats) +} + +// TagRPC calls the StubStatsHandler's TagRPCF, if set. +func (ssh *StubStatsHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + if ssh.TagRPCF != nil { + return ssh.TagRPCF(ctx, info) + } + return ctx +} + +// HandleRPC calls the StubStatsHandler's HandleRPCF, if set. +func (ssh *StubStatsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { + if ssh.HandleRPCF != nil { + ssh.HandleRPCF(ctx, rs) + } +} + +// TagConn calls the StubStatsHandler's TagConnF, if set. +func (ssh *StubStatsHandler) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context { + if ssh.TagConnF != nil { + return ssh.TagConnF(ctx, info) + } + return ctx +} + +// HandleConn calls the StubStatsHandler's HandleConnF, if set. +func (ssh *StubStatsHandler) HandleConn(ctx context.Context, cs stats.ConnStats) { + if ssh.HandleConnF != nil { + ssh.HandleConnF(ctx, cs) + } +} diff --git a/internal/testutils/tls_creds.go b/internal/testutils/tls_creds.go new file mode 100644 index 0000000..680f688 --- /dev/null +++ b/internal/testutils/tls_creds.go @@ -0,0 +1,156 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package testutils + +import ( + "crypto/tls" + "crypto/x509" + "os" + "testing" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/testdata" +) + +// CreateClientTLSCredentials creates client-side TLS transport credentials +// using certificate and key files from testdata/x509 directory. +func CreateClientTLSCredentials(t *testing.T) credentials.TransportCredentials { + t.Helper() + + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/client1_cert.pem"), testdata.Path("x509/client1_key.pem")) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(x509/client1_cert.pem, x509/client1_key.pem) failed: %v", err) + } + b, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem")) + if err != nil { + t.Fatalf("os.ReadFile(x509/server_ca_cert.pem) failed: %v", err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(b) { + t.Fatal("Failed to append certificates") + } + return credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: roots, + ServerName: "x.test.example.com", + }) +} + +// CreateServerTLSCredentials creates server-side TLS transport credentials +// using certificate and key files from testdata/x509 directory. +func CreateServerTLSCredentials(t *testing.T, clientAuth tls.ClientAuthType) credentials.TransportCredentials { + t.Helper() + + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(x509/server1_cert.pem, x509/server1_key.pem) failed: %v", err) + } + b, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem")) + if err != nil { + t.Fatalf("os.ReadFile(x509/client_ca_cert.pem) failed: %v", err) + } + ca := x509.NewCertPool() + if !ca.AppendCertsFromPEM(b) { + t.Fatal("Failed to append certificates") + } + return credentials.NewTLS(&tls.Config{ + ClientAuth: clientAuth, + Certificates: []tls.Certificate{cert}, + ClientCAs: ca, + }) +} + +// CreateServerTLSCredentialsCompatibleWithSPIFFE creates server-side TLS +// transport credentials using certificate and key files from the +// testdata/spiffe_end2end directory. These credentials are compatible with the +// SPIFFE trust bundles used on the client side. +func CreateServerTLSCredentialsCompatibleWithSPIFFE(t *testing.T, clientAuth tls.ClientAuthType) credentials.TransportCredentials { + t.Helper() + + cert, err := tls.LoadX509KeyPair(testdata.Path("spiffe_end2end/server_spiffe.pem"), testdata.Path("spiffe_end2end/server.key")) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(spiffe_end2end/server_spiffe.pem, spiffe_end2end/server.key) failed: %v", err) + } + b, err := os.ReadFile(testdata.Path("spiffe_end2end/ca.pem")) + if err != nil { + t.Fatalf("os.ReadFile(spiffe_end2end/ca.pem) failed: %v", err) + } + ca := x509.NewCertPool() + if !ca.AppendCertsFromPEM(b) { + t.Fatal("Failed to append certificates") + } + return credentials.NewTLS(&tls.Config{ + ClientAuth: clientAuth, + Certificates: []tls.Certificate{cert}, + ClientCAs: ca, + }) +} + +// CreateServerTLSCredentialsCompatibleWithSPIFFEChain creates server-side TLS +// transport credentials using a certificate chain and key files from the +// testdata/spiffe_end2end directory. These credentials are compatible with the +// SPIFFE trust bundles used on the client side. +func CreateServerTLSCredentialsCompatibleWithSPIFFEChain(t *testing.T, clientAuth tls.ClientAuthType) credentials.TransportCredentials { + t.Helper() + + certs, err := tls.LoadX509KeyPair(testdata.Path("spiffe_end2end/leaf_and_intermediate_chain.pem"), testdata.Path("spiffe_end2end/leaf_signed_by_intermediate.key")) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(spiffe_end2end/leaf_and_intermediate_chain.pem, spiffe_end2end/leaf_signed_by_intermediate.key) failed: %v", err) + } + b, err := os.ReadFile(testdata.Path("spiffe_end2end/ca.pem")) + if err != nil { + t.Fatalf("os.ReadFile(spiffe_end2end/ca.pem) failed: %v", err) + } + ca := x509.NewCertPool() + if !ca.AppendCertsFromPEM(b) { + t.Fatal("Failed to append certificates") + } + return credentials.NewTLS(&tls.Config{ + ClientAuth: clientAuth, + Certificates: []tls.Certificate{certs}, + ClientCAs: ca, + }) +} + +// CreateServerTLSCredentialsValidSPIFFEButWrongCA creates server-side TLS +// transport credentials using certificate and key files from the +// testdata/spiffe directory rather than the testdata/spiffe_end2end directory. +// These credentials have the expected trust domains and SPIFFE IDs that are +// compatible with testdata/spiffe_end2end client files, but they are signed by +// a different CA and will thus fail the connection. +func CreateServerTLSCredentialsValidSPIFFEButWrongCA(t *testing.T, clientAuth tls.ClientAuthType) credentials.TransportCredentials { + t.Helper() + + cert, err := tls.LoadX509KeyPair(testdata.Path("spiffe/server1_spiffe.pem"), testdata.Path("server1.key")) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(spiffe/server1_spiffe.pem, spiffe/server.key) failed: %v", err) + } + b, err := os.ReadFile(testdata.Path("spiffe_end2end/ca.pem")) + if err != nil { + t.Fatalf("os.ReadFile(spiffe_end2end/ca.pem) failed: %v", err) + } + ca := x509.NewCertPool() + if !ca.AppendCertsFromPEM(b) { + t.Fatal("Failed to append certificates") + } + return credentials.NewTLS(&tls.Config{ + ClientAuth: clientAuth, + Certificates: []tls.Certificate{cert}, + ClientCAs: ca, + }) +} diff --git a/internal/testutils/wrappers.go b/internal/testutils/wrappers.go new file mode 100644 index 0000000..1770b52 --- /dev/null +++ b/internal/testutils/wrappers.go @@ -0,0 +1,74 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "net" + "testing" +) + +// ConnWrapper wraps a net.Conn and pushes on a channel when closed. +type ConnWrapper struct { + net.Conn + CloseCh *Channel +} + +// Close closes the connection and sends a value on the close channel. +func (cw *ConnWrapper) Close() error { + err := cw.Conn.Close() + cw.CloseCh.Replace(nil) + return err +} + +// ListenerWrapper wraps a net.Listener and the returned net.Conn. +// +// It pushes on a channel whenever it accepts a new connection. +type ListenerWrapper struct { + net.Listener + NewConnCh *Channel +} + +// Accept wraps the Listener Accept and sends the accepted connection on a +// channel. +func (l *ListenerWrapper) Accept() (net.Conn, error) { + c, err := l.Listener.Accept() + if err != nil { + return nil, err + } + closeCh := NewChannel() + conn := &ConnWrapper{Conn: c, CloseCh: closeCh} + l.NewConnCh.Replace(conn) + return conn, nil +} + +// NewListenerWrapper returns a ListenerWrapper. +func NewListenerWrapper(t *testing.T, lis net.Listener) *ListenerWrapper { + if lis == nil { + var err error + lis, err = LocalTCPListener() + if err != nil { + t.Fatal(err) + } + } + + return &ListenerWrapper{ + Listener: lis, + NewConnCh: NewChannel(), + } +} diff --git a/internal/testutils/wrr.go b/internal/testutils/wrr.go new file mode 100644 index 0000000..5dcc7a8 --- /dev/null +++ b/internal/testutils/wrr.go @@ -0,0 +1,73 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "fmt" + "sync" + + "google.golang.org/grpc/internal/wrr" +) + +// testWRR is a deterministic WRR implementation. +// +// The real implementation does random WRR. testWRR makes the balancer behavior +// deterministic and easier to test. +// +// With {a: 2, b: 3}, the Next() results will be {a, a, b, b, b}. +type testWRR struct { + itemsWithWeight []struct { + item any + weight int64 + } + length int + + mu sync.Mutex + idx int // The index of the item that will be picked + count int64 // The number of times the current item has been picked. +} + +// NewTestWRR return a WRR for testing. It's deterministic instead of random. +func NewTestWRR() wrr.WRR { + return &testWRR{} +} + +func (twrr *testWRR) Add(item any, weight int64) { + twrr.itemsWithWeight = append(twrr.itemsWithWeight, struct { + item any + weight int64 + }{item: item, weight: weight}) + twrr.length++ +} + +func (twrr *testWRR) Next() any { + twrr.mu.Lock() + iww := twrr.itemsWithWeight[twrr.idx] + twrr.count++ + if twrr.count >= iww.weight { + twrr.idx = (twrr.idx + 1) % twrr.length + twrr.count = 0 + } + twrr.mu.Unlock() + return iww.item +} + +func (twrr *testWRR) String() string { + return fmt.Sprint(twrr.itemsWithWeight) +} diff --git a/internal/testutils/xds/e2e/bootstrap.go b/internal/testutils/xds/e2e/bootstrap.go new file mode 100644 index 0000000..768d32b --- /dev/null +++ b/internal/testutils/xds/e2e/bootstrap.go @@ -0,0 +1,196 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +import ( + "encoding/json" + "fmt" + "os" + "path" + "testing" + + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/testdata" +) + +// DefaultFileWatcherConfig is a helper function to create a default certificate +// provider plugin configuration. The test is expected to have setup the files +// appropriately before this configuration is used to instantiate providers. +func DefaultFileWatcherConfig(certPath, keyPath, caPath string) json.RawMessage { + return json.RawMessage(fmt.Sprintf(`{ + "plugin_name": "file_watcher", + "config": { + "certificate_file": %q, + "private_key_file": %q, + "ca_certificate_file": %q, + "refresh_interval": "600s" + } + }`, certPath, keyPath, caPath)) +} + +// SPIFFEFileWatcherConfig is a helper function to create a default certificate +// provider plugin configuration. The test is expected to have setup the files +// appropriately before this configuration is used to instantiate providers. +func SPIFFEFileWatcherConfig(certPath, keyPath, caPath, spiffeBundleMapPath string) json.RawMessage { + return json.RawMessage(fmt.Sprintf(`{ + "plugin_name": "file_watcher", + "config": { + "certificate_file": %q, + "private_key_file": %q, + "ca_certificate_file": %q, + "spiffe_trust_bundle_map_file": %q, + "refresh_interval": "600s" + } + }`, certPath, keyPath, caPath, spiffeBundleMapPath)) +} + +// SPIFFEBootstrapContents creates a bootstrap configuration with the given node +// ID and server URI. It also creates certificate provider configuration using +// SPIFFE certificates and sets the listener resource name template to be used +// on the server side. +func SPIFFEBootstrapContents(t *testing.T, nodeID, serverURI string) []byte { + t.Helper() + + // Create a directory to hold certs and key files used on the server side. + serverDir, err := createTmpDirWithCerts("testServerSideXDSSPIFFE*", "spiffe_end2end/server_spiffe.pem", "spiffe_end2end/server.key", "spiffe_end2end/ca.pem", "spiffe_end2end/server_spiffebundle.json") + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create a directory to hold certs and key files used on the client side. + clientDir, err := createTmpDirWithCerts("testClientSideXDSSPIFFE*", "spiffe_end2end/client_spiffe.pem", "spiffe_end2end/client.key", "spiffe_end2end/ca.pem", "spiffe_end2end/client_spiffebundle.json") + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create certificate providers section of the bootstrap config with entries + // for both the client and server sides. + cpc := map[string]json.RawMessage{ + ServerSideCertProviderInstance: SPIFFEFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile), path.Join(serverDir, spiffeBundleMapFile)), + ClientSideCertProviderInstance: SPIFFEFileWatcherConfig(path.Join(clientDir, certFile), path.Join(clientDir, keyFile), path.Join(clientDir, rootFile), path.Join(clientDir, spiffeBundleMapFile)), + } + + // Create the bootstrap configuration. + bs, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": "passthrough:///%s", + "channel_creds": [{"type": "insecure"}] + }]`, serverURI)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + CertificateProviders: cpc, + ServerListenerResourceNameTemplate: ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + return bs + +} + +// DefaultBootstrapContents creates a default bootstrap configuration with the +// given node ID and server URI. It also creates certificate provider +// configuration and sets the listener resource name template to be used on the +// server side. +func DefaultBootstrapContents(t *testing.T, nodeID, serverURI string) []byte { + t.Helper() + + // Create a directory to hold certs and key files used on the server side. + serverDir, err := createTmpDirWithCerts("testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem", "") + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create a directory to hold certs and key files used on the client side. + clientDir, err := createTmpDirWithCerts("testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem", "") + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create certificate providers section of the bootstrap config with entries + // for both the client and server sides. + cpc := map[string]json.RawMessage{ + ServerSideCertProviderInstance: DefaultFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile)), + ClientSideCertProviderInstance: DefaultFileWatcherConfig(path.Join(clientDir, certFile), path.Join(clientDir, keyFile), path.Join(clientDir, rootFile)), + } + + // Create the bootstrap configuration. + bs, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": "passthrough:///%s", + "channel_creds": [{"type": "insecure"}], + "server_features": ["trusted_xds_server"] + }]`, serverURI)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + CertificateProviders: cpc, + ServerListenerResourceNameTemplate: ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + return bs +} + +const ( + // Names of files inside tempdir, for certprovider plugin to watch. + certFile = "cert.pem" + keyFile = "key.pem" + rootFile = "ca.pem" + spiffeBundleMapFile = "spiffe_bundle_map.json" +) + +func createTmpFile(src, dst string) error { + data, err := os.ReadFile(src) + if err != nil { + return fmt.Errorf("os.ReadFile(%q) failed: %v", src, err) + } + if err := os.WriteFile(dst, data, os.ModePerm); err != nil { + return fmt.Errorf("os.WriteFile(%q) failed: %v", dst, err) + } + return nil +} + +// createTmpDirWithCerts creates a temporary directory under the system default +// tempDir with the given dirPattern. It also reads from certSrc, keySrc and +// rootSrc files and creates appropriate files under the newly create tempDir. +// Returns the path of the created tempDir if successful, and an error +// otherwise. +func createTmpDirWithCerts(dirPattern, certSrc, keySrc, rootSrc, spiffeBundleMapSrc string) (string, error) { + // Create a temp directory. Passing an empty string for the first argument + // uses the system temp directory. + dir, err := os.MkdirTemp("", dirPattern) + if err != nil { + return "", fmt.Errorf("os.MkdirTemp() failed: %v", err) + } + + if err := createTmpFile(testdata.Path(certSrc), path.Join(dir, certFile)); err != nil { + return "", err + } + if err := createTmpFile(testdata.Path(keySrc), path.Join(dir, keyFile)); err != nil { + return "", err + } + if err := createTmpFile(testdata.Path(rootSrc), path.Join(dir, rootFile)); err != nil { + return "", err + } + if spiffeBundleMapSrc != "" { + if err := createTmpFile(testdata.Path(spiffeBundleMapSrc), path.Join(dir, spiffeBundleMapFile)); err != nil { + return "", err + } + } + return dir, nil +} diff --git a/internal/testutils/xds/e2e/clientresources.go b/internal/testutils/xds/e2e/clientresources.go new file mode 100644 index 0000000..6ce8402 --- /dev/null +++ b/internal/testutils/xds/e2e/clientresources.go @@ -0,0 +1,846 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +import ( + "fmt" + "net" + "strconv" + + "github.com/envoyproxy/go-control-plane/pkg/wellknown" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +const ( + // ServerListenerResourceNameTemplate is the Listener resource name template + // used on the server side. + ServerListenerResourceNameTemplate = "grpc/server?xds.resource.listening_address=%s" + // ClientSideCertProviderInstance is the certificate provider instance name + // used in the Cluster resource on the client side. + ClientSideCertProviderInstance = "client-side-certificate-provider-instance" + // ServerSideCertProviderInstance is the certificate provider instance name + // used in the Listener resource on the server side. + ServerSideCertProviderInstance = "server-side-certificate-provider-instance" +) + +// SecurityLevel allows the test to control the security level to be used in the +// resource returned by this package. +type SecurityLevel int + +const ( + // SecurityLevelNone is used when no security configuration is required. + SecurityLevelNone SecurityLevel = iota + // SecurityLevelTLS is used when security configuration corresponding to TLS + // is required. Only the server presents an identity certificate in this + // configuration. + SecurityLevelTLS + // SecurityLevelMTLS is used when security configuration corresponding to + // mTLS is required. Both client and server present identity certificates in + // this configuration. + SecurityLevelMTLS + // SecurityLevelTLSWithSystemRootCerts is used when security configuration + // corresponding to TLS is required. Only the server presents an identity + // certificate in this configuration and the client uses system root certs + // to validate the server certificate. + SecurityLevelTLSWithSystemRootCerts +) + +// ResourceParams wraps the arguments to be passed to DefaultClientResources. +type ResourceParams struct { + // DialTarget is the client's dial target. This is used as the name of the + // Listener resource. + DialTarget string + // NodeID is the id of the xdsClient to which this update is to be pushed. + NodeID string + // Host is the host of the default Endpoint resource. + Host string + // port is the port of the default Endpoint resource. + Port uint32 + // SecLevel controls the security configuration in the Cluster resource. + SecLevel SecurityLevel +} + +// DefaultClientResources returns a set of resources (LDS, RDS, CDS, EDS) for a +// client to generically connect to one server. +func DefaultClientResources(params ResourceParams) UpdateOptions { + routeConfigName := "route-" + params.DialTarget + clusterName := "cluster-" + params.DialTarget + endpointsName := "endpoints-" + params.DialTarget + return UpdateOptions{ + NodeID: params.NodeID, + Listeners: []*v3listenerpb.Listener{DefaultClientListener(params.DialTarget, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{DefaultRouteConfig(routeConfigName, params.DialTarget, clusterName)}, + Clusters: []*v3clusterpb.Cluster{DefaultCluster(clusterName, endpointsName, params.SecLevel)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{DefaultEndpoint(endpointsName, params.Host, []uint32{params.Port})}, + } +} + +// RouterHTTPFilter is the HTTP Filter configuration for the Router filter. +var RouterHTTPFilter = HTTPFilter("router", &v3routerpb.Router{}) + +// DefaultClientListener returns a basic xds Listener resource to be used on +// the client side. +func DefaultClientListener(target, routeName string) *v3listenerpb.Listener { + hcm := marshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: routeName, + }}, + HttpFilters: []*v3httppb.HttpFilter{HTTPFilter("router", &v3routerpb.Router{})}, // router fields are unused by grpc + }) + return &v3listenerpb.Listener{ + Name: target, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + } +} + +func marshalAny(m proto.Message) *anypb.Any { + a, err := anypb.New(m) + if err != nil { + panic(fmt.Sprintf("anypb.New(%+v) failed: %v", m, err)) + } + return a +} + +// DefaultServerListener returns a basic xds Listener resource to be used on the +// server side. The returned Listener resource contains an inline route +// configuration with the name of routeName. +func DefaultServerListener(host string, port uint32, secLevel SecurityLevel, routeName string) *v3listenerpb.Listener { + return defaultServerListenerCommon(host, port, secLevel, routeName, true) +} + +func defaultServerListenerCommon(host string, port uint32, secLevel SecurityLevel, routeName string, inlineRouteConfig bool) *v3listenerpb.Listener { + var hcm *v3httppb.HttpConnectionManager + if inlineRouteConfig { + hcm = &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + // This "*" string matches on any incoming authority. This is to ensure any + // incoming RPC matches to Route_NonForwardingAction and will proceed as + // normal. + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}}, + }, + HttpFilters: []*v3httppb.HttpFilter{RouterHTTPFilter}, + } + } else { + hcm = &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: routeName, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{RouterHTTPFilter}, + } + } + + var tlsContext *v3tlspb.DownstreamTlsContext + switch secLevel { + case SecurityLevelNone: + case SecurityLevelTLS: + tlsContext = &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: ServerSideCertProviderInstance, + }, + }, + } + case SecurityLevelMTLS: + tlsContext = &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: ServerSideCertProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: ServerSideCertProviderInstance, + }, + }, + }, + } + } + + var ts *v3corepb.TransportSocket + if tlsContext != nil { + ts = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: marshalAny(tlsContext), + }, + } + } + return &v3listenerpb.Listener{ + Name: fmt.Sprintf(ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "v4-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: marshalAny(hcm)}, + }, + }, + TransportSocket: ts, + }, + { + Name: "v6-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: marshalAny(hcm)}, + }, + }, + TransportSocket: ts, + }, + }, + } +} + +// HTTPFilter constructs an xds HttpFilter with the provided name and config. +func HTTPFilter(name string, config proto.Message) *v3httppb.HttpFilter { + return &v3httppb.HttpFilter{ + Name: name, + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: marshalAny(config), + }, + } +} + +// DefaultRouteConfig returns a basic xds RouteConfig resource. +func DefaultRouteConfig(routeName, vhDomain, clusterName string) *v3routepb.RouteConfiguration { + return &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{vhDomain}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: clusterName, + Weight: &wrapperspb.UInt32Value{Value: 100}, + }, + }, + }}, + }}, + }}, + }}, + } +} + +// RouteConfigClusterSpecifierType determines the cluster specifier type for the +// route actions configured in the returned RouteConfiguration resource. +type RouteConfigClusterSpecifierType int + +const ( + // RouteConfigClusterSpecifierTypeCluster results in the cluster specifier + // being set to a RouteAction_Cluster. + RouteConfigClusterSpecifierTypeCluster RouteConfigClusterSpecifierType = iota + // RouteConfigClusterSpecifierTypeWeightedCluster results in the cluster + // specifier being set to RouteAction_WeightedClusters. + RouteConfigClusterSpecifierTypeWeightedCluster + // RouteConfigClusterSpecifierTypeClusterSpecifierPlugin results in the + // cluster specifier being set to a RouteAction_ClusterSpecifierPlugin. + RouteConfigClusterSpecifierTypeClusterSpecifierPlugin +) + +// RouteConfigOptions contains options to configure a RouteConfiguration +// resource. +type RouteConfigOptions struct { + // RouteConfigName is the name of the RouteConfiguration resource. + RouteConfigName string + // ListenerName is the name of the Listener resource which uses this + // RouteConfiguration. + ListenerName string + // ClusterSpecifierType determines the cluster specifier type. + ClusterSpecifierType RouteConfigClusterSpecifierType + // ClusterName is name of the cluster resource used when the cluster + // specifier type is set to RouteConfigClusterSpecifierTypeCluster. + // + // Default value of "A" is used if left unspecified. + ClusterName string + // WeightedClusters is a map from cluster name to weights, and is used when + // the cluster specifier type is set to + // RouteConfigClusterSpecifierTypeWeightedCluster. + // + // Default value of {"A": 75, "B": 25} is used if left unspecified. + WeightedClusters map[string]int + // The below two fields specify the name of the cluster specifier plugin and + // its configuration, and are used when the cluster specifier type is set to + // RouteConfigClusterSpecifierTypeClusterSpecifierPlugin. Tests are expected + // to provide valid values for these fields when appropriate. + ClusterSpecifierPluginName string + ClusterSpecifierPluginConfig *anypb.Any +} + +// RouteConfigResourceWithOptions returns a RouteConfiguration resource +// configured with the provided options. +func RouteConfigResourceWithOptions(opts RouteConfigOptions) *v3routepb.RouteConfiguration { + switch opts.ClusterSpecifierType { + case RouteConfigClusterSpecifierTypeCluster: + clusterName := opts.ClusterName + if clusterName == "" { + clusterName = "A" + } + return &v3routepb.RouteConfiguration{ + Name: opts.RouteConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{opts.ListenerName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }}, + }}, + }}, + } + case RouteConfigClusterSpecifierTypeWeightedCluster: + weightedClusters := opts.WeightedClusters + if weightedClusters == nil { + weightedClusters = map[string]int{"A": 75, "B": 25} + } + clusters := []*v3routepb.WeightedCluster_ClusterWeight{} + for name, weight := range weightedClusters { + clusters = append(clusters, &v3routepb.WeightedCluster_ClusterWeight{ + Name: name, + Weight: &wrapperspb.UInt32Value{Value: uint32(weight)}, + }) + } + return &v3routepb.RouteConfiguration{ + Name: opts.RouteConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{opts.ListenerName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{WeightedClusters: &v3routepb.WeightedCluster{Clusters: clusters}}, + }}, + }}, + }}, + } + case RouteConfigClusterSpecifierTypeClusterSpecifierPlugin: + return &v3routepb.RouteConfiguration{ + Name: opts.RouteConfigName, + ClusterSpecifierPlugins: []*v3routepb.ClusterSpecifierPlugin{{ + Extension: &v3corepb.TypedExtensionConfig{ + Name: opts.ClusterSpecifierPluginName, + TypedConfig: opts.ClusterSpecifierPluginConfig, + }}, + }, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{opts.ListenerName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_ClusterSpecifierPlugin{ClusterSpecifierPlugin: opts.ClusterSpecifierPluginName}, + }}, + }}, + }}, + } + default: + panic(fmt.Sprintf("unsupported cluster specifier plugin type: %v", opts.ClusterSpecifierType)) + } +} + +// DefaultCluster returns a basic xds Cluster resource. +func DefaultCluster(clusterName, edsServiceName string, secLevel SecurityLevel) *v3clusterpb.Cluster { + return ClusterResourceWithOptions(ClusterOptions{ + ClusterName: clusterName, + ServiceName: edsServiceName, + Policy: LoadBalancingPolicyRoundRobin, + SecurityLevel: secLevel, + }) +} + +// LoadBalancingPolicy determines the policy used for balancing load across +// endpoints in the Cluster. +type LoadBalancingPolicy int + +const ( + // LoadBalancingPolicyRoundRobin results in the use of the weighted_target + // LB policy to balance load across localities and endpoints in the cluster. + LoadBalancingPolicyRoundRobin LoadBalancingPolicy = iota + // LoadBalancingPolicyRingHash results in the use of the ring_hash LB policy + // as the leaf policy. + LoadBalancingPolicyRingHash +) + +// ClusterType specifies the type of the Cluster resource. +type ClusterType int + +const ( + // ClusterTypeEDS specifies a Cluster that uses EDS to resolve endpoints. + ClusterTypeEDS ClusterType = iota + // ClusterTypeLogicalDNS specifies a Cluster that uses DNS to resolve + // endpoints. + ClusterTypeLogicalDNS + // ClusterTypeAggregate specifies a Cluster that is made up of child + // clusters. + ClusterTypeAggregate +) + +// ClusterOptions contains options to configure a Cluster resource. +type ClusterOptions struct { + Type ClusterType + // ClusterName is the name of the Cluster resource. + ClusterName string + // ServiceName is the EDS service name of the Cluster. Applicable only when + // cluster type is EDS. + ServiceName string + // ChildNames is the list of child Cluster names. Applicable only when + // cluster type is Aggregate. + ChildNames []string + // DNSHostName is the dns host name of the Cluster. Applicable only when the + // cluster type is DNS. + DNSHostName string + // DNSPort is the port number of the Cluster. Applicable only when the + // cluster type is DNS. + DNSPort uint32 + // Policy is the LB policy to be used. + Policy LoadBalancingPolicy + // SecurityLevel determines the security configuration for the Cluster. + SecurityLevel SecurityLevel + // EnableLRS adds a load reporting configuration with a config source + // pointing to self. + EnableLRS bool +} + +// ClusterResourceWithOptions returns an xDS Cluster resource configured with +// the provided options. +func ClusterResourceWithOptions(opts ClusterOptions) *v3clusterpb.Cluster { + var tlsContext *v3tlspb.UpstreamTlsContext + switch opts.SecurityLevel { + case SecurityLevelNone: + case SecurityLevelTLS: + tlsContext = &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: ClientSideCertProviderInstance, + }, + }, + }, + } + case SecurityLevelMTLS: + tlsContext = &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: ClientSideCertProviderInstance, + }, + }, + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: ClientSideCertProviderInstance, + }, + }, + } + case SecurityLevelTLSWithSystemRootCerts: + tlsContext = &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + SystemRootCerts: &v3tlspb.CertificateValidationContext_SystemRootCerts{}, + }, + }, + }, + } + } + + var lbPolicy v3clusterpb.Cluster_LbPolicy + switch opts.Policy { + case LoadBalancingPolicyRoundRobin: + lbPolicy = v3clusterpb.Cluster_ROUND_ROBIN + case LoadBalancingPolicyRingHash: + lbPolicy = v3clusterpb.Cluster_RING_HASH + } + cluster := &v3clusterpb.Cluster{ + Name: opts.ClusterName, + LbPolicy: lbPolicy, + } + switch opts.Type { + case ClusterTypeEDS: + cluster.ClusterDiscoveryType = &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS} + cluster.EdsClusterConfig = &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: opts.ServiceName, + } + case ClusterTypeLogicalDNS: + cluster.ClusterDiscoveryType = &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_LOGICAL_DNS} + cluster.LoadAssignment = &v3endpointpb.ClusterLoadAssignment{ + Endpoints: []*v3endpointpb.LocalityLbEndpoints{{ + LbEndpoints: []*v3endpointpb.LbEndpoint{{ + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: opts.DNSHostName, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: opts.DNSPort, + }, + }, + }, + }, + }, + }, + }}, + }}, + } + case ClusterTypeAggregate: + cluster.ClusterDiscoveryType = &v3clusterpb.Cluster_ClusterType{ + ClusterType: &v3clusterpb.Cluster_CustomClusterType{ + Name: "envoy.clusters.aggregate", + TypedConfig: marshalAny(&v3aggregateclusterpb.ClusterConfig{ + Clusters: opts.ChildNames, + }), + }, + } + } + if tlsContext != nil { + cluster.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: marshalAny(tlsContext), + }, + } + } + if opts.EnableLRS { + cluster.LrsServer = &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + } + } + return cluster +} + +// LocalityID represents a locality identifier. +type LocalityID struct { + Region string + Zone string + SubZone string +} + +// LocalityOptions contains options to configure a Locality. +type LocalityOptions struct { + // Name is the unique locality name. + Name string + // Weight is the weight of the locality, used for load balancing. + Weight uint32 + // Backends is a set of backends belonging to this locality. + Backends []BackendOptions + // Priority is the priority of the locality. Defaults to 0. + Priority uint32 + // Locality is the locality identifier. If not specified, a random + // identifier is generated. + Locality LocalityID +} + +// BackendOptions contains options to configure individual backends in a +// locality. +type BackendOptions struct { + // Ports on which the backend is accepting connections. All backends + // are expected to run on localhost, hence host name is not stored here. + Ports []uint32 + // Health status of the backend. Default is UNKNOWN which is treated the + // same as HEALTHY. + HealthStatus v3corepb.HealthStatus + // Weight sets the backend weight. Defaults to 1. + Weight uint32 + // Hostname sets the endpoint hostname for authority rewriting. + Hostname string + // Metadata sets the LB endpoint metadata (envoy.lb FilterMetadata field). + // See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/base.proto#envoy-v3-api-msg-config-core-v3-metadata + Metadata map[string]any +} + +// EndpointOptions contains options to configure an Endpoint (or +// ClusterLoadAssignment) resource. +type EndpointOptions struct { + // ClusterName is the name of the Cluster resource (or EDS service name) + // containing the endpoints specified below. + ClusterName string + // Host is the hostname of the endpoints. In our e2e tests, hostname must + // always be "localhost". + Host string + // Localities is a set of localities belonging to this resource. + Localities []LocalityOptions + // DropPercents is a map from drop category to a drop percentage. If unset, + // no drops are configured. + DropPercents map[string]int +} + +// DefaultEndpoint returns a basic xds Endpoint resource. +func DefaultEndpoint(clusterName string, host string, ports []uint32) *v3endpointpb.ClusterLoadAssignment { + var bOpts []BackendOptions + for _, p := range ports { + bOpts = append(bOpts, BackendOptions{Ports: []uint32{p}, Weight: 1}) + } + return EndpointResourceWithOptions(EndpointOptions{ + ClusterName: clusterName, + Host: host, + Localities: []LocalityOptions{ + { + Backends: bOpts, + Weight: 1, + }, + }, + }) +} + +// EndpointResourceWithOptions returns an xds Endpoint resource configured with +// the provided options. +func EndpointResourceWithOptions(opts EndpointOptions) *v3endpointpb.ClusterLoadAssignment { + var endpoints []*v3endpointpb.LocalityLbEndpoints + for i, locality := range opts.Localities { + var lbEndpoints []*v3endpointpb.LbEndpoint + for _, b := range locality.Backends { + // Weight defaults to 1. + if b.Weight == 0 { + b.Weight = 1 + } + additionalAddresses := make([]*v3endpointpb.Endpoint_AdditionalAddress, len(b.Ports)-1) + for i, p := range b.Ports[1:] { + additionalAddresses[i] = &v3endpointpb.Endpoint_AdditionalAddress{ + Address: &v3corepb.Address{Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Protocol: v3corepb.SocketAddress_TCP, + Address: opts.Host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: p}, + }}, + }, + } + } + lbEndpoint := &v3endpointpb.LbEndpoint{ + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Protocol: v3corepb.SocketAddress_TCP, + Address: opts.Host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: b.Ports[0]}, + }, + }}, + Hostname: b.Hostname, + AdditionalAddresses: additionalAddresses, + }}, + HealthStatus: b.HealthStatus, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: b.Weight}, + } + + if b.Metadata != nil { + metadata, err := structpb.NewStruct(b.Metadata) + if err != nil { + panic(fmt.Sprintf("failed to marshal metadata: %v", err)) + } + lbEndpoint.Metadata = &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "envoy.lb": metadata, + }, + } + } + lbEndpoints = append(lbEndpoints, lbEndpoint) + } + + l := locality.Locality + if l == (LocalityID{}) { + l = LocalityID{ + Region: fmt.Sprintf("region-%d", i+1), + Zone: fmt.Sprintf("zone-%d", i+1), + SubZone: fmt.Sprintf("subzone-%d", i+1), + } + } + endpoints = append(endpoints, &v3endpointpb.LocalityLbEndpoints{ + Locality: &v3corepb.Locality{Region: l.Region, Zone: l.Zone, SubZone: l.SubZone}, + LbEndpoints: lbEndpoints, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: locality.Weight}, + Priority: locality.Priority, + }) + } + + cla := &v3endpointpb.ClusterLoadAssignment{ + ClusterName: opts.ClusterName, + Endpoints: endpoints, + } + + var drops []*v3endpointpb.ClusterLoadAssignment_Policy_DropOverload + for category, val := range opts.DropPercents { + drops = append(drops, &v3endpointpb.ClusterLoadAssignment_Policy_DropOverload{ + Category: category, + DropPercentage: &v3typepb.FractionalPercent{ + Numerator: uint32(val), + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }) + } + if len(drops) != 0 { + cla.Policy = &v3endpointpb.ClusterLoadAssignment_Policy{ + DropOverloads: drops, + } + } + return cla +} + +// DefaultServerListenerWithRouteConfigName returns a basic xds Listener +// resource to be used on the server side. The returned Listener resource +// contains a RouteConfiguration resource name that needs to be resolved. +func DefaultServerListenerWithRouteConfigName(host string, port uint32, secLevel SecurityLevel, routeName string) *v3listenerpb.Listener { + return defaultServerListenerCommon(host, port, secLevel, routeName, false) +} + +// RouteConfigNoRouteMatch returns an xDS RouteConfig resource which a route +// with no route match. This will be NACKed by the xDS Client. +func RouteConfigNoRouteMatch(routeName string) *v3routepb.RouteConfiguration { + return &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + // This "*" string matches on any incoming authority. This is to ensure any + // incoming RPC matches to Route_NonForwardingAction and will proceed as + // normal. + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}} +} + +// RouteConfigNonForwardingAction returns an xDS RouteConfig resource which +// specifies to route to a route specifying non forwarding action. This is +// intended to be used on the server side for RDS requests, and corresponds to +// the inline route configuration in DefaultServerListener. +func RouteConfigNonForwardingAction(routeName string) *v3routepb.RouteConfiguration { + return &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + // This "*" string matches on any incoming authority. This is to ensure any + // incoming RPC matches to Route_NonForwardingAction and will proceed as + // normal. + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}} +} + +// RouteConfigFilterAction returns an xDS RouteConfig resource which specifies +// to route to a route specifying route filter action. Since this is not type +// non forwarding action, this should fail requests that match to this server +// side. +func RouteConfigFilterAction(routeName string) *v3routepb.RouteConfiguration { + return &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + // This "*" string matches on any incoming authority. This is to + // ensure any incoming RPC matches to Route_Route and will fail with + // UNAVAILABLE. + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_FilterAction{}, + }}}}} +} diff --git a/internal/testutils/xds/e2e/logging.go b/internal/testutils/xds/e2e/logging.go new file mode 100644 index 0000000..ed3a603 --- /dev/null +++ b/internal/testutils/xds/e2e/logging.go @@ -0,0 +1,40 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +// serverLogger implements the Logger interface defined at +// envoyproxy/go-control-plane/pkg/log. This is passed to the Snapshot cache. +type serverLogger struct { + logger interface { + Logf(format string, args ...any) + } +} + +func (l serverLogger) Debugf(format string, args ...any) { + l.logger.Logf(format, args...) +} +func (l serverLogger) Infof(format string, args ...any) { + l.logger.Logf(format, args...) +} +func (l serverLogger) Warnf(format string, args ...any) { + l.logger.Logf(format, args...) +} +func (l serverLogger) Errorf(format string, args ...any) { + l.logger.Logf(format, args...) +} diff --git a/internal/testutils/xds/e2e/server.go b/internal/testutils/xds/e2e/server.go new file mode 100644 index 0000000..0ae551f --- /dev/null +++ b/internal/testutils/xds/e2e/server.go @@ -0,0 +1,256 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package e2e provides utilities for end2end testing of xDS functionality. +package e2e + +import ( + "context" + "fmt" + "net" + "reflect" + "strconv" + "testing" + + "github.com/envoyproxy/go-control-plane/pkg/cache/types" + "google.golang.org/grpc" + "google.golang.org/grpc/internal/testutils/xds/fakeserver" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverygrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + v3cache "github.com/envoyproxy/go-control-plane/pkg/cache/v3" + v3resource "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + v3server "github.com/envoyproxy/go-control-plane/pkg/server/v3" +) + +// ManagementServer is a thin wrapper around the xDS control plane +// implementation provided by envoyproxy/go-control-plane. +type ManagementServer struct { + // Address is the host:port on which the management server is listening for + // new connections. + Address string + + // LRSServer points to the fake LRS server implementation. Set only if the + // SupportLoadReportingService option was set to true when creating this + // management server. + LRSServer *fakeserver.Server + + cancel context.CancelFunc // To stop the v3 ADS service. + xs v3server.Server // v3 implementation of ADS. + gs *grpc.Server // gRPC server which exports the ADS service. + cache v3cache.SnapshotCache // Resource snapshot. + version int // Version of resource snapshot. + + // A logging interface, usually supplied from *testing.T. + logger interface { + Logf(format string, args ...any) + } +} + +// ManagementServerOptions contains options to be passed to the management +// server during creation. +type ManagementServerOptions struct { + // Listener to accept connections on. If nil, a TPC listener on a local port + // will be created and used. + Listener net.Listener + + // SupportLoadReportingService, if set, results in the load reporting + // service being registered on the same port as that of ADS. + SupportLoadReportingService bool + + // AllowResourceSubSet allows the management server to respond to requests + // before all configured resources are explicitly named in the request. The + // default behavior that we want is for the management server to wait for + // all configured resources to be requested before responding to any of + // them, since this is how we have run our tests historically, and should be + // set to true only for tests which explicitly require the other behavior. + AllowResourceSubset bool + + // ServerFeaturesIgnoreResourceDeletion, if set, results in a bootstrap config + // where the server features list contains `ignore_resource_deletion`. This + // results in gRPC ignoring resource deletions from the management server, as + // per A53. + ServerFeaturesIgnoreResourceDeletion bool + + // The callbacks defined below correspond to the state of the world (sotw) + // version of the xDS API on the management server. + + // OnStreamOpen is called when an xDS stream is opened. The callback is + // invoked with the assigned stream ID and the type URL from the incoming + // request (or "" for ADS). + // + // Returning an error from this callback will end processing and close the + // stream. OnStreamClosed will still be called. + OnStreamOpen func(context.Context, int64, string) error + + // OnStreamClosed is called immediately prior to closing an xDS stream. The + // callback is invoked with the stream ID of the stream being closed. + OnStreamClosed func(int64, *v3corepb.Node) + + // OnStreamRequest is called when a request is received on the stream. The + // callback is invoked with the stream ID of the stream on which the request + // was received and the received request. + // + // Returning an error from this callback will end processing and close the + // stream. OnStreamClosed will still be called. + OnStreamRequest func(int64, *v3discoverypb.DiscoveryRequest) error + + // OnStreamResponse is called immediately prior to sending a response on the + // stream. The callback is invoked with the stream ID of the stream on which + // the response is being sent along with the incoming request and the outgoing + // response. + OnStreamResponse func(context.Context, int64, *v3discoverypb.DiscoveryRequest, *v3discoverypb.DiscoveryResponse) +} + +// StartManagementServer initializes a management server which implements the +// AggregatedDiscoveryService endpoint. The management server is initialized +// with no resources. Tests should call the Update() method to change the +// resource snapshot held by the management server, as per by the test logic. +// +// Registers a cleanup function on t to stop the management server. +func StartManagementServer(t *testing.T, opts ManagementServerOptions) *ManagementServer { + t.Helper() + + // Create a snapshot cache. The first parameter to NewSnapshotCache() + // controls whether the server should wait for all resources to be + // explicitly named in the request before responding to any of them. + wait := !opts.AllowResourceSubset + cache := v3cache.NewSnapshotCache(wait, v3cache.IDHash{}, serverLogger{t}) + t.Logf("Created new snapshot cache...") + + lis := opts.Listener + if lis == nil { + var err error + lis, err = net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen on localhost:0: %v", err) + } + } + + // Cancelling the context passed to the server is the only way of stopping it + // at the end of the test. + ctx, cancel := context.WithCancel(context.Background()) + callbacks := v3server.CallbackFuncs{ + StreamOpenFunc: opts.OnStreamOpen, + StreamClosedFunc: opts.OnStreamClosed, + StreamRequestFunc: opts.OnStreamRequest, + StreamResponseFunc: opts.OnStreamResponse, + } + + // Create an xDS management server and register the ADS implementation + // provided by it on a gRPC server. + xs := v3server.NewServer(ctx, cache, callbacks) + gs := grpc.NewServer() + v3discoverygrpc.RegisterAggregatedDiscoveryServiceServer(gs, xs) + t.Logf("Registered Aggregated Discovery Service (ADS)...") + + mgmtServer := &ManagementServer{ + Address: lis.Addr().String(), + cancel: cancel, + version: 0, + gs: gs, + xs: xs, + cache: cache, + logger: t, + } + if opts.SupportLoadReportingService { + lrs := fakeserver.NewServer(lis.Addr().String()) + v3lrsgrpc.RegisterLoadReportingServiceServer(gs, lrs) + mgmtServer.LRSServer = lrs + t.Logf("Registered Load Reporting Service (LRS)...") + } + + // Start serving. + go gs.Serve(lis) + t.Logf("xDS management server serving at: %v...", lis.Addr().String()) + t.Cleanup(mgmtServer.Stop) + return mgmtServer +} + +// UpdateOptions wraps parameters to be passed to the Update() method. +type UpdateOptions struct { + // NodeID is the id of the client to which this update is to be pushed. + NodeID string + // Endpoints, Clusters, Routes, and Listeners are the updated list of xds + // resources for the server. All must be provided with each Update. + Endpoints []*v3endpointpb.ClusterLoadAssignment + Clusters []*v3clusterpb.Cluster + Routes []*v3routepb.RouteConfiguration + Listeners []*v3listenerpb.Listener + // SkipValidation indicates whether we want to skip validation (by not + // calling snapshot.Consistent()). It can be useful for negative tests, + // where we send updates that the client will NACK. + SkipValidation bool +} + +// Update changes the resource snapshot held by the management server, which +// updates connected clients as required. +func (s *ManagementServer) Update(ctx context.Context, opts UpdateOptions) error { + s.version++ + + // Create a snapshot with the passed in resources. + resources := map[v3resource.Type][]types.Resource{ + v3resource.ListenerType: resourceSlice(opts.Listeners), + v3resource.RouteType: resourceSlice(opts.Routes), + v3resource.ClusterType: resourceSlice(opts.Clusters), + v3resource.EndpointType: resourceSlice(opts.Endpoints), + } + snapshot, err := v3cache.NewSnapshot(strconv.Itoa(s.version), resources) + if err != nil { + return fmt.Errorf("failed to create new snapshot cache: %v", err) + + } + if !opts.SkipValidation { + if err := snapshot.Consistent(); err != nil { + return fmt.Errorf("failed to create new resource snapshot: %v", err) + } + } + s.logger.Logf("Created new resource snapshot...") + + // Update the cache with the new resource snapshot. + if err := s.cache.SetSnapshot(ctx, opts.NodeID, snapshot); err != nil { + return fmt.Errorf("failed to update resource snapshot in management server: %v", err) + } + s.logger.Logf("Updated snapshot cache with resource snapshot...") + return nil +} + +// Stop stops the management server. +func (s *ManagementServer) Stop() { + if s.cancel != nil { + s.cancel() + } + s.gs.Stop() +} + +// resourceSlice accepts a slice of any type of proto messages and returns a +// slice of types.Resource. Will panic if there is an input type mismatch. +func resourceSlice(i any) []types.Resource { + v := reflect.ValueOf(i) + rs := make([]types.Resource, v.Len()) + for i := 0; i < v.Len(); i++ { + rs[i] = v.Index(i).Interface().(types.Resource) + } + return rs +} diff --git a/internal/testutils/xds/e2e/setup/setup.go b/internal/testutils/xds/e2e/setup/setup.go new file mode 100644 index 0000000..f1ac21b --- /dev/null +++ b/internal/testutils/xds/e2e/setup/setup.go @@ -0,0 +1,85 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package setup implements setup helpers for xDS e2e tests. +package setup + +import ( + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/resolver" + _ "google.golang.org/grpc/xds" // Register the xds_resolver. +) + +// ManagementServerAndResolver sets up an xDS management server, creates +// bootstrap configuration pointing to that server and creates an xDS resolver +// using that configuration. +// +// Registers a cleanup function on t to stop the management server. +// +// Returns the following: +// - the xDS management server +// - the node ID to use when talking to this management server +// - bootstrap configuration to use (if creating an xDS-enabled gRPC server) +// - xDS resolver builder (if creating an xDS-enabled gRPC client) +func ManagementServerAndResolver(t *testing.T) (*e2e.ManagementServer, string, []byte, resolver.Builder) { + // Start an xDS management server. + xdsServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, xdsServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + if internal.NewXDSResolverWithConfigForTesting == nil { + t.Fatalf("internal.NewXDSResolverWithConfigForTesting is nil") + } + r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + return xdsServer, nodeID, bc, r +} + +// ManagementServerAndResolverWithSPIFFE is exactly the same as +// ManagementServerAndResolver, except that it uses a bootstrap configuration +// containing certificate providers utilizing SPIFFE test certificates. +func ManagementServerAndResolverWithSPIFFE(t *testing.T) (*e2e.ManagementServer, + string, []byte, resolver.Builder) { + // Start an xDS management server. + xdsServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.SPIFFEBootstrapContents(t, nodeID, xdsServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + if internal.NewXDSResolverWithConfigForTesting == nil { + t.Fatalf("internal.NewXDSResolverWithConfigForTesting is nil") + } + r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + return xdsServer, nodeID, bc, r +} diff --git a/internal/testutils/xds/fakeserver/server.go b/internal/testutils/xds/fakeserver/server.go new file mode 100644 index 0000000..241f993 --- /dev/null +++ b/internal/testutils/xds/fakeserver/server.go @@ -0,0 +1,254 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package fakeserver provides a fake implementation of the management server. +// +// This package is recommended only for scenarios which cannot be tested using +// the xDS management server (which uses envoy-go-control-plane) provided by the +// `internal/testutils/xds/e2e` package. +package fakeserver + +import ( + "fmt" + "io" + "net" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + v3discoverygrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" +) + +const ( + // TODO: Make this a var or a field in the server if there is a need to use a + // value other than this default. + defaultChannelBufferSize = 50 + defaultDialTimeout = 5 * time.Second +) + +// Request wraps the request protobuf (xds/LRS) and error received by the +// Server in a call to stream.Recv(). +type Request struct { + Req proto.Message + Err error +} + +// Response wraps the response protobuf (xds/LRS) and error that the Server +// should send out to the client through a call to stream.Send() +type Response struct { + Resp proto.Message + Err error +} + +// Server is a fake implementation of xDS and LRS protocols. It listens on the +// same port for both services and exposes a bunch of channels to send/receive +// messages. +// +// This server is recommended only for scenarios which cannot be tested using +// the xDS management server (which uses envoy-go-control-plane) provided by the +// `internal/testutils/xds/e2e` package. +type Server struct { + // XDSRequestChan is a channel on which received xDS requests are made + // available to the users of this Server. + XDSRequestChan *testutils.Channel + // XDSResponseChan is a channel on which the Server accepts xDS responses + // to be sent to the client. + XDSResponseChan chan *Response + // LRSRequestChan is a channel on which received LRS requests are made + // available to the users of this Server. + LRSRequestChan *testutils.Channel + // LRSResponseChan is a channel on which the Server accepts the LRS + // response to be sent to the client. + LRSResponseChan chan *Response + // LRSStreamOpenChan is a channel on which the Server sends notifications + // when a new LRS stream is created. + LRSStreamOpenChan *testutils.Channel + // LRSStreamCloseChan is a channel on which the Server sends notifications + // when an existing LRS stream is closed. + LRSStreamCloseChan *testutils.Channel + // NewConnChan is a channel on which the fake server notifies receipt of new + // connection attempts. Tests can gate on this event before proceeding to + // other actions which depend on a connection to the fake server being up. + NewConnChan *testutils.Channel + // Address is the host:port on which the Server is listening for requests. + Address string + + // The underlying fake implementation of xDS and LRS. + *xdsServerV3 + *lrsServerV3 +} + +type wrappedListener struct { + net.Listener + server *Server +} + +func (wl *wrappedListener) Accept() (net.Conn, error) { + c, err := wl.Listener.Accept() + if err != nil { + return nil, err + } + wl.server.NewConnChan.Send(struct{}{}) + return c, err +} + +// StartServer makes a new Server and gets it to start listening on the given +// net.Listener. If the given net.Listener is nil, a new one is created on a +// local port for gRPC requests. The returned cancel function should be invoked +// by the caller upon completion of the test. +func StartServer(lis net.Listener) (*Server, func(), error) { + if lis == nil { + var err error + lis, err = net.Listen("tcp", "localhost:0") + if err != nil { + return nil, func() {}, fmt.Errorf("net.Listen() failed: %v", err) + } + } + + s := NewServer(lis.Addr().String()) + wp := &wrappedListener{ + Listener: lis, + server: s, + } + + server := grpc.NewServer() + v3lrsgrpc.RegisterLoadReportingServiceServer(server, s) + v3discoverygrpc.RegisterAggregatedDiscoveryServiceServer(server, s) + go server.Serve(wp) + + return s, func() { server.Stop() }, nil +} + +// NewServer returns a new instance of Server, set to accept requests on addr. +// It is the responsibility of the caller to register the exported ADS and LRS +// services on an appropriate gRPC server. Most usages should prefer +// StartServer() instead of this. +func NewServer(addr string) *Server { + s := &Server{ + XDSRequestChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + LRSRequestChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + NewConnChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + XDSResponseChan: make(chan *Response, defaultChannelBufferSize), + LRSResponseChan: make(chan *Response, 1), // The server only ever sends one response. + LRSStreamOpenChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + LRSStreamCloseChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + Address: addr, + } + s.xdsServerV3 = &xdsServerV3{reqChan: s.XDSRequestChan, respChan: s.XDSResponseChan} + s.lrsServerV3 = &lrsServerV3{reqChan: s.LRSRequestChan, respChan: s.LRSResponseChan, streamOpenChan: s.LRSStreamOpenChan, streamCloseChan: s.LRSStreamCloseChan} + return s +} + +type xdsServerV3 struct { + reqChan *testutils.Channel + respChan chan *Response +} + +func (xdsS *xdsServerV3) StreamAggregatedResources(s v3discoverygrpc.AggregatedDiscoveryService_StreamAggregatedResourcesServer) error { + errCh := make(chan error, 2) + go func() { + for { + req, err := s.Recv() + if err != nil { + errCh <- err + return + } + xdsS.reqChan.Send(&Request{req, err}) + } + }() + go func() { + var retErr error + defer func() { + errCh <- retErr + }() + + for { + select { + case r := <-xdsS.respChan: + if r.Err != nil { + retErr = r.Err + return + } + if err := s.Send(r.Resp.(*v3discoverypb.DiscoveryResponse)); err != nil { + retErr = err + return + } + case <-s.Context().Done(): + retErr = s.Context().Err() + return + } + } + }() + + if err := <-errCh; err != nil { + return err + } + return nil +} + +func (xdsS *xdsServerV3) DeltaAggregatedResources(v3discoverygrpc.AggregatedDiscoveryService_DeltaAggregatedResourcesServer) error { + return status.Error(codes.Unimplemented, "") +} + +type lrsServerV3 struct { + reqChan *testutils.Channel + respChan chan *Response + streamOpenChan *testutils.Channel + streamCloseChan *testutils.Channel +} + +func (lrsS *lrsServerV3) StreamLoadStats(s v3lrsgrpc.LoadReportingService_StreamLoadStatsServer) error { + lrsS.streamOpenChan.Send(nil) + defer lrsS.streamCloseChan.Send(nil) + + req, err := s.Recv() + lrsS.reqChan.Send(&Request{req, err}) + if err != nil { + return err + } + + select { + case r := <-lrsS.respChan: + if r.Err != nil { + return r.Err + } + if err := s.Send(r.Resp.(*v3lrspb.LoadStatsResponse)); err != nil { + return err + } + case <-s.Context().Done(): + return s.Context().Err() + } + + for { + req, err := s.Recv() + lrsS.reqChan.Send(&Request{req, err}) + if err != nil { + if err == io.EOF { + return nil + } + return err + } + } +} diff --git a/internal/testutils/xds_bootstrap.go b/internal/testutils/xds_bootstrap.go new file mode 100644 index 0000000..22cb696 --- /dev/null +++ b/internal/testutils/xds_bootstrap.go @@ -0,0 +1,52 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "os" + "testing" + + "google.golang.org/grpc/internal/envconfig" +) + +// CreateBootstrapFileForTesting creates a temporary file with the provided +// bootstrap contents, and updates the bootstrap environment variable to point +// to this file. +// +// Registers a cleanup function on the provided testing.T, that deletes the +// temporary file and resets the bootstrap environment variable. +func CreateBootstrapFileForTesting(t *testing.T, bootstrapContents []byte) { + t.Helper() + + f, err := os.CreateTemp("", "test_xds_bootstrap_*") + if err != nil { + t.Fatalf("Failed to created bootstrap file: %v", err) + } + + if err := os.WriteFile(f.Name(), bootstrapContents, 0644); err != nil { + t.Fatalf("Failed to created bootstrap file: %v", err) + } + + origBootstrapFileName := envconfig.XDSBootstrapFileName + envconfig.XDSBootstrapFileName = f.Name() + t.Cleanup(func() { + os.Remove(f.Name()) + envconfig.XDSBootstrapFileName = origBootstrapFileName + }) +} diff --git a/internal/transport/bdp_estimator.go b/internal/transport/bdp_estimator.go new file mode 100644 index 0000000..070680e --- /dev/null +++ b/internal/transport/bdp_estimator.go @@ -0,0 +1,141 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "sync" + "time" +) + +const ( + // bdpLimit is the maximum value the flow control windows will be increased + // to. TCP typically limits this to 4MB, but some systems go up to 16MB. + // Since this is only a limit, it is safe to make it optimistic. + bdpLimit = (1 << 20) * 16 + // alpha is a constant factor used to keep a moving average + // of RTTs. + alpha = 0.9 + // If the current bdp sample is greater than or equal to + // our beta * our estimated bdp and the current bandwidth + // sample is the maximum bandwidth observed so far, we + // increase our bbp estimate by a factor of gamma. + beta = 0.66 + // To put our bdp to be smaller than or equal to twice the real BDP, + // we should multiply our current sample with 4/3, however to round things out + // we use 2 as the multiplication factor. + gamma = 2 +) + +// Adding arbitrary data to ping so that its ack can be identified. +// Easter-egg: what does the ping message say? +var bdpPing = &ping{data: [8]byte{2, 4, 16, 16, 9, 14, 7, 7}} + +type bdpEstimator struct { + // sentAt is the time when the ping was sent. + sentAt time.Time + + mu sync.Mutex + // bdp is the current bdp estimate. + bdp uint32 + // sample is the number of bytes received in one measurement cycle. + sample uint32 + // bwMax is the maximum bandwidth noted so far (bytes/sec). + bwMax float64 + // bool to keep track of the beginning of a new measurement cycle. + isSent bool + // Callback to update the window sizes. + updateFlowControl func(n uint32) + // sampleCount is the number of samples taken so far. + sampleCount uint64 + // round trip time (seconds) + rtt float64 +} + +// timesnap registers the time bdp ping was sent out so that +// network rtt can be calculated when its ack is received. +// It is called (by controller) when the bdpPing is +// being written on the wire. +func (b *bdpEstimator) timesnap(d [8]byte) { + if bdpPing.data != d { + return + } + b.sentAt = time.Now() +} + +// add adds bytes to the current sample for calculating bdp. +// It returns true only if a ping must be sent. This can be used +// by the caller (handleData) to make decision about batching +// a window update with it. +func (b *bdpEstimator) add(n uint32) bool { + b.mu.Lock() + defer b.mu.Unlock() + if b.bdp == bdpLimit { + return false + } + if !b.isSent { + b.isSent = true + b.sample = n + b.sentAt = time.Time{} + b.sampleCount++ + return true + } + b.sample += n + return false +} + +// calculate is called when an ack for a bdp ping is received. +// Here we calculate the current bdp and bandwidth sample and +// decide if the flow control windows should go up. +func (b *bdpEstimator) calculate(d [8]byte) { + // Check if the ping acked for was the bdp ping. + if bdpPing.data != d { + return + } + b.mu.Lock() + rttSample := time.Since(b.sentAt).Seconds() + if b.sampleCount < 10 { + // Bootstrap rtt with an average of first 10 rtt samples. + b.rtt += (rttSample - b.rtt) / float64(b.sampleCount) + } else { + // Heed to the recent past more. + b.rtt += (rttSample - b.rtt) * float64(alpha) + } + b.isSent = false + // The number of bytes accumulated so far in the sample is smaller + // than or equal to 1.5 times the real BDP on a saturated connection. + bwCurrent := float64(b.sample) / (b.rtt * float64(1.5)) + if bwCurrent > b.bwMax { + b.bwMax = bwCurrent + } + // If the current sample (which is smaller than or equal to the 1.5 times the real BDP) is + // greater than or equal to 2/3rd our perceived bdp AND this is the maximum bandwidth seen so far, we + // should update our perception of the network BDP. + if float64(b.sample) >= beta*float64(b.bdp) && bwCurrent == b.bwMax && b.bdp != bdpLimit { + sampleFloat := float64(b.sample) + b.bdp = uint32(gamma * sampleFloat) + if b.bdp > bdpLimit { + b.bdp = bdpLimit + } + bdp := b.bdp + b.mu.Unlock() + b.updateFlowControl(bdp) + return + } + b.mu.Unlock() +} diff --git a/internal/transport/client_stream.go b/internal/transport/client_stream.go new file mode 100644 index 0000000..046f0a5 --- /dev/null +++ b/internal/transport/client_stream.go @@ -0,0 +1,189 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "fmt" + "sync/atomic" + + "golang.org/x/net/http2" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +// nonGRPCDataMaxLen is the maximum length of nonGRPCDataBuf. +// +// NOTE: If changed this value, you MUST update the corresponding test in: +// - /test/end2end_test.go:TestHTTPServerSendsNonGRPCHeaderSurfaceFurtherData +const nonGRPCDataMaxLen = 1024 + +// ClientStream implements streaming functionality for a gRPC client. +type ClientStream struct { + Stream // Embed for common stream functionality. + + ct *http2Client + done chan struct{} // closed at the end of stream to unblock writers. + + headerChan chan struct{} // closed to indicate the end of header metadata. + header metadata.MD // the received header metadata + + status *status.Status // the status error received from the server + + // Non-pointer fields are at the end to optimize GC allocations. + + // headerValid indicates whether a valid header was received. Only + // meaningful after headerChan is closed (always call waitOnHeader() before + // reading its value). + headerValid bool + + nonGRPCStatus *status.Status // the initial status from the non-gRPC response header, finalized with collected data before closing. + nonGRPCDataBuf []byte // stores the data of a non-gRPC response. + + noHeaders bool // set if the client never received headers (set only after the stream is done). + headerChanClosed uint32 // set when headerChan is closed. Used to avoid closing headerChan multiple times. + bytesReceived atomic.Bool // indicates whether any bytes have been received on this stream + unprocessed atomic.Bool // set if the server sends a refused stream or GOAWAY including this stream + statsHandler stats.Handler // nil for internal streams (e.g., health check, ORCA) where telemetry is not supported. +} + +func (s *ClientStream) startNonGRPCDataCollection(st *status.Status) { + s.nonGRPCStatus = st + s.nonGRPCDataBuf = make([]byte, 0, nonGRPCDataMaxLen) +} + +// finalizeNonGRPCStatus builds the terminal status by appending the collected +// response body to the original non-gRPC status message. +func (s *ClientStream) finalizeNonGRPCStatus() *status.Status { + msg := fmt.Sprintf("%s\ndata: %q", s.nonGRPCStatus.Message(), s.nonGRPCDataBuf) + return status.New(s.nonGRPCStatus.Code(), msg) +} + +// handleNonGRPCData collects non-gRPC body from the given data frame. +// It returns non-nil value when the stream should be closed with it. +func (s *ClientStream) handleNonGRPCData(f *parsedDataFrame) *status.Status { + n := min(f.data.Len(), nonGRPCDataMaxLen-len(s.nonGRPCDataBuf)) + s.nonGRPCDataBuf = append(s.nonGRPCDataBuf, f.data.ReadOnlyData()[0:n]...) + if len(s.nonGRPCDataBuf) >= nonGRPCDataMaxLen || f.StreamEnded() { + return s.finalizeNonGRPCStatus() + } + return nil +} + +// Read reads an n byte message from the input stream. +func (s *ClientStream) Read(n int) (mem.BufferSlice, error) { + b, err := s.Stream.read(n) + if err == nil { + s.ct.incrMsgRecv() + } + return b, err +} + +// Close closes the stream and propagates err to any readers. +func (s *ClientStream) Close(err error) { + var ( + rst bool + rstCode http2.ErrCode + ) + if err != nil { + rst = true + rstCode = http2.ErrCodeCancel + } + s.ct.closeStream(s, err, rst, rstCode, status.Convert(err), nil, false) +} + +// Write writes the hdr and data bytes to the output stream. +func (s *ClientStream) Write(hdr []byte, data mem.BufferSlice, opts *WriteOptions) error { + return s.ct.write(s, hdr, data, opts) +} + +// BytesReceived indicates whether any bytes have been received on this stream. +func (s *ClientStream) BytesReceived() bool { + return s.bytesReceived.Load() +} + +// Unprocessed indicates whether the server did not process this stream -- +// i.e. it sent a refused stream or GOAWAY including this stream ID. +func (s *ClientStream) Unprocessed() bool { + return s.unprocessed.Load() +} + +func (s *ClientStream) waitOnHeader() { + select { + case <-s.ctx.Done(): + // Close the stream to prevent headers/trailers from changing after + // this function returns. + s.Close(ContextErr(s.ctx.Err())) + // headerChan could possibly not be closed yet if closeStream raced + // with operateHeaders; wait until it is closed explicitly here. + <-s.headerChan + case <-s.headerChan: + } +} + +// RecvCompress returns the compression algorithm applied to the inbound +// message. It is empty string if there is no compression applied. +func (s *ClientStream) RecvCompress() string { + s.waitOnHeader() + return s.recvCompress +} + +// Done returns a channel which is closed when it receives the final status +// from the server. +func (s *ClientStream) Done() <-chan struct{} { + return s.done +} + +// Header returns the header metadata of the stream. Acquires the key-value +// pairs of header metadata once it is available. It blocks until i) the +// metadata is ready or ii) there is no header metadata or iii) the stream is +// canceled/expired. +func (s *ClientStream) Header() (metadata.MD, error) { + s.waitOnHeader() + + if !s.headerValid || s.noHeaders { + return nil, s.status.Err() + } + + return s.header.Copy(), nil +} + +// TrailersOnly blocks until a header or trailers-only frame is received and +// then returns true if the stream was trailers-only. If the stream ends +// before headers are received, returns true, nil. +func (s *ClientStream) TrailersOnly() bool { + s.waitOnHeader() + return s.noHeaders +} + +// Status returns the status received from the server. +// Status can be read safely only after the stream has ended, +// that is, after Done() is closed. +func (s *ClientStream) Status() *status.Status { + return s.status +} + +func (s *ClientStream) requestRead(n int) { + s.ct.adjustWindow(s, uint32(n)) +} + +func (s *ClientStream) updateWindow(n int) { + s.ct.updateWindow(s, uint32(n)) +} diff --git a/internal/transport/controlbuf.go b/internal/transport/controlbuf.go new file mode 100644 index 0000000..89c938f --- /dev/null +++ b/internal/transport/controlbuf.go @@ -0,0 +1,1051 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "bytes" + "errors" + "fmt" + "net" + "runtime" + "sync" + "sync/atomic" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/mem" +) + +var updateHeaderTblSize = func(e *hpack.Encoder, v uint32) { + e.SetMaxDynamicTableSizeLimit(v) +} + +// itemNodePool is used to reduce heap allocations. +var itemNodePool = sync.Pool{ + New: func() any { + return &itemNode{} + }, +} + +type itemNode struct { + it any + next *itemNode +} + +type itemList struct { + head *itemNode + tail *itemNode +} + +func (il *itemList) enqueue(i any) { + n := itemNodePool.Get().(*itemNode) + n.next = nil + n.it = i + if il.tail == nil { + il.head, il.tail = n, n + return + } + il.tail.next = n + il.tail = n +} + +// peek returns the first item in the list without removing it from the +// list. +func (il *itemList) peek() any { + return il.head.it +} + +func (il *itemList) dequeue() any { + if il.head == nil { + return nil + } + i := il.head.it + temp := il.head + il.head = il.head.next + itemNodePool.Put(temp) + if il.head == nil { + il.tail = nil + } + return i +} + +func (il *itemList) dequeueAll() *itemNode { + h := il.head + il.head, il.tail = nil, nil + return h +} + +func (il *itemList) isEmpty() bool { + return il.head == nil +} + +// maxQueuedControlBufferItems is the maximum number of frames (other than +// HEADERS and DATA) that we will buffer before preventing new reads from +// occurring on the transport. These are control frames sent in response to +// client requests, or frames that result in work being scheduled, such as +// RST_STREAM due to bad headers or settings acks. +var maxQueuedControlBufferItems = int(envconfig.ControlBufferThrottleLimit) + +type cbItem interface { + isThrottled() bool +} + +// throttledItem represents every item in the controlBuffer to which the overall +// throttling limit applies, other than outgoing HEADERS and DATA frames. +type throttledItem struct{} + +func (throttledItem) isThrottled() bool { return true } + +// The following defines various control items which could flow through +// the control buffer of transport. They represent different aspects of +// control tasks, e.g., flow control, settings, streaming resetting, etc. + +// registerStream is used to register an incoming stream with loopy writer. +type registerStream struct { + throttledItem + streamID uint32 + wq *writeQuota +} + +type clientHeaders struct { + streamID uint32 + hf []hpack.HeaderField + initStream func(uint32) error + onWrite func() + wq *writeQuota + onOrphaned func(error) +} + +func (*clientHeaders) isThrottled() bool { return false } + +type serverHeaders struct { + streamID uint32 + hf []hpack.HeaderField + endStream bool + onWrite func() + cleanup *cleanupStream +} + +func (h *serverHeaders) isThrottled() bool { return false } + +type cleanupStream struct { + throttledItem + streamID uint32 + rst bool + rstCode http2.ErrCode + onWrite func() +} + +type earlyAbortStream struct { + throttledItem + streamID uint32 + rst bool + hf []hpack.HeaderField // Pre-built header fields +} + +type dataFrame struct { + streamID uint32 + endStream bool + h []byte + data mem.BufferSlice + processing bool + // onEachWrite is called every time + // a part of data is written out. + onEachWrite func() +} + +func (*dataFrame) isThrottled() bool { return false } + +type incomingWindowUpdate struct { + throttledItem + streamID uint32 + increment uint32 +} + +type outgoingWindowUpdate struct { + throttledItem + streamID uint32 + increment uint32 +} + +type incomingSettings struct { + throttledItem + ss []http2.Setting +} + +type outgoingSettings struct { + throttledItem + ss []http2.Setting +} + +type incomingGoAway struct { + throttledItem +} + +type goAway struct { + throttledItem + code http2.ErrCode + debugData []byte + headsUp bool + closeConn error // if set, loopyWriter will exit with this error +} + +type ping struct { + throttledItem + ack bool + data [8]byte +} + +type outFlowControlSizeRequest struct { + throttledItem + resp chan uint32 +} + +// outStreamRequestForTesting is used by tests to retrieve a pointer to an outStream +// safely inside the loopyWriter goroutine without racing on loopyWriter.estdStreams. +// By holding the *outStream pointer while streams and transports close down, tests +// can inspect settled accounting values (such as bytesOutStanding) after loopyWriter +// has run to completion without any data races. +type outStreamRequestForTesting struct { + throttledItem + streamID uint32 + resp chan *outStream +} + +// closeConnection is an instruction to tell the loopy writer to flush the +// framer and exit, which will cause the transport's connection to be closed +// (by the client or server). The transport itself will close after the reader +// encounters the EOF caused by the connection closure. +type closeConnection struct { + throttledItem +} + +type outStreamState int + +const ( + active outStreamState = iota + empty + waitingOnStreamQuota +) + +type outStream struct { + id uint32 + state outStreamState + itl *itemList + bytesOutStanding int + wq *writeQuota + reader mem.Reader + + next *outStream + prev *outStream +} + +func (s *outStream) deleteSelf() { + if s.prev != nil { + s.prev.next = s.next + } + if s.next != nil { + s.next.prev = s.prev + } + s.next, s.prev = nil, nil +} + +type outStreamList struct { + // Following are sentinel objects that mark the + // beginning and end of the list. They do not + // contain any item lists. All valid objects are + // inserted in between them. + // This is needed so that an outStream object can + // deleteSelf() in O(1) time without knowing which + // list it belongs to. + head *outStream + tail *outStream +} + +func newOutStreamList() *outStreamList { + head, tail := new(outStream), new(outStream) + head.next = tail + tail.prev = head + return &outStreamList{ + head: head, + tail: tail, + } +} + +func (l *outStreamList) enqueue(s *outStream) { + e := l.tail.prev + e.next = s + s.prev = e + s.next = l.tail + l.tail.prev = s +} + +// remove from the beginning of the list. +func (l *outStreamList) dequeue() *outStream { + b := l.head.next + if b == l.tail { + return nil + } + b.deleteSelf() + return b +} + +// controlBuffer is a way to pass information to loopy. +// +// Information is passed as specific struct types called control frames. A +// control frame not only represents data, messages or headers to be sent out +// but can also be used to instruct loopy to update its internal state. It +// shouldn't be confused with an HTTP2 frame, although some of the control +// frames like dataFrame and headerFrame do go out on wire as HTTP2 frames. +type controlBuffer struct { + wakeupCh chan struct{} // Unblocks readers waiting for something to read. + done <-chan struct{} // Closed when the transport is done. + + // Mutex guards all the fields below, except trfChan which can be read + // atomically without holding mu. + mu sync.Mutex + consumerWaiting bool // True when readers are blocked waiting for new data. + closed bool // True when the controlbuf is finished. + list *itemList // List of queued control frames. + + // transportResponseFrames counts the number of queued items that represent + // the response of an action initiated by the peer. trfChan is created + // when transportResponseFrames >= maxQueuedTransportResponseFrames and is + // closed and nilled when transportResponseFrames drops below the + // threshold. Both fields are protected by mu. + transportResponseFrames int + trfChan atomic.Pointer[chan struct{}] +} + +func newControlBuffer(done <-chan struct{}) *controlBuffer { + return &controlBuffer{ + wakeupCh: make(chan struct{}, 1), + list: &itemList{}, + done: done, + } +} + +// throttle blocks if there are too many frames in the control buf that +// represent the response of an action initiated by the peer, like +// incomingSettings cleanupStreams etc. +func (c *controlBuffer) throttle() { + if ch := c.trfChan.Load(); ch != nil { + select { + case <-(*ch): + case <-c.done: + } + } +} + +// put adds an item to the controlbuf. +func (c *controlBuffer) put(it cbItem) error { + _, err := c.executeAndPut(nil, it) + return err +} + +// executeAndPut runs f, and if the return value is true, adds the given item to +// the controlbuf. The item could be nil, in which case, this method simply +// executes f and does not add the item to the controlbuf. +// +// The first return value indicates whether the item was successfully added to +// the control buffer. A non-nil error, specifically ErrConnClosing, is returned +// if the control buffer is already closed. +func (c *controlBuffer) executeAndPut(f func() bool, it cbItem) (bool, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.closed { + return false, ErrConnClosing + } + if f != nil { + if !f() { // f wasn't successful + return false, nil + } + } + if it == nil { + return true, nil + } + + var wakeUp bool + if c.consumerWaiting { + wakeUp = true + c.consumerWaiting = false + } + c.list.enqueue(it) + if it.isThrottled() { + c.transportResponseFrames++ + if c.transportResponseFrames == maxQueuedControlBufferItems { + // We are adding the frame that puts us over the threshold; create + // a throttling channel. + ch := make(chan struct{}) + c.trfChan.Store(&ch) + } + } + if wakeUp { + select { + case c.wakeupCh <- struct{}{}: + default: + } + } + return true, nil +} + +// get returns the next control frame from the control buffer. If block is true +// **and** there are no control frames in the control buffer, the call blocks +// until one of the conditions is met: there is a frame to return or the +// transport is closed. +func (c *controlBuffer) get(block bool) (any, error) { + for { + c.mu.Lock() + frame, err := c.getOnceLocked() + if frame != nil || err != nil || !block { + // If we read a frame or an error, we can return to the caller. The + // call to getOnceLocked() returns a nil frame and a nil error if + // there is nothing to read, and in that case, if the caller asked + // us not to block, we can return now as well. + c.mu.Unlock() + return frame, err + } + c.consumerWaiting = true + c.mu.Unlock() + + // Release the lock above and wait to be woken up. + select { + case <-c.wakeupCh: + case <-c.done: + return nil, errors.New("transport closed by client") + } + } +} + +// Callers must not use this method, but should instead use get(). +// +// Caller must hold c.mu. +func (c *controlBuffer) getOnceLocked() (any, error) { + if c.closed { + return false, ErrConnClosing + } + if c.list.isEmpty() { + return nil, nil + } + h := c.list.dequeue().(cbItem) + if h.isThrottled() { + if c.transportResponseFrames == maxQueuedControlBufferItems { + // We are removing the frame that put us over the + // threshold; close and clear the throttling channel. + ch := c.trfChan.Swap(nil) + close(*ch) + } + c.transportResponseFrames-- + } + return h, nil +} + +// finish closes the control buffer, cleaning up any streams that have queued +// header frames. Once this method returns, no more frames can be added to the +// control buffer, and attempts to do so will return ErrConnClosing. +func (c *controlBuffer) finish() { + c.mu.Lock() + defer c.mu.Unlock() + + if c.closed { + return + } + c.closed = true + // There may be headers for streams in the control buffer. + // These streams need to be cleaned out since the transport + // is still not aware of these yet. + for head := c.list.dequeueAll(); head != nil; head = head.next { + switch v := head.it.(type) { + case *clientHeaders: + v.onOrphaned(ErrConnClosing) + case *dataFrame: + if !v.processing { + v.data.Free() + } + } + } + + // In case throttle() is currently in flight, it needs to be unblocked. + // Otherwise, the transport may not close, since the transport is closed by + // the reader encountering the connection error. + ch := c.trfChan.Swap(nil) + if ch != nil { + close(*ch) + } +} + +type side int + +const ( + clientSide side = iota + serverSide +) + +// maxWriteBufSize is the maximum length (number of elements) the cached +// writeBuf can grow to. The length depends on the number of buffers +// contained within the BufferSlice produced by the codec, which is +// generally small. +// +// If a writeBuf larger than this limit is required, it will be allocated +// and freed after use, rather than being cached. This avoids holding +// on to large amounts of memory. +const maxWriteBufSize = 64 + +// Loopy receives frames from the control buffer. +// Each frame is handled individually; most of the work done by loopy goes +// into handling data frames. Loopy maintains a queue of active streams, and each +// stream maintains a queue of data frames; as loopy receives data frames +// it gets added to the queue of the relevant stream. +// Loopy goes over this list of active streams by processing one node every iteration, +// thereby closely resembling a round-robin scheduling over all streams. While +// processing a stream, loopy writes out data bytes from this stream capped by the min +// of http2MaxFrameLen, connection-level flow control and stream-level flow control. +type loopyWriter struct { + side side + cbuf *controlBuffer + sendQuota uint32 + oiws uint32 // outbound initial window size. + // estdStreams is map of all established streams that are not cleaned-up yet. + // On client-side, this is all streams whose headers were sent out. + // On server-side, this is all streams whose headers were received. + estdStreams map[uint32]*outStream // Established streams. + // activeStreams is a linked-list of all streams that have data to send and some + // stream-level flow control quota. + // Each of these streams internally have a list of data items(and perhaps trailers + // on the server-side) to be sent out. + activeStreams *outStreamList + framer *framer + hBuf *bytes.Buffer // The buffer for HPACK encoding. + hEnc *hpack.Encoder // HPACK encoder. + bdpEst *bdpEstimator + draining bool + conn net.Conn + logger *grpclog.PrefixLogger + bufferPool mem.BufferPool + + // Side-specific handlers + ssGoAwayHandler func(*goAway) (bool, error) + + writeBuf [][]byte // cached slice to avoid heap allocations for calls to mem.Reader.Peek. +} + +func newLoopyWriter(s side, fr *framer, cbuf *controlBuffer, bdpEst *bdpEstimator, conn net.Conn, logger *grpclog.PrefixLogger, goAwayHandler func(*goAway) (bool, error), bufferPool mem.BufferPool) *loopyWriter { + var buf bytes.Buffer + l := &loopyWriter{ + side: s, + cbuf: cbuf, + sendQuota: defaultWindowSize, + oiws: defaultWindowSize, + estdStreams: make(map[uint32]*outStream), + activeStreams: newOutStreamList(), + framer: fr, + hBuf: &buf, + hEnc: hpack.NewEncoder(&buf), + bdpEst: bdpEst, + conn: conn, + logger: logger, + ssGoAwayHandler: goAwayHandler, + bufferPool: bufferPool, + } + return l +} + +const minBatchSize = 1000 + +// run should be run in a separate goroutine. +// It reads control frames from controlBuf and processes them by: +// 1. Updating loopy's internal state, or/and +// 2. Writing out HTTP2 frames on the wire. +// +// Loopy keeps all active streams with data to send in a linked-list. +// All streams in the activeStreams linked-list must have both: +// 1. Data to send, and +// 2. Stream level flow control quota available. +// +// In each iteration of run loop, other than processing the incoming control +// frame, loopy calls processData, which processes one node from the +// activeStreams linked-list. This results in writing of HTTP2 frames into an +// underlying write buffer. When there's no more control frames to read from +// controlBuf, loopy flushes the write buffer. As an optimization, to increase +// the batch size for each flush, loopy yields the processor, once if the batch +// size is too low to give stream goroutines a chance to fill it up. +// +// Upon exiting, if the error causing the exit is not an I/O error, run() +// flushes the underlying connection. The connection is always left open to +// allow different closing behavior on the client and server. +func (l *loopyWriter) run() (err error) { + defer func() { + if l.logger.V(logLevel) { + l.logger.Infof("loopyWriter exiting with error: %v", err) + } + if !isIOError(err) { + l.framer.writer.Flush() + } + l.cbuf.finish() + }() + for { + it, err := l.cbuf.get(true) + if err != nil { + return err + } + if err = l.handle(it); err != nil { + return err + } + if _, err = l.processData(); err != nil { + return err + } + gosched := true + hasdata: + for { + it, err := l.cbuf.get(false) + if err != nil { + return err + } + if it != nil { + if err = l.handle(it); err != nil { + return err + } + if _, err = l.processData(); err != nil { + return err + } + continue hasdata + } + isEmpty, err := l.processData() + if err != nil { + return err + } + if !isEmpty { + continue hasdata + } + if gosched { + gosched = false + if l.framer.writer.offset < minBatchSize { + runtime.Gosched() + continue hasdata + } + } + l.framer.writer.Flush() + break hasdata + } + } +} + +func (l *loopyWriter) outgoingWindowUpdateHandler(w *outgoingWindowUpdate) error { + return l.framer.fr.WriteWindowUpdate(w.streamID, w.increment) +} + +func (l *loopyWriter) incomingWindowUpdateHandler(w *incomingWindowUpdate) { + // Otherwise update the quota. + if w.streamID == 0 { + l.sendQuota += w.increment + return + } + // Find the stream and update it. + if str, ok := l.estdStreams[w.streamID]; ok { + str.bytesOutStanding -= int(w.increment) + if strQuota := int(l.oiws) - str.bytesOutStanding; strQuota > 0 && str.state == waitingOnStreamQuota { + str.state = active + l.activeStreams.enqueue(str) + return + } + } +} + +func (l *loopyWriter) outgoingSettingsHandler(s *outgoingSettings) error { + return l.framer.fr.WriteSettings(s.ss...) +} + +func (l *loopyWriter) incomingSettingsHandler(s *incomingSettings) error { + l.applySettings(s.ss) + return l.framer.fr.WriteSettingsAck() +} + +func (l *loopyWriter) registerStreamHandler(h *registerStream) { + str := &outStream{ + id: h.streamID, + state: empty, + itl: &itemList{}, + wq: h.wq, + } + l.estdStreams[h.streamID] = str +} + +func (l *loopyWriter) serverHeaderHandler(hdr *serverHeaders) error { + str, ok := l.estdStreams[hdr.streamID] + if !ok { + if l.logger.V(logLevel) { + l.logger.Infof("Unrecognized streamID %d in loopyWriter", hdr.streamID) + } + return nil + } + + // Case 1: Server is responding back with headers. + if !hdr.endStream { + return l.writeHeader(hdr.streamID, hdr.endStream, hdr.hf, hdr.onWrite) + } + + // Case 2: Server is closing the stream. + if str.state != empty { // either active or waiting on stream quota. + str.itl.enqueue(hdr) + return nil + } + if err := l.writeHeader(hdr.streamID, hdr.endStream, hdr.hf, hdr.onWrite); err != nil { + return err + } + return l.cleanupStreamHandler(hdr.cleanup) +} + +func (l *loopyWriter) clientHeaderHandler(hdr *clientHeaders) error { + str := &outStream{ + id: hdr.streamID, + state: empty, + itl: &itemList{}, + wq: hdr.wq, + } + // l.draining is set when handling GoAway. In which case, we want to avoid + // creating new streams. + if l.draining { + // TODO: provide a better error with the reason we are in draining. + hdr.onOrphaned(errStreamDrain) + return nil + } + if err := hdr.initStream(str.id); err != nil { + return err + } + if err := l.writeHeader(str.id, false, hdr.hf, hdr.onWrite); err != nil { + return err + } + l.estdStreams[str.id] = str + return nil +} + +func (l *loopyWriter) writeHeader(streamID uint32, endStream bool, hf []hpack.HeaderField, onWrite func()) error { + if onWrite != nil { + onWrite() + } + l.hBuf.Reset() + for _, f := range hf { + if err := l.hEnc.WriteField(f); err != nil { + if l.logger.V(logLevel) { + l.logger.Warningf("Encountered error while encoding headers: %v", err) + } + } + } + var ( + err error + endHeaders, first bool + ) + first = true + for !endHeaders { + size := l.hBuf.Len() + if size > http2MaxFrameLen { + size = http2MaxFrameLen + } else { + endHeaders = true + } + if first { + first = false + err = l.framer.fr.WriteHeaders(http2.HeadersFrameParam{ + StreamID: streamID, + BlockFragment: l.hBuf.Next(size), + EndStream: endStream, + EndHeaders: endHeaders, + }) + } else { + err = l.framer.fr.WriteContinuation( + streamID, + endHeaders, + l.hBuf.Next(size), + ) + } + if err != nil { + return err + } + } + return nil +} + +func (l *loopyWriter) preprocessData(df *dataFrame) { + str, ok := l.estdStreams[df.streamID] + if !ok { + return + } + // If we got data for a stream it means that + // stream was originated and the headers were sent out. + str.itl.enqueue(df) + if str.state == empty { + str.state = active + l.activeStreams.enqueue(str) + } +} + +func (l *loopyWriter) pingHandler(p *ping) error { + if !p.ack { + l.bdpEst.timesnap(p.data) + } + return l.framer.fr.WritePing(p.ack, p.data) + +} + +func (l *loopyWriter) outFlowControlSizeRequestHandler(o *outFlowControlSizeRequest) { + o.resp <- l.sendQuota +} + +func (l *loopyWriter) outStreamRequestHandler(o *outStreamRequestForTesting) { + o.resp <- l.estdStreams[o.streamID] +} + +func (l *loopyWriter) cleanupStreamHandler(c *cleanupStream) error { + c.onWrite() + if str, ok := l.estdStreams[c.streamID]; ok { + // On the server side it could be a trailers-only response or + // a RST_STREAM before stream initialization thus the stream might + // not be established yet. + delete(l.estdStreams, c.streamID) + str.reader.Close() + str.deleteSelf() + for head := str.itl.dequeueAll(); head != nil; head = head.next { + if df, ok := head.it.(*dataFrame); ok { + if !df.processing { + df.data.Free() + } + } + } + } + if c.rst { // If RST_STREAM needs to be sent. + if err := l.framer.fr.WriteRSTStream(c.streamID, c.rstCode); err != nil { + return err + } + } + if l.draining && len(l.estdStreams) == 0 { + // Flush and close the connection; we are done with it. + return errors.New("finished processing active streams while in draining mode") + } + return nil +} + +func (l *loopyWriter) earlyAbortStreamHandler(eas *earlyAbortStream) error { + if l.side == clientSide { + return errors.New("earlyAbortStream not handled on client") + } + if err := l.writeHeader(eas.streamID, true, eas.hf, nil); err != nil { + return err + } + if eas.rst { + if err := l.framer.fr.WriteRSTStream(eas.streamID, http2.ErrCodeNo); err != nil { + return err + } + } + return nil +} + +func (l *loopyWriter) incomingGoAwayHandler(*incomingGoAway) error { + if l.side == clientSide { + l.draining = true + if len(l.estdStreams) == 0 { + // Flush and close the connection; we are done with it. + return errors.New("received GOAWAY with no active streams") + } + } + return nil +} + +func (l *loopyWriter) goAwayHandler(g *goAway) error { + // Handling of outgoing GoAway is very specific to side. + if l.ssGoAwayHandler != nil { + draining, err := l.ssGoAwayHandler(g) + if err != nil { + return err + } + l.draining = draining + } + return nil +} + +func (l *loopyWriter) handle(i any) error { + switch i := i.(type) { + case *incomingWindowUpdate: + l.incomingWindowUpdateHandler(i) + case *outgoingWindowUpdate: + return l.outgoingWindowUpdateHandler(i) + case *incomingSettings: + return l.incomingSettingsHandler(i) + case *outgoingSettings: + return l.outgoingSettingsHandler(i) + case *clientHeaders: + return l.clientHeaderHandler(i) + case *serverHeaders: + return l.serverHeaderHandler(i) + case *registerStream: + l.registerStreamHandler(i) + case *cleanupStream: + return l.cleanupStreamHandler(i) + case *earlyAbortStream: + return l.earlyAbortStreamHandler(i) + case *incomingGoAway: + return l.incomingGoAwayHandler(i) + case *dataFrame: + l.preprocessData(i) + case *ping: + return l.pingHandler(i) + case *goAway: + return l.goAwayHandler(i) + case *outFlowControlSizeRequest: + l.outFlowControlSizeRequestHandler(i) + case *outStreamRequestForTesting: + l.outStreamRequestHandler(i) + case closeConnection: + // Just return a non-I/O error and run() will flush and close the + // connection. + return ErrConnClosing + default: + return fmt.Errorf("transport: unknown control message type %T", i) + } + return nil +} + +func (l *loopyWriter) applySettings(ss []http2.Setting) { + for _, s := range ss { + switch s.ID { + case http2.SettingInitialWindowSize: + o := l.oiws + l.oiws = s.Val + if o < l.oiws { + // If the new limit is greater make all depleted streams active. + for _, stream := range l.estdStreams { + if stream.state == waitingOnStreamQuota { + stream.state = active + l.activeStreams.enqueue(stream) + } + } + } + case http2.SettingHeaderTableSize: + updateHeaderTblSize(l.hEnc, s.Val) + } + } +} + +// processData removes the first stream from active streams, writes out at most 16KB +// of its data and then puts it at the end of activeStreams if there's still more data +// to be sent and stream has some stream-level flow control. +func (l *loopyWriter) processData() (bool, error) { + if l.sendQuota == 0 { + return true, nil + } + str := l.activeStreams.dequeue() // Remove the first stream. + if str == nil { + return true, nil + } + reader := &str.reader + dataItem := str.itl.peek().(*dataFrame) // Peek at the first data item this stream. + if !dataItem.processing { + dataItem.processing = true + reader.Reset(dataItem.data) + dataItem.data.Free() + } + // A data item is represented by a dataFrame, since it later translates into + // multiple HTTP2 data frames. + // Every dataFrame has two buffers; h that keeps grpc-message header and data + // that is the actual message. As an optimization to keep wire traffic low, data + // from data is copied to h to make as big as the maximum possible HTTP2 frame + // size. + + isEmpty := len(dataItem.h) == 0 && reader.Remaining() == 0 + // Figure out the maximum size we can send + maxSize := http2MaxFrameLen + strQuota := int(l.oiws) - str.bytesOutStanding + if strQuota <= 0 && !isEmpty { // stream-level flow control. + str.state = waitingOnStreamQuota + return false, nil + } + maxSize = min(maxSize, max(strQuota, 0)) + maxSize = min(maxSize, int(l.sendQuota)) // connection-level flow control. + // Compute how much of the header and data we can send within quota and max frame length + hSize := min(maxSize, len(dataItem.h)) + dSize := min(maxSize-hSize, reader.Remaining()) + remainingBytes := len(dataItem.h) + reader.Remaining() - hSize - dSize + size := hSize + dSize + + l.writeBuf = l.writeBuf[:0] + if hSize > 0 { + l.writeBuf = append(l.writeBuf, dataItem.h[:hSize]) + } + if dSize > 0 { + var err error + l.writeBuf, err = reader.Peek(dSize, l.writeBuf) + if err != nil { + // This must never happen since the reader must have at least dSize + // bytes. + // Log an error to fail tests. + l.logger.Errorf("unexpected error while reading Data frame payload: %v", err) + return false, err + } + } + + // Now that outgoing flow controls are checked we can replenish str's write quota + str.wq.replenish(size) + var endStream bool + // If this is the last data message on this stream and all of it can be written in this iteration. + if dataItem.endStream && remainingBytes == 0 { + endStream = true + } + if dataItem.onEachWrite != nil { + dataItem.onEachWrite() + } + err := l.framer.writeData(dataItem.streamID, endStream, l.writeBuf) + reader.Discard(dSize) + if cap(l.writeBuf) > maxWriteBufSize { + l.writeBuf = nil + } else { + clear(l.writeBuf) + } + if err != nil { + return false, err + } + str.bytesOutStanding += size + l.sendQuota -= uint32(size) + dataItem.h = dataItem.h[hSize:] + + if remainingBytes == 0 { // All the data from that message was written out. + reader.Close() + str.itl.dequeue() + } + return false, l.updateStreamAfterWrite(str) +} + +func (l *loopyWriter) updateStreamAfterWrite(str *outStream) error { + if str.itl.isEmpty() { + str.state = empty + } else if trailer, ok := str.itl.peek().(*serverHeaders); ok { // the next item is trailers. + if err := l.writeHeader(trailer.streamID, trailer.endStream, trailer.hf, trailer.onWrite); err != nil { + return err + } + if err := l.cleanupStreamHandler(trailer.cleanup); err != nil { + return err + } + } else if int(l.oiws)-str.bytesOutStanding <= 0 { // Ran out of stream quota. + str.state = waitingOnStreamQuota + } else { // Otherwise add it back to the list of active streams. + l.activeStreams.enqueue(str) + } + return nil +} diff --git a/internal/transport/controlbuf_test.go b/internal/transport/controlbuf_test.go new file mode 100644 index 0000000..eb71ddf --- /dev/null +++ b/internal/transport/controlbuf_test.go @@ -0,0 +1,85 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "testing" + "time" +) + +func (s) TestControlBuffer_Throttle(t *testing.T) { + done := make(chan struct{}) + defer close(done) + cb := newControlBuffer(done) + + // Fill the control buffer up to the limit with throttled items. + for i := 0; i < maxQueuedControlBufferItems; i++ { + cb.put(&ping{}) + } + + // The next call to throttle should block. + throttleDone := make(chan struct{}) + go func() { + cb.throttle() + close(throttleDone) + }() + + select { + case <-throttleDone: + t.Fatal("throttle() did not block when the buffer was full") + case <-time.After(defaultTestShortTimeout): + } + + // Consume one item from the control buffer. + if _, err := cb.get(true); err != nil { + t.Fatalf("cb.get(true) failed: %v", err) + } + + // Now throttle() should unblock. + select { + case <-throttleDone: + case <-time.After(time.Second): + t.Fatal("throttle() did not unblock after an item was consumed") + } +} + +func (s) TestControlBuffer_NoThrottleForNonThrottledItems(t *testing.T) { + done := make(chan struct{}) + defer close(done) + cb := newControlBuffer(done) + + // Fill the control buffer with many more than limit number of non-throttled + // items. + for i := 0; i < maxQueuedControlBufferItems+10; i++ { + cb.put(&dataFrame{}) + } + + // throttle() should not block. + throttled := make(chan struct{}) + go func() { + cb.throttle() + close(throttled) + }() + + select { + case <-throttled: + case <-time.After(defaultTestShortTimeout): + t.Fatal("throttle() blocked for non-throttled items") + } +} diff --git a/internal/transport/defaults.go b/internal/transport/defaults.go new file mode 100644 index 0000000..0b2269a --- /dev/null +++ b/internal/transport/defaults.go @@ -0,0 +1,56 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "math" + "time" +) + +const ( + // The default value of flow control window size in HTTP2 spec. + defaultWindowSize = 65535 + // The initial window size for flow control. + initialWindowSize = defaultWindowSize // for an RPC + infinity = time.Duration(math.MaxInt64) + defaultClientKeepaliveTime = infinity + defaultClientKeepaliveTimeout = 20 * time.Second + defaultMaxStreamsClient = 100 + defaultMaxConnectionIdle = infinity + defaultMaxConnectionAge = infinity + defaultMaxConnectionAgeGrace = infinity + defaultServerKeepaliveTime = 2 * time.Hour + defaultServerKeepaliveTimeout = 20 * time.Second + defaultKeepalivePolicyMinTime = 5 * time.Minute + // max window limit set by HTTP2 Specs. + maxWindowSize = math.MaxInt32 + // defaultWriteQuota is the default value for number of data + // bytes that each stream can schedule before some of it being + // flushed out. + defaultWriteQuota = 64 * 1024 + defaultClientMaxHeaderListSize = uint32(16 << 20) + defaultServerMaxHeaderListSize = uint32(16 << 20) + upcomingDefaultHeaderListSize = uint32(8 << 10) +) + +// MaxStreamID is the upper bound for the stream ID before the current +// transport gracefully closes and new transport is created for subsequent RPCs. +// This is set to 75% of 2^31-1. Streams are identified with an unsigned 31-bit +// integer. It's exported so that tests can override it. +var MaxStreamID = uint32(math.MaxInt32 * 3 / 4) diff --git a/internal/transport/flowcontrol.go b/internal/transport/flowcontrol.go new file mode 100644 index 0000000..98cef9e --- /dev/null +++ b/internal/transport/flowcontrol.go @@ -0,0 +1,211 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "fmt" + "math" + "sync" + "sync/atomic" +) + +// writeQuota is a soft limit on the amount of data a stream can +// schedule before some of it is written out. +type writeQuota struct { + _ noCopy + // get waits on read from when quota goes less than or equal to zero. + // replenish writes on it when quota goes positive again. + ch chan struct{} + // done is triggered in error case. + done <-chan struct{} + // replenish is called by loopyWriter to give quota back to. + // It is implemented as a field so that it can be updated + // by tests. + replenish func(n int) + quota int32 +} + +// init allows a writeQuota to be initialized in-place, which is useful for +// resetting a buffer or for avoiding a heap allocation when the buffer is +// embedded in another struct. +func (w *writeQuota) init(sz int32, done <-chan struct{}) { + w.quota = sz + w.ch = make(chan struct{}, 1) + w.done = done + w.replenish = w.realReplenish +} + +func (w *writeQuota) get(sz int32) error { + for { + if atomic.LoadInt32(&w.quota) > 0 { + atomic.AddInt32(&w.quota, -sz) + return nil + } + select { + case <-w.ch: + continue + case <-w.done: + return errStreamDone + } + } +} + +func (w *writeQuota) realReplenish(n int) { + sz := int32(n) + newQuota := atomic.AddInt32(&w.quota, sz) + previousQuota := newQuota - sz + if previousQuota <= 0 && newQuota > 0 { + select { + case w.ch <- struct{}{}: + default: + } + } +} + +type trInFlow struct { + limit uint32 + unacked uint32 + effectiveWindowSize uint32 +} + +func (f *trInFlow) newLimit(n uint32) uint32 { + d := n - f.limit + f.limit = n + f.updateEffectiveWindowSize() + return d +} + +func (f *trInFlow) onData(n uint32) uint32 { + f.unacked += n + if f.unacked < f.limit/4 { + f.updateEffectiveWindowSize() + return 0 + } + return f.reset() +} + +func (f *trInFlow) reset() uint32 { + w := f.unacked + f.unacked = 0 + f.updateEffectiveWindowSize() + return w +} + +func (f *trInFlow) updateEffectiveWindowSize() { + atomic.StoreUint32(&f.effectiveWindowSize, f.limit-f.unacked) +} + +func (f *trInFlow) getSize() uint32 { + return atomic.LoadUint32(&f.effectiveWindowSize) +} + +// inFlow deals with inbound flow control +type inFlow struct { + mu sync.Mutex + // The inbound flow control limit for pending data. + limit uint32 + // pendingData is the overall data which have been received but not been + // consumed by applications. + pendingData uint32 + // The amount of data the application has consumed but grpc has not sent + // window update for them. Used to reduce window update frequency. + pendingUpdate uint32 + // delta is the extra window update given by receiver when an application + // is reading data bigger in size than the inFlow limit. + delta uint32 +} + +// newLimit updates the inflow window to a new value n. +// It assumes that n is always greater than the old limit. +func (f *inFlow) newLimit(n uint32) { + f.mu.Lock() + f.limit = n + f.mu.Unlock() +} + +func (f *inFlow) maybeAdjust(n uint32) uint32 { + if n > uint32(math.MaxInt32) { + n = uint32(math.MaxInt32) + } + f.mu.Lock() + defer f.mu.Unlock() + // estSenderQuota is the receiver's view of the maximum number of bytes the sender + // can send without a window update. + estSenderQuota := int32(f.limit - (f.pendingData + f.pendingUpdate)) + // estUntransmittedData is the maximum number of bytes the sends might not have put + // on the wire yet. A value of 0 or less means that we have already received all or + // more bytes than the application is requesting to read. + estUntransmittedData := int32(n - f.pendingData) // Casting into int32 since it could be negative. + // This implies that unless we send a window update, the sender won't be able to send all the bytes + // for this message. Therefore we must send an update over the limit since there's an active read + // request from the application. + if estUntransmittedData > estSenderQuota { + // Sender's window shouldn't go more than 2^31 - 1 as specified in the HTTP spec. + if f.limit+n > maxWindowSize { + f.delta = maxWindowSize - f.limit + } else { + // Send a window update for the whole message and not just the difference between + // estUntransmittedData and estSenderQuota. This will be helpful in case the message + // is padded; We will fallback on the current available window(at least a 1/4th of the limit). + f.delta = n + } + return f.delta + } + return 0 +} + +// onData is invoked when some data frame is received. It updates pendingData. +func (f *inFlow) onData(n uint32) error { + f.mu.Lock() + defer f.mu.Unlock() + + f.pendingData += n + if f.pendingData+f.pendingUpdate > f.limit+f.delta { + limit := f.limit + rcvd := f.pendingData + f.pendingUpdate + return fmt.Errorf("received %d-bytes data exceeding the limit %d bytes", rcvd, limit) + } + return nil +} + +// onRead is invoked when the application reads the data. It returns the window size +// to be sent to the peer. +func (f *inFlow) onRead(n uint32) uint32 { + f.mu.Lock() + defer f.mu.Unlock() + + if f.pendingData == 0 { + return 0 + } + f.pendingData -= n + if n > f.delta { + n -= f.delta + f.delta = 0 + } else { + f.delta -= n + n = 0 + } + f.pendingUpdate += n + if f.pendingUpdate >= f.limit/4 { + wu := f.pendingUpdate + f.pendingUpdate = 0 + return wu + } + return 0 +} diff --git a/internal/transport/handler_server.go b/internal/transport/handler_server.go new file mode 100644 index 0000000..9cd8d28 --- /dev/null +++ b/internal/transport/handler_server.go @@ -0,0 +1,506 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// This file is the implementation of a gRPC server using HTTP/2 which +// uses the standard Go http2 Server implementation (via the +// http.Handler interface), rather than speaking low-level HTTP/2 +// frames itself. It is the implementation of *grpc.Server.ServeHTTP. + +package transport + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "strings" + "sync" + "time" + + "golang.org/x/net/http2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcutil" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +// NewServerHandlerTransport returns a ServerTransport handling gRPC from +// inside an http.Handler, or writes an HTTP error to w and returns an error. +// It requires that the http Server supports HTTP/2. +func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request, stats stats.Handler, bufferPool mem.BufferPool) (ServerTransport, error) { + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + msg := fmt.Sprintf("invalid gRPC request method %q", r.Method) + http.Error(w, msg, http.StatusMethodNotAllowed) + return nil, errors.New(msg) + } + contentType := r.Header.Get("Content-Type") + // TODO: do we assume contentType is lowercase? we did before + contentSubtype, validContentType := grpcutil.ContentSubtype(contentType) + if !validContentType { + msg := fmt.Sprintf("invalid gRPC request content-type %q", contentType) + http.Error(w, msg, http.StatusUnsupportedMediaType) + return nil, errors.New(msg) + } + if r.ProtoMajor != 2 { + msg := "gRPC requires HTTP/2" + http.Error(w, msg, http.StatusHTTPVersionNotSupported) + return nil, errors.New(msg) + } + if _, ok := w.(http.Flusher); !ok { + msg := "gRPC requires a ResponseWriter supporting http.Flusher" + http.Error(w, msg, http.StatusInternalServerError) + return nil, errors.New(msg) + } + + var localAddr net.Addr + if la := r.Context().Value(http.LocalAddrContextKey); la != nil { + localAddr, _ = la.(net.Addr) + } + var authInfo credentials.AuthInfo + if r.TLS != nil { + authInfo = credentials.TLSInfo{State: *r.TLS, CommonAuthInfo: credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}} + } + p := peer.Peer{ + Addr: strAddr(r.RemoteAddr), + LocalAddr: localAddr, + AuthInfo: authInfo, + } + st := &serverHandlerTransport{ + rw: w, + req: r, + closedCh: make(chan struct{}), + writes: make(chan func()), + peer: p, + contentType: contentType, + contentSubtype: contentSubtype, + stats: stats, + bufferPool: bufferPool, + } + st.logger = prefixLoggerForServerHandlerTransport(st) + + if v := r.Header.Get("grpc-timeout"); v != "" { + to, err := decodeTimeout(v) + if err != nil { + msg := fmt.Sprintf("malformed grpc-timeout: %v", err) + http.Error(w, msg, http.StatusBadRequest) + return nil, status.Error(codes.Internal, msg) + } + st.timeoutSet = true + st.timeout = to + } + + metakv := []string{"content-type", contentType} + if r.Host != "" { + metakv = append(metakv, ":authority", r.Host) + } + for k, vv := range r.Header { + k = strings.ToLower(k) + if isReservedHeader(k) && !isWhitelistedHeader(k) { + continue + } + for _, v := range vv { + v, err := decodeMetadataHeader(k, v) + if err != nil { + msg := fmt.Sprintf("malformed binary metadata %q in header %q: %v", v, k, err) + http.Error(w, msg, http.StatusBadRequest) + return nil, status.Error(codes.Internal, msg) + } + metakv = append(metakv, k, v) + } + } + st.headerMD = metadata.Pairs(metakv...) + + return st, nil +} + +// serverHandlerTransport is an implementation of ServerTransport +// which replies to exactly one gRPC request (exactly one HTTP request), +// using the net/http.Handler interface. This http.Handler is guaranteed +// at this point to be speaking over HTTP/2, so it's able to speak valid +// gRPC. +type serverHandlerTransport struct { + rw http.ResponseWriter + req *http.Request + timeoutSet bool + timeout time.Duration + + headerMD metadata.MD + + peer peer.Peer + + closeOnce sync.Once + closedCh chan struct{} // closed on Close + + // writes is a channel of code to run serialized in the + // ServeHTTP (HandleStreams) goroutine. The channel is closed + // when WriteStatus is called. + writes chan func() + + // block concurrent WriteStatus calls + // e.g. grpc/(*serverStream).SendMsg/RecvMsg + writeStatusMu sync.Mutex + + // we just mirror the request content-type + contentType string + // we store both contentType and contentSubtype so we don't keep recreating them + // TODO make sure this is consistent across handler_server and http2_server + contentSubtype string + + stats stats.Handler + logger *grpclog.PrefixLogger + + bufferPool mem.BufferPool +} + +func (ht *serverHandlerTransport) Close(err error) { + ht.closeOnce.Do(func() { + if ht.logger.V(logLevel) { + ht.logger.Infof("Closing: %v", err) + } + close(ht.closedCh) + }) +} + +func (ht *serverHandlerTransport) Peer() *peer.Peer { + return &peer.Peer{ + Addr: ht.peer.Addr, + LocalAddr: ht.peer.LocalAddr, + AuthInfo: ht.peer.AuthInfo, + } +} + +// strAddr is a net.Addr backed by either a TCP "ip:port" string, or +// the empty string if unknown. +type strAddr string + +func (a strAddr) Network() string { + if a != "" { + // Per the documentation on net/http.Request.RemoteAddr, if this is + // set, it's set to the IP:port of the peer (hence, TCP): + // https://golang.org/pkg/net/http/#Request + // + // If we want to support Unix sockets later, we can + // add our own grpc-specific convention within the + // grpc codebase to set RemoteAddr to a different + // format, or probably better: we can attach it to the + // context and use that from serverHandlerTransport.RemoteAddr. + return "tcp" + } + return "" +} + +func (a strAddr) String() string { return string(a) } + +// do runs fn in the ServeHTTP goroutine. +func (ht *serverHandlerTransport) do(fn func()) error { + select { + case <-ht.closedCh: + return ErrConnClosing + case ht.writes <- fn: + return nil + } +} + +func (ht *serverHandlerTransport) writeStatus(s *ServerStream, st *status.Status) error { + ht.writeStatusMu.Lock() + defer ht.writeStatusMu.Unlock() + + headersWritten := s.updateHeaderSent() + err := ht.do(func() { + if !headersWritten { + ht.writePendingHeaders(s) + } + + // And flush, in case no header or body has been sent yet. + // This forces a separation of headers and trailers if this is the + // first call (for example, in end2end tests's TestNoService). + ht.rw.(http.Flusher).Flush() + + h := ht.rw.Header() + h.Set("Grpc-Status", fmt.Sprintf("%d", st.Code())) + if m := st.Message(); m != "" { + h.Set("Grpc-Message", encodeGrpcMessage(m)) + } + + s.hdrMu.Lock() + defer s.hdrMu.Unlock() + if p := st.Proto(); p != nil && len(p.Details) > 0 { + delete(s.trailer, grpcStatusDetailsBinHeader) + stBytes, err := proto.Marshal(p) + if err != nil { + // TODO: return error instead, when callers are able to handle it. + panic(err) + } + + h.Set(grpcStatusDetailsBinHeader, encodeBinHeader(stBytes)) + } + + if len(s.trailer) > 0 { + for k, vv := range s.trailer { + // Clients don't tolerate reading restricted headers after some non restricted ones were sent. + if isReservedHeader(k) { + continue + } + for _, v := range vv { + // http2 ResponseWriter mechanism to send undeclared Trailers after + // the headers have possibly been written. + h.Add(http2.TrailerPrefix+k, encodeMetadataHeader(k, v)) + } + } + } + }) + + if err == nil && ht.stats != nil { // transport has not been closed + // Note: The trailer fields are compressed with hpack after this call returns. + // No WireLength field is set here. + s.hdrMu.Lock() + ht.stats.HandleRPC(s.Context(), &stats.OutTrailer{ + Trailer: s.trailer.Copy(), + }) + s.hdrMu.Unlock() + } + ht.Close(errors.New("finished writing status")) + return err +} + +// writePendingHeaders sets common and custom headers on the first +// write call (Write, WriteHeader, or WriteStatus) +func (ht *serverHandlerTransport) writePendingHeaders(s *ServerStream) { + ht.writeCommonHeaders(s) + ht.writeCustomHeaders(s) +} + +// writeCommonHeaders sets common headers on the first write +// call (Write, WriteHeader, or WriteStatus). +func (ht *serverHandlerTransport) writeCommonHeaders(s *ServerStream) { + h := ht.rw.Header() + h["Date"] = nil // suppress Date to make tests happy; TODO: restore + h.Set("Content-Type", ht.contentType) + + // Predeclare trailers we'll set later in WriteStatus (after the body). + // This is a SHOULD in the HTTP RFC, and the way you add (known) + // Trailers per the net/http.ResponseWriter contract. + // See https://golang.org/pkg/net/http/#ResponseWriter + // and https://golang.org/pkg/net/http/#example_ResponseWriter_trailers + h.Add("Trailer", "Grpc-Status") + h.Add("Trailer", "Grpc-Message") + h.Add("Trailer", "Grpc-Status-Details-Bin") + + if s.sendCompress != "" { + h.Set("Grpc-Encoding", s.sendCompress) + } +} + +// writeCustomHeaders sets custom headers set on the stream via SetHeader +// on the first write call (Write, WriteHeader, or WriteStatus) +func (ht *serverHandlerTransport) writeCustomHeaders(s *ServerStream) { + h := ht.rw.Header() + + s.hdrMu.Lock() + for k, vv := range s.header { + if isReservedHeader(k) { + continue + } + for _, v := range vv { + h.Add(k, encodeMetadataHeader(k, v)) + } + } + + s.hdrMu.Unlock() +} + +func (ht *serverHandlerTransport) write(s *ServerStream, hdr []byte, data mem.BufferSlice, _ *WriteOptions) error { + // Always take a reference because otherwise there is no guarantee the data will + // be available after this function returns. This is what callers to Write + // expect. + data.Ref() + headersWritten := s.updateHeaderSent() + err := ht.do(func() { + defer data.Free() + if !headersWritten { + ht.writePendingHeaders(s) + } + ht.rw.Write(hdr) + for _, b := range data { + _, _ = ht.rw.Write(b.ReadOnlyData()) + } + ht.rw.(http.Flusher).Flush() + }) + if err != nil { + data.Free() + return err + } + return nil +} + +func (ht *serverHandlerTransport) writeHeader(s *ServerStream, md metadata.MD) error { + if err := s.SetHeader(md); err != nil { + return err + } + + headersWritten := s.updateHeaderSent() + err := ht.do(func() { + if !headersWritten { + ht.writePendingHeaders(s) + } + + ht.rw.WriteHeader(200) + ht.rw.(http.Flusher).Flush() + }) + + if err == nil && ht.stats != nil { + // Note: The header fields are compressed with hpack after this call returns. + // No WireLength field is set here. + ht.stats.HandleRPC(s.Context(), &stats.OutHeader{ + Header: md.Copy(), + Compression: s.sendCompress, + }) + } + return err +} + +func (ht *serverHandlerTransport) adjustWindow(*ServerStream, uint32) { +} + +func (ht *serverHandlerTransport) updateWindow(*ServerStream, uint32) { +} + +func (ht *serverHandlerTransport) HandleStreams(ctx context.Context, startStream func(*ServerStream)) { + // With this transport type there will be exactly 1 stream: this HTTP request. + var cancel context.CancelFunc + if ht.timeoutSet { + ctx, cancel = context.WithTimeout(ctx, ht.timeout) + } else { + ctx, cancel = context.WithCancel(ctx) + } + + // requestOver is closed when the status has been written via WriteStatus. + requestOver := make(chan struct{}) + go func() { + select { + case <-requestOver: + case <-ht.closedCh: + case <-ht.req.Context().Done(): + } + cancel() + ht.Close(errors.New("request is done processing")) + }() + + ctx = metadata.NewIncomingContext(ctx, ht.headerMD) + req := ht.req + s := &ServerStream{ + Stream: Stream{ + id: 0, // irrelevant + ctx: ctx, + method: req.URL.Path, + recvCompress: req.Header.Get("grpc-encoding"), + contentSubtype: ht.contentSubtype, + }, + cancel: cancel, + st: ht, + headerWireLength: 0, // won't have access to header wire length until golang/go#18997. + } + s.Stream.buf.init(ht.bufferPool) + s.readRequester = s + s.trReader = transportReader{ + reader: recvBufferReader{ctx: s.ctx, ctxDone: s.ctx.Done(), recv: &s.buf}, + windowHandler: s, + } + + // readerDone is closed when the Body.Read-ing goroutine exits. + readerDone := make(chan struct{}) + go func() { + defer close(readerDone) + + for { + buf := ht.bufferPool.Get(http2MaxFrameLen) + n, err := req.Body.Read(*buf) + if n > 0 { + *buf = (*buf)[:n] + s.buf.put(recvMsg{buffer: mem.NewBuffer(buf, ht.bufferPool)}) + } else { + ht.bufferPool.Put(buf) + } + if err != nil { + s.buf.put(recvMsg{err: mapRecvMsgError(err)}) + return + } + } + }() + + // startStream is provided by the *grpc.Server's serveStreams. + // It starts a goroutine serving s and exits immediately. + // The goroutine that is started is the one that then calls + // into ht, calling WriteHeader, Write, WriteStatus, Close, etc. + startStream(s) + + ht.runStream() + close(requestOver) + + // Wait for reading goroutine to finish. + req.Body.Close() + <-readerDone +} + +func (ht *serverHandlerTransport) runStream() { + for { + select { + case fn := <-ht.writes: + fn() + case <-ht.closedCh: + return + } + } +} + +func (ht *serverHandlerTransport) incrMsgRecv() {} + +func (ht *serverHandlerTransport) Drain(s string) { + ht.Close(errors.New(s)) +} + +// mapRecvMsgError returns the non-nil err into the appropriate +// error value as expected by callers of *grpc.parser.recvMsg. +// In particular, in can only be: +// - io.EOF +// - io.ErrUnexpectedEOF +// - of type transport.ConnectionError +// - an error from the status package +func mapRecvMsgError(err error) error { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return err + } + if se, ok := err.(http2.StreamError); ok { + if code, ok := http2ErrConvTab[se.Code]; ok { + return status.Error(code, se.Error()) + } + } + if strings.Contains(err.Error(), "body closed by handler") { + return status.Error(codes.Canceled, err.Error()) + } + return connectionErrorf(true, err, "%s", err.Error()) +} diff --git a/internal/transport/handler_server_test.go b/internal/transport/handler_server_test.go new file mode 100644 index 0000000..1610863 --- /dev/null +++ b/internal/transport/handler_server_test.go @@ -0,0 +1,614 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "sync" + "testing" + "time" + + epb "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/protoadapt" + "google.golang.org/protobuf/types/known/durationpb" +) + +func (s) TestHandlerTransport_NewServerHandlerTransport(t *testing.T) { + type testCase struct { + name string + req *http.Request + wantErr string + wantErrCode int + modrw func(http.ResponseWriter) http.ResponseWriter + check func(*serverHandlerTransport, *testCase) error + } + tests := []testCase{ + { + name: "bad method", + req: &http.Request{ + ProtoMajor: 2, + Method: "GET", + Header: http.Header{}, + }, + wantErr: `invalid gRPC request method "GET"`, + wantErrCode: http.StatusMethodNotAllowed, + }, + { + name: "bad content type", + req: &http.Request{ + ProtoMajor: 2, + Method: "POST", + Header: http.Header{ + "Content-Type": {"application/foo"}, + }, + }, + wantErr: `invalid gRPC request content-type "application/foo"`, + wantErrCode: http.StatusUnsupportedMediaType, + }, + { + name: "http/1.1", + req: &http.Request{ + ProtoMajor: 1, + ProtoMinor: 1, + Method: "POST", + Header: http.Header{"Content-Type": []string{"application/grpc"}}, + }, + wantErr: "gRPC requires HTTP/2", + wantErrCode: http.StatusHTTPVersionNotSupported, + }, + { + name: "not flusher", + req: &http.Request{ + ProtoMajor: 2, + Method: "POST", + Header: http.Header{ + "Content-Type": {"application/grpc"}, + }, + }, + modrw: func(w http.ResponseWriter) http.ResponseWriter { + // Return w without its Flush method + type onlyCloseNotifier interface { + http.ResponseWriter + } + return struct{ onlyCloseNotifier }{w.(onlyCloseNotifier)} + }, + wantErr: "gRPC requires a ResponseWriter supporting http.Flusher", + wantErrCode: http.StatusInternalServerError, + }, + { + name: "valid", + req: &http.Request{ + ProtoMajor: 2, + Method: "POST", + Header: http.Header{ + "Content-Type": {"application/grpc"}, + }, + URL: &url.URL{ + Path: "/service/foo.bar", + }, + }, + check: func(t *serverHandlerTransport, tt *testCase) error { + if t.req != tt.req { + return fmt.Errorf("t.req = %p; want %p", t.req, tt.req) + } + if t.rw == nil { + return errors.New("t.rw = nil; want non-nil") + } + return nil + }, + }, + { + name: "with timeout", + req: &http.Request{ + ProtoMajor: 2, + Method: "POST", + Header: http.Header{ + "Content-Type": []string{"application/grpc"}, + "Grpc-Timeout": {"200m"}, + }, + URL: &url.URL{ + Path: "/service/foo.bar", + }, + }, + check: func(t *serverHandlerTransport, _ *testCase) error { + if !t.timeoutSet { + return errors.New("timeout not set") + } + if want := 200 * time.Millisecond; t.timeout != want { + return fmt.Errorf("timeout = %v; want %v", t.timeout, want) + } + return nil + }, + }, + { + name: "with bad timeout", + req: &http.Request{ + ProtoMajor: 2, + Method: "POST", + Header: http.Header{ + "Content-Type": []string{"application/grpc"}, + "Grpc-Timeout": {"tomorrow"}, + }, + URL: &url.URL{ + Path: "/service/foo.bar", + }, + }, + wantErr: `rpc error: code = Internal desc = malformed grpc-timeout: transport: timeout unit is not recognized: "tomorrow"`, + wantErrCode: http.StatusBadRequest, + }, + { + name: "with metadata", + req: &http.Request{ + ProtoMajor: 2, + Method: "POST", + Header: http.Header{ + "Content-Type": []string{"application/grpc"}, + "meta-foo": {"foo-val"}, + "meta-bar": {"bar-val1", "bar-val2"}, + "user-agent": {"x/y a/b"}, + }, + URL: &url.URL{ + Path: "/service/foo.bar", + }, + }, + check: func(ht *serverHandlerTransport, _ *testCase) error { + want := metadata.MD{ + "meta-bar": {"bar-val1", "bar-val2"}, + "user-agent": {"x/y a/b"}, + "meta-foo": {"foo-val"}, + "content-type": {"application/grpc"}, + } + + if !reflect.DeepEqual(ht.headerMD, want) { + return fmt.Errorf("metadata = %#v; want %#v", ht.headerMD, want) + } + return nil + }, + }, + } + + for _, tt := range tests { + rrec := httptest.NewRecorder() + rw := http.ResponseWriter(testHandlerResponseWriter{ + ResponseRecorder: rrec, + }) + + if tt.modrw != nil { + rw = tt.modrw(rw) + } + got, gotErr := NewServerHandlerTransport(rw, tt.req, nil, mem.DefaultBufferPool()) + if (gotErr != nil) != (tt.wantErr != "") || (gotErr != nil && gotErr.Error() != tt.wantErr) { + t.Errorf("%s: error = %q; want %q", tt.name, gotErr.Error(), tt.wantErr) + continue + } + if tt.wantErrCode == 0 { + tt.wantErrCode = http.StatusOK + } + if rrec.Code != tt.wantErrCode { + t.Errorf("%s: code = %d; want %d", tt.name, rrec.Code, tt.wantErrCode) + continue + } + if gotErr != nil { + continue + } + if tt.check != nil { + if err := tt.check(got.(*serverHandlerTransport), &tt); err != nil { + t.Errorf("%s: %v", tt.name, err) + } + } + } +} + +type testHandlerResponseWriter struct { + *httptest.ResponseRecorder +} + +func (w testHandlerResponseWriter) Flush() {} + +func newTestHandlerResponseWriter() http.ResponseWriter { + return testHandlerResponseWriter{ + ResponseRecorder: httptest.NewRecorder(), + } +} + +type handleStreamTest struct { + t *testing.T + bodyw *io.PipeWriter + rw testHandlerResponseWriter + ht *serverHandlerTransport +} + +type mockStatsHandler struct { + rpcStatsCh chan stats.RPCStats +} + +func (h *mockStatsHandler) TagRPC(ctx context.Context, _ *stats.RPCTagInfo) context.Context { + return ctx +} + +func (h *mockStatsHandler) HandleRPC(_ context.Context, s stats.RPCStats) { + h.rpcStatsCh <- s +} + +func (h *mockStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +func (h *mockStatsHandler) HandleConn(context.Context, stats.ConnStats) { +} + +func newHandleStreamTest(t *testing.T, statsHandler stats.Handler) *handleStreamTest { + bodyr, bodyw := io.Pipe() + req := &http.Request{ + ProtoMajor: 2, + Method: "POST", + Header: http.Header{ + "Content-Type": {"application/grpc"}, + }, + URL: &url.URL{ + Path: "/service/foo.bar", + }, + Body: bodyr, + } + rw := newTestHandlerResponseWriter().(testHandlerResponseWriter) + ht, err := NewServerHandlerTransport(rw, req, statsHandler, mem.DefaultBufferPool()) + if err != nil { + t.Fatal(err) + } + return &handleStreamTest{ + t: t, + bodyw: bodyw, + ht: ht.(*serverHandlerTransport), + rw: rw, + } +} + +func (s) TestHandlerTransport_HandleStreams(t *testing.T) { + st := newHandleStreamTest(t, nil) + handleStream := func(s *ServerStream) { + if want := "/service/foo.bar"; s.method != want { + t.Errorf("stream method = %q; want %q", s.method, want) + } + + if err := s.SetHeader(metadata.Pairs("custom-header", "Custom header value")); err != nil { + t.Error(err) + } + + if err := s.SetTrailer(metadata.Pairs("custom-trailer", "Custom trailer value")); err != nil { + t.Error(err) + } + + if err := s.SetSendCompress("gzip"); err != nil { + t.Error(err) + } + + md := metadata.Pairs("custom-header", "Another custom header value") + if err := s.SendHeader(md); err != nil { + t.Error(err) + } + delete(md, "custom-header") + + if err := s.SetHeader(metadata.Pairs("too-late", "Header value that should be ignored")); err == nil { + t.Error("expected SetHeader call after SendHeader to fail") + } + + if err := s.SendHeader(metadata.Pairs("too-late", "This header value should be ignored as well")); err == nil { + t.Error("expected second SendHeader call to fail") + } + + if err := s.SetSendCompress("snappy"); err == nil { + t.Error("expected second SetSendCompress call to fail") + } + + st.bodyw.Close() // no body + s.WriteStatus(status.New(codes.OK, "")) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + st.ht.HandleStreams( + ctx, func(s *ServerStream) { go handleStream(s) }, + ) + wantHeader := http.Header{ + "Date": nil, + "Content-Type": {"application/grpc"}, + "Trailer": {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"}, + "Custom-Header": {"Custom header value", "Another custom header value"}, + "Grpc-Encoding": {"gzip"}, + } + wantTrailer := http.Header{ + "Grpc-Status": {"0"}, + "Custom-Trailer": {"Custom trailer value"}, + } + checkHeaderAndTrailer(t, st.rw, wantHeader, wantTrailer) +} + +// Tests that codes.Unimplemented will close the body, per comment in handler_server.go. +func (s) TestHandlerTransport_HandleStreams_Unimplemented(t *testing.T) { + handleStreamCloseBodyTest(t, codes.Unimplemented, "thingy is unimplemented") +} + +// Tests that codes.InvalidArgument will close the body, per comment in handler_server.go. +func (s) TestHandlerTransport_HandleStreams_InvalidArgument(t *testing.T) { + handleStreamCloseBodyTest(t, codes.InvalidArgument, "bad arg") +} + +func handleStreamCloseBodyTest(t *testing.T, statusCode codes.Code, msg string) { + st := newHandleStreamTest(t, nil) + + handleStream := func(s *ServerStream) { + s.WriteStatus(status.New(statusCode, msg)) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + st.ht.HandleStreams( + ctx, func(s *ServerStream) { go handleStream(s) }, + ) + wantHeader := http.Header{ + "Date": nil, + "Content-Type": {"application/grpc"}, + "Trailer": {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"}, + } + wantTrailer := http.Header{ + "Grpc-Status": {fmt.Sprint(uint32(statusCode))}, + "Grpc-Message": {encodeGrpcMessage(msg)}, + } + checkHeaderAndTrailer(t, st.rw, wantHeader, wantTrailer) +} + +func (s) TestHandlerTransport_HandleStreams_Timeout(t *testing.T) { + bodyr, bodyw := io.Pipe() + req := &http.Request{ + ProtoMajor: 2, + Method: "POST", + Header: http.Header{ + "Content-Type": {"application/grpc"}, + "Grpc-Timeout": {"200m"}, + }, + URL: &url.URL{ + Path: "/service/foo.bar", + }, + Body: bodyr, + } + rw := newTestHandlerResponseWriter().(testHandlerResponseWriter) + ht, err := NewServerHandlerTransport(rw, req, nil, mem.DefaultBufferPool()) + if err != nil { + t.Fatal(err) + } + runStream := func(s *ServerStream) { + defer bodyw.Close() + select { + case <-s.ctx.Done(): + case <-time.After(5 * time.Second): + t.Errorf("timeout waiting for ctx.Done") + return + } + err := s.ctx.Err() + if err != context.DeadlineExceeded { + t.Errorf("ctx.Err = %v; want %v", err, context.DeadlineExceeded) + return + } + s.WriteStatus(status.New(codes.DeadlineExceeded, "too slow")) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ht.HandleStreams( + ctx, func(s *ServerStream) { go runStream(s) }, + ) + wantHeader := http.Header{ + "Date": nil, + "Content-Type": {"application/grpc"}, + "Trailer": {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"}, + } + wantTrailer := http.Header{ + "Grpc-Status": {"4"}, + "Grpc-Message": {encodeGrpcMessage("too slow")}, + } + checkHeaderAndTrailer(t, rw, wantHeader, wantTrailer) +} + +// TestHandlerTransport_HandleStreams_MultiWriteStatus ensures that +// concurrent "WriteStatus"s do not panic writing to closed "writes" channel. +func (s) TestHandlerTransport_HandleStreams_MultiWriteStatus(t *testing.T) { + testHandlerTransportHandleStreams(t, func(st *handleStreamTest, s *ServerStream) { + if want := "/service/foo.bar"; s.method != want { + t.Errorf("stream method = %q; want %q", s.method, want) + } + st.bodyw.Close() // no body + + var wg sync.WaitGroup + wg.Add(5) + for i := 0; i < 5; i++ { + go func() { + defer wg.Done() + s.WriteStatus(status.New(codes.OK, "")) + }() + } + wg.Wait() + }) +} + +// TestHandlerTransport_HandleStreams_WriteStatusWrite ensures that "Write" +// following "WriteStatus" does not panic writing to closed "writes" channel. +func (s) TestHandlerTransport_HandleStreams_WriteStatusWrite(t *testing.T) { + testHandlerTransportHandleStreams(t, func(st *handleStreamTest, s *ServerStream) { + if want := "/service/foo.bar"; s.method != want { + t.Errorf("stream method = %q; want %q", s.method, want) + } + st.bodyw.Close() // no body + + s.WriteStatus(status.New(codes.OK, "")) + s.Write([]byte("hdr"), newBufferSlice([]byte("data")), &WriteOptions{}) + }) +} + +func testHandlerTransportHandleStreams(t *testing.T, handleStream func(st *handleStreamTest, s *ServerStream)) { + st := newHandleStreamTest(t, nil) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + t.Cleanup(cancel) + st.ht.HandleStreams( + ctx, func(s *ServerStream) { go handleStream(st, s) }, + ) +} + +func (s) TestHandlerTransport_HandleStreams_ErrDetails(t *testing.T) { + errDetails := []protoadapt.MessageV1{ + &epb.RetryInfo{ + RetryDelay: &durationpb.Duration{Seconds: 60}, + }, + &epb.ResourceInfo{ + ResourceType: "foo bar", + ResourceName: "service.foo.bar", + Owner: "User", + }, + } + + statusCode := codes.ResourceExhausted + msg := "you are being throttled" + st, err := status.New(statusCode, msg).WithDetails(errDetails...) + if err != nil { + t.Fatal(err) + } + + stBytes, err := proto.Marshal(st.Proto()) + if err != nil { + t.Fatal(err) + } + + hst := newHandleStreamTest(t, nil) + handleStream := func(s *ServerStream) { + s.WriteStatus(st) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + hst.ht.HandleStreams( + ctx, func(s *ServerStream) { go handleStream(s) }, + ) + wantHeader := http.Header{ + "Date": nil, + "Content-Type": {"application/grpc"}, + "Trailer": {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"}, + } + wantTrailer := http.Header{ + "Grpc-Status": {fmt.Sprint(uint32(statusCode))}, + "Grpc-Message": {encodeGrpcMessage(msg)}, + "Grpc-Status-Details-Bin": {encodeBinHeader(stBytes)}, + } + + checkHeaderAndTrailer(t, hst.rw, wantHeader, wantTrailer) +} + +// Tests the use of stats handlers and ensures there are no data races while +// accessing trailers. +func (s) TestHandlerTransport_HandleStreams_StatsHandlers(t *testing.T) { + errDetails := []protoadapt.MessageV1{ + &epb.RetryInfo{ + RetryDelay: &durationpb.Duration{Seconds: 60}, + }, + &epb.ResourceInfo{ + ResourceType: "foo bar", + ResourceName: "service.foo.bar", + Owner: "User", + }, + } + + statusCode := codes.ResourceExhausted + msg := "you are being throttled" + st, err := status.New(statusCode, msg).WithDetails(errDetails...) + if err != nil { + t.Fatal(err) + } + + stBytes, err := proto.Marshal(st.Proto()) + if err != nil { + t.Fatal(err) + } + // Add mock stats handlers to exercise the stats handler code path. + statsHandler := &mockStatsHandler{ + rpcStatsCh: make(chan stats.RPCStats, 2), + } + hst := newHandleStreamTest(t, statsHandler) + handleStream := func(s *ServerStream) { + if err := s.SendHeader(metadata.New(map[string]string{})); err != nil { + t.Error(err) + } + if err := s.SetTrailer(metadata.Pairs("custom-trailer", "Custom trailer value")); err != nil { + t.Error(err) + } + s.WriteStatus(st) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + hst.ht.HandleStreams( + ctx, func(s *ServerStream) { go handleStream(s) }, + ) + wantHeader := http.Header{ + "Date": nil, + "Content-Type": {"application/grpc"}, + "Trailer": {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"}, + } + wantTrailer := http.Header{ + "Grpc-Status": {fmt.Sprint(uint32(statusCode))}, + "Grpc-Message": {encodeGrpcMessage(msg)}, + "Grpc-Status-Details-Bin": {encodeBinHeader(stBytes)}, + "Custom-Trailer": []string{"Custom trailer value"}, + } + + checkHeaderAndTrailer(t, hst.rw, wantHeader, wantTrailer) + wantStatTypes := []stats.RPCStats{&stats.OutHeader{}, &stats.OutTrailer{}} + for _, wantType := range wantStatTypes { + select { + case <-ctx.Done(): + t.Fatal("Context timed out waiting for statsHandler.HandleRPC() to be called.") + case s := <-statsHandler.rpcStatsCh: + if reflect.TypeOf(s) != reflect.TypeOf(wantType) { + t.Fatalf("Received RPCStats of type %T, want %T", s, wantType) + } + } + } +} + +// checkHeaderAndTrailer checks that the resulting header and trailer matches the expectation. +func checkHeaderAndTrailer(t *testing.T, rw testHandlerResponseWriter, wantHeader, wantTrailer http.Header) { + // For trailer-only responses, the trailer values might be reported as part of the Header. They will however + // be present in Trailer in either case. Hence, normalize the header by removing all trailer values. + actualHeader := rw.Result().Header.Clone() + for _, trailerKey := range actualHeader["Trailer"] { + actualHeader.Del(trailerKey) + } + + if !reflect.DeepEqual(actualHeader, wantHeader) { + t.Errorf("Header mismatch.\n got: %#v\n want: %#v", actualHeader, wantHeader) + } + if actualTrailer := rw.Result().Trailer; !reflect.DeepEqual(actualTrailer, wantTrailer) { + t.Errorf("Trailer mismatch.\n got: %#v\n want: %#v", actualTrailer, wantTrailer) + } +} diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go new file mode 100644 index 0000000..41a51a8 --- /dev/null +++ b/internal/transport/http2_client.go @@ -0,0 +1,1921 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "context" + "fmt" + "io" + "math" + "net" + "net/http" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/channelz" + icredentials "google.golang.org/grpc/internal/credentials" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpcutil" + imetadata "google.golang.org/grpc/internal/metadata" + "google.golang.org/grpc/internal/proxyattributes" + istats "google.golang.org/grpc/internal/stats" + istatus "google.golang.org/grpc/internal/status" + isyscall "google.golang.org/grpc/internal/syscall" + "google.golang.org/grpc/internal/transport/networktype" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +// clientConnectionCounter counts the number of connections a client has +// initiated (equal to the number of http2Clients created). Must be accessed +// atomically. +var clientConnectionCounter uint64 + +var goAwayLoopyWriterTimeout = 5 * time.Second + +var metadataFromOutgoingContextRaw = internal.FromOutgoingContextRaw.(func(context.Context) (metadata.MD, [][]string, bool)) + +// http2Client implements the ClientTransport interface with HTTP2. +type http2Client struct { + lastRead int64 // Keep this field 64-bit aligned. Accessed atomically. + ctx context.Context + cancel context.CancelFunc + ctxDone <-chan struct{} // Cache the ctx.Done() chan. + userAgent string + // address contains the resolver returned address for this transport. + // If the `ServerName` field is set, it takes precedence over `CallHdr.Host` + // passed to `NewStream`, when determining the :authority header. + address resolver.Address + md metadata.MD + conn net.Conn // underlying communication channel + loopy *loopyWriter + remoteAddr net.Addr + localAddr net.Addr + authInfo credentials.AuthInfo // auth info about the connection + + readerDone chan struct{} // sync point to enable testing. + writerDone chan struct{} // sync point to enable testing. + // goAway is closed to notify the upper layer (i.e., addrConn.transportMonitor) + // that the server sent GoAway on this transport. + goAway chan struct{} + keepaliveDone chan struct{} // Closed when the keepalive goroutine exits. + framer *framer + // controlBuf delivers all the control related tasks (e.g., window + // updates, reset streams, and various settings) to the controller. + // Do not access controlBuf with mu held. + controlBuf *controlBuffer + fc *trInFlow + // The scheme used: https if TLS is on, http otherwise. + scheme string + + isSecure bool + + perRPCCreds []credentials.PerRPCCredentials + + kp keepalive.ClientParameters + keepaliveEnabled bool + + statsHandler stats.Handler + + initialWindowSize int32 + + // configured by peer through SETTINGS_MAX_HEADER_LIST_SIZE + maxSendHeaderListSize *uint32 + + bdpEst *bdpEstimator + + maxConcurrentStreams uint32 + streamQuota int64 + streamsQuotaAvailable chan struct{} + waitingStreams uint32 + registeredCompressors string + + // Do not access controlBuf with mu held. + mu sync.Mutex // guard the following variables + nextID uint32 + state transportState + activeStreams map[uint32]*ClientStream + // prevGoAway ID records the Last-Stream-ID in the previous GOAway frame. + prevGoAwayID uint32 + // goAwayReason records the http2.ErrCode and debug data received with the + // GoAway frame. + goAwayReason GoAwayReason + // goAwayDebugMessage contains a detailed human readable string about a + // GoAway frame, useful for error messages. + goAwayDebugMessage string + // goAwayCode records the http2.ErrCode received with the GoAway frame. + goAwayCode http2.ErrCode + // A condition variable used to signal when the keepalive goroutine should + // go dormant. The condition for dormancy is based on the number of active + // streams and the `PermitWithoutStream` keepalive client parameter. And + // since the number of active streams is guarded by the above mutex, we use + // the same for this condition variable as well. + kpDormancyCond *sync.Cond + // A boolean to track whether the keepalive goroutine is dormant or not. + // This is checked before attempting to signal the above condition + // variable. + kpDormant bool + + channelz *channelz.Socket + + onClose OnCloseFunc + + bufferPool mem.BufferPool + + connectionID uint64 + logger *grpclog.PrefixLogger +} + +func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error), addr resolver.Address, grpcUA string) (net.Conn, error) { + address := addr.Addr + networkType, ok := networktype.Get(addr) + if fn != nil { + // Special handling for unix scheme with custom dialer. Back in the day, + // we did not have a unix resolver and therefore targets with a unix + // scheme would end up using the passthrough resolver. So, user's used a + // custom dialer in this case and expected the original dial target to + // be passed to the custom dialer. Now, we have a unix resolver. But if + // a custom dialer is specified, we want to retain the old behavior in + // terms of the address being passed to the custom dialer. + if networkType == "unix" && !strings.HasPrefix(address, "\x00") { + // Supported unix targets are either "unix://absolute-path" or + // "unix:relative-path". + if filepath.IsAbs(address) { + return fn(ctx, "unix://"+address) + } + return fn(ctx, "unix:"+address) + } + return fn(ctx, address) + } + if !ok { + networkType, address = ParseDialTarget(address) + } + if opts, present := proxyattributes.Get(addr); present { + return proxyDial(ctx, addr, grpcUA, opts) + } + return internal.NetDialerWithTCPKeepalive().DialContext(ctx, networkType, address) +} + +func isTemporary(err error) bool { + switch err := err.(type) { + case interface { + Temporary() bool + }: + return err.Temporary() + case interface { + Timeout() bool + }: + // Timeouts may be resolved upon retry, and are thus treated as + // temporary. + return err.Timeout() + } + return true +} + +// NewHTTP2Client constructs a connected ClientTransport to addr based on HTTP2 +// and starts to receive messages on it. Non-nil error returns if construction +// fails. +func NewHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts ConnectOptions, onClose OnCloseFunc) (_ ClientTransport, err error) { + scheme := "http" + ctx, cancel := context.WithCancel(ctx) + defer func() { + if err != nil { + cancel() + } + }() + + // gRPC, resolver, balancer etc. can specify arbitrary data in the + // Attributes field of resolver.Address, which is shoved into connectCtx + // and passed to the dialer and credential handshaker. This makes it possible for + // address specific arbitrary data to reach custom dialers and credential handshakers. + connectCtx = icredentials.NewClientHandshakeInfoContext(connectCtx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes}) + + conn, err := dial(connectCtx, opts.Dialer, addr, opts.UserAgent) + if err != nil { + if opts.FailOnNonTempDialError { + return nil, connectionErrorf(isTemporary(err), err, "transport: error while dialing: %v", err) + } + return nil, connectionErrorf(true, err, "transport: Error while dialing: %v", err) + } + + // Any further errors will close the underlying connection + defer func(conn net.Conn) { + if err != nil { + conn.Close() + } + }(conn) + + // The following defer and goroutine monitor the connectCtx for cancellation + // and deadline. On context expiration, the connection is hard closed and + // this function will naturally fail as a result. Otherwise, the defer + // waits for the goroutine to exit to prevent the context from being + // monitored (and to prevent the connection from ever being closed) after + // returning from this function. + ctxMonitorDone := grpcsync.NewEvent() + newClientCtx, newClientDone := context.WithCancel(connectCtx) + defer func() { + newClientDone() // Awaken the goroutine below if connectCtx hasn't expired. + <-ctxMonitorDone.Done() // Wait for the goroutine below to exit. + }() + go func(conn net.Conn) { + defer ctxMonitorDone.Fire() // Signal this goroutine has exited. + <-newClientCtx.Done() // Block until connectCtx expires or the defer above executes. + if err := connectCtx.Err(); err != nil { + // connectCtx expired before exiting the function. Hard close the connection. + if logger.V(logLevel) { + logger.Infof("Aborting due to connect deadline expiring: %v", err) + } + conn.Close() + } + }(conn) + + kp := opts.KeepaliveParams + // Validate keepalive parameters. + if kp.Time == 0 { + kp.Time = defaultClientKeepaliveTime + } + if kp.Timeout == 0 { + kp.Timeout = defaultClientKeepaliveTimeout + } + keepaliveEnabled := false + if kp.Time != infinity { + if err = isyscall.SetTCPUserTimeout(conn, kp.Timeout); err != nil { + return nil, connectionErrorf(false, err, "transport: failed to set TCP_USER_TIMEOUT: %v", err) + } + keepaliveEnabled = true + } + var ( + isSecure bool + authInfo credentials.AuthInfo + ) + transportCreds := opts.TransportCredentials + perRPCCreds := opts.PerRPCCredentials + + if b := opts.CredsBundle; b != nil { + if t := b.TransportCredentials(); t != nil { + transportCreds = t + } + if t := b.PerRPCCredentials(); t != nil { + perRPCCreds = append(perRPCCreds, t) + } + } + if transportCreds != nil { + conn, authInfo, err = transportCreds.ClientHandshake(connectCtx, addr.ServerName, conn) + if err != nil { + return nil, connectionErrorf(isTemporary(err), err, "transport: authentication handshake failed: %v", err) + } + for _, cd := range perRPCCreds { + if cd.RequireTransportSecurity() { + if ci, ok := authInfo.(interface { + GetCommonAuthInfo() credentials.CommonAuthInfo + }); ok { + secLevel := ci.GetCommonAuthInfo().SecurityLevel + if secLevel != credentials.InvalidSecurityLevel && secLevel < credentials.PrivacyAndIntegrity { + return nil, connectionErrorf(true, nil, "transport: cannot send secure credentials on an insecure connection") + } + } + } + } + isSecure = true + if transportCreds.Info().SecurityProtocol == "tls" { + scheme = "https" + } + } + icwz := int32(initialWindowSize) + if opts.InitialConnWindowSize >= defaultWindowSize { + icwz = opts.InitialConnWindowSize + } + writeBufSize := opts.WriteBufferSize + readBufSize := opts.ReadBufferSize + // The default header list size is moving from 16MB to 8KB. The 8KB limit + // is only used if Enable8KBDefaultHeaderListSize is true; otherwise, the + // old 16MB default is used. User-specified options always take precedence. + maxHeaderListSize := defaultClientMaxHeaderListSize + if envconfig.Enable8KBDefaultHeaderListSize { + maxHeaderListSize = upcomingDefaultHeaderListSize + } + if opts.MaxHeaderListSize != nil { + maxHeaderListSize = *opts.MaxHeaderListSize + } + + t := &http2Client{ + ctx: ctx, + ctxDone: ctx.Done(), // Cache Done chan. + cancel: cancel, + userAgent: opts.UserAgent, + registeredCompressors: grpcutil.RegisteredCompressors(), + address: addr, + conn: conn, + remoteAddr: conn.RemoteAddr(), + localAddr: conn.LocalAddr(), + authInfo: authInfo, + readerDone: make(chan struct{}), + writerDone: make(chan struct{}), + goAway: make(chan struct{}), + keepaliveDone: make(chan struct{}), + framer: newFramer(conn, writeBufSize, readBufSize, opts.SharedWriteBuffer, maxHeaderListSize, opts.BufferPool), + fc: &trInFlow{limit: uint32(icwz)}, + scheme: scheme, + activeStreams: make(map[uint32]*ClientStream), + isSecure: isSecure, + perRPCCreds: perRPCCreds, + kp: kp, + statsHandler: istats.NewCombinedHandler(opts.StatsHandlers...), + initialWindowSize: initialWindowSize, + nextID: 1, + maxConcurrentStreams: defaultMaxStreamsClient, + streamQuota: defaultMaxStreamsClient, + streamsQuotaAvailable: make(chan struct{}, 1), + keepaliveEnabled: keepaliveEnabled, + bufferPool: opts.BufferPool, + onClose: onClose, + } + var czSecurity credentials.ChannelzSecurityValue + if au, ok := authInfo.(credentials.ChannelzSecurityInfo); ok { + czSecurity = au.GetSecurityValue() + } + t.channelz = channelz.RegisterSocket( + &channelz.Socket{ + SocketType: channelz.SocketTypeNormal, + Parent: opts.ChannelzParent, + SocketMetrics: channelz.SocketMetrics{}, + EphemeralMetrics: t.socketMetrics, + LocalAddr: t.localAddr, + RemoteAddr: t.remoteAddr, + SocketOptions: channelz.GetSocketOption(t.conn), + Security: czSecurity, + }) + t.logger = prefixLoggerForClientTransport(t) + // Add peer information to the http2client context. + t.ctx = peer.NewContext(t.ctx, t.Peer()) + + if md, ok := addr.Metadata.(*metadata.MD); ok { + t.md = *md + } else if md := imetadata.Get(addr); md != nil { + t.md = md + } + t.controlBuf = newControlBuffer(t.ctxDone) + if opts.InitialWindowSize >= defaultWindowSize { + t.initialWindowSize = opts.InitialWindowSize + } + if !opts.StaticWindowSize { + t.bdpEst = &bdpEstimator{ + bdp: initialWindowSize, + updateFlowControl: t.updateFlowControl, + } + } + if t.statsHandler != nil { + t.ctx = t.statsHandler.TagConn(t.ctx, &stats.ConnTagInfo{ + RemoteAddr: t.remoteAddr, + LocalAddr: t.localAddr, + }) + t.statsHandler.HandleConn(t.ctx, &stats.ConnBegin{ + Client: true, + }) + } + if t.keepaliveEnabled { + t.kpDormancyCond = sync.NewCond(&t.mu) + go t.keepalive() + } + + // Start the reader goroutine for incoming messages. Each transport has a + // dedicated goroutine which reads HTTP2 frames from the network. Then it + // dispatches the frame to the corresponding stream entity. When the + // server preface is received, readerErrCh is closed. If an error occurs + // first, an error is pushed to the channel. This must be checked before + // returning from this function. + readerErrCh := make(chan error, 1) + go t.reader(readerErrCh) + defer func() { + if err != nil { + // writerDone should be closed since the loopy goroutine + // wouldn't have started in the case this function returns an error. + close(t.writerDone) + t.Close(err) + } + }() + + // Send connection preface to server. + n, err := t.conn.Write(clientPreface) + if err != nil { + err = connectionErrorf(true, err, "transport: failed to write client preface: %v", err) + return nil, err + } + if n != len(clientPreface) { + err = connectionErrorf(true, nil, "transport: preface mismatch, wrote %d bytes; want %d", n, len(clientPreface)) + return nil, err + } + var ss []http2.Setting + + if t.initialWindowSize != defaultWindowSize { + ss = append(ss, http2.Setting{ + ID: http2.SettingInitialWindowSize, + Val: uint32(t.initialWindowSize), + }) + } + if opts.MaxHeaderListSize != nil { + ss = append(ss, http2.Setting{ + ID: http2.SettingMaxHeaderListSize, + Val: *opts.MaxHeaderListSize, + }) + } + err = t.framer.fr.WriteSettings(ss...) + if err != nil { + err = connectionErrorf(true, err, "transport: failed to write initial settings frame: %v", err) + return nil, err + } + // Adjust the connection flow control window if needed. + if delta := uint32(icwz - defaultWindowSize); delta > 0 { + if err := t.framer.fr.WriteWindowUpdate(0, delta); err != nil { + err = connectionErrorf(true, err, "transport: failed to write window update: %v", err) + return nil, err + } + } + + t.connectionID = atomic.AddUint64(&clientConnectionCounter, 1) + + if err := t.framer.writer.Flush(); err != nil { + return nil, err + } + // Block until the server preface is received successfully or an error occurs. + if err = <-readerErrCh; err != nil { + return nil, err + } + go func() { + t.loopy = newLoopyWriter(clientSide, t.framer, t.controlBuf, t.bdpEst, t.conn, t.logger, t.outgoingGoAwayHandler, t.bufferPool) + if err := t.loopy.run(); !isIOError(err) { + // Immediately close the connection, as the loopy writer returns + // when there are no more active streams and we were draining (the + // server sent a GOAWAY). For I/O errors, the reader will hit it + // after draining any remaining incoming data. + t.conn.Close() + } + close(t.writerDone) + }() + return t, nil +} + +func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr, handler stats.Handler) *ClientStream { + // TODO(zhaoq): Handle uint32 overflow of Stream.id. + s := &ClientStream{ + Stream: Stream{ + method: callHdr.Method, + sendCompress: callHdr.SendCompress, + contentSubtype: callHdr.ContentSubtype, + }, + ct: t, + done: make(chan struct{}), + headerChan: make(chan struct{}), + statsHandler: handler, + } + s.Stream.buf.init(t.bufferPool) + s.Stream.wq.init(defaultWriteQuota, s.done) + s.readRequester = s + // The client side stream context should have exactly the same life cycle with the user provided context. + // That means, s.ctx should be read-only. And s.ctx is done iff ctx is done. + // So we use the original context here instead of creating a copy. + s.ctx = ctx + s.trReader = transportReader{ + reader: recvBufferReader{ + ctx: s.ctx, + ctxDone: s.ctx.Done(), + recv: &s.buf, + clientStream: s, + }, + windowHandler: s, + } + return s +} + +func (t *http2Client) Peer() *peer.Peer { + return &peer.Peer{ + Addr: t.remoteAddr, + AuthInfo: t.authInfo, // Can be nil + LocalAddr: t.localAddr, + } +} + +// OutgoingGoAwayHandler writes a GOAWAY to the connection. Always returns (false, err) as we want the GoAway +// to be the last frame loopy writes to the transport. +func (t *http2Client) outgoingGoAwayHandler(g *goAway) (bool, error) { + t.mu.Lock() + maxStreamID := t.nextID - 2 + t.mu.Unlock() + if err := t.framer.fr.WriteGoAway(maxStreamID, http2.ErrCodeNo, g.debugData); err != nil { + return false, err + } + return false, g.closeConn +} + +func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr) ([]hpack.HeaderField, error) { + aud := t.createAudience(callHdr) + ri := credentials.RequestInfo{ + Method: callHdr.Method, + AuthInfo: t.authInfo, + } + ctxWithRequestInfo := credentials.NewContextWithRequestInfo(ctx, ri) + authData, err := t.getTrAuthData(ctxWithRequestInfo, aud) + if err != nil { + return nil, err + } + callAuthData, err := t.getCallAuthData(ctxWithRequestInfo, aud, callHdr) + if err != nil { + return nil, err + } + // Make the slice of certain predictable size to reduce allocations made by append. + hfLen := 7 // :method, :scheme, :path, :authority, content-type, user-agent, te + hfLen += len(authData) + len(callAuthData) + registeredCompressors := t.registeredCompressors + if callHdr.AcceptedCompressors != nil { + registeredCompressors = *callHdr.AcceptedCompressors + } + if callHdr.PreviousAttempts > 0 { + hfLen++ + } + if callHdr.SendCompress != "" { + hfLen++ + } + if registeredCompressors != "" { + hfLen++ + } + if _, ok := ctx.Deadline(); ok { + hfLen++ + } + // Count the metadata header fields as well so the slice is not reallocated + // while they are appended below. Reserved headers are dropped when writing, + // so this may slightly over-count, which is preferable to growing the slice. + md, added, mdOK := metadataFromOutgoingContextRaw(ctx) + if mdOK { + for _, vv := range md { + hfLen += len(vv) + } + for _, vv := range added { + hfLen += len(vv) / 2 + } + } + for _, vv := range t.md { + hfLen += len(vv) + } + headerFields := make([]hpack.HeaderField, 0, hfLen) + headerFields = append(headerFields, hpack.HeaderField{Name: ":method", Value: "POST"}) + headerFields = append(headerFields, hpack.HeaderField{Name: ":scheme", Value: t.scheme}) + headerFields = append(headerFields, hpack.HeaderField{Name: ":path", Value: callHdr.Method}) + headerFields = append(headerFields, hpack.HeaderField{Name: ":authority", Value: callHdr.Host}) + headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: grpcutil.ContentType(callHdr.ContentSubtype)}) + headerFields = append(headerFields, hpack.HeaderField{Name: "user-agent", Value: t.userAgent}) + headerFields = append(headerFields, hpack.HeaderField{Name: "te", Value: "trailers"}) + if callHdr.PreviousAttempts > 0 { + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-previous-rpc-attempts", Value: strconv.Itoa(callHdr.PreviousAttempts)}) + } + + if callHdr.SendCompress != "" { + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-encoding", Value: callHdr.SendCompress}) + // Include the outgoing compressor name when compressor is not registered + // via encoding.RegisterCompressor. This is possible when client uses + // WithCompressor dial option. + if !grpcutil.IsCompressorNameRegistered(callHdr.SendCompress) { + if registeredCompressors != "" { + registeredCompressors += "," + } + registeredCompressors += callHdr.SendCompress + } + } + + if registeredCompressors != "" { + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-accept-encoding", Value: registeredCompressors}) + } + if dl, ok := ctx.Deadline(); ok { + // Send out timeout regardless its value. The server can detect timeout context by itself. + // TODO(mmukhi): Perhaps this field should be updated when actually writing out to the wire. + timeout := time.Until(dl) + if timeout <= 0 { + return nil, status.Error(codes.DeadlineExceeded, context.DeadlineExceeded.Error()) + } + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-timeout", Value: grpcutil.EncodeDuration(timeout)}) + } + for k, v := range authData { + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) + } + for k, v := range callAuthData { + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) + } + + if mdOK { + var k string + for k, vv := range md { + // HTTP doesn't allow you to set pseudoheaders after non pseudoheaders were set. + if isReservedHeader(k) { + continue + } + for _, v := range vv { + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) + } + } + for _, vv := range added { + for i, v := range vv { + if i%2 == 0 { + k = strings.ToLower(v) + continue + } + // HTTP doesn't allow you to set pseudoheaders after non pseudoheaders were set. + if isReservedHeader(k) { + continue + } + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) + } + } + } + for k, vv := range t.md { + if isReservedHeader(k) { + continue + } + if err := imetadata.ValidatePair(k, vv...); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + for _, v := range vv { + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) + } + } + return headerFields, nil +} + +func (t *http2Client) createAudience(callHdr *CallHdr) string { + // Create an audience string only if needed. + if len(t.perRPCCreds) == 0 && callHdr.Creds == nil { + return "" + } + // Construct URI required to get auth request metadata. + // Omit port if it is the default one. + host := strings.TrimSuffix(callHdr.Host, ":443") + pos := strings.LastIndex(callHdr.Method, "/") + if pos == -1 { + pos = len(callHdr.Method) + } + return "https://" + host + callHdr.Method[:pos] +} + +func (t *http2Client) getTrAuthData(ctx context.Context, audience string) (map[string]string, error) { + if len(t.perRPCCreds) == 0 { + return nil, nil + } + authData := map[string]string{} + for _, c := range t.perRPCCreds { + data, err := c.GetRequestMetadata(ctx, audience) + if err != nil { + if st, ok := status.FromError(err); ok { + // Restrict the code to the list allowed by gRFC A54. + if istatus.IsRestrictedControlPlaneCode(st) { + err = status.Errorf(codes.Internal, "transport: received per-RPC creds error with illegal status: %v", err) + } + return nil, err + } + + return nil, status.Errorf(codes.Unauthenticated, "transport: per-RPC creds failed due to error: %v", err) + } + for k, v := range data { + // Capital header names are illegal in HTTP/2. + k = strings.ToLower(k) + if err := imetadata.ValidatePair(k, v); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + authData[k] = v + } + } + return authData, nil +} + +func (t *http2Client) getCallAuthData(ctx context.Context, audience string, callHdr *CallHdr) (map[string]string, error) { + var callAuthData map[string]string + // Check if credentials.PerRPCCredentials were provided via call options. + // Note: if these credentials are provided both via dial options and call + // options, then both sets of credentials will be applied. + if callCreds := callHdr.Creds; callCreds != nil { + if callCreds.RequireTransportSecurity() { + ri, _ := credentials.RequestInfoFromContext(ctx) + if !t.isSecure || credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity) != nil { + return nil, status.Error(codes.Unauthenticated, "transport: cannot send secure credentials on an insecure connection") + } + } + data, err := callCreds.GetRequestMetadata(ctx, audience) + if err != nil { + if st, ok := status.FromError(err); ok { + // Restrict the code to the list allowed by gRFC A54. + if istatus.IsRestrictedControlPlaneCode(st) { + err = status.Errorf(codes.Internal, "transport: received per-RPC creds error with illegal status: %v", err) + } + return nil, err + } + return nil, status.Errorf(codes.Internal, "transport: per-RPC creds failed due to error: %v", err) + } + callAuthData = make(map[string]string, len(data)) + for k, v := range data { + // Capital header names are illegal in HTTP/2 + k = strings.ToLower(k) + if err := imetadata.ValidatePair(k, v); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + callAuthData[k] = v + } + } + return callAuthData, nil +} + +// NewStreamError wraps an error and reports additional information. Typically +// NewStream errors result in transparent retry, as they mean nothing went onto +// the wire. However, there are two notable exceptions: +// +// 1. If the stream headers violate the max header list size allowed by the +// server. It's possible this could succeed on another transport, even if +// it's unlikely, but do not transparently retry. +// 2. If the credentials errored when requesting their headers. In this case, +// it's possible a retry can fix the problem, but indefinitely transparently +// retrying is not appropriate as it is likely the credentials, if they can +// eventually succeed, would need I/O to do so. +type NewStreamError struct { + Err error + + AllowTransparentRetry bool +} + +func (e NewStreamError) Error() string { + return e.Err.Error() +} + +// NewStream creates a stream and registers it into the transport as "active" +// streams. All non-nil errors returned will be *NewStreamError. +func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr, handler stats.Handler) (*ClientStream, error) { + ctx = peer.NewContext(ctx, t.Peer()) + + // ServerName field of the resolver returned address takes precedence over + // Host field of CallHdr to determine the :authority header. This is because, + // the ServerName field takes precedence for server authentication during + // TLS handshake, and the :authority header should match the value used + // for server authentication. + if t.address.ServerName != "" { + newCallHdr := *callHdr + newCallHdr.Host = t.address.ServerName + callHdr = &newCallHdr + } + + // The authority specified via the `CallAuthority` CallOption takes the + // highest precedence when determining the `:authority` header. It overrides + // any value present in the Host field of CallHdr. Before applying this + // override, the authority string is validated. If the credentials do not + // implement the AuthorityValidator interface, or if validation fails, the + // RPC is failed with a status code of `UNAVAILABLE`. + if callHdr.Authority != "" { + auth, ok := t.authInfo.(credentials.AuthorityValidator) + if !ok { + return nil, &NewStreamError{Err: status.Errorf(codes.Unavailable, "credentials type %q does not implement the AuthorityValidator interface, but authority override specified with CallAuthority call option", t.authInfo.AuthType())} + } + if err := auth.ValidateAuthority(callHdr.Authority); err != nil { + return nil, &NewStreamError{Err: status.Errorf(codes.Unavailable, "failed to validate authority %q : %v", callHdr.Authority, err)} + } + newCallHdr := *callHdr + newCallHdr.Host = callHdr.Authority + callHdr = &newCallHdr + } + + headerFields, err := t.createHeaderFields(ctx, callHdr) + if err != nil { + return nil, &NewStreamError{Err: err, AllowTransparentRetry: false} + } + s := t.newStream(ctx, callHdr, handler) + cleanup := func(err error) { + if s.swapState(streamDone) == streamDone { + // If it was already done, return. + return + } + // The stream was unprocessed by the server. + s.unprocessed.Store(true) + s.write(recvMsg{err: err}) + close(s.done) + // If headerChan isn't closed, then close it. + if atomic.CompareAndSwapUint32(&s.headerChanClosed, 0, 1) { + close(s.headerChan) + } + } + hdr := &clientHeaders{ + hf: headerFields, + initStream: func(uint32) error { + t.mu.Lock() + // TODO: handle transport closure in loopy instead and remove this + // initStream is never called when transport is draining. + if t.state == closing { + t.mu.Unlock() + cleanup(ErrConnClosing) + return ErrConnClosing + } + if channelz.IsOn() { + t.channelz.SocketMetrics.StreamsStarted.Add(1) + t.channelz.SocketMetrics.LastLocalStreamCreatedTimestamp.Store(time.Now().UnixNano()) + } + // If the keepalive goroutine has gone dormant, wake it up. + if t.kpDormant { + t.kpDormancyCond.Signal() + } + t.mu.Unlock() + return nil + }, + onOrphaned: cleanup, + wq: &s.wq, + } + firstTry := true + var ch chan struct{} + transportDrainRequired := false + checkForStreamQuota := func() bool { + if t.streamQuota <= 0 { // Can go negative if server decreases it. + if firstTry { + t.waitingStreams++ + } + ch = t.streamsQuotaAvailable + return false + } + if !firstTry { + t.waitingStreams-- + } + t.streamQuota-- + + t.mu.Lock() + if t.state == draining || t.activeStreams == nil { // Can be niled from Close(). + t.mu.Unlock() + return false // Don't create a stream if the transport is already closed. + } + + hdr.streamID = t.nextID + t.nextID += 2 + // Drain client transport if nextID > MaxStreamID which signals gRPC that + // the connection is closed and a new one must be created for subsequent RPCs. + transportDrainRequired = t.nextID > MaxStreamID + + s.id = hdr.streamID + s.fc = inFlow{limit: uint32(t.initialWindowSize)} + t.activeStreams[s.id] = s + t.mu.Unlock() + + if t.streamQuota > 0 && t.waitingStreams > 0 { + select { + case t.streamsQuotaAvailable <- struct{}{}: + default: + } + } + return true + } + var hdrListSizeErr error + checkForHeaderListSize := func() bool { + if t.maxSendHeaderListSize == nil { + return true + } + var sz int64 + for _, f := range hdr.hf { + sz += int64(f.Size()) + if sz > int64(*t.maxSendHeaderListSize) { + hdrListSizeErr = status.Errorf(codes.Internal, "header list size to send violates the maximum size (%d bytes) set by server", *t.maxSendHeaderListSize) + return false + } + } + if !envconfig.Enable8KBDefaultHeaderListSize && sz > int64(upcomingDefaultHeaderListSize) { + t.logger.Warningf("Header list size to send (%d bytes) is larger than the upcoming default limit (%d bytes). In release v1.82.0, GRPC_GO_EXPERIMENTAL_ENABLE_8KB_DEFAULT_HEADER_LIST_SIZE will be enabled by default, enforcing this limit.", sz, upcomingDefaultHeaderListSize) + } + return true + } + for { + success, err := t.controlBuf.executeAndPut(func() bool { + return checkForHeaderListSize() && checkForStreamQuota() + }, hdr) + if err != nil { + // Connection closed. + return nil, &NewStreamError{Err: err, AllowTransparentRetry: true} + } + if success { + break + } + if hdrListSizeErr != nil { + return nil, &NewStreamError{Err: hdrListSizeErr} + } + firstTry = false + select { + case <-ch: + case <-ctx.Done(): + return nil, &NewStreamError{Err: ContextErr(ctx.Err())} + case <-t.goAway: + return nil, &NewStreamError{Err: errStreamDrain, AllowTransparentRetry: true} + case <-t.ctx.Done(): + return nil, &NewStreamError{Err: ErrConnClosing, AllowTransparentRetry: true} + } + } + if s.statsHandler != nil { + header, ok := metadata.FromOutgoingContext(ctx) + if ok { + header.Set("user-agent", t.userAgent) + } else { + header = metadata.Pairs("user-agent", t.userAgent) + } + // Note: The header fields are compressed with hpack after this call returns. + // No WireLength field is set here. + s.statsHandler.HandleRPC(s.ctx, &stats.OutHeader{ + Client: true, + FullMethod: callHdr.Method, + RemoteAddr: t.remoteAddr, + LocalAddr: t.localAddr, + Compression: callHdr.SendCompress, + Header: header, + Authority: callHdr.Host, + }) + } + if transportDrainRequired { + if t.logger.V(logLevel) { + t.logger.Infof("Draining transport: t.nextID > MaxStreamID") + } + t.GracefulClose() + } + return s, nil +} + +func (t *http2Client) closeStream(s *ClientStream, err error, rst bool, rstCode http2.ErrCode, st *status.Status, mdata map[string][]string, eosReceived bool) { + // Set stream status to done. + if s.swapState(streamDone) == streamDone { + // If it was already done, return. If multiple closeStream calls + // happen simultaneously, wait for the first to finish. + <-s.done + return + } + // status and trailers can be updated here without any synchronization because the stream goroutine will + // only read it after it sees an io.EOF error from read or write and we'll write those errors + // only after updating this. + s.status = st + if len(mdata) > 0 { + s.trailer = mdata + } + if err != nil { + // This will unblock reads eventually. + s.write(recvMsg{err: err}) + } + // If headerChan isn't closed, then close it. + if atomic.CompareAndSwapUint32(&s.headerChanClosed, 0, 1) { + s.noHeaders = true + close(s.headerChan) + } + cleanup := &cleanupStream{ + streamID: s.id, + onWrite: func() { + t.mu.Lock() + if t.activeStreams != nil { + delete(t.activeStreams, s.id) + } + t.mu.Unlock() + if channelz.IsOn() { + if eosReceived { + t.channelz.SocketMetrics.StreamsSucceeded.Add(1) + } else { + t.channelz.SocketMetrics.StreamsFailed.Add(1) + } + } + }, + rst: rst, + rstCode: rstCode, + } + addBackStreamQuota := func() bool { + t.streamQuota++ + if t.streamQuota > 0 && t.waitingStreams > 0 { + select { + case t.streamsQuotaAvailable <- struct{}{}: + default: + } + } + return true + } + t.controlBuf.executeAndPut(addBackStreamQuota, cleanup) + // This will unblock write. + close(s.done) +} + +// Close kicks off the shutdown process of the transport. This should be called +// only once on a transport. Once it is called, the transport should not be +// accessed anymore. +func (t *http2Client) Close(err error) { + t.conn.SetWriteDeadline(time.Now().Add(time.Second * 10)) + // For background on the deadline value chosen here, see + // https://github.com/grpc/grpc-go/issues/8425#issuecomment-3057938248 . + t.conn.SetReadDeadline(time.Now().Add(time.Second)) + t.mu.Lock() + // Make sure we only close once. + if t.state == closing { + t.mu.Unlock() + return + } + if t.logger.V(logLevel) { + t.logger.Infof("Closing: %v", err) + } + // Call t.onClose ASAP to prevent the client from attempting to create new + // streams. + if t.state != draining { + t.onClose(GoAwayInfo{Reason: GoAwayInvalid, GoAwayCode: http2.ErrCodeNo, Err: err}) + } + t.state = closing + streams := t.activeStreams + t.activeStreams = nil + if t.kpDormant { + // If the keepalive goroutine is blocked on this condition variable, we + // should unblock it so that the goroutine eventually exits. + t.kpDormancyCond.Signal() + } + // Append info about previous goaways if there were any, since this may be important + // for understanding the root cause for this connection to be closed. + goAwayDebugMessage := t.goAwayDebugMessage + t.mu.Unlock() + + // Per HTTP/2 spec, a GOAWAY frame must be sent before closing the + // connection. See https://httpwg.org/specs/rfc7540.html#GOAWAY. It + // also waits for loopyWriter to be closed with a timer to avoid the + // long blocking in case the connection is blackholed, i.e. TCP is + // just stuck. + t.controlBuf.put(&goAway{code: http2.ErrCodeNo, debugData: []byte("client transport shutdown"), closeConn: err}) + timer := time.NewTimer(goAwayLoopyWriterTimeout) + defer timer.Stop() + select { + case <-t.writerDone: // success + case <-timer.C: + t.logger.Infof("Failed to write a GOAWAY frame as part of connection close after %s. Giving up and closing the transport.", goAwayLoopyWriterTimeout) + } + t.cancel() + t.conn.Close() + // Waits for the reader and keepalive goroutines to exit before returning to + // ensure all resources are cleaned up before Close can return. + <-t.readerDone + if t.keepaliveEnabled { + <-t.keepaliveDone + } + channelz.RemoveEntry(t.channelz.ID) + var st *status.Status + if len(goAwayDebugMessage) > 0 { + st = status.Newf(codes.Unavailable, "closing transport due to: %v, received prior goaway: %v", err, goAwayDebugMessage) + err = st.Err() + } else { + st = status.New(codes.Unavailable, err.Error()) + } + + // Notify all active streams. + for _, s := range streams { + t.closeStream(s, err, false, http2.ErrCodeNo, st, nil, false) + } + if t.statsHandler != nil { + t.statsHandler.HandleConn(t.ctx, &stats.ConnEnd{ + Client: true, + }) + } +} + +// GracefulClose sets the state to draining, which prevents new streams from +// being created and causes the transport to be closed when the last active +// stream is closed. If there are no active streams, the transport is closed +// immediately. This does nothing if the transport is already draining or +// closing. +func (t *http2Client) GracefulClose() { + t.mu.Lock() + // Make sure we move to draining only from active. + if t.state == draining || t.state == closing { + t.mu.Unlock() + return + } + if t.logger.V(logLevel) { + t.logger.Infof("GracefulClose called") + } + t.onClose(GoAwayInfo{Reason: GoAwayInvalid, GoAwayCode: http2.ErrCodeNo}) + t.state = draining + active := len(t.activeStreams) + t.mu.Unlock() + if active == 0 { + t.Close(connectionErrorf(true, nil, "no active streams left to process while draining")) + return + } + t.controlBuf.put(&incomingGoAway{}) +} + +// Write formats the data into HTTP2 data frame(s) and sends it out. The caller +// should proceed only if Write returns nil. +func (t *http2Client) write(s *ClientStream, hdr []byte, data mem.BufferSlice, opts *WriteOptions) error { + if opts.Last { + // If it's the last message, update stream state. + if !s.compareAndSwapState(streamActive, streamWriteDone) { + return errStreamDone + } + } else if s.getState() != streamActive { + return errStreamDone + } + df := &dataFrame{ + streamID: s.id, + endStream: opts.Last, + h: hdr, + data: data, + } + dataLen := data.Len() + if hdr != nil || dataLen != 0 { // If it's not an empty data frame, check quota. + if err := s.wq.get(int32(len(hdr) + dataLen)); err != nil { + return err + } + } + data.Ref() + if err := t.controlBuf.put(df); err != nil { + data.Free() + return err + } + t.incrMsgSent() + return nil +} + +func (t *http2Client) getStream(f http2.Frame) *ClientStream { + t.mu.Lock() + s := t.activeStreams[f.Header().StreamID] + t.mu.Unlock() + return s +} + +// adjustWindow sends out extra window update over the initial window size +// of stream if the application is requesting data larger in size than +// the window. +func (t *http2Client) adjustWindow(s *ClientStream, n uint32) { + if w := s.fc.maybeAdjust(n); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{streamID: s.id, increment: w}) + } +} + +// updateWindow adjusts the inbound quota for the stream. +// Window updates will be sent out when the cumulative quota +// exceeds the corresponding threshold. +func (t *http2Client) updateWindow(s *ClientStream, n uint32) { + if w := s.fc.onRead(n); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{streamID: s.id, increment: w}) + } +} + +// updateFlowControl updates the incoming flow control windows +// for the transport and the stream based on the current bdp +// estimation. +func (t *http2Client) updateFlowControl(n uint32) { + updateIWS := func() bool { + t.initialWindowSize = int32(n) + t.mu.Lock() + for _, s := range t.activeStreams { + s.fc.newLimit(n) + } + t.mu.Unlock() + return true + } + t.controlBuf.executeAndPut(updateIWS, &outgoingWindowUpdate{streamID: 0, increment: t.fc.newLimit(n)}) + t.controlBuf.put(&outgoingSettings{ + ss: []http2.Setting{ + { + ID: http2.SettingInitialWindowSize, + Val: n, + }, + }, + }) +} + +func (t *http2Client) handleData(f *parsedDataFrame) { + size := f.Header().Length + var sendBDPPing bool + if t.bdpEst != nil { + sendBDPPing = t.bdpEst.add(size) + } + // Decouple connection's flow control from application's read. + // An update on connection's flow control should not depend on + // whether user application has read the data or not. Such a + // restriction is already imposed on the stream's flow control, + // and therefore the sender will be blocked anyways. + // Decoupling the connection flow control will prevent other + // active(fast) streams from starving in presence of slow or + // inactive streams. + // + if w := t.fc.onData(size); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: w, + }) + } + if sendBDPPing { + // Avoid excessive ping detection (e.g. in an L7 proxy) + // by sending a window update prior to the BDP ping. + + if w := t.fc.reset(); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: w, + }) + } + + t.controlBuf.put(bdpPing) + } + // Select the right stream to dispatch. + s := t.getStream(f) + if s == nil { + return + } + if size > 0 { + if err := s.fc.onData(size); err != nil { + t.closeStream(s, io.EOF, true, http2.ErrCodeFlowControl, status.New(codes.Internal, err.Error()), nil, false) + return + } + } + + if s.nonGRPCStatus != nil { + // The frame should be handled as a non-gRPC response body. A non-nil + // status also covers END_STREAM, so no separate handling is needed below. + st := s.handleNonGRPCData(f) + if st != nil { + t.closeStream(s, st.Err(), true, http2.ErrCodeProtocol, st, nil, true) + return + } + if w := s.fc.onRead(size); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: s.id, + increment: w, + }) + } + return + } + + if size > 0 { + dataLen := f.data.Len() + if f.Header().Flags.Has(http2.FlagDataPadded) { + if w := s.fc.onRead(size - uint32(dataLen)); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: s.id, + increment: w, + }) + } + } + if dataLen > 0 { + f.data.Ref() + s.write(recvMsg{buffer: f.data}) + } + } + // The server has closed the stream without sending trailers. Record that + // the read direction is closed, and set the status appropriately. + if f.StreamEnded() { + // If client received END_STREAM from server while stream was still + // active, send RST_STREAM. + rstStream := s.getState() == streamActive + t.closeStream(s, io.EOF, rstStream, http2.ErrCodeNo, status.New(codes.Internal, "server closed the stream without sending trailers"), nil, true) + } +} + +func (t *http2Client) handleRSTStream(f *http2.RSTStreamFrame) { + s := t.getStream(f) + if s == nil { + return + } + if f.ErrCode == http2.ErrCodeRefusedStream { + // The stream was unprocessed by the server. + s.unprocessed.Store(true) + } + statusCode, ok := http2ErrConvTab[f.ErrCode] + if !ok { + if t.logger.V(logLevel) { + t.logger.Infof("Received a RST_STREAM frame with code %q, but found no mapped gRPC status", f.ErrCode) + } + statusCode = codes.Unknown + } + if statusCode == codes.Canceled { + if d, ok := s.ctx.Deadline(); ok && !d.After(time.Now()) { + // Our deadline was already exceeded, and that was likely the cause + // of this cancellation. Alter the status code accordingly. + statusCode = codes.DeadlineExceeded + } + } + st := status.Newf(statusCode, "stream terminated by RST_STREAM with error code: %v", f.ErrCode) + t.closeStream(s, st.Err(), false, http2.ErrCodeNo, st, nil, false) +} + +func (t *http2Client) handleSettings(f *http2.SettingsFrame, isFirst bool) { + if f.IsAck() { + return + } + var maxStreams *uint32 + var ss []http2.Setting + var updateFuncs []func() + f.ForeachSetting(func(s http2.Setting) error { + switch s.ID { + case http2.SettingMaxConcurrentStreams: + maxStreams = new(uint32) + *maxStreams = s.Val + case http2.SettingMaxHeaderListSize: + updateFuncs = append(updateFuncs, func() { + t.maxSendHeaderListSize = new(uint32) + *t.maxSendHeaderListSize = s.Val + }) + default: + ss = append(ss, s) + } + return nil + }) + if isFirst && maxStreams == nil { + maxStreams = new(uint32) + *maxStreams = math.MaxUint32 + } + sf := &incomingSettings{ + ss: ss, + } + if maxStreams != nil { + updateStreamQuota := func() { + delta := int64(*maxStreams) - int64(t.maxConcurrentStreams) + t.maxConcurrentStreams = *maxStreams + t.streamQuota += delta + if delta > 0 && t.waitingStreams > 0 { + close(t.streamsQuotaAvailable) // wake all of them up. + t.streamsQuotaAvailable = make(chan struct{}, 1) + } + } + updateFuncs = append(updateFuncs, updateStreamQuota) + } + t.controlBuf.executeAndPut(func() bool { + for _, f := range updateFuncs { + f() + } + return true + }, sf) +} + +func (t *http2Client) handlePing(f *http2.PingFrame) { + if f.IsAck() { + // Maybe it's a BDP ping. + if t.bdpEst != nil { + t.bdpEst.calculate(f.Data) + } + return + } + pingAck := &ping{ack: true} + copy(pingAck.data[:], f.Data[:]) + t.controlBuf.put(pingAck) +} + +func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) error { + t.mu.Lock() + if t.state == closing { + t.mu.Unlock() + return nil + } + if f.ErrCode == http2.ErrCodeEnhanceYourCalm && string(f.DebugData()) == "too_many_pings" { + // When a client receives a GOAWAY with error code ENHANCE_YOUR_CALM and debug + // data equal to ASCII "too_many_pings", it should log the occurrence at a log level that is + // enabled by default and double the configure KEEPALIVE_TIME used for new connections + // on that channel. + logger.Errorf("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\".") + } + id := f.LastStreamID + if id > 0 && id%2 == 0 { + t.mu.Unlock() + return connectionErrorf(true, nil, "received goaway with non-zero even-numbered stream id: %v", id) + } + // A client can receive multiple GoAways from the server (see + // https://github.com/grpc/grpc-go/issues/1387). The idea is that the first + // GoAway will be sent with an ID of MaxInt32 and the second GoAway will be + // sent after an RTT delay with the ID of the last stream the server will + // process. + // + // Therefore, when we get the first GoAway we don't necessarily close any + // streams. While in case of second GoAway we close all streams created after + // the GoAwayId. This way streams that were in-flight while the GoAway from + // server was being sent don't get killed. + select { + case <-t.goAway: // t.goAway has been closed (i.e.,multiple GoAways). + // If there are multiple GoAways the first one should always have an ID greater than the following ones. + if id > t.prevGoAwayID { + t.mu.Unlock() + return connectionErrorf(true, nil, "received goaway with stream id: %v, which exceeds stream id of previous goaway: %v", id, t.prevGoAwayID) + } + default: + t.setGoAwayReason(f) + close(t.goAway) + defer t.controlBuf.put(&incomingGoAway{}) // Defer as t.mu is currently held. + // Notify the clientconn about the GOAWAY before we set the state to + // draining, to allow the client to stop attempting to create streams + // before disallowing new streams on this connection. + if t.state != draining { + t.onClose(GoAwayInfo{Reason: t.goAwayReason, GoAwayCode: t.goAwayCode}) + t.state = draining + } + } + // All streams with IDs greater than the GoAwayId + // and smaller than the previous GoAway ID should be killed. + upperLimit := t.prevGoAwayID + if upperLimit == 0 { // This is the first GoAway Frame. + upperLimit = math.MaxUint32 // Kill all streams after the GoAway ID. + } + + t.prevGoAwayID = id + if len(t.activeStreams) == 0 { + t.mu.Unlock() + return connectionErrorf(true, nil, "received goaway and there are no active streams") + } + + streamsToClose := make([]*ClientStream, 0) + for streamID, stream := range t.activeStreams { + if streamID > id && streamID <= upperLimit { + // The stream was unprocessed by the server. + stream.unprocessed.Store(true) + streamsToClose = append(streamsToClose, stream) + } + } + t.mu.Unlock() + // Called outside t.mu because closeStream can take controlBuf's mu, which + // could induce deadlock and is not allowed. + for _, stream := range streamsToClose { + t.closeStream(stream, errStreamDrain, false, http2.ErrCodeNo, statusGoAway, nil, false) + } + return nil +} + +// setGoAwayReason sets the value of t.goAwayReason based +// on the GoAway frame received. +// It expects a lock on transport's mutex to be held by +// the caller. +func (t *http2Client) setGoAwayReason(f *http2.GoAwayFrame) { + t.goAwayReason = GoAwayNoReason + if f.ErrCode == http2.ErrCodeEnhanceYourCalm { + if string(f.DebugData()) == "too_many_pings" { + t.goAwayReason = GoAwayTooManyPings + } + } + if len(f.DebugData()) == 0 { + t.goAwayDebugMessage = fmt.Sprintf("code: %s", f.ErrCode) + } else { + t.goAwayDebugMessage = fmt.Sprintf("code: %s, debug data: %q", f.ErrCode, string(f.DebugData())) + } + t.goAwayCode = f.ErrCode +} + +func (t *http2Client) GetGoAwayReason() (GoAwayReason, string) { + t.mu.Lock() + defer t.mu.Unlock() + return t.goAwayReason, t.goAwayDebugMessage +} + +func (t *http2Client) handleWindowUpdate(f *http2.WindowUpdateFrame) { + t.controlBuf.put(&incomingWindowUpdate{ + streamID: f.Header().StreamID, + increment: f.Increment, + }) +} + +// operateHeaders takes action on the decoded headers. +func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) { + s := t.getStream(frame) + if s == nil { + return + } + endStream := frame.StreamEnded() + s.bytesReceived.Store(true) + initialHeader := atomic.LoadUint32(&s.headerChanClosed) == 0 + + if !initialHeader && !endStream { + // As specified by gRPC over HTTP2, a HEADERS frame (and associated CONTINUATION frames) can only appear at the start or end of a stream. Therefore, second HEADERS frame must have EOS bit set. + st := status.New(codes.Internal, "a HEADERS frame cannot appear in the middle of a stream") + t.closeStream(s, st.Err(), true, http2.ErrCodeProtocol, st, nil, false) + return + } + + // frame.Truncated is set to true when framer detects that the current header + // list size hits MaxHeaderListSize limit. + if frame.Truncated { + se := status.New(codes.Internal, "peer header list size exceeded limit") + t.closeStream(s, se.Err(), true, http2.ErrCodeFrameSize, se, nil, endStream) + return + } + + // If we are collecting non-gRPC response data and receive a trailing + // HEADERS frame with END_STREAM, finalize the buffered data and close + // the stream. + if s.nonGRPCStatus != nil { + if endStream { + st := s.finalizeNonGRPCStatus() + t.closeStream(s, st.Err(), true, http2.ErrCodeProtocol, st, nil, true) + } + return + } + + var ( + // If a gRPC Response-Headers has already been received, then it means + // that the peer is speaking gRPC and we are in gRPC mode. + isGRPC = !initialHeader + mdata = make(map[string][]string) + contentTypeErr = "malformed header: missing HTTP content-type" + grpcMessage string + recvCompress string + httpStatusErr string + // the code from the grpc-status header, if present + grpcStatusCode = codes.Unknown + // headerError is set if an error is encountered while parsing the headers + headerError string + httpStatus string + ) + + for _, hf := range frame.Fields { + switch hf.Name { + case "content-type": + if _, validContentType := grpcutil.ContentSubtype(hf.Value); !validContentType { + contentTypeErr = fmt.Sprintf("transport: received unexpected content-type %q", hf.Value) + break + } + contentTypeErr = "" + mdata[hf.Name] = append(mdata[hf.Name], hf.Value) + isGRPC = true + case "grpc-encoding": + recvCompress = hf.Value + case "grpc-status": + code, err := strconv.ParseInt(hf.Value, 10, 32) + if err != nil { + se := status.New(codes.Unknown, fmt.Sprintf("transport: malformed grpc-status: %v", err)) + t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, endStream) + return + } + grpcStatusCode = codes.Code(uint32(code)) + case "grpc-message": + grpcMessage = decodeGrpcMessage(hf.Value) + case ":status": + httpStatus = hf.Value + default: + if isReservedHeader(hf.Name) && !isWhitelistedHeader(hf.Name) { + break + } + v, err := decodeMetadataHeader(hf.Name, hf.Value) + if err != nil { + headerError = fmt.Sprintf("transport: malformed %s: %v", hf.Name, err) + logger.Warningf("Failed to decode metadata header (%q, %q): %v", hf.Name, hf.Value, err) + break + } + mdata[hf.Name] = append(mdata[hf.Name], v) + } + } + + // If a non-gRPC response is received, then evaluate the HTTP status to + // process the response and close the stream. + // In case http status doesn't provide any error information (status : 200), + // then evalute response code to be Unknown. + if !isGRPC { + var grpcErrorCode = codes.Internal + if httpStatus == "" { + httpStatusErr = "malformed header: missing HTTP status" + } else { + // Parse the status codes (e.g. "200", 404"). + statusCode, err := strconv.Atoi(httpStatus) + if err != nil { + se := status.New(grpcErrorCode, fmt.Sprintf("transport: malformed http-status: %v", err)) + t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, endStream) + return + } + if statusCode >= 100 && statusCode < 200 { + if endStream { + se := status.New(codes.Internal, fmt.Sprintf( + "protocol error: informational header with status code %d must not have END_STREAM set", statusCode)) + t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, endStream) + } + // In case of informational headers, return. + return + } + httpStatusErr = fmt.Sprintf( + "unexpected HTTP status code received from server: %d (%s)", + statusCode, + http.StatusText(statusCode), + ) + var ok bool + grpcErrorCode, ok = HTTPStatusConvTab[statusCode] + if !ok { + grpcErrorCode = codes.Unknown + } + } + var errs []string + if httpStatusErr != "" { + errs = append(errs, httpStatusErr) + } + + if contentTypeErr != "" { + errs = append(errs, contentTypeErr) + } + + se := status.New(grpcErrorCode, strings.Join(errs, "; ")) + if endStream { + t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, true) + return + } + + s.startNonGRPCDataCollection(se) + return + } + + if headerError != "" { + se := status.New(codes.Internal, headerError) + t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, endStream) + return + } + + // For headers, set them in s.header and close headerChan. For trailers or + // trailers-only, closeStream will set the trailers and close headerChan as + // needed. + if !endStream { + // If headerChan hasn't been closed yet (expected, given we checked it + // above, but something else could have potentially closed the whole + // stream). + if atomic.CompareAndSwapUint32(&s.headerChanClosed, 0, 1) { + s.headerValid = true + // These values can be set without any synchronization because + // stream goroutine will read it only after seeing a closed + // headerChan which we'll close after setting this. + s.recvCompress = recvCompress + if len(mdata) > 0 { + s.header = mdata + } + close(s.headerChan) + } + } + + if s.statsHandler != nil { + if !endStream { + s.statsHandler.HandleRPC(s.ctx, &stats.InHeader{ + Client: true, + WireLength: int(frame.Header().Length), + Header: metadata.MD(mdata).Copy(), + Compression: s.recvCompress, + }) + } else { + s.statsHandler.HandleRPC(s.ctx, &stats.InTrailer{ + Client: true, + WireLength: int(frame.Header().Length), + Trailer: metadata.MD(mdata).Copy(), + }) + } + } + + if !endStream { + return + } + + status := istatus.NewWithProto(grpcStatusCode, grpcMessage, mdata[grpcStatusDetailsBinHeader]) + + // If client received END_STREAM from server while stream was still active, + // send RST_STREAM. + rstStream := s.getState() == streamActive + t.closeStream(s, io.EOF, rstStream, http2.ErrCodeNo, status, mdata, true) +} + +// readServerPreface reads and handles the initial settings frame from the +// server. +func (t *http2Client) readServerPreface() error { + frame, err := t.framer.fr.ReadFrame() + if err != nil { + return connectionErrorf(true, err, "error reading server preface: %v", err) + } + sf, ok := frame.(*http2.SettingsFrame) + if !ok { + return connectionErrorf(true, nil, "initial http2 frame from server is not a settings frame: %T", frame) + } + t.handleSettings(sf, true) + return nil +} + +// reader verifies the server preface and reads all subsequent data from +// network connection. If the server preface is not read successfully, an +// error is pushed to errCh; otherwise errCh is closed with no error. +func (t *http2Client) reader(errCh chan<- error) { + var errClose error + defer func() { + close(t.readerDone) + if errClose != nil { + t.Close(errClose) + } + }() + + if err := t.readServerPreface(); err != nil { + errCh <- err + return + } + close(errCh) + if t.keepaliveEnabled { + atomic.StoreInt64(&t.lastRead, time.Now().UnixNano()) + } + + // loop to keep reading incoming messages on this transport. + for { + t.controlBuf.throttle() + frame, err := t.framer.readFrame() + if t.keepaliveEnabled { + atomic.StoreInt64(&t.lastRead, time.Now().UnixNano()) + } + if err != nil { + // Abort an active stream if the http2.Framer returns a + // http2.StreamError. This can happen only if the server's response + // is malformed http2. + if se, ok := err.(http2.StreamError); ok { + t.mu.Lock() + s := t.activeStreams[se.StreamID] + t.mu.Unlock() + if s != nil { + // use error detail to provide better err message + code := http2ErrConvTab[se.Code] + errorDetail := t.framer.errorDetail() + var msg string + if errorDetail != nil { + msg = errorDetail.Error() + } else { + msg = "received invalid frame" + } + t.closeStream(s, status.Error(code, msg), true, http2.ErrCodeProtocol, status.New(code, msg), nil, false) + } + continue + } + // Transport error. + errClose = connectionErrorf(true, err, "error reading from server: %v", err) + return + } + switch frame := frame.(type) { + case *http2.MetaHeadersFrame: + t.operateHeaders(frame) + case *parsedDataFrame: + t.handleData(frame) + frame.data.Free() + case *http2.RSTStreamFrame: + t.handleRSTStream(frame) + case *http2.SettingsFrame: + t.handleSettings(frame, false) + case *http2.PingFrame: + t.handlePing(frame) + case *http2.GoAwayFrame: + errClose = t.handleGoAway(frame) + case *http2.WindowUpdateFrame: + t.handleWindowUpdate(frame) + default: + if logger.V(logLevel) { + logger.Errorf("transport: http2Client.reader got unhandled frame type %v.", frame) + } + } + } +} + +// keepalive running in a separate goroutine makes sure the connection is alive by sending pings. +func (t *http2Client) keepalive() { + var err error + defer func() { + close(t.keepaliveDone) + if err != nil { + t.Close(err) + } + }() + p := &ping{data: [8]byte{}} + // True iff a ping has been sent, and no data has been received since then. + outstandingPing := false + // Amount of time remaining before which we should receive an ACK for the + // last sent ping. + timeoutLeft := time.Duration(0) + // Records the last value of t.lastRead before we go block on the timer. + // This is required to check for read activity since then. + prevNano := time.Now().UnixNano() + timer := time.NewTimer(t.kp.Time) + for { + select { + case <-timer.C: + lastRead := atomic.LoadInt64(&t.lastRead) + if lastRead > prevNano { + // There has been read activity since the last time we were here. + outstandingPing = false + // Next timer should fire at kp.Time seconds from lastRead time. + timer.Reset(time.Duration(lastRead) + t.kp.Time - time.Duration(time.Now().UnixNano())) + prevNano = lastRead + continue + } + if outstandingPing && timeoutLeft <= 0 { + err = connectionErrorf(true, nil, "keepalive ping failed to receive ACK within timeout") + return + } + t.mu.Lock() + if t.state == closing { + // If the transport is closing, we should exit from the + // keepalive goroutine here. If not, we could have a race + // between the call to Signal() from Close() and the call to + // Wait() here, whereby the keepalive goroutine ends up + // blocking on the condition variable which will never be + // signalled again. + t.mu.Unlock() + return + } + if len(t.activeStreams) < 1 && !t.kp.PermitWithoutStream { + // If a ping was sent out previously (because there were active + // streams at that point) which wasn't acked and its timeout + // hadn't fired, but we got here and are about to go dormant, + // we should make sure that we unconditionally send a ping once + // we awaken. + outstandingPing = false + t.kpDormant = true + t.kpDormancyCond.Wait() + } + t.kpDormant = false + t.mu.Unlock() + + // We get here either because we were dormant and a new stream was + // created which unblocked the Wait() call, or because the + // keepalive timer expired. In both cases, we need to send a ping. + if !outstandingPing { + if channelz.IsOn() { + t.channelz.SocketMetrics.KeepAlivesSent.Add(1) + } + t.controlBuf.put(p) + timeoutLeft = t.kp.Timeout + outstandingPing = true + } + // The amount of time to sleep here is the minimum of kp.Time and + // timeoutLeft. This will ensure that we wait only for kp.Time + // before sending out the next ping (for cases where the ping is + // acked). + sleepDuration := min(t.kp.Time, timeoutLeft) + timeoutLeft -= sleepDuration + timer.Reset(sleepDuration) + case <-t.ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return + } + } +} + +func (t *http2Client) Error() <-chan struct{} { + return t.ctx.Done() +} + +func (t *http2Client) GoAway() <-chan struct{} { + return t.goAway +} + +func (t *http2Client) socketMetrics() *channelz.EphemeralSocketMetrics { + return &channelz.EphemeralSocketMetrics{ + LocalFlowControlWindow: int64(t.fc.getSize()), + RemoteFlowControlWindow: t.getOutFlowWindow(), + } +} + +func (t *http2Client) incrMsgSent() { + if channelz.IsOn() { + t.channelz.SocketMetrics.MessagesSent.Add(1) + t.channelz.SocketMetrics.LastMessageSentTimestamp.Store(time.Now().UnixNano()) + } +} + +func (t *http2Client) incrMsgRecv() { + if channelz.IsOn() { + t.channelz.SocketMetrics.MessagesReceived.Add(1) + t.channelz.SocketMetrics.LastMessageReceivedTimestamp.Store(time.Now().UnixNano()) + } +} + +func (t *http2Client) getOutFlowWindow() int64 { + resp := make(chan uint32, 1) + timer := time.NewTimer(time.Second) + defer timer.Stop() + t.controlBuf.put(&outFlowControlSizeRequest{resp: resp}) + select { + case sz := <-resp: + return int64(sz) + case <-t.ctxDone: + return -1 + case <-timer.C: + return -2 + } +} + +func (t *http2Client) stateForTesting() transportState { + t.mu.Lock() + defer t.mu.Unlock() + return t.state +} diff --git a/internal/transport/http2_client_header_bench_test.go b/internal/transport/http2_client_header_bench_test.go new file mode 100644 index 0000000..2d4d0c9 --- /dev/null +++ b/internal/transport/http2_client_header_bench_test.go @@ -0,0 +1,96 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "context" + "fmt" + "testing" + "time" + + "google.golang.org/grpc/metadata" +) + +// BenchmarkCreateHeaderFields measures header-slice construction as the amount +// of outgoing metadata varies. +func BenchmarkCreateHeaderFields(b *testing.B) { + for _, mdCount := range []int{0, 4, 12} { + b.Run(fmt.Sprintf("mdCount=%d", mdCount), func(b *testing.B) { + t := &http2Client{ + scheme: "https", + userAgent: "grpc-go/benchmark", + md: metadata.MD{}, + } + ctx, cancel := context.WithTimeout(context.Background(), time.Hour) + defer cancel() + if mdCount > 0 { + md := make(metadata.MD, mdCount) + for i := 0; i < mdCount; i++ { + md[fmt.Sprintf("header-%d", i)] = []string{fmt.Sprintf("value-%d", i)} + } + ctx = metadata.NewOutgoingContext(ctx, md) + } + callHdr := &CallHdr{ + Method: "/grpc.testing.BenchmarkService/UnaryCall", + Host: "server:443", + } + + b.ReportAllocs() + for b.Loop() { + hf, err := t.createHeaderFields(ctx, callHdr) + if err != nil { + b.Fatal(err) + } + if len(hf) == 0 { + b.Fatal("no header fields produced") + } + } + }) + + b.Run(fmt.Sprintf("mdCount=%d-appended", mdCount), func(b *testing.B) { + t := &http2Client{ + scheme: "https", + userAgent: "grpc-go/benchmark", + md: metadata.MD{}, + } + ctx, cancel := context.WithTimeout(context.Background(), time.Hour) + defer cancel() + kvs := make([]string, 0, mdCount*2) + for i := 0; i < mdCount; i++ { + kvs = append(kvs, fmt.Sprintf("header-%d", i), fmt.Sprintf("value-%d", i)) + } + ctx = metadata.AppendToOutgoingContext(ctx, kvs...) + callHdr := &CallHdr{ + Method: "/grpc.testing.BenchmarkService/UnaryCall", + Host: "server:443", + } + + b.ReportAllocs() + for b.Loop() { + hf, err := t.createHeaderFields(ctx, callHdr) + if err != nil { + b.Fatal(err) + } + if len(hf) == 0 { + b.Fatal("no header fields produced") + } + } + }) + } +} diff --git a/internal/transport/http2_server.go b/internal/transport/http2_server.go new file mode 100644 index 0000000..3d7145c --- /dev/null +++ b/internal/transport/http2_server.go @@ -0,0 +1,1532 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "math" + rand "math/rand/v2" + "net" + "net/http" + "strconv" + "sync" + "sync/atomic" + "time" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" + "google.golang.org/protobuf/proto" + + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcutil" + "google.golang.org/grpc/internal/pretty" + istatus "google.golang.org/grpc/internal/status" + "google.golang.org/grpc/internal/syscall" + transportinternal "google.golang.org/grpc/internal/transport/internal" + "google.golang.org/grpc/mem" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" + "google.golang.org/grpc/tap" +) + +var ( + // ErrIllegalHeaderWrite indicates that setting header is illegal because of + // the stream's state. + ErrIllegalHeaderWrite = status.Error(codes.Internal, "transport: SendHeader called multiple times") + // ErrHeaderListSizeLimitViolation indicates that the header list size is larger + // than the limit set by peer. + ErrHeaderListSizeLimitViolation = status.Error(codes.Internal, "transport: trying to send header list size larger than the limit set by peer") +) + +// serverConnectionCounter counts the number of connections a server has seen +// (equal to the number of http2Servers created). Must be accessed atomically. +var serverConnectionCounter uint64 + +// http2Server implements the ServerTransport interface with HTTP2. +type http2Server struct { + lastRead int64 // Keep this field 64-bit aligned. Accessed atomically. + done chan struct{} + conn net.Conn + loopy *loopyWriter + readerDone chan struct{} // sync point to enable testing. + loopyWriterDone chan struct{} + peer peer.Peer + inTapHandle tap.ServerInHandle + framer *framer + // The max number of concurrent streams. + maxStreams uint32 + // controlBuf delivers all the control related tasks (e.g., window + // updates, reset streams, and various settings) to the controller. + controlBuf *controlBuffer + fc *trInFlow + stats stats.Handler + // Keepalive and max-age parameters for the server. + kp keepalive.ServerParameters + // Keepalive enforcement policy. + kep keepalive.EnforcementPolicy + // The time instance last ping was received. + lastPingAt time.Time + // Number of times the client has violated keepalive ping policy so far. + pingStrikes uint8 + // Flag to signify that number of ping strikes should be reset to 0. + // This is set whenever data or header frames are sent. + // 1 means yes. + resetPingStrikes uint32 // Accessed atomically. + initialWindowSize int32 + bdpEst *bdpEstimator + maxSendHeaderListSize *uint32 + + mu sync.Mutex // guard the following + + // drainEvent is initialized when Drain() is called the first time. After + // which the server writes out the first GoAway(with ID 2^31-1) frame. Then + // an independent goroutine will be launched to later send the second + // GoAway. During this time we don't want to write another first GoAway(with + // ID 2^31 -1) frame. Thus call to Drain() will be a no-op if drainEvent is + // already initialized since draining is already underway. + drainEvent *grpcsync.Event + state transportState + activeStreams map[uint32]*ServerStream + // idle is the time instant when the connection went idle. + // This is either the beginning of the connection or when the number of + // RPCs go down to 0. + // When the connection is busy, this value is set to 0. + idle time.Time + + // Fields below are for channelz metric collection. + channelz *channelz.Socket + bufferPool mem.BufferPool + + connectionID uint64 + + // maxStreamMu guards the maximum stream ID + // This lock may not be taken if mu is already held. + maxStreamMu sync.Mutex + maxStreamID uint32 // max stream ID ever seen + + logger *grpclog.PrefixLogger + // setResetPingStrikes is stored as a closure instead of making this a + // method on http2Server to avoid a heap allocation when converting a method + // to a closure for passing to frames objects. + setResetPingStrikes func() +} + +// NewServerTransport creates a http2 transport with conn and configuration +// options from config. +// +// It returns a non-nil transport and a nil error on success. On failure, it +// returns a nil transport and a non-nil error. For a special case where the +// underlying conn gets closed before the client preface could be read, it +// returns a nil transport and a nil error. +func NewServerTransport(conn net.Conn, config *ServerConfig) (_ ServerTransport, err error) { + var authInfo credentials.AuthInfo + rawConn := conn + if config.Credentials != nil { + var err error + conn, authInfo, err = config.Credentials.ServerHandshake(rawConn) + if err != nil { + // ErrConnDispatched means that the connection was dispatched away + // from gRPC; those connections should be left open. io.EOF means + // the connection was closed before handshaking completed, which can + // happen naturally from probers. Return these errors directly. + if err == credentials.ErrConnDispatched || err == io.EOF { + return nil, err + } + return nil, connectionErrorf(false, err, "ServerHandshake(%q) failed: %v", rawConn.RemoteAddr(), err) + } + } + writeBufSize := config.WriteBufferSize + readBufSize := config.ReadBufferSize + // The default header list size is moving from 16MB to 8KB. The 8KB limit + // is only used if Enable8KBDefaultHeaderListSize is true; otherwise, the + // old 16MB default is used. User-specified options always take precedence. + maxHeaderListSize := defaultServerMaxHeaderListSize + if envconfig.Enable8KBDefaultHeaderListSize { + maxHeaderListSize = upcomingDefaultHeaderListSize + } + if config.MaxHeaderListSize != nil { + maxHeaderListSize = *config.MaxHeaderListSize + } + framer := newFramer(conn, writeBufSize, readBufSize, config.SharedWriteBuffer, maxHeaderListSize, config.BufferPool) + // Send initial settings as connection preface to client. + isettings := []http2.Setting{{ + ID: http2.SettingMaxFrameSize, + Val: http2MaxFrameLen, + }} + if config.MaxStreams != math.MaxUint32 { + isettings = append(isettings, http2.Setting{ + ID: http2.SettingMaxConcurrentStreams, + Val: config.MaxStreams, + }) + } + iwz := int32(initialWindowSize) + if config.InitialWindowSize >= defaultWindowSize { + iwz = config.InitialWindowSize + } + icwz := int32(initialWindowSize) + if config.InitialConnWindowSize >= defaultWindowSize { + icwz = config.InitialConnWindowSize + } + if iwz != defaultWindowSize { + isettings = append(isettings, http2.Setting{ + ID: http2.SettingInitialWindowSize, + Val: uint32(iwz)}) + } + if config.MaxHeaderListSize != nil { + isettings = append(isettings, http2.Setting{ + ID: http2.SettingMaxHeaderListSize, + Val: *config.MaxHeaderListSize, + }) + } + if config.HeaderTableSize != nil { + isettings = append(isettings, http2.Setting{ + ID: http2.SettingHeaderTableSize, + Val: *config.HeaderTableSize, + }) + } + if err := framer.fr.WriteSettings(isettings...); err != nil { + return nil, connectionErrorf(false, err, "transport: %v", err) + } + // Adjust the connection flow control window if needed. + if delta := uint32(icwz - defaultWindowSize); delta > 0 { + if err := framer.fr.WriteWindowUpdate(0, delta); err != nil { + return nil, connectionErrorf(false, err, "transport: %v", err) + } + } + kp := config.KeepaliveParams + if kp.MaxConnectionIdle == 0 { + kp.MaxConnectionIdle = defaultMaxConnectionIdle + } + if kp.MaxConnectionAge == 0 { + kp.MaxConnectionAge = defaultMaxConnectionAge + } + // Add a jitter to MaxConnectionAge. + kp.MaxConnectionAge += getJitter(kp.MaxConnectionAge) + if kp.MaxConnectionAgeGrace == 0 { + kp.MaxConnectionAgeGrace = defaultMaxConnectionAgeGrace + } + if kp.Time == 0 { + kp.Time = defaultServerKeepaliveTime + } + if kp.Timeout == 0 { + kp.Timeout = defaultServerKeepaliveTimeout + } + if kp.Time != infinity { + if err = syscall.SetTCPUserTimeout(rawConn, kp.Timeout); err != nil { + return nil, connectionErrorf(false, err, "transport: failed to set TCP_USER_TIMEOUT: %v", err) + } + } + kep := config.KeepalivePolicy + if kep.MinTime == 0 { + kep.MinTime = defaultKeepalivePolicyMinTime + } + + done := make(chan struct{}) + peer := peer.Peer{ + Addr: conn.RemoteAddr(), + LocalAddr: conn.LocalAddr(), + AuthInfo: authInfo, + } + t := &http2Server{ + done: done, + conn: conn, + peer: peer, + framer: framer, + readerDone: make(chan struct{}), + loopyWriterDone: make(chan struct{}), + maxStreams: config.MaxStreams, + inTapHandle: config.InTapHandle, + fc: &trInFlow{limit: uint32(icwz)}, + state: reachable, + activeStreams: make(map[uint32]*ServerStream), + stats: config.StatsHandler, + kp: kp, + idle: time.Now(), + kep: kep, + initialWindowSize: iwz, + bufferPool: config.BufferPool, + } + t.setResetPingStrikes = func() { + atomic.StoreUint32(&t.resetPingStrikes, 1) + } + var czSecurity credentials.ChannelzSecurityValue + if au, ok := authInfo.(credentials.ChannelzSecurityInfo); ok { + czSecurity = au.GetSecurityValue() + } + t.channelz = channelz.RegisterSocket( + &channelz.Socket{ + SocketType: channelz.SocketTypeNormal, + Parent: config.ChannelzParent, + SocketMetrics: channelz.SocketMetrics{}, + EphemeralMetrics: t.socketMetrics, + LocalAddr: t.peer.LocalAddr, + RemoteAddr: t.peer.Addr, + SocketOptions: channelz.GetSocketOption(t.conn), + Security: czSecurity, + }, + ) + t.logger = prefixLoggerForServerTransport(t) + + t.controlBuf = newControlBuffer(t.done) + if !config.StaticWindowSize { + t.bdpEst = &bdpEstimator{ + bdp: initialWindowSize, + updateFlowControl: t.updateFlowControl, + } + } + + t.connectionID = atomic.AddUint64(&serverConnectionCounter, 1) + t.framer.writer.Flush() + + defer func() { + if err != nil { + t.Close(err) + } + }() + + // Check the validity of client preface. + preface := make([]byte, len(clientPreface)) + if _, err := io.ReadFull(t.conn, preface); err != nil { + // In deployments where a gRPC server runs behind a cloud load balancer + // which performs regular TCP level health checks, the connection is + // closed immediately by the latter. Returning io.EOF here allows the + // grpc server implementation to recognize this scenario and suppress + // logging to reduce spam. + if err == io.EOF { + return nil, io.EOF + } + return nil, connectionErrorf(false, err, "transport: http2Server.HandleStreams failed to receive the preface from client: %v", err) + } + if !bytes.Equal(preface, clientPreface) { + return nil, connectionErrorf(false, nil, "transport: http2Server.HandleStreams received bogus greeting from client: %q", preface) + } + + frame, err := t.framer.fr.ReadFrame() + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil, err + } + if err != nil { + return nil, connectionErrorf(false, err, "transport: http2Server.HandleStreams failed to read initial settings frame: %v", err) + } + atomic.StoreInt64(&t.lastRead, time.Now().UnixNano()) + sf, ok := frame.(*http2.SettingsFrame) + if !ok { + return nil, connectionErrorf(false, nil, "transport: http2Server.HandleStreams saw invalid preface type %T from client", frame) + } + t.handleSettings(sf) + + go func() { + t.loopy = newLoopyWriter(serverSide, t.framer, t.controlBuf, t.bdpEst, t.conn, t.logger, t.outgoingGoAwayHandler, t.bufferPool) + err := t.loopy.run() + close(t.loopyWriterDone) + if !isIOError(err) { + // Close the connection if a non-I/O error occurs (for I/O errors + // the reader will also encounter the error and close). Wait 1 + // second before closing the connection, or when the reader is done + // (i.e. the client already closed the connection or a connection + // error occurred). This avoids the potential problem where there + // is unread data on the receive side of the connection, which, if + // closed, would lead to a TCP RST instead of FIN, and the client + // encountering errors. For more info: + // https://github.com/grpc/grpc-go/issues/5358 + timer := time.NewTimer(time.Second) + defer timer.Stop() + select { + case <-t.readerDone: + case <-timer.C: + } + t.conn.Close() + } + }() + go t.keepalive() + return t, nil +} + +// operateHeaders takes action on the decoded headers. Returns an error if fatal +// error encountered and transport needs to close, otherwise returns nil. +func (t *http2Server) operateHeaders(ctx context.Context, frame *http2.MetaHeadersFrame, handle func(*ServerStream)) error { + // Acquire max stream ID lock for entire duration + t.maxStreamMu.Lock() + defer t.maxStreamMu.Unlock() + + streamID := frame.Header().StreamID + + // frame.Truncated is set to true when framer detects that the current header + // list size hits MaxHeaderListSize limit. + if frame.Truncated { + t.controlBuf.put(&cleanupStream{ + streamID: streamID, + rst: true, + rstCode: http2.ErrCodeFrameSize, + onWrite: func() {}, + }) + return nil + } + + if streamID%2 != 1 || streamID <= t.maxStreamID { + // illegal gRPC stream id. + return fmt.Errorf("received an illegal stream id: %v. headers frame: %+v", streamID, frame) + } + t.maxStreamID = streamID + + s := &ServerStream{ + Stream: Stream{ + id: streamID, + fc: inFlow{limit: uint32(t.initialWindowSize)}, + }, + st: t, + headerWireLength: int(frame.Header().Length), + } + s.Stream.buf.init(t.bufferPool) + var ( + // if false, content-type was missing or invalid + isGRPC = false + contentType = "" + mdata = make(metadata.MD, len(frame.Fields)) + httpMethod string + // these are set if an error is encountered while parsing the headers + protocolError bool + headerError *status.Status + + timeoutSet bool + timeout time.Duration + ) + + for _, hf := range frame.Fields { + switch hf.Name { + case "content-type": + contentSubtype, validContentType := grpcutil.ContentSubtype(hf.Value) + if !validContentType { + contentType = hf.Value + break + } + mdata[hf.Name] = append(mdata[hf.Name], hf.Value) + s.contentSubtype = contentSubtype + isGRPC = true + + case "grpc-accept-encoding": + mdata[hf.Name] = append(mdata[hf.Name], hf.Value) + if hf.Value == "" { + continue + } + compressors := hf.Value + if s.clientAdvertisedCompressors != "" { + compressors = s.clientAdvertisedCompressors + "," + compressors + } + s.clientAdvertisedCompressors = compressors + case "grpc-encoding": + s.recvCompress = hf.Value + case ":method": + httpMethod = hf.Value + case ":path": + s.method = hf.Value + case "grpc-timeout": + timeoutSet = true + var err error + if timeout, err = decodeTimeout(hf.Value); err != nil { + headerError = status.Newf(codes.Internal, "malformed grpc-timeout: %v", err) + } + // "Transports must consider requests containing the Connection header + // as malformed." - A41 + case "connection": + if t.logger.V(logLevel) { + t.logger.Infof("Received a HEADERS frame with a :connection header which makes the request malformed, as per the HTTP/2 spec") + } + protocolError = true + default: + if isReservedHeader(hf.Name) && !isWhitelistedHeader(hf.Name) { + break + } + v, err := decodeMetadataHeader(hf.Name, hf.Value) + if err != nil { + headerError = status.Newf(codes.Internal, "malformed binary metadata %q in header %q: %v", hf.Value, hf.Name, err) + t.logger.Warningf("Failed to decode metadata header (%q, %q): %v", hf.Name, hf.Value, err) + break + } + mdata[hf.Name] = append(mdata[hf.Name], v) + } + } + + // "If multiple Host headers or multiple :authority headers are present, the + // request must be rejected with an HTTP status code 400 as required by Host + // validation in RFC 7230 §5.4, gRPC status code INTERNAL, or RST_STREAM + // with HTTP/2 error code PROTOCOL_ERROR." - A41. Since this is a HTTP/2 + // error, this takes precedence over a client not speaking gRPC. + if len(mdata[":authority"]) > 1 || len(mdata["host"]) > 1 { + errMsg := fmt.Sprintf("num values of :authority: %v, num values of host: %v, both must only have 1 value as per HTTP/2 spec", len(mdata[":authority"]), len(mdata["host"])) + if t.logger.V(logLevel) { + t.logger.Infof("Aborting the stream early: %v", errMsg) + } + t.writeEarlyAbort(streamID, s.contentSubtype, status.New(codes.Internal, errMsg), http.StatusBadRequest, !frame.StreamEnded()) + return nil + } + + if protocolError { + t.controlBuf.put(&cleanupStream{ + streamID: streamID, + rst: true, + rstCode: http2.ErrCodeProtocol, + onWrite: func() {}, + }) + return nil + } + if !isGRPC { + t.writeEarlyAbort(streamID, s.contentSubtype, status.Newf(codes.InvalidArgument, "invalid gRPC request content-type %q", contentType), http.StatusUnsupportedMediaType, !frame.StreamEnded()) + return nil + } + if headerError != nil { + t.writeEarlyAbort(streamID, s.contentSubtype, headerError, http.StatusBadRequest, !frame.StreamEnded()) + return nil + } + + // "If :authority is missing, Host must be renamed to :authority." - A41 + if len(mdata[":authority"]) == 0 { + // No-op if host isn't present, no eventual :authority header is a valid + // RPC. + if host, ok := mdata["host"]; ok { + mdata[":authority"] = host + delete(mdata, "host") + } + } else { + // "If :authority is present, Host must be discarded" - A41 + delete(mdata, "host") + } + + // If :authority is still missing, i.e. no host or :authority header is + // present, reject the request as invalid. + if len(mdata[":authority"]) == 0 { + t.writeEarlyAbort(streamID, s.contentSubtype, status.New(codes.Internal, "no host or :authority header present"), http.StatusBadRequest, !frame.StreamEnded()) + return nil + } + if frame.StreamEnded() { + // s is just created by the caller. No lock needed. + s.state = streamReadDone + s.write(recvMsg{err: io.EOF}) + } + if timeoutSet { + s.ctx, s.cancel = context.WithTimeout(ctx, timeout) + } else { + s.ctx, s.cancel = context.WithCancel(ctx) + } + + // Attach the received metadata to the context. + if len(mdata) > 0 { + s.ctx = metadata.NewIncomingContext(s.ctx, mdata) + } + t.mu.Lock() + if t.state != reachable { + t.mu.Unlock() + s.cancel() + return nil + } + if uint32(len(t.activeStreams)) >= t.maxStreams { + t.mu.Unlock() + t.controlBuf.put(&cleanupStream{ + streamID: streamID, + rst: true, + rstCode: http2.ErrCodeRefusedStream, + onWrite: func() {}, + }) + s.cancel() + return nil + } + if httpMethod != http.MethodPost { + t.mu.Unlock() + errMsg := fmt.Sprintf("Received a HEADERS frame with :method %q which should be POST", httpMethod) + if t.logger.V(logLevel) { + t.logger.Infof("Aborting the stream early: %v", errMsg) + } + t.writeEarlyAbort(streamID, s.contentSubtype, status.New(codes.Internal, errMsg), http.StatusMethodNotAllowed, !frame.StreamEnded()) + s.cancel() + return nil + } + if t.inTapHandle != nil { + var err error + if s.ctx, err = t.inTapHandle(s.ctx, &tap.Info{FullMethodName: s.method, Header: mdata}); err != nil { + t.mu.Unlock() + if t.logger.V(logLevel) { + t.logger.Infof("Aborting the stream early due to InTapHandle failure: %v", err) + } + stat, ok := status.FromError(err) + if !ok { + stat = status.New(codes.PermissionDenied, err.Error()) + } + t.writeEarlyAbort(s.id, s.contentSubtype, stat, http.StatusOK, !frame.StreamEnded()) + return nil + } + } + + if s.ctx.Err() != nil { + t.mu.Unlock() + st := status.New(codes.DeadlineExceeded, context.DeadlineExceeded.Error()) + // Early abort in case the timeout was zero or so low it already fired. + t.writeEarlyAbort(s.id, s.contentSubtype, st, http.StatusOK, !frame.StreamEnded()) + return nil + } + + t.activeStreams[streamID] = s + if len(t.activeStreams) == 1 { + t.idle = time.Time{} + } + + // Start a timer to close the stream on reaching the deadline. + if timeoutSet { + // We need to wait for s.cancel to be updated before calling + // t.closeStream to avoid data races. + cancelUpdated := make(chan struct{}) + timer := internal.TimeAfterFunc(timeout, func() { + <-cancelUpdated + t.closeStream(s, true, http2.ErrCodeCancel, false) + }) + oldCancel := s.cancel + s.cancel = func() { + oldCancel() + timer.Stop() + } + close(cancelUpdated) + } + t.mu.Unlock() + if channelz.IsOn() { + t.channelz.SocketMetrics.StreamsStarted.Add(1) + t.channelz.SocketMetrics.LastRemoteStreamCreatedTimestamp.Store(time.Now().UnixNano()) + } + s.readRequester = s + s.ctxDone = s.ctx.Done() + s.Stream.wq.init(defaultWriteQuota, s.ctxDone) + s.trReader = transportReader{ + reader: recvBufferReader{ + ctx: s.ctx, + ctxDone: s.ctxDone, + recv: &s.buf, + }, + windowHandler: s, + } + // Register the stream with loopy. + t.controlBuf.put(®isterStream{ + streamID: s.id, + wq: &s.wq, + }) + handle(s) + return nil +} + +// HandleStreams receives incoming streams using the given handler. This is +// typically run in a separate goroutine. +// traceCtx attaches trace to ctx and returns the new context. +func (t *http2Server) HandleStreams(ctx context.Context, handle func(*ServerStream)) { + defer func() { + close(t.readerDone) + <-t.loopyWriterDone + }() + for { + t.controlBuf.throttle() + frame, err := t.framer.readFrame() + atomic.StoreInt64(&t.lastRead, time.Now().UnixNano()) + if err != nil { + if se, ok := err.(http2.StreamError); ok { + if t.logger.V(logLevel) { + t.logger.Warningf("Encountered http2.StreamError: %v", se) + } + t.mu.Lock() + s := t.activeStreams[se.StreamID] + t.mu.Unlock() + if s != nil { + t.closeStream(s, true, se.Code, false) + } else { + t.controlBuf.put(&cleanupStream{ + streamID: se.StreamID, + rst: true, + rstCode: se.Code, + onWrite: func() {}, + }) + } + continue + } + t.Close(err) + return + } + switch frame := frame.(type) { + case *http2.MetaHeadersFrame: + if err := t.operateHeaders(ctx, frame, handle); err != nil { + // Any error processing client headers, e.g. invalid stream ID, + // is considered a protocol violation. + t.controlBuf.put(&goAway{ + code: http2.ErrCodeProtocol, + debugData: []byte(err.Error()), + closeConn: err, + }) + continue + } + case *parsedDataFrame: + t.handleData(frame) + frame.data.Free() + case *http2.RSTStreamFrame: + t.handleRSTStream(frame) + case *http2.SettingsFrame: + t.handleSettings(frame) + case *http2.PingFrame: + t.handlePing(frame) + case *http2.WindowUpdateFrame: + t.handleWindowUpdate(frame) + case *http2.GoAwayFrame: + // TODO: Handle GoAway from the client appropriately. + default: + if t.logger.V(logLevel) { + t.logger.Infof("Received unsupported frame type %T", frame) + } + } + } +} + +func (t *http2Server) getStream(f http2.Frame) (*ServerStream, bool) { + t.mu.Lock() + defer t.mu.Unlock() + if t.activeStreams == nil { + // The transport is closing. + return nil, false + } + s, ok := t.activeStreams[f.Header().StreamID] + if !ok { + // The stream is already done. + return nil, false + } + return s, true +} + +// adjustWindow sends out extra window update over the initial window size +// of stream if the application is requesting data larger in size than +// the window. +func (t *http2Server) adjustWindow(s *ServerStream, n uint32) { + if w := s.fc.maybeAdjust(n); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{streamID: s.id, increment: w}) + } + +} + +// updateWindow adjusts the inbound quota for the stream and the transport. +// Window updates will deliver to the controller for sending when +// the cumulative quota exceeds the corresponding threshold. +func (t *http2Server) updateWindow(s *ServerStream, n uint32) { + if w := s.fc.onRead(n); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{streamID: s.id, + increment: w, + }) + } +} + +// updateFlowControl updates the incoming flow control windows +// for the transport and the stream based on the current bdp +// estimation. +func (t *http2Server) updateFlowControl(n uint32) { + t.mu.Lock() + for _, s := range t.activeStreams { + s.fc.newLimit(n) + } + t.initialWindowSize = int32(n) + t.mu.Unlock() + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: t.fc.newLimit(n), + }) + t.controlBuf.put(&outgoingSettings{ + ss: []http2.Setting{ + { + ID: http2.SettingInitialWindowSize, + Val: n, + }, + }, + }) + +} + +func (t *http2Server) handleData(f *parsedDataFrame) { + size := f.Header().Length + var sendBDPPing bool + if t.bdpEst != nil { + sendBDPPing = t.bdpEst.add(size) + } + // Decouple connection's flow control from application's read. + // An update on connection's flow control should not depend on + // whether user application has read the data or not. Such a + // restriction is already imposed on the stream's flow control, + // and therefore the sender will be blocked anyways. + // Decoupling the connection flow control will prevent other + // active(fast) streams from starving in presence of slow or + // inactive streams. + if w := t.fc.onData(size); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: w, + }) + } + if sendBDPPing { + // Avoid excessive ping detection (e.g. in an L7 proxy) + // by sending a window update prior to the BDP ping. + if w := t.fc.reset(); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: 0, + increment: w, + }) + } + t.controlBuf.put(bdpPing) + } + // Select the right stream to dispatch. + s, ok := t.getStream(f) + if !ok { + return + } + if s.getState() == streamReadDone { + t.closeStream(s, true, http2.ErrCodeStreamClosed, false) + return + } + if size > 0 { + if err := s.fc.onData(size); err != nil { + t.closeStream(s, true, http2.ErrCodeFlowControl, false) + return + } + dataLen := f.data.Len() + if f.Header().Flags.Has(http2.FlagDataPadded) { + if w := s.fc.onRead(size - uint32(dataLen)); w > 0 { + t.controlBuf.put(&outgoingWindowUpdate{ + streamID: s.id, + increment: w, + }) + } + } + if dataLen > 0 { + f.data.Ref() + s.write(recvMsg{buffer: f.data}) + } + } + if f.StreamEnded() { + // Received the end of stream from the client. + s.compareAndSwapState(streamActive, streamReadDone) + s.write(recvMsg{err: io.EOF}) + } +} + +func (t *http2Server) handleRSTStream(f *http2.RSTStreamFrame) { + // If the stream is not deleted from the transport's active streams map, then do a regular close stream. + if s, ok := t.getStream(f); ok { + t.closeStream(s, false, 0, false) + return + } + // If the stream is already deleted from the active streams map, then put a cleanupStream item into controlbuf to delete the stream from loopy writer's established streams map. + t.controlBuf.put(&cleanupStream{ + streamID: f.Header().StreamID, + rst: false, + rstCode: 0, + onWrite: func() {}, + }) +} + +func (t *http2Server) handleSettings(f *http2.SettingsFrame) { + if f.IsAck() { + return + } + var ss []http2.Setting + var updateFuncs []func() + f.ForeachSetting(func(s http2.Setting) error { + switch s.ID { + case http2.SettingMaxHeaderListSize: + updateFuncs = append(updateFuncs, func() { + t.maxSendHeaderListSize = new(uint32) + *t.maxSendHeaderListSize = s.Val + }) + default: + ss = append(ss, s) + } + return nil + }) + t.controlBuf.executeAndPut(func() bool { + for _, f := range updateFuncs { + f() + } + return true + }, &incomingSettings{ + ss: ss, + }) +} + +const ( + maxPingStrikes = 2 + defaultPingTimeout = 2 * time.Hour +) + +func (t *http2Server) handlePing(f *http2.PingFrame) { + if f.IsAck() { + if f.Data == goAwayPing.data && t.drainEvent != nil { + t.drainEvent.Fire() + return + } + // Maybe it's a BDP ping. + if t.bdpEst != nil { + t.bdpEst.calculate(f.Data) + } + return + } + pingAck := &ping{ack: true} + copy(pingAck.data[:], f.Data[:]) + t.controlBuf.put(pingAck) + + now := time.Now() + defer func() { + t.lastPingAt = now + }() + // A reset ping strikes means that we don't need to check for policy + // violation for this ping and the pingStrikes counter should be set + // to 0. + if atomic.CompareAndSwapUint32(&t.resetPingStrikes, 1, 0) { + t.pingStrikes = 0 + return + } + t.mu.Lock() + ns := len(t.activeStreams) + t.mu.Unlock() + if ns < 1 && !t.kep.PermitWithoutStream { + // Keepalive shouldn't be active thus, this new ping should + // have come after at least defaultPingTimeout. + if t.lastPingAt.Add(defaultPingTimeout).After(now) { + t.pingStrikes++ + } + } else { + // Check if keepalive policy is respected. + if t.lastPingAt.Add(t.kep.MinTime).After(now) { + t.pingStrikes++ + } + } + + if t.pingStrikes > maxPingStrikes { + // Send goaway and close the connection. + t.controlBuf.put(&goAway{code: http2.ErrCodeEnhanceYourCalm, debugData: []byte("too_many_pings"), closeConn: errors.New("got too many pings from the client")}) + } +} + +func (t *http2Server) handleWindowUpdate(f *http2.WindowUpdateFrame) { + t.controlBuf.put(&incomingWindowUpdate{ + streamID: f.Header().StreamID, + increment: f.Increment, + }) +} + +func appendHeaderFieldsFromMD(headerFields []hpack.HeaderField, md metadata.MD) []hpack.HeaderField { + for k, vv := range md { + if isReservedHeader(k) { + // Clients don't tolerate reading restricted headers after some non restricted ones were sent. + continue + } + for _, v := range vv { + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) + } + } + return headerFields +} + +func (t *http2Server) checkForHeaderListSize(hf []hpack.HeaderField) bool { + if t.maxSendHeaderListSize == nil { + return true + } + var sz int64 + for _, f := range hf { + sz += int64(f.Size()) + if sz > int64(*t.maxSendHeaderListSize) { + if t.logger.V(logLevel) { + t.logger.Infof("Header list size to send violates the maximum size (%d bytes) set by client", *t.maxSendHeaderListSize) + } + return false + } + } + if !envconfig.Enable8KBDefaultHeaderListSize && sz > int64(upcomingDefaultHeaderListSize) { + t.logger.Warningf("Header list size to send (%d bytes) is larger than the upcoming default limit (%d bytes). In release v1.82.0, GRPC_GO_EXPERIMENTAL_ENABLE_8KB_DEFAULT_HEADER_LIST_SIZE will be enabled by default, enforcing this limit.", sz, upcomingDefaultHeaderListSize) + } + return true +} + +// writeEarlyAbort sends an early abort response with the given HTTP status and +// gRPC status. If the header list size exceeds the peer's limit, it sends a +// RST_STREAM instead. +func (t *http2Server) writeEarlyAbort(streamID uint32, contentSubtype string, stat *status.Status, httpStatus uint32, rst bool) { + hf := []hpack.HeaderField{ + {Name: ":status", Value: strconv.Itoa(int(httpStatus))}, + {Name: "content-type", Value: grpcutil.ContentType(contentSubtype)}, + {Name: "grpc-status", Value: strconv.Itoa(int(stat.Code()))}, + {Name: "grpc-message", Value: encodeGrpcMessage(stat.Message())}, + } + if p := istatus.RawStatusProto(stat); len(p.GetDetails()) > 0 { + stBytes, err := proto.Marshal(p) + if err != nil { + t.logger.Errorf("Failed to marshal rpc status: %s, error: %v", pretty.ToJSON(p), err) + } + if err == nil { + hf = append(hf, hpack.HeaderField{Name: grpcStatusDetailsBinHeader, Value: encodeBinHeader(stBytes)}) + } + } + success, _ := t.controlBuf.executeAndPut(func() bool { + return t.checkForHeaderListSize(hf) + }, &earlyAbortStream{ + streamID: streamID, + rst: rst, + hf: hf, + }) + if !success { + t.controlBuf.put(&cleanupStream{ + streamID: streamID, + rst: true, + rstCode: http2.ErrCodeInternal, + onWrite: func() {}, + }) + } +} + +func (t *http2Server) streamContextErr(s *ServerStream) error { + select { + case <-t.done: + return ErrConnClosing + default: + } + err := s.ctx.Err() + if err == nil { + // In closeStream and finishStream, the stream state is transitioned to + // streamDone before s.cancel() is invoked so that concurrent operations + // immediately observe the terminal state. A concurrent caller (such as + // write or writeHeader) can see streamDone and invoke streamContextErr + // before s.cancel() runs or propagates to s.ctx. Default to + // context.Canceled rather than passing nil to ContextErr. + err = context.Canceled + } + return ContextErr(err) +} + +// WriteHeader sends the header metadata md back to the client. +func (t *http2Server) writeHeader(s *ServerStream, md metadata.MD) error { + s.hdrMu.Lock() + defer s.hdrMu.Unlock() + if s.getState() == streamDone { + return t.streamContextErr(s) + } + + if s.updateHeaderSent() { + return ErrIllegalHeaderWrite + } + + if md.Len() > 0 { + if s.header.Len() > 0 { + s.header = metadata.Join(s.header, md) + } else { + s.header = md + } + } + if err := t.writeHeaderLocked(s); err != nil { + switch e := err.(type) { + case ConnectionError: + return status.Error(codes.Unavailable, e.Desc) + default: + return status.Convert(err).Err() + } + } + return nil +} + +func (t *http2Server) writeHeaderLocked(s *ServerStream) error { + // TODO(mmukhi): Benchmark if the performance gets better if count the metadata and other header fields + // first and create a slice of that exact size. + headerFields := make([]hpack.HeaderField, 0, 2) // at least :status, content-type will be there if none else. + headerFields = append(headerFields, hpack.HeaderField{Name: ":status", Value: "200"}) + headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: grpcutil.ContentType(s.contentSubtype)}) + if s.sendCompress != "" { + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-encoding", Value: s.sendCompress}) + } + headerFields = appendHeaderFieldsFromMD(headerFields, s.header) + hf := &serverHeaders{ + streamID: s.id, + hf: headerFields, + endStream: false, + onWrite: t.setResetPingStrikes, + } + success, err := t.controlBuf.executeAndPut(func() bool { return t.checkForHeaderListSize(hf.hf) }, hf) + if !success { + if err != nil { + return err + } + t.closeStream(s, true, http2.ErrCodeInternal, false) + return ErrHeaderListSizeLimitViolation + } + if t.stats != nil { + // Note: Headers are compressed with hpack after this call returns. + // No WireLength field is set here. + t.stats.HandleRPC(s.Context(), &stats.OutHeader{ + Header: s.header.Copy(), + Compression: s.sendCompress, + }) + } + return nil +} + +// writeStatus sends stream status to the client and terminates the stream. +// There is no further I/O operations being able to perform on this stream. +// TODO(zhaoq): Now it indicates the end of entire stream. Revisit if early +// OK is adopted. +func (t *http2Server) writeStatus(s *ServerStream, st *status.Status) error { + s.hdrMu.Lock() + defer s.hdrMu.Unlock() + + if s.getState() == streamDone { + return nil + } + + // TODO(mmukhi): Benchmark if the performance gets better if count the metadata and other header fields + // first and create a slice of that exact size. + headerFields := make([]hpack.HeaderField, 0, 2) // grpc-status and grpc-message will be there if none else. + if !s.updateHeaderSent() { // No headers have been sent. + if len(s.header) > 0 { // Send a separate header frame. + if err := t.writeHeaderLocked(s); err != nil { + return err + } + } else { // Send a trailer only response. + headerFields = append(headerFields, hpack.HeaderField{Name: ":status", Value: "200"}) + headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: grpcutil.ContentType(s.contentSubtype)}) + } + } + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-status", Value: strconv.Itoa(int(st.Code()))}) + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-message", Value: encodeGrpcMessage(st.Message())}) + + if p := istatus.RawStatusProto(st); len(p.GetDetails()) > 0 { + // Do not use the user's grpc-status-details-bin (if present) if we are + // even attempting to set our own. + delete(s.trailer, grpcStatusDetailsBinHeader) + stBytes, err := proto.Marshal(p) + if err != nil { + // TODO: return error instead, when callers are able to handle it. + t.logger.Errorf("Failed to marshal rpc status: %s, error: %v", pretty.ToJSON(p), err) + } else { + headerFields = append(headerFields, hpack.HeaderField{Name: grpcStatusDetailsBinHeader, Value: encodeBinHeader(stBytes)}) + } + } + + // Attach the trailer metadata. + headerFields = appendHeaderFieldsFromMD(headerFields, s.trailer) + trailingHeader := &serverHeaders{ + streamID: s.id, + hf: headerFields, + endStream: true, + onWrite: t.setResetPingStrikes, + } + + success, err := t.controlBuf.executeAndPut(func() bool { + return t.checkForHeaderListSize(trailingHeader.hf) + }, nil) + if !success { + if err != nil { + return err + } + t.closeStream(s, true, http2.ErrCodeInternal, false) + return ErrHeaderListSizeLimitViolation + } + // Send a RST_STREAM after the trailers if the client has not already half-closed. + rst := s.getState() == streamActive + t.finishStream(s, rst, http2.ErrCodeNo, trailingHeader, true) + if t.stats != nil { + // Note: The trailer fields are compressed with hpack after this call returns. + // No WireLength field is set here. + t.stats.HandleRPC(s.Context(), &stats.OutTrailer{ + Trailer: s.trailer.Copy(), + }) + } + return nil +} + +// Write converts the data into HTTP2 data frame and sends it out. Non-nil error +// is returns if it fails (e.g., framing error, transport error). +func (t *http2Server) write(s *ServerStream, hdr []byte, data mem.BufferSlice, _ *WriteOptions) error { + if !s.isHeaderSent() { // Headers haven't been written yet. + if err := t.writeHeader(s, nil); err != nil { + return err + } + } else { + // Writing headers checks for this condition. + if s.getState() == streamDone { + return t.streamContextErr(s) + } + } + + df := &dataFrame{ + streamID: s.id, + h: hdr, + data: data, + onEachWrite: t.setResetPingStrikes, + } + dataLen := data.Len() + if err := s.wq.get(int32(len(hdr) + dataLen)); err != nil { + return t.streamContextErr(s) + } + data.Ref() + if err := t.controlBuf.put(df); err != nil { + data.Free() + return err + } + t.incrMsgSent() + return nil +} + +// keepalive running in a separate goroutine does the following: +// 1. Gracefully closes an idle connection after a duration of keepalive.MaxConnectionIdle. +// 2. Gracefully closes any connection after a duration of keepalive.MaxConnectionAge. +// 3. Forcibly closes a connection after an additive period of keepalive.MaxConnectionAgeGrace over keepalive.MaxConnectionAge. +// 4. Makes sure a connection is alive by sending pings with a frequency of keepalive.Time and closes a non-responsive connection +// after an additional duration of keepalive.Timeout. +func (t *http2Server) keepalive() { + p := &ping{} + // True iff a ping has been sent, and no data has been received since then. + outstandingPing := false + // Amount of time remaining before which we should receive an ACK for the + // last sent ping. + kpTimeoutLeft := time.Duration(0) + // Records the last value of t.lastRead before we go block on the timer. + // This is required to check for read activity since then. + prevNano := time.Now().UnixNano() + // Initialize the different timers to their default values. + idleTimer := time.NewTimer(t.kp.MaxConnectionIdle) + ageTimer := time.NewTimer(t.kp.MaxConnectionAge) + kpTimer := time.NewTimer(t.kp.Time) + defer func() { + // We need to drain the underlying channel in these timers after a call + // to Stop(), only if we are interested in resetting them. Clearly we + // are not interested in resetting them here. + idleTimer.Stop() + ageTimer.Stop() + kpTimer.Stop() + }() + + for { + select { + case <-idleTimer.C: + t.mu.Lock() + idle := t.idle + if idle.IsZero() { // The connection is non-idle. + t.mu.Unlock() + idleTimer.Reset(t.kp.MaxConnectionIdle) + continue + } + val := t.kp.MaxConnectionIdle - time.Since(idle) + t.mu.Unlock() + if val <= 0 { + // The connection has been idle for a duration of keepalive.MaxConnectionIdle or more. + // Gracefully close the connection. + t.Drain("max_idle") + return + } + idleTimer.Reset(val) + case <-ageTimer.C: + t.Drain("max_age") + ageTimer.Reset(t.kp.MaxConnectionAgeGrace) + select { + case <-ageTimer.C: + // Close the connection after grace period. + if t.logger.V(logLevel) { + t.logger.Infof("Closing server transport due to maximum connection age") + } + t.controlBuf.put(closeConnection{}) + case <-t.done: + } + return + case <-kpTimer.C: + lastRead := atomic.LoadInt64(&t.lastRead) + if lastRead > prevNano { + // There has been read activity since the last time we were + // here. Setup the timer to fire at kp.Time seconds from + // lastRead time and continue. + outstandingPing = false + kpTimer.Reset(time.Duration(lastRead) + t.kp.Time - time.Duration(time.Now().UnixNano())) + prevNano = lastRead + continue + } + if outstandingPing && kpTimeoutLeft <= 0 { + t.Close(fmt.Errorf("keepalive ping not acked within timeout %s", t.kp.Timeout)) + return + } + if !outstandingPing { + if channelz.IsOn() { + t.channelz.SocketMetrics.KeepAlivesSent.Add(1) + } + t.controlBuf.put(p) + kpTimeoutLeft = t.kp.Timeout + outstandingPing = true + } + // The amount of time to sleep here is the minimum of kp.Time and + // timeoutLeft. This will ensure that we wait only for kp.Time + // before sending out the next ping (for cases where the ping is + // acked). + sleepDuration := min(t.kp.Time, kpTimeoutLeft) + kpTimeoutLeft -= sleepDuration + kpTimer.Reset(sleepDuration) + case <-t.done: + return + } + } +} + +// Close starts shutting down the http2Server transport. +// TODO(zhaoq): Now the destruction is not blocked on any pending streams. This +// could cause some resource issue. Revisit this later. +func (t *http2Server) Close(err error) { + t.mu.Lock() + if t.state == closing { + t.mu.Unlock() + return + } + if t.logger.V(logLevel) { + t.logger.Infof("Closing: %v", err) + } + t.state = closing + streams := t.activeStreams + t.activeStreams = nil + t.mu.Unlock() + t.controlBuf.finish() + close(t.done) + if err := t.conn.Close(); err != nil && t.logger.V(logLevel) { + t.logger.Infof("Error closing underlying net.Conn during Close: %v", err) + } + channelz.RemoveEntry(t.channelz.ID) + // Cancel all active streams. + for _, s := range streams { + s.cancel() + } +} + +// deleteStream deletes the stream s from transport's active streams. +func (t *http2Server) deleteStream(s *ServerStream, eosReceived bool) { + t.mu.Lock() + _, isActive := t.activeStreams[s.id] + if isActive { + delete(t.activeStreams, s.id) + if len(t.activeStreams) == 0 { + t.idle = time.Now() + } + } + t.mu.Unlock() + + if isActive && channelz.IsOn() { + if eosReceived { + t.channelz.SocketMetrics.StreamsSucceeded.Add(1) + } else { + t.channelz.SocketMetrics.StreamsFailed.Add(1) + } + } +} + +// finishStream closes the stream and puts the trailing headerFrame into controlbuf. +func (t *http2Server) finishStream(s *ServerStream, rst bool, rstCode http2.ErrCode, hdr *serverHeaders, eosReceived bool) { + + oldState := s.swapState(streamDone) + if oldState == streamDone { + // If the stream was already done, return. + return + } + + // In case stream sending and receiving are invoked in separate + // goroutines (e.g., bi-directional streaming), cancel needs to be + // called to interrupt the potential blocking on other goroutines. + s.cancel() + + hdr.cleanup = &cleanupStream{ + streamID: s.id, + rst: rst, + rstCode: rstCode, + onWrite: func() { + t.deleteStream(s, eosReceived) + }, + } + t.controlBuf.put(hdr) +} + +// closeStream clears the footprint of a stream when the stream is not needed any more. +func (t *http2Server) closeStream(s *ServerStream, rst bool, rstCode http2.ErrCode, eosReceived bool) { + // We can't return early even if the stream's state is "done" as the state + // might have been set by the `finishStream` method. Deleting the stream via + // `finishStream` can get blocked on flow control. + s.swapState(streamDone) + + // In case stream sending and receiving are invoked in separate + // goroutines (e.g., bi-directional streaming), cancel needs to be + // called to interrupt the potential blocking on other goroutines. + s.cancel() + + t.deleteStream(s, eosReceived) + + t.controlBuf.put(&cleanupStream{ + streamID: s.id, + rst: rst, + rstCode: rstCode, + onWrite: func() {}, + }) +} + +func (t *http2Server) Drain(debugData string) { + t.mu.Lock() + defer t.mu.Unlock() + if t.drainEvent != nil { + return + } + t.drainEvent = grpcsync.NewEvent() + t.controlBuf.put(&goAway{code: http2.ErrCodeNo, debugData: []byte(debugData), headsUp: true}) +} + +var goAwayPing = &ping{data: [8]byte{1, 6, 1, 8, 0, 3, 3, 9}} + +// Handles outgoing GoAway and returns true if loopy needs to put itself +// in draining mode. +func (t *http2Server) outgoingGoAwayHandler(g *goAway) (bool, error) { + t.maxStreamMu.Lock() + t.mu.Lock() + if t.state == closing { // TODO(mmukhi): This seems unnecessary. + t.mu.Unlock() + t.maxStreamMu.Unlock() + // The transport is closing. + return false, ErrConnClosing + } + if !g.headsUp { + // Stop accepting more streams now. + t.state = draining + sid := t.maxStreamID + retErr := g.closeConn + if len(t.activeStreams) == 0 { + retErr = errors.New("second GOAWAY written and no active streams left to process") + } + t.mu.Unlock() + t.maxStreamMu.Unlock() + if err := t.framer.fr.WriteGoAway(sid, g.code, g.debugData); err != nil { + return false, err + } + t.framer.writer.Flush() + if retErr != nil { + return false, retErr + } + return true, nil + } + t.mu.Unlock() + t.maxStreamMu.Unlock() + // For a graceful close, send out a GoAway with stream ID of MaxUInt32, + // Follow that with a ping and wait for the ack to come back or a timer + // to expire. During this time accept new streams since they might have + // originated before the GoAway reaches the client. + // After getting the ack or timer expiration send out another GoAway this + // time with an ID of the max stream server intends to process. + if err := t.framer.fr.WriteGoAway(math.MaxUint32, http2.ErrCodeNo, g.debugData); err != nil { + return false, err + } + if err := t.framer.fr.WritePing(false, goAwayPing.data); err != nil { + return false, err + } + go func() { + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + select { + case <-t.drainEvent.Done(): + case <-timer.C: + case <-t.done: + return + } + t.controlBuf.put(&goAway{code: g.code, debugData: g.debugData}) + }() + return false, nil +} + +func (t *http2Server) socketMetrics() *channelz.EphemeralSocketMetrics { + return &channelz.EphemeralSocketMetrics{ + LocalFlowControlWindow: int64(t.fc.getSize()), + RemoteFlowControlWindow: t.getOutFlowWindow(), + } +} + +func (t *http2Server) incrMsgSent() { + if channelz.IsOn() { + t.channelz.SocketMetrics.MessagesSent.Add(1) + t.channelz.SocketMetrics.LastMessageSentTimestamp.Store(transportinternal.TimeNowFunc()) + } +} + +func (t *http2Server) incrMsgRecv() { + if channelz.IsOn() { + t.channelz.SocketMetrics.MessagesReceived.Add(1) + t.channelz.SocketMetrics.LastMessageReceivedTimestamp.Store(transportinternal.TimeNowFunc()) + } +} + +func (t *http2Server) getOutFlowWindow() int64 { + resp := make(chan uint32, 1) + timer := time.NewTimer(time.Second) + defer timer.Stop() + t.controlBuf.put(&outFlowControlSizeRequest{resp: resp}) + select { + case sz := <-resp: + return int64(sz) + case <-t.done: + return -1 + case <-timer.C: + return -2 + } +} + +// Peer returns the peer of the transport. +func (t *http2Server) Peer() *peer.Peer { + return &peer.Peer{ + Addr: t.peer.Addr, + LocalAddr: t.peer.LocalAddr, + AuthInfo: t.peer.AuthInfo, // Can be nil + } +} + +func getJitter(v time.Duration) time.Duration { + if v == infinity { + return 0 + } + // Generate a jitter between +/- 10% of the value. + r := int64(v / 10) + j := rand.Int64N(2*r) - r + return time.Duration(j) +} + +type connectionKey struct{} + +// GetConnection gets the connection from the context. +func GetConnection(ctx context.Context) net.Conn { + conn, _ := ctx.Value(connectionKey{}).(net.Conn) + return conn +} + +// SetConnection adds the connection to the context to be able to get +// information about the destination ip and port for an incoming RPC. This also +// allows any unary or streaming interceptors to see the connection. +func SetConnection(ctx context.Context, conn net.Conn) context.Context { + return context.WithValue(ctx, connectionKey{}, conn) +} diff --git a/internal/transport/http_util.go b/internal/transport/http_util.go new file mode 100644 index 0000000..faa42f8 --- /dev/null +++ b/internal/transport/http_util.go @@ -0,0 +1,643 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "bufio" + "encoding/base64" + "errors" + "fmt" + "io" + "math" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + "unicode/utf8" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/envconfig" + imem "google.golang.org/grpc/internal/mem" + "google.golang.org/grpc/internal/transport/readyreader" + "google.golang.org/grpc/mem" +) + +const ( + // http2MaxFrameLen specifies the max length of a HTTP2 frame. + http2MaxFrameLen = 16384 // 16KB frame + // https://httpwg.org/specs/rfc7540.html#SettingValues + http2InitHeaderTableSize = 4096 +) + +var ( + clientPreface = []byte(http2.ClientPreface) + http2ErrConvTab = map[http2.ErrCode]codes.Code{ + http2.ErrCodeNo: codes.Internal, + http2.ErrCodeProtocol: codes.Internal, + http2.ErrCodeInternal: codes.Internal, + http2.ErrCodeFlowControl: codes.ResourceExhausted, + http2.ErrCodeSettingsTimeout: codes.Internal, + http2.ErrCodeStreamClosed: codes.Internal, + http2.ErrCodeFrameSize: codes.Internal, + http2.ErrCodeRefusedStream: codes.Unavailable, + http2.ErrCodeCancel: codes.Canceled, + http2.ErrCodeCompression: codes.Internal, + http2.ErrCodeConnect: codes.Internal, + http2.ErrCodeEnhanceYourCalm: codes.ResourceExhausted, + http2.ErrCodeInadequateSecurity: codes.PermissionDenied, + http2.ErrCodeHTTP11Required: codes.Internal, + } + // HTTPStatusConvTab is the HTTP status code to gRPC error code conversion table. + HTTPStatusConvTab = map[int]codes.Code{ + // 400 Bad Request - INTERNAL. + http.StatusBadRequest: codes.Internal, + // 401 Unauthorized - UNAUTHENTICATED. + http.StatusUnauthorized: codes.Unauthenticated, + // 403 Forbidden - PERMISSION_DENIED. + http.StatusForbidden: codes.PermissionDenied, + // 404 Not Found - UNIMPLEMENTED. + http.StatusNotFound: codes.Unimplemented, + // 429 Too Many Requests - UNAVAILABLE. + http.StatusTooManyRequests: codes.Unavailable, + // 502 Bad Gateway - UNAVAILABLE. + http.StatusBadGateway: codes.Unavailable, + // 503 Service Unavailable - UNAVAILABLE. + http.StatusServiceUnavailable: codes.Unavailable, + // 504 Gateway timeout - UNAVAILABLE. + http.StatusGatewayTimeout: codes.Unavailable, + } +) + +var grpcStatusDetailsBinHeader = "grpc-status-details-bin" + +// isReservedHeader checks whether hdr belongs to HTTP2 headers +// reserved by gRPC protocol. Any other headers are classified as the +// user-specified metadata. +func isReservedHeader(hdr string) bool { + if hdr != "" && hdr[0] == ':' { + return true + } + switch hdr { + case "content-type", + "user-agent", + "grpc-message-type", + "grpc-encoding", + "grpc-message", + "grpc-status", + "grpc-timeout", + // Intentionally exclude grpc-previous-rpc-attempts and + // grpc-retry-pushback-ms, which are "reserved", but their API + // intentionally works via metadata. + "te": + return true + default: + return false + } +} + +// isWhitelistedHeader checks whether hdr should be propagated into metadata +// visible to users, even though it is classified as "reserved", above. +func isWhitelistedHeader(hdr string) bool { + switch hdr { + case ":authority", "user-agent": + return true + default: + return false + } +} + +const binHdrSuffix = "-bin" + +func encodeBinHeader(v []byte) string { + return base64.RawStdEncoding.EncodeToString(v) +} + +func decodeBinHeader(v string) ([]byte, error) { + if len(v)%4 == 0 { + // Input was padded, or padding was not necessary. + return base64.StdEncoding.DecodeString(v) + } + return base64.RawStdEncoding.DecodeString(v) +} + +func encodeMetadataHeader(k, v string) string { + if strings.HasSuffix(k, binHdrSuffix) { + return encodeBinHeader(([]byte)(v)) + } + return v +} + +func decodeMetadataHeader(k, v string) (string, error) { + if strings.HasSuffix(k, binHdrSuffix) { + b, err := decodeBinHeader(v) + return string(b), err + } + return v, nil +} + +type timeoutUnit uint8 + +const ( + hour timeoutUnit = 'H' + minute timeoutUnit = 'M' + second timeoutUnit = 'S' + millisecond timeoutUnit = 'm' + microsecond timeoutUnit = 'u' + nanosecond timeoutUnit = 'n' +) + +func timeoutUnitToDuration(u timeoutUnit) (d time.Duration, ok bool) { + switch u { + case hour: + return time.Hour, true + case minute: + return time.Minute, true + case second: + return time.Second, true + case millisecond: + return time.Millisecond, true + case microsecond: + return time.Microsecond, true + case nanosecond: + return time.Nanosecond, true + default: + } + return +} + +func decodeTimeout(s string) (time.Duration, error) { + size := len(s) + if size < 2 { + return 0, fmt.Errorf("transport: timeout string is too short: %q", s) + } + if size > 9 { + // Spec allows for 8 digits plus the unit. + return 0, fmt.Errorf("transport: timeout string is too long: %q", s) + } + unit := timeoutUnit(s[size-1]) + d, ok := timeoutUnitToDuration(unit) + if !ok { + return 0, fmt.Errorf("transport: timeout unit is not recognized: %q", s) + } + t, err := strconv.ParseUint(s[:size-1], 10, 64) + if err != nil { + return 0, err + } + const maxHours = math.MaxInt64 / uint64(time.Hour) + if d == time.Hour && t > maxHours { + // This timeout would overflow math.MaxInt64; clamp it. + return time.Duration(math.MaxInt64), nil + } + return d * time.Duration(t), nil +} + +const ( + spaceByte = ' ' + tildeByte = '~' + percentByte = '%' +) + +// encodeGrpcMessage is used to encode status code in header field +// "grpc-message". It does percent encoding and also replaces invalid utf-8 +// characters with Unicode replacement character. +// +// It checks to see if each individual byte in msg is an allowable byte, and +// then either percent encoding or passing it through. When percent encoding, +// the byte is converted into hexadecimal notation with a '%' prepended. +func encodeGrpcMessage(msg string) string { + if msg == "" { + return "" + } + lenMsg := len(msg) + for i := 0; i < lenMsg; i++ { + c := msg[i] + if !(c >= spaceByte && c <= tildeByte && c != percentByte) { + return encodeGrpcMessageUnchecked(msg) + } + } + return msg +} + +func encodeGrpcMessageUnchecked(msg string) string { + var sb strings.Builder + for len(msg) > 0 { + r, size := utf8.DecodeRuneInString(msg) + for _, b := range []byte(string(r)) { + if size > 1 { + // If size > 1, r is not ascii. Always do percent encoding. + fmt.Fprintf(&sb, "%%%02X", b) + continue + } + + // The for loop is necessary even if size == 1. r could be + // utf8.RuneError. + // + // fmt.Sprintf("%%%02X", utf8.RuneError) gives "%FFFD". + if b >= spaceByte && b <= tildeByte && b != percentByte { + sb.WriteByte(b) + } else { + fmt.Fprintf(&sb, "%%%02X", b) + } + } + msg = msg[size:] + } + return sb.String() +} + +// decodeGrpcMessage decodes the msg encoded by encodeGrpcMessage. +func decodeGrpcMessage(msg string) string { + if msg == "" { + return "" + } + lenMsg := len(msg) + for i := 0; i < lenMsg; i++ { + if msg[i] == percentByte && i+2 < lenMsg { + return decodeGrpcMessageUnchecked(msg) + } + } + return msg +} + +func decodeGrpcMessageUnchecked(msg string) string { + var sb strings.Builder + lenMsg := len(msg) + for i := 0; i < lenMsg; i++ { + c := msg[i] + if c == percentByte && i+2 < lenMsg { + parsed, err := strconv.ParseUint(msg[i+1:i+3], 16, 8) + if err != nil { + sb.WriteByte(c) + } else { + sb.WriteByte(byte(parsed)) + i += 2 + } + } else { + sb.WriteByte(c) + } + } + return sb.String() +} + +type bufWriter struct { + pool *imem.SimpleBufferPool + bufHandle *[]byte + offset int + batchSize int + conn io.Writer + err error +} + +// unsharedBufWriter keeps the unshared slice header in the writer allocation. +type unsharedBufWriter struct { + bufWriter + buf []byte +} + +func newBufWriter(conn io.Writer, batchSize int, pool *imem.SimpleBufferPool) *bufWriter { + if pool == nil && batchSize > 0 { + w := &unsharedBufWriter{ + bufWriter: bufWriter{batchSize: batchSize, conn: conn}, + buf: make([]byte, batchSize), + } + w.bufHandle = &w.buf + return &w.bufWriter + } + return &bufWriter{batchSize: batchSize, conn: conn, pool: pool} +} + +func (w *bufWriter) Write(b []byte) (int, error) { + if w.err != nil { + return 0, w.err + } + if w.batchSize == 0 { // Buffer has been disabled. + n, err := w.conn.Write(b) + return n, toIOError(err) + } + if w.bufHandle == nil { + w.bufHandle = w.pool.Get(w.batchSize) + } + buf := *w.bufHandle + written := 0 + for len(b) > 0 { + copied := copy(buf[w.offset:], b) + b = b[copied:] + written += copied + w.offset += copied + if w.offset < w.batchSize { + continue + } + if err := w.flushKeepBuffer(); err != nil { + return written, err + } + } + return written, nil +} + +func (w *bufWriter) Flush() error { + err := w.flushKeepBuffer() + w.releaseBuffer() + return err +} + +func (w *bufWriter) releaseBuffer() { + if w.pool != nil && w.bufHandle != nil { + w.pool.Put(w.bufHandle) + w.bufHandle = nil + } +} + +func (w *bufWriter) flushKeepBuffer() error { + if w.err != nil { + return w.err + } + if w.offset == 0 { + return nil + } + buf := *w.bufHandle + _, w.err = w.conn.Write(buf[:w.offset]) + w.err = toIOError(w.err) + w.offset = 0 + if w.err != nil { + w.releaseBuffer() + } + return w.err +} + +type ioError struct { + error +} + +func (i ioError) Unwrap() error { + return i.error +} + +func isIOError(err error) bool { + return errors.As(err, &ioError{}) +} + +func toIOError(err error) error { + if err == nil { + return nil + } + return ioError{error: err} +} + +type parsedDataFrame struct { + http2.FrameHeader + data mem.Buffer +} + +func (df *parsedDataFrame) StreamEnded() bool { + return df.FrameHeader.Flags.Has(http2.FlagDataEndStream) +} + +type framer struct { + writer *bufWriter + fr *http2.Framer + headerBuf []byte // cached slice for framer headers to reduce heap allocs. + reader io.Reader + dataFrame parsedDataFrame // Cached data frame to avoid heap allocations. + pool mem.BufferPool + errDetail error +} + +var ioBufferPoolMap = make(map[int]*imem.SimpleBufferPool) +var ioBufferMutex sync.Mutex + +func bufferedReader(r io.Reader, bufSize int) io.Reader { + if bufSize <= 0 { + return r + } + if envconfig.EnableHTTPFramerReadBufferPooling { + if rr := readyreader.NewNonBlocking(r); rr != nil { + readPool := ioBufferPool(bufSize) + return readyreader.NewBuffered(rr, bufSize, readPool) + } + } + return bufio.NewReaderSize(r, bufSize) +} + +func newFramer(conn io.ReadWriter, writeBufferSize, readBufferSize int, sharedWriteBuffer bool, maxHeaderListSize uint32, memPool mem.BufferPool) *framer { + if writeBufferSize < 0 { + writeBufferSize = 0 + } + r := bufferedReader(conn, readBufferSize) + var writePool *imem.SimpleBufferPool + if sharedWriteBuffer { + writePool = ioBufferPool(writeBufferSize) + } + w := newBufWriter(conn, writeBufferSize, writePool) + f := &framer{ + writer: w, + fr: http2.NewFramer(w, r), + reader: r, + pool: memPool, + } + f.fr.SetMaxReadFrameSize(http2MaxFrameLen) + // Opt-in to Frame reuse API on framer to reduce garbage. + // Frames aren't safe to read from after a subsequent call to ReadFrame. + f.fr.SetReuseFrames() + f.fr.MaxHeaderListSize = maxHeaderListSize + f.fr.ReadMetaHeaders = hpack.NewDecoder(http2InitHeaderTableSize, nil) + return f +} + +// writeData writes a DATA frame. +// +// It is the caller's responsibility not to violate the maximum frame size. +func (f *framer) writeData(streamID uint32, endStream bool, data [][]byte) error { + var flags http2.Flags + if endStream { + flags = http2.FlagDataEndStream + } + length := uint32(0) + for _, d := range data { + length += uint32(len(d)) + } + // TODO: Replace the header write with the framer API being added in + // https://github.com/golang/go/issues/66655. + f.headerBuf = append(f.headerBuf[:0], + byte(length>>16), + byte(length>>8), + byte(length), + byte(http2.FrameData), + byte(flags), + byte(streamID>>24), + byte(streamID>>16), + byte(streamID>>8), + byte(streamID)) + if _, err := f.writer.Write(f.headerBuf); err != nil { + return err + } + for _, d := range data { + if _, err := f.writer.Write(d); err != nil { + return err + } + } + return nil +} + +// readFrame reads a single frame. The returned Frame is only valid +// until the next call to readFrame. +func (f *framer) readFrame() (any, error) { + f.errDetail = nil + fh, err := f.fr.ReadFrameHeader() + if err != nil { + f.errDetail = f.fr.ErrorDetail() + return nil, err + } + // Read the data frame directly from the underlying io.Reader to avoid + // copies. + if fh.Type == http2.FrameData { + err = f.readDataFrame(fh) + return &f.dataFrame, err + } + fr, err := f.fr.ReadFrameForHeader(fh) + if err != nil { + f.errDetail = f.fr.ErrorDetail() + return nil, err + } + return fr, err +} + +// errorDetail returns a more detailed error of the last error +// returned by framer.readFrame. For instance, if readFrame +// returns a StreamError with code PROTOCOL_ERROR, errorDetail +// will say exactly what was invalid. errorDetail is not guaranteed +// to return a non-nil value. +// errorDetail is reset after the next call to readFrame. +func (f *framer) errorDetail() error { + return f.errDetail +} + +func (f *framer) readDataFrame(fh http2.FrameHeader) (err error) { + if fh.StreamID == 0 { + // DATA frames MUST be associated with a stream. If a + // DATA frame is received whose stream identifier + // field is 0x0, the recipient MUST respond with a + // connection error (Section 5.4.1) of type + // PROTOCOL_ERROR. + f.errDetail = errors.New("DATA frame with stream ID 0") + return http2.ConnectionError(http2.ErrCodeProtocol) + } + // Converting a *[]byte to a mem.SliceBuffer incurs a heap allocation. This + // conversion is performed by mem.NewBuffer. To avoid the extra allocation + // a []byte is allocated directly if required and cast to a mem.SliceBuffer. + var buf []byte + // poolHandle is the pointer returned by the buffer pool (if it's used.). + var poolHandle *[]byte + useBufferPool := !mem.IsBelowBufferPoolingThreshold(int(fh.Length)) + if useBufferPool { + poolHandle = f.pool.Get(int(fh.Length)) + buf = *poolHandle + defer func() { + if err != nil { + f.pool.Put(poolHandle) + } + }() + } else { + buf = make([]byte, int(fh.Length)) + } + if fh.Flags.Has(http2.FlagDataPadded) { + if fh.Length == 0 { + return io.ErrUnexpectedEOF + } + // This initial 1-byte read can be inefficient for unbuffered readers, + // but it allows the rest of the payload to be read directly to the + // start of the destination slice. This makes it easy to return the + // original slice back to the buffer pool. + if _, err := io.ReadFull(f.reader, buf[:1]); err != nil { + return err + } + padSize := buf[0] + buf = buf[:len(buf)-1] + if int(padSize) > len(buf) { + // If the length of the padding is greater than the + // length of the frame payload, the recipient MUST + // treat this as a connection error. + // Filed: https://github.com/http2/http2-spec/issues/610 + f.errDetail = errors.New("pad size larger than data payload") + return http2.ConnectionError(http2.ErrCodeProtocol) + } + if _, err := io.ReadFull(f.reader, buf); err != nil { + return err + } + buf = buf[:len(buf)-int(padSize)] + } else if _, err := io.ReadFull(f.reader, buf); err != nil { + return err + } + + f.dataFrame.FrameHeader = fh + if useBufferPool { + // Update the handle to point to the (potentially re-sliced) buf. + *poolHandle = buf + f.dataFrame.data = mem.NewBuffer(poolHandle, f.pool) + } else { + f.dataFrame.data = mem.SliceBuffer(buf) + } + return nil +} + +func (df *parsedDataFrame) Header() http2.FrameHeader { + return df.FrameHeader +} + +func ioBufferPool(size int) *imem.SimpleBufferPool { + ioBufferMutex.Lock() + defer ioBufferMutex.Unlock() + pool, ok := ioBufferPoolMap[size] + if ok { + return pool + } + pool = imem.NewDirtySimplePool() + ioBufferPoolMap[size] = pool + return pool +} + +// ParseDialTarget returns the network and address to pass to dialer. +func ParseDialTarget(target string) (string, string) { + net := "tcp" + m1 := strings.Index(target, ":") + m2 := strings.Index(target, ":/") + // handle unix:addr which will fail with url.Parse + if m1 >= 0 && m2 < 0 { + if n := target[0:m1]; n == "unix" { + return n, target[m1+1:] + } + } + if m2 >= 0 { + t, err := url.Parse(target) + if err != nil { + return net, target + } + scheme := t.Scheme + addr := t.Path + if scheme == "unix" { + if addr == "" { + addr = t.Host + } + return scheme, addr + } + } + return net, target +} diff --git a/internal/transport/http_util_test.go b/internal/transport/http_util_test.go new file mode 100644 index 0000000..0fbf7ba --- /dev/null +++ b/internal/transport/http_util_test.go @@ -0,0 +1,637 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "math" + "net" + "reflect" + "strings" + "testing" + "time" + + "golang.org/x/net/http2" + "google.golang.org/grpc/internal/envconfig" + imem "google.golang.org/grpc/internal/mem" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/transport/readyreader" + "google.golang.org/grpc/mem" +) + +func (s) TestDecodeTimeout(t *testing.T) { + for _, test := range []struct { + // input + s string + // output + d time.Duration + wantErr bool + }{ + + {"00000001n", time.Nanosecond, false}, + {"10u", time.Microsecond * 10, false}, + {"00000010m", time.Millisecond * 10, false}, + {"1234S", time.Second * 1234, false}, + {"00000001M", time.Minute, false}, + {"09999999S", time.Second * 9999999, false}, + {"99999999S", time.Second * 99999999, false}, + {"99999999M", time.Minute * 99999999, false}, + {"2562047H", time.Hour * 2562047, false}, + {"2562048H", time.Duration(math.MaxInt64), false}, + {"99999999H", time.Duration(math.MaxInt64), false}, + {"-1S", 0, true}, + {"1234x", 0, true}, + {"1234s", 0, true}, + {"1234", 0, true}, + {"1", 0, true}, + {"", 0, true}, + {"9a1S", 0, true}, + {"0S", 0, false}, // PROTOCOL-HTTP2.md requires positive integers, but we allow it to timeout instead + {"00000000S", 0, false}, + {"000000000S", 0, true}, // PROTOCOL-HTTP2.md allows at most 8 digits + } { + d, err := decodeTimeout(test.s) + gotErr := err != nil + if d != test.d || gotErr != test.wantErr { + t.Errorf("timeoutDecode(%q) = %d, %v, want %d, wantErr=%v", + test.s, int64(d), err, int64(test.d), test.wantErr) + } + } +} + +func (s) TestEncodeGrpcMessage(t *testing.T) { + for _, tt := range []struct { + input string + expected string + }{ + {"", ""}, + {"Hello", "Hello"}, + {"\u0000", "%00"}, + {"%", "%25"}, + {"系统", "%E7%B3%BB%E7%BB%9F"}, + {string([]byte{0xff, 0xfe, 0xfd}), "%EF%BF%BD%EF%BF%BD%EF%BF%BD"}, + } { + actual := encodeGrpcMessage(tt.input) + if tt.expected != actual { + t.Errorf("encodeGrpcMessage(%q) = %q, want %q", tt.input, actual, tt.expected) + } + } + + // make sure that all the visible ASCII chars except '%' are not percent encoded. + for i := ' '; i <= '~' && i != '%'; i++ { + output := encodeGrpcMessage(string(i)) + if output != string(i) { + t.Errorf("encodeGrpcMessage(%v) = %v, want %v", string(i), output, string(i)) + } + } + + // make sure that all the invisible ASCII chars and '%' are percent encoded. + for i := rune(0); i == '%' || (i >= rune(0) && i < ' ') || (i > '~' && i <= rune(127)); i++ { + output := encodeGrpcMessage(string(i)) + expected := fmt.Sprintf("%%%02X", i) + if output != expected { + t.Errorf("encodeGrpcMessage(%v) = %v, want %v", string(i), output, expected) + } + } +} + +func (s) TestDecodeGrpcMessage(t *testing.T) { + for _, tt := range []struct { + input string + expected string + }{ + {"", ""}, + {"Hello", "Hello"}, + {"H%61o", "Hao"}, + {"H%6", "H%6"}, + {"%G0", "%G0"}, + {"%E7%B3%BB%E7%BB%9F", "系统"}, + {"%EF%BF%BD", "�"}, + } { + actual := decodeGrpcMessage(tt.input) + if tt.expected != actual { + t.Errorf("decodeGrpcMessage(%q) = %q, want %q", tt.input, actual, tt.expected) + } + } + + // make sure that all the visible ASCII chars except '%' are not percent decoded. + for i := ' '; i <= '~' && i != '%'; i++ { + output := decodeGrpcMessage(string(i)) + if output != string(i) { + t.Errorf("decodeGrpcMessage(%v) = %v, want %v", string(i), output, string(i)) + } + } + + // make sure that all the invisible ASCII chars and '%' are percent decoded. + for i := rune(0); i == '%' || (i >= rune(0) && i < ' ') || (i > '~' && i <= rune(127)); i++ { + output := decodeGrpcMessage(fmt.Sprintf("%%%02X", i)) + if output != string(i) { + t.Errorf("decodeGrpcMessage(%v) = %v, want %v", fmt.Sprintf("%%%02X", i), output, string(i)) + } + } +} + +// Decode an encoded string should get the same thing back, except for invalid +// utf8 chars. +func (s) TestDecodeEncodeGrpcMessage(t *testing.T) { + testCases := []struct { + orig string + want string + }{ + {"", ""}, + {"hello", "hello"}, + {"h%6", "h%6"}, + {"%G0", "%G0"}, + {"系统", "系统"}, + {"Hello, 世界", "Hello, 世界"}, + + {string([]byte{0xff, 0xfe, 0xfd}), "���"}, + {string([]byte{0xff}) + "Hello" + string([]byte{0xfe}) + "世界" + string([]byte{0xfd}), "�Hello�世界�"}, + } + for _, tC := range testCases { + got := decodeGrpcMessage(encodeGrpcMessage(tC.orig)) + if got != tC.want { + t.Errorf("decodeGrpcMessage(encodeGrpcMessage(%q)) = %q, want %q", tC.orig, got, tC.want) + } + } +} + +const binaryValue = "\u0080" + +func (s) TestEncodeMetadataHeader(t *testing.T) { + for _, test := range []struct { + // input + kin string + vin string + // output + vout string + }{ + {"key", "abc", "abc"}, + {"KEY", "abc", "abc"}, + {"key-bin", "abc", "YWJj"}, + {"key-bin", binaryValue, "woA"}, + } { + v := encodeMetadataHeader(test.kin, test.vin) + if !reflect.DeepEqual(v, test.vout) { + t.Fatalf("encodeMetadataHeader(%q, %q) = %q, want %q", test.kin, test.vin, v, test.vout) + } + } +} + +func (s) TestDecodeMetadataHeader(t *testing.T) { + for _, test := range []struct { + // input + kin string + vin string + // output + vout string + err error + }{ + {"a", "abc", "abc", nil}, + {"key-bin", "Zm9vAGJhcg==", "foo\x00bar", nil}, + {"key-bin", "Zm9vAGJhcg", "foo\x00bar", nil}, + {"key-bin", "woA=", binaryValue, nil}, + {"a", "abc,efg", "abc,efg", nil}, + } { + v, err := decodeMetadataHeader(test.kin, test.vin) + if !reflect.DeepEqual(v, test.vout) || !reflect.DeepEqual(err, test.err) { + t.Fatalf("decodeMetadataHeader(%q, %q) = %q, %v, want %q, %v", test.kin, test.vin, v, err, test.vout, test.err) + } + } +} + +func (s) TestParseDialTarget(t *testing.T) { + for _, test := range []struct { + target, wantNet, wantAddr string + }{ + {"unix:a", "unix", "a"}, + {"unix:a/b/c", "unix", "a/b/c"}, + {"unix:/a", "unix", "/a"}, + {"unix:/a/b/c", "unix", "/a/b/c"}, + {"unix://a", "unix", "a"}, + {"unix://a/b/c", "unix", "/b/c"}, + {"unix:///a", "unix", "/a"}, + {"unix:///a/b/c", "unix", "/a/b/c"}, + {"unix:etcd:0", "unix", "etcd:0"}, + {"unix:///tmp/unix-3", "unix", "/tmp/unix-3"}, + {"unix://domain", "unix", "domain"}, + {"unix://etcd:0", "unix", "etcd:0"}, + {"unix:///etcd:0", "unix", "/etcd:0"}, + {"passthrough://unix://domain", "tcp", "passthrough://unix://domain"}, + {"https://google.com:443", "tcp", "https://google.com:443"}, + {"dns:///google.com", "tcp", "dns:///google.com"}, + {"/unix/socket/address", "tcp", "/unix/socket/address"}, + } { + gotNet, gotAddr := ParseDialTarget(test.target) + if gotNet != test.wantNet || gotAddr != test.wantAddr { + t.Errorf("ParseDialTarget(%q) = %s, %s want %s, %s", test.target, gotNet, gotAddr, test.wantNet, test.wantAddr) + } + } +} + +type badNetworkConn struct { + net.Conn +} + +func (c *badNetworkConn) Write([]byte) (int, error) { + return 0, io.EOF +} + +// This test ensures Write() on a broken network connection does not lead to +// an infinite loop. See https://github.com/grpc/grpc-go/issues/7389 for more details. +func (s) TestWriteBadConnection(t *testing.T) { + data := []byte("test_data") + // Configure the bufWriter with a batchsize that results in data being flushed + // to the underlying conn, midway through Write(). + writeBufferSize := (len(data) - 1) / 2 + writer := newBufWriter(&badNetworkConn{}, writeBufferSize, ioBufferPool(writeBufferSize)) + + errCh := make(chan error, 1) + go func() { + _, err := writer.Write(data) + errCh <- err + }() + + select { + case <-time.After(time.Second): + t.Fatalf("Write() did not return in time") + case err := <-errCh: + if !errors.Is(err, io.EOF) { + t.Fatalf("Write() = %v, want error presence = %v", err, io.EOF) + } + } + if writer.bufHandle != nil { + t.Fatalf("failed Write() retained pooled buffer: handle=%p", writer.bufHandle) + } +} + +func (s) TestBufWriterSharedFlushAllocations(t *testing.T) { + pool := imem.NewDirtySimplePool() + writer := newBufWriter(io.Discard, defaultWriteBufSize, pool) + payload := make([]byte, 128) + + allocs := testing.AllocsPerRun(1000, func() { + if _, err := writer.Write(payload); err != nil { + t.Fatalf("Write() failed: %v", err) + } + if err := writer.Flush(); err != nil { + t.Fatalf("Flush() failed: %v", err) + } + }) + if allocs != 0 { + t.Fatalf("shared Write()+Flush() allocated %v times, want 0", allocs) + } +} + +func (s) TestBufWriterSharedBufferOwnership(t *testing.T) { + const batchSize = 8 + pool := imem.NewDirtySimplePool() + var conn bytes.Buffer + writer := newBufWriter(&conn, batchSize, pool) + + if _, err := writer.Write([]byte("abc")); err != nil { + t.Fatalf("Write() failed: %v", err) + } + if writer.bufHandle == nil { + t.Fatal("Write() did not retain the pooled buffer handle") + } + if err := writer.Flush(); err != nil { + t.Fatalf("Flush() failed: %v", err) + } + if writer.bufHandle != nil { + t.Fatalf("Flush() retained pooled buffer: handle=%p", writer.bufHandle) + } + if got := conn.String(); got != "abc" { + t.Fatalf("conn data = %q, want %q", got, "abc") + } + if err := writer.Flush(); err != nil { + t.Fatalf("second Flush() failed: %v", err) + } + + if _, err := writer.Write([]byte("12345678")); err != nil { + t.Fatalf("full-buffer Write() failed: %v", err) + } + if writer.bufHandle == nil { + t.Fatal("full-buffer Write() released the handle during its internal flush") + } + if writer.offset != 0 { + t.Fatalf("full-buffer Write() offset = %d, want 0", writer.offset) + } + if err := writer.Flush(); err != nil { + t.Fatalf("Flush() after full-buffer Write() failed: %v", err) + } + if writer.bufHandle != nil { + t.Fatalf("Flush() retained pooled buffer: handle=%p", writer.bufHandle) + } +} + +func (s) TestBufWriterFlushErrorReleasesSharedBuffer(t *testing.T) { + pool := imem.NewDirtySimplePool() + writer := newBufWriter(&badNetworkConn{}, 8, pool) + + if _, err := writer.Write([]byte("abc")); err != nil { + t.Fatalf("Write() failed before flush: %v", err) + } + if writer.bufHandle == nil { + t.Fatal("Write() did not retain the pooled buffer handle") + } + if err := writer.Flush(); !errors.Is(err, io.EOF) { + t.Fatalf("Flush() error = %v, want %v", err, io.EOF) + } + if writer.bufHandle != nil { + t.Fatalf("failed Flush() retained pooled buffer: handle=%p", writer.bufHandle) + } + if _, err := writer.Write([]byte("def")); !errors.Is(err, io.EOF) { + t.Fatalf("Write() after failed Flush() error = %v, want %v", err, io.EOF) + } + if writer.bufHandle != nil { + t.Fatal("Write() after failed Flush() acquired another pooled buffer") + } +} + +func BenchmarkBufWriter(b *testing.B) { + const payloadSize = 128 + payload := make([]byte, payloadSize) + + for _, test := range []struct { + name string + pool *imem.SimpleBufferPool + }{ + {name: "Shared", pool: imem.NewDirtySimplePool()}, + {name: "Unshared"}, + } { + b.Run(test.name, func(b *testing.B) { + b.Run("New", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + writer := newBufWriter(io.Discard, defaultWriteBufSize, test.pool) + _ = writer + } + }) + + b.Run("WriteAndFlush", func(b *testing.B) { + writer := newBufWriter(io.Discard, defaultWriteBufSize, test.pool) + b.ReportAllocs() + b.SetBytes(payloadSize) + for b.Loop() { + if _, err := writer.Write(payload); err != nil { + b.Fatal(err) + } + if err := writer.Flush(); err != nil { + b.Fatal(err) + } + } + }) + + b.Run("EmptyFlush", func(b *testing.B) { + writer := newBufWriter(io.Discard, defaultWriteBufSize, test.pool) + b.ReportAllocs() + for b.Loop() { + if err := writer.Flush(); err != nil { + b.Fatal(err) + } + } + }) + + b.Run("FullBufferWrite", func(b *testing.B) { + writer := newBufWriter(io.Discard, defaultWriteBufSize, test.pool) + fullBuffer := make([]byte, defaultWriteBufSize) + b.ReportAllocs() + b.SetBytes(defaultWriteBufSize) + for b.Loop() { + if _, err := writer.Write(fullBuffer); err != nil { + b.Fatal(err) + } + } + }) + }) + } +} + +func BenchmarkDecodeGrpcMessage(b *testing.B) { + input := "Hello, %E4%B8%96%E7%95%8C" + want := "Hello, 世界" + b.ReportAllocs() + for i := 0; i < b.N; i++ { + got := decodeGrpcMessage(input) + if got != want { + b.Fatalf("decodeGrpcMessage(%q) = %s, want %s", input, got, want) + } + } +} + +func BenchmarkEncodeGrpcMessage(b *testing.B) { + input := "Hello, 世界" + want := "Hello, %E4%B8%96%E7%95%8C" + b.ReportAllocs() + for i := 0; i < b.N; i++ { + got := encodeGrpcMessage(input) + if got != want { + b.Fatalf("encodeGrpcMessage(%q) = %s, want %s", input, got, want) + } + } +} + +func buildDataFrame(h http2.FrameHeader, payload []byte) []byte { + buf := new(bytes.Buffer) + buf.Write([]byte{ + byte(h.Length >> 16), + byte(h.Length >> 8), + byte(h.Length), + byte(h.Type), + byte(h.Flags), + byte(h.StreamID >> 24), + byte(h.StreamID >> 16), + byte(h.StreamID >> 8), + byte(h.StreamID), + }) + buf.Write(payload) + return buf.Bytes() +} + +func (s) TestFramer_ParseDataFrame(t *testing.T) { + tests := []struct { + name string + wire []byte // from frame header onward + wantData []byte + wantErr error + wantErrDetailSubstr string + }{ + { + name: "good_padded", + wire: buildDataFrame(http2.FrameHeader{ + Type: http2.FrameData, Length: 6, StreamID: 1, Flags: http2.FlagDataPadded, + }, []byte{ + 2, // pad length + 'f', 'o', 'o', // data + 0, 0, // padding + }), + wantData: []byte("foo"), + }, + { + name: "good_unpadded", + wire: buildDataFrame(http2.FrameHeader{ + Type: http2.FrameData, Length: 3, StreamID: 1, Flags: 0, + }, []byte("foo")), + wantData: []byte("foo"), + }, + { + name: "stream_id_0", + wire: buildDataFrame(http2.FrameHeader{ + Type: http2.FrameData, Length: 1, StreamID: 0, Flags: 0, + }, []byte{0}), + wantErr: http2.ConnectionError(http2.ErrCodeProtocol), + wantErrDetailSubstr: "DATA frame with stream ID 0", + }, + { + name: "pad_size_bigger_than_payload", + wire: buildDataFrame(http2.FrameHeader{ + Type: http2.FrameData, Length: 4, StreamID: 1, Flags: http2.FlagDataPadded, + }, []byte{ + 4, // pad length of 4 + 'f', 'o', // data 'fo' is 2 bytes + 0, // padding 0 is 1 byte. + }), // pad length 4 but only 3 bytes for data+padding available in payload after pad length byte + wantErr: http2.ConnectionError(http2.ErrCodeProtocol), + wantErrDetailSubstr: "pad size larger than data payload", + }, + { + name: "padded_zero_data_some_padding", + wire: buildDataFrame(http2.FrameHeader{ + Type: http2.FrameData, Length: 3, StreamID: 1, Flags: http2.FlagDataPadded, + }, []byte{ + 2, // pad length 2 + 0, 0, // padding + }), + wantData: []byte{}, + }, + { + name: "padded_short_payload_reading_pad_flag", + wire: buildDataFrame(http2.FrameHeader{ + Type: http2.FrameData, Length: 0, StreamID: 1, Flags: http2.FlagDataPadded, + }, []byte{}), + wantErr: io.ErrUnexpectedEOF, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fr := newFramer(bytes.NewBuffer(tc.wire), defaultWriteBufSize, defaultReadBufSize, false, defaultClientMaxHeaderListSize, mem.DefaultBufferPool()) + f, err := fr.readFrame() + + if err != tc.wantErr { + t.Fatalf("readFrame() returned unexpected error: %v, want %v", err, tc.wantErr) + } + gotErrDetailStr := "" + if fr.errDetail != nil { + gotErrDetailStr = fr.errDetail.Error() + } + if !strings.Contains(gotErrDetailStr, tc.wantErrDetailSubstr) { + t.Fatalf("errorDetail() returned unexpected error string: %q, want substring %q", gotErrDetailStr, tc.wantErrDetailSubstr) + } + + if tc.wantErr != nil { + return + } + df, ok := f.(*parsedDataFrame) + if !ok { + t.Fatalf("readFrame() returned %T, want *parsedDataFrame", f) + } + if gotData := df.data.ReadOnlyData(); !bytes.Equal(gotData, tc.wantData) { + t.Fatalf("parsedDataFrame.Data() = %q, want %q", gotData, tc.wantData) + } + df.data.Free() + }) + } +} + +type testReadyReader struct { + readyreader.Reader +} + +func (t *testReadyReader) Read([]byte) (int, error) { + return 0, io.EOF +} + +func (s) TestBufferedReader(t *testing.T) { + normalReader := bytes.NewReader(nil) + + tests := []struct { + name string + reader io.Reader + bufSize int + enablePooling bool + wantTypeOf any + }{ + { + name: "bufSize_0", + reader: normalReader, + bufSize: 0, + enablePooling: true, + wantTypeOf: (*bytes.Reader)(nil), + }, + { + name: "env_var_disabled_normal_reader", + reader: normalReader, + bufSize: 10, + enablePooling: false, + wantTypeOf: (*bufio.Reader)(nil), + }, + { + name: "env_var_disabled_ready_reader", + reader: &testReadyReader{}, + bufSize: 10, + enablePooling: false, + wantTypeOf: (*bufio.Reader)(nil), + }, + { + name: "env_var_enabled_normal_reader", + reader: normalReader, + bufSize: 10, + enablePooling: true, + wantTypeOf: (*bufio.Reader)(nil), + }, + { + name: "env_var_enabled_ready_reader", + reader: &testReadyReader{}, + bufSize: 10, + enablePooling: true, + wantTypeOf: readyreader.NewBuffered(nil, 10, mem.DefaultBufferPool()), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnableHTTPFramerReadBufferPooling, tt.enablePooling) + + got := bufferedReader(tt.reader, tt.bufSize) + + gotType := reflect.TypeOf(got) + wantType := reflect.TypeOf(tt.wantTypeOf) + + if gotType != wantType { + t.Errorf("bufferedReader() type = %v, want %v", gotType, wantType) + } + }) + } +} diff --git a/internal/transport/internal/internal.go b/internal/transport/internal/internal.go new file mode 100644 index 0000000..a7c7c7d --- /dev/null +++ b/internal/transport/internal/internal.go @@ -0,0 +1,25 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains functionality internal to the transport package. +package internal + +// TimeNowFunc is a variable that can be set to override the default behavior of +// getting the current time in nanoseconds. It is used in transport code to set +// channelz timestamps, and is exposed here for testing purposes. +var TimeNowFunc func() int64 diff --git a/internal/transport/keepalive_test.go b/internal/transport/keepalive_test.go new file mode 100644 index 0000000..85582a6 --- /dev/null +++ b/internal/transport/keepalive_test.go @@ -0,0 +1,860 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// This file contains tests related to the following proposals: +// https://github.com/grpc/proposal/blob/master/A8-client-side-keepalive.md +// https://github.com/grpc/proposal/blob/master/A9-server-side-conn-mgt.md +// https://github.com/grpc/proposal/blob/master/A18-tcp-user-timeout.md +package transport + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net" + "os" + "strings" + "testing" + "time" + + "golang.org/x/net/http2" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/syscall" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/testdata" +) + +const defaultTestTimeout = 10 * time.Second +const defaultTestShortTimeout = 10 * time.Millisecond + +// TestMaxConnectionIdle tests that a server will send GoAway to an idle +// client. An idle client is one who doesn't make any RPC calls for a duration +// of MaxConnectionIdle time. +func (s) TestMaxConnectionIdle(t *testing.T) { + serverConfig := &ServerConfig{ + BufferPool: mem.DefaultBufferPool(), + KeepaliveParams: keepalive.ServerParameters{ + MaxConnectionIdle: 30 * time.Millisecond, + }, + } + copts := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + } + server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, copts) + defer func() { + client.Close(fmt.Errorf("closed manually by test")) + server.stop() + cancel() + }() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := client.NewStream(ctx, &CallHdr{}, nil) + if err != nil { + t.Fatalf("client.NewStream() failed: %v", err) + } + stream.Close(io.EOF) + + // Verify the server sends a GoAway to client after MaxConnectionIdle timeout + // kicks in. + select { + case <-ctx.Done(): + t.Fatalf("context expired before receiving GoAway from the server.") + case <-client.GoAway(): + reason, debugMsg := client.GetGoAwayReason() + if reason != GoAwayNoReason { + t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayNoReason) + } + if !strings.Contains(debugMsg, "max_idle") { + t.Fatalf("GoAwayDebugMessage is %v, want %v", debugMsg, "max_idle") + } + } +} + +// TestMaxConnectionIdleBusyClient tests that a server will not send GoAway to +// a busy client. +func (s) TestMaxConnectionIdleBusyClient(t *testing.T) { + serverConfig := &ServerConfig{ + KeepaliveParams: keepalive.ServerParameters{ + MaxConnectionIdle: 100 * time.Millisecond, + }, + } + copts := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + } + server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, copts) + defer func() { + client.Close(fmt.Errorf("closed manually by test")) + server.stop() + cancel() + }() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := client.NewStream(ctx, &CallHdr{}, nil) + if err != nil { + t.Fatalf("client.NewStream() failed: %v", err) + } + + // Verify the server does not send a GoAway to client even after MaxConnectionIdle + // timeout kicks in. + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + defer cancel() + select { + case <-client.GoAway(): + t.Fatalf("A busy client received a GoAway.") + case <-ctx.Done(): + } +} + +// TestMaxConnectionAge tests that a server will send GoAway after a duration +// of MaxConnectionAge. +func (s) TestMaxConnectionAge(t *testing.T) { + maxConnAge := 100 * time.Millisecond + serverConfig := &ServerConfig{ + BufferPool: mem.DefaultBufferPool(), + KeepaliveParams: keepalive.ServerParameters{ + MaxConnectionAge: maxConnAge, + MaxConnectionAgeGrace: 10 * time.Millisecond, + }, + } + copts := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + } + server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, copts) + defer func() { + client.Close(fmt.Errorf("closed manually by test")) + server.stop() + cancel() + }() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := client.NewStream(ctx, &CallHdr{}, nil); err != nil { + t.Fatalf("client.NewStream() failed: %v", err) + } + + // Verify the server sends a GoAway to client even after client remains idle + // for more than MaxConnectionIdle time. + select { + case <-client.GoAway(): + reason, debugMsg := client.GetGoAwayReason() + if reason != GoAwayNoReason { + t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayNoReason) + } + if !strings.Contains(debugMsg, "max_age") { + t.Fatalf("GoAwayDebugMessage is %v, want %v", debugMsg, "max_age") + } + case <-ctx.Done(): + t.Fatalf("timed out before getting a GoAway from the server.") + } +} + +const ( + defaultWriteBufSize = 32 * 1024 + defaultReadBufSize = 32 * 1024 +) + +// TestKeepaliveServerClosesUnresponsiveClient tests that a server closes +// the connection with a client that doesn't respond to keepalive pings. +// +// This test creates a regular net.Conn connection to the server and sends the +// clientPreface and the initial Settings frame, and then remains unresponsive. +func (s) TestKeepaliveServerClosesUnresponsiveClient(t *testing.T) { + serverConfig := &ServerConfig{ + BufferPool: mem.DefaultBufferPool(), + KeepaliveParams: keepalive.ServerParameters{ + Time: 100 * time.Millisecond, + Timeout: 10 * time.Millisecond, + }, + } + copts := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + } + server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, copts) + defer func() { + client.Close(fmt.Errorf("closed manually by test")) + server.stop() + cancel() + }() + + addr := server.addr() + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatalf("net.Dial(tcp, %v) failed: %v", addr, err) + } + defer conn.Close() + + if n, err := conn.Write(clientPreface); err != nil || n != len(clientPreface) { + t.Fatalf("conn.Write(clientPreface) failed: n=%v, err=%v", n, err) + } + framer := newFramer(conn, defaultWriteBufSize, defaultReadBufSize, false, 0, mem.DefaultBufferPool()) + if err := framer.fr.WriteSettings(http2.Setting{}); err != nil { + t.Fatal("framer.WriteSettings(http2.Setting{}) failed:", err) + } + framer.writer.Flush() + + // We read from the net.Conn till we get an error, which is expected when + // the server closes the connection as part of the keepalive logic. + errCh := make(chan error, 1) + go func() { + b := make([]byte, 24) + for { + if _, err = conn.Read(b); err != nil { + errCh <- err + return + } + } + }() + + // Server waits for KeepaliveParams.Time seconds before sending out a ping, + // and then waits for KeepaliveParams.Timeout for a ping ack. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case err := <-errCh: + if err != io.EOF { + t.Fatalf("client.Read(_) = _,%v, want io.EOF", err) + + } + case <-ctx.Done(): + t.Fatalf("Test timed out before server closed the connection.") + } +} + +// TestKeepaliveServerWithResponsiveClient tests that a server doesn't close +// the connection with a client that responds to keepalive pings. +func (s) TestKeepaliveServerWithResponsiveClient(t *testing.T) { + serverConfig := &ServerConfig{ + BufferPool: mem.DefaultBufferPool(), + KeepaliveParams: keepalive.ServerParameters{ + Time: 100 * time.Millisecond, + Timeout: 100 * time.Millisecond, + }, + } + copts := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + } + server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, copts) + defer func() { + client.Close(fmt.Errorf("closed manually by test")) + server.stop() + cancel() + }() + + // Give keepalive logic some time by sleeping. + time.Sleep(500 * time.Millisecond) + + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) + } +} + +func channelzSubChannel(t *testing.T) *channelz.SubChannel { + ch := channelz.RegisterChannel(nil, "test chan") + sc := channelz.RegisterSubChannel(ch, "test subchan") + t.Cleanup(func() { + channelz.RemoveEntry(sc.ID) + channelz.RemoveEntry(ch.ID) + }) + return sc +} + +// TestKeepaliveClientClosesUnresponsiveServer creates a server which does not +// respond to keepalive pings, and makes sure that the client closes the +// transport once the keepalive logic kicks in. Here, we set the +// `PermitWithoutStream` parameter to true which ensures that the keepalive +// logic is running even without any active streams. +func (s) TestKeepaliveClientClosesUnresponsiveServer(t *testing.T) { + connCh := make(chan net.Conn, 1) + copts := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + ChannelzParent: channelzSubChannel(t), + KeepaliveParams: keepalive.ClientParameters{ + Time: 10 * time.Millisecond, + Timeout: 10 * time.Millisecond, + PermitWithoutStream: true, + }, + } + server, client, cancel := setUpControllablePingServer(t, copts, connCh) + server.setPingAck(false) + defer cancel() + defer client.Close(fmt.Errorf("closed manually by test")) + + conn, ok := <-connCh + if !ok { + t.Fatalf("Server didn't return connection object") + } + defer conn.Close() + + if err := pollForStreamCreationError(client); err != nil { + t.Fatal(err) + } +} + +// TestKeepaliveClientOpenWithUnresponsiveServer creates a server which does +// not respond to keepalive pings, and makes sure that the client does not +// close the transport. Here, we do not set the `PermitWithoutStream` parameter +// to true which ensures that the keepalive logic is turned off without any +// active streams, and therefore the transport stays open. +func (s) TestKeepaliveClientOpenWithUnresponsiveServer(t *testing.T) { + connCh := make(chan net.Conn, 1) + copts := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + ChannelzParent: channelzSubChannel(t), + KeepaliveParams: keepalive.ClientParameters{ + Time: 10 * time.Millisecond, + Timeout: 10 * time.Millisecond, + }, + } + server, client, cancel := setUpControllablePingServer(t, copts, connCh) + server.setPingAck(false) + defer cancel() + defer client.Close(fmt.Errorf("closed manually by test")) + + conn, ok := <-connCh + if !ok { + t.Fatalf("Server didn't return connection object") + } + defer conn.Close() + + // Give keepalive some time. + time.Sleep(500 * time.Millisecond) + + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) + } +} + +// TestKeepaliveClientClosesWithActiveStreams creates a responsive server and +// then stops responding to keepalive pings. It makes sure that the client +// closes the transport even when there is an active stream. +func (s) TestKeepaliveClientClosesWithActiveStreams(t *testing.T) { + connCh := make(chan net.Conn, 1) + copts := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + ChannelzParent: channelzSubChannel(t), + KeepaliveParams: keepalive.ClientParameters{ + Time: 10 * time.Millisecond, + Timeout: 100 * time.Millisecond, + }, + } + server, client, cancel := setUpControllablePingServer(t, copts, connCh) + + defer cancel() + defer client.Close(fmt.Errorf("closed manually by test")) + + conn, ok := <-connCh + if !ok { + t.Fatalf("Server didn't return connection object") + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Create a stream, but send no data on it. + if _, err := client.NewStream(ctx, &CallHdr{}, nil); err != nil { + t.Fatalf("Stream creation failed: %v", err) + } + + // Now that we have an active stream and verified the connection is stable, + // we want to simulate a "no-ping" server. + server.setPingAck(false) + + if err := pollForStreamCreationError(client); err != nil { + t.Fatal(err) + } +} + +// TestKeepaliveClientStaysHealthyWithResponsiveServer creates a server which +// responds to keepalive pings, and makes sure than a client transport stays +// healthy without any active streams. +func (s) TestKeepaliveClientStaysHealthyWithResponsiveServer(t *testing.T) { + server, client, cancel := setUpWithOptions(t, 0, + &ServerConfig{ + BufferPool: mem.DefaultBufferPool(), + KeepalivePolicy: keepalive.EnforcementPolicy{ + MinTime: 50 * time.Millisecond, + PermitWithoutStream: true, + }, + }, + normal, + ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + KeepaliveParams: keepalive.ClientParameters{ + Time: 55 * time.Millisecond, + Timeout: time.Second, + PermitWithoutStream: true, + }}) + defer func() { + client.Close(fmt.Errorf("closed manually by test")) + server.stop() + cancel() + }() + + // Give keepalive some time. + time.Sleep(500 * time.Millisecond) + + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) + } +} + +// TestKeepaliveClientFrequency creates a server which expects at most 1 client +// ping for every 100 ms, while the client is configured to send a ping +// every 50 ms. So, this configuration should end up with the client +// transport being closed. But we had a bug wherein the client was sending one +// ping every [Time+Timeout] instead of every [Time] period, and this test +// explicitly makes sure the fix works and the client sends a ping every [Time] +// period. +func (s) TestKeepaliveClientFrequency(t *testing.T) { + grpctest.ExpectError("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\"") + + serverConfig := &ServerConfig{ + BufferPool: mem.DefaultBufferPool(), + KeepalivePolicy: keepalive.EnforcementPolicy{ + MinTime: 100 * time.Millisecond, + PermitWithoutStream: true, + }, + } + clientOptions := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + KeepaliveParams: keepalive.ClientParameters{ + Time: 50 * time.Millisecond, + Timeout: time.Second, + PermitWithoutStream: true, + }, + } + server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions) + defer func() { + client.Close(fmt.Errorf("closed manually by test")) + server.stop() + cancel() + }() + + if err := waitForGoAwayTooManyPings(client); err != nil { + t.Fatal(err) + } +} + +// TestKeepaliveServerEnforcementWithAbusiveClientNoRPC verifies that the +// server closes a client transport when it sends too many keepalive pings +// (when there are no active streams), based on the configured +// EnforcementPolicy. +func (s) TestKeepaliveServerEnforcementWithAbusiveClientNoRPC(t *testing.T) { + grpctest.ExpectError("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\"") + + serverConfig := &ServerConfig{ + BufferPool: mem.DefaultBufferPool(), + KeepalivePolicy: keepalive.EnforcementPolicy{ + MinTime: time.Second, + }, + } + clientOptions := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + KeepaliveParams: keepalive.ClientParameters{ + Time: 20 * time.Millisecond, + Timeout: 100 * time.Millisecond, + PermitWithoutStream: true, + }, + } + server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions) + defer func() { + client.Close(fmt.Errorf("closed manually by test")) + server.stop() + cancel() + }() + + if err := waitForGoAwayTooManyPings(client); err != nil { + t.Fatal(err) + } +} + +// TestKeepaliveServerEnforcementWithAbusiveClientWithRPC verifies that the +// server closes a client transport when it sends too many keepalive pings +// (even when there is an active stream), based on the configured +// EnforcementPolicy. +func (s) TestKeepaliveServerEnforcementWithAbusiveClientWithRPC(t *testing.T) { + grpctest.ExpectError("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\"") + + serverConfig := &ServerConfig{ + BufferPool: mem.DefaultBufferPool(), + KeepalivePolicy: keepalive.EnforcementPolicy{ + MinTime: time.Second, + }, + } + clientOptions := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + KeepaliveParams: keepalive.ClientParameters{ + Time: 50 * time.Millisecond, + Timeout: 100 * time.Millisecond, + }, + } + server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, clientOptions) + defer func() { + client.Close(fmt.Errorf("closed manually by test")) + server.stop() + cancel() + }() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := client.NewStream(ctx, &CallHdr{}, nil); err != nil { + t.Fatalf("Stream creation failed: %v", err) + } + + if err := waitForGoAwayTooManyPings(client); err != nil { + t.Fatal(err) + } +} + +// TestKeepaliveServerEnforcementWithObeyingClientNoRPC verifies that the +// server does not close a client transport (with no active streams) which +// sends keepalive pings in accordance to the configured keepalive +// EnforcementPolicy. +func (s) TestKeepaliveServerEnforcementWithObeyingClientNoRPC(t *testing.T) { + serverConfig := &ServerConfig{ + BufferPool: mem.DefaultBufferPool(), + KeepalivePolicy: keepalive.EnforcementPolicy{ + MinTime: 40 * time.Millisecond, + PermitWithoutStream: true, + }, + } + clientOptions := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + KeepaliveParams: keepalive.ClientParameters{ + Time: 50 * time.Millisecond, + Timeout: time.Second, + PermitWithoutStream: true, + }, + } + server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions) + defer func() { + client.Close(fmt.Errorf("closed manually by test")) + server.stop() + cancel() + }() + + // Sleep for client to send ~10 keepalive pings. + time.Sleep(500 * time.Millisecond) + + // Verify that the server does not close the client transport. + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) + } +} + +// TestKeepaliveServerEnforcementWithObeyingClientWithRPC verifies that the +// server does not close a client transport (with active streams) which +// sends keepalive pings in accordance to the configured keepalive +// EnforcementPolicy. +func (s) TestKeepaliveServerEnforcementWithObeyingClientWithRPC(t *testing.T) { + serverConfig := &ServerConfig{ + BufferPool: mem.DefaultBufferPool(), + KeepalivePolicy: keepalive.EnforcementPolicy{ + MinTime: 40 * time.Millisecond, + }, + } + clientOptions := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + KeepaliveParams: keepalive.ClientParameters{ + Time: 50 * time.Millisecond, + }, + } + server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, clientOptions) + defer func() { + client.Close(fmt.Errorf("closed manually by test")) + server.stop() + cancel() + }() + + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) + } + + // Give keepalive enough time. + time.Sleep(500 * time.Millisecond) + + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) + } +} + +// TestKeepaliveServerEnforcementWithDormantKeepaliveOnClient verifies that the +// server does not closes a client transport, which has been configured to send +// more pings than allowed by the server's EnforcementPolicy. This client +// transport does not have any active streams and `PermitWithoutStream` is set +// to false. This should ensure that the keepalive functionality on the client +// side enters a dormant state. +func (s) TestKeepaliveServerEnforcementWithDormantKeepaliveOnClient(t *testing.T) { + serverConfig := &ServerConfig{ + BufferPool: mem.DefaultBufferPool(), + KeepalivePolicy: keepalive.EnforcementPolicy{ + MinTime: 100 * time.Millisecond, + }, + } + clientOptions := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + KeepaliveParams: keepalive.ClientParameters{ + Time: 10 * time.Millisecond, + Timeout: 10 * time.Millisecond, + }, + } + server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions) + defer func() { + client.Close(fmt.Errorf("closed manually by test")) + server.stop() + cancel() + }() + + // No active streams on the client. Give keepalive enough time. + time.Sleep(500 * time.Millisecond) + + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) + } +} + +// TestTCPUserTimeout tests that the TCP_USER_TIMEOUT socket option is set to +// the keepalive timeout, as detailed in proposal A18. +func (s) TestTCPUserTimeout(t *testing.T) { + tests := []struct { + tls bool + time time.Duration + timeout time.Duration + clientWantTimeout time.Duration + serverWantTimeout time.Duration + }{ + { + false, + 10 * time.Second, + 10 * time.Second, + 10 * 1000 * time.Millisecond, + 10 * 1000 * time.Millisecond, + }, + { + false, + 0, + 0, + 0, + 20 * 1000 * time.Millisecond, + }, + { + false, + infinity, + infinity, + 0, + 0, + }, + { + true, + 10 * time.Second, + 10 * time.Second, + 10 * 1000 * time.Millisecond, + 10 * 1000 * time.Millisecond, + }, + { + true, + 0, + 0, + 0, + 20 * 1000 * time.Millisecond, + }, + { + true, + infinity, + infinity, + 0, + 0, + }, + } + for _, tt := range tests { + sopts := &ServerConfig{ + BufferPool: mem.DefaultBufferPool(), + KeepaliveParams: keepalive.ServerParameters{ + Time: tt.time, + Timeout: tt.timeout, + }, + } + + copts := ConnectOptions{ + BufferPool: mem.DefaultBufferPool(), + KeepaliveParams: keepalive.ClientParameters{ + Time: tt.time, + Timeout: tt.timeout, + }, + } + + if tt.tls { + copts.TransportCredentials = makeTLSCreds(t, "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem") + sopts.Credentials = makeTLSCreds(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + + } + + server, client, cancel := setUpWithOptions( + t, + 0, + sopts, + normal, + copts, + ) + defer func() { + client.Close(fmt.Errorf("closed manually by test")) + server.stop() + cancel() + }() + + var sc *http2Server + var srawConn net.Conn + // Wait until the server transport is setup. + for { + server.mu.Lock() + if len(server.conns) == 0 { + server.mu.Unlock() + time.Sleep(time.Millisecond) + continue + } + for k := range server.conns { + var ok bool + sc, ok = k.(*http2Server) + if !ok { + t.Fatalf("Failed to convert %v to *http2Server", k) + } + srawConn = server.conns[k] + } + server.mu.Unlock() + break + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := client.NewStream(ctx, &CallHdr{}, nil) + if err != nil { + t.Fatalf("client.NewStream() failed: %v", err) + } + stream.Close(io.EOF) + + // check client TCP user timeout only when non TLS + // TODO : find a way to get the underlying conn for client when TLS + if !tt.tls { + cltOpt, err := syscall.GetTCPUserTimeout(client.conn) + if err != nil { + t.Fatalf("syscall.GetTCPUserTimeout() failed: %v", err) + } + if cltOpt < 0 { + t.Skipf("skipping test on unsupported environment") + } + if gotTimeout := time.Duration(cltOpt) * time.Millisecond; gotTimeout != tt.clientWantTimeout { + t.Fatalf("syscall.GetTCPUserTimeout() = %d, want %d", gotTimeout, tt.clientWantTimeout) + } + } + scConn := sc.conn + if tt.tls { + if _, ok := sc.conn.(*net.TCPConn); ok { + t.Fatalf("sc.conn is should have wrapped conn with TLS") + } + scConn = srawConn + } + // verify the type of scConn (on which TCP user timeout will be got) + if _, ok := scConn.(*net.TCPConn); !ok { + t.Fatalf("server underlying conn is of type %T, want net.TCPConn", scConn) + } + srvOpt, err := syscall.GetTCPUserTimeout(scConn) + if err != nil { + t.Fatalf("syscall.GetTCPUserTimeout() failed: %v", err) + } + if gotTimeout := time.Duration(srvOpt) * time.Millisecond; gotTimeout != tt.serverWantTimeout { + t.Fatalf("syscall.GetTCPUserTimeout() = %d, want %d", gotTimeout, tt.serverWantTimeout) + } + + } +} + +func makeTLSCreds(t *testing.T, certPath, keyPath, rootsPath string) credentials.TransportCredentials { + cert, err := tls.LoadX509KeyPair(testdata.Path(certPath), testdata.Path(keyPath)) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(%q, %q) failed: %v", certPath, keyPath, err) + } + b, err := os.ReadFile(testdata.Path(rootsPath)) + if err != nil { + t.Fatalf("os.ReadFile(%q) failed: %v", rootsPath, err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(b) { + t.Fatal("failed to append certificates") + } + return credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: roots, + InsecureSkipVerify: true, + }) +} + +// checkForHealthyStream attempts to create a stream and return error if any. +// The stream created is closed right after to avoid any leakages. +func checkForHealthyStream(client *http2Client) error { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := client.NewStream(ctx, &CallHdr{}, nil) + if err != nil { + return err + } + stream.Close(nil) + return nil +} + +func pollForStreamCreationError(client *http2Client) error { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for { + if _, err := client.NewStream(ctx, &CallHdr{}, nil); err != nil { + break + } + time.Sleep(50 * time.Millisecond) + } + if ctx.Err() != nil { + return fmt.Errorf("test timed out before stream creation returned an error") + } + return nil +} + +// waitForGoAwayTooManyPings waits for client to receive a GoAwayTooManyPings +// from server. It also asserts that stream creation fails after receiving a +// GoAway. +func waitForGoAwayTooManyPings(client *http2Client) error { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-client.GoAway(): + if reason, _ := client.GetGoAwayReason(); reason != GoAwayTooManyPings { + return fmt.Errorf("goAwayReason is %v, want %v", reason, GoAwayTooManyPings) + } + case <-ctx.Done(): + return fmt.Errorf("test timed out before getting GoAway with reason:GoAwayTooManyPings from server") + } + + if _, err := client.NewStream(ctx, &CallHdr{}, nil); err == nil { + return fmt.Errorf("stream creation succeeded after receiving a GoAway from the server") + } + return nil +} diff --git a/internal/transport/logging.go b/internal/transport/logging.go new file mode 100644 index 0000000..42ed2b0 --- /dev/null +++ b/internal/transport/logging.go @@ -0,0 +1,40 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +var logger = grpclog.Component("transport") + +func prefixLoggerForServerTransport(p *http2Server) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[server-transport %p] ", p)) +} + +func prefixLoggerForServerHandlerTransport(p *serverHandlerTransport) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[server-handler-transport %p] ", p)) +} + +func prefixLoggerForClientTransport(p *http2Client) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[client-transport %p] ", p)) +} diff --git a/internal/transport/networktype/networktype.go b/internal/transport/networktype/networktype.go new file mode 100644 index 0000000..c11b527 --- /dev/null +++ b/internal/transport/networktype/networktype.go @@ -0,0 +1,46 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package networktype declares the network type to be used in the default +// dialer. Attribute of a resolver.Address. +package networktype + +import ( + "google.golang.org/grpc/resolver" +) + +// keyType is the key to use for storing State in Attributes. +type keyType string + +const key = keyType("grpc.internal.transport.networktype") + +// Set returns a copy of the provided address with attributes containing networkType. +func Set(address resolver.Address, networkType string) resolver.Address { + address.Attributes = address.Attributes.WithValue(key, networkType) + return address +} + +// Get returns the network type in the resolver.Address and true, or "", false +// if not present. +func Get(address resolver.Address) (string, bool) { + v := address.Attributes.Value(key) + if v == nil { + return "", false + } + return v.(string), true +} diff --git a/internal/transport/proxy.go b/internal/transport/proxy.go new file mode 100644 index 0000000..d773845 --- /dev/null +++ b/internal/transport/proxy.go @@ -0,0 +1,116 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "bufio" + "context" + "encoding/base64" + "fmt" + "io" + "net" + "net/http" + "net/http/httputil" + "net/url" + + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/proxyattributes" + "google.golang.org/grpc/resolver" +) + +const proxyAuthHeaderKey = "Proxy-Authorization" + +// To read a response from a net.Conn, http.ReadResponse() takes a bufio.Reader. +// It's possible that this reader reads more than what's need for the response +// and stores those bytes in the buffer. bufConn wraps the original net.Conn +// and the bufio.Reader to make sure we don't lose the bytes in the buffer. +type bufConn struct { + net.Conn + r io.Reader +} + +func (c *bufConn) Read(b []byte) (int, error) { + return c.r.Read(b) +} + +func basicAuth(username, password string) string { + auth := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(auth)) +} + +func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, grpcUA string, opts proxyattributes.Options) (_ net.Conn, err error) { + defer func() { + if err != nil { + conn.Close() + } + }() + + req := &http.Request{ + Method: http.MethodConnect, + URL: &url.URL{Host: opts.ConnectAddr}, + Header: map[string][]string{"User-Agent": {grpcUA}}, + } + if user := opts.User; user != nil { + u := user.Username() + p, _ := user.Password() + req.Header.Add(proxyAuthHeaderKey, "Basic "+basicAuth(u, p)) + } + if err := sendHTTPRequest(ctx, req, conn); err != nil { + return nil, fmt.Errorf("failed to write the HTTP request: %v", err) + } + + r := bufio.NewReader(conn) + resp, err := http.ReadResponse(r, req) + if err != nil { + return nil, fmt.Errorf("reading server HTTP response: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + dump, err := httputil.DumpResponse(resp, true) + if err != nil { + return nil, fmt.Errorf("failed to do connect handshake, status code: %s", resp.Status) + } + return nil, fmt.Errorf("failed to do connect handshake, response: %q", dump) + } + // The buffer could contain extra bytes from the target server, so we can't + // discard it. However, in many cases where the server waits for the client + // to send the first message (e.g. when TLS is being used), the buffer will + // be empty, so we can avoid the overhead of reading through this buffer. + if r.Buffered() != 0 { + return &bufConn{Conn: conn, r: r}, nil + } + return conn, nil +} + +// proxyDial establishes a TCP connection to the specified address and performs an HTTP CONNECT handshake. +func proxyDial(ctx context.Context, addr resolver.Address, grpcUA string, opts proxyattributes.Options) (net.Conn, error) { + conn, err := internal.NetDialerWithTCPKeepalive().DialContext(ctx, "tcp", addr.Addr) + if err != nil { + return nil, err + } + return doHTTPConnectHandshake(ctx, conn, grpcUA, opts) +} + +func sendHTTPRequest(ctx context.Context, req *http.Request, conn net.Conn) error { + req = req.WithContext(ctx) + if err := req.Write(conn); err != nil { + return fmt.Errorf("failed to write the HTTP request: %v", err) + } + return nil +} diff --git a/internal/transport/proxy_ext_test.go b/internal/transport/proxy_ext_test.go new file mode 100644 index 0000000..c24f5a7 --- /dev/null +++ b/internal/transport/proxy_ext_test.go @@ -0,0 +1,441 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport_test + +import ( + "context" + "encoding/base64" + "fmt" + "net" + "net/http" + "net/netip" + "net/url" + "testing" + "time" + + "golang.org/x/net/http/httpproxy" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/resolver/delegatingresolver" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/proxyserver" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func startBackendServer(t *testing.T) *stubserver.StubServer { + t.Helper() + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + t.Cleanup(backend.Stop) + return backend +} + +func isIPAddr(addr string) bool { + _, err := netip.ParseAddr(addr) + return err == nil +} + +func overrideTestHTTPSProxy(t *testing.T, proxyAddr string) { + t.Helper() + hpfe := func(*http.Request) (*url.URL, error) { + return &url.URL{ + Scheme: "https", + Host: proxyAddr, + }, nil + } + originalhpfe := delegatingresolver.HTTPSProxyFromEnvironment + delegatingresolver.HTTPSProxyFromEnvironment = hpfe + t.Cleanup(func() { delegatingresolver.HTTPSProxyFromEnvironment = originalhpfe }) +} + +// Tests the scenario where grpc.Dial is performed using a proxy with the +// default resolver in the target URI. The test verifies that the connection is +// established to the proxy server, sends the unresolved target URI in the HTTP +// CONNECT request and is successfully connected to the backend server. +func (s) TestGRPCDialWithProxy(t *testing.T) { + backend := startBackendServer(t) + unresolvedTargetURI := fmt.Sprintf("localhost:%d", testutils.ParsePort(t, backend.Address)) + proxyCalled := false + reqCheck := func(req *http.Request) { + proxyCalled = true + host, _, err := net.SplitHostPort(req.URL.Host) + if err != nil { + t.Error(err) + } + if got, want := host, "localhost"; got != want { + t.Errorf(" Unexpected request host: %s, want = %s ", got, want) + } + } + pServer := proxyserver.New(t, reqCheck, false) + // Use "localhost:" to verify the proxy address is handled + // correctly by the delegating resolver and connects to the proxy server + // correctly even when unresolved. + pAddr := fmt.Sprintf("localhost:%d", testutils.ParsePort(t, pServer.Addr)) + + overrideTestHTTPSProxy(t, pAddr) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, err := grpc.Dial(unresolvedTargetURI, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial(%s) failed: %v", unresolvedTargetURI, err) + } + defer conn.Close() + + // Send an empty RPC to the backend through the proxy. + client := testgrpc.NewTestServiceClient(conn) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } + + if !proxyCalled { + t.Fatalf("Proxy not connected") + } +} + +// Tests the scenario where `grpc.Dial` is performed with a proxy and the "dns" +// scheme for the target. The test verifies that the proxy URI is correctly +// resolved and that the target URI resolution on the client preserves the +// original behavior of `grpc.Dial`. It also ensures that a connection is +// established to the proxy server, with the resolved target URI sent in the +// HTTP CONNECT request, successfully connecting to the backend server. +func (s) TestGRPCDialWithDNSAndProxy(t *testing.T) { + backend := startBackendServer(t) + unresolvedTargetURI := fmt.Sprintf("localhost:%d", testutils.ParsePort(t, backend.Address)) + proxyCalled := false + reqCheck := func(req *http.Request) { + proxyCalled = true + + host, _, err := net.SplitHostPort(req.URL.Host) + if err != nil { + t.Error(err) + } + if got, want := isIPAddr(host), true; got != want { + t.Errorf("isIPAddr(%q) = %t, want = %t", host, got, want) + } + } + pServer := proxyserver.New(t, reqCheck, false) + + overrideTestHTTPSProxy(t, pServer.Addr) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, err := grpc.Dial("dns:///"+unresolvedTargetURI, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial(%s) failed: %v", "dns:///"+unresolvedTargetURI, err) + } + defer conn.Close() + + // Send an empty RPC to the backend through the proxy. + client := testgrpc.NewTestServiceClient(conn) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } + + if !proxyCalled { + t.Fatalf("Proxy not connected") + } +} + +// Tests the scenario where `grpc.NewClient` is used with the default DNS +// resolver for the target URI and a proxy is configured. The test verifies +// that the client resolves proxy URI, connects to the proxy server, sends the +// unresolved target URI in the HTTP CONNECT request, and successfully +// establishes a connection to the backend server. +func (s) TestNewClientWithProxy(t *testing.T) { + backend := startBackendServer(t) + unresolvedTargetURI := fmt.Sprintf("localhost:%d", testutils.ParsePort(t, backend.Address)) + proxyCalled := false + reqCheck := func(req *http.Request) { + proxyCalled = true + host, _, err := net.SplitHostPort(req.URL.Host) + if err != nil { + t.Error(err) + } + if got, want := host, "localhost"; got != want { + t.Errorf(" Unexpected request host: %s, want = %s ", got, want) + } + } + pServer := proxyserver.New(t, reqCheck, false) + // Use "localhost:" to verify the proxy address is handled + // correctly by the delegating resolver and connects to the proxy server + // correctly even when unresolved. + pAddr := fmt.Sprintf("localhost:%d", testutils.ParsePort(t, pServer.Addr)) + + overrideTestHTTPSProxy(t, pAddr) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, err := grpc.NewClient(unresolvedTargetURI, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v", unresolvedTargetURI, err) + } + defer conn.Close() + + // Send an empty RPC to the backend through the proxy. + client := testgrpc.NewTestServiceClient(conn) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } + if !proxyCalled { + t.Fatalf("Proxy not connected") + } +} + +// Tests the scenario where grpc.NewClient is used with a custom target URI +// scheme and a proxy is configured. The test verifies that the client +// successfully connects to the proxy server, resolves the proxy URI correctly, +// includes the resolved target URI in the HTTP CONNECT request, and +// establishes a connection to the backend server. +func (s) TestNewClientWithProxyAndCustomResolver(t *testing.T) { + backend := startBackendServer(t) + unresolvedTargetURI := fmt.Sprintf("localhost:%d", testutils.ParsePort(t, backend.Address)) + proxyCalled := false + reqCheck := func(req *http.Request) { + proxyCalled = true + host, _, err := net.SplitHostPort(req.URL.Host) + if err != nil { + t.Error(err) + } + if got, want := isIPAddr(host), true; got != want { + t.Errorf("isIPAddr(%q) = %t, want = %t", host, got, want) + } + } + pServer := proxyserver.New(t, reqCheck, false) + + overrideTestHTTPSProxy(t, pServer.Addr) + + // Create and update a custom resolver for target URI. + targetResolver := manual.NewBuilderWithScheme("test") + resolver.Register(targetResolver) + targetResolver.InitialState(resolver.State{Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: backend.Address}}}}}) + + // Dial to the proxy server. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, err := grpc.NewClient(targetResolver.Scheme()+":///"+unresolvedTargetURI, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v", targetResolver.Scheme()+":///"+unresolvedTargetURI, err) + } + defer conn.Close() + + // Send an empty RPC to the backend through the proxy. + client := testgrpc.NewTestServiceClient(conn) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + if !proxyCalled { + t.Fatalf("Proxy not connected") + } +} + +// Tests the scenario where grpc.NewClient is used with the default "dns" +// resolver and the dial option grpc.WithLocalDNSResolution() is set, +// enabling target resolution on the client. The test verifies that target +// resolution happens on the client by sending resolved target URI in HTTP +// CONNECT request, the proxy URI is resolved correctly, and the connection is +// successfully established with the backend server through the proxy. +func (s) TestNewClientWithProxyAndTargetResolutionEnabled(t *testing.T) { + backend := startBackendServer(t) + unresolvedTargetURI := fmt.Sprintf("localhost:%d", testutils.ParsePort(t, backend.Address)) + proxyCalled := false + reqCheck := func(req *http.Request) { + proxyCalled = true + host, _, err := net.SplitHostPort(req.URL.Host) + if err != nil { + t.Error(err) + } + if got, want := isIPAddr(host), true; got != want { + t.Errorf("isIPAddr(%q) = %t, want = %t", host, got, want) + } + } + pServer := proxyserver.New(t, reqCheck, false) + + overrideTestHTTPSProxy(t, pServer.Addr) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, err := grpc.NewClient(unresolvedTargetURI, grpc.WithLocalDNSResolution(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v", unresolvedTargetURI, err) + } + defer conn.Close() + + // Send an empty RPC to the backend through the proxy. + client := testgrpc.NewTestServiceClient(conn) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } + + if !proxyCalled { + t.Fatalf("Proxy not connected") + } +} + +// Tests the scenario where grpc.NewClient is used with grpc.WithNoProxy() set, +// explicitly disabling proxy usage. The test verifies that the client does not +// dial the proxy but directly connects to the backend server. It also checks +// that the proxy resolution function is not called and that the proxy server +// never receives a connection request. +func (s) TestNewClientWithNoProxy(t *testing.T) { + backend := startBackendServer(t) + unresolvedTargetURI := fmt.Sprintf("localhost:%d", testutils.ParsePort(t, backend.Address)) + reqCheck := func(_ *http.Request) { t.Error("proxy server should not have received a Connect request") } + pServer := proxyserver.New(t, reqCheck, false) + + overrideTestHTTPSProxy(t, pServer.Addr) + + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithNoProxy(), // Disable proxy. + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, err := grpc.NewClient(unresolvedTargetURI, dopts...) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v", unresolvedTargetURI, err) + } + defer conn.Close() + + // Create a test service client and make an RPC call. + client := testgrpc.NewTestServiceClient(conn) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} + +// Tests the scenario where grpc.NewClient is used with grpc.WithContextDialer() +// set. The test verifies that the client bypasses proxy dialing and uses the +// custom dialer instead. It ensures that the proxy server is never dialed, the +// proxy resolution function is not triggered, and the custom dialer is invoked +// as expected. +func (s) TestNewClientWithContextDialer(t *testing.T) { + backend := startBackendServer(t) + unresolvedTargetURI := fmt.Sprintf("localhost:%d", testutils.ParsePort(t, backend.Address)) + reqCheck := func(_ *http.Request) { t.Error("proxy server should not have received a Connect request") } + pServer := proxyserver.New(t, reqCheck, false) + + overrideTestHTTPSProxy(t, pServer.Addr) + + // Create a custom dialer that directly dials the backend. + customDialer := func(_ context.Context, unresolvedTargetURI string) (net.Conn, error) { + return net.Dial("tcp", unresolvedTargetURI) + } + + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(customDialer), // Use a custom dialer. + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, err := grpc.NewClient(unresolvedTargetURI, dopts...) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v", unresolvedTargetURI, err) + } + defer conn.Close() + + client := testgrpc.NewTestServiceClient(conn) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} + +// Tests the scenario where grpc.NewClient is used with the default DNS resolver +// for targetURI and a proxy. The test verifies that the client connects to the +// proxy server, sends the unresolved target URI in the HTTP CONNECT request, +// and successfully connects to the backend. Additionally, it checks that the +// correct user information is included in the Proxy-Authorization header of +// the CONNECT request. The test also ensures that target resolution does not +// happen on the client. +func (s) TestBasicAuthInNewClientWithProxy(t *testing.T) { + unresolvedTargetURI := "example.test" + const ( + user = "notAUser" + password = "notAPassword" + ) + proxyCalled := false + reqCheck := func(req *http.Request) { + proxyCalled = true + if got, want := req.URL.Host, "example.test:443"; got != want { + t.Errorf(" Unexpected request host: %s, want = %s ", got, want) + } + wantProxyAuthStr := "Basic " + base64.StdEncoding.EncodeToString([]byte(user+":"+password)) + if got := req.Header.Get("Proxy-Authorization"); got != wantProxyAuthStr { + gotDecoded, err := base64.StdEncoding.DecodeString(got) + if err != nil { + t.Errorf("failed to decode Proxy-Authorization header: %v", err) + } + wantDecoded, _ := base64.StdEncoding.DecodeString(wantProxyAuthStr) + t.Errorf("unexpected auth %q (%q), want %q (%q)", got, gotDecoded, wantProxyAuthStr, wantDecoded) + } + } + pServer := proxyserver.New(t, reqCheck, false) + + t.Setenv("HTTPS_PROXY", user+":"+password+"@"+pServer.Addr) + + // Use the httpproxy package functions instead of `http.ProxyFromEnvironment` + // because the latter reads proxy-related environment variables only once at + // initialization. This behavior causes issues when running test multiple + // times, as changes to environment variables during tests would be ignored. + // By using `httpproxy.FromEnvironment()`, we ensure proxy settings are read dynamically. + origHTTPSProxyFromEnvironment := delegatingresolver.HTTPSProxyFromEnvironment + delegatingresolver.HTTPSProxyFromEnvironment = func(req *http.Request) (*url.URL, error) { + return httpproxy.FromEnvironment().ProxyFunc()(req.URL) + } + defer func() { + delegatingresolver.HTTPSProxyFromEnvironment = origHTTPSProxyFromEnvironment + }() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, err := grpc.NewClient(unresolvedTargetURI, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v", unresolvedTargetURI, err) + } + defer conn.Close() + + // Send an empty RPC to the backend through the proxy. + client := testgrpc.NewTestServiceClient(conn) + client.EmptyCall(ctx, &testpb.Empty{}) + + if !proxyCalled { + t.Fatalf("Proxy not connected") + } +} diff --git a/internal/transport/proxy_test.go b/internal/transport/proxy_test.go new file mode 100644 index 0000000..442d2cb --- /dev/null +++ b/internal/transport/proxy_test.go @@ -0,0 +1,105 @@ +//go:build !race +// +build !race + +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "context" + "net" + "net/http" + "net/netip" + "testing" + "time" + + "google.golang.org/grpc/internal/proxyattributes" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/proxyserver" + "google.golang.org/grpc/resolver" +) + +func (s) TestHTTPConnectWithServerHello(t *testing.T) { + serverMessage := []byte("server-hello") + blis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + reqCheck := func(req *http.Request) { + if req.Method != http.MethodConnect { + t.Errorf("unexpected Method %q, want %q", req.Method, http.MethodConnect) + } + host, _, err := net.SplitHostPort(req.URL.Host) + if err != nil { + t.Error(err) + } + _, err = netip.ParseAddr(host) + if err != nil { + t.Error(err) + } + } + pServer := proxyserver.New(t, reqCheck, true) + + msg := []byte{4, 3, 5, 2} + recvBuf := make([]byte, len(msg)) + done := make(chan error, 1) + go func() { + in, err := blis.Accept() + if err != nil { + done <- err + return + } + defer in.Close() + in.Write(serverMessage) + in.Read(recvBuf) + done <- nil + }() + + // Dial to proxy server. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + c, err := proxyDial(ctx, resolver.Address{Addr: pServer.Addr}, "test", proxyattributes.Options{ConnectAddr: blis.Addr().String()}) + if err != nil { + t.Fatalf("HTTP connect Dial failed: %v", err) + } + defer c.Close() + c.SetDeadline(time.Now().Add(defaultTestTimeout)) + + // Send msg on the connection. + c.Write(msg) + if err := <-done; err != nil { + t.Fatalf("Failed to accept: %v", err) + } + + // Check received msg. + if string(recvBuf) != string(msg) { + t.Fatalf("Received msg: %v, want %v", recvBuf, msg) + } + + if len(serverMessage) > 0 { + gotServerMessage := make([]byte, len(serverMessage)) + if _, err := c.Read(gotServerMessage); err != nil { + t.Errorf("Got error while reading message from server: %v", err) + return + } + if string(gotServerMessage) != string(serverMessage) { + t.Errorf("Message from server: %v, want %v", gotServerMessage, serverMessage) + } + } +} diff --git a/internal/transport/readyreader/raw_conn_linux.go b/internal/transport/readyreader/raw_conn_linux.go new file mode 100644 index 0000000..56906c3 --- /dev/null +++ b/internal/transport/readyreader/raw_conn_linux.go @@ -0,0 +1,39 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package readyreader + +import "syscall" + +func isRawConnSupported() bool { + return true +} + +// sysRead uses the standard syscall package rather than the modern unix package +// to avoid triggering the race detector. Because both packages perform sync +// operations on a local variable to satisfy the race detector, mixing them +// for read and write syscalls causes data races. We use syscall here to remain +// consistent with net.Conn implementations in standard library. +func sysRead(fd uintptr, p []byte) (int, error) { + return syscall.Read(int(fd), p) +} + +// wouldBlock checks standard Unix non-blocking errors. +func wouldBlock(err error) bool { + return err == syscall.EAGAIN || err == syscall.EWOULDBLOCK +} diff --git a/internal/transport/readyreader/raw_conn_nonlinux.go b/internal/transport/readyreader/raw_conn_nonlinux.go new file mode 100644 index 0000000..4d1f330 --- /dev/null +++ b/internal/transport/readyreader/raw_conn_nonlinux.go @@ -0,0 +1,35 @@ +//go:build !linux + +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package readyreader + +func isRawConnSupported() bool { + return false +} + +// sysRead is not implemented. Support can be added in the future if necessary. +func sysRead(uintptr, []byte) (int, error) { + panic("RawConn functionality is not implemented for non-unix platforms.") +} + +// wouldBlock is not implemented. Support can be added in the future if necessary. +func wouldBlock(error) bool { + panic("RawConn functionality is not implemented for non-unix platforms.") +} diff --git a/internal/transport/readyreader/ready_reader.go b/internal/transport/readyreader/ready_reader.go new file mode 100644 index 0000000..250a300 --- /dev/null +++ b/internal/transport/readyreader/ready_reader.go @@ -0,0 +1,253 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package readyreader provides utilities to perform non-memory-pinning reads. +package readyreader + +import ( + "io" + "net" + "syscall" + + "google.golang.org/grpc/mem" +) + +// Reader is an optional interface that can be implemented by [net.Conn] +// implementations to enable gRPC to perform non-memory-pinning reads. +type Reader interface { + // ReadOnReady waits for data to arrive, fetches a buffer, and performs a + // read. When the underlying IO is readable, it allocates a buffer of size + // bufSize from the pool and reads up to bufSize bytes into the buffer. + // + // It returns a pointer to the buffer so it can be returned to the pool + // later, the number of bytes read, and an error. + // + // Callers should always process the n > 0 bytes returned before considering + // the error. Doing so correctly handles I/O errors that happen after + // reading some bytes, as well as both of the allowed EOF behaviors. + ReadOnReady(bufSize int, pool mem.BufferPool) (b *[]byte, n int, err error) +} + +// nonBlockingReader is optimized for non-memory-pinning reads using the RawConn +// interface. +type nonBlockingReader struct { + raw syscall.RawConn + // The following fields are stored as field to avoid heap allocations. + state readState + doRead func(fd uintptr) bool +} + +type readState struct { + // Request params. + bufSize int + pool mem.BufferPool + + // Response params. + readError error + bytesRead int + buf *[]byte +} + +// NewNonBlocking returns a ReadyReader if the passed reader supports +// non-memory-pinning reads, else nil. +func NewNonBlocking(r io.Reader) Reader { + if rr, ok := r.(Reader); ok { + return rr + } + if !isRawConnSupported() { + return nil + } + // We restrict the types before asserting syscall.Conn. The credentials + // package may return a wrapper that implements syscall.Conn by embedding + // both the raw connection and the encrypted connection. If the code + // attempts to read directly from the raw syscall.RawConn, it would read + // encrypted data. + switch r.(type) { + case *net.TCPConn, *net.UDPConn, *net.UnixConn, *net.IPConn: + default: + return nil + } + sysConn, ok := r.(syscall.Conn) + if !ok { + return nil + } + raw, err := sysConn.SyscallConn() + if err != nil { + return nil + } + rr := &nonBlockingReader{raw: raw} + rr.doRead = func(fd uintptr) bool { + s := &rr.state + + s.buf = s.pool.Get(s.bufSize) + s.bytesRead, s.readError = sysRead(fd, *s.buf) + + if s.readError != nil { + s.pool.Put(s.buf) + s.buf = nil + } + return !wouldBlock(s.readError) + } + return rr +} + +func (c *nonBlockingReader) ReadOnReady(bufSize int, pool mem.BufferPool) (*[]byte, int, error) { + c.state = readState{ + pool: pool, + bufSize: bufSize, + } + err := c.raw.Read(c.doRead) + + buf := c.state.buf + n := c.state.bytesRead + readErr := c.state.readError + c.state = readState{} + + if err != nil { + if buf != nil { + pool.Put(buf) + } + return nil, 0, err + } + if readErr != nil { + // buffer is already released in the callback. + return nil, 0, readErr + } + if n == 0 { + // syscall.Read doesn't consider a graceful socket closure to be an + // error condition, but Go's io.Reader expects an EOF error. + pool.Put(buf) + return nil, 0, io.EOF + } + return buf, n, nil +} + +type blockingReader struct { + reader io.Reader +} + +func (c *blockingReader) ReadOnReady(bufSize int, pool mem.BufferPool) (*[]byte, int, error) { + buf := pool.Get(bufSize) + n, err := c.reader.Read(*buf) + if err != nil { + pool.Put(buf) + return nil, 0, err + } + return buf, n, nil +} + +// New detects if [syscall.RawConn] is available for non-memory-pinning reads. +// If [syscall.RawConn] is unavailable, it falls back to using the simpler +// [io.Reader] interface for reads. +func New(r io.Reader) Reader { + if r := NewNonBlocking(r); r != nil { + return r + } + return &blockingReader{reader: r} +} + +// bufReadyReader implements buffering for a ReadyReader object. +// A new bufReadyReader is created by calling [NewBuffered]. +type bufReadyReader struct { + buf *[]byte + pool mem.BufferPool + bufSize int + rd Reader // reader provided by the caller + r, w int // buf read and write positions + err error + constPool constBufferPool // stored as a field to avoid heap allocations. +} + +// NewBuffered returns a new [io.Reader] with a buffer of the specified size +// which is allocated from the provided pool. +func NewBuffered(rd Reader, size int, pool mem.BufferPool) io.Reader { + return &bufReadyReader{ + rd: rd, + pool: pool, + bufSize: size, + } +} + +func (b *bufReadyReader) readErr() error { + err := b.err + b.err = nil + return err +} + +func (b *bufReadyReader) buffered() int { return b.w - b.r } + +// Read reads data into p. It returns the number of bytes read into p. The +// bytes are taken from at most one Read on the underlying [ReadyReader], +// hence n may be less than len(p). If the underlying [ReadyReader] can return +// a non-zero count with io.EOF, then this Read method can do so as well; see +// the [io.Reader] docs. +func (b *bufReadyReader) Read(p []byte) (n int, err error) { + n = len(p) + if n == 0 { + if b.buffered() > 0 { + return 0, nil + } + return 0, b.readErr() + } + if b.r == b.w { + if b.err != nil { + return 0, b.readErr() + } + if len(p) >= b.bufSize { + // Large read, empty buffer. + // Read directly into p to avoid copy. + b.constPool.buffer = p + _, n, b.err = b.rd.ReadOnReady(len(p), &b.constPool) + return n, b.readErr() + } + // One read. + b.r = 0 + b.w = 0 + b.buf, n, b.err = b.rd.ReadOnReady(b.bufSize, b.pool) + if n == 0 { + if b.buf != nil { + b.pool.Put(b.buf) + b.buf = nil + } + return 0, b.readErr() + } + b.w += n + } + + // copy as much as we can + // b.buf must be non-nil since b.r != b.w. + buf := *b.buf + n = copy(p, buf[b.r:b.w]) + b.r += n + if b.r == b.w { + // Consumed entire buffer, release it. + b.pool.Put(b.buf) + b.buf = nil + } + return n, nil +} + +type constBufferPool struct { + buffer []byte +} + +func (p *constBufferPool) Get(int) *[]byte { + return &p.buffer +} + +func (p *constBufferPool) Put(*[]byte) {} diff --git a/internal/transport/readyreader/ready_reader_ext_test.go b/internal/transport/readyreader/ready_reader_ext_test.go new file mode 100644 index 0000000..cd686fd --- /dev/null +++ b/internal/transport/readyreader/ready_reader_ext_test.go @@ -0,0 +1,414 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package readyreader_test + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "runtime" + "strings" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/transport/readyreader" + "google.golang.org/grpc/mem" +) + +var ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestReadyReader_NonRawConn verifies that ReadOnReady correctly reads data +// from a net.Conn that doesn't support non-memory-pinning reads. +func (s) TestReadyReader_NonRawConn(t *testing.T) { + data := []byte("hello world") + reader, writer := net.Pipe() + go writer.Write(data) + + pool := mem.DefaultBufferPool() + readyReader := readyreader.New(reader) + + bufHandle, n, err := readyReader.ReadOnReady(1024, pool) + if err != nil { + t.Fatalf("ReadOnReady() failed: %v", err) + } + defer pool.Put(bufHandle) + + if n != len(data) { + t.Errorf("n = %d; want %d", n, len(data)) + } + if !bytes.Equal((*bufHandle)[:n], data) { + t.Errorf("Read data = %s; want %s", string((*bufHandle)[:n]), string(data)) + } +} + +func (s) TestReadyReader_EOF(t *testing.T) { + ln, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen failed: %v", err) + } + defer ln.Close() + + data := []byte("hello") + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + if _, err := conn.Write(data); err != nil { + t.Errorf("Failed to write data: %v", err) + return + } + conn.Close() + }() + + conn, err := net.Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("net.Dial failed: %v", err) + } + defer conn.Close() + + pool := mem.DefaultBufferPool() + rr := readyreader.New(conn) + res, _, err := rr.ReadOnReady(len(data), pool) + if err != nil { + t.Errorf("Failed to read: %v", err) + return + } + + if !bytes.Equal(*res, data) { + t.Errorf("Read data = %s; want %s", string(*res), string(data)) + } + pool.Put(res) + + // Since the server closes the TCP connection, the next read should return + // an io.EOF. + res, _, err = rr.ReadOnReady(len(data), pool) + if err != io.EOF { + t.Errorf("Read after server connection close returned err = %v; want io.EOF", err) + } + if res != nil { + t.Error("ReadOnReady() returned non-nil buffer.") + } +} + +// Tests the behavior of readers wrapping TCP connections. It ensures that read +// operations do not hold onto an allocated buffer while the connection is idle. +// Buffers should only be permanently allocated from the pool when data is +// actually ready on the socket, optimizing memory usage for idle connections. +func (s) TestReadyReader_TCP(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("This test is only applicable for Linux, as RawConn functionality is not implemented for non-linux platforms.") + } + tests := []struct { + name string + read func(conn net.Conn, pool *trackingBufferPool, readBufSize int) ([]byte, error) + }{ + { + name: "ReadyReader", + read: func(conn net.Conn, pool *trackingBufferPool, readBufSize int) ([]byte, error) { + rr := readyreader.New(conn) + bufHandle, n, err := rr.ReadOnReady(readBufSize, pool) + if err != nil { + return nil, err + } + defer pool.Put(bufHandle) + res := make([]byte, n) + copy(res, (*bufHandle)[:n]) + return res, nil + }, + }, + { + name: "BufReadyReader", + read: func(conn net.Conn, pool *trackingBufferPool, readBufSize int) ([]byte, error) { + rr := readyreader.New(conn) + bufRR := readyreader.NewBuffered(rr, readBufSize, pool) + buf := make([]byte, 100) + n, err := bufRR.Read(buf) + if err != nil { + return nil, err + } + return buf[:n], nil + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), defaultTestTimeout) + defer cancel() + ln, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen failed: %v", err) + } + defer ln.Close() + + data := []byte("hello tcp delayed") + connCh := make(chan net.Conn) + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + connCh <- conn + }() + + conn, err := net.Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("net.Dial failed: %v", err) + } + defer conn.Close() + + serverConn := <-connCh + defer serverConn.Close() + + pool := newTrackingPool(mem.DefaultBufferPool()) + const readBufSize = 1024 + + resCh := make(chan []byte) + + go func() { + res, err := tc.read(conn, pool, readBufSize) + if err != nil { + t.Errorf("Failed to read: %v", err) + return + } + resCh <- res + }() + + // Verify that no read buffer is allocated for a short while. If it + // is allocated (e.g. for a probe), it must be returned immediately. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if n, err := pool.requestChan.Receive(sCtx); err == nil { + if n != readBufSize { + t.Fatalf("Unexpected request buffer size: got %d, want %d", n, readBufSize) + } + if n, err := pool.putChan.Receive(ctx); err != nil { + t.Fatal("Read buffer allocated and NOT returned while idle.") + } else if n != readBufSize { + t.Fatalf("Unexpected returned buffer size: got %d, want %d", n, readBufSize) + } + } + + // Write data to unblock. + serverConn.Write(data) + + if n, err := pool.requestChan.Receive(ctx); err != nil { + t.Fatal("Context timed out while waiting for a read buffer to be allocated.") + } else if n != readBufSize { + t.Fatalf("Unexpected request buffer size: got %d, want %d", n, readBufSize) + } + + var res []byte + select { + case res = <-resCh: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for read to complete.") + } + if !bytes.Equal(res, data) { + t.Errorf("Read data = %s; want %s", string(res), string(data)) + } + }) + } +} + +// trackingBufferPool wraps a mem.BufferPool and provides channels to track +// when buffers are requested and returned, useful for verifying allocation +// behavior in tests. +type trackingBufferPool struct { + mem.BufferPool + requestChan testutils.Channel + putChan testutils.Channel +} + +func newTrackingPool(pool mem.BufferPool) *trackingBufferPool { + return &trackingBufferPool{ + BufferPool: pool, + requestChan: *testutils.NewChannelWithSize(1), + putChan: *testutils.NewChannelWithSize(1), + } +} + +func (p *trackingBufferPool) Get(size int) *[]byte { + p.requestChan.Replace(size) + return p.BufferPool.Get(size) +} + +func (p *trackingBufferPool) Put(b *[]byte) { + p.putChan.Replace(len(*b)) + p.BufferPool.Put(b) +} + +// Call Read to accumulate the text of a file +func reads(buf io.Reader, m int) string { + var b [1000]byte + nb := 0 + for { + n, err := buf.Read(b[nb : nb+m]) + nb += n + if err == io.EOF { + break + } + } + return string(b[0:nb]) +} + +type bufReader struct { + name string + fn func(io.Reader) string +} + +var bufreaders = []bufReader{ + {"1", func(b io.Reader) string { return reads(b, 1) }}, + {"2", func(b io.Reader) string { return reads(b, 2) }}, + {"3", func(b io.Reader) string { return reads(b, 3) }}, + {"4", func(b io.Reader) string { return reads(b, 4) }}, + {"5", func(b io.Reader) string { return reads(b, 5) }}, + {"7", func(b io.Reader) string { return reads(b, 7) }}, +} + +const minReadBufferSize = 16 + +var bufsizes = []int{ + 0, minReadBufferSize, 23, 32, 46, 64, 93, 128, 1024, 4096, +} + +func (s) TestBufReader(t *testing.T) { + var texts [31]string + str := "" + all := "" + for i := range len(texts) - 1 { + texts[i] = str + "\n" + all += texts[i] + str += string(rune(i%26 + 'a')) + } + texts[len(texts)-1] = all + + for _, text := range texts { + for _, bufreader := range bufreaders { + for _, bufsize := range bufsizes { + // We don't use t.Run() here to avoid excessive logging due to + // the large number of subtests. + read := readyreader.New(strings.NewReader(text)) + buf := readyreader.NewBuffered(read, bufsize, mem.DefaultBufferPool()) + s := bufreader.fn(buf) + if s != text { + t.Errorf("fn=%s bufsize=%d want=%q got=%q", bufreader.name, bufsize, text, s) + } + } + } + } +} + +func (s) TestBufReader_ReadEmptyBuffer(t *testing.T) { + rr := readyreader.New(new(bytes.Buffer)) + l := readyreader.NewBuffered(rr, minReadBufferSize, mem.DefaultBufferPool()) + b := make([]byte, 100) + n, err := l.Read(b) + if err != io.EOF { + t.Errorf("expected EOF from Read, got %q %v", b[:n], err) + } +} + +type errorThenGoodReader struct { + didErr bool + nread int +} + +var errFake = errors.New("fake error") + +func (r *errorThenGoodReader) Read(p []byte) (int, error) { + r.nread++ + if !r.didErr { + r.didErr = true + return 0, errFake + } + return len(p), nil +} + +func (s) TestBufReader_ClearError(t *testing.T) { + r := &errorThenGoodReader{} + b := readyreader.NewBuffered(readyreader.New(r), minReadBufferSize, mem.DefaultBufferPool()) + buf := make([]byte, 1) + if _, err := b.Read(nil); err != nil { + t.Fatalf("1st nil Read = %v; want nil", err) + } + if _, err := b.Read(buf); err != errFake { + t.Fatalf("1st Read = %v; want errFake", err) + } + if _, err := b.Read(nil); err != nil { + t.Fatalf("2nd nil Read = %v; want nil", err) + } + if _, err := b.Read(buf); err != nil { + t.Fatalf("3rd Read with buffer = %v; want nil", err) + } + if r.nread != 2 { + t.Errorf("num reads = %d; want 2", r.nread) + } +} + +type emptyThenNonEmptyReader struct { + r io.Reader + n int +} + +func (r *emptyThenNonEmptyReader) Read(p []byte) (int, error) { + if r.n <= 0 { + return r.r.Read(p) + } + r.n-- + return 0, nil +} + +func (s) TestBufReader_ReadZero(t *testing.T) { + for _, size := range []int{100, 2} { + t.Run(fmt.Sprintf("bufsize=%d", size), func(t *testing.T) { + r := io.MultiReader(strings.NewReader("abc"), &emptyThenNonEmptyReader{r: strings.NewReader("def"), n: 1}) + br := readyreader.NewBuffered(readyreader.New(r), size, mem.DefaultBufferPool()) + want := func(s string, wantErr error) { + p := make([]byte, 50) + n, err := br.Read(p) + if err != wantErr || n != len(s) || string(p[:n]) != s { + t.Fatalf("read(%d) = %q, %v, want %q, %v", len(p), string(p[:n]), err, s, wantErr) + } + t.Logf("read(%d) = %q, %v", len(p), string(p[:n]), err) + } + want("abc", nil) + want("", nil) + want("def", nil) + want("", io.EOF) + }) + } +} diff --git a/internal/transport/server_metrics_ext_test.go b/internal/transport/server_metrics_ext_test.go new file mode 100644 index 0000000..b3db78f --- /dev/null +++ b/internal/transport/server_metrics_ext_test.go @@ -0,0 +1,185 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport_test + +import ( + "context" + "fmt" + "io" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/stubserver" + transportinternal "google.golang.org/grpc/internal/transport/internal" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func overrideTimeNowFunc(t *testing.T, f func() int64) { + orig := transportinternal.TimeNowFunc + transportinternal.TimeNowFunc = f + t.Cleanup(func() { transportinternal.TimeNowFunc = orig }) +} + +func verifyResultWithDelay(ctx context.Context, f func() (bool, error)) error { + var lastErr error + for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) { + var ok bool + if ok, lastErr = f(); ok { + return nil + } + } + return fmt.Errorf("Timeout when waiting for channelz to contain expected data. Last seen error: %v", lastErr) +} + +func (s) TestChannelz_ServerSocketMetricsLastMessageTimestamps(t *testing.T) { + channelz.TurnOn() + defer internal.ChannelzTurnOffForTesting() + + // Override timeNowFunc for the duration of the test. + var curTime atomic.Int64 + overrideTimeNowFunc(t, func() int64 { + curTime.Add(1) + return curTime.Load() + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start a stub server that echoes back messages sent by the client. This + // allows us to test both LastMessageReceivedTimestamp and + // LastMessageSentTimestamp on the server side. + var blockBeforeSend = make(chan struct{}, 1) + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + req, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + select { + case <-blockBeforeSend: + case <-ctx.Done(): + return ctx.Err() + } + if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: req.GetPayload()}); err != nil { + return err + } + } + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Failed to start test server: %v", err) + } + defer ss.Stop() + + // Wait for the server to be up and registered with channelz. + var svrID int64 + if err := verifyResultWithDelay(ctx, func() (bool, error) { + ssrvs, _ := channelz.GetServers(0, 0) + for _, s := range ssrvs { + for id := range s.ListenSockets() { + skt := channelz.GetSocket(id) + if skt != nil && skt.LocalAddr.String() == ss.Address { + svrID = s.ID + return true, nil + } + } + } + return false, fmt.Errorf("could not find server with address %s in channelz", ss.Address) + }); err != nil { + t.Fatal(err) + } + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall() failed: %v", err) + } + + // Wait for the message from the client to be received on the server and + // timestamp updated. + wantTimestamp := int64(1) + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("stream.Send(_) = %v, want ", err) + } + if err := verifyResultWithDelay(ctx, func() (bool, error) { + ns, _ := channelz.GetServerSockets(svrID, 0, 0) + if len(ns) == 0 { + return false, fmt.Errorf("no server sockets found") + } + sktData := &ns[0].SocketMetrics + if sktData.MessagesReceived.Load() < 1 { + return false, fmt.Errorf("server has not received any messages yet") + } + if gotTimestamp := sktData.LastMessageReceivedTimestamp.Load(); gotTimestamp != wantTimestamp { + return false, fmt.Errorf("LastMessageReceivedTimestamp is %d, want %d", gotTimestamp, wantTimestamp) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + blockBeforeSend <- struct{}{} + + // Wait for the message to be sent by the server and timestamp updated. + if _, err := stream.Recv(); err != nil { + t.Fatalf("stream.Recv() = %v, want ", err) + } + wantTimestamp = int64(2) + if err := verifyResultWithDelay(ctx, func() (bool, error) { + ns, _ := channelz.GetServerSockets(svrID, 0, 0) + sktData := &ns[0].SocketMetrics + if sktData.MessagesSent.Load() == 0 { + return false, fmt.Errorf("server has not sent any messages yet") + } + if gotTimestamp := sktData.LastMessageSentTimestamp.Load(); gotTimestamp != wantTimestamp { + return false, fmt.Errorf("LastMessageSentTimestamp = %d, want = %d", gotTimestamp, wantTimestamp) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + // Send another message to verify that LastMessageReceivedTimestamp is + // updated again. + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("stream.Send(_) = %v, want ", err) + } + wantTimestamp = int64(3) + if err := verifyResultWithDelay(ctx, func() (bool, error) { + ns, _ := channelz.GetServerSockets(svrID, 0, 0) + sktData := &ns[0].SocketMetrics + if sktData.MessagesReceived.Load() < 2 { + return false, fmt.Errorf("server has not received second message yet") + } + if gotTimestamp := sktData.LastMessageReceivedTimestamp.Load(); gotTimestamp != wantTimestamp { + return false, fmt.Errorf("LastMessageReceivedTimestamp is %d, want %d", gotTimestamp, wantTimestamp) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} diff --git a/internal/transport/server_stream.go b/internal/transport/server_stream.go new file mode 100644 index 0000000..ed6a13b --- /dev/null +++ b/internal/transport/server_stream.go @@ -0,0 +1,189 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "context" + "errors" + "strings" + "sync" + "sync/atomic" + + "google.golang.org/grpc/mem" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +// ServerStream implements streaming functionality for a gRPC server. +type ServerStream struct { + Stream // Embed for common stream functionality. + + st internalServerTransport + ctxDone <-chan struct{} // closed at the end of stream. Cache of ctx.Done() (for performance) + // cancel is invoked at the end of stream to cancel ctx. It also stops the + // timer for monitoring the rpc deadline if configured. + cancel func() + + // Holds compressor names passed in grpc-accept-encoding metadata from the + // client. + clientAdvertisedCompressors string + + // hdrMu protects outgoing header and trailer metadata. + hdrMu sync.Mutex + header metadata.MD // the outgoing header metadata. Updated by WriteHeader. + headerSent atomic.Bool // atomically set when the headers are sent out. + + headerWireLength int +} + +// Read reads an n byte message from the input stream. +func (s *ServerStream) Read(n int) (mem.BufferSlice, error) { + b, err := s.Stream.read(n) + if err == nil { + s.st.incrMsgRecv() + } + return b, err +} + +// SendHeader sends the header metadata for the given stream. +func (s *ServerStream) SendHeader(md metadata.MD) error { + return s.st.writeHeader(s, md) +} + +// Write writes the hdr and data bytes to the output stream. +func (s *ServerStream) Write(hdr []byte, data mem.BufferSlice, opts *WriteOptions) error { + return s.st.write(s, hdr, data, opts) +} + +// WriteStatus sends the status of a stream to the client. WriteStatus is +// the final call made on a stream and always occurs. +func (s *ServerStream) WriteStatus(st *status.Status) error { + return s.st.writeStatus(s, st) +} + +// isHeaderSent indicates whether headers have been sent. +func (s *ServerStream) isHeaderSent() bool { + return s.headerSent.Load() +} + +// updateHeaderSent updates headerSent and returns true +// if it was already set. +func (s *ServerStream) updateHeaderSent() bool { + return s.headerSent.Swap(true) +} + +// RecvCompress returns the compression algorithm applied to the inbound +// message. It is empty string if there is no compression applied. +func (s *ServerStream) RecvCompress() string { + return s.recvCompress +} + +// SendCompress returns the send compressor name. +func (s *ServerStream) SendCompress() string { + return s.sendCompress +} + +// ContentSubtype returns the content-subtype for a request. For example, a +// content-subtype of "proto" will result in a content-type of +// "application/grpc+proto". This will always be lowercase. See +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +func (s *ServerStream) ContentSubtype() string { + return s.contentSubtype +} + +// SetSendCompress sets the compression algorithm to the stream. +func (s *ServerStream) SetSendCompress(name string) error { + if s.isHeaderSent() || s.getState() == streamDone { + return errors.New("transport: set send compressor called after headers sent or stream done") + } + + s.sendCompress = name + return nil +} + +// SetContext sets the context of the stream. This will be deleted once the +// stats handler callouts all move to gRPC layer. +func (s *ServerStream) SetContext(ctx context.Context) { + s.ctx = ctx +} + +// ClientAdvertisedCompressors returns the compressor names advertised by the +// client via grpc-accept-encoding header. +func (s *ServerStream) ClientAdvertisedCompressors() []string { + values := strings.Split(s.clientAdvertisedCompressors, ",") + for i, v := range values { + values[i] = strings.TrimSpace(v) + } + return values +} + +// Header returns the header metadata of the stream. It returns the out header +// after t.WriteHeader is called. It does not block and must not be called +// until after WriteHeader. +func (s *ServerStream) Header() (metadata.MD, error) { + // Return the header in stream. It will be the out + // header after t.WriteHeader is called. + return s.header.Copy(), nil +} + +// HeaderWireLength returns the size of the headers of the stream as received +// from the wire. +func (s *ServerStream) HeaderWireLength() int { + return s.headerWireLength +} + +// SetHeader sets the header metadata. This can be called multiple times. +// This should not be called in parallel to other data writes. +func (s *ServerStream) SetHeader(md metadata.MD) error { + if md.Len() == 0 { + return nil + } + if s.isHeaderSent() || s.getState() == streamDone { + return ErrIllegalHeaderWrite + } + s.hdrMu.Lock() + s.header = metadata.Join(s.header, md) + s.hdrMu.Unlock() + return nil +} + +// SetTrailer sets the trailer metadata which will be sent with the RPC status +// by the server. This can be called multiple times. +// This should not be called parallel to other data writes. +func (s *ServerStream) SetTrailer(md metadata.MD) error { + if md.Len() == 0 { + return nil + } + if s.getState() == streamDone { + return ErrIllegalHeaderWrite + } + s.hdrMu.Lock() + s.trailer = metadata.Join(s.trailer, md) + s.hdrMu.Unlock() + return nil +} + +func (s *ServerStream) requestRead(n int) { + s.st.adjustWindow(s, uint32(n)) +} + +func (s *ServerStream) updateWindow(n int) { + s.st.updateWindow(s, uint32(n)) +} diff --git a/internal/transport/transport.go b/internal/transport/transport.go new file mode 100644 index 0000000..fc656b3 --- /dev/null +++ b/internal/transport/transport.go @@ -0,0 +1,878 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package transport defines and implements message oriented communication +// channel to complete various transactions (e.g., an RPC). It is meant for +// grpc-internal usage and is not intended to be imported directly by users. +package transport + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "sync" + "sync/atomic" + "time" + "unsafe" + + "golang.org/x/net/http2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/envconfig" + imem "google.golang.org/grpc/internal/mem" + "google.golang.org/grpc/internal/transport/internal" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" + "google.golang.org/grpc/tap" +) + +const ( + logLevel = 2 + // recvMsgSize estimates the memory overhead of a recvMsg in the backlog. + // It accounts for the recvMsg struct itself and the slice header of the + // underlying buffer's data. + recvMsgSize = int(unsafe.Sizeof(recvMsg{}) + unsafe.Sizeof([]byte{})) + + // utilizationFactor controls when we consider memory utilization acceptable. + // When backlogHeapSize / payloadSize <= utilizationFactor (meaning at least + // 50% of the heap memory is actual payload data), compaction is skipped. + utilizationFactor = 2 +) + +var ( + // compactionThreshold is approx 57KB (on 64-bit systems). It allows + // accumulating up to 1024 1-byte payloads before triggering compaction. + // + // Because individual payloads <= 1024 bytes are allocated on the heap + // outside mem.BufferPool, waiting for at least 1024 bytes to accumulate + // ensures that compaction coalesces those small heap allocations into a + // single large buffer from mem.BufferPool, enabling buffer reuse while + // avoiding frequent copying for small bursts of frames. + compactionThreshold = imem.BufferPoolingThreshold * (recvMsgSize + 1) +) + +func init() { + internal.TimeNowFunc = func() int64 { return time.Now().UnixNano() } +} + +// recvMsg represents the received msg from the transport. All transport +// protocol specific info has been removed. +type recvMsg struct { + buffer mem.Buffer + // nil: received some data + // io.EOF: stream is completed. data is nil. + // other non-nil error: transport failure. data is nil. + err error +} + +// recvBuffer is an unbounded channel of recvMsg structs. +// +// Note: recvBuffer differs from buffer.Unbounded in that it provides in-place +// value initialization (via init) to avoid struct pointer allocations on +// streams, and automatically frees pooled mem.Buffer payloads when stream +// errors occur. +type recvBuffer struct { + c chan recvMsg + mu sync.Mutex + backlog []recvMsg + // uncompactedSuffixLen tracks the number of consecutive data messages at + // the tail of backlog that have not been compacted. + uncompactedSuffixLen int + // uncompactedBytes tracks the total payload bytes across the trailing + // uncompactedSuffixLen messages. + uncompactedBytes int + err error + bufPool mem.BufferPool +} + +// init allows a recvBuffer to be initialized in-place, which is useful +// for resetting a buffer or for avoiding a heap allocation when the buffer +// is embedded in another struct. +func (b *recvBuffer) init(pool mem.BufferPool) { + b.c = make(chan recvMsg, 1) + b.bufPool = pool +} + +func (b *recvBuffer) put(r recvMsg) { + b.mu.Lock() + defer b.mu.Unlock() + if b.err != nil { + // drop the buffer on the floor. Since b.err is not nil, any subsequent reads + // will always return an error, making this buffer inaccessible. + r.buffer.Free() + // An error had occurred earlier, don't accept more + // data or errors. + return + } + b.err = r.err + if len(b.backlog) == 0 { + select { + case b.c <- r: + return + default: + } + } + b.backlog = append(b.backlog, r) + b.compactBacklogLocked(r) +} + +func (b *recvBuffer) compactBacklogLocked(r recvMsg) { + if !envconfig.EnableReceiveBufferCompaction { + return + } + if r.buffer == nil { + b.uncompactedBytes = 0 + b.uncompactedSuffixLen = 0 + return + } + + b.uncompactedSuffixLen++ + b.uncompactedBytes += r.buffer.Len() + backlogHeapSize := b.uncompactedSuffixLen*recvMsgSize + b.uncompactedBytes + + // If the memory overhead is less than 50% of the heap usage (e.g., because + // a large DATA frame arrived), the average message size in the suffix is + // large enough that memory bloat is not a concern. Reset suffix tracking. + if backlogHeapSize <= utilizationFactor*b.uncompactedBytes { + b.uncompactedBytes = 0 + b.uncompactedSuffixLen = 0 + return + } + // Avoid compacting too frequently for short bursts of small frames. + // Wait until we have accumulated at least ~1024 small messages (~57 KB). + if backlogHeapSize <= compactionThreshold { + // Still can accumulate more payloads. + return + } + + // Since the memory utilization is less than 50%, the average payload size + // of each recvMsg must be less than recvMsgSize (approx 56 bytes). + // In the worst case for bytes copied (where the average payload is just + // below recvMsgSize), compaction will occur once every: + // compactionThreshold / (recvMsgSize + avg_payload) = ~520 messages, + // copying ~29KB of data. + + start := 0 + newBuf := b.bufPool.Get(b.uncompactedBytes) + startIdx := len(b.backlog) - b.uncompactedSuffixLen + + for i := startIdx; i < len(b.backlog); i++ { + m := b.backlog[i] + b.backlog[i] = recvMsg{} + start += copy((*newBuf)[start:], m.buffer.ReadOnlyData()) + m.buffer.Free() + } + b.backlog[startIdx] = recvMsg{ + buffer: mem.NewBuffer(newBuf, b.bufPool), + } + b.backlog = b.backlog[:startIdx+1] + // After compaction, the suffix is replaced with a single message containing + // the combined payload. The new utilization is close to 1.0 (overhead of + // one recvMsg relative to the large compacted payload), which is well + // below the utilization factor of 2. + b.uncompactedBytes = 0 + b.uncompactedSuffixLen = 0 +} + +func (b *recvBuffer) load() { + b.mu.Lock() + if len(b.backlog) > 0 { + select { + case b.c <- b.backlog[0]: + // backlog[0] is only part of the tracked uncompacted suffix if the + // entire backlog currently consists of the suffix. If an earlier + // compaction or reset occurred, backlog[0] is already compacted. + if envconfig.EnableReceiveBufferCompaction && b.uncompactedSuffixLen == len(b.backlog) { + b.uncompactedSuffixLen-- + b.uncompactedBytes -= b.backlog[0].buffer.Len() + } + b.backlog[0] = recvMsg{} + b.backlog = b.backlog[1:] + default: + } + } + b.mu.Unlock() +} + +// get returns the channel that receives a recvMsg in the buffer. +// +// Upon receipt of a recvMsg, the caller should call load to send another +// recvMsg onto the channel if there is any. +func (b *recvBuffer) get() <-chan recvMsg { + return b.c +} + +// recvBufferReader implements io.Reader interface to read the data from +// recvBuffer. +type recvBufferReader struct { + _ noCopy + clientStream *ClientStream // The client transport stream is closed with a status representing ctx.Err() and nil trailer metadata. + ctx context.Context + ctxDone <-chan struct{} // cache of ctx.Done() (for performance). + recv *recvBuffer + last mem.Buffer // Stores the remaining data in the previous calls. + err error +} + +func (r *recvBufferReader) ReadMessageHeader(header []byte) (n int, err error) { + if r.err != nil { + return 0, r.err + } + if r.last != nil { + n, r.last = mem.ReadUnsafe(header, r.last) + return n, nil + } + if r.clientStream != nil { + n, r.err = r.readMessageHeaderClient(header) + } else { + n, r.err = r.readMessageHeader(header) + } + return n, r.err +} + +// Read reads the next n bytes from last. If last is drained, it tries to read +// additional data from recv. It blocks if there no additional data available in +// recv. If Read returns any non-nil error, it will continue to return that +// error. +func (r *recvBufferReader) Read(n int) (buf mem.Buffer, err error) { + if r.err != nil { + return nil, r.err + } + if r.last != nil { + buf = r.last + if r.last.Len() > n { + buf, r.last = mem.SplitUnsafe(buf, n) + } else { + r.last = nil + } + return buf, nil + } + if r.clientStream != nil { + buf, r.err = r.readClient(n) + } else { + buf, r.err = r.read(n) + } + return buf, r.err +} + +func (r *recvBufferReader) readMessageHeader(header []byte) (n int, err error) { + select { + case <-r.ctxDone: + return 0, ContextErr(r.ctx.Err()) + case m := <-r.recv.get(): + return r.readMessageHeaderAdditional(m, header) + } +} + +func (r *recvBufferReader) read(n int) (buf mem.Buffer, err error) { + select { + case <-r.ctxDone: + return nil, ContextErr(r.ctx.Err()) + case m := <-r.recv.get(): + return r.readAdditional(m, n) + } +} + +func (r *recvBufferReader) readMessageHeaderClient(header []byte) (n int, err error) { + // If the context is canceled, then closes the stream with nil metadata. + // closeStream writes its error parameter to r.recv as a recvMsg. + // r.readAdditional acts on that message and returns the necessary error. + select { + case <-r.ctxDone: + // Note that this adds the ctx error to the end of recv buffer, and + // reads from the head. This will delay the error until recv buffer is + // empty, thus will delay ctx cancellation in Recv(). + // + // It's done this way to fix a race between ctx cancel and trailer. The + // race was, stream.Recv() may return ctx error if ctxDone wins the + // race, but stream.Trailer() may return a non-nil md because the stream + // was not marked as done when trailer is received. This closeStream + // call will mark stream as done, thus fix the race. + // + // TODO: delaying ctx error seems like a unnecessary side effect. What + // we really want is to mark the stream as done, and return ctx error + // faster. + r.clientStream.Close(ContextErr(r.ctx.Err())) + m := <-r.recv.get() + return r.readMessageHeaderAdditional(m, header) + case m := <-r.recv.get(): + return r.readMessageHeaderAdditional(m, header) + } +} + +func (r *recvBufferReader) readClient(n int) (buf mem.Buffer, err error) { + // If the context is canceled, then closes the stream with nil metadata. + // closeStream writes its error parameter to r.recv as a recvMsg. + // r.readAdditional acts on that message and returns the necessary error. + select { + case <-r.ctxDone: + // Note that this adds the ctx error to the end of recv buffer, and + // reads from the head. This will delay the error until recv buffer is + // empty, thus will delay ctx cancellation in Recv(). + // + // It's done this way to fix a race between ctx cancel and trailer. The + // race was, stream.Recv() may return ctx error if ctxDone wins the + // race, but stream.Trailer() may return a non-nil md because the stream + // was not marked as done when trailer is received. This closeStream + // call will mark stream as done, thus fix the race. + // + // TODO: delaying ctx error seems like a unnecessary side effect. What + // we really want is to mark the stream as done, and return ctx error + // faster. + r.clientStream.Close(ContextErr(r.ctx.Err())) + m := <-r.recv.get() + return r.readAdditional(m, n) + case m := <-r.recv.get(): + return r.readAdditional(m, n) + } +} + +func (r *recvBufferReader) readMessageHeaderAdditional(m recvMsg, header []byte) (n int, err error) { + r.recv.load() + if m.err != nil { + if m.buffer != nil { + m.buffer.Free() + } + return 0, m.err + } + + n, r.last = mem.ReadUnsafe(header, m.buffer) + + return n, nil +} + +func (r *recvBufferReader) readAdditional(m recvMsg, n int) (b mem.Buffer, err error) { + r.recv.load() + if m.err != nil { + if m.buffer != nil { + m.buffer.Free() + } + return nil, m.err + } + + if m.buffer.Len() > n { + m.buffer, r.last = mem.SplitUnsafe(m.buffer, n) + } + + return m.buffer, nil +} + +type streamState uint32 + +const ( + streamActive streamState = iota + streamWriteDone // EndStream sent + streamReadDone // EndStream received + streamDone // the entire stream is finished. +) + +// Stream represents an RPC in the transport layer. +type Stream struct { + ctx context.Context // the associated context of the stream + method string // the associated RPC method of the stream + recvCompress string + sendCompress string + + readRequester readRequester + + // contentSubtype is the content-subtype for requests. + // this must be lowercase or the behavior is undefined. + contentSubtype string + + trailer metadata.MD // the key-value map of trailer metadata. + + // Non-pointer fields are at the end to optimize GC performance. + state streamState + id uint32 + buf recvBuffer + trReader transportReader + fc inFlow + wq writeQuota +} + +// readRequester is used to state application's intentions to read data. This +// is used to adjust flow control, if needed. +type readRequester interface { + requestRead(int) +} + +func (s *Stream) swapState(st streamState) streamState { + return streamState(atomic.SwapUint32((*uint32)(&s.state), uint32(st))) +} + +func (s *Stream) compareAndSwapState(oldState, newState streamState) bool { + return atomic.CompareAndSwapUint32((*uint32)(&s.state), uint32(oldState), uint32(newState)) +} + +func (s *Stream) getState() streamState { + return streamState(atomic.LoadUint32((*uint32)(&s.state))) +} + +// Trailer returns the cached trailer metadata. Note that if it is not called +// after the entire stream is done, it could return an empty MD. +// It can be safely read only after stream has ended that is either read +// or write have returned io.EOF. +func (s *Stream) Trailer() metadata.MD { + return s.trailer.Copy() +} + +// Context returns the context of the stream. +func (s *Stream) Context() context.Context { + return s.ctx +} + +// Method returns the method for the stream. +func (s *Stream) Method() string { + return s.method +} + +func (s *Stream) write(m recvMsg) { + s.buf.put(m) +} + +// ReadMessageHeader reads data into the provided header slice from the stream. +// It first checks if there was an error during a previous read operation and +// returns it if present. It then requests a read operation for the length of +// the header. It continues to read from the stream until the entire header +// slice is filled or an error occurs. If an `io.EOF` error is encountered with +// partially read data, it is converted to `io.ErrUnexpectedEOF` to indicate an +// unexpected end of the stream. The method returns any error encountered during +// the read process or nil if the header was successfully read. +func (s *Stream) ReadMessageHeader(header []byte) (err error) { + // Don't request a read if there was an error earlier + if er := s.trReader.er; er != nil { + return er + } + s.readRequester.requestRead(len(header)) + bytesRead := 0 + for len(header) != 0 { + n, err := s.trReader.ReadMessageHeader(header) + bytesRead += n + header = header[n:] + if len(header) == 0 { + err = nil + } + if err != nil { + if bytesRead > 0 && err == io.EOF { + err = io.ErrUnexpectedEOF + s.trReader.er = err + } + return err + } + } + return nil +} + +// ceil returns the ceil after dividing the numerator and denominator while +// avoiding integer overflows. +func ceil(numerator, denominator int) int { + if numerator == 0 { + return 0 + } + return (numerator-1)/denominator + 1 +} + +// Read reads n bytes from the wire for this stream. +func (s *Stream) read(n int) (data mem.BufferSlice, err error) { + // Don't request a read if there was an error earlier + if er := s.trReader.er; er != nil { + return nil, er + } + // gRPC Go accepts data frames with a maximum length of 16KB. Larger + // messages must be split into multiple frames. We pre-allocate the + // buffer to avoid resizing during the read loop, but cap the initial + // capacity to 128 frames (2MB) to prevent over-allocation or panics + // when reading extremely large streams. + allocCap := min(ceil(n, http2MaxFrameLen), 128) + data = make(mem.BufferSlice, 0, allocCap) + s.readRequester.requestRead(n) + bytesRead := 0 + for n != 0 { + buf, err := s.trReader.Read(n) + var bufLen int + if buf != nil { + bufLen = buf.Len() + } + bytesRead += bufLen + n -= bufLen + if n == 0 { + err = nil + } + if err != nil { + if bytesRead > 0 && err == io.EOF { + err = io.ErrUnexpectedEOF + s.trReader.er = err + } + data.Free() + return nil, err + } + data = append(data, buf) + } + return data, nil +} + +// noCopy may be embedded into structs which must not be copied +// after the first use. +// +// See https://golang.org/issues/8005#issuecomment-190753527 +// for details. +type noCopy struct { +} + +func (*noCopy) Lock() {} +func (*noCopy) Unlock() {} + +// transportReader reads all the data available for this Stream from the transport and +// passes them into the decoder, which converts them into a gRPC message stream. +// The error is io.EOF when the stream is done or another non-nil error if +// the stream broke. +type transportReader struct { + _ noCopy + // The handler to control the window update procedure for both this + // particular stream and the associated transport. + windowHandler windowHandler + er error + reader recvBufferReader +} + +// The handler to control the window update procedure for both this +// particular stream and the associated transport. +type windowHandler interface { + updateWindow(int) +} + +func (t *transportReader) ReadMessageHeader(header []byte) (int, error) { + n, err := t.reader.ReadMessageHeader(header) + if err != nil { + t.er = err + return 0, err + } + t.windowHandler.updateWindow(n) + return n, nil +} + +func (t *transportReader) Read(n int) (mem.Buffer, error) { + buf, err := t.reader.Read(n) + if err != nil { + t.er = err + return buf, err + } + t.windowHandler.updateWindow(buf.Len()) + return buf, nil +} + +// GoString is implemented by Stream so context.String() won't +// race when printing %#v. +func (s *Stream) GoString() string { + return fmt.Sprintf("", s, s.method) +} + +// state of transport +type transportState int + +const ( + reachable transportState = iota + closing + draining +) + +// ServerConfig consists of all the configurations to establish a server transport. +type ServerConfig struct { + MaxStreams uint32 + ConnectionTimeout time.Duration + Credentials credentials.TransportCredentials + InTapHandle tap.ServerInHandle + StatsHandler stats.Handler + KeepaliveParams keepalive.ServerParameters + KeepalivePolicy keepalive.EnforcementPolicy + InitialWindowSize int32 + InitialConnWindowSize int32 + WriteBufferSize int + ReadBufferSize int + SharedWriteBuffer bool + ChannelzParent *channelz.Server + MaxHeaderListSize *uint32 + HeaderTableSize *uint32 + BufferPool mem.BufferPool + StaticWindowSize bool +} + +// ConnectOptions covers all relevant options for communicating with the server. +type ConnectOptions struct { + // UserAgent is the application user agent. + UserAgent string + // Dialer specifies how to dial a network address. + Dialer func(context.Context, string) (net.Conn, error) + // FailOnNonTempDialError specifies if gRPC fails on non-temporary dial errors. + FailOnNonTempDialError bool + // PerRPCCredentials stores the PerRPCCredentials required to issue RPCs. + PerRPCCredentials []credentials.PerRPCCredentials + // TransportCredentials stores the Authenticator required to setup a client + // connection. Only one of TransportCredentials and CredsBundle is non-nil. + TransportCredentials credentials.TransportCredentials + // CredsBundle is the credentials bundle to be used. Only one of + // TransportCredentials and CredsBundle is non-nil. + CredsBundle credentials.Bundle + // KeepaliveParams stores the keepalive parameters. + KeepaliveParams keepalive.ClientParameters + // StatsHandlers stores the handler for stats. + StatsHandlers []stats.Handler + // InitialWindowSize sets the initial window size for a stream. + InitialWindowSize int32 + // InitialConnWindowSize sets the initial window size for a connection. + InitialConnWindowSize int32 + // WriteBufferSize sets the size of write buffer which in turn determines how much data can be batched before it's written on the wire. + WriteBufferSize int + // ReadBufferSize sets the size of read buffer, which in turn determines how much data can be read at most for one read syscall. + ReadBufferSize int + // SharedWriteBuffer indicates whether connections should reuse write buffer + SharedWriteBuffer bool + // ChannelzParent sets the addrConn id which initiated the creation of this client transport. + ChannelzParent *channelz.SubChannel + // MaxHeaderListSize sets the max (uncompressed) size of header list that is prepared to be received. + MaxHeaderListSize *uint32 + // The mem.BufferPool to use when reading/writing to the wire. + BufferPool mem.BufferPool + // StaticWindowSize controls whether dynamic window sizing is enabled. + StaticWindowSize bool +} + +// WriteOptions provides additional hints and information for message +// transmission. +type WriteOptions struct { + // Last indicates whether this write is the last piece for + // this stream. + Last bool +} + +// CallHdr carries the information of a particular RPC. +type CallHdr struct { + // Host specifies the peer's host. + Host string + + // Method specifies the operation to perform. + Method string + + // SendCompress specifies the compression algorithm applied on + // outbound message. + SendCompress string + + // AcceptedCompressors overrides the grpc-accept-encoding header for this + // call. When nil, the transport advertises the default set of registered + // compressors. A non-nil pointer overrides that value (including the empty + // string to advertise none). + AcceptedCompressors *string + + // Creds specifies credentials.PerRPCCredentials for a call. + Creds credentials.PerRPCCredentials + + // ContentSubtype specifies the content-subtype for a request. For example, a + // content-subtype of "proto" will result in a content-type of + // "application/grpc+proto". The value of ContentSubtype must be all + // lowercase, otherwise the behavior is undefined. See + // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests + // for more details. + ContentSubtype string + + PreviousAttempts int // value of grpc-previous-rpc-attempts header to set + + // Authority is used to explicitly override the `:authority` header. + // + // This value comes from one of two sources: + // 1. The `CallAuthority` call option, if specified by the user. + // 2. An override provided by the LB picker (e.g. xDS authority rewriting). + // + // The `CallAuthority` call option always takes precedence over the LB + // picker override. + Authority string +} + +// ClientTransport is the common interface for all gRPC client-side transport +// implementations. +type ClientTransport interface { + // Close tears down this transport. Once it returns, the transport + // should not be accessed any more. The caller must make sure this + // is called only once. + Close(err error) + + // GracefulClose starts to tear down the transport: the transport will stop + // accepting new RPCs and NewStream will return error. Once all streams are + // finished, the transport will close. + // + // It does not block. + GracefulClose() + + // NewStream creates a Stream for an RPC. + NewStream(ctx context.Context, callHdr *CallHdr, handler stats.Handler) (*ClientStream, error) + + // Error returns a channel that is closed when some I/O error + // happens. Typically the caller should have a goroutine to monitor + // this in order to take action (e.g., close the current transport + // and create a new one) in error case. It should not return nil + // once the transport is initiated. + Error() <-chan struct{} + + // GoAway returns a channel that is closed when ClientTransport + // receives the draining signal from the server (e.g., GOAWAY frame in + // HTTP/2). + GoAway() <-chan struct{} + + // GetGoAwayReason returns the reason why GoAway frame was received, along + // with a human readable string with debug info. + GetGoAwayReason() (GoAwayReason, string) + + // Peer returns information about the peer associated with the Transport. + // The returned information includes authentication and network address details. + Peer() *peer.Peer +} + +// ServerTransport is the common interface for all gRPC server-side transport +// implementations. +// +// Methods may be called concurrently from multiple goroutines, but +// Write methods for a given Stream will be called serially. +type ServerTransport interface { + // HandleStreams receives incoming streams using the given handler. + HandleStreams(context.Context, func(*ServerStream)) + + // Close tears down the transport. Once it is called, the transport + // should not be accessed any more. All the pending streams and their + // handlers will be terminated asynchronously. + Close(err error) + + // Peer returns the peer of the server transport. + Peer() *peer.Peer + + // Drain notifies the client this ServerTransport stops accepting new RPCs. + Drain(debugData string) +} + +type internalServerTransport interface { + ServerTransport + writeHeader(s *ServerStream, md metadata.MD) error + write(s *ServerStream, hdr []byte, data mem.BufferSlice, opts *WriteOptions) error + writeStatus(s *ServerStream, st *status.Status) error + incrMsgRecv() + adjustWindow(s *ServerStream, n uint32) + updateWindow(s *ServerStream, n uint32) +} + +// connectionErrorf creates an ConnectionError with the specified error description. +func connectionErrorf(temp bool, e error, format string, a ...any) ConnectionError { + return ConnectionError{ + Desc: fmt.Sprintf(format, a...), + temp: temp, + err: e, + } +} + +// ConnectionError is an error that results in the termination of the +// entire connection and the retry of all the active streams. +type ConnectionError struct { + Desc string + temp bool + err error +} + +func (e ConnectionError) Error() string { + return fmt.Sprintf("connection error: desc = %q", e.Desc) +} + +// Temporary indicates if this connection error is temporary or fatal. +func (e ConnectionError) Temporary() bool { + return e.temp +} + +// Origin returns the original error of this connection error. +func (e ConnectionError) Origin() error { + // Never return nil error here. + // If the original error is nil, return itself. + if e.err == nil { + return e + } + return e.err +} + +// Unwrap returns the original error of this connection error or nil when the +// origin is nil. +func (e ConnectionError) Unwrap() error { + return e.err +} + +var ( + // ErrConnClosing indicates that the transport is closing. + ErrConnClosing = connectionErrorf(true, nil, "transport is closing") + // errStreamDrain indicates that the stream is rejected because the + // connection is draining. This could be caused by goaway or balancer + // removing the address. + errStreamDrain = status.Error(codes.Unavailable, "the connection is draining") + // errStreamDone is returned from write at the client side to indicate application + // layer of an error. + errStreamDone = errors.New("the stream is done") + // StatusGoAway indicates that the server sent a GOAWAY that included this + // stream's ID in unprocessed RPCs. + statusGoAway = status.New(codes.Unavailable, "the stream is rejected because server is draining the connection") +) + +// GoAwayReason contains the reason for the GoAway frame received. +type GoAwayReason uint8 + +const ( + // GoAwayInvalid indicates that no GoAway frame is received. + GoAwayInvalid GoAwayReason = 0 + // GoAwayNoReason is the default value when GoAway frame is received. + GoAwayNoReason GoAwayReason = 1 + // GoAwayTooManyPings indicates that a GoAway frame with + // ErrCodeEnhanceYourCalm was received and that the debug data said + // "too_many_pings". + GoAwayTooManyPings GoAwayReason = 2 +) + +// GoAwayInfo contains metadata about why a connection was closed. +type GoAwayInfo struct { + // Reason is the parsed reason for an HTTP/2 GOAWAY frame. + Reason GoAwayReason + // GoAwayCode is the raw HTTP/2 error code received in a GOAWAY frame. + GoAwayCode http2.ErrCode + // Err is the underlying error that caused the connection to close. It is + // populated if the connection was closed due to a socket error or context + // cancellation without receiving a GOAWAY frame. If the connection was + // closed due to a GOAWAY frame, this field will be nil. + Err error +} + +// OnCloseFunc is a callback invoked when a ClientTransport closes. +type OnCloseFunc func(GoAwayInfo) + +// ContextErr converts the error from context package into a status error. +func ContextErr(err error) error { + switch err { + case context.DeadlineExceeded: + return status.Error(codes.DeadlineExceeded, err.Error()) + case context.Canceled: + return status.Error(codes.Canceled, err.Error()) + } + return status.Errorf(codes.Internal, "Unexpected error from context packet: %v", err) +} diff --git a/internal/wrr/edf.go b/internal/wrr/edf.go new file mode 100644 index 0000000..a06656f --- /dev/null +++ b/internal/wrr/edf.go @@ -0,0 +1,92 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wrr + +import ( + "container/heap" + "sync" +) + +// edfWrr is a struct for EDF weighted round robin implementation. +type edfWrr struct { + lock sync.Mutex + items edfPriorityQueue + currentOrderOffset uint64 + currentTime float64 +} + +// NewEDF creates Earliest Deadline First (EDF) +// (https://en.wikipedia.org/wiki/Earliest_deadline_first_scheduling) implementation for weighted round robin. +// Each pick from the schedule has the earliest deadline entry selected. Entries have deadlines set +// at current time + 1 / weight, providing weighted round robin behavior with O(log n) pick time. +func NewEDF() WRR { + return &edfWrr{} +} + +// edfEntry is an internal wrapper for item that also stores weight and relative position in the queue. +type edfEntry struct { + deadline float64 + weight int64 + orderOffset uint64 + item any +} + +// edfPriorityQueue is a heap.Interface implementation for edfEntry elements. +type edfPriorityQueue []*edfEntry + +func (pq edfPriorityQueue) Len() int { return len(pq) } +func (pq edfPriorityQueue) Less(i, j int) bool { + return pq[i].deadline < pq[j].deadline || pq[i].deadline == pq[j].deadline && pq[i].orderOffset < pq[j].orderOffset +} +func (pq edfPriorityQueue) Swap(i, j int) { pq[i], pq[j] = pq[j], pq[i] } + +func (pq *edfPriorityQueue) Push(x any) { + *pq = append(*pq, x.(*edfEntry)) +} + +func (pq *edfPriorityQueue) Pop() any { + old := *pq + *pq = old[0 : len(old)-1] + return old[len(old)-1] +} + +func (edf *edfWrr) Add(item any, weight int64) { + edf.lock.Lock() + defer edf.lock.Unlock() + entry := edfEntry{ + deadline: edf.currentTime + 1.0/float64(weight), + weight: weight, + item: item, + orderOffset: edf.currentOrderOffset, + } + edf.currentOrderOffset++ + heap.Push(&edf.items, &entry) +} + +func (edf *edfWrr) Next() any { + edf.lock.Lock() + defer edf.lock.Unlock() + if len(edf.items) == 0 { + return nil + } + item := edf.items[0] + edf.currentTime = item.deadline + item.deadline = edf.currentTime + 1.0/float64(item.weight) + heap.Fix(&edf.items, 0) + return item.item +} diff --git a/internal/wrr/edf_test.go b/internal/wrr/edf_test.go new file mode 100644 index 0000000..c3d0c54 --- /dev/null +++ b/internal/wrr/edf_test.go @@ -0,0 +1,35 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package wrr + +import ( + "testing" +) + +func (s) TestEDFOnEndpointsWithSameWeight(t *testing.T) { + wrr := NewEDF() + wrr.Add("1", 1) + wrr.Add("2", 1) + wrr.Add("3", 1) + expected := []string{"1", "2", "3", "1", "2", "3", "1", "2", "3", "1", "2", "3"} + for i := 0; i < len(expected); i++ { + item := wrr.Next().(string) + if item != expected[i] { + t.Errorf("wrr Next=%s, want=%s", item, expected[i]) + } + } +} diff --git a/internal/wrr/random.go b/internal/wrr/random.go new file mode 100644 index 0000000..0913ed6 --- /dev/null +++ b/internal/wrr/random.go @@ -0,0 +1,84 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wrr + +import ( + "fmt" + rand "math/rand/v2" + "sort" +) + +// weightedItem is a wrapped weighted item that is used to implement weighted random algorithm. +type weightedItem struct { + item any + weight int64 + accumulatedWeight int64 +} + +func (w *weightedItem) String() string { + return fmt.Sprint(*w) +} + +// randomWRR is a struct that contains weighted items implement weighted random algorithm. +type randomWRR struct { + items []*weightedItem + // Are all item's weights equal + equalWeights bool +} + +// NewRandom creates a new WRR with random. +func NewRandom() WRR { + return &randomWRR{} +} + +var randInt64n = rand.Int64N + +func (rw *randomWRR) Next() (item any) { + if len(rw.items) == 0 { + return nil + } + if rw.equalWeights { + return rw.items[randInt64n(int64(len(rw.items)))].item + } + + sumOfWeights := rw.items[len(rw.items)-1].accumulatedWeight + // Random number in [0, sumOfWeights). + randomWeight := randInt64n(sumOfWeights) + // Item's accumulated weights are in ascending order, because item's weight >= 0. + // Binary search rw.items to find first item whose accumulatedWeight > randomWeight + // The return i is guaranteed to be in range [0, len(rw.items)) because randomWeight < last item's accumulatedWeight + i := sort.Search(len(rw.items), func(i int) bool { return rw.items[i].accumulatedWeight > randomWeight }) + return rw.items[i].item +} + +func (rw *randomWRR) Add(item any, weight int64) { + accumulatedWeight := weight + equalWeights := true + if len(rw.items) > 0 { + lastItem := rw.items[len(rw.items)-1] + accumulatedWeight = lastItem.accumulatedWeight + weight + equalWeights = rw.equalWeights && weight == lastItem.weight + } + rw.equalWeights = equalWeights + rItem := &weightedItem{item: item, weight: weight, accumulatedWeight: accumulatedWeight} + rw.items = append(rw.items, rItem) +} + +func (rw *randomWRR) String() string { + return fmt.Sprint(rw.items) +} diff --git a/internal/wrr/wrr.go b/internal/wrr/wrr.go new file mode 100644 index 0000000..91a40d7 --- /dev/null +++ b/internal/wrr/wrr.go @@ -0,0 +1,32 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package wrr contains the interface and common implementations of wrr +// algorithms. +package wrr + +// WRR defines an interface that implements weighted round robin. +type WRR interface { + // Add adds an item with weight to the WRR set. Add must be only called + // before any calls to Next. + Add(item any, weight int64) + // Next returns the next picked item. + // + // Next needs to be thread safe. Add may not be called after any call to + // Next. + Next() any +} diff --git a/internal/wrr/wrr_test.go b/internal/wrr/wrr_test.go new file mode 100644 index 0000000..4a7d81e --- /dev/null +++ b/internal/wrr/wrr_test.go @@ -0,0 +1,193 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wrr + +import ( + "errors" + "math" + rand "math/rand/v2" + "strconv" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const iterCount = 10000 + +func equalApproximate(a, b float64) error { + opt := cmp.Comparer(func(x, y float64) bool { + delta := math.Abs(x - y) + mean := math.Abs(x+y) / 2.0 + return delta/mean < 0.05 + }) + if !cmp.Equal(a, b, opt) { + return errors.New(cmp.Diff(a, b)) + } + return nil +} + +func testWRRNext(t *testing.T, newWRR func() WRR) { + tests := []struct { + name string + weights []int64 + }{ + { + name: "1-1-1", + weights: []int64{1, 1, 1}, + }, + { + name: "1-2-3", + weights: []int64{1, 2, 3}, + }, + { + name: "5-3-2", + weights: []int64{5, 3, 2}, + }, + { + name: "17-23-37", + weights: []int64{17, 23, 37}, + }, + { + name: "no items", + weights: []int64{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := newWRR() + if len(tt.weights) == 0 { + if next := w.Next(); next != nil { + t.Fatalf("w.Next returns non nil value:%v when there is no item", next) + } + return + } + + var sumOfWeights int64 + for i, weight := range tt.weights { + w.Add(i, weight) + sumOfWeights += weight + } + + results := make(map[int]int) + for i := 0; i < iterCount; i++ { + results[w.Next().(int)]++ + } + + wantRatio := make([]float64, len(tt.weights)) + for i, weight := range tt.weights { + wantRatio[i] = float64(weight) / float64(sumOfWeights) + } + gotRatio := make([]float64, len(tt.weights)) + for i, count := range results { + gotRatio[i] = float64(count) / iterCount + } + + for i := range wantRatio { + if err := equalApproximate(gotRatio[i], wantRatio[i]); err != nil { + t.Errorf("%v not equal %v", i, err) + } + } + }) + } +} + +func (s) TestRandomWRRNext(t *testing.T) { + testWRRNext(t, NewRandom) +} + +func (s) TestEdfWrrNext(t *testing.T) { + testWRRNext(t, NewEDF) +} + +func BenchmarkRandomWRRNext(b *testing.B) { + for _, n := range []int{100, 500, 1000} { + b.Run("equal-weights-"+strconv.Itoa(n)+"-items", func(b *testing.B) { + w := NewRandom() + sumOfWeights := n + for i := 0; i < n; i++ { + w.Add(i, 1) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + for i := 0; i < sumOfWeights; i++ { + w.Next() + } + } + }) + } + + var maxWeight int64 = 1024 + for _, n := range []int{100, 500, 1000} { + b.Run("random-weights-"+strconv.Itoa(n)+"-items", func(b *testing.B) { + w := NewRandom() + var sumOfWeights int64 + for i := 0; i < n; i++ { + weight := rand.Int64N(maxWeight + 1) + w.Add(i, weight) + sumOfWeights += weight + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + for i := 0; i < int(sumOfWeights); i++ { + w.Next() + } + } + }) + } + + itemsNum := 200 + heavyWeight := int64(itemsNum) + lightWeight := int64(1) + heavyIndices := []int{0, itemsNum / 2, itemsNum - 1} + for _, heavyIndex := range heavyIndices { + b.Run("skew-weights-heavy-index-"+strconv.Itoa(heavyIndex), func(b *testing.B) { + w := NewRandom() + var sumOfWeights int64 + for i := 0; i < itemsNum; i++ { + var weight int64 + if i == heavyIndex { + weight = heavyWeight + } else { + weight = lightWeight + } + sumOfWeights += weight + w.Add(i, weight) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + for i := 0; i < int(sumOfWeights); i++ { + w.Next() + } + } + }) + } +} + +func init() { + r := rand.New(rand.NewPCG(0, 0)) + randInt64n = r.Int64N +} diff --git a/internal/xds/balancer/balancer.go b/internal/xds/balancer/balancer.go new file mode 100644 index 0000000..c61fd9d --- /dev/null +++ b/internal/xds/balancer/balancer.go @@ -0,0 +1,30 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package balancer installs all the xds balancers. +package balancer + +import ( + _ "google.golang.org/grpc/balancer/leastrequest" // Register the least_request_experimental balancer + _ "google.golang.org/grpc/balancer/weightedtarget" // Register the weighted_target balancer + _ "google.golang.org/grpc/internal/xds/balancer/cdsbalancer" // Register the CDS balancer + _ "google.golang.org/grpc/internal/xds/balancer/clusterimpl" // Register the xds_cluster_impl balancer + _ "google.golang.org/grpc/internal/xds/balancer/clustermanager" // Register the xds_cluster_manager balancer + _ "google.golang.org/grpc/internal/xds/balancer/outlierdetection" // Register the outlier_detection balancer + _ "google.golang.org/grpc/internal/xds/balancer/priority" // Register the priority balancer +) diff --git a/internal/xds/balancer/cdsbalancer/aggregate_cluster_test.go b/internal/xds/balancer/cdsbalancer/aggregate_cluster_test.go new file mode 100644 index 0000000..9cb040d --- /dev/null +++ b/internal/xds/balancer/cdsbalancer/aggregate_cluster_test.go @@ -0,0 +1,932 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cdsbalancer + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/balancer/priority" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/balancer/pickfirst" + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter. + _ "google.golang.org/grpc/internal/xds/resolver" // Register the xds resolver +) + +// makeAggregateClusterResource returns an aggregate cluster resource with the +// given name and list of child names. +func makeAggregateClusterResource(name string, childNames []string) *v3clusterpb.Cluster { + return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: name, + Type: e2e.ClusterTypeAggregate, + ChildNames: childNames, + }) +} + +// makeLogicalDNSClusterResource returns a LOGICAL_DNS cluster resource with the +// given name and given DNS host and port. +func makeLogicalDNSClusterResource(name, dnsHost string, dnsPort uint32) *v3clusterpb.Cluster { + return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: name, + Type: e2e.ClusterTypeLogicalDNS, + DNSHostName: dnsHost, + DNSPort: dnsPort, + }) +} + +// verifyDNSResolution verifies that the DNS resolver is started for the expected target. +func verifyDNSResolution(ctx context.Context, t *testing.T, dnsTargetCh chan resolver.Target, dnsR *manual.Resolver, host string, port uint32) { + t.Helper() + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for DNS watch") + case target := <-dnsTargetCh: + addr := fmt.Sprintf("%s:%d", host, port) + if target.Endpoint() != addr { + t.Fatalf("DNS resolution started for target %q, want %q", target.Endpoint(), addr) + } + dnsR.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: addr}}}}, + }) + } +} + +// Tests the case where the cluster resource requested is a leaf cluster. The +// management server sends two updates for the same leaf cluster resource. The +// test verifies that the load balancing configuration pushed to the priority LB +// policy contains the expected discovery mechanism corresponding to the leaf +// cluster, on both occasions. +func (s) TestAggregateClusterSuccess_LeafNode(t *testing.T) { + tests := []struct { + name string + firstClusterResource *v3clusterpb.Cluster + secondClusterResource *v3clusterpb.Cluster + wantFirstChildCfg serviceconfig.LoadBalancingConfig + wantSecondChildCfg serviceconfig.LoadBalancingConfig + }{ + { + name: "eds", + firstClusterResource: e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone), + secondClusterResource: e2e.DefaultCluster(clusterName, serviceName+"-new", e2e.SecurityLevelNone), + wantFirstChildCfg: &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: createPriorityConfig(clusterName), + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0"}, + }, + wantSecondChildCfg: &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-1-0": { + Config: createPriorityConfig(clusterName), + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-1-0"}, + }, + }, + { + name: "dns", + firstClusterResource: makeLogicalDNSClusterResource(clusterName, "dns_host", uint32(port)), + secondClusterResource: makeLogicalDNSClusterResource(clusterName, "dns_host_new", uint32(port)), + wantFirstChildCfg: &priority.LBConfig{ + Children: map[string]*priority.Child{"priority-0": {Config: createPriorityConfig(clusterName)}}, + Priorities: []string{"priority-0"}, + }, + wantSecondChildCfg: &priority.LBConfig{ + Children: map[string]*priority.Child{"priority-1": {Config: createPriorityConfig(clusterName)}}, + Priorities: []string{"priority-1"}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t) + mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil) + + // Push the first cluster resource through the management server and + // verify the configuration pushed to the child policy. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{test.firstClusterResource}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := compareLoadBalancingConfig(ctx, lbCfgCh, test.wantFirstChildCfg); err != nil { + t.Fatal(err) + } + + // Push the second cluster resource through the management server and + // verify the configuration pushed to the child policy. + resources.Clusters[0] = test.secondClusterResource + resources.Endpoints[0] = e2e.DefaultEndpoint(serviceName+"-new", host, []uint32{port}) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := compareLoadBalancingConfig(ctx, lbCfgCh, test.wantSecondChildCfg); err != nil { + t.Fatal(err) + } + }) + } +} + +// Tests the case where the cluster resource requested is an aggregate cluster +// root pointing to two child clusters, one of type EDS and the other of type +// LogicalDNS. The test verifies that load balancing configuration is pushed to +// the priority LB policy only when all child clusters are resolved and it also +// verifies that the pushed configuration contains the expected priority config. +// The test then updates the aggregate cluster to point to two child clusters, +// the same leaf cluster of type EDS and a different leaf cluster of type +// LogicalDNS and verifies that the load balancing configuration pushed to the +// priority LB policy contains the expected config. +func (s) TestAggregateClusterSuccess_ThenUpdateChildClusters(t *testing.T) { + dnsTargetCh, dnsR := setupDNS(t) + lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t) + mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil) + + // Configure the management server with the aggregate cluster resource + // pointing to two child clusters, one EDS and one LogicalDNS. Include the + // resource corresponding to the EDS cluster here, but don't include + // resource corresponding to the LogicalDNS cluster yet. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})}, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that no configuration is pushed to the child policy yet, because + // not all clusters making up the aggregate cluster have been resolved yet. + select { + case cfg := <-lbCfgCh: + t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg)) + case <-time.After(defaultTestShortTimeout): + } + + // Now configure the LogicalDNS cluster in the management server. This + // should result in configuration being pushed down to the child policy. + resources.Clusters = append(resources.Clusters, makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort)) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + verifyDNSResolution(ctx, t, dnsTargetCh, dnsR, dnsHostName, dnsPort) + + wantChildCfg := &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: createPriorityConfig(edsClusterName), + IgnoreReresolutionRequests: true, + }, + "priority-1": {Config: createPriorityConfig(dnsClusterName)}, + }, + Priorities: []string{"priority-0-0", "priority-1"}, + } + if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil { + t.Fatal(err) + } + + const dnsClusterNameNew = dnsClusterName + "-new" + const dnsHostNameNew = dnsHostName + "-new" + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterNameNew}), + e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + makeLogicalDNSClusterResource(dnsClusterNameNew, dnsHostNameNew, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + verifyDNSResolution(ctx, t, dnsTargetCh, dnsR, dnsHostNameNew, dnsPort) + + wantChildCfg = &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: createPriorityConfig(edsClusterName), + IgnoreReresolutionRequests: true, + }, + "priority-2": {Config: createPriorityConfig(dnsClusterNameNew)}, + }, + Priorities: []string{"priority-0-0", "priority-2"}, + } + if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil { + t.Fatal(err) + } +} + +// Tests the case where the cluster resource requested is an aggregate cluster +// root pointing to two child clusters, one of type EDS and the other of type +// LogicalDNS. The test verifies that the load balancing configuration pushed to +// the priority LB policy contains the discovery mechanisms for both child +// clusters. The test then updates the root cluster resource requested by the +// cds LB policy to a leaf cluster of type EDS and verifies the load balancing +// configuration pushed to the priority LB policy contains a single discovery +// mechanism. +func (s) TestAggregateClusterSuccess_ThenChangeRootToEDS(t *testing.T) { + dnsTargetCh, dnsR := setupDNS(t) + lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t) + mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil) + + // Configure the management server with the aggregate cluster resource + // pointing to two child clusters. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})}, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + verifyDNSResolution(ctx, t, dnsTargetCh, dnsR, dnsHostName, dnsPort) + + wantChildCfg := &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: createPriorityConfig(edsClusterName), + IgnoreReresolutionRequests: true, + }, + "priority-1": {Config: createPriorityConfig(dnsClusterName)}, + }, + Priorities: []string{"priority-0-0", "priority-1"}, + } + if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil { + t.Fatal(err) + } + + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + // Since the service name of the EDS cluster remains same, same priority name + // is used. + wantChildCfg = &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: createPriorityConfig(clusterName), + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0"}, + } + if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil { + t.Fatal(err) + } +} + +// Tests the case where a requested cluster resource switches between being a +// leaf and an aggregate cluster pointing to an EDS and LogicalDNS child +// cluster. In each of these cases, the test verifies that the load balancing +// configuration pushed to the priority LB policy contains the expected config. +func (s) TestAggregatedClusterSuccess_SwitchBetweenLeafAndAggregate(t *testing.T) { + dnsTargetCh, dnsR := setupDNS(t) + lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t) + mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil) + + // Start off with the requested cluster being a leaf EDS cluster. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})}, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + wantChildCfg := &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: createPriorityConfig(clusterName), + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0"}, + } + if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil { + t.Fatal(err) + } + + // Switch the requested cluster to be an aggregate cluster pointing to two + // child clusters. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + verifyDNSResolution(ctx, t, dnsTargetCh, dnsR, dnsHostName, dnsPort) + + wantChildCfg = &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: createPriorityConfig(edsClusterName), + IgnoreReresolutionRequests: true, + }, + "priority-1": {Config: createPriorityConfig(dnsClusterName)}, + }, + Priorities: []string{"priority-0-0", "priority-1"}, + } + if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil { + t.Fatal(err) + } + + // Switch the cluster back to a leaf EDS cluster. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + wantChildCfg = &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: createPriorityConfig(clusterName), + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0"}, + } + if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil { + t.Fatal(err) + } +} + +// Tests the scenario where an aggregate cluster exceeds the maximum depth, +// which is 16. Verifies that the channel moves to TRANSIENT_FAILURE, and the +// error is propagated to RPC callers. The test then modifies the graph to no +// longer exceed maximum depth, but be at the maximum allowed depth, and +// verifies that an RPC can be made successfully. +func (s) TestAggregatedClusterFailure_ExceedsMaxStackDepth(t *testing.T) { + mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil) + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{clusterName + "-1"}), + makeAggregateClusterResource(clusterName+"-1", []string{clusterName + "-2"}), + makeAggregateClusterResource(clusterName+"-2", []string{clusterName + "-3"}), + makeAggregateClusterResource(clusterName+"-3", []string{clusterName + "-4"}), + makeAggregateClusterResource(clusterName+"-4", []string{clusterName + "-5"}), + makeAggregateClusterResource(clusterName+"-5", []string{clusterName + "-6"}), + makeAggregateClusterResource(clusterName+"-6", []string{clusterName + "-7"}), + makeAggregateClusterResource(clusterName+"-7", []string{clusterName + "-8"}), + makeAggregateClusterResource(clusterName+"-8", []string{clusterName + "-9"}), + makeAggregateClusterResource(clusterName+"-9", []string{clusterName + "-10"}), + makeAggregateClusterResource(clusterName+"-10", []string{clusterName + "-11"}), + makeAggregateClusterResource(clusterName+"-11", []string{clusterName + "-12"}), + makeAggregateClusterResource(clusterName+"-12", []string{clusterName + "-13"}), + makeAggregateClusterResource(clusterName+"-13", []string{clusterName + "-14"}), + makeAggregateClusterResource(clusterName+"-14", []string{clusterName + "-15"}), + makeAggregateClusterResource(clusterName+"-15", []string{clusterName + "-16"}), + e2e.DefaultCluster(clusterName+"-16", serviceName, e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})}, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + const wantErr = "aggregate cluster graph exceeds max depth" + client := testgrpc.NewTestServiceClient(cc) + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if code := status.Code(err); code != codes.Unavailable { + t.Fatalf("EmptyCall() failed with code: %v, want %v", code, codes.Unavailable) + } + if err != nil && !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall() failed with err: %v, want err containing: %v", err, wantErr) + } + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + + // Update the aggregate cluster resource to no longer exceed max depth, and + // be at the maximum depth allowed. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{clusterName + "-1"}), + makeAggregateClusterResource(clusterName+"-1", []string{clusterName + "-2"}), + makeAggregateClusterResource(clusterName+"-2", []string{clusterName + "-3"}), + makeAggregateClusterResource(clusterName+"-3", []string{clusterName + "-4"}), + makeAggregateClusterResource(clusterName+"-4", []string{clusterName + "-5"}), + makeAggregateClusterResource(clusterName+"-5", []string{clusterName + "-6"}), + makeAggregateClusterResource(clusterName+"-6", []string{clusterName + "-7"}), + makeAggregateClusterResource(clusterName+"-7", []string{clusterName + "-8"}), + makeAggregateClusterResource(clusterName+"-8", []string{clusterName + "-9"}), + makeAggregateClusterResource(clusterName+"-9", []string{clusterName + "-10"}), + makeAggregateClusterResource(clusterName+"-10", []string{clusterName + "-11"}), + makeAggregateClusterResource(clusterName+"-11", []string{clusterName + "-12"}), + makeAggregateClusterResource(clusterName+"-12", []string{clusterName + "-13"}), + makeAggregateClusterResource(clusterName+"-13", []string{clusterName + "-14"}), + makeAggregateClusterResource(clusterName+"-14", []string{clusterName + "-15"}), + e2e.DefaultCluster(clusterName+"-15", serviceName, e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{testutils.ParsePort(t, server.Address)})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} + +// Tests a diamond shaped aggregate cluster (A->[B,C]; B->D; C->D). Verifies +// that the load balancing configuration pushed to the priority LB +// policy specifies cluster D only once. Also verifies that configuration is +// pushed only after all child clusters are resolved. +func (s) TestAggregatedClusterSuccess_DiamondDependency(t *testing.T) { + lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t) + mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil) + + // Configure the management server with an aggregate cluster resource having + // a diamond dependency pattern, (A->[B,C]; B->D; C->D). Includes resources + // for cluster A, B and D, but don't include the resource for cluster C yet. + // This will help us verify that no configuration is pushed to the child + // policy until the whole cluster graph is resolved. + const ( + clusterNameA = clusterName // cluster name in cds LB policy config + clusterNameB = clusterName + "-B" + clusterNameC = clusterName + "-C" + clusterNameD = clusterName + "-D" + ) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterNameA, []string{clusterNameB, clusterNameC}), + makeAggregateClusterResource(clusterNameB, []string{clusterNameD}), + e2e.DefaultCluster(clusterNameD, serviceName, e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})}, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that no configuration is pushed to the child policy yet, because + // not all clusters making up the aggregate cluster have been resolved yet. + select { + case cfg := <-lbCfgCh: + t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg)) + case <-time.After(defaultTestShortTimeout): + } + + // Now configure the resource for cluster C in the management server, + // thereby completing the cluster graph. This should result in configuration + // being pushed down to the child policy. + resources.Clusters = append(resources.Clusters, makeAggregateClusterResource(clusterNameC, []string{clusterNameD})) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + wantChildCfg := &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: createPriorityConfig(clusterNameD), + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0"}, + } + if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil { + t.Fatal(err) + } +} + +// Tests the case where the aggregate cluster graph contains duplicates (A->[B, +// C]; B->[C, D]). Verifies that the load balancing configuration pushed to the +// priority LB policy does not contain duplicates, and that the priority +// corresponding to cluster C is higher than that for cluster D. Also verifies +// that the configuration is pushed only after all child clusters are resolved. +func (s) TestAggregatedClusterSuccess_IgnoreDups(t *testing.T) { + lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t) + mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil) + + // Configure the management server with an aggregate cluster resource that + // has duplicates in the graph, (A->[B, C]; B->[C, D]). Include resources + // for clusters A, B and D, but don't configure the resource for cluster C + // yet. This will help us verify that no configuration is pushed to the + // child policy until the whole cluster graph is resolved. + const ( + clusterNameA = clusterName // cluster name in cds LB policy config + clusterNameB = clusterName + "-B" + clusterNameC = clusterName + "-C" + clusterNameD = clusterName + "-D" + ) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterNameA, []string{clusterNameB, clusterNameC}), + makeAggregateClusterResource(clusterNameB, []string{clusterNameC, clusterNameD}), + e2e.DefaultCluster(clusterNameD, serviceName, e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})}, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that no configuration is pushed to the child policy yet, because + // not all clusters making up the aggregate cluster have been resolved yet. + select { + case cfg := <-lbCfgCh: + t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg)) + case <-time.After(defaultTestShortTimeout): + } + + // Now configure the resource for cluster C in the management server, + // thereby completing the cluster graph. This should result in configuration + // being pushed down to the child policy. + resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterNameC, edsClusterName, e2e.SecurityLevelNone)) + resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(edsClusterName, "localhost", []uint32{1234})) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + wantChildCfg := &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: createPriorityConfig(clusterNameC), + IgnoreReresolutionRequests: true, + }, + "priority-1-0": { + Config: createPriorityConfig(clusterNameD), + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0", "priority-1-0"}, + } + if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil { + t.Fatal(err) + } +} + +// Tests the scenario where the aggregate cluster graph has a node that has +// child node of itself. The case for this is A -> A, and since there is no base +// cluster (EDS or Logical DNS), no configuration should be pushed to the child +// policy. The channel is expected to move to TRANSIENT_FAILURE and RPCs are +// expected to fail with code UNAVAILABLE and an error message specifying that +// the aggregate cluster graph has no leaf clusters. Then the test updates A -> B, +// where B is a leaf EDS cluster. Verifies that configuration is pushed to the +// child policy and that an RPC can be successfully made. +func (s) TestAggregatedCluster_NodeChildOfItself(t *testing.T) { + lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t) + mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil) + + const ( + clusterNameA = clusterName // cluster name in cds LB policy config + clusterNameB = clusterName + "-B" + ) + // Configure the management server with an aggregate cluster resource whose + // child is itself. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{makeAggregateClusterResource(clusterNameA, []string{clusterNameA})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + select { + case cfg := <-lbCfgCh: + t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg)) + case <-time.After(defaultTestShortTimeout): + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Verify that the RPC fails with expected code. + client := testgrpc.NewTestServiceClient(cc) + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if gotCode, wantCode := status.Code(err), codes.Unavailable; gotCode != wantCode { + t.Fatalf("EmptyCall() failed with code: %v, want %v", gotCode, wantCode) + } + const wantErr = "aggregate cluster graph has no leaf clusters" + if !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall() failed with err: %v, want error containing %s", err, wantErr) + } + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + + // Update the aggregate cluster to point to a leaf EDS cluster. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterNameA, []string{clusterNameB}), + e2e.DefaultCluster(clusterNameB, serviceName, e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{testutils.ParsePort(t, server.Address)})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify the configuration pushed to the child policy. + wantChildCfg := &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: createPriorityConfig(clusterNameB), + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0"}, + } + if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} + +// Tests the scenario where the aggregate cluster graph contains a cycle and +// contains no leaf clusters. The case used here is [A -> B, B -> A]. As there +// are no leaf clusters in this graph, no configuration should be pushed to the +// child policy. The channel is expected to move to TRANSIENT_FAILURE and RPCs +// are expected to fail with code UNAVAILABLE and an error message specifying +// that the aggregate cluster graph has no leaf clusters. +func (s) TestAggregatedCluster_CycleWithNoLeafNode(t *testing.T) { + lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t) + mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil) + + const ( + clusterNameA = clusterName // cluster name in cds LB policy config + clusterNameB = clusterName + "-B" + ) + // Configure the management server with an aggregate cluster resource graph + // that contains a cycle and no leaf clusters. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterNameA, []string{clusterNameB}), + makeAggregateClusterResource(clusterNameB, []string{clusterNameA}), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + select { + case cfg := <-lbCfgCh: + t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg)) + case <-time.After(defaultTestShortTimeout): + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Verify that the RPC fails with expected code. + client := testgrpc.NewTestServiceClient(cc) + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if gotCode, wantCode := status.Code(err), codes.Unavailable; gotCode != wantCode { + t.Fatalf("EmptyCall() failed with code: %v, want %v", gotCode, wantCode) + } + const wantErr = "aggregate cluster graph has no leaf clusters" + if !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall() failed with err: %v, want %s", err, wantErr) + } +} + +// Tests the scenario where the aggregate cluster graph contains a cycle and +// also contains a leaf cluster. The case used here is [A -> B, B -> A, C]. As +// there is a leaf cluster in this graph , configuration should be pushed to the +// child policy and RPCs should get routed to that leaf cluster. +func (s) TestAggregatedCluster_CycleWithLeafNode(t *testing.T) { + lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t) + mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil) + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + + const ( + clusterNameA = clusterName // cluster name in cds LB policy config + clusterNameB = clusterName + "-B" + clusterNameC = clusterName + "-C" + ) + // Configure the management server with an aggregate cluster resource graph + // that contains a cycle, but also contains a leaf cluster. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterNameA, []string{clusterNameB}), + makeAggregateClusterResource(clusterNameB, []string{clusterNameA, clusterNameC}), + e2e.DefaultCluster(clusterNameC, serviceName, e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify the configuration pushed to the child policy. + wantChildCfg := &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: createPriorityConfig(clusterNameC), + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0"}, + } + if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} + +// Tests the scenario where the cluster tree changes, and verifies that the +// watchers for the cds balancer are updated accordingly. That is the cluster +// removed from the tree no longer has a watcher and the new cluster added has a +// new watcher. +func (s) TestWatchers(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cdsResourceRequestedCh := make(chan []string, 1) + onStreamReq := func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3ClusterURL { + if len(req.GetResourceNames()) > 0 { + select { + case cdsResourceRequestedCh <- req.GetResourceNames(): + case <-ctx.Done(): + } + } + } + return nil + } + mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, onStreamReq) + + const ( + clusterA = clusterName + clusterB = clusterName + "-B" + clusterC = clusterName + "-C" + clusterD = clusterName + "-D" + ) + + // Initial CDS resources: A -> B, C + initialResources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterA)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterA, []string{clusterB, clusterC}), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, initialResources); err != nil { + t.Fatalf("Update failed: %v", err) + } + wantNames := []string{clusterA, clusterB, clusterC} + if err := waitForResourceNames(ctx, cdsResourceRequestedCh, wantNames); err != nil { + t.Fatal(err) + } + + // Update the CDS resources to remove cluster C and add cluster D. + initialResources.Clusters = []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterA, []string{clusterB, clusterD}), + } + if err := mgmtServer.Update(ctx, initialResources); err != nil { + t.Fatalf("Update failed: %v", err) + } + wantNames = []string{clusterA, clusterB, clusterD} + if err := waitForResourceNames(ctx, cdsResourceRequestedCh, wantNames); err != nil { + t.Fatal(err) + } +} diff --git a/internal/xds/balancer/cdsbalancer/cdsbalancer.go b/internal/xds/balancer/cdsbalancer/cdsbalancer.go new file mode 100644 index 0000000..08b6b45 --- /dev/null +++ b/internal/xds/balancer/cdsbalancer/cdsbalancer.go @@ -0,0 +1,472 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package cdsbalancer implements a balancer to handle CDS responses. +package cdsbalancer + +import ( + "encoding/json" + "fmt" + + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/nop" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/pretty" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/xds/balancer/outlierdetection" + "google.golang.org/grpc/internal/xds/balancer/priority" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/internal/xds/xdsdepmgr" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const cdsName = "cds_experimental" + +var ( + // newChildBalancer is a helper function to build a new priority balancer + // and will be overridden in unittests. + newChildBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) { + builder := balancer.Get(priority.Name) + if builder == nil { + return nil, fmt.Errorf("xds: no balancer builder with name %v", priority.Name) + } + // We directly pass the parent clientConn to the underlying priority + // balancer because the cdsBalancer does not deal with subConns. + return builder.Build(cc, opts), nil + } +) + +func init() { + balancer.Register(bb{}) +} + +// bb implements the balancer.Builder interface to help build a cdsBalancer. +// It also implements the balancer.ConfigParser interface to help parse the +// JSON service config, to be passed to the cdsBalancer. +type bb struct{} + +// Build creates a new CDS balancer with the ClientConn. +func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + builder := balancer.Get(priority.Name) + if builder == nil { + // Shouldn't happen, registered through imported Priority builder. Still, + // defensive programming. + logger.Errorf("%q LB policy is needed but not registered", priority.Name) + return nop.NewBalancer(cc, fmt.Errorf("%q LB policy is needed but not registered", priority.Name)) + } + parser, ok := builder.(balancer.ConfigParser) + if !ok { + // Shouldn't happen, imported Priority builder has this method. + logger.Errorf("%q LB policy does not implement a config parser", priority.Name) + return nop.NewBalancer(cc, fmt.Errorf("%q LB policy does not implement a config parser", priority.Name)) + } + + b := &cdsBalancer{ + bOpts: opts, + childConfigParser: parser, + clusterConfigs: make(map[string]*xdsresource.ClusterResult), + priorityConfigs: make(map[string]*priorityConfig), + cc: cc, + } + b.logger = prefixLogger(b) + b.logger.Infof("Created") + return b +} + +// Name returns the name of balancers built by this builder. +func (bb) Name() string { + return cdsName +} + +// lbConfig represents the loadBalancingConfig section of the service config +// for the cdsBalancer. +type lbConfig struct { + serviceconfig.LoadBalancingConfig + ClusterName string `json:"cluster"` + IsDynamic bool `json:"isDynamic"` +} + +// ParseConfig parses the JSON load balancer config provided into an +// internal form or returns an error if the config is invalid. +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + var cfg lbConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, fmt.Errorf("xds: unable to unmarshal lbconfig: %s, error: %v", string(c), err) + } + return &cfg, nil +} + +// cdsBalancer implements a CDS based LB policy. It instantiates a +// cluster_resolver balancer to further resolve the serviceName received from +// CDS, into localities and endpoints. Implements the balancer.Balancer +// interface which is exposed to gRPC and implements the balancer.ClientConn +// interface which is exposed to the cluster_resolver balancer. +type cdsBalancer struct { + // The following fields are initialized at build time and are either + // read-only after that or provide their own synchronization, and therefore + // do not need to be guarded by a mutex. + cc balancer.ClientConn // ClientConn interface passed to child LB. + bOpts balancer.BuildOptions // BuildOptions passed to child LB. + childConfigParser balancer.ConfigParser // Config parser for cluster_resolver LB policy. + logger *grpclog.PrefixLogger // Prefix logger for all logging. + + // All fields below are accessed only from methods implementing the + // balancer.Balancer interface. Since gRPC guarantees that these methods are + // never invoked concurrently, no additional synchronization is required to + // protect access to these fields. + xdsClient xdsclient.XDSClient + childLB balancer.Balancer // Child policy, built upon resolution of the cluster graph. + clusterConfigs map[string]*xdsresource.ClusterResult // Cluster name to the last received result for that cluster. + priorityConfigs map[string]*priorityConfig // Hostname to priority config for that leaf cluster. + lbCfg *lbConfig // Current load balancing configuration. + priorities []*priorityConfig // List of priorities in the order. + unsubscribe func() // For dynamic cluster unsubscription. + isSubscribed bool // True if a dynamic cluster has been subscribed to. + clusterSubscriber xdsdepmgr.ClusterSubscriber // To subscribe to dynamic cluster resource. + xdsLBPolicy internalserviceconfig.BalancerConfig // Stores the locality and endpoint picking policy. + attributes *attributes.Attributes // Attributes from resolver state. + serviceConfig *serviceconfig.ParseResult + // Each new leaf cluster needs a child name generator to reuse child policy + // names. But to make sure the names across leaf clusters doesn't conflict, + // we need a seq ID. This ID is incremented for each new cluster. + childNameGeneratorSeqID uint64 +} + +// UpdateClientConnState receives the serviceConfig, xdsConfig, +// ClusterSubscriber and the xdsClient object from the xdsResolver. If an error +// is encountered, the parent (clustermanager) sets the corresponding cluster’s +// picker to transient_failure. Otherwise, the received configuration is +// processed and forwarded to the appropriate child policy. +func (b *cdsBalancer) UpdateClientConnState(state balancer.ClientConnState) error { + if b.xdsClient == nil { + c := xdsclient.FromResolverState(state.ResolverState) + if c == nil { + b.logger.Warningf("Received balancer config with no xDS client") + return balancer.ErrBadResolverState + } + b.xdsClient = c + } + b.logger.Infof("Received balancer config update: %s", pretty.ToJSON(state.BalancerConfig)) + + xdsConfig := xdsresource.XDSConfigFromResolverState(state.ResolverState) + if xdsConfig == nil { + b.logger.Warningf("Received balancer config with no xDS config") + return balancer.ErrBadResolverState + } + b.clusterConfigs = xdsConfig.Clusters + b.clusterSubscriber = xdsdepmgr.XDSClusterSubscriberFromResolverState(state.ResolverState) + if b.clusterSubscriber == nil { + b.logger.Warningf("Received balancer config with no cluster subscriber") + return balancer.ErrBadResolverState + } + // The errors checked here should ideally never happen because the + // ServiceConfig in this case is prepared by the xdsResolver and is not + // something that is received on the wire. + lbCfg, ok := state.BalancerConfig.(*lbConfig) + if !ok { + b.logger.Warningf("Received unexpected balancer config type: %T", state.BalancerConfig) + return balancer.ErrBadResolverState + } + if lbCfg.ClusterName == "" { + b.logger.Warningf("Received balancer config with no cluster name") + return balancer.ErrBadResolverState + } + + b.lbCfg = lbCfg + b.serviceConfig = state.ResolverState.ServiceConfig + b.attributes = state.ResolverState.Attributes + return b.handleXDSConfigUpdate() +} + +// handleXDSConfigUpdate processes the XDSConfig update from the xDS resolver. +func (b *cdsBalancer) handleXDSConfigUpdate() error { + clusterName := b.lbCfg.ClusterName + + // If the cluster is dynamic and we dont have a subscription yet, create + // one. + if b.lbCfg.IsDynamic && !b.isSubscribed { + b.unsubscribe = b.clusterSubscriber.SubscribeToCluster(clusterName) + b.isSubscribed = true + return nil + } + + clusterUpdate, ok := b.clusterConfigs[clusterName] + if !ok { + // If the cluster is missing from the config, check if it is dynamic. + // For dynamic clusters, the xDS config may be updated before the + // corresponding cluster resource is received. This should never occur + // for static clusters. + if b.lbCfg.IsDynamic { + return nil + } + return b.annotateErrorWithNodeID(fmt.Errorf("did not find the cluster %q in XDSConfig", clusterName)) + } + // If the cluster resource has an error, return the error. + if clusterUpdate.Err != nil { + return clusterUpdate.Err + } + return b.handleClusterUpdate() +} + +// handleClusterUpdate handles a good XDSConfig update from the xDS resolver. +// Builds the child policy config and pushes it down. +func (b *cdsBalancer) handleClusterUpdate() error { + clusterName := b.lbCfg.ClusterName + clusterConfig := b.clusterConfigs[clusterName].Config + + var newPriorities []*priorityConfig + switch clusterConfig.Cluster.ClusterType { + case xdsresource.ClusterTypeEDS, xdsresource.ClusterTypeLogicalDNS: + p := b.updatePriorityConfig(clusterName, &clusterConfig) + newPriorities = append(newPriorities, p) + case xdsresource.ClusterTypeAggregate: + for _, leaf := range clusterConfig.AggregateConfig.LeafClusters { + leafCluster := b.clusterConfigs[leaf] + // Update priority config for leaf clusters. + p := b.updatePriorityConfig(leaf, &leafCluster.Config) + newPriorities = append(newPriorities, p) + } + } + b.priorities = newPriorities + + if err := b.updateOutlierDetection(); err != nil { + return b.annotateErrorWithNodeID(fmt.Errorf("failed to correctly update Outlier Detection config %v", err)) + } + + // The LB policy is configured by the root cluster. + if err := json.Unmarshal(clusterConfig.Cluster.LBPolicy, &b.xdsLBPolicy); err != nil { + return b.annotateErrorWithNodeID(fmt.Errorf("error unmarshalling xDS LB Policy: %v", err)) + } + if err := b.updateChildConfig(); err != nil { + return b.annotateErrorWithNodeID(err) + } + return nil +} + +// updateChildConfig builds child policy configuration using endpoint addresses +// returned from the XDSConfig and child policy configuration. +// +// A child policy is created if one doesn't already exist. The newly built +// configuration is then pushed to the child policy. +func (b *cdsBalancer) updateChildConfig() error { + if b.childLB == nil { + childLB, err := newChildBalancer(b.cc, b.bOpts) + if err != nil { + return fmt.Errorf("failed to create child policy of type %s: %v", priority.Name, err) + } + b.childLB = childLB + } + + childCfgBytes, endpoints, err := buildPriorityConfigJSON(b.priorities, &b.xdsLBPolicy) + if err != nil { + return fmt.Errorf("failed to build child policy config: %v", err) + } + childCfg, err := b.childConfigParser.ParseConfig(childCfgBytes) + if err != nil { + return fmt.Errorf("failed to parse child policy config. This should never happen because the config was generated: %v", err) + } + if b.logger.V(2) { + b.logger.Infof("Built child policy config: %s", pretty.ToJSON(childCfg)) + } + + for i := range endpoints { + for j := range endpoints[i].Addresses { + addr := endpoints[i].Addresses[j] + addr.BalancerAttributes = endpoints[i].Attributes + // BalancerAttributes are used for the following: + // * Authority Override. + // * grpc.lb.backend_service metric label propagation. + // See https://github.com/grpc/grpc-go/issues/6472 + endpoints[i].Addresses[j] = addr + } + } + if err := b.childLB.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: endpoints, + ServiceConfig: b.serviceConfig, + Attributes: b.attributes, + }, + BalancerConfig: childCfg, + }); err != nil { + return fmt.Errorf("failed to push config to child policy: %v", err) + } + return nil +} + +// updatePriorityConfig updates the priority configuration for the specified EDS +// or DNS cluster, creating it if it does not already exist. +func (b *cdsBalancer) updatePriorityConfig(clusterName string, clusterConfig *xdsresource.ClusterConfig) *priorityConfig { + name := hostName(clusterName, *clusterConfig.Cluster) + pc, ok := b.priorityConfigs[name] + if !ok { + pc = &priorityConfig{ + childNameGen: newNameGenerator(b.childNameGeneratorSeqID), + } + b.priorityConfigs[name] = pc + // Increment the seq ID for the next new cluster. This is done to make + // sure that the child policy names generated for different clusters + // don't conflict with each other. + b.childNameGeneratorSeqID++ + } + pc.clusterConfig = clusterConfig + return pc +} + +// updateOutlierDetection updates Outlier Detection config for all priorities. +func (b *cdsBalancer) updateOutlierDetection() error { + odBuilder := balancer.Get(outlierdetection.Name) + if odBuilder == nil { + // Shouldn't happen, registered through imported Outlier Detection, + // defensive programming. + return fmt.Errorf("%q LB policy is needed but not registered", outlierdetection.Name) + } + + odParser, ok := odBuilder.(balancer.ConfigParser) + if !ok { + // Shouldn't happen, imported Outlier Detection builder has this method. + return fmt.Errorf("%q LB policy does not implement a config parser", outlierdetection.Name) + } + + for _, p := range b.priorities { + // Update Outlier Detection Config. + odJSON := p.clusterConfig.Cluster.OutlierDetection + if odJSON == nil { + odJSON = json.RawMessage(`{}`) + } + + lbCfg, err := odParser.ParseConfig(odJSON) + if err != nil { + return fmt.Errorf("error parsing Outlier Detection config %v: %v", odJSON, err) + } + + odCfg, ok := lbCfg.(*outlierdetection.LBConfig) + if !ok { + // Shouldn't happen, Parser built at build time with Outlier + // Detection builder pulled from gRPC LB Registry. + return fmt.Errorf("config parser for Outlier Detection returned config with unexpected type %T: %v", lbCfg, lbCfg) + } + p.outlierDetection = *odCfg + } + return nil +} + +// ResolverError handles errors reported by the xdsResolver. +func (b *cdsBalancer) ResolverError(err error) { + // Missing Listener or RouteConfiguration on the management server + // results in a 'resource not found' error from the xDS resolver. In + // these cases, we should report transient failure. + if xdsresource.ErrType(err) == xdsresource.ErrorTypeResourceNotFound { + b.closeChildPolicyAndReportTF(err) + return + } + var root string + if b.lbCfg != nil { + root = b.lbCfg.ClusterName + } + b.onClusterError(root, err) +} + +// UpdateSubConnState handles subConn updates from gRPC. +func (b *cdsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +// closeChildPolicyAndReportTF closes the child policy, if it exists, and +// updates the connectivity state of the channel to TransientFailure with an +// error picker. +func (b *cdsBalancer) closeChildPolicyAndReportTF(err error) { + if b.childLB != nil { + b.childLB.Close() + b.childLB = nil + } + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(err), + }) +} + +// Close closes the child policy, unsubscribes to the dynamic cluster, and +// closes the cdsBalancer. +func (b *cdsBalancer) Close() { + if b.childLB != nil { + b.childLB.Close() + b.childLB = nil + } + if b.unsubscribe != nil { + b.unsubscribe() + } + b.logger.Infof("Shutdown") +} + +func (b *cdsBalancer) ExitIdle() { + if b.childLB == nil { + b.logger.Warningf("Received ExitIdle with no child policy") + return + } + // This implementation assumes the child balancer supports + // ExitIdle (but still checks for the interface's existence to + // avoid a panic if not). If the child does not, no subconns + // will be connected. + b.childLB.ExitIdle() +} + +// Node ID needs to be manually added to errors generated in the following +// scenarios: +// - resource-does-not-exist: since the xDS watch API uses a separate callback +// instead of returning an error value. TODO(gRFC A88): Once A88 is +// implemented, the xDS client will be able to add the node ID to +// resource-does-not-exist errors as well, and we can get rid of this +// special handling. +// - received a good update from the xDS client, but the update either contains +// an invalid security configuration or contains invalid aggragate cluster +// config. +func (b *cdsBalancer) annotateErrorWithNodeID(err error) error { + nodeID := b.xdsClient.BootstrapConfig().Node().GetId() + return fmt.Errorf("[xDS node id: %v]: %w", nodeID, err) +} + +// onClusterAmbientError handles an ambient error, if a childLB already has a +// good update, it should continue using that. +func (b *cdsBalancer) onClusterAmbientError(name string, err error) { + b.logger.Warningf("Cluster resource %q received ambient error update: %v", name, err) + + if xdsresource.ErrType(err) != xdsresource.ErrorTypeConnection && b.childLB != nil { + // Connection errors will be sent to the child balancers directly. + // There's no need to forward them. + b.childLB.ResolverError(err) + } +} + +// onClusterResourceError handles errors to stop using the previously seen +// resource. Propagates the error down to the child policy if one exists, and +// puts the channel in TRANSIENT_FAILURE. +func (b *cdsBalancer) onClusterResourceError(name string, err error) { + b.logger.Warningf("CDS watch for resource %q reported resource error", name) + b.closeChildPolicyAndReportTF(err) +} + +func (b *cdsBalancer) onClusterError(name string, err error) { + if b.childLB != nil { + b.onClusterAmbientError(name, err) + } else { + b.onClusterResourceError(name, err) + } +} diff --git a/internal/xds/balancer/cdsbalancer/cdsbalancer_test.go b/internal/xds/balancer/cdsbalancer/cdsbalancer_test.go new file mode 100644 index 0000000..7e1a9b2 --- /dev/null +++ b/internal/xds/balancer/cdsbalancer/cdsbalancer_test.go @@ -0,0 +1,1135 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cdsbalancer + +import ( + "context" + "encoding/json" + "fmt" + "net" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + iringhash "google.golang.org/grpc/internal/ringhash" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/balancer/clusterimpl" + "google.golang.org/grpc/internal/xds/balancer/outlierdetection" + "google.golang.org/grpc/internal/xds/balancer/priority" + "google.golang.org/grpc/internal/xds/balancer/wrrlocality" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + "google.golang.org/grpc/balancer/ringhash" + "google.golang.org/grpc/balancer/roundrobin" + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter. + _ "google.golang.org/grpc/internal/xds/resolver" // Register the xds resolver +) + +const ( + target = "test.service" + routeName = "test_route" + clusterName = "cluster1" + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + serviceName = "service1" + dnsHostName = "dns_host" + host = "localhost" + port = 8080 + dnsPort = uint32(8080) + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func waitForResourceNames(ctx context.Context, resourceNamesCh chan []string, wantNames []string) error { + for ctx.Err() == nil { + select { + case <-ctx.Done(): + case gotNames := <-resourceNamesCh: + // Sort both slices before comparing them, as the order of clusters + // does not matter. + if cmp.Equal(gotNames, wantNames, cmpopts.SortSlices(func(a, b string) bool { return a < b })) { + return nil + } + } + } + if ctx.Err() != nil { + return fmt.Errorf("Timeout when waiting for appropriate Cluster resources to be requested") + } + return nil +} + +// Registers a wrapped priority LB policy (child policy of the cds LB +// policy) for the duration of this test that retains all the functionality of +// the former, but makes certain events available for inspection by the test. +// +// Returns the following: +// - a channel to read received load balancing configuration +// - a channel to read received resolver error +// - a channel that is closed when ExitIdle() is called +// - a channel that is closed when the balancer is closed +func registerWrappedPriorityPolicy(t *testing.T) (chan serviceconfig.LoadBalancingConfig, chan error, chan struct{}, chan struct{}) { + priorityBuilder := balancer.Get(priority.Name) + internal.BalancerUnregister(priorityBuilder.Name()) + + lbCfgCh := make(chan serviceconfig.LoadBalancingConfig, 1) + resolverErrCh := make(chan error, 1) + exitIdleCh := make(chan struct{}) + closeCh := make(chan struct{}) + + stub.Register(priority.Name, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = priorityBuilder.Build(bd.ClientConn, bd.BuildOptions) + }, + ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return priorityBuilder.(balancer.ConfigParser).ParseConfig(lbCfg) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + select { + case lbCfgCh <- ccs.BalancerConfig: + default: + } + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + ResolverError: func(bd *stub.BalancerData, err error) { + select { + case resolverErrCh <- err: + default: + } + bd.ChildBalancer.ResolverError(err) + }, + ExitIdle: func(bd *stub.BalancerData) { + bd.ChildBalancer.ExitIdle() + close(exitIdleCh) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + close(closeCh) + }, + }) + t.Cleanup(func() { balancer.Register(priorityBuilder) }) + + return lbCfgCh, resolverErrCh, exitIdleCh, closeCh +} + +// setupDNS unregisters the DNS resolver and registers a manual resolver for the +// same scheme. This allows the test to fake the DNS resolution by supplying the +// addresses of the test backends. +// +// Returns the following: +// - a channel onto which the DNS target being resolved is written to by the +// fake DNS resolver +// - a manual resolver which is used to fake the actual DNS resolution +func setupDNS(t *testing.T) (chan resolver.Target, *manual.Resolver) { + targetCh := make(chan resolver.Target, 1) + + mr := manual.NewBuilderWithScheme("dns") + mr.BuildCallback = func(target resolver.Target, _ resolver.ClientConn, _ resolver.BuildOptions) { targetCh <- target } + + dnsResolverBuilder := resolver.Get("dns") + resolver.Register(mr) + + t.Cleanup(func() { resolver.Register(dnsResolverBuilder) }) + return targetCh, mr +} + +// Performs the following setup required for tests: +// - Spins up an xDS management server and the provided onStreamRequest +// function is set to be called for every incoming request on the ADS stream. +// - Creates an xDS client talking to this management server +// - Creates a gRPC channel with grpc.NewClient that create a xds resolver. +// +// Returns the following: +// - the xDS management server +// - the nodeID expected by the management server +// - the grpc channel to the test backend service +// - the xDS client used the grpc channel +func setupWithManagementServer(t *testing.T, lis net.Listener, onStreamRequest func(int64, *v3discoverypb.DiscoveryRequest) error) (*e2e.ManagementServer, string, *grpc.ClientConn) { + t.Helper() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + OnStreamRequest: onStreamRequest, + // Required for aggregate clusters as all resources cannot be requested + // at once. + AllowResourceSubset: true, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", target), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient(\"xds:///%s\") = %v", target, err) + } + cc.Connect() + t.Cleanup(func() { cc.Close() }) + + return mgmtServer, nodeID, cc +} + +// Helper function to compare the load balancing configuration received on the +// channel with the expected one. Both configs are marshalled to JSON and then +// compared. +// +// Returns an error if marshalling to JSON fails, or if the load balancing +// configurations don't match, or if the context deadline expires before reading +// a child policy configuration off of the lbCfgCh. +func compareLoadBalancingConfig(ctx context.Context, lbCfgCh chan serviceconfig.LoadBalancingConfig, wantChildCfg serviceconfig.LoadBalancingConfig) error { + wantJSON, err := json.Marshal(wantChildCfg) + if err != nil { + return fmt.Errorf("failed to marshal expected child config to JSON: %v", err) + } + select { + case lbCfg := <-lbCfgCh: + gotJSON, err := json.Marshal(lbCfg) + if err != nil { + return fmt.Errorf("failed to marshal received LB config into JSON: %v", err) + } + if diff := cmp.Diff(wantJSON, gotJSON); diff != "" { + return fmt.Errorf("child policy received unexpected diff in config (-want +got):\n%s", diff) + } + case <-ctx.Done(): + return fmt.Errorf("timeout when waiting for child policy to receive its configuration") + } + return nil +} + +func verifyRPCError(gotErr error, wantCode codes.Code, wantErr, wantNodeID string) error { + if gotErr == nil { + return fmt.Errorf("RPC succeeded when expecting an error with code %v, message %q and nodeID %q", wantCode, wantErr, wantNodeID) + } + if gotCode := status.Code(gotErr); gotCode != wantCode { + return fmt.Errorf("RPC failed with code: %v, want code %v", gotCode, wantCode) + } + if !strings.Contains(gotErr.Error(), wantErr) { + return fmt.Errorf("RPC failed with error: %v, want %q", gotErr, wantErr) + } + if !strings.Contains(gotErr.Error(), wantNodeID) { + return fmt.Errorf("RPC failed with error: %v, want nodeID %q", gotErr, wantNodeID) + } + return nil +} + +// createPriorityConfig creates priority config for both EDS and DNS cluster. +func createPriorityConfig(cluster string) *iserviceconfig.BalancerConfig { + return &iserviceconfig.BalancerConfig{ + Name: outlierdetection.Name, + Config: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: cluster, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ChildPolicy: &iserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + }, + }, + }, + } +} + +// Tests the case where a configuration with an empty cluster name is pushed to +// the CDS LB policy. Verifies that ErrBadResolverState is returned. +func (s) TestConfigurationUpdate_EmptyCluster(t *testing.T) { + // Setup a management server and an xDS client to talk to it. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + xdsClient, xdsClose, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + t.Cleanup(xdsClose) + + // Create a manual resolver that configures the CDS LB policy as the + // top-level LB policy on the channel, and pushes a configuration with an + // empty cluster name. Also, register a callback with the manual resolver to + // receive the error returned by the balancer when a configuration with an + // empty cluster name is pushed. + r := manual.NewBuilderWithScheme("whatever") + updateStateErrCh := make(chan error, 1) + r.UpdateStateCallback = func(err error) { updateStateErrCh <- err } + jsonSC := `{ + "loadBalancingConfig":[{ + "cds_experimental":{ + "cluster": "" + } + }] + }` + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, xdsClient)) + + // Create a ClientConn with the above manual resolver. + cc, err := grpc.NewClient(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + t.Cleanup(func() { cc.Close() }) + + select { + case <-time.After(defaultTestTimeout): + t.Fatalf("Timed out waiting for error from the LB policy") + case err := <-updateStateErrCh: + if err != balancer.ErrBadResolverState { + t.Fatalf("For a configuration update with an empty cluster name, got error %v from the LB policy, want %v", err, balancer.ErrBadResolverState) + } + } +} + +// Tests the case where a configuration with a missing xDS client is pushed to +// the CDS LB policy. Verifies that ErrBadResolverState is returned. +func (s) TestConfigurationUpdate_MissingXdsClient(t *testing.T) { + // Create a manual resolver that configures the CDS LB policy as the + // top-level LB policy on the channel, and pushes a configuration that is + // missing the xDS client. Also, register a callback with the manual + // resolver to receive the error returned by the balancer. + r := manual.NewBuilderWithScheme("whatever") + updateStateErrCh := make(chan error, 1) + r.UpdateStateCallback = func(err error) { updateStateErrCh <- err } + jsonSC := `{ + "loadBalancingConfig":[{ + "cds_experimental":{ + "cluster": "foo" + } + }] + }` + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(resolver.State{ServiceConfig: scpr}) + + // Create a ClientConn with the above manual resolver. + cc, err := grpc.NewClient(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + t.Cleanup(func() { cc.Close() }) + + select { + case <-time.After(defaultTestTimeout): + t.Fatalf("Timed out waiting for error from the LB policy") + case err := <-updateStateErrCh: + if err != balancer.ErrBadResolverState { + t.Fatalf("For a configuration update missing the xDS client, got error %v from the LB policy, want %v", err, balancer.ErrBadResolverState) + } + } +} + +// Tests success scenarios where the cds LB policy receives a cluster resource +// from the management server. Verifies that the load balancing configuration +// pushed to the child is as expected. +func (s) TestClusterUpdate_Success(t *testing.T) { + tests := []struct { + name string + clusterResource *v3clusterpb.Cluster + wantChildCfg serviceconfig.LoadBalancingConfig + }{ + { + name: "happy-case-with-circuit-breakers", + clusterResource: func() *v3clusterpb.Cluster { + c := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone) + c.CircuitBreakers = &v3clusterpb.CircuitBreakers{ + Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{ + { + Priority: v3corepb.RoutingPriority_DEFAULT, + MaxRequests: wrapperspb.UInt32(512), + }, + { + Priority: v3corepb.RoutingPriority_HIGH, + MaxRequests: nil, + }, + }, + } + return c + }(), + wantChildCfg: &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: &iserviceconfig.BalancerConfig{ + Name: outlierdetection.Name, + Config: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: clusterName, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ChildPolicy: &iserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + }, + }, + }, + }, + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0"}, + }, + }, + { + name: "happy-case-with-ring-hash-lb-policy", + clusterResource: func() *v3clusterpb.Cluster { + c := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: serviceName, + SecurityLevel: e2e.SecurityLevelNone, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + c.LbConfig = &v3clusterpb.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &v3clusterpb.Cluster_RingHashLbConfig{ + MinimumRingSize: &wrapperspb.UInt64Value{Value: 100}, + MaximumRingSize: &wrapperspb.UInt64Value{Value: 1000}, + }, + } + return c + }(), + wantChildCfg: &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: &iserviceconfig.BalancerConfig{ + Name: outlierdetection.Name, + Config: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: clusterName, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: ringhash.Name, + Config: &iringhash.LBConfig{ + MinRingSize: 100, + MaxRingSize: 1000, + }, + }, + }, + }, + }, + }, + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0"}, + }, + }, + { + name: "happy-case-outlier-detection-xds-defaults", // OD proto set but no proto fields set + clusterResource: func() *v3clusterpb.Cluster { + c := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: serviceName, + SecurityLevel: e2e.SecurityLevelNone, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + c.OutlierDetection = &v3clusterpb.OutlierDetection{} + return c + }(), + wantChildCfg: &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: &iserviceconfig.BalancerConfig{ + Name: outlierdetection.Name, + Config: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + SuccessRateEjection: &outlierdetection.SuccessRateEjection{ + StdevFactor: 1900, + EnforcementPercentage: 100, + MinimumHosts: 5, + RequestVolume: 100, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: clusterName, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: ringhash.Name, + Config: &iringhash.LBConfig{ + MinRingSize: 1024, // default sizes + MaxRingSize: 4096, + }, + }, + }, + }, + }, + }, + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0"}, + }, + }, + { + name: "happy-case-outlier-detection-all-fields-set", + clusterResource: func() *v3clusterpb.Cluster { + c := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: serviceName, + SecurityLevel: e2e.SecurityLevelNone, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + c.OutlierDetection = &v3clusterpb.OutlierDetection{ + Interval: durationpb.New(10 * time.Second), + BaseEjectionTime: durationpb.New(30 * time.Second), + MaxEjectionTime: durationpb.New(300 * time.Second), + MaxEjectionPercent: wrapperspb.UInt32(10), + SuccessRateStdevFactor: wrapperspb.UInt32(1900), + EnforcingSuccessRate: wrapperspb.UInt32(100), + SuccessRateMinimumHosts: wrapperspb.UInt32(5), + SuccessRateRequestVolume: wrapperspb.UInt32(100), + FailurePercentageThreshold: wrapperspb.UInt32(85), + EnforcingFailurePercentage: wrapperspb.UInt32(5), + FailurePercentageMinimumHosts: wrapperspb.UInt32(5), + FailurePercentageRequestVolume: wrapperspb.UInt32(50), + } + return c + }(), + wantChildCfg: &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: &iserviceconfig.BalancerConfig{ + Name: outlierdetection.Name, + Config: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + SuccessRateEjection: &outlierdetection.SuccessRateEjection{ + StdevFactor: 1900, + EnforcementPercentage: 100, + MinimumHosts: 5, + RequestVolume: 100, + }, + FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{ + Threshold: 85, + EnforcementPercentage: 5, + MinimumHosts: 5, + RequestVolume: 50, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: clusterName, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: ringhash.Name, + Config: &iringhash.LBConfig{ + MinRingSize: 1024, // default sizes + MaxRingSize: 4096, + }, + }, + }, + }, + }, + }, + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0"}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t) + mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, test.clusterResource.Name)}, + Clusters: []*v3clusterpb.Cluster{test.clusterResource}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})}, + }); err != nil { + t.Fatal(err) + } + + if err := compareLoadBalancingConfig(ctx, lbCfgCh, test.wantChildCfg); err != nil { + t.Fatal(err) + } + }) + } +} + +// Tests scenarios for a bad cluster update received from the management server. +// +// - when a bad cluster resource update is received without any previous good +// update from the management server, the cds LB policy is expected to put +// the channel in TRANSIENT_FAILURE. +// - when a bad cluster resource update is received after a previous good +// update from the management server, the cds LB policy is expected to +// continue using the previous good update. +func (s) TestClusterUpdate_Failure(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + cdsResourceCanceledCh := make(chan struct{}, 1) + onStreamReq := func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3ClusterURL { + if len(req.GetResourceNames()) == 0 { + select { + case cdsResourceCanceledCh <- struct{}{}: + case <-ctx.Done(): + } + } + } + return nil + } + mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, onStreamReq) + + // Configure the management server to return a cluster resource that + // contains a config_source_specifier for the `lrs_server` field which is not + // set to `self`, and hence is expected to be NACKed by the client. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: target, + NodeID: nodeID, + Host: host, + Port: port, + }) + resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{}} + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that the watch for the cluster resource is not cancelled. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-cdsResourceCanceledCh: + t.Fatal("Watch for cluster resource is cancelled when not expected to") + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Ensure that the NACK error and the xDS node ID are propagated to the RPC + // caller. + const wantClusterNACKErr = "unsupported config_source_specifier" + client := testgrpc.NewTestServiceClient(cc) + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err := verifyRPCError(err, codes.Unavailable, wantClusterNACKErr, nodeID); err != nil { + t.Fatal(err) + } + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + + // Configure correct cluster and endpoints resources in the management server. + resources = e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: target, + NodeID: nodeID, + Host: host, + Port: testutils.ParsePort(t, server.Address), + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Send the bad cluster resource again. + resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{}} + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that the watch for the cluster resource is not cancelled. + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-cdsResourceCanceledCh: + t.Fatal("Watch for cluster resource is cancelled when not expected to") + } + + // Verify that a successful RPC can be made, using the previously received + // good configuration. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} + +// Tests the following scenarios for resolver errors: +// - when a resolver error is received without any previous good update from the +// management server, the cds LB policy is expected to put the channel in +// TRANSIENT_FAILURE. +// - when a resolver error is received (one that is not a resource-not-found +// error), with a previous good update from the management server, the cds LB +// policy is expected to continue to use the previously received good +// configuration. +// - when a resolver error is received (one that is a resource-not-found +// error, which is usually the case when the LDS resource is removed), +// with a previous good update from the management server, the cds LB policy +// is expected to push the error down the child policy and put the channel in +// TRANSIENT_FAILURE. It is also expected to cancel the CDS watch. +func (s) TestResolverError(t *testing.T) { + registerWrappedPriorityPolicy(t) + mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil) + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + + // Host and port for the backend. + host := "localhost" + port := testutils.ParsePort(t, server.Address) + + // Push a resolver error (Bad Listener) on startup. Since there are no + // active clusters, xdsResolver should call ReportError. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: target, + NodeID: nodeID, + Host: host, + Port: port, + }) + resources.Listeners[0].ApiListener.ApiListener = nil + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that the channel enters TRANSIENT_FAILURE. + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Verify that RPCs fail. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Fatalf("EmptyCall() succeeded, want failure due to bad listener") + } + + // Configure good resources. + resources = e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: target, + NodeID: nodeID, + Host: host, + Port: port, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that the channel enters READY state. + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Verify that a successful RPC can be made. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Push a Bad Listener again (NACK). xdsResolver has active clusters, so it + // should NOT call ReportError, but keep old config. This should be treated + // as an ambient error. + resources.Listeners[0].ApiListener.ApiListener = nil + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made, using the previously received + // good configuration. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Remove the Listener (Resource Not Found). + resources.Listeners = nil + resources.SkipValidation = true + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that RPCs eventually fail. + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Ensure that the resolver error is propagated to the RPC caller. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Fatal("EmptyCall() succeeded, want failure due to removed listener") + } +} + +// Tests the scenario for resolver errors: when a resolver resource not found +// error is received when the LDS resource is removed with a previous good +// update from the management server, the cds LB policy is expected to push the +// error down the child policy and put the channel in TRANSIENT_FAILURE. It is +// also expected to cancel the CDS watch. +func (s) TestResourceNotFoundResolverError(t *testing.T) { + _, _, _, childPolicyCloseCh := registerWrappedPriorityPolicy(t) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cdsResourceCanceledCh := make(chan struct{}, 1) + onStreamReq := func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3ClusterURL { + if len(req.GetResourceNames()) == 0 { + select { + case cdsResourceCanceledCh <- struct{}{}: + case <-ctx.Done(): + } + } + } + return nil + } + mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, onStreamReq) + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + + // Configure default resources in the management server. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: target, + NodeID: nodeID, + Host: host, + Port: testutils.ParsePort(t, server.Address), + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Push a resource-not-found type error by removing the LDS resource. + resources.Listeners = nil + resources.SkipValidation = true + mgmtServer.Update(ctx, resources) + + // Verify that the resolver error is pushed to the child policy. + select { + case <-childPolicyCloseCh: + case <-ctx.Done(): + t.Fatal("Timeout when waiting for child policy to be closed") + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Ensure that the resolver error is propagated to the RPC caller. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for error from RPC") + default: + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err := verifyRPCError(err, codes.Unavailable, "", nodeID); err == nil { + return + } + } +} + +// Tests scenarios involving removal of a cluster resource from the management +// server. +// +// - when the cluster resource is removed after a previous good +// update from the management server, the cds LB policy is expected to put +// the channel in TRANSIENT_FAILURE. +// - when the cluster resource is re-sent by the management server, RPCs +// should start succeeding. +func (s) TestClusterUpdate_ResourceNotFound(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cdsResourceCanceledCh := make(chan struct{}, 1) + onStreamReq := func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3ClusterURL { + if len(req.GetResourceNames()) == 0 { + select { + case cdsResourceCanceledCh <- struct{}{}: + case <-ctx.Done(): + } + } + } + return nil + } + mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, onStreamReq) + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + + // Configure default resources in the management server. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: target, + NodeID: nodeID, + Host: host, + Port: testutils.ParsePort(t, server.Address), + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Remove the cluster resource from the management server, triggering a + // resource-not-found error. + oldClusters := resources.Clusters + resources.Clusters = nil + resources.SkipValidation = true + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that the watch for the cluster resource is not cancelled. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-cdsResourceCanceledCh: + t.Fatal("Watch for cluster resource is cancelled when not expected to") + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Ensure RPC fails with Unavailable status code and the error message is + // meaningful and contains the xDS node ID. + wantErr := fmt.Sprintf("resource %q of type %q has been removed", oldClusters[0].Name, "ClusterResource") + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err := verifyRPCError(err, codes.Unavailable, wantErr, nodeID); err != nil { + t.Fatal(err) + } + + // Re-add the cluster resource to the management server. + resources.Clusters = oldClusters + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} + +// Tests that closing the cds LB policy results in the the child policy being +// closed. +func (s) TestClose(t *testing.T) { + _, _, _, childPolicyCloseCh := registerWrappedPriorityPolicy(t) + mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil) + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + + // Configure resources in the management server. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: target, + NodeID: nodeID, + Host: host, + Port: testutils.ParsePort(t, server.Address), + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Close the gRPC ClientConn, which will close the cds LB policy. + cc.Close() + + // Wait for the child policy to be closed. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for the child policy to be closed") + case <-childPolicyCloseCh: + } +} + +// Tests that calling ExitIdle on the cds LB policy results in the call being +// propagated to the child policy. +func (s) TestExitIdle(t *testing.T) { + _, _, exitIdleCh, _ := registerWrappedPriorityPolicy(t) + mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil) + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + + // Configure resources in the management server. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: target, + NodeID: nodeID, + Host: host, + Port: testutils.ParsePort(t, server.Address), + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Connect to gRPC channel that will call ExitIdle on the cds LB policy. + cc.Connect() + + // Wait for ExitIdle to be called on the child policy. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for the child policy to be closed") + case <-exitIdleCh: + } +} + +// TestParseConfig verifies the ParseConfig() method in the CDS balancer. +func (s) TestParseConfig(t *testing.T) { + bb := balancer.Get(cdsName) + if bb == nil { + t.Fatalf("balancer.Get(%q) returned nil", cdsName) + } + parser, ok := bb.(balancer.ConfigParser) + if !ok { + t.Fatalf("balancer %q does not implement the ConfigParser interface", cdsName) + } + + tests := []struct { + name string + input json.RawMessage + wantCfg serviceconfig.LoadBalancingConfig + wantErr bool + }{ + { + name: "good-config", + input: json.RawMessage(`{"cluster": "cluster1"}`), + wantCfg: &lbConfig{ClusterName: "cluster1"}, + }, + { + name: "good-config-with-is-dynamic", + input: json.RawMessage(`{"cluster": "cluster1","isDynamic":true}`), + wantCfg: &lbConfig{ClusterName: "cluster1", IsDynamic: true}, + }, + { + name: "unknown-fields-in-config", + input: json.RawMessage(`{"Unknown": "foobar"}`), + wantCfg: &lbConfig{ClusterName: ""}, + }, + { + name: "empty-config", + input: json.RawMessage(""), + wantErr: true, + }, + { + name: "bad-config", + input: json.RawMessage(`{"cluster": 5}`), + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotCfg, gotErr := parser.ParseConfig(test.input) + if (gotErr != nil) != test.wantErr { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", string(test.input), gotErr, test.wantErr) + } + if test.wantErr { + return + } + if !cmp.Equal(gotCfg, test.wantCfg) { + t.Fatalf("ParseConfig(%v) = %v, want %v", string(test.input), gotCfg, test.wantCfg) + } + }) + } +} + +func newUint32(i uint32) *uint32 { + return &i +} diff --git a/internal/xds/balancer/cdsbalancer/configbuilder.go b/internal/xds/balancer/cdsbalancer/configbuilder.go new file mode 100644 index 0000000..6efeeb1 --- /dev/null +++ b/internal/xds/balancer/cdsbalancer/configbuilder.go @@ -0,0 +1,375 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package cdsbalancer + +import ( + "encoding/json" + "fmt" + "maps" + "slices" + + "google.golang.org/grpc/experimental/balancer/hostname" + "google.golang.org/grpc/experimental/balancer/weight" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/hierarchy" + "google.golang.org/grpc/internal/proxyattributes" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + xdsinternal "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/internal/xds/balancer/clusterimpl" + "google.golang.org/grpc/internal/xds/balancer/outlierdetection" + "google.golang.org/grpc/internal/xds/balancer/priority" + "google.golang.org/grpc/internal/xds/balancer/wrrlocality" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/resolver" +) + +const million = 1000000 + +// priorityConfig is config for one priority. For example, if there's an EDS and +// a DNS, the priority list will be [priorityConfig{EDS}, priorityConfig{DNS}]. +// +// Each priorityConfig corresponds to one leaf cluster retrieved from XDSConfig +// for the top-level cluster. +type priorityConfig struct { + // clusterConfig has the cluster update as well as EDS or DNS endpoints + // depending on the leaf cluster type. + clusterConfig *xdsresource.ClusterConfig + // outlierDetection is the Outlier Detection LB configuration for this + // priority. + outlierDetection outlierdetection.LBConfig + // Each leaf cluster has a name generator so that the child policies can + // reuse names between updates (EDS updates for example). + childNameGen *nameGenerator +} + +// hostName returns the name of the host for the given cluster. +// +// For EDS, it's the EDSServiceName (or ClusterName if empty). +// For DNS, it's the DNSHostName. +func hostName(clusterName string, update xdsresource.ClusterUpdate) string { + switch update.ClusterType { + case xdsresource.ClusterTypeEDS: + if update.EDSServiceName != "" { + return update.EDSServiceName + } + return clusterName + case xdsresource.ClusterTypeLogicalDNS: + return update.DNSHostName + default: + return "" + } +} + +// buildPriorityConfigJSON builds balancer config for the passed in +// priorities. +// +// The built tree of balancers (see test for the output struct). +// +// ┌────────┐ +// │priority│ +// └┬──────┬┘ +// │ │ +// ┌──────────▼─┐ ┌─▼──────────┐ +// │cluster_impl│ │cluster_impl│ +// └──────┬─────┘ └─────┬──────┘ +// │ │ +// ┌──────▼─────┐ ┌─────▼──────┐ +// │xDSLBPolicy │ │xDSLBPolicy │ (Locality and Endpoint picking layer) +// └────────────┘ └────────────┘ +func buildPriorityConfigJSON(priorities []*priorityConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) ([]byte, []resolver.Endpoint, error) { + pc, endpoints, err := buildPriorityConfig(priorities, xdsLBPolicy) + if err != nil { + return nil, nil, fmt.Errorf("failed to build priority config: %v", err) + } + ret, err := json.Marshal(pc) + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal built priority config struct into json: %v", err) + } + return ret, endpoints, nil +} + +func buildPriorityConfig(priorities []*priorityConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) (*priority.LBConfig, []resolver.Endpoint, error) { + var ( + retConfig = &priority.LBConfig{Children: make(map[string]*priority.Child)} + retEndpoints []resolver.Endpoint + ) + for _, p := range priorities { + clusterUpdate := p.clusterConfig.Cluster + switch clusterUpdate.ClusterType { + case xdsresource.ClusterTypeEDS: + names, configs, endpoints, err := buildClusterImplConfigForEDS(p.childNameGen, p.clusterConfig, xdsLBPolicy) + if err != nil { + return nil, nil, err + } + retConfig.Priorities = append(retConfig.Priorities, names...) + retEndpoints = append(retEndpoints, endpoints...) + odCfgs := convertClusterImplMapToOutlierDetection(configs, p.outlierDetection) + for n, c := range odCfgs { + retConfig.Children[n] = &priority.Child{ + Config: &internalserviceconfig.BalancerConfig{Name: outlierdetection.Name, Config: c}, + // Ignore all re-resolution from EDS children. + IgnoreReresolutionRequests: true, + } + } + continue + case xdsresource.ClusterTypeLogicalDNS: + name, config, endpoints := buildClusterImplConfigForDNS(p.childNameGen, p.clusterConfig, xdsLBPolicy) + retConfig.Priorities = append(retConfig.Priorities, name) + retEndpoints = append(retEndpoints, endpoints...) + odCfg := makeClusterImplOutlierDetectionChild(config, p.outlierDetection) + retConfig.Children[name] = &priority.Child{ + Config: &internalserviceconfig.BalancerConfig{Name: outlierdetection.Name, Config: odCfg}, + // Not ignore re-resolution from DNS children, they will trigger + // DNS to re-resolve. + IgnoreReresolutionRequests: false, + } + continue + } + } + return retConfig, retEndpoints, nil +} + +func convertClusterImplMapToOutlierDetection(ciCfgs map[string]*clusterimpl.LBConfig, odCfg outlierdetection.LBConfig) map[string]*outlierdetection.LBConfig { + odCfgs := make(map[string]*outlierdetection.LBConfig, len(ciCfgs)) + for n, c := range ciCfgs { + odCfgs[n] = makeClusterImplOutlierDetectionChild(c, odCfg) + } + return odCfgs +} + +func makeClusterImplOutlierDetectionChild(ciCfg *clusterimpl.LBConfig, odCfg outlierdetection.LBConfig) *outlierdetection.LBConfig { + odCfgRet := odCfg + odCfgRet.ChildPolicy = &internalserviceconfig.BalancerConfig{Name: clusterimpl.Name, Config: ciCfg} + return &odCfgRet +} + +func buildClusterImplConfigForDNS(g *nameGenerator, config *xdsresource.ClusterConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) (string, *clusterimpl.LBConfig, []resolver.Endpoint) { + pName := fmt.Sprintf("priority-%v", g.prefix) + clusterUpdate := config.Cluster + lbconfig := &clusterimpl.LBConfig{ + Cluster: clusterUpdate.ClusterName, + ChildPolicy: xdsLBPolicy, + } + endpoints := config.EndpointConfig.DNSEndpoints.Endpoints + if len(endpoints) == 0 { + return pName, lbconfig, nil + } + var retEndpoint resolver.Endpoint + for _, e := range endpoints { + // LOGICAL_DNS requires all resolved addresses to be grouped into a + // single logical endpoint. We iterate over the input endpoints and + // aggregate their addresses into a new endpoint variable. + retEndpoint.Addresses = append(retEndpoint.Addresses, e.Addresses...) + } + // Even though localities are not a thing for the LOGICAL_DNS cluster and + // its endpoint(s), we add an empty locality attribute here to ensure that + // LB policies that rely on locality information (like weighted_target) + // continue to work. + localityStr := xdsinternal.LocalityString(clients.Locality{}) + retEndpoint = hostname.Set(hierarchy.SetInEndpoint(retEndpoint, []string{pName, localityStr}), clusterUpdate.DNSHostName) + // Set the locality weight to 1. This is required because the child policy + // like weighted_target which relies on locality weights to distribute + // traffic. These policies may drop traffic if the weight is 0. + retEndpoint = wrrlocality.SetAddrInfo(retEndpoint, wrrlocality.AddrInfo{LocalityWeight: 1}) + return pName, lbconfig, []resolver.Endpoint{retEndpoint} +} + +// buildClusterImplConfigForEDS returns a list of cluster_impl configs, one for +// each priority, sorted by priority, and the addresses for each priority (with +// hierarchy attributes set). +// +// For example, if there are two priorities, the returned values will be +// - ["p0", "p1"] +// - map{"p0":p0_config, "p1":p1_config} +// - [p0_address_0, p0_address_1, p1_address_0, p1_address_1] +// - p0 addresses' hierarchy attributes are set to p0 +func buildClusterImplConfigForEDS(g *nameGenerator, config *xdsresource.ClusterConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) ([]string, map[string]*clusterimpl.LBConfig, []resolver.Endpoint, error) { + edsUpdate := config.EndpointConfig.EDSUpdate + + // Localities of length 0 is triggered by an NACK or resource-not-found + // error before update, or an empty localities list in an update. In either + // case want to create a priority, and send down empty address list, causing + // TF for that priority. "If any discovery mechanism instance experiences an + // error retrieving data, and it has not previously reported any results, it + // should report a result that is a single priority with no endpoints." - + // A37 + priorities := [][]xdsresource.Locality{{}} + if len(edsUpdate.Localities) != 0 { + priorities = groupLocalitiesByPriority(edsUpdate.Localities) + } + retNames := g.generate(priorities) + retConfigs := make(map[string]*clusterimpl.LBConfig, len(retNames)) + var retEndpoints []resolver.Endpoint + for i, pName := range retNames { + priorityLocalities := priorities[i] + cfg, endpoints, err := priorityLocalitiesToClusterImpl(priorityLocalities, pName, *config.Cluster, xdsLBPolicy) + if err != nil { + return nil, nil, nil, err + } + retConfigs[pName] = cfg + retEndpoints = append(retEndpoints, endpoints...) + } + return retNames, retConfigs, retEndpoints, nil +} + +// groupLocalitiesByPriority returns the localities grouped by priority. +// +// The returned list is sorted from higher priority to lower. Each item in the +// list is a group of localities. +// +// For example, for L0-p0, L1-p0, L2-p1, results will be +// - [[L0, L1], [L2]] +func groupLocalitiesByPriority(localities []xdsresource.Locality) [][]xdsresource.Locality { + priorities := make(map[int][]xdsresource.Locality) + for _, locality := range localities { + priority := int(locality.Priority) + priorities[priority] = append(priorities[priority], locality) + } + // Sort the priorities based on the int value, deduplicate, and then turn + // the sorted list into a string list. This will be child names, in priority + // order. + priorityIntSlice := slices.Sorted(maps.Keys(priorities)) + ret := make([][]xdsresource.Locality, 0, len(priorityIntSlice)) + for _, p := range priorityIntSlice { + ret = append(ret, priorities[p]) + } + return ret +} + +// priorityLocalitiesToClusterImpl takes a list of localities (with the same +// priority), and generates a cluster impl policy config, and a list of +// addresses with their path hierarchy set to [priority-name, locality-name], so +// priority and the xDS LB Policy know which child policy each address is for. +func priorityLocalitiesToClusterImpl(localities []xdsresource.Locality, priorityName string, clusterUpdate xdsresource.ClusterUpdate, xdsLBPolicy *internalserviceconfig.BalancerConfig) (*clusterimpl.LBConfig, []resolver.Endpoint, error) { + var retEndpoints []resolver.Endpoint + + // Compute the sum of locality weights to normalize locality weights. The + // xDS client guarantees that the sum of locality weights (within a + // priority) will not exceed MaxUint32. + var localityWeightSum uint32 + for _, locality := range localities { + localityWeightSum += locality.Weight + } + + for _, locality := range localities { + // Compute the sum of endpoint weights to normalize endpoint weights. + // The xDS client does not currently guarantee that the sum of endpoint + // weights (within a locality) will not exceed MaxUint32. TODO(i/8862): + // Once the xDS client guarantees that the sum of endpoint weights does + // not exceed MaxUint32, we can change the type of this variable from + // uint64 to uint32. + var endpointWeightSum uint64 + for _, endpoint := range locality.Endpoints { + endpointWeightSum += uint64(endpoint.Weight) + } + + localityStr := xdsinternal.LocalityString(locality.ID) + for _, endpoint := range locality.Endpoints { + // Filter out all "unhealthy" endpoints (unknown and healthy are + // both considered to be healthy: + // https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/core/health_check.proto#envoy-api-enum-core-healthstatus). + if endpoint.HealthStatus != xdsresource.EndpointHealthStatusHealthy && endpoint.HealthStatus != xdsresource.EndpointHealthStatusUnknown { + continue + } + + // Create a copy of endpoint.ResolverEndpoint to avoid race between + // the xDS Client (which owns this shared object in its cache) and + // the Cluster Resolver (which is trying to modify attributes). + resolverEndpoint := endpoint.ResolverEndpoint + resolverEndpoint.Addresses = slices.Clone(endpoint.ResolverEndpoint.Addresses) + + if clusterUpdate.IsHTTP11ProxyEnabled { + var proxyAddrStr string + if val, ok := endpoint.Metadata["envoy.http11_proxy_transport_socket.proxy_address"].(xdsresource.ProxyAddressMetadataValue); ok { + proxyAddrStr = val.Address + } else if val, ok := locality.Metadata["envoy.http11_proxy_transport_socket.proxy_address"].(xdsresource.ProxyAddressMetadataValue); ok { + proxyAddrStr = val.Address + } + if proxyAddrStr != "" { + for idx, addr := range resolverEndpoint.Addresses { + connectAddr := addr.Addr + addr.Addr = proxyAddrStr + resolverEndpoint.Addresses[idx] = proxyattributes.Set(addr, proxyattributes.Options{ + ConnectAddr: connectAddr, + }) + } + } + } + + resolverEndpoint = hierarchy.SetInEndpoint(resolverEndpoint, []string{priorityName, localityStr}) + resolverEndpoint = xdsinternal.SetLocalityIDInEndpoint(resolverEndpoint, locality.ID) + // "To provide the xds_wrr_locality load balancer information about + // locality weights received from EDS, the cluster resolver will + // populate a new locality weight attribute for each address The + // attribute will have the weight (as an integer) of the locality + // the address is part of." - A52 + resolverEndpoint = wrrlocality.SetAddrInfo(resolverEndpoint, wrrlocality.AddrInfo{LocalityWeight: locality.Weight}) + + if envconfig.PickFirstWeightedShuffling { + normalizedLocalityWeight := fractionToFixedPoint(uint64(locality.Weight), uint64(localityWeightSum)) + normalizedEndpointWeight := fractionToFixedPoint(uint64(endpoint.Weight), endpointWeightSum) + endpointWeight := fixedPointMultiply(normalizedEndpointWeight, normalizedLocalityWeight) + if endpointWeight == 0 { + endpointWeight = 1 + } + resolverEndpoint = weight.Set(resolverEndpoint, weight.EndpointInfo{Weight: endpointWeight}) + } else { + resolverEndpoint = weight.Set(resolverEndpoint, weight.EndpointInfo{Weight: locality.Weight * endpoint.Weight}) + } + retEndpoints = append(retEndpoints, resolverEndpoint) + } + } + return &clusterimpl.LBConfig{ + Cluster: clusterUpdate.ClusterName, + ChildPolicy: xdsLBPolicy, + }, retEndpoints, nil +} + +// fixedPointFractionalBits is the number of bits used for the fractional part +// of normalized endpoint and locality weights. +// +// We use the UQ1.31 fixed-point format (Unsigned, 1 integer bit, 31 fractional bits). +// This allows representing values in the range [0.0, 2.0) with a precision +// of 2^-31. +// +// Bit Layout: +// [ 31 ] [ 30 ................. 0 ] +// +// | | +// | +--- Fractional Part (31 bits) +// +------------------ Integer Part (1 bit) +// +// See gRFC A113 for more details. +const fixedPointFractionalBits = 31 + +// fractionToFixedPoint converts a fraction represented by numerator and +// denominator to a fixed-point number between 0 and 1 represented as a uint32. +// +// The xDS client guarantees that the sum of locality weights (within a +// priority) will not exceed MaxUint32. TODO(i/8862): Once the xDS client +// guarantees that the sum of endpoint weights does not exceed MaxUint32, we can +// change the types of this function's arguments from uint64 to uint32. +func fractionToFixedPoint(numerator, denominator uint64) uint32 { + return uint32(uint64(numerator) << fixedPointFractionalBits / uint64(denominator)) +} + +func fixedPointMultiply(a, b uint32) uint32 { + return uint32((uint64(a) * uint64(b)) >> fixedPointFractionalBits) +} diff --git a/internal/xds/balancer/cdsbalancer/configbuilder_childname.go b/internal/xds/balancer/cdsbalancer/configbuilder_childname.go new file mode 100644 index 0000000..2b0fedd --- /dev/null +++ b/internal/xds/balancer/cdsbalancer/configbuilder_childname.go @@ -0,0 +1,111 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cdsbalancer + +import ( + "fmt" + + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" +) + +// nameGenerator generates a child name for a list of priorities (each priority +// is a list of localities). +// +// The purpose of this generator is to reuse names between updates. So the +// struct keeps state between generate() calls, and a later generate() might +// return names returned by the previous call. +type nameGenerator struct { + prevLocalitiesToChildNames map[clients.Locality]string // locality to child name mapping generated for the previous update + prevChildNames []string // prioritized list of child names generated for the previous update + prefix uint64 + nextID uint64 +} + +func newNameGenerator(prefix uint64) *nameGenerator { + return &nameGenerator{prefix: prefix} +} + +// generate returns a list of names for the given list of priorities. +// +// Each priority is a list of localities. The name for the priority is picked as +// - for each locality in this priority, if it exists in the existing names, +// this priority will reuse the name +// - if no reusable name is found for this priority, a new name is generated +// +// For example: +// - update 1: [[L1], [L2], [L3]] --> ["0", "1", "2"] +// - update 2: [[L1], [L2], [L3]] --> ["0", "1", "2"] +// - update 3: [[L1, L2], [L3]] --> ["0", "2"] (Two priorities were merged) +// - update 4: [[L1], [L4]] --> ["0", "3",] (A priority was split, and a new priority was added) +func (ng *nameGenerator) generate(priorities [][]xdsresource.Locality) []string { + ret := make([]string, len(priorities)) + usedNames := make(map[string]bool) + newNames := make(map[clients.Locality]string) + + // Pass 1: Same priority index match. + for i, priority := range priorities { + if i >= len(ng.prevChildNames) { + continue + } + targetName := ng.prevChildNames[i] + for _, locality := range priority { + if name, ok := ng.prevLocalitiesToChildNames[locality.ID]; ok && name == targetName { + ret[i] = targetName + usedNames[targetName] = true + break + } + } + } + + // Pass 2: Greedy reuse. + for i, priority := range priorities { + if ret[i] != "" { + continue + } + for _, locality := range priority { + if name, ok := ng.prevLocalitiesToChildNames[locality.ID]; ok { + if !usedNames[name] { + ret[i] = name + usedNames[name] = true + break + } + } + } + } + + // Pass 3: New name. + for i, name := range ret { + if name == "" { + newID := fmt.Sprintf("priority-%d-%d", ng.prefix, ng.nextID) + ng.nextID++ + ret[i] = newID + usedNames[newID] = true + } + } + + // Update state. + for i, priority := range priorities { + for _, l := range priority { + newNames[l.ID] = ret[i] + } + } + ng.prevLocalitiesToChildNames = newNames + ng.prevChildNames = ret + return ret +} diff --git a/internal/xds/balancer/cdsbalancer/configbuilder_childname_test.go b/internal/xds/balancer/cdsbalancer/configbuilder_childname_test.go new file mode 100644 index 0000000..7275f55 --- /dev/null +++ b/internal/xds/balancer/cdsbalancer/configbuilder_childname_test.go @@ -0,0 +1,311 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cdsbalancer + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" +) + +func (s) Test_nameGenerator_generate(t *testing.T) { + tests := []struct { + name string + prefix uint64 + steps []struct { + input [][]xdsresource.Locality + want []string + } + }{ + { + name: "init, two new priorities", + prefix: 3, + steps: []struct { + input [][]xdsresource.Locality + want []string + }{ + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}}, + {{ID: clients.Locality{Zone: "L1"}}}, + }, + want: []string{"priority-3-0", "priority-3-1"}, + }, + }, + }, + { + name: "one new priority", + prefix: 1, + steps: []struct { + input [][]xdsresource.Locality + want []string + }{ + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}}, + }, + want: []string{"priority-1-0"}, + }, + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}}, + {{ID: clients.Locality{Zone: "L1"}}}, + }, + want: []string{"priority-1-0", "priority-1-1"}, + }, + }, + }, + { + name: "merge two priorities", + prefix: 4, + steps: []struct { + input [][]xdsresource.Locality + want []string + }{ + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}}, + {{ID: clients.Locality{Zone: "L1"}}}, + {{ID: clients.Locality{Zone: "L2"}}}, + }, + want: []string{"priority-4-0", "priority-4-1", "priority-4-2"}, + }, + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}}, + {{ID: clients.Locality{Zone: "L2"}}}, + }, + want: []string{"priority-4-0", "priority-4-2"}, + }, + }, + }, + { + name: "swap two priorities", + steps: []struct { + input [][]xdsresource.Locality + want []string + }{ + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}}, + {{ID: clients.Locality{Zone: "L1"}}}, + {{ID: clients.Locality{Zone: "L2"}}}, + }, + want: []string{"priority-0-0", "priority-0-1", "priority-0-2"}, + }, + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L1"}}}, + {{ID: clients.Locality{Zone: "L0"}}}, + {{ID: clients.Locality{Zone: "L2"}}}, + }, + want: []string{"priority-0-1", "priority-0-0", "priority-0-2"}, + }, + }, + }, + { + name: "split priority", + steps: []struct { + input [][]xdsresource.Locality + want []string + }{ + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}}, + {{ID: clients.Locality{Zone: "L2"}}}, + }, + want: []string{"priority-0-0", "priority-0-1"}, + }, + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}}, + {{ID: clients.Locality{Zone: "L1"}}}, + {{ID: clients.Locality{Zone: "L2"}}}, + }, + want: []string{"priority-0-0", "priority-0-2", "priority-0-1"}, + }, + }, + }, + { + name: "priority index preference", + steps: []struct { + input [][]xdsresource.Locality + want []string + }{ + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}}, + {{ID: clients.Locality{Zone: "L1"}}}, + }, + want: []string{"priority-0-0", "priority-0-1"}, + }, + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L0"}}}, + }, + want: []string{"priority-0-0"}, + }, + }, + }, + { + name: "merge partial", + steps: []struct { + input [][]xdsresource.Locality + want []string + }{ + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}}, + {{ID: clients.Locality{Zone: "L2"}}, {ID: clients.Locality{Zone: "L3"}}}, + }, + want: []string{"priority-0-0", "priority-0-1"}, + }, + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L2"}}}, + {{ID: clients.Locality{Zone: "L3"}}}, + }, + want: []string{"priority-0-0", "priority-0-1"}, + }, + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}}, + {{ID: clients.Locality{Zone: "L2"}}, {ID: clients.Locality{Zone: "L3"}}}, + }, + want: []string{"priority-0-0", "priority-0-1"}, + }, + }, + }, + { + name: "swap shift", + steps: []struct { + input [][]xdsresource.Locality + want []string + }{ + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}}, + {{ID: clients.Locality{Zone: "L2"}}, {ID: clients.Locality{Zone: "L3"}}}, + }, + want: []string{"priority-0-0", "priority-0-1"}, + }, + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L2"}}}, + {{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}}, + }, + want: []string{"priority-0-1", "priority-0-0"}, + }, + }, + }, + { + name: "replace priority", + steps: []struct { + input [][]xdsresource.Locality + want []string + }{ + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}}, + {{ID: clients.Locality{Zone: "L2"}}, {ID: clients.Locality{Zone: "L3"}}}, + }, + want: []string{"priority-0-0", "priority-0-1"}, + }, + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}}, + {{ID: clients.Locality{Zone: "L5"}}}, + }, + want: []string{"priority-0-0", "priority-0-2"}, + }, + }, + }, + { + name: "reordered merge", + steps: []struct { + input [][]xdsresource.Locality + want []string + }{ + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L2"}}}, + {{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L3"}}}, + }, + want: []string{"priority-0-0", "priority-0-1"}, + }, + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L2"}}}, + {{ID: clients.Locality{Zone: "L3"}}}, + }, + want: []string{"priority-0-0", "priority-0-1"}, + }, + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L2"}}}, + {{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L3"}}}, + }, + want: []string{"priority-0-0", "priority-0-1"}, + }, + }, + }, + { + name: "three-step shift stability", + steps: []struct { + input [][]xdsresource.Locality + want []string + }{ + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}}, + {{ID: clients.Locality{Zone: "L1"}}}, + }, + want: []string{"priority-0-0", "priority-0-1"}, + }, + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L2"}}}, + {{ID: clients.Locality{Zone: "L0"}}}, + {{ID: clients.Locality{Zone: "L1"}}}, + }, + want: []string{"priority-0-2", "priority-0-0", "priority-0-1"}, + }, + { + input: [][]xdsresource.Locality{ + {{ID: clients.Locality{Zone: "L0"}}}, + {{ID: clients.Locality{Zone: "L1"}}}, + }, + want: []string{"priority-0-0", "priority-0-1"}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ng := newNameGenerator(tt.prefix) + for i, step := range tt.steps { + got := ng.generate(step.input) + if diff := cmp.Diff(got, step.want); diff != "" { + t.Errorf("step %d: generate() = got: %v, want: %v, diff (-got +want): %s", i, got, step.want, diff) + } + } + }) + } +} diff --git a/internal/xds/balancer/cdsbalancer/configbuilder_test.go b/internal/xds/balancer/cdsbalancer/configbuilder_test.go new file mode 100644 index 0000000..282e2a4 --- /dev/null +++ b/internal/xds/balancer/cdsbalancer/configbuilder_test.go @@ -0,0 +1,1445 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package cdsbalancer + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/balancer/ringhash" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/experimental/balancer/weight" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/hierarchy" + "google.golang.org/grpc/internal/proxyattributes" + iringhash "google.golang.org/grpc/internal/ringhash" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + xdsinternal "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/internal/xds/balancer/clusterimpl" + "google.golang.org/grpc/internal/xds/balancer/outlierdetection" + "google.golang.org/grpc/internal/xds/balancer/priority" + "google.golang.org/grpc/internal/xds/balancer/wrrlocality" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/resolver" +) + +const ( + testClusterName = "test-cluster-name" + testClusterName2 = "google_cfe_some-name" + testMaxRequests = 314 + testEDSServiceName = "service-name-from-parent" + testDropCategory = "test-drops" + testDropOverMillion = 1 +) + +var ( + endpointCmpOpts = cmp.Options{ + cmp.AllowUnexported(attributes.Attributes{}), + cmp.Transformer("SortEndpoints", func(in []resolver.Endpoint) []resolver.Endpoint { + out := append([]resolver.Endpoint(nil), in...) // Copy input to avoid mutating it + sort.Slice(out, func(i, j int) bool { + return out[i].Addresses[0].Addr < out[j].Addresses[0].Addr + }) + return out + }), + } + + noopODCfg = outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + } +) + +// makeLocalityID creates a clients.Locality with Zone set to +// "test-zone-{idx}". +func makeLocalityID(idx int) clients.Locality { + return clients.Locality{Zone: fmt.Sprintf("test-zone-%d", idx)} +} + +// makeEndpoint creates a test xdsresource.Endpoint with a healthy status, the +// specified endpoint weight, and three addresses: +// "addr-{localityIdx}-{endpointIdx}", +// "addr-{localityIdx}-{endpointIdx}-additional-1", and +// "addr-{localityIdx}-{endpointIdx}-additional-2". +func makeEndpoint(localityIdx, endpointIdx int, endpointWeight uint32) xdsresource.Endpoint { + addr := fmt.Sprintf("addr-%d-%d", localityIdx, endpointIdx) + return xdsresource.Endpoint{ + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{ + {Addr: addr}, + {Addr: fmt.Sprintf("%s-additional-1", addr)}, + {Addr: fmt.Sprintf("%s-additional-2", addr)}, + }, + }, + Weight: endpointWeight, + } +} + +// makeResolverEndpoint creates a resolver.Endpoint with a single address +// "addr-{localityIdx}-{endpointIdx}". +func makeResolverEndpoint(localityIdx, endpointIdx int) resolver.Endpoint { + return resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: fmt.Sprintf("addr-%d-%d", localityIdx, endpointIdx)}}, + } +} + +// makeLocality creates an xdsresource.Locality with endpointCount endpoints, +// each with an endpoint weight of 1. The locality has the specified weight +// and priority. The locality ID and endpoint addresses are derived from +// localityIdx. +func makeLocality(localityIdx int, localityWeight, priority uint32, endpointCount int) xdsresource.Locality { + endpoints := make([]xdsresource.Endpoint, endpointCount) + for j := range endpointCount { + endpoints[j] = makeEndpoint(localityIdx, j, 1) + } + return xdsresource.Locality{ + Endpoints: endpoints, + ID: makeLocalityID(localityIdx), + Weight: localityWeight, + Priority: priority, + } +} + +// TestBuildPriorityConfigJSON is a sanity check that the built balancer config +// can be parsed. The behavior test is covered by TestBuildPriorityConfig. +func (s) TestBuildPriorityConfigJSON(t *testing.T) { + testLRSServerConfig, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{ + URI: "trafficdirector.googleapis.com:443", + ChannelCreds: []bootstrap.ChannelCreds{{Type: "google_default"}}, + }) + if err != nil { + t.Fatalf("Failed to create LRS server config for testing: %v", err) + } + + gotConfig, _, err := buildPriorityConfigJSON([]*priorityConfig{ + { + clusterConfig: &xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterName: testClusterName, + ClusterType: xdsresource.ClusterTypeEDS, + EDSServiceName: testEDSServiceName, + MaxRequests: newUint32(testMaxRequests), + LRSServerConfig: testLRSServerConfig, + }, + EndpointConfig: &xdsresource.EndpointConfig{ + EDSUpdate: &xdsresource.EndpointsUpdate{ + Drops: []xdsresource.OverloadDropConfig{{ + Category: testDropCategory, + Numerator: testDropOverMillion, + Denominator: million, + }}, + Localities: []xdsresource.Locality{ + makeLocality(0, 20, 0, 2), + makeLocality(1, 80, 0, 2), + makeLocality(2, 20, 1, 2), + makeLocality(3, 80, 1, 2), + }, + }, + }, + }, + childNameGen: newNameGenerator(0), + }, + { + clusterConfig: &xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeLogicalDNS, + }, + EndpointConfig: &xdsresource.EndpointConfig{ + DNSEndpoints: &xdsresource.DNSUpdate{Endpoints: []resolver.Endpoint{makeResolverEndpoint(4, 0), makeResolverEndpoint(4, 1)}}, + }, + }, + childNameGen: newNameGenerator(1), + }, + }, nil) + if err != nil { + t.Fatalf("buildPriorityConfigJSON(...) failed: %v", err) + } + + var prettyGot bytes.Buffer + if err := json.Indent(&prettyGot, gotConfig, ">>> ", " "); err != nil { + t.Fatalf("json.Indent() failed: %v", err) + } + // Print the indented json if this test fails. + t.Log(prettyGot.String()) + + priorityB := balancer.Get(priority.Name) + if _, err = priorityB.(balancer.ConfigParser).ParseConfig(gotConfig); err != nil { + t.Fatalf("ParseConfig(%+v) failed: %v", gotConfig, err) + } +} + +// TestBuildPriorityConfig tests the priority config generation. Each top level +// balancer per priority should be an Outlier Detection balancer, with a Cluster +// Impl Balancer as a child. +func (s) TestBuildPriorityConfig(t *testing.T) { + gotConfig, _, _ := buildPriorityConfig([]*priorityConfig{ + { + // EDS - OD config should be the top level for both of the EDS + // priorities balancer This EDS priority will have multiple sub + // priorities. The Outlier Detection configuration specified in the + // Discovery Mechanism should be the top level for each sub + // priorities balancer. + clusterConfig: &xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterName: testClusterName, + ClusterType: xdsresource.ClusterTypeEDS, + EDSServiceName: testEDSServiceName, + }, + EndpointConfig: &xdsresource.EndpointConfig{ + EDSUpdate: &xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + makeLocality(0, 20, 0, 2), + makeLocality(1, 80, 0, 2), + makeLocality(2, 20, 1, 2), + makeLocality(3, 80, 1, 2), + }, + }, + }, + }, + outlierDetection: noopODCfg, + childNameGen: newNameGenerator(0), + }, + { + // This OD config should wrap the Logical DNS priorities balancer. + clusterConfig: &xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterName: testClusterName2, + ClusterType: xdsresource.ClusterTypeLogicalDNS, + }, + EndpointConfig: &xdsresource.EndpointConfig{ + DNSEndpoints: &xdsresource.DNSUpdate{ + Endpoints: []resolver.Endpoint{makeResolverEndpoint(4, 0), makeResolverEndpoint(4, 1)}, + }, + }, + }, + outlierDetection: noopODCfg, + childNameGen: newNameGenerator(1), + }, + }, nil) + + wantConfig := &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: &iserviceconfig.BalancerConfig{ + Name: outlierdetection.Name, + Config: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{Cluster: testClusterName}, + }, + }, + }, + IgnoreReresolutionRequests: true, + }, + "priority-0-1": { + Config: &iserviceconfig.BalancerConfig{ + Name: outlierdetection.Name, + Config: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{Cluster: testClusterName}, + }, + }, + }, + IgnoreReresolutionRequests: true, + }, + "priority-1": { + Config: &iserviceconfig.BalancerConfig{ + Name: outlierdetection.Name, + Config: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: testClusterName2, + }, + }, + }, + }, + IgnoreReresolutionRequests: false, + }, + }, + Priorities: []string{"priority-0-0", "priority-0-1", "priority-1"}, + } + if diff := cmp.Diff(gotConfig, wantConfig); diff != "" { + t.Errorf("buildPriorityConfig() diff (-got +want) %v", diff) + } +} + +func testEndpointForDNS(endpoints []resolver.Endpoint, localityWeight uint32, path []string) resolver.Endpoint { + retEndpoint := resolver.Endpoint{} + for _, e := range endpoints { + retEndpoint.Addresses = append(retEndpoint.Addresses, e.Addresses...) + } + retEndpoint = hierarchy.SetInEndpoint(retEndpoint, path) + retEndpoint = wrrlocality.SetAddrInfo(retEndpoint, wrrlocality.AddrInfo{LocalityWeight: localityWeight}) + return retEndpoint +} + +func (s) TestBuildClusterImplConfigForDNS(t *testing.T) { + for _, tt := range []struct { + name string + endpoints []resolver.Endpoint + xdsLBPolicy *iserviceconfig.BalancerConfig + }{ + { + name: "one_endpoint_one_address", + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "addr-0-0"}}}}, + xdsLBPolicy: &iserviceconfig.BalancerConfig{Name: pickfirst.Name}, + }, + { + name: "one_endpoint_multiple_addresses", + endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{ + {Addr: "addr-0-0"}, + {Addr: "addr-0-1"}, + }}}, + xdsLBPolicy: &iserviceconfig.BalancerConfig{Name: wrrlocality.Name}, + }, + { + name: "multiple_endpoints_one_address_each", + endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "addr-0-0"}}}, + {Addresses: []resolver.Address{{Addr: "addr-0-1"}}}, + }, + xdsLBPolicy: &iserviceconfig.BalancerConfig{Name: roundrobin.Name}, + }, + { + name: "multiple_endpoints_multiple_addresses", + endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{ + {Addr: "addr-0-0"}, + {Addr: "addr-0-1"}, + }}, + {Addresses: []resolver.Address{ + {Addr: "addr-1-0"}, + {Addr: "addr-1-1"}, + }}, + }, + xdsLBPolicy: &iserviceconfig.BalancerConfig{Name: roundrobin.Name}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + gotName, gotConfig, gotEndpoints := buildClusterImplConfigForDNS(newNameGenerator(3), + &xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterName: testClusterName2, + ClusterType: xdsresource.ClusterTypeLogicalDNS, + }, + EndpointConfig: &xdsresource.EndpointConfig{DNSEndpoints: &xdsresource.DNSUpdate{Endpoints: tt.endpoints}}, + }, + tt.xdsLBPolicy) + const wantName = "priority-3" + if diff := cmp.Diff(wantName, gotName); diff != "" { + t.Errorf("buildClusterImplConfigForDNS() diff (-want +got) %v", diff) + } + + wantConfig := &clusterimpl.LBConfig{ + Cluster: testClusterName2, + ChildPolicy: tt.xdsLBPolicy, + } + if diff := cmp.Diff(wantConfig, gotConfig); diff != "" { + t.Errorf("buildClusterImplConfigForDNS() diff (-want +got) %v", diff) + } + + wantEndpoints := []resolver.Endpoint{testEndpointForDNS(tt.endpoints, 1, []string{wantName, xdsinternal.LocalityString(clients.Locality{})})} + if diff := cmp.Diff(wantEndpoints, gotEndpoints, endpointCmpOpts); diff != "" { + t.Errorf("buildClusterImplConfigForDNS() diff (-want +got) %v", diff) + } + }) + } +} + +func (s) TestBuildClusterImplConfigForEDS_PickFirstWeightedShuffling_Disabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.PickFirstWeightedShuffling, false) + + testLRSServerConfig, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{ + URI: "trafficdirector.googleapis.com:443", + ChannelCreds: []bootstrap.ChannelCreds{{Type: "google_default"}}, + }) + if err != nil { + t.Fatalf("Failed to create LRS server config for testing: %v", err) + } + + // Create test localities with 2 endpoints each. + // Localities are passed in shuffled order to verify sorting by priority. + loc0 := makeLocality(0, 20, 0, 2) + loc1 := makeLocality(1, 80, 0, 2) + loc2 := makeLocality(2, 20, 1, 2) + loc3 := makeLocality(3, 80, 1, 2) + + gotNames, gotConfigs, gotEndpoints, _ := buildClusterImplConfigForEDS( + newNameGenerator(2), + &xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterName: testClusterName, + ClusterType: xdsresource.ClusterTypeEDS, + EDSServiceName: testEDSServiceName, + LRSServerConfig: testLRSServerConfig, + MaxRequests: newUint32(testMaxRequests), + }, + EndpointConfig: &xdsresource.EndpointConfig{ + EDSUpdate: &xdsresource.EndpointsUpdate{ + Drops: []xdsresource.OverloadDropConfig{{ + Category: testDropCategory, + Numerator: testDropOverMillion, + Denominator: million, + }}, + Localities: []xdsresource.Locality{ + makeLocality(3, 80, 1, 2), + makeLocality(1, 80, 0, 2), + makeLocality(2, 20, 1, 2), + makeLocality(0, 20, 0, 2), + }, + }, + }, + }, + nil, + ) + + wantNames := []string{"priority-2-0", "priority-2-1"} + wantConfigs := map[string]*clusterimpl.LBConfig{ + "priority-2-0": {Cluster: testClusterName}, + "priority-2-1": {Cluster: testClusterName}, + } + // Endpoint weight is the product of locality weight and endpoint weight. + wantEndpoints := []resolver.Endpoint{ + testEndpointWithAttrs(loc0.Endpoints[0].ResolverEndpoint, 20, 20*1, "priority-2-0", &loc0.ID), + testEndpointWithAttrs(loc0.Endpoints[1].ResolverEndpoint, 20, 20*1, "priority-2-0", &loc0.ID), + testEndpointWithAttrs(loc1.Endpoints[0].ResolverEndpoint, 80, 80*1, "priority-2-0", &loc1.ID), + testEndpointWithAttrs(loc1.Endpoints[1].ResolverEndpoint, 80, 80*1, "priority-2-0", &loc1.ID), + testEndpointWithAttrs(loc2.Endpoints[0].ResolverEndpoint, 20, 20*1, "priority-2-1", &loc2.ID), + testEndpointWithAttrs(loc2.Endpoints[1].ResolverEndpoint, 20, 20*1, "priority-2-1", &loc2.ID), + testEndpointWithAttrs(loc3.Endpoints[0].ResolverEndpoint, 80, 80*1, "priority-2-1", &loc3.ID), + testEndpointWithAttrs(loc3.Endpoints[1].ResolverEndpoint, 80, 80*1, "priority-2-1", &loc3.ID), + } + + if diff := cmp.Diff(wantNames, gotNames); diff != "" { + t.Errorf("buildClusterImplConfigForEDS() diff (-want +got) %v", diff) + } + if diff := cmp.Diff(wantConfigs, gotConfigs); diff != "" { + t.Errorf("buildClusterImplConfigForEDS() diff (-want +got) %v", diff) + } + if diff := cmp.Diff(wantEndpoints, gotEndpoints, endpointCmpOpts); diff != "" { + t.Errorf("buildClusterImplConfigForEDS() diff (-want +got) %v", diff) + } +} + +func (s) TestBuildClusterImplConfigForEDS_PickFirstWeightedShuffling_Enabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.PickFirstWeightedShuffling, true) + + testLRSServerConfig, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{ + URI: "trafficdirector.googleapis.com:443", + ChannelCreds: []bootstrap.ChannelCreds{{Type: "google_default"}}, + }) + if err != nil { + t.Fatalf("Failed to create LRS server config for testing: %v", err) + } + + // Create test localities with 2 endpoints each. + // Localities are passed in shuffled order to verify sorting by priority. + loc0 := makeLocality(0, 20, 0, 2) + loc1 := makeLocality(1, 80, 0, 2) + loc2 := makeLocality(2, 20, 1, 2) + loc3 := makeLocality(3, 80, 1, 2) + + gotNames, gotConfigs, gotEndpoints, _ := buildClusterImplConfigForEDS( + newNameGenerator(2), + &xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterName: testClusterName, + ClusterType: xdsresource.ClusterTypeEDS, + EDSServiceName: testEDSServiceName, + LRSServerConfig: testLRSServerConfig, + MaxRequests: newUint32(testMaxRequests), + }, + EndpointConfig: &xdsresource.EndpointConfig{ + EDSUpdate: &xdsresource.EndpointsUpdate{ + Drops: []xdsresource.OverloadDropConfig{{ + Category: testDropCategory, + Numerator: testDropOverMillion, + Denominator: million, + }}, + Localities: []xdsresource.Locality{ + makeLocality(3, 80, 1, 2), + makeLocality(1, 80, 0, 2), + makeLocality(2, 20, 1, 2), + makeLocality(0, 20, 0, 2), + }, + }, + }, + }, + nil, + ) + + wantNames := []string{"priority-2-0", "priority-2-1"} + wantConfigs := map[string]*clusterimpl.LBConfig{ + "priority-2-0": {Cluster: testClusterName}, + "priority-2-1": {Cluster: testClusterName}, + } + // Endpoints weights are the product of normalized locality weight and + // endpoint weight, represented as a fixed-point number in uQ1.31 format. + // Locality weights are normalized as: + // P1: locality 3: 80 / (100) = 0.8 + // P0: locality 1: 80 / (100) = 0.8 + // P1: locality 2: 20 / (100) = 0.2 + // P0: locality 0: 20 / (100) = 0.2 + // In fixed-point uQ1.31 format, the weights are: + // locality 3: 0.8 * 2^31 = 1717986918 + // locality 1: 0.8 * 2^31 = 1717986918 + // locality 2: 0.2 * 2^31 = 429496729 + // locality 0: 0.2 * 2^31 = 429496729 + // + // There are two endpoints in each locality, each with weight 1. So, their + // normalized weights are 0.5 each. And the final endpoint weights are a + // product of their locality weights and 0.5, which turns out to be either + // 1717986918 * 0.5 = 858993459, or, + // 429496729 * 0.5 = 214748364 + wantEndpoints := []resolver.Endpoint{ + testEndpointWithAttrs(loc0.Endpoints[0].ResolverEndpoint, 20, 214748364, "priority-2-0", &loc0.ID), + testEndpointWithAttrs(loc0.Endpoints[1].ResolverEndpoint, 20, 214748364, "priority-2-0", &loc0.ID), + testEndpointWithAttrs(loc1.Endpoints[0].ResolverEndpoint, 80, 858993459, "priority-2-0", &loc1.ID), + testEndpointWithAttrs(loc1.Endpoints[1].ResolverEndpoint, 80, 858993459, "priority-2-0", &loc1.ID), + testEndpointWithAttrs(loc2.Endpoints[0].ResolverEndpoint, 20, 214748364, "priority-2-1", &loc2.ID), + testEndpointWithAttrs(loc2.Endpoints[1].ResolverEndpoint, 20, 214748364, "priority-2-1", &loc2.ID), + testEndpointWithAttrs(loc3.Endpoints[0].ResolverEndpoint, 80, 858993459, "priority-2-1", &loc3.ID), + testEndpointWithAttrs(loc3.Endpoints[1].ResolverEndpoint, 80, 858993459, "priority-2-1", &loc3.ID), + } + + if diff := cmp.Diff(gotNames, wantNames); diff != "" { + t.Errorf("buildClusterImplConfigForEDS() diff (-got +want) %v", diff) + } + if diff := cmp.Diff(gotConfigs, wantConfigs); diff != "" { + t.Errorf("buildClusterImplConfigForEDS() diff (-got +want) %v", diff) + } + if diff := cmp.Diff(gotEndpoints, wantEndpoints, endpointCmpOpts); diff != "" { + t.Errorf("buildClusterImplConfigForEDS() diff (-got +want) %v", diff) + } +} + +func (s) TestGroupLocalitiesByPriority(t *testing.T) { + // Create localities for two priorities (p0 and p1). + p0Loc0 := makeLocality(0, 20, 0, 2) + p0Loc1 := makeLocality(1, 80, 0, 2) + p1Loc0 := makeLocality(2, 20, 1, 2) + p1Loc1 := makeLocality(3, 80, 1, 2) + + tests := []struct { + name string + localities []xdsresource.Locality + wantLocalities [][]xdsresource.Locality + }{ + { + name: "1 locality 1 priority", + localities: []xdsresource.Locality{p0Loc0}, + wantLocalities: [][]xdsresource.Locality{ + {p0Loc0}, + }, + }, + { + name: "2 locality 1 priority", + localities: []xdsresource.Locality{p0Loc0, p0Loc1}, + wantLocalities: [][]xdsresource.Locality{ + {p0Loc0, p0Loc1}, + }, + }, + { + name: "1 locality in each", + localities: []xdsresource.Locality{p0Loc0, p1Loc0}, + wantLocalities: [][]xdsresource.Locality{ + {p0Loc0}, + {p1Loc0}, + }, + }, + { + name: "2 localities in each sorted", + localities: []xdsresource.Locality{ + p0Loc0, p0Loc1, + p1Loc0, p1Loc1}, + wantLocalities: [][]xdsresource.Locality{ + {p0Loc0, p0Loc1}, + {p1Loc0, p1Loc1}, + }, + }, + { + // The localities are given in order [p1, p0, p1, p0], but the + // returned priority list must be sorted [p0, p1], because the list + // order is the priority order. + name: "2 localities in each needs to sort", + localities: []xdsresource.Locality{ + p1Loc1, p0Loc1, + p1Loc0, p0Loc0}, + wantLocalities: [][]xdsresource.Locality{ + {p0Loc1, p0Loc0}, + {p1Loc1, p1Loc0}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotLocalities := groupLocalitiesByPriority(tt.localities) + if diff := cmp.Diff(tt.wantLocalities, gotLocalities); diff != "" { + t.Errorf("groupLocalitiesByPriority() diff(-want +got) %v", diff) + } + }) + } +} + +func (s) TestPriorityLocalitiesToClusterImpl_PickFirstWeightedShuffling_Disabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.PickFirstWeightedShuffling, false) + tests := []struct { + name string + localities []xdsresource.Locality + priorityName string + clusterUpdate xdsresource.ClusterUpdate + childPolicy *iserviceconfig.BalancerConfig + wantConfig *clusterimpl.LBConfig + wantEndpoints []resolver.Endpoint + wantErr bool + }{ + { + name: "round_robin_as_child_no_LRS", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-1"}}}, + Weight: 90, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-2"}}}, + Weight: 10, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + }, + ID: clients.Locality{Zone: "test-zone-1"}, + Weight: 20, + }, + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-1"}}}, + Weight: 90, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-2"}}}, + Weight: 10, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + }, + ID: clients.Locality{Zone: "test-zone-2"}, + Weight: 80, + }, + }, + priorityName: "test-priority", + childPolicy: &iserviceconfig.BalancerConfig{Name: roundrobin.Name}, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterName: testClusterName, + ClusterType: xdsresource.ClusterTypeEDS, + EDSServiceName: testEDSServiceName, + }, + // lrsServer is nil, so LRS policy will not be used. + wantConfig: &clusterimpl.LBConfig{ + Cluster: testClusterName, + ChildPolicy: &iserviceconfig.BalancerConfig{Name: roundrobin.Name}, + }, + // Endpoint weight is the product of locality weight and endpoint weight. + wantEndpoints: []resolver.Endpoint{ + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-1"}}}, 20, 20*90, "test-priority", &clients.Locality{Zone: "test-zone-1"}), + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-2"}}}, 20, 20*10, "test-priority", &clients.Locality{Zone: "test-zone-1"}), + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-1"}}}, 80, 80*90, "test-priority", &clients.Locality{Zone: "test-zone-2"}), + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-2"}}}, 80, 80*10, "test-priority", &clients.Locality{Zone: "test-zone-2"}), + }, + }, + { + name: "ring_hash_as_child", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-1"}}}, + Weight: 90, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-2"}}}, + Weight: 10, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + }, + ID: clients.Locality{Zone: "test-zone-1"}, + Weight: 20, + }, + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-1"}}}, + Weight: 90, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-2"}}}, + Weight: 10, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + }, + ID: clients.Locality{Zone: "test-zone-2"}, + Weight: 80, + }, + }, + priorityName: "test-priority", + childPolicy: &iserviceconfig.BalancerConfig{Name: ringhash.Name, Config: &iringhash.LBConfig{MinRingSize: 1, MaxRingSize: 2}}, + // lrsServer is nil, so LRS policy will not be used. + wantConfig: &clusterimpl.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: ringhash.Name, + Config: &iringhash.LBConfig{MinRingSize: 1, MaxRingSize: 2}, + }, + }, + // Endpoint weight is the product of locality weight and endpoint weight. + wantEndpoints: []resolver.Endpoint{ + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-1"}}}, 20, 20*90, "test-priority", &clients.Locality{Zone: "test-zone-1"}), + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-2"}}}, 20, 20*10, "test-priority", &clients.Locality{Zone: "test-zone-1"}), + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-1"}}}, 80, 80*90, "test-priority", &clients.Locality{Zone: "test-zone-2"}), + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-2"}}}, 80, 80*10, "test-priority", &clients.Locality{Zone: "test-zone-2"}), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotConfig, gotEndpoints, err := priorityLocalitiesToClusterImpl(tt.localities, tt.priorityName, tt.clusterUpdate, tt.childPolicy) + if (err != nil) != tt.wantErr { + t.Fatalf("priorityLocalitiesToClusterImpl() error = %v, wantErr %v", err, tt.wantErr) + } + if diff := cmp.Diff(gotConfig, tt.wantConfig); diff != "" { + t.Errorf("priorityLocalitiesToClusterImpl() diff (-got +want) %v", diff) + } + if diff := cmp.Diff(gotEndpoints, tt.wantEndpoints, cmp.AllowUnexported(attributes.Attributes{})); diff != "" { + t.Errorf("priorityLocalitiesToClusterImpl() diff (-got +want) %v", diff) + } + }) + } +} + +func (s) TestPriorityLocalitiesToClusterImpl_PickFirstWeightedShuffling_Enabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.PickFirstWeightedShuffling, true) + tests := []struct { + name string + localities []xdsresource.Locality + priorityName string + clusterUpdate xdsresource.ClusterUpdate + childPolicy *iserviceconfig.BalancerConfig + wantConfig *clusterimpl.LBConfig + wantEndpoints []resolver.Endpoint + wantErr bool + }{ + { + name: "round_robin_as_child_no_LRS", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-1"}}}, + Weight: 90, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-2"}}}, + Weight: 10, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + }, + ID: clients.Locality{Zone: "test-zone-1"}, + Weight: 20, + }, + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-1"}}}, + Weight: 90, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-2"}}}, + Weight: 10, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + }, + ID: clients.Locality{Zone: "test-zone-2"}, + Weight: 80, + }, + }, + priorityName: "test-priority", + childPolicy: &iserviceconfig.BalancerConfig{Name: roundrobin.Name}, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterName: testClusterName, + ClusterType: xdsresource.ClusterTypeEDS, + EDSServiceName: testEDSServiceName, + }, + // lrsServer is nil, so LRS policy will not be used. + wantConfig: &clusterimpl.LBConfig{ + Cluster: testClusterName, + ChildPolicy: &iserviceconfig.BalancerConfig{Name: roundrobin.Name}, + }, + // Endpoints weights are the product of normalized locality weight and + // endpoint weight, represented as a fixed-point number in uQ1.31 format. + // Locality weights are normalized as: + // locality 0: 20 / (100) = 0.2 + // locality 1: 80 / (100) = 0.8 + // In fixed-point uQ1.31 format, the weights are: + // locality 0: 0.2 * 2^31 = 429496729 + // locality 1: 0.8 * 2^31 = 1717986918 + // + // The normalized weights of endpoints in each locality are: + // locality 0: endpoint 0: 90 / (100) = 0.9, endpoint 1: 10 / (100) = 0.1 + // locality 1: endpoint 0: 90 / (100) = 0.9, endpoint 1: 10 / (100) = 0.1 + // + // The final endpoint weights are a product of the above normalized weights, + // which turns out to be: + // locality 0, endpoint 0: 0.2 * 0.9 = 386547056 + // locality 0, endpoint 1: 0.2 * 0.1 = 42949672 + // locality 1, endpoint 0: 0.8 * 0.9 = 1546188226 + // locality 1, endpoint 1: 0.8 * 0.1 = 171798691 + wantEndpoints: []resolver.Endpoint{ + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-1"}}}, 20, 386547056, "test-priority", &clients.Locality{Zone: "test-zone-1"}), + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-2"}}}, 20, 42949672, "test-priority", &clients.Locality{Zone: "test-zone-1"}), + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-1"}}}, 80, 1546188226, "test-priority", &clients.Locality{Zone: "test-zone-2"}), + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-2"}}}, 80, 171798691, "test-priority", &clients.Locality{Zone: "test-zone-2"}), + }, + }, + { + name: "ring_hash_as_child", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-1"}}}, + Weight: 90, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-2"}}}, + Weight: 10, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + }, + ID: clients.Locality{Zone: "test-zone-1"}, + Weight: 20, + }, + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-1"}}}, + Weight: 90, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-2"}}}, + Weight: 10, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }, + }, + ID: clients.Locality{Zone: "test-zone-2"}, + Weight: 80, + }, + }, + priorityName: "test-priority", + childPolicy: &iserviceconfig.BalancerConfig{Name: ringhash.Name, Config: &iringhash.LBConfig{MinRingSize: 1, MaxRingSize: 2}}, + // lrsServer is nil, so LRS policy will not be used. + wantConfig: &clusterimpl.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: ringhash.Name, + Config: &iringhash.LBConfig{MinRingSize: 1, MaxRingSize: 2}, + }, + }, + // Endpoints weights are the product of normalized locality weight and + // endpoint weight, represented as a fixed-point number in uQ1.31 format. + // Locality weights are normalized as: + // locality 0: 20 / (100) = 0.2 + // locality 1: 80 / (100) = 0.8 + // In fixed-point uQ1.31 format, the weights are: + // locality 0: 0.2 * 2^31 = 429496729 + // locality 1: 0.8 * 2^31 = 1717986918 + // + // The normalized weights of endpoints in each locality are: + // locality 0: endpoint 0: 90 / (100) = 0.9, endpoint 1: 10 / (100) = 0.1 + // locality 1: endpoint 0: 90 / (100) = 0.9, endpoint 1: 10 / (100) = 0.1 + // + // The final endpoint weights are a product of the above normalized weights, + // which turns out to be: + // locality 0, endpoint 0: 0.2 * 0.9 = 386547056 + // locality 0, endpoint 1: 0.2 * 0.1 = 42949672 + // locality 1, endpoint 0: 0.8 * 0.9 = 1546188226 + // locality 1, endpoint 1: 0.8 * 0.1 = 171798691 + wantEndpoints: []resolver.Endpoint{ + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-1"}}}, 20, 386547056, "test-priority", &clients.Locality{Zone: "test-zone-1"}), + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-1-2"}}}, 20, 42949672, "test-priority", &clients.Locality{Zone: "test-zone-1"}), + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-1"}}}, 80, 1546188226, "test-priority", &clients.Locality{Zone: "test-zone-2"}), + testEndpointWithAttrs(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "addr-2-2"}}}, 80, 171798691, "test-priority", &clients.Locality{Zone: "test-zone-2"}), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotConfig, gotEndpoints, err := priorityLocalitiesToClusterImpl(tt.localities, tt.priorityName, tt.clusterUpdate, tt.childPolicy) + if (err != nil) != tt.wantErr { + t.Fatalf("priorityLocalitiesToClusterImpl() error = %v, wantErr %v", err, tt.wantErr) + } + if diff := cmp.Diff(gotConfig, tt.wantConfig); diff != "" { + t.Errorf("priorityLocalitiesToClusterImpl() diff (-got +want) %v", diff) + } + if diff := cmp.Diff(gotEndpoints, tt.wantEndpoints, cmp.AllowUnexported(attributes.Attributes{})); diff != "" { + t.Errorf("priorityLocalitiesToClusterImpl() diff (-got +want) %v", diff) + } + }) + } +} + +// testEndpointWithAttrs creates a resolver.Endpoint with the attributes that +// priorityLocalitiesToClusterImpl is expected to set: hierarchy path, locality +// ID, locality weight (for xds_wrr_locality), and final endpoint weight +// (for pick_first weighted shuffling). The endpointWeight should be the +// expected final computed weight, not the raw endpoint weight from xDS. +func testEndpointWithAttrs(endpoint resolver.Endpoint, localityWeight, endpointWeight uint32, priority string, lID *clients.Locality) resolver.Endpoint { + path := []string{priority} + if lID != nil { + path = append(path, xdsinternal.LocalityString(*lID)) + endpoint = xdsinternal.SetLocalityIDInEndpoint(endpoint, *lID) + } + endpoint = hierarchy.SetInEndpoint(endpoint, path) + endpoint = wrrlocality.SetAddrInfo(endpoint, wrrlocality.AddrInfo{LocalityWeight: localityWeight}) + endpoint = weight.Set(endpoint, weight.EndpointInfo{Weight: endpointWeight}) + return endpoint +} + +func (s) TestConvertClusterImplMapToOutlierDetection(t *testing.T) { + tests := []struct { + name string + ciCfgsMap map[string]*clusterimpl.LBConfig + odCfg outlierdetection.LBConfig + wantODCfgs map[string]*outlierdetection.LBConfig + }{ + { + name: "single-entry-noop", + ciCfgsMap: map[string]*clusterimpl.LBConfig{ + "child1": { + Cluster: "cluster1", + }, + }, + odCfg: outlierdetection.LBConfig{ + Interval: 1<<63 - 1, + }, + wantODCfgs: map[string]*outlierdetection.LBConfig{ + "child1": { + Interval: 1<<63 - 1, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: "cluster1", + }, + }, + }, + }, + }, + { + name: "multiple-entries-noop", + ciCfgsMap: map[string]*clusterimpl.LBConfig{ + "child1": { + Cluster: "cluster1", + }, + "child2": { + Cluster: "cluster2", + }, + }, + odCfg: outlierdetection.LBConfig{ + Interval: 1<<63 - 1, + }, + wantODCfgs: map[string]*outlierdetection.LBConfig{ + "child1": { + Interval: 1<<63 - 1, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: "cluster1", + }, + }, + }, + "child2": { + Interval: 1<<63 - 1, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: "cluster2", + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := convertClusterImplMapToOutlierDetection(test.ciCfgsMap, test.odCfg) + if diff := cmp.Diff(test.wantODCfgs, got); diff != "" { + t.Fatalf("convertClusterImplMapToOutlierDetection() diff(-want +got) %v", diff) + } + }) + } +} + +// Test tests that when HTTP11Proxy is enabled, endpoints are rewritten to +// the proxy address with the original address preserved in proxyattributes, +// validating fallback and precedence rules. +func (s) TestPriorityLocalitiesToClusterImpl_HTTP11Proxy(t *testing.T) { + tests := []struct { + name string + localities []xdsresource.Locality + clusterUpdate xdsresource.ClusterUpdate + wantEndpoints []resolver.Endpoint + wantConnectAddr []string + }{ + { + name: "endpoint_metadata_only", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.5:443"}}, + }, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + Weight: 1, + Metadata: map[string]any{ + "envoy.http11_proxy_transport_socket.proxy_address": xdsresource.ProxyAddressMetadataValue{ + Address: "192.168.1.1:8080", + }, + }, + }, + }, + ID: clients.Locality{SubZone: "testzone-1"}, + Weight: 1, + }, + }, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterName: "test-cluster", + IsHTTP11ProxyEnabled: true, + }, + wantEndpoints: []resolver.Endpoint{ + testEndpointWithAttrs( + resolver.Endpoint{ + Addresses: []resolver.Address{ + proxyattributes.Set(resolver.Address{Addr: "192.168.1.1:8080"}, proxyattributes.Options{ConnectAddr: "10.0.0.5:443"}), + }, + }, + 1, 2147483648, "priority-0", &clients.Locality{SubZone: "testzone-1"}, + ), + }, + wantConnectAddr: []string{"10.0.0.5:443"}, + }, + { + name: "locality_metadata_fallback", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.5:443"}}, + }, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + Weight: 1, + }, + }, + ID: clients.Locality{SubZone: "subzone-1"}, + Weight: 1, + Metadata: map[string]any{ + "envoy.http11_proxy_transport_socket.proxy_address": xdsresource.ProxyAddressMetadataValue{ + Address: "192.168.1.1:8080", + }, + }, + }, + }, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterName: "test-cluster", + IsHTTP11ProxyEnabled: true, + }, + wantEndpoints: []resolver.Endpoint{ + testEndpointWithAttrs( + resolver.Endpoint{ + Addresses: []resolver.Address{ + proxyattributes.Set(resolver.Address{Addr: "192.168.1.1:8080"}, proxyattributes.Options{ConnectAddr: "10.0.0.5:443"}), + }, + }, + 1, 2147483648, "priority-0", &clients.Locality{SubZone: "subzone-1"}, + ), + }, + wantConnectAddr: []string{"10.0.0.5:443"}, + }, + { + name: "endpoint_metadata_precedence_over_locality_metadata", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.5:443"}}, + }, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + Weight: 1, + Metadata: map[string]any{ + "envoy.http11_proxy_transport_socket.proxy_address": xdsresource.ProxyAddressMetadataValue{ + Address: "192.168.1.1:8080", + }, + }, + }, + }, + ID: clients.Locality{SubZone: "subzone-1"}, + Weight: 1, + Metadata: map[string]any{ + "envoy.http11_proxy_transport_socket.proxy_address": xdsresource.ProxyAddressMetadataValue{ + Address: "192.168.1.99:8080", + }, + }, + }, + }, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterName: "test-cluster", + IsHTTP11ProxyEnabled: true, + }, + wantEndpoints: []resolver.Endpoint{ + testEndpointWithAttrs( + resolver.Endpoint{ + Addresses: []resolver.Address{proxyattributes.Set(resolver.Address{Addr: "192.168.1.1:8080"}, proxyattributes.Options{ConnectAddr: "10.0.0.5:443"})}, + }, + 1, 2147483648, "priority-0", &clients.Locality{SubZone: "subzone-1"}), + }, + wantConnectAddr: []string{"10.0.0.5:443"}, + }, + { + name: "multiple_localities_each_having_one_endpoint", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.5:443"}}, + }, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + Weight: 1, + }, + }, + ID: clients.Locality{SubZone: "subzone-1"}, + Weight: 1, + Metadata: map[string]any{ + "envoy.http11_proxy_transport_socket.proxy_address": xdsresource.ProxyAddressMetadataValue{ + Address: "192.168.1.1:8080", + }, + }, + }, + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.6:443"}}, + }, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + Weight: 1, + }, + }, + ID: clients.Locality{SubZone: "subzone-2"}, + Weight: 1, + Metadata: map[string]any{ + "envoy.http11_proxy_transport_socket.proxy_address": xdsresource.ProxyAddressMetadataValue{ + Address: "192.168.1.2:8080", + }, + }, + }, + }, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterName: "test-cluster", + IsHTTP11ProxyEnabled: true, + }, + wantEndpoints: []resolver.Endpoint{ + testEndpointWithAttrs( + resolver.Endpoint{ + Addresses: []resolver.Address{proxyattributes.Set(resolver.Address{Addr: "192.168.1.1:8080"}, proxyattributes.Options{ConnectAddr: "10.0.0.5:443"})}, + }, + 1, 1073741824, "priority-0", &clients.Locality{SubZone: "subzone-1"}, + ), + testEndpointWithAttrs( + resolver.Endpoint{ + Addresses: []resolver.Address{proxyattributes.Set(resolver.Address{Addr: "192.168.1.2:8080"}, proxyattributes.Options{ConnectAddr: "10.0.0.6:443"})}, + }, + 1, 1073741824, "priority-0", &clients.Locality{SubZone: "subzone-2"}, + ), + }, + wantConnectAddr: []string{"10.0.0.5:443", "10.0.0.6:443"}, + }, + { + name: " multiple_localities_each_having_multiple_endpoints", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.5:443"}}, + }, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + Weight: 1, + }, + { + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.6:443"}}, + }, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + Weight: 1, + Metadata: map[string]any{ + "envoy.http11_proxy_transport_socket.proxy_address": xdsresource.ProxyAddressMetadataValue{ + Address: "192.168.1.2:8080", + }, + }, + }, + }, + ID: clients.Locality{SubZone: "subzone-1"}, + Weight: 1, + Metadata: map[string]any{ + "envoy.http11_proxy_transport_socket.proxy_address": xdsresource.ProxyAddressMetadataValue{ + Address: "192.168.1.1:8080", + }, + }, + }, + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.7:443"}}, + }, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + Weight: 1, + Metadata: map[string]any{ + "envoy.http11_proxy_transport_socket.proxy_address": xdsresource.ProxyAddressMetadataValue{ + Address: "192.168.1.3:8080", + }, + }, + }, + { + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.8:443"}}, + }, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + Weight: 1, + Metadata: map[string]any{ + "envoy.http11_proxy_transport_socket.proxy_address": xdsresource.ProxyAddressMetadataValue{ + Address: "192.168.1.4:8080", + }, + }, + }, + }, + ID: clients.Locality{SubZone: "subzone-2"}, + Weight: 1, + }, + }, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterName: "test-cluster", + IsHTTP11ProxyEnabled: true, + }, + wantEndpoints: []resolver.Endpoint{ + testEndpointWithAttrs( + resolver.Endpoint{ + Addresses: []resolver.Address{ + proxyattributes.Set(resolver.Address{Addr: "192.168.1.1:8080"}, proxyattributes.Options{ConnectAddr: "10.0.0.5:443"}), + }, + }, + 1, 536870912, "priority-0", &clients.Locality{SubZone: "subzone-1"}, + ), + testEndpointWithAttrs( + resolver.Endpoint{ + Addresses: []resolver.Address{ + proxyattributes.Set(resolver.Address{Addr: "192.168.1.2:8080"}, proxyattributes.Options{ConnectAddr: "10.0.0.6:443"}), + }, + }, + 1, 536870912, "priority-0", &clients.Locality{SubZone: "subzone-1"}, + ), + testEndpointWithAttrs( + resolver.Endpoint{ + Addresses: []resolver.Address{ + proxyattributes.Set(resolver.Address{Addr: "192.168.1.3:8080"}, proxyattributes.Options{ConnectAddr: "10.0.0.7:443"}), + }, + }, + 1, 536870912, "priority-0", &clients.Locality{SubZone: "subzone-2"}, + ), + testEndpointWithAttrs( + resolver.Endpoint{ + Addresses: []resolver.Address{ + proxyattributes.Set(resolver.Address{Addr: "192.168.1.4:8080"}, proxyattributes.Options{ConnectAddr: "10.0.0.8:443"}), + }, + }, + 1, 536870912, "priority-0", &clients.Locality{SubZone: "subzone-2"}, + ), + }, + wantConnectAddr: []string{"10.0.0.5:443", "10.0.0.6:443", "10.0.0.7:443", "10.0.0.8:443"}, + }, + { + name: "missing_proxy_address_metadata", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.5:443"}}, + }, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + Weight: 1, + }, + }, + ID: clients.Locality{SubZone: "subzone-1"}, + Weight: 1, + }, + }, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterName: "test-cluster", + IsHTTP11ProxyEnabled: true, + }, + wantEndpoints: []resolver.Endpoint{ + testEndpointWithAttrs( + resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.5:443"}}, + }, + 1, 2147483648, "priority-0", &clients.Locality{SubZone: "subzone-1"}, + ), + }, + wantConnectAddr: []string{}, + }, + { + name: "invalid_proxy_address_metadata_type", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.5:443"}}, + }, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + Weight: 1, + Metadata: map[string]any{ + "envoy.http11_proxy_transport_socket.proxy_address": "192.168.1.1:8080", // string type instead of ProxyAddressMetadataValue struct + }, + }, + }, + ID: clients.Locality{SubZone: "subzone-1"}, + Weight: 1, + }, + }, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterName: "test-cluster", + IsHTTP11ProxyEnabled: true, + }, + wantEndpoints: []resolver.Endpoint{ + testEndpointWithAttrs( + resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.5:443"}}, + }, + 1, 2147483648, "priority-0", &clients.Locality{SubZone: "subzone-1"}, + ), + }, + }, + { + name: "http11proxy_disabled", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.5:443"}}, + }, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + Weight: 1, + Metadata: map[string]any{ + "envoy.http11_proxy_transport_socket.proxy_address": xdsresource.ProxyAddressMetadataValue{ + Address: "192.168.1.1:8080", + }, + }, + }, + }, + ID: clients.Locality{SubZone: "testzone-1"}, + Weight: 1, + }, + }, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterName: "test-cluster", + IsHTTP11ProxyEnabled: false, + }, + wantEndpoints: []resolver.Endpoint{ + testEndpointWithAttrs( + resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "10.0.0.5:443"}}, + }, + 1, 2147483648, "priority-0", &clients.Locality{SubZone: "testzone-1"}, + ), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, retEndpoints, err := priorityLocalitiesToClusterImpl(tt.localities, "priority-0", tt.clusterUpdate, nil) + if err != nil { + t.Fatalf("priorityLocalitiesToClusterImpl() failed: %v", err) + } + + if diff := cmp.Diff(retEndpoints, tt.wantEndpoints, cmp.AllowUnexported(attributes.Attributes{})); diff != "" { + t.Fatalf("priorityLocalitiesToClusterImpl() diff (-got +want) %v", diff) + } + + for idx, addr := range tt.wantConnectAddr { + ep := retEndpoints[idx].Addresses[0] + opts, ok := proxyattributes.Get(ep) + if !ok { + t.Fatalf("Expected proxyattributes to be set") + } + if opts.ConnectAddr != addr { + t.Fatalf("Unexpected ConnectAddr in proxyattributes, got: %s want: %s", opts.ConnectAddr, addr) + } + } + }) + } +} diff --git a/internal/xds/balancer/cdsbalancer/e2e_test/aggregate_cluster_test.go b/internal/xds/balancer/cdsbalancer/e2e_test/aggregate_cluster_test.go new file mode 100644 index 0000000..ea0c321 --- /dev/null +++ b/internal/xds/balancer/cdsbalancer/e2e_test/aggregate_cluster_test.go @@ -0,0 +1,1467 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e_test + +import ( + "context" + "fmt" + "net" + "slices" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils/pickfirst" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/fakeserver" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter +) + +// makeAggregateClusterResource returns an aggregate cluster resource with the +// given name and list of child names. +func makeAggregateClusterResource(name string, childNames []string) *v3clusterpb.Cluster { + return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: name, + Type: e2e.ClusterTypeAggregate, + ChildNames: childNames, + }) +} + +// makeLogicalDNSClusterResource returns a LOGICAL_DNS cluster resource with the +// given name and given DNS host and port. +func makeLogicalDNSClusterResource(name, dnsHost string, dnsPort uint32) *v3clusterpb.Cluster { + return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: name, + Type: e2e.ClusterTypeLogicalDNS, + DNSHostName: dnsHost, + DNSPort: dnsPort, + }) +} + +// setupDNS unregisters the DNS resolver and registers a manual resolver for the +// same scheme. This allows the test to mock the DNS resolution by supplying the +// addresses of the test backends. +// +// Returns the following: +// - a channel onto which the DNS target being resolved is written to by the +// mock DNS resolver +// - a manual resolver which is used to mock the actual DNS resolution +func setupDNS(t *testing.T) (chan resolver.Target, *manual.Resolver) { + targetCh := make(chan resolver.Target, 1) + + mr := manual.NewBuilderWithScheme("dns") + mr.BuildCallback = func(target resolver.Target, _ resolver.ClientConn, _ resolver.BuildOptions) { targetCh <- target } + + dnsResolverBuilder := resolver.Get("dns") + resolver.Register(mr) + + t.Cleanup(func() { resolver.Register(dnsResolverBuilder) }) + return targetCh, mr +} + +// TestAggregateCluster_WithTwoEDSClusters tests the case where the top-level +// cluster resource is an aggregate cluster. It verifies that RPCs fail when the +// management server has not responded to all requested EDS resources, and also +// that RPCs are routed to the highest priority cluster once all requested EDS +// resources have been sent by the management server. +func (s) TestAggregateCluster_WithTwoEDSClusters(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const clusterName1 = clusterName + "-cluster-1" + const clusterName2 = clusterName + "-cluster-2" + + // gotBothEDSRequests is fired when the management server receives EDS + // requests for both clusterName1 and clusterName2. This is used to block + // the test until both EDS requests have been received. + gotBothEDSRequests := grpcsync.NewEvent() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() != version.V3EndpointsURL { + return nil + } + if len(req.GetResourceNames()) == 0 { + // This happens at the end of the test when the grpc channel is + // being shut down and it is no longer interested in xDS + // resources. + return nil + } + + // Check if we have a request for both EDS resources. If so, fire + // the event to unblock the test. + names := req.GetResourceNames() + sortedNames := slices.Clone(names) + slices.Sort(sortedNames) + if cmp.Equal(sortedNames, []string{clusterName1, clusterName2}) { + gotBothEDSRequests.Fire() + } + return nil + }, + AllowResourceSubset: true, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start two test backends and extract their host and port. The first + // backend belongs to EDS cluster "cluster-1", while the second backend + // belongs to EDS cluster "cluster-2". + servers, cleanup2 := startTestServiceBackends(t, 2) + defer cleanup2() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Configure an aggregate cluster, two EDS clusters and only one endpoints + // resource (corresponding to the first EDS cluster) in the management + // server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{clusterName1, clusterName2}), + e2e.DefaultCluster(clusterName1, "", e2e.SecurityLevelNone), + e2e.DefaultCluster(clusterName2, "", e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterName1, "localhost", []uint32{uint32(ports[0])})}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Wait for both EDS resources to be requested. + select { + case <-gotBothEDSRequests.Done(): + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for all EDS resources %v to be requested", []string{clusterName1, clusterName2}) + } + + // Make an RPC with a short deadline. We expect this RPC to not succeed + // because the management server has not responded with all EDS resources + // requested. + client := testgrpc.NewTestServiceClient(cc) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() code %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + + // Update the management server with the second EDS resource. + resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterName2, "localhost", []uint32{uint32(ports[1])})) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Make an RPC and ensure that it gets routed to cluster-1, implicitly + // higher priority than cluster-2. + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } +} + +// TestAggregateCluster_WithTwoEDSClusters_PrioritiesChange tests the case where +// the top-level cluster resource is an aggregate cluster. It verifies that RPCs +// are routed to the highest priority EDS cluster. +func (s) TestAggregateCluster_WithTwoEDSClusters_PrioritiesChange(t *testing.T) { + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start two test backends and extract their host and port. The first + // backend belongs to EDS cluster "cluster-1", while the second backend + // belongs to EDS cluster "cluster-2". + servers, cleanup2 := startTestServiceBackends(t, 2) + defer cleanup2() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Configure an aggregate cluster, two EDS clusters and the corresponding + // endpoints resources in the management server. + const clusterName1 = clusterName + "cluster-1" + const clusterName2 = clusterName + "cluster-2" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{clusterName1, clusterName2}), + e2e.DefaultCluster(clusterName1, "", e2e.SecurityLevelNone), + e2e.DefaultCluster(clusterName2, "", e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(clusterName1, "localhost", []uint32{uint32(ports[0])}), + e2e.DefaultEndpoint(clusterName2, "localhost", []uint32{uint32(ports[1])}), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Make an RPC and ensure that it gets routed to cluster-1, implicitly + // higher priority than cluster-2. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } + + // Swap the priorities of the EDS clusters in the aggregate cluster. + resources.Clusters = []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{clusterName2, clusterName1}), + e2e.DefaultCluster(clusterName1, "", e2e.SecurityLevelNone), + e2e.DefaultCluster(clusterName2, "", e2e.SecurityLevelNone), + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for RPCs to get routed to cluster-2, which is now implicitly higher + // priority than cluster-1, after the priority switch above. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() == addrs[1].Addr { + break + } + } + if ctx.Err() != nil { + t.Fatal("Timeout waiting for RPCs to be routed to cluster-2 after priority switch") + } +} + +func hostAndPortFromAddress(t *testing.T, addr string) (string, uint32) { + t.Helper() + + host, p, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("Invalid serving address: %v", addr) + } + port, err := strconv.ParseUint(p, 10, 32) + if err != nil { + t.Fatalf("Invalid serving port %q: %v", p, err) + } + return host, uint32(port) +} + +// TestAggregateCluster_WithOneDNSCluster tests the case where the top-level +// cluster resource is an aggregate cluster that resolves to a single +// LOGICAL_DNS cluster. The test verifies that RPCs can be made to backends that +// make up the LOGICAL_DNS cluster. +func (s) TestAggregateCluster_WithOneDNSCluster(t *testing.T) { + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + host, port := hostAndPortFromAddress(t, server.Address) + + // Configure an aggregate cluster pointing to a single LOGICAL_DNS cluster. + const dnsClusterName = clusterName + "-dns" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{dnsClusterName}), + makeLogicalDNSClusterResource(dnsClusterName, host, uint32(port)), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Make an RPC and ensure that it gets routed to the first backend since the + // child policy for a LOGICAL_DNS cluster is pick_first by default. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != server.Address { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, server.Address) + } +} + +// Tests the case where the top-level cluster resource is an aggregate cluster +// that resolves to a single LOGICAL_DNS cluster. The specified dns hostname is +// expected to fail url parsing. The test verifies that the channel moves to +// TRANSIENT_FAILURE. +func (s) TestAggregateCluster_WithOneDNSCluster_ParseFailure(t *testing.T) { + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Configure an aggregate cluster pointing to a single LOGICAL_DNS cluster. + const dnsClusterName = clusterName + "-dns" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{dnsClusterName}), + makeLogicalDNSClusterResource(dnsClusterName, "%gh&%ij", uint32(8080)), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure that the ClientConn moves to TransientFailure. + for state := cc.GetState(); state != connectivity.TransientFailure; state = cc.GetState() { + if !cc.WaitForStateChange(ctx, state) { + t.Fatalf("Timed out waiting for state change. got %v; want %v", state, connectivity.TransientFailure) + } + } +} + +// Tests the case where the top-level cluster resource is an aggregate cluster +// that resolves to a single LOGICAL_DNS cluster. The test verifies that RPCs +// can be made to backends that make up the LOGICAL_DNS cluster. The hostname of +// the LOGICAL_DNS cluster is updated, and the test verifies that RPCs can be +// made to backends that the new hostname resolves to. +func (s) TestAggregateCluster_WithOneDNSCluster_HostnameChange(t *testing.T) { + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start two test backends and extract their host and port. The first + // backend is used initially for the LOGICAL_DNS cluster and an update + // switches the cluster to use the second backend. + servers, cleanup2 := startTestServiceBackends(t, 2) + defer cleanup2() + + // Configure an aggregate cluster pointing to a single LOGICAL_DNS cluster. + const dnsClusterName = clusterName + "-dns" + dnsHostName, dnsPort := hostAndPortFromAddress(t, servers[0].Address) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{dnsClusterName}), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Make an RPC and ensure that it gets routed to the first backend since the + // child policy for a LOGICAL_DNS cluster is pick_first by default. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != servers[0].Address { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, servers[0].Address) + } + + // Update the LOGICAL_DNS cluster's hostname to point to the second backend. + dnsHostName, dnsPort = hostAndPortFromAddress(t, servers[1].Address) + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{dnsClusterName}), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that traffic moves to the second backend eventually. + for ctx.Err() == nil { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() == servers[1].Address { + break + } + } + if ctx.Err() != nil { + t.Fatal("Timeout when waiting for RPCs to switch to the second backend") + } +} + +// TestAggregateCluster_WithEDSAndDNS tests the case where the top-level cluster +// resource is an aggregate cluster that resolves to an EDS and a LOGICAL_DNS +// cluster. The test verifies that RPCs fail until both clusters are resolved to +// endpoints, and RPCs are routed to the higher priority EDS cluster. +func (s) TestAggregateCluster_WithEDSAndDNS(t *testing.T) { + dnsTargetCh, dnsR := setupDNS(t) + + // Start an xDS management server that pushes the name of the requested EDS + // resource onto a channel. + edsResourceCh := make(chan string, 1) + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() != version.V3EndpointsURL { + return nil + } + if len(req.GetResourceNames()) == 0 { + // This happens at the end of the test when the grpc channel is + // being shut down and it is no longer interested in xDS + // resources. + return nil + } + select { + case edsResourceCh <- req.GetResourceNames()[0]: + default: + } + return nil + }, + AllowResourceSubset: true, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start two test backends and extract their host and port. The first + // backend is used for the EDS cluster and the second backend is used for + // the LOGICAL_DNS cluster. + servers, cleanup3 := startTestServiceBackends(t, 2) + defer cleanup3() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Configure an aggregate cluster pointing to an EDS and DNS cluster. Also + // configure an endpoints resource for the EDS cluster. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + dnsHostName = "dns_host" + dnsPort = uint32(8080) + ) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, "", e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsClusterName, "localhost", []uint32{uint32(ports[0])})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure that an EDS request is sent for the expected resource name. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for EDS request to be received on the management server") + case name := <-edsResourceCh: + if name != edsClusterName { + t.Fatalf("Received EDS request with resource name %q, want %q", name, edsClusterName) + } + } + + // Ensure that the DNS resolver is started for the expected target. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for DNS resolver to be started") + case target := <-dnsTargetCh: + got, want := target.Endpoint(), fmt.Sprintf("%s:%d", dnsHostName, dnsPort) + if got != want { + t.Fatalf("DNS resolution started for target %q, want %q", got, want) + } + } + + // Make an RPC with a short deadline. We expect this RPC to not succeed + // because the DNS resolver has not responded with endpoint addresses. + client := testgrpc.NewTestServiceClient(cc) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() code %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + + // Update DNS resolver with test backend addresses. + dnsR.UpdateState(resolver.State{Endpoints: addrsToEndpoints(addrs[1:])}) + + // Make an RPC and ensure that it gets routed to the first backend since the + // EDS cluster is of higher priority than the LOGICAL_DNS cluster. + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } +} + +// TestAggregateCluster_SwitchEDSAndDNS tests the case where the top-level +// cluster resource is an aggregate cluster. It initially resolves to a single +// EDS cluster. The test verifies that RPCs are routed to backends in the EDS +// cluster. Subsequently, the aggregate cluster resolves to a single DNS +// cluster. The test verifies that RPCs are successful, this time to backends in +// the DNS cluster. +func (s) TestAggregateCluster_SwitchEDSAndDNS(t *testing.T) { + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start two test backends and extract their host and port. The first + // backend is used for the EDS cluster and the second backend is used for + // the LOGICAL_DNS cluster. + servers, cleanup3 := startTestServiceBackends(t, 2) + defer cleanup3() + addrs, ports := backendAddressesAndPorts(t, servers) + dnsHostName, dnsPort := hostAndPortFromAddress(t, addrs[1].Addr) + + // Configure an aggregate cluster pointing to a single EDS cluster. Also, + // configure the underlying EDS cluster (and the corresponding endpoints + // resource) and DNS cluster (will be used later in the test). + const dnsClusterName = clusterName + "-dns" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsServiceName}), + e2e.DefaultCluster(edsServiceName, "", e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{uint32(ports[0])})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure that the RPC is routed to the appropriate backend. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } + + // Update the aggregate cluster to point to a single DNS cluster. + resources.Clusters = []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{dnsClusterName}), + e2e.DefaultCluster(edsServiceName, "", e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that start getting routed to the backend corresponding to the + // LOGICAL_DNS cluster. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)) + if peer.Addr.String() == addrs[1].Addr { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for RPCs to be routed to backend %q in the DNS cluster", addrs[1].Addr) + } +} + +// TestAggregateCluster_BadEDS_GoodToBadDNS tests the case where the top-level +// cluster is an aggregate cluster that resolves to an EDS and LOGICAL_DNS +// cluster. The test first asserts that no RPCs can be made after receiving an +// EDS response with zero endpoints because no update has been received from the +// DNS resolver yet. Once the DNS resolver pushes an update, the test verifies +// that we switch to the DNS cluster and can make a successful RPC. At this +// point when the DNS cluster returns an error, the test verifies that RPCs are +// still successful. This is the expected behavior because the cluster resolver +// policy eats errors from DNS Resolver after it has returned an error. +func (s) TestAggregateCluster_BadEDS_GoodToBadDNS(t *testing.T) { + dnsTargetCh, dnsR := setupDNS(t) + + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start two test backends. + servers, cleanup := startTestServiceBackends(t, 2) + defer cleanup() + addrs, _ := backendAddressesAndPorts(t, servers) + + // Configure an aggregate cluster pointing to an EDS and LOGICAL_DNS + // cluster. Also configure an endpoints resource for the EDS cluster which + // triggers a NACK. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + dnsHostName = "dns_host" + dnsPort = uint32(8080) + ) + emptyEndpointResource := e2e.DefaultEndpoint(edsServiceName, "localhost", nil) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, edsServiceName, e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{emptyEndpointResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Make an RPC with a short deadline. We expect this RPC to not succeed + // because the EDS resource came back with no endpoints, and we are yet to + // push an update through the DNS resolver. + client := testgrpc.NewTestServiceClient(cc) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() code %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + + // Ensure that the DNS resolver is started for the expected target. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for DNS resolver to be started") + case target := <-dnsTargetCh: + got, want := target.Endpoint(), fmt.Sprintf("%s:%d", dnsHostName, dnsPort) + if got != want { + t.Fatalf("DNS resolution started for target %q, want %q", got, want) + } + } + + // Update DNS resolver with test backend addresses. + dnsR.UpdateState(resolver.State{Endpoints: addrsToEndpoints(addrs)}) + + // Ensure that RPCs start getting routed to the first backend since the + // child policy for a LOGICAL_DNS cluster is pick_first by default. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Logf("EmptyCall() failed: %v", err) + continue + } + if peer.Addr.String() == addrs[0].Addr { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for RPCs to be routed to backend %q in the DNS cluster", addrs[0].Addr) + } + + // Push an error from the DNS resolver as well. + dnsErr := fmt.Errorf("DNS error") + dnsR.CC().ReportError(dnsErr) + + // Ensure that RPCs continue to succeed for the next second. + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } + } +} + +// TestAggregateCluster_BadEDS_GoodToBadDNS tests the case where the top-level +// cluster is an aggregate cluster that resolves to an EDS and LOGICAL_DNS +// cluster. The test first sends an EDS response which triggers an NACK. Once +// the DNS resolver pushes an update, the test verifies that we switch to the +// DNS cluster and can make a successful RPC. +func (s) TestAggregateCluster_BadEDSFromError_GoodToBadDNS(t *testing.T) { + // Start an xDS management server and block it from continuously re-sending + // the same EDS response that triggers a NACK. This is to ensure that the + // test can proceed to the point of verifying that we fall back to the + // LOGICAL_DNS cluster. More more details see: + // https://github.com/grpc/grpc-go/issues/8994. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + edsResponseSeen := atomic.Bool{} + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + if resp.GetTypeUrl() != version.V3EndpointsURL { + return + } + if edsResponseSeen.Load() { + t.Logf("Received EDS response again, blocking to prevent test from busylooping on the same EDS response: %v", resp) + <-ctx.Done() + return + } + edsResponseSeen.Store(true) + }, + AllowResourceSubset: true, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + dnsHostName, dnsPort := hostAndPortFromAddress(t, server.Address) + + // Configure an aggregate cluster pointing to an EDS and LOGICAL_DNS + // cluster. Also configure an empty endpoints resource for the EDS cluster + // that contains no endpoints. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + ) + nackEndpointResource := e2e.DefaultEndpoint(edsServiceName, "localhost", nil) + nackEndpointResource.Endpoints = []*v3endpointpb.LocalityLbEndpoints{ + { + LoadBalancingWeight: &wrapperspb.UInt32Value{ + Value: 0, // causes an NACK + }, + }, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, edsServiceName, e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{nackEndpointResource}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure that RPCs start getting routed to the first backend since the + // child policy for a LOGICAL_DNS cluster is pick_first by default. + pickfirst.CheckRPCsToBackend(ctx, cc, resolver.Address{Addr: server.Address}) +} + +// TestAggregateCluster_BadDNS_GoodEDS tests the case where the top-level +// cluster is an aggregate cluster that resolves to an LOGICAL_DNS and EDS +// cluster. When the DNS Resolver returns an error and EDS cluster returns a +// good update, this test verifies the cluster_resolver balancer correctly falls +// back from the LOGICAL_DNS cluster to the EDS cluster. +func (s) TestAggregateCluster_BadDNS_GoodEDS(t *testing.T) { + dnsTargetCh, dnsR := setupDNS(t) + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + _, edsPort := hostAndPortFromAddress(t, server.Address) + + // Configure an aggregate cluster pointing to an LOGICAL_DNS and EDS + // cluster. Also configure an endpoints resource for the EDS cluster. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + dnsHostName = "bad.ip.v4.address" + dnsPort = 8080 + ) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{dnsClusterName, edsClusterName}), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + e2e.DefaultCluster(edsClusterName, edsServiceName, e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{uint32(edsPort)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure that the DNS resolver is started for the expected target. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for DNS resolver to be started") + case target := <-dnsTargetCh: + got, want := target.Endpoint(), fmt.Sprintf("%s:%d", dnsHostName, dnsPort) + if got != want { + t.Fatalf("DNS resolution started for target %q, want %q", got, want) + } + } + + // Produce a bad resolver update from the DNS resolver. + dnsErr := fmt.Errorf("DNS error") + dnsR.CC().ReportError(dnsErr) + + // RPCs should work, higher level DNS cluster errors so should fallback to + // EDS cluster. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != server.Address { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, server.Address) + } +} + +// TestAggregateCluster_BadEDS_BadDNS tests the case where the top-level cluster +// is an aggregate cluster that resolves to an EDS and LOGICAL_DNS cluster. When +// the EDS request returns a resource that contains no endpoints, the test +// verifies that we switch to the DNS cluster. When the DNS cluster returns an +// error, the test verifies that RPCs fail with the error triggered by the DNS +// Discovery Mechanism (from sending an empty address list down). +func (s) TestAggregateCluster_BadEDS_BadDNS(t *testing.T) { + dnsTargetCh, dnsR := setupDNS(t) + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Configure an aggregate cluster pointing to an EDS and LOGICAL_DNS + // cluster. Also configure an empty endpoints resource for the EDS cluster + // that contains no endpoints. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + dnsHostName = "bad.ip.v4.address" + dnsPort = 8080 + ) + emptyEndpointResource := e2e.DefaultEndpoint(edsServiceName, "localhost", nil) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, edsServiceName, e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{emptyEndpointResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), 50*defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure that the DNS resolver is started for the expected target. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for DNS resolver to be started") + case target := <-dnsTargetCh: + got, want := target.Endpoint(), fmt.Sprintf("%s:%d", dnsHostName, dnsPort) + if got != want { + t.Fatalf("DNS resolution started for target %q, want %q", got, want) + } + } + + // Produce a bad resolver update from the DNS resolver. + dnsR.CC().ReportError(fmt.Errorf("DNS error")) + + // Ensure that the error from the DNS Resolver leads to an empty address + // update for both priorities. + client := testgrpc.NewTestServiceClient(cc) + for ctx.Err() == nil { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Fatal("EmptyCall() succeeded when expected to fail") + } + if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), "no targets to pick from") { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for RPCs to fail with expected code and error") + } +} + +// TestAggregateCluster_NoFallback_EDSNackedWithPreviousGoodUpdate tests the +// scenario where the top-level cluster is an aggregate cluster that resolves to +// an EDS and LOGICAL_DNS cluster. The management server first sends a good EDS +// response for the EDS cluster and the test verifies that RPCs get routed to +// the EDS cluster. The management server then sends a bad EDS response. The +// test verifies that the cluster_resolver LB policy continues to use the +// previously received good update and that RPCs still get routed to the EDS +// cluster. +func (s) TestAggregateCluster_NoFallback_EDSNackedWithPreviousGoodUpdate(t *testing.T) { + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start two test backends and extract their host and port. The first + // backend is used for the EDS cluster and the second backend is used for + // the LOGICAL_DNS cluster. + servers, cleanup3 := startTestServiceBackends(t, 2) + defer cleanup3() + addrs, ports := backendAddressesAndPorts(t, servers) + dnsHostName, dnsPort := hostAndPortFromAddress(t, servers[1].Address) + + // Configure an aggregate cluster pointing to an EDS and DNS cluster. Also + // configure an endpoints resource for the EDS cluster. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + ) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, "", e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsClusterName, "localhost", []uint32{uint32(ports[0])})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Make an RPC and ensure that it gets routed to the first backend since the + // EDS cluster is of higher priority than the LOGICAL_DNS cluster. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } + + // Push an EDS resource from the management server that is expected to be + // NACKed by the xDS client. Since the cluster_resolver LB policy has a + // previously received good EDS resource, it will continue to use that. + resources.Endpoints[0].Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that RPCs continue to get routed to the EDS cluster for the next + // second. + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } + } +} + +// TestAggregateCluster_Fallback_EDSNackedWithoutPreviousGoodUpdate tests the +// scenario where the top-level cluster is an aggregate cluster that resolves to +// an EDS and LOGICAL_DNS cluster. The management server sends a bad EDS +// response. The test verifies that the cluster_resolver LB policy falls back to +// the LOGICAL_DNS cluster, because it is supposed to treat the bad EDS response +// as though it received an update with no endpoints. +func (s) TestAggregateCluster_Fallback_EDSNackedWithoutPreviousGoodUpdate(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server and block it from continuously re-sending + // the same EDS response that triggers a NACK. This is to ensure that the + // test can proceed to the point of verifying that we fall back to the + // LOGICAL_DNS cluster. More more details see: + // https://github.com/grpc/grpc-go/issues/8994. + edsResponseSeen := atomic.Bool{} + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + if resp.GetTypeUrl() != version.V3EndpointsURL { + return + } + if edsResponseSeen.Load() { + t.Logf("Received EDS response again, blocking to prevent test from busylooping on the same EDS response: %v", resp) + <-ctx.Done() + return + } + edsResponseSeen.Store(true) + }, + AllowResourceSubset: true, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start two test backends and extract their host and port. The first + // backend is used for the EDS cluster and the second backend is used for + // the LOGICAL_DNS cluster. + servers, cleanup := startTestServiceBackends(t, 2) + defer cleanup() + addrs, ports := backendAddressesAndPorts(t, servers) + dnsHostName, dnsPort := hostAndPortFromAddress(t, servers[1].Address) + + // Configure an aggregate cluster pointing to an EDS and DNS cluster. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + ) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, "", e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsClusterName, "localhost", []uint32{uint32(ports[0])})}, + SkipValidation: true, + } + + // Set a load balancing weight of 0 for the backend in the EDS resource. + // This is expected to be NACKed by the xDS client. Since the + resources.Endpoints[0].Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Make an RPC and ensure that it gets routed to the LOGICAL_DNS cluster. + peer := &peer.Peer{} + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[1].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[1].Addr) + } +} + +// TestAggregateCluster_Fallback_EDS_ResourceNotFound tests the scenario where +// the top-level cluster is an aggregate cluster that resolves to an EDS and +// LOGICAL_DNS cluster. The management server does not respond with the EDS +// cluster. The test verifies that the cluster_resolver LB policy falls back to +// the LOGICAL_DNS cluster in this case. +func (s) TestAggregateCluster_Fallback_EDS_ResourceNotFound(t *testing.T) { + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start a test backend for the LOGICAL_DNS cluster. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + dnsHostName, dnsPort := hostAndPortFromAddress(t, server.Address) + + // Configure an aggregate cluster pointing to an EDS and DNS cluster. No + // endpoints are configured for the EDS cluster. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + ) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, "", e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client talking to the above management server, configured + // with a short watch expiry timeout. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + // Create a new xDS client with a short watch expiry to quickly detect the + // missing endpoints resource. + xdsClient, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + if internal.NewXDSResolverWithClientForTesting == nil { + t.Fatalf("internal.NewXDSResolverWithClientForTesting is nil") + } + r, err := internal.NewXDSResolverWithClientForTesting.(func(xdsclient.XDSClient) (resolver.Builder, error))(xdsClient) + if err != nil { + t.Fatalf("failed to create resolver") + } + cc, err := grpc.NewClient("xds:///"+serviceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + defer cc.Close() + // Make an RPC and ensure that it gets routed to the LOGICAL_DNS cluster. + // Even though the EDS cluster is of higher priority, since the management + // server does not respond with an EDS resource, the cluster_resolver LB + // policy is expected to fallback to the LOGICAL_DNS cluster once the watch + // timeout expires. + peer := &peer.Peer{} + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != server.Address { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, server.Address) + } +} + +func addrsToEndpoints(addrs []resolver.Address) []resolver.Endpoint { + endpoints := make([]resolver.Endpoint, len(addrs)) + for i, addr := range addrs { + endpoints[i] = resolver.Endpoint{Addresses: []resolver.Address{addr}} + } + return endpoints +} + +// Tests that when an aggregate cluster (Root -> [PrimaryEDS, SecondaryEDS]) has +// LRS enabled on both leaf clusters: +// 1. LRS reports are sent only for the primary leaf cluster while healthy. +// 2. When the primary leaf cluster fails, traffic shifts to the secondary leaf +// cluster and LRS stats are reported for the secondary leaf cluster. +func (s) TestAggregateCluster_LRS(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const ( + clusterName1 = clusterName + "-cluster-1" + clusterName2 = clusterName + "-cluster-2" + ) + + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + AllowResourceSubset: true, + SupportLoadReportingService: true, + }) + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + servers, cleanup := startTestServiceBackends(t, 2) + defer cleanup() + _, ports := backendAddressesAndPorts(t, servers) + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{clusterName1, clusterName2}), + e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName1, + Type: e2e.ClusterTypeEDS, + EnableLRS: true, + }), + e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName2, + Type: e2e.ClusterTypeEDS, + EnableLRS: true, + }), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(clusterName1, "localhost", []uint32{uint32(ports[0])}), + e2e.DefaultEndpoint(clusterName2, "localhost", []uint32{uint32(ports[1])}), + }, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + client := testgrpc.NewTestServiceClient(cc) + + // Ensure LRS stream is opened. + if _, err := managementServer.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Timeout waiting for LRS stream open: %v", err) + } + if _, err := managementServer.LRSServer.LRSRequestChan.Receive(ctx); err != nil { + t.Fatalf("Timeout waiting for initial LRS request: %v", err) + } + + // Make RPC calls and verify every call reaches the primary cluster + // (clusterName1). + for i := 0; i < 10; i++ { + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if got, want := peer.Addr.String(), servers[0].Address; got != want { + t.Fatalf("EmptyCall() call #%d routed to %q, want %q", i, got, want) + } + } + + // Send LRS response requesting load reporting every 10ms. + managementServer.LRSServer.LRSResponseChan <- &fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + SendAllClusters: true, + LoadReportingInterval: durationpb.New(defaultLoadReportingInterval), + }, + } + + // Verify LRS reports load for clusterName1 and 0 load for clusterName2. + wantClusterStats := &v3endpointpb.ClusterStats{ + ClusterName: clusterName1, + UpstreamLocalityStats: []*v3endpointpb.UpstreamLocalityStats{ + { + Locality: &v3corepb.Locality{Region: "region-1", Zone: "zone-1", SubZone: "subzone-1"}, + TotalSuccessfulRequests: 10, + TotalIssuedRequests: 10, + }, + }, + } + +waitForPrimaryStats: + for { + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for LRS load report for %q", clusterName1) + case req := <-managementServer.LRSServer.LRSRequestChan.C: + loadStats := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) + if len(loadStats.ClusterStats) == 0 { + continue + } + opts := []cmp.Option{ + protocmp.Transform(), + protocmp.IgnoreFields(&v3endpointpb.ClusterStats{}, "load_report_interval"), + } + if diff := cmp.Diff([]*v3endpointpb.ClusterStats{wantClusterStats}, loadStats.ClusterStats, opts...); diff != "" { + t.Fatalf("Unexpected diff in LRS ClusterStats for %q (-want +got):\n%s", clusterName1, diff) + } + break waitForPrimaryStats + } + } + + // Fail primary cluster by clearing endpoints for clusterName1. + resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(clusterName1, "localhost", nil), + e2e.DefaultEndpoint(clusterName2, "localhost", []uint32{uint32(ports[1])}), + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Make RPCs and verify failover to secondary backend. + peer := &peer.Peer{} + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err == nil && peer.Addr.String() == servers[1].Address { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout waiting for RPCs to switch to secondary backend %q", servers[1].Address) + } + + // Verify LRS report now includes load for secondary cluster (clusterName2). + wantClusterStats = &v3endpointpb.ClusterStats{ + ClusterName: clusterName2, + UpstreamLocalityStats: []*v3endpointpb.UpstreamLocalityStats{ + { + Locality: &v3corepb.Locality{Region: "region-1", Zone: "zone-1", SubZone: "subzone-1"}, + TotalSuccessfulRequests: 1, + }, + }, + } + for { + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for LRS load report for secondary cluster %q", clusterName2) + case req := <-managementServer.LRSServer.LRSRequestChan.C: + loadStats := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) + for _, load := range loadStats.ClusterStats { + if load.ClusterName != clusterName2 { + continue + } + // The periodic LRS timer may fire while a failover RPC is + // in flight, producing a transitional report with 0 + // successful requests. Skip such reports until the report + // with the completed RPC arrives. + if len(load.UpstreamLocalityStats) == 0 || load.UpstreamLocalityStats[0].TotalSuccessfulRequests == 0 { + continue + } + opts := []cmp.Option{ + protocmp.Transform(), + protocmp.IgnoreFields(&v3endpointpb.ClusterStats{}, "load_report_interval"), + protocmp.IgnoreFields(&v3endpointpb.UpstreamLocalityStats{}, "total_issued_requests", "total_requests_in_progress"), + } + if diff := cmp.Diff(wantClusterStats, load, opts...); diff != "" { + t.Fatalf("Unexpected diff in LRS ClusterStats for %q (-want +got):\n%s", clusterName2, diff) + } + return + } + } + } +} diff --git a/internal/xds/balancer/cdsbalancer/e2e_test/balancer_test.go b/internal/xds/balancer/cdsbalancer/e2e_test/balancer_test.go new file mode 100644 index 0000000..6b7df2f --- /dev/null +++ b/internal/xds/balancer/cdsbalancer/e2e_test/balancer_test.go @@ -0,0 +1,420 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e_test + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/balancer/clusterimpl" + "google.golang.org/grpc/internal/xds/balancer/outlierdetection" + "google.golang.org/grpc/internal/xds/balancer/priority" + "google.golang.org/grpc/internal/xds/balancer/wrrlocality" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/internal/xds/balancer/cdsbalancer" // Register the "cds_experimental" LB policy. + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter + _ "google.golang.org/grpc/internal/xds/resolver" // Register the xds resolver +) + +// setupAndDial performs common setup across all tests +// +// - creates an xDS client with the passed in bootstrap contents +// - creates a xds resolver to be used +// - creates a ClientConn to talk to the test backends +// +// Returns a function to close the ClientConn and the xDS client. +func setupAndDial(t *testing.T, bootstrapContents []byte) (*grpc.ClientConn, func()) { + t.Helper() + // Create an xDS resolver with the above bootstrap configuration. + r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("xDS resolver creation failed: %v", err) + } + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient("xds:///"+serviceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + return cc, func() { + cc.Close() + } +} + +// TestErrorFromParentLB_ConnectionError tests the case where a connection error +// is sent by the CDS LB Policy. The CDS LB policy sends a connection error when +// the ADS stream to the management server breaks. The test verifies that there +// is no perceivable effect because of this connection error, and that RPCs +// continue to work (because the LB policies are expected to use previously +// received xDS resources). +func (s) TestErrorFromParentLB_ConnectionError(t *testing.T) { + // Create a listener to be used by the management server. The test will + // close this listener to simulate ADS stream breakage. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + + // Start an xDS management server with the above restartable listener, and + // push a channel when the stream is closed. + streamClosedCh := make(chan struct{}, 1) + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + OnStreamClosed: func(int64, *v3corepb.Node) { + select { + case streamClosedCh <- struct{}{}: + default: + } + }, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: serviceName, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Close the listener and ensure that the ADS stream breaks. + lis.Close() + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for ADS stream to close") + default: + } + + // Ensure that RPCs continue to succeed for the next second. + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + } +} + +// TestErrorFromParentLB_ResourceNotFound tests the case where a +// resource-not-found error is received by the CDS LB policy for a cluster. The +// resource-not-found error is received when the cluster resource associated +// with these LB policies is removed by the management server. The test verifies +// that the associated EDS is canceled and RPCs fail. It also ensures that when +// the Cluster resource is added back, the EDS resource is re-requested and RPCs +// being to succeed. +func (s) TestErrorFromParentLB_ResourceNotFound(t *testing.T) { + // Start an xDS management server that uses a couple of channels to + // notify the test about the following events: + // - an EDS requested with the expected resource name is requested + // - EDS resource is unrequested, i.e, an EDS request with no resource name + // is received, which indicates that we are no longer interested in that + // resource. + edsResourceRequestedCh := make(chan struct{}, 1) + edsResourceCanceledCh := make(chan struct{}, 1) + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3EndpointsURL { + switch len(req.GetResourceNames()) { + case 0: + select { + case edsResourceCanceledCh <- struct{}{}: + default: + } + case 1: + if req.GetResourceNames()[0] == edsServiceName { + select { + case edsResourceRequestedCh <- struct{}{}: + default: + } + } + default: + t.Errorf("Unexpected number of resources, %d, in an EDS request", len(req.GetResourceNames())) + } + } + return nil + }, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure cluster and endpoints resources in the management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Wait for the EDS resource to be requested. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for EDS resource to be requested") + case <-edsResourceRequestedCh: + } + + // Ensure that a successful RPC can be made. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + oldCluster := resources.Clusters + // Delete the cluster resource from the management server. + resources.Clusters = nil + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the EDS resource to be not requested anymore. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for EDS resource to not be requested") + case <-edsResourceCanceledCh: + } + + // Ensure that RPCs start to fail with expected error. + wantErr := fmt.Sprintf("resource %q of type %q has been removed", clusterName, "ClusterResource") + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + _, err := client.EmptyCall(sCtx, &testpb.Empty{}) + if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), wantErr) { + break + } + if err != nil { + t.Logf("EmptyCall failed: %v", err) + } + } + if ctx.Err() != nil { + t.Fatalf("RPCs did not fail after removal of Cluster resource") + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Add the cluster resource back to the management server. + resources.Clusters = oldCluster + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the EDS resource to be requested again. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for EDS resource to be requested") + case <-edsResourceRequestedCh: + } + + // Ensure that a successful RPC can be made. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); err != nil { + t.Logf("EmptyCall failed: %v", err) + continue + } + break + } + if ctx.Err() != nil { + t.Fatalf("RPCs did not fail after removal of Cluster resource") + } +} + +// Test verifies that when the received Cluster resource contains outlier +// detection configuration, the LB config pushed to the priority policy contains +// the appropriate configuration for the outlier detection LB policy. +func (s) TestOutlierDetectionConfigPropagationToChildPolicy(t *testing.T) { + // Unregister the priority balancer builder for the duration of this test, + // and register a policy under the same name that makes the LB config + // pushed to it available to the test. + priorityBuilder := balancer.Get(priority.Name) + internal.BalancerUnregister(priorityBuilder.Name()) + lbCfgCh := make(chan serviceconfig.LoadBalancingConfig, 1) + stub.Register(priority.Name, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = priorityBuilder.Build(bd.ClientConn, bd.BuildOptions) + }, + ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return priorityBuilder.(balancer.ConfigParser).ParseConfig(lbCfg) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + select { + case lbCfgCh <- ccs.BalancerConfig: + default: + } + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + }) + defer balancer.Register(priorityBuilder) + + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure cluster and endpoints resources in the management server. + cluster := e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone) + cluster.OutlierDetection = &v3clusterpb.OutlierDetection{ + Interval: durationpb.New(10 * time.Second), + BaseEjectionTime: durationpb.New(30 * time.Second), + MaxEjectionTime: durationpb.New(300 * time.Second), + MaxEjectionPercent: wrapperspb.UInt32(10), + SuccessRateStdevFactor: wrapperspb.UInt32(2000), + EnforcingSuccessRate: wrapperspb.UInt32(50), + SuccessRateMinimumHosts: wrapperspb.UInt32(10), + SuccessRateRequestVolume: wrapperspb.UInt32(50), + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + _, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // The priority configuration generated should have Outlier Detection as a + // direct child due to Outlier Detection being turned on. + wantCfg := &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: &iserviceconfig.BalancerConfig{ + Name: outlierdetection.Name, + Config: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + SuccessRateEjection: &outlierdetection.SuccessRateEjection{ + StdevFactor: 2000, + EnforcementPercentage: 50, + MinimumHosts: 10, + RequestVolume: 50, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: clusterName, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }, + }, + }, + }, + }, + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0"}, + } + + select { + case lbCfg := <-lbCfgCh: + gotCfg := lbCfg.(*priority.LBConfig) + if diff := cmp.Diff(wantCfg, gotCfg); diff != "" { + t.Fatalf("Child policy received unexpected diff in config (-want +got):\n%s", diff) + } + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for child policy to receive its configuration") + } +} diff --git a/internal/xds/balancer/cdsbalancer/e2e_test/dns_impl_test.go b/internal/xds/balancer/cdsbalancer/e2e_test/dns_impl_test.go new file mode 100644 index 0000000..902317e --- /dev/null +++ b/internal/xds/balancer/cdsbalancer/e2e_test/dns_impl_test.go @@ -0,0 +1,136 @@ +/* + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e_test + +import ( + "context" + "fmt" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/internal/xds/balancer/priority" // Register priority LB policy. +) + +// TestLogicalDNS_MultipleEndpoints tests the priority LB policy using a +// LOGICAL_DNS discovery mechanism. +// +// The test verifies that multiple addresses returned by the DNS resolver are +// grouped into a single endpoint (as per gRFC A61). Because the round_robin LB +// policy sees only one endpoint, it should not rotate traffic between the +// addresses. Instead, the single endpoint is picked, and connects to the first +// address. +func (s) TestLogicalDNS_MultipleEndpoints(t *testing.T) { + // Spin up a management server to receive xDS resources from. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start backend servers which provide an implementation of the TestService. + server1 := stubserver.StartTestService(t, nil) + defer server1.Stop() + server2 := stubserver.StartTestService(t, nil) + defer server2.Stop() + + // Register a manual resolver with the "dns" scheme to override DNS resolution. + // This global override is safe because connection to the xDS management + // server uses the passthrough scheme instead and therefore overriding + // the DNS resolver does not affect it in any way. + const dnsScheme = "dns" + dnsR := manual.NewBuilderWithScheme(dnsScheme) + originalDNS := resolver.Get("dns") + resolver.Register(dnsR) + t.Cleanup(func() { resolver.Register(originalDNS) }) + + // For LOGICAL_DNS, this updates the SINGLE endpoint to have 2 IPs. + dnsR.InitialState(resolver.State{ + Endpoints: []resolver.Endpoint{{ + Addresses: []resolver.Address{ + {Addr: server1.Address}, + {Addr: server2.Address}, + }}}, + }) + + const ( + serviceName = "test-xds-service" + clusterName = "cluster-test-xds-service" + endpointsName = "endpoints-test-xds-service" + rdsName = "route-test-xds-service" + ) + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: endpointsName, + Type: e2e.ClusterTypeLogicalDNS, + DNSHostName: "dns", + DNSPort: uint32(8080), + Policy: e2e.LoadBalancingPolicyRoundRobin, + })}, + Endpoints: nil, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server: %v", err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///"+serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create new client for local test server: %v", err) + } + defer cc.Close() + + // Ensure the RPC is routed to the first backend. + testClient := testgrpc.NewTestServiceClient(cc) + for i := 0; i < 10; i++ { + var peer peer.Peer + if _, err := testClient.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + t.Fatalf("RPC failed: %v", err) + } + + if got, want := peer.Addr.String(), server1.Address; got != want { + t.Errorf("peer.Addr = %q, want = %q", got, want) + } + } +} diff --git a/internal/xds/balancer/cdsbalancer/e2e_test/eds_impl_test.go b/internal/xds/balancer/cdsbalancer/e2e_test/eds_impl_test.go new file mode 100644 index 0000000..9997db9 --- /dev/null +++ b/internal/xds/balancer/cdsbalancer/e2e_test/eds_impl_test.go @@ -0,0 +1,1105 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e_test + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + rrutil "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + "google.golang.org/grpc/internal/xds/balancer/priority" +) + +const ( + serviceName = "listener-my-service-client-side-xds" + routeName = "route-my-service-client-side-xds" + clusterName = "cluster-my-service-client-side-xds" + edsServiceName = "endpoints-my-service-client-side-xds" + localityName1 = "my-locality-1" + localityName2 = "my-locality-2" + localityName3 = "my-locality-3" + + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + defaultTestWatchExpiryTimeout = 500 * time.Millisecond + defaultLoadReportingInterval = 10 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// backendAddressesAndPorts extracts the address and port of each of the +// StubServers passed in and returns them. Fails the test if any of the +// StubServers passed have an invalid address. +func backendAddressesAndPorts(t *testing.T, servers []*stubserver.StubServer) ([]resolver.Address, []uint32) { + addrs := make([]resolver.Address, len(servers)) + ports := make([]uint32, len(servers)) + for i := 0; i < len(servers); i++ { + addrs[i] = resolver.Address{Addr: servers[i].Address} + ports[i] = testutils.ParsePort(t, servers[i].Address) + } + return addrs, ports +} + +func startTestServiceBackends(t *testing.T, numBackends int) ([]*stubserver.StubServer, func()) { + var servers []*stubserver.StubServer + for i := 0; i < numBackends; i++ { + servers = append(servers, stubserver.StartTestService(t, nil)) + servers[i].StartServer() + } + + return servers, func() { + for _, server := range servers { + server.Stop() + } + } +} + +// clientResources returns complete resources for the specified nodeID, +// service name and localities. +func clientResources(nodeID, edsServiceName string, localities []e2e.LocalityOptions) e2e.UpdateOptions { + return e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: edsServiceName, + Host: "localhost", + Localities: localities, + })}, + SkipValidation: true, + } +} + +// TestEDS_OneLocality tests the following scenarios: +// 1. Single backend. Test verifies that RPCs reach this backend. +// 2. Add a backend. Test verifies that RPCs are roundrobined across the two +// backends. +// 3. Remove one backend. Test verifies that all RPCs reach the other backend. +// 4. Replace the backend. Test verifies that all RPCs reach the new backend. +func (s) TestEDS_OneLocality(t *testing.T) { + // Spin up a management server to receive xDS resources from. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start backend servers which provide an implementation of the TestService. + servers, cleanup2 := startTestServiceBackends(t, 3) + defer cleanup2() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Create xDS resources for consumption by the test. We start off with a + // single backend in a single EDS locality. + resources := clientResources(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[0]}}}, + }}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + // Ensure RPCs are being roundrobined across the single backend. + testClient := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, testClient, addrs[:1]); err != nil { + t.Fatal(err) + } + + // Add a backend to the same locality, and ensure RPCs are sent in a + // roundrobin fashion across the two backends. + resources = clientResources(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[0]}}, {Ports: []uint32{ports[1]}}}, + }}) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckRoundRobinRPCs(ctx, testClient, addrs[:2]); err != nil { + t.Fatal(err) + } + + // Remove the first backend, and ensure all RPCs are sent to the second + // backend. + resources = clientResources(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[1]}}}, + }}) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckRoundRobinRPCs(ctx, testClient, addrs[1:2]); err != nil { + t.Fatal(err) + } + + // Replace the backend, and ensure all RPCs are sent to the new backend. + resources = clientResources(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[2]}}}, + }}) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckRoundRobinRPCs(ctx, testClient, addrs[2:3]); err != nil { + t.Fatal(err) + } +} + +// TestEDS_MultipleLocalities tests the cluster_resolver LB policy using an EDS +// resource with multiple localities. The following scenarios are tested: +// 1. Two localities, each with a single backend. Test verifies that RPCs are +// weighted roundrobined across these two backends. +// 2. Add another locality, with a single backend. Test verifies that RPCs are +// weighted roundrobined across all the backends. +// 3. Remove one locality. Test verifies that RPCs are weighted roundrobined +// across backends from the remaining localities. +// 4. Add a backend to one locality. Test verifies that RPCs are weighted +// roundrobined across localities. +// 5. Change the weight of one of the localities. Test verifies that RPCs are +// weighted roundrobined across the localities. +// +// In our LB policy tree, one of the descendants is the "weighted_target" LB +// policy which performs weighted roundrobin across localities (and this has a +// randomness component associated with it). Therefore, the moment we have +// backends from more than one locality, RPCs are weighted roundrobined across +// them. +func (s) TestEDS_MultipleLocalities(t *testing.T) { + // Spin up a management server to receive xDS resources from. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start backend servers which provide an implementation of the TestService. + servers, cleanup := startTestServiceBackends(t, 4) + defer cleanup() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Create xDS resources for consumption by the test. We start off with two + // localities, and single backend in each of them. + resources := clientResources(nodeID, edsServiceName, []e2e.LocalityOptions{ + { + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[0]}}}, + }, + { + Name: localityName2, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[1]}}}, + }, + }) + // Use a 20 second timeout since validating WRR requires sending 500+ unary + // RPCs. + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure RPCs are being weighted roundrobined across the two backends. + testClient := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckWeightedRoundRobinRPCs(ctx, t, testClient, addrs[0:2]); err != nil { + t.Fatal(err) + } + + // Add another locality with a single backend, and ensure RPCs are being + // weighted roundrobined across the three backends. + resources = clientResources(nodeID, edsServiceName, []e2e.LocalityOptions{ + { + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[0]}}}, + }, + { + Name: localityName2, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[1]}}}, + }, + { + Name: localityName3, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[2]}}}, + }, + }) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckWeightedRoundRobinRPCs(ctx, t, testClient, addrs[0:3]); err != nil { + t.Fatal(err) + } + + // Remove the first locality, and ensure RPCs are being weighted + // roundrobined across the remaining two backends. + resources = clientResources(nodeID, edsServiceName, []e2e.LocalityOptions{ + { + Name: localityName2, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[1]}}}, + }, + { + Name: localityName3, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[2]}}}, + }, + }) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckWeightedRoundRobinRPCs(ctx, t, testClient, addrs[1:3]); err != nil { + t.Fatal(err) + } + + // Add a backend to one locality, and ensure weighted roundrobin. Since RPCs + // are weighted-roundrobined across localities, locality2's backend will + // receive twice the traffic. + resources = clientResources(nodeID, edsServiceName, []e2e.LocalityOptions{ + { + Name: localityName2, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[1]}}}, + }, + { + Name: localityName3, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[2]}}, {Ports: []uint32{ports[3]}}}, + }, + }) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + wantAddrs := []resolver.Address{addrs[1], addrs[1], addrs[2], addrs[3]} + if err := rrutil.CheckWeightedRoundRobinRPCs(ctx, t, testClient, wantAddrs); err != nil { + t.Fatal(err) + } +} + +// TestEDS_EndpointsHealth tests the scenario where an EDS resource specifying +// endpoint health information is received, and verifies that traffic is routed +// only to backends deemed capable of receiving traffic. +func (s) TestEDS_EndpointsHealth(t *testing.T) { + // Spin up a management server to receive xDS resources from. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start backend servers which provide an implementation of the TestService. + servers, cleanup2 := startTestServiceBackends(t, 12) + defer cleanup2() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Create xDS resources for consumption by the test. Two localities with + // six backends each, with two of the six backends being healthy. Both + // UNKNOWN and HEALTHY are considered by gRPC for load balancing. + resources := clientResources(nodeID, edsServiceName, []e2e.LocalityOptions{ + { + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{ + {Ports: []uint32{ports[0]}, HealthStatus: v3corepb.HealthStatus_UNKNOWN}, + {Ports: []uint32{ports[1]}, HealthStatus: v3corepb.HealthStatus_HEALTHY}, + {Ports: []uint32{ports[2]}, HealthStatus: v3corepb.HealthStatus_UNHEALTHY}, + {Ports: []uint32{ports[3]}, HealthStatus: v3corepb.HealthStatus_DRAINING}, + {Ports: []uint32{ports[4]}, HealthStatus: v3corepb.HealthStatus_TIMEOUT}, + {Ports: []uint32{ports[5]}, HealthStatus: v3corepb.HealthStatus_DEGRADED}, + }, + }, + { + Name: localityName2, + Weight: 1, + Backends: []e2e.BackendOptions{ + {Ports: []uint32{ports[6]}, HealthStatus: v3corepb.HealthStatus_UNKNOWN}, + {Ports: []uint32{ports[7]}, HealthStatus: v3corepb.HealthStatus_HEALTHY}, + {Ports: []uint32{ports[8]}, HealthStatus: v3corepb.HealthStatus_UNHEALTHY}, + {Ports: []uint32{ports[9]}, HealthStatus: v3corepb.HealthStatus_DRAINING}, + {Ports: []uint32{ports[10]}, HealthStatus: v3corepb.HealthStatus_TIMEOUT}, + {Ports: []uint32{ports[11]}, HealthStatus: v3corepb.HealthStatus_DEGRADED}, + }, + }, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure RPCs are being weighted roundrobined across healthy backends from + // both localities. + testClient := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckWeightedRoundRobinRPCs(ctx, t, testClient, append(addrs[0:2], addrs[6:8]...)); err != nil { + t.Fatal(err) + } +} + +// TestEDS_EmptyUpdate tests when an EDS resource with no localities is received +// and verifies that RPCs fail with "all priorities removed" error. +func (s) TestEDS_EmptyUpdate(t *testing.T) { + // Spin up a management server to receive xDS resources from. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start backend servers which provide an implementation of the TestService. + servers, cleanup2 := startTestServiceBackends(t, 4) + defer cleanup2() + addrs, ports := backendAddressesAndPorts(t, servers) + + oldCacheTimeout := priority.DefaultSubBalancerCloseTimeout + priority.DefaultSubBalancerCloseTimeout = 100 * time.Microsecond + defer func() { priority.DefaultSubBalancerCloseTimeout = oldCacheTimeout }() + + // Create xDS resources for consumption by the test. The first update is an + // empty update. This should put the channel in TRANSIENT_FAILURE. + resources := clientResources(nodeID, edsServiceName, nil) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + testClient := testgrpc.NewTestServiceClient(cc) + if err := waitForProducedZeroAddressesError(ctx, t, testClient); err != nil { + t.Fatal(err) + } + + // Add a locality with one backend and ensure RPCs are successful. + resources = clientResources(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[0]}}}, + }}) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckRoundRobinRPCs(ctx, testClient, addrs[:1]); err != nil { + t.Fatal(err) + } + + // Push another empty update and ensure that RPCs fail with the "all priorities + // removed" error again. + resources = clientResources(nodeID, edsServiceName, nil) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := waitForProducedZeroAddressesError(ctx, t, testClient); err != nil { + t.Fatal(err) + } +} + +// TestEDS_ResourceRemoved tests the case where the EDS resource requested is +// removed from the management server. The test verifies that the EDS watch is +// not canceled and that RPCs continue to succeed with the previously received +// configuration. +func (s) TestEDS_ResourceRemoved(t *testing.T) { + // Start an xDS management server that uses a couple of channels to + // notify the test about the following events: + // - an EDS requested with the expected resource name is requested + // - EDS resource is unrequested, i.e, an EDS request with no resource name + // is received, which indicates that we are no longer interested in that + // resource. + edsResourceRequestedCh := make(chan struct{}, 1) + edsResourceCanceledCh := make(chan struct{}, 1) + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3EndpointsURL { + switch len(req.GetResourceNames()) { + case 0: + select { + case edsResourceCanceledCh <- struct{}{}: + default: + } + case 1: + if req.GetResourceNames()[0] == edsServiceName { + select { + case edsResourceRequestedCh <- struct{}{}: + default: + } + } + default: + t.Errorf("Unexpected number of resources, %d, in an EDS request", len(req.GetResourceNames())) + } + } + return nil + }, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure cluster and endpoints resources in the management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Delete the endpoints resource from the management server. + resources.Endpoints = nil + resources.SkipValidation = true + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that RPCs continue to succeed for the next second, and that the + // EDS watch is not canceled. + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + select { + case <-edsResourceCanceledCh: + t.Fatal("EDS watch canceled when not expected to be canceled") + default: + } + } +} + +// TestEDS_ClusterResourceDoesNotContainEDSServiceName tests the case where the +// Cluster resource sent by the management server does not contain an EDS +// service name. The test verifies that the cluster name is used for the EDS +// resource. +func (s) TestEDS_ClusterResourceDoesNotContainEDSServiceName(t *testing.T) { + edsResourceCh := make(chan string, 1) + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() != version.V3EndpointsURL { + return nil + } + if len(req.GetResourceNames()) > 0 { + select { + case edsResourceCh <- req.GetResourceNames()[0]: + default: + } + } + return nil + }, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure cluster and endpoints resources with the same name in the management server. The cluster resource does not specify an EDS service name. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, "", e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for EDS request to be received on the management server") + case name := <-edsResourceCh: + if name != clusterName { + t.Fatalf("Received EDS request with resource name %q, want %q", name, clusterName) + } + } +} + +// TestEDS_ClusterResourceUpdates verifies different scenarios with regards to +// cluster resource updates. +// +// - The first cluster resource contains an eds_service_name. The test verifies +// that an EDS request is sent for the received eds_service_name. It also +// verifies that a subsequent RPC gets routed to a backend belonging to that +// service name. +// - The next cluster resource update contains no eds_service_name. The test +// verifies that a subsequent EDS request is sent for the cluster_name and +// that the previously received eds_service_name is no longer requested. It +// also verifies that a subsequent RPC gets routed to a backend belonging to +// the service represented by the cluster_name. +// - The next cluster resource update changes the circuit breaking +// configuration, but does not change the service name. The test verifies +// that a subsequent RPC gets routed to the same backend as before. +func (s) TestEDS_ClusterResourceUpdates(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server that fires off events when EDS resources are + // requested. + edsServiceNameRequested := grpcsync.NewEvent() + clusterNameRequested := grpcsync.NewEvent() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() != version.V3EndpointsURL { + return nil + } + if len(req.GetResourceNames()) == 1 { + if req.GetResourceNames()[0] == edsServiceName { + edsServiceNameRequested.Fire() + } else if req.GetResourceNames()[0] == clusterName { + clusterNameRequested.Fire() + } + } + return nil + }, + AllowResourceSubset: true, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start two test backends and extract their host and port. The first + // backend is used for the EDS resource identified by the eds_service_name, + // and the second backend is used for the EDS resource identified by the + // cluster_name. + servers, cleanup2 := startTestServiceBackends(t, 2) + defer cleanup2() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Configure cluster and endpoints resources in the management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{uint32(ports[0])}), + e2e.DefaultEndpoint(clusterName, "localhost", []uint32{uint32(ports[1])}), + }, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } + + // Ensure EDS watch is registered for eds_service_name. + select { + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for EDS request for resource %q", edsServiceName) + case <-edsServiceNameRequested.Done(): + } + + // Change the cluster resource to not contain an eds_service_name. + resources.Clusters = []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, "", e2e.SecurityLevelNone)} + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that an EDS watch for eds_service_name is canceled and new watch + // for cluster_name is registered. The actual order in which this happens is + // not deterministic, i.e the watch for old resource could be canceled + // before the new one is registered or vice-versa. In either case, + // eventually, we want to see a request to the management server for just + // the cluster_name. + select { + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for EDS request for resource %q", clusterName) + case <-clusterNameRequested.Done(): + } + + // Make an RPC, and ensure that it gets routed to second backend, + // corresponding to the cluster_name. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + continue + } + if peer.Addr.String() == addrs[1].Addr { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for EmptyCall() to be routed to correct backend %q", addrs[1].Addr) + } + + // Change cluster resource circuit breaking count. + resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{ + Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{ + { + Priority: v3corepb.RoutingPriority_DEFAULT, + MaxRequests: wrapperspb.UInt32(512), + }, + }, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that RPCs continue to get routed to the second backend for the + // next second. + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[1].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[1].Addr) + } + } +} + +// TestEDS_BadUpdateWithoutPreviousGoodUpdate tests the case where the +// management server sends a bad update (one that is NACKed by the xDS client). +// Since the xDS client does not have a previously received good update, it is +// expected to treat this bad update as though it received an update with no +// endpoints. Hence RPCs are expected to fail with "all priorities removed" +// error. +func (s) TestEDS_BadUpdateWithoutPreviousGoodUpdate(t *testing.T) { + // Spin up a management server to receive xDS resources from. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start a backend server that implements the TestService. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Create an EDS resource with a load balancing weight of 0. This will + // result in the resource being NACKed by the xDS client. Since the + // cluster_resolver LB policy does not have a previously received good EDS + // update, it should treat this update as an empty EDS update. + resources := clientResources(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, server.Address)}}}, + }}) + resources.Endpoints[0].Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, cancel := setupAndDial(t, bootstrapContents) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if err := waitForProducedZeroAddressesError(ctx, t, client); err != nil { + t.Fatal(err) + } +} + +// TestEDS_BadUpdateWithPreviousGoodUpdate tests the case where the +// cluster_resolver LB policy receives a good EDS update from the management +// server and the test verifies that RPCs are successful. Then, a bad update is +// received from the management server (one that is NACKed by the xDS client). +// The test verifies that the previously received good update is still being +// used and that RPCs are still successful. +func (s) TestEDS_BadUpdateWithPreviousGoodUpdate(t *testing.T) { + // Spin up a management server to receive xDS resources from. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Start a backend server that implements the TestService. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Create an EDS resource for consumption by the test. + resources := clientResources(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, server.Address)}}}, + }}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, xdsResolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure RPCs are being roundrobined across the single backend. + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, []resolver.Address{{Addr: server.Address}}); err != nil { + t.Fatal(err) + } + + // Update the endpoints resource in the management server with a load + // balancing weight of 0. This will result in the resource being NACKed by + // the xDS client. But since the cluster_resolver LB policy has a previously + // received good EDS update, it should continue using it. + resources.Endpoints[0].Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that RPCs continue to succeed for the next second. + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + if err := rrutil.CheckRoundRobinRPCs(ctx, client, []resolver.Address{{Addr: server.Address}}); err != nil { + t.Fatal(err) + } + } +} + +// TestEDS_ResourceNotFound tests the case where the requested EDS resource does +// not exist on the management server. Once the watch timer associated with the +// requested resource expires, the cluster_resolver LB policy receives a +// "resource-not-found" callback from the xDS client and is expected to treat it +// as though it received an update with no endpoints. Hence RPCs are expected to +// fail with "all priorities removed" error. +func (s) TestEDS_ResourceNotFound(t *testing.T) { + // Spin up a management server to receive xDS resources from. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create an xDS client talking to the above management server, configured + // with a short watch expiry timeout. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + xdsClient, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Configure no eds resources on the management server. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: serviceName, + Host: "localhost", + Port: 8080, + }) + resources.Endpoints = nil + resources.SkipValidation = true + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + if internal.NewXDSResolverWithClientForTesting == nil { + t.Fatalf("internal.NewXDSResolverWithClientForTesting is nil") + } + r, err := internal.NewXDSResolverWithClientForTesting.(func(xdsclient.XDSClient) (resolver.Builder, error))(xdsClient) + if err != nil { + t.Fatalf("failed to create resolver") + } + cc, err := grpc.NewClient("xds:///"+serviceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + if err := waitForProducedZeroAddressesError(ctx, t, client); err != nil { + t.Fatal(err) + } +} + +// waitForAllPrioritiesRemovedError repeatedly makes RPCs using the +// TestServiceClient until they fail with an error which indicates that no +// resolver addresses have been produced. A non-nil error is returned if the +// context expires before RPCs fail with the expected error. +func waitForProducedZeroAddressesError(ctx context.Context, t *testing.T, client testgrpc.TestServiceClient) error { + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Log("EmptyCall() succeeded after error in Discovery Mechanism") + continue + } + if code := status.Code(err); code != codes.Unavailable { + t.Logf("EmptyCall() returned code: %v, want: %v", code, codes.Unavailable) + continue + } + if !strings.Contains(err.Error(), "no targets to pick from") { + t.Logf("EmptyCall() = %v, want %v", err, "no targets to pick from") + continue + } + return nil + } + return errors.New("timeout when waiting for RPCs to fail with UNAVAILABLE status and produced zero addresses") +} + +// Test runs a server which listens on multiple ports. The test updates xds resouce +// cache to contain a single endpoint with multiple addresses. The test intercepts +// the resolver updates sent to the petiole policy and verifies that the +// additional endpoint addresses are correctly propagated. +func (s) TestEDS_EndpointWithMultipleAddresses(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start a backend server which listens to multiple ports to simulate a + // backend with multiple addresses. + server := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + } + lis1, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + defer lis1.Close() + lis2, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + defer lis2.Close() + lis3, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + defer lis3.Close() + + server.Listener = lis1 + if err := server.StartServer(); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + go server.S.Serve(lis2) + go server.S.Serve(lis3) + + t.Logf("Started test service backend at addresses %q, %q, %q", lis1.Addr(), lis2.Addr(), lis3.Addr()) + + ports := []uint32{ + testutils.ParsePort(t, lis1.Addr().String()), + testutils.ParsePort(t, lis2.Addr().String()), + testutils.ParsePort(t, lis3.Addr().String()), + } + + testCases := []struct { + name string + dualstackEndpointsEnabled bool + wantEndpointPorts []uint32 + }{ + { + name: "flag_enabled", + dualstackEndpointsEnabled: true, + wantEndpointPorts: ports, + }, + { + name: "flag_disabled", + wantEndpointPorts: ports[:1], + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + origDualstackEndpointsEnabled := envconfig.XDSDualstackEndpointsEnabled + defer func() { + envconfig.XDSDualstackEndpointsEnabled = origDualstackEndpointsEnabled + }() + envconfig.XDSDualstackEndpointsEnabled = tc.dualstackEndpointsEnabled + + // Wrap the round robin balancer to intercept resolver updates. + originalRRBuilder := balancer.Get(roundrobin.Name) + defer func() { + balancer.Register(originalRRBuilder) + }() + resolverState := atomic.Pointer[resolver.State]{} + resolverState.Store(&resolver.State{}) + stub.Register(roundrobin.Name, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = originalRRBuilder.Build(bd.ClientConn, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + resolverState.Store(&ccs.ResolverState) + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + }) + + // Spin up a management server to receive xDS resources from. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + + // Create xDS resources for consumption by the test. We start off with a + // single backend in a single EDS locality. + resources := clientResources(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{ + Ports: ports, + }}, + }}) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client talking to the above management server, configured + // with a short watch expiry timeout. + xdsClient, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + if internal.NewXDSResolverWithClientForTesting == nil { + t.Fatalf("internal.NewXDSResolverWithClientForTesting is nil") + } + r, err := internal.NewXDSResolverWithClientForTesting.(func(xdsclient.XDSClient) (resolver.Builder, error))(xdsClient) + if err != nil { + t.Fatalf("failed to create resolver") + } + cc, err := grpc.NewClient("xds:///"+serviceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, []resolver.Address{{Addr: lis1.Addr().String()}}); err != nil { + t.Fatal(err) + } + + gotState := resolverState.Load() + + gotEndpointPorts := []uint32{} + for _, a := range gotState.Endpoints[0].Addresses { + gotEndpointPorts = append(gotEndpointPorts, testutils.ParsePort(t, a.Addr)) + } + if diff := cmp.Diff(gotEndpointPorts, tc.wantEndpointPorts); diff != "" { + t.Errorf("Unexpected endpoint address ports in resolver update, diff (-got +want): %v", diff) + } + }) + } +} diff --git a/internal/xds/balancer/cdsbalancer/logging.go b/internal/xds/balancer/cdsbalancer/logging.go new file mode 100644 index 0000000..e179bb1 --- /dev/null +++ b/internal/xds/balancer/cdsbalancer/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package cdsbalancer + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[cds-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *cdsBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/internal/xds/balancer/clusterimpl/balancer_test.go b/internal/xds/balancer/clusterimpl/balancer_test.go new file mode 100644 index 0000000..8e118f6 --- /dev/null +++ b/internal/xds/balancer/clusterimpl/balancer_test.go @@ -0,0 +1,356 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl + +import ( + "context" + "errors" + "testing" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + xdsinternal "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/internal/xds/testutils/fakeclient" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/resolver" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultShortTestTimeout = 100 * time.Microsecond + + testClusterName = "test-cluster" + testServiceName = "test-eds-service" +) + +var ( + testBackendEndpoints = []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}} +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func init() { + NewRandomWRR = testutils.NewTestWRR +} + +// TestPickerUpdateAfterClose covers the case where a child policy sends a +// picker update after the cluster_impl policy is closed. Because picker updates +// are handled in the run() goroutine, which exits before Close() returns, we +// expect the above picker update to be dropped. +func (s) TestPickerUpdateAfterClose(t *testing.T) { + builder := balancer.Get(Name) + cc := testutils.NewBalancerClientConn(t) + b := builder.Build(cc, balancer.BuildOptions{}) + + // Create a stub balancer which waits for the cluster_impl policy to be + // closed before sending a picker update (upon receipt of a subConn state + // change). + closeCh := make(chan struct{}) + const childPolicyName = "stubBalancer-TestPickerUpdateAfterClose" + stub.Register(childPolicyName, stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + // Create a subConn which will be used later on to test the race + // between StateListener() and Close(). + sc, err := bd.ClientConn.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{ + StateListener: func(balancer.SubConnState) { + go func() { + // Wait for Close() to be called on the parent policy before + // sending the picker update. + <-closeCh + bd.ClientConn.UpdateState(balancer.State{ + Picker: base.NewErrPicker(errors.New("dummy error picker")), + }) + }() + }, + }) + if err != nil { + return err + } + sc.Connect() + return nil + }, + }) + + var maxRequest uint32 = 50 + xdsC := fakeclient.NewClient() + state := xdsclient.SetClient(resolver.State{Endpoints: testBackendEndpoints}, xdsC) + state = xdsresource.SetXDSConfig(state, &xdsresource.XDSConfig{ + Clusters: map[string]*xdsresource.ClusterResult{ + testClusterName: { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterName: testClusterName, + ClusterType: xdsresource.ClusterTypeEDS, + EDSServiceName: testServiceName, + MaxRequests: &maxRequest, + }, + EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}}, + }, + }, + }, + }) + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: state, + BalancerConfig: &LBConfig{ + Cluster: testClusterName, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: childPolicyName, + }, + }, + }); err != nil { + b.Close() + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + + // Send a subConn state change to trigger a picker update. The stub balancer + // that we use as the child policy will not send a picker update until the + // parent policy is closed. + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + b.Close() + close(closeCh) + + select { + case <-cc.NewPickerCh: + t.Fatalf("unexpected picker update after balancer is closed") + case <-time.After(defaultShortTestTimeout): + } +} + +// TestClusterNameInAddressAttributes covers the case that cluster name is +// attached to the subconn address attributes. +func (s) TestClusterNameInAddressAttributes(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + builder := balancer.Get(Name) + cc := testutils.NewBalancerClientConn(t) + b := builder.Build(cc, balancer.BuildOptions{}) + defer b.Close() + + xdsC := fakeclient.NewClient() + state := xdsclient.SetClient(resolver.State{Endpoints: testBackendEndpoints}, xdsC) + state = xdsresource.SetXDSConfig(state, &xdsresource.XDSConfig{ + Clusters: map[string]*xdsresource.ClusterResult{ + testClusterName: { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: testClusterName, + EDSServiceName: testServiceName, + }, + EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}}, + }, + }, + }, + }) + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: state, + BalancerConfig: &LBConfig{ + Cluster: testClusterName, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }); err != nil { + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + // This should get the connecting picker. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendEndpoints[0].Addresses[0].Addr; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + cn, ok := xdsinternal.GetXDSHandshakeClusterName(addrs1[0].Attributes) + if !ok || cn != testClusterName { + t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn, ok, testClusterName) + } + + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + // Test pick with one backend. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + const testClusterName2 = "test-cluster-2" + var addr2 = resolver.Address{Addr: "2.2.2.2"} + state2 := xdsclient.SetClient(resolver.State{Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{addr2}}}}, xdsC) + state2 = xdsresource.SetXDSConfig(state2, &xdsresource.XDSConfig{ + Clusters: map[string]*xdsresource.ClusterResult{ + testClusterName2: { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: testClusterName2, + EDSServiceName: testServiceName, + }, + EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}}, + }, + }, + }, + }) + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: state2, + BalancerConfig: &LBConfig{ + Cluster: testClusterName2, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }); err != nil { + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, addr2.Addr; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + // New addresses should have the new cluster name. + cn2, ok := xdsinternal.GetXDSHandshakeClusterName(addrs2[0].Attributes) + if !ok || cn2 != testClusterName2 { + t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn2, ok, testClusterName2) + } +} + +// Test verify that the case picker is updated synchronously on receipt of +// configuration update. +func (s) TestPickerUpdatedSynchronouslyOnConfigUpdate(t *testing.T) { + // Override the pickerUpdateHook to be notified that picker was updated. + pickerUpdated := make(chan struct{}, 1) + origNewPickerUpdated := pickerUpdateHook + pickerUpdateHook = func() { + pickerUpdated <- struct{}{} + } + defer func() { pickerUpdateHook = origNewPickerUpdated }() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Override the clientConnUpdateHook to ensure client conn was updated. + clientConnUpdateDone := make(chan struct{}, 1) + origClientConnUpdateHook := clientConnUpdateHook + clientConnUpdateHook = func() { + // Verify that picker was updated before the completion of + // client conn update. + select { + case <-pickerUpdated: + case <-ctx.Done(): + t.Fatal("Client conn update completed before picker update.") + } + clientConnUpdateDone <- struct{}{} + } + defer func() { clientConnUpdateHook = origClientConnUpdateHook }() + + builder := balancer.Get(Name) + cc := testutils.NewBalancerClientConn(t) + b := builder.Build(cc, balancer.BuildOptions{}) + defer b.Close() + + // Create a stub balancer which waits for the cluster_impl policy to be + // closed before sending a picker update (upon receipt of a resolver + // update). + stub.Register(t.Name(), stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + bd.ClientConn.UpdateState(balancer.State{ + Picker: base.NewErrPicker(errors.New("dummy error picker")), + }) + return nil + }, + }) + + xdsC := fakeclient.NewClient() + state := xdsclient.SetClient(resolver.State{Endpoints: testBackendEndpoints}, xdsC) + state = xdsresource.SetXDSConfig(state, &xdsresource.XDSConfig{ + Clusters: map[string]*xdsresource.ClusterResult{ + testClusterName: { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterName: testClusterName, + ClusterType: xdsresource.ClusterTypeEDS, + EDSServiceName: testServiceName, + }, + EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}}, + }, + }, + }, + }) + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: state, + BalancerConfig: &LBConfig{ + Cluster: testClusterName, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: t.Name(), + }, + }, + }); err != nil { + t.Fatalf("Unexpected error from UpdateClientConnState: %v", err) + } + + select { + case <-clientConnUpdateDone: + case <-ctx.Done(): + t.Fatal("Timed out waiting for client conn update to be completed.") + } +} + +func (s) TestDropRequestsPerMillion(t *testing.T) { + tests := []struct { + name string + numerator uint32 + denominator uint32 + want uint32 + }{ + {name: "zero", numerator: 0, denominator: million, want: 0}, + {name: "five percent hundred", numerator: 5, denominator: 100, want: 50000}, + // numerator*million overflows a uint32 once numerator reaches 4295 with + // the million denominator, which is only a 0.43% drop. + {name: "point four three percent million", numerator: 4295, denominator: million, want: 4295}, + {name: "one percent million", numerator: 10000, denominator: million, want: 10000}, + {name: "fifty percent million", numerator: 500000, denominator: million, want: 500000}, + {name: "hundred percent million", numerator: million, denominator: million, want: million}, + // A fraction above 100% is capped at a million. + {name: "over hundred percent", numerator: 150, denominator: 100, want: million}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := dropRequestsPerMillion(tt.numerator, tt.denominator); got != tt.want { + t.Errorf("dropRequestsPerMillion(%d, %d) = %d, want %d", tt.numerator, tt.denominator, got, tt.want) + } + }) + } +} diff --git a/internal/xds/balancer/clusterimpl/clusterimpl.go b/internal/xds/balancer/clusterimpl/clusterimpl.go new file mode 100644 index 0000000..36b84c0 --- /dev/null +++ b/internal/xds/balancer/clusterimpl/clusterimpl.go @@ -0,0 +1,644 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package clusterimpl implements the xds_cluster_impl balancing policy. It +// handles the cluster features (e.g. circuit_breaking, RPC dropping). +// +// Note that it doesn't handle name resolution, which is done by policy +// xds_cluster_resolver. +package clusterimpl + +import ( + "context" + "encoding/json" + "fmt" + "net" + "slices" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal/balancer/gracefulswitch" + "google.golang.org/grpc/internal/credentials/xds" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/pretty" + xdsinternal "google.golang.org/grpc/internal/xds" + + "google.golang.org/grpc/internal/xds/balancer/clusterimpl/internal" + "google.golang.org/grpc/internal/xds/balancer/loadstore" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/lrsclient" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const ( + // Name is the name of the cluster_impl balancer. + Name = "xds_cluster_impl_experimental" + defaultRequestCountMax = 1024 + loadStoreStopTimeout = 1 * time.Second +) + +var ( + // Below function is no-op in actual code, but can be overridden in + // tests to give tests visibility into exactly when certain events happen. + clientConnUpdateHook = func() {} + pickerUpdateHook = func() {} + buildProvider = buildProviderFunc +) + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &clusterImplBalancer{ + ClientConn: cc, + loadWrapper: loadstore.NewWrapper(), + requestCountMax: defaultRequestCountMax, + } + b.xdsHIPtr.Store(xds.NewHandshakeInfo(nil, nil, nil, "", false, false, false)) + b.logger = prefixLogger(b) + b.child = gracefulswitch.NewBalancer(b, bOpts) + b.logger.Infof("Created") + + var creds credentials.TransportCredentials + switch { + case bOpts.DialCreds != nil: + creds = bOpts.DialCreds + case bOpts.CredsBundle != nil: + creds = bOpts.CredsBundle.TransportCredentials() + } + if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() { + b.xdsCredsInUse = true + } + b.logger.Infof("xDS credentials in use: %v", b.xdsCredsInUse) + return b +} + +func (bb) Name() string { + return Name +} + +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(c) +} + +type clusterImplBalancer struct { + balancer.ClientConn + + // The following fields are set at creation time, and are read-only after + // that, and therefore need not be protected by a mutex. + logger *grpclog.PrefixLogger + // TODO: #8366 - Refactor usage of loadWrapper to easily plugin a test + // load reporter from tests. + loadWrapper *loadstore.Wrapper + + // The following fields are only accessed from balancer API methods, which + // are guaranteed to be called serially by gRPC. + xdsClient xdsclient.XDSClient // Sent down in ResolverState attributes. + cancelLoadReport func(context.Context) // To stop reporting load through the above xDS client. + edsServiceName string // EDS service name to report load for. + lrsServer *bootstrap.ServerConfig // Load reporting server configuration. + dropCategories []DropConfig // The categories for drops. + child *gracefulswitch.Balancer + xdsHIPtr atomic.Pointer[grpcsync.RefCounted[*xds.HandshakeInfo]] // Accessed atomically as it is shared between the balancer and the transport. + xdsCredsInUse bool + + // The active security configuration received from the control plane. + securityConfig *xdsresource.SecurityConfig + + // The following fields are protected by mu, since they are accessed in + // balancer API methods and in methods called from the child policy. + mu sync.Mutex + clusterName string // The cluster name for credentials handshaking. + inhibitPickerUpdates bool // Inhibits state updates from child policy when processing an update from the parent. + pendingPickerUpdates bool // True if a picker update from the child policy was inhibited when processing an update from the parent. + childState balancer.State // Most recent state update from the child policy. + drops []*dropper // Drops implementation. + requestCounterCluster string // The cluster name for the request counter, from LB config. + requestCounterService string // The service name for the request counter, from LB config. + requestCountMax uint32 // Max concurrent requests, from LB config. + requestCounter *xdsclient.ClusterRequestsCounter // Tracks total inflight requests for a given service. + telemetryLabels map[string]string // Telemetry labels to set on picks, from LB config. + lrsReportEndpointMetrics *xdsresource.LRSReportEndpointMetricsConfig // LRS metrics to propagate. +} + +// dropRequestsPerMillion scales a drop overload's numerator and denominator to +// a number of requests to drop per million. numerator can be as large as a +// million (a 100% drop), so the multiplication is done in uint64 to avoid +// overflowing a uint32, and the result is capped at a million because a drop +// ratio above 100% is treated as 100%. +func dropRequestsPerMillion(numerator, denominator uint32) uint32 { + rpm := uint64(numerator) * million / uint64(denominator) + if rpm > million { + rpm = million + } + return uint32(rpm) +} + +// handleClusterConfigLocked updates the internal state of the balancer with the +// new cluster configuration. It returns true if a new picker needs to be +// generated as a result of these changes. It must be called with b.mu held. +func (b *clusterImplBalancer) handleClusterConfigLocked(clusterConfig xdsresource.ClusterConfig) bool { + clusterUpdate := clusterConfig.Cluster + var updatePicker bool + + b.telemetryLabels = clusterUpdate.TelemetryLabels + + if !b.lrsReportEndpointMetrics.Equal(clusterUpdate.LRSReportEndpointMetrics) { + b.lrsReportEndpointMetrics = clusterUpdate.LRSReportEndpointMetrics + updatePicker = true + } + + var newDrops []DropConfig + if clusterUpdate.ClusterType == xdsresource.ClusterTypeEDS { + edsUpdate := clusterConfig.EndpointConfig.EDSUpdate + newDrops = make([]DropConfig, 0, len(edsUpdate.Drops)) + for _, d := range edsUpdate.Drops { + newDrops = append(newDrops, DropConfig{ + Category: d.Category, + RequestsPerMillion: dropRequestsPerMillion(d.Numerator, d.Denominator), + }) + } + if !slices.Equal(b.dropCategories, newDrops) { + b.dropCategories = newDrops + b.drops = make([]*dropper, 0, len(newDrops)) + for _, c := range newDrops { + b.drops = append(b.drops, newDropper(c)) + } + updatePicker = true + } + } + + if b.requestCounterCluster != clusterUpdate.ClusterName || b.requestCounterService != clusterUpdate.EDSServiceName { + b.requestCounterCluster = clusterUpdate.ClusterName + b.requestCounterService = clusterUpdate.EDSServiceName + b.requestCounter = xdsclient.GetClusterRequestsCounter(clusterUpdate.ClusterName, clusterUpdate.EDSServiceName) + updatePicker = true + } + var newRequestCountMax uint32 = 1024 + if clusterUpdate.MaxRequests != nil { + newRequestCountMax = *clusterUpdate.MaxRequests + } + if b.requestCountMax != newRequestCountMax { + b.requestCountMax = newRequestCountMax + updatePicker = true + } + + return updatePicker +} + +func (b *clusterImplBalancer) newPickerLocked() *picker { + return &picker{ + drops: b.drops, + s: b.childState, + loadStore: b.loadWrapper, + counter: b.requestCounter, + countMax: b.requestCountMax, + telemetryLabels: b.telemetryLabels, + clusterName: b.clusterName, + metrics: b.lrsReportEndpointMetrics, + } +} + +// updateLoadStore checks the config for load store, and decides whether it +// needs to restart the load reporting stream. +func (b *clusterImplBalancer) updateLoadStore(clusterUpdate *xdsresource.ClusterUpdate) error { + var updateLoadClusterAndService bool + + // ClusterName is different, restart. ClusterName is from ClusterName and + // EDSServiceName. + clusterName := b.getClusterName() + if clusterName != clusterUpdate.ClusterName { + updateLoadClusterAndService = true + b.setClusterName(clusterUpdate.ClusterName) + clusterName = clusterUpdate.ClusterName + } + if b.edsServiceName != clusterUpdate.EDSServiceName { + updateLoadClusterAndService = true + b.edsServiceName = clusterUpdate.EDSServiceName + } + if updateLoadClusterAndService { + // This updates the clusterName and serviceName that will be reported + // for the loads. The update here is too early, the perfect timing is + // when the picker is updated with the new connection. But from this + // balancer's point of view, it's impossible to tell. + // + // On the other hand, this will almost never happen. Each LRS policy + // shouldn't get updated config. The parent should do a graceful switch + // when the clusterName or serviceName is changed. + b.loadWrapper.UpdateClusterAndService(clusterName, b.edsServiceName) + } + + var ( + stopOldLoadReport bool + startNewLoadReport bool + ) + + // Check if it's necessary to restart load report. + if b.lrsServer == nil { + if clusterUpdate.LRSServerConfig != nil { + // Old is nil, new is not nil, start new LRS. + b.lrsServer = clusterUpdate.LRSServerConfig + startNewLoadReport = true + } + // Old is nil, new is nil, do nothing. + } else if clusterUpdate.LRSServerConfig == nil { + // Old is not nil, new is nil, stop old, don't start new. + b.lrsServer = nil + stopOldLoadReport = true + } else { + // Old is not nil, new is not nil, compare string values, if + // different, stop old and start new. + if !b.lrsServer.Equal(clusterUpdate.LRSServerConfig) { + b.lrsServer = clusterUpdate.LRSServerConfig + stopOldLoadReport = true + startNewLoadReport = true + } + } + + if stopOldLoadReport { + if b.cancelLoadReport != nil { + stopCtx, stopCancel := context.WithTimeout(context.Background(), loadStoreStopTimeout) + defer stopCancel() + b.cancelLoadReport(stopCtx) + b.cancelLoadReport = nil + if !startNewLoadReport { + // If a new LRS stream will be started later, no need to update + // it to nil here. + b.loadWrapper.UpdateLoadStore(nil) + } + } + } + if startNewLoadReport { + var loadStore *lrsclient.LoadStore + if b.xdsClient != nil { + loadStore, b.cancelLoadReport = b.xdsClient.ReportLoad(b.lrsServer) + } + b.loadWrapper.UpdateLoadStore(loadStore) + } + + return nil +} + +func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) { + cfg := configs[instanceName] + provider, err := cfg.Build(certprovider.BuildOptions{ + CertName: certName, + WantIdentity: wantIdentity, + WantRoot: wantRoot, + }) + if err != nil { + // This error is not expected since the bootstrap process parses the + // config and makes sure that it is acceptable to the plugin. Still, it + // is possible that the plugin parses the config successfully, but its + // Build() method errors out. + return nil, fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", cfg, err) + } + return provider, nil +} + +func (b *clusterImplBalancer) buildProviders(config *xdsresource.SecurityConfig) (certprovider.Provider, certprovider.Provider, error) { + cpc := b.xdsClient.BootstrapConfig().CertProviderConfigs() + var rootProvider certprovider.Provider + if config.UseSystemRootCerts { + rootProvider = systemRootCertsProvider{} + } else { + rp, err := buildProvider(cpc, config.RootInstanceName, config.RootCertName, false, true) + if err != nil { + return nil, nil, err + } + rootProvider = rp + } + + var identityProvider certprovider.Provider + if name, cert := config.IdentityInstanceName, config.IdentityCertName; name != "" { + ip, err := buildProvider(cpc, name, cert, true, false) + if err != nil { + if rootProvider != nil { + rootProvider.Close() + } + return nil, nil, err + } + identityProvider = ip + } + return rootProvider, identityProvider, nil +} + +// handleSecurityConfig processes the security configuration received from the +// management server, creates appropriate certificate provider plugins, and +// updates the HandshakeInfo which is added as an address attribute in +// NewSubConn() calls. +func (b *clusterImplBalancer) handleSecurityConfig(config *xdsresource.SecurityConfig) error { + // If xdsCredentials are not in use, i.e, the user did not want to get + // security configuration from an xDS server, we should not be acting on the + // received security config here. Doing so poses a security threat. + if !b.xdsCredsInUse || config.Equal(b.securityConfig) { + return nil + } + + // Security config being nil is a valid case where the management server has + // not sent any security configuration. The xdsCredentials implementation + // handles this by delegating to its fallback credentials. + if config == nil { + b.securityConfig = nil + oldHI := b.xdsHIPtr.Swap(xds.NewHandshakeInfo(nil, nil, nil, "", false, false, false)) + if oldHI != nil { + oldHI.Decrement() + } + return nil + } + + rootProvider, identityProvider, err := b.buildProviders(config) + if err != nil { + return err + } + + newHI := xds.NewHandshakeInfo(rootProvider, identityProvider, config.SubjectAltNameMatchers, config.SNI, false, config.AutoSNISANValidation, config.UseAutoHostSNI) + b.securityConfig = config + oldHI := b.xdsHIPtr.Swap(newHI) + if oldHI != nil { + oldHI.Decrement() + } + return nil +} + +func (b *clusterImplBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + defer clientConnUpdateHook() + + b.mu.Lock() + b.inhibitPickerUpdates = true + b.mu.Unlock() + if b.logger.V(2) { + b.logger.Infof("Received configuration: %s", pretty.ToJSON(s.BalancerConfig)) + } + newConfig, ok := s.BalancerConfig.(*LBConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) + } + + // Need to check for potential errors at the beginning of this function, so + // that on errors, we reject the whole config, instead of applying part of + // it. + bb := balancer.Get(newConfig.ChildPolicy.Name) + if bb == nil { + return fmt.Errorf("child policy %q not registered", newConfig.ChildPolicy.Name) + } + + if b.xdsClient == nil { + c := xdsclient.FromResolverState(s.ResolverState) + if c == nil { + return balancer.ErrBadResolverState + } + b.xdsClient = c + } + + xdsConfig := xdsresource.XDSConfigFromResolverState(s.ResolverState) + if xdsConfig == nil { + b.logger.Warningf("Received balancer config with no xDS config") + return balancer.ErrBadResolverState + } + clusterCfg := xdsConfig.Clusters[newConfig.Cluster] + clusterUpdate := clusterCfg.Config.Cluster + if err := b.handleSecurityConfig(clusterUpdate.SecurityCfg); err != nil { + // If the security config is invalid, for example, if the provider + // instance is not found in the bootstrap config, we need to put the + // channel in transient failure. + return fmt.Errorf("received Cluster resource that contains invalid security config: %v", err) + + } + // Update load reporting config. This needs to be done before updating the + // child policy because we need the loadStore from the updated client to be + // passed to the ccWrapper, so that the next picker from the child policy + // will pick up the new loadStore. + if err := b.updateLoadStore(clusterUpdate); err != nil { + return err + } + + // Build config for the gracefulswitch balancer. It is safe to ignore JSON + // marshaling errors here, since the config was already validated as part of + // ParseConfig(). + cfg := []map[string]any{{newConfig.ChildPolicy.Name: newConfig.ChildPolicy.Config}} + cfgJSON, _ := json.Marshal(cfg) + parsedCfg, err := gracefulswitch.ParseConfig(cfgJSON) + if err != nil { + return err + } + + // Addresses and sub-balancer config are sent to sub-balancer. + err = b.child.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: weightedroundrobin.SetBackendService(s.ResolverState, b.clusterName), + BalancerConfig: parsedCfg, + }) + + b.mu.Lock() + updatePicker := b.handleClusterConfigLocked(clusterCfg.Config) + // We want to send a picker update to the parent if one of the two + // conditions are met: + // - drop/request count config or LRS metrics config has changed *and* there + // is already a picker from the child, or + // - there is a pending picker update from the child (and this covers the + // case where the drop/request config has not changed, but the child sent + // a picker update while we were still processing config from our parent). + if (updatePicker && b.childState.Picker != nil) || b.pendingPickerUpdates { + b.pendingPickerUpdates = false + b.ClientConn.UpdateState(balancer.State{ + ConnectivityState: b.childState.ConnectivityState, + Picker: b.newPickerLocked(), + }) + } + b.inhibitPickerUpdates = false + b.mu.Unlock() + pickerUpdateHook() + return err +} + +func (b *clusterImplBalancer) ResolverError(err error) { + b.child.ResolverError(err) +} + +func (b *clusterImplBalancer) updateSubConnState(_ balancer.SubConn, s balancer.SubConnState, cb func(balancer.SubConnState)) { + // Trigger re-resolution when a SubConn turns transient failure. This is + // necessary for the LogicalDNS in cluster_resolver policy to re-resolve. + // + // Note that this happens not only for the addresses from DNS, but also for + // EDS (cluster_impl doesn't know if it's DNS or EDS, only the parent + // knows). The parent priority policy is configured to ignore re-resolution + // signal from the EDS children. + if s.ConnectivityState == connectivity.TransientFailure { + b.ClientConn.ResolveNow(resolver.ResolveNowOptions{}) + } + + if cb != nil { + cb(s) + } +} + +func (b *clusterImplBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, s) +} + +func (b *clusterImplBalancer) Close() { + b.child.Close() + b.childState = balancer.State{} + + if b.cancelLoadReport != nil { + stopCtx, stopCancel := context.WithTimeout(context.Background(), loadStoreStopTimeout) + defer stopCancel() + b.cancelLoadReport(stopCtx) + b.cancelLoadReport = nil + } + oldHI := b.xdsHIPtr.Swap(nil) + if oldHI != nil { + oldHI.Decrement() + } + b.securityConfig = nil + b.logger.Infof("Shutdown") +} + +func (b *clusterImplBalancer) ExitIdle() { + b.child.ExitIdle() +} + +// Override methods to accept updates from the child LB. + +func (b *clusterImplBalancer) UpdateState(state balancer.State) { + b.mu.Lock() + defer b.mu.Unlock() + + // Inhibit sending a picker update to our parent as part of handling new + // state from the child, if we are currently handling an update from our + // parent. Update the childState field regardless. + b.childState = state + if b.inhibitPickerUpdates { + b.pendingPickerUpdates = true + if b.logger.V(2) { + b.logger.Infof("Received a picker update from the child when processing an update from the parent") + } + return + } + + b.ClientConn.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: b.newPickerLocked(), + }) + pickerUpdateHook() +} + +func (b *clusterImplBalancer) setClusterName(n string) { + b.mu.Lock() + defer b.mu.Unlock() + b.clusterName = n +} + +func (b *clusterImplBalancer) getClusterName() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.clusterName +} + +// scWrapper is a wrapper of SubConn with locality ID. The locality ID can be +// retrieved from the addresses when creating SubConn. +// +// All SubConns passed to the child policies are wrapped in this, so that the +// picker can get the localityID from the picked SubConn, and do load reporting. +// +// After wrapping, all SubConns to and from the parent ClientConn (e.g. for +// SubConn state update, update/remove SubConn) must be the original SubConns. +// All SubConns to and from the child policy (NewSubConn, forwarding SubConn +// state update) must be the wrapper. The balancer keeps a map from the original +// SubConn to the wrapper for this purpose. +type scWrapper struct { + balancer.SubConn + + localityID clients.Locality + + hostname string +} + +func (b *clusterImplBalancer) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + clusterName := b.getClusterName() + newAddrs := make([]resolver.Address, len(addrs)) + for i, addr := range addrs { + newAddrs[i] = xdsinternal.SetXDSHandshakeClusterName(addr, clusterName) + newAddrs[i] = xds.SetHandshakeInfo(newAddrs[i], &b.xdsHIPtr) + + host := xdsresource.Hostname(addr) + // If the hostname contains a port, strip it. Per [RFC 6066, Section + // 3](https://www.rfc-editor.org/rfc/rfc6066.html#section-3), the SNI + // may only contain a qualified DNS hostname, which excludes port + // numbers. + h, _, err := net.SplitHostPort(host) + if err == nil { + host = h + } + // Store hostname in the address attributes, so that it can be used in + // the client handshake. + newAddrs[i] = xds.SetAddressHostname(newAddrs[i], host) + } + var sc balancer.SubConn + scw := &scWrapper{} + if len(addrs) > 0 { + scw.hostname = xdsresource.Hostname(addrs[0]) + scw.localityID = xdsinternal.GetLocalityID(addrs[0]) + } + oldListener := opts.StateListener + opts.StateListener = func(state balancer.SubConnState) { + b.updateSubConnState(sc, state, oldListener) + } + sc, err := b.ClientConn.NewSubConn(newAddrs, opts) + if err != nil { + return nil, err + } + scw.SubConn = sc + return scw, nil +} + +func (b *clusterImplBalancer) RemoveSubConn(sc balancer.SubConn) { + b.logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc) +} + +func (b *clusterImplBalancer) UpdateAddresses(sc balancer.SubConn, _ []resolver.Address) { + b.logger.Errorf("UpdateAddresses(%v) called unexpectedly", sc) +} + +// systemRootCertsProvider implements a certprovider.Provider that returns the +// system default root certificates for validation. +type systemRootCertsProvider struct{} + +func (systemRootCertsProvider) Close() {} + +func (systemRootCertsProvider) KeyMaterial(context.Context) (*certprovider.KeyMaterial, error) { + rootCAs, err := internal.X509SystemCertPoolFunc() + if err != nil { + return nil, err + } + return &certprovider.KeyMaterial{Roots: rootCAs}, nil +} diff --git a/internal/xds/balancer/clusterimpl/config.go b/internal/xds/balancer/clusterimpl/config.go new file mode 100644 index 0000000..47be86e --- /dev/null +++ b/internal/xds/balancer/clusterimpl/config.go @@ -0,0 +1,48 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl + +import ( + "encoding/json" + + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +// DropConfig contains the category, and drop ratio. +type DropConfig struct { + Category string + RequestsPerMillion uint32 +} + +// LBConfig is the balancer config for cluster_impl balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + Cluster string `json:"cluster,omitempty"` + ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"` +} + +func parseConfig(c json.RawMessage) (*LBConfig, error) { + var cfg LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} diff --git a/internal/xds/balancer/clusterimpl/config_test.go b/internal/xds/balancer/clusterimpl/config_test.go new file mode 100644 index 0000000..75163e0 --- /dev/null +++ b/internal/xds/balancer/clusterimpl/config_test.go @@ -0,0 +1,132 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + _ "google.golang.org/grpc/balancer/roundrobin" + _ "google.golang.org/grpc/balancer/weightedtarget" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" +) + +const ( + testJSONConfig = `{ + "cluster": "test_cluster", + "edsServiceName": "test-eds", + "lrsLoadReportingServer": { + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }, + "maxConcurrentRequests": 123, + "dropCategories": [ + { + "category": "drop-1", + "requestsPerMillion": 314 + }, + { + "category": "drop-2", + "requestsPerMillion": 159 + } + ], + "childPolicy": [ + { + "weighted_target_experimental": { + "targets": { + "wt-child-1": { + "weight": 75, + "childPolicy":[{"round_robin":{}}] + }, + "wt-child-2": { + "weight": 25, + "childPolicy":[{"round_robin":{}}] + } + } + } + } + ] +}` + + wtName = "weighted_target_experimental" +) + +var ( + wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser) + wtConfigJSON = `{ + "targets": { + "wt-child-1": { + "weight": 75, + "childPolicy":[{"round_robin":{}}] + }, + "wt-child-2": { + "weight": 25, + "childPolicy":[{"round_robin":{}}] + } + } +}` + + wtConfig, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON)) +) + +func (s) TestParseConfig(t *testing.T) { + tests := []struct { + name string + js string + want *LBConfig + wantErr bool + }{ + { + name: "empty json", + js: "", + want: nil, + wantErr: true, + }, + { + name: "bad json", + js: "{", + want: nil, + wantErr: true, + }, + { + name: "OK", + js: testJSONConfig, + want: &LBConfig{ + Cluster: "test_cluster", + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: wtName, + Config: wtConfig, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseConfig([]byte(tt.js)) + if (err != nil) != tt.wantErr { + t.Fatalf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + } + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("parseConfig() got unexpected diff (-want, +got): %v", diff) + } + }) + } +} diff --git a/internal/xds/balancer/clusterimpl/internal/internal.go b/internal/xds/balancer/clusterimpl/internal/internal.go new file mode 100644 index 0000000..711896b --- /dev/null +++ b/internal/xds/balancer/clusterimpl/internal/internal.go @@ -0,0 +1,25 @@ +/* + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains code internal to the clusterimpl package. +package internal + +import "crypto/x509" + +// X509SystemCertPoolFunc is used for overriding the system cert pool for +// tests. +var X509SystemCertPoolFunc = x509.SystemCertPool diff --git a/internal/xds/balancer/clusterimpl/logging.go b/internal/xds/balancer/clusterimpl/logging.go new file mode 100644 index 0000000..3bbd1b0 --- /dev/null +++ b/internal/xds/balancer/clusterimpl/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[xds-cluster-impl-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *clusterImplBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/internal/xds/balancer/clusterimpl/picker.go b/internal/xds/balancer/clusterimpl/picker.go new file mode 100644 index 0000000..00ed6da --- /dev/null +++ b/internal/xds/balancer/clusterimpl/picker.go @@ -0,0 +1,239 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl + +import ( + "context" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/stats" + "google.golang.org/grpc/internal/wrr" + xdsinternal "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +// NewRandomWRR is used when calculating drops. It's exported so that tests can +// override it. +var NewRandomWRR = wrr.NewRandom + +const million = 1000000 + +type dropper struct { + category string + w wrr.WRR +} + +// greatest common divisor (GCD) via Euclidean algorithm +func gcd(a, b uint32) uint32 { + for b != 0 { + t := b + b = a % b + a = t + } + return a +} + +func newDropper(c DropConfig) *dropper { + w := NewRandomWRR() + gcdv := gcd(c.RequestsPerMillion, million) + // Return true for RequestPerMillion, false for the rest. + w.Add(true, int64(c.RequestsPerMillion/gcdv)) + w.Add(false, int64((million-c.RequestsPerMillion)/gcdv)) + + return &dropper{ + category: c.Category, + w: w, + } +} + +func (d *dropper) drop() (ret bool) { + return d.w.Next().(bool) +} + +// loadReporter wraps the methods from the loadStore that are used here. +type loadReporter interface { + CallStarted(locality clients.Locality) + CallFinished(locality clients.Locality, err error) + CallServerLoad(locality clients.Locality, name string, val float64) + CallDropped(category string) +} + +// Picker implements RPC drop, circuit breaking drop and load reporting. +type picker struct { + drops []*dropper + s balancer.State + loadStore loadReporter + counter *xdsclient.ClusterRequestsCounter + countMax uint32 + telemetryLabels map[string]string + clusterName string + metrics *xdsresource.LRSReportEndpointMetricsConfig +} + +func (d *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + // Unconditionally update labels if present, even dropped or queued RPC's can + // use these labels. + stats.UpdateLabels(info.Ctx, d.telemetryLabels) + + // Don't drop unless the inner picker is READY. Similar to + // https://github.com/grpc/grpc-go/issues/2622. + if d.s.ConnectivityState == connectivity.Ready { + // Check if this RPC should be dropped by category. + for _, dp := range d.drops { + if dp.drop() { + if d.loadStore != nil { + d.loadStore.CallDropped(dp.category) + } + return balancer.PickResult{}, status.Errorf(codes.Unavailable, "RPC is dropped") + } + } + } + + // Check if this RPC should be dropped by circuit breaking. + if d.counter != nil { + if err := d.counter.StartRequest(d.countMax); err != nil { + // Drops by circuit breaking are reported with empty category. They + // will be reported only in total drops, but not in per category. + if d.loadStore != nil { + d.loadStore.CallDropped("") + } + return balancer.PickResult{}, status.Error(codes.Unavailable, err.Error()) + } + } + + var lID clients.Locality + pr, err := d.s.Picker.Pick(info) + if scw, ok := pr.SubConn.(*scWrapper); ok { + // This OK check also covers the case err!=nil, because SubConn will be + // nil. + pr.SubConn = scw.SubConn + // If locality ID isn't found in the wrapper, an empty locality ID will + // be used. + lID = scw.localityID + + if scw.hostname != "" && autoHostRewriteEnabled(info.Ctx) { + if pr.Metadata == nil { + pr.Metadata = metadata.Pairs(":authority", scw.hostname) + } else { + pr.Metadata.Set(":authority", scw.hostname) + } + } + } + + if err != nil { + if d.counter != nil { + // Release one request count if this pick fails. + d.counter.EndRequest() + } + return pr, err + } + + stats.UpdateLabels( + info.Ctx, + map[string]string{ + "grpc.lb.locality": xdsinternal.LocalityString(lID), + "grpc.lb.backend_service": d.clusterName, + }, + ) + + if d.loadStore != nil { + locality := clients.Locality{Region: lID.Region, Zone: lID.Zone, SubZone: lID.SubZone} + d.loadStore.CallStarted(locality) + oldDone := pr.Done + pr.Done = func(info balancer.DoneInfo) { + if oldDone != nil { + oldDone(info) + } + d.loadStore.CallFinished(locality, info.Err) + + load, ok := info.ServerLoad.(*v3orcapb.OrcaLoadReport) + if !ok || load == nil { + return + } + + if envconfig.XDSORCAToLRSPropEnabled { + if d.metrics != nil { + if d.metrics.CPUUtilization { + d.loadStore.CallServerLoad(locality, "cpu_utilization", load.CpuUtilization) + } + if d.metrics.MemUtilization { + d.loadStore.CallServerLoad(locality, "mem_utilization", load.MemUtilization) + } + if d.metrics.ApplicationUtilization { + d.loadStore.CallServerLoad(locality, "application_utilization", load.ApplicationUtilization) + } + for n, c := range load.NamedMetrics { + _, ok := d.metrics.NamedMetrics[n] + if d.metrics.NamedMetricsAll || ok { + d.loadStore.CallServerLoad(locality, "named_metrics."+n, c) + } + } + } + } else { + for n, c := range load.NamedMetrics { + d.loadStore.CallServerLoad(locality, n, c) + } + } + } + } + + if d.counter != nil { + // Update Done() so that when the RPC finishes, the request count will + // be released. + oldDone := pr.Done + pr.Done = func(doneInfo balancer.DoneInfo) { + d.counter.EndRequest() + if oldDone != nil { + oldDone(doneInfo) + } + } + } + + return pr, err +} + +// autoHostRewriteKey is the context key used to store the value of +// route's autoHostRewrite in the RPC context. +type autoHostRewriteKey struct{} + +// autoHostRewriteEnabled retrieves the autoHostRewrite value from the provided context. +func autoHostRewriteEnabled(ctx context.Context) bool { + v, _ := ctx.Value(autoHostRewriteKey{}).(bool) + return v +} + +// AutoHostRewriteEnabledForTesting returns the value of autoHostRewrite field; +// to be used for testing only. +func AutoHostRewriteEnabledForTesting(ctx context.Context) bool { + return autoHostRewriteEnabled(ctx) +} + +// EnableAutoHostRewrite adds the autoHostRewrite value to the context for +// the xds_cluster_impl LB policy to pick. +func EnableAutoHostRewrite(ctx context.Context) context.Context { + return context.WithValue(ctx, autoHostRewriteKey{}, true) +} diff --git a/internal/xds/balancer/clusterimpl/tests/balancer_test.go b/internal/xds/balancer/clusterimpl/tests/balancer_test.go new file mode 100644 index 0000000..4a044ac --- /dev/null +++ b/internal/xds/balancer/clusterimpl/tests/balancer_test.go @@ -0,0 +1,1650 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl_test + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "math" + "net" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/fakeserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3" + v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + xdscreds "google.golang.org/grpc/credentials/xds" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/orca" + "google.golang.org/protobuf/types/known/structpb" + + _ "google.golang.org/grpc/xds" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 100 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestConfigUpdateWithSameLoadReportingServerConfig tests the scenario where +// the clusterimpl LB policy receives a config update with no change in the load +// reporting server configuration. The test verifies that the existing load +// reporting stream is not terminated and that a new load reporting stream is not +// created. +func (s) TestConfigUpdateWithSameLoadReportingServerConfig(t *testing.T) { + // Create an xDS management server that serves ADS and LRS requests. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a server backend exposing the test service. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure the xDS management server with default resources. Override the + // default cluster to include an LRS server config pointing to self. + const serviceName = "my-test-xds-service" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // Ensure that an LRS stream is created. + if _, err := mgmtServer.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Failure when waiting for an LRS stream to be opened: %v", err) + } + + // Configure a new resource on the management server with drop config that + // drops all RPCs, but with no change in the load reporting server config. + resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{ + e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: "endpoints-" + serviceName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, server.Address)}}}, + Weight: 1, + }, + }, + DropPercents: map[string]int{"test-drop-everything": 100}, + }), + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Repeatedly send RPCs until we sees that they are getting dropped, or the + // test context deadline expires. The former indicates that new config with + // drops has been applied. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err != nil && status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), "RPC is dropped") { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for RPCs to be dropped after config update") + } + + // Ensure that the old LRS stream is not closed. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := mgmtServer.LRSServer.LRSStreamCloseChan.Receive(sCtx); err == nil { + t.Fatal("LRS stream closed when expected not to") + } + + // Also ensure that a new LRS stream is not created. + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := mgmtServer.LRSServer.LRSStreamOpenChan.Receive(sCtx); err == nil { + t.Fatal("New LRS stream created when expected not to") + } +} + +// Tests whether load is reported correctly when using pickfirst with endpoints +// in multiple localities. +func (s) TestLoadReportingPickFirstMultiLocality(t *testing.T) { + // Create an xDS management server that serves ADS and LRS requests. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start two server backends exposing the test service. + server1 := stubserver.StartTestService(t, nil) + defer server1.Stop() + + server2 := stubserver.StartTestService(t, nil) + defer server2.Stop() + + // Configure the xDS management server. + const serviceName = "my-test-xds-service" + routeConfigName := "route-" + serviceName + clusterName := "cluster-" + serviceName + endpointsName := "endpoints-" + serviceName + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + { + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: endpointsName, + }, + // Specify a custom load balancing policy to use pickfirst. + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3pickfirstpb.PickFirst{}), + }, + }, + }, + }, + // Include a fake LRS server config pointing to self. + LrsServer: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + }, + }, + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: endpointsName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Backends: []e2e.BackendOptions{ + {Ports: []uint32{testutils.ParsePort(t, server1.Address)}}, + }, + Weight: 1, + }, + { + Backends: []e2e.BackendOptions{ + {Ports: []uint32{testutils.ParsePort(t, server2.Address)}}, + }, + Weight: 2, + }, + }, + })}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + var peer peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // Verify that the request was sent to server 1. + if got, want := peer.Addr.String(), server1.Address; got != want { + t.Errorf("peer.Addr = %q, want = %q", got, want) + } + + // Ensure that an LRS stream is created. + if _, err = mgmtServer.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Failure when waiting for an LRS stream to be opened: %v", err) + } + + // Handle the initial LRS request from the xDS client. + if _, err = mgmtServer.LRSServer.LRSRequestChan.Receive(ctx); err != nil { + t.Fatalf("Failure waiting for initial LRS request: %v", err) + } + + resp := fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + SendAllClusters: true, + LoadReportingInterval: durationpb.New(10 * time.Millisecond), + }, + } + mgmtServer.LRSServer.LRSResponseChan <- &resp + + // Wait for load to be reported for locality of server 1. + if err := waitForSuccessfulLoadReport(ctx, mgmtServer.LRSServer, "region-1"); err != nil { + t.Fatalf("Server 1 did not receive load due to error: %v", err) + } + + // Stop server 1 and send one more rpc. Now the request should go to server 2. + server1.Stop() + + // Wait for the balancer to pick up the server state change. + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // Verify that the request was sent to server 2. + if got, want := peer.Addr.String(), server2.Address; got != want { + t.Errorf("peer.Addr = %q, want = %q", got, want) + } + + // Wait for load to be reported for locality of server 2. + if err := waitForSuccessfulLoadReport(ctx, mgmtServer.LRSServer, "region-2"); err != nil { + t.Fatalf("Server 2 did not receive load due to error: %v", err) + } +} + +// waitForSuccessfulLoadReport waits for a successful request to be reported for +// the specified locality region. +func waitForSuccessfulLoadReport(ctx context.Context, lrsServer *fakeserver.Server, region string) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case req := <-lrsServer.LRSRequestChan.C: + loadStats := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) + for _, load := range loadStats.ClusterStats { + for _, locality := range load.UpstreamLocalityStats { + if locality.TotalSuccessfulRequests > 0 && locality.Locality.Region == region { + return nil + } + } + } + } + } +} + +// Tests that circuit breaking limits RPCs E2E. +func (s) TestCircuitBreaking(t *testing.T) { + // Create an xDS management server that serves ADS and LRS requests. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a server backend exposing the test service. + f := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + if _, err := stream.Recv(); err != nil { + return err + } + } + }, + } + server := stubserver.StartTestService(t, f) + defer server.Stop() + + // Configure xDS resources on the management server with a circuit breaking + // policy that limits the maximum number of concurrent requests to 3. + const serviceName = "my-test-xds-service" + const maxRequests = 3 + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + }) + resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{ + Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{ + { + Priority: v3corepb.RoutingPriority_DEFAULT, + MaxRequests: wrapperspb.UInt32(maxRequests), + }, + }, + } + resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + // Start maxRequests streams. + for range maxRequests { + if _, err := client.FullDuplexCall(ctx); err != nil { + t.Fatalf("rpc FullDuplexCall() failed: %v", err) + } + } + + // Since we are at the max, new streams should fail. It's possible some are + // allowed due to inherent raciness in the tracking, however. + const droppedRPCCount = 100 + for i := 0; i < droppedRPCCount; i++ { + if ctx.Err() != nil { + t.Fatalf("Context error: %v", ctx.Err()) + } + if _, err := client.FullDuplexCall(ctx); status.Code(err) != codes.Unavailable { + t.Fatalf("client.FullDuplexCall() returned status %q, want %q", status.Code(err), codes.Unavailable) + } + } + + if _, err = mgmtServer.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Failure when waiting for an LRS stream to be opened: %v", err) + } + + if _, err = mgmtServer.LRSServer.LRSRequestChan.Receive(ctx); err != nil { + t.Fatalf("Failure waiting for initial LRS request: %v", err) + } + + mgmtServer.LRSServer.LRSResponseChan <- &fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + SendAllClusters: true, + LoadReportingInterval: durationpb.New(10 * time.Millisecond), + }, + } + + select { + case req := <-mgmtServer.LRSServer.LRSRequestChan.C: + clusterStats := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if l := len(clusterStats); l != 1 { + t.Fatalf("Received load for %d clusters, want 1", l) + } + clusterStats[0].LoadReportInterval = nil + wantLoad := &v3endpointpb.ClusterStats{ + ClusterName: "cluster-my-test-xds-service", + ClusterServiceName: "endpoints-my-test-xds-service", + UpstreamLocalityStats: []*v3endpointpb.UpstreamLocalityStats{ + { + Locality: &v3corepb.Locality{Region: "region-1", Zone: "zone-1", SubZone: "subzone-1"}, + TotalIssuedRequests: maxRequests, + TotalRequestsInProgress: maxRequests, + }, + }, + TotalDroppedRequests: droppedRPCCount, + } + if diff := cmp.Diff(wantLoad, clusterStats[0], protocmp.Transform()); diff != "" { + t.Errorf("Failed with unexpected diff (-want +got):\n%s", diff) + } + case <-ctx.Done(): + t.Fatalf("Timeout while waiting for load report on LRS stream: %v", ctx.Err()) + } +} + +// ComputeIdealNumRpcs calculates the ideal number of RPCs to send to test +// probabilistic behaviors. +// +// It's based on the idea of a binomial distribution approximated by a normal +// distribution. The function aims to find a number of RPCs such that +// the observed probability is within a certain error_tolerance of the expected +// probability (p). +func computeIdealNumRpcs(p, errorTolerance float64) uint64 { + return uint64(math.Ceil(p * (1 - p) * 5.00 * 5.00 / errorTolerance / errorTolerance)) +} + +func verifyDropRateByCategory(ctx context.Context, client testgrpc.TestServiceClient, rpcCount int, lrsServer *fakeserver.Server, wantCluster, wantDropCategory string, wantDropRate, errorTolerance float64) error { + for i := 0; i < rpcCount; i++ { + if ctx.Err() != nil { + return fmt.Errorf("context error: %v", ctx.Err()) + } + stream, err := client.FullDuplexCall(ctx) + switch { + case err == nil: + // Close stream to avoid exceeding limits. + stream.CloseSend() + case status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), "RPC is dropped"): + continue + default: + return fmt.Errorf("client.FullDuplexCall(_) failed with unexpected error = %v", err) + } + } + + for { + if ctx.Err() != nil { + return fmt.Errorf("timeout when waiting for load stats") + } + req, err := lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + continue + } + loadStats := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if len(loadStats) != 1 || loadStats[0].ClusterName != wantCluster { + continue + } + if loadStats[0].ClusterName != wantCluster { + return fmt.Errorf("Received stats for unexpected cluster got: %q, want: %q", loadStats[0].ClusterName, wantCluster) + } + + for _, cs := range loadStats { + found := false + for _, dr := range cs.DroppedRequests { + if dr.Category == wantDropCategory { + found = true + gotRPCDropRate := float64(dr.DroppedCount) / float64(rpcCount) + if (math.Trunc(math.Abs(gotRPCDropRate-wantDropRate)*100) / 100) > errorTolerance { + return fmt.Errorf("Drop rate goes out of errortolerance got: %v, want: %v, totalDroppedRequest: %v, totalIssuesRequest: %v", (math.Trunc(math.Abs(gotRPCDropRate-wantDropRate)*100) / 100), errorTolerance, cs.TotalDroppedRequests, cs.UpstreamLocalityStats[0].TotalIssuedRequests) + } + } + } + if !found { + continue + } + } + break + } + return nil +} + +// TestDropByCategory verifies that the balancer correctly drops the picks, and +// that the drops are reported. +func (s) TestDropByCategory(t *testing.T) { + // Create an xDS management server that serves ADS and LRS requests. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a server backend exposing the test service. + f := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + if _, err := stream.Recv(); err != nil { + return err + } + } + }, + } + server := stubserver.StartTestService(t, f) + defer server.Stop() + + // Configure xDS resources on the management server with drops + // configuration that drops one RPC for every 50 RPCs made. + const ( + dropReason = "test-dropping-category" + dropNumerator = 1 + dropDenominator = 50 + serviceName = "my-test-xds-service" + errorTolerance = 0.05 + ) + wantRPCDropRate := float64(dropNumerator) / float64(dropDenominator) + rpcCount := computeIdealNumRpcs(wantRPCDropRate, errorTolerance) + t.Logf("About to send %d RPCs to test drop rate of %v", rpcCount, wantRPCDropRate) + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + }) + resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + } + resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{ + e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: "endpoints-" + serviceName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, server.Address)}}}, + Weight: 1, + }, + }, + DropPercents: map[string]int{ + dropReason: dropNumerator * 100 / dropDenominator, + }, + })} + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Ensure the gRPC channel is READY before issuing RPCs to get accurate + // drop count. + cc.Connect() + client := testgrpc.NewTestServiceClient(cc) + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if _, err = mgmtServer.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Failure when waiting for an LRS stream to be opened: %v", err) + } + + resp := fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + Clusters: []string{resources.Clusters[0].Name}, + LoadReportingInterval: durationpb.New(50 * time.Millisecond), + }, + } + mgmtServer.LRSServer.LRSResponseChan <- &resp + + if err := verifyDropRateByCategory(ctx, client, int(rpcCount), mgmtServer.LRSServer, resources.Clusters[0].Name, dropReason, wantRPCDropRate, errorTolerance); err != nil { + t.Fatal(err) + } + + // Update the drop configuration to drop 1 out of every 40 RPCs, + // and verify that the observed drop rate matches this new config. + const ( + dropReason2 = "test-dropping-category-2" + dropNumerator2 = 1 + dropDenominator2 = 40 + ) + + resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{ + e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: "endpoints-" + serviceName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, server.Address)}}}, + Weight: 1, + }, + }, + DropPercents: map[string]int{ + dropReason2: dropNumerator2 * 100 / dropDenominator2, // drops one RPC for every 40 RPCs made + }, + }), + } + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + wantRPCDropRate = float64(dropNumerator2) / float64(dropDenominator2) + rpcCount = computeIdealNumRpcs(wantRPCDropRate, errorTolerance) + t.Logf("About to send %d RPCs to test drop rate of %v", rpcCount, wantRPCDropRate) + + if err := verifyDropRateByCategory(ctx, client, int(rpcCount), mgmtServer.LRSServer, resources.Clusters[0].Name, dropReason2, wantRPCDropRate, errorTolerance); err != nil { + t.Fatal(err) + } +} + +// Tests that circuit breaking limits RPCs in LOGICAL_DNS clusters E2E. +func (s) TestCircuitBreakingLogicalDNS(t *testing.T) { + // Create an xDS management server that serves ADS requests. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a server backend exposing the test service. + f := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + if _, err := stream.Recv(); err != nil { + return err + } + } + }, + } + server := stubserver.StartTestService(t, f) + defer server.Stop() + host, port := hostAndPortFromAddress(t, server.Address) + + // Configure the xDS management server with default resources. Override the + // default cluster to include a circuit breaking config. + const serviceName = "my-test-xds-service" + const maxRequests = 3 + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + }) + resources.Clusters = []*v3clusterpb.Cluster{ + e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: "cluster-" + serviceName, + Type: e2e.ClusterTypeLogicalDNS, + DNSHostName: host, + DNSPort: port, + }), + } + resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{ + Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{ + { + Priority: v3corepb.RoutingPriority_DEFAULT, + MaxRequests: wrapperspb.UInt32(maxRequests), + }, + }, + } + resources.Endpoints = nil + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + // Start maxRequests streams. + for range maxRequests { + if _, err := client.FullDuplexCall(ctx); err != nil { + t.Fatalf("rpc FullDuplexCall() failed: %v", err) + } + } + + // Since we are at the max, new streams should fail. It's possible some are + // allowed due to inherent raciness in the tracking, however. + for i := 0; i < 100; i++ { + stream, err := client.FullDuplexCall(ctx) + if status.Code(err) == codes.Unavailable { + return + } + if err == nil { + // Terminate the stream (the server immediately exits upon a client + // CloseSend) to ensure we never go over the limit. + stream.CloseSend() + stream.Recv() + } + time.Sleep(10 * time.Millisecond) + } + + t.Fatalf("RPCs unexpectedly allowed beyond circuit breaking maximum") +} + +func hostAndPortFromAddress(t *testing.T, addr string) (string, uint32) { + t.Helper() + + host, p, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("Invalid serving address: %v", addr) + } + port, err := strconv.ParseUint(p, 10, 32) + if err != nil { + t.Fatalf("Invalid serving port %q: %v", p, err) + } + return host, uint32(port) +} + +// Tests that LRS works correctly in a LOGICAL_DNS cluster. +func (s) TestLRSLogicalDNS(t *testing.T) { + // Create an xDS management server that serves ADS and LRS requests. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a server backend exposing the test service. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + host, port := hostAndPortFromAddress(t, server.Address) + + // Configure the xDS management server with default resources. Override the + // default cluster to include an LRS server config pointing to self. + const serviceName = "my-test-xds-service" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + resources.Clusters = []*v3clusterpb.Cluster{ + e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: "cluster-" + serviceName, + Type: e2e.ClusterTypeLogicalDNS, + DNSHostName: host, + DNSPort: port, + }), + } + resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + } + resources.Endpoints = nil + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // Ensure that an LRS stream is created. + if _, err := mgmtServer.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Failure when waiting for an LRS stream to be opened: %v", err) + } + + // Handle the initial LRS request from the xDS client. + if _, err = mgmtServer.LRSServer.LRSRequestChan.Receive(ctx); err != nil { + t.Fatalf("Failure waiting for initial LRS request: %v", err) + } + + resp := fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + SendAllClusters: true, + LoadReportingInterval: durationpb.New(10 * time.Millisecond), + }, + } + mgmtServer.LRSServer.LRSResponseChan <- &resp + + // Wait for load to be reported for locality of server 1. + if err := waitForSuccessfulLoadReport(ctx, mgmtServer.LRSServer, ""); err != nil { + t.Fatalf("Server did not receive load due to error: %v", err) + } +} + +// TestReResolution verifies that when a SubConn turns transient failure, +// re-resolution is triggered. +func (s) TestReResolutionAfterTransientFailure(t *testing.T) { + // Create an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer mgmtServer.Stop() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Create a restartable listener to simulate server being down. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + server := stubserver.StartTestService(t, &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + }) + defer server.Stop() + host, port := hostAndPortFromAddress(t, server.Address) + + const ( + listenerName = "test-listener" + routeName = "test-route" + clusterName = "test-aggregate-cluster" + dnsCluster = "logical-dns-cluster" + ) + + // Configure xDS resources. + ldnsCluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + Type: e2e.ClusterTypeLogicalDNS, + ClusterName: dnsCluster, + DNSHostName: host, + DNSPort: port, + }) + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + Type: e2e.ClusterTypeAggregate, + ChildNames: []string{dnsCluster}, + }) + updateOpts := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, listenerName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{cluster, ldnsCluster}, + Endpoints: nil, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Replace DNS resolver with a wrapped resolver to capture ResolveNow calls. + // We expect two ResolveNow calls: + // 1. When the server goes down and the transport is closed on the client, + // the gRPC channel invokes ResolveNow on the resolver. + // 2. Subsequently, the subchannel enters IDLE, reconnection attempt takes + // place and since the server is down, the subchannel eventually moves + // enters TRANSIENT_FAILURE, at which point the clusterimpl policy + // invokes ResolveNow. + resolveNowCh := make(chan struct{}, 2) + dnsR := manual.NewBuilderWithScheme("dns") + dnsResolverBuilder := resolver.Get("dns") + resolver.Register(dnsR) + defer resolver.Register(dnsResolverBuilder) + dnsR.ResolveNowCallback = func(resolver.ResolveNowOptions) { + select { + case resolveNowCh <- struct{}{}: + case <-ctx.Done(): + } + } + dnsR.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: fmt.Sprintf("%s:%d", host, port)}}}}, + }) + + if err := mgmtServer.Update(ctx, updateOpts); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + conn, err := grpc.NewClient(fmt.Sprintf("xds:///%s", listenerName), grpc.WithResolvers(resolverBuilder), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + defer conn.Close() + + client := testgrpc.NewTestServiceClient(conn) + + // Verify initial RPC routes correctly to backend. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("client.EmptyCall() failed: %v", err) + } + + // Stopping the server listener will close the transport on the client, + // which will lead to the channel eventually moving to TRANSIENT_FAILURE. + lis.Stop() + testutils.AwaitState(ctx, t, conn, connectivity.TransientFailure) + + // Expect two calls to the resolver's ResolveNow method. + for range 2 { + select { + case <-resolveNowCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for ResolveNow call after TransientFailure") + } + } + + // Restart the listener and expected to reconnect on its own and come out + // of TRANSIENT_FAILURE, even without an RPC attempt. + lis.Restart() + testutils.AwaitState(ctx, t, conn, connectivity.Ready) + + // An RPC at this point is expected to succeed. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("client.EmptyCall() failed: %v", err) + } +} + +// TestUpdateLRSServerToNil verifies that updating the cluster's LRS server +// config from 'Self' to nil correctly closes the LRS stream and ensures no +// more LRS reports are sent. +func (s) TestUpdateLRSServerToNil(t *testing.T) { + // Create an xDS management server that serves ADS and LRS requests. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true}) + defer mgmtServer.Stop() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a server backend exposing the test service. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure the xDS management server with default resources. + const serviceName = "my-test-xds-service" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("client.EmptyCall() failed: %v", err) + } + + // Ensure that an LRS stream is created. + if _, err := mgmtServer.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Error waiting for initial LRS stream open: %v", err) + } + if _, err := mgmtServer.LRSServer.LRSRequestChan.Receive(ctx); err != nil { + t.Fatalf("Error waiting for initial LRS report: %v", err) + } + + // Update LRS Server to nil + resources.Clusters[0].LrsServer = nil + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that the old LRS stream is closed. + if _, err := mgmtServer.LRSServer.LRSStreamCloseChan.Receive(ctx); err != nil { + t.Fatalf("Error waiting for initial LRS stream close : %v", err) + } + + // Also ensure that a new LRS stream is not created. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := mgmtServer.LRSServer.LRSRequestChan.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Expected no LRS reports after disable, got %v want %v", err, context.DeadlineExceeded) + } +} + +// Test verifies that child policy was updated on receipt of +// configuration update. +func (s) TestChildPolicyChangeOnConfigUpdate(t *testing.T) { + const customLBPolicy = "test_custom_lb_policy" + + // Create an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer mgmtServer.Stop() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a server backend exposing the test service. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const serviceName = "test-child-policy" + + // Configure the xDS management server with default resources. Cluster + // corresponding to this resource will be configured with "round_robin" + // as the endpoint picking policy + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithResolvers(resolverBuilder), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("client.EmptyCall() failed: %v", err) + } + + // Register stub customLBPolicy LB policy so that we can catch config changes. + pfBuilder := balancer.Get(pickfirst.Name) + lbCfgCh := make(chan serviceconfig.LoadBalancingConfig, 1) + var updatedChildPolicy atomic.Value + + stub.Register(customLBPolicy, stub.BalancerFuncs{ + ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return pfBuilder.(balancer.ConfigParser).ParseConfig(lbCfg) + }, + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = pfBuilder.Build(bd.ClientConn, bd.BuildOptions) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + // name := customLBPolicy + updatedChildPolicy.Store(customLBPolicy) + select { + case lbCfgCh <- ccs.BalancerConfig: + case <-ctx.Done(): + t.Error("Timed out while waiting for BalancerConfig, context deadline exceeded") + } + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + }) + + defer internal.BalancerUnregister(customLBPolicy) + + // Update the cluster to use customLBPolicy as the endpoint picking policy. + resources.Clusters[0].LoadBalancingPolicy = &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{{ + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/" + customLBPolicy, + Value: &structpb.Struct{}, + }), + }, + }}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for child policy config") + case <-lbCfgCh: + } + + if p, ok := updatedChildPolicy.Load().(string); !ok || p != customLBPolicy { + t.Fatalf("Unexpected child policy after config update, got %q (ok: %v), want %q", p, ok, customLBPolicy) + } + + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for successful RPC after policy update.") + case <-ticker.C: + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err == nil { + return + } + } + } +} + +// Test verifies that config update fails if child policy config +// failed to parse. +func (s) TestFailedToParseChildPolicyConfig(t *testing.T) { + // Create an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer mgmtServer.Stop() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a server backend exposing the test service. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const serviceName = "test-child-policy" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + }) + resources.Clusters[0].LoadBalancingPolicy = &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{{ + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3pickfirstpb.PickFirst{}), + }, + }}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + // Register stub pickfirst LB policy so that we can catch parsing errors. + pfBuilder := balancer.Get(pickfirst.Name) + internal.BalancerUnregister(pfBuilder.Name()) + const parseConfigError = "failed to parse config" + stub.Register(pfBuilder.Name(), stub.BalancerFuncs{ + ParseConfig: func(_ json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return nil, errors.New(parseConfigError) + }, + }) + defer balancer.Register(pfBuilder) + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithResolvers(resolverBuilder), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err == nil || !strings.Contains(err.Error(), parseConfigError) { + t.Fatal("EmptyCall RPC succeeded when expected to fail") + } +} + +// setupManagementServerAndResolver sets up an xDS management server and returns +// the management server, resolver builder and Node ID. +func setupManagementServerAndResolver(t *testing.T) (*e2e.ManagementServer, resolver.Builder, string) { + t.Helper() + + nodeID := uuid.New().String() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + contents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(contents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + return mgmtServer, resolverBuilder, nodeID +} + +// configureXDSResources configures the management server with a route that +// enables auto_host_rewrite and an endpoint with the specified hostname. +func configureXDSResources(ctx context.Context, t *testing.T, mgmtServer *e2e.ManagementServer, nodeID, serverAddr, endpointHostname string, secLevel e2e.SecurityLevel, clusterType e2e.ClusterType) { + t.Helper() + + const ( + serviceName = "my-test-xds-service" + routeName = "route-my-test-xds-service" + clusterName = "cluster-my-test-xds-service" + endpointName = "endpoints-my-test-xds-service" + ) + + port := testutils.ParsePort(t, serverAddr) + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: port, + SecLevel: secLevel, + }) + + if clusterType == e2e.ClusterTypeLogicalDNS { + resources.Clusters = []*v3clusterpb.Cluster{ + e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + Type: e2e.ClusterTypeLogicalDNS, + DNSHostName: endpointHostname, + DNSPort: port, + }), + } + resources.Endpoints = nil + } else { + // Set the endpoint hostname for authority rewriting. + resources.Endpoints[0].Endpoints[0].LbEndpoints[0].GetEndpoint().Hostname = endpointHostname + } + + // Modify the route to enable AutoHostRewrite. + resources.Routes[0].VirtualHosts[0].Routes[0].GetRoute().HostRewriteSpecifier = &v3routepb.RouteAction_AutoHostRewrite{ + AutoHostRewrite: &wrapperspb.BoolValue{Value: true}, + } + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } +} + +// TestAuthorityOverriding verifies that the :authority header is correctly +// rewritten to the endpoint's hostname. Also verifies that CallAuthority +// call option takes precedence. +func (s) TestAuthorityOverriding(t *testing.T) { + tests := []struct { + name string + clusterType e2e.ClusterType + }{ + { + name: "EDS", + clusterType: e2e.ClusterTypeEDS, + }, + { + name: "LogicalDNS", + clusterType: e2e.ClusterTypeLogicalDNS, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSAuthorityRewrite, true) + mgmtServer, resolverBuilder, nodeID := setupManagementServerAndResolver(t) + + // Start a server backend exposing the test service. + gotAuthority := atomic.Pointer[string]{} + f := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + if md, ok := metadata.FromIncomingContext(ctx); ok { + if authVals := md.Get(":authority"); len(authVals) > 0 { + gotAuthority.Store(&authVals[0]) + } + } + return &testpb.Empty{}, nil + }, + } + server := stubserver.StartTestService(t, f) + defer server.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + var hostname string + if test.clusterType == e2e.ClusterTypeEDS { + hostname = "rewritten.example.com" + } else { + hostname, _ = hostAndPortFromAddress(t, server.Address) + } + configureXDSResources(ctx, t, mgmtServer, nodeID, server.Address, hostname, e2e.SecurityLevelNone, test.clusterType) + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient("xds:///my-test-xds-service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("client.EmptyCall() failed: %v", err) + } + + var wantAuthority string + if test.clusterType == e2e.ClusterTypeEDS { + wantAuthority = "rewritten.example.com" + } else { + wantAuthority = server.Address + } + if got, want := *gotAuthority.Load(), wantAuthority; got != want { + t.Errorf("invalid authority got: %q, want: %q", got, want) + } + + // The authority specified via the `CallAuthority` CallOption takes the + // highest precedence when determining the `:authority` header. + const userAuthorityOverride = "user-override.com" + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.CallAuthority(userAuthorityOverride)); err != nil { + t.Fatalf("client.EmptyCall() failed: %v", err) + } + + if got, want := *gotAuthority.Load(), userAuthorityOverride; got != want { + t.Errorf("Server received authority %q, want %q (user override)", got, want) + } + }) + } +} + +// TestAuthorityOverridingWithTLS verifies the interaction between xDS Authority +// Rewriting and TLS Secure Naming. It ensures that when the :authority header +// is rewritten by the clusterimpl picker, the new authority is correctly +// validated against the server's TLS certificate before the RPC proceeds. +// Also check that RPC fails when the rewritten authority does not match the +// server's certificate due to secure naming validation. +func (s) TestAuthorityOverridingWithTLS(t *testing.T) { + tests := []struct { + name string + xdsAuthorityOverride string + wantSuccess bool + }{ + { + name: "Valid_Authority_Rewrite", + xdsAuthorityOverride: "x.test.example.com", + wantSuccess: true, + }, + { + name: "Authority_Rewrite_Mismatch", + xdsAuthorityOverride: "xyz.exmaple.com", + wantSuccess: false, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSAuthorityRewrite, true) + mgmtServer, resolverBuilder, nodeID := setupManagementServerAndResolver(t) + + serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert) + + // Start a server backend exposing the test service. + var gotAuthority string + f := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + if md, ok := metadata.FromIncomingContext(ctx); ok { + if authVals := md.Get(":authority"); len(authVals) > 0 { + gotAuthority = authVals[0] + } + } + return &testpb.Empty{}, nil + }, + } + f.StartServer(grpc.Creds(serverCreds)) + defer f.Stop() + + clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("Failed to create client credentials: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + configureXDSResources(ctx, t, mgmtServer, nodeID, f.Address, test.xdsAuthorityOverride, e2e.SecurityLevelMTLS, e2e.ClusterTypeEDS) + + // Create ClientConn with TLS + cc, err := grpc.NewClient("xds:///my-test-xds-service", grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + _, err = client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)) + + if test.wantSuccess { + if err != nil { + t.Fatalf("RPC failed unexpectedly: %v", err) + } + if gotAuthority != test.xdsAuthorityOverride { + t.Errorf("invalid authority got: %q, want: %q", gotAuthority, test.xdsAuthorityOverride) + } + } else { + if status.Code(err) != codes.Unavailable { + t.Fatalf("Expected TLS failure due to authority mismatch, got: %q want: %q", codes.Unavailable, status.Code(err)) + } + } + }) + } +} + +// Tests that configured LRS metrics are successfully propagated from the +// backend to the LRS server. +func (s) TestLoadReporting_CustomMetricsPropagation(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSORCAToLRSPropEnabled, true) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true}) + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + smr := orca.NewServerMetricsRecorder() + smr.SetCPUUtilization(0.8) + + f := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + cmr := orca.CallMetricsRecorderFromContext(ctx) + if cmr != nil { + cmr.SetNamedMetric("db_cost", 50.0) + cmr.SetNamedMetric("ignored_metric", 10.0) + } + return &testpb.Empty{}, nil + }, + } + server := stubserver.StartTestService(t, f, orca.CallMetricsServerOption(smr)) + defer server.Stop() + const serviceName = "my-test-xds-service" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + } + resources.Clusters[0].LrsReportEndpointMetrics = []string{ + "cpu_utilization", + "named_metrics.db_cost", + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + if _, err = mgmtServer.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Failure when waiting for an LRS stream to be opened: %v", err) + } + if _, err = mgmtServer.LRSServer.LRSRequestChan.Receive(ctx); err != nil { + t.Fatalf("Failure waiting for initial LRS request: %v", err) + } + + mgmtServer.LRSServer.LRSResponseChan <- &fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + SendAllClusters: true, + LoadReportingInterval: durationpb.New(10 * time.Millisecond), + }, + } + + for { + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for LRS load report: %v", ctx.Err()) + case req := <-mgmtServer.LRSServer.LRSRequestChan.C: + loadStats := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) + for _, load := range loadStats.ClusterStats { + for _, locality := range load.UpstreamLocalityStats { + if locality.TotalSuccessfulRequests > 0 { + foundDBCost := false + + for _, m := range locality.LoadMetricStats { + if m.MetricName == "named_metrics.db_cost" { + foundDBCost = true + if got, want := m.NumRequestsFinishedWithMetric, uint64(1); got != want { + t.Errorf("db_cost NumRequestsFinishedWithMetric = %v, want %v", got, want) + } + if got, want := m.TotalMetricValue, 50.0; got != want { + t.Errorf("db_cost TotalMetricValue = %v, want %v", got, want) + } + } + if m.MetricName == "named_metrics.ignored_metric" { + t.Errorf("ignored_metric should not have been reported") + } + } + + if !foundDBCost { + t.Errorf("Expected to find named_metrics.db_cost inside load reports") + } + + if locality.CpuUtilization == nil { + t.Errorf("Expected CpuUtilization to be explicitly set but got nil") + } else { + if got, want := locality.CpuUtilization.NumRequestsFinishedWithMetric, uint64(1); got != want { + t.Errorf("CpuUtilization NumRequestsFinishedWithMetric = %v, want %v", got, want) + } + if got, want := locality.CpuUtilization.TotalMetricValue, 0.8; got != want { + t.Errorf("CpuUtilization TotalMetricValue = %v, want %v", got, want) + } + } + + if locality.MemUtilization != nil { + t.Errorf("MemUtilization metric set to %v when expected to be unset", locality.MemUtilization) + } + + return + } + } + } + } + } +} diff --git a/internal/xds/balancer/clusterimpl/tests/clusterimpl_security_test.go b/internal/xds/balancer/clusterimpl/tests/clusterimpl_security_test.go new file mode 100644 index 0000000..bbd92d7 --- /dev/null +++ b/internal/xds/balancer/clusterimpl/tests/clusterimpl_security_test.go @@ -0,0 +1,783 @@ +/* + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package clusterimpl_test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + iclusterimpl "google.golang.org/grpc/internal/xds/balancer/clusterimpl/internal" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/testdata" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/credentials/tls/certprovider/pemfile" // Register the file watcher certificate provider plugin. + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter. + _ "google.golang.org/grpc/internal/xds/resolver" // Register the xds resolver +) + +// Common setup for security tests: +// - creates an xDS client with the specified bootstrap configuration +// - creates a gRPC channel that uses the passed in client creds +// - creates a test server that uses the passed in server creds +// +// Returns the following: +// - a client channel to make RPCs +// - address of the test backend server +func setupForSecurityTests(t *testing.T, bootstrapContents []byte, clientCreds, serverCreds credentials.TransportCredentials) (*grpc.ClientConn, string) { + t.Helper() + + r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Create a ClientConn with the specified transport credentials. + cc, err := grpc.NewClient(r.Scheme()+":///test.service", grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + t.Cleanup(func() { cc.Close() }) + + // Start a test service backend with the specified transport credentials. + sOpts := []grpc.ServerOption{} + if serverCreds != nil { + sOpts = append(sOpts, grpc.Creds(serverCreds)) + } + server := stubserver.StartTestService(t, nil, sOpts...) + t.Cleanup(server.Stop) + + return cc, server.Address +} + +// Creates transport credentials to be used on the client side that rely on xDS +// to provide the security configuration. It falls back to insecure creds if no +// security information is received from the management server. +func xdsClientCredsWithInsecureFallback(t *testing.T) credentials.TransportCredentials { + t.Helper() + + xdsCreds, err := xds.NewClientCredentials(xds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("Failed to create xDS credentials: %v", err) + } + return xdsCreds +} + +// Creates transport credentials to be used on the server side from certificate +// files in testdata/x509. +// +// The certificate returned by this function has a CommonName of "test-server1". +func tlsServerCreds(t *testing.T) credentials.TransportCredentials { + t.Helper() + + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("Failed to load server cert and key: %v", err) + + } + pemData, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem")) + if err != nil { + t.Fatalf("Failed to read client CA cert: %v", err) + } + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(pemData) + cfg := &tls.Config{ + Certificates: []tls.Certificate{cert}, + ClientCAs: roots, + } + return credentials.NewTLS(cfg) +} + +// Checks the AuthInfo available in the peer if it matches the expected security +// level of the connection. +func verifySecurityInformationFromPeer(t *testing.T, pr *peer.Peer, wantSecLevel e2e.SecurityLevel) { + // This is not a true helper in the Go sense, because it does not perform + // setup or cleanup tasks. Marking it a helper is to ensure that when the + // test fails, the line information of the caller is outputted instead of + // from here. + // + // And this function directly calls t.Fatalf() instead of returning an error + // and letting the caller decide what to do with it. This is also OK since + // all callers will simply end up calling t.Fatalf() with the returned + // error, and can't add any contextual information of value to the error + // message. + t.Helper() + + switch wantSecLevel { + case e2e.SecurityLevelNone: + if pr.AuthInfo.AuthType() != "insecure" { + t.Fatalf("AuthType() is %s, want insecure", pr.AuthInfo.AuthType()) + } + case e2e.SecurityLevelMTLS: + ai, ok := pr.AuthInfo.(credentials.TLSInfo) + if !ok { + t.Fatalf("AuthInfo type is %T, want %T", pr.AuthInfo, credentials.TLSInfo{}) + } + if len(ai.State.PeerCertificates) != 1 { + t.Fatalf("Number of peer certificates is %d, want 1", len(ai.State.PeerCertificates)) + } + cert := ai.State.PeerCertificates[0] + const wantCommonName = "test-server1" + if cn := cert.Subject.CommonName; cn != wantCommonName { + t.Fatalf("Common name in peer certificate is %s, want %s", cn, wantCommonName) + } + } +} + +// makeAggregateClusterResource returns an aggregate cluster resource with the +// given name and list of child names. +func makeAggregateClusterResource(name string, childNames []string) *v3clusterpb.Cluster { + return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: name, + Type: e2e.ClusterTypeAggregate, + ChildNames: childNames, + }) +} + +// Tests the case where xDS credentials are not in use, but the cds LB policy +// receives a Cluster update with security configuration. Verifies that the +// connection between client and server is insecure. +func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) { + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create a grpc channel with insecure creds talking to a test server with + // insecure credentials. + cc, serverAddress := setupForSecurityTests(t, bc, insecure.NewCredentials(), nil) + + // Configure default resources in the management server. The + // cluster resource is configured to return security configuration. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: "test.service", + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, serverAddress), + SecLevel: e2e.SecurityLevelMTLS, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made. + peer := &peer.Peer{} + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelNone) +} + +// Tests the case where xDS credentials are in use, but the cds LB policy +// receives a Cluster update without security configuration. Verifies that the +// connection between client and server is insecure. +func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) { + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create a grpc channel with xDS creds talking to a test server with + // insecure credentials. + cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), nil) + + // Configure default resources in the management server. The + // cluster resource is not configured to return any security configuration. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: "test.service", + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, serverAddress), + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made. + peer := &peer.Peer{} + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelNone) +} + +// Tests the case where the security config returned by the management server +// cannot be resolved based on the contents of the bootstrap config. Verifies +// that the cds LB policy puts the channel in TRANSIENT_FAILURE. +func (s) TestSecurityConfigNotFoundInBootstrap(t *testing.T) { + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server, + // and one that does not have certificate providers configuration. + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create a grpc channel with xDS creds. + cc, _ := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), nil) + + // Configure a cluster resource that contains security configuration, in the + // management server. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: "test.service", + NodeID: nodeID, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelMTLS, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) +} + +// A certificate provider builder that returns a nil Provider from the starter +// func passed to certprovider.NewBuildableConfig(). +type errCertProviderBuilder struct{} + +const errCertProviderName = "err-cert-provider" + +func (e errCertProviderBuilder) ParseConfig(any) (*certprovider.BuildableConfig, error) { + // Returning a nil Provider simulates the case where an error is encountered + // at the time of building the Provider. + bc := certprovider.NewBuildableConfig(errCertProviderName, nil, func(certprovider.BuildOptions) certprovider.Provider { return nil }) + return bc, nil +} + +func (e errCertProviderBuilder) Name() string { + return errCertProviderName +} + +func init() { + certprovider.Register(errCertProviderBuilder{}) +} + +// Tests the case where the certprovider.Store returns an error when the cds LB +// policy attempts to build a certificate provider. Verifies that the cds LB +// policy puts the channel in TRANSIENT_FAILURE. +func (s) TestCertproviderStoreError(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server + // and one that includes certificate providers configuration for + // errCertProviderBuilder. + nodeID := uuid.New().String() + providerCfg := json.RawMessage(fmt.Sprintf(`{ + "plugin_name": "%s", + "config": {} + }`, errCertProviderName)) + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + CertificateProviders: map[string]json.RawMessage{e2e.ClientSideCertProviderInstance: providerCfg}, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create a grpc channel with xDS creds. + cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), nil) + + // Configure a cluster resource that contains security configuration, in the + // management server. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: "test.service", + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, serverAddress), + SecLevel: e2e.SecurityLevelMTLS, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) +} + +// Tests the case where the cds LB policy receives security configuration as +// part of the Cluster resource that can be successfully resolved using the +// bootstrap file contents. Verifies that the connection between the client and +// the server is secure. +func (s) TestGoodSecurityConfig(t *testing.T) { + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server + // and one that includes certificate providers configuration. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create a grpc channel with xDS creds talking to a test server with TLS + // credentials. + cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + + // Configure default resources in the management server. The + // cluster resource is configured to return security configuration. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: "test.service", + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, serverAddress), + SecLevel: e2e.SecurityLevelMTLS, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made over a secure connection. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS) +} + +// Tests the case where the cds LB policy receives security configuration as +// part of the Cluster resource that contains a certificate provider instance +// that is missing in the bootstrap file. Verifies that the channel moves to +// TRANSIENT_FAILURE. Subsequently, the cds LB policy receives a cluster +// resource that contains a certificate provider that is present in the +// bootstrap file. Verifies that the connection between the client and the +// server is secure. +func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) { + var ( + target = "test.service" + routeName = "route1" + clusterName = "cluster1" + serviceName = "service1" + ) + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create a grpc channel with xDS creds talking to a test server with TLS + // credentials. + cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + + // The cluster resource contains security configuration with a certificate + // provider instance that is missing in the bootstrap configuration. + cluster := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone) + cluster.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "unknown-certificate-provider-instance", + }, + }, + }, + }), + }, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})}, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with initial resources: %v", err) + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Update the management server with a Cluster resource that contains a + // certificate provider instance that is present in the bootstrap + // configuration. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelMTLS)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with valid resources: %v", err) + } + + // Verify that a successful RPC can be made over a secure connection. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS) +} + +// Tests the case where the cds LB policy receives security configuration as +// part of the Cluster resource that can be successfully resolved using the +// bootstrap file contents. Verifies that the connection between the client and +// the server is secure. Subsequently, the cds LB policy receives a cluster +// resource without security configuration. Verifies that this results in the +// use of fallback credentials, which in this case is insecure creds. +func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) { + target := "test.service" + + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create a grpc channel with xDS creds talking to a test server with TLS + // credentials. + cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + + // Configure default resources in the management server. The + // cluster resource is configured to return security configuration. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: target, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, serverAddress), + SecLevel: e2e.SecurityLevelMTLS, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made over a secure connection. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS) + + // Start a test service backend that does not expect a secure connection. + insecureServer := stubserver.StartTestService(t, nil) + t.Cleanup(insecureServer.Stop) + + // Update the resources in the management server to contain no security + // configuration. This should result in the use of fallback credentials, + // which is insecure in our case. + resources = e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: target, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, insecureServer.Address), + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the connection to move to the new backend that expects + // connections without security. + for ctx.Err() == nil { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Logf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() == insecureServer.Address { + break + } + } + if ctx.Err() != nil { + t.Fatal("Timed out when waiting for connection to switch to second backend") + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelNone) +} + +// Tests the case where the cds LB policy receives security configuration as +// part of the Cluster resource that can be successfully resolved using the +// bootstrap file contents. Verifies that the connection between the client and +// the server is secure. Subsequently, the cds LB policy receives a cluster +// resource that is NACKed by the xDS client. Test verifies that the cds LB +// policy continues to use the previous good configuration. +func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) { + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create a grpc channel with xDS creds talking to a test server with TLS + // credentials. + cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + + // Configure default resources in the management server. The + // cluster resource is configured to return security configuration. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: "test.service", + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, serverAddress), + SecLevel: e2e.SecurityLevelMTLS, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made over a secure connection. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS) + + // The cluster resource contains security configuration with a certificate + // provider instance that is missing in the bootstrap configuration. + resources.Clusters[0] = e2e.DefaultCluster(resources.Clusters[0].Name, resources.Endpoints[0].ClusterName, e2e.SecurityLevelNone) + resources.Clusters[0].TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "unknown-certificate-provider-instance", + }, + }, + }, + }), + }, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made over a secure connection. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS) +} + +// Tests the case where the cds LB policy receives security configuration as +// part of the Cluster resource that specifies the use system root certs. +// Verifies that the connection between the client and the server is secure. +func (s) TestSystemRootCertsSecurityConfig(t *testing.T) { + origFlag := envconfig.XDSSystemRootCertsEnabled + origSRCF := iclusterimpl.X509SystemCertPoolFunc + defer func() { + envconfig.XDSSystemRootCertsEnabled = origFlag + iclusterimpl.X509SystemCertPoolFunc = origSRCF + }() + envconfig.XDSSystemRootCertsEnabled = true + + systemRootCertsFuncCalled := false + iclusterimpl.X509SystemCertPoolFunc = func() (*x509.CertPool, error) { + certData, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem")) + if err != nil { + return nil, fmt.Errorf("failed to read certificate file: %w", err) + } + certPool := x509.NewCertPool() + + if ok := certPool.AppendCertsFromPEM(certData); !ok { + return nil, fmt.Errorf("failed to append certificate to cert pool") + } + systemRootCertsFuncCalled = true + return certPool, nil + } + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server + // and one that includes certificate providers configuration. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create a grpc channel with xDS creds talking to a test server with TLS + // credentials. + cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + + // Configure default resources in the management server. The cluster + // resource is configured to return security configuration. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: "test.service", + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, serverAddress), + SecLevel: e2e.SecurityLevelTLSWithSystemRootCerts, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made over a secure connection. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS) + + if systemRootCertsFuncCalled != true { + t.Errorf("System root certs were not used during the test.") + } +} + +// TestAggregateClusterSecurityConfig tests the case where the CDS LB policy +// receives a top-level aggregate cluster pointing to a child EDS cluster. The +// test verifies that the CDS LB policy correctly ignores the aggregate +// cluster's security configuration and uses the child EDS cluster's security +// configuration to establish the mTLS connection. +func (s) TestAggregateClusterSecurityConfig(t *testing.T) { + var ( + target = "test.service" + routeName = "route1" + clusterName = "cluster1" + serviceName = "service1" + edsClusterName = clusterName + "-eds" + ) + + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server + // and one that includes certificate providers configuration. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create a grpc channel with xDS creds talking to a test server with TLS + // credentials. + cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + + // Configures an aggregate cluster with a security config that is not NACKed + // by the xdsclient but would fail the handshake as it has an invalid SAN + // matcher. + aggCluster := makeAggregateClusterResource(clusterName, []string{edsClusterName}) + aggCluster.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "bad-cn"}}, + }, + }, + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + }, + }, + }, + }, + }), + }, + } + + // Create the child EDS cluster with a valid mTLS security configuration. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)}, + Clusters: []*v3clusterpb.Cluster{ + aggCluster, + e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelMTLS), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made over a secure connection since + // the EDS cluster has a valid mTLS configuration even though the top level + // cluster has an incorrect security configuration. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS) +} diff --git a/internal/xds/balancer/clustermanager/balancerstateaggregator.go b/internal/xds/balancer/clustermanager/balancerstateaggregator.go new file mode 100644 index 0000000..92c69f5 --- /dev/null +++ b/internal/xds/balancer/clustermanager/balancerstateaggregator.go @@ -0,0 +1,196 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clustermanager + +import ( + "fmt" + "sync" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/grpclog" +) + +type subBalancerState struct { + state balancer.State + // stateToAggregate is the connectivity state used only for state + // aggregation. It could be different from state.ConnectivityState. For + // example when a sub-balancer transitions from TransientFailure to + // connecting, state.ConnectivityState is Connecting, but stateToAggregate + // is still TransientFailure. + stateToAggregate connectivity.State +} + +func (s *subBalancerState) String() string { + return fmt.Sprintf("picker:%p,state:%v,stateToAggregate:%v", s.state.Picker, s.state.ConnectivityState, s.stateToAggregate) +} + +type balancerStateAggregator struct { + cc balancer.ClientConn + logger *grpclog.PrefixLogger + csEval *balancer.ConnectivityStateEvaluator + + mu sync.Mutex + // This field is used to ensure that no updates are forwarded to the parent + // CC once the aggregator is closed. A closed sub-balancer could still send + // pickers to this aggregator. + closed bool + // Map from child policy name to last reported state. + idToPickerState map[string]*subBalancerState + // Set when UpdateState call propagation is paused. + pauseUpdateState bool + // Set when UpdateState call propagation is paused and an UpdateState call + // is suppressed. + needUpdateStateOnResume bool +} + +func newBalancerStateAggregator(cc balancer.ClientConn, logger *grpclog.PrefixLogger) *balancerStateAggregator { + return &balancerStateAggregator{ + cc: cc, + logger: logger, + csEval: &balancer.ConnectivityStateEvaluator{}, + idToPickerState: make(map[string]*subBalancerState), + } +} + +func (bsa *balancerStateAggregator) close() { + bsa.mu.Lock() + defer bsa.mu.Unlock() + bsa.closed = true +} + +// add adds a sub-balancer in CONNECTING state. +// +// This is called when there's a new child. +func (bsa *balancerStateAggregator) add(id string) { + bsa.mu.Lock() + defer bsa.mu.Unlock() + + bsa.idToPickerState[id] = &subBalancerState{ + // Start everything in CONNECTING, so if one of the sub-balancers + // reports TransientFailure, the RPCs will still wait for the other + // sub-balancers. + state: balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + }, + stateToAggregate: connectivity.Connecting, + } + bsa.csEval.RecordTransition(connectivity.Shutdown, connectivity.Connecting) + bsa.buildAndUpdateLocked() +} + +// remove removes the sub-balancer state. Future updates from this sub-balancer, +// if any, will be ignored. +// +// This is called when a child is removed. +func (bsa *balancerStateAggregator) remove(id string) { + bsa.mu.Lock() + defer bsa.mu.Unlock() + if _, ok := bsa.idToPickerState[id]; !ok { + return + } + // Setting the state of the deleted sub-balancer to Shutdown will get + // csEvltr to remove the previous state for any aggregated state + // evaluations. Transitions to and from connectivity.Shutdown are ignored + // by csEvltr. + bsa.csEval.RecordTransition(bsa.idToPickerState[id].stateToAggregate, connectivity.Shutdown) + // Remove id and picker from picker map. This also results in future updates + // for this ID to be ignored. + delete(bsa.idToPickerState, id) + bsa.buildAndUpdateLocked() +} + +// pauseStateUpdates causes UpdateState calls to not propagate to the parent +// ClientConn. The last state will be remembered and propagated when +// ResumeStateUpdates is called. +func (bsa *balancerStateAggregator) pauseStateUpdates() { + bsa.mu.Lock() + defer bsa.mu.Unlock() + bsa.pauseUpdateState = true + bsa.needUpdateStateOnResume = false +} + +// resumeStateUpdates will resume propagating UpdateState calls to the parent, +// and call UpdateState on the parent if any UpdateState call was suppressed. +func (bsa *balancerStateAggregator) resumeStateUpdates() { + bsa.mu.Lock() + defer bsa.mu.Unlock() + bsa.pauseUpdateState = false + if bsa.needUpdateStateOnResume { + bsa.cc.UpdateState(bsa.buildLocked()) + } +} + +// UpdateState is called to report a balancer state change from sub-balancer. +// It's usually called by the balancer group. +// +// It calls parent ClientConn's UpdateState with the new aggregated state. +func (bsa *balancerStateAggregator) UpdateState(id string, state balancer.State) { + bsa.logger.Infof("State update from sub-balancer %q: %+v", id, state) + + bsa.mu.Lock() + defer bsa.mu.Unlock() + pickerSt, ok := bsa.idToPickerState[id] + if !ok { + // All state starts with an entry in pickStateMap. If ID is not in map, + // it's either removed, or never existed. + return + } + if !(pickerSt.state.ConnectivityState == connectivity.TransientFailure && state.ConnectivityState == connectivity.Connecting) { + // If old state is TransientFailure, and new state is Connecting, don't + // update the state, to prevent the aggregated state from being always + // CONNECTING. Otherwise, stateToAggregate is the same as + // state.ConnectivityState. + bsa.csEval.RecordTransition(pickerSt.stateToAggregate, state.ConnectivityState) + pickerSt.stateToAggregate = state.ConnectivityState + } + pickerSt.state = state + bsa.buildAndUpdateLocked() +} + +// buildAndUpdateLocked combines the sub-state from each sub-balancer into one +// state, and sends a picker update to the parent ClientConn. +func (bsa *balancerStateAggregator) buildAndUpdateLocked() { + if bsa.closed { + return + } + if bsa.pauseUpdateState { + // If updates are paused, do not call UpdateState, but remember that we + // need to call it when they are resumed. + bsa.needUpdateStateOnResume = true + return + } + bsa.cc.UpdateState(bsa.buildLocked()) +} + +// buildLocked combines sub-states into one. +func (bsa *balancerStateAggregator) buildLocked() balancer.State { + // The picker's return error might not be consistent with the + // aggregatedState. Because for this LB policy, we want to always build + // picker with all sub-pickers (not only ready sub-pickers), so even if the + // overall state is Ready, pick for certain RPCs can behave like Connecting + // or TransientFailure. + bsa.logger.Infof("Child pickers: %+v", bsa.idToPickerState) + return balancer.State{ + ConnectivityState: bsa.csEval.CurrentState(), + Picker: newPickerGroup(bsa.idToPickerState), + } +} diff --git a/internal/xds/balancer/clustermanager/clustermanager.go b/internal/xds/balancer/clustermanager/clustermanager.go new file mode 100644 index 0000000..c5300a6 --- /dev/null +++ b/internal/xds/balancer/clustermanager/clustermanager.go @@ -0,0 +1,205 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package clustermanager implements the cluster manager LB policy for xds. +package clustermanager + +import ( + "encoding/json" + "fmt" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/balancergroup" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/hierarchy" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const balancerName = "xds_cluster_manager_experimental" + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + b := &bal{} + b.logger = prefixLogger(b) + b.stateAggregator = newBalancerStateAggregator(cc, b.logger) + b.bg = balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: opts, + StateAggregator: b.stateAggregator, + Logger: b.logger, + SubBalancerCloseTimeout: time.Duration(0), // Disable caching of removed child policies + }) + b.logger.Infof("Created") + return b +} + +func (bb) Name() string { + return balancerName +} + +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(c) +} + +type bal struct { + logger *internalgrpclog.PrefixLogger + bg *balancergroup.BalancerGroup + stateAggregator *balancerStateAggregator + + children map[string]childConfig +} + +func (b *bal) setErrorPickerForChild(childName string, err error) { + b.stateAggregator.UpdateState(childName, balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(err), + }) +} + +func (b *bal) updateChildren(s balancer.ClientConnState, newConfig *lbConfig) error { + endpointsSplit := hierarchy.Group(s.ResolverState.Endpoints) + + // Remove sub-balancers that are not in the new list from the aggregator and + // balancergroup. + for name := range b.children { + if _, ok := newConfig.Children[name]; !ok { + b.stateAggregator.remove(name) + b.bg.Remove(name) + } + } + + var retErr error + for childName, childCfg := range newConfig.Children { + lbCfg := childCfg.ChildPolicy.Config + if _, ok := b.children[childName]; !ok { + // Add new sub-balancers to the aggregator and balancergroup. + b.stateAggregator.add(childName) + b.bg.Add(childName, balancer.Get(childCfg.ChildPolicy.Name)) + } else { + // If the child policy type has changed for existing sub-balancers, + // parse the new config and send down the config update to the + // balancergroup, which will take care of gracefully switching the + // child over to the new policy. + // + // If we run into errors here, we need to ensure that RPCs to this + // child fail, while RPCs to other children with good configs + // continue to succeed. + newPolicyName, oldPolicyName := childCfg.ChildPolicy.Name, b.children[childName].ChildPolicy.Name + if newPolicyName != oldPolicyName { + var err error + var cfgJSON []byte + cfgJSON, err = childCfg.ChildPolicy.MarshalJSON() + if err != nil { + retErr = fmt.Errorf("failed to JSON marshal load balancing policy for child %q: %v", childName, err) + b.setErrorPickerForChild(childName, retErr) + continue + } + // This overwrites lbCfg to be in the format expected by the + // gracefulswitch balancer. So, when this config is pushed to + // the child (below), it will result in a graceful switch to the + // new child policy. + lbCfg, err = balancergroup.ParseConfig(cfgJSON) + if err != nil { + retErr = fmt.Errorf("failed to parse load balancing policy for child %q: %v", childName, err) + b.setErrorPickerForChild(childName, retErr) + continue + } + } + } + + if err := b.bg.UpdateClientConnState(childName, balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: endpointsSplit[childName], + ServiceConfig: s.ResolverState.ServiceConfig, + Attributes: s.ResolverState.Attributes, + }, + BalancerConfig: lbCfg, + }); err != nil { + retErr = fmt.Errorf("failed to push new configuration %v to child %q: %v", childCfg.ChildPolicy.Config, childName, err) + b.setErrorPickerForChild(childName, retErr) + } + + // Picker update is sent to the parent ClientConn only after the + // new child policy returns a picker. So, there is no need to + // set needUpdateStateOnResume to true here. + } + + b.children = newConfig.Children + + // If multiple sub-balancers run into errors, we will return only the last + // one, which is still good enough, since the grpc channel will anyways + // return this error as balancer.ErrBadResolver to the name resolver, + // resulting in re-resolution attempts. + return retErr + + // Adding or removing a sub-balancer will result in the + // needUpdateStateOnResume bit to true which results in a picker update once + // resumeStateUpdates() is called. +} + +func (b *bal) UpdateClientConnState(s balancer.ClientConnState) error { + if b.logger.V(2) { + b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig)) + } + + newConfig, ok := s.BalancerConfig.(*lbConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) + } + + b.stateAggregator.pauseStateUpdates() + defer b.stateAggregator.resumeStateUpdates() + return b.updateChildren(s, newConfig) +} + +func (b *bal) ResolverError(err error) { + b.bg.ResolverError(err) +} + +func (b *bal) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *bal) Close() { + b.stateAggregator.close() + b.bg.Close() + b.logger.Infof("Shutdown") +} + +func (b *bal) ExitIdle() { + b.bg.ExitIdle() +} + +const prefix = "[xds-cluster-manager-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *bal) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/internal/xds/balancer/clustermanager/clustermanager_test.go b/internal/xds/balancer/clustermanager/clustermanager_test.go new file mode 100644 index 0000000..3051845 --- /dev/null +++ b/internal/xds/balancer/clustermanager/clustermanager_test.go @@ -0,0 +1,775 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clustermanager + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/hierarchy" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + testBackendAddrsCount = 12 +) + +var testBackendAddrStrs []string + +func init() { + for i := 0; i < testBackendAddrsCount; i++ { + testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)) + } +} + +func testPick(t *testing.T, p balancer.Picker, info balancer.PickInfo, wantSC balancer.SubConn, wantErr error) { + t.Helper() + for i := 0; i < 5; i++ { + gotSCSt, err := p.Pick(info) + if fmt.Sprint(err) != fmt.Sprint(wantErr) { + t.Fatalf("picker.Pick(%+v), got error %v, want %v", info, err, wantErr) + } + if gotSCSt.SubConn != wantSC { + t.Fatalf("picker.Pick(%+v), got %v, want SubConn=%v", info, gotSCSt, wantSC) + } + } +} + +func (s) TestClusterPicks(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, balancer.BuildOptions{}) + + configJSON1 := `{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"round_robin":""}] }, + "cds:cluster_2":{ "childPolicy": [{"round_robin":""}] } +} +}` + config1, err := parser.ParseConfig([]byte(configJSON1)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0], BalancerAttributes: nil}, + {Addr: testBackendAddrStrs[1], BalancerAttributes: nil}, + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + m1 := make(map[resolver.Address]balancer.SubConn) + // Verify that a subconn is created with the address. + for range wantAddrs { + addrs := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + // Clear the attributes before adding to map. + addrs[0].BalancerAttributes = nil + m1[addrs[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + p1 := <-cc.NewPickerCh + for _, tt := range []struct { + pickInfo balancer.PickInfo + wantSC balancer.SubConn + wantErr error + }{ + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:cluster_1"), + }, + wantSC: m1[wantAddrs[0]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:cluster_2"), + }, + wantSC: m1[wantAddrs[1]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "notacluster"), + }, + wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "notacluster"`), + }, + } { + testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr) + } +} + +// TestConfigUpdateAddCluster covers the cases the balancer receives config +// update with extra clusters. +func (s) TestConfigUpdateAddCluster(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, balancer.BuildOptions{}) + + configJSON1 := `{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"round_robin":""}] }, + "cds:cluster_2":{ "childPolicy": [{"round_robin":""}] } +} +}` + config1, err := parser.ParseConfig([]byte(configJSON1)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0], BalancerAttributes: nil}, + {Addr: testBackendAddrStrs[1], BalancerAttributes: nil}, + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + m1 := make(map[resolver.Address]balancer.SubConn) + // Verify that a subconn is created with the address. + for range wantAddrs { + addrs := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + // Clear the attributes before adding to map. + addrs[0].BalancerAttributes = nil + m1[addrs[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + p1 := <-cc.NewPickerCh + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for _, tt := range []struct { + pickInfo balancer.PickInfo + wantSC balancer.SubConn + wantErr error + }{ + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:cluster_1"), + }, + wantSC: m1[wantAddrs[0]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:cluster_2"), + }, + wantSC: m1[wantAddrs[1]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:notacluster"), + }, + wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`), + }, + } { + testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr) + } + + // A config update with different routes, and different actions. Expect a + // new subconn and a picker update. + configJSON2 := `{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"round_robin":""}] }, + "cds:cluster_2":{ "childPolicy": [{"round_robin":""}] }, + "cds:cluster_3":{ "childPolicy": [{"round_robin":""}] } +} +}` + config2, err := parser.ParseConfig([]byte(configJSON2)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + wantAddrs = append(wantAddrs, resolver.Address{Addr: testBackendAddrStrs[2], BalancerAttributes: nil}) + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[2]}}, []string{"cds:cluster_3"}), + }}, + BalancerConfig: config2, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Expect exactly one new subconn. + addrs := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + // Clear the attributes before adding to map. + addrs[0].BalancerAttributes = nil + m1[addrs[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Should have no more newSubConn. + select { + case <-time.After(time.Millisecond * 500): + case <-cc.NewSubConnCh: + addrs := <-cc.NewSubConnAddrsCh + t.Fatalf("unexpected NewSubConn with address %v", addrs) + } + + p2 := <-cc.NewPickerCh + for _, tt := range []struct { + pickInfo balancer.PickInfo + wantSC balancer.SubConn + wantErr error + }{ + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:cluster_1"), + }, + wantSC: m1[wantAddrs[0]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:cluster_2"), + }, + wantSC: m1[wantAddrs[1]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:cluster_3"), + }, + wantSC: m1[wantAddrs[2]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:notacluster"), + }, + wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`), + }, + } { + testPick(t, p2, tt.pickInfo, tt.wantSC, tt.wantErr) + } +} + +// TestRoutingConfigUpdateDeleteAll covers the cases the balancer receives +// config update with no clusters. Pick should fail with details in error. +func (s) TestRoutingConfigUpdateDeleteAll(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, balancer.BuildOptions{}) + + configJSON1 := `{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"round_robin":""}] }, + "cds:cluster_2":{ "childPolicy": [{"round_robin":""}] } +} +}` + config1, err := parser.ParseConfig([]byte(configJSON1)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0], BalancerAttributes: nil}, + {Addr: testBackendAddrStrs[1], BalancerAttributes: nil}, + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + m1 := make(map[resolver.Address]balancer.SubConn) + // Verify that a subconn is created with the address, and the hierarchy path + // in the address is cleared. + for range wantAddrs { + addrs := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + // Clear the attributes before adding to map. + addrs[0].BalancerAttributes = nil + m1[addrs[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + p1 := <-cc.NewPickerCh + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for _, tt := range []struct { + pickInfo balancer.PickInfo + wantSC balancer.SubConn + wantErr error + }{ + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:cluster_1"), + }, + wantSC: m1[wantAddrs[0]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:cluster_2"), + }, + wantSC: m1[wantAddrs[1]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:notacluster"), + }, + wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`), + }, + } { + testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr) + } + + // A config update with no clusters. + configJSON2 := `{}` + config2, err := parser.ParseConfig([]byte(configJSON2)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: config2, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Expect two removed subconns. + for range wantAddrs { + select { + case <-time.After(time.Millisecond * 500): + t.Fatalf("timeout waiting for remove subconn") + case <-cc.ShutdownSubConnCh: + } + } + + p2 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, err := p2.Pick(balancer.PickInfo{Ctx: SetPickedCluster(ctx, "cds:notacluster")}) + if fmt.Sprint(err) != status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`).Error() { + t.Fatalf("picker.Pick, got %v, %v, want error %v", gotSCSt, err, `unknown cluster selected for RPC: "cds:notacluster"`) + } + } + + // Resend the previous config with clusters + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + m2 := make(map[resolver.Address]balancer.SubConn) + // Verify that a subconn is created with the address, and the hierarchy path + // in the address is cleared. + for range wantAddrs { + addrs := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + // Clear the attributes before adding to map. + addrs[0].BalancerAttributes = nil + m2[addrs[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + p3 := <-cc.NewPickerCh + for _, tt := range []struct { + pickInfo balancer.PickInfo + wantSC balancer.SubConn + wantErr error + }{ + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:cluster_1"), + }, + wantSC: m2[wantAddrs[0]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:cluster_2"), + }, + wantSC: m2[wantAddrs[1]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(ctx, "cds:notacluster"), + }, + wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`), + }, + } { + testPick(t, p3, tt.pickInfo, tt.wantSC, tt.wantErr) + } +} + +func (s) TestClusterManagerForwardsBalancerBuildOptions(t *testing.T) { + const ( + userAgent = "ua" + defaultTestTimeout = 1 * time.Second + ) + + // Setup the stub balancer such that we can read the build options passed to + // it in the UpdateClientConnState method. + ccsCh := testutils.NewChannel() + bOpts := balancer.BuildOptions{ + DialCreds: insecure.NewCredentials(), + CustomUserAgent: userAgent, + } + stub.Register(t.Name(), stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + if !cmp.Equal(bd.BuildOptions, bOpts) { + err := fmt.Errorf("buildOptions in child balancer: %v, want %v", bd, bOpts) + ccsCh.Send(err) + return err + } + ccsCh.Send(nil) + return nil + }, + }) + + cc := testutils.NewBalancerClientConn(t) + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, bOpts) + + configJSON1 := fmt.Sprintf(`{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"%s":""}] } +} +}`, t.Name()) + config1, err := parser.ParseConfig([]byte(configJSON1)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + if err := bal.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: config1}); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + v, err := ccsCh.Receive(ctx) + if err != nil { + t.Fatalf("timed out waiting for UpdateClientConnState result: %v", err) + } + if v != nil { + t.Fatal(v) + } +} + +const initIdleBalancerName = "test-init-idle-balancer" + +var errTestInitIdle = fmt.Errorf("init Idle balancer error 0") + +func init() { + stub.Register(initIdleBalancerName, stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, opts balancer.ClientConnState) error { + sc, err := bd.ClientConn.NewSubConn(opts.ResolverState.Addresses, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + err := fmt.Errorf("wrong picker error") + if state.ConnectivityState == connectivity.Idle { + err = errTestInitIdle + } + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &testutils.TestConstPicker{Err: err}, + }) + }, + }) + if err != nil { + return err + } + sc.Connect() + return nil + }, + }) +} + +// TestInitialIdle covers the case that if the child reports Idle, the overall +// state will be Idle. +func (s) TestInitialIdle(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, balancer.BuildOptions{}) + + configJSON1 := `{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"test-init-idle-balancer":""}] } +} +}` + config1, err := parser.ParseConfig([]byte(configJSON1)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0], BalancerAttributes: nil}, + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Verify that a subconn is created with the address, and the hierarchy path + // in the address is cleared. + for range wantAddrs { + sc := <-cc.NewSubConnCh + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + } + + if state1 := <-cc.NewStateCh; state1 != connectivity.Idle { + t.Fatalf("Received aggregated state: %v, want Idle", state1) + } +} + +// TestClusterGracefulSwitch tests the graceful switch functionality for a child +// of the cluster_manager. The test performs the following steps: +// 1. Configures a single child for the cluster_manager with a round_robin LB +// policy. Ensures the child sends a picker update when the subchannel moves +// to READY. +// 2. Updates the exiting child with a different LB policy (pick_first), and a +// different endpoint. This should trigger the graceful switch process, and +// cluster_manager should not send a picker update until the child has +// completed the graceful switch process. +// 3. The child should complete the graceful switch process once it receives an +// update that moves its subchannel to READY. At this point, the cluster_manager +// should send a picker update reflecting the new pick_first balancer's picker. +func (s) TestClusterGracefulSwitch(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, balancer.BuildOptions{}) + defer bal.Close() + + // Push round_robin config with one endpoint to the cluster_manager. + rrConfigJSON := ` + { + "children": { + "csp:cluster":{ "childPolicy": [{"round_robin": {}}] } + } + }` + rrConfig, err := parser.ParseConfig([]byte(rrConfigJSON)) + if err != nil { + t.Fatalf("Failed to parse round_robin config: %v", err) + } + rrEndpoint := hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "rr-backend"}}}, []string{"csp:cluster"}) + ccs := balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{rrEndpoint}}, + BalancerConfig: rrConfig, + } + if err := bal.UpdateClientConnState(ccs); err != nil { + t.Fatalf("Failed to send round_robin config update to cluster_manager: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Wait for a subchannel to be created. + var sc1 *testutils.TestSubConn + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for subchannel to be created by the round_robin LB policy") + case sc1 = <-cc.NewSubConnCh: + } + + // Update the subchannel's state to READY, and ensure we get a picker update. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + var p1 balancer.Picker + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for picker update from cluster_manager LB policy") + case p1 = <-cc.NewPickerCh: + } + + // Verify the picker picks the expected subchannel. + pi := balancer.PickInfo{Ctx: SetPickedCluster(ctx, "csp:cluster")} + testPick(t, p1, pi, sc1, nil) + + // Update the config to use pick_first, and a different endpoint. + pfConfigJSON := ` + { + "children": { + "csp:cluster":{ "childPolicy": [{"pick_first": {}}] } + } + }` + pfConfig, err := parser.ParseConfig([]byte(pfConfigJSON)) + if err != nil { + t.Fatalf("Failed to parse pick_first config: %v", err) + } + pfEndpoint := hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "pf-backend"}}}, []string{"csp:cluster"}) + ccs = balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{pfEndpoint}}, + BalancerConfig: pfConfig, + } + if err := bal.UpdateClientConnState(ccs); err != nil { + t.Fatalf("Failed to send pick_first config update to cluster_manager: %v", err) + } + + // Wait for a subchannel to be created. + var sc2 *testutils.TestSubConn + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for subchannel to be created by the pick_first LB policy") + case sc2 = <-cc.NewSubConnCh: + } + + // Move the pick_first LB policy's subchannel to CONNECTING. This will cause + // the pick_first LB policy to send a connecting picker. But this should not + // result in the cluster_manager sending a picker update to the channel as + // the graceful switch process is not yet complete. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + select { + case <-cc.NewPickerCh: + t.Fatalf("Received picker from cluster_manager when none expected during graceful switch process") + case <-sCtx.Done(): + } + + // Move the pick_first LB policy's subchannel to READY. This will cause the + // pick_first LB policy to send a ready picker and complete the graceful + // switch process. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + var p2 balancer.Picker + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for picker update from cluster_manager LB policy") + case p2 = <-cc.NewPickerCh: + } + + // Verify the picker picks the expected subchannel. + testPick(t, p2, pi, sc2, nil) + + // Completion of the graceful switch process should result in the old + // subchannel being removed. + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for subchannels belonging to old child to be deleted") + case rsc := <-cc.ShutdownSubConnCh: + if rsc != sc1 { + t.Fatalf("Unexpected subchannel %v being shutdown, want %v", rsc, sc1) + } + } +} + +// tcc wraps a testutils.TestClientConn but stores all state transitions in a +// slice. +type tcc struct { + *testutils.BalancerClientConn + states []balancer.State +} + +func (t *tcc) UpdateState(bs balancer.State) { + t.states = append(t.states, bs) + t.BalancerClientConn.UpdateState(bs) +} + +func (s) TestUpdateStatePauses(t *testing.T) { + cc := &tcc{BalancerClientConn: testutils.NewBalancerClientConn(t)} + + balFuncs := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: nil}) + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: nil}) + return nil + }, + } + stub.Register("update_state_balancer", balFuncs) + + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, balancer.BuildOptions{}) + defer bal.Close() + + configJSON1 := `{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"update_state_balancer":""}] } +} +}` + config1, err := parser.ParseConfig([]byte(configJSON1)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0], BalancerAttributes: nil}, + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Verify that the only state update is the second one called by the child. + if len(cc.states) != 1 || cc.states[0].ConnectivityState != connectivity.Ready { + t.Fatalf("cc.states = %v; want [connectivity.Ready]", cc.states) + } +} diff --git a/internal/xds/balancer/clustermanager/config.go b/internal/xds/balancer/clustermanager/config.go new file mode 100644 index 0000000..3a5625f --- /dev/null +++ b/internal/xds/balancer/clustermanager/config.go @@ -0,0 +1,46 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clustermanager + +import ( + "encoding/json" + + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +type childConfig struct { + // ChildPolicy is the child policy and it's config. + ChildPolicy *internalserviceconfig.BalancerConfig +} + +// lbConfig is the balancer config for xds routing policy. +type lbConfig struct { + serviceconfig.LoadBalancingConfig + Children map[string]childConfig +} + +func parseConfig(c json.RawMessage) (*lbConfig, error) { + cfg := &lbConfig{} + if err := json.Unmarshal(c, cfg); err != nil { + return nil, err + } + + return cfg, nil +} diff --git a/internal/xds/balancer/clustermanager/config_test.go b/internal/xds/balancer/clustermanager/config_test.go new file mode 100644 index 0000000..fd815df --- /dev/null +++ b/internal/xds/balancer/clustermanager/config_test.go @@ -0,0 +1,144 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clustermanager + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + _ "google.golang.org/grpc/balancer/weightedtarget" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + _ "google.golang.org/grpc/internal/xds/balancer/cdsbalancer" +) + +const ( + testJSONConfig = `{ + "children":{ + "cds:cluster_1":{ + "childPolicy":[{ + "cds_experimental":{"cluster":"cluster_1"} + }] + }, + "weighted:cluster_1_cluster_2_1":{ + "childPolicy":[{ + "weighted_target_experimental":{ + "targets": { + "cluster_1" : { + "weight":75, + "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] + }, + "cluster_2" : { + "weight":25, + "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] + } + } + } + }] + }, + "weighted:cluster_1_cluster_3_1":{ + "childPolicy":[{ + "weighted_target_experimental":{ + "targets": { + "cluster_1": { + "weight":99, + "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] + }, + "cluster_3": { + "weight":1, + "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}] + } + } + } + }] + } + } +} +` + + cdsName = "cds_experimental" + wtName = "weighted_target_experimental" +) + +var ( + cdsConfigParser = balancer.Get(cdsName).(balancer.ConfigParser) + cdsConfigJSON1 = `{"cluster":"cluster_1"}` + cdsConfig1, _ = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON1)) + + wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser) + wtConfigJSON1 = `{ + "targets": { + "cluster_1" : { "weight":75, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] }, + "cluster_2" : { "weight":25, "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] } + } }` + wtConfig1, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON1)) + wtConfigJSON2 = `{ + "targets": { + "cluster_1": { "weight":99, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] }, + "cluster_3": { "weight":1, "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}] } + } }` + wtConfig2, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON2)) +) + +func (s) Test_parseConfig(t *testing.T) { + tests := []struct { + name string + js string + want *lbConfig + wantErr bool + }{ + { + name: "empty json", + js: "", + want: nil, + wantErr: true, + }, + { + name: "OK", + js: testJSONConfig, + want: &lbConfig{ + Children: map[string]childConfig{ + "cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: cdsName, Config: cdsConfig1}, + }, + "weighted:cluster_1_cluster_2_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: wtName, Config: wtConfig1}, + }, + "weighted:cluster_1_cluster_3_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: wtName, Config: wtConfig2}, + }, + }, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseConfig([]byte(tt.js)) + if (err != nil) != tt.wantErr { + t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(got, tt.want, cmp.AllowUnexported(lbConfig{})); d != "" { + t.Errorf("parseConfig() got unexpected result, diff: %v", d) + } + }) + } +} diff --git a/internal/xds/balancer/clustermanager/e2e_test/clustermanager_test.go b/internal/xds/balancer/clustermanager/e2e_test/clustermanager_test.go new file mode 100644 index 0000000..a55c9c5 --- /dev/null +++ b/internal/xds/balancer/clustermanager/e2e_test/clustermanager_test.go @@ -0,0 +1,342 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e_test + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" + + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3" + "github.com/google/uuid" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/xds" // Register the xDS name resolver and related LB policies. +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +func makeEmptyCallRPCAndVerifyPeer(ctx context.Context, client testgrpc.TestServiceClient, wantPeer string) error { + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + return fmt.Errorf("EmptyCall() failed: %v", err) + } + if gotPeer := peer.Addr.String(); gotPeer != wantPeer { + return fmt.Errorf("EmptyCall() routed to %q, want to be routed to: %q", gotPeer, wantPeer) + } + return nil +} + +func makeUnaryCallRPCAndVerifyPeer(ctx context.Context, client testgrpc.TestServiceClient, wantPeer string) error { + peer := &peer.Peer{} + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil { + return fmt.Errorf("UnaryCall() failed: %v", err) + } + if gotPeer := peer.Addr.String(); gotPeer != wantPeer { + return fmt.Errorf("EmptyCall() routed to %q, want to be routed to: %q", gotPeer, wantPeer) + } + return nil +} + +func (s) TestConfigUpdate_ChildPolicyChange(t *testing.T) { + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + testutils.CreateBootstrapFileForTesting(t, bc) + + // Create an xDS resolver with the above bootstrap configuration. + var resolverBuilder resolver.Builder + var err error + if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { + resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } + + // Configure client side xDS resources on the management server. + const ( + serviceName = "my-service-client-side-xds" + routeConfigName = "route-" + serviceName + clusterName1 = "cluster1-" + serviceName + clusterName2 = "cluster2-" + serviceName + endpointsName1 = "endpoints1-" + serviceName + endpointsName2 = "endpoints2-" + serviceName + endpointsName3 = "endpoints3-" + serviceName + ) + // A single Listener resource pointing to the following Route + // configuration: + // - "/grpc.testing.TestService/EmptyCall" --> cluster1 + // - "/grpc.testing.TestService/UnaryCall" --> cluster2 + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)} + routes := []*v3routepb.RouteConfiguration{{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}, + }}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}, + }}, + }, + }, + }}, + }} + // Two cluster resources corresponding to the ones mentioned in the above + // route configuration resource. These are configured with round_robin as + // their endpoint picking policy. + clusters := []*v3clusterpb.Cluster{ + e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelNone), + e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelNone), + } + // Spin up two test backends, one for each cluster below. + server1 := stubserver.StartTestService(t, nil) + defer server1.Stop() + server2 := stubserver.StartTestService(t, nil) + defer server2.Stop() + // Two endpoints resources, each with one backend from above. + endpoints := []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}), + e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}), + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Make an EmptyCall RPC and verify that it is routed to cluster1. + client := testgrpc.NewTestServiceClient(cc) + if err := makeEmptyCallRPCAndVerifyPeer(ctx, client, server1.Address); err != nil { + t.Fatal(err) + } + + // Make a UnaryCall RPC and verify that it is routed to cluster2. + if err := makeUnaryCallRPCAndVerifyPeer(ctx, client, server2.Address); err != nil { + t.Fatal(err) + } + + // Create a wrapped pickfirst LB policy. When the endpoint picking policy on + // the cluster resource is changed to pickfirst, this will allow us to + // verify that load balancing configuration is pushed to it. + pfBuilder := balancer.Get(pickfirst.Name) + internal.BalancerUnregister(pfBuilder.Name()) + + lbCfgCh := make(chan serviceconfig.LoadBalancingConfig, 1) + stub.Register(pfBuilder.Name(), stub.BalancerFuncs{ + ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return pfBuilder.(balancer.ConfigParser).ParseConfig(lbCfg) + }, + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = pfBuilder.Build(bd.ClientConn, bd.BuildOptions) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + select { + case lbCfgCh <- ccs.BalancerConfig: + default: + } + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + }) + + // Send a config update that changes the child policy configuration for one + // of the clusters to pickfirst. The endpoints resource is also changed here + // to ensure that we can verify that the new child policy + cluster2 := &v3clusterpb.Cluster{ + Name: clusterName2, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: endpointsName3, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3pickfirstpb.PickFirst{ + ShuffleAddressList: true, + }), + }, + }, + }, + }, + } + server3 := stubserver.StartTestService(t, nil) + defer server3.Stop() + endpoints3 := e2e.DefaultEndpoint(endpointsName3, "localhost", []uint32{testutils.ParsePort(t, server3.Address)}) + resources.Clusters = append(resources.Clusters, cluster2) + resources.Endpoints = append(resources.Endpoints, endpoints3) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + select { + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for configuration to be pushed to the new pickfirst child policy") + case <-lbCfgCh: + } + + // Ensure RPCs are still succeeding. + + // Make an EmptyCall RPC and verify that it is routed to cluster1. + if err := makeEmptyCallRPCAndVerifyPeer(ctx, client, server1.Address); err != nil { + t.Fatal(err) + } + + // Make a UnaryCall RPC and verify that it is routed to cluster2, and the + // new endpoints resource. + for ; ctx.Err() != nil; <-time.After(defaultTestShortTimeout) { + if err := makeUnaryCallRPCAndVerifyPeer(ctx, client, server3.Address); err == nil { + break + } + t.Log(err) + } + if ctx.Err() != nil { + t.Fatal("Timeout when waiting for RPCs to cluster2 to be routed to the new endpoints resource") + } + + // Send a config update that changes the child policy configuration for one + // of the clusters to an unsupported LB policy. This should result in + // failure of RPCs to that cluster. + cluster2 = &v3clusterpb.Cluster{ + Name: clusterName2, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: endpointsName3, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + // The type not registered in gRPC Policy registry. + TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/myorg.ThisTypeDoesNotExist", + Value: &structpb.Struct{}, + }), + }, + }, + }, + }, + } + resources.Clusters[1] = cluster2 + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // At this point, RPCs to cluster1 should continue to succeed, while RPCs to + // cluster2 should start to fail. + + // Make an EmptyCall RPC and verify that it is routed to cluster1. + if err := makeEmptyCallRPCAndVerifyPeer(ctx, client, server1.Address); err != nil { + t.Fatal(err) + } + + // Make a UnaryCall RPC and verify that it starts to fail. + for ; ctx.Err() != nil; <-time.After(defaultTestShortTimeout) { + _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}) + got := status.Code(err) + if got == codes.Unavailable { + break + } + t.Logf("UnaryCall() returned code: %v, want %v", got, codes.Unavailable) + } + if ctx.Err() != nil { + t.Fatal("Timeout when waiting for RPCs to cluster2 to start failing") + } + + // Channel should still be READY. + if got, want := cc.GetState(), connectivity.Ready; got != want { + t.Fatalf("grpc.ClientConn in state %v, want %v", got, want) + } +} diff --git a/internal/xds/balancer/clustermanager/picker.go b/internal/xds/balancer/clustermanager/picker.go new file mode 100644 index 0000000..97a1670 --- /dev/null +++ b/internal/xds/balancer/clustermanager/picker.go @@ -0,0 +1,65 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clustermanager + +import ( + "context" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// pickerGroup contains a list of pickers. If the picker isn't ready, the pick +// will be queued. +type pickerGroup struct { + pickers map[string]balancer.Picker +} + +func newPickerGroup(idToPickerState map[string]*subBalancerState) *pickerGroup { + pickers := make(map[string]balancer.Picker) + for id, st := range idToPickerState { + pickers[id] = st.state.Picker + } + return &pickerGroup{ + pickers: pickers, + } +} + +func (pg *pickerGroup) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + cluster := PickedCluster(info.Ctx) + if p := pg.pickers[cluster]; p != nil { + return p.Pick(info) + } + return balancer.PickResult{}, status.Errorf(codes.Unavailable, "unknown cluster selected for RPC: %q", cluster) +} + +type clusterKey struct{} + +// PickedCluster returns the cluster name stored in the context. +func PickedCluster(ctx context.Context) string { + cluster, _ := ctx.Value(clusterKey{}).(string) + return cluster +} + +// SetPickedCluster adds the selected cluster to the context for the +// xds_cluster_manager LB policy to pick. +func SetPickedCluster(ctx context.Context, cluster string) context.Context { + return context.WithValue(ctx, clusterKey{}, cluster) +} diff --git a/internal/xds/balancer/loadstore/load_store_wrapper.go b/internal/xds/balancer/loadstore/load_store_wrapper.go new file mode 100644 index 0000000..89d5ad8 --- /dev/null +++ b/internal/xds/balancer/loadstore/load_store_wrapper.go @@ -0,0 +1,128 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package loadstore contains the loadStoreWrapper shared by the balancers. +package loadstore + +import ( + "sync" + + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/lrsclient" +) + +// NewWrapper creates a Wrapper. +func NewWrapper() *Wrapper { + return &Wrapper{} +} + +// Wrapper wraps a load store with cluster and edsService. +// +// It's store and cluster/edsService can be updated separately. And it will +// update its internal perCluster store so that new stats will be added to the +// correct perCluster. +// +// Note that this struct is a temporary workaround before we implement graceful +// switch for EDS. Any update to the clusterName and serviceName is too early, +// the perfect timing is when the picker is updated with the new connection. +// This early update could cause picks for the old SubConn being reported to the +// new services. +// +// When the graceful switch in EDS is done, there should be no need for this +// struct. The policies that record/report load shouldn't need to handle update +// of lrsServerName/cluster/edsService. Its parent should do a graceful switch +// of the whole tree when one of that changes. +type Wrapper struct { + mu sync.RWMutex + cluster string + edsService string + // store and perCluster are initialized as nil. They are only set by the + // balancer when LRS is enabled. Before that, all functions to record loads + // are no-op. + store *lrsclient.LoadStore + perCluster *lrsclient.PerClusterReporter +} + +// UpdateClusterAndService updates the cluster name and eds service for this +// wrapper. If any one of them is changed from before, the perCluster store in +// this wrapper will also be updated. +func (lsw *Wrapper) UpdateClusterAndService(cluster, edsService string) { + lsw.mu.Lock() + defer lsw.mu.Unlock() + if cluster == lsw.cluster && edsService == lsw.edsService { + return + } + lsw.cluster = cluster + lsw.edsService = edsService + if lsw.store == nil { + return + } + lsw.perCluster = lsw.store.ReporterForCluster(lsw.cluster, lsw.edsService) +} + +// UpdateLoadStore updates the load store for this wrapper. If it is changed +// from before, the perCluster store in this wrapper will also be updated. +func (lsw *Wrapper) UpdateLoadStore(store *lrsclient.LoadStore) { + lsw.mu.Lock() + defer lsw.mu.Unlock() + if store == lsw.store { + return + } + lsw.store = store + if lsw.store == nil { + lsw.perCluster = nil + return + } + lsw.perCluster = lsw.store.ReporterForCluster(lsw.cluster, lsw.edsService) +} + +// CallStarted records a call started in the store. +func (lsw *Wrapper) CallStarted(locality clients.Locality) { + lsw.mu.RLock() + defer lsw.mu.RUnlock() + if lsw.perCluster != nil { + lsw.perCluster.CallStarted(locality) + } +} + +// CallFinished records a call finished in the store. +func (lsw *Wrapper) CallFinished(locality clients.Locality, err error) { + lsw.mu.RLock() + defer lsw.mu.RUnlock() + if lsw.perCluster != nil { + lsw.perCluster.CallFinished(locality, err) + } +} + +// CallServerLoad records the server load in the store. +func (lsw *Wrapper) CallServerLoad(locality clients.Locality, name string, val float64) { + lsw.mu.RLock() + defer lsw.mu.RUnlock() + if lsw.perCluster != nil { + lsw.perCluster.CallServerLoad(locality, name, val) + } +} + +// CallDropped records a call dropped in the store. +func (lsw *Wrapper) CallDropped(category string) { + lsw.mu.RLock() + defer lsw.mu.RUnlock() + if lsw.perCluster != nil { + lsw.perCluster.CallDropped(category) + } +} diff --git a/internal/xds/balancer/outlierdetection/balancer.go b/internal/xds/balancer/outlierdetection/balancer.go new file mode 100644 index 0000000..0904c5d --- /dev/null +++ b/internal/xds/balancer/outlierdetection/balancer.go @@ -0,0 +1,927 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package outlierdetection provides an implementation of the outlier detection +// LB policy, as defined in +// https://github.com/grpc/proposal/blob/master/A50-xds-outlier-detection.md. +package outlierdetection + +import ( + "encoding/json" + "fmt" + "math" + rand "math/rand/v2" + "strings" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal/balancer/gracefulswitch" + "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Globals to stub out in tests. +var ( + afterFunc = time.AfterFunc + now = time.Now +) + +// Name is the name of the outlier detection balancer. +const Name = "outlier_detection_experimental" + +var ( + ejectionsEnforcedMetric = estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "grpc.lb.outlier_detection.ejections_enforced", + Description: "EXPERIMENTAL. Number of outlier ejections enforced by detection method", + Unit: "{ejection}", + Labels: []string{"grpc.target", "grpc.lb.outlier_detection.detection_method"}, + Default: false, + }) + + ejectionsUnenforcedMetric = estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "grpc.lb.outlier_detection.ejections_unenforced", + Description: "EXPERIMENTAL. Number of unenforced outlier ejections due to either `max_ejection_percentage` or `enforcement_percentage`", + Unit: "{ejection}", + Labels: []string{"grpc.target", "grpc.lb.outlier_detection.detection_method", "grpc.lb.outlier_detection.unenforced_reason"}, + Default: false, + }) +) + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &outlierDetectionBalancer{ + ClientConn: cc, + closed: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + addrs: make(map[string]*endpointInfo), + scUpdateCh: buffer.NewUnbounded[any](), + pickerUpdateCh: buffer.NewUnbounded[any](), + channelzParent: bOpts.ChannelzParent, + endpoints: resolver.NewEndpointMap[*endpointInfo](), + metricsRecorder: cc.MetricsRecorder(), // we use an explicit field instead of using cc.MetricsRecorder() so we can override the metric recorder in tests. + target: bOpts.Target.String(), + } + b.logger = prefixLogger(b) + b.logger.Infof("Created") + b.child = synchronizingBalancerWrapper{lb: gracefulswitch.NewBalancer(b, bOpts)} + go b.run() + return b +} + +func (bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + lbCfg := &LBConfig{ + // Default top layer values as documented in A50. + Interval: iserviceconfig.Duration(10 * time.Second), + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + } + + // This unmarshalling handles underlying layers sre and fpe which have their + // own defaults for their fields if either sre or fpe are present. + if err := json.Unmarshal(s, lbCfg); err != nil { // Validates child config if present as well. + return nil, fmt.Errorf("xds: unable to unmarshal LBconfig: %s, error: %v", string(s), err) + } + + // Note: in the xds flow, these validations will never fail. The xdsclient + // performs the same validations as here on the xds Outlier Detection + // resource before parsing resource into JSON which this function gets + // called with. A50 defines two separate places for these validations to + // take place, the xdsclient and this ParseConfig method. "When parsing a + // config from JSON, if any of these requirements is violated, that should + // be treated as a parsing error." - A50 + switch { + // "The google.protobuf.Duration fields interval, base_ejection_time, and + // max_ejection_time must obey the restrictions in the + // google.protobuf.Duration documentation and they must have non-negative + // values." - A50 + // Approximately 290 years is the maximum time that time.Duration (int64) + // can represent. The restrictions on the protobuf.Duration field are to be + // within +-10000 years. Thus, just check for negative values. + case lbCfg.Interval < 0: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.interval = %s; must be >= 0", lbCfg.Interval) + case lbCfg.BaseEjectionTime < 0: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.base_ejection_time = %s; must be >= 0", lbCfg.BaseEjectionTime) + case lbCfg.MaxEjectionTime < 0: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.max_ejection_time = %s; must be >= 0", lbCfg.MaxEjectionTime) + + // "The fields max_ejection_percent, + // success_rate_ejection.enforcement_percentage, + // failure_percentage_ejection.threshold, and + // failure_percentage.enforcement_percentage must have values less than or + // equal to 100." - A50 + case lbCfg.MaxEjectionPercent > 100: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.max_ejection_percent = %v; must be <= 100", lbCfg.MaxEjectionPercent) + case lbCfg.SuccessRateEjection != nil && lbCfg.SuccessRateEjection.EnforcementPercentage > 100: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.SuccessRateEjection.enforcement_percentage = %v; must be <= 100", lbCfg.SuccessRateEjection.EnforcementPercentage) + case lbCfg.FailurePercentageEjection != nil && lbCfg.FailurePercentageEjection.Threshold > 100: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.threshold = %v; must be <= 100", lbCfg.FailurePercentageEjection.Threshold) + case lbCfg.FailurePercentageEjection != nil && lbCfg.FailurePercentageEjection.EnforcementPercentage > 100: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.enforcement_percentage = %v; must be <= 100", lbCfg.FailurePercentageEjection.EnforcementPercentage) + } + return lbCfg, nil +} + +func (bb) Name() string { + return Name +} + +// scUpdate wraps a subConn update to be sent to the child balancer. +type scUpdate struct { + scw *subConnWrapper + state balancer.SubConnState +} + +type ejectionUpdate struct { + scw *subConnWrapper + isEjected bool // true for ejected, false for unejected +} + +type lbCfgUpdate struct { + lbCfg *LBConfig + // to make sure picker is updated synchronously. + done chan struct{} +} + +type scHealthUpdate struct { + scw *subConnWrapper + state balancer.SubConnState +} + +type outlierDetectionBalancer struct { + balancer.ClientConn + // These fields are safe to be accessed without holding any mutex because + // they are synchronized in run(), which makes these field accesses happen + // serially. + // + // childState is the latest balancer state received from the child. + childState balancer.State + // recentPickerNoop represents whether the most recent picker sent upward to + // the balancer.ClientConn is a noop picker, which doesn't count RPC's. Used + // to suppress redundant picker updates. + recentPickerNoop bool + + closed *grpcsync.Event + done *grpcsync.Event + logger *grpclog.PrefixLogger + channelzParent channelz.Identifier + metricsRecorder estats.MetricsRecorder + target string + + child synchronizingBalancerWrapper + + // mu guards access to the following fields. It also helps to synchronize + // behaviors of the following events: config updates, firing of the interval + // timer, SubConn State updates, SubConn address updates, and child state + // updates. + // + // For example, when we receive a config update in the middle of the + // interval timer algorithm, which uses knobs present in the config, the + // balancer will wait for the interval timer algorithm to finish before + // persisting the new configuration. + // + // Another example would be the updating of the endpoints or addrs map, such + // as from a SubConn address update in the middle of the interval timer + // algorithm which uses endpoints. This balancer waits for the interval + // timer algorithm to finish before making the update to the endpoints map. + // + // This mutex is never held when calling methods on the child policy + // (within the context of a single goroutine). + mu sync.Mutex + // endpoints stores pointers to endpointInfo objects for each endpoint. + endpoints *resolver.EndpointMap[*endpointInfo] + // addrs stores pointers to endpointInfo objects for each address. Addresses + // belonging to the same endpoint point to the same object. + addrs map[string]*endpointInfo + cfg *LBConfig + timerStartTime time.Time + intervalTimer *time.Timer + inhibitPickerUpdates bool + updateUnconditionally bool + numEndpointsEjected int // For fast calculations of percentage of endpoints ejected + + scUpdateCh *buffer.Unbounded[any] + pickerUpdateCh *buffer.Unbounded[any] +} + +// noopConfig returns whether this balancer is configured with a logical no-op +// configuration or not. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) noopConfig() bool { + return b.cfg.SuccessRateEjection == nil && b.cfg.FailurePercentageEjection == nil +} + +// onIntervalConfig handles logic required specifically on the receipt of a +// configuration which specifies to count RPC's and periodically perform passive +// health checking based on heuristics defined in configuration every configured +// interval. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) onIntervalConfig() { + var interval time.Duration + if b.timerStartTime.IsZero() { + b.timerStartTime = time.Now() + for _, epInfo := range b.endpoints.All() { + epInfo.callCounter.clear() + } + interval = time.Duration(b.cfg.Interval) + } else { + interval = time.Duration(b.cfg.Interval) - now().Sub(b.timerStartTime) + if interval < 0 { + interval = 0 + } + } + b.intervalTimer = afterFunc(interval, b.intervalTimerAlgorithm) +} + +// onNoopConfig handles logic required specifically on the receipt of a +// configuration which specifies the balancer to be a noop. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) onNoopConfig() { + // "If a config is provided with both the `success_rate_ejection` and + // `failure_percentage_ejection` fields unset, skip starting the timer and + // do the following:" + // "Unset the timer start timestamp." + b.timerStartTime = time.Time{} + for _, epInfo := range b.endpoints.All() { + // "Uneject all currently ejected endpoints." + if !epInfo.latestEjectionTimestamp.IsZero() { + b.unejectEndpoint(epInfo) + } + // "Reset each endpoint's ejection time multiplier to 0." + epInfo.ejectionTimeMultiplier = 0 + } +} + +func (b *outlierDetectionBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + lbCfg, ok := s.BalancerConfig.(*LBConfig) + if !ok { + b.logger.Errorf("received config with unexpected type %T: %v", s.BalancerConfig, s.BalancerConfig) + return balancer.ErrBadResolverState + } + + // Reject whole config if child policy doesn't exist, don't persist it for + // later. + bb := balancer.Get(lbCfg.ChildPolicy.Name) + if bb == nil { + return fmt.Errorf("outlier detection: child balancer %q not registered", lbCfg.ChildPolicy.Name) + } + + // It is safe to read b.cfg here without holding the mutex, as the only + // write to b.cfg happens later in this function. This function is part of + // the balancer.Balancer API, so it is guaranteed to be called in a + // synchronous manner, so it cannot race with this read. + if b.cfg == nil || b.cfg.ChildPolicy.Name != lbCfg.ChildPolicy.Name { + if err := b.child.switchTo(bb); err != nil { + return fmt.Errorf("outlier detection: error switching to child of type %q: %v", lbCfg.ChildPolicy.Name, err) + } + } + + b.mu.Lock() + // Inhibit child picker updates until this UpdateClientConnState() call + // completes. If needed, a picker update containing the no-op config bit + // determined from this config and most recent state from the child will be + // sent synchronously upward at the end of this UpdateClientConnState() + // call. + b.inhibitPickerUpdates = true + b.updateUnconditionally = false + b.cfg = lbCfg + + newEndpoints := resolver.NewEndpointMap[bool]() + for _, ep := range s.ResolverState.Endpoints { + newEndpoints.Set(ep, true) + if _, ok := b.endpoints.Get(ep); !ok { + b.endpoints.Set(ep, newEndpointInfo()) + } + } + + for ep := range b.endpoints.All() { + if _, ok := newEndpoints.Get(ep); !ok { + b.endpoints.Delete(ep) + } + } + + // populate the addrs map. + b.addrs = map[string]*endpointInfo{} + for _, ep := range s.ResolverState.Endpoints { + epInfo, _ := b.endpoints.Get(ep) + for _, addr := range ep.Addresses { + if _, ok := b.addrs[addr.Addr]; ok { + b.logger.Errorf("Endpoints contain duplicate address %q", addr.Addr) + continue + } + b.addrs[addr.Addr] = epInfo + } + } + + if b.intervalTimer != nil { + b.intervalTimer.Stop() + } + + if b.noopConfig() { + b.onNoopConfig() + } else { + b.onIntervalConfig() + } + b.mu.Unlock() + + err := b.child.updateClientConnState(balancer.ClientConnState{ + ResolverState: s.ResolverState, + BalancerConfig: b.cfg.ChildPolicy.Config, + }) + + done := make(chan struct{}) + b.pickerUpdateCh.Put(lbCfgUpdate{ + lbCfg: lbCfg, + done: done, + }) + <-done + + return err +} + +func (b *outlierDetectionBalancer) ResolverError(err error) { + b.child.resolverError(err) +} + +func (b *outlierDetectionBalancer) updateSubConnState(scw *subConnWrapper, state balancer.SubConnState) { + b.mu.Lock() + defer b.mu.Unlock() + scw.setLatestConnectivityState(state.ConnectivityState) + b.scUpdateCh.Put(&scUpdate{ + scw: scw, + state: state, + }) +} + +func (b *outlierDetectionBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *outlierDetectionBalancer) Close() { + b.closed.Fire() + <-b.done.Done() + b.child.closeLB() + + b.scUpdateCh.Close() + b.pickerUpdateCh.Close() + + b.mu.Lock() + defer b.mu.Unlock() + if b.intervalTimer != nil { + b.intervalTimer.Stop() + } +} + +func (b *outlierDetectionBalancer) ExitIdle() { + b.child.exitIdle() +} + +// wrappedPicker delegates to the child policy's picker, and when the request +// finishes, it increments the corresponding counter in the map entry referenced +// by the subConnWrapper that was picked. If both the `success_rate_ejection` +// and `failure_percentage_ejection` fields are unset in the configuration, this +// picker will not count. +type wrappedPicker struct { + childPicker balancer.Picker + noopPicker bool +} + +func (wp *wrappedPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + pr, err := wp.childPicker.Pick(info) + if err != nil { + return balancer.PickResult{}, err + } + + done := func(di balancer.DoneInfo) { + if !wp.noopPicker { + incrementCounter(pr.SubConn, di) + } + if pr.Done != nil { + pr.Done(di) + } + } + scw, ok := pr.SubConn.(*subConnWrapper) + if !ok { + // This can never happen, but check is present for defensive + // programming. + logger.Errorf("Picked SubConn from child picker is not a SubConnWrapper") + return balancer.PickResult{ + SubConn: pr.SubConn, + Done: done, + Metadata: pr.Metadata, + }, nil + } + return balancer.PickResult{ + SubConn: scw.SubConn, + Done: done, + Metadata: pr.Metadata, + }, nil +} + +func incrementCounter(sc balancer.SubConn, info balancer.DoneInfo) { + scw, ok := sc.(*subConnWrapper) + if !ok { + // Shouldn't happen, as comes from child + return + } + + // After reading callCounter.activeBucket in this picker a swap call can + // concurrently change what activeBucket points to. A50 says to swap the + // pointer, which will cause this race to write to deprecated memory the + // interval timer algorithm will never read, which makes this race alright. + epInfo := scw.endpointInfo + if epInfo == nil { + return + } + ab := epInfo.callCounter.activeBucket.Load() + + if info.Err == nil { + atomic.AddUint32(&ab.numSuccesses, 1) + } else { + atomic.AddUint32(&ab.numFailures, 1) + } +} + +func (b *outlierDetectionBalancer) UpdateState(s balancer.State) { + b.pickerUpdateCh.Put(s) +} + +func (b *outlierDetectionBalancer) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + oldListener := opts.StateListener + scw := &subConnWrapper{ + addresses: addrs, + scUpdateCh: b.scUpdateCh, + listener: oldListener, + latestHealthState: balancer.SubConnState{ConnectivityState: connectivity.Connecting}, + } + opts.StateListener = func(state balancer.SubConnState) { b.updateSubConnState(scw, state) } + b.mu.Lock() + defer b.mu.Unlock() + sc, err := b.ClientConn.NewSubConn(addrs, opts) + if err != nil { + return nil, err + } + scw.SubConn = sc + if len(addrs) != 1 { + return scw, nil + } + epInfo, ok := b.addrs[addrs[0].Addr] + if !ok { + return scw, nil + } + epInfo.sws = append(epInfo.sws, scw) + scw.endpointInfo = epInfo + if !epInfo.latestEjectionTimestamp.IsZero() { + scw.eject() + } + return scw, nil +} + +func (b *outlierDetectionBalancer) RemoveSubConn(sc balancer.SubConn) { + b.logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc) +} + +// removeSubConnFromEndpointMapEntry removes the scw from its map entry if +// present. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) removeSubConnFromEndpointMapEntry(scw *subConnWrapper) { + epInfo := scw.endpointInfo + if epInfo == nil { + return + } + for i, sw := range epInfo.sws { + if scw == sw { + epInfo.sws = append(epInfo.sws[:i], epInfo.sws[i+1:]...) + return + } + } +} + +func (b *outlierDetectionBalancer) UpdateAddresses(sc balancer.SubConn, _ []resolver.Address) { + b.logger.Errorf("UpdateAddresses(%v) called unexpectedly", sc) +} + +// handleSubConnUpdate stores the recent state and forward the update. +func (b *outlierDetectionBalancer) handleSubConnUpdate(u *scUpdate) { + scw := u.scw + scw.clearHealthListener() + b.child.updateSubConnState(scw, u.state) + if u.state.ConnectivityState == connectivity.Shutdown { + b.mu.Lock() + b.removeSubConnFromEndpointMapEntry(scw) + b.mu.Unlock() + } +} + +func (b *outlierDetectionBalancer) handleSubConnHealthUpdate(u *scHealthUpdate) { + b.child.updateSubConnHealthState(u.scw, u.state) +} + +// handleEjectedUpdate handles any SubConns that get ejected/unejected, and +// forwards the appropriate corresponding subConnState to the child policy. +func (b *outlierDetectionBalancer) handleEjectedUpdate(u *ejectionUpdate) { + b.child.handleEjectionUpdate(u) +} + +// handleChildStateUpdate forwards the picker update wrapped in a wrapped picker +// with the noop picker bit present. +func (b *outlierDetectionBalancer) handleChildStateUpdate(u balancer.State) { + b.childState = u + b.mu.Lock() + if b.inhibitPickerUpdates { + // If a child's state is updated during the suppression of child + // updates, the synchronous handleLBConfigUpdate function with respect + // to UpdateClientConnState should return a picker unconditionally. + b.updateUnconditionally = true + b.mu.Unlock() + return + } + noopCfg := b.noopConfig() + b.mu.Unlock() + b.recentPickerNoop = noopCfg + b.ClientConn.UpdateState(balancer.State{ + ConnectivityState: b.childState.ConnectivityState, + Picker: &wrappedPicker{ + childPicker: b.childState.Picker, + noopPicker: noopCfg, + }, + }) +} + +// handleLBConfigUpdate compares whether the new config is a noop config or not, +// to the noop bit in the picker if present. It updates the picker if this bit +// changed compared to the picker currently in use. +func (b *outlierDetectionBalancer) handleLBConfigUpdate(u lbCfgUpdate) { + lbCfg := u.lbCfg + noopCfg := lbCfg.SuccessRateEjection == nil && lbCfg.FailurePercentageEjection == nil + // If the child has sent its first update and this config flips the noop + // bit compared to the most recent picker update sent upward, then a new + // picker with this updated bit needs to be forwarded upward. If a child + // update was received during the suppression of child updates within + // UpdateClientConnState(), then a new picker needs to be forwarded with + // this updated state, irregardless of whether this new configuration flips + // the bit. + if b.childState.Picker != nil && noopCfg != b.recentPickerNoop || b.updateUnconditionally { + b.recentPickerNoop = noopCfg + b.ClientConn.UpdateState(balancer.State{ + ConnectivityState: b.childState.ConnectivityState, + Picker: &wrappedPicker{ + childPicker: b.childState.Picker, + noopPicker: noopCfg, + }, + }) + } + b.inhibitPickerUpdates = false + b.updateUnconditionally = false + close(u.done) +} + +func (b *outlierDetectionBalancer) run() { + defer b.done.Fire() + for { + select { + case update, ok := <-b.scUpdateCh.Get(): + if !ok { + return + } + b.scUpdateCh.Load() + if b.closed.HasFired() { // don't send SubConn updates to child after the balancer has been closed + return + } + switch u := update.(type) { + case *scUpdate: + b.handleSubConnUpdate(u) + case *ejectionUpdate: + b.handleEjectedUpdate(u) + case *scHealthUpdate: + b.handleSubConnHealthUpdate(u) + } + case update, ok := <-b.pickerUpdateCh.Get(): + if !ok { + return + } + b.pickerUpdateCh.Load() + if b.closed.HasFired() { // don't send picker updates to grpc after the balancer has been closed + return + } + switch u := update.(type) { + case balancer.State: + b.handleChildStateUpdate(u) + case lbCfgUpdate: + b.handleLBConfigUpdate(u) + } + case <-b.closed.Done(): + return + } + } +} + +// intervalTimerAlgorithm ejects and unejects endpoints based on the Outlier +// Detection configuration and data about each endpoint from the previous +// interval. +func (b *outlierDetectionBalancer) intervalTimerAlgorithm() { + b.mu.Lock() + defer b.mu.Unlock() + b.timerStartTime = time.Now() + + for _, epInfo := range b.endpoints.All() { + epInfo.callCounter.swap() + } + + if b.cfg.SuccessRateEjection != nil { + b.successRateAlgorithm() + } + + if b.cfg.FailurePercentageEjection != nil { + b.failurePercentageAlgorithm() + } + + for _, epInfo := range b.endpoints.All() { + if epInfo.latestEjectionTimestamp.IsZero() && epInfo.ejectionTimeMultiplier > 0 { + epInfo.ejectionTimeMultiplier-- + continue + } + if epInfo.latestEjectionTimestamp.IsZero() { + // Endpoint is already not ejected, so no need to check for whether + // to uneject the endpoint below. + continue + } + et := time.Duration(b.cfg.BaseEjectionTime) * time.Duration(epInfo.ejectionTimeMultiplier) + met := max(time.Duration(b.cfg.BaseEjectionTime), time.Duration(b.cfg.MaxEjectionTime)) + uet := epInfo.latestEjectionTimestamp.Add(min(et, met)) + if now().After(uet) { + b.unejectEndpoint(epInfo) + } + } + + // This conditional only for testing (since the interval timer algorithm is + // called manually), will never hit in production. + if b.intervalTimer != nil { + b.intervalTimer.Stop() + } + b.intervalTimer = afterFunc(time.Duration(b.cfg.Interval), b.intervalTimerAlgorithm) +} + +// endpointsWithAtLeastRequestVolume returns a slice of endpoint information of +// all endpoints with at least request volume passed in. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) endpointsWithAtLeastRequestVolume(requestVolume uint32) []*endpointInfo { + var endpoints []*endpointInfo + for _, epInfo := range b.endpoints.All() { + bucket1 := epInfo.callCounter.inactiveBucket + rv := bucket1.numSuccesses + bucket1.numFailures + if rv >= requestVolume { + endpoints = append(endpoints, epInfo) + } + } + return endpoints +} + +// meanAndStdDev returns the mean and std dev of the fractions of successful +// requests of the endpoints passed in. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) meanAndStdDev(endpoints []*endpointInfo) (float64, float64) { + var totalFractionOfSuccessfulRequests float64 + var mean float64 + for _, epInfo := range endpoints { + bucket := epInfo.callCounter.inactiveBucket + rv := bucket.numSuccesses + bucket.numFailures + totalFractionOfSuccessfulRequests += float64(bucket.numSuccesses) / float64(rv) + } + mean = totalFractionOfSuccessfulRequests / float64(len(endpoints)) + var sumOfSquares float64 + for _, epInfo := range endpoints { + bucket := epInfo.callCounter.inactiveBucket + rv := bucket.numSuccesses + bucket.numFailures + devFromMean := (float64(bucket.numSuccesses) / float64(rv)) - mean + sumOfSquares += devFromMean * devFromMean + } + variance := sumOfSquares / float64(len(endpoints)) + return mean, math.Sqrt(variance) +} + +// successRateAlgorithm ejects any endpoints where the success rate falls below +// the other endpoints according to mean and standard deviation, and if overall +// applicable from other set heuristics. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) successRateAlgorithm() { + endpointsToConsider := b.endpointsWithAtLeastRequestVolume(b.cfg.SuccessRateEjection.RequestVolume) + if len(endpointsToConsider) < int(b.cfg.SuccessRateEjection.MinimumHosts) { + return + } + mean, stddev := b.meanAndStdDev(endpointsToConsider) + ejectionCfg := b.cfg.SuccessRateEjection + for _, epInfo := range endpointsToConsider { + bucket := epInfo.callCounter.inactiveBucket + successRate := float64(bucket.numSuccesses) / float64(bucket.numSuccesses+bucket.numFailures) + requiredSuccessRate := mean - stddev*(float64(ejectionCfg.StdevFactor)/1000) + if successRate < requiredSuccessRate { + channelz.Infof(logger, b.channelzParent, "SuccessRate algorithm detected outlier: %s. Parameters: successRate=%f, mean=%f, stddev=%f, requiredSuccessRate=%f", epInfo, successRate, mean, stddev, requiredSuccessRate) + // Check if max ejection percentage would prevent ejection. + if float64(b.numEndpointsEjected)/float64(b.endpoints.Len())*100 >= float64(b.cfg.MaxEjectionPercent) { + // Record unenforced ejection due to max ejection percentage. + ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "success_rate", "max_ejection_overflow") + continue + } + if uint32(rand.Int32N(100)) < ejectionCfg.EnforcementPercentage { + b.ejectEndpoint(epInfo, "success_rate") + } else { + // Record unenforced ejection due to enforcement percentage. + ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "success_rate", "enforcement_percentage") + } + } + } +} + +// failurePercentageAlgorithm ejects any endpoints where the failure percentage +// rate exceeds a set enforcement percentage, if overall applicable from other +// set heuristics. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) failurePercentageAlgorithm() { + endpointsToConsider := b.endpointsWithAtLeastRequestVolume(b.cfg.FailurePercentageEjection.RequestVolume) + if len(endpointsToConsider) < int(b.cfg.FailurePercentageEjection.MinimumHosts) { + return + } + + ejectionCfg := b.cfg.FailurePercentageEjection + for _, epInfo := range endpointsToConsider { + bucket := epInfo.callCounter.inactiveBucket + failurePercentage := (float64(bucket.numFailures) / float64(bucket.numSuccesses+bucket.numFailures)) * 100 + if failurePercentage > float64(b.cfg.FailurePercentageEjection.Threshold) { + channelz.Infof(logger, b.channelzParent, "FailurePercentage algorithm detected outlier: %s, failurePercentage=%f", epInfo, failurePercentage) + // Check if max ejection percentage would prevent ejection. + if float64(b.numEndpointsEjected)/float64(b.endpoints.Len())*100 >= float64(b.cfg.MaxEjectionPercent) { + // Record unenforced ejection due to max ejection percentage. + ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "failure_percentage", "max_ejection_overflow") + continue + } + if uint32(rand.Int32N(100)) < ejectionCfg.EnforcementPercentage { + b.ejectEndpoint(epInfo, "failure_percentage") + } else { + // Record unenforced ejection due to enforcement percentage. + ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "failure_percentage", "enforcement_percentage") + } + } + } +} + +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) ejectEndpoint(epInfo *endpointInfo, detectionMethod string) { + b.numEndpointsEjected++ + epInfo.latestEjectionTimestamp = b.timerStartTime + epInfo.ejectionTimeMultiplier++ + for _, sbw := range epInfo.sws { + sbw.eject() + channelz.Infof(logger, b.channelzParent, "Subchannel ejected: %s", sbw) + } + + // Record the enforced ejection metric. + ejectionsEnforcedMetric.Record(b.metricsRecorder, 1, b.target, detectionMethod) +} + +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) unejectEndpoint(epInfo *endpointInfo) { + b.numEndpointsEjected-- + epInfo.latestEjectionTimestamp = time.Time{} + for _, sbw := range epInfo.sws { + sbw.uneject() + channelz.Infof(logger, b.channelzParent, "Subchannel unejected: %s", sbw) + } +} + +// synchronizingBalancerWrapper serializes calls into balancer (to uphold the +// balancer.Balancer API guarantee of synchronous calls). It also ensures a +// consistent order of locking mutexes when using SubConn listeners to avoid +// deadlocks. +type synchronizingBalancerWrapper struct { + // mu should not be used directly from outside this struct, instead use + // methods defined on the struct. + mu sync.Mutex + lb *gracefulswitch.Balancer +} + +func (sbw *synchronizingBalancerWrapper) switchTo(builder balancer.Builder) error { + sbw.mu.Lock() + defer sbw.mu.Unlock() + return sbw.lb.SwitchTo(builder) +} + +func (sbw *synchronizingBalancerWrapper) updateClientConnState(state balancer.ClientConnState) error { + sbw.mu.Lock() + defer sbw.mu.Unlock() + return sbw.lb.UpdateClientConnState(state) +} + +func (sbw *synchronizingBalancerWrapper) resolverError(err error) { + sbw.mu.Lock() + defer sbw.mu.Unlock() + sbw.lb.ResolverError(err) +} + +func (sbw *synchronizingBalancerWrapper) closeLB() { + sbw.mu.Lock() + defer sbw.mu.Unlock() + sbw.lb.Close() +} + +func (sbw *synchronizingBalancerWrapper) exitIdle() { + sbw.mu.Lock() + defer sbw.mu.Unlock() + sbw.lb.ExitIdle() +} + +func (sbw *synchronizingBalancerWrapper) updateSubConnHealthState(scw *subConnWrapper, scs balancer.SubConnState) { + sbw.mu.Lock() + defer sbw.mu.Unlock() + scw.updateSubConnHealthState(scs) +} + +func (sbw *synchronizingBalancerWrapper) updateSubConnState(scw *subConnWrapper, scs balancer.SubConnState) { + sbw.mu.Lock() + defer sbw.mu.Unlock() + scw.updateSubConnConnectivityState(scs) +} + +func (sbw *synchronizingBalancerWrapper) handleEjectionUpdate(u *ejectionUpdate) { + sbw.mu.Lock() + defer sbw.mu.Unlock() + if u.isEjected { + u.scw.handleEjection() + } else { + u.scw.handleUnejection() + } +} + +// endpointInfo contains the runtime information about an endpoint that pertains +// to Outlier Detection. This struct and all of its fields is protected by +// outlierDetectionBalancer.mu in the case where it is accessed through the +// address or endpoint map. In the case of Picker callbacks, the writes to the +// activeBucket of callCounter are protected by atomically loading and storing +// unsafe.Pointers (see further explanation in incrementCounter()). +type endpointInfo struct { + // The call result counter object. + callCounter *callCounter + + // The latest ejection timestamp, or zero if the endpoint is currently not + // ejected. + latestEjectionTimestamp time.Time + + // The current ejection time multiplier, starting at 0. + ejectionTimeMultiplier int64 + + // A list of subchannel wrapper objects that correspond to this endpoint. + sws []*subConnWrapper +} + +func (a *endpointInfo) String() string { + var res strings.Builder + res.WriteString("[") + for _, sw := range a.sws { + res.WriteString(sw.String()) + } + res.WriteString("]") + return res.String() +} + +func newEndpointInfo() *endpointInfo { + return &endpointInfo{ + callCounter: newCallCounter(), + } +} diff --git a/internal/xds/balancer/outlierdetection/balancer_ext_test.go b/internal/xds/balancer/outlierdetection/balancer_ext_test.go new file mode 100644 index 0000000..fead684 --- /dev/null +++ b/internal/xds/balancer/outlierdetection/balancer_ext_test.go @@ -0,0 +1,518 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package outlierdetection_test + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/xds/balancer/clusterimpl" + "google.golang.org/grpc/internal/xds/balancer/outlierdetection" + "google.golang.org/grpc/serviceconfig" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestParseConfig verifies the ParseConfig() method in the Outlier Detection +// Balancer. +func (s) TestParseConfig(t *testing.T) { + const errParseConfigName = "errParseConfigBalancer" + stub.Register(errParseConfigName, stub.BalancerFuncs{ + ParseConfig: func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return nil, errors.New("some error") + }, + }) + + parser := balancer.Get(outlierdetection.Name).(balancer.ConfigParser) + const ( + defaultInterval = iserviceconfig.Duration(10 * time.Second) + defaultBaseEjectionTime = iserviceconfig.Duration(30 * time.Second) + defaultMaxEjectionTime = iserviceconfig.Duration(300 * time.Second) + defaultMaxEjectionPercent = 10 + defaultSuccessRateStdevFactor = 1900 + defaultEnforcingSuccessRate = 100 + defaultSuccessRateMinimumHosts = 5 + defaultSuccessRateRequestVolume = 100 + defaultFailurePercentageThreshold = 85 + defaultEnforcingFailurePercentage = 0 + defaultFailurePercentageMinimumHosts = 5 + defaultFailurePercentageRequestVolume = 50 + ) + tests := []struct { + name string + input string + wantCfg serviceconfig.LoadBalancingConfig + wantErr string + }{ + { + name: "Default", + input: `{ + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + wantCfg: &outlierdetection.LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + + { + name: "TopLevelSet", + input: `{ + "interval": "15s", + "maxEjectionTime": "350s", + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + // Should get set fields + defaults for unset fields. + wantCfg: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(15 * time.Second), + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: iserviceconfig.Duration(350 * time.Second), + MaxEjectionPercent: defaultMaxEjectionPercent, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "SuccessRate_Defaults", + input: `{ + "successRateEjection": {}, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + // Should get defaults of success-rate-ejection struct. + wantCfg: &outlierdetection.LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + SuccessRateEjection: &outlierdetection.SuccessRateEjection{ + StdevFactor: defaultSuccessRateStdevFactor, + EnforcementPercentage: defaultEnforcingSuccessRate, + MinimumHosts: defaultSuccessRateMinimumHosts, + RequestVolume: defaultSuccessRateRequestVolume, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "SuccessRate_Partial", + input: `{ + "successRateEjection": { + "stdevFactor": 1000, + "minimumHosts": 5 + }, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + // Should get set fields + defaults for others in success rate + // ejection layer. + wantCfg: &outlierdetection.LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + SuccessRateEjection: &outlierdetection.SuccessRateEjection{ + StdevFactor: 1000, + EnforcementPercentage: defaultEnforcingSuccessRate, + MinimumHosts: 5, + RequestVolume: defaultSuccessRateRequestVolume, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "SuccessRate_Full", + input: `{ + "successRateEjection": { + "stdevFactor": 1000, + "enforcementPercentage": 50, + "minimumHosts": 5, + "requestVolume": 50 + }, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + wantCfg: &outlierdetection.LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + SuccessRateEjection: &outlierdetection.SuccessRateEjection{ + StdevFactor: 1000, + EnforcementPercentage: 50, + MinimumHosts: 5, + RequestVolume: 50, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "FailurePercentage_Defaults", + input: `{ + "failurePercentageEjection": {}, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + // Should get defaults of failure percentage ejection layer. + wantCfg: &outlierdetection.LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{ + Threshold: defaultFailurePercentageThreshold, + EnforcementPercentage: defaultEnforcingFailurePercentage, + MinimumHosts: defaultFailurePercentageMinimumHosts, + RequestVolume: defaultFailurePercentageRequestVolume, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "FailurePercentage_Partial", + input: `{ + "failurePercentageEjection": { + "threshold": 80, + "minimumHosts": 10 + }, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + // Should get set fields + defaults for others in success rate + // ejection layer. + wantCfg: &outlierdetection.LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{ + Threshold: 80, + EnforcementPercentage: defaultEnforcingFailurePercentage, + MinimumHosts: 10, + RequestVolume: defaultFailurePercentageRequestVolume, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "FailurePercentage_Full", + input: `{ + "failurePercentageEjection": { + "threshold": 80, + "enforcementPercentage": 100, + "minimumHosts": 10, + "requestVolume": 40 + }, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + wantCfg: &outlierdetection.LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{ + Threshold: 80, + EnforcementPercentage: 100, + MinimumHosts: 10, + RequestVolume: 40, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { // to make sure zero values aren't overwritten by defaults + name: "AllFields_Zero", + input: `{ + "interval": "0s", + "baseEjectionTime": "0s", + "maxEjectionTime": "0s", + "maxEjectionPercent": 0, + "successRateEjection": { + "stdevFactor": 0, + "enforcementPercentage": 0, + "minimumHosts": 0, + "requestVolume": 0 + }, + "failurePercentageEjection": { + "threshold": 0, + "enforcementPercentage": 0, + "minimumHosts": 0, + "requestVolume": 0 + }, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + wantCfg: &outlierdetection.LBConfig{ + SuccessRateEjection: &outlierdetection.SuccessRateEjection{}, + FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{}, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "AllFields_Set", + input: `{ + "interval": "10s", + "baseEjectionTime": "30s", + "maxEjectionTime": "300s", + "maxEjectionPercent": 10, + "successRateEjection": { + "stdevFactor": 1900, + "enforcementPercentage": 100, + "minimumHosts": 5, + "requestVolume": 100 + }, + "failurePercentageEjection": { + "threshold": 85, + "enforcementPercentage": 5, + "minimumHosts": 5, + "requestVolume": 50 + }, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + wantCfg: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + SuccessRateEjection: &outlierdetection.SuccessRateEjection{ + StdevFactor: 1900, + EnforcementPercentage: 100, + MinimumHosts: 5, + RequestVolume: 100, + }, + FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{ + Threshold: 85, + EnforcementPercentage: 5, + MinimumHosts: 5, + RequestVolume: 50, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "Interval_Negative", + input: `{"interval": "-10s"}`, + wantErr: "OutlierDetectionLoadBalancingConfig.interval = -10s; must be >= 0", + }, + { + name: "BaseEjectionTime_Negative", + input: `{"baseEjectionTime": "-10s"}`, + wantErr: "OutlierDetectionLoadBalancingConfig.base_ejection_time = -10s; must be >= 0", + }, + { + name: "MaxEjectionTime_Negative", + input: `{"maxEjectionTime": "-10s"}`, + wantErr: "OutlierDetectionLoadBalancingConfig.max_ejection_time = -10s; must be >= 0", + }, + { + name: "MaxEjectionPercent_TooHigh", + input: `{"maxEjectionPercent": 150}`, + wantErr: "OutlierDetectionLoadBalancingConfig.max_ejection_percent = 150; must be <= 100", + }, + { + name: "SuccessRate_Enforcement_TooHigh", + input: `{ + "successRateEjection": { + "enforcementPercentage": 150 + } + }`, + wantErr: "OutlierDetectionLoadBalancingConfig.SuccessRateEjection.enforcement_percentage = 150; must be <= 100", + }, + { + name: "FailurePercentage_Threshold_TooHigh", + input: `{ + "failurePercentageEjection": { + "threshold": 150 + } + }`, + wantErr: "OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.threshold = 150; must be <= 100", + }, + { + name: "FailurePercentage_Enforcement_TooHigh", + input: `{ + "failurePercentageEjection": { + "enforcementPercentage": 150 + } + }`, + wantErr: "OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.enforcement_percentage = 150; must be <= 100", + }, + { + name: "ChildPolicy_ParseError", + input: `{ + "childPolicy": [ + { + "errParseConfigBalancer": { + "cluster": "test_cluster" + } + } + ] + }`, + wantErr: "error parsing loadBalancingConfig for policy \"errParseConfigBalancer\"", + }, + { + name: "ChildPolicy_NotSupported", + input: `{ + "childPolicy": [ + { + "doesNotExistBalancer": { + "cluster": "test_cluster" + } + } + ] + }`, + wantErr: "invalid loadBalancingConfig: no supported policies found", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotCfg, gotErr := parser.ParseConfig(json.RawMessage(test.input)) + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if test.wantErr != "" { + return + } + if diff := cmp.Diff(gotCfg, test.wantCfg); diff != "" { + t.Fatalf("parseConfig(%v) got unexpected output, diff (-got +want): %v", string(test.input), diff) + } + }) + } +} diff --git a/internal/xds/balancer/outlierdetection/balancer_test.go b/internal/xds/balancer/outlierdetection/balancer_test.go new file mode 100644 index 0000000..ad3e4c8 --- /dev/null +++ b/internal/xds/balancer/outlierdetection/balancer_test.go @@ -0,0 +1,1533 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package outlierdetection + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/internal/testutils/stats" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (lbc *LBConfig) Equal(lbc2 *LBConfig) bool { + if !lbc.EqualIgnoringChildPolicy(lbc2) { + return false + } + return cmp.Equal(lbc.ChildPolicy, lbc2.ChildPolicy) +} + +type subConnWithState struct { + sc balancer.SubConn + state balancer.SubConnState +} + +func setup(t *testing.T) (*outlierDetectionBalancer, *testutils.BalancerClientConn, func()) { + t.Helper() + builder := balancer.Get(Name) + if builder == nil { + t.Fatalf("balancer.Get(%q) returned nil", Name) + } + tcc := testutils.NewBalancerClientConn(t) + ch := channelz.RegisterChannel(nil, "test channel") + t.Cleanup(func() { channelz.RemoveEntry(ch.ID) }) + odB := builder.Build(tcc, balancer.BuildOptions{ChannelzParent: ch}) + return odB.(*outlierDetectionBalancer), tcc, odB.Close +} + +type emptyChildConfig struct { + serviceconfig.LoadBalancingConfig +} + +// TestChildBasicOperations tests basic operations of the Outlier Detection +// Balancer and its interaction with its child. The following scenarios are +// tested, in a step by step fashion: +// 1. The Outlier Detection Balancer receives it's first good configuration. The +// balancer is expected to create a child and sent the child it's configuration. +// 2. The Outlier Detection Balancer receives new configuration that specifies a +// child's type, and the new type immediately reports READY inline. The first +// child balancer should be closed and the second child balancer should receive +// a config update. +// 3. The Outlier Detection Balancer is closed. The second child balancer should +// be closed. +func (s) TestChildBasicOperations(t *testing.T) { + bc := emptyChildConfig{} + + ccsCh := testutils.NewChannel() + closeCh := testutils.NewChannel() + + stub.Register(t.Name()+"child1", stub.BalancerFuncs{ + UpdateClientConnState: func(_ *stub.BalancerData, ccs balancer.ClientConnState) error { + ccsCh.Send(ccs.BalancerConfig) + return nil + }, + Close: func(*stub.BalancerData) { + closeCh.Send(nil) + }, + }) + + stub.Register(t.Name()+"child2", stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + // UpdateState inline to READY to complete graceful switch process + // synchronously from any UpdateClientConnState call. + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &testutils.TestConstPicker{}, + }) + ccsCh.Send(nil) + return nil + }, + Close: func(*stub.BalancerData) { + closeCh.Send(nil) + }, + }) + + od, tcc, _ := setup(t) + + // This first config update should cause a child to be built and forwarded + // its first update. + od.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: &LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name() + "child1", + Config: bc, + }, + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cr, err := ccsCh.Receive(ctx) + if err != nil { + t.Fatalf("timed out waiting for UpdateClientConnState on the first child balancer: %v", err) + } + if _, ok := cr.(emptyChildConfig); !ok { + t.Fatalf("Received child policy config of type %T, want %T", cr, emptyChildConfig{}) + } + + // This Update Client Conn State call should cause the first child balancer + // to close, and a new child to be created and also forwarded its first + // config update. + od.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: &LBConfig{ + Interval: math.MaxInt64, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name() + "child2", + Config: emptyChildConfig{}, + }, + }, + }) + + // Verify inline UpdateState() call from the new child eventually makes it's + // way to the Test Client Conn. + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case state := <-tcc.NewStateCh: + if state != connectivity.Ready { + t.Fatalf("ClientConn received connectivity state %v, want %v", state, connectivity.Ready) + } + } + + // Verify the first child balancer closed. + if _, err = closeCh.Receive(ctx); err != nil { + t.Fatalf("timed out waiting for the first child balancer to be closed: %v", err) + } + // Verify the second child balancer received its first config update. + if _, err = ccsCh.Receive(ctx); err != nil { + t.Fatalf("timed out waiting for UpdateClientConnState on the second child balancer: %v", err) + } + // Closing the Outlier Detection Balancer should close the newly created + // child. + od.Close() + if _, err = closeCh.Receive(ctx); err != nil { + t.Fatalf("timed out waiting for the second child balancer to be closed: %v", err) + } +} + +func scwsEqual(gotSCWS subConnWithState, wantSCWS subConnWithState) error { + if gotSCWS.sc != wantSCWS.sc || !cmp.Equal(gotSCWS.state, wantSCWS.state, cmp.AllowUnexported(subConnWrapper{}, endpointInfo{}, balancer.SubConnState{}), cmpopts.IgnoreFields(subConnWrapper{}, "scUpdateCh")) { + return fmt.Errorf("received SubConnState: %+v, want %+v", gotSCWS, wantSCWS) + } + return nil +} + +type rrPicker struct { + scs []balancer.SubConn + next int +} + +func (rrp *rrPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + sc := rrp.scs[rrp.next] + rrp.next = (rrp.next + 1) % len(rrp.scs) + return balancer.PickResult{SubConn: sc}, nil +} + +// TestDurationOfInterval tests the configured interval timer. +// The following scenarios are tested: +// 1. The Outlier Detection Balancer receives it's first config. The balancer +// should configure the timer with whatever is directly specified on the config. +// 2. The Outlier Detection Balancer receives a subsequent config. The balancer +// should configure with whatever interval is configured minus the difference +// between the current time and the previous start timestamp. +// 3. The Outlier Detection Balancer receives a no-op configuration. The +// balancer should not configure a timer at all. +func (s) TestDurationOfInterval(t *testing.T) { + stub.Register(t.Name(), stub.BalancerFuncs{}) + + od, _, cleanup := setup(t) + defer func(af func(d time.Duration, f func()) *time.Timer) { + cleanup() + afterFunc = af + }(afterFunc) + + durationChan := testutils.NewChannel() + afterFunc = func(dur time.Duration, _ func()) *time.Timer { + durationChan.Send(dur) + return time.NewTimer(math.MaxInt64) + } + + od.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: &LBConfig{ + Interval: iserviceconfig.Duration(8 * time.Second), + SuccessRateEjection: &SuccessRateEjection{ + StdevFactor: 1900, + EnforcementPercentage: 100, + MinimumHosts: 5, + RequestVolume: 100, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + d, err := durationChan.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving duration from afterFunc() call: %v", err) + } + dur := d.(time.Duration) + // The configured duration should be 8 seconds - what the balancer was + // configured with. + if dur != 8*time.Second { + t.Fatalf("configured duration should have been 8 seconds to start timer") + } + + // Override time.Now to time.Now() + 5 seconds. This will represent 5 + // seconds already passing for the next check in UpdateClientConnState. + defer func(n func() time.Time) { + now = n + }(now) + now = func() time.Time { + return time.Now().Add(time.Second * 5) + } + + // UpdateClientConnState with an interval of 9 seconds. Due to 5 seconds + // already passing (from overridden time.Now function), this should start an + // interval timer of ~4 seconds. + od.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: &LBConfig{ + Interval: iserviceconfig.Duration(9 * time.Second), + SuccessRateEjection: &SuccessRateEjection{ + StdevFactor: 1900, + EnforcementPercentage: 100, + MinimumHosts: 5, + RequestVolume: 100, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + + d, err = durationChan.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving duration from afterFunc() call: %v", err) + } + dur = d.(time.Duration) + if dur.Seconds() < 3.5 || 4.5 < dur.Seconds() { + t.Fatalf("configured duration should have been around 4 seconds to start timer") + } + + // UpdateClientConnState with a no-op config. This shouldn't configure the + // interval timer at all due to it being a no-op. + od.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: &LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + + // No timer should have been started. + sCtx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err = durationChan.Receive(sCtx); err == nil { + t.Fatal("No timer should have started.") + } +} + +// TestEjectUnejectSuccessRate tests the functionality of the interval timer +// algorithm when configured with SuccessRateEjection. The Outlier Detection +// Balancer will be set up with N SubConns, each with a different address. +// It tests the following scenarios, in a step by step fashion: +// 1. The N addresses each have 5 successes. The interval timer algorithm +// should not eject any of the addresses. +// 2. N - `wantFailures` of the addresses have 5 successes but the remaining +// address has 5 failures. The internal algorithm +// should attempt to eject the address based on the outlier detection lb +// config. Only `wantEjections` addresses should be ejected. +// 3. The interval timer algorithm is run at a later time past max ejection +// time. The interval timer algorithm should uneject all. +func (s) TestEjectUnejectSuccessRate(t *testing.T) { + tests := []struct { + name string + lbConfig LBConfig + numberOfConns int + wantFailures int + wantEjections int + }{ + { + name: "three_upstreams_one_failure", + lbConfig: LBConfig{ + Interval: math.MaxInt64, // so the interval will never run unless called manually in test. + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: 50, + EnforcementPercentage: 100, + MinimumHosts: 3, + RequestVolume: 3, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "three_upstreams_one_failure", + Config: emptyChildConfig{}, + }, + }, + numberOfConns: 3, + wantFailures: 1, + wantEjections: 1, + }, + { + name: "three_upstreams_no_failure", + lbConfig: LBConfig{ + Interval: math.MaxInt64, // so the interval will never run unless called manually in test. + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: 50, + EnforcementPercentage: 100, + MinimumHosts: 3, + RequestVolume: 3, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "three_upstreams_no_failure", + Config: emptyChildConfig{}, + }, + }, + numberOfConns: 3, + wantFailures: 0, + wantEjections: 0, + }, + { + name: "three_upstreams_one_failure_no_ejection_enforcement_perc", + lbConfig: LBConfig{ + Interval: math.MaxInt64, // so the interval will never run unless called manually in test. + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: 50, + EnforcementPercentage: 0, + MinimumHosts: 3, + RequestVolume: 3, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "three_upstreams_one_failure_no_ejection_enforcement_perc", + Config: emptyChildConfig{}, + }, + }, + numberOfConns: 3, + wantFailures: 1, + wantEjections: 0, + }, + { + name: "three_upstreams_one_failure_no_ejection_max_ejection_perc", + lbConfig: LBConfig{ + Interval: math.MaxInt64, // so the interval will never run unless called manually in test. + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 0, + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: 50, + EnforcementPercentage: 100, + MinimumHosts: 3, + RequestVolume: 3, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "three_upstreams_one_failure_no_ejection_max_ejection_perc", + Config: emptyChildConfig{}, + }, + }, + numberOfConns: 3, + wantFailures: 1, + wantEjections: 0, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + scsCh := testutils.NewChannel() + connectivityCh := make(chan struct{}) + var allSubConns = make([]balancer.SubConn, test.numberOfConns) + stub.Register(test.name, stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + for i := range test.numberOfConns { + scw, err := bd.ClientConn.NewSubConn(ccs.ResolverState.Endpoints[i].Addresses, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + if state.ConnectivityState == connectivity.Ready { + connectivityCh <- struct{}{} + } + }, + }) + if err != nil { + t.Errorf("NewSubConn(%v) failed: %v", ccs.ResolverState.Endpoints[i].Addresses, err) + } + scw.Connect() + allSubConns[i] = scw + } + + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &rrPicker{ + scs: allSubConns, + }, + }) + return nil + }, + }) + + od, tcc, cleanup := setup(t) + defer cleanup() + endpoints := make([]resolver.Endpoint, test.numberOfConns) + for i := range test.numberOfConns { + endpoints[i] = resolver.Endpoint{Addresses: []resolver.Address{{Addr: fmt.Sprintf("address%d", i+1)}}} + } + od.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: endpoints, + }, + BalancerConfig: &test.lbConfig, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Transition the SubConns to READY so that they can register health + // listeners. + for range test.numberOfConns { + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for creation of new SubConn.") + case sc := <-tcc.NewSubConnCh: + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + } + + // Register health listeners after all the connectivity updates are + // processed to avoid data races while accessing the health listener within + // the TestClientConn. + connectionsReady := 0 + for connectionsReady < test.numberOfConns { + select { + case <-ctx.Done(): + t.Fatal("Context timed out waiting for all SubConns to become READY.") + case <-connectivityCh: + connectionsReady++ + } + } + + for i := range test.numberOfConns { + allSubConns[i].RegisterHealthListener(func(healthState balancer.SubConnState) { + scsCh.Send(subConnWithState{sc: allSubConns[i], state: healthState}) + }) + } + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case picker := <-tcc.NewPickerCh: + // Set each of the three upstream addresses to have five successes each. + // This should cause none of the addresses to be ejected as none of them + // are outliers according to the success rate algorithm. + for i := 0; i < test.numberOfConns; i++ { + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("picker.Pick failed with error: %v", err) + } + for c := 0; c < 5; c++ { + pi.Done(balancer.DoneInfo{}) + } + } + + // Create test metrics recorder + tmr := stats.NewTestMetricsRecorder() + od.metricsRecorder = tmr + od.intervalTimerAlgorithm() + + // verify no StateListener() call on the child, as no addresses got + // ejected (ejected address will cause an StateListener call). + sCtx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := scsCh.Receive(sCtx); err == nil { + t.Fatalf("no SubConn update should have been sent (no SubConn got ejected)") + } + if got, _ := tmr.Metric("grpc.lb.outlier_detection.ejections_enforced"); got != 0 { + t.Errorf("Metric grpc.lb.outlier_detection.ejections_enforced: got %f, want 0", got) + } + if got, _ := tmr.Metric("grpc.lb.outlier_detection.ejections_unenforced"); got != 0 { + t.Errorf("Metric grpc.lb.outlier_detection.ejections_unenforced: got %f, want 0", got) + } + + // Since no addresses are ejected, a SubConn update should forward down + // to the child. + newState := balancer.SubConnState{ConnectivityState: connectivity.Connecting} + wantSC := allSubConns[0] + od.scUpdateCh.Put(&scHealthUpdate{ + scw: wantSC.(*subConnWrapper), + state: newState, + }) + + gotSCWS, err := scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for Sub Conn update: %v", err) + } + if err = scwsEqual(gotSCWS.(subConnWithState), subConnWithState{ + sc: wantSC, + state: newState, + }); err != nil { + t.Fatalf("Error in Sub Conn update: %v", err) + } + + // Set all the upstream before the offset to have five successes, but + // the remaining addresses to have failures. + offset := test.numberOfConns - test.wantFailures + for i := 0; i < test.numberOfConns; i++ { + pickerDone := balancer.DoneInfo{} + if i >= offset { + pickerDone = balancer.DoneInfo{Err: errors.New("some error")} + } + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("picker.Pick failed with error: %v", err) + } + for c := 0; c < 5; c++ { + pi.Done(pickerDone) + } + + } + + // should eject address that always errored. + od.intervalTimerAlgorithm() + + // Due to the address being ejected, the SubConn with that address + // should be ejected, meaning a TRANSIENT_FAILURE connectivity state + // gets reported to the child. + for i := 0; i < test.wantEjections; i++ { + got, err := scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for SubConn to be ejected: %v", err) + } + if err = scwsEqual(got.(subConnWithState), subConnWithState{ + sc: allSubConns[len(allSubConns)-1-i], + state: balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}, + }); err != nil { + t.Fatalf("Unexpected subconnection with state: %v", err) + } + } + + sCtx, cancel2 := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel2() + if _, err := scsCh.Receive(sCtx); err == nil { + t.Fatalf("Only one SubConn update should have been sent (only one SubConn got ejected)") + } + + if got, _ := tmr.Metric("grpc.lb.outlier_detection.ejections_enforced"); got != float64(test.wantEjections) { + t.Errorf("Metric grpc.lb.outlier_detection.ejections_enforced: got %f, want %f", got, float64(test.wantEjections)) + } + if got, _ := tmr.Metric("grpc.lb.outlier_detection.ejections_unenforced"); got != float64(test.wantFailures-test.wantEjections) { + t.Errorf("Metric grpc.lb.outlier_detection.ejections_unenforced: got %f, want %f", got, float64(test.wantFailures-test.wantEjections)) + } + + for i := 0; i < test.wantEjections; i++ { + // Now that an address is ejected, SubConn updates for SubConns using + // that address should not be forwarded downward. These SubConn updates + // will be cached to update the child sometime in the future when the + // address gets unejected. + scw := allSubConns[len(allSubConns)-1-i] + od.scUpdateCh.Put(&scHealthUpdate{ + scw: scw.(*subConnWrapper), + state: balancer.SubConnState{ConnectivityState: connectivity.Connecting}, + }) + sCtx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := scsCh.Receive(sCtx); err == nil { + t.Fatalf("SubConn update should not have been forwarded (the SubConn is ejected)") + } + } + + // Override now to cause the interval timer algorithm to always uneject + // the ejected address. This will always uneject the ejected address + // because this time is set way past the max ejection time set in the + // configuration, which will make the next interval timer algorithm run + // uneject any ejected addresses. + defer func(n func() time.Time) { + now = n + }(now) + now = func() time.Time { + return time.Now().Add(time.Second * 1000) + } + od.intervalTimerAlgorithm() + for i := 0; i < test.wantEjections; i++ { + scw := allSubConns[len(allSubConns)-1-i] + // unejected SubConn should report latest persisted state - which is + // connecting from earlier. + gotSCWS, err = scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for Sub Conn update: %v", err) + } + if err = scwsEqual(gotSCWS.(subConnWithState), subConnWithState{ + sc: scw, + state: balancer.SubConnState{ConnectivityState: connectivity.Connecting}, + }); err != nil { + t.Fatalf("Error in Sub Conn update: %v", err) + } + } + } + }) + } +} + +// TestEjectFailureRate tests the functionality of the interval timer algorithm +// when configured with FailurePercentageEjection, and also the functionality of +// noop configuration. The Outlier Detection Balancer will be set up with 3 +// SubConns, each with a different address. It tests the following scenarios, in +// a step by step fashion: +// 1. The three addresses each have 5 successes. The interval timer algorithm +// should not eject any of the addresses. +// 2. Two of the addresses have 5 successes, the third has five failures. The +// interval timer algorithm should eject the third address with five failures. +// 3. The Outlier Detection Balancer receives a subsequent noop config update. +// The balancer should uneject all ejected addresses. +func (s) TestEjectFailureRate(t *testing.T) { + scsCh := testutils.NewChannel() + var scw1, scw2, scw3 balancer.SubConn + var err error + connectivityCh := make(chan struct{}) + stub.Register(t.Name(), stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + if scw1 != nil { // UpdateClientConnState was already called, no need to recreate SubConns. + return nil + } + scw1, err = bd.ClientConn.NewSubConn([]resolver.Address{{Addr: "address1"}}, balancer.NewSubConnOptions{ + StateListener: func(balancer.SubConnState) {}, + }) + if err != nil { + t.Errorf("error in od.NewSubConn call: %v", err) + } + scw1.Connect() + scw2, err = bd.ClientConn.NewSubConn([]resolver.Address{{Addr: "address2"}}, balancer.NewSubConnOptions{ + StateListener: func(balancer.SubConnState) {}, + }) + if err != nil { + t.Errorf("error in od.NewSubConn call: %v", err) + } + scw2.Connect() + scw3, err = bd.ClientConn.NewSubConn([]resolver.Address{{Addr: "address3"}}, balancer.NewSubConnOptions{ + StateListener: func(scs balancer.SubConnState) { + if scs.ConnectivityState == connectivity.Ready { + close(connectivityCh) + } + }, + }) + if err != nil { + t.Errorf("error in od.NewSubConn call: %v", err) + } + scw3.Connect() + return nil + }, + }) + + od, tcc, cleanup := setup(t) + defer func() { + cleanup() + }() + + od.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "address1"}}}, + {Addresses: []resolver.Address{{Addr: "address2"}}}, + {Addresses: []resolver.Address{{Addr: "address3"}}}, + }, + }, + BalancerConfig: &LBConfig{ + Interval: math.MaxInt64, // so the interval will never run unless called manually in test. + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + SuccessRateEjection: &SuccessRateEjection{ + StdevFactor: 500, + EnforcementPercentage: 100, + MinimumHosts: 3, + RequestVolume: 3, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + + od.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &rrPicker{ + scs: []balancer.SubConn{scw1, scw2, scw3}, + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Transition the SubConns to READY so that they can register health + // listeners. + for range 3 { + select { + case <-ctx.Done(): + t.Fatal("Timed out waiting for creation of new SubConn.") + case sc := <-tcc.NewSubConnCh: + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + } + // Register health listeners after all the connectivity updates are + // processed to avoid data races while accessing the health listener within + // the TestClientConn. + select { + case <-ctx.Done(): + t.Fatal("Context timed out waiting for all SubConns to become READY.") + case <-connectivityCh: + } + + scw1.RegisterHealthListener(func(healthState balancer.SubConnState) { + scsCh.Send(subConnWithState{sc: scw1, state: healthState}) + }) + scw2.RegisterHealthListener(func(healthState balancer.SubConnState) { + scsCh.Send(subConnWithState{sc: scw2, state: healthState}) + }) + scw3.RegisterHealthListener(func(healthState balancer.SubConnState) { + scsCh.Send(subConnWithState{sc: scw3, state: healthState}) + }) + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case picker := <-tcc.NewPickerCh: + // Set each upstream address to have five successes each. This should + // cause none of the addresses to be ejected as none of them are below + // the failure percentage threshold. + for i := 0; i < 3; i++ { + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("picker.Pick failed with error: %v", err) + } + for c := 0; c < 5; c++ { + pi.Done(balancer.DoneInfo{}) + } + } + tmr := stats.NewTestMetricsRecorder() + od.metricsRecorder = tmr + + od.intervalTimerAlgorithm() + sCtx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := scsCh.Receive(sCtx); err == nil { + t.Fatalf("Received unexpected subchannel state change when expecting none") + } + if got, _ := tmr.Metric("grpc.lb.outlier_detection.ejections_enforced"); got != 0 { + t.Errorf("Metric grpc.lb.outlier_detection.ejections_enforced: got %v, want 0", got) + } + if got, _ := tmr.Metric("grpc.lb.outlier_detection.ejections_unenforced"); got != 0 { + t.Errorf("Metric grpc.lb.outlier_detection.ejections_unenforced: got %v, want 0", got) + } + + // Set two upstream addresses to have five successes each, and one + // upstream address to have five failures. This should cause the address + // with five failures to be ejected according to the Failure Percentage + // Algorithm. + for i := 0; i < 2; i++ { + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("picker.Pick failed with error: %v", err) + } + for c := 0; c < 5; c++ { + pi.Done(balancer.DoneInfo{}) + } + } + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("picker.Pick failed with error: %v", err) + } + for c := 0; c < 5; c++ { + pi.Done(balancer.DoneInfo{Err: errors.New("some error")}) + } + + // should eject address that always errored. + od.intervalTimerAlgorithm() + + // verify StateListener() got called with TRANSIENT_FAILURE for child + // in address that was ejected. + gotSCWS, err := scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for Sub Conn update: %v", err) + } + if err = scwsEqual(gotSCWS.(subConnWithState), subConnWithState{ + sc: scw3, + state: balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}, + }); err != nil { + t.Fatalf("Error in Sub Conn update: %v", err) + } + + // verify only one address got ejected. + sCtx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := scsCh.Receive(sCtx); err == nil { + t.Fatalf("Received unexpected subchannel state change when expecting none") + } + if got, _ := tmr.Metric("grpc.lb.outlier_detection.ejections_enforced"); got != 1 { + t.Errorf("Metric grpc.lb.outlier_detection.ejections_enforced: got %v, want 1", got) + } + if got, _ := tmr.Metric("grpc.lb.outlier_detection.ejections_unenforced"); got != 0 { + t.Errorf("Metric grpc.lb.outlier_detection.ejections_unenforced: got %v, want 0", got) + } + + // upon the Outlier Detection balancer being reconfigured with a noop + // configuration, every ejected SubConn should be unejected. + od.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "address1"}}}, + {Addresses: []resolver.Address{{Addr: "address2"}}}, + {Addresses: []resolver.Address{{Addr: "address3"}}}, + }, + }, + BalancerConfig: &LBConfig{ + Interval: math.MaxInt64, + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + gotSCWS, err = scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for Sub Conn update: %v", err) + } + if err = scwsEqual(gotSCWS.(subConnWithState), subConnWithState{ + sc: scw3, + state: balancer.SubConnState{ConnectivityState: connectivity.Connecting}, + }); err != nil { + t.Fatalf("Error in Sub Conn update: %v", err) + } + } +} + +// TestConcurrentOperations calls different operations on the balancer in +// separate goroutines to test for any race conditions and deadlocks. It also +// uses a child balancer which verifies that no operations on the child get +// called after the child balancer is closed. +func (s) TestConcurrentOperations(t *testing.T) { + closed := grpcsync.NewEvent() + stub.Register(t.Name(), stub.BalancerFuncs{ + UpdateClientConnState: func(*stub.BalancerData, balancer.ClientConnState) error { + if closed.HasFired() { + t.Error("UpdateClientConnState was called after Close(), which breaks the balancer API") + } + return nil + }, + ResolverError: func(*stub.BalancerData, error) { + if closed.HasFired() { + t.Error("ResolverError was called after Close(), which breaks the balancer API") + } + }, + Close: func(*stub.BalancerData) { + closed.Fire() + }, + ExitIdle: func(*stub.BalancerData) { + if closed.HasFired() { + t.Error("ExitIdle was called after Close(), which breaks the balancer API") + } + }, + }) + + od, tcc, cleanup := setup(t) + defer func() { + cleanup() + }() + + od.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "address1"}}}, + {Addresses: []resolver.Address{{Addr: "address2"}}}, + {Addresses: []resolver.Address{{Addr: "address3"}}}, + }, + }, + BalancerConfig: &LBConfig{ + Interval: math.MaxInt64, // so the interval will never run unless called manually in test. + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + SuccessRateEjection: &SuccessRateEjection{ // Have both Success Rate and Failure Percentage to step through all the interval timer code + StdevFactor: 500, + EnforcementPercentage: 100, + MinimumHosts: 3, + RequestVolume: 3, + }, + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: 50, + EnforcementPercentage: 100, + MinimumHosts: 3, + RequestVolume: 3, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + scw1, err := od.NewSubConn([]resolver.Address{{Addr: "address1"}}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error in od.NewSubConn call: %v", err) + } + if err != nil { + t.Fatalf("error in od.NewSubConn call: %v", err) + } + + scw2, err := od.NewSubConn([]resolver.Address{{Addr: "address2"}}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error in od.NewSubConn call: %v", err) + } + + scw3, err := od.NewSubConn([]resolver.Address{{Addr: "address3"}}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error in od.NewSubConn call: %v", err) + } + + od.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &rrPicker{ + scs: []balancer.SubConn{scw2, scw3}, + }, + }) + + var picker balancer.Picker + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case picker = <-tcc.NewPickerCh: + } + + finished := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-finished: + return + default: + } + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + continue + } + pi.Done(balancer.DoneInfo{}) + pi.Done(balancer.DoneInfo{Err: errors.New("some error")}) + time.Sleep(1 * time.Nanosecond) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-finished: + return + default: + } + od.intervalTimerAlgorithm() + } + }() + + // call Outlier Detection's balancer.ClientConn operations asynchronously. + // balancer.ClientConn operations have no guarantee from the API to be + // called synchronously. + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-finished: + return + default: + } + od.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &rrPicker{ + scs: []balancer.SubConn{scw2, scw3}, + }, + }) + time.Sleep(1 * time.Nanosecond) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + od.NewSubConn([]resolver.Address{{Addr: "address4"}}, balancer.NewSubConnOptions{}) + }() + + wg.Add(1) + go func() { + defer wg.Done() + scw1.Shutdown() + }() + + // Call balancer.Balancers synchronously in this goroutine, upholding the + // balancer.Balancer API guarantee of synchronous calls. + od.UpdateClientConnState(balancer.ClientConnState{ // This will delete addresses and flip to no op + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "address1"}}}}, + }, + BalancerConfig: &LBConfig{ + Interval: math.MaxInt64, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + + // Call balancer.Balancers synchronously in this goroutine, upholding the + // balancer.Balancer API guarantee. + od.updateSubConnState(scw1.(*subConnWrapper), balancer.SubConnState{ + ConnectivityState: connectivity.Connecting, + }) + od.ResolverError(errors.New("some error")) + od.ExitIdle() + od.Close() + close(finished) + wg.Wait() +} + +// Tests handling of endpoints with multiple addresses. The test creates two +// endpoints, each with two addresses. The first endpoint has a backend that +// always returns errors. The test verifies that the first endpoint is ejected +// after running the intervalTimerAlgorithm. The test stops the unhealthy +// backend and verifies that the second backend in the first endpoint is dialed +// but it doesn't receive requests due to its ejection status. The test stops +// the connected backend in the second endpoint and verifies that requests +// start going to the second address in the second endpoint. The test reduces +// the ejection interval and runs the intervalTimerAlgorithm again. The test +// verifies that the first endpoint is unejected and requests reach both +// endpoints. +func (s) TestMultipleAddressesPerEndpoint(t *testing.T) { + unhealthyBackend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return nil, errors.New("some error") + }, + } + if err := unhealthyBackend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + defer unhealthyBackend.Stop() + t.Logf("Started unhealthy TestService backend at: %q", unhealthyBackend.Address) + + healthyBackends := make([]*stubserver.StubServer, 3) + for i := 0; i < 3; i++ { + healthyBackends[i] = stubserver.StartTestService(t, nil) + defer healthyBackends[i].Stop() + } + + wrrCfg, err := balancer.Get(weightedroundrobin.Name).(balancer.ConfigParser).ParseConfig(json.RawMessage("{}")) + if err != nil { + t.Fatalf("Failed to parse %q config: %v", weightedroundrobin.Name, err) + } + // The interval is intentionally kept very large, the interval algorithm + // will be triggered manually. + odCfg := &LBConfig{ + Interval: iserviceconfig.Duration(300 * time.Second), + BaseEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: 50, + EnforcementPercentage: 100, + MinimumHosts: 0, + RequestVolume: 2, + }, + MaxEjectionPercent: 100, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: weightedroundrobin.Name, + Config: wrrCfg, + }, + } + + lbChan := make(chan *outlierDetectionBalancer, 1) + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = balancer.Get(Name).Build(bd.ClientConn, bd.BuildOptions) + lbChan <- bd.ChildBalancer.(*outlierDetectionBalancer) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + ccs.BalancerConfig = odCfg + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + } + + stub.Register(t.Name(), bf) + r := manual.NewBuilderWithScheme("whatever") + endpoints := []resolver.Endpoint{ + { + Addresses: []resolver.Address{ + {Addr: unhealthyBackend.Address}, + {Addr: healthyBackends[0].Address}, + }, + }, + { + Addresses: []resolver.Address{ + {Addr: healthyBackends[1].Address}, + {Addr: healthyBackends[2].Address}, + }, + }, + } + + r.InitialState(resolver.State{ + Endpoints: endpoints, + }) + dialer := testutils.NewBlockingDialer() + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{%q: {}}] }`, t.Name())), + grpc.WithResolvers(r), + grpc.WithContextDialer(dialer.DialContext), + } + cc, err := grpc.NewClient(r.Scheme()+":///", opts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + client.EmptyCall(ctx, &testpb.Empty{}) + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Wait until both endpoints start receiving requests. + addrsSeen := map[string]bool{} + for ; ctx.Err() == nil && len(addrsSeen) < 2; <-time.After(time.Millisecond) { + var peer peer.Peer + client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)) + addrsSeen[peer.String()] = true + } + + if len(addrsSeen) < 2 { + t.Fatalf("Context timed out waiting for requests to reach both endpoints.") + } + + // Make 2 requests to each endpoint and verify the first endpoint gets + // ejected. + for i := 0; i < 2*len(endpoints); i++ { + client.EmptyCall(ctx, &testpb.Empty{}) + } + var od *outlierDetectionBalancer + select { + case <-ctx.Done(): + t.Fatal("Timed out waiting for the outlier detection LB policy to be built.") + case od = <-lbChan: + } + od.intervalTimerAlgorithm() + + // The first endpoint should be ejected, requests should only go to + // endpoints[1]. + if err := roundrobin.CheckRoundRobinRPCs(ctx, client, []resolver.Address{endpoints[1].Addresses[0]}); err != nil { + t.Fatalf("RPCs didn't go to the second endpoint: %v", err) + } + + // Shutdown the unhealthy backend. The second address in the endpoint should + // be connected, but it should be ejected by outlier detection. + hold := dialer.Hold(healthyBackends[0].Address) + unhealthyBackend.Stop() + if hold.Wait(ctx) != true { + t.Fatalf("Timeout waiting for second address in endpoint[0] with address %q to be contacted", healthyBackends[0].Address) + } + hold.Resume() + + // Verify requests go only to healthyBackends[1] for a short time. + shortCtx, cancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer cancel() + for ; shortCtx.Err() == nil; <-time.After(time.Millisecond) { + var peer peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + if status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() returned unexpected error %v", err) + } + break + } + if got, want := peer.Addr.String(), healthyBackends[1].Address; got != want { + t.Fatalf("EmptyCall() went to unexpected backend: got %q, want %q", got, want) + } + } + + // shutdown the connected backend in endpoints[1], requests should start + // going to the second address in the same endpoint. + healthyBackends[1].Stop() + if err := roundrobin.CheckRoundRobinRPCs(ctx, client, []resolver.Address{endpoints[1].Addresses[1]}); err != nil { + t.Fatalf("RPCs didn't go to second address in the second endpoint: %v", err) + } + + // Reduce the ejection interval and run the interval algorithm again, it + // should uneject endpoints[0]. + odCfg.MaxEjectionTime = 0 + odCfg.BaseEjectionTime = 0 + <-time.After(time.Millisecond) + r.UpdateState(resolver.State{Endpoints: endpoints}) + od.intervalTimerAlgorithm() + if err := roundrobin.CheckRoundRobinRPCs(ctx, client, []resolver.Address{endpoints[0].Addresses[1], endpoints[1].Addresses[1]}); err != nil { + t.Fatalf("RPCs didn't go to the second addresses of both endpoints: %v", err) + } +} + +// Tests that removing an address from an endpoint resets its ejection state. +// The test creates two endpoints, each with two addresses. The first endpoint +// has a backend that always returns errors. The test verifies that the first +// endpoint is ejected after running the intervalTimerAlgorithm. The test sends +// a resolver update that removes the first address in the ejected endpoint. The +// test verifies that requests start reaching the remaining address from the +// first endpoint. +func (s) TestEjectionStateResetsWhenEndpointAddressesChange(t *testing.T) { + unhealthyBackend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return nil, errors.New("some error") + }, + } + if err := unhealthyBackend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + defer unhealthyBackend.Stop() + t.Logf("Started unhealthy TestService backend at: %q", unhealthyBackend.Address) + + healthyBackends := make([]*stubserver.StubServer, 3) + for i := 0; i < 3; i++ { + healthyBackends[i] = stubserver.StartTestService(t, nil) + defer healthyBackends[i].Stop() + } + + wrrCfg, err := balancer.Get(weightedroundrobin.Name).(balancer.ConfigParser).ParseConfig(json.RawMessage("{}")) + if err != nil { + t.Fatalf("Failed to parse %q config: %v", weightedroundrobin.Name, err) + } + // The interval is intentionally kept very large, the interval algorithm + // will be triggered manually. + odCfg := &LBConfig{ + Interval: iserviceconfig.Duration(300 * time.Second), + BaseEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: 50, + EnforcementPercentage: 100, + MinimumHosts: 0, + RequestVolume: 2, + }, + MaxEjectionPercent: 100, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: weightedroundrobin.Name, + Config: wrrCfg, + }, + } + + lbChan := make(chan *outlierDetectionBalancer, 1) + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.ChildBalancer = balancer.Get(Name).Build(bd.ClientConn, bd.BuildOptions) + lbChan <- bd.ChildBalancer.(*outlierDetectionBalancer) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + ccs.BalancerConfig = odCfg + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + } + + stub.Register(t.Name(), bf) + r := manual.NewBuilderWithScheme("whatever") + endpoints := []resolver.Endpoint{ + { + Addresses: []resolver.Address{ + {Addr: unhealthyBackend.Address}, + {Addr: healthyBackends[0].Address}, + }, + }, + { + Addresses: []resolver.Address{ + {Addr: healthyBackends[1].Address}, + {Addr: healthyBackends[2].Address}, + }, + }, + } + + r.InitialState(resolver.State{ + Endpoints: endpoints, + }) + dialer := testutils.NewBlockingDialer() + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{%q: {}}] }`, t.Name())), + grpc.WithResolvers(r), + grpc.WithContextDialer(dialer.DialContext), + } + cc, err := grpc.NewClient(r.Scheme()+":///", opts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + client.EmptyCall(ctx, &testpb.Empty{}) + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Wait until both endpoints start receiving requests. + addrsSeen := map[string]bool{} + for ; ctx.Err() == nil && len(addrsSeen) < 2; <-time.After(time.Millisecond) { + var peer peer.Peer + client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)) + addrsSeen[peer.String()] = true + } + + if len(addrsSeen) < 2 { + t.Fatalf("Context timed out waiting for requests to reach both endpoints.") + } + + // Make 2 requests to each endpoint and verify the first endpoint gets + // ejected. + for i := 0; i < 2*len(endpoints); i++ { + client.EmptyCall(ctx, &testpb.Empty{}) + } + var od *outlierDetectionBalancer + select { + case <-ctx.Done(): + t.Fatal("Timed out waiting for the outlier detection LB policy to be built.") + case od = <-lbChan: + } + od.intervalTimerAlgorithm() + + // The first endpoint should be ejected, requests should only go to + // endpoints[1]. + if err := roundrobin.CheckRoundRobinRPCs(ctx, client, []resolver.Address{endpoints[1].Addresses[0]}); err != nil { + t.Fatalf("RPCs didn't go to the second endpoint: %v", err) + } + + // Remove the first address from the first endpoint. This makes the first + // endpoint a new endpoint for outlier detection, resetting its ejection + // status. + r.UpdateState(resolver.State{Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{endpoints[0].Addresses[1]}}, + endpoints[1], + }}) + od.intervalTimerAlgorithm() + if err := roundrobin.CheckRoundRobinRPCs(ctx, client, []resolver.Address{endpoints[0].Addresses[1], endpoints[1].Addresses[0]}); err != nil { + t.Fatalf("RPCs didn't go to the second addresses of both endpoints: %v", err) + } +} + +// TestSubConnShutdownRemovesFromEndpointMap tests that when a subconn is shut +// down, it's removed from the endpoint map. +func (s) TestSubConnShutdownRemovesFromEndpointMap(t *testing.T) { + childBalancerUpdateCh := testutils.NewChannel() + childBalancerNewSubConnCh := testutils.NewChannel() + + stub.Register(t.Name(), stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + var sc balancer.SubConn + opts := balancer.NewSubConnOptions{ + StateListener: func(scs balancer.SubConnState) { + childBalancerUpdateCh.Send(subConnWithState{sc: sc, state: scs}) + }, + } + sc, err := bd.ClientConn.NewSubConn(ccs.ResolverState.Endpoints[0].Addresses, opts) + if err != nil { + return err + } + childBalancerNewSubConnCh.Send(sc) + sc.Connect() + return nil + }, + }) + + od, tcc, cleanup := setup(t) + defer cleanup() + + addr := "address1" + ep := resolver.Endpoint{Addresses: []resolver.Address{{Addr: addr}}} + od.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ep}, + }, + BalancerConfig: &LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // The child balancer creates a subconn. + sc, err := childBalancerNewSubConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for child balancer to create a subconn: %v", err) + } + scw := sc.(*subConnWrapper) + + // The OD balancer creates an underlying subconn. + testSC := <-tcc.NewSubConnCh + + // Verify the subconn wrapper is in the endpoint info. + od.mu.Lock() + epInfo, ok := od.endpoints.Get(ep) + if !ok { + od.mu.Unlock() + t.Fatalf("epInfo not found for endpoint %v", ep) + } + if len(epInfo.sws) != 1 || epInfo.sws[0] != scw { + od.mu.Unlock() + t.Fatalf("subConnWrapper not found in endpointInfo.sws, got: %v", epInfo.sws) + } + od.mu.Unlock() + + // Simulate SHUTDOWN state update for the subconn. This is done by calling + // UpdateState on the underlying TestSubConn. + testSC.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) + + // The OD balancer will forward this update to the child. + update, err := childBalancerUpdateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for child balancer to receive subconn update: %v", err) + } + gotUpdate := update.(subConnWithState) + if gotUpdate.sc != scw { + t.Fatalf("Child balancer received update for unexpected subconn: got %v, want %v", gotUpdate.sc, scw) + } + if gotUpdate.state.ConnectivityState != connectivity.Shutdown { + t.Fatalf("Child balancer received unexpected subconn state: got %v, want %v", gotUpdate.state.ConnectivityState, connectivity.Shutdown) + } + + // Now we need to verify that the subconn wrapper is removed. + // We'll poll for a short time. + for { + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for subconn to be removed from endpoint map") + default: + } + od.mu.Lock() + epInfo, _ := od.endpoints.Get(ep) + if epInfo == nil || len(epInfo.sws) == 0 { + od.mu.Unlock() + return // Success + } + od.mu.Unlock() + <-time.After(time.Millisecond) + } +} diff --git a/internal/xds/balancer/outlierdetection/callcounter.go b/internal/xds/balancer/outlierdetection/callcounter.go new file mode 100644 index 0000000..1dd0f7e --- /dev/null +++ b/internal/xds/balancer/outlierdetection/callcounter.go @@ -0,0 +1,66 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package outlierdetection + +import ( + "sync/atomic" +) + +type bucket struct { + numSuccesses uint32 + numFailures uint32 +} + +func newCallCounter() *callCounter { + cc := &callCounter{ + inactiveBucket: &bucket{}, + } + cc.activeBucket.Store(&bucket{}) + return cc +} + +// callCounter has two buckets, which each count successful and failing RPC's. +// The activeBucket is used to actively count any finished RPC's, and the +// inactiveBucket is populated with this activeBucket's data every interval for +// use by the Outlier Detection algorithm. +type callCounter struct { + // activeBucket updates every time a call finishes (from picker passed to + // Client Conn), so protect pointer read with atomic load of the pointer + // so picker does not have to grab a mutex per RPC, the critical path. + activeBucket atomic.Pointer[bucket] + inactiveBucket *bucket +} + +func (cc *callCounter) clear() { + cc.activeBucket.Store(&bucket{}) + cc.inactiveBucket = &bucket{} +} + +// "When the timer triggers, the inactive bucket is zeroed and swapped with the +// active bucket. Then the inactive bucket contains the number of successes and +// failures since the last time the timer triggered. Those numbers are used to +// evaluate the ejection criteria." - A50. +func (cc *callCounter) swap() { + ib := cc.inactiveBucket + *ib = bucket{} + ab := cc.activeBucket.Swap(ib) + cc.inactiveBucket = &bucket{ + numSuccesses: atomic.LoadUint32(&ab.numSuccesses), + numFailures: atomic.LoadUint32(&ab.numFailures), + } +} diff --git a/internal/xds/balancer/outlierdetection/callcounter_test.go b/internal/xds/balancer/outlierdetection/callcounter_test.go new file mode 100644 index 0000000..a95666f --- /dev/null +++ b/internal/xds/balancer/outlierdetection/callcounter_test.go @@ -0,0 +1,92 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package outlierdetection + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func (b1 *bucket) Equal(b2 *bucket) bool { + if b1 == nil && b2 == nil { + return true + } + if (b1 != nil) != (b2 != nil) { + return false + } + if b1.numSuccesses != b2.numSuccesses { + return false + } + return b1.numFailures == b2.numFailures +} + +func (cc *callCounter) Equal(cc2 *callCounter) bool { + if cc == nil && cc2 == nil { + return true + } + if (cc != nil) != (cc2 != nil) { + return false + } + ab1 := cc.activeBucket.Load() + ab2 := cc2.activeBucket.Load() + if !ab1.Equal(ab2) { + return false + } + return cc.inactiveBucket.Equal(cc2.inactiveBucket) +} + +// TestClear tests that clear on the call counter clears (everything set to 0) +// the active and inactive buckets. +func (s) TestClear(t *testing.T) { + cc := newCallCounter() + ab := cc.activeBucket.Load() + ab.numSuccesses = 1 + ab.numFailures = 2 + cc.inactiveBucket.numSuccesses = 4 + cc.inactiveBucket.numFailures = 5 + cc.clear() + // Both the active and inactive buckets should be cleared. + ccWant := newCallCounter() + if diff := cmp.Diff(cc, ccWant); diff != "" { + t.Fatalf("callCounter is different than expected, diff (-got +want): %v", diff) + } +} + +// TestSwap tests that swap() on the callCounter successfully has the desired +// end result of inactive bucket containing the previous active buckets data, +// and the active bucket being cleared. +func (s) TestSwap(t *testing.T) { + cc := newCallCounter() + ab := cc.activeBucket.Load() + ab.numSuccesses = 1 + ab.numFailures = 2 + cc.inactiveBucket.numSuccesses = 4 + cc.inactiveBucket.numFailures = 5 + ib := cc.inactiveBucket + cc.swap() + // Inactive should pick up active's data, active should be swapped to zeroed + // inactive. + ccWant := newCallCounter() + ccWant.inactiveBucket.numSuccesses = 1 + ccWant.inactiveBucket.numFailures = 2 + ccWant.activeBucket.Store(ib) + if diff := cmp.Diff(cc, ccWant); diff != "" { + t.Fatalf("callCounter is different than expected, diff (-got +want): %v", diff) + } +} diff --git a/internal/xds/balancer/outlierdetection/config.go b/internal/xds/balancer/outlierdetection/config.go new file mode 100644 index 0000000..162337f --- /dev/null +++ b/internal/xds/balancer/outlierdetection/config.go @@ -0,0 +1,240 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package outlierdetection + +import ( + "encoding/json" + "time" + + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +// SuccessRateEjection is parameters for the success rate ejection algorithm. +// This algorithm monitors the request success rate for all endpoints and ejects +// individual endpoints whose success rates are statistical outliers. +type SuccessRateEjection struct { + // StddevFactor is used to determine the ejection threshold for + // success rate outlier ejection. The ejection threshold is the difference + // between the mean success rate, and the product of this factor and the + // standard deviation of the mean success rate: mean - (stdev * + // success_rate_stdev_factor). This factor is divided by a thousand to get a + // double. That is, if the desired factor is 1.9, the runtime value should + // be 1900. Defaults to 1900. + StdevFactor uint32 `json:"stdevFactor,omitempty"` + // EnforcementPercentage is the % chance that a host will be actually ejected + // when an outlier status is detected through success rate statistics. This + // setting can be used to disable ejection or to ramp it up slowly. Defaults + // to 100. + EnforcementPercentage uint32 `json:"enforcementPercentage,omitempty"` + // MinimumHosts is the number of hosts in a cluster that must have enough + // request volume to detect success rate outliers. If the number of hosts is + // less than this setting, outlier detection via success rate statistics is + // not performed for any host in the cluster. Defaults to 5. + MinimumHosts uint32 `json:"minimumHosts,omitempty"` + // RequestVolume is the minimum number of total requests that must be + // collected in one interval (as defined by the interval duration above) to + // include this host in success rate based outlier detection. If the volume + // is lower than this setting, outlier detection via success rate statistics + // is not performed for that host. Defaults to 100. + RequestVolume uint32 `json:"requestVolume,omitempty"` +} + +// For UnmarshalJSON to work correctly and set defaults without infinite +// recursion. +type successRateEjection SuccessRateEjection + +// UnmarshalJSON unmarshals JSON into SuccessRateEjection. If a +// SuccessRateEjection field is not set, that field will get its default value. +func (sre *SuccessRateEjection) UnmarshalJSON(j []byte) error { + sre.StdevFactor = 1900 + sre.EnforcementPercentage = 100 + sre.MinimumHosts = 5 + sre.RequestVolume = 100 + // Unmarshal JSON on a type with zero values for methods, including + // UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to + // avoid infinite recursion by not recalling this function and causing stack + // overflow. + return json.Unmarshal(j, (*successRateEjection)(sre)) +} + +// Equal returns whether the SuccessRateEjection is the same with the parameter. +func (sre *SuccessRateEjection) Equal(sre2 *SuccessRateEjection) bool { + if sre == nil && sre2 == nil { + return true + } + if (sre != nil) != (sre2 != nil) { + return false + } + if sre.StdevFactor != sre2.StdevFactor { + return false + } + if sre.EnforcementPercentage != sre2.EnforcementPercentage { + return false + } + if sre.MinimumHosts != sre2.MinimumHosts { + return false + } + return sre.RequestVolume == sre2.RequestVolume +} + +// FailurePercentageEjection is parameters for the failure percentage algorithm. +// This algorithm ejects individual endpoints whose failure rate is greater than +// some threshold, independently of any other endpoint. +type FailurePercentageEjection struct { + // Threshold is the failure percentage to use when determining failure + // percentage-based outlier detection. If the failure percentage of a given + // host is greater than or equal to this value, it will be ejected. Defaults + // to 85. + Threshold uint32 `json:"threshold,omitempty"` + // EnforcementPercentage is the % chance that a host will be actually + // ejected when an outlier status is detected through failure percentage + // statistics. This setting can be used to disable ejection or to ramp it up + // slowly. Defaults to 0. + EnforcementPercentage uint32 `json:"enforcementPercentage,omitempty"` + // MinimumHosts is the minimum number of hosts in a cluster in order to + // perform failure percentage-based ejection. If the total number of hosts + // in the cluster is less than this value, failure percentage-based ejection + // will not be performed. Defaults to 5. + MinimumHosts uint32 `json:"minimumHosts,omitempty"` + // RequestVolume is the minimum number of total requests that must be + // collected in one interval (as defined by the interval duration above) to + // perform failure percentage-based ejection for this host. If the volume is + // lower than this setting, failure percentage-based ejection will not be + // performed for this host. Defaults to 50. + RequestVolume uint32 `json:"requestVolume,omitempty"` +} + +// For UnmarshalJSON to work correctly and set defaults without infinite +// recursion. +type failurePercentageEjection FailurePercentageEjection + +// UnmarshalJSON unmarshals JSON into FailurePercentageEjection. If a +// FailurePercentageEjection field is not set, that field will get its default +// value. +func (fpe *FailurePercentageEjection) UnmarshalJSON(j []byte) error { + fpe.Threshold = 85 + fpe.EnforcementPercentage = 0 + fpe.MinimumHosts = 5 + fpe.RequestVolume = 50 + // Unmarshal JSON on a type with zero values for methods, including + // UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to + // avoid infinite recursion by not recalling this function and causing stack + // overflow. + return json.Unmarshal(j, (*failurePercentageEjection)(fpe)) +} + +// Equal returns whether the FailurePercentageEjection is the same with the +// parameter. +func (fpe *FailurePercentageEjection) Equal(fpe2 *FailurePercentageEjection) bool { + if fpe == nil && fpe2 == nil { + return true + } + if (fpe != nil) != (fpe2 != nil) { + return false + } + if fpe.Threshold != fpe2.Threshold { + return false + } + if fpe.EnforcementPercentage != fpe2.EnforcementPercentage { + return false + } + if fpe.MinimumHosts != fpe2.MinimumHosts { + return false + } + return fpe.RequestVolume == fpe2.RequestVolume +} + +// LBConfig is the config for the outlier detection balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + // Interval is the time interval between ejection analysis sweeps. This can + // result in both new ejections as well as addresses being returned to + // service. Defaults to 10s. + Interval iserviceconfig.Duration `json:"interval,omitempty"` + // BaseEjectionTime is the base time that a host is ejected for. The real + // time is equal to the base time multiplied by the number of times the host + // has been ejected and is capped by MaxEjectionTime. Defaults to 30s. + BaseEjectionTime iserviceconfig.Duration `json:"baseEjectionTime,omitempty"` + // MaxEjectionTime is the maximum time that an endpoint is ejected for. If + // not specified, the default value (300s) or the BaseEjectionTime value is + // applied, whichever is larger. + MaxEjectionTime iserviceconfig.Duration `json:"maxEjectionTime,omitempty"` + // MaxEjectionPercent is the maximum % of an upstream cluster that can be + // ejected due to outlier detection. Defaults to 10% but will eject at least + // one host regardless of the value. + MaxEjectionPercent uint32 `json:"maxEjectionPercent,omitempty"` + // SuccessRateEjection is the parameters for the success rate ejection + // algorithm. If set, success rate ejections will be performed. + SuccessRateEjection *SuccessRateEjection `json:"successRateEjection,omitempty"` + // FailurePercentageEjection is the parameters for the failure percentage + // algorithm. If set, failure rate ejections will be performed. + FailurePercentageEjection *FailurePercentageEjection `json:"failurePercentageEjection,omitempty"` + // ChildPolicy is the config for the child policy. + ChildPolicy *iserviceconfig.BalancerConfig `json:"childPolicy,omitempty"` +} + +// For UnmarshalJSON to work correctly and set defaults without infinite +// recursion. +type lbConfig LBConfig + +// UnmarshalJSON unmarshals JSON into LBConfig. If a top level LBConfig field +// (i.e. not next layer sre or fpe) is not set, that field will get its default +// value. If sre or fpe is not set, it will stay unset, otherwise it will +// unmarshal on those types populating with default values for their fields if +// needed. +func (lbc *LBConfig) UnmarshalJSON(j []byte) error { + // Default top layer values as documented in A50. + lbc.Interval = iserviceconfig.Duration(10 * time.Second) + lbc.BaseEjectionTime = iserviceconfig.Duration(30 * time.Second) + lbc.MaxEjectionTime = iserviceconfig.Duration(300 * time.Second) + lbc.MaxEjectionPercent = 10 + // Unmarshal JSON on a type with zero values for methods, including + // UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to + // avoid infinite recursion by not recalling this function and causing stack + // overflow. + return json.Unmarshal(j, (*lbConfig)(lbc)) +} + +// EqualIgnoringChildPolicy returns whether the LBConfig is same with the +// parameter outside of the child policy, only comparing the Outlier Detection +// specific configuration. +func (lbc *LBConfig) EqualIgnoringChildPolicy(lbc2 *LBConfig) bool { + if lbc == nil && lbc2 == nil { + return true + } + if (lbc != nil) != (lbc2 != nil) { + return false + } + if lbc.Interval != lbc2.Interval { + return false + } + if lbc.BaseEjectionTime != lbc2.BaseEjectionTime { + return false + } + if lbc.MaxEjectionTime != lbc2.MaxEjectionTime { + return false + } + if lbc.MaxEjectionPercent != lbc2.MaxEjectionPercent { + return false + } + if !lbc.SuccessRateEjection.Equal(lbc2.SuccessRateEjection) { + return false + } + return lbc.FailurePercentageEjection.Equal(lbc2.FailurePercentageEjection) +} diff --git a/internal/xds/balancer/outlierdetection/config_test.go b/internal/xds/balancer/outlierdetection/config_test.go new file mode 100644 index 0000000..ce924dc --- /dev/null +++ b/internal/xds/balancer/outlierdetection/config_test.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package outlierdetection + +import ( + "reflect" + "testing" +) + +func TestSuccessRateEjection(t *testing.T) { + fields := map[string]bool{ + "StdevFactor": true, + "EnforcementPercentage": true, + "MinimumHosts": true, + "RequestVolume": true, + } + typ := reflect.TypeOf(SuccessRateEjection{}) + for i := 0; i < typ.NumField(); i++ { + if n := typ.Field(i).Name; !fields[n] { + t.Errorf("New field in SuccessRateEjection %q, update this test and Equal", n) + } + } +} + +func TestEqualFieldsFailurePercentageEjection(t *testing.T) { + fields := map[string]bool{ + "Threshold": true, + "EnforcementPercentage": true, + "MinimumHosts": true, + "RequestVolume": true, + } + typ := reflect.TypeOf(FailurePercentageEjection{}) + for i := 0; i < typ.NumField(); i++ { + if n := typ.Field(i).Name; !fields[n] { + t.Errorf("New field in FailurePercentageEjection %q, update this test and Equal", n) + } + } +} + +func TestEqualFieldsLBConfig(t *testing.T) { + fields := map[string]bool{ + "LoadBalancingConfig": true, + "Interval": true, + "BaseEjectionTime": true, + "MaxEjectionTime": true, + "MaxEjectionPercent": true, + "SuccessRateEjection": true, + "FailurePercentageEjection": true, + "ChildPolicy": true, + } + typ := reflect.TypeOf(LBConfig{}) + for i := 0; i < typ.NumField(); i++ { + if n := typ.Field(i).Name; !fields[n] { + t.Errorf("New field in LBConfig %q, update this test and EqualIgnoringChildPolicy", n) + } + } +} diff --git a/internal/xds/balancer/outlierdetection/e2e_test/outlierdetection_test.go b/internal/xds/balancer/outlierdetection/e2e_test/outlierdetection_test.go new file mode 100644 index 0000000..7fd67d2 --- /dev/null +++ b/internal/xds/balancer/outlierdetection/e2e_test/outlierdetection_test.go @@ -0,0 +1,425 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package e2e_test contains e2e test cases for the Outlier Detection LB Policy. +package e2e_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/internal/xds/balancer/outlierdetection" // To register helper functions which register/unregister Outlier Detection LB Policy. +) + +var ( + defaultTestTimeout = 5 * time.Second + leafPolicyName = "round_robin" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// Setup spins up three test backends, each listening on a port on localhost. +// Two of the backends are configured to always reply with an empty response and +// no error and one is configured to always return an error. +func setupBackends(t *testing.T) ([]string, func()) { + t.Helper() + + backends := make([]*stubserver.StubServer, 3) + addresses := make([]string, 3) + // Construct and start 2 working backends. + for i := 0; i < 2; i++ { + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started good TestService backend at: %q", backend.Address) + backends[i] = backend + addresses[i] = backend.Address + } + + // Construct and start a failing backend. + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return nil, errors.New("some error") + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started bad TestService backend at: %q", backend.Address) + backends[2] = backend + addresses[2] = backend.Address + cancel := func() { + for _, backend := range backends { + backend.Stop() + } + } + return addresses, cancel +} + +// checkRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn, +// connected to a server exposing the test.grpc_testing.TestService, are +// roundrobined across the given backend addresses. +// +// Returns a non-nil error if context deadline expires before RPCs start to get +// roundrobined across the given backends. +func checkRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error { + wantAddrCount := make(map[string]int) + for _, addr := range addrs { + wantAddrCount[addr.Addr]++ + } + gotAddrCount := make(map[string]int) + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + gotAddrCount = make(map[string]int) + // Perform 3 iterations. + var iterations [][]string + for i := 0; i < 3; i++ { + iteration := make([]string, len(addrs)) + for c := 0; c < len(addrs); c++ { + var peer peer.Peer + client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)) + if peer.Addr != nil { + iteration[c] = peer.Addr.String() + } + } + iterations = append(iterations, iteration) + } + // Ensure the first iteration contains all addresses in addrs. + for _, addr := range iterations[0] { + gotAddrCount[addr]++ + } + if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" { + continue + } + // Ensure all three iterations contain the same addresses. + if !cmp.Equal(iterations[0], iterations[1]) || !cmp.Equal(iterations[0], iterations[2]) { + continue + } + return nil + } + return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v; got: %v", addrs, gotAddrCount) +} + +// TestPickFirstHealthListenerDisabled verifies that outlier detection doesn't +// eject subchannels created by pick_first when it is not under a petiole +// policy. In that configuration, pick_first does not register a health +// listener and ejection updates must not affect connectivity. +func (s) TestPickFirstHealthListenerDisabled(t *testing.T) { + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return nil, errors.New("some error") + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + defer backend.Stop() + + const serviceConfig = `{ + "loadBalancingConfig": [{ + "outlier_detection_experimental": { + "interval": "0.050s", + "failurePercentageEjection": { + "threshold": 50, + "enforcementPercentage": 100, + "minimumHosts": 0, + "requestVolume": 2 + }, + "childPolicy": [{"pick_first": {}}] + } + }] + }` + cc, err := grpc.NewClient(backend.Address, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(serviceConfig)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + client.EmptyCall(ctx, &testpb.Empty{}) + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + for range 10 { + client.EmptyCall(ctx, &testpb.Empty{}) + } + + // Wait for three outlier detection intervals to verify that the subchannel is not ejected. + sCtx, sCancel := context.WithTimeout(ctx, 150*time.Millisecond) + defer sCancel() + testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready) +} + +// TestOutlierDetectionAlgorithmsE2E tests the Outlier Detection Success Rate +// and Failure Percentage algorithms in an e2e fashion. The Outlier Detection +// Balancer is configured as the top level LB Policy of the channel with a Round +// Robin child, and connects to three upstreams. Two of the upstreams are healthy and +// one is unhealthy. The two algorithms should at some point eject the failing +// upstream, causing RPC's to not be routed to that upstream, and only be +// Round Robined across the two healthy upstreams. Other than the intervals the +// unhealthy upstream is ejected, RPC's should regularly round robin +// across all three upstreams. +func (s) TestOutlierDetectionAlgorithmsE2E(t *testing.T) { + tests := []struct { + name string + odscJSON string + }{ + { + name: "Success Rate Algorithm", + odscJSON: fmt.Sprintf(` + { + "loadBalancingConfig": [ + { + "outlier_detection_experimental": { + "interval": "0.050s", + "baseEjectionTime": "0.100s", + "maxEjectionTime": "300s", + "maxEjectionPercent": 33, + "successRateEjection": { + "stdevFactor": 50, + "enforcementPercentage": 100, + "minimumHosts": 3, + "requestVolume": 5 + }, + "childPolicy": [{"%s": {}}] + } + } + ] + }`, leafPolicyName), + }, + { + name: "Failure Percentage Algorithm", + odscJSON: fmt.Sprintf(` + { + "loadBalancingConfig": [ + { + "outlier_detection_experimental": { + "interval": "0.050s", + "baseEjectionTime": "0.100s", + "maxEjectionTime": "300s", + "maxEjectionPercent": 33, + "failurePercentageEjection": { + "threshold": 50, + "enforcementPercentage": 100, + "minimumHosts": 3, + "requestVolume": 5 + }, + "childPolicy": [{"%s": {}} + ] + } + } + ] + }`, leafPolicyName), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + addresses, cancel := setupBackends(t) + defer cancel() + + mr := manual.NewBuilderWithScheme("od-e2e") + defer mr.Close() + + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(test.odscJSON) + // The full list of addresses. + fullAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + {Addr: addresses[2]}, + } + mr.InitialState(resolver.State{ + Addresses: fullAddresses, + ServiceConfig: sc, + }) + + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + + // At first, due to no statistics on each of the backends, the 3 + // upstreams should all be round robined across. + if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // The addresses which don't return errors. + okAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + } + // After calling the three upstreams, one of them constantly error + // and should eventually be ejected for a period of time. This + // period of time should cause the RPC's to be round robined only + // across the two that are healthy. + if err = checkRoundRobinRPCs(ctx, testServiceClient, okAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // The failing upstream isn't ejected indefinitely, and eventually + // should be unejected in subsequent iterations of the interval + // algorithm as per the spec for the two specific algorithms. + if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + }) + } +} + +// TestNoopConfiguration tests the Outlier Detection Balancer configured with a +// noop configuration. The noop configuration should cause the Outlier Detection +// Balancer to not count RPC's, and thus never eject any upstreams and continue +// to route to every upstream connected to, even if they continuously error. +// Once the Outlier Detection Balancer gets reconfigured with configuration +// requiring counting RPC's, the Outlier Detection Balancer should start +// ejecting any upstreams as specified in the configuration. +func (s) TestNoopConfiguration(t *testing.T) { + addresses, cancel := setupBackends(t) + defer cancel() + + mr := manual.NewBuilderWithScheme("od-e2e") + defer mr.Close() + + noopODServiceConfigJSON := fmt.Sprintf(` + { + "loadBalancingConfig": [ + { + "outlier_detection_experimental": { + "interval": "0.050s", + "baseEjectionTime": "0.100s", + "maxEjectionTime": "300s", + "maxEjectionPercent": 33, + "childPolicy": [{"%s": {}}] + } + } + ] + }`, leafPolicyName) + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(noopODServiceConfigJSON) + // The full list of addresses. + fullAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + {Addr: addresses[2]}, + } + mr.InitialState(resolver.State{ + Addresses: fullAddresses, + ServiceConfig: sc, + }) + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + + for i := 0; i < 2; i++ { + // Since the Outlier Detection Balancer starts with a noop + // configuration, it shouldn't count RPCs or eject any upstreams. Thus, + // even though an upstream it connects to constantly errors, it should + // continue to Round Robin across every upstream. + if err := checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + } + + // Reconfigure the Outlier Detection Balancer with a configuration that + // specifies to count RPC's and eject upstreams. Due to the balancer no + // longer being a noop, it should eject any unhealthy addresses as specified + // by the failure percentage portion of the configuration. + countingODServiceConfigJSON := fmt.Sprintf(` + { + "loadBalancingConfig": [ + { + "outlier_detection_experimental": { + "interval": "0.050s", + "baseEjectionTime": "0.100s", + "maxEjectionTime": "300s", + "maxEjectionPercent": 33, + "failurePercentageEjection": { + "threshold": 50, + "enforcementPercentage": 100, + "minimumHosts": 3, + "requestVolume": 5 + }, + "childPolicy": [{"%s": {}}] + } + } + ] + }`, leafPolicyName) + sc = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(countingODServiceConfigJSON) + + mr.UpdateState(resolver.State{ + Addresses: fullAddresses, + ServiceConfig: sc, + }) + + // At first on the reconfigured balancer, the balancer has no stats + // collected about upstreams. Thus, it should at first route across the full + // upstream list. + if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // The addresses which don't return errors. + okAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + } + // Now that the reconfigured balancer has data about the failing upstream, + // it should eject the upstream and only route across the two healthy + // upstreams. + if err = checkRoundRobinRPCs(ctx, testServiceClient, okAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } +} diff --git a/internal/xds/balancer/outlierdetection/logging.go b/internal/xds/balancer/outlierdetection/logging.go new file mode 100644 index 0000000..705b0cb --- /dev/null +++ b/internal/xds/balancer/outlierdetection/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package outlierdetection + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[outlier-detection-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *outlierDetectionBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/internal/xds/balancer/outlierdetection/subconn_wrapper.go b/internal/xds/balancer/outlierdetection/subconn_wrapper.go new file mode 100644 index 0000000..ed9d963 --- /dev/null +++ b/internal/xds/balancer/outlierdetection/subconn_wrapper.go @@ -0,0 +1,175 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package outlierdetection + +import ( + "fmt" + "sync" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/resolver" +) + +// subConnWrapper wraps every created SubConn in the Outlier Detection Balancer, +// to help track the latest state update from the underlying SubConn, and also +// whether or not this SubConn is ejected. +type subConnWrapper struct { + balancer.SubConn + // endpointInfo is a pointer to the subConnWrapper's corresponding endpoint + // map entry, if the map entry exists. + endpointInfo *endpointInfo + // The following fields are set during object creation and read-only after + // that. + + listener func(balancer.SubConnState) + + scUpdateCh *buffer.Unbounded[any] + + // The following fields are only referenced in the context of a work + // serializing buffer and don't need to be protected by a mutex. + + // These two pieces of state will reach eventual consistency due to sync in + // run(), and child will always have the correctly updated SubConnState. + + ejected bool + + // addresses is the list of address(es) this SubConn was created with to + // help support any change in address(es) + addresses []resolver.Address + // latestHealthState is tracked to update the child policy during + // unejection. + latestHealthState balancer.SubConnState + + // Access to the following fields are protected by a mutex. These fields + // should not be accessed from outside this file, instead use methods + // defined on the struct. + mu sync.Mutex + healthListener func(balancer.SubConnState) + // latestReceivedConnectivityState is the SubConn's most recent connectivity + // state received. It may not be delivered to the child balancer yet. It is + // used to ensure a health listener is registered with the SubConn only when + // the SubConn is READY. + latestReceivedConnectivityState connectivity.State +} + +// eject causes the wrapper to report a state update with the TRANSIENT_FAILURE +// state, and to stop passing along updates from the underlying subchannel. +func (scw *subConnWrapper) eject() { + scw.scUpdateCh.Put(&ejectionUpdate{ + scw: scw, + isEjected: true, + }) +} + +// uneject causes the wrapper to report a state update with the latest update +// from the underlying subchannel, and resume passing along updates from the +// underlying subchannel. +func (scw *subConnWrapper) uneject() { + scw.scUpdateCh.Put(&ejectionUpdate{ + scw: scw, + isEjected: false, + }) +} + +func (scw *subConnWrapper) String() string { + return fmt.Sprintf("%+v", scw.addresses) +} + +func (scw *subConnWrapper) RegisterHealthListener(listener func(balancer.SubConnState)) { + // gRPC currently supports two mechanisms that provide a health signal for + // a connection: client-side health checking and outlier detection. Earlier + // both these mechanisms signaled unhealthiness by setting the subchannel + // state to TRANSIENT_FAILURE. As part of the dualstack changes to make + // pick_first the universal leaf policy (see A61), both these mechanisms + // started using the new health listener to make health signal visible to + // the petiole policies without affecting the underlying connectivity + // management of the pick_first policy. + scw.mu.Lock() + defer scw.mu.Unlock() + + if scw.latestReceivedConnectivityState != connectivity.Ready { + return + } + scw.healthListener = listener + if listener == nil { + scw.SubConn.RegisterHealthListener(nil) + return + } + + scw.SubConn.RegisterHealthListener(func(scs balancer.SubConnState) { + scw.scUpdateCh.Put(&scHealthUpdate{ + scw: scw, + state: scs, + }) + }) +} + +// updateSubConnHealthState stores the latest health state for unejection and +// sends updates the health listener. +func (scw *subConnWrapper) updateSubConnHealthState(scs balancer.SubConnState) { + scw.latestHealthState = scs + if scw.ejected { + return + } + scw.mu.Lock() + defer scw.mu.Unlock() + if scw.healthListener != nil { + scw.healthListener(scs) + } +} + +// updateSubConnConnectivityState stores the latest connectivity state for +// unejection and updates the raw connectivity listener. +func (scw *subConnWrapper) updateSubConnConnectivityState(scs balancer.SubConnState) { + if scw.listener != nil { + scw.listener(scs) + } +} + +func (scw *subConnWrapper) clearHealthListener() { + scw.mu.Lock() + defer scw.mu.Unlock() + scw.healthListener = nil +} + +func (scw *subConnWrapper) handleUnejection() { + scw.ejected = false + // If scw.latestHealthState has never been written to will use the health + // state CONNECTING set during object creation. + scw.updateSubConnHealthState(scw.latestHealthState) +} + +func (scw *subConnWrapper) handleEjection() { + scw.ejected = true + stateToUpdate := balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + } + scw.mu.Lock() + defer scw.mu.Unlock() + if scw.healthListener != nil { + scw.healthListener(stateToUpdate) + } +} + +func (scw *subConnWrapper) setLatestConnectivityState(state connectivity.State) { + scw.mu.Lock() + defer scw.mu.Unlock() + scw.latestReceivedConnectivityState = state +} diff --git a/internal/xds/balancer/priority/balancer.go b/internal/xds/balancer/priority/balancer.go new file mode 100644 index 0000000..7ce430e --- /dev/null +++ b/internal/xds/balancer/priority/balancer.go @@ -0,0 +1,292 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package priority implements the priority balancer. +// +// This balancer will be kept in internal until we use it in the xds balancers, +// and are confident its functionalities are stable. It will then be exported +// for more users. +package priority + +import ( + "encoding/json" + "fmt" + "sync" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancergroup" + "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/hierarchy" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Name is the name of the priority balancer. +const Name = "priority_experimental" + +// DefaultSubBalancerCloseTimeout is defined as a variable instead of const for +// testing. +var DefaultSubBalancerCloseTimeout = 15 * time.Minute + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &priorityBalancer{ + cc: cc, + done: grpcsync.NewEvent(), + children: make(map[string]*childBalancer), + childBalancerStateUpdate: buffer.NewUnbounded[any](), + } + + b.logger = prefixLogger(b) + b.bg = balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: bOpts, + StateAggregator: b, + Logger: b.logger, + SubBalancerCloseTimeout: DefaultSubBalancerCloseTimeout, + }) + go b.run() + b.logger.Infof("Created") + return b +} + +func (b bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(s) +} + +func (bb) Name() string { + return Name +} + +// timerWrapper wraps a timer with a boolean. So that when a race happens +// between AfterFunc and Stop, the func is guaranteed to not execute. +type timerWrapper struct { + stopped bool + timer *time.Timer +} + +type priorityBalancer struct { + logger *grpclog.PrefixLogger + cc balancer.ClientConn + bg *balancergroup.BalancerGroup + done *grpcsync.Event + childBalancerStateUpdate *buffer.Unbounded[any] + + mu sync.Mutex + childInUse string + // priorities is a list of child names from higher to lower priority. + priorities []string + // children is a map from child name to sub-balancers. + children map[string]*childBalancer + + // Set during UpdateClientConnState when calling into sub-balancers. + // Prevents child updates from recomputing the active priority or sending + // an update of the aggregated picker to the parent. Cleared after all + // sub-balancers have finished UpdateClientConnState, after which + // syncPriority is called manually. + inhibitPickerUpdates bool +} + +func (b *priorityBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + if b.logger.V(2) { + b.logger.Infof("Received an update with balancer config: %+v", pretty.ToJSON(s.BalancerConfig)) + } + newConfig, ok := s.BalancerConfig.(*LBConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) + } + endpointsSplit := hierarchy.Group(s.ResolverState.Endpoints) + + b.mu.Lock() + // Create and remove children, since we know all children from the config + // are used by some priority. + for name, newSubConfig := range newConfig.Children { + bb := balancer.Get(newSubConfig.Config.Name) + if bb == nil { + b.logger.Errorf("balancer name %v from config is not registered", newSubConfig.Config.Name) + continue + } + + currentChild, ok := b.children[name] + if !ok { + // This is a new child, add it to the children list. But note that + // the balancer isn't built, because this child can be a low + // priority. If necessary, it will be built when syncing priorities. + cb := newChildBalancer(name, b, bb.Name(), b.cc) + cb.updateConfig(newSubConfig, resolver.State{ + Endpoints: endpointsSplit[name], + ServiceConfig: s.ResolverState.ServiceConfig, + Attributes: s.ResolverState.Attributes, + }) + b.children[name] = cb + continue + } + + // This is not a new child. But the config/addresses could change. + + // The balancing policy name is changed, close the old child. But don't + // rebuild, rebuild will happen when syncing priorities. + if currentChild.balancerName != bb.Name() { + currentChild.stop(true) + currentChild.updateBalancerName(bb.Name()) + } + + // Update config and address, but note that this doesn't send the + // updates to non-started child balancers (the child balancer might not + // be built, if it's a low priority). + currentChild.updateConfig(newSubConfig, resolver.State{ + Endpoints: endpointsSplit[name], + ServiceConfig: s.ResolverState.ServiceConfig, + Attributes: s.ResolverState.Attributes, + }) + } + // Cleanup resources used by children removed from the config. + for name, oldChild := range b.children { + if _, ok := newConfig.Children[name]; !ok { + oldChild.stop(!envconfig.EnablePriorityLBChildPolicyCache) + delete(b.children, name) + } + } + + // Update priorities and handle priority changes. + b.priorities = newConfig.Priorities + + // Everything was removed by the update. + if len(b.priorities) == 0 { + b.childInUse = "" + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(ErrAllPrioritiesRemoved), + }) + b.mu.Unlock() + return nil + } + + // This will sync the states of all children to the new updated + // priorities. Includes starting/stopping child balancers when necessary. + // Block picker updates until all children have had a chance to call + // UpdateState to prevent races where, e.g., the active priority reports + // transient failure but a higher priority may have reported something that + // made it active, and if the transient failure update is handled first, + // RPCs could fail. + b.inhibitPickerUpdates = true + // Add an item to queue to notify us when the current items in the queue + // are done and syncPriority has been called. + done := make(chan struct{}) + b.childBalancerStateUpdate.Put(resumePickerUpdates{done: done}) + b.mu.Unlock() + <-done + + return nil +} + +func (b *priorityBalancer) ResolverError(err error) { + if b.logger.V(2) { + b.logger.Infof("Received error from the resolver: %v", err) + } + b.bg.ResolverError(err) +} + +func (b *priorityBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *priorityBalancer) Close() { + b.bg.Close() + b.childBalancerStateUpdate.Close() + + b.mu.Lock() + defer b.mu.Unlock() + b.done.Fire() + // Clear states of the current child in use, so if there's a race in picker + // update, it will be dropped. + b.childInUse = "" + // Stop the child policies, this is necessary to stop the init timers in the + // children. + for _, child := range b.children { + child.stop(true) + } +} + +func (b *priorityBalancer) ExitIdle() { + b.bg.ExitIdle() +} + +// UpdateState implements balancergroup.BalancerStateAggregator interface. The +// balancer group sends new connectivity state and picker here. +func (b *priorityBalancer) UpdateState(childName string, state balancer.State) { + b.childBalancerStateUpdate.Put(childBalancerState{ + name: childName, + s: state, + }) +} + +type childBalancerState struct { + name string + s balancer.State +} + +type resumePickerUpdates struct { + done chan struct{} +} + +// run handles child update in a separate goroutine, so if the child sends +// updates inline (when called by parent), it won't cause deadlocks (by trying +// to hold the same mutex). +func (b *priorityBalancer) run() { + for { + select { + case u, ok := <-b.childBalancerStateUpdate.Get(): + if !ok { + return + } + b.childBalancerStateUpdate.Load() + // Needs to handle state update in a goroutine, because each state + // update needs to start/close child policy, could result in + // deadlock. + b.mu.Lock() + if b.done.HasFired() { + b.mu.Unlock() + return + } + switch s := u.(type) { + case childBalancerState: + b.handleChildStateUpdate(s.name, s.s) + case resumePickerUpdates: + b.inhibitPickerUpdates = false + b.syncPriority(b.childInUse) + close(s.done) + } + b.mu.Unlock() + case <-b.done.Done(): + return + } + } +} diff --git a/internal/xds/balancer/priority/balancer_child.go b/internal/xds/balancer/priority/balancer_child.go new file mode 100644 index 0000000..ad5b5a8 --- /dev/null +++ b/internal/xds/balancer/priority/balancer_child.go @@ -0,0 +1,186 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +var timeAfterFunc = time.AfterFunc + +type childBalancer struct { + name string + parent *priorityBalancer + parentCC balancer.ClientConn + balancerName string + cc *ignoreResolveNowClientConn + + ignoreReresolutionRequests bool + config serviceconfig.LoadBalancingConfig + rState resolver.State + + started bool + // This is set when the child reports TransientFailure, and unset when it + // reports Ready or Idle. It is used to decide whether the failover timer + // should start when the child is transitioning into Connecting. The timer + // will be restarted if the child has not reported TF more recently than it + // reported Ready or Idle. + reportedTF bool + // The latest state the child balancer provided. + state balancer.State + // The timer to give a priority some time to connect. And if the priority + // doesn't go into Ready/Failure, the next priority will be started. + initTimer *timerWrapper +} + +// newChildBalancer creates a child balancer place holder, but doesn't +// build/start the child balancer. +func newChildBalancer(name string, parent *priorityBalancer, balancerName string, cc balancer.ClientConn) *childBalancer { + return &childBalancer{ + name: name, + parent: parent, + parentCC: cc, + balancerName: balancerName, + cc: newIgnoreResolveNowClientConn(cc, false), + started: false, + // Start with the connecting state and picker with re-pick error, so + // that when a priority switch causes this child picked before it's + // balancing policy is created, a re-pick will happen. + state: balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + }, + } +} + +// updateBalancerName updates balancer name for the child, but doesn't build a +// new one. The parent priority LB always closes the child policy before +// updating the balancer name, and the new balancer is built when it gets added +// to the balancergroup as part of start(). +func (cb *childBalancer) updateBalancerName(balancerName string) { + cb.balancerName = balancerName + cb.cc = newIgnoreResolveNowClientConn(cb.parentCC, cb.ignoreReresolutionRequests) +} + +// updateConfig sets childBalancer's config and state, but doesn't send update to +// the child balancer unless it is started. +func (cb *childBalancer) updateConfig(child *Child, rState resolver.State) { + cb.ignoreReresolutionRequests = child.IgnoreReresolutionRequests + cb.config = child.Config.Config + cb.rState = rState + if cb.started { + cb.sendUpdate() + } +} + +// start builds the child balancer if it's not already started. +// +// It doesn't do it directly. It asks the balancer group to build it. +func (cb *childBalancer) start() { + if cb.started { + return + } + cb.started = true + cb.parent.bg.AddWithClientConn(cb.name, cb.balancerName, cb.cc) + cb.startInitTimer() + cb.sendUpdate() +} + +// sendUpdate sends the addresses and config to the child balancer. +func (cb *childBalancer) sendUpdate() { + cb.cc.updateIgnoreResolveNow(cb.ignoreReresolutionRequests) + // TODO: return and aggregate the returned error in the parent. + err := cb.parent.bg.UpdateClientConnState(cb.name, balancer.ClientConnState{ + ResolverState: cb.rState, + BalancerConfig: cb.config, + }) + // Report TF if update to the child fails. + if err != nil { + cb.parent.logger.Warningf("Failed to update state for child policy %q: %v", cb.name, err) + cb.reportedTF = true + cb.state = balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(err), + } + cb.parent.handleChildStateUpdate(cb.name, cb.state) + } +} + +// stop stops the child balancer and resets the state. +// +// It doesn't do it directly. It asks the balancer group to remove it. +// +// Note that the underlying balancer group could keep the child in a cache. +func (cb *childBalancer) stop(immediate bool) { + if !cb.started { + return + } + cb.stopInitTimer() + if immediate { + cb.parent.bg.RemoveImmediately(cb.name) + } else { + cb.parent.bg.Remove(cb.name) + } + cb.started = false + cb.state = balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + } + // Clear child.reportedTF, so that if this child is started later, it will + // be given time to connect. + cb.reportedTF = false +} + +func (cb *childBalancer) startInitTimer() { + if cb.initTimer != nil { + return + } + // Need this local variable to capture timerW in the AfterFunc closure + // to check the stopped boolean. + timerW := &timerWrapper{} + cb.initTimer = timerW + timerW.timer = timeAfterFunc(DefaultPriorityInitTimeout, func() { + cb.parent.mu.Lock() + defer cb.parent.mu.Unlock() + if timerW.stopped { + return + } + cb.initTimer = nil + // Re-sync the priority. This will switch to the next priority if + // there's any. Note that it's important sync() is called after setting + // initTimer to nil. + cb.parent.syncPriority("") + }) +} + +func (cb *childBalancer) stopInitTimer() { + timerW := cb.initTimer + if timerW == nil { + return + } + cb.initTimer = nil + timerW.stopped = true + timerW.timer.Stop() +} diff --git a/internal/xds/balancer/priority/balancer_priority.go b/internal/xds/balancer/priority/balancer_priority.go new file mode 100644 index 0000000..3181ce2 --- /dev/null +++ b/internal/xds/balancer/priority/balancer_priority.go @@ -0,0 +1,223 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "errors" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" +) + +var ( + // ErrAllPrioritiesRemoved is returned by the picker when there's no priority available. + ErrAllPrioritiesRemoved = errors.New("no priority is provided, all priorities are removed") + // DefaultPriorityInitTimeout is the timeout after which if a priority is + // not READY, the next will be started. It's exported to be overridden by + // tests. + DefaultPriorityInitTimeout = 10 * time.Second +) + +// syncPriority handles priority after a config update or a child balancer +// connectivity state update. It makes sure the balancer state (started or not) +// is in sync with the priorities (even in tricky cases where a child is moved +// from a priority to another). +// +// It's guaranteed that after this function returns: +// +// If some child is READY, it is childInUse, and all lower priorities are +// closed. +// +// If some child is newly started(in Connecting for the first time), it is +// childInUse, and all lower priorities are closed. +// +// Otherwise, the lowest priority is childInUse (none of the children is +// ready, and the overall state is not ready). +// +// Steps: +// +// If all priorities were deleted, unset childInUse (to an empty string), and +// set parent ClientConn to TransientFailure +// +// Otherwise, Scan all children from p0, and check balancer stats: +// +// For any of the following cases: +// +// If balancer is not started (not built), this is either a new child with +// high priority, or a new builder for an existing child. +// +// If balancer is Connecting and has non-nil initTimer (meaning it +// transitioned from Ready or Idle to connecting, not from TF, so we +// should give it init-time to connect). +// +// If balancer is READY or IDLE +// +// If this is the lowest priority +// +// do the following: +// +// if this is not the old childInUse, override picker so old picker is no +// longer used. +// +// switch to it (because all higher priorities are neither new or Ready) +// +// forward the new addresses and config +// +// Caller must hold b.mu. +func (b *priorityBalancer) syncPriority(childUpdating string) { + if b.inhibitPickerUpdates { + if b.logger.V(2) { + b.logger.Infof("Skipping update from child policy %q", childUpdating) + } + return + } + for p, name := range b.priorities { + child, ok := b.children[name] + if !ok { + b.logger.Warningf("Priority name %q is not found in list of child policies", name) + continue + } + + if !child.started || + child.state.ConnectivityState == connectivity.Ready || + child.state.ConnectivityState == connectivity.Idle || + (child.state.ConnectivityState == connectivity.Connecting && child.initTimer != nil) || + p == len(b.priorities)-1 { + if b.childInUse != child.name || child.name == childUpdating { + if b.logger.V(2) { + b.logger.Infof("childInUse, childUpdating: %q, %q", b.childInUse, child.name) + } + // If we switch children or the child in use just updated its + // picker, push the child's picker to the parent. + b.cc.UpdateState(child.state) + } + if b.logger.V(2) { + b.logger.Infof("Switching to (%q, %v) in syncPriority", child.name, p) + } + b.switchToChild(child, p) + break + } + } +} + +// Stop priorities [p+1, lowest]. +// +// Caller must hold b.mu. +func (b *priorityBalancer) stopSubBalancersLowerThanPriority(p int) { + for i := p + 1; i < len(b.priorities); i++ { + name := b.priorities[i] + child, ok := b.children[name] + if !ok { + b.logger.Warningf("Priority name %q is not found in list of child policies", name) + continue + } + child.stop(false) + } +} + +// switchToChild does the following: +// - stop all child with lower priorities +// - if childInUse is not this child +// - set childInUse to this child +// - if this child is not started, start it +// +// Note that it does NOT send the current child state (picker) to the parent +// ClientConn. The caller needs to send it if necessary. +// +// this can be called when +// 1. first update, start p0 +// 2. an update moves a READY child from a lower priority to higher +// 2. a different builder is updated for this child +// 3. a high priority goes Failure, start next +// 4. a high priority init timeout, start next +// +// Caller must hold b.mu. +func (b *priorityBalancer) switchToChild(child *childBalancer, priority int) { + // Stop lower priorities even if childInUse is same as this child. It's + // possible this child was moved from a priority to another. + b.stopSubBalancersLowerThanPriority(priority) + + // If this child is already in use, do nothing. + // + // This can happen: + // - all priorities are not READY, an config update always triggers switch + // to the lowest. In this case, the lowest child could still be connecting, + // so we don't stop the init timer. + // - a high priority is READY, an config update always triggers switch to + // it. + if b.childInUse == child.name && child.started { + return + } + b.childInUse = child.name + + if !child.started { + child.start() + } +} + +// handleChildStateUpdate start/close priorities based on the connectivity +// state. +func (b *priorityBalancer) handleChildStateUpdate(childName string, newState balancer.State) { + // Update state in child. The updated picker will be sent to parent later if + // necessary. + child, ok := b.children[childName] + if !ok { + b.logger.Warningf("Child policy not found for %q", childName) + return + } + if !child.started { + b.logger.Warningf("Ignoring update from child policy %q which is not in started state: %+v", childName, newState) + return + } + oldState := child.state + child.state = newState + + // We start/stop the init timer of this child based on the new connectivity + // state. syncPriority() later will need the init timer (to check if it's + // nil or not) to decide which child to switch to. + switch newState.ConnectivityState { + case connectivity.Ready, connectivity.Idle: + child.reportedTF = false + child.stopInitTimer() + case connectivity.TransientFailure: + child.reportedTF = true + child.stopInitTimer() + case connectivity.Connecting: + // The init timer is created when the child is created and is reset when + // it reports Ready or Idle. Most child policies start off in + // Connecting, but ring_hash starts off in Idle and moves to Connecting + // when a request comes in. To support such cases, we restart the init + // timer when we see Connecting, but only if the child has not reported + // TransientFailure more recently than it reported Ready or Idle. See + // gRFC A42 for details on why ring_hash is special and what provisions + // are required to make it work as a child of the priority LB policy. + // + // We don't want to restart the timer if the child was already in + // Connecting, because we want failover to happen once the timer elapses + // even when the child is still in Connecting. + if !child.reportedTF && oldState.ConnectivityState != connectivity.Connecting { + child.startInitTimer() + } + default: + // New state is Shutdown, should never happen. Don't forward. + } + + child.parent.syncPriority(childName) +} diff --git a/internal/xds/balancer/priority/balancer_test.go b/internal/xds/balancer/priority/balancer_test.go new file mode 100644 index 0000000..c1c1572 --- /dev/null +++ b/internal/xds/balancer/priority/balancer_test.go @@ -0,0 +1,2366 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/hierarchy" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 100 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +var testBackendAddrStrs []string + +const ( + testBackendAddrsCount = 12 + testRRBalancerName = "another-round-robin" +) + +type anotherRR struct { + balancer.Builder +} + +func (*anotherRR) Name() string { + return testRRBalancerName +} + +func init() { + for i := 0; i < testBackendAddrsCount; i++ { + testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)) + } + balancer.Register(&anotherRR{Builder: balancer.Get(roundrobin.Name)}) +} + +func overrideInitTimeout(t *testing.T, val time.Duration) { + orig := DefaultPriorityInitTimeout + DefaultPriorityInitTimeout = val + t.Cleanup(func() { DefaultPriorityInitTimeout = orig }) +} + +// When a high priority is ready, adding/removing lower locality doesn't cause +// changes. +// +// Init 0 and 1; 0 is up, use 0; add 2, use 0; remove 2, use 0. +func (s) TestPriority_HighPriorityReady(t *testing.T) { + t.Run("caching_enabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, true) + testPriorityHighPriorityReady(t) + }) + t.Run("caching_disabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, false) + testPriorityHighPriorityReady(t) + }) +} + +func testPriorityHighPriorityReady(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two children, with priorities [0, 1], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // p0 is ready. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Add p2, it shouldn't cause any updates. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[2]}}}, []string{"child-2"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1", "child-2"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + select { + case sc := <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn: %s", sc) + case sc := <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn: %v", sc) + case <-time.After(time.Millisecond * 100): + } + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Remove p2, no updates. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + select { + case <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn") + case <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn") + case <-time.After(time.Millisecond * 100): + } + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } +} + +// Lower priority is used when higher priority is not ready. +// +// Init 0 and 1; 0 is up, use 0; 0 is down, 1 is up, use 1; add 2, use 1; 1 is +// down, use 2; remove 2, use 1. +func (s) TestPriority_SwitchPriority(t *testing.T) { + t.Run("caching_enabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, true) + testPrioritySwitchPriority(t) + }) + t.Run("caching_disabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, false) + testPrioritySwitchPriority(t) + }) +} + +func testPrioritySwitchPriority(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + t.Log("Two localities, with priorities [0, 1], each with one backend.") + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + t.Log("Make p0 ready.") + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Turn down 0, will start and use 1.") + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Handle SubConn creation from 1.") + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + <-sc1.ConnectCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 1. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Add p2, it shouldn't cause any updates.") + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[2]}}}, []string{"child-2"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1", "child-2"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + select { + case sc := <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn, %s", sc) + case <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn") + case <-time.After(time.Millisecond * 100): + } + + t.Log("Turn down 1, use 2.") + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + <-sc1.ConnectCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: errors.New("test error"), + }) + + // Before 2 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, testBackendAddrStrs[2]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc2 := <-cc.NewSubConnCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 2. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Remove 2, use 1.") + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // p2 will be closed immediately only if the cache is not enabled. + if !envconfig.EnablePriorityLBChildPolicyCache { + // p2 SubConns are shut down. + scToShutdown := <-cc.ShutdownSubConnCh + // The same SubConn is closed by gracefulswitch and pickfirstleaf when + // they are closed. Remove duplicate events. + // TODO: https://github.com/grpc/grpc-go/issues/6472 - Remove this + // workaround once pickfirst is the only leaf policy and responsible for + // shutting down SubConns. + <-cc.ShutdownSubConnCh + if scToShutdown != sc2 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc2, scToShutdown) + } + } + + // Should get an update with 1's old transient failure picker, to override + // 2's old picker. + if err := cc.WaitForErrPicker(ctx); err != nil { + t.Fatal(err.Error()) + } + <-cc.NewStateCh // Drain to match picker + + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + // Does not change the aggregate state, because round robin does not leave + // TRANSIENT_FAILURE if a subconn goes CONNECTING. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } +} + +// Lower priority is used when higher priority turns Connecting from Ready. +// Because changing from Ready to Connecting is a failure. +// +// Init 0 and 1; 0 is up, use 0; 0 is connecting, 1 is up, use 1; 0 is ready, +// use 0. +// +// Env var GRPC_EXPERIMENTAL_ENABLE_PRIORITY_LB_CHILD_POLICY_CACHE does not +// affect whether a lower priority child policy is cached when it is removed +// from the balancer group. +func (s) TestPriority_HighPriorityToConnectingFromReady(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + t.Log("Two localities, with priorities [0, 1], each with one backend.") + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + t.Log("Make p0 ready.") + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Turn down 0, will start and use 1.") + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Handle SubConn creation from 1.") + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 1. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + t.Logf("Turn 0 back to Ready.") + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // At this point p1 is removed from the balancer group, but it gets cached. + // Therefore its subchannels will not be removed immediately. So, we do not + // check for the shutdown of p1's subchannels here. Instead, we check that + // p0 is used again after it becomes ready, which indicates that the + // priority switch is successful. + if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { + t.Fatal(err.Error()) + } +} + +// Add a lower priority while the higher priority is down. +// +// Init 0 and 1; 0 and 1 both down; add 2, use 2. +func (s) TestPriority_HigherDownWhileAddingLower(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two localities, with different priorities, each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + t.Log("Turn down 0, 1 is used.") + testErr := errors.New("test error") + sc0.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: testErr, + }) + + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + t.Log("Turn down 1, pick should error.") + sc1.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: testErr, + }) + + // Test pick failure. + if err := cc.WaitForPickerWithErr(ctx, testErr); err != nil { + t.Fatal(err.Error()) + } + <-cc.NewStateCh // Drain to match picker + + t.Log("Add p2, it should create a new SubConn.") + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[2]}}}, []string{"child-2"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1", "child-2"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // A new connecting picker should be updated for the new priority. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, testBackendAddrStrs[2]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc2 := <-cc.NewSubConnCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 2. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } +} + +// When a higher priority becomes available, all lower priorities are closed. +// +// Init 0,1,2; 0 and 1 down, use 2; 0 up, close 1 and 2. +func (s) TestPriority_HigherReadyCloseAllLower(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Three localities, with priorities [0,1,2], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[2]}}}, []string{"child-2"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1", "child-2"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // Turn down 0, 1 is used. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // Turn down 1, 2 is used. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // Before 2 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, testBackendAddrStrs[2]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc2 := <-cc.NewSubConnCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 2. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } + + // When 0 becomes ready, 0 should be used, 1 and 2 should all be closed. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 0. + if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { + t.Fatal(err.Error()) + } +} + +// At init, start the next lower priority after timeout if the higher priority +// doesn't get ready. +// +// Init 0,1; 0 is not ready (in connecting), after timeout, use 1. +func (s) TestPriority_InitTimeout(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const testPriorityInitTimeout = 200 * time.Millisecond + overrideInitTimeout(t, testPriorityInitTimeout) + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two localities, with different priorities, each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // Keep 0 in connecting, 1 will be used after init timeout. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + + // Make sure new SubConn is created before timeout. + select { + case <-time.After(testPriorityInitTimeout * 3 / 4): + case <-cc.NewSubConnAddrsCh: + t.Fatalf("Got a new SubConn too early (Within timeout). Expect a new SubConn only after timeout") + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // After the init timer of p0, when switching to p1, a connecting picker + // will be sent to the parent. Clear it here. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 1. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } +} + +// EDS removes all priorities, and re-adds them. +func (s) TestPriority_RemovesAllPriorities(t *testing.T) { + t.Run("caching_enabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, true) + testPriorityRemovesAllPriorities(t) + }) + t.Run("caching_disabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, false) + testPriorityRemovesAllPriorities(t) + }) +} + +func testPriorityRemovesAllPriorities(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const testPriorityInitTimeout = 200 * time.Millisecond + overrideInitTimeout(t, testPriorityInitTimeout) + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two localities, with different priorities, each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { + t.Fatal(err.Error()) + } + + // Remove all priorities. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: nil, + }, + BalancerConfig: &LBConfig{ + Children: nil, + Priorities: nil, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // p0 will be closed immediately only if the cache is not enabled. + if !envconfig.EnablePriorityLBChildPolicyCache { + // p0 SubConns are shut down. + scToShutdown := <-cc.ShutdownSubConnCh + // The same SubConn is closed by gracefulswitch and pickfirstleaf when + // they are closed. Remove duplicate events. + // TODO: https://github.com/grpc/grpc-go/issues/6472 - Remove this + // workaround once pickfirst is the only leaf policy and responsible for + // shutting down SubConns. + <-cc.ShutdownSubConnCh + if scToShutdown != sc0 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc0, scToShutdown) + } + } + + // Test pick return TransientFailure. + if err := cc.WaitForPickerWithErr(ctx, ErrAllPrioritiesRemoved); err != nil { + t.Fatal(err.Error()) + } + + // Re-add two localities, with previous priorities, but different backends. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[2]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[3]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs01 := <-cc.NewSubConnAddrsCh + if got, want := addrs01[0].Addr, testBackendAddrStrs[2]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc01 := <-cc.NewSubConnCh + + if envconfig.EnablePriorityLBChildPolicyCache { + // The old p0 SubConn is shut down. + scToShutdown := <-cc.ShutdownSubConnCh + if scToShutdown != sc0 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc0, scToShutdown) + } + } + + // Don't send any update to p0, so to not override the old state of p0. + // Later, connect to p1 and then remove p1. This will fallback to p0, and + // will send p0's old picker if they are not correctly removed. + + // p1 will be used after priority init timeout. + addrs11 := <-cc.NewSubConnAddrsCh + if got, want := addrs11[0].Addr, testBackendAddrStrs[3]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc11 := <-cc.NewSubConnCh + sc11.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc11.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p1 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc11); err != nil { + t.Fatal(err.Error()) + } + + // Remove p1, to fallback to p0. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[2]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // p1 will be closed immediately only if the cache is not enabled. + if !envconfig.EnablePriorityLBChildPolicyCache { + // p1 SubConns are shut down. + scToShutdown := <-cc.ShutdownSubConnCh + // The same SubConn is closed by gracefulswitch and pickfirstleaf when + // they are closed. Remove duplicate events. + // TODO: https://github.com/grpc/grpc-go/issues/6472 - Remove this + // workaround once pickfirst is the only leaf policy and responsible for + // shutting down SubConns. + <-cc.ShutdownSubConnCh + if scToShutdown != sc11 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc11, scToShutdown) + } + } + + // Test pick return NoSubConn. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + // Send an ready update for the p0 sc that was received when re-adding + // priorities. + sc01.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc01.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc01); err != nil { + t.Fatal(err.Error()) + } + + select { + case <-cc.NewPickerCh: + t.Fatalf("got unexpected new picker") + case <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn") + case <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn") + case <-time.After(time.Millisecond * 100): + } +} + +// Test the case where the high priority contains no backends. The low priority +// will be used. +func (s) TestPriority_HighPriorityNoEndpoints(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two localities, with priorities [0, 1], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // p0 is ready. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Remove addresses from priority 0, should use p1. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // p0 will shutdown the subconn, and ClientConn will send a sc update to + // shutdown. + scToShutdown := <-cc.ShutdownSubConnCh + scToShutdown.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) + + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc2 := <-cc.NewSubConnCh + + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + // p1 is ready. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p1 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } +} + +// Test the case where the first and only priority is removed. +func (s) TestPriority_FirstPriorityUnavailable(t *testing.T) { + t.Run("caching_enabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, true) + testPriorityFirstPriorityUnavailable(t) + }) + t.Run("caching_disabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, false) + testPriorityFirstPriorityUnavailable(t) + }) +} + +func testPriorityFirstPriorityUnavailable(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One localities, with priorities [0], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // Remove the only localities. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: nil, + }, + BalancerConfig: &LBConfig{ + Children: nil, + Priorities: nil, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // p0 will be closed immediately only if the cache is not enabled. + if !envconfig.EnablePriorityLBChildPolicyCache { + // p0 SubConns are shut down. + scToShutdown := <-cc.ShutdownSubConnCh + // The same SubConn is closed by gracefulswitch and pickfirstleaf when + // they are closed. Remove duplicate events. + // TODO: https://github.com/grpc/grpc-go/issues/6472 - Remove this + // workaround once pickfirst is the only leaf policy and responsible for + // shutting down SubConns. + <-cc.ShutdownSubConnCh + if scToShutdown != sc0 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc0, scToShutdown) + } + } + + // Test pick return TransientFailure. + if err := cc.WaitForPickerWithErr(ctx, ErrAllPrioritiesRemoved); err != nil { + t.Fatal(err.Error()) + } +} + +// When a child is moved from low priority to high. +// +// Init a(p0) and b(p1); a(p0) is up, use a; move b to p0, a to p1, use b. +func (s) TestPriority_MoveChildToHigherPriority(t *testing.T) { + t.Run("caching_enabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, true) + testPriorityMoveChildToHigherPriority(t) + }) + t.Run("caching_disabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, false) + testPriorityMoveChildToHigherPriority(t) + }) +} + +func testPriorityMoveChildToHigherPriority(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two children, with priorities [0, 1], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // p0 is ready. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Swap child with p0 and p1, the child at lower priority should now be the + // higher priority, and be used. The old SubConn should be closed. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-1", "child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // When the new child for p0 is changed from the previous child, the + // balancer should immediately update the picker so the picker from old + // child is not used. In this case, the picker becomes a + // no-subconn-available picker because this child is just started. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc2 := <-cc.NewSubConnCh + + // New p0 child is ready. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only new subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } +} + +// When a child is in lower priority, and in use (because higher is down), +// move it from low priority to high. +// +// Init a(p0) and b(p1); a(p0) is down, use b; move b to p0, a to p1, use b. +func (s) TestPriority_MoveReadyChildToHigherPriority(t *testing.T) { + t.Run("caching_enabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, true) + testPriorityMoveReadyChildToHigherPriority(t) + }) + t.Run("caching_disabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, false) + testPriorityMoveReadyChildToHigherPriority(t) + }) +} + +func testPriorityMoveReadyChildToHigherPriority(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two children, with priorities [0, 1], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // p0 is down. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p1 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Swap child with p0 and p1, the child at lower priority should now be the + // higher priority, and be used. The old SubConn should be closed. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-1", "child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Because this was a ready child moved to a higher priority, no new subconn + // or picker should be updated. + select { + case <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn") + case <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn") + case <-time.After(time.Millisecond * 100): + } +} + +// When the lowest child is in use, and is removed, should use the higher +// priority child even though it's not ready. +// +// Init a(p0) and b(p1); a(p0) is down, use b; move b to p0, a to p1, use b. +func (s) TestPriority_RemoveReadyLowestChild(t *testing.T) { + t.Run("caching_enabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, true) + testPriorityRemoveReadyLowestChild(t) + }) + t.Run("caching_disabled", func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, false) + testPriorityRemoveReadyLowestChild(t) + }) +} + +func testPriorityRemoveReadyLowestChild(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two children, with priorities [0, 1], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // p0 is down. + sc0.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: errors.New("test error"), + }) + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p1 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Remove child with p1, the child at higher priority should now be used. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + if !envconfig.EnablePriorityLBChildPolicyCache { + // Old subconn from child-1 should be shut down. + scToShutdown := <-cc.ShutdownSubConnCh + // The same SubConn is closed by gracefulswitch and pickfirstleaf when they + // are closed. Remove duplicate events. + // TODO: https://github.com/grpc/grpc-go/issues/6472 - Remove this + // workaround once pickfirst is the only leaf policy and responsible for + // shutting down SubConns. + <-cc.ShutdownSubConnCh + if scToShutdown != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scToShutdown) + } + } + + if err := cc.WaitForErrPicker(ctx); err != nil { + t.Fatal(err.Error()) + } + <-cc.NewStateCh // Drain to match picker + + // Because there was no new child, no new subconn should be created. + select { + case <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn") + case <-time.After(time.Millisecond * 100): + } +} + +// When a ready child is removed, it's kept in cache. Re-adding doesn't create subconns. +// +// Init 0; 0 is up, use 0; remove 0, only picker is updated, no subconn is +// removed; re-add 0, picker is updated. +func (s) TestPriority_RemoveOnlyReadyChild_CachingEnabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, true) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One children, with priorities [0], with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // p0 is ready. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Remove the child. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{}, + BalancerConfig: &LBConfig{}, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + if err := cc.WaitForPickerWithErr(ctx, ErrAllPrioritiesRemoved); err != nil { + t.Fatal(err.Error()) + } + + // But no conn changes should happen. Child balancer is in cache. + select { + case sc := <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn: %s", sc) + case sc := <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn: %v", sc) + case <-time.After(time.Millisecond * 100): + } + + // Re-add the child. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // But no conn changes should happen. Child balancer is just taken out from + // the cache. + select { + case sc := <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn: %s", sc) + case sc := <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn: %v", sc) + case <-time.After(time.Millisecond * 100): + } +} + +// When the only ready child is removed, it is deleted and subchannels are shut +// down. Re-adding results in subchannels being recreated. +// +// Init 0; 0 is up, use 0; remove 0, re-add 0. +func (s) TestPriority_RemoveOnlyReadyChild_CachingDisabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.EnablePriorityLBChildPolicyCache, false) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One children, with priorities [0], with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // p0 is ready. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Remove the child. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{}, + BalancerConfig: &LBConfig{}, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // p0 SubConns are shut down. + scToShutdown := <-cc.ShutdownSubConnCh + // The same SubConn is closed by gracefulswitch and pickfirstleaf when + // they are closed. Remove duplicate events. + // TODO: https://github.com/grpc/grpc-go/issues/6472 - Remove this + // workaround once pickfirst is the only leaf policy and responsible for + // shutting down SubConns. + <-cc.ShutdownSubConnCh + if scToShutdown != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scToShutdown) + } + + if err := cc.WaitForPickerWithErr(ctx, ErrAllPrioritiesRemoved); err != nil { + t.Fatal(err.Error()) + } + + // Re-add the child. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs1 = <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 = <-cc.NewSubConnCh + + // p0 is ready. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } +} + +// When the policy of a child is changed. +// +// Init 0; 0 is up, use 0; change 0's policy, 0 is used. +func (s) TestPriority_ChildPolicyChange(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One children, with priorities [0], with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // p0 is ready. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Change the policy for the child (still roundrobin, but with a different + // name). + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: testRRBalancerName}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Old subconn should be shut down. + scToShutdown := <-cc.ShutdownSubConnCh + // The same SubConn is closed by gracefulswitch and pickfirstleaf when they + // are closed. Remove duplicate events. + // TODO: https://github.com/grpc/grpc-go/issues/6472 - Remove this + // workaround once pickfirst is the only leaf policy and responsible for + // shutting down SubConns. + <-cc.ShutdownSubConnCh + if scToShutdown != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scToShutdown) + } + + // A new subconn should be created. + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc2 := <-cc.NewSubConnCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pickfirst with the new subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } +} + +const inlineUpdateBalancerName = "test-inline-update-balancer" + +var errTestInlineStateUpdate = fmt.Errorf("don't like addresses, empty or not") + +func init() { + stub.Register(inlineUpdateBalancerName, stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &testutils.TestConstPicker{Err: errTestInlineStateUpdate}, + }) + return nil + }, + }) +} + +// When the child policy update picker inline in a handleClientUpdate call +// (e.g., roundrobin handling empty addresses). There could be deadlock caused +// by acquiring a locked mutex. +func (s) TestPriority_ChildPolicyUpdatePickerInline(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One children, with priorities [0], with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: inlineUpdateBalancerName}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + if err := cc.WaitForPickerWithErr(ctx, errTestInlineStateUpdate); err != nil { + t.Fatal(err.Error()) + } +} + +// TestPriority_IgnoreReresolutionRequest tests the case where the priority +// policy has a single child policy. The test verifies that ResolveNow() calls +// from the child policy are ignored based on the value of the +// IgnoreReresolutionRequests field in the configuration. +func (s) TestPriority_IgnoreReresolutionRequest(t *testing.T) { + // Register a stub balancer to act the child policy of the priority policy. + // Provide an init function to the stub balancer to capture the ClientConn + // passed to the child policy. + ccCh := testutils.NewChannel() + childPolicyName := t.Name() + stub.Register(childPolicyName, stub.BalancerFuncs{ + Init: func(data *stub.BalancerData) { + ccCh.Send(data.ClientConn) + }, + }) + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One children, with priorities [0], with one backend, reresolution is + // ignored. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": { + Config: &internalserviceconfig.BalancerConfig{Name: childPolicyName}, + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Retrieve the ClientConn passed to the child policy. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := ccCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout waiting for ClientConn from the child policy") + } + balancerCC := val.(balancer.ClientConn) + + // Since IgnoreReresolutionRequests was set to true, all ResolveNow() calls + // should be ignored. + for i := 0; i < 5; i++ { + balancerCC.ResolveNow(resolver.ResolveNowOptions{}) + } + select { + case <-cc.ResolveNowCh: + t.Fatalf("got unexpected ResolveNow() call") + case <-time.After(defaultTestShortTimeout): + } + + // Send another update to set IgnoreReresolutionRequests to false. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": { + Config: &internalserviceconfig.BalancerConfig{Name: childPolicyName}, + IgnoreReresolutionRequests: false, + }, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Call ResolveNow() on the CC, it should be forwarded. + balancerCC.ResolveNow(resolver.ResolveNowOptions{}) + select { + case <-cc.ResolveNowCh: + case <-time.After(time.Second): + t.Fatalf("timeout waiting for ResolveNow()") + } +} + +// TestPriority_IgnoreReresolutionRequestTwoChildren tests the case where the +// priority policy has two child policies, one of them has the +// IgnoreReresolutionRequests field set to true while the other one has it set +// to false. The test verifies that ResolveNow() calls from the child which is +// set to ignore reresolution requests are ignored, while calls from the other +// child are processed. +func (s) TestPriority_IgnoreReresolutionRequestTwoChildren(t *testing.T) { + // Register a stub balancer to act the child policy of the priority policy. + // Provide an init function to the stub balancer to capture the ClientConn + // passed to the child policy. + ccCh := testutils.NewChannel() + childPolicyName := t.Name() + stub.Register(childPolicyName, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + ccCh.Send(bd.ClientConn) + bd.ChildBalancer = balancer.Get(roundrobin.Name).Build(bd.ClientConn, bd.BuildOptions) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + }) + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One children, with priorities [0, 1], each with one backend. + // Reresolution is ignored for p0. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": { + Config: &internalserviceconfig.BalancerConfig{Name: childPolicyName}, + IgnoreReresolutionRequests: true, + }, + "child-1": { + Config: &internalserviceconfig.BalancerConfig{Name: childPolicyName}, + }, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Retrieve the ClientConn passed to the child policy from p0. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := ccCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout waiting for ClientConn from the child policy") + } + balancerCC0 := val.(balancer.ClientConn) + + // Set p0 to transient failure, p1 will be started. + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + + // Retrieve the ClientConn passed to the child policy from p1. + val, err = ccCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout waiting for ClientConn from the child policy") + } + balancerCC1 := val.(balancer.ClientConn) + + // Since IgnoreReresolutionRequests was set to true for p0, ResolveNow() + // from p0 should all be ignored. + for i := 0; i < 5; i++ { + balancerCC0.ResolveNow(resolver.ResolveNowOptions{}) + } + select { + case <-cc.ResolveNowCh: + t.Fatalf("got unexpected ResolveNow() call") + case <-time.After(defaultTestShortTimeout): + } + + // But IgnoreReresolutionRequests was false for p1, ResolveNow() from p1 + // should be forwarded. + balancerCC1.ResolveNow(resolver.ResolveNowOptions{}) + select { + case <-cc.ResolveNowCh: + case <-time.After(defaultTestShortTimeout): + t.Fatalf("timeout waiting for ResolveNow()") + } +} + +const initIdleBalancerName = "test-init-idle-balancer" + +var errsTestInitIdle = []error{ + fmt.Errorf("init Idle balancer error 0"), + fmt.Errorf("init Idle balancer error 1"), +} + +func init() { + for i := 0; i < 2; i++ { + ii := i + stub.Register(fmt.Sprintf("%s-%d", initIdleBalancerName, ii), stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, opts balancer.ClientConnState) error { + lis := func(state balancer.SubConnState) { + err := fmt.Errorf("wrong picker error") + if state.ConnectivityState == connectivity.Idle { + err = errsTestInitIdle[ii] + } + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &testutils.TestConstPicker{Err: err}, + }) + } + + sc, err := bd.ClientConn.NewSubConn(opts.ResolverState.Endpoints[0].Addresses, balancer.NewSubConnOptions{StateListener: lis}) + if err != nil { + return err + } + sc.Connect() + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &testutils.TestConstPicker{Err: balancer.ErrNoSubConnAvailable}, + }) + return nil + }, + }) + } +} + +// If the high priorities send initial pickers with Idle state, their pickers +// should get picks, because policies like ringhash starts in Idle, and doesn't +// connect. +// +// Init 0, 1; 0 is Idle, use 0; 0 is down, start 1; 1 is Idle, use 1. +func (s) TestPriority_HighPriorityInitIdle(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two children, with priorities [0, 1], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 0)}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 1)}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // Send an Idle state update to trigger an Idle picker update. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + if err := cc.WaitForPickerWithErr(ctx, errsTestInitIdle[0]); err != nil { + t.Fatal(err.Error()) + } + + // Turn p0 down, to start p1. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + // Idle picker from p1 should also be forwarded. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + if err := cc.WaitForPickerWithErr(ctx, errsTestInitIdle[1]); err != nil { + t.Fatal(err.Error()) + } +} + +// If the high priorities send initial pickers with Idle state, their pickers +// should get picks, because policies like ringhash starts in Idle, and doesn't +// connect. In this case, if a lower priority is added, it shouldn't switch to +// the lower priority. +// +// Init 0; 0 is Idle, use 0; add 1, use 0. +func (s) TestPriority_AddLowPriorityWhenHighIsInIdle(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One child, with priorities [0], one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 0)}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // Send an Idle state update to trigger an Idle picker update. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + if err := cc.WaitForPickerWithErr(ctx, errsTestInitIdle[0]); err != nil { + t.Fatal(err.Error()) + } + + // Add 1, should keep using 0. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 0)}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 1)}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // The ClientConn state update triggers a priority switch, from p0 -> p0 + // (since p0 is still in use). Along with this the update, p0 also gets a + // ClientConn state update, with the addresses, which didn't change in this + // test (this update to the child is necessary in case the addresses are + // different). + // + // The test child policy, initIdleBalancer, blindly calls NewSubConn with + // all the addresses it receives, so this will trigger a NewSubConn with the + // old p0 addresses. (Note that in a real balancer, like roundrobin, no new + // SubConn will be created because the addresses didn't change). + // + // The check below makes sure that the addresses are still from p0, and not + // from p1. This is good enough for the purpose of this test. + addrsNew := <-cc.NewSubConnAddrsCh + if got, want := addrsNew[0].Addr, testBackendAddrStrs[0]; got != want { + // Fail if p1 is started and creates a SubConn. + t.Fatalf("got unexpected call to NewSubConn with addr: %v, want %v", addrsNew, want) + } +} + +// Lower priority is used when higher priority is not ready; higher priority +// still gets updates. +// +// Init 0 and 1; 0 is down, 1 is up, use 1; update 0; 0 is up, use 0 +func (s) TestPriority_HighPriorityUpdatesWhenLowInUse(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + t.Log("Two localities, with priorities [0, 1], each with one backend.") + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + t.Log("Make p0 fail.") + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Make p1 ready.") + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 1. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + // Does not change the aggregate state, because round robin does not leave + // TRANSIENT_FAILURE if a subconn goes CONNECTING. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Change p0 to use new address.") + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[2]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[3]}}}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Two new subconns are created by the previous update; one by p0 and one + // by p1. They don't happen concurrently, but they could happen in any + // order. + t.Log("Make p0 and p1 both ready; p0 should be used.") + var sc2, sc3 balancer.SubConn + for i := 0; i < 2; i++ { + addr := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + switch addr[0].Addr { + case testBackendAddrStrs[2]: + sc2 = sc + case testBackendAddrStrs[3]: + sc3 = sc + default: + t.Fatalf("sc is created with addr %v, want %v or %v", addr[0].Addr, testBackendAddrStrs[2], testBackendAddrStrs[3]) + } + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + if sc2 == nil { + t.Fatalf("sc not created with addr %v", testBackendAddrStrs[2]) + } + if sc3 == nil { + t.Fatalf("sc not created with addr %v", testBackendAddrStrs[3]) + } + + // Test pick with 0. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } +} + +// Tests that the priority balancer's init timer is not restarted when its child +// reports a state transition from CONNECTING to CONNECTING. +func (s) TestPriority_InitTimerNotRestarted_OnConnectingToConnecting(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + initTimerStarted := make(chan struct{}, 1) + origTimeAfterFunc := timeAfterFunc + timeAfterFunc = func(d time.Duration, f func()) *time.Timer { + select { + case initTimerStarted <- struct{}{}: + case <-ctx.Done(): + t.Errorf("Timeout waiting to send init timer started signal") + } + return time.AfterFunc(d, f) + } + defer func() { timeAfterFunc = origTimeAfterFunc }() + + cc := testutils.NewBalancerClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One child, with two backends. + ccs := balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[0]}}}, []string{"child-0"}), + hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: testBackendAddrStrs[1]}}}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + } + if err := pb.UpdateClientConnState(ccs); err != nil { + t.Fatalf("UpdateClientConnState(%+v) failed: %v", ccs, err) + } + + // Wait for child-0 to be started and two subchannels to be created. + var sc0, sc1 *testutils.TestSubConn + for i := range 2 { + var addrs []resolver.Address + select { + case addrs = <-cc.NewSubConnAddrsCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for subconn %d to be created", i) + } + switch got := addrs[0].Addr; got { + case testBackendAddrStrs[0]: + sc0 = <-cc.NewSubConnCh + case testBackendAddrStrs[1]: + sc1 = <-cc.NewSubConnCh + default: + t.Fatalf("Got unexpected new subconn addr: %q, want %q or %q", got, testBackendAddrStrs[0], testBackendAddrStrs[1]) + } + } + + // Move both subchannels to CONNECTING. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + + // Ensure that the init timer is started only once. + select { + case <-initTimerStarted: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for init timer to start") + } + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-initTimerStarted: + t.Fatalf("Init timer started when second subchannel moved to CONNECTING") + case <-sCtx.Done(): + } + + // Simulate the connection succeeding (subchannel becoming Ready), and then + // failing (subchannel moving to Idle). RR will immediately start connecting + // on the failed subchannel, and will therefore reporting an overall state + // of Connecting. This should trigger a restart of the init timer. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + select { + case <-initTimerStarted: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for init timer to start") + } + + // Move the subchannel to CONNECTING again, and ensure that the init timer + // is not restarted. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-initTimerStarted: + t.Fatalf("Init timer restarted when subchannel moved from Ready to Idle") + case <-sCtx.Done(): + } +} diff --git a/internal/xds/balancer/priority/config.go b/internal/xds/balancer/priority/config.go new file mode 100644 index 0000000..37f1c9a --- /dev/null +++ b/internal/xds/balancer/priority/config.go @@ -0,0 +1,67 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "encoding/json" + "fmt" + + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +// Child is a child of priority balancer. +type Child struct { + Config *internalserviceconfig.BalancerConfig `json:"config,omitempty"` + IgnoreReresolutionRequests bool `json:"ignoreReresolutionRequests,omitempty"` +} + +// LBConfig represents priority balancer's config. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + // Children is a map from the child balancer names to their configs. Child + // names can be found in field Priorities. + Children map[string]*Child `json:"children,omitempty"` + // Priorities is a list of child balancer names. They are sorted from + // highest priority to low. The type/config for each child can be found in + // field Children, with the balancer name as the key. + Priorities []string `json:"priorities,omitempty"` +} + +func parseConfig(c json.RawMessage) (*LBConfig, error) { + var cfg LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + + prioritiesSet := make(map[string]bool) + for _, name := range cfg.Priorities { + if _, ok := cfg.Children[name]; !ok { + return nil, fmt.Errorf("LB policy name %q found in Priorities field (%v) is not found in Children field (%+v)", name, cfg.Priorities, cfg.Children) + } + prioritiesSet[name] = true + } + for name := range cfg.Children { + if _, ok := prioritiesSet[name]; !ok { + return nil, fmt.Errorf("LB policy name %q found in Children field (%v) is not found in Priorities field (%+v)", name, cfg.Children, cfg.Priorities) + } + } + return &cfg, nil +} diff --git a/internal/xds/balancer/priority/config_test.go b/internal/xds/balancer/priority/config_test.go new file mode 100644 index 0000000..8316224 --- /dev/null +++ b/internal/xds/balancer/priority/config_test.go @@ -0,0 +1,108 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer/roundrobin" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" +) + +func TestParseConfig(t *testing.T) { + tests := []struct { + name string + js string + want *LBConfig + wantErr bool + }{ + { + name: "child not found", + js: `{ + "priorities": ["child-1", "child-2", "child-3"], + "children": { + "child-1": {"config": [{"round_robin":{}}]}, + "child-3": {"config": [{"round_robin":{}}]} + } +} + `, + wantErr: true, + }, + { + name: "child not used", + js: `{ + "priorities": ["child-1", "child-2"], + "children": { + "child-1": {"config": [{"round_robin":{}}]}, + "child-2": {"config": [{"round_robin":{}}]}, + "child-3": {"config": [{"round_robin":{}}]} + } +} + `, + wantErr: true, + }, + { + name: "good", + js: `{ + "priorities": ["child-1", "child-2", "child-3"], + "children": { + "child-1": {"config": [{"round_robin":{}}], "ignoreReresolutionRequests": true}, + "child-2": {"config": [{"round_robin":{}}]}, + "child-3": {"config": [{"round_robin":{}}]} + } +} + `, + want: &LBConfig{ + Children: map[string]*Child{ + "child-1": { + Config: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + IgnoreReresolutionRequests: true, + }, + "child-2": { + Config: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + "child-3": { + Config: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }, + Priorities: []string{"child-1", "child-2", "child-3"}, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseConfig([]byte(tt.js)) + if (err != nil) != tt.wantErr { + t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("parseConfig() got = %v, want %v, diff: %s", got, tt.want, diff) + } + }) + } +} diff --git a/internal/xds/balancer/priority/ignore_resolve_now.go b/internal/xds/balancer/priority/ignore_resolve_now.go new file mode 100644 index 0000000..df9ec46 --- /dev/null +++ b/internal/xds/balancer/priority/ignore_resolve_now.go @@ -0,0 +1,50 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "sync/atomic" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" +) + +// ignoreResolveNowClientConn wraps a balancer.ClientConn and overrides the +// ResolveNow() method to ignore those calls if the ignoreResolveNow bit is set. +type ignoreResolveNowClientConn struct { + balancer.ClientConn + ignoreResolveNow atomic.Bool +} + +func newIgnoreResolveNowClientConn(cc balancer.ClientConn, ignore bool) *ignoreResolveNowClientConn { + ret := &ignoreResolveNowClientConn{ClientConn: cc} + ret.updateIgnoreResolveNow(ignore) + return ret +} + +func (i *ignoreResolveNowClientConn) updateIgnoreResolveNow(b bool) { + i.ignoreResolveNow.Store(b) +} + +func (i *ignoreResolveNowClientConn) ResolveNow(o resolver.ResolveNowOptions) { + if i.ignoreResolveNow.Load() { + return + } + i.ClientConn.ResolveNow(o) +} diff --git a/internal/xds/balancer/priority/ignore_resolve_now_test.go b/internal/xds/balancer/priority/ignore_resolve_now_test.go new file mode 100644 index 0000000..661e9d0 --- /dev/null +++ b/internal/xds/balancer/priority/ignore_resolve_now_test.go @@ -0,0 +1,66 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "context" + "testing" + "time" + + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" +) + +func (s) TestIgnoreResolveNowClientConn(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + ignoreCC := newIgnoreResolveNowClientConn(cc, false) + + // Call ResolveNow() on the CC, it should be forwarded. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + ignoreCC.ResolveNow(resolver.ResolveNowOptions{}) + select { + case <-cc.ResolveNowCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for ResolveNow()") + } + + // Update ignoreResolveNow to true, call ResolveNow() on the CC, they should + // all be ignored. + ignoreCC.updateIgnoreResolveNow(true) + for i := 0; i < 5; i++ { + ignoreCC.ResolveNow(resolver.ResolveNowOptions{}) + } + select { + case <-cc.ResolveNowCh: + t.Fatalf("got unexpected ResolveNow() call") + case <-time.After(defaultTestShortTimeout): + } + + // Update ignoreResolveNow to false, new ResolveNow() calls should be + // forwarded. + ignoreCC.updateIgnoreResolveNow(false) + ignoreCC.ResolveNow(resolver.ResolveNowOptions{}) + select { + case <-cc.ResolveNowCh: + case <-ctx.Done(): + t.Fatalf("timeout waiting for ResolveNow()") + } +} diff --git a/internal/xds/balancer/priority/logging.go b/internal/xds/balancer/priority/logging.go new file mode 100644 index 0000000..2fb8d2d --- /dev/null +++ b/internal/xds/balancer/priority/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[priority-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *priorityBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/internal/xds/balancer/wrrlocality/balancer.go b/internal/xds/balancer/wrrlocality/balancer.go new file mode 100644 index 0000000..f5f2a89 --- /dev/null +++ b/internal/xds/balancer/wrrlocality/balancer.go @@ -0,0 +1,201 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package wrrlocality provides an implementation of the wrr locality LB policy, +// as defined in [A52 - xDS Custom LB Policies]. +// +// [A52 - xDS Custom LB Policies]: https://github.com/grpc/proposal/blob/master/A52-xds-custom-lb-policies.md +package wrrlocality + +import ( + "encoding/json" + "errors" + "fmt" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/weightedtarget" + "google.golang.org/grpc/internal/grpclog" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + xdsinternal "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Name is the name of wrr_locality balancer. +const Name = "xds_wrr_locality_experimental" + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Name() string { + return Name +} + +// LBConfig is the config for the wrr locality balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + // ChildPolicy is the config for the child policy. + ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"` +} + +// To plumb in a different child in tests. +var weightedTargetName = weightedtarget.Name + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + builder := balancer.Get(weightedTargetName) + if builder == nil { + // Shouldn't happen, registered through imported weighted target, + // defensive programming. + return nil + } + + // Doesn't need to intercept any balancer.ClientConn operations; pass + // through by just giving cc to child balancer. + wtb := builder.Build(cc, bOpts) + if wtb == nil { + // shouldn't happen, defensive programming. + return nil + } + wtbCfgParser, ok := builder.(balancer.ConfigParser) + if !ok { + // Shouldn't happen, imported weighted target builder has this method. + return nil + } + wrrL := &wrrLocalityBalancer{ + child: wtb, + childParser: wtbCfgParser, + } + + wrrL.logger = prefixLogger(wrrL) + wrrL.logger.Infof("Created") + return wrrL +} + +func (bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + var lbCfg *LBConfig + if err := json.Unmarshal(s, &lbCfg); err != nil { + return nil, fmt.Errorf("xds_wrr_locality: invalid LBConfig: %s, error: %v", string(s), err) + } + if lbCfg == nil || lbCfg.ChildPolicy == nil { + return nil, errors.New("xds_wrr_locality: invalid LBConfig: child policy field must be set") + } + return lbCfg, nil +} + +type attributeKey struct{} + +// Equal allows the values to be compared by Attributes.Equal. +func (a AddrInfo) Equal(o any) bool { + oa, ok := o.(AddrInfo) + return ok && oa.LocalityWeight == a.LocalityWeight +} + +// AddrInfo is the locality weight of the locality an address is a part of. +type AddrInfo struct { + LocalityWeight uint32 +} + +// SetAddrInfo returns a copy of endpoint in which the Attributes field is +// updated with AddrInfo. +func SetAddrInfo(endpoint resolver.Endpoint, addrInfo AddrInfo) resolver.Endpoint { + endpoint.Attributes = endpoint.Attributes.WithValue(attributeKey{}, addrInfo) + return endpoint +} + +func (a AddrInfo) String() string { + return fmt.Sprintf("Locality Weight: %d", a.LocalityWeight) +} + +// getAddrInfo returns the AddrInfo stored in the Attributes field of +// ep. Returns false if no AddrInfo found. +func getAddrInfo(ep resolver.Endpoint) (AddrInfo, bool) { + v := ep.Attributes.Value(attributeKey{}) + ai, ok := v.(AddrInfo) + return ai, ok +} + +// wrrLocalityBalancer wraps a weighted target balancer, and builds +// configuration for the weighted target once it receives configuration +// specifying the weighted target child balancer and locality weight +// information. +type wrrLocalityBalancer struct { + // child will be a weighted target balancer, and will be built it at + // wrrLocalityBalancer build time. Other than preparing configuration, other + // balancer operations are simply pass through. + child balancer.Balancer + + childParser balancer.ConfigParser + + logger *grpclog.PrefixLogger +} + +func (b *wrrLocalityBalancer) ExitIdle() { + b.child.ExitIdle() +} + +func (b *wrrLocalityBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + lbCfg, ok := s.BalancerConfig.(*LBConfig) + if !ok { + b.logger.Errorf("Received config with unexpected type %T: %v", s.BalancerConfig, s.BalancerConfig) + return balancer.ErrBadResolverState + } + + weightedTargets := make(map[string]weightedtarget.Target) + for _, ep := range s.ResolverState.Endpoints { + // This get of LocalityID could potentially return a zero value. This + // shouldn't happen though (this attribute that is set actually gets + // used to build localities in the first place), and thus don't error + // out, and just build a weighted target with undefined behavior. + locality := xdsinternal.LocalityString(xdsinternal.LocalityIDFromEndpoint(ep)) + ai, ok := getAddrInfo(ep) + if !ok { + return fmt.Errorf("xds_wrr_locality: missing locality weight information in endpoint %q", ep) + } + weightedTargets[locality] = weightedtarget.Target{Weight: ai.LocalityWeight, ChildPolicy: lbCfg.ChildPolicy} + } + wtCfg := &weightedtarget.LBConfig{Targets: weightedTargets} + wtCfgJSON, err := json.Marshal(wtCfg) + if err != nil { + // Shouldn't happen. + return fmt.Errorf("xds_wrr_locality: error marshalling prepared config: %v", wtCfg) + } + var sc serviceconfig.LoadBalancingConfig + if sc, err = b.childParser.ParseConfig(wtCfgJSON); err != nil { + return fmt.Errorf("xds_wrr_locality: config generated %v is invalid: %v", wtCfgJSON, err) + } + + return b.child.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: s.ResolverState, + BalancerConfig: sc, + }) +} + +func (b *wrrLocalityBalancer) ResolverError(err error) { + b.child.ResolverError(err) +} + +func (b *wrrLocalityBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *wrrLocalityBalancer) Close() { + b.child.Close() +} diff --git a/internal/xds/balancer/wrrlocality/balancer_test.go b/internal/xds/balancer/wrrlocality/balancer_test.go new file mode 100644 index 0000000..31cfec8 --- /dev/null +++ b/internal/xds/balancer/wrrlocality/balancer_test.go @@ -0,0 +1,250 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package wrrlocality + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/balancer/weightedtarget" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + xdsinternal "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const ( + defaultTestTimeout = 5 * time.Second +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestParseConfig(t *testing.T) { + const errParseConfigName = "errParseConfigBalancer" + stub.Register(errParseConfigName, stub.BalancerFuncs{ + ParseConfig: func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return nil, errors.New("some error") + }, + }) + + parser := bb{} + tests := []struct { + name string + input string + wantCfg serviceconfig.LoadBalancingConfig + wantErr string + }{ + { + name: "happy-case-round robin-child", + input: `{"childPolicy": [{"round_robin": {}}]}`, + wantCfg: &LBConfig{ + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }, + { + name: "invalid-json", + input: "{{invalidjson{{", + wantErr: "invalid character", + }, + + { + name: "child-policy-field-isn't-set", + input: `{}`, + wantErr: "child policy field must be set", + }, + { + name: "child-policy-type-is-empty", + input: `{"childPolicy": []}`, + wantErr: "invalid loadBalancingConfig: no supported policies found in []", + }, + { + name: "child-policy-empty-config", + input: `{"childPolicy": [{"": {}}]}`, + wantErr: "invalid loadBalancingConfig: no supported policies found in []", + }, + { + name: "child-policy-type-isn't-registered", + input: `{"childPolicy": [{"doesNotExistBalancer": {"cluster": "test_cluster"}}]}`, + wantErr: "invalid loadBalancingConfig: no supported policies found in [doesNotExistBalancer]", + }, + { + name: "child-policy-config-is-invalid", + input: `{"childPolicy": [{"errParseConfigBalancer": {"cluster": "test_cluster"}}]}`, + wantErr: "error parsing loadBalancingConfig for policy \"errParseConfigBalancer\"", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotCfg, gotErr := parser.ParseConfig(json.RawMessage(test.input)) + // Substring match makes this very tightly coupled to the + // internalserviceconfig.BalancerConfig error strings. However, it + // is important to distinguish the different types of error messages + // possible as the parser has a few defined buckets of ways it can + // error out. + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if test.wantErr != "" { + return + } + if diff := cmp.Diff(gotCfg, test.wantCfg); diff != "" { + t.Fatalf("ParseConfig(%v) got unexpected output, diff (-got +want): %v", test.input, diff) + } + }) + } +} + +// TestUpdateClientConnState tests the UpdateClientConnState method of the +// wrr_locality_experimental balancer. This UpdateClientConn operation should +// take the localities and their weights in the addresses passed in, alongside +// the endpoint picking policy defined in the Balancer Config and construct a +// weighted target configuration corresponding to these inputs. +func (s) TestUpdateClientConnState(t *testing.T) { + // Configure the stub balancer defined below as the child policy of + // wrrLocalityBalancer. + cfgCh := testutils.NewChannel() + oldWeightedTargetName := weightedTargetName + defer func() { + weightedTargetName = oldWeightedTargetName + }() + weightedTargetName = "fake_weighted_target" + stub.Register("fake_weighted_target", stub.BalancerFuncs{ + ParseConfig: func(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + var cfg weightedtarget.LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + return &cfg, nil + }, + UpdateClientConnState: func(_ *stub.BalancerData, ccs balancer.ClientConnState) error { + wtCfg, ok := ccs.BalancerConfig.(*weightedtarget.LBConfig) + if !ok { + return errors.New("child received config that was not a weighted target config") + } + defer cfgCh.Send(wtCfg) + return nil + }, + }) + + builder := balancer.Get(Name) + if builder == nil { + t.Fatalf("balancer.Get(%q) returned nil", Name) + } + tcc := testutils.NewBalancerClientConn(t) + bal := builder.Build(tcc, balancer.BuildOptions{}) + defer bal.Close() + wrrL := bal.(*wrrLocalityBalancer) + + // Create the addresses with two localities with certain locality weights. + // This represents what addresses the wrr_locality balancer will receive in + // UpdateClientConnState. + ep1 := resolver.Endpoint{Addresses: []resolver.Address{{Addr: "locality-1"}}} + ep1 = xdsinternal.SetLocalityIDInEndpoint(ep1, clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }) + ep1 = SetAddrInfo(ep1, AddrInfo{LocalityWeight: 2}) + + ep2 := resolver.Endpoint{Addresses: []resolver.Address{{Addr: "locality-2"}}} + ep2 = xdsinternal.SetLocalityIDInEndpoint(ep2, clients.Locality{ + Region: "region-2", + Zone: "zone-2", + SubZone: "subzone-2", + }) + ep2 = SetAddrInfo(ep2, AddrInfo{LocalityWeight: 1}) + + eps := []resolver.Endpoint{ep1, ep2} + + err := wrrL.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: &LBConfig{ + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + ResolverState: resolver.State{ + Endpoints: eps, + }, + }) + if err != nil { + t.Fatalf("Unexpected error from UpdateClientConnState: %v", err) + } + + // Note that these inline strings declared as the key in Targets built from + // Locality ID are not exactly what is shown in the example in the gRFC. + // However, this is an implementation detail that does not affect + // correctness (confirmed with Java team). The important thing is to get + // those three pieces of information region, zone, and subzone down to the + // child layer. + wantWtCfg := &weightedtarget.LBConfig{ + Targets: map[string]weightedtarget.Target{ + "{region=\"region-1\", zone=\"zone-1\", sub_zone=\"subzone-1\"}": { + Weight: 2, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + "{region=\"region-2\", zone=\"zone-2\", sub_zone=\"subzone-2\"}": { + Weight: 1, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cfg, err := cfgCh.Receive(ctx) + if err != nil { + t.Fatalf("No signal received from UpdateClientConnState() on the child: %v", err) + } + + gotWtCfg, ok := cfg.(*weightedtarget.LBConfig) + if !ok { + // Shouldn't happen - only sends a config on this channel. + t.Fatalf("Unexpected config type: %T", gotWtCfg) + } + + if diff := cmp.Diff(gotWtCfg, wantWtCfg); diff != "" { + t.Fatalf("Child received unexpected config, diff (-got, +want): %v", diff) + } +} diff --git a/internal/xds/balancer/wrrlocality/logging.go b/internal/xds/balancer/wrrlocality/logging.go new file mode 100644 index 0000000..42ccea0 --- /dev/null +++ b/internal/xds/balancer/wrrlocality/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package wrrlocality + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[wrrlocality-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *wrrLocalityBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/internal/xds/bootstrap/bootstrap.go b/internal/xds/bootstrap/bootstrap.go new file mode 100644 index 0000000..3a517a0 --- /dev/null +++ b/internal/xds/bootstrap/bootstrap.go @@ -0,0 +1,1099 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package bootstrap provides the functionality to initialize certain aspects +// of an xDS client by reading a bootstrap file. +package bootstrap + +import ( + "bytes" + "encoding/json" + "fmt" + "maps" + "net/url" + "os" + "slices" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + xdscreds "google.golang.org/grpc/internal/xds/credentials" + "google.golang.org/grpc/xds/bootstrap" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" +) + +const ( + serverFeaturesIgnoreResourceDeletion = "ignore_resource_deletion" + serverFeaturesTrustedXDSServer = "trusted_xds_server" + gRPCUserAgentName = "gRPC Go" + clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" + clientFeatureResourceWrapper = "xds.config.resource-in-sotw" +) + +// For overriding in unit tests. +var bootstrapFileReadFunc = os.ReadFile + +// ChannelCreds contains the credentials to be used while communicating with an +// xDS server. It is also used to dedup servers with the same server URI. +// +// This type does not implement custom JSON marshal/unmarshal logic because it +// is straightforward to accomplish the same with json struct tags. +type ChannelCreds struct { + // Type contains a unique name identifying the credentials type. The only + // supported types currently are "google_default" and "insecure". + Type string `json:"type,omitempty"` + // Config contains the JSON configuration associated with the credentials. + Config json.RawMessage `json:"config,omitempty"` +} + +// Equal reports whether cc and other are considered equal. +func (cc ChannelCreds) Equal(other ChannelCreds) bool { + return cc.Type == other.Type && bytes.Equal(cc.Config, other.Config) +} + +// String returns a string representation of the credentials. It contains the +// type and the config (if non-nil) separated by a "-". +func (cc ChannelCreds) String() string { + if cc.Config == nil { + return cc.Type + } + + // We do not expect the Marshal call to fail since we wrote to cc.Config + // after a successful unmarshalling from JSON configuration. Therefore, + // it is safe to ignore the error here. + b, _ := json.Marshal(cc.Config) + return cc.Type + "-" + string(b) +} + +// CallCredsConfig contains the call credentials configuration to be used on +// RPCs to the management server. +type CallCredsConfig struct { + // Type contains a name identifying the call credentials type. + Type string `json:"type,omitempty"` + // Config contains the JSON configuration for this call credentials. + // Optional as per gRFC A97. + Config json.RawMessage `json:"config,omitempty"` +} + +// Equal reports whether cc and other are considered equal. +func (cc CallCredsConfig) Equal(other CallCredsConfig) bool { + return cc.Type == other.Type && bytes.Equal(cc.Config, other.Config) +} + +func (cc CallCredsConfig) String() string { + if len(cc.Config) == 0 { + return cc.Type + } + // We do not expect the Marshal call to fail since we wrote to cc.Config. + b, _ := json.Marshal(cc.Config) + return cc.Type + "-" + string(b) +} + +// CallCredsConfigs represents a collection of call credentials configurations. +type CallCredsConfigs []CallCredsConfig + +func (ccs CallCredsConfigs) String() string { + var creds []string + for _, cc := range ccs { + creds = append(creds, cc.String()) + } + return strings.Join(creds, ",") +} + +// AllowedGRPCService contains credentials config for an allowed gRPC service. +type AllowedGRPCService struct { + // targetURI is the fully-qualified side-channel target this entry + // applies to. It is the map key under which the service is stored, + // copied in during parsing so the service is self-describing. + targetURI string + // channelCreds is the list of channel-credential configs from the + // bootstrap JSON. Kept for MarshalJSON. + channelCreds []ChannelCreds + // callCredsConfigs is the list of call-credential configs from the + // bootstrap JSON. Kept for MarshalJSON. + callCredsConfigs []CallCredsConfig + // sideChannelCreds is the credentials bundle built from the first + // channel-creds entry whose type the client supports, paired with its + // identity. + sideChannelCreds *xdscreds.ChannelCreds + // sideCallCreds are the call credentials built from the supported + // call-creds configs, paired with their identities, preserving order. + sideCallCreds []*xdscreds.CallCreds + // cleanups release resources (credential bundles, file watchers) built + // for this service; run when the owning Config is no longer needed. + cleanups []func() +} + +// TargetURI returns the fully-qualified target URI this service applies to. +func (a *AllowedGRPCService) TargetURI() string { + return a.targetURI +} + +// SideChannelCredentials returns the channel and call credentials configured +// for this service, paired with their identities, for use when creating the +// side channel to it. The returned credentials are owned by the bootstrap +// config: their cleanups are nil, and the underlying resources are released +// via Cleanups when the config is no longer needed. +func (a *AllowedGRPCService) SideChannelCredentials() (*xdscreds.ChannelCreds, []*xdscreds.CallCreds) { + return a.sideChannelCreds, a.sideCallCreds +} + +// Cleanups returns cleanups to run when the service is no longer needed. +func (a *AllowedGRPCService) Cleanups() []func() { + return a.cleanups +} + +// Equal reports whether a and other are considered equal: the same target +// with the same built channel and call credential identities. +func (a *AllowedGRPCService) Equal(other *AllowedGRPCService) bool { + if a == nil && other == nil { + return true + } + if a == nil || other == nil { + return false + } + if a.targetURI != other.targetURI { + return false + } + if !a.sideChannelCreds.Equal(other.sideChannelCreds) { + return false + } + return slices.EqualFunc(a.sideCallCreds, other.sideCallCreds, (*xdscreds.CallCreds).Equal) +} + +type allowedGRPCServiceJSON struct { + ChannelCreds []ChannelCreds `json:"channel_creds,omitempty"` + CallCredsConfigs []CallCredsConfig `json:"call_creds,omitempty"` +} + +// allowedGRPCServicesEnabled reports whether any feature that consumes the +// allowed_grpc_services field is enabled. Per gRFC A102 the field has no +// dedicated env var; it is guarded by the env vars of its consuming features +// (currently only ext_proc on the client; OR in ext_authz/RLQS guards as +// those features are implemented). +func allowedGRPCServicesEnabled() bool { + return envconfig.XDSClientExtProcEnabled +} + +// AllowedGRPCServices maps a target URI to the credentials gRPC may use for +// that side channel when the xDS server that configured it is untrusted +// (gRFC A102). +type AllowedGRPCServices map[string]*AllowedGRPCService + +// UnmarshalJSON parses and validates the allowed_grpc_services field. The +// field is parsed only when a consuming feature is enabled, so a disabled +// feature can neither build credentials nor fail bootstrap. +func (s *AllowedGRPCServices) UnmarshalJSON(data []byte) error { + if !allowedGRPCServicesEnabled() { + return nil + } + services := make(map[string]*AllowedGRPCService) + if err := json.Unmarshal(data, &services); err != nil { + return err + } + // A JSON null decodes to a nil entry without calling UnmarshalJSON; reject + // it since channel_creds is required, and stamp the target on each service. + for target, svc := range services { + if svc == nil { + return fmt.Errorf("xds: allowed_grpc_services entry %q has no configuration in bootstrap config", target) + } + svc.targetURI = target + } + *s = services + return nil +} + +// UnmarshalJSON parses the JSON data and validates its credentials. +func (a *AllowedGRPCService) UnmarshalJSON(data []byte) (err error) { + var jsonS allowedGRPCServiceJSON + if err := json.Unmarshal(data, &jsonS); err != nil { + return err + } + + // Build into locals and assign the receiver only on success, so a + // mid-parse error never leaves the receiver partially mutated. Credentials + // built before a failure are released by this deferred cleanup. + var cleanups []func() + defer func() { + if err != nil { + for _, f := range cleanups { + f() + } + } + }() + + // The built credentials are paired with their (JSON) identities but the + // pairs carry no cleanups: the resources built here are owned by the + // bootstrap config and released via the cleanups collected below. + var sideChannelCreds *xdscreds.ChannelCreds + for _, cc := range jsonS.ChannelCreds { + c := bootstrap.GetChannelCredentials(cc.Type) + if c == nil { + continue + } + bundle, cancel, err := c.Build(cc.Config) + if err != nil { + return fmt.Errorf("xds: failed to build credentials bundle from bootstrap for allowed grpc service: type %q, err: %v", cc.Type, err) + } + identity := xdscreds.Identity{Type: cc.Type, Data: cc.Config} + sideChannelCreds = xdscreds.NewChannelCreds(bundle, identity, nil) + cleanups = append(cleanups, cancel) + break + } + + // If no channel-creds type in the list was supported, sideChannelCreds is + // still nil after the loop; that is a validation error. + if sideChannelCreds == nil { + return fmt.Errorf("xds: no supported channel credentials found for allowed grpc service in config:\n%s", string(data)) + } + + var sideCallCreds []*xdscreds.CallCreds + for _, cfg := range jsonS.CallCredsConfigs { + c := bootstrap.GetCallCredentials(cfg.Type) + if c == nil { + continue + } + callCreds, cancel, err := c.Build(cfg.Config) + if err != nil { + return fmt.Errorf("xds: failed to build call credentials from bootstrap for allowed grpc service: type %q, err: %v", cfg.Type, err) + } + identity := xdscreds.Identity{Type: cfg.Type, Data: cfg.Config} + sideCallCreds = append(sideCallCreds, xdscreds.NewCallCreds(callCreds, identity, nil)) + cleanups = append(cleanups, cancel) + } + + a.channelCreds = jsonS.ChannelCreds + a.callCredsConfigs = jsonS.CallCredsConfigs + a.sideChannelCreds = sideChannelCreds + a.sideCallCreds = sideCallCreds + a.cleanups = cleanups + return nil +} + +// MarshalJSON marshals the allowed gRPC service into JSON format. +func (a *AllowedGRPCService) MarshalJSON() ([]byte, error) { + return json.Marshal(allowedGRPCServiceJSON{ + ChannelCreds: a.channelCreds, + CallCredsConfigs: a.callCredsConfigs, + }) +} + +// ServerConfigs represents a collection of server configurations. +type ServerConfigs []*ServerConfig + +// Equal returns true if scs equals other. +func (scs *ServerConfigs) Equal(other *ServerConfigs) bool { + if len(*scs) != len(*other) { + return false + } + for i := range *scs { + if !(*scs)[i].Equal((*other)[i]) { + return false + } + } + return true +} + +// UnmarshalJSON takes the json data (a list of server configurations) and +// unmarshals it to the struct. +func (scs *ServerConfigs) UnmarshalJSON(data []byte) error { + servers := []*ServerConfig{} + if err := json.Unmarshal(data, &servers); err != nil { + return fmt.Errorf("xds: failed to JSON unmarshal server configurations during bootstrap: %v, config:\n%s", err, string(data)) + } + *scs = servers + return nil +} + +// String returns a string representation of the ServerConfigs, by concatenating +// the string representations of the underlying server configs. +func (scs *ServerConfigs) String() string { + ret := "" + for i, sc := range *scs { + if i > 0 { + ret += ", " + } + ret += sc.String() + } + return ret +} + +// Authority contains configuration for an xDS control plane authority. +// +// This type does not implement custom JSON marshal/unmarshal logic because it +// is straightforward to accomplish the same with json struct tags. +type Authority struct { + // ClientListenerResourceNameTemplate is template for the name of the + // Listener resource to subscribe to for a gRPC client channel. Used only + // when the channel is created using an "xds:" URI with this authority name. + // + // The token "%s", if present in this string, will be replaced + // with %-encoded service authority (i.e., the path part of the target + // URI used to create the gRPC channel). + // + // Must start with "xdstp:///". If it does not, + // that is considered a bootstrap file parsing error. + // + // If not present in the bootstrap file, defaults to + // "xdstp:///envoy.config.listener.v3.Listener/%s". + ClientListenerResourceNameTemplate string `json:"client_listener_resource_name_template,omitempty"` + // XDSServers contains the list of server configurations for this authority. + XDSServers ServerConfigs `json:"xds_servers,omitempty"` +} + +// Equal returns true if a equals other. +func (a *Authority) Equal(other *Authority) bool { + switch { + case a == nil && other == nil: + return true + case (a != nil) != (other != nil): + return false + case a.ClientListenerResourceNameTemplate != other.ClientListenerResourceNameTemplate: + return false + case !a.XDSServers.Equal(&other.XDSServers): + return false + } + return true +} + +// ServerConfig contains the configuration to connect to a server. +type ServerConfig struct { + serverURI string + // TODO: rename ChannelCreds to ChannelCredsConfigs for consistency with + // CallCredsConfigs. + channelCreds []ChannelCreds + callCredsConfigs []CallCredsConfig + serverFeatures []string + + // As part of unmarshalling the JSON config into this struct, we ensure that + // the credentials config is valid by building an instance of the specified + // credentials and store it here for easy access. + selectedChannelCreds ChannelCreds + selectedCallCreds []credentials.PerRPCCredentials + credsDialOption grpc.DialOption + extraDialOptions []grpc.DialOption + + cleanups []func() +} + +// ServerURI returns the URI of the management server to connect to. +func (sc *ServerConfig) ServerURI() string { + return sc.serverURI +} + +// ChannelCreds returns the credentials configuration to use when communicating +// with this server. Also used to dedup servers with the same server URI. +func (sc *ServerConfig) ChannelCreds() []ChannelCreds { + return sc.channelCreds +} + +// ServerFeatures returns the list of features supported by this server. Also +// used to dedup servers with the same server URI and channel creds. +func (sc *ServerConfig) ServerFeatures() []string { + return sc.serverFeatures +} + +// CallCredsConfigs returns the call credentials configuration for this server. +func (sc *ServerConfig) CallCredsConfigs() CallCredsConfigs { + return sc.callCredsConfigs +} + +// ServerFeaturesIgnoreResourceDeletion returns true if this server supports a +// feature where the xDS client can ignore resource deletions from this server, +// as described in gRFC A53. +// +// This feature controls the behavior of the xDS client when the server deletes +// a previously sent Listener or Cluster resource. If set, the xDS client will +// not invoke the watchers' ResourceError() method when a resource is +// deleted, nor will it remove the existing resource value from its cache. +func (sc *ServerConfig) ServerFeaturesIgnoreResourceDeletion() bool { + for _, sf := range sc.serverFeatures { + if sf == serverFeaturesIgnoreResourceDeletion { + return true + } + } + return false +} + +// ServerFeaturesTrustedXDSServer returns true if this server is trusted, +// and gRPC should accept security-config-affecting fields from the server +// as described in gRFC A81. +func (sc *ServerConfig) ServerFeaturesTrustedXDSServer() bool { + for _, sf := range sc.serverFeatures { + if sf == serverFeaturesTrustedXDSServer { + return true + } + } + return false +} + +// SelectedChannelCreds returns the selected credentials configuration for +// communicating with this server. +func (sc *ServerConfig) SelectedChannelCreds() ChannelCreds { + return sc.selectedChannelCreds +} + +// DialOptions returns a slice of all the configured dial options for this +// server except grpc.WithCredentialsBundle(). +func (sc *ServerConfig) DialOptions() []grpc.DialOption { + var dopts []grpc.DialOption + if sc.extraDialOptions != nil { + dopts = append(dopts, sc.extraDialOptions...) + } + return dopts +} + +// Cleanups returns a collection of functions to be called when the xDS client +// for this server is closed. Allows cleaning up resources created specifically +// for this server. +func (sc *ServerConfig) Cleanups() []func() { + return sc.cleanups +} + +// Equal reports whether sc and other are considered equal. +func (sc *ServerConfig) Equal(other *ServerConfig) bool { + switch { + case sc == nil && other == nil: + return true + case (sc != nil) != (other != nil): + return false + case sc.serverURI != other.serverURI: + return false + case !slices.EqualFunc(sc.channelCreds, other.channelCreds, func(a, b ChannelCreds) bool { return a.Equal(b) }): + return false + case !slices.EqualFunc(sc.callCredsConfigs, other.callCredsConfigs, func(a, b CallCredsConfig) bool { return a.Equal(b) }): + return false + case !slices.Equal(sc.serverFeatures, other.serverFeatures): + return false + } + return true +} + +// String returns the string representation of the ServerConfig. +func (sc *ServerConfig) String() string { + if len(sc.serverFeatures) == 0 { + return strings.Join([]string{sc.serverURI, sc.selectedChannelCreds.String(), sc.CallCredsConfigs().String()}, "-") + } + features := strings.Join(sc.serverFeatures, "-") + return strings.Join([]string{sc.serverURI, sc.selectedChannelCreds.String(), features, sc.CallCredsConfigs().String()}, "-") +} + +// The following fields correspond 1:1 with the JSON schema for ServerConfig. +type serverConfigJSON struct { + ServerURI string `json:"server_uri,omitempty"` + ChannelCreds []ChannelCreds `json:"channel_creds,omitempty"` + CallCredsConfigs []CallCredsConfig `json:"call_creds,omitempty"` + ServerFeatures []string `json:"server_features,omitempty"` +} + +// MarshalJSON returns marshaled JSON bytes corresponding to this server config. +func (sc *ServerConfig) MarshalJSON() ([]byte, error) { + server := &serverConfigJSON{ + ServerURI: sc.serverURI, + ChannelCreds: sc.channelCreds, + CallCredsConfigs: sc.callCredsConfigs, + ServerFeatures: sc.serverFeatures, + } + return json.Marshal(server) +} + +// extraDialOptions captures custom dial options specified via +// credentials.Bundle. +type extraDialOptions interface { + DialOptions() []grpc.DialOption +} + +// UnmarshalJSON takes the json data (a server) and unmarshals it to the struct. +func (sc *ServerConfig) UnmarshalJSON(data []byte) error { + server := serverConfigJSON{} + if err := json.Unmarshal(data, &server); err != nil { + return fmt.Errorf("xds: failed to JSON unmarshal server configuration during bootstrap: %v, config:\n%s", err, string(data)) + } + + sc.serverURI = server.ServerURI + sc.channelCreds = server.ChannelCreds + sc.callCredsConfigs = server.CallCredsConfigs + sc.serverFeatures = server.ServerFeatures + + for _, cc := range server.ChannelCreds { + // We stop at the first credential type that we support. + c := bootstrap.GetChannelCredentials(cc.Type) + if c == nil { + continue + } + bundle, cancel, err := c.Build(cc.Config) + if err != nil { + return fmt.Errorf("failed to build credentials bundle from bootstrap for %q: %v", cc.Type, err) + } + sc.selectedChannelCreds = cc + sc.credsDialOption = grpc.WithCredentialsBundle(bundle) + if d, ok := bundle.(extraDialOptions); ok { + sc.extraDialOptions = d.DialOptions() + } + sc.cleanups = append(sc.cleanups, cancel) + break + } + + if envconfig.XDSBootstrapCallCredsEnabled { + // Process call credentials - unlike channel creds, we use ALL supported + // types. Also, call credentials are optional as per gRFC A97. + for _, cfg := range server.CallCredsConfigs { + c := bootstrap.GetCallCredentials(cfg.Type) + if c == nil { + // Skip unsupported call credential types (don't fail bootstrap). + continue + } + callCreds, cancel, err := c.Build(cfg.Config) + if err != nil { + // Call credential validation failed - this should fail bootstrap. + return fmt.Errorf("failed to build call credentials from bootstrap for %q: %v", cfg.Type, err) + } + sc.selectedCallCreds = append(sc.selectedCallCreds, callCreds) + sc.extraDialOptions = append(sc.extraDialOptions, grpc.WithPerRPCCredentials(callCreds)) + sc.cleanups = append(sc.cleanups, cancel) + } + } + + if sc.serverURI == "" { + return fmt.Errorf("xds: `server_uri` field in server config cannot be empty: %s", string(data)) + } + if sc.credsDialOption == nil { + return fmt.Errorf("xds: `channel_creds` field in server config cannot be empty: %s", string(data)) + } + return nil +} + +// ServerConfigTestingOptions specifies options for creating a new ServerConfig +// for testing purposes. +// +// # Testing-Only +type ServerConfigTestingOptions struct { + // URI is the name of the server corresponding to this server config. + URI string + // ChannelCreds contains a list of channel credentials to use when talking + // to this server. If unspecified, `insecure` credentials will be used. + ChannelCreds []ChannelCreds + // CallCredsConfigs contains a list of call credentials to use for individual RPCs + // to this server. Optional. + CallCredsConfigs []CallCredsConfig + // ServerFeatures represents the list of features supported by this server. + ServerFeatures []string +} + +// ServerConfigForTesting creates a new ServerConfig from the passed in options, +// for testing purposes. +// +// # Testing-Only +func ServerConfigForTesting(opts ServerConfigTestingOptions) (*ServerConfig, error) { + cc := opts.ChannelCreds + if cc == nil { + cc = []ChannelCreds{{Type: "insecure"}} + } + scInternal := &serverConfigJSON{ + ServerURI: opts.URI, + ChannelCreds: cc, + CallCredsConfigs: opts.CallCredsConfigs, + ServerFeatures: opts.ServerFeatures, + } + scJSON, err := json.Marshal(scInternal) + if err != nil { + return nil, err + } + + sc := new(ServerConfig) + if err := sc.UnmarshalJSON(scJSON); err != nil { + return nil, err + } + return sc, nil +} + +// Config is the internal representation of the bootstrap configuration provided +// to the xDS client. +type Config struct { + xDSServers ServerConfigs + cpcs map[string]certproviderNameAndConfig + serverListenerResourceNameTemplate string + clientDefaultListenerResourceNameTemplate string + authorities map[string]*Authority + node node + + // A map from certprovider instance names to parsed buildable configs. + certProviderConfigs map[string]*certprovider.BuildableConfig + + // allowedGRPCServices is the side-channel allowlist parsed from the + // allowed_grpc_services field (gRFC A102). + allowedGRPCServices AllowedGRPCServices +} + +// AllowedGRPCServices returns the allowlist of gRPC services. +// Callers must not modify the returned map. +func (c *Config) AllowedGRPCServices() AllowedGRPCServices { + return c.allowedGRPCServices +} + +// AllowedGRPCService returns the allowed gRPC service configured for the +// given target URI, or nil if there is none. +func (c *Config) AllowedGRPCService(targetURI string) *AllowedGRPCService { + return c.allowedGRPCServices[targetURI] +} + +// XDSServers returns the top-level list of management servers to connect to, +// ordered by priority. +func (c *Config) XDSServers() ServerConfigs { + return c.xDSServers +} + +// CertProviderConfigs returns a map from certificate provider plugin instance +// name to their configuration. Callers must not modify the returned map. +func (c *Config) CertProviderConfigs() map[string]*certprovider.BuildableConfig { + return c.certProviderConfigs +} + +// ServerListenerResourceNameTemplate returns template for the name of the +// Listener resource to subscribe to for a gRPC server. +// +// If starts with "xdstp:", will be interpreted as a new-style name, +// in which case the authority of the URI will be used to select the +// relevant configuration in the "authorities" map. +// +// The token "%s", if present in this string, will be replaced with the IP +// and port on which the server is listening. (e.g., "0.0.0.0:8080", +// "[::]:8080"). For example, a value of "example/resource/%s" could become +// "example/resource/0.0.0.0:8080". If the template starts with "xdstp:", +// the replaced string will be %-encoded. +// +// There is no default; if unset, xDS-based server creation fails. +func (c *Config) ServerListenerResourceNameTemplate() string { + return c.serverListenerResourceNameTemplate +} + +// ClientDefaultListenerResourceNameTemplate returns a template for the name of +// the Listener resource to subscribe to for a gRPC client channel. Used only +// when the channel is created with an "xds:" URI with no authority. +// +// If starts with "xdstp:", will be interpreted as a new-style name, +// in which case the authority of the URI will be used to select the +// relevant configuration in the "authorities" map. +// +// The token "%s", if present in this string, will be replaced with +// the service authority (i.e., the path part of the target URI +// used to create the gRPC channel). If the template starts with +// "xdstp:", the replaced string will be %-encoded. +// +// Defaults to "%s". +func (c *Config) ClientDefaultListenerResourceNameTemplate() string { + return c.clientDefaultListenerResourceNameTemplate +} + +// Authorities returns a map of authority name to corresponding configuration. +// Callers must not modify the returned map. +// +// This is used in the following cases: +// - A gRPC client channel is created using an "xds:" URI that includes +// an authority. +// - A gRPC client channel is created using an "xds:" URI with no +// authority, but the "client_default_listener_resource_name_template" +// field above turns it into an "xdstp:" URI. +// - A gRPC server is created and the +// "server_listener_resource_name_template" field is an "xdstp:" URI. +// +// In any of those cases, it is an error if the specified authority is +// not present in this map. +func (c *Config) Authorities() map[string]*Authority { + return c.authorities +} + +// Node returns xDS a v3 Node proto corresponding to the node field in the +// bootstrap configuration, which identifies a specific gRPC instance. +func (c *Config) Node() *v3corepb.Node { + return c.node.toProto() +} + +// Equal returns true if c equals other. +func (c *Config) Equal(other *Config) bool { + switch { + case c == nil && other == nil: + return true + case (c != nil) != (other != nil): + return false + case !c.xDSServers.Equal(&other.xDSServers): + return false + case !maps.EqualFunc(c.certProviderConfigs, other.certProviderConfigs, func(a, b *certprovider.BuildableConfig) bool { return a.String() == b.String() }): + return false + case c.serverListenerResourceNameTemplate != other.serverListenerResourceNameTemplate: + return false + case c.clientDefaultListenerResourceNameTemplate != other.clientDefaultListenerResourceNameTemplate: + return false + case !maps.EqualFunc(c.authorities, other.authorities, (*Authority).Equal): + return false + case !maps.EqualFunc(c.allowedGRPCServices, other.allowedGRPCServices, (*AllowedGRPCService).Equal): + return false + case !c.node.Equal(other.node): + return false + } + return true +} + +// String returns a string representation of the Config. +func (c *Config) String() string { + s, _ := c.MarshalJSON() + return string(s) +} + +// The following fields correspond 1:1 with the JSON schema for Config. +type configJSON struct { + XDSServers ServerConfigs `json:"xds_servers,omitempty"` + CertificateProviders map[string]certproviderNameAndConfig `json:"certificate_providers,omitempty"` + ServerListenerResourceNameTemplate string `json:"server_listener_resource_name_template,omitempty"` + ClientDefaultListenerResourceNameTemplate string `json:"client_default_listener_resource_name_template,omitempty"` + Authorities map[string]*Authority `json:"authorities,omitempty"` + Node node `json:"node,omitempty"` + AllowedGRPCServices AllowedGRPCServices `json:"allowed_grpc_services,omitempty"` +} + +// MarshalJSON returns marshaled JSON bytes corresponding to this config. +func (c *Config) MarshalJSON() ([]byte, error) { + config := &configJSON{ + XDSServers: c.xDSServers, + CertificateProviders: c.cpcs, + ServerListenerResourceNameTemplate: c.serverListenerResourceNameTemplate, + ClientDefaultListenerResourceNameTemplate: c.clientDefaultListenerResourceNameTemplate, + Authorities: c.authorities, + Node: c.node, + AllowedGRPCServices: c.allowedGRPCServices, + } + return json.MarshalIndent(config, " ", " ") +} + +// UnmarshalJSON takes the json data (the complete bootstrap configuration) and +// unmarshals it to the struct. +func (c *Config) UnmarshalJSON(data []byte) error { + // Initialize the node field with client controlled values. This ensures + // even if the bootstrap configuration did not contain the node field, we + // will have a node field with client controlled fields alone. + config := configJSON{Node: newNode()} + + // Credentials (bundles, file watchers) eagerly built during unmarshaling + // are intentionally not released if parsing fails below: a failed + // bootstrap is parsed once per process (the xDS client pool memoizes + // GetConfiguration via sync.OnceValues) and is treated as fatal, so the + // leak is bounded and one-time. + if err := json.Unmarshal(data, &config); err != nil { + return fmt.Errorf("xds: json.Unmarshal(%s) failed during bootstrap: %v", string(data), err) + } + + c.xDSServers = config.XDSServers + c.cpcs = config.CertificateProviders + c.serverListenerResourceNameTemplate = config.ServerListenerResourceNameTemplate + c.clientDefaultListenerResourceNameTemplate = config.ClientDefaultListenerResourceNameTemplate + c.authorities = config.Authorities + c.node = config.Node + c.allowedGRPCServices = config.AllowedGRPCServices + + // Build the certificate providers configuration to ensure that it is valid. + cpcCfgs := make(map[string]*certprovider.BuildableConfig) + getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder) + for instance, nameAndConfig := range c.cpcs { + name := nameAndConfig.PluginName + parser := getBuilder(nameAndConfig.PluginName) + if parser == nil { + // We ignore plugins that we do not know about. + continue + } + bc, err := parser.ParseConfig(nameAndConfig.Config) + if err != nil { + return fmt.Errorf("xds: config parsing for certificate provider plugin %q failed during bootstrap: %v", name, err) + } + cpcCfgs[instance] = bc + } + c.certProviderConfigs = cpcCfgs + + // Default value of the default client listener name template is "%s". + if c.clientDefaultListenerResourceNameTemplate == "" { + c.clientDefaultListenerResourceNameTemplate = "%s" + } + if len(c.xDSServers) == 0 { + return fmt.Errorf("xds: required field `xds_servers` not found in bootstrap configuration: %s", string(data)) + } + + // Post-process the authorities' client listener resource template field: + // - if set, it must start with "xdstp:///" + // - if not set, it defaults to "xdstp:///envoy.config.listener.v3.Listener/%s" + for name, authority := range c.authorities { + prefix := fmt.Sprintf("xdstp://%s", url.PathEscape(name)) + if authority.ClientListenerResourceNameTemplate == "" { + authority.ClientListenerResourceNameTemplate = prefix + "/envoy.config.listener.v3.Listener/%s" + continue + } + if !strings.HasPrefix(authority.ClientListenerResourceNameTemplate, prefix) { + return fmt.Errorf("xds: field clientListenerResourceNameTemplate %q of authority %q doesn't start with prefix %q", authority.ClientListenerResourceNameTemplate, name, prefix) + } + } + return nil +} + +// GetConfiguration returns the bootstrap configuration initialized by reading +// the bootstrap file found at ${GRPC_XDS_BOOTSTRAP} or bootstrap contents +// specified at ${GRPC_XDS_BOOTSTRAP_CONFIG}. If both env vars are set, the +// former is preferred. +// +// This function tries to process as much of the bootstrap file as possible (in +// the presence of the errors) and may return a Config object with certain +// fields left unspecified, in which case the caller should use some sane +// defaults. +// +// This function returns an error if it's unable to parse the contents of the +// bootstrap config. It returns (nil, nil) if none of the env vars are set. +func GetConfiguration() (*Config, error) { + fName := envconfig.XDSBootstrapFileName + fContent := envconfig.XDSBootstrapFileContent + + if fName != "" { + if logger.V(2) { + logger.Infof("Using bootstrap file with name %q from GRPC_XDS_BOOTSTRAP environment variable", fName) + } + cfg, err := bootstrapFileReadFunc(fName) + if err != nil { + return nil, fmt.Errorf("xds: failed to read bootstrap config from file %q: %v", fName, err) + } + return NewConfigFromContents(cfg) + } + + if fContent != "" { + if logger.V(2) { + logger.Infof("Using bootstrap contents from GRPC_XDS_BOOTSTRAP_CONFIG environment variable") + } + return NewConfigFromContents([]byte(fContent)) + } + + return nil, nil +} + +// NewConfigFromContents creates a new bootstrap configuration from the provided +// contents. +func NewConfigFromContents(data []byte) (*Config, error) { + // Normalize the input configuration. + buf := bytes.Buffer{} + err := json.Indent(&buf, data, "", "") + if err != nil { + return nil, fmt.Errorf("xds: error normalizing JSON bootstrap configuration: %v", err) + } + data = bytes.TrimSpace(buf.Bytes()) + + config := &Config{} + if err := config.UnmarshalJSON(data); err != nil { + return nil, err + } + return config, nil +} + +// ConfigOptionsForTesting specifies options for creating a new bootstrap +// configuration for testing purposes. +// +// # Testing-Only +type ConfigOptionsForTesting struct { + // Servers is the top-level xDS server configuration. It contains a list of + // server configurations. + Servers json.RawMessage + // CertificateProviders is the certificate providers configuration. + CertificateProviders map[string]json.RawMessage + // ServerListenerResourceNameTemplate is the listener resource name template + // to be used on the gRPC server. + ServerListenerResourceNameTemplate string + // ClientDefaultListenerResourceNameTemplate is the default listener + // resource name template to be used on the gRPC client. + ClientDefaultListenerResourceNameTemplate string + // Authorities is a list of non-default authorities. + Authorities map[string]json.RawMessage + // Node identifies the gRPC client/server node in the + // proxyless service mesh. + Node json.RawMessage + // AllowedGRPCServices is the allowlist of gRPC services. + AllowedGRPCServices json.RawMessage +} + +// NewContentsForTesting creates a new bootstrap configuration from the passed in +// options, for testing purposes. +// +// # Testing-Only +func NewContentsForTesting(opts ConfigOptionsForTesting) ([]byte, error) { + var servers ServerConfigs + if err := json.Unmarshal(opts.Servers, &servers); err != nil { + return nil, err + } + certProviders := make(map[string]certproviderNameAndConfig) + for k, v := range opts.CertificateProviders { + cp := certproviderNameAndConfig{} + if err := json.Unmarshal(v, &cp); err != nil { + return nil, fmt.Errorf("failed to unmarshal certificate provider configuration for %s: %s", k, string(v)) + } + certProviders[k] = cp + } + authorities := make(map[string]*Authority) + for k, v := range opts.Authorities { + a := &Authority{} + if err := json.Unmarshal(v, a); err != nil { + return nil, fmt.Errorf("failed to unmarshal authority configuration for %s: %s", k, string(v)) + } + authorities[k] = a + } + node := newNode() + if err := json.Unmarshal(opts.Node, &node); err != nil { + return nil, fmt.Errorf("failed to unmarshal node configuration %s: %v", string(opts.Node), err) + } + allowedGRPCServices := make(AllowedGRPCServices) + if len(opts.AllowedGRPCServices) > 0 { + if err := json.Unmarshal(opts.AllowedGRPCServices, &allowedGRPCServices); err != nil { + return nil, fmt.Errorf("failed to unmarshal allowed_grpc_services configuration: %v", err) + } + // Only the parsed config is needed to marshal the bootstrap JSON below; + // release the credentials built during unmarshal so this test helper + // does not leak file watchers. + for _, svc := range allowedGRPCServices { + if svc == nil { + continue + } + for _, cleanup := range svc.Cleanups() { + cleanup() + } + } + } + cfgJSON := configJSON{ + XDSServers: servers, + CertificateProviders: certProviders, + ServerListenerResourceNameTemplate: opts.ServerListenerResourceNameTemplate, + ClientDefaultListenerResourceNameTemplate: opts.ClientDefaultListenerResourceNameTemplate, + Authorities: authorities, + Node: node, + AllowedGRPCServices: allowedGRPCServices, + } + contents, err := json.MarshalIndent(cfgJSON, " ", " ") + if err != nil { + return nil, fmt.Errorf("failed to marshal bootstrap configuration for provided options %+v: %v", opts, err) + } + return contents, nil +} + +// certproviderNameAndConfig is the internal representation of +// the`certificate_providers` field in the bootstrap configuration. +type certproviderNameAndConfig struct { + PluginName string `json:"plugin_name"` + Config json.RawMessage `json:"config"` +} + +// locality is the internal representation of the locality field within node. +type locality struct { + Region string `json:"region,omitempty"` + Zone string `json:"zone,omitempty"` + SubZone string `json:"sub_zone,omitempty"` +} + +func (l locality) Equal(other locality) bool { + return l.Region == other.Region && l.Zone == other.Zone && l.SubZone == other.SubZone +} + +func (l locality) isEmpty() bool { + return l.Equal(locality{}) +} + +type userAgentVersion struct { + UserAgentVersion string `json:"user_agent_version,omitempty"` +} + +// node is the internal representation of the node field in the bootstrap +// configuration. +type node struct { + ID string `json:"id,omitempty"` + Cluster string `json:"cluster,omitempty"` + Locality locality `json:"locality,omitempty"` + Metadata *structpb.Struct `json:"metadata,omitempty"` + + // The following fields are controlled by the client implementation and + // should not unmarshaled from JSON. + userAgentName string + userAgentVersionType userAgentVersion + clientFeatures []string +} + +// newNode is a convenience function to create a new node instance with fields +// controlled by the client implementation set to the desired values. +func newNode() node { + return node{ + userAgentName: gRPCUserAgentName, + userAgentVersionType: userAgentVersion{UserAgentVersion: grpc.Version}, + clientFeatures: []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper}, + } +} + +func (n node) Equal(other node) bool { + switch { + case n.ID != other.ID: + return false + case n.Cluster != other.Cluster: + return false + case !n.Locality.Equal(other.Locality): + return false + case n.userAgentName != other.userAgentName: + return false + case n.userAgentVersionType != other.userAgentVersionType: + return false + } + + // Consider failures in JSON marshaling as being unable to perform the + // comparison, and hence return false. + nMetadata, err := n.Metadata.MarshalJSON() + if err != nil { + return false + } + otherMetadata, err := other.Metadata.MarshalJSON() + if err != nil { + return false + } + if !bytes.Equal(nMetadata, otherMetadata) { + return false + } + + return slices.Equal(n.clientFeatures, other.clientFeatures) +} + +func (n node) toProto() *v3corepb.Node { + return &v3corepb.Node{ + Id: n.ID, + Cluster: n.Cluster, + Locality: func() *v3corepb.Locality { + if n.Locality.isEmpty() { + return nil + } + return &v3corepb.Locality{ + Region: n.Locality.Region, + Zone: n.Locality.Zone, + SubZone: n.Locality.SubZone, + } + }(), + Metadata: proto.Clone(n.Metadata).(*structpb.Struct), + UserAgentName: n.userAgentName, + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: n.userAgentVersionType.UserAgentVersion}, + ClientFeatures: slices.Clone(n.clientFeatures), + } +} diff --git a/internal/xds/bootstrap/bootstrap_test.go b/internal/xds/bootstrap/bootstrap_test.go new file mode 100644 index 0000000..26b77de --- /dev/null +++ b/internal/xds/bootstrap/bootstrap_test.go @@ -0,0 +1,1947 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package bootstrap + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "testing" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/jwt" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + xdscreds "google.golang.org/grpc/internal/xds/credentials" + "google.golang.org/grpc/xds/bootstrap" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/structpb" +) + +var ( + v3BootstrapFileMap = map[string]string{ + "serverFeaturesIncludesXDSV3": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["xds_v3"] + }] + }`, + "serverFeaturesExcludesXDSV3": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ] + }] + }`, + "emptyNodeProto": ` + { + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ] + }] + }`, + "unknownTopLevelFieldInFile": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ] + }], + "unknownField": "foobar" + }`, + "unknownFieldInNodeProto": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "unknownField": "foobar", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ] + }] + }`, + "unknownFieldInXdsServer": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ], + "unknownField": "foobar" + }] + }`, + "multipleChannelCreds": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "not-google-default" }, + { "type": "google_default" } + ], + "server_features": ["xds_v3"] + }] + }`, + "goodBootstrap": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features": ["xds_v3"] + }] + }`, + "multipleXDSServers": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [ + { + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [{ "type": "google_default" }], + "server_features": ["xds_v3"] + }, + { + "server_uri": "backup.never.use.com:1234", + "channel_creds": [{ "type": "google_default" }] + } + ] + }`, + "serverSupportsIgnoreResourceDeletion": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["ignore_resource_deletion", "xds_v3"] + }] + }`, + // example data seeded from + // https://github.com/istio/istio/blob/877e8df49d7ead6040ae812ae03ce1bad9ea2bfb/pkg/istio-agent/testdata/grpc-bootstrap.json + "istioStyleInsecureWithJWTCallCreds": ` + { + "node": { + "id": "sidecar~127.0.0.1~pod1.fake-namespace~fake-namespace.svc.cluster.local", + "metadata": { + "GENERATOR": "grpc", + "INSTANCE_IPS": "127.0.0.1", + "ISTIO_VERSION": "1.26.2", + "WORKLOAD_IDENTITY_SOCKET_FILE": "socket" + }, + "locality": {} + }, + "xds_servers" : [{ + "server_uri": "unix:///etc/istio/XDS", + "channel_creds": [ + { "type": "insecure" } + ], + "call_creds": [ + { "type": "jwt_token_file", "config": {"jwt_token_file": "/var/run/secrets/tokens/istio-token"} } + ], + "server_features" : ["xds_v3"] + }] + }`, + "istioStyleInsecureWithoutCallCreds": ` + { + "node": { + "id": "sidecar~127.0.0.1~pod1.fake-namespace~fake-namespace.svc.cluster.local", + "metadata": { + "GENERATOR": "grpc", + "INSTANCE_IPS": "127.0.0.1", + "ISTIO_VERSION": "1.26.2", + "WORKLOAD_IDENTITY_SOCKET_FILE": "socket" + }, + "locality": {} + }, + "xds_servers" : [{ + "server_uri": "unix:///etc/istio/XDS", + "channel_creds": [ + { "type": "insecure" } + ], + "server_features" : ["xds_v3"] + }] + }`, + "istioStyleWithTLSAndJWT": ` + { + "node": { + "id": "sidecar~127.0.0.1~pod1.fake-namespace~fake-namespace.svc.cluster.local", + "metadata": { + "GENERATOR": "grpc", + "INSTANCE_IPS": "127.0.0.1", + "ISTIO_VERSION": "1.26.2", + "WORKLOAD_IDENTITY_SOCKET_FILE": "socket" + }, + "locality": {} + }, + "xds_servers" : [{ + "server_uri": "unix:///etc/istio/XDS", + "channel_creds": [ + { "type": "tls", "config": {} } + ], + "call_creds": [ + { "type": "jwt_token_file", "config": {"jwt_token_file": "/var/run/secrets/tokens/istio-token"} } + ], + "server_features" : ["xds_v3"] + }] + }`, + "serverSupportsTrustedXDSServer": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["trusted_xds_server", "xds_v3"] + }] + }`, + } + metadata = &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "TRAFFICDIRECTOR_GRPC_HOSTNAME": { + Kind: &structpb.Value_StringValue{StringValue: "trafficdirector"}, + }, + }, + } + v3Node = node{ + ID: "ENVOY_NODE_ID", + Metadata: metadata, + userAgentName: gRPCUserAgentName, + userAgentVersionType: userAgentVersion{UserAgentVersion: grpc.Version}, + clientFeatures: []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper}, + } + configWithInsecureCreds = &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "insecure"}}, + selectedChannelCreds: ChannelCreds{Type: "insecure"}, + }}, + node: v3Node, + clientDefaultListenerResourceNameTemplate: "%s", + } + configWithMultipleChannelCredsAndV3 = &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "not-google-default"}, {Type: "google_default"}}, + serverFeatures: []string{"xds_v3"}, + selectedChannelCreds: ChannelCreds{Type: "google_default"}, + }}, + node: v3Node, + clientDefaultListenerResourceNameTemplate: "%s", + } + configWithGoogleDefaultCredsAndV3 = &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "google_default"}}, + serverFeatures: []string{"xds_v3"}, + selectedChannelCreds: ChannelCreds{Type: "google_default"}, + }}, + node: v3Node, + clientDefaultListenerResourceNameTemplate: "%s", + } + configWithMultipleServers = &Config{ + xDSServers: []*ServerConfig{ + { + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "google_default"}}, + serverFeatures: []string{"xds_v3"}, + selectedChannelCreds: ChannelCreds{Type: "google_default"}, + }, + { + serverURI: "backup.never.use.com:1234", + channelCreds: []ChannelCreds{{Type: "google_default"}}, + selectedChannelCreds: ChannelCreds{Type: "google_default"}, + }, + }, + node: v3Node, + clientDefaultListenerResourceNameTemplate: "%s", + } + configWithGoogleDefaultCredsAndIgnoreResourceDeletion = &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "google_default"}}, + serverFeatures: []string{"ignore_resource_deletion", "xds_v3"}, + selectedChannelCreds: ChannelCreds{Type: "google_default"}, + }}, + node: v3Node, + clientDefaultListenerResourceNameTemplate: "%s", + } + configWithGoogleDefaultCredsAndTrustedXDSServer = &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "google_default"}}, + serverFeatures: []string{"trusted_xds_server", "xds_v3"}, + selectedChannelCreds: ChannelCreds{Type: "google_default"}, + }}, + node: v3Node, + clientDefaultListenerResourceNameTemplate: "%s", + } + configWithGoogleDefaultCredsAndNoServerFeatures = &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "google_default"}}, + selectedChannelCreds: ChannelCreds{Type: "google_default"}, + }}, + node: v3Node, + clientDefaultListenerResourceNameTemplate: "%s", + } + + istioNodeMetadata = &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "GENERATOR": { + Kind: &structpb.Value_StringValue{StringValue: "grpc"}, + }, + "INSTANCE_IPS": { + Kind: &structpb.Value_StringValue{StringValue: "127.0.0.1"}, + }, + "ISTIO_VERSION": { + Kind: &structpb.Value_StringValue{StringValue: "1.26.2"}, + }, + "WORKLOAD_IDENTITY_SOCKET_FILE": { + Kind: &structpb.Value_StringValue{StringValue: "socket"}, + }, + }, + } + jwtCallCreds, _ = jwt.NewTokenFileCallCredentials("/var/run/secrets/tokens/istio-token") + selectedJWTCallCreds = []credentials.PerRPCCredentials{jwtCallCreds} + configWithIstioStyleNoCallCreds = &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "unix:///etc/istio/XDS", + channelCreds: []ChannelCreds{{Type: "insecure"}}, + serverFeatures: []string{"xds_v3"}, + selectedChannelCreds: ChannelCreds{Type: "insecure"}, + }}, + node: node{ + ID: "sidecar~127.0.0.1~pod1.fake-namespace~fake-namespace.svc.cluster.local", + Metadata: istioNodeMetadata, + userAgentName: gRPCUserAgentName, + userAgentVersionType: userAgentVersion{UserAgentVersion: grpc.Version}, + clientFeatures: []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper}, + }, + certProviderConfigs: map[string]*certprovider.BuildableConfig{}, + clientDefaultListenerResourceNameTemplate: "%s", + } +) + +func fileReadFromFileMap(bootstrapFileMap map[string]string, name string) ([]byte, error) { + if b, ok := bootstrapFileMap[name]; ok { + return []byte(b), nil + } + return nil, os.ErrNotExist +} + +func setupBootstrapOverride(bootstrapFileMap map[string]string) func() { + oldFileReadFunc := bootstrapFileReadFunc + bootstrapFileReadFunc = func(filename string) ([]byte, error) { + return fileReadFromFileMap(bootstrapFileMap, filename) + } + return func() { bootstrapFileReadFunc = oldFileReadFunc } +} + +// This function overrides the bootstrap file NAME env variable, to test the +// code that reads file with the given fileName. +func testGetConfigurationWithFileNameEnv(t *testing.T, fileName string, wantError bool, wantConfig *Config) { + origBootstrapFileName := envconfig.XDSBootstrapFileName + envconfig.XDSBootstrapFileName = fileName + defer func() { envconfig.XDSBootstrapFileName = origBootstrapFileName }() + + c, err := GetConfiguration() + if (err != nil) != wantError { + t.Fatalf("GetConfiguration() returned error %v, wantError: %v", err, wantError) + } + if wantError { + return + } + if diff := cmp.Diff(wantConfig, c); diff != "" { + t.Fatalf("Unexpected diff in bootstrap configuration (-want, +got):\n%s", diff) + } +} + +// This function overrides the bootstrap file CONTENT env variable, to test the +// code that uses the content from env directly. +func testGetConfigurationWithFileContentEnv(t *testing.T, fileName string, wantError bool, wantConfig *Config) { + t.Helper() + b, err := bootstrapFileReadFunc(fileName) + if err != nil { + t.Skip(err) + } + origBootstrapContent := envconfig.XDSBootstrapFileContent + envconfig.XDSBootstrapFileContent = string(b) + defer func() { envconfig.XDSBootstrapFileContent = origBootstrapContent }() + + c, err := GetConfiguration() + if (err != nil) != wantError { + t.Fatalf("GetConfiguration() returned error %v, wantError: %v", err, wantError) + } + if wantError { + return + } + if diff := cmp.Diff(wantConfig, c); diff != "" { + t.Fatalf("Unexpected diff in bootstrap configuration (-want, +got):\n%s", diff) + } +} + +// Tests GetConfiguration with bootstrap file contents that are expected to +// fail. +func (s) TestGetConfiguration_Failure(t *testing.T) { + bootstrapFileMap := map[string]string{ + "empty": "", + "badJSON": `["test": 123]`, + "noBalancerName": `{"node": {"id": "ENVOY_NODE_ID"}}`, + "emptyXdsServer": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + } + }`, + "emptyChannelCreds": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443" + }] + }`, + "nonGoogleDefaultCreds": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "not-google-default" } + ] + }] + }`, + } + cancel := setupBootstrapOverride(bootstrapFileMap) + defer cancel() + + for _, name := range []string{"nonExistentBootstrapFile", "badJSON", "noBalancerName", "emptyXdsServer"} { + t.Run(name, func(t *testing.T) { + testGetConfigurationWithFileNameEnv(t, name, true, nil) + testGetConfigurationWithFileContentEnv(t, name, true, nil) + }) + } + const name = "empty" + t.Run(name, func(t *testing.T) { + testGetConfigurationWithFileNameEnv(t, name, true, nil) + // If both the env vars are empty, a nil config with a nil error must be + // returned. + testGetConfigurationWithFileContentEnv(t, name, false, nil) + }) +} + +// Tests the functionality in GetConfiguration with different bootstrap file +// contents. It overrides the fileReadFunc by returning bootstrap file contents +// defined in this test, instead of reading from a file. +func (s) TestGetConfiguration_Success(t *testing.T) { + cancel := setupBootstrapOverride(v3BootstrapFileMap) + defer cancel() + + tests := []struct { + name string + wantConfig *Config + }{ + { + name: "emptyNodeProto", + wantConfig: &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "insecure"}}, + selectedChannelCreds: ChannelCreds{Type: "insecure"}, + }}, + node: node{ + userAgentName: gRPCUserAgentName, + userAgentVersionType: userAgentVersion{UserAgentVersion: grpc.Version}, + clientFeatures: []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper}, + }, + clientDefaultListenerResourceNameTemplate: "%s", + }, + }, + {"unknownTopLevelFieldInFile", configWithInsecureCreds}, + {"unknownFieldInNodeProto", configWithInsecureCreds}, + {"unknownFieldInXdsServer", configWithInsecureCreds}, + {"multipleChannelCreds", configWithMultipleChannelCredsAndV3}, + {"goodBootstrap", configWithGoogleDefaultCredsAndV3}, + {"multipleXDSServers", configWithMultipleServers}, + {"serverSupportsIgnoreResourceDeletion", configWithGoogleDefaultCredsAndIgnoreResourceDeletion}, + {"serverSupportsTrustedXDSServer", configWithGoogleDefaultCredsAndTrustedXDSServer}, + {"istioStyleInsecureWithoutCallCreds", configWithIstioStyleNoCallCreds}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testGetConfigurationWithFileNameEnv(t, test.name, false, test.wantConfig) + testGetConfigurationWithFileContentEnv(t, test.name, false, test.wantConfig) + }) + } +} + +// Tests Istio-style bootstrap configurations with JWT call credentials. +func (s) TestGetConfiguration_IstioStyleWithCallCreds(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSBootstrapCallCredsEnabled, true) + cancel := setupBootstrapOverride(v3BootstrapFileMap) + defer cancel() + + configWithIstioJWTCallCreds := &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "unix:///etc/istio/XDS", + channelCreds: []ChannelCreds{{Type: "insecure"}}, + callCredsConfigs: []CallCredsConfig{{Type: "jwt_token_file", Config: json.RawMessage("{\n\"jwt_token_file\": \"/var/run/secrets/tokens/istio-token\"\n}")}}, + serverFeatures: []string{"xds_v3"}, + selectedChannelCreds: ChannelCreds{Type: "insecure"}, + selectedCallCreds: selectedJWTCallCreds, + }}, + node: node{ + ID: "sidecar~127.0.0.1~pod1.fake-namespace~fake-namespace.svc.cluster.local", + Metadata: istioNodeMetadata, + userAgentName: gRPCUserAgentName, + userAgentVersionType: userAgentVersion{UserAgentVersion: grpc.Version}, + clientFeatures: []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper}, + }, + certProviderConfigs: map[string]*certprovider.BuildableConfig{}, + clientDefaultListenerResourceNameTemplate: "%s", + } + configWithIstioStyleWithTLSAndJWT := &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "unix:///etc/istio/XDS", + channelCreds: []ChannelCreds{{Type: "tls", Config: json.RawMessage("{}")}}, + callCredsConfigs: []CallCredsConfig{{Type: "jwt_token_file", Config: json.RawMessage("{\n\"jwt_token_file\": \"/var/run/secrets/tokens/istio-token\"\n}")}}, + serverFeatures: []string{"xds_v3"}, + selectedChannelCreds: ChannelCreds{Type: "tls", Config: json.RawMessage("{}")}, + selectedCallCreds: selectedJWTCallCreds, + }}, + node: node{ + ID: "sidecar~127.0.0.1~pod1.fake-namespace~fake-namespace.svc.cluster.local", + Metadata: istioNodeMetadata, + userAgentName: gRPCUserAgentName, + userAgentVersionType: userAgentVersion{UserAgentVersion: grpc.Version}, + clientFeatures: []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper}, + }, + certProviderConfigs: map[string]*certprovider.BuildableConfig{}, + clientDefaultListenerResourceNameTemplate: "%s", + } + + tests := []struct { + name string + wantConfig *Config + }{ + {"istioStyleInsecureWithJWTCallCreds", configWithIstioJWTCallCreds}, + {"istioStyleWithTLSAndJWT", configWithIstioStyleWithTLSAndJWT}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testGetConfigurationWithFileNameEnv(t, test.name, false, test.wantConfig) + testGetConfigurationWithFileContentEnv(t, test.name, false, test.wantConfig) + }) + } +} + +// Tests that the two bootstrap env variables are read in correct priority. +// +// "GRPC_XDS_BOOTSTRAP" which specifies the file name containing the bootstrap +// configuration takes precedence over "GRPC_XDS_BOOTSTRAP_CONFIG", which +// directly specifies the bootstrap configuration in itself. +func (s) TestGetConfiguration_BootstrapEnvPriority(t *testing.T) { + oldFileReadFunc := bootstrapFileReadFunc + bootstrapFileReadFunc = func(filename string) ([]byte, error) { + return fileReadFromFileMap(v3BootstrapFileMap, filename) + } + defer func() { bootstrapFileReadFunc = oldFileReadFunc }() + + goodFileName1 := "serverFeaturesIncludesXDSV3" + goodConfig1 := configWithGoogleDefaultCredsAndV3 + + goodFileName2 := "serverFeaturesExcludesXDSV3" + goodFileContent2 := v3BootstrapFileMap[goodFileName2] + goodConfig2 := configWithGoogleDefaultCredsAndNoServerFeatures + + origBootstrapFileName := envconfig.XDSBootstrapFileName + envconfig.XDSBootstrapFileName = "" + defer func() { envconfig.XDSBootstrapFileName = origBootstrapFileName }() + + origBootstrapContent := envconfig.XDSBootstrapFileContent + envconfig.XDSBootstrapFileContent = "" + defer func() { envconfig.XDSBootstrapFileContent = origBootstrapContent }() + + // When both env variables are empty, GetConfiguration should return nil. + if cfg, err := GetConfiguration(); err != nil || cfg != nil { + t.Errorf("GetConfiguration() returned (%v, %v), want (, )", cfg, err) + } + + // When one of them is set, it should be used. + envconfig.XDSBootstrapFileName = goodFileName1 + envconfig.XDSBootstrapFileContent = "" + c, err := GetConfiguration() + if err != nil { + t.Errorf("GetConfiguration() failed: %v", err) + } + if diff := cmp.Diff(goodConfig1, c); diff != "" { + t.Errorf("Unexpected diff in bootstrap configuration (-want, +got):\n%s", diff) + } + + envconfig.XDSBootstrapFileName = "" + envconfig.XDSBootstrapFileContent = goodFileContent2 + c, err = GetConfiguration() + if err != nil { + t.Errorf("GetConfiguration() failed: %v", err) + } + if diff := cmp.Diff(goodConfig2, c); diff != "" { + t.Errorf("Unexpected diff in bootstrap configuration (-want, +got):\n%s", diff) + } + + // Set both, file name should be read. + envconfig.XDSBootstrapFileName = goodFileName1 + envconfig.XDSBootstrapFileContent = goodFileContent2 + c, err = GetConfiguration() + if err != nil { + t.Errorf("GetConfiguration() failed: %v", err) + } + if diff := cmp.Diff(goodConfig1, c); diff != "" { + t.Errorf("Unexpected diff in bootstrap configuration (-want, +got):\n%s", diff) + } +} + +func init() { + certprovider.Register(&fakeCertProviderBuilder{}) +} + +const fakeCertProviderName = "fake-certificate-provider" + +// fakeCertProviderBuilder builds new instances of fakeCertProvider and +// interprets the config provided to it as JSON with a single key and value. +type fakeCertProviderBuilder struct{} + +// ParseConfig expects input in JSON format containing a map from string to +// string, with a single entry and mapKey being "configKey". +func (b *fakeCertProviderBuilder) ParseConfig(cfg any) (*certprovider.BuildableConfig, error) { + config, ok := cfg.(json.RawMessage) + if !ok { + return nil, fmt.Errorf("fakeCertProviderBuilder received config of type %T, want []byte", config) + } + var cfgData map[string]string + if err := json.Unmarshal(config, &cfgData); err != nil { + return nil, fmt.Errorf("fakeCertProviderBuilder config parsing failed: %v", err) + } + if len(cfgData) != 1 || cfgData["configKey"] == "" { + return nil, errors.New("fakeCertProviderBuilder received invalid config") + } + fc := &fakeStableConfig{config: cfgData} + return certprovider.NewBuildableConfig(fakeCertProviderName, fc.canonical(), func(certprovider.BuildOptions) certprovider.Provider { + return &fakeCertProvider{} + }), nil +} + +func (b *fakeCertProviderBuilder) Name() string { + return fakeCertProviderName +} + +type fakeStableConfig struct { + config map[string]string +} + +func (c *fakeStableConfig) canonical() []byte { + var cfg string + for k, v := range c.config { + cfg = fmt.Sprintf("%s:%s", k, v) + } + return []byte(cfg) +} + +// fakeCertProvider is an empty implementation of the Provider interface. +type fakeCertProvider struct { + certprovider.Provider +} + +func (s) TestGetConfiguration_CertificateProviders(t *testing.T) { + bootstrapFileMap := map[string]string{ + "badJSONCertProviderConfig": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["foo", "bar", "xds_v3"], + }], + "certificate_providers": "bad JSON" + }`, + "allUnknownCertProviders": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["xds_v3"] + }], + "certificate_providers": { + "unknownProviderInstance1": { + "plugin_name": "foo", + "config": {"foo": "bar"} + }, + "unknownProviderInstance2": { + "plugin_name": "bar", + "config": {"foo": "bar"} + } + } + }`, + "badCertProviderConfig": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["xds_v3"], + }], + "certificate_providers": { + "unknownProviderInstance": { + "plugin_name": "foo", + "config": {"foo": "bar"} + }, + "fakeProviderInstanceBad": { + "plugin_name": "fake-certificate-provider", + "config": {"configKey": 666} + } + } + }`, + "goodCertProviderConfig": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ], + "server_features" : ["xds_v3"] + }], + "certificate_providers": { + "unknownProviderInstance": { + "plugin_name": "foo", + "config": {"foo": "bar"} + }, + "fakeProviderInstance": { + "plugin_name": "fake-certificate-provider", + "config": {"configKey": "configValue"} + } + } + }`, + } + + getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder) + parser := getBuilder(fakeCertProviderName) + if parser == nil { + t.Fatalf("Missing certprovider plugin %q", fakeCertProviderName) + } + wantCfg, err := parser.ParseConfig(json.RawMessage(`{"configKey": "configValue"}`)) + if err != nil { + t.Fatalf("config parsing for plugin %q failed: %v", fakeCertProviderName, err) + } + + cancel := setupBootstrapOverride(bootstrapFileMap) + defer cancel() + + goodConfig := &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "insecure"}}, + serverFeatures: []string{"xds_v3"}, + selectedChannelCreds: ChannelCreds{Type: "insecure"}, + }}, + certProviderConfigs: map[string]*certprovider.BuildableConfig{ + "fakeProviderInstance": wantCfg, + }, + clientDefaultListenerResourceNameTemplate: "%s", + node: v3Node, + } + tests := []struct { + name string + wantConfig *Config + wantErr bool + }{ + { + name: "badJSONCertProviderConfig", + wantErr: true, + }, + { + + name: "badCertProviderConfig", + wantErr: true, + }, + { + + name: "allUnknownCertProviders", + wantConfig: configWithGoogleDefaultCredsAndV3, + }, + { + name: "goodCertProviderConfig", + wantConfig: goodConfig, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testGetConfigurationWithFileNameEnv(t, test.name, test.wantErr, test.wantConfig) + testGetConfigurationWithFileContentEnv(t, test.name, test.wantErr, test.wantConfig) + }) + } +} + +func (s) TestGetConfiguration_ServerListenerResourceNameTemplate(t *testing.T) { + cancel := setupBootstrapOverride(map[string]string{ + "badServerListenerResourceNameTemplate:": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ] + }], + "server_listener_resource_name_template": 123456789 + }`, + "goodServerListenerResourceNameTemplate": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ] + }], + "server_listener_resource_name_template": "grpc/server?xds.resource.listening_address=%s" + }`, + }) + defer cancel() + + tests := []struct { + name string + wantConfig *Config + wantErr bool + }{ + { + name: "badServerListenerResourceNameTemplate", + wantErr: true, + }, + { + name: "goodServerListenerResourceNameTemplate", + wantConfig: &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "google_default"}}, + selectedChannelCreds: ChannelCreds{Type: "google_default"}, + }}, + node: v3Node, + serverListenerResourceNameTemplate: "grpc/server?xds.resource.listening_address=%s", + clientDefaultListenerResourceNameTemplate: "%s", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testGetConfigurationWithFileNameEnv(t, test.name, test.wantErr, test.wantConfig) + testGetConfigurationWithFileContentEnv(t, test.name, test.wantErr, test.wantConfig) + }) + } +} + +func (s) TestGetConfiguration_Federation(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true) + cancel := setupBootstrapOverride(map[string]string{ + "badclientListenerResourceNameTemplate": ` + { + "node": { "id": "ENVOY_NODE_ID" }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443" + }], + "client_default_listener_resource_name_template": 123456789 + }`, + "badclientListenerResourceNameTemplatePerAuthority": ` + { + "node": { "id": "ENVOY_NODE_ID" }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }], + "authorities": { + "xds.td.com": { + "client_listener_resource_name_template": "some/template/%s", + "xds_servers": [{ + "server_uri": "td.com", + "channel_creds": [ { "type": "google_default" } ], + "server_features" : ["foo", "bar", "xds_v3"] + }] + } + } + }`, + "good": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }], + "server_listener_resource_name_template": "xdstp://xds.example.com/envoy.config.listener.v3.Listener/grpc/server?listening_address=%s", + "client_default_listener_resource_name_template": "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + "authorities": { + "xds.td.com": { + "client_listener_resource_name_template": "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s", + "xds_servers": [{ + "server_uri": "td.com", + "channel_creds": [ { "type": "google_default" } ], + "server_features" : ["xds_v3"] + }] + } + }, + "allowed_grpc_services": { + "dns:///whitelisted-ext-proc:443": { + "channel_creds": [ + { "type": "insecure" } + ], + "call_creds": [ + { "type": "jwt_token_file", "config": {"jwt_token_file":"/var/run/secrets/tokens/istio-token"} } + ] + } + } + }`, + // If client_default_listener_resource_name_template is not set, it + // defaults to "%s". + "goodWithDefaultDefaultClientListenerTemplate": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }] + }`, + // If client_listener_resource_name_template in authority is not set, it + // defaults to + // "xdstp:///envoy.config.listener.v3.Listener/%s". + "goodWithDefaultClientListenerTemplatePerAuthority": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }], + "client_default_listener_resource_name_template": "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + "authorities": { + "xds.td.com": { }, + "#.com": { } + } + }`, + // It's OK for an authority to not have servers. The top-level server + // will be used. + "goodWithNoServerPerAuthority": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }], + "client_default_listener_resource_name_template": "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + "authorities": { + "xds.td.com": { + "client_listener_resource_name_template": "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s" + } + } + }`, + }) + defer cancel() + + tests := []struct { + name string + wantConfig *Config + wantErr bool + }{ + { + name: "badclientListenerResourceNameTemplate", + wantErr: true, + }, + { + name: "badclientListenerResourceNameTemplatePerAuthority", + wantErr: true, + }, + { + name: "good", + wantConfig: &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "google_default"}}, + selectedChannelCreds: ChannelCreds{Type: "google_default"}, + }}, + node: v3Node, + serverListenerResourceNameTemplate: "xdstp://xds.example.com/envoy.config.listener.v3.Listener/grpc/server?listening_address=%s", + clientDefaultListenerResourceNameTemplate: "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + authorities: map[string]*Authority{ + "xds.td.com": { + ClientListenerResourceNameTemplate: "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s", + XDSServers: []*ServerConfig{{ + serverURI: "td.com", + channelCreds: []ChannelCreds{{Type: "google_default"}}, + serverFeatures: []string{"xds_v3"}, + selectedChannelCreds: ChannelCreds{Type: "google_default"}, + }}, + }, + }, + allowedGRPCServices: map[string]*AllowedGRPCService{ + "dns:///whitelisted-ext-proc:443": { + targetURI: "dns:///whitelisted-ext-proc:443", + channelCreds: []ChannelCreds{{Type: "insecure"}}, + callCredsConfigs: []CallCredsConfig{{ + Type: "jwt_token_file", + Config: json.RawMessage("{\n\"jwt_token_file\": \"/var/run/secrets/tokens/istio-token\"\n}"), + }}, + // Equal compares the built credentials by identity, so + // the fixture carries identity-only pairs. + sideChannelCreds: xdscreds.NewChannelCreds(nil, xdscreds.Identity{Type: "insecure"}, nil), + sideCallCreds: []*xdscreds.CallCreds{xdscreds.NewCallCreds(nil, xdscreds.Identity{ + Type: "jwt_token_file", + Data: json.RawMessage("{\n\"jwt_token_file\": \"/var/run/secrets/tokens/istio-token\"\n}"), + }, nil)}, + }, + }, + }, + }, + { + name: "goodWithDefaultDefaultClientListenerTemplate", + wantConfig: &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "google_default"}}, + selectedChannelCreds: ChannelCreds{Type: "google_default"}, + }}, + node: v3Node, + clientDefaultListenerResourceNameTemplate: "%s", + }, + }, + { + name: "goodWithDefaultClientListenerTemplatePerAuthority", + wantConfig: &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "google_default"}}, + selectedChannelCreds: ChannelCreds{Type: "google_default"}, + }}, + node: v3Node, + clientDefaultListenerResourceNameTemplate: "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + authorities: map[string]*Authority{ + "xds.td.com": { + ClientListenerResourceNameTemplate: "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s", + }, + "#.com": { + ClientListenerResourceNameTemplate: "xdstp://%23.com/envoy.config.listener.v3.Listener/%s", + }, + }, + }, + }, + { + name: "goodWithNoServerPerAuthority", + wantConfig: &Config{ + xDSServers: []*ServerConfig{{ + serverURI: "trafficdirector.googleapis.com:443", + channelCreds: []ChannelCreds{{Type: "google_default"}}, + selectedChannelCreds: ChannelCreds{Type: "google_default"}, + }}, + node: v3Node, + clientDefaultListenerResourceNameTemplate: "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + authorities: map[string]*Authority{ + "xds.td.com": { + ClientListenerResourceNameTemplate: "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s", + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testGetConfigurationWithFileNameEnv(t, test.name, test.wantErr, test.wantConfig) + testGetConfigurationWithFileContentEnv(t, test.name, test.wantErr, test.wantConfig) + }) + } +} + +func (s) TestServerConfigMarshalAndUnmarshal(t *testing.T) { + origConfig, err := ServerConfigForTesting(ServerConfigTestingOptions{URI: "test-server", ServerFeatures: []string{"xds_v3"}}) + if err != nil { + t.Fatalf("Failed to create server config for testing: %v", err) + } + marshaledCfg, err := json.Marshal(origConfig) + if err != nil { + t.Fatalf("failed to marshal: %v", err) + } + + unmarshaledConfig := new(ServerConfig) + if err := json.Unmarshal(marshaledCfg, unmarshaledConfig); err != nil { + t.Fatalf("failed to unmarshal: %v", err) + } + if diff := cmp.Diff(origConfig, unmarshaledConfig); diff != "" { + t.Fatalf("Unexpected diff in server config (-want, +got):\n%s", diff) + } +} + +func (s) TestDefaultBundles(t *testing.T) { + tests := []string{"google_default", "insecure", "tls"} + + for _, typename := range tests { + t.Run(typename, func(t *testing.T) { + if c := bootstrap.GetChannelCredentials(typename); c == nil { + t.Errorf(`bootstrap.GetCredentials(%s) credential is nil, want non-nil`, typename) + } + }) + } +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestCallCreds_Equal(t *testing.T) { + tests := []struct { + name string + cc1 CallCredsConfig + cc2 CallCredsConfig + want bool + }{ + { + name: "identical_configs", + cc1: CallCredsConfig{Type: "jwt_token_file", Config: json.RawMessage(`{"jwt_token_file": "/path/to/token"}`)}, + cc2: CallCredsConfig{Type: "jwt_token_file", Config: json.RawMessage(`{"jwt_token_file": "/path/to/token"}`)}, + want: true, + }, + { + name: "different_types", + cc1: CallCredsConfig{Type: "jwt_token_file", Config: json.RawMessage(`{"jwt_token_file": "/path/to/token"}`)}, + cc2: CallCredsConfig{Type: "other_type", Config: json.RawMessage(`{"jwt_token_file": "/path/to/token"}`)}, + want: false, + }, + { + name: "different_configs", + cc1: CallCredsConfig{Type: "jwt_token_file", Config: json.RawMessage(`{"jwt_token_file": "/path/to/token"}`)}, + cc2: CallCredsConfig{Type: "jwt_token_file", Config: json.RawMessage(`{"jwt_token_file": "/different/path"}`)}, + want: false, + }, + { + name: "nil_vs_non-nil_configs", + cc1: CallCredsConfig{Type: "jwt_token_file", Config: nil}, + cc2: CallCredsConfig{Type: "jwt_token_file", Config: json.RawMessage(`{"jwt_token_file": "/path/to/token"}`)}, + want: false, + }, + { + name: "both_nil_configs", + cc1: CallCredsConfig{Type: "jwt_token_file", Config: nil}, + cc2: CallCredsConfig{Type: "jwt_token_file", Config: nil}, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := test.cc1.Equal(test.cc2); got != test.want { + t.Errorf("CallCreds.Equal() = %v, want %v", got, test.want) + } + }) + } +} + +func (s) TestServerConfig_UnmarshalJSON_WithCallCreds(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSBootstrapCallCredsEnabled, true) + tests := []struct { + name string + json string + wantCallCreds CallCredsConfigs + }{ + { + name: "valid_call_creds_with_jwt_token_file", + json: `{ + "server_uri": "xds-server:443", + "channel_creds": [{"type": "insecure"}], + "call_creds": [ + { + "type": "jwt_token_file", + "config": {"jwt_token_file": "/path/to/token.jwt"} + } + ] + }`, + wantCallCreds: []CallCredsConfig{{ + Type: "jwt_token_file", + Config: json.RawMessage(`{"jwt_token_file": "/path/to/token.jwt"}`), + }}, + }, + { + name: "multiple_call_creds_types", + json: `{ + "server_uri": "xds-server:443", + "channel_creds": [{"type": "insecure"}], + "call_creds": [ + {"type": "jwt_token_file", "config": {"jwt_token_file": "/token1.jwt"}}, + {"type": "unsupported_type", "config": {}} + ] + }`, + wantCallCreds: []CallCredsConfig{ + {Type: "jwt_token_file", Config: json.RawMessage(`{"jwt_token_file": "/token1.jwt"}`)}, + {Type: "unsupported_type", Config: json.RawMessage(`{}`)}, + }, + }, + { + name: "empty_call_creds_array", + json: `{ + "server_uri": "xds-server:443", + "channel_creds": [{"type": "insecure"}], + "call_creds": [] + }`, + wantCallCreds: []CallCredsConfig{}, + }, + { + name: "unspecified_call_creds_field", + json: `{ + "server_uri": "xds-server:443", + "channel_creds": [{"type": "insecure"}] + }`, + wantCallCreds: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var sc ServerConfig + err := sc.UnmarshalJSON([]byte(test.json)) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if diff := cmp.Diff(test.wantCallCreds, sc.CallCredsConfigs()); diff != "" { + t.Errorf("CallCreds mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func (s) TestServerConfig_Equal_WithCallCreds(t *testing.T) { + callCreds := []CallCredsConfig{{ + Type: "jwt_token_file", + Config: json.RawMessage(`{"jwt_token_file": "/test/token.jwt"}`), + }} + sc1 := &ServerConfig{ + serverURI: "server1", + channelCreds: []ChannelCreds{{Type: "insecure"}}, + callCredsConfigs: callCreds, + serverFeatures: []string{"feature1"}, + } + sc2 := &ServerConfig{ + serverURI: "server1", + channelCreds: []ChannelCreds{{Type: "insecure"}}, + callCredsConfigs: callCreds, + serverFeatures: []string{"feature1"}, + } + sc3 := &ServerConfig{ + serverURI: "server1", + channelCreds: []ChannelCreds{{Type: "insecure"}}, + callCredsConfigs: []CallCredsConfig{{Type: "different"}}, + serverFeatures: []string{"feature1"}, + } + + if !sc1.Equal(sc2) { + t.Error("Equal ServerConfigs with same call creds should be equal") + } + if sc1.Equal(sc3) { + t.Error("ServerConfigs with different call creds should not be equal") + } +} + +func (s) TestServerConfig_MarshalJSON_WithCallCreds(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSBootstrapCallCredsEnabled, true) + sc := &ServerConfig{ + serverURI: "test-server:443", + channelCreds: []ChannelCreds{{Type: "insecure"}}, + callCredsConfigs: []CallCredsConfig{{ + Type: "jwt_token_file", + Config: json.RawMessage(`{"jwt_token_file":"/test/token.jwt"}`), + }}, + serverFeatures: []string{"test_feature"}, + } + + data, err := sc.MarshalJSON() + if err != nil { + t.Fatalf("MarshalJSON failed: %v", err) + } + + // Check Marshal/Unmarshal symmetry. + var unmarshaled ServerConfig + if err := json.Unmarshal(data, &unmarshaled); err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + if diff := cmp.Diff(sc.CallCredsConfigs(), unmarshaled.CallCredsConfigs()); diff != "" { + t.Errorf("Marshal/Unmarshal call credentials produces differences:\n%s", diff) + } +} + +func newStructProtoFromMap(t *testing.T, input map[string]any) *structpb.Struct { + t.Helper() + + ret, err := structpb.NewStruct(input) + if err != nil { + t.Fatalf("Failed to create new struct proto from map %v: %v", input, err) + } + return ret +} + +func (s) TestNode_MarshalAndUnmarshal(t *testing.T) { + tests := []struct { + desc string + inputJSON []byte + wantNode node + }{ + { + desc: "basic happy case", + inputJSON: []byte(`{ + "id": "id", + "cluster": "cluster", + "locality": { + "region": "region", + "zone": "zone", + "sub_zone": "sub_zone" + }, + "metadata": { + "k1": "v1", + "k2": 101, + "k3": 280.0 + } +}`), + wantNode: node{ + ID: "id", + Cluster: "cluster", + Locality: locality{ + Region: "region", + Zone: "zone", + SubZone: "sub_zone", + }, + Metadata: newStructProtoFromMap(t, map[string]any{ + "k1": "v1", + "k2": 101, + "k3": 280.0, + }), + userAgentName: "gRPC Go", + userAgentVersionType: userAgentVersion{UserAgentVersion: grpc.Version}, + clientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + }, + }, + { + desc: "client controlled fields", + inputJSON: []byte(`{ + "id": "id", + "cluster": "cluster", + "user_agent_name": "user_agent_name", + "user_agent_version_type": { + "user_agent_version": "version" + }, + "client_features": ["feature1", "feature2"] +}`), + wantNode: node{ + ID: "id", + Cluster: "cluster", + userAgentName: "gRPC Go", + userAgentVersionType: userAgentVersion{UserAgentVersion: grpc.Version}, + clientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // Unmarshal the input JSON into a node struct and check if it + // matches expectations. + unmarshaledNode := newNode() + if err := json.Unmarshal([]byte(test.inputJSON), &unmarshaledNode); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.wantNode, unmarshaledNode); diff != "" { + t.Fatalf("Unexpected diff in node: (-want, +got):\n%s", diff) + } + + // Marshal the recently unmarshaled node struct into JSON and + // remarshal it into another node struct, and check that it still + // matches expectations. + marshaledJSON, err := json.Marshal(unmarshaledNode) + if err != nil { + t.Fatalf("node.MarshalJSON() failed: %v", err) + } + reUnmarshaledNode := newNode() + if err := json.Unmarshal([]byte(marshaledJSON), &reUnmarshaledNode); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.wantNode, reUnmarshaledNode); diff != "" { + t.Fatalf("Unexpected diff in node: (-want, +got):\n%s", diff) + } + }) + } +} + +func (s) TestNode_ToProto(t *testing.T) { + tests := []struct { + desc string + inputNode node + wantProto *v3corepb.Node + }{ + { + desc: "all fields set", + inputNode: func() node { + n := newNode() + n.ID = "id" + n.Cluster = "cluster" + n.Locality = locality{ + Region: "region", + Zone: "zone", + SubZone: "sub_zone", + } + n.Metadata = newStructProtoFromMap(t, map[string]any{ + "k1": "v1", + "k2": 101, + "k3": 280.0, + }) + return n + }(), + wantProto: &v3corepb.Node{ + Id: "id", + Cluster: "cluster", + Locality: &v3corepb.Locality{ + Region: "region", + Zone: "zone", + SubZone: "sub_zone", + }, + Metadata: newStructProtoFromMap(t, map[string]any{ + "k1": "v1", + "k2": 101, + "k3": 280.0, + }), + UserAgentName: "gRPC Go", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + }, + }, + { + desc: "some fields unset", + inputNode: func() node { + n := newNode() + n.ID = "id" + return n + }(), + wantProto: &v3corepb.Node{ + Id: "id", + UserAgentName: "gRPC Go", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + gotProto := test.inputNode.toProto() + if diff := cmp.Diff(test.wantProto, gotProto, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in node proto: (-want, +got):\n%s", diff) + } + }) + } +} + +func (s) TestBootstrap_SelectedChannelCredsAndCallCreds(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSBootstrapCallCredsEnabled, true) + tests := []struct { + name string + bootstrapConfig string + wantDialOpts int + wantTransportType string + }{ + { + name: "JWT_call_creds_with_TLS_channel_creds", + bootstrapConfig: `{ + "server_uri": "xds-server:443", + "channel_creds": [{"type": "tls", "config": {}}], + "call_creds": [ + { + "type": "jwt_token_file", + "config": {"jwt_token_file": "/token.jwt"} + } + ] + }`, + wantDialOpts: 1, + wantTransportType: "tls", + }, + { + name: "JWT_call_creds_with_multiple_channel_creds", + bootstrapConfig: `{ + "server_uri": "xds-server:443", + "channel_creds": [{"type": "tls", "config": {}}, {"type": "insecure"}], + "call_creds": [ + { + "type": "jwt_token_file", + "config": {"jwt_token_file": "/token.jwt"} + }, + { + "type": "jwt_token_file", + "config": {"jwt_token_file": "/token2.jwt"} + } + ] + }`, + wantDialOpts: 2, + wantTransportType: "tls", // The first channel creds is selected. + }, + { + name: "JWT_call_creds_with_insecure_channel_creds", + bootstrapConfig: `{ + "server_uri": "xds-server:443", + "channel_creds": [{"type": "insecure"}], + "call_creds": [ + { + "type": "jwt_token_file", + "config": {"jwt_token_file": "/token.jwt"} + } + ] + }`, + wantDialOpts: 1, + wantTransportType: "insecure", + }, + { + name: "No_call_creds", + bootstrapConfig: `{ + "server_uri": "xds-server:443", + "channel_creds": [{"type": "insecure"}] + }`, + wantDialOpts: 0, + wantTransportType: "insecure", + }, + { + name: "No_call_creds_multiple_channel_creds", + bootstrapConfig: `{ + "server_uri": "xds-server:443", + "channel_creds": [{"type": "insecure"}, {"type": "tls", "config": {}}] + }`, + wantDialOpts: 0, + wantTransportType: "insecure", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var sc ServerConfig + err := sc.UnmarshalJSON([]byte(test.bootstrapConfig)) + if err != nil { + t.Fatalf("Failed to unmarshal bootstrap config: %v", err) + } + + // Verify call credentials processing. + callCredsConfig := sc.CallCredsConfigs() + dialOpts := sc.DialOptions() + if len(callCredsConfig) != test.wantDialOpts { + t.Errorf("Call creds configs count = %d, want %d", len(callCredsConfig), test.wantDialOpts) + } + if len(dialOpts) != test.wantDialOpts { + t.Errorf("Call creds count = %d, want %d", len(dialOpts), test.wantDialOpts) + } + // Verify transport credentials are properly selected. + if sc.SelectedChannelCreds().Type != test.wantTransportType { + t.Errorf("Selected transport creds type = %q, want %q", sc.SelectedChannelCreds().Type, test.wantTransportType) + } + }) + } +} + +func (s) TestBootstrap_SelectedCallCreds_WhenNotCCNotEnabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSBootstrapCallCredsEnabled, false) + config := `{ + "server_uri": "xds-server:443", + "channel_creds": [{"type": "tls", "config": {}}], + "call_creds": [ + { + "type": "jwt_token_file", + "config": {"jwt_token_file": "/token.jwt"} + } + ] + }` + + var sc ServerConfig + err := sc.UnmarshalJSON([]byte(config)) + if err != nil { + t.Fatalf("Failed to unmarshal bootstrap config: %v", err) + } + + // Verify call credentials processing. + callCredsConfig := sc.CallCredsConfigs() + dialOpts := sc.DialOptions() + if len(callCredsConfig) != 1 { + t.Errorf("Call creds configs count = %d, want %d", len(callCredsConfig), 1) + } + // Even though we have parsed the call creds configs, we are not using them + // because the env variable is not enabled. + if len(dialOpts) != 0 { + t.Errorf("Call creds count = %d, want %d", len(dialOpts), 0) + } +} + +// TestAllowedGRPCServices_UnmarshalJSON verifies parsing of the +// allowed_grpc_services field, operating directly on the AllowedGRPCServices +// type. +func (s) TestAllowedGRPCServices_UnmarshalJSON(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true) + const target = "dns:///sharding-service:443" + tests := []struct { + name string + json string + // want carries the expected target and credential identities; + // comparisons use the Equal methods via cmp.Diff. + want AllowedGRPCServices + // wantCallCreds is the number of call credentials expected to be + // built for the target. + wantCallCreds int + }{ + { + name: "insecure_channel_creds", + json: `{"dns:///sharding-service:443": {"channel_creds": [{"type": "insecure"}]}}`, + want: AllowedGRPCServices{target: { + targetURI: target, + sideChannelCreds: xdscreds.NewChannelCreds(nil, xdscreds.Identity{Type: "insecure"}, nil), + }}, + }, + { + name: "with_call_creds", + json: `{"dns:///sharding-service:443": {"channel_creds": [{"type": "insecure"}], "call_creds": [{"type": "jwt_token_file", "config": {"jwt_token_file": "/var/run/secrets/tokens/istio-token"}}]}}`, + want: AllowedGRPCServices{target: { + targetURI: target, + sideChannelCreds: xdscreds.NewChannelCreds(nil, xdscreds.Identity{Type: "insecure"}, nil), + sideCallCreds: []*xdscreds.CallCreds{xdscreds.NewCallCreds(nil, xdscreds.Identity{ + Type: "jwt_token_file", + Data: json.RawMessage(`{"jwt_token_file": "/var/run/secrets/tokens/istio-token"}`), + }, nil)}, + }}, + wantCallCreds: 1, + }, + { + // Unsupported call-creds types are skipped without error, so no + // call credentials are built. + name: "unsupported_call_creds_skipped", + json: `{"dns:///sharding-service:443": {"channel_creds": [{"type": "insecure"}], "call_creds": [{"type": "unsupported_call_creds_type"}]}}`, + want: AllowedGRPCServices{target: { + targetURI: target, + sideChannelCreds: xdscreds.NewChannelCreds(nil, xdscreds.Identity{Type: "insecure"}, nil), + }}, + }, + { + // One call credential is built for each supported call-creds + // config, preserving order. + name: "multiple_supported_call_creds", + json: `{"dns:///sharding-service:443": {"channel_creds": [{"type": "insecure"}], "call_creds": [{"type": "jwt_token_file", "config": {"jwt_token_file": "/tokens/token-one"}}, {"type": "jwt_token_file", "config": {"jwt_token_file": "/tokens/token-two"}}]}}`, + want: AllowedGRPCServices{target: { + targetURI: target, + sideChannelCreds: xdscreds.NewChannelCreds(nil, xdscreds.Identity{Type: "insecure"}, nil), + sideCallCreds: []*xdscreds.CallCreds{ + xdscreds.NewCallCreds(nil, xdscreds.Identity{ + Type: "jwt_token_file", + Data: json.RawMessage(`{"jwt_token_file": "/tokens/token-one"}`), + }, nil), + xdscreds.NewCallCreds(nil, xdscreds.Identity{ + Type: "jwt_token_file", + Data: json.RawMessage(`{"jwt_token_file": "/tokens/token-two"}`), + }, nil), + }, + }}, + wantCallCreds: 2, + }, + { + name: "tls_channel_creds", + json: `{"dns:///sharding-service:443": {"channel_creds": [{"type": "tls", "config": {}}]}}`, + want: AllowedGRPCServices{target: { + targetURI: target, + sideChannelCreds: xdscreds.NewChannelCreds(nil, xdscreds.Identity{Type: "tls", Data: json.RawMessage("{}")}, nil), + }}, + }, + { + name: "skips_unsupported_channel_creds", + json: `{"dns:///sharding-service:443": {"channel_creds": [{"type": "unsupported_cred_type"}, {"type": "insecure"}]}}`, + want: AllowedGRPCServices{target: { + targetURI: target, + sideChannelCreds: xdscreds.NewChannelCreds(nil, xdscreds.Identity{Type: "insecure"}, nil), + }}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var got AllowedGRPCServices + if err := json.Unmarshal([]byte(test.json), &got); err != nil { + t.Fatalf("AllowedGRPCServices unmarshal failed: %v", err) + } + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("AllowedGRPCServices unmarshal returned unexpected diff (-want +got):\n%s", diff) + } + // Equal compares credentials by identity only, so it cannot tell + // a built credential from a nil one; verify the credentials were + // built. + chanCreds, callCreds := got[target].SideChannelCredentials() + if chanCreds == nil || chanCreds.Bundle() == nil { + t.Error("SideChannelCredentials() returned no built channel credentials") + } + if len(callCreds) != test.wantCallCreds { + t.Errorf("SideChannelCredentials() returned %d call credentials, want %d", len(callCreds), test.wantCallCreds) + } + for i, cc := range callCreds { + if cc.Credentials() == nil { + t.Errorf("SideChannelCredentials() call credentials[%d] have no built credentials", i) + } + } + }) + } +} + +// TestAllowedGRPCServices_UnmarshalJSON_Errors verifies that invalid +// allowed_grpc_services entries are rejected while parsing the field. +func (s) TestAllowedGRPCServices_UnmarshalJSON_Errors(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true) + tests := []struct { + name string + json string + }{ + { + name: "no_supported_channel_creds", + json: `{"dns:///sharding-service:443": {"channel_creds": [{"type": "unsupported_cred_type"}]}}`, + }, + { + name: "null_entry", + json: `{"dns:///sharding-service:443": null}`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var got AllowedGRPCServices + if err := json.Unmarshal([]byte(test.json), &got); err == nil { + t.Fatalf("AllowedGRPCServices unmarshal succeeded, want error") + } + }) + } +} + +// TestBootstrap_AllowedGRPCServices_NullEntryRejected verifies that a null +// allowed_grpc_services entry is rejected rather than silently accepted as a +// credential-less allowlisted target. gRFC A102 requires channel_creds on +// every entry, but a JSON null map value is decoded without invoking +// UnmarshalJSON, so it would otherwise bypass that validation. +func (s) TestBootstrap_AllowedGRPCServices_NullEntryRejected(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true) + cfg := `{ + "xds_servers": [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [{"type": "insecure"}] + }], + "allowed_grpc_services": {"dns:///side-channel:443": null} + }` + var c Config + if err := c.UnmarshalJSON([]byte(cfg)); err == nil { + t.Fatalf("Config.UnmarshalJSON() succeeded; want error for null allowed_grpc_services entry") + } +} + +// TestAllowedGRPCService_Accessors exercises TargetURI and Equal, including +// inequality when only the target URI differs. +func (s) TestAllowedGRPCService_Accessors(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true) + const target = "dns:///sharding-service:443" + cfg := `{"dns:///sharding-service:443": {"channel_creds": [{"type": "insecure"}]}}` + var s1, s2 AllowedGRPCServices + if err := json.Unmarshal([]byte(cfg), &s1); err != nil { + t.Fatalf("AllowedGRPCServices unmarshal failed: %v", err) + } + if err := json.Unmarshal([]byte(cfg), &s2); err != nil { + t.Fatalf("AllowedGRPCServices unmarshal failed: %v", err) + } + svc := s1[target] + if got := svc.TargetURI(); got != target { + t.Errorf("TargetURI() = %q, want %q", got, target) + } + // Two independent parses of the same config must compare equal. + if !svc.Equal(s2[target]) { + t.Errorf("Equal() = false for identical configs, want true") + } + // Two services identical except for target URI must not be equal. + diff := *svc + diff.targetURI = "dns:///different:443" + if svc.Equal(&diff) { + t.Errorf("Equal() = true for services with different target URIs, want false") + } +} + +// TestAllowedGRPCServices_Disabled verifies that the allowed_grpc_services +// field is ignored entirely when no consuming feature is enabled: nothing is +// parsed, and even a malformed entry does not fail parsing. +func (s) TestAllowedGRPCServices_Disabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, false) + cfg := `{"dns:///side-channel:443": {"channel_creds": [{"type": "unsupported_cred_type"}]}}` + var got AllowedGRPCServices + if err := json.Unmarshal([]byte(cfg), &got); err != nil { + t.Fatalf("AllowedGRPCServices unmarshal failed: %v", err) + } + if got != nil { + t.Errorf("AllowedGRPCServices = %v, want nil when disabled", got) + } +} diff --git a/internal/xds/bootstrap/jwtcreds/call_creds.go b/internal/xds/bootstrap/jwtcreds/call_creds.go new file mode 100644 index 0000000..f4b4c7d --- /dev/null +++ b/internal/xds/bootstrap/jwtcreds/call_creds.go @@ -0,0 +1,54 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package jwtcreds implements JWT CallCredentials for XDS, configured via xDS +// Bootstrap File. For more details, see gRFC A97: +// https://github.com/grpc/proposal/blob/master/A97-xds-jwt-call-creds.md +package jwtcreds + +import ( + "encoding/json" + "fmt" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/jwt" +) + +// NewCallCredentials returns a new JWT token based call credentials. The input +// config must match the structure specified in gRFC A97. +// +// The caller is expected to invoke the cancel function when they are done using +// the returned call creds. This cancel function is idempotent. +func NewCallCredentials(configJSON json.RawMessage) (c credentials.PerRPCCredentials, cancel func(), err error) { + var cfg struct { + JWTTokenFile string `json:"jwt_token_file"` + } + emptyFn := func() {} + + if err := json.Unmarshal(configJSON, &cfg); err != nil { + return nil, emptyFn, fmt.Errorf("failed to unmarshal JWT call credentials config: %v", err) + } + if cfg.JWTTokenFile == "" { + return nil, emptyFn, fmt.Errorf("jwt_token_file is required in JWT call credentials config") + } + callCreds, err := jwt.NewTokenFileCallCredentials(cfg.JWTTokenFile) + if err != nil { + return nil, emptyFn, fmt.Errorf("failed to create JWT call credentials: %v", err) + } + return callCreds, emptyFn, nil +} diff --git a/internal/xds/bootstrap/jwtcreds/call_creds_test.go b/internal/xds/bootstrap/jwtcreds/call_creds_test.go new file mode 100644 index 0000000..e3a536f --- /dev/null +++ b/internal/xds/bootstrap/jwtcreds/call_creds_test.go @@ -0,0 +1,168 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package jwtcreds + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestNewCallCredentialsWithInvalidConfig(t *testing.T) { + tests := []struct { + name string + config string + }{ + { + name: "empty_file", + config: `""`, + }, + { + name: "empty_config", + config: `{}`, + }, + { + name: "empty_path", + config: `{"jwt_token_file": ""}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + callCreds, cleanup, err := NewCallCredentials(json.RawMessage(tt.config)) + if err == nil { + t.Fatalf("NewCallCredentials(%s): got nil, want error", tt.config) + } + if callCreds != nil { + t.Errorf("NewCallCredentials(%s): returned non-nil call credentials", tt.config) + } + if cleanup == nil { + t.Errorf("NewCallCredentials(%s): returned nil cleanup function", tt.config) + } + }) + } +} + +func (s) TestNewCallCredentialsWithValidConfig(t *testing.T) { + token := createTestJWT(t) + tokenFile := writeTempFile(t, token) + config := `{"jwt_token_file": "` + tokenFile + `"}` + + callCreds, cleanup, err := NewCallCredentials(json.RawMessage(config)) + if err != nil { + t.Fatalf("NewCallCredentials(%s) failed: %v", config, err) + } + if callCreds == nil { + t.Fatalf("NewCallCredentials(%s): returned nil credentials", config) + } + if cleanup == nil { + t.Errorf("NewCallCredentials(%s): returned nil cleanup function", config) + } else { + defer cleanup() + } + + // Test that call credentials get used. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &testAuthInfo{secLevel: credentials.PrivacyAndIntegrity}, + }) + metadata, err := callCreds.GetRequestMetadata(ctx) + if err != nil { + t.Fatalf("GetRequestMetadata failed: %v", err) + } + if len(metadata) == 0 { + t.Fatal("GetRequestMetadata: returned empty metadata") + } + authHeader, ok := metadata["authorization"] + if !ok { + t.Fatal("GetRequestMetadata: returned empty authorization header in metadata") + } + if !strings.HasPrefix(authHeader, "Bearer ") { + t.Errorf("GetRequestMetadata: Authorization header should start with 'Bearer ', got %q", authHeader) + } +} + +func (s) TestCallCredentials_Cleanup(t *testing.T) { + token := createTestJWT(t) + tokenFile := writeTempFile(t, token) + config := `{"jwt_token_file": "` + tokenFile + `"}` + _, cleanup, err := NewCallCredentials(json.RawMessage(config)) + if err != nil { + t.Fatalf("NewCallCredentials(%s) failed: %v", config, err) + } + if cleanup == nil { + t.Errorf("NewCallCredentials(%s): returned nil cleanup function", config) + } + + // Cleanup should not panic. Multiple cleanup calls should be safe + cleanup() + cleanup() +} + +// testAuthInfo implements credentials.AuthInfo for testing. +type testAuthInfo struct { + secLevel credentials.SecurityLevel +} + +func (t *testAuthInfo) AuthType() string { + return "test" +} + +func (t *testAuthInfo) GetCommonAuthInfo() credentials.CommonAuthInfo { + return credentials.CommonAuthInfo{SecurityLevel: t.secLevel} +} + +// createTestJWT creates a test JWT token for testing. +func createTestJWT(t *testing.T) string { + t.Helper() + + // Header: {"typ":"JWT","alg":"HS256"} + header := "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9" + // Claims: {"aud":"https://example.com","exp":future_timestamp} + claims := "eyJhdWQiOiJodHRwczovL2V4YW1wbGUuY29tIiwiZXhwIjoyMDAwMDAwMDAwfQ" + signature := "fake_signature_for_testing" + + return header + "." + claims + "." + signature +} + +func writeTempFile(t *testing.T, content string) string { + t.Helper() + tempDir := t.TempDir() + filePath := filepath.Join(tempDir, "jwt_token") + if err := os.WriteFile(filePath, []byte(content), 0600); err != nil { + t.Fatalf("Failed to write temp file: %v", err) + } + return filePath +} diff --git a/internal/xds/bootstrap/logging.go b/internal/xds/bootstrap/logging.go new file mode 100644 index 0000000..fdd811d --- /dev/null +++ b/internal/xds/bootstrap/logging.go @@ -0,0 +1,28 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package bootstrap + +import ( + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[xds-bootstrap] " + +var logger = internalgrpclog.NewPrefixLogger(grpclog.Component("xds"), prefix) diff --git a/internal/xds/bootstrap/template.go b/internal/xds/bootstrap/template.go new file mode 100644 index 0000000..ec1a309 --- /dev/null +++ b/internal/xds/bootstrap/template.go @@ -0,0 +1,47 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bootstrap + +import ( + "net/url" + "strings" +) + +// PopulateResourceTemplate populates the given template using the target +// string. "%s", if exists in the template, will be replaced with target. +// +// If the template starts with "xdstp:", the replaced string will be %-encoded. +// But note that "/" is not percent encoded. +func PopulateResourceTemplate(template, target string) string { + if !strings.Contains(template, "%s") { + return template + } + if strings.HasPrefix(template, "xdstp:") { + target = percentEncode(target) + } + return strings.ReplaceAll(template, "%s", target) +} + +// percentEncode percent encode t, except for "/". See the tests for examples. +func percentEncode(t string) string { + segs := strings.Split(t, "/") + for i := range segs { + segs[i] = url.PathEscape(segs[i]) + } + return strings.Join(segs, "/") +} diff --git a/internal/xds/bootstrap/template_test.go b/internal/xds/bootstrap/template_test.go new file mode 100644 index 0000000..bc12eb4 --- /dev/null +++ b/internal/xds/bootstrap/template_test.go @@ -0,0 +1,97 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bootstrap + +import "testing" + +func Test_percentEncode(t *testing.T) { + tests := []struct { + name string + target string + want string + }{ + { + name: "normal name", + target: "server.example.com", + want: "server.example.com", + }, + { + name: "ipv4", + target: "0.0.0.0:8080", + want: "0.0.0.0:8080", + }, + { + name: "ipv6", + target: "[::1]:8080", + want: "%5B::1%5D:8080", // [ and ] are percent encoded. + }, + { + name: "/ should not be percent encoded", + target: "my/service/region", + want: "my/service/region", // "/"s are kept. + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := percentEncode(tt.target); got != tt.want { + t.Errorf("percentEncode() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPopulateResourceTemplate(t *testing.T) { + tests := []struct { + name string + template string + target string + want string + }{ + { + name: "no %s", + template: "/name/template", + target: "[::1]:8080", + want: "/name/template", + }, + { + name: "with %s, no xdstp: prefix, ipv6", + template: "/name/template/%s", + target: "[::1]:8080", + want: "/name/template/[::1]:8080", + }, + { + name: "with %s, with xdstp: prefix", + template: "xdstp://authority.com/%s", + target: "0.0.0.0:8080", + want: "xdstp://authority.com/0.0.0.0:8080", + }, + { + name: "with %s, with xdstp: prefix, and ipv6", + template: "xdstp://authority.com/%s", + target: "[::1]:8080", + want: "xdstp://authority.com/%5B::1%5D:8080", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := PopulateResourceTemplate(tt.template, tt.target); got != tt.want { + t.Errorf("PopulateResourceTemplate() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/xds/bootstrap/tlscreds/bundle.go b/internal/xds/bootstrap/tlscreds/bundle.go new file mode 100644 index 0000000..dda2331 --- /dev/null +++ b/internal/xds/bootstrap/tlscreds/bundle.go @@ -0,0 +1,194 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package tlscreds implements mTLS Credentials in xDS Bootstrap File. +// See gRFC A65: github.com/grpc/proposal/blob/master/A65-xds-mtls-creds-in-bootstrap.md. +package tlscreds + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "net" + "sync" + "time" + + "github.com/spiffe/go-spiffe/v2/bundle/spiffebundle" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/credentials/tls/certprovider/pemfile" + "google.golang.org/grpc/internal/credentials/spiffe" + "google.golang.org/grpc/internal/envconfig" +) + +// bundle is an implementation of credentials.Bundle which implements mTLS +// Credentials in xDS Bootstrap File. +type bundle struct { + transportCredentials credentials.TransportCredentials +} + +// NewBundle returns a credentials.Bundle which implements mTLS Credentials in xDS +// Bootstrap File. It delegates certificate loading to a file_watcher provider +// if either client certificates or server root CA is specified. The second +// return value is a close func that should be called when the caller no longer +// needs this bundle. +// See gRFC A65: github.com/grpc/proposal/blob/master/A65-xds-mtls-creds-in-bootstrap.md +func NewBundle(jd json.RawMessage) (credentials.Bundle, func(), error) { + cfg := &struct { + CertificateFile string `json:"certificate_file"` + CACertificateFile string `json:"ca_certificate_file"` + PrivateKeyFile string `json:"private_key_file"` + SPIFFETrustBundleMapFile string `json:"spiffe_trust_bundle_map_file"` + }{} + + if jd != nil { + if err := json.Unmarshal(jd, cfg); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal config: %v", err) + } + } // Else the config field is absent. Treat it as an empty config. + + if !envconfig.XDSSPIFFEEnabled { + cfg.SPIFFETrustBundleMapFile = "" + } + if cfg.CACertificateFile == "" && cfg.CertificateFile == "" && cfg.PrivateKeyFile == "" && cfg.SPIFFETrustBundleMapFile == "" { + // We cannot use (and do not need) a file_watcher provider in this case, + // and can simply directly use the TLS transport credentials. + // Quoting A65: + // + // > The only difference between the file-watcher certificate provider + // > config and this one is that in the file-watcher certificate + // > provider, at least one of the "certificate_file" or + // > "ca_certificate_file" fields must be specified, whereas in this + // > configuration, it is acceptable to specify neither one. + // Further, with the introduction of SPIFFE Trust Map support, we also + // check for this value. + return &bundle{transportCredentials: credentials.NewTLS(&tls.Config{})}, func() {}, nil + } + // Otherwise we need to use a file_watcher provider to watch the CA, + // private and public keys. + + // The pemfile plugin (file_watcher) currently ignores BuildOptions. + provider, err := certprovider.GetProvider(pemfile.PluginName, jd, certprovider.BuildOptions{}) + if err != nil { + return nil, nil, err + } + return &bundle{ + transportCredentials: &reloadingCreds{provider: provider}, + }, sync.OnceFunc(func() { provider.Close() }), nil +} + +func (t *bundle) TransportCredentials() credentials.TransportCredentials { + return t.transportCredentials +} + +func (t *bundle) PerRPCCredentials() credentials.PerRPCCredentials { + // mTLS provides transport credentials only. There are no per-RPC + // credentials. + return nil +} + +func (t *bundle) NewWithMode(string) (credentials.Bundle, error) { + // This bundle has a single mode which only uses TLS transport credentials, + // so there is no legitimate case where callers would call NewWithMode. + return nil, fmt.Errorf("xDS TLS credentials only support one mode") +} + +// reloadingCreds is a credentials.TransportCredentials for client +// side mTLS that reloads the server root CA certificate and the client +// certificates from the provider on every client handshake. This is necessary +// because the standard TLS credentials do not support reloading CA +// certificates. +type reloadingCreds struct { + provider certprovider.Provider +} + +func (c *reloadingCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + km, err := c.provider.KeyMaterial(ctx) + if err != nil { + return nil, nil, err + } + var config *tls.Config + if km.SPIFFEBundleMap != nil { + config = &tls.Config{ + InsecureSkipVerify: true, + VerifyPeerCertificate: buildSPIFFEVerifyFunc(km.SPIFFEBundleMap), + Certificates: km.Certs, + } + } else { + config = &tls.Config{ + RootCAs: km.Roots, + Certificates: km.Certs, + } + } + return credentials.NewTLS(config).ClientHandshake(ctx, authority, rawConn) +} + +func (c *reloadingCreds) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{SecurityProtocol: "tls"} +} + +func (c *reloadingCreds) Clone() credentials.TransportCredentials { + return &reloadingCreds{provider: c.provider} +} + +func (c *reloadingCreds) OverrideServerName(string) error { + return errors.New("overriding server name is not supported by xDS client TLS credentials") +} + +func (c *reloadingCreds) ServerHandshake(net.Conn) (net.Conn, credentials.AuthInfo, error) { + return nil, nil, errors.New("server handshake is not supported by xDS client TLS credentials") +} + +func buildSPIFFEVerifyFunc(spiffeBundleMap map[string]*spiffebundle.Bundle) func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + return func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + rawCertList := make([]*x509.Certificate, len(rawCerts)) + for i, asn1Data := range rawCerts { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + return fmt.Errorf("spiffe: verify function could not parse input certificate: %v", err) + } + rawCertList[i] = cert + } + if len(rawCertList) == 0 { + return fmt.Errorf("spiffe: verify function has no valid input certificates") + } + leafCert := rawCertList[0] + roots, err := spiffe.GetRootsFromSPIFFEBundleMap(spiffeBundleMap, leafCert) + if err != nil { + return err + } + + opts := x509.VerifyOptions{ + Roots: roots, + CurrentTime: time.Now(), + Intermediates: x509.NewCertPool(), + } + + for _, cert := range rawCertList[1:] { + opts.Intermediates.AddCert(cert) + } + // The verified chain is (surprisingly) unused. + if _, err = rawCertList[0].Verify(opts); err != nil { + return fmt.Errorf("spiffe: x509 certificate Verify failed: %v", err) + } + return nil + } +} diff --git a/internal/xds/bootstrap/tlscreds/bundle_ext_test.go b/internal/xds/bootstrap/tlscreds/bundle_ext_test.go new file mode 100644 index 0000000..155376e --- /dev/null +++ b/internal/xds/bootstrap/tlscreds/bundle_ext_test.go @@ -0,0 +1,520 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package tlscreds_test + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "os" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/bootstrap/tlscreds" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" +) + +const defaultTestTimeout = 5 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type Closable interface { + Close() +} + +func (s) TestValidTlsBuilder(t *testing.T) { + caCert := testdata.Path("x509/server_ca_cert.pem") + clientCert := testdata.Path("x509/client1_cert.pem") + clientKey := testdata.Path("x509/client1_key.pem") + clientSpiffeBundle := testdata.Path("spiffe_end2end/client_spiffebundle.json") + tests := []struct { + name string + jd string + }{ + { + name: "Absent configuration", + jd: `null`, + }, + { + name: "Empty configuration", + jd: `{}`, + }, + { + name: "Only CA certificate chain", + jd: fmt.Sprintf(`{"ca_certificate_file": "%s"}`, caCert), + }, + { + name: "Only private key and certificate chain", + jd: fmt.Sprintf(`{"certificate_file":"%s","private_key_file":"%s"}`, clientCert, clientKey), + }, + { + name: "CA chain, private key and certificate chain", + jd: fmt.Sprintf(`{"ca_certificate_file":"%s","certificate_file":"%s","private_key_file":"%s"}`, caCert, clientCert, clientKey), + }, + { + name: "Only refresh interval", jd: `{"refresh_interval": "1s"}`, + }, + { + name: "Refresh interval and CA certificate chain", + jd: fmt.Sprintf(`{"refresh_interval": "1s","ca_certificate_file": "%s"}`, caCert), + }, + { + name: "Refresh interval, private key and certificate chain", + jd: fmt.Sprintf(`{"refresh_interval": "1s","certificate_file":"%s","private_key_file":"%s"}`, clientCert, clientKey), + }, + { + name: "Refresh interval, CA chain, private key and certificate chain", + jd: fmt.Sprintf(`{"refresh_interval": "1s","ca_certificate_file":"%s","certificate_file":"%s","private_key_file":"%s"}`, caCert, clientCert, clientKey), + }, + { + name: "Refresh interval, CA chain, private key, certificate chain, spiffe bundle", + jd: fmt.Sprintf(`{"refresh_interval": "1s","ca_certificate_file":"%s","certificate_file":"%s","private_key_file":"%s","spiffe_trust_bundle_map_file":"%s"}`, caCert, clientCert, clientKey, clientSpiffeBundle), + }, + { + name: "Unknown field", + jd: `{"unknown_field": "foo"}`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + msg := json.RawMessage(test.jd) + _, stop, err := tlscreds.NewBundle(msg) + if err != nil { + t.Fatalf("NewBundle(%s) returned error %s when expected to succeed", test.jd, err) + } + stop() + }) + } +} + +func (s) TestInvalidTlsBuilder(t *testing.T) { + tests := []struct { + name, jd, wantErrPrefix string + }{ + { + name: "Wrong type in json", + jd: `{"ca_certificate_file": 1}`, + wantErrPrefix: "failed to unmarshal config:"}, + { + name: "Missing private key", + jd: fmt.Sprintf(`{"certificate_file":"%s"}`, testdata.Path("x509/server_cert.pem")), + wantErrPrefix: "pemfile: private key file and identity cert file should be both specified or not specified", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + msg := json.RawMessage(test.jd) + _, stop, err := tlscreds.NewBundle(msg) + if err == nil || !strings.HasPrefix(err.Error(), test.wantErrPrefix) { + if stop != nil { + stop() + } + t.Fatalf("NewBundle(%s): got error %s, want an error with prefix %s", msg, err, test.wantErrPrefix) + } + }) + } +} + +func (s) TestCaReloading(t *testing.T) { + serverCa, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem")) + if err != nil { + t.Fatalf("Failed to read test CA cert: %s", err) + } + + // Write CA certs to a temporary file so that we can modify it later. + caPath := t.TempDir() + "/ca.pem" + if err = os.WriteFile(caPath, serverCa, 0644); err != nil { + t.Fatalf("Failed to write test CA cert: %v", err) + } + cfg := fmt.Sprintf(`{ + "ca_certificate_file": "%s", + "refresh_interval": ".01s" + }`, caPath) + tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg)) + if err != nil { + t.Fatalf("Failed to create TLS bundle: %v", err) + } + defer stop() + + serverCredentials := grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.NoClientCert)) + server := stubserver.StartTestService(t, nil, serverCredentials) + + conn, err := grpc.NewClient( + server.Address, + grpc.WithCredentialsBundle(tlsBundle), + grpc.WithAuthority("x.test.example.com"), + ) + if err != nil { + t.Fatalf("Error dialing: %v", err) + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + client := testgrpc.NewTestServiceClient(conn) + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("Error calling EmptyCall: %v", err) + } + // close the server and create a new one to force client to do a new + // handshake. + server.Stop() + + invalidCa, err := os.ReadFile(testdata.Path("ca.pem")) + if err != nil { + t.Fatalf("Failed to read test CA cert: %v", err) + } + // unload root cert + err = os.WriteFile(caPath, invalidCa, 0644) + if err != nil { + t.Fatalf("Failed to write test CA cert: %v", err) + } + for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) { + ss := stubserver.StubServer{ + Address: server.Address, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + server = stubserver.StartTestService(t, &ss, serverCredentials) + + // Client handshake should eventually fail because the client CA was + // reloaded, and thus the server cert is signed by an unknown CA. + t.Log(server) + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + const wantErr = "certificate signed by unknown authority" + if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), wantErr) { + // Certs have reloaded. + server.Stop() + break + } + t.Logf("EmptyCall() got err: %s, want code: %s, want err: %s", err, codes.Unavailable, wantErr) + server.Stop() + } + if ctx.Err() != nil { + t.Errorf("Timed out waiting for CA certs reloading") + } +} + +// Test_SPIFFE_Reloading sets up a client and server. The client is configured +// to use a SPIFFE bundle map, and the server is configured to use TLS creds +// compatible with this bundle. A handshake is performed and connection is +// expected to be successful. Then we change the client's SPIFFE Bundle Map file +// on disk to one that should fail with the server's credentials. This change +// should be picked up by the client via our file reloading. Another handshake +// is performed and checked for failure, ensuring that gRPC is correctly using +// the changed-on-disk bundle map. +func (s) Test_SPIFFE_Reloading(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true) + clientSPIFFEBundle, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffebundle.json")) + if err != nil { + t.Fatalf("Failed to read test SPIFFE bundle: %v", err) + } + + // Write CA certs to a temporary file so that we can modify it later. + spiffePath := t.TempDir() + "/client_spiffe.json" + if err = os.WriteFile(spiffePath, clientSPIFFEBundle, 0644); err != nil { + t.Fatalf("Failed to write test SPIFFE Bundle %v: %v", clientSPIFFEBundle, err) + } + cfg := fmt.Sprintf(`{ + "spiffe_trust_bundle_map_file": "%s", + "refresh_interval": ".01s" + }`, spiffePath) + tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg)) + if err != nil { + t.Fatalf("Failed to create TLS bundle: %v", err) + } + defer stop() + + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + defer lis.Close() + ss := stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + + serverCredentials := grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.NoClientCert)) + server := stubserver.StartTestService(t, &ss, serverCredentials) + + defer server.Stop() + + conn, err := grpc.NewClient( + server.Address, + grpc.WithCredentialsBundle(tlsBundle), + grpc.WithAuthority("x.test.example.com"), + ) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", server.Address, err) + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + client := testgrpc.NewTestServiceClient(conn) + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("Error calling EmptyCall: %v", err) + } + + // Setup the wrong bundle to be reloaded + wrongBundle, err := os.ReadFile(testdata.Path("spiffe_end2end/server_spiffebundle.json")) + if err != nil { + t.Fatalf("Failed to read test spiffe bundle %v: %v", "spiffe_end2end/server_spiffebundle.json", err) + } + // Write the bundle that will fail to the tmp file path to be reloaded + err = os.WriteFile(spiffePath, wrongBundle, 0644) + if err != nil { + t.Fatalf("Failed to write test spiffe bundle %v: %v", "spiffe_end2end/server_spiffebundle.json", err) + } + + for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) { + // Stop and restart the listener to force new handshakes + lis.Stop() + lis.Restart() + // Client handshake should eventually fail because the client CA was + // reloaded, and thus the server cert is signed by an unknown CA. + t.Log(server) + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + const wantErr = "no bundle found for peer certificates trust domain" + if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), wantErr) { + // Certs have reloaded. + server.Stop() + break + } + t.Logf("EmptyCall() got err: %s, want code: %s, want err: %s", err, codes.Unavailable, wantErr) + } + if ctx.Err() != nil { + t.Errorf("Timed out waiting for CA certs reloading") + } +} + +func (s) TestMTLS(t *testing.T) { + s := stubserver.StartTestService(t, nil, grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert))) + defer s.Stop() + + cfg := fmt.Sprintf(`{ + "ca_certificate_file": "%s", + "certificate_file": "%s", + "private_key_file": "%s" + }`, + testdata.Path("x509/server_ca_cert.pem"), + testdata.Path("x509/client1_cert.pem"), + testdata.Path("x509/client1_key.pem")) + tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg)) + if err != nil { + t.Fatalf("Failed to create TLS bundle: %v", err) + } + defer stop() + conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com")) + if err != nil { + t.Fatalf("Error dialing: %v", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("EmptyCall(): got error %v when expected to succeed", err) + } +} + +// Test_MTLS_SPIFFE configures a client and server. The server has a certificate +// chain that is compatible with the client's configured SPIFFE bundle map. An +// MTLS connection is attempted between the two and checked for success. +func (s) Test_MTLS_SPIFFE(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true) + tests := []struct { + name string + serverOption grpc.ServerOption + }{ + { + name: "MTLS SPIFFE", + serverOption: grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert)), + }, + { + name: "MTLS SPIFFE Chain", + serverOption: grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFEChain(t, tls.RequireAndVerifyClientCert)), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := stubserver.StartTestService(t, nil, tc.serverOption) + defer s.Stop() + + cfg := fmt.Sprintf(`{ + "certificate_file": "%s", + "private_key_file": "%s", + "spiffe_trust_bundle_map_file": "%s" +}`, + testdata.Path("spiffe_end2end/client_spiffe.pem"), + testdata.Path("spiffe_end2end/client.key"), + testdata.Path("spiffe_end2end/client_spiffebundle.json")) + tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg)) + if err != nil { + t.Fatalf("Failed to create TLS bundle: %v", err) + } + defer stop() + conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com")) + if err != nil { + t.Fatalf("Error dialing: %v", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("EmptyCall(): got error %v when expected to succeed", err) + } + }) + } +} + +// Test_MTLS_SPIFFE_FlagDisabled configures a client and server. The server has +// a certificate chain that is compatible with the client's configured SPIFFE +// bundle map. However, the XDS flag that enabled SPIFFE usage is disabled. An +// MTLS connection is attempted between the two and checked for failure. +func (s) Test_MTLS_SPIFFE_FlagDisabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, false) + serverOption := grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert)) + s := stubserver.StartTestService(t, nil, serverOption) + defer s.Stop() + + cfg := fmt.Sprintf(`{ +"certificate_file": "%s", +"private_key_file": "%s", +"spiffe_trust_bundle_map_file": "%s" +}`, + testdata.Path("spiffe_end2end/client_spiffe.pem"), + testdata.Path("spiffe_end2end/client.key"), + testdata.Path("spiffe_end2end/client_spiffebundle.json")) + tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg)) + if err != nil { + t.Fatalf("Failed to create TLS bundle: %v", err) + } + defer stop() + conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com")) + if err != nil { + t.Fatalf("Error dialing: %v", err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Errorf("EmptyCall(): got success want failure") + } +} + +func (s) Test_MTLS_SPIFFE_Failure(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true) + tests := []struct { + name string + certFile string + keyFile string + spiffeBundleFile string + serverOption grpc.ServerOption + wantErrContains string + wantErrCode codes.Code + }{ + { + name: "No matching trust domain in bundle", + certFile: "spiffe_end2end/client_spiffe.pem", + keyFile: "spiffe_end2end/client.key", + spiffeBundleFile: "spiffe_end2end/server_spiffebundle.json", + serverOption: grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert)), + wantErrContains: "spiffe: no bundle found for peer certificates", + wantErrCode: codes.Unavailable, + }, + { + name: "Server cert has no valid SPIFFE URIs", + certFile: "spiffe_end2end/client_spiffe.pem", + keyFile: "spiffe_end2end/client.key", + spiffeBundleFile: "spiffe_end2end/client_spiffebundle.json", + serverOption: grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)), + wantErrContains: "spiffe: could not get spiffe ID from peer leaf cert", + wantErrCode: codes.Unavailable, + }, + { + name: "Server cert has valid spiffe ID but doesn't chain to the root CA", + certFile: "spiffe_end2end/client_spiffe.pem", + keyFile: "spiffe_end2end/client.key", + spiffeBundleFile: "spiffe_end2end/client_spiffebundle.json", + serverOption: grpc.Creds(testutils.CreateServerTLSCredentialsValidSPIFFEButWrongCA(t, tls.RequireAndVerifyClientCert)), + wantErrContains: "spiffe: x509 certificate Verify failed: x509: certificate signed by unknown authority", + wantErrCode: codes.Unavailable, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := stubserver.StartTestService(t, nil, tc.serverOption) + defer s.Stop() + cfg := fmt.Sprintf(`{ +"certificate_file": "%s", +"private_key_file": "%s", +"spiffe_trust_bundle_map_file": "%s" +}`, + testdata.Path(tc.certFile), + testdata.Path(tc.keyFile), + testdata.Path(tc.spiffeBundleFile)) + tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg)) + if err != nil { + t.Fatalf("Failed to create TLS bundle: %v", err) + } + defer stop() + conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com")) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", s.Address, err) + } + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Errorf("EmptyCall(): got success. want failure") + } + if status.Code(err) != tc.wantErrCode { + t.Errorf("EmptyCall(): failed with wrong error. got code %v. want code: %v", status.Code(err), tc.wantErrCode) + } + if !strings.Contains(err.Error(), tc.wantErrContains) { + t.Errorf("EmptyCall(): failed with wrong error. got %v. want contains: %v", err, tc.wantErrContains) + } + }) + } +} diff --git a/internal/xds/bootstrap/tlscreds/bundle_test.go b/internal/xds/bootstrap/tlscreds/bundle_test.go new file mode 100644 index 0000000..9e2e449 --- /dev/null +++ b/internal/xds/bootstrap/tlscreds/bundle_test.go @@ -0,0 +1,162 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package tlscreds + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal/credentials/spiffe" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const defaultTestTimeout = 5 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type failingProvider struct{} + +func (f failingProvider) KeyMaterial(context.Context) (*certprovider.KeyMaterial, error) { + return nil, errors.New("test error") +} + +func (f failingProvider) Close() {} + +func (s) TestFailingProvider(t *testing.T) { + s := stubserver.StartTestService(t, nil, grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert))) + defer s.Stop() + + cfg := fmt.Sprintf(`{ + "ca_certificate_file": "%s", + "certificate_file": "%s", + "private_key_file": "%s", + "spiffe_trust_bundle_map_file": "%s" + }`, + testdata.Path("x509/server_ca_cert.pem"), + testdata.Path("x509/client1_cert.pem"), + testdata.Path("x509/client1_key.pem"), + testdata.Path("spiffe_end2end/client_spiffebundle.json")) + tlsBundle, stop, err := NewBundle([]byte(cfg)) + if err != nil { + t.Fatalf("Failed to create TLS bundle: %v", err) + } + stop() + + // Force a provider that returns an error, and make sure the client fails + // the handshake. + creds, ok := tlsBundle.TransportCredentials().(*reloadingCreds) + if !ok { + t.Fatalf("Got %T, expected reloadingCreds", tlsBundle.TransportCredentials()) + } + creds.provider = &failingProvider{} + + conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com")) + if err != nil { + t.Fatalf("Error dialing: %v", err) + } + defer conn.Close() + + client := testgrpc.NewTestServiceClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if wantErr := "test error"; err == nil || !strings.Contains(err.Error(), wantErr) { + t.Errorf("EmptyCall() got err: %s, want err to contain: %s", err, wantErr) + } +} + +func rawCertsFromFile(t *testing.T, filePath string) [][]byte { + t.Helper() + rawCert, err := os.ReadFile(testdata.Path(filePath)) + if err != nil { + t.Fatalf("Reading certificate file failed: %v", err) + } + block, _ := pem.Decode(rawCert) + if block == nil || block.Type != "CERTIFICATE" { + t.Fatalf("pem.Decode() failed to decode certificate in file %q", "spiffe/server1_spiffe.pem") + } + return [][]byte{block.Bytes} +} + +func (s) TestSPIFFEVerifyFuncMismatchedCert(t *testing.T) { + spiffeBundleBytes, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffebundle.json")) + if err != nil { + t.Fatalf("Reading spiffebundle file failed: %v", err) + } + spiffeBundle, err := spiffe.BundleMapFromBytes(spiffeBundleBytes) + if err != nil { + t.Fatalf("spiffe.BundleMapFromBytes() failed: %v", err) + } + verifyFunc := buildSPIFFEVerifyFunc(spiffeBundle) + verifiedChains := [][]*x509.Certificate{} + tests := []struct { + name string + rawCerts [][]byte + wantErrContains string + }{ + { + name: "mismathed cert", + rawCerts: rawCertsFromFile(t, "spiffe/server1_spiffe.pem"), + wantErrContains: "spiffe: x509 certificate Verify failed", + }, + { + name: "bad input cert", + rawCerts: [][]byte{[]byte("NOT_GOOD_DATA")}, + wantErrContains: "spiffe: verify function could not parse input certificate", + }, + { + name: "no input bytes", + rawCerts: nil, + wantErrContains: "no valid input certificates", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err = verifyFunc(tc.rawCerts, verifiedChains) + if err == nil { + t.Fatalf("buildSPIFFEVerifyFunc call succeeded. want failure") + } + if !strings.Contains(err.Error(), tc.wantErrContains) { + t.Fatalf("buildSPIFFEVerifyFunc got err %v want err to contain %v", err, tc.wantErrContains) + } + }) + } +} diff --git a/internal/xds/clients/config.go b/internal/xds/clients/config.go new file mode 100644 index 0000000..aa89d2b --- /dev/null +++ b/internal/xds/clients/config.go @@ -0,0 +1,140 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package clients provides implementations of the clients to interact with +// xDS and LRS servers. +// +// # xDS Client +// +// The xDS client allows applications to: +// - Create client instances with in-memory configurations. +// - Register watches for named resources. +// - Receive resources via the ADS (Aggregated Discovery Service) stream. +// +// This enables applications to dynamically discover and configure resources +// such as listeners, routes, clusters, and endpoints from an xDS management +// server. +// +// # LRS Client +// +// The LRS (Load Reporting Service) client allows applications to report load +// data to an LRS server via the LRS stream. This data can be used for +// monitoring, traffic management, and other purposes. +// +// # Experimental +// +// NOTICE: This package is EXPERIMENTAL and may be changed or removed +// in a later release. +package clients + +// ServerIdentifier holds identifying information for connecting to an xDS +// management or LRS server. +type ServerIdentifier struct { + // ServerURI is the target URI of the server. + ServerURI string + + // Extensions can be populated with arbitrary data to be passed to the + // TransportBuilder and/or xDS Client's ResourceType implementations. + // This field can be used to provide additional configuration or context + // specific to the user's needs. + // + // The xDS and LRS clients do not interpret the contents of this field. + // It is the responsibility of the user's custom TransportBuilder and/or + // ResourceType implementations to handle and interpret these extensions. + // + // For example, a custom TransportBuilder might use this field to + // configure a specific security credentials. + // + // Extensions may be any type that is comparable, as they are used as map + // keys internally. If Extensions are not able to be used as a map key, + // the client may panic. + // + // See: https://go.dev/ref/spec#Comparison_operators + // + // Any equivalent extensions in all ServerIdentifiers present in a single + // client's configuration should have the same value. Not following this + // restriction may result in excess resource usage. + Extensions any +} + +// Node represents the identity of the xDS client, allowing xDS and LRS servers +// to identify the source of xDS requests. +type Node struct { + // ID is a string identifier of the application. + ID string + // Cluster is the name of the cluster the application belongs to. + Cluster string + // Locality is the location of the application including region, zone, + // sub-zone. + Locality Locality + // Metadata provides additional context about the application by associating + // arbitrary key-value pairs with it. + Metadata any + // UserAgentName is the user agent name of application. + UserAgentName string + // UserAgentVersion is the user agent version of application. + UserAgentVersion string +} + +// Locality represents the location of the xDS client application. +type Locality struct { + // Region is the region of the xDS client application. + Region string + // Zone is the area within a region. + Zone string + // SubZone is the further subdivision within a zone. + SubZone string +} + +// MetricsReporter is used by the XDSClient to report metrics. +type MetricsReporter interface { + // ReportMetric reports a metric. The metric will be one of the predefined + // set of types depending on the client (XDSClient or LRSClient). + // + // Each client will produce different metrics. Please see the client's + // documentation for a list of possible metrics events. + ReportMetric(metric any) + + // RegisterAsyncReporter registers a reporter to produce metric values for + // the set of metrics supported by the client. See the metrics sub-package + // for the specific client (e.g. internal/xds/clients/xdsclient/metrics/metrics.go) + // for the list of supported metrics. The returned function must be called + // when the metrics are no longer needed, which will remove the reporter. + // The function is expected to be idempotent. + // + // Once the returned cancel function is called, the Report method on the + // registered reporter is guaranteed not to be called again. + RegisterAsyncReporter(reporter AsyncReporter) func() +} + +// AsyncReporter records metrics asynchronously. +// Implementations must be concurrent-safe. +// The metric will be recorded once per collection cycle, rather than every time +// its value changes. +type AsyncReporter interface { + // Report records metric values using the provided recorder. + Report(AsyncMetricsRecorder) error +} + +// AsyncMetricsRecorder is a recorder for async metrics (i.e the metric will be +// recorded once per collection cycle, rather than every time its value changes). +type AsyncMetricsRecorder interface { + // ReportMetric reports a metric. The metric will be one of the predefined + // set of types in the internal/xds/clients/xdsclient/metrics/metrics.go file. + ReportMetric(metric any) +} diff --git a/internal/xds/clients/grpctransport/examples_test.go b/internal/xds/clients/grpctransport/examples_test.go new file mode 100644 index 0000000..09f9678 --- /dev/null +++ b/internal/xds/clients/grpctransport/examples_test.go @@ -0,0 +1,39 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpctransport_test + +import ( + "fmt" + + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" +) + +// ExampleServerIdentifierExtension demonstrates how to create +// clients.ServerIdentifier with grpctransport.ServerIdentifierExtension as +// its extensions. +// +// This example is creating clients.ServerIdentifier to connect to server at +// localhost:5678 using the config named "local". Note that "local" must +// exist as an entry in the provided configs to grpctransport.Builder. +func ExampleServerIdentifierExtension() { + // Note the Extensions field is set by value and not by pointer. + fmt.Printf("%+v", clients.ServerIdentifier{ServerURI: "localhost:5678", Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "local"}}) + // Output: {ServerURI:localhost:5678 Extensions:{ConfigName:local}} +} diff --git a/internal/xds/clients/grpctransport/grpc_transport.go b/internal/xds/clients/grpctransport/grpc_transport.go new file mode 100644 index 0000000..4cbaf7e --- /dev/null +++ b/internal/xds/clients/grpctransport/grpc_transport.go @@ -0,0 +1,237 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpctransport provides an implementation of the +// clients.TransportBuilder interface using gRPC. +package grpctransport + +import ( + "context" + "fmt" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/keepalive" +) + +var ( + logger = grpclog.Component("grpctransport") +) + +// ServerIdentifierExtension holds settings for connecting to a gRPC server, +// such as an xDS management or an LRS server. +// +// It must be set by value (not pointer) in the +// clients.ServerIdentifier.Extensions field (See Example). +type ServerIdentifierExtension struct { + // ConfigName is the name of the configuration to use for this transport. + // It must be present as a key in the map of configs passed to NewBuilder. + ConfigName string +} + +// Builder creates gRPC-based Transports. It must be paired with ServerIdentifiers +// that contain an Extension field of type ServerIdentifierExtension. +type Builder struct { + // configs is a map of configuration names to their respective Config. + configs map[string]Config + + mu sync.Mutex + // connections is a map of clients.ServerIdentifiers in use by the Builder + // to connect to different servers. + connections map[clients.ServerIdentifier]*grpc.ClientConn + // refs tracks the number of active references to each connection. + refs map[clients.ServerIdentifier]int +} + +// Config defines the configuration for connecting to a gRPC server, including +// credentials and an optional custom new client function. +type Config struct { + // Credentials is the credentials bundle to be used for the connection. + Credentials credentials.Bundle + // GRPCNewClient is an optional custom function to establish a gRPC connection. + // If nil, grpc.NewClient will be used as the default. + GRPCNewClient func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) +} + +// NewBuilder provides a builder for creating gRPC-based Transports using +// the credentials from provided map of credentials names to +// credentials.Bundle. +func NewBuilder(configs map[string]Config) *Builder { + return &Builder{ + configs: configs, + connections: make(map[clients.ServerIdentifier]*grpc.ClientConn), + refs: make(map[clients.ServerIdentifier]int), + } +} + +// Build returns a gRPC-based clients.Transport. +// +// The Extension field of the ServerIdentifier must be a ServerIdentifierExtension. +func (b *Builder) Build(si clients.ServerIdentifier) (clients.Transport, error) { + if si.ServerURI == "" { + return nil, fmt.Errorf("grpctransport: ServerURI is not set in ServerIdentifier") + } + if si.Extensions == nil { + return nil, fmt.Errorf("grpctransport: Extensions is not set in ServerIdentifier") + } + sce, ok := si.Extensions.(ServerIdentifierExtension) + if !ok { + return nil, fmt.Errorf("grpctransport: Extensions field is %T, but must be %T in ServerIdentifier", si.Extensions, ServerIdentifierExtension{}) + } + + config, ok := b.configs[sce.ConfigName] + if !ok { + return nil, fmt.Errorf("grpctransport: unknown config name %q specified in ServerIdentifierExtension", sce.ConfigName) + } + if config.Credentials == nil { + return nil, fmt.Errorf("grpctransport: config %q has nil credentials bundle", sce.ConfigName) + } + + b.mu.Lock() + defer b.mu.Unlock() + + if cc, ok := b.connections[si]; ok { + if logger.V(2) { + logger.Infof("Reusing existing connection to the server for ServerIdentifier: %v", si) + } + b.refs[si]++ + tr := &grpcTransport{cc: cc} + tr.cleanup = b.cleanupFunc(si, tr) + return tr, nil + } + + // Create a new gRPC client/channel for the server with the provided + // credentials, server URI, and a byte codec to send and receive messages. + // Also set a static keepalive configuration that is common across gRPC + // language implementations. + kpCfg := grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: 5 * time.Minute, + Timeout: 20 * time.Second, + }) + dopts := []grpc.DialOption{kpCfg, grpc.WithCredentialsBundle(config.Credentials), grpc.WithDefaultCallOptions(grpc.ForceCodec(&byteCodec{}))} + newClientFunc := grpc.NewClient + if config.GRPCNewClient != nil { + newClientFunc = config.GRPCNewClient + } + cc, err := newClientFunc(si.ServerURI, dopts...) + if err != nil { + return nil, fmt.Errorf("grpctransport: failed to create connection to server %q: %v", si.ServerURI, err) + } + tr := &grpcTransport{cc: cc} + // Register a cleanup function that decrements the refs to the gRPC + // transport each time Close() is called to close it and remove from + // transports and connections map if last reference is being released. + tr.cleanup = b.cleanupFunc(si, tr) + + // Add the newly created connection to the maps to re-use the transport + // channel and track references. + b.connections[si] = cc + b.refs[si] = 1 + + if logger.V(2) { + logger.Infof("Created a new transport to the server for ServerIdentifier: %v", si) + } + return tr, nil +} + +func (b *Builder) cleanupFunc(si clients.ServerIdentifier, tr *grpcTransport) func() { + return sync.OnceFunc(func() { + b.mu.Lock() + defer b.mu.Unlock() + + b.refs[si]-- + if b.refs[si] != 0 { + return + } + + tr.cc.Close() + tr.cc = nil + delete(b.connections, si) + delete(b.refs, si) + }) +} + +type grpcTransport struct { + cc *grpc.ClientConn + + // cleanup is the function to be invoked for releasing the references to + // the gRPC transport each time Close() is called. + cleanup func() +} + +// NewStream creates a new gRPC stream to the server for the specified method. +func (g *grpcTransport) NewStream(ctx context.Context, method string) (clients.Stream, error) { + s, err := g.cc.NewStream(ctx, &grpc.StreamDesc{ClientStreams: true, ServerStreams: true}, method) + if err != nil { + return nil, err + } + return &stream{stream: s}, nil +} + +// Close closes the gRPC channel to the server. +func (g *grpcTransport) Close() { + g.cleanup() +} + +type stream struct { + stream grpc.ClientStream +} + +// Send sends a message to the server. +func (s *stream) Send(msg []byte) error { + return s.stream.SendMsg(msg) +} + +// Recv receives a message from the server. +func (s *stream) Recv() ([]byte, error) { + var typedRes []byte + + if err := s.stream.RecvMsg(&typedRes); err != nil { + return nil, err + } + return typedRes, nil +} + +// byteCodec here is still sending proto messages. It's just they are +// in []byte form. +type byteCodec struct{} + +func (c *byteCodec) Marshal(v any) ([]byte, error) { + if b, ok := v.([]byte); ok { + return b, nil + } + return nil, fmt.Errorf("grpctransport: message is %T, but must be a []byte", v) +} + +func (c *byteCodec) Unmarshal(data []byte, v any) error { + if b, ok := v.(*[]byte); ok { + *b = data + return nil + } + return fmt.Errorf("grpctransport: target is %T, but must be *[]byte", v) +} + +func (c *byteCodec) Name() string { + // Return "" to ensure the Content-Type header is "application/grpc", + // which is expected by standard gRPC servers for protobuf messages. + return "" +} diff --git a/internal/xds/clients/grpctransport/grpc_transport_ext_test.go b/internal/xds/clients/grpctransport/grpc_transport_ext_test.go new file mode 100644 index 0000000..5c3050f --- /dev/null +++ b/internal/xds/clients/grpctransport/grpc_transport_ext_test.go @@ -0,0 +1,316 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpctransport_test + +import ( + "context" + "sync" + "testing" + "time" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/local" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/internal/testutils" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e" +) + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type testCredentials struct { + credentials.Bundle + transportCredentials credentials.TransportCredentials +} + +func (tc *testCredentials) TransportCredentials() credentials.TransportCredentials { + return tc.transportCredentials +} +func (tc *testCredentials) PerRPCCredentials() credentials.PerRPCCredentials { + return nil +} + +// TestBuild_Single tests that multiple calls to Build() with the same +// clients.ServerIdentifier returns the same transport. Also verifies that +// only when all references to the newly created transport are released, +// the underlying transport is closed. +func (s) TestBuild_Single(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + lis := testutils.NewListenerWrapper(t, nil) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis}) + + serverID := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "local"}, + } + configs := map[string]grpctransport.Config{ + "local": {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}}, + } + + // Calling Build() first time should create new gRPC transport. + builder := grpctransport.NewBuilder(configs) + tr, err := builder.Build(serverID) + if err != nil { + t.Fatalf("Failed to build transport: %v", err) + } + // Create a new stream to the server and verify that a new transport is + // created. + if _, err = tr.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil { + t.Fatalf("Failed to create stream: %v", err) + } + val, err := lis.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err) + } + conn := val.(*testutils.ConnWrapper) + + // Calling Build() again should not create new gRPC transport. + const count = 9 + transports := make([]clients.Transport, count) + for i := 0; i < count; i++ { + func() { + transports[i], err = builder.Build(serverID) + if err != nil { + t.Fatalf("Failed to build transport: %v", err) + } + // Create a new stream to the server and verify that no connection + // is established to the management server at this point. A new + // transport is created only when an existing connection for + // serverID does not exist. + if _, err = tr.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil { + t.Fatalf("Failed to create stream: %v", err) + } + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } + }() + } + + // Call Close() multiple times on each of the transport received in the + // above for loop. Close() calls are idempotent. The underlying gRPC + // transport is removed after the Close() call but calling close second + // time should not panic and underlying gRPC transport should not be + // closed. + for i := 0; i < count; i++ { + func() { + transports[i].Close() + transports[i].Close() + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected transport closure to management server") + } + }() + } + + // Call the last Close(). The underlying gRPC transport should be closed + // because calls in the above for loop have released all references. + tr.Close() + if _, err := conn.CloseCh.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for connection to management server to be closed") + } + + // Calling Build() again, after the previous transport was actually closed, + // should create a new one. + tr2, err := builder.Build(serverID) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer tr2.Close() + // Create a new stream to the server and verify that a new transport is + // created. + if _, err = tr2.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil { + t.Fatalf("Failed to create stream: %v", err) + } + if _, err := lis.NewConnCh.Receive(ctx); err != nil { + t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err) + } +} + +// TestBuild_Multiple tests the scenario where there are multiple calls to +// Build() with different clients.ServerIdentifier. Verifies that reference +// counts are tracked correctly for each transport and that only when all +// references are released for a transport, it is closed. +func (s) TestBuild_Multiple(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + lis := testutils.NewListenerWrapper(t, nil) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis}) + + serverID1 := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "local"}, + } + serverID2 := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + configs := map[string]grpctransport.Config{ + "local": {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}}, + "insecure": {Credentials: insecure.NewBundle()}, + } + + // Create two gRPC transports. + builder := grpctransport.NewBuilder(configs) + + tr1, err := builder.Build(serverID1) + if err != nil { + t.Fatalf("Failed to build transport: %v", err) + } + // Create a new stream to the server and verify that a new transport is + // created. + if _, err = tr1.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil { + t.Fatalf("Failed to create stream: %v", err) + } + val, err := lis.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err) + } + conn1 := val.(*testutils.ConnWrapper) + + tr2, err := builder.Build(serverID2) + if err != nil { + t.Fatalf("Failed to build transport: %v", err) + } + // Create a new stream to the server and verify that a new transport is + // created because credentials are different. + if _, err = tr2.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil { + t.Fatalf("Failed to create stream: %v", err) + } + val, err = lis.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err) + } + conn2 := val.(*testutils.ConnWrapper) + + // Create N more references to each of the two transports. + const count = 9 + transports1 := make([]clients.Transport, count) + transports2 := make([]clients.Transport, count) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; i < count; i++ { + var err error + transports1[i], err = builder.Build(serverID1) + if err != nil { + t.Errorf("Failed to build transport: %v", err) + } + // Create a new stream to the server and verify that no connection + // is established to the management server at this point. A new + // transport is created only when an existing connection for + // serverID does not exist. + if _, err = transports1[i].NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil { + t.Errorf("Failed to create stream: %v", err) + } + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Error("Unexpected new transport created to management server") + } + } + }() + go func() { + defer wg.Done() + for i := 0; i < count; i++ { + var err error + transports2[i], err = builder.Build(serverID2) + if err != nil { + t.Errorf("%d-th call to Build() failed with error: %v", i, err) + } + // Create a new stream to the server and verify that no connection + // is established to the management server at this point. A new + // transport is created only when an existing connection for + // serverID does not exist. + if _, err = transports2[i].NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil { + t.Errorf("Failed to create stream: %v", err) + } + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Error("Unexpected new transport created to management server") + } + } + }() + wg.Wait() + if t.Failed() { + t.FailNow() + } + + // Call Close() multiple times on each of the transport received in the + // above for loop. Close() calls are idempotent. The underlying gRPC + // transport is removed after the Close() call but calling close second + // time should not panic and underlying gRPC transport should not be + // closed. + for i := 0; i < count; i++ { + transports1[i].Close() + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := conn1.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected transport closure to management server") + } + transports1[i].Close() + } + // Call the last Close(). The underlying gRPC transport should be closed + // because calls in the above for loop have released all references. + tr1.Close() + if _, err := conn1.CloseCh.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for connection to management server to be closed") + } + + // Call Close() multiple times on each of the transport received in the + // above for loop. Close() calls are idempotent. The underlying gRPC + // transport is removed after the Close() call but calling close second + // time should not panic and underlying gRPC transport should not be + // closed. + for i := 0; i < count; i++ { + transports2[i].Close() + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := conn2.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected transport closure to management server") + } + transports2[i].Close() + } + // Call the last Close(). The underlying gRPC transport should be closed + // because calls in the above for loop have released all references. + tr2.Close() + if _, err := conn2.CloseCh.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for connection to management server to be closed") + } +} diff --git a/internal/xds/clients/grpctransport/grpc_transport_test.go b/internal/xds/clients/grpctransport/grpc_transport_test.go new file mode 100644 index 0000000..e33e889 --- /dev/null +++ b/internal/xds/clients/grpctransport/grpc_transport_test.go @@ -0,0 +1,425 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpctransport + +import ( + "context" + "io" + "net" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/local" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" + + v3discoverygrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// testServer implements the AggregatedDiscoveryServiceServer interface to test +// the gRPC transport implementation. +type testServer struct { + v3discoverygrpc.UnimplementedAggregatedDiscoveryServiceServer + + address string // address of the server + requestChan chan *v3discoverypb.DiscoveryRequest // channel to send the received requests on for verification + response *v3discoverypb.DiscoveryResponse // response to send back to the client from handler +} + +// setupTestServer set up the gRPC server for AggregatedDiscoveryService. It +// creates an instance of testServer that returns the provided response from +// the StreamAggregatedResources() handler and registers it with a gRPC server. +func setupTestServer(t *testing.T, response *v3discoverypb.DiscoveryResponse) *testServer { + t.Helper() + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen on localhost:0: %v", err) + } + ts := &testServer{ + requestChan: make(chan *v3discoverypb.DiscoveryRequest), + address: lis.Addr().String(), + response: response, + } + + s := grpc.NewServer() + + v3discoverygrpc.RegisterAggregatedDiscoveryServiceServer(s, ts) + go s.Serve(lis) + t.Cleanup(s.Stop) + + return ts +} + +// StreamAggregatedResources handles bidirectional streaming of +// DiscoveryRequest and DiscoveryResponse. It waits for a message from the +// client on the stream, and then sends a discovery response message back to +// the client. It also put the received message in requestChan for client to +// verify if the correct request was received. It continues until the client +// closes the stream. +func (s *testServer) StreamAggregatedResources(stream v3discoverygrpc.AggregatedDiscoveryService_StreamAggregatedResourcesServer) error { + ctx := stream.Context() + + for { + // Receive a DiscoveryRequest from the client + req, err := stream.Recv() + if err == io.EOF { + return nil // Stream closed by client + } + if err != nil { + return err // Handle other errors + } + + // Push received request for client to verify the correct request was + // received. + select { + case s.requestChan <- req: + case <-ctx.Done(): + return ctx.Err() + } + + // Send the response back to the client + if err := stream.Send(s.response); err != nil { + return err + } + } +} + +type testCredentials struct { + credentials.Bundle + transportCredentials credentials.TransportCredentials +} + +func (tc *testCredentials) TransportCredentials() credentials.TransportCredentials { + return tc.transportCredentials +} +func (tc *testCredentials) PerRPCCredentials() credentials.PerRPCCredentials { + return nil +} + +// TestBuild_Success verifies that the Builder successfully creates a new +// Transport in both cases when provided clients.ServerIdentifer is same +// one of the existing transport or a new one. +func (s) TestBuild_Success(t *testing.T) { + configs := map[string]Config{ + "local": {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}}, + "insecure": {Credentials: insecure.NewBundle()}, + } + b := NewBuilder(configs) + + serverID1 := clients.ServerIdentifier{ + ServerURI: "server-address", + Extensions: ServerIdentifierExtension{ConfigName: "local"}, + } + tr1, err := b.Build(serverID1) + if err != nil { + t.Fatalf("Build(serverID1) call failed: %v", err) + } + defer tr1.Close() + + serverID2 := clients.ServerIdentifier{ + ServerURI: "server-address", + Extensions: ServerIdentifierExtension{ConfigName: "local"}, + } + tr2, err := b.Build(serverID2) + if err != nil { + t.Fatalf("Build(serverID2) call failed: %v", err) + } + defer tr2.Close() + + serverID3 := clients.ServerIdentifier{ + ServerURI: "server-address", + Extensions: ServerIdentifierExtension{ConfigName: "insecure"}, + } + tr3, err := b.Build(serverID3) + if err != nil { + t.Fatalf("Build(serverID3) call failed: %v", err) + } + defer tr3.Close() +} + +// TestBuild_Failure verifies that the Builder returns error when incorrect +// ServerIdentifier is provided. +// +// It covers the following scenarios: +// - ServerURI is empty. +// - Extensions is nil. +// - Extensions is not ServerIdentifierExtension. +// - Credentials are nil. +func (s) TestBuild_Failure(t *testing.T) { + tests := []struct { + name string + serverID clients.ServerIdentifier + }{ + { + name: "ServerURI is empty", + serverID: clients.ServerIdentifier{ + ServerURI: "", + Extensions: ServerIdentifierExtension{ConfigName: "local"}, + }, + }, + { + name: "Extensions is nil", + serverID: clients.ServerIdentifier{ServerURI: "server-address"}, + }, + { + name: "Extensions is not a ServerIdentifierExtension", + serverID: clients.ServerIdentifier{ + ServerURI: "server-address", + Extensions: 1, + }, + }, + { + name: "ServerIdentifierExtension without ConfigName", + serverID: clients.ServerIdentifier{ + ServerURI: "server-address", + Extensions: ServerIdentifierExtension{}, + }, + }, + { + name: "ServerIdentifierExtension ConfigName is not present", + serverID: clients.ServerIdentifier{ + ServerURI: "server-address", + Extensions: ServerIdentifierExtension{ConfigName: "unknown"}, + }, + }, + { + name: "ServerIdentifierExtension ConfigName maps to nil credentials", + serverID: clients.ServerIdentifier{ + ServerURI: "server-address", + Extensions: ServerIdentifierExtension{ConfigName: "nil-credentials"}, + }, + }, + { + name: "ServerIdentifierExtension is added as pointer", + serverID: clients.ServerIdentifier{ + ServerURI: "server-address", + Extensions: &ServerIdentifierExtension{ConfigName: "local"}, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + configs := map[string]Config{ + "local": {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}}, + "nil-credentials": {Credentials: nil}, + } + b := NewBuilder(configs) + tr, err := b.Build(test.serverID) + if err == nil { + t.Fatalf("Build() succeeded, want error") + } + if tr != nil { + t.Fatalf("Got non-nil transport from Build(), want nil") + } + }) + } +} + +// TestNewStream_Success verifies that NewStream() successfully creates a new +// client stream for the server when provided a valid server URI and a config +// with valid credentials. +func (s) TestNewStream_Success(t *testing.T) { + ts := setupTestServer(t, &v3discoverypb.DiscoveryResponse{VersionInfo: "1"}) + + serverCfg := clients.ServerIdentifier{ + ServerURI: ts.address, + Extensions: ServerIdentifierExtension{ConfigName: "local"}, + } + configs := map[string]Config{ + "local": {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}}, + } + builder := NewBuilder(configs) + transport, err := builder.Build(serverCfg) + if err != nil { + t.Fatalf("Failed to build transport: %v", err) + } + defer transport.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = transport.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil { + t.Fatalf("transport.NewStream() failed: %v", err) + } +} + +// TestNewStream_Success_WithCustomGRPCNewClient verifies that NewStream() +// successfully creates a new client stream for the server when provided a +// valid server URI and a config with valid credentials and a custom gRPC +// NewClient function. +func (s) TestNewStream_Success_WithCustomGRPCNewClient(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + ts := setupTestServer(t, &v3discoverypb.DiscoveryResponse{VersionInfo: "1"}) + + // Create a custom dialer function that will be used by the gRPC client. + customDialerCalled := make(chan struct{}, 1) + customGRPCNewClient := func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + customDialerCalled <- struct{}{} + return grpc.NewClient(target, opts...) + } + + configs := map[string]Config{ + "custom-dialer-config": { + Credentials: &testCredentials{transportCredentials: local.NewCredentials()}, + GRPCNewClient: customGRPCNewClient, + }, + } + builder := NewBuilder(configs) + + serverID := clients.ServerIdentifier{ + ServerURI: ts.address, + Extensions: ServerIdentifierExtension{ConfigName: "custom-dialer-config"}, + } + + transport, err := builder.Build(serverID) + if err != nil { + t.Fatalf("builder.Build(%+v) failed: %v", serverID, err) + } + defer transport.Close() + + select { + case <-customDialerCalled: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for custom dialer to be called: %v", ctx.Err()) + } + + // Verify that the transport works by creating a stream. + if _, err = transport.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil { + t.Fatalf("transport.NewStream() failed with custom dialer: %v", err) + } +} + +// TestNewStream_Error verifies that NewStream() returns an error +// when attempting to create a stream with an invalid server URI. +func (s) TestNewStream_Error(t *testing.T) { + serverCfg := clients.ServerIdentifier{ + ServerURI: "invalid-server-uri", + Extensions: ServerIdentifierExtension{ConfigName: "local"}, + } + configs := map[string]Config{ + "local": {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}}, + } + builder := NewBuilder(configs) + transport, err := builder.Build(serverCfg) + if err != nil { + t.Fatalf("Failed to build transport: %v", err) + } + defer transport.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = transport.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err == nil { + t.Fatal("transport.NewStream() succeeded, want failure") + } +} + +// TestStream_SendAndRecv verifies that Send() and Recv() successfully send +// and receive messages on the stream to and from the gRPC server. +// +// It starts a gRPC test server using setupTestServer(). The test then sends a +// testDiscoverRequest on the stream and verifies that the received discovery +// request on the server is same as sent. It then wait to receive a +// testDiscoverResponse from the server and verifies that the received +// discovery response is same as sent from the server. +func (s) TestStream_SendAndRecv(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + ts := setupTestServer(t, &v3discoverypb.DiscoveryResponse{VersionInfo: "1"}) + + // Build a grpc-based transport to the above server. + serverCfg := clients.ServerIdentifier{ + ServerURI: ts.address, + Extensions: ServerIdentifierExtension{ConfigName: "local"}, + } + configs := map[string]Config{ + "local": {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}}, + } + builder := NewBuilder(configs) + transport, err := builder.Build(serverCfg) + if err != nil { + t.Fatalf("Failed to build transport: %v", err) + } + defer transport.Close() + + // Create a new stream to the server. + stream, err := transport.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources") + if err != nil { + t.Fatalf("Failed to create stream: %v", err) + } + + // Send a discovery request message on the stream. + testDiscoverRequest := &v3discoverypb.DiscoveryRequest{VersionInfo: "1"} + msg, err := proto.Marshal(testDiscoverRequest) + if err != nil { + t.Fatalf("Failed to marshal DiscoveryRequest: %v", err) + } + if err := stream.Send(msg); err != nil { + t.Fatalf("Failed to send message: %v", err) + } + + // Verify that the DiscoveryRequest received on the server was same as + // sent. + select { + case gotReq := <-ts.requestChan: + if diff := cmp.Diff(testDiscoverRequest, gotReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in request received on server (-want +got):\n%s", diff) + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for request to reach server") + } + + // Wait until response message is received from the server. + res, err := stream.Recv() + if err != nil { + t.Fatalf("Failed to receive message: %v", err) + } + + // Verify that the DiscoveryResponse received was same as sent from the + // server. + var gotRes v3discoverypb.DiscoveryResponse + if err := proto.Unmarshal(res, &gotRes); err != nil { + t.Fatalf("Failed to unmarshal response from server to DiscoveryResponse: %v", err) + } + if diff := cmp.Diff(ts.response, &gotRes, protocmp.Transform()); diff != "" { + t.Fatalf("proto.Unmarshal(res, &gotRes) returned unexpected diff (-want +got):\n%s", diff) + } +} diff --git a/internal/xds/clients/internal/backoff/backoff.go b/internal/xds/clients/internal/backoff/backoff.go new file mode 100644 index 0000000..c07e155 --- /dev/null +++ b/internal/xds/clients/internal/backoff/backoff.go @@ -0,0 +1,124 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package backoff implements the backoff strategy for clients. +// +// This is kept in internal until the clients project decides whether or not to +// allow alternative backoff strategies. +package backoff + +import ( + "context" + "errors" + rand "math/rand/v2" + "time" +) + +// config defines the configuration options for backoff. +type config struct { + // baseDelay is the amount of time to backoff after the first failure. + baseDelay time.Duration + // multiplier is the factor with which to multiply backoffs after a + // failed retry. Should ideally be greater than 1. + multiplier float64 + // jitter is the factor with which backoffs are randomized. + jitter float64 + // maxDelay is the upper bound of backoff delay. + maxDelay time.Duration +} + +// defaultConfig is a backoff configuration with the default values specified +// at https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. +// +// This should be useful for callers who want to configure backoff with +// non-default values only for a subset of the options. +var defaultConfig = config{ + baseDelay: 1.0 * time.Second, + multiplier: 1.6, + jitter: 0.2, + maxDelay: 120 * time.Second, +} + +// DefaultExponential is an exponential backoff implementation using the +// default values for all the configurable knobs defined in +// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. +var DefaultExponential = exponential{config: defaultConfig} + +// exponential implements exponential backoff algorithm as defined in +// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. +type exponential struct { + // Config contains all options to configure the backoff algorithm. + config config +} + +// Backoff returns the amount of time to wait before the next retry given the +// number of retries. +func (bc exponential) Backoff(retries int) time.Duration { + if retries == 0 { + return bc.config.baseDelay + } + backoff, max := float64(bc.config.baseDelay), float64(bc.config.maxDelay) + for backoff < max && retries > 0 { + backoff *= bc.config.multiplier + retries-- + } + if backoff > max { + backoff = max + } + // Randomize backoff delays so that if a cluster of requests start at + // the same time, they won't operate in lockstep. + backoff *= 1 + bc.config.jitter*(rand.Float64()*2-1) + if backoff < 0 { + return 0 + } + return time.Duration(backoff) +} + +// ErrResetBackoff is the error to be returned by the function executed by RunF, +// to instruct the latter to reset its backoff state. +var ErrResetBackoff = errors.New("reset backoff state") + +// RunF provides a convenient way to run a function f repeatedly until the +// context expires or f returns a non-nil error that is not ErrResetBackoff. +// When f returns ErrResetBackoff, RunF continues to run f, but resets its +// backoff state before doing so. backoff accepts an integer representing the +// number of retries, and returns the amount of time to backoff. +func RunF(ctx context.Context, f func() error, backoff func(int) time.Duration) { + attempt := 0 + timer := time.NewTimer(0) + for ctx.Err() == nil { + select { + case <-timer.C: + case <-ctx.Done(): + timer.Stop() + return + } + + err := f() + if errors.Is(err, ErrResetBackoff) { + timer.Reset(0) + attempt = 0 + continue + } + if err != nil { + return + } + timer.Reset(backoff(attempt)) + attempt++ + } +} diff --git a/internal/xds/clients/internal/buffer/unbounded.go b/internal/xds/clients/internal/buffer/unbounded.go new file mode 100644 index 0000000..4d3527d --- /dev/null +++ b/internal/xds/clients/internal/buffer/unbounded.go @@ -0,0 +1,115 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package buffer provides an implementation of an unbounded buffer. +package buffer + +import ( + "errors" + "sync" +) + +// Unbounded is an implementation of an unbounded buffer which does not use +// extra goroutines. This is typically used for passing updates from one entity +// to another within gRPC. +// +// To avoid extra memory allocations and type assertions, using any on +// performance-critical code paths is discouraged. Use concrete types wherever +// possible when instantiating Unbounded. +// +// All methods on this type are thread-safe and don't block on anything except +// the underlying mutex used for synchronization. +type Unbounded[T any] struct { + c chan T + closed bool + closing bool + mu sync.Mutex + backlog []T +} + +// NewUnbounded returns a new instance of Unbounded. +func NewUnbounded[T any]() *Unbounded[T] { + return &Unbounded[T]{c: make(chan T, 1)} +} + +var errBufferClosed = errors.New("Put() called on closed buffer.Unbounded") + +// Put adds t to the unbounded buffer. +func (b *Unbounded[T]) Put(t T) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.closing { + return errBufferClosed + } + if len(b.backlog) == 0 { + select { + case b.c <- t: + return nil + default: + } + } + b.backlog = append(b.backlog, t) + return nil +} + +// Load sends the earliest buffered data, if any, onto the read channel returned +// by Get(). Users are expected to call this every time they successfully read a +// value from the read channel. +func (b *Unbounded[T]) Load() { + b.mu.Lock() + defer b.mu.Unlock() + if len(b.backlog) > 0 { + select { + case b.c <- b.backlog[0]: + var zero T + b.backlog[0] = zero + b.backlog = b.backlog[1:] + default: + } + } else if b.closing && !b.closed { + b.closed = true + close(b.c) + } +} + +// Get returns a read channel on which values added to the buffer, via Put(), +// are sent on. +// +// Upon reading a value from this channel, users are expected to call Load() to +// send the next buffered value onto the channel if there is any. +// +// If the unbounded buffer is closed, the read channel returned by this method +// is closed after all data is drained. +func (b *Unbounded[T]) Get() <-chan T { + return b.c +} + +// Close closes the unbounded buffer. No subsequent data may be Put(), and the +// channel returned from Get() will be closed after all the data is read and +// Load() is called for the final time. +func (b *Unbounded[T]) Close() { + b.mu.Lock() + defer b.mu.Unlock() + if b.closing { + return + } + b.closing = true + if len(b.backlog) == 0 { + b.closed = true + close(b.c) + } +} diff --git a/internal/xds/clients/internal/buffer/unbounded_test.go b/internal/xds/clients/internal/buffer/unbounded_test.go new file mode 100644 index 0000000..8ccf133 --- /dev/null +++ b/internal/xds/clients/internal/buffer/unbounded_test.go @@ -0,0 +1,148 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package buffer + +import ( + "sort" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" +) + +const ( + numWriters = 10 + numWrites = 10 +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// wantReads contains the set of values expected to be read by the reader +// goroutine in the tests. +var wantReads []int + +func init() { + for i := 0; i < numWriters; i++ { + for j := 0; j < numWrites; j++ { + wantReads = append(wantReads, i) + } + } +} + +// TestSingleWriter starts one reader and one writer goroutine and makes sure +// that the reader gets all the values added to the buffer by the writer. +func (s) TestSingleWriter(t *testing.T) { + ub := NewUnbounded[int]() + reads := []int{} + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + ch := ub.Get() + for i := 0; i < numWriters*numWrites; i++ { + r := <-ch + reads = append(reads, r) + ub.Load() + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < numWriters; i++ { + for j := 0; j < numWrites; j++ { + ub.Put(i) + } + } + }() + + wg.Wait() + if !cmp.Equal(reads, wantReads) { + t.Errorf("reads: %#v, wantReads: %#v", reads, wantReads) + } +} + +// TestMultipleWriters starts multiple writers and one reader goroutine and +// makes sure that the reader gets all the data written by all writers. +func (s) TestMultipleWriters(t *testing.T) { + ub := NewUnbounded[int]() + reads := []int{} + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + ch := ub.Get() + for i := 0; i < numWriters*numWrites; i++ { + r := <-ch + reads = append(reads, r) + ub.Load() + } + }() + + wg.Add(numWriters) + for i := 0; i < numWriters; i++ { + go func(index int) { + defer wg.Done() + for j := 0; j < numWrites; j++ { + ub.Put(index) + } + }(i) + } + + wg.Wait() + sort.Ints(reads) + if !cmp.Equal(reads, wantReads) { + t.Errorf("reads: %#v, wantReads: %#v", reads, wantReads) + } +} + +// TestClose closes the buffer and makes sure that nothing is sent after the +// buffer is closed. +func (s) TestClose(t *testing.T) { + ub := NewUnbounded[int]() + if err := ub.Put(1); err != nil { + t.Fatalf("Unbounded.Put() = %v; want nil", err) + } + ub.Close() + if err := ub.Put(1); err == nil { + t.Fatalf("Unbounded.Put() = ; want non-nil error") + } + if v, ok := <-ub.Get(); !ok { + t.Errorf("Unbounded.Get() = %v, %v, want %v, %v", v, ok, 1, true) + } + if err := ub.Put(1); err == nil { + t.Fatalf("Unbounded.Put() = ; want non-nil error") + } + ub.Load() + if v, ok := <-ub.Get(); ok { + t.Errorf("Unbounded.Get() = %v, want closed channel", v) + } + if err := ub.Put(1); err == nil { + t.Fatalf("Unbounded.Put() = ; want non-nil error") + } + ub.Close() // ignored +} diff --git a/internal/xds/clients/internal/internal.go b/internal/xds/clients/internal/internal.go new file mode 100644 index 0000000..60cf82b --- /dev/null +++ b/internal/xds/clients/internal/internal.go @@ -0,0 +1,71 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains helpers for xDS and LRS clients. +package internal + +import ( + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" +) + +// NodeProto returns a protobuf representation of clients.Node n. +// +// This function is intended to be used by the client implementation to convert +// the user-provided Node configuration to its protobuf representation. +func NodeProto(n clients.Node) *v3corepb.Node { + return &v3corepb.Node{ + Id: n.ID, + Cluster: n.Cluster, + Locality: func() *v3corepb.Locality { + if isLocalityEmpty(n.Locality) { + return nil + } + return &v3corepb.Locality{ + Region: n.Locality.Region, + Zone: n.Locality.Zone, + SubZone: n.Locality.SubZone, + } + }(), + Metadata: func() *structpb.Struct { + if n.Metadata == nil { + return nil + } + if md, ok := n.Metadata.(*structpb.Struct); ok { + return proto.Clone(md).(*structpb.Struct) + } + return nil + }(), + UserAgentName: n.UserAgentName, + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: n.UserAgentVersion}, + } +} + +// isLocalityEqual reports whether clients.Locality l is considered empty. +func isLocalityEmpty(l clients.Locality) bool { + return isLocalityEqual(l, clients.Locality{}) +} + +// isLocalityEqual returns true if clients.Locality l1 and l2 are considered +// equal. +func isLocalityEqual(l1, l2 clients.Locality) bool { + return l1.Region == l2.Region && l1.Zone == l2.Zone && l1.SubZone == l2.SubZone +} diff --git a/internal/xds/clients/internal/internal_test.go b/internal/xds/clients/internal/internal_test.go new file mode 100644 index 0000000..99deb5c --- /dev/null +++ b/internal/xds/clients/internal/internal_test.go @@ -0,0 +1,213 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package internal + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/structpb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func newStructProtoFromMap(t *testing.T, input map[string]any) *structpb.Struct { + t.Helper() + + ret, err := structpb.NewStruct(input) + if err != nil { + t.Fatalf("Failed to create new struct proto from map %v: %v", input, err) + } + return ret +} + +func (s) TestIsLocalityEmpty(t *testing.T) { + tests := []struct { + name string + locality clients.Locality + want bool + }{ + { + name: "empty_locality", + locality: clients.Locality{}, + want: true, + }, + { + name: "non_empty_region", + locality: clients.Locality{Region: "region"}, + want: false, + }, + { + name: "non_empty_zone", + locality: clients.Locality{Zone: "zone"}, + want: false, + }, + { + name: "non_empty_subzone", + locality: clients.Locality{SubZone: "subzone"}, + want: false, + }, + { + name: "non_empty_all_fields", + locality: clients.Locality{Region: "region", Zone: "zone", SubZone: "subzone"}, + want: false, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := isLocalityEmpty(test.locality); got != test.want { + t.Errorf("IsEmpty() = %v, want %v", got, test.want) + } + }) + } +} + +func (s) TestIsLocalityEqual(t *testing.T) { + tests := []struct { + name string + l1 clients.Locality + l2 clients.Locality + wantEq bool + }{ + { + name: "both_equal", + l1: clients.Locality{Region: "region", Zone: "zone", SubZone: "subzone"}, + l2: clients.Locality{Region: "region", Zone: "zone", SubZone: "subzone"}, + wantEq: true, + }, + { + name: "different_regions", + l1: clients.Locality{Region: "region1", Zone: "zone", SubZone: "subzone"}, + l2: clients.Locality{Region: "region2", Zone: "zone", SubZone: "subzone"}, + wantEq: false, + }, + + { + name: "different_zones", + l1: clients.Locality{Region: "region", Zone: "zone1", SubZone: "subzone"}, + l2: clients.Locality{Region: "region", Zone: "zone2", SubZone: "subzone"}, + wantEq: false, + }, + { + name: "different_subzones", + l1: clients.Locality{Region: "region", Zone: "zone", SubZone: "subzone1"}, + l2: clients.Locality{Region: "region", Zone: "zone", SubZone: "subzone2"}, + wantEq: false, + }, + { + name: "one_empty", + l1: clients.Locality{}, + l2: clients.Locality{Region: "region", Zone: "zone", SubZone: "subzone"}, + wantEq: false, + }, + { + name: "both_empty", + l1: clients.Locality{}, + l2: clients.Locality{}, + wantEq: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if gotEq := isLocalityEqual(test.l1, test.l2); gotEq != test.wantEq { + t.Errorf("Equal() = %v, want %v", gotEq, test.wantEq) + } + }) + } +} + +func (s) TestNodeProto(t *testing.T) { + tests := []struct { + desc string + inputNode clients.Node + wantProto *v3corepb.Node + }{ + { + desc: "all_fields_set", + inputNode: clients.Node{ + ID: "id", + Cluster: "cluster", + Locality: clients.Locality{ + Region: "region", + Zone: "zone", + SubZone: "sub_zone", + }, + Metadata: newStructProtoFromMap(t, map[string]any{"k1": "v1", "k2": 101, "k3": 280.0}), + UserAgentName: "user agent", + UserAgentVersion: "version", + }, + wantProto: &v3corepb.Node{ + Id: "id", + Cluster: "cluster", + Locality: &v3corepb.Locality{ + Region: "region", + Zone: "zone", + SubZone: "sub_zone", + }, + Metadata: newStructProtoFromMap(t, map[string]any{"k1": "v1", "k2": 101, "k3": 280.0}), + UserAgentName: "user agent", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "version"}, + }, + }, + { + desc: "some_fields_unset", + inputNode: clients.Node{ + ID: "id", + }, + wantProto: &v3corepb.Node{ + Id: "id", + UserAgentName: "", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: ""}, + }, + }, + { + desc: "empty_locality", + inputNode: clients.Node{ + ID: "id", + Locality: clients.Locality{}, + }, + wantProto: &v3corepb.Node{ + Id: "id", + UserAgentName: "", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: ""}, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + gotProto := NodeProto(test.inputNode) + if diff := cmp.Diff(test.wantProto, gotProto, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in node proto: (-want, +got):\n%s", diff) + } + }) + } +} diff --git a/internal/xds/clients/internal/pretty/pretty.go b/internal/xds/clients/internal/pretty/pretty.go new file mode 100644 index 0000000..dbee7a6 --- /dev/null +++ b/internal/xds/clients/internal/pretty/pretty.go @@ -0,0 +1,73 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package pretty defines helper functions to pretty-print structs for logging. +package pretty + +import ( + "bytes" + "encoding/json" + "fmt" + + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/protoadapt" +) + +const jsonIndent = " " + +// ToJSON marshals the input into a json string. +// +// If marshal fails, it falls back to fmt.Sprintf("%+v"). +func ToJSON(e any) string { + if ee, ok := e.(protoadapt.MessageV1); ok { + e = protoadapt.MessageV2Of(ee) + } + + if ee, ok := e.(protoadapt.MessageV2); ok { + mm := protojson.MarshalOptions{ + Indent: jsonIndent, + Multiline: true, + } + ret, err := mm.Marshal(ee) + if err != nil { + // This may fail for proto.Anys, e.g. for xDS v2, LDS, the v2 + // messages are not imported, and this will fail because the message + // is not found. + return fmt.Sprintf("%+v", ee) + } + return string(ret) + } + + ret, err := json.MarshalIndent(e, "", jsonIndent) + if err != nil { + return fmt.Sprintf("%+v", e) + } + return string(ret) +} + +// FormatJSON formats the input json bytes with indentation. +// +// If Indent fails, it returns the unchanged input as string. +func FormatJSON(b []byte) string { + var out bytes.Buffer + err := json.Indent(&out, b, "", jsonIndent) + if err != nil { + return string(b) + } + return out.String() +} diff --git a/internal/xds/clients/internal/syncutil/callback_serializer.go b/internal/xds/clients/internal/syncutil/callback_serializer.go new file mode 100644 index 0000000..54dc6e4 --- /dev/null +++ b/internal/xds/clients/internal/syncutil/callback_serializer.go @@ -0,0 +1,98 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package syncutil + +import ( + "context" + + "google.golang.org/grpc/internal/xds/clients/internal/buffer" +) + +// CallbackSerializer provides a mechanism to schedule callbacks in a +// synchronized manner. It provides a FIFO guarantee on the order of execution +// of scheduled callbacks. New callbacks can be scheduled by invoking the +// Schedule() method. +// +// This type is safe for concurrent access. +type CallbackSerializer struct { + // done is closed once the serializer is shut down completely, i.e all + // scheduled callbacks are executed and the serializer has deallocated all + // its resources. + done chan struct{} + + callbacks *buffer.Unbounded[func(context.Context)] +} + +// NewCallbackSerializer returns a new CallbackSerializer instance. The provided +// context will be passed to the scheduled callbacks. Users should cancel the +// provided context to shutdown the CallbackSerializer. It is guaranteed that no +// callbacks will be added once this context is canceled, and any pending un-run +// callbacks will be executed before the serializer is shut down. +func NewCallbackSerializer(ctx context.Context) *CallbackSerializer { + cs := &CallbackSerializer{ + done: make(chan struct{}), + callbacks: buffer.NewUnbounded[func(context.Context)](), + } + go cs.run(ctx) + return cs +} + +// TrySchedule tries to schedule the provided callback function f to be +// executed in the order it was added. This is a best-effort operation. If the +// context passed to NewCallbackSerializer was canceled before this method is +// called, the callback will not be scheduled. +// +// Callbacks are expected to honor the context when performing any blocking +// operations, and should return early when the context is canceled. +func (cs *CallbackSerializer) TrySchedule(f func(ctx context.Context)) { + cs.callbacks.Put(f) +} + +// ScheduleOr schedules the provided callback function f to be executed in the +// order it was added. If the context passed to NewCallbackSerializer has been +// canceled before this method is called, the onFailure callback will be +// executed inline instead. +// +// Callbacks are expected to honor the context when performing any blocking +// operations, and should return early when the context is canceled. +func (cs *CallbackSerializer) ScheduleOr(f func(ctx context.Context), onFailure func()) { + if cs.callbacks.Put(f) != nil { + onFailure() + } +} + +func (cs *CallbackSerializer) run(ctx context.Context) { + defer close(cs.done) + + // Close the buffer when the context is canceled + // to prevent new callbacks from being added. + context.AfterFunc(ctx, cs.callbacks.Close) + + // Run all callbacks. + for cb := range cs.callbacks.Get() { + cs.callbacks.Load() + cb(ctx) + } +} + +// Done returns a channel that is closed after the context passed to +// NewCallbackSerializer is canceled and all callbacks have been executed. +func (cs *CallbackSerializer) Done() <-chan struct{} { + return cs.done +} diff --git a/internal/xds/clients/internal/syncutil/callback_serializer_test.go b/internal/xds/clients/internal/syncutil/callback_serializer_test.go new file mode 100644 index 0000000..a517115 --- /dev/null +++ b/internal/xds/clients/internal/syncutil/callback_serializer_test.go @@ -0,0 +1,206 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package syncutil + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) + +// TestCallbackSerializer_Schedule_FIFO verifies that callbacks are executed in +// the same order in which they were scheduled. +func (s) TestCallbackSerializer_Schedule_FIFO(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + cs := NewCallbackSerializer(ctx) + defer cancel() + + // We have two channels, one to record the order of scheduling, and the + // other to record the order of execution. We spawn a bunch of goroutines + // which record the order of scheduling and call the actual Schedule() + // method as well. The callbacks record the order of execution. + // + // We need to grab a lock to record order of scheduling to guarantee that + // the act of recording and the act of calling Schedule() happen atomically. + const numCallbacks = 100 + var mu sync.Mutex + scheduleOrderCh := make(chan int, numCallbacks) + executionOrderCh := make(chan int, numCallbacks) + for i := 0; i < numCallbacks; i++ { + go func(id int) { + mu.Lock() + defer mu.Unlock() + scheduleOrderCh <- id + cs.TrySchedule(func(ctx context.Context) { + select { + case <-ctx.Done(): + return + case executionOrderCh <- id: + } + }) + }(i) + } + + // Spawn a couple of goroutines to capture the order or scheduling and the + // order of execution. + scheduleOrder := make([]int, numCallbacks) + executionOrder := make([]int, numCallbacks) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; i < numCallbacks; i++ { + select { + case <-ctx.Done(): + return + case id := <-scheduleOrderCh: + scheduleOrder[i] = id + } + } + }() + go func() { + defer wg.Done() + for i := 0; i < numCallbacks; i++ { + select { + case <-ctx.Done(): + return + case id := <-executionOrderCh: + executionOrder[i] = id + } + } + }() + wg.Wait() + + if diff := cmp.Diff(executionOrder, scheduleOrder); diff != "" { + t.Fatalf("Callbacks are not executed in scheduled order. diff(-want, +got):\n%s", diff) + } +} + +// TestCallbackSerializer_Schedule_Concurrent verifies that all concurrently +// scheduled callbacks get executed. +func (s) TestCallbackSerializer_Schedule_Concurrent(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + cs := NewCallbackSerializer(ctx) + defer cancel() + + // Schedule callbacks concurrently by calling Schedule() from goroutines. + // The execution of the callbacks call Done() on the waitgroup, which + // eventually unblocks the test and allows it to complete. + const numCallbacks = 100 + var wg sync.WaitGroup + wg.Add(numCallbacks) + for i := 0; i < numCallbacks; i++ { + go func() { + cs.TrySchedule(func(context.Context) { + wg.Done() + }) + }() + } + + // We call Wait() on the waitgroup from a goroutine so that we can select on + // the Wait() being unblocked and the overall test deadline expiring. + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for all scheduled callbacks to be executed") + case <-done: + } +} + +// TestCallbackSerializer_Schedule_Close verifies that callbacks in the queue +// are not executed once Close() returns. +func (s) TestCallbackSerializer_Schedule_Close(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + serializerCtx, serializerCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + cs := NewCallbackSerializer(serializerCtx) + + // Schedule a callback which blocks until the context passed to it is + // canceled. It also closes a channel to signal that it has started. + firstCallbackStartedCh := make(chan struct{}) + cs.TrySchedule(func(ctx context.Context) { + close(firstCallbackStartedCh) + <-ctx.Done() + }) + + // Schedule a bunch of callbacks. These should be executed since they are + // scheduled before the serializer is closed. + const numCallbacks = 10 + callbackCh := make(chan int, numCallbacks) + for i := 0; i < numCallbacks; i++ { + num := i + callback := func(context.Context) { callbackCh <- num } + onFailure := func() { t.Fatal("Schedule failed to accept a callback when the serializer is yet to be closed") } + cs.ScheduleOr(callback, onFailure) + } + + // Ensure that none of the newer callbacks are executed at this point. + select { + case <-time.After(defaultTestShortTimeout): + case <-callbackCh: + t.Fatal("Newer callback executed when older one is still executing") + } + + // Wait for the first callback to start before closing the scheduler. + <-firstCallbackStartedCh + + // Cancel the context which will unblock the first callback. All of the + // other callbacks (which have not started executing at this point) should + // be executed after this. + serializerCancel() + + // Ensure that the newer callbacks are executed. + for i := 0; i < numCallbacks; i++ { + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for callback scheduled before close to be executed") + case num := <-callbackCh: + if num != i { + t.Fatalf("Executing callback %d, want %d", num, i) + } + } + } + <-cs.Done() + + // Ensure that a callback cannot be scheduled after the serializer is + // closed. + done := make(chan struct{}) + callback := func(context.Context) { t.Fatal("Scheduled a callback after closing the serializer") } + onFailure := func() { close(done) } + cs.ScheduleOr(callback, onFailure) + select { + case <-time.After(defaultTestTimeout): + t.Fatal("Successfully scheduled callback after serializer is closed") + case <-done: + } +} diff --git a/internal/xds/clients/internal/syncutil/event.go b/internal/xds/clients/internal/syncutil/event.go new file mode 100644 index 0000000..1850ca3 --- /dev/null +++ b/internal/xds/clients/internal/syncutil/event.go @@ -0,0 +1,61 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package syncutil implements additional synchronization primitives built upon +// the sync package. +package syncutil + +import ( + "sync" + "sync/atomic" +) + +// Event represents a one-time event that may occur in the future. +type Event struct { + fired int32 + c chan struct{} + o sync.Once +} + +// Fire causes e to complete. It is safe to call multiple times, and +// concurrently. It returns true iff this call to Fire caused the signaling +// channel returned by Done to close. +func (e *Event) Fire() bool { + ret := false + e.o.Do(func() { + atomic.StoreInt32(&e.fired, 1) + close(e.c) + ret = true + }) + return ret +} + +// Done returns a channel that will be closed when Fire is called. +func (e *Event) Done() <-chan struct{} { + return e.c +} + +// HasFired returns true if Fire has been called. +func (e *Event) HasFired() bool { + return atomic.LoadInt32(&e.fired) == 1 +} + +// NewEvent returns a new, ready-to-use Event. +func NewEvent() *Event { + return &Event{c: make(chan struct{})} +} diff --git a/internal/xds/clients/internal/syncutil/event_test.go b/internal/xds/clients/internal/syncutil/event_test.go new file mode 100644 index 0000000..1eb1daf --- /dev/null +++ b/internal/xds/clients/internal/syncutil/event_test.go @@ -0,0 +1,81 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package syncutil + +import ( + "testing" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestEventHasFired(t *testing.T) { + e := NewEvent() + if e.HasFired() { + t.Fatal("e.HasFired() = true; want false") + } + if !e.Fire() { + t.Fatal("e.Fire() = false; want true") + } + if !e.HasFired() { + t.Fatal("e.HasFired() = false; want true") + } +} + +func (s) TestEventDoneChannel(t *testing.T) { + e := NewEvent() + select { + case <-e.Done(): + t.Fatal("e.HasFired() = true; want false") + default: + } + if !e.Fire() { + t.Fatal("e.Fire() = false; want true") + } + select { + case <-e.Done(): + default: + t.Fatal("e.HasFired() = false; want true") + } +} + +func (s) TestEventMultipleFires(t *testing.T) { + e := NewEvent() + if e.HasFired() { + t.Fatal("e.HasFired() = true; want false") + } + if !e.Fire() { + t.Fatal("e.Fire() = false; want true") + } + for i := 0; i < 3; i++ { + if !e.HasFired() { + t.Fatal("e.HasFired() = false; want true") + } + if e.Fire() { + t.Fatal("e.Fire() = true; want false") + } + } +} diff --git a/internal/xds/clients/internal/testutils/channel.go b/internal/xds/clients/internal/testutils/channel.go new file mode 100644 index 0000000..ac77d55 --- /dev/null +++ b/internal/xds/clients/internal/testutils/channel.go @@ -0,0 +1,87 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package testutils contains testing helpers for xDS and LRS clients. +package testutils + +import "context" + +// Channel wraps a generic channel and provides a timed receive operation. +type Channel struct { + // C is the underlying channel on which values sent using the SendXxx() + // methods are delivered. Tests which cannot use ReceiveXxx() for whatever + // reasons can use C to read the values. + C chan any +} + +// Send sends value on the underlying channel. +func (c *Channel) Send(value any) { + c.C <- value +} + +// Receive returns the value received on the underlying channel, or the error +// returned by ctx if it is closed or cancelled. +func (c *Channel) Receive(ctx context.Context) (any, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case got := <-c.C: + return got, nil + } +} + +// Replace clears the value on the underlying channel, and sends the new value. +// +// It's expected to be used with a size-1 channel, to only keep the most +// up-to-date item. This method is inherently racy when invoked concurrently +// from multiple goroutines. +func (c *Channel) Replace(value any) { + for { + select { + case c.C <- value: + return + case <-c.C: + } + } +} + +// SendContext sends value on the underlying channel, or returns an error if +// the context expires. +func (c *Channel) SendContext(ctx context.Context, value any) error { + select { + case c.C <- value: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// Drain drains the channel by repeatedly reading from it until it is empty. +func (c *Channel) Drain() { + for { + select { + case <-c.C: + default: + return + } + } +} + +// NewChannelWithSize returns a new Channel with a buffer of bufSize. +func NewChannelWithSize(bufSize int) *Channel { + return &Channel{C: make(chan any, bufSize)} +} diff --git a/internal/xds/clients/internal/testutils/e2e/clientresources.go b/internal/xds/clients/internal/testutils/e2e/clientresources.go new file mode 100644 index 0000000..6724f1a --- /dev/null +++ b/internal/xds/clients/internal/testutils/e2e/clientresources.go @@ -0,0 +1,75 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +import ( + "fmt" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" +) + +const ( + // ClientSideCertProviderInstance is the certificate provider instance name + // used in the Cluster resource on the client side. + ClientSideCertProviderInstance = "client-side-certificate-provider-instance" +) + +// RouterHTTPFilter is the HTTP Filter configuration for the Router filter. +var RouterHTTPFilter = HTTPFilter("router", &v3routerpb.Router{}) + +// DefaultClientListener returns a basic xds Listener resource to be used on +// the client side. +func DefaultClientListener(target, routeName string) *v3listenerpb.Listener { + hcm := marshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: routeName, + }}, + }) + return &v3listenerpb.Listener{ + Name: target, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + } +} + +func marshalAny(m proto.Message) *anypb.Any { + a, err := anypb.New(m) + if err != nil { + panic(fmt.Sprintf("anypb.New(%+v) failed: %v", m, err)) + } + return a +} + +// HTTPFilter constructs an xds HttpFilter with the provided name and config. +func HTTPFilter(name string, config proto.Message) *v3httppb.HttpFilter { + return &v3httppb.HttpFilter{ + Name: name, + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: marshalAny(config), + }, + } +} diff --git a/internal/xds/clients/internal/testutils/e2e/logging.go b/internal/xds/clients/internal/testutils/e2e/logging.go new file mode 100644 index 0000000..ed3a603 --- /dev/null +++ b/internal/xds/clients/internal/testutils/e2e/logging.go @@ -0,0 +1,40 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +// serverLogger implements the Logger interface defined at +// envoyproxy/go-control-plane/pkg/log. This is passed to the Snapshot cache. +type serverLogger struct { + logger interface { + Logf(format string, args ...any) + } +} + +func (l serverLogger) Debugf(format string, args ...any) { + l.logger.Logf(format, args...) +} +func (l serverLogger) Infof(format string, args ...any) { + l.logger.Logf(format, args...) +} +func (l serverLogger) Warnf(format string, args ...any) { + l.logger.Logf(format, args...) +} +func (l serverLogger) Errorf(format string, args ...any) { + l.logger.Logf(format, args...) +} diff --git a/internal/xds/clients/internal/testutils/e2e/server.go b/internal/xds/clients/internal/testutils/e2e/server.go new file mode 100644 index 0000000..3426bbc --- /dev/null +++ b/internal/xds/clients/internal/testutils/e2e/server.go @@ -0,0 +1,257 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package e2e provides utilities for end2end testing of xDS and LRS clients +// functionalities. +package e2e + +import ( + "context" + "fmt" + "net" + "reflect" + "strconv" + "testing" + + "github.com/envoyproxy/go-control-plane/pkg/cache/types" + "google.golang.org/grpc" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverygrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + v3cache "github.com/envoyproxy/go-control-plane/pkg/cache/v3" + v3resource "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + v3server "github.com/envoyproxy/go-control-plane/pkg/server/v3" +) + +// ManagementServer is a thin wrapper around the xDS control plane +// implementation provided by envoyproxy/go-control-plane. +type ManagementServer struct { + // Address is the host:port on which the management server is listening for + // new connections. + Address string + + // LRSServer points to the fake LRS server implementation. Set only if the + // SupportLoadReportingService option was set to true when creating this + // management server. + LRSServer *fakeserver.Server + + cancel context.CancelFunc // To stop the v3 ADS service. + xs v3server.Server // v3 implementation of ADS. + gs *grpc.Server // gRPC server which exports the ADS service. + cache v3cache.SnapshotCache // Resource snapshot. + version int // Version of resource snapshot. + + // A logging interface, usually supplied from *testing.T. + logger interface { + Logf(format string, args ...any) + } +} + +// ManagementServerOptions contains options to be passed to the management +// server during creation. +type ManagementServerOptions struct { + // Listener to accept connections on. If nil, a TPC listener on a local port + // will be created and used. + Listener net.Listener + + // SupportLoadReportingService, if set, results in the load reporting + // service being registered on the same port as that of ADS. + SupportLoadReportingService bool + + // AllowResourceSubSet allows the management server to respond to requests + // before all configured resources are explicitly named in the request. The + // default behavior that we want is for the management server to wait for + // all configured resources to be requested before responding to any of + // them, since this is how we have run our tests historically, and should be + // set to true only for tests which explicitly require the other behavior. + AllowResourceSubset bool + + // ServerFeaturesIgnoreResourceDeletion, if set, results in a bootstrap config + // where the server features list contains `ignore_resource_deletion`. This + // results in gRPC ignoring resource deletions from the management server, as + // per A53. + ServerFeaturesIgnoreResourceDeletion bool + + // The callbacks defined below correspond to the state of the world (sotw) + // version of the xDS API on the management server. + + // OnStreamOpen is called when an xDS stream is opened. The callback is + // invoked with the assigned stream ID and the type URL from the incoming + // request (or "" for ADS). + // + // Returning an error from this callback will end processing and close the + // stream. OnStreamClosed will still be called. + OnStreamOpen func(context.Context, int64, string) error + + // OnStreamClosed is called immediately prior to closing an xDS stream. The + // callback is invoked with the stream ID of the stream being closed. + OnStreamClosed func(int64, *v3corepb.Node) + + // OnStreamRequest is called when a request is received on the stream. The + // callback is invoked with the stream ID of the stream on which the request + // was received and the received request. + // + // Returning an error from this callback will end processing and close the + // stream. OnStreamClosed will still be called. + OnStreamRequest func(int64, *v3discoverypb.DiscoveryRequest) error + + // OnStreamResponse is called immediately prior to sending a response on the + // stream. The callback is invoked with the stream ID of the stream on which + // the response is being sent along with the incoming request and the outgoing + // response. + OnStreamResponse func(context.Context, int64, *v3discoverypb.DiscoveryRequest, *v3discoverypb.DiscoveryResponse) +} + +// StartManagementServer initializes a management server which implements the +// AggregatedDiscoveryService endpoint. The management server is initialized +// with no resources. Tests should call the Update() method to change the +// resource snapshot held by the management server, as per by the test logic. +// +// Registers a cleanup function on t to stop the management server. +func StartManagementServer(t *testing.T, opts ManagementServerOptions) *ManagementServer { + t.Helper() + + // Create a snapshot cache. The first parameter to NewSnapshotCache() + // controls whether the server should wait for all resources to be + // explicitly named in the request before responding to any of them. + wait := !opts.AllowResourceSubset + cache := v3cache.NewSnapshotCache(wait, v3cache.IDHash{}, serverLogger{t}) + t.Logf("Created new snapshot cache...") + + lis := opts.Listener + if lis == nil { + var err error + lis, err = net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen on localhost:0: %v", err) + } + } + + // Cancelling the context passed to the server is the only way of stopping it + // at the end of the test. + ctx, cancel := context.WithCancel(context.Background()) + callbacks := v3server.CallbackFuncs{ + StreamOpenFunc: opts.OnStreamOpen, + StreamClosedFunc: opts.OnStreamClosed, + StreamRequestFunc: opts.OnStreamRequest, + StreamResponseFunc: opts.OnStreamResponse, + } + + // Create an xDS management server and register the ADS implementation + // provided by it on a gRPC server. + xs := v3server.NewServer(ctx, cache, callbacks) + gs := grpc.NewServer() + v3discoverygrpc.RegisterAggregatedDiscoveryServiceServer(gs, xs) + t.Logf("Registered Aggregated Discovery Service (ADS)...") + + mgmtServer := &ManagementServer{ + Address: lis.Addr().String(), + cancel: cancel, + version: 0, + gs: gs, + xs: xs, + cache: cache, + logger: t, + } + if opts.SupportLoadReportingService { + lrs := fakeserver.NewServer(lis.Addr().String()) + v3lrsgrpc.RegisterLoadReportingServiceServer(gs, lrs) + mgmtServer.LRSServer = lrs + t.Logf("Registered Load Reporting Service (LRS)...") + } + + // Start serving. + go gs.Serve(lis) + t.Logf("xDS management server serving at: %v...", lis.Addr().String()) + t.Cleanup(mgmtServer.Stop) + return mgmtServer +} + +// UpdateOptions wraps parameters to be passed to the Update() method. +type UpdateOptions struct { + // NodeID is the id of the client to which this update is to be pushed. + NodeID string + // Endpoints, Clusters, Routes, and Listeners are the updated list of xds + // resources for the server. All must be provided with each Update. + Endpoints []*v3endpointpb.ClusterLoadAssignment + Clusters []*v3clusterpb.Cluster + Routes []*v3routepb.RouteConfiguration + Listeners []*v3listenerpb.Listener + // SkipValidation indicates whether we want to skip validation (by not + // calling snapshot.Consistent()). It can be useful for negative tests, + // where we send updates that the client will NACK. + SkipValidation bool +} + +// Update changes the resource snapshot held by the management server, which +// updates connected clients as required. +func (s *ManagementServer) Update(ctx context.Context, opts UpdateOptions) error { + s.version++ + + // Create a snapshot with the passed in resources. + resources := map[v3resource.Type][]types.Resource{ + v3resource.ListenerType: resourceSlice(opts.Listeners), + v3resource.RouteType: resourceSlice(opts.Routes), + v3resource.ClusterType: resourceSlice(opts.Clusters), + v3resource.EndpointType: resourceSlice(opts.Endpoints), + } + snapshot, err := v3cache.NewSnapshot(strconv.Itoa(s.version), resources) + if err != nil { + return fmt.Errorf("failed to create new snapshot cache: %v", err) + + } + if !opts.SkipValidation { + if err := snapshot.Consistent(); err != nil { + return fmt.Errorf("failed to create new resource snapshot: %v", err) + } + } + s.logger.Logf("Created new resource snapshot...") + + // Update the cache with the new resource snapshot. + if err := s.cache.SetSnapshot(ctx, opts.NodeID, snapshot); err != nil { + return fmt.Errorf("failed to update resource snapshot in management server: %v", err) + } + s.logger.Logf("Updated snapshot cache with resource snapshot...") + return nil +} + +// Stop stops the management server. +func (s *ManagementServer) Stop() { + if s.cancel != nil { + s.cancel() + } + s.gs.Stop() +} + +// resourceSlice accepts a slice of any type of proto messages and returns a +// slice of types.Resource. Will panic if there is an input type mismatch. +func resourceSlice(i any) []types.Resource { + v := reflect.ValueOf(i) + rs := make([]types.Resource, v.Len()) + for i := 0; i < v.Len(); i++ { + rs[i] = v.Index(i).Interface().(types.Resource) + } + return rs +} diff --git a/internal/xds/clients/internal/testutils/fakeserver/server.go b/internal/xds/clients/internal/testutils/fakeserver/server.go new file mode 100644 index 0000000..8adb2e8 --- /dev/null +++ b/internal/xds/clients/internal/testutils/fakeserver/server.go @@ -0,0 +1,254 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package fakeserver provides a fake implementation of the management server. +// +// This package is recommended only for scenarios which cannot be tested using +// the xDS management server (which uses envoy-go-control-plane) provided by the +// `internal/testutils/e2e` package. +package fakeserver + +import ( + "fmt" + "io" + "net" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/xds/clients/internal/testutils" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + v3discoverygrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" +) + +const ( + // TODO: Make this a var or a field in the server if there is a need to use a + // value other than this default. + defaultChannelBufferSize = 50 + defaultDialTimeout = 5 * time.Second +) + +// Request wraps the request protobuf (xds/LRS) and error received by the +// Server in a call to stream.Recv(). +type Request struct { + Req proto.Message + Err error +} + +// Response wraps the response protobuf (xds/LRS) and error that the Server +// should send out to the client through a call to stream.Send() +type Response struct { + Resp proto.Message + Err error +} + +// Server is a fake implementation of xDS and LRS protocols. It listens on the +// same port for both services and exposes a bunch of channels to send/receive +// messages. +// +// This server is recommended only for scenarios which cannot be tested using +// the xDS management server (which uses envoy-go-control-plane) provided by the +// `internal/testutils/xds/e2e` package. +type Server struct { + // XDSRequestChan is a channel on which received xDS requests are made + // available to the users of this Server. + XDSRequestChan *testutils.Channel + // XDSResponseChan is a channel on which the Server accepts xDS responses + // to be sent to the client. + XDSResponseChan chan *Response + // LRSRequestChan is a channel on which received LRS requests are made + // available to the users of this Server. + LRSRequestChan *testutils.Channel + // LRSResponseChan is a channel on which the Server accepts the LRS + // response to be sent to the client. + LRSResponseChan chan *Response + // LRSStreamOpenChan is a channel on which the Server sends notifications + // when a new LRS stream is created. + LRSStreamOpenChan *testutils.Channel + // LRSStreamCloseChan is a channel on which the Server sends notifications + // when an existing LRS stream is closed. + LRSStreamCloseChan *testutils.Channel + // NewConnChan is a channel on which the fake server notifies receipt of new + // connection attempts. Tests can gate on this event before proceeding to + // other actions which depend on a connection to the fake server being up. + NewConnChan *testutils.Channel + // Address is the host:port on which the Server is listening for requests. + Address string + + // The underlying fake implementation of xDS and LRS. + *xdsServer + *lrsServer +} + +type wrappedListener struct { + net.Listener + server *Server +} + +func (wl *wrappedListener) Accept() (net.Conn, error) { + c, err := wl.Listener.Accept() + if err != nil { + return nil, err + } + wl.server.NewConnChan.Send(struct{}{}) + return c, err +} + +// StartServer makes a new Server and gets it to start listening on the given +// net.Listener. If the given net.Listener is nil, a new one is created on a +// local port for gRPC requests. The returned cancel function should be invoked +// by the caller upon completion of the test. +func StartServer(lis net.Listener) (*Server, func(), error) { + if lis == nil { + var err error + lis, err = net.Listen("tcp", "localhost:0") + if err != nil { + return nil, func() {}, fmt.Errorf("net.Listen() failed: %v", err) + } + } + + s := NewServer(lis.Addr().String()) + wp := &wrappedListener{ + Listener: lis, + server: s, + } + + server := grpc.NewServer() + v3lrsgrpc.RegisterLoadReportingServiceServer(server, s) + v3discoverygrpc.RegisterAggregatedDiscoveryServiceServer(server, s) + go server.Serve(wp) + + return s, func() { server.Stop() }, nil +} + +// NewServer returns a new instance of Server, set to accept requests on addr. +// It is the responsibility of the caller to register the exported ADS and LRS +// services on an appropriate gRPC server. Most usages should prefer +// StartServer() instead of this. +func NewServer(addr string) *Server { + s := &Server{ + XDSRequestChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + LRSRequestChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + NewConnChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + XDSResponseChan: make(chan *Response, defaultChannelBufferSize), + LRSResponseChan: make(chan *Response, 1), // The server only ever sends one response. + LRSStreamOpenChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + LRSStreamCloseChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + Address: addr, + } + s.xdsServer = &xdsServer{reqChan: s.XDSRequestChan, respChan: s.XDSResponseChan} + s.lrsServer = &lrsServer{reqChan: s.LRSRequestChan, respChan: s.LRSResponseChan, streamOpenChan: s.LRSStreamOpenChan, streamCloseChan: s.LRSStreamCloseChan} + return s +} + +type xdsServer struct { + reqChan *testutils.Channel + respChan chan *Response +} + +func (xdsS *xdsServer) StreamAggregatedResources(s v3discoverygrpc.AggregatedDiscoveryService_StreamAggregatedResourcesServer) error { + errCh := make(chan error, 2) + go func() { + for { + req, err := s.Recv() + if err != nil { + errCh <- err + return + } + xdsS.reqChan.Send(&Request{req, err}) + } + }() + go func() { + var retErr error + defer func() { + errCh <- retErr + }() + + for { + select { + case r := <-xdsS.respChan: + if r.Err != nil { + retErr = r.Err + return + } + if err := s.Send(r.Resp.(*v3discoverypb.DiscoveryResponse)); err != nil { + retErr = err + return + } + case <-s.Context().Done(): + retErr = s.Context().Err() + return + } + } + }() + + if err := <-errCh; err != nil { + return err + } + return nil +} + +func (xdsS *xdsServer) DeltaAggregatedResources(v3discoverygrpc.AggregatedDiscoveryService_DeltaAggregatedResourcesServer) error { + return status.Error(codes.Unimplemented, "") +} + +type lrsServer struct { + reqChan *testutils.Channel + respChan chan *Response + streamOpenChan *testutils.Channel + streamCloseChan *testutils.Channel +} + +func (lrsS *lrsServer) StreamLoadStats(s v3lrsgrpc.LoadReportingService_StreamLoadStatsServer) error { + lrsS.streamOpenChan.Send(nil) + defer lrsS.streamCloseChan.Send(nil) + + req, err := s.Recv() + lrsS.reqChan.Send(&Request{req, err}) + if err != nil { + return err + } + + select { + case r := <-lrsS.respChan: + if r.Err != nil { + return r.Err + } + if err := s.Send(r.Resp.(*v3lrspb.LoadStatsResponse)); err != nil { + return err + } + case <-s.Context().Done(): + return s.Context().Err() + } + + for { + req, err := s.Recv() + lrsS.reqChan.Send(&Request{req, err}) + if err != nil { + if err == io.EOF { + return nil + } + return err + } + } +} diff --git a/internal/xds/clients/internal/testutils/faketransport/xds_fake_transport.go b/internal/xds/clients/internal/testutils/faketransport/xds_fake_transport.go new file mode 100644 index 0000000..30c4757 --- /dev/null +++ b/internal/xds/clients/internal/testutils/faketransport/xds_fake_transport.go @@ -0,0 +1,252 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package faketransport provides a fake implementation of the xDS client's +// transport layer. It implements the clients.TransportBuilder, +// clients.Transport and clients.Stream interfaces for testing purposes. +package faketransport + +import ( + "context" + "fmt" + "sync" + "time" + + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/protobuf/proto" + + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +// This compile time checks ensures that the Builder, transport and stream +// implementations satisfy the required interfaces. +var _ clients.TransportBuilder = &Builder{} +var _ clients.Transport = &transport{} +var _ clients.Stream = &stream{} + +// Builder implements clients.TransportBuilder. +type Builder struct { + mu sync.Mutex + activeTransports map[string]*transport // Tracks created transports for the fuzzer to interact with. + activeTransportsChan map[string]chan struct{} // Notifies when transport and stream are ready +} + +// NewBuilder creates a new Builder. +func NewBuilder() *Builder { + return &Builder{ + activeTransports: make(map[string]*transport), + activeTransportsChan: make(map[string]chan struct{}), + } +} + +// Build creates a new Transport for the given server identifier. +func (b *Builder) Build(serverIdentifier clients.ServerIdentifier) (clients.Transport, error) { + b.mu.Lock() + defer b.mu.Unlock() + + if at, ok := b.activeTransports[serverIdentifier.ServerURI]; ok { + return at, nil + } + + streamReadyCh, ok := b.activeTransportsChan[serverIdentifier.ServerURI] + if !ok { + streamReadyCh = make(chan struct{}) + b.activeTransportsChan[serverIdentifier.ServerURI] = streamReadyCh + } + + ft := newTransport(streamReadyCh) + b.activeTransports[serverIdentifier.ServerURI] = ft + return ft, nil +} + +// Close closes the transport for the given server identifier. +func (b *Builder) Close(serverURI string) { + b.mu.Lock() + t, ok := b.activeTransports[serverURI] + b.mu.Unlock() + if ok { + t.Close() + } +} + +// Transport returns the active transport for a given server URI. +func (b *Builder) Transport(ctx context.Context, serverURI string) (*ServerHandle, error) { + b.mu.Lock() + if t, ok := b.activeTransports[serverURI]; ok && t.serverHandle() != nil { + b.mu.Unlock() + return t.serverHandle(), nil + } + + ch, ok := b.activeTransportsChan[serverURI] + if !ok { + ch = make(chan struct{}) + b.activeTransportsChan[serverURI] = ch + } + b.mu.Unlock() + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-ch: + b.mu.Lock() + defer b.mu.Unlock() + return b.activeTransports[serverURI].serverHandle(), nil + } +} + +// transport implements clients.Transport. +type transport struct { + mu sync.Mutex + activeADSStream *stream + closed bool + streamReady func() +} + +func newTransport(streamReady chan struct{}) *transport { + return &transport{ + streamReady: sync.OnceFunc(func() { + close(streamReady) + }), + } +} + +// serverHandle returns a serverhandle for testing. +func (t *transport) serverHandle() *ServerHandle { + t.mu.Lock() + defer t.mu.Unlock() + if t.activeADSStream == nil { + return nil + } + return &ServerHandle{fs: t.activeADSStream} +} + +// NewStream creates a new stream to the server. +func (t *transport) NewStream(ctx context.Context, _ string) (clients.Stream, error) { + t.mu.Lock() + defer t.mu.Unlock() + + if t.closed { + return nil, fmt.Errorf("transport is closed") + } + + fs := newStream(ctx) + t.activeADSStream = fs + t.streamReady() + return fs, nil +} + +// Close closes the stream. +func (t *transport) Close() { + t.mu.Lock() + t.closed = true + stream := t.activeADSStream + t.mu.Unlock() + if stream != nil { + stream.close() + } +} + +// stream implements clients.Stream. +type stream struct { + ctx context.Context + cancel context.CancelFunc + + reqChan chan []byte + respChan chan []byte +} + +func newStream(ctx context.Context) *stream { + c, cancel := context.WithCancel(ctx) + return &stream{ + ctx: c, + cancel: cancel, + reqChan: make(chan []byte), + respChan: make(chan []byte), + } +} + +// Send sends the provided message on the stream. It puts the request into the +// reqChan for consumption. +func (s *stream) Send(data []byte) error { + // A short timeout prevents the client from deadlocking if there is no + // receiver on the server-side to consume the request (e.g., if the xDS + // client sends an ACK after creating a stream but the server handle is + // not actively receiving). + ctx, cancel := context.WithTimeout(s.ctx, 10*time.Millisecond) + defer cancel() + select { + case <-ctx.Done(): + return ctx.Err() + case s.reqChan <- data: + return nil + } +} + +// Recv blocks until the next message is received on the stream. It blocks until +// a response is available in the respChan or the context is canceled. +func (s *stream) Recv() ([]byte, error) { + select { + case data := <-s.respChan: + return data, nil + case <-s.ctx.Done(): + return nil, s.ctx.Err() + } +} + +// Close closes the stream. +func (s *stream) close() { + s.cancel() +} + +// ServerHandle provides the server-side send/recv methods to interact with +// the stream. +type ServerHandle struct { + fs *stream +} + +// Recv reads the next request from the reqChan. It blocks until a +// request is available or the context expires. It returns an error if the +// context expires or if the request cannot be unmarshaled. +func (h *ServerHandle) Recv(ctx context.Context) (*v3discoverypb.DiscoveryRequest, error) { + select { + case data := <-h.fs.reqChan: + req := &v3discoverypb.DiscoveryRequest{} + if err := proto.Unmarshal(data, req); err != nil { + return nil, fmt.Errorf("failed to unmarshal request: %v", err) + } + return req, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// Send simulates a server response. It marshals the provided +// DiscoveryResponse, puts it in the respChan to notify that a response +// is available for the client to Recv. +func (h *ServerHandle) Send(ctx context.Context, resp *v3discoverypb.DiscoveryResponse) error { + data, err := proto.Marshal(resp) + if err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case h.fs.respChan <- data: + return nil + } +} diff --git a/internal/xds/clients/internal/testutils/marshal_any.go b/internal/xds/clients/internal/testutils/marshal_any.go new file mode 100644 index 0000000..12ce832 --- /dev/null +++ b/internal/xds/clients/internal/testutils/marshal_any.go @@ -0,0 +1,37 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package testutils + +import ( + "testing" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +// MarshalAny is a convenience function to marshal protobuf messages into any +// protos. function will fail the test with a fatal error if the marshaling fails. +func MarshalAny(t *testing.T, m proto.Message) *anypb.Any { + t.Helper() + + a, err := anypb.New(m) + if err != nil { + t.Fatalf("Failed to marshal proto %+v into an Any: %v", m, err) + } + return a +} diff --git a/internal/xds/clients/internal/testutils/restartable_listener.go b/internal/xds/clients/internal/testutils/restartable_listener.go new file mode 100644 index 0000000..b6c31a1 --- /dev/null +++ b/internal/xds/clients/internal/testutils/restartable_listener.go @@ -0,0 +1,106 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "net" + "sync" + + "google.golang.org/grpc/grpclog" +) + +var logger = grpclog.Component("testutils") + +type tempError struct{} + +func (*tempError) Error() string { + return "restartable listener temporary error" +} +func (*tempError) Temporary() bool { + return true +} + +// RestartableListener wraps a net.Listener and supports stopping and restarting +// the latter. +type RestartableListener struct { + lis net.Listener + + mu sync.Mutex + stopped bool + conns []net.Conn +} + +// NewRestartableListener returns a new RestartableListener wrapping l. +func NewRestartableListener(l net.Listener) *RestartableListener { + return &RestartableListener{lis: l} +} + +// Accept waits for and returns the next connection to the listener. +// +// If the listener is currently not accepting new connections, because `Stop` +// was called on it, the connection is immediately closed after accepting +// without any bytes being sent on it. +func (l *RestartableListener) Accept() (net.Conn, error) { + conn, err := l.lis.Accept() + if err != nil { + return nil, err + } + + l.mu.Lock() + defer l.mu.Unlock() + if l.stopped { + conn.Close() + return nil, &tempError{} + } + l.conns = append(l.conns, conn) + return conn, nil +} + +// Close closes the listener. +func (l *RestartableListener) Close() error { + return l.lis.Close() +} + +// Addr returns the listener's network address. +func (l *RestartableListener) Addr() net.Addr { + return l.lis.Addr() +} + +// Stop closes existing connections on the listener and prevents new connections +// from being accepted. +func (l *RestartableListener) Stop() { + logger.Infof("Stopping restartable listener %q", l.Addr()) + + l.mu.Lock() + l.stopped = true + for _, conn := range l.conns { + conn.Close() + } + l.conns = nil + l.mu.Unlock() +} + +// Restart gets a previously stopped listener to start accepting connections. +func (l *RestartableListener) Restart() { + logger.Infof("Restarting listener %q", l.Addr()) + + l.mu.Lock() + l.stopped = false + l.mu.Unlock() +} diff --git a/internal/xds/clients/internal/testutils/wrappers.go b/internal/xds/clients/internal/testutils/wrappers.go new file mode 100644 index 0000000..c7d5690 --- /dev/null +++ b/internal/xds/clients/internal/testutils/wrappers.go @@ -0,0 +1,74 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "net" + "testing" +) + +// ConnWrapper wraps a net.Conn and pushes on a channel when closed. +type ConnWrapper struct { + net.Conn + CloseCh *Channel +} + +// Close closes the connection and sends a value on the close channel. +func (cw *ConnWrapper) Close() error { + err := cw.Conn.Close() + cw.CloseCh.Replace(nil) + return err +} + +// ListenerWrapper wraps a net.Listener and the returned net.Conn. +// +// It pushes on a channel whenever it accepts a new connection. +type ListenerWrapper struct { + net.Listener + NewConnCh *Channel +} + +// Accept wraps the Listener Accept and sends the accepted connection on a +// channel. +func (l *ListenerWrapper) Accept() (net.Conn, error) { + c, err := l.Listener.Accept() + if err != nil { + return nil, err + } + closeCh := NewChannelWithSize(1) + conn := &ConnWrapper{Conn: c, CloseCh: closeCh} + l.NewConnCh.Replace(conn) + return conn, nil +} + +// NewListenerWrapper returns a ListenerWrapper. +func NewListenerWrapper(t *testing.T, lis net.Listener) *ListenerWrapper { + if lis == nil { + var err error + lis, err = net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + } + + return &ListenerWrapper{ + Listener: lis, + NewConnCh: NewChannelWithSize(1), + } +} diff --git a/internal/xds/clients/lrsclient/internal/internal.go b/internal/xds/clients/lrsclient/internal/internal.go new file mode 100644 index 0000000..86eb14e --- /dev/null +++ b/internal/xds/clients/lrsclient/internal/internal.go @@ -0,0 +1,26 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package internal contains functionality internal to the lrsclient package. +package internal + +import "time" + +var ( + // TimeNow is used to get the current time. It can be overridden in tests. + TimeNow func() time.Time +) diff --git a/internal/xds/clients/lrsclient/load_store.go b/internal/xds/clients/lrsclient/load_store.go new file mode 100644 index 0000000..fcc6247 --- /dev/null +++ b/internal/xds/clients/lrsclient/load_store.go @@ -0,0 +1,438 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package lrsclient + +import ( + "context" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/internal/xds/clients" + lrsclientinternal "google.golang.org/grpc/internal/xds/clients/lrsclient/internal" +) + +// A LoadStore aggregates loads for multiple clusters and services that are +// intended to be reported via LRS. +// +// LoadStore stores loads reported to a single LRS server. Use multiple stores +// for multiple servers. +// +// It is safe for concurrent use. +type LoadStore struct { + // stop is the function to call to Stop the LoadStore reporting. + stop func(ctx context.Context) + + // mu only protects the map (2 layers). The read/write to + // *PerClusterReporter doesn't need to hold the mu. + mu sync.Mutex + // clusters is a map with cluster name as the key. The second layer is a + // map with service name as the key. Each value (PerClusterReporter) + // contains data for a (cluster, service) pair. + // + // Note that new entries are added to this map, but never removed. This is + // potentially a memory leak. But the memory is allocated for each new + // (cluster,service) pair, and the memory allocated is just pointers and + // maps. So this shouldn't get too bad. + clusters map[string]map[string]*PerClusterReporter +} + +func init() { + lrsclientinternal.TimeNow = time.Now +} + +// newLoadStore creates a LoadStore. +func newLoadStore() *LoadStore { + return &LoadStore{ + clusters: make(map[string]map[string]*PerClusterReporter), + } +} + +// Stop signals the LoadStore to stop reporting. +// +// Before closing the underlying LRS stream, this method may block until a +// final load report send attempt completes or the provided context `ctx` +// expires. +// +// The provided context must have a deadline or timeout set to prevent Stop +// from blocking indefinitely if the final send attempt fails to complete. +// +// Calling Stop on an already stopped LoadStore is a no-op. +func (ls *LoadStore) Stop(ctx context.Context) { + ls.stop(ctx) +} + +// ReporterForCluster returns the PerClusterReporter for the given cluster and +// service. +func (ls *LoadStore) ReporterForCluster(clusterName, serviceName string) *PerClusterReporter { + ls.mu.Lock() + defer ls.mu.Unlock() + c, ok := ls.clusters[clusterName] + if !ok { + c = make(map[string]*PerClusterReporter) + ls.clusters[clusterName] = c + } + + if p, ok := c[serviceName]; ok { + return p + } + p := &PerClusterReporter{ + cluster: clusterName, + service: serviceName, + lastLoadReportAt: lrsclientinternal.TimeNow(), + } + c[serviceName] = p + return p +} + +// stats returns the load data for the given cluster names. Data is returned in +// a slice with no specific order. +// +// If no clusterName is given (an empty slice), all data for all known clusters +// is returned. +// +// If a cluster's loadData is empty (no load to report), it's not appended to +// the returned slice. +func (ls *LoadStore) stats(clusterNames []string) []*loadData { + ls.mu.Lock() + defer ls.mu.Unlock() + + var ret []*loadData + if len(clusterNames) == 0 { + for _, c := range ls.clusters { + ret = appendClusterStats(ret, c) + } + return ret + } + for _, n := range clusterNames { + if c, ok := ls.clusters[n]; ok { + ret = appendClusterStats(ret, c) + } + } + + return ret +} + +// PerClusterReporter records load data pertaining to a single cluster. It +// provides methods to record call starts, finishes, server-reported loads, +// and dropped calls. +// +// It is safe for concurrent use. +// +// TODO(purnesh42h): Use regular maps with mutexes instead of sync.Map here. +// The latter is optimized for two common use cases: (1) when the entry for a +// given key is only ever written once but read many times, as in caches that +// only grow, or (2) when multiple goroutines read, write, and overwrite +// entries for disjoint sets of keys. In these two cases, use of a Map may +// significantly reduce lock contention compared to a Go map paired with a +// separate Mutex or RWMutex. +// Neither of these conditions are met here, and we should transition to a +// regular map with a mutex for better type safety. +type PerClusterReporter struct { + cluster, service string + drops sync.Map // map[string]*uint64 + localityRPCCount sync.Map // map[clients.Locality]*rpcCountData + + mu sync.Mutex + lastLoadReportAt time.Time +} + +// CallStarted records a call started in the LoadStore. +func (p *PerClusterReporter) CallStarted(locality clients.Locality) { + s, ok := p.localityRPCCount.Load(locality) + if !ok { + tp := newRPCCountData() + s, _ = p.localityRPCCount.LoadOrStore(locality, tp) + } + s.(*rpcCountData).incrInProgress() + s.(*rpcCountData).incrIssued() +} + +// CallFinished records a call finished in the LoadStore. +func (p *PerClusterReporter) CallFinished(locality clients.Locality, err error) { + f, ok := p.localityRPCCount.Load(locality) + if !ok { + // The map is never cleared, only values in the map are reset. So the + // case where entry for call-finish is not found should never happen. + return + } + f.(*rpcCountData).decrInProgress() + if err == nil { + f.(*rpcCountData).incrSucceeded() + } else { + f.(*rpcCountData).incrErrored() + } +} + +// CallServerLoad records the server load in the LoadStore. +func (p *PerClusterReporter) CallServerLoad(locality clients.Locality, name string, val float64) { + s, ok := p.localityRPCCount.Load(locality) + if !ok { + // The map is never cleared, only values in the map are reset. So the + // case where entry for callServerLoad is not found should never happen. + return + } + s.(*rpcCountData).addServerLoad(name, val) +} + +// CallDropped records a call dropped in the LoadStore. +func (p *PerClusterReporter) CallDropped(category string) { + d, ok := p.drops.Load(category) + if !ok { + tp := new(uint64) + d, _ = p.drops.LoadOrStore(category, tp) + } + atomic.AddUint64(d.(*uint64), 1) +} + +// stats returns and resets all loads reported to the store, except inProgress +// rpc counts. +// +// It returns nil if the store doesn't contain any (new) data. +func (p *PerClusterReporter) stats() *loadData { + sd := newLoadData(p.cluster, p.service) + p.drops.Range(func(key, val any) bool { + d := atomic.SwapUint64(val.(*uint64), 0) + if d == 0 { + return true + } + sd.totalDrops += d + keyStr := key.(string) + if keyStr != "" { + // Skip drops without category. They are counted in total_drops, but + // not in per category. One example is drops by circuit breaking. + sd.drops[keyStr] = d + } + return true + }) + p.localityRPCCount.Range(func(key, val any) bool { + countData := val.(*rpcCountData) + succeeded := countData.loadAndClearSucceeded() + inProgress := countData.loadInProgress() + errored := countData.loadAndClearErrored() + issued := countData.loadAndClearIssued() + if succeeded == 0 && inProgress == 0 && errored == 0 && issued == 0 { + return true + } + + ld := localityData{ + requestStats: requestData{ + succeeded: succeeded, + errored: errored, + inProgress: inProgress, + issued: issued, + }, + loadStats: make(map[string]serverLoadData), + } + countData.serverLoads.Range(func(key, val any) bool { + sum, count := val.(*rpcLoadData).loadAndClear() + if count == 0 { + return true + } + ld.loadStats[key.(string)] = serverLoadData{ + count: count, + sum: sum, + } + return true + }) + sd.localityStats[key.(clients.Locality)] = ld + return true + }) + + p.mu.Lock() + sd.reportInterval = lrsclientinternal.TimeNow().Sub(p.lastLoadReportAt) + p.lastLoadReportAt = lrsclientinternal.TimeNow() + p.mu.Unlock() + + if sd.totalDrops == 0 && len(sd.drops) == 0 && len(sd.localityStats) == 0 { + return nil + } + return sd +} + +// loadData contains all load data reported to the LoadStore since the most recent +// call to stats(). +type loadData struct { + // cluster is the name of the cluster this data is for. + cluster string + // service is the name of the EDS service this data is for. + service string + // totalDrops is the total number of dropped requests. + totalDrops uint64 + // drops is the number of dropped requests per category. + drops map[string]uint64 + // localityStats contains load reports per locality. + localityStats map[clients.Locality]localityData + // reportInternal is the duration since last time load was reported (stats() + // was called). + reportInterval time.Duration +} + +// localityData contains load data for a single locality. +type localityData struct { + // requestStats contains counts of requests made to the locality. + requestStats requestData + // loadStats contains server load data for requests made to the locality, + // indexed by the load type. + loadStats map[string]serverLoadData +} + +// requestData contains request counts. +type requestData struct { + // succeeded is the number of succeeded requests. + succeeded uint64 + // errored is the number of requests which ran into errors. + errored uint64 + // inProgress is the number of requests in flight. + inProgress uint64 + // issued is the total number requests that were sent. + issued uint64 +} + +// serverLoadData contains server load data. +type serverLoadData struct { + // count is the number of load reports. + count uint64 + // sum is the total value of all load reports. + sum float64 +} + +// appendClusterStats gets the Data for all the given clusters, append to ret, +// and return the new slice. +// +// Data is only appended to ret if it's not empty. +func appendClusterStats(ret []*loadData, clusters map[string]*PerClusterReporter) []*loadData { + for _, d := range clusters { + data := d.stats() + if data == nil { + // Skip this data if it doesn't contain any information. + continue + } + ret = append(ret, data) + } + return ret +} + +func newLoadData(cluster, service string) *loadData { + return &loadData{ + cluster: cluster, + service: service, + drops: make(map[string]uint64), + localityStats: make(map[clients.Locality]localityData), + } +} + +type rpcCountData struct { + // Only atomic accesses are allowed for the fields. + succeeded *uint64 + errored *uint64 + inProgress *uint64 + issued *uint64 + + // Map from load desc to load data (sum+count). Loading data from map is + // atomic, but updating data takes a lock, which could cause contention when + // multiple RPCs try to report loads for the same desc. + // + // To fix the contention, shard this map. + serverLoads sync.Map // map[string]*rpcLoadData +} + +func newRPCCountData() *rpcCountData { + return &rpcCountData{ + succeeded: new(uint64), + errored: new(uint64), + inProgress: new(uint64), + issued: new(uint64), + } +} + +func (rcd *rpcCountData) incrSucceeded() { + atomic.AddUint64(rcd.succeeded, 1) +} + +func (rcd *rpcCountData) loadAndClearSucceeded() uint64 { + return atomic.SwapUint64(rcd.succeeded, 0) +} + +func (rcd *rpcCountData) incrErrored() { + atomic.AddUint64(rcd.errored, 1) +} + +func (rcd *rpcCountData) loadAndClearErrored() uint64 { + return atomic.SwapUint64(rcd.errored, 0) +} + +func (rcd *rpcCountData) incrInProgress() { + atomic.AddUint64(rcd.inProgress, 1) +} + +func (rcd *rpcCountData) decrInProgress() { + atomic.AddUint64(rcd.inProgress, ^uint64(0)) // atomic.Add(x, -1) +} + +func (rcd *rpcCountData) loadInProgress() uint64 { + return atomic.LoadUint64(rcd.inProgress) // InProgress count is not clear when reading. +} + +func (rcd *rpcCountData) incrIssued() { + atomic.AddUint64(rcd.issued, 1) +} + +func (rcd *rpcCountData) loadAndClearIssued() uint64 { + return atomic.SwapUint64(rcd.issued, 0) +} + +func (rcd *rpcCountData) addServerLoad(name string, d float64) { + loads, ok := rcd.serverLoads.Load(name) + if !ok { + tl := newRPCLoadData() + loads, _ = rcd.serverLoads.LoadOrStore(name, tl) + } + loads.(*rpcLoadData).add(d) +} + +// rpcLoadData is data for server loads (from trailers or oob). Fields in this +// struct must be updated consistently. +// +// The current solution is to hold a lock, which could cause contention. To fix, +// shard serverLoads map in rpcCountData. +type rpcLoadData struct { + mu sync.Mutex + sum float64 + count uint64 +} + +func newRPCLoadData() *rpcLoadData { + return &rpcLoadData{} +} + +func (rld *rpcLoadData) add(v float64) { + rld.mu.Lock() + rld.sum += v + rld.count++ + rld.mu.Unlock() +} + +func (rld *rpcLoadData) loadAndClear() (s float64, c uint64) { + rld.mu.Lock() + s, rld.sum = rld.sum, 0 + c, rld.count = rld.count, 0 + rld.mu.Unlock() + return s, c +} diff --git a/internal/xds/clients/lrsclient/load_store_test.go b/internal/xds/clients/lrsclient/load_store_test.go new file mode 100644 index 0000000..abe1bc4 --- /dev/null +++ b/internal/xds/clients/lrsclient/load_store_test.go @@ -0,0 +1,525 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package lrsclient + +import ( + "fmt" + "sort" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/xds/clients" + lrsclientinternal "google.golang.org/grpc/internal/xds/clients/lrsclient/internal" +) + +var ( + dropCategories = []string{"drop_for_real", "drop_for_fun"} + localities = []clients.Locality{{Region: "locality-A"}, {Region: "locality-B"}} + errTest = fmt.Errorf("test error") +) + +// rpcData wraps the rpc counts and load data to be pushed to the store. +type rpcData struct { + start, success, failure int + serverData map[string]float64 // Will be reported with successful RPCs. +} + +func verifyLoadStoreData(wantStoreData, gotStoreData []*loadData) error { + if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmp.AllowUnexported(loadData{}, localityData{}, requestData{}, serverLoadData{}), cmpopts.IgnoreFields(loadData{}, "reportInterval"), sortDataSlice); diff != "" { + return fmt.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } + return nil +} + +// TestDrops spawns a bunch of goroutines which report drop data. After the +// goroutines have exited, the test dumps the stats from the Store and makes +// sure they are as expected. +func TestDrops(t *testing.T) { + var ( + drops = map[string]int{ + dropCategories[0]: 30, + dropCategories[1]: 40, + "": 10, + } + wantStoreData = &loadData{ + totalDrops: 80, + drops: map[string]uint64{ + dropCategories[0]: 30, + dropCategories[1]: 40, + }, + } + ) + + ls := PerClusterReporter{} + var wg sync.WaitGroup + for category, count := range drops { + for i := 0; i < count; i++ { + wg.Add(1) + go func(c string) { + ls.CallDropped(c) + wg.Done() + }(category) + } + } + wg.Wait() + + gotStoreData := ls.stats() + if err := verifyLoadStoreData([]*loadData{wantStoreData}, []*loadData{gotStoreData}); err != nil { + t.Error(err) + } +} + +// TestLocalityStats spawns a bunch of goroutines which report rpc and load +// data. After the goroutines have exited, the test dumps the stats from the +// Store and makes sure they are as expected. +func TestLocalityStats(t *testing.T) { + var ( + ld = map[clients.Locality]rpcData{ + localities[0]: { + start: 40, + success: 20, + failure: 10, + serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4}, + }, + localities[1]: { + start: 80, + success: 40, + failure: 20, + serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4}, + }, + } + wantStoreData = &loadData{ + localityStats: map[clients.Locality]localityData{ + localities[0]: { + requestStats: requestData{ + succeeded: 20, + errored: 10, + inProgress: 10, + issued: 40, + }, + loadStats: map[string]serverLoadData{ + "net": {count: 20, sum: 20}, + "disk": {count: 20, sum: 40}, + "cpu": {count: 20, sum: 60}, + "mem": {count: 20, sum: 80}, + }, + }, + localities[1]: { + requestStats: requestData{ + succeeded: 40, + errored: 20, + inProgress: 20, + issued: 80, + }, + loadStats: map[string]serverLoadData{ + "net": {count: 40, sum: 40}, + "disk": {count: 40, sum: 80}, + "cpu": {count: 40, sum: 120}, + "mem": {count: 40, sum: 160}, + }, + }, + }, + } + ) + + ls := PerClusterReporter{} + var wg sync.WaitGroup + for locality, data := range ld { + wg.Add(data.start) + for i := 0; i < data.start; i++ { + go func(l clients.Locality) { + ls.CallStarted(l) + wg.Done() + }(locality) + } + // The calls to callStarted() need to happen before the other calls are + // made. Hence the wait here. + wg.Wait() + + wg.Add(data.success) + for i := 0; i < data.success; i++ { + go func(l clients.Locality, serverData map[string]float64) { + ls.CallFinished(l, nil) + for n, d := range serverData { + ls.CallServerLoad(l, n, d) + } + wg.Done() + }(locality, data.serverData) + } + wg.Add(data.failure) + for i := 0; i < data.failure; i++ { + go func(l clients.Locality) { + ls.CallFinished(l, errTest) + wg.Done() + }(locality) + } + wg.Wait() + } + + gotStoreData := ls.stats() + if err := verifyLoadStoreData([]*loadData{wantStoreData}, []*loadData{gotStoreData}); err != nil { + t.Error(err) + } +} + +func TestResetAfterStats(t *testing.T) { + // Push a bunch of drops, call stats and load stats, and leave inProgress to be non-zero. + // Dump the stats. Verify expected + // Push the same set of loads as before + // Now dump and verify the newly expected ones. + var ( + drops = map[string]int{ + dropCategories[0]: 30, + dropCategories[1]: 40, + } + ld = map[clients.Locality]rpcData{ + localities[0]: { + start: 40, + success: 20, + failure: 10, + serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4}, + }, + localities[1]: { + start: 80, + success: 40, + failure: 20, + serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4}, + }, + } + wantStoreData = &loadData{ + totalDrops: 70, + drops: map[string]uint64{ + dropCategories[0]: 30, + dropCategories[1]: 40, + }, + localityStats: map[clients.Locality]localityData{ + localities[0]: { + requestStats: requestData{ + succeeded: 20, + errored: 10, + inProgress: 10, + issued: 40, + }, + + loadStats: map[string]serverLoadData{ + "net": {count: 20, sum: 20}, + "disk": {count: 20, sum: 40}, + "cpu": {count: 20, sum: 60}, + "mem": {count: 20, sum: 80}, + }, + }, + localities[1]: { + requestStats: requestData{ + succeeded: 40, + errored: 20, + inProgress: 20, + issued: 80, + }, + + loadStats: map[string]serverLoadData{ + "net": {count: 40, sum: 40}, + "disk": {count: 40, sum: 80}, + "cpu": {count: 40, sum: 120}, + "mem": {count: 40, sum: 160}, + }, + }, + }, + } + ) + + reportLoad := func(ls *PerClusterReporter) { + for category, count := range drops { + for i := 0; i < count; i++ { + ls.CallDropped(category) + } + } + for locality, data := range ld { + for i := 0; i < data.start; i++ { + ls.CallStarted(locality) + } + for i := 0; i < data.success; i++ { + ls.CallFinished(locality, nil) + for n, d := range data.serverData { + ls.CallServerLoad(locality, n, d) + } + } + for i := 0; i < data.failure; i++ { + ls.CallFinished(locality, errTest) + } + } + } + + ls := PerClusterReporter{} + reportLoad(&ls) + gotStoreData := ls.stats() + if err := verifyLoadStoreData([]*loadData{wantStoreData}, []*loadData{gotStoreData}); err != nil { + t.Error(err) + } + + // The above call to stats() should have reset all load reports except the + // inProgress rpc count. We are now going to push the same load data into + // the store. So, we should expect to see twice the count for inProgress. + for _, l := range localities { + ls := wantStoreData.localityStats[l] + ls.requestStats.inProgress *= 2 + wantStoreData.localityStats[l] = ls + } + reportLoad(&ls) + gotStoreData = ls.stats() + if err := verifyLoadStoreData([]*loadData{wantStoreData}, []*loadData{gotStoreData}); err != nil { + t.Error(err) + } +} + +var sortDataSlice = cmp.Transformer("SortDataSlice", func(in []*loadData) []*loadData { + out := append([]*loadData(nil), in...) // Copy input to avoid mutating it + sort.Slice(out, + func(i, j int) bool { + if out[i].cluster < out[j].cluster { + return true + } + if out[i].cluster == out[j].cluster { + return out[i].service < out[j].service + } + return false + }, + ) + return out +}) + +// Test all load are returned for the given clusters, and all clusters are +// reported if no cluster is specified. +func TestStoreStats(t *testing.T) { + var ( + testClusters = []string{"c0", "c1", "c2"} + testServices = []string{"s0", "s1"} + testLocality = clients.Locality{Region: "test-locality"} + ) + + store := newLoadStore() + for _, c := range testClusters { + for _, s := range testServices { + store.ReporterForCluster(c, s).CallStarted(testLocality) + store.ReporterForCluster(c, s).CallServerLoad(testLocality, "abc", 123) + store.ReporterForCluster(c, s).CallDropped("dropped") + store.ReporterForCluster(c, s).CallFinished(testLocality, nil) + } + } + + wantC0 := []*loadData{ + { + cluster: "c0", service: "s0", + totalDrops: 1, drops: map[string]uint64{"dropped": 1}, + localityStats: map[clients.Locality]localityData{ + testLocality: { + requestStats: requestData{succeeded: 1, issued: 1}, + loadStats: map[string]serverLoadData{"abc": {count: 1, sum: 123}}, + }, + }, + }, + { + cluster: "c0", service: "s1", + totalDrops: 1, drops: map[string]uint64{"dropped": 1}, + localityStats: map[clients.Locality]localityData{ + testLocality: { + requestStats: requestData{succeeded: 1, issued: 1}, + loadStats: map[string]serverLoadData{"abc": {count: 1, sum: 123}}, + }, + }, + }, + } + // Call Stats with just "c0", this should return data for "c0", and not + // touch data for other clusters. + gotC0 := store.stats([]string{"c0"}) + verifyLoadStoreData(wantC0, gotC0) + + wantOther := []*loadData{ + { + cluster: "c1", service: "s0", + totalDrops: 1, drops: map[string]uint64{"dropped": 1}, + localityStats: map[clients.Locality]localityData{ + testLocality: { + requestStats: requestData{succeeded: 1, issued: 1}, + loadStats: map[string]serverLoadData{"abc": {count: 1, sum: 123}}, + }, + }, + }, + { + cluster: "c1", service: "s1", + totalDrops: 1, drops: map[string]uint64{"dropped": 1}, + localityStats: map[clients.Locality]localityData{ + testLocality: { + requestStats: requestData{succeeded: 1, issued: 1}, + loadStats: map[string]serverLoadData{"abc": {count: 1, sum: 123}}, + }, + }, + }, + { + cluster: "c2", service: "s0", + totalDrops: 1, drops: map[string]uint64{"dropped": 1}, + localityStats: map[clients.Locality]localityData{ + testLocality: { + requestStats: requestData{succeeded: 1, issued: 1}, + loadStats: map[string]serverLoadData{"abc": {count: 1, sum: 123}}, + }, + }, + }, + { + cluster: "c2", service: "s1", + totalDrops: 1, drops: map[string]uint64{"dropped": 1}, + localityStats: map[clients.Locality]localityData{ + testLocality: { + requestStats: requestData{succeeded: 1, issued: 1}, + loadStats: map[string]serverLoadData{"abc": {count: 1, sum: 123}}, + }, + }, + }, + } + // Call Stats with empty slice, this should return data for all the + // remaining clusters, and not include c0 (because c0 data was cleared). + gotOther := store.stats(nil) + if err := verifyLoadStoreData(wantOther, gotOther); err != nil { + t.Error(err) + } +} + +// Test the cases that if a cluster doesn't have load to report, its data is not +// appended to the slice returned by Stats(). +func TestStoreStatsEmptyDataNotReported(t *testing.T) { + var ( + testServices = []string{"s0", "s1"} + testLocality = clients.Locality{Region: "test-locality"} + ) + + store := newLoadStore() + // "c0"'s RPCs all finish with success. + for _, s := range testServices { + store.ReporterForCluster("c0", s).CallStarted(testLocality) + store.ReporterForCluster("c0", s).CallFinished(testLocality, nil) + } + // "c1"'s RPCs never finish (always inprocess). + for _, s := range testServices { + store.ReporterForCluster("c1", s).CallStarted(testLocality) + } + + want0 := []*loadData{ + { + cluster: "c0", service: "s0", + localityStats: map[clients.Locality]localityData{ + testLocality: {requestStats: requestData{succeeded: 1, issued: 1}}, + }, + }, + { + cluster: "c0", service: "s1", + localityStats: map[clients.Locality]localityData{ + testLocality: {requestStats: requestData{succeeded: 1, issued: 1}}, + }, + }, + { + cluster: "c1", service: "s0", + localityStats: map[clients.Locality]localityData{ + testLocality: {requestStats: requestData{inProgress: 1, issued: 1}}, + }, + }, + { + cluster: "c1", service: "s1", + localityStats: map[clients.Locality]localityData{ + testLocality: {requestStats: requestData{inProgress: 1, issued: 1}}, + }, + }, + } + // Call Stats with empty slice, this should return data for all the + // clusters. + got0 := store.stats(nil) + if err := verifyLoadStoreData(want0, got0); err != nil { + t.Error(err) + } + + want1 := []*loadData{ + { + cluster: "c1", service: "s0", + localityStats: map[clients.Locality]localityData{ + testLocality: {requestStats: requestData{inProgress: 1}}, + }, + }, + { + cluster: "c1", service: "s1", + localityStats: map[clients.Locality]localityData{ + testLocality: {requestStats: requestData{inProgress: 1}}, + }, + }, + } + // Call Stats with empty slice again, this should return data only for "c1", + // because "c0" data was cleared, but "c1" has in-progress RPCs. + got1 := store.stats(nil) + if err := verifyLoadStoreData(want1, got1); err != nil { + t.Error(err) + } +} + +// TestStoreReportInterval verify that the load report interval gets +// calculated at every stats() call and is the duration between start of last +// load reporting to next stats() call. +func TestStoreReportInterval(t *testing.T) { + originalTimeNow := lrsclientinternal.TimeNow + t.Cleanup(func() { lrsclientinternal.TimeNow = originalTimeNow }) + + // Initial time for reporter creation + currentTime := time.Now() + lrsclientinternal.TimeNow = func() time.Time { + return currentTime + } + + store := newLoadStore() + reporter := store.ReporterForCluster("test-cluster", "test-service") + // Report dummy drop to ensure stats1 is not nil. + reporter.CallDropped("dummy-category") + + // Update currentTime to simulate the passage of time between the reporter + // creation and first stats() call. + currentTime = currentTime.Add(5 * time.Second) + stats1 := reporter.stats() + + if stats1 == nil { + t.Fatalf("stats1 is nil after reporting a drop, want non-nil") + } + // Verify stats() call calculate the report interval from the time of + // reporter creation. + if got, want := stats1.reportInterval, 5*time.Second; got != want { + t.Errorf("stats1.reportInterval = %v, want %v", stats1.reportInterval, want) + } + + // Update currentTime to simulate the passage of time between the first + // and second stats() call. + currentTime = currentTime.Add(10 * time.Second) + // Report another dummy drop to ensure stats2 is not nil. + reporter.CallDropped("dummy-category-2") + stats2 := reporter.stats() + + if stats2 == nil { + t.Fatalf("stats2 is nil after reporting a drop, want non-nil") + } + // Verify stats() call calculate the report interval from the time of first + // stats() call. + if got, want := stats2.reportInterval, 10*time.Second; got != want { + t.Errorf("stats2.reportInterval = %v, want %v", stats2.reportInterval, want) + } +} diff --git a/internal/xds/clients/lrsclient/loadreport_test.go b/internal/xds/clients/lrsclient/loadreport_test.go new file mode 100644 index 0000000..9754cdb --- /dev/null +++ b/internal/xds/clients/lrsclient/loadreport_test.go @@ -0,0 +1,730 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package lrsclient_test + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + grpctestutils "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/internal/testutils" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver" + "google.golang.org/grpc/internal/xds/clients/lrsclient" + lrsclientinternal "google.golang.org/grpc/internal/xds/clients/lrsclient/internal" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/durationpb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + testKey1 = "test-key1" + testKey2 = "test-key2" + defaultTestWatchExpiryTimeout = 100 * time.Millisecond + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) + +var ( + testLocality1 = clients.Locality{Region: "test-region1"} + testLocality2 = clients.Locality{Region: "test-region2"} + toleranceCmpOpt = cmpopts.EquateApprox(0, 1e-5) + ignoreOrderCmpOpt = protocmp.FilterField(&v3endpointpb.ClusterStats{}, "upstream_locality_stats", + cmpopts.SortSlices(func(a, b protocmp.Message) bool { + return a.String() < b.String() + }), + ) +) + +type wrappedListener struct { + net.Listener + newConnChan *testutils.Channel // Connection attempts are pushed here. +} + +func (wl *wrappedListener) Accept() (net.Conn, error) { + c, err := wl.Listener.Accept() + if err != nil { + return nil, err + } + wl.newConnChan.Send(struct{}{}) + return c, err +} + +// Tests a load reporting scenario where the LRS client is reporting loads to +// multiple servers. Verifies the following: +// - calling the load reporting API with different server configuration +// results in connections being created to those corresponding servers +// - the same load.Store is not returned when the load reporting API called +// with different server configurations +// - canceling the load reporting from the client results in the LRS stream +// being canceled on the server +func (s) TestReportLoad_ConnectionCreation(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create two management servers that also serve LRS. + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + newConnChan1 := testutils.NewChannelWithSize(1) + lis1 := &wrappedListener{ + Listener: l, + newConnChan: newConnChan1, + } + mgmtServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis1, + SupportLoadReportingService: true, + }) + l, err = net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + newConnChan2 := testutils.NewChannelWithSize(1) + lis2 := &wrappedListener{ + Listener: l, + newConnChan: newConnChan2, + } + mgmtServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis2, + SupportLoadReportingService: true, + }) + + // Create an LRS client with a configuration that contains both of + // the above two servers. The authority name is immaterial here since load + // reporting is per-server and not per-authority. + nodeID := uuid.New().String() + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + config := lrsclient.Config{ + Node: clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"}, + TransportBuilder: grpctransport.NewBuilder(configs), + } + client, err := lrsclient.New(config) + if err != nil { + t.Fatalf("lrsclient.New() failed: %v", err) + } + + serverIdentifier1 := clients.ServerIdentifier{ServerURI: mgmtServer1.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}} + loadStore1, err := client.ReportLoad(serverIdentifier1) + if err != nil { + t.Fatalf("client.ReportLoad() failed: %v", err) + } + ssCtx, ssCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer ssCancel() + defer loadStore1.Stop(ssCtx) + + // Call the load reporting API to report load to the first management + // server, and ensure that a connection to the server is created. + if _, err := newConnChan1.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for a connection to the first management server, after starting load reporting") + } + if _, err := mgmtServer1.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for LRS stream to be created") + } + + // Call the load reporting API to report load to the first management + // server, and ensure that a connection to the server is created. + serverIdentifier2 := clients.ServerIdentifier{ServerURI: mgmtServer2.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}} + loadStore2, err := client.ReportLoad(serverIdentifier2) + if err != nil { + t.Fatalf("client.ReportLoad() failed: %v", err) + } + if _, err := newConnChan2.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for a connection to the second management server, after starting load reporting") + } + if _, err := mgmtServer2.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for LRS stream to be created") + } + + if loadStore1 == loadStore2 { + t.Fatalf("Got same store for different servers, want different") + } + + // Push some loads on the received store. + loadStore2.ReporterForCluster("cluster", "eds").CallDropped("test") + + // Ensure the initial load reporting request is received at the server. + lrsServer := mgmtServer2.LRSServer + req, err := lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for initial LRS request: %v", err) + } + gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) + nodeProto := &v3corepb.Node{ + Id: nodeID, + UserAgentName: "user-agent", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"}, + } + wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto} + if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff) + } + + // Send a response from the server with a small deadline. + lrsServer.LRSResponseChan <- &fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + SendAllClusters: true, + LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms + }, + } + + // Ensure that loads are seen on the server. + req, err = lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for LRS request with loads: %v", err) + } + gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if l := len(gotLoad); l != 1 { + t.Fatalf("Received load for %d clusters, want 1", l) + } + + // This field is set by the client to indicate the actual time elapsed since + // the last report was sent. We cannot deterministically compare this, and + // we cannot use the cmpopts.IgnoreFields() option on proto structs, since + // we already use the protocmp.Transform() which marshals the struct into + // another message. Hence setting this field to nil is the best option here. + gotLoad[0].LoadReportInterval = nil + wantLoad := &v3endpointpb.ClusterStats{ + ClusterName: "cluster", + ClusterServiceName: "eds", + TotalDroppedRequests: 1, + DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}}, + } + if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" { + t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff) + } + + // Stop this load reporting stream, server should see error canceled. + ssCtx, ssCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer ssCancel() + loadStore2.Stop(ssCtx) + + // Server should receive a stream canceled error. There may be additional + // load reports from the client in the channel. + for { + if ctx.Err() != nil { + t.Fatal("Timeout when waiting for the LRS stream to be canceled on the server") + } + u, err := lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + continue + } + // Ignore load reports sent before the stream was cancelled. + if u.(*fakeserver.Request).Err == nil { + continue + } + if status.Code(u.(*fakeserver.Request).Err) != codes.Canceled { + t.Fatalf("Unexpected LRS request: %v, want error canceled", u) + } + break + } +} + +// Tests a load reporting scenario where the load reporting API is called +// multiple times for the same server. The test verifies the following: +// - calling the load reporting API the second time for the same server +// configuration does not create a new LRS stream +// - the LRS stream is closed *only* after all the API calls invoke their +// cancel functions +// - creating new streams after the previous one was closed works +func (s) TestReportLoad_StreamCreation(t *testing.T) { + grpctestutils.SetEnvConfig(t, &envconfig.XDSORCAToLRSPropEnabled, true) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create a management server that serves LRS. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true}) + + // Create an LRS client with configuration pointing to the above server. + nodeID := uuid.New().String() + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + config := lrsclient.Config{ + Node: clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"}, + TransportBuilder: grpctransport.NewBuilder(configs), + } + client, err := lrsclient.New(config) + if err != nil { + t.Fatalf("lrsclient.New() failed: %v", err) + } + + // Call the load reporting API, and ensure that an LRS stream is created. + serverIdentifier := clients.ServerIdentifier{ServerURI: mgmtServer.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}} + loadStore1, err := client.ReportLoad(serverIdentifier) + if err != nil { + t.Fatalf("client.ReportLoad() failed: %v", err) + } + lrsServer := mgmtServer.LRSServer + if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err) + } + + // Push some loads on the received store. + loadStore1.ReporterForCluster("cluster1", "eds1").CallDropped("test") + loadStore1.ReporterForCluster("cluster1", "eds1").CallStarted(testLocality1) + loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, testKey1, 3.14) + loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, testKey1, 2.718) + loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, "cpu_utilization", 1.5) + loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, "mem_utilization", 2.0) + loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, "application_utilization", 3.0) + loadStore1.ReporterForCluster("cluster1", "eds1").CallFinished(testLocality1, nil) + loadStore1.ReporterForCluster("cluster1", "eds1").CallStarted(testLocality2) + loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality2, testKey2, 1.618) + loadStore1.ReporterForCluster("cluster1", "eds1").CallFinished(testLocality2, nil) + + // Ensure the initial load reporting request is received at the server. + req, err := lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for initial LRS request: %v", err) + } + gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) + nodeProto := &v3corepb.Node{ + Id: nodeID, + UserAgentName: "user-agent", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"}, + } + wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto} + if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff) + } + + // Send a response from the server with a small deadline. + lrsServer.LRSResponseChan <- &fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + SendAllClusters: true, + LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms + }, + } + + // Ensure that loads are seen on the server. + req, err = lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for LRS request with loads") + } + gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if l := len(gotLoad); l != 1 { + t.Fatalf("Received load for %d clusters, want 1", l) + } + + // This field is set by the client to indicate the actual time elapsed since + // the last report was sent. We cannot deterministically compare this, and + // we cannot use the cmpopts.IgnoreFields() option on proto structs, since + // we already use the protocmp.Transform() which marshals the struct into + // another message. Hence setting this field to nil is the best option here. + gotLoad[0].LoadReportInterval = nil + wantLoad := &v3endpointpb.ClusterStats{ + ClusterName: "cluster1", + ClusterServiceName: "eds1", + TotalDroppedRequests: 1, + DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}}, + UpstreamLocalityStats: []*v3endpointpb.UpstreamLocalityStats{ + { + Locality: &v3corepb.Locality{Region: "test-region1"}, + LoadMetricStats: []*v3endpointpb.EndpointLoadMetricStats{ + // TotalMetricValue is the aggregation of 3.14 + 2.718 = 5.858 + {MetricName: testKey1, NumRequestsFinishedWithMetric: 2, TotalMetricValue: 5.858}}, + CpuUtilization: &v3endpointpb.UnnamedEndpointLoadMetricStats{ + NumRequestsFinishedWithMetric: 1, TotalMetricValue: 1.5, + }, + MemUtilization: &v3endpointpb.UnnamedEndpointLoadMetricStats{ + NumRequestsFinishedWithMetric: 1, TotalMetricValue: 2.0, + }, + ApplicationUtilization: &v3endpointpb.UnnamedEndpointLoadMetricStats{ + NumRequestsFinishedWithMetric: 1, TotalMetricValue: 3.0, + }, + TotalSuccessfulRequests: 1, + TotalIssuedRequests: 1, + }, + { + Locality: &v3corepb.Locality{Region: "test-region2"}, + LoadMetricStats: []*v3endpointpb.EndpointLoadMetricStats{ + {MetricName: testKey2, NumRequestsFinishedWithMetric: 1, TotalMetricValue: 1.618}}, + TotalSuccessfulRequests: 1, + TotalIssuedRequests: 1, + }, + }, + } + if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" { + t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff) + } + + // Make another call to the load reporting API, and ensure that a new LRS + // stream is not created. + loadStore2, err := client.ReportLoad(serverIdentifier) + if err != nil { + t.Fatalf("client.ReportLoad() failed: %v", err) + } + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := lrsServer.LRSStreamOpenChan.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("New LRS stream created when expected to use an existing one") + } + + // Push more loads. + loadStore2.ReporterForCluster("cluster2", "eds2").CallDropped("test") + + // Ensure that loads are seen on the server. We need a loop here because + // there could have been some requests from the client in the time between + // us reading the first request and now. Those would have been queued in the + // request channel that we read out of. + for { + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for new loads to be seen on the server") + } + + req, err = lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + continue + } + gotLoad = req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if l := len(gotLoad); l != 1 { + continue + } + gotLoad[0].LoadReportInterval = nil + wantLoad := &v3endpointpb.ClusterStats{ + ClusterName: "cluster2", + ClusterServiceName: "eds2", + TotalDroppedRequests: 1, + DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}}, + } + if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform()); diff != "" { + t.Logf("Unexpected diff in LRS request (-got, +want):\n%s", diff) + continue + } + break + } + + // Cancel the first load reporting call, and ensure that the stream does not + // close (because we have another call open). + ssCtx, ssCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer ssCancel() + loadStore1.Stop(ssCtx) + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := lrsServer.LRSStreamCloseChan.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("LRS stream closed when expected to stay open") + } + + // Stop the second load reporting call, and ensure the stream is closed. + ssCtx, ssCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer ssCancel() + loadStore2.Stop(ssCtx) + if _, err := lrsServer.LRSStreamCloseChan.Receive(ctx); err != nil { + t.Fatal("Timeout waiting for LRS stream to close") + } + + // Calling the load reporting API again should result in the creation of a + // new LRS stream. This ensures that creating and closing multiple streams + // works smoothly. + loadStore3, err := client.ReportLoad(serverIdentifier) + if err != nil { + t.Fatalf("client.ReportLoad() failed: %v", err) + } + if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err) + } + ssCtx, ssCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer ssCancel() + loadStore3.Stop(ssCtx) +} + +// TestReportLoad_StopWithContext tests the behavior of LoadStore.Stop() when +// called with a context. It verifies that: +// - Stop() blocks until the context expires or final load send attempt is +// made. +// - Final load report is seen on the server after stop is called. +// - The stream is closed after Stop() returns. +func (s) TestReportLoad_StopWithContext(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create a management server that serves LRS. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true}) + + // Create an LRS client with configuration pointing to the above server. + nodeID := uuid.New().String() + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + config := lrsclient.Config{ + Node: clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"}, + TransportBuilder: grpctransport.NewBuilder(configs), + } + client, err := lrsclient.New(config) + if err != nil { + t.Fatalf("lrsclient.New() failed: %v", err) + } + + // Call the load reporting API, and ensure that an LRS stream is created. + serverIdentifier := clients.ServerIdentifier{ServerURI: mgmtServer.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}} + loadStore, err := client.ReportLoad(serverIdentifier) + if err != nil { + t.Fatalf("client.ReportLoad() failed: %v", err) + } + lrsServer := mgmtServer.LRSServer + if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err) + } + + // Push some loads on the received store. + loadStore.ReporterForCluster("cluster1", "eds1").CallDropped("test") + + // Ensure the initial load reporting request is received at the server. + req, err := lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for initial LRS request: %v", err) + } + gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) + nodeProto := &v3corepb.Node{ + Id: nodeID, + UserAgentName: "user-agent", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"}, + } + wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto} + if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff) + } + + // Send a response from the server with a small deadline. + lrsServer.LRSResponseChan <- &fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + SendAllClusters: true, + LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms + }, + } + + // Ensure that loads are seen on the server. + req, err = lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for LRS request with loads") + } + gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if l := len(gotLoad); l != 1 { + t.Fatalf("Received load for %d clusters, want 1", l) + } + + // This field is set by the client to indicate the actual time elapsed since + // the last report was sent. We cannot deterministically compare this, and + // we cannot use the cmpopts.IgnoreFields() option on proto structs, since + // we already use the protocmp.Transform() which marshals the struct into + // another message. Hence setting this field to nil is the best option here. + gotLoad[0].LoadReportInterval = nil + wantLoad := &v3endpointpb.ClusterStats{ + ClusterName: "cluster1", + ClusterServiceName: "eds1", + TotalDroppedRequests: 1, + DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}}, + } + if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" { + t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff) + } + + // Create a context for Stop() that remains until the end of test to ensure + // that only possibility of Stop()s to finish is if final load send attempt + // is made. If final load attempt is not made, test will timeout. + stopCtx, stopCancel := context.WithCancel(ctx) + defer stopCancel() + + // Push more loads. + loadStore.ReporterForCluster("cluster2", "eds2").CallDropped("test") + + stopCalled := make(chan struct{}) + // Call Stop in a separate goroutine. It will block until + // final load send attempt is made. + go func() { + loadStore.Stop(stopCtx) + close(stopCalled) + }() + <-stopCalled + + // Ensure that loads are seen on the server. We need a loop here because + // there could have been some requests from the client in the time between + // us reading the first request and now. Those would have been queued in the + // request channel that we read out of. + for { + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for new loads to be seen on the server") + } + + req, err = lrsServer.LRSRequestChan.Receive(ctx) + if err != nil || req.(*fakeserver.Request).Err != nil { + continue + } + if req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) == nil { + // This can happen due to a race: + // 1. Load for "cluster2" is reported just before Stop(). + // 2. The periodic ticker might send this load before Stop()'s + // final send mechanism processes it, clearing the data. + // 3. Stop()'s final send might then send an empty report. + // This is acceptable for this test because we only need to verify + // if the final load report send attempt was made. + t.Logf("Empty final load report sent on server") + break + } + gotLoad = req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if l := len(gotLoad); l != 1 { + continue + } + gotLoad[0].LoadReportInterval = nil + wantLoad := &v3endpointpb.ClusterStats{ + ClusterName: "cluster2", + ClusterServiceName: "eds2", + TotalDroppedRequests: 1, + DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}}, + } + if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform()); diff != "" { + t.Logf("Unexpected diff in LRS request (-got, +want):\n%s", diff) + continue + } + break + } + + // Verify the stream is eventually closed on the server side. + if _, err := lrsServer.LRSStreamCloseChan.Receive(ctx); err != nil { + t.Fatal("Timeout waiting for LRS stream to close") + } +} + +// TestReportLoad_LoadReportInterval tests verify that the load report interval +// received by the LRS server is the duration between start of last load +// reporting by the client and the time when the load is reported to the LRS +// server. +func (s) TestReportLoad_LoadReportInterval(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + originalTimeNow := lrsclientinternal.TimeNow + t.Cleanup(func() { lrsclientinternal.TimeNow = originalTimeNow }) + + // Create a management server that serves LRS. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true}) + + // Create an LRS client with configuration pointing to the above server. + nodeID := uuid.New().String() + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + config := lrsclient.Config{ + Node: clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"}, + TransportBuilder: grpctransport.NewBuilder(configs), + } + client, err := lrsclient.New(config) + if err != nil { + t.Fatalf("lrsclient.New() failed: %v", err) + } + + // Call the load reporting API, and ensure that an LRS stream is created. + serverIdentifier := clients.ServerIdentifier{ServerURI: mgmtServer.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}} + loadStore1, err := client.ReportLoad(serverIdentifier) + if err != nil { + t.Fatalf("client.ReportLoad() failed: %v", err) + } + lrsServer := mgmtServer.LRSServer + if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err) + } + + stateMu := sync.Mutex{} + // Initial time for reporter creation + currentTime := time.Now() + lrsclientinternal.TimeNow = func() time.Time { + stateMu.Lock() + defer stateMu.Unlock() + return currentTime + } + + // Report dummy drop to ensure stats is not nil. + loadStore1.ReporterForCluster("cluster1", "eds1").CallDropped("test") + + // Update currentTime to simulate the passage of time between the reporter + // creation and first stats() call. + stateMu.Lock() + currentTime = currentTime.Add(5 * time.Second) + stateMu.Unlock() + + // Ensure the initial load reporting request is received at the server. + req, err := lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for initial LRS request: %v", err) + } + gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) + nodeProto := &v3corepb.Node{ + Id: nodeID, + UserAgentName: "user-agent", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"}, + } + wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto} + if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff) + } + + // Send a response from the server with a small deadline. + lrsServer.LRSResponseChan <- &fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + SendAllClusters: true, + LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms + }, + } + + // Ensure that loads are seen on the server. + req, err = lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for LRS request with loads") + } + gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if l := len(gotLoad); l != 1 { + t.Fatalf("Received load for %d clusters, want 1", l) + } + // Verify load received at LRS server has load report interval calculated + // from the time of reporter creation. + if got, want := gotLoad[0].GetLoadReportInterval().AsDuration(), 5*time.Second; got != want { + t.Errorf("Got load report interval %v, want %v", got, want) + } + + ssCtx, ssCancel := context.WithTimeout(context.Background(), time.Millisecond) + defer ssCancel() + loadStore1.Stop(ssCtx) +} diff --git a/internal/xds/clients/lrsclient/logging.go b/internal/xds/clients/lrsclient/logging.go new file mode 100644 index 0000000..032b56b --- /dev/null +++ b/internal/xds/clients/lrsclient/logging.go @@ -0,0 +1,36 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package lrsclient + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +var logger = grpclog.Component("xds") + +func prefixLogger(c *LRSClient) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, clientPrefix(c)) +} + +func clientPrefix(c *LRSClient) string { + return fmt.Sprintf("[lrs-client %p] ", c) +} diff --git a/internal/xds/clients/lrsclient/lrs_stream.go b/internal/xds/clients/lrsclient/lrs_stream.go new file mode 100644 index 0000000..9a45bea --- /dev/null +++ b/internal/xds/clients/lrsclient/lrs_stream.go @@ -0,0 +1,334 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package lrsclient + +import ( + "context" + "fmt" + "io" + "time" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/envconfig" + igrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" +) + +// Any per-RPC level logs which print complete request or response messages +// should be gated at this verbosity level. Other per-RPC level logs which print +// terse output should be at `INFO` and verbosity 2. +const perRPCVerbosityLevel = 9 + +// streamImpl provides all the functionality associated with an LRS (Load +// Reporting Service) stream on the client-side. It manages the lifecycle of +// the LRS stream, including starting, stopping, and retrying the stream. It +// also provides a LoadStore that can be used to report load, with a Stop +// function that should be called when the load reporting is no longer +// needed. +type streamImpl struct { + // The following fields are initialized when a stream instance is created + // and are read-only afterwards, and hence can be accessed without a mutex. + transport clients.Transport // Transport to use for LRS stream. + backoff func(int) time.Duration // Backoff for retries, after stream failures. + nodeProto *v3corepb.Node // Identifies the gRPC application. + doneCh chan struct{} // To notify exit of LRS goroutine. + logger *igrpclog.PrefixLogger + + cancelStream context.CancelFunc // Cancel the stream. If nil, the stream is not active. + loadStore *LoadStore // LoadStore returned to user for pushing loads. + + finalSendRequest chan struct{} // To request for the final attempt to send loads. + finalSendDone chan error // To signal completion of the final attempt of sending loads. +} + +// streamOpts holds the options for creating an lrsStream. +type streamOpts struct { + transport clients.Transport // xDS transport to create the stream on. + backoff func(int) time.Duration // Backoff for retries, after stream failures. + nodeProto *v3corepb.Node // Node proto to identify the gRPC application. + logPrefix string // Prefix to be used for log messages. +} + +// newStreamImpl creates a new StreamImpl with the provided options. +// +// The actual streaming RPC call is initiated when the first call to ReportLoad +// is made, and is terminated when the last call to ReportLoad is canceled. +func newStreamImpl(opts streamOpts) *streamImpl { + ctx, cancel := context.WithCancel(context.Background()) + + lrs := &streamImpl{ + transport: opts.transport, + backoff: opts.backoff, + nodeProto: opts.nodeProto, + cancelStream: cancel, + doneCh: make(chan struct{}), + finalSendRequest: make(chan struct{}, 1), + finalSendDone: make(chan error, 1), + } + + l := grpclog.Component("xds") + lrs.logger = igrpclog.NewPrefixLogger(l, opts.logPrefix+fmt.Sprintf("[lrs-stream %p] ", lrs)) + lrs.loadStore = newLoadStore() + go lrs.runner(ctx) + return lrs +} + +// runner is responsible for managing the lifetime of an LRS streaming call. It +// creates the stream, sends the initial LoadStatsRequest, receives the first +// LoadStatsResponse, and then starts a goroutine to periodically send +// LoadStatsRequests. The runner will restart the stream if it encounters any +// errors. +func (lrs *streamImpl) runner(ctx context.Context) { + defer close(lrs.doneCh) + + // This feature indicates that the client supports the + // LoadStatsResponse.send_all_clusters field in the LRS response. + node := proto.Clone(lrs.nodeProto).(*v3corepb.Node) + node.ClientFeatures = append(node.ClientFeatures, "envoy.lrs.supports_send_all_clusters") + + runLoadReportStream := func() error { + // streamCtx is created and canceled in case we terminate the stream + // early for any reason, to avoid gRPC-Go leaking the RPC's monitoring + // goroutine. + streamCtx, cancel := context.WithCancel(ctx) + defer cancel() + + stream, err := lrs.transport.NewStream(streamCtx, "/envoy.service.load_stats.v3.LoadReportingService/StreamLoadStats") + if err != nil { + lrs.logger.Warningf("Failed to create new LRS streaming RPC: %v", err) + return nil + } + if lrs.logger.V(2) { + lrs.logger.Infof("LRS stream created") + } + + if err := lrs.sendFirstLoadStatsRequest(stream, node); err != nil { + lrs.logger.Warningf("Sending first LRS request failed: %v", err) + return nil + } + + clusters, interval, err := lrs.recvFirstLoadStatsResponse(stream) + if err != nil { + lrs.logger.Warningf("Reading from LRS streaming RPC failed: %v", err) + return nil + } + + // We reset backoff state when we successfully receive at least one + // message from the server. + lrs.sendLoads(streamCtx, stream, clusters, interval) + return backoff.ErrResetBackoff + } + backoff.RunF(ctx, runLoadReportStream, lrs.backoff) +} + +// sendLoads is responsible for periodically sending load reports to the LRS +// server at the specified interval for the specified clusters, until the passed +// in context is canceled. +func (lrs *streamImpl) sendLoads(ctx context.Context, stream clients.Stream, clusterNames []string, interval time.Duration) { + tick := time.NewTicker(interval) + defer tick.Stop() + for { + select { + case <-tick.C: + case <-ctx.Done(): + return + case <-lrs.finalSendRequest: + var finalSendErr error + if lrs.logger.V(2) { + lrs.logger.Infof("Final send request received. Attempting final LRS report.") + } + if err := lrs.sendLoadStatsRequest(stream, lrs.loadStore.stats(clusterNames)); err != nil { + lrs.logger.Warningf("Failed to send final load report. Writing to LRS stream failed: %v", err) + finalSendErr = err + } + if lrs.logger.V(2) { + lrs.logger.Infof("Successfully sent final load report.") + } + lrs.finalSendDone <- finalSendErr + return + } + + if err := lrs.sendLoadStatsRequest(stream, lrs.loadStore.stats(clusterNames)); err != nil { + lrs.logger.Warningf("Failed to send periodic load report. Writing to LRS stream failed: %v", err) + return + } + } +} + +func (lrs *streamImpl) sendFirstLoadStatsRequest(stream clients.Stream, node *v3corepb.Node) error { + req := &v3lrspb.LoadStatsRequest{Node: node} + if lrs.logger.V(perRPCVerbosityLevel) { + lrs.logger.Infof("Sending initial LoadStatsRequest: %s", pretty.ToJSON(req)) + } + msg, err := proto.Marshal(req) + if err != nil { + lrs.logger.Warningf("Failed to marshal LoadStatsRequest: %v", err) + return err + } + err = stream.Send(msg) + if err == io.EOF { + return getStreamError(stream) + } + return err +} + +// recvFirstLoadStatsResponse receives the first LoadStatsResponse from the LRS +// server. Returns the following: +// - a list of cluster names requested by the server or an empty slice if the +// server requested for load from all clusters +// - the load reporting interval, and +// - any error encountered +func (lrs *streamImpl) recvFirstLoadStatsResponse(stream clients.Stream) ([]string, time.Duration, error) { + r, err := stream.Recv() + if err != nil { + return nil, 0, fmt.Errorf("lrs: failed to receive first LoadStatsResponse: %v", err) + } + var resp v3lrspb.LoadStatsResponse + if err := proto.Unmarshal(r, &resp); err != nil { + if lrs.logger.V(2) { + lrs.logger.Infof("Failed to unmarshal response to LoadStatsResponse: %v", err) + } + return nil, time.Duration(0), fmt.Errorf("lrs: unexpected message type %T", r) + } + if lrs.logger.V(perRPCVerbosityLevel) { + lrs.logger.Infof("Received first LoadStatsResponse: %s", pretty.ToJSON(&resp)) + } + + internal := resp.GetLoadReportingInterval() + if internal.CheckValid() != nil { + return nil, 0, fmt.Errorf("lrs: invalid load_reporting_interval: %v", err) + } + loadReportingInterval := internal.AsDuration() + + clusters := resp.Clusters + if resp.SendAllClusters { + // Return an empty slice to send stats for all clusters. + clusters = []string{} + } + + return clusters, loadReportingInterval, nil +} + +func (lrs *streamImpl) sendLoadStatsRequest(stream clients.Stream, loads []*loadData) error { + clusterStats := make([]*v3endpointpb.ClusterStats, 0, len(loads)) + for _, sd := range loads { + droppedReqs := make([]*v3endpointpb.ClusterStats_DroppedRequests, 0, len(sd.drops)) + for category, count := range sd.drops { + droppedReqs = append(droppedReqs, &v3endpointpb.ClusterStats_DroppedRequests{ + Category: category, + DroppedCount: count, + }) + } + localityStats := make([]*v3endpointpb.UpstreamLocalityStats, 0, len(sd.localityStats)) + for lid, localityData := range sd.localityStats { + loadMetricStats := make([]*v3endpointpb.EndpointLoadMetricStats, 0, len(localityData.loadStats)) + var cpuUtilization, memUtilization, appUtilization *v3endpointpb.UnnamedEndpointLoadMetricStats + for name, loadData := range localityData.loadStats { + if envconfig.XDSORCAToLRSPropEnabled { + switch name { + case "cpu_utilization": + cpuUtilization = &v3endpointpb.UnnamedEndpointLoadMetricStats{ + NumRequestsFinishedWithMetric: loadData.count, + TotalMetricValue: loadData.sum, + } + case "mem_utilization": + memUtilization = &v3endpointpb.UnnamedEndpointLoadMetricStats{ + NumRequestsFinishedWithMetric: loadData.count, + TotalMetricValue: loadData.sum, + } + case "application_utilization": + appUtilization = &v3endpointpb.UnnamedEndpointLoadMetricStats{ + NumRequestsFinishedWithMetric: loadData.count, + TotalMetricValue: loadData.sum, + } + default: + loadMetricStats = append(loadMetricStats, &v3endpointpb.EndpointLoadMetricStats{ + MetricName: name, + NumRequestsFinishedWithMetric: loadData.count, + TotalMetricValue: loadData.sum, + }) + } + } else { + loadMetricStats = append(loadMetricStats, &v3endpointpb.EndpointLoadMetricStats{ + MetricName: name, + NumRequestsFinishedWithMetric: loadData.count, + TotalMetricValue: loadData.sum, + }) + } + } + localityStats = append(localityStats, &v3endpointpb.UpstreamLocalityStats{ + Locality: &v3corepb.Locality{ + Region: lid.Region, + Zone: lid.Zone, + SubZone: lid.SubZone, + }, + TotalSuccessfulRequests: localityData.requestStats.succeeded, + TotalRequestsInProgress: localityData.requestStats.inProgress, + TotalErrorRequests: localityData.requestStats.errored, + TotalIssuedRequests: localityData.requestStats.issued, + CpuUtilization: cpuUtilization, + MemUtilization: memUtilization, + ApplicationUtilization: appUtilization, + LoadMetricStats: loadMetricStats, + UpstreamEndpointStats: nil, // TODO: populate for per endpoint loads. + }) + } + + clusterStats = append(clusterStats, &v3endpointpb.ClusterStats{ + ClusterName: sd.cluster, + ClusterServiceName: sd.service, + UpstreamLocalityStats: localityStats, + TotalDroppedRequests: sd.totalDrops, + DroppedRequests: droppedReqs, + LoadReportInterval: durationpb.New(sd.reportInterval), + }) + } + + req := &v3lrspb.LoadStatsRequest{ClusterStats: clusterStats} + if lrs.logger.V(perRPCVerbosityLevel) { + lrs.logger.Infof("Sending LRS loads: %s", pretty.ToJSON(req)) + } + msg, err := proto.Marshal(req) + if err != nil { + if lrs.logger.V(2) { + lrs.logger.Infof("Failed to marshal LoadStatsRequest: %v", err) + } + return err + } + err = stream.Send(msg) + if err == io.EOF { + return getStreamError(stream) + } + return err +} + +func getStreamError(stream clients.Stream) error { + for { + if _, err := stream.Recv(); err != nil { + return err + } + } +} diff --git a/internal/xds/clients/lrsclient/lrsclient.go b/internal/xds/clients/lrsclient/lrsclient.go new file mode 100644 index 0000000..424df81 --- /dev/null +++ b/internal/xds/clients/lrsclient/lrsclient.go @@ -0,0 +1,175 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package lrsclient provides an LRS (Load Reporting Service) client. +// +// See: https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/load_stats/v3/lrs.proto +package lrsclient + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + igrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/xds/clients" + clientsinternal "google.golang.org/grpc/internal/xds/clients/internal" + "google.golang.org/grpc/internal/xds/clients/internal/backoff" +) + +const ( + clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" + clientFeatureResourceWrapper = "xds.config.resource-in-sotw" +) + +var ( + defaultExponentialBackoff = backoff.DefaultExponential.Backoff +) + +// LRSClient is an LRS (Load Reporting Service) client. +type LRSClient struct { + transportBuilder clients.TransportBuilder + node clients.Node + backoff func(int) time.Duration // Backoff for LRS stream failures. + logger *igrpclog.PrefixLogger + + // The LRSClient owns a bunch of streams to individual LRS servers. + // + // Once all references to a stream are dropped, the stream is closed. + mu sync.Mutex + lrsStreams map[clients.ServerIdentifier]*streamImpl // Map from server config to in-use streamImpls. + lrsRefs map[clients.ServerIdentifier]int // Map from server config to number of references. +} + +// New returns a new LRS Client configured with the provided config. +func New(config Config) (*LRSClient, error) { + if config.TransportBuilder == nil { + return nil, errors.New("lrsclient: transport builder is nil") + } + + c := &LRSClient{ + transportBuilder: config.TransportBuilder, + node: config.Node, + backoff: defaultExponentialBackoff, + lrsStreams: make(map[clients.ServerIdentifier]*streamImpl), + lrsRefs: make(map[clients.ServerIdentifier]int), + } + c.logger = prefixLogger(c) + return c, nil +} + +// ReportLoad creates and returns a LoadStore for the caller to report loads +// using a LoadReportingStream. +// +// Caller must call Stop on the returned LoadStore when they are done reporting +// load to this server. +func (c *LRSClient) ReportLoad(si clients.ServerIdentifier) (*LoadStore, error) { + lrs, err := c.getOrCreateLRSStream(si) + if err != nil { + return nil, err + } + return lrs.loadStore, nil +} + +// getOrCreateLRSStream returns an lrs stream for the given server identifier. +// +// If an active lrs stream exists for the given server identifier, it is +// returned. Otherwise, a new lrs stream is created and returned. +func (c *LRSClient) getOrCreateLRSStream(serverIdentifier clients.ServerIdentifier) (*streamImpl, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.logger.V(2) { + c.logger.Infof("Received request for a reference to an lrs stream for server identifier %q", serverIdentifier) + } + + // Use an existing stream, if one exists for this server identifier. + if s, ok := c.lrsStreams[serverIdentifier]; ok { + if c.logger.V(2) { + c.logger.Infof("Reusing an existing lrs stream for server identifier %q", serverIdentifier) + } + c.lrsRefs[serverIdentifier]++ + return s, nil + } + + if c.logger.V(2) { + c.logger.Infof("Creating a new lrs stream for server identifier %q", serverIdentifier) + } + + // Create a new transport and create a new lrs stream, and add it to the + // map of lrs streams. + tr, err := c.transportBuilder.Build(serverIdentifier) + if err != nil { + return nil, fmt.Errorf("lrsclient: failed to create transport for server identifier %s: %v", serverIdentifier, err) + } + + nodeProto := clientsinternal.NodeProto(c.node) + nodeProto.ClientFeatures = []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper} + lrs := newStreamImpl(streamOpts{ + transport: tr, + backoff: c.backoff, + nodeProto: nodeProto, + logPrefix: clientPrefix(c), + }) + + // Register a stop function that decrements the reference count, stops + // the LRS stream when the last reference is removed and closes the + // transport and removes the lrs stream and its references from the + // respective maps. Before closing the stream, it waits for the provided + // context to be done (timeout or cancellation). + stop := func(ctx context.Context) { + c.mu.Lock() + defer c.mu.Unlock() + + if r, ok := c.lrsRefs[serverIdentifier]; !ok || r == 0 { + c.logger.Errorf("Attempting to stop already stopped StreamImpl") + return + } + c.lrsRefs[serverIdentifier]-- + if c.lrsRefs[serverIdentifier] != 0 { + return + } + + lrs.finalSendRequest <- struct{}{} + + select { + case err := <-lrs.finalSendDone: + if err != nil { + c.logger.Warningf("Final send attempt failed: %v", err) + } + case <-ctx.Done(): + c.logger.Warningf("Context canceled before finishing the final send attempt: %v", err) + } + + lrs.cancelStream() + lrs.cancelStream = nil + lrs.logger.Infof("Stopping LRS stream") + <-lrs.doneCh + + delete(c.lrsStreams, serverIdentifier) + tr.Close() + } + lrs.loadStore.stop = stop + + c.lrsStreams[serverIdentifier] = lrs + c.lrsRefs[serverIdentifier] = 1 + + return lrs, nil +} diff --git a/internal/xds/clients/lrsclient/lrsconfig.go b/internal/xds/clients/lrsclient/lrsconfig.go new file mode 100644 index 0000000..6dde1b4 --- /dev/null +++ b/internal/xds/clients/lrsclient/lrsconfig.go @@ -0,0 +1,35 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package lrsclient + +import ( + "google.golang.org/grpc/internal/xds/clients" +) + +// Config is used to configure an LRS client. After one has been passed to the +// LRS client's New function, no part of it may modified. A Config may be +// reused; the lrsclient package will also not modify it. +type Config struct { + // Node is the identity of the client application reporting load to the + // LRS server. + Node clients.Node + + // TransportBuilder is used to connect to the LRS server. + TransportBuilder clients.TransportBuilder +} diff --git a/internal/xds/clients/transport_builder.go b/internal/xds/clients/transport_builder.go new file mode 100644 index 0000000..f940675 --- /dev/null +++ b/internal/xds/clients/transport_builder.go @@ -0,0 +1,53 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clients + +import ( + "context" +) + +// TransportBuilder provides the functionality to create a communication +// channel to an xDS or LRS server. +type TransportBuilder interface { + // Build creates a new Transport instance to the server based on the + // provided ServerIdentifier. + Build(serverIdentifier ServerIdentifier) (Transport, error) +} + +// Transport provides the functionality to communicate with an xDS or LRS +// server using streaming calls. +type Transport interface { + // NewStream creates a new streaming call to the server for the specific + // RPC method name. The returned Stream interface can be used to send and + // receive messages on the stream. + NewStream(context.Context, string) (Stream, error) + + // Close closes the Transport. + Close() +} + +// Stream provides methods to send and receive messages on a stream. Messages +// are represented as a byte slice. +type Stream interface { + // Send sends the provided message on the stream. + Send([]byte) error + + // Recv blocks until the next message is received on the stream. + Recv() ([]byte, error) +} diff --git a/internal/xds/clients/xdsclient/ads_stream.go b/internal/xds/clients/xdsclient/ads_stream.go new file mode 100644 index 0000000..95c5d88 --- /dev/null +++ b/internal/xds/clients/xdsclient/ads_stream.go @@ -0,0 +1,733 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/grpclog" + igrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/internal/backoff" + "google.golang.org/grpc/internal/xds/clients/internal/pretty" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + cpb "google.golang.org/genproto/googleapis/rpc/code" + statuspb "google.golang.org/genproto/googleapis/rpc/status" +) + +const ( + // Any per-RPC level logs which print complete request or response messages + // should be gated at this verbosity level. Other per-RPC level logs which print + // terse output should be at `INFO` and verbosity 2. + perRPCVerbosityLevel = 9 +) + +// request represents a queued request message to be sent on the ADS stream. It +// contains the type of the resource and the list of resource names to be sent. +type request struct { + typ ResourceType + resourceNames []string +} + +// response represents a response received on the ADS stream. It contains the +// type URL, version, and resources for the response. +type response struct { + typeURL string + version string + resources []*anypb.Any +} + +// dataAndErrTuple is a struct that holds a resource and an error. It is used to +// return a resource and any associated error from a function. +type dataAndErrTuple struct { + Resource ResourceData + Err error +} + +// adsStreamEventHandler is an interface that defines the callbacks for events that +// occur on the ADS stream. Methods on this interface may be invoked +// concurrently and implementations need to handle them in a thread-safe manner. +type adsStreamEventHandler interface { + onStreamError(error) // Called when the ADS stream breaks. + onWatchExpiry(ResourceType, string) // Called when the watch timer expires for a resource. + onResponse(response, func()) ([]string, error) // Called when a response is received on the ADS stream. +} + +// state corresponding to a resource type. +type resourceTypeState struct { + version string // Last acked version. Should not be reset when the stream breaks. + nonce string // Last received nonce. Should be reset when the stream breaks. + subscribedResources map[string]*xdsresource.ResourceWatchState // Map of subscribed resource names to their state. +} + +// adsStreamImpl provides the functionality associated with an ADS (Aggregated +// Discovery Service) stream on the client side. It manages the lifecycle of the +// ADS stream, including creating the stream, sending requests, and handling +// responses. It also handles flow control and retries for the stream. +type adsStreamImpl struct { + // The following fields are initialized from arguments passed to the + // constructor and are read-only afterwards, and hence can be accessed + // without a mutex. + transport clients.Transport // Transport to use for ADS stream. + eventHandler adsStreamEventHandler // Callbacks into the xdsChannel. + backoff func(int) time.Duration // Backoff for retries, after stream failures. + nodeProto *v3corepb.Node // Identifies the gRPC application. + watchExpiryTimeout time.Duration // Resource watch expiry timeout + logger *igrpclog.PrefixLogger + + // The following fields are initialized in the constructor and are not + // written to afterwards, and hence can be accessed without a mutex. + streamCh chan clients.Stream // New ADS streams are pushed here. + runnerDoneCh chan struct{} // Notify completion of runner goroutine. + cancel context.CancelFunc // To cancel the context passed to the runner goroutine. + fc *adsFlowControl // Flow control for ADS stream. + notifySender chan struct{} // To notify the sending goroutine of a pending request. + + // Guards access to the below fields (and to the contents of the map). + mu sync.Mutex + resourceTypeState map[ResourceType]*resourceTypeState // Map of resource types to their state. + pendingRequests []request // Subscriptions and unsubscriptions are pushed here. + + // The following fields are accessed atomically. + firstRequest atomic.Bool // False after the first request is sent out. + firstStreamCreated atomic.Bool // Set to true after the very first ADS stream is created. + streamEstablished atomic.Bool // Set to true when an ADS stream is established and a response is received, except for the very first stream which is set to true immediately. +} + +// adsStreamOpts contains the options for creating a new ADS Stream. +type adsStreamOpts struct { + transport clients.Transport // xDS transport to create the stream on. + eventHandler adsStreamEventHandler // Callbacks for stream events. + backoff func(int) time.Duration // Backoff for retries, after stream failures. + nodeProto *v3corepb.Node // Node proto to identify the gRPC application. + watchExpiryTimeout time.Duration // Resource watch expiry timeout. + logPrefix string // Prefix to be used for log messages. +} + +// newADSStreamImpl initializes a new adsStreamImpl instance using the given +// parameters. It also launches goroutines responsible for managing reads and +// writes for messages of the underlying stream. +func newADSStreamImpl(opts adsStreamOpts) *adsStreamImpl { + s := &adsStreamImpl{ + transport: opts.transport, + eventHandler: opts.eventHandler, + backoff: opts.backoff, + nodeProto: opts.nodeProto, + watchExpiryTimeout: opts.watchExpiryTimeout, + + streamCh: make(chan clients.Stream, 1), + runnerDoneCh: make(chan struct{}), + fc: newADSFlowControl(), + notifySender: make(chan struct{}, 1), + resourceTypeState: make(map[ResourceType]*resourceTypeState), + } + + l := grpclog.Component("xds") + s.logger = igrpclog.NewPrefixLogger(l, opts.logPrefix+fmt.Sprintf("[ads-stream %p] ", s)) + + ctx, cancel := context.WithCancel(context.Background()) + s.cancel = cancel + go s.runner(ctx) + return s +} + +// Stop blocks until the stream is closed and all spawned goroutines exit. +func (s *adsStreamImpl) Stop() { + s.cancel() + s.fc.stop() + <-s.runnerDoneCh + s.logger.Infof("Shutdown ADS stream") +} + +// subscribe subscribes to the given resource. It is assumed that multiple +// subscriptions for the same resource is deduped at the caller. A discovery +// request is sent out on the underlying stream, for the resource type with the +// newly subscribed resource. +func (s *adsStreamImpl) subscribe(typ ResourceType, name string) { + if s.logger.V(2) { + s.logger.Infof("Subscribing to resource %q of type %q", name, typ.TypeName) + } + + s.mu.Lock() + state, ok := s.resourceTypeState[typ] + if !ok { + // An entry in the type state map is created as part of the first + // subscription request for this type. + state = &resourceTypeState{subscribedResources: make(map[string]*xdsresource.ResourceWatchState)} + s.resourceTypeState[typ] = state + } + + // Create state for the newly subscribed resource. The watch timer will + // be started when a request for this resource is actually sent out. + state.subscribedResources[name] = &xdsresource.ResourceWatchState{State: xdsresource.ResourceWatchStateStarted} + + // Send a request for the resource type with updated subscriptions. + s.pendingRequests = append(s.pendingRequests, request{typ: typ, resourceNames: resourceNames(state.subscribedResources)}) + s.mu.Unlock() + + select { + case s.notifySender <- struct{}{}: + default: + } +} + +// unsubscribe cancels the subscription to the given resource. It is a no-op if +// the given resource does not exist. The watch expiry timer associated with the +// resource is stopped if one is active. A discovery request is sent out on the +// stream for the resource type with the updated set of resource names. +func (s *adsStreamImpl) unsubscribe(typ ResourceType, name string) { + if s.logger.V(2) { + s.logger.Infof("Unsubscribing to resource %q of type %q", name, typ.TypeName) + } + + s.mu.Lock() + state, ok := s.resourceTypeState[typ] + if !ok { + s.mu.Unlock() + return + } + rs, ok := state.subscribedResources[name] + if !ok { + s.mu.Unlock() + return + } + if rs.ExpiryTimer != nil { + rs.ExpiryTimer.Stop() + } + delete(state.subscribedResources, name) + + // Send a request for the resource type with updated subscriptions. + s.pendingRequests = append(s.pendingRequests, request{typ: typ, resourceNames: resourceNames(state.subscribedResources)}) + s.mu.Unlock() + + select { + case s.notifySender <- struct{}{}: + default: + } +} + +// runner is a long-running goroutine that handles the lifecycle of the ADS +// stream. It spawns another goroutine to handle writes of discovery request +// messages on the stream. Whenever an existing stream fails, it performs +// exponential backoff (if no messages were received on that stream) before +// creating a new stream. +func (s *adsStreamImpl) runner(ctx context.Context) { + defer close(s.runnerDoneCh) + + go s.send(ctx) + + runStreamWithBackoff := func() error { + stream, err := s.transport.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources") + if err != nil { + s.logger.Warningf("Failed to create a new ADS streaming RPC: %v", err) + s.streamEstablished.Store(false) + s.onError(err, false) + return nil + } + if s.logger.V(2) { + s.logger.Infof("ADS stream created") + } + + s.firstRequest.Store(true) + if !s.firstStreamCreated.Load() { + s.streamEstablished.Store(true) + s.firstStreamCreated.Store(true) + } + + // Ensure that the most recently created stream is pushed on the + // channel for the `send` goroutine to consume. + select { + case <-s.streamCh: + default: + } + s.streamCh <- stream + + // Backoff state is reset upon successful receipt of at least one + // message from the server. + if s.recv(stream) { + return backoff.ErrResetBackoff + } + return nil + } + backoff.RunF(ctx, runStreamWithBackoff, s.backoff) +} + +// send is a long running goroutine that handles sending discovery requests for +// two scenarios: +// - a new subscription or unsubscription request is received +// - a new stream is created after the previous one failed +func (s *adsStreamImpl) send(ctx context.Context) { + // Stores the most recent stream instance received on streamCh. + var stream clients.Stream + for { + select { + case <-ctx.Done(): + return + case stream = <-s.streamCh: + if err := s.sendExisting(stream); err != nil { + // Send failed, clear the current stream. Attempt to resend will + // only be made after a new stream is created. + stream = nil + continue + } + case <-s.notifySender: + // If there's no stream yet, skip the request. This request will be resent + // when a new stream is created. If no stream is created, the watcher will + // timeout (same as server not sending response back). + if stream == nil { + continue + } + + // Resetting the pendingRequests slice to nil works for both cases: + // - When we successfully sends the requests out on the wire. + // - When sending fails. This can happen only when the stream fails, + // and in this case, we rely on the `sendExisting` to send out + // requests for all subscriptions when the stream is recreated. + s.mu.Lock() + if err := s.sendNewLocked(stream, s.pendingRequests); err != nil { + stream = nil + } + s.pendingRequests = nil + s.mu.Unlock() + } + } +} + +// sendNewLocked attempts to send a discovery request based on a new subscription or +// unsubscription. This method also starts the watch expiry timer for resources +// that were sent in the request for the first time, i.e. their watch state is +// `watchStateStarted`. +// +// Caller needs to hold c.mu. +func (s *adsStreamImpl) sendNewLocked(stream clients.Stream, requests []request) error { + for _, req := range requests { + state := s.resourceTypeState[req.typ] + if err := s.sendMessageLocked(stream, req.resourceNames, req.typ.TypeURL, state.version, state.nonce, nil); err != nil { + return err + } + s.startWatchTimersLocked(req.typ, req.resourceNames) + } + return nil +} + +// sendExisting sends out discovery requests for existing resources when +// recovering from a broken stream. +// +// The stream argument is guaranteed to be non-nil. +func (s *adsStreamImpl) sendExisting(stream clients.Stream) error { + s.mu.Lock() + defer s.mu.Unlock() + + // Clear any queued requests. Previously subscribed to resources will be + // resent below. + s.pendingRequests = nil + + for typ, state := range s.resourceTypeState { + // Reset only the nonces map when the stream restarts. + // + // xDS spec says the following. See section: + // https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#ack-nack-and-resource-type-instance-version + // + // Note that the version for a resource type is not a property of an + // individual xDS stream but rather a property of the resources + // themselves. If the stream becomes broken and the client creates a new + // stream, the client’s initial request on the new stream should + // indicate the most recent version seen by the client on the previous + // stream + state.nonce = "" + + if len(state.subscribedResources) == 0 { + continue + } + + names := resourceNames(state.subscribedResources) + if err := s.sendMessageLocked(stream, names, typ.TypeURL, state.version, state.nonce, nil); err != nil { + return err + } + s.startWatchTimersLocked(typ, names) + } + return nil +} + +// sendMessageLocked sends a discovery request to the server, populating the +// different fields of the message with the given parameters. Returns a non-nil +// error if the request could not be sent. +// +// Caller needs to hold c.mu. +func (s *adsStreamImpl) sendMessageLocked(stream clients.Stream, names []string, url, version, nonce string, nackErr error) error { + req := &v3discoverypb.DiscoveryRequest{ + ResourceNames: names, + TypeUrl: url, + VersionInfo: version, + ResponseNonce: nonce, + } + + // The xDS protocol only requires that we send the node proto in the first + // discovery request on every stream. Sending the node proto in every + // request wastes CPU resources on the client and the server. + if s.firstRequest.Load() { + req.Node = s.nodeProto + } + + if nackErr != nil { + req.ErrorDetail = &statuspb.Status{ + Code: int32(cpb.Code_INVALID_ARGUMENT), Message: nackErr.Error(), + } + } + + msg, err := proto.Marshal(req) + if err != nil { + s.logger.Warningf("Failed to marshal DiscoveryRequest: %v", err) + return err + } + if err := stream.Send(msg); err != nil { + s.logger.Warningf("Sending ADS request for type %q, resources: %v, version: %q, nonce: %q failed: %v", url, names, version, nonce, err) + return err + } + s.firstRequest.Store(false) + + if s.logger.V(perRPCVerbosityLevel) { + s.logger.Infof("ADS request sent: %v", pretty.ToJSON(req)) + } else if s.logger.V(2) { + s.logger.Infof("ADS request sent for type %q, resources: %v, version: %q, nonce: %q", url, names, version, nonce) + } + + return nil +} + +// recv is responsible for receiving messages from the ADS stream. +// +// It performs the following actions: +// - Waits for local flow control to be available before it receives a message +// from the ADS stream. If an error is encountered here, it is handled by +// the onError method which propagates the error to all watchers. +// - Invokes the event handler's OnADSResponse method to process the message. +// - Sends an ACK or NACK to the server based on the response. +// +// It returns a boolean indicating whether at least one message was received +// from the server. +func (s *adsStreamImpl) recv(stream clients.Stream) bool { + msgReceived := false + for { + // Wait for ADS stream level flow control to be available. + if s.fc.wait() { + if s.logger.V(2) { + s.logger.Infof("ADS stream stopped while waiting for flow control") + } + return msgReceived + } + + resources, url, version, nonce, err := s.recvMessage(stream) + if err != nil { + if !msgReceived { + s.streamEstablished.Store(false) + } + s.onError(err, msgReceived) + s.logger.Warningf("ADS stream closed: %v", err) + return msgReceived + } + s.streamEstablished.Store(true) + msgReceived = true + + // Invoke the onResponse event handler to parse the incoming message and + // decide whether to send an ACK or NACK. + resp := response{ + resources: resources, + typeURL: url, + version: version, + } + var resourceNames []string + var nackErr error + s.fc.setPending(true) + resourceNames, nackErr = s.eventHandler.onResponse(resp, sync.OnceFunc(func() { s.fc.setPending(false) })) + if xdsresource.ErrType(nackErr) == xdsresource.ErrorTypeResourceTypeUnsupported { + // A general guiding principle is that if the server sends + // something the client didn't actually subscribe to, then the + // client ignores it. Here, we have received a response with + // resources of a type that we don't know about. + // + // Sending a NACK doesn't really seem appropriate here, since we're + // not actually validating what the server sent and therefore don't + // know that it's invalid. But we shouldn't ACK either, because we + // don't know that it is valid. + s.logger.Warningf("%v", nackErr) + continue + } + + s.onRecv(stream, resourceNames, url, version, nonce, nackErr) + } +} + +func (s *adsStreamImpl) recvMessage(stream clients.Stream) (resources []*anypb.Any, url, version, nonce string, err error) { + r, err := stream.Recv() + if err != nil { + return nil, "", "", "", err + } + var resp v3discoverypb.DiscoveryResponse + if err := proto.Unmarshal(r, &resp); err != nil { + s.logger.Infof("Failed to unmarshal response to DiscoveryResponse: %v", err) + return nil, "", "", "", fmt.Errorf("unexpected message type %T", r) + } + if s.logger.V(perRPCVerbosityLevel) { + s.logger.Infof("ADS response received: %v", pretty.ToJSON(&resp)) + } else if s.logger.V(2) { + s.logger.Infof("ADS response received for type %q, version %q, nonce %q", resp.GetTypeUrl(), resp.GetVersionInfo(), resp.GetNonce()) + } + return resp.GetResources(), resp.GetTypeUrl(), resp.GetVersionInfo(), resp.GetNonce(), nil +} + +// onRecv is invoked when a response is received from the server. The arguments +// passed to this method correspond to the most recently received response. +// +// It performs the following actions: +// - updates resource type specific state +// - updates resource specific state for resources in the response +// - sends an ACK or NACK to the server based on the response +func (s *adsStreamImpl) onRecv(stream clients.Stream, names []string, url, version, nonce string, nackErr error) { + s.mu.Lock() + defer s.mu.Unlock() + + // Lookup the resource type specific state based on the type URL. + var typ ResourceType + for t := range s.resourceTypeState { + if t.TypeURL == url { + typ = t + break + } + } + typeState, ok := s.resourceTypeState[typ] + if !ok { + s.logger.Warningf("ADS stream received a response for type %q, but no state exists for it", url) + return + } + + // Update the resource type specific state. This includes: + // - updating the nonce unconditionally + // - updating the version only if the response is to be ACKed + previousVersion := typeState.version + typeState.nonce = nonce + if nackErr == nil { + typeState.version = version + } + + // Update the resource specific state. For all resources received as + // part of this response that are in state `started` or `requested`, + // this includes: + // - setting the watch state to watchstateReceived + // - stopping the expiry timer, if one exists + for _, name := range names { + rs, ok := typeState.subscribedResources[name] + if !ok { + s.logger.Warningf("ADS stream received a response for resource %q, but no state exists for it", name) + continue + } + if ws := rs.State; ws == xdsresource.ResourceWatchStateStarted || ws == xdsresource.ResourceWatchStateRequested { + rs.State = xdsresource.ResourceWatchStateReceived + if rs.ExpiryTimer != nil { + rs.ExpiryTimer.Stop() + rs.ExpiryTimer = nil + } + } + } + + // Send an ACK or NACK. + subscribedResourceNames := resourceNames(typeState.subscribedResources) + if nackErr != nil { + s.logger.Warningf("Sending NACK for resource type: %q, version: %q, nonce: %q, reason: %v", url, version, nonce, nackErr) + s.sendMessageLocked(stream, subscribedResourceNames, url, previousVersion, nonce, nackErr) + return + } + + if s.logger.V(2) { + s.logger.Infof("Sending ACK for resource type: %q, version: %q, nonce: %q", url, version, nonce) + } + s.sendMessageLocked(stream, subscribedResourceNames, url, version, nonce, nil) +} + +// onError is called when an error occurs on the ADS stream. It stops any +// outstanding resource timers and resets the watch state to started for any +// resources that were in the requested state. It also handles the case where +// the ADS stream was closed after receiving a response, which is not +// considered an error. +func (s *adsStreamImpl) onError(err error, msgReceived bool) { + // For resources that been requested but not yet responded to by the + // management server, stop the resource timers and reset the watch state to + // watchStateStarted. This is because we don't want the expiry timer to be + // running when we don't have a stream open to the management server. + s.mu.Lock() + for _, state := range s.resourceTypeState { + for _, rs := range state.subscribedResources { + if rs.State != xdsresource.ResourceWatchStateRequested { + continue + } + if rs.ExpiryTimer != nil { + rs.ExpiryTimer.Stop() + rs.ExpiryTimer = nil + } + rs.State = xdsresource.ResourceWatchStateStarted + } + } + s.mu.Unlock() + + // Note that we do not consider it an error if the ADS stream was closed + // after having received a response on the stream. This is because there + // are legitimate reasons why the server may need to close the stream during + // normal operations, such as needing to rebalance load or the underlying + // connection hitting its max connection age limit. + // (see [gRFC A9](https://github.com/grpc/proposal/blob/master/A9-server-side-conn-mgt.md)). + if msgReceived { + err = xdsresource.NewError(xdsresource.ErrTypeStreamFailedAfterRecv, err.Error()) + } + + s.eventHandler.onStreamError(err) +} + +// startWatchTimersLocked starts the expiry timers for the given resource names +// of the specified resource type. For each resource name, if the resource +// watch state is in the "started" state, it transitions the state to +// "requested" and starts an expiry timer. When the timer expires, the resource +// watch state is set to "timeout" and the event handler callback is called. +// +// The caller must hold the s.mu lock. +func (s *adsStreamImpl) startWatchTimersLocked(typ ResourceType, names []string) { + typeState := s.resourceTypeState[typ] + for _, name := range names { + resourceState, ok := typeState.subscribedResources[name] + if !ok { + continue + } + if resourceState.State != xdsresource.ResourceWatchStateStarted { + continue + } + resourceState.State = xdsresource.ResourceWatchStateRequested + + rs := resourceState + resourceState.ExpiryTimer = time.AfterFunc(s.watchExpiryTimeout, func() { + s.mu.Lock() + rs.State = xdsresource.ResourceWatchStateTimeout + rs.ExpiryTimer = nil + s.mu.Unlock() + s.eventHandler.onWatchExpiry(typ, name) + }) + } +} + +func (s *adsStreamImpl) adsResourceWatchStateForTesting(rType ResourceType, resourceName string) (xdsresource.ResourceWatchState, error) { + s.mu.Lock() + defer s.mu.Unlock() + + state, ok := s.resourceTypeState[rType] + if !ok { + return xdsresource.ResourceWatchState{}, fmt.Errorf("unknown resource type: %v", rType) + } + resourceState, ok := state.subscribedResources[resourceName] + if !ok { + return xdsresource.ResourceWatchState{}, fmt.Errorf("unknown resource name: %v", resourceName) + } + return *resourceState, nil +} + +func resourceNames(m map[string]*xdsresource.ResourceWatchState) []string { + ret := make([]string, len(m)) + idx := 0 + for name := range m { + ret[idx] = name + idx++ + } + return ret +} + +// adsFlowControl implements ADS stream level flow control that enables the ADS +// stream to block the reading of the next message until the previous update is +// consumed by all watchers. +// +// The lifetime of the flow control is tied to the lifetime of the stream. When +// the stream is closed, it is the responsibility of the caller to stop the flow +// control. This ensures that any goroutine blocked on the flow control's wait +// method is unblocked. +type adsFlowControl struct { + mu sync.Mutex + // cond is used to signal when the most recent update has been consumed, or + // the flow control has been stopped (in which case, waiters should be + // unblocked as well). + cond *sync.Cond + pending bool // indicates if the most recent update is pending consumption + stopped bool // indicates if the ADS stream has been stopped +} + +// newADSFlowControl returns a new adsFlowControl. +func newADSFlowControl() *adsFlowControl { + fc := &adsFlowControl{} + fc.cond = sync.NewCond(&fc.mu) + return fc +} + +// stop marks the flow control as stopped and signals the condition variable to +// unblock any goroutine waiting on it. +func (fc *adsFlowControl) stop() { + fc.mu.Lock() + defer fc.mu.Unlock() + + fc.stopped = true + fc.cond.Broadcast() +} + +// setPending changes the internal state to indicate whether there is an update +// pending consumption by all watchers. If there is no longer a pending update, +// the condition variable is signaled to allow the recv method to proceed. +func (fc *adsFlowControl) setPending(pending bool) { + fc.mu.Lock() + defer fc.mu.Unlock() + + if fc.stopped { + return + } + + fc.pending = pending + if !pending { + fc.cond.Broadcast() + } +} + +// wait blocks until all the watchers have consumed the most recent update. +// Returns true if the flow control was stopped while waiting, false otherwise. +func (fc *adsFlowControl) wait() bool { + fc.mu.Lock() + defer fc.mu.Unlock() + + for fc.pending && !fc.stopped { + fc.cond.Wait() + } + + return fc.stopped +} + +func (s *adsStreamImpl) isStreamEstablished() bool { + return s.streamEstablished.Load() +} diff --git a/internal/xds/clients/xdsclient/authority.go b/internal/xds/clients/xdsclient/authority.go new file mode 100644 index 0000000..f5d78eb --- /dev/null +++ b/internal/xds/clients/xdsclient/authority.go @@ -0,0 +1,996 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + + "google.golang.org/grpc/grpclog" + igrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/internal/syncutil" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + "google.golang.org/grpc/internal/xds/clients/xdsclient/metrics" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/timestamppb" + + v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3" + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" +) + +type resourceState struct { + watchers map[ResourceWatcher]bool // Set of watchers for this resource. + cache ResourceData // Most recent ACKed update for this resource. + md xdsresource.UpdateMetadata // Metadata for the most recent update. + deletionIgnored bool // True, if resource deletion was ignored for a prior update. + xdsChannelConfigs map[*xdsChannelWithConfig]bool // Set of xdsChannels where this resource is subscribed. +} + +// xdsChannelForADS is used to acquire a reference to an xdsChannel. This +// functionality is provided by the xdsClient. +// +// The arguments to the function are as follows: +// - the server config for the xdsChannel +// - the calling authority on which a set of callbacks are invoked by the +// xdsChannel on ADS stream events +// +// Returns a reference to the xdsChannel and a function to release the same. A +// non-nil error is returned if the channel creation fails and the first two +// return values are meaningless in this case. +type xdsChannelForADS func(*ServerConfig, *authority) (*xdsChannel, func(), error) + +// xdsChannelWithConfig is a struct that holds an xdsChannel and its associated +// ServerConfig, along with a cleanup function to release the xdsChannel. +type xdsChannelWithConfig struct { + channel *xdsChannel + serverConfig *ServerConfig + cleanup func() +} + +// authority provides the functionality required to communicate with a +// management server corresponding to an authority name specified in the +// xDS client configuration. +// +// It holds references to one or more xdsChannels, one for each server +// configuration in the config, to allow fallback from a primary management +// server to a secondary management server. Authorities that contain similar +// server configuration entries will end up sharing the xdsChannel for that +// server configuration. The xdsChannels are owned and managed by the xdsClient. +// +// It also contains a cache of resource state for resources requested from +// management server(s). This cache contains the list of registered watchers and +// the most recent resource configuration received from the management server. +type authority struct { + // The following fields are initialized at creation time and are read-only + // afterwards, and therefore don't need to be protected with a mutex. + name string // Name of the authority from xDS client configuration. + watcherCallbackSerializer *syncutil.CallbackSerializer // Serializer to run watcher callbacks, owned by the xDS client implementation. + getChannelForADS xdsChannelForADS // Function to get an xdsChannel for ADS, provided by the xDS client implementation. + xdsClientSerializer *syncutil.CallbackSerializer // Serializer to run call ins from the xDS client, owned by this authority. + xdsClientSerializerClose func() // Function to close the above serializer. + logger *igrpclog.PrefixLogger // Logger for this authority. + target string // The gRPC Channel target. + metricsReporter clients.MetricsReporter + + // The below defined fields must only be accessed in the context of the + // serializer callback, owned by this authority. + + // A two level map containing the state of all the resources being watched. + // + // The first level map key is the ResourceType (Listener, Route etc). This + // allows us to have a single map for all resources instead of having per + // resource-type maps. + // + // The second level map key is the resource name, with the value being the + // actual state of the resource. + resources map[ResourceType]map[string]*resourceState + + // An ordered list of xdsChannels corresponding to the list of server + // configurations specified for this authority in the config. The + // ordering specifies the order in which these channels are preferred for + // fallback. + xdsChannelConfigs []*xdsChannelWithConfig + + // The current active xdsChannel. Here, active does not mean that the + // channel has a working connection to the server. It simply points to the + // channel that we are trying to work with, based on fallback logic. + activeXDSChannel *xdsChannelWithConfig +} + +// authorityBuildOptions wraps arguments required to create a new authority. +type authorityBuildOptions struct { + serverConfigs []ServerConfig // Server configs for the authority + name string // Name of the authority + serializer *syncutil.CallbackSerializer // Callback serializer for invoking watch callbacks + getChannelForADS xdsChannelForADS // Function to acquire a reference to an xdsChannel + logPrefix string // Prefix for logging + target string // Target for the gRPC Channel that owns xDS Client/Authority + metricsReporter clients.MetricsReporter // Metrics reporter for the authority +} + +// newAuthority creates a new authority instance with the provided +// configuration. The authority is responsible for managing the state of +// resources requested from the management server, as well as acquiring and +// releasing references to channels used to communicate with the management +// server. +// +// Note that no channels to management servers are created at this time. Instead +// a channel to the first server configuration is created when the first watch +// is registered, and more channels are created as needed by the fallback logic. +func newAuthority(args authorityBuildOptions) *authority { + ctx, cancel := context.WithCancel(context.Background()) + l := grpclog.Component("xds") + logPrefix := args.logPrefix + fmt.Sprintf("[authority %q] ", args.name) + ret := &authority{ + name: args.name, + watcherCallbackSerializer: args.serializer, + getChannelForADS: args.getChannelForADS, + xdsClientSerializer: syncutil.NewCallbackSerializer(ctx), + xdsClientSerializerClose: cancel, + logger: igrpclog.NewPrefixLogger(l, logPrefix), + resources: make(map[ResourceType]map[string]*resourceState), + target: args.target, + metricsReporter: args.metricsReporter, + } + + // Create an ordered list of xdsChannels with their server configs. The + // actual channel to the first server configuration is created when the + // first watch is registered, and channels to other server configurations + // are created as needed to support fallback. + for _, sc := range args.serverConfigs { + ret.xdsChannelConfigs = append(ret.xdsChannelConfigs, &xdsChannelWithConfig{serverConfig: &sc}) + } + return ret +} + +// adsStreamFailure is called to notify the authority about an ADS stream +// failure on an xdsChannel to the management server identified by the provided +// server config. The error is forwarded to all the resource watchers. +// +// This method is called by the xDS client implementation (on all interested +// authorities) when a stream error is reported by an xdsChannel. +// +// Errors of type xdsresource.ErrTypeStreamFailedAfterRecv are ignored. +func (a *authority) adsStreamFailure(serverConfig *ServerConfig, err error) { + a.xdsClientSerializer.TrySchedule(func(context.Context) { + a.handleADSStreamFailure(serverConfig, err) + }) +} + +// Handles ADS stream failure by invoking watch callbacks and triggering +// fallback if the associated conditions are met. +// +// Only executed in the context of a serializer callback. +func (a *authority) handleADSStreamFailure(serverConfig *ServerConfig, err error) { + if a.logger.V(2) { + a.logger.Infof("Connection to server %s failed with error: %v", serverConfig, err) + } + + // We do not consider it an error if the ADS stream was closed after having + // received a response on the stream. This is because there are legitimate + // reasons why the server may need to close the stream during normal + // operations, such as needing to rebalance load or the underlying + // connection hitting its max connection age limit. See gRFC A57 for more + // details. + if xdsresource.ErrType(err) == xdsresource.ErrTypeStreamFailedAfterRecv { + a.logger.Warningf("Watchers not notified since ADS stream failed after having received at least one response: %v", err) + return + } + + // Two conditions need to be met for fallback to be triggered: + // 1. There is a connectivity failure on the ADS stream, as described in + // gRFC A57. For us, this means that the ADS stream was closed before the + // first server response was received. We already checked that condition + // earlier in this method. + // 2. There is at least one watcher for a resource that is not cached. + // Cached resources include ones that + // - have been successfully received and can be used. + // - are considered non-existent according to xDS Protocol Specification. + if !a.watcherExistsForUncachedResource() { + if a.logger.V(2) { + a.logger.Infof("No watchers for uncached resources. Not triggering fallback") + } + // Since we are not triggering fallback, propagate the connectivity + // error to all watchers and return early. + a.propagateConnectivityErrorToAllWatchers(err) + return + } + + // Attempt to fallback to servers with lower priority than the failing one. + currentServerIdx := a.serverIndexForConfig(serverConfig) + for i := currentServerIdx + 1; i < len(a.xdsChannelConfigs); i++ { + if a.fallbackToServer(a.xdsChannelConfigs[i]) { + // Since we have successfully triggered fallback, we don't have to + // notify watchers about the connectivity error. + return + } + } + + // Having exhausted all available servers, we must notify watchers of the + // connectivity error - A71. + a.propagateConnectivityErrorToAllWatchers(err) +} + +// propagateConnectivityErrorToAllWatchers propagates the given connection error +// to all watchers of all resources. +// +// Only executed in the context of a serializer callback. +func (a *authority) propagateConnectivityErrorToAllWatchers(err error) { + for _, rType := range a.resources { + for _, state := range rType { + for watcher := range state.watchers { + if state.cache == nil { + a.watcherCallbackSerializer.TrySchedule(func(context.Context) { + watcher.ResourceError(xdsresource.NewErrorf(xdsresource.ErrorTypeConnection, "xds: error received from xDS stream: %v", err), func() {}) + }) + } else { + a.watcherCallbackSerializer.TrySchedule(func(context.Context) { + watcher.AmbientError(xdsresource.NewErrorf(xdsresource.ErrorTypeConnection, "xds: error received from xDS stream: %v", err), func() {}) + }) + } + } + } + } +} + +// serverIndexForConfig returns the index of the xdsChannelConfig matching the +// provided server config, panicking if no match is found (which indicates a +// programming error). +func (a *authority) serverIndexForConfig(sc *ServerConfig) int { + for i, cfg := range a.xdsChannelConfigs { + if isServerConfigEqual(sc, cfg.serverConfig) { + return i + } + } + panic(fmt.Sprintf("no server config matching %v found", sc)) +} + +// Determines the server to fallback to and triggers fallback to the same. If +// required, creates an xdsChannel to that server, and re-subscribes to all +// existing resources. +// +// Only executed in the context of a serializer callback. +func (a *authority) fallbackToServer(xc *xdsChannelWithConfig) bool { + if a.logger.V(2) { + a.logger.Infof("Attempting to initiate fallback to server %q", xc.serverConfig) + } + + if xc.channel != nil { + if a.logger.V(2) { + a.logger.Infof("Channel to the next server in the list %q already exists", xc.serverConfig) + } + return false + } + + channel, cleanup, err := a.getChannelForADS(xc.serverConfig, a) + if err != nil { + a.logger.Errorf("Failed to create xDS channel: %v", err) + return false + } + xc.channel = channel + xc.cleanup = cleanup + a.activeXDSChannel = xc + + // Subscribe to all existing resources from the new management server. + for typ, resources := range a.resources { + for name, state := range resources { + if a.logger.V(2) { + a.logger.Infof("Resubscribing to resource of type %q and name %q", typ.TypeName, name) + } + xc.channel.subscribe(typ, name) + + // Add the new channel to the list of xdsChannels from which this + // resource has been requested from. Retain the cached resource and + // the set of existing watchers (and other metadata fields) in the + // resource state. + state.xdsChannelConfigs[xc] = true + } + } + return true +} + +// adsResourceUpdate is called to notify the authority about a resource update +// received on the ADS stream. +// +// This method is called by the xDS client implementation (on all interested +// authorities) when a stream error is reported by an xdsChannel. +func (a *authority) adsResourceUpdate(serverConfig *ServerConfig, rType ResourceType, updates map[string]dataAndErrTuple, md xdsresource.UpdateMetadata, onDone func()) { + a.xdsClientSerializer.TrySchedule(func(context.Context) { + a.handleADSResourceUpdate(serverConfig, rType, updates, md, onDone) + }) +} + +// handleADSResourceUpdate processes an update from the xDS client, updating the +// resource cache and notifying any registered watchers of the update. +// +// If the update is received from a higher priority xdsChannel that was +// previously down, we revert to it and close all lower priority xdsChannels. +// +// Once the update has been processed by all watchers, the authority is expected +// to invoke the onDone callback. +// +// Only executed in the context of a serializer callback. +func (a *authority) handleADSResourceUpdate(serverConfig *ServerConfig, rType ResourceType, updates map[string]dataAndErrTuple, md xdsresource.UpdateMetadata, onDone func()) { + if !a.handleRevertingToPrimaryOnUpdate(serverConfig) { + return + } + + // We build a list of callback funcs to invoke, and invoke them at the end + // of this method instead of inline (when handling the update for a + // particular resource), because we want to make sure that all calls to + // increment watcherCnt happen before any callbacks are invoked. This will + // ensure that the onDone callback is never invoked before all watcher + // callbacks are invoked, and the watchers have processed the update. + watcherCnt := new(atomic.Int64) + done := func() { + if watcherCnt.Add(-1) == 0 { + onDone() + } + } + var funcsToSchedule []func(context.Context) + defer func() { + if len(funcsToSchedule) == 0 { + // When there are no watchers for the resources received as part of + // this update, invoke onDone explicitly to unblock the next read on + // the ADS stream. + onDone() + return + } + for _, f := range funcsToSchedule { + a.watcherCallbackSerializer.ScheduleOr(f, onDone) + } + }() + + resourceStates := a.resources[rType] + for name, uErr := range updates { + state, ok := resourceStates[name] + if !ok { + continue + } + + if err := uErr.Err; err != nil { + if a.metricsReporter != nil { + a.metricsReporter.ReportMetric(&metrics.ResourceUpdateInvalid{ + ServerURI: serverConfig.ServerIdentifier.ServerURI, ResourceType: rType.TypeName, + }) + } + + // Notify watchers only if this is not a duplicated error from the previous update. + if errState := state.md.ErrState; errState == nil || errState.Err == nil || state.md.ErrState.Err.Error() != err.Error() { + for watcher := range state.watchers { + watcherCnt.Add(1) + if state.cache == nil { + funcsToSchedule = append(funcsToSchedule, func(context.Context) { watcher.ResourceError(err, done) }) + } else { + funcsToSchedule = append(funcsToSchedule, func(context.Context) { watcher.AmbientError(err, done) }) + } + } + } + + // On error, keep previous version of the resource. But update + // status to NACKed and capture the specific per-resource error (and + // not the overall error for the update) in the ErrState field. + state.md.Status = xdsresource.ServiceStatusNACKed + state.md.ErrState = &xdsresource.UpdateErrorMetadata{ + Version: md.ErrState.Version, + Err: err, + Timestamp: md.ErrState.Timestamp, + } + + continue + } + + if a.metricsReporter != nil { + a.metricsReporter.ReportMetric(&metrics.ResourceUpdateValid{ + ServerURI: serverConfig.ServerIdentifier.ServerURI, ResourceType: rType.TypeName, + }) + } + + if state.deletionIgnored { + state.deletionIgnored = false + a.logger.Infof("A valid update was received for resource %q of type %q after previously ignoring a deletion", name, rType.TypeName) + } + // Notify watchers if any of these conditions are met: + // - this is the first update for this resource + // - this update is different from the one currently cached + // - the previous update for this resource was NACKed, but the update + // before that was the same as this update. + if state.cache == nil || !state.cache.Equal(uErr.Resource) || state.md.ErrState != nil { + // Update the resource cache. + if a.logger.V(2) { + a.logger.Infof("Resource type %q with name %q added to cache", rType.TypeName, name) + } + state.cache = uErr.Resource + + for watcher := range state.watchers { + resource := uErr.Resource + watcherCnt.Add(1) + funcsToSchedule = append(funcsToSchedule, func(context.Context) { watcher.ResourceChanged(resource, done) }) + } + } + + // Set status to ACK, and clear error state. The metadata might be a + // NACK metadata because some other resources in the same response + // are invalid. + state.md = md + state.md.ErrState = nil + state.md.Status = xdsresource.ServiceStatusACKed + if md.ErrState != nil { + state.md.Version = md.ErrState.Version + } + } + + // If this resource type requires that all resources be present in every + // SotW response from the server, a response that does not include a + // previously seen resource will be interpreted as a deletion of that + // resource unless ignore_resource_deletion option was set in the server + // config. + if !rType.AllResourcesRequiredInSotW { + return + } + for name, state := range resourceStates { + if state.cache == nil { + // If the resource state does not contain a cached update, which can + // happen when: + // - resource was newly requested but has not yet been received, or, + // - resource was removed as part of a previous update, + // we don't want to generate an error for the watchers. + // + // For the first of the above two conditions, this ADS response may + // be in reaction to an earlier request that did not yet request the + // new resource, so its absence from the response does not + // necessarily indicate that the resource does not exist. For that + // case, we rely on the request timeout instead. + // + // For the second of the above two conditions, we already generated + // an error when we received the first response which removed this + // resource. So, there is no need to generate another one. + continue + } + if _, ok := updates[name]; ok { + // If the resource was present in the response, move on. + continue + } + if state.md.Status == xdsresource.ServiceStatusNotExist { + // The metadata status is set to "ServiceStatusNotExist" if a + // previous update deleted this resource, in which case we do not + // want to repeatedly call the watch callbacks with a + // "resource-not-found" error. + continue + } + if serverConfig.SupportsServerFeature(ServerFeatureIgnoreResourceDeletion) { + // Per A53, resource deletions are ignored if the + // `ignore_resource_deletion` server feature is enabled through the + // xDS client configuration. If the resource deletion is to be + // ignored, the resource is not removed from the cache and the + // corresponding OnResourceDoesNotExist() callback is not invoked on + // the watchers. + if !state.deletionIgnored { + state.deletionIgnored = true + a.logger.Warningf("Ignoring resource deletion for resource %q of type %q", name, rType.TypeName) + } + continue + } + + // If we get here, it means that the resource exists in cache, but not + // in the new update. Delete the resource from cache, and send a + // resource not found error to indicate that the resource has been + // removed. Metadata for the resource is still maintained, as this is + // required by CSDS. + state.cache = nil + state.md = xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist} + for watcher := range state.watchers { + watcherCnt.Add(1) + funcsToSchedule = append(funcsToSchedule, func(context.Context) { + watcher.ResourceError(xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "xds: resource %q of type %q has been removed", name, rType.TypeName), done) + }) + } + } +} + +// adsResourceDoesNotExist is called by the xDS client implementation (on all +// interested authorities) to notify the authority that a subscribed resource +// does not exist. +func (a *authority) adsResourceDoesNotExist(rType ResourceType, resourceName string) { + a.xdsClientSerializer.TrySchedule(func(context.Context) { + a.handleADSResourceDoesNotExist(rType, resourceName) + }) +} + +// handleADSResourceDoesNotExist is called when a subscribed resource does not +// exist. It removes the resource from the cache, updates the metadata status +// to ServiceStatusNotExist, and notifies all watchers that the resource does +// not exist. +func (a *authority) handleADSResourceDoesNotExist(rType ResourceType, resourceName string) { + if a.logger.V(2) { + a.logger.Infof("Watch for resource %q of type %s timed out", resourceName, rType.TypeName) + } + + resourceStates := a.resources[rType] + if resourceStates == nil { + if a.logger.V(2) { + a.logger.Infof("Resource %q of type %s currently not being watched", resourceName, rType.TypeName) + } + return + } + state, ok := resourceStates[resourceName] + if !ok { + if a.logger.V(2) { + a.logger.Infof("Resource %q of type %s currently not being watched", resourceName, rType.TypeName) + } + return + } + + state.cache = nil + state.md = xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist} + for watcher := range state.watchers { + watcher := watcher + a.watcherCallbackSerializer.TrySchedule(func(context.Context) { + watcher.ResourceError(xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "xds: resource %q of type %q has been removed", resourceName, rType.TypeName), func() {}) + }) + } +} + +// handleRevertingToPrimaryOnUpdate is called when a resource update is received +// from the xDS client. +// +// If the update is from the currently active server, nothing is done. Else, all +// lower priority servers are closed and the active server is reverted to the +// highest priority server that sent the update. +// +// The return value indicates whether subsequent processing of the resource +// update should continue or not. +// +// This method is only executed in the context of a serializer callback. +func (a *authority) handleRevertingToPrimaryOnUpdate(serverConfig *ServerConfig) bool { + if a.activeXDSChannel == nil { + // This can happen only when all watches on this authority have been + // removed, and the xdsChannels have been closed. This update should + // have been received prior to closing of the channel, and therefore + // must be ignored. + return false + } + + if isServerConfigEqual(serverConfig, a.activeXDSChannel.serverConfig) { + // If the resource update is from the current active server, nothing + // needs to be done from fallback point of view. + return true + } + + // If the resource update is not from the current active server, it means + // that we have received an update either from: + // - a server that has a higher priority than the current active server and + // therefore we need to revert back to it and close all lower priority + // servers, or, + // - a server that has a lower priority than the current active server. This + // can happen when the server close and the response race against each + // other. We can safely ignore this update, since we have already reverted + // to the higher priority server, and closed all lower priority servers. + serverIdx := a.serverIndexForConfig(serverConfig) + activeServerIdx := a.serverIndexForConfig(a.activeXDSChannel.serverConfig) + if activeServerIdx < serverIdx { + if a.logger.V(2) { + a.logger.Infof("Ignoring update from lower priority server [%d] %q", serverIdx, serverConfig) + } + return false + } + if a.logger.V(2) { + a.logger.Infof("Received update from higher priority server [%d] %q", serverIdx, serverConfig) + } + + // At this point, we are guaranteed that we have received a response from a + // higher priority server compared to the current active server. So, we + // revert to the higher priorty server and close all lower priority ones. + a.activeXDSChannel = a.xdsChannelConfigs[serverIdx] + + // Close all lower priority channels. + // + // But before closing any channel, we need to unsubscribe from any resources + // that were subscribed to on this channel. Resources could be subscribed to + // from multiple channels as we fallback to lower priority servers. But when + // a higher priority one comes back up, we need to unsubscribe from all + // lower priority ones before releasing the reference to them. + for i := serverIdx + 1; i < len(a.xdsChannelConfigs); i++ { + cfg := a.xdsChannelConfigs[i] + + for rType, rState := range a.resources { + for resourceName, state := range rState { + for xcc := range state.xdsChannelConfigs { + if xcc != cfg { + continue + } + // If the current resource is subscribed to on this channel, + // unsubscribe, and remove the channel from the list of + // channels that this resource is subscribed to. + xcc.channel.unsubscribe(rType, resourceName) + delete(state.xdsChannelConfigs, xcc) + } + } + } + + // Release the reference to the channel. + if cfg.cleanup != nil { + if a.logger.V(2) { + a.logger.Infof("Closing lower priority server [%d] %q", i, cfg.serverConfig) + } + cfg.cleanup() + cfg.cleanup = nil + } + cfg.channel = nil + } + return true +} + +// watchResource registers a new watcher for the specified resource type and +// name. It returns a function that can be called to cancel the watch. +// +// If this is the first watch for any resource on this authority, an xdsChannel +// to the first management server (from the list of server configurations) will +// be created. +// +// If this is the first watch for the given resource name, it will subscribe to +// the resource with the xdsChannel. If a cached copy of the resource exists, it +// will immediately notify the new watcher. When the last watcher for a resource +// is removed, it will unsubscribe the resource from the xdsChannel. +func (a *authority) watchResource(rType ResourceType, resourceName string, watcher ResourceWatcher) func() { + cleanup := func() {} + done := make(chan struct{}) + + a.xdsClientSerializer.ScheduleOr(func(context.Context) { + defer close(done) + + if a.logger.V(2) { + a.logger.Infof("New watch for type %q, resource name %q", rType.TypeName, resourceName) + } + + xdsChannel, err := a.xdsChannelToUse() + if err != nil { + a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.ResourceError(err, func() {}) }) + return + } + + // Lookup the entry for the resource type in the top-level map. If there is + // no entry for this resource type, create one. + resources := a.resources[rType] + if resources == nil { + resources = make(map[string]*resourceState) + a.resources[rType] = resources + } + + // Lookup the resource state for the particular resource name that the watch + // is being registered for. If this is the first watch for this resource + // name, request it from the management server. + state := resources[resourceName] + if state == nil { + if a.logger.V(2) { + a.logger.Infof("First watch for type %q, resource name %q", rType.TypeName, resourceName) + } + state = &resourceState{ + watchers: make(map[ResourceWatcher]bool), + md: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusRequested}, + xdsChannelConfigs: map[*xdsChannelWithConfig]bool{xdsChannel: true}, + } + resources[resourceName] = state + xdsChannel.channel.subscribe(rType, resourceName) + } + // Always add the new watcher to the set of watchers. + state.watchers[watcher] = true + + // If we have a cached copy of the resource, notify the new watcher + // immediately. + if state.cache != nil { + if a.logger.V(2) { + a.logger.Infof("Resource type %q with resource name %q found in cache: %v", rType.TypeName, resourceName, state.cache) + } + // state can only be accessed in the context of an + // xdsClientSerializer callback. Hence making a copy of the cached + // resource here for watchCallbackSerializer. + resource := state.cache + a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.ResourceChanged(resource, func() {}) }) + } + // If last update was NACK'd, notify the new watcher of error + // immediately as well. + if state.md.Status == xdsresource.ServiceStatusNACKed { + if a.logger.V(2) { + a.logger.Infof("Resource type %q with resource name %q was NACKed", rType.TypeName, resourceName) + } + // state can only be accessed in the context of an + // xdsClientSerializer callback. Hence making a copy of the error + // here for watchCallbackSerializer. + err := state.md.ErrState.Err + if state.cache == nil { + a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.ResourceError(err, func() {}) }) + } else { + a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.AmbientError(err, func() {}) }) + } + } + // If the metadata field is updated to indicate that the management + // server does not have this resource, notify the new watcher. + if state.md.Status == xdsresource.ServiceStatusNotExist { + a.watcherCallbackSerializer.TrySchedule(func(context.Context) { + watcher.ResourceError(xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "xds: resource %q of type %q has been removed", resourceName, rType.TypeName), func() {}) + }) + } + cleanup = a.unwatchResource(rType, resourceName, watcher) + }, func() { + if a.logger.V(2) { + a.logger.Infof("Failed to schedule a watch for type %q, resource name %q, because the xDS client is closed", rType.TypeName, resourceName) + } + close(done) + }) + <-done + return cleanup +} + +func (a *authority) unwatchResource(rType ResourceType, resourceName string, watcher ResourceWatcher) func() { + return sync.OnceFunc(func() { + done := make(chan struct{}) + a.xdsClientSerializer.ScheduleOr(func(context.Context) { + defer close(done) + + if a.logger.V(2) { + a.logger.Infof("Canceling a watch for type %q, resource name %q", rType.TypeName, resourceName) + } + + // Lookup the resource type from the resource cache. The entry is + // guaranteed to be present, since *we* were the ones who added it in + // there when the watch was registered. + resources := a.resources[rType] + state := resources[resourceName] + + // Delete this particular watcher from the list of watchers, so that its + // callback will not be invoked in the future. + delete(state.watchers, watcher) + if len(state.watchers) > 0 { + if a.logger.V(2) { + a.logger.Infof("Other watchers exist for type %q, resource name %q", rType.TypeName, resourceName) + } + return + } + + // There are no more watchers for this resource. Unsubscribe this + // resource from all channels where it was subscribed to and delete + // the state associated with it. + if a.logger.V(2) { + a.logger.Infof("Removing last watch for resource name %q", resourceName) + } + for xcc := range state.xdsChannelConfigs { + xcc.channel.unsubscribe(rType, resourceName) + } + delete(resources, resourceName) + + // If there are no more watchers for this resource type, delete the + // resource type from the top-level map. + if len(resources) == 0 { + if a.logger.V(2) { + a.logger.Infof("Removing last watch for resource type %q", rType.TypeName) + } + delete(a.resources, rType) + } + // If there are no more watchers for any resource type, release the + // reference to the xdsChannels. + if len(a.resources) == 0 { + if a.logger.V(2) { + a.logger.Infof("Removing last watch for any resource type, releasing reference to the xdsChannel") + } + a.closeXDSChannels() + } + }, func() { close(done) }) + <-done + }) +} + +// xdsChannelToUse returns the xdsChannel to use for communicating with the +// management server. If an active channel is available, it returns that. +// Otherwise, it creates a new channel using the first server configuration in +// the list of configurations, and returns that. +// +// A non-nil error is returned if the channel creation fails. +// +// Only executed in the context of a serializer callback. +func (a *authority) xdsChannelToUse() (*xdsChannelWithConfig, error) { + if a.activeXDSChannel != nil { + return a.activeXDSChannel, nil + } + + sc := a.xdsChannelConfigs[0].serverConfig + xc, cleanup, err := a.getChannelForADS(sc, a) + if err != nil { + return nil, err + } + a.xdsChannelConfigs[0].channel = xc + a.xdsChannelConfigs[0].cleanup = cleanup + a.activeXDSChannel = a.xdsChannelConfigs[0] + return a.activeXDSChannel, nil +} + +// closeXDSChannels closes all the xDS channels associated with this authority, +// when there are no more watchers for any resource type. +// +// Only executed in the context of a serializer callback. +func (a *authority) closeXDSChannels() { + for _, xcc := range a.xdsChannelConfigs { + if xcc.cleanup != nil { + xcc.cleanup() + xcc.cleanup = nil + } + xcc.channel = nil + } + a.activeXDSChannel = nil +} + +// watcherExistsForUncachedResource returns true if there is at least one +// watcher for a resource that has not yet been cached. +// +// Only executed in the context of a serializer callback. +func (a *authority) watcherExistsForUncachedResource() bool { + for _, resourceStates := range a.resources { + for _, state := range resourceStates { + if state.md.Status == xdsresource.ServiceStatusRequested { + return true + } + } + } + return false +} + +// dumpResources returns a dump of the resource configuration cached by this +// authority, for CSDS purposes. +func (a *authority) dumpResources() []*v3statuspb.ClientConfig_GenericXdsConfig { + var ret []*v3statuspb.ClientConfig_GenericXdsConfig + done := make(chan struct{}) + + a.xdsClientSerializer.ScheduleOr(func(context.Context) { + defer close(done) + ret = a.resourceConfig() + }, func() { close(done) }) + <-done + return ret +} + +// resourceConfig returns a slice of GenericXdsConfig objects representing the +// current state of all resources managed by this authority. This is used for +// reporting the current state of the xDS client. +// +// Only executed in the context of a serializer callback. +func (a *authority) resourceConfig() []*v3statuspb.ClientConfig_GenericXdsConfig { + var ret []*v3statuspb.ClientConfig_GenericXdsConfig + for rType, resourceStates := range a.resources { + typeURL := rType.TypeURL + for name, state := range resourceStates { + var raw *anypb.Any + if state.cache != nil { + raw = &anypb.Any{TypeUrl: typeURL, Value: state.cache.Bytes()} + } + config := &v3statuspb.ClientConfig_GenericXdsConfig{ + TypeUrl: typeURL, + Name: name, + VersionInfo: state.md.Version, + XdsConfig: raw, + LastUpdated: timestamppb.New(state.md.Timestamp), + ClientStatus: serviceStatusToProto(state.md.Status), + } + if errState := state.md.ErrState; errState != nil { + config.ErrorState = &v3adminpb.UpdateFailureState{ + LastUpdateAttempt: timestamppb.New(errState.Timestamp), + Details: errState.Err.Error(), + VersionInfo: errState.Version, + } + } + ret = append(ret, config) + } + } + return ret +} + +func (a *authority) close() { + a.xdsClientSerializerClose() + <-a.xdsClientSerializer.Done() + if a.logger.V(2) { + a.logger.Infof("Closed") + } +} + +func serviceStatusToProto(serviceStatus xdsresource.ServiceStatus) v3adminpb.ClientResourceStatus { + switch serviceStatus { + case xdsresource.ServiceStatusUnknown: + return v3adminpb.ClientResourceStatus_UNKNOWN + case xdsresource.ServiceStatusRequested: + return v3adminpb.ClientResourceStatus_REQUESTED + case xdsresource.ServiceStatusNotExist: + return v3adminpb.ClientResourceStatus_DOES_NOT_EXIST + case xdsresource.ServiceStatusACKed: + return v3adminpb.ClientResourceStatus_ACKED + case xdsresource.ServiceStatusNACKed: + return v3adminpb.ClientResourceStatus_NACKED + default: + return v3adminpb.ClientResourceStatus_UNKNOWN + } +} + +func (a *authority) resourceWatchStateForTesting(rType ResourceType, resourceName string) (state xdsresource.ResourceWatchState, err error) { + done := make(chan struct{}) + a.xdsClientSerializer.ScheduleOr(func(context.Context) { + state, err = a.activeXDSChannel.channel.ads.adsResourceWatchStateForTesting(rType, resourceName) + close(done) + }, func() { + err = errors.New("failed to retrieve resource watch state because the xDS client is closed") + close(done) + }) + <-done + + return state, err +} + +// resourceStats returns a snapshot of the current state of all resources watched +// by this authority. The return value is a nested map where: +// - The outer map's key is the resource type name (e.g., "ListenerResource"). +// - The inner map's key is the cache state of the resource (e.g., "requested", +// "acked", "nacked", "does_not_exist"). +// - The inner map's value is the total count of resources in that specific state. +func (a *authority) resourceStats() map[string]map[string]int { + ret := make(chan map[string]map[string]int, 1) + op := func(context.Context) { + summary := make(map[string]map[string]int) + for rType, resourceMap := range a.resources { + typeName := rType.TypeName + if _, ok := summary[typeName]; !ok { + summary[typeName] = make(map[string]int) + } + for _, state := range resourceMap { + s := cacheState(state) + summary[typeName][s]++ + } + } + + ret <- summary + } + a.xdsClientSerializer.ScheduleOr(op, func() { + ret <- nil + }) + + return <-ret +} + +// cacheState determines the metrics label string for a given resource state. +func cacheState(r *resourceState) string { + switch r.md.Status { + case xdsresource.ServiceStatusRequested: + return "requested" + case xdsresource.ServiceStatusNotExist: + return "does_not_exist" + case xdsresource.ServiceStatusACKed: + return "acked" + case xdsresource.ServiceStatusNACKed: + // If the status is NACKed, it means the *latest* update failed. + // However, if 'r.cache' is not nil, it means we are still holding onto + // a previously ACKed version of the resource. + if r.cache != nil { + return "nacked_but_cached" + } + return "nacked" + default: + // Fallback for initialization states + return "requested" + } +} diff --git a/internal/xds/clients/xdsclient/channel.go b/internal/xds/clients/xdsclient/channel.go new file mode 100644 index 0000000..354c049 --- /dev/null +++ b/internal/xds/clients/xdsclient/channel.go @@ -0,0 +1,329 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "errors" + "fmt" + "strings" + "time" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/envconfig" + igrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/internal" + "google.golang.org/grpc/internal/xds/clients/internal/backoff" + "google.golang.org/grpc/internal/xds/clients/internal/syncutil" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" +) + +const ( + clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" + clientFeatureResourceWrapper = "xds.config.resource-in-sotw" +) + +// xdsChannelEventHandler wraps callbacks used to notify the xDS client about +// events on the xdsChannel. Methods in this interface may be invoked +// concurrently and the xDS client implementation needs to handle them in a +// thread-safe manner. +type xdsChannelEventHandler interface { + // adsStreamFailure is called when the xdsChannel encounters an ADS stream + // failure. + adsStreamFailure(error) + + // adsResourceUpdate is called when the xdsChannel receives an ADS response + // from the xDS management server. The callback is provided with the + // following: + // - the resource type of the resources in the response + // - a map of resources in the response, keyed by resource name + // - the metadata associated with the response + // - a callback to be invoked when the updated is processed + adsResourceUpdate(ResourceType, map[string]dataAndErrTuple, xdsresource.UpdateMetadata, func()) + + // adsResourceDoesNotExist is called when the xdsChannel determines that a + // requested ADS resource does not exist. + adsResourceDoesNotExist(ResourceType, string) +} + +// xdsChannelOpts holds the options for creating a new xdsChannel. +type xdsChannelOpts struct { + transport clients.Transport // Takes ownership of this transport. + serverConfig *ServerConfig // Configuration of the server to connect to. + clientConfig *Config // Complete xDS client configuration, used to decode resources. + eventHandler xdsChannelEventHandler // Callbacks for ADS stream events. + backoff func(int) time.Duration // Backoff function to use for stream retries. Defaults to exponential backoff, if unset. + watchExpiryTimeout time.Duration // Timeout for ADS resource watch expiry. + logPrefix string // Prefix to use for logging. +} + +// newXDSChannel creates a new xdsChannel instance with the provided options. +// It performs basic validation on the provided options and initializes the +// xdsChannel with the necessary components. +func newXDSChannel(opts xdsChannelOpts) (*xdsChannel, error) { + switch { + case opts.transport == nil: + return nil, errors.New("xdsclient: transport is nil") + case opts.serverConfig == nil: + return nil, errors.New("xdsclient: serverConfig is nil") + case opts.clientConfig == nil: + return nil, errors.New("xdsclient: clientConfig is nil") + case opts.eventHandler == nil: + return nil, errors.New("xdsclient: eventHandler is nil") + } + + xc := &xdsChannel{ + transport: opts.transport, + serverConfig: opts.serverConfig, + clientConfig: opts.clientConfig, + eventHandler: opts.eventHandler, + closed: syncutil.NewEvent(), + } + + l := grpclog.Component("xds") + logPrefix := opts.logPrefix + fmt.Sprintf("[xds-channel %p] ", xc) + xc.logger = igrpclog.NewPrefixLogger(l, logPrefix) + + if opts.backoff == nil { + opts.backoff = backoff.DefaultExponential.Backoff + } + np := internal.NodeProto(opts.clientConfig.Node) + np.ClientFeatures = []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper} + xc.ads = newADSStreamImpl(adsStreamOpts{ + transport: opts.transport, + eventHandler: xc, + backoff: opts.backoff, + nodeProto: np, + watchExpiryTimeout: opts.watchExpiryTimeout, + logPrefix: logPrefix, + }) + if xc.logger.V(2) { + xc.logger.Infof("xdsChannel is created for ServerConfig %v", opts.serverConfig) + } + return xc, nil +} + +// xdsChannel represents a client channel to a management server, and is +// responsible for managing the lifecycle of the ADS and LRS streams. It invokes +// callbacks on the registered event handler for various ADS stream events. +// +// It is safe for concurrent use. +type xdsChannel struct { + // The following fields are initialized at creation time and are read-only + // after that, and hence need not be guarded by a mutex. + transport clients.Transport // Takes ownership of this transport (used to make streaming calls). + ads *adsStreamImpl // An ADS stream to the management server. + serverConfig *ServerConfig // Configuration of the server to connect to. + clientConfig *Config // Complete xDS client configuration, used to decode resources. + eventHandler xdsChannelEventHandler // Callbacks for ADS stream events. + logger *igrpclog.PrefixLogger // Logger to use for logging. + closed *syncutil.Event // Fired when the channel is closed. +} + +func (xc *xdsChannel) close() { + xc.closed.Fire() + xc.ads.Stop() + xc.transport.Close() + xc.logger.Infof("Shutdown") +} + +// subscribe adds a subscription for the given resource name of the given +// resource type on the ADS stream. +func (xc *xdsChannel) subscribe(typ ResourceType, name string) { + if xc.closed.HasFired() { + if xc.logger.V(2) { + xc.logger.Infof("Attempt to subscribe to an xDS resource of type %s and name %q on a closed channel", typ.TypeName, name) + } + return + } + xc.ads.subscribe(typ, name) +} + +// unsubscribe removes the subscription for the given resource name of the given +// resource type from the ADS stream. +func (xc *xdsChannel) unsubscribe(typ ResourceType, name string) { + if xc.closed.HasFired() { + if xc.logger.V(2) { + xc.logger.Infof("Attempt to unsubscribe to an xDS resource of type %s and name %q on a closed channel", typ.TypeName, name) + } + return + } + xc.ads.unsubscribe(typ, name) +} + +// The following onADSXxx() methods implement the StreamEventHandler interface +// and are invoked by the ADS stream implementation. + +// onStreamError is invoked when an error occurs on the ADS stream. It +// propagates the update to the xDS client. +func (xc *xdsChannel) onStreamError(err error) { + if xc.closed.HasFired() { + if xc.logger.V(2) { + xc.logger.Infof("Received ADS stream error on a closed xdsChannel: %v", err) + } + return + } + xc.eventHandler.adsStreamFailure(err) +} + +// onWatchExpiry is invoked when a watch for a resource expires. It +// propagates the update to the xDS client. +func (xc *xdsChannel) onWatchExpiry(typ ResourceType, name string) { + if xc.closed.HasFired() { + if xc.logger.V(2) { + xc.logger.Infof("Received ADS resource watch expiry for resource %q on a closed xdsChannel", name) + } + return + } + xc.eventHandler.adsResourceDoesNotExist(typ, name) +} + +// onResponse is invoked when a response is received on the ADS stream. It +// decodes the resources in the response, and propagates the updates to the xDS +// client. +// +// It returns the list of resource names in the response and any errors +// encountered during decoding. +func (xc *xdsChannel) onResponse(resp response, onDone func()) ([]string, error) { + if xc.closed.HasFired() { + if xc.logger.V(2) { + xc.logger.Infof("Received an update from the ADS stream on closed ADS stream") + } + return nil, errors.New("xdsChannel is closed") + } + + // Lookup the resource parser based on the resource type. + rType, ok := xc.clientConfig.ResourceTypes[resp.typeURL] + if !ok { + return nil, xdsresource.NewErrorf(xdsresource.ErrorTypeResourceTypeUnsupported, "Resource type URL %q unknown in response from server", resp.typeURL) + } + + updates, md, err := xc.decodeResponse(&rType, resp) + var names []string + for name := range updates { + names = append(names, name) + } + + xc.eventHandler.adsResourceUpdate(rType, updates, md, onDone) + return names, err +} + +// decodeResponse decodes the resources in the given ADS response. +// +// The opts parameter provides configuration options for decoding the resources. +// The rType parameter specifies the resource type parser to use for decoding +// the resources. +// +// The returned map contains a key for each resource in the response, with the +// value being either the decoded resource data or an error if decoding failed. +// The returned metadata includes the version of the response, the timestamp of +// the update, and the status of the update (ACKed or NACKed). +// +// If there are any errors decoding the resources, the metadata will indicate +// that the update was NACKed, and the returned error will contain information +// about all errors encountered by this function. +func (xc *xdsChannel) decodeResponse(rType *ResourceType, resp response) (map[string]dataAndErrTuple, xdsresource.UpdateMetadata, error) { + timestamp := time.Now() + md := xdsresource.UpdateMetadata{ + Version: resp.version, + Timestamp: timestamp, + } + opts := &DecodeOptions{ + Config: xc.clientConfig, + ServerConfig: xc.serverConfig, + } + + topLevelErrors := make([]error, 0) // Tracks deserialization errors, where we don't have a resource name. + perResourceErrors := make(map[string]error) // Tracks resource validation errors, where we have a resource name. + ret := make(map[string]dataAndErrTuple) // Return result, a map from resource name to either resource data or error. + for _, r := range resp.resources { + result, err := func() (res *DecodeResult, err error) { + defer func() { + if envconfig.XDSRecoverPanicInResourceParsing { + if p := recover(); p != nil { + err = fmt.Errorf("recovered from panic during resource parsing, resource: %v, panic: %v", r, p) + } + } + }() + return rType.Decoder.Decode(NewAnyProto(r), *opts) + }() + + // Name field of the result is left unpopulated only when resource + // deserialization fails. + name := "" + if result == nil && err == nil { + xc.logger.Errorf("Decode() returned nil result and nil error for resource: %v", r) + continue + } + if result != nil { + name = xdsresource.ParseName(result.Name).String() + } + if err == nil { + ret[name] = dataAndErrTuple{Resource: result.Resource} + continue + } + if name == "" { + topLevelErrors = append(topLevelErrors, err) + continue + } + perResourceErrors[name] = err + // Add place holder in the map so we know this resource name was in + // the response. + ret[name] = dataAndErrTuple{Err: xdsresource.NewError(xdsresource.ErrorTypeNACKed, err.Error())} + } + + if len(topLevelErrors) == 0 && len(perResourceErrors) == 0 { + md.Status = xdsresource.ServiceStatusACKed + return ret, md, nil + } + + md.Status = xdsresource.ServiceStatusNACKed + errRet := combineErrors(rType.TypeName, topLevelErrors, perResourceErrors) + md.ErrState = &xdsresource.UpdateErrorMetadata{ + Version: resp.version, + Err: xdsresource.NewError(xdsresource.ErrorTypeNACKed, errRet.Error()), + Timestamp: timestamp, + } + return ret, md, errRet +} + +func combineErrors(rType string, topLevelErrors []error, perResourceErrors map[string]error) error { + var errStrB strings.Builder + errStrB.WriteString(fmt.Sprintf("error parsing %q response: ", rType)) + if len(topLevelErrors) > 0 { + errStrB.WriteString("top level errors: ") + for i, err := range topLevelErrors { + if i != 0 { + errStrB.WriteString(";\n") + } + errStrB.WriteString(err.Error()) + } + } + if len(perResourceErrors) > 0 { + var i int + for name, err := range perResourceErrors { + if i != 0 { + errStrB.WriteString(";\n") + } + i++ + errStrB.WriteString(fmt.Sprintf("resource %q: %v", name, err.Error())) + } + } + return errors.New(errStrB.String()) +} diff --git a/internal/xds/clients/xdsclient/channel_test.go b/internal/xds/clients/xdsclient/channel_test.go new file mode 100644 index 0000000..b471a2a --- /dev/null +++ b/internal/xds/clients/xdsclient/channel_test.go @@ -0,0 +1,821 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "context" + "fmt" + "net" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + xdstestutils "google.golang.org/grpc/internal/xds/clients/internal/testutils" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +// xdsChannelForTest creates an xdsChannel to the specified serverURI for +// testing purposes. +func xdsChannelForTest(t *testing.T, serverURI, nodeID string, watchExpiryTimeout time.Duration) *xdsChannel { + t.Helper() + + // Create a grpc transport to the above management server. + si := clients.ServerIdentifier{ + ServerURI: serverURI, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + tr, err := (grpctransport.NewBuilder(configs)).Build(si) + if err != nil { + t.Fatalf("Failed to create a transport for server config %v: %v", si, err) + } + + serverCfg := ServerConfig{ + ServerIdentifier: si, + } + clientConfig := Config{ + Servers: []ServerConfig{serverCfg}, + Node: clients.Node{ID: nodeID}, + ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType}, + } + // Create an xdsChannel that uses everything set up above. + xc, err := newXDSChannel(xdsChannelOpts{ + transport: tr, + serverConfig: &serverCfg, + clientConfig: &clientConfig, + eventHandler: newTestEventHandler(), + watchExpiryTimeout: watchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create xdsChannel: %v", err) + } + t.Cleanup(func() { xc.close() }) + return xc +} + +// verifyUpdateAndMetadata verifies that the event handler received the expected +// updates and metadata. It checks that the received resource type matches the +// expected type, and that the received updates and metadata match the expected +// values. The function ignores the timestamp fields in the metadata, as those +// are expected to be different. +func verifyUpdateAndMetadata(ctx context.Context, t *testing.T, eh *testEventHandler, wantUpdates map[string]dataAndErrTuple, wantMD xdsresource.UpdateMetadata) { + t.Helper() + + gotTyp, gotUpdates, gotMD, err := eh.waitForUpdate(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for update callback to be invoked on the event handler") + } + + if gotTyp != listenerType { + t.Fatalf("Got resource type %v, want %v", gotTyp, listenerType) + } + opts := cmp.Options{ + protocmp.Transform(), + cmpopts.EquateEmpty(), + cmpopts.EquateErrors(), + cmpopts.IgnoreFields(xdsresource.UpdateMetadata{}, "Timestamp"), + cmpopts.IgnoreFields(xdsresource.UpdateErrorMetadata{}, "Timestamp"), + } + if diff := cmp.Diff(wantUpdates, gotUpdates, opts); diff != "" { + t.Fatalf("Got unexpected diff in update (-want +got):\n%s\n want: %+v\n got: %+v", diff, wantUpdates, gotUpdates) + } + if diff := cmp.Diff(wantMD, gotMD, opts); diff != "" { + t.Fatalf("Got unexpected diff in update (-want +got):\n%s\n want: %v\n got: %v", diff, wantMD, gotMD) + } +} + +// Tests different failure cases when creating a new xdsChannel. It checks that +// the xdsChannel creation fails when any of the required options (transport, +// serverConfig, bootstrapConfig, or resourceTypeGetter) are missing or nil. +func (s) TestChannel_New_FailureCases(t *testing.T) { + type fakeTransport struct { + clients.Transport + } + + tests := []struct { + name string + opts xdsChannelOpts + wantErrStr string + }{ + { + name: "emptyTransport", + opts: xdsChannelOpts{}, + wantErrStr: "transport is nil", + }, + { + name: "emptyServerConfig", + opts: xdsChannelOpts{transport: &fakeTransport{}}, + wantErrStr: "serverConfig is nil", + }, + { + name: "emptyCConfig", + opts: xdsChannelOpts{ + transport: &fakeTransport{}, + serverConfig: &ServerConfig{}, + }, + wantErrStr: "clientConfig is nil", + }, + { + name: "emptyEventHandler", + opts: xdsChannelOpts{ + transport: &fakeTransport{}, + serverConfig: &ServerConfig{}, + clientConfig: &Config{}, + }, + wantErrStr: "eventHandler is nil", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := newXDSChannel(test.opts); err == nil || !strings.Contains(err.Error(), test.wantErrStr) { + t.Fatalf("newXDSChannel() = %v, want %q", err, test.wantErrStr) + } + }) + } +} + +// Tests different scenarios of the xdsChannel receiving a response from the +// management server. In all scenarios, the xdsChannel is expected to pass the +// received responses as-is to the resource parsing functionality specified by +// the resourceTypeGetter. +func (s) TestChannel_ADS_HandleResponseFromManagementServer(t *testing.T) { + const ( + listenerName1 = "listener-name-1" + listenerName2 = "listener-name-2" + routeName = "route-name" + clusterName = "cluster-name" + ) + var ( + badlyMarshaledResource = &anypb.Any{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Value: []byte{1, 2, 3, 4}, + } + apiListener = &v3listenerpb.ApiListener{ + ApiListener: xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: routeName}, + }, + }), + } + listener1 = xdstestutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: listenerName1, + ApiListener: apiListener, + }) + listener2 = xdstestutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: listenerName2, + ApiListener: apiListener, + }) + ) + + tests := []struct { + desc string + resourceNamesToRequest []string + managementServerResponse *v3discoverypb.DiscoveryResponse + wantUpdates map[string]dataAndErrTuple + wantMD xdsresource.UpdateMetadata + wantErr error + }{ + { + desc: "one bad resource - deserialization failure", + resourceNamesToRequest: []string{listenerName1}, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + VersionInfo: "0", + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Resources: []*anypb.Any{badlyMarshaledResource}, + }, + wantUpdates: nil, // No updates expected as the response runs into unmarshaling errors. + wantMD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusNACKed, + Version: "0", + ErrState: &xdsresource.UpdateErrorMetadata{ + Version: "0", + Err: cmpopts.AnyError, + }, + }, + wantErr: cmpopts.AnyError, + }, + { + desc: "one bad resource - validation failure", + resourceNamesToRequest: []string{listenerName1}, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + VersionInfo: "0", + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Resources: []*anypb.Any{xdstestutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: listenerName1, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{}, + }), + }, + })}, + }, + wantUpdates: map[string]dataAndErrTuple{ + listenerName1: { + Err: cmpopts.AnyError, + }, + }, + wantMD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusNACKed, + Version: "0", + ErrState: &xdsresource.UpdateErrorMetadata{ + Version: "0", + Err: cmpopts.AnyError, + }, + }, + }, + { + desc: "two bad resources", + resourceNamesToRequest: []string{listenerName1, listenerName2}, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + VersionInfo: "0", + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Resources: []*anypb.Any{ + badlyMarshaledResource, + xdstestutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: listenerName2, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{}, + }), + }, + }), + }, + }, + wantUpdates: map[string]dataAndErrTuple{ + listenerName2: { + Err: cmpopts.AnyError, + }, + }, + wantMD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusNACKed, + Version: "0", + ErrState: &xdsresource.UpdateErrorMetadata{ + Version: "0", + Err: cmpopts.AnyError, + }, + }, + }, + { + desc: "one good resource", + resourceNamesToRequest: []string{listenerName1}, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + VersionInfo: "0", + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Resources: []*anypb.Any{listener1}, + }, + wantUpdates: map[string]dataAndErrTuple{ + listenerName1: { + Resource: &listenerResourceData{Resource: listenerUpdate{ + RouteConfigName: routeName, + Raw: listener1.GetValue(), + }}, + }, + }, + wantMD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusACKed, + Version: "0", + }, + }, + { + desc: "one good and one bad - deserialization failure", + resourceNamesToRequest: []string{listenerName1, listenerName2}, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + VersionInfo: "0", + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Resources: []*anypb.Any{ + badlyMarshaledResource, + listener2, + }, + }, + wantUpdates: map[string]dataAndErrTuple{ + listenerName2: { + Resource: &listenerResourceData{Resource: listenerUpdate{ + RouteConfigName: routeName, + Raw: listener2.GetValue(), + }}, + }, + }, + wantMD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusNACKed, + Version: "0", + ErrState: &xdsresource.UpdateErrorMetadata{ + Version: "0", + Err: cmpopts.AnyError, + }, + }, + }, + { + desc: "one good and one bad - validation failure", + resourceNamesToRequest: []string{listenerName1, listenerName2}, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + VersionInfo: "0", + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Resources: []*anypb.Any{ + xdstestutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: listenerName1, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{}, + }), + }, + }), + listener2, + }, + }, + wantUpdates: map[string]dataAndErrTuple{ + listenerName1: {Err: cmpopts.AnyError}, + listenerName2: { + Resource: &listenerResourceData{Resource: listenerUpdate{ + RouteConfigName: routeName, + Raw: listener2.GetValue(), + }}, + }, + }, + wantMD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusNACKed, + Version: "0", + ErrState: &xdsresource.UpdateErrorMetadata{ + Version: "0", + Err: cmpopts.AnyError, + }, + }, + }, + { + desc: "two good resources", + resourceNamesToRequest: []string{listenerName1, listenerName2}, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + VersionInfo: "0", + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Resources: []*anypb.Any{listener1, listener2}, + }, + wantUpdates: map[string]dataAndErrTuple{ + listenerName1: { + Resource: &listenerResourceData{Resource: listenerUpdate{ + RouteConfigName: routeName, + Raw: listener1.GetValue(), + }}, + }, + listenerName2: { + Resource: &listenerResourceData{Resource: listenerUpdate{ + RouteConfigName: routeName, + Raw: listener2.GetValue(), + }}, + }, + }, + wantMD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusACKed, + Version: "0", + }, + }, + { + desc: "two resources when we requested one", + resourceNamesToRequest: []string{listenerName1}, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + VersionInfo: "0", + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Resources: []*anypb.Any{listener1, listener2}, + }, + wantUpdates: map[string]dataAndErrTuple{ + listenerName1: { + Resource: &listenerResourceData{Resource: listenerUpdate{ + RouteConfigName: routeName, + Raw: listener1.GetValue(), + }}, + }, + listenerName2: { + Resource: &listenerResourceData{Resource: listenerUpdate{ + RouteConfigName: routeName, + Raw: listener2.GetValue(), + }}, + }, + }, + wantMD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusACKed, + Version: "0", + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start a fake xDS management server and configure the response it + // would send to its client. + mgmtServer, cleanup, err := fakeserver.StartServer(nil) + if err != nil { + t.Fatalf("Failed to start fake xDS server: %v", err) + } + defer cleanup() + t.Logf("Started xDS management server on %s", mgmtServer.Address) + mgmtServer.XDSResponseChan <- &fakeserver.Response{Resp: test.managementServerResponse} + + // Create an xdsChannel for the test with a long watch expiry timer + // to ensure that watches don't expire for the duration of the test. + nodeID := uuid.New().String() + xc := xdsChannelForTest(t, mgmtServer.Address, nodeID, 2*defaultTestTimeout) + defer xc.close() + + // Subscribe to the resources specified in the test table. + for _, name := range test.resourceNamesToRequest { + xc.subscribe(listenerType, name) + } + + // Wait for an update callback on the event handler and verify the + // contents of the update and the metadata. + verifyUpdateAndMetadata(ctx, t, xc.eventHandler.(*testEventHandler), test.wantUpdates, test.wantMD) + }) + } +} + +// Tests that the xdsChannel correctly handles the expiry of a watch for a +// resource by ensuring that the watch expiry callback is invoked on the event +// handler with the expected resource type and name. +func (s) TestChannel_ADS_HandleResponseWatchExpiry(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server, but do not configure any resources on it. + // This will result in the watch for a resource to timeout. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create an xdsChannel for the test with a short watch expiry timer to + // ensure that the test does not run very long, as it needs to wait for the + // watch to expire. + nodeID := uuid.New().String() + xc := xdsChannelForTest(t, mgmtServer.Address, nodeID, 2*defaultTestShortTimeout) + defer xc.close() + + // Subscribe to a listener resource. + const listenerName = "listener-name" + xc.subscribe(listenerType, listenerName) + + // Wait for the watch expiry callback on the authority to be invoked and + // verify that the watch expired for the expected resource name and type. + eventHandler := xc.eventHandler.(*testEventHandler) + gotTyp, gotName, err := eventHandler.waitForResourceDoesNotExist(ctx) + if err != nil { + t.Fatal("Timeout when waiting for the watch expiry callback to be invoked on the xDS client") + } + + if gotTyp != listenerType { + t.Fatalf("Got type %v, want %v", gotTyp, listenerType) + } + if gotName != listenerName { + t.Fatalf("Got name %v, want %v", gotName, listenerName) + } +} + +// Tests that the xdsChannel correctly handles stream failures by ensuring that +// the stream failure callback is invoked on the event handler. +func (s) TestChannel_ADS_StreamFailure(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20000*defaultTestTimeout) + defer cancel() + + // Start an xDS management server with a restartable listener to simulate + // connection failures. + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + lis := xdstestutils.NewRestartableListener(l) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis}) + + // Configure a listener resource on the management server. + const listenerResourceName = "test-listener-resource" + const routeConfigurationName = "test-route-configuration-resource" + nodeID := uuid.New().String() + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Create an xdsChannel for the test with a long watch expiry timer + // to ensure that watches don't expire for the duration of the test. + xc := xdsChannelForTest(t, mgmtServer.Address, nodeID, 2000*defaultTestTimeout) + defer xc.close() + + // Subscribe to the resource created above. + xc.subscribe(listenerType, listenerResourceName) + + // Wait for an update callback on the event handler and verify the + // contents of the update and the metadata. + hcm := xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: routeConfigurationName, + }}, + }) + listenerResource, err := anypb.New(&v3listenerpb.Listener{ + Name: listenerResourceName, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + }) + if err != nil { + t.Fatalf("Failed to create listener resource: %v", err) + } + + wantUpdates := map[string]dataAndErrTuple{ + listenerResourceName: { + Resource: &listenerResourceData{ + Resource: listenerUpdate{ + RouteConfigName: routeConfigurationName, + Raw: listenerResource.GetValue(), + }, + }, + }, + } + wantMD := xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusACKed, + Version: "1", + } + + eventHandler := xc.eventHandler.(*testEventHandler) + verifyUpdateAndMetadata(ctx, t, eventHandler, wantUpdates, wantMD) + + lis.Stop() + if err := eventHandler.waitForStreamFailure(ctx); err != nil { + t.Fatalf("Timeout when waiting for the stream failure callback to be invoked on the xDS client: %v", err) + } +} + +// Tests the behavior of the xdsChannel when a resource is unsubscribed. +// Verifies that when a previously subscribed resource is unsubscribed, a +// request is sent without the previously subscribed resource name. +func (s) TestChannel_ADS_ResourceUnsubscribe(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server that uses a channel to inform the test + // about the specific LDS resource names being requested. + ldsResourcesCh := make(chan []string, 1) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + t.Logf("Received request for resources: %v of type %s", req.GetResourceNames(), req.GetTypeUrl()) + + if req.TypeUrl != xdsresource.V3ListenerURL { + return fmt.Errorf("unexpected resource type URL: %q", req.TypeUrl) + } + + // Make the most recently requested names available to the test. + ldsResourcesCh <- req.GetResourceNames() + return nil + }, + }) + + // Configure two listener resources on the management server. + const listenerResourceName1 = "test-listener-resource-1" + const routeConfigurationName1 = "test-route-configuration-resource-1" + const listenerResourceName2 = "test-listener-resource-2" + const routeConfigurationName2 = "test-route-configuration-resource-2" + nodeID := uuid.New().String() + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(listenerResourceName1, routeConfigurationName1), + e2e.DefaultClientListener(listenerResourceName2, routeConfigurationName2), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Create an xdsChannel for the test with a long watch expiry timer + // to ensure that watches don't expire for the duration of the test. + xc := xdsChannelForTest(t, mgmtServer.Address, nodeID, 2*defaultTestTimeout) + defer xc.close() + + // Subscribe to the resources created above and verify that a request is + // sent for the same. + xc.subscribe(listenerType, listenerResourceName1) + xc.subscribe(listenerType, listenerResourceName2) + if err := waitForResourceNames(ctx, ldsResourcesCh, []string{listenerResourceName1, listenerResourceName2}); err != nil { + t.Fatal(err) + } + + // Wait for the above resources to be ACKed. + if err := waitForResourceNames(ctx, ldsResourcesCh, []string{listenerResourceName1, listenerResourceName2}); err != nil { + t.Fatal(err) + } + + // Unsubscribe to one of the resources created above, and ensure that the + // other resource is still being requested. + xc.unsubscribe(listenerType, listenerResourceName1) + if err := waitForResourceNames(ctx, ldsResourcesCh, []string{listenerResourceName2}); err != nil { + t.Fatal(err) + } + + // Since the version on the management server for the above resource is not + // changed, we will not receive an update from it for the one resource that + // we are still requesting. + + // Unsubscribe to the remaining resource, and ensure that no more resources + // are being requested. + xc.unsubscribe(listenerType, listenerResourceName2) + if err := waitForResourceNames(ctx, ldsResourcesCh, []string{}); err != nil { + t.Fatal(err) + } +} + +// waitForResourceNames waits for the wantNames to be received on namesCh. +// Returns a non-nil error if the context expires before that. +func waitForResourceNames(ctx context.Context, namesCh chan []string, wantNames []string) error { + var lastRequestedNames []string + for ; ; <-time.After(defaultTestShortTimeout) { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for resources %v to be requested from the management server. Last requested resources: %v", wantNames, lastRequestedNames) + case gotNames := <-namesCh: + if cmp.Equal(gotNames, wantNames, cmpopts.EquateEmpty(), cmpopts.SortSlices(func(s1, s2 string) bool { return s1 < s2 })) { + return nil + } + lastRequestedNames = gotNames + } + } +} + +// newTestEventHandler creates a new testEventHandler instance with the +// necessary channels for testing the xdsChannel. +func newTestEventHandler() *testEventHandler { + return &testEventHandler{ + typeCh: make(chan ResourceType, 1), + updateCh: make(chan map[string]dataAndErrTuple, 1), + mdCh: make(chan xdsresource.UpdateMetadata, 1), + nameCh: make(chan string, 1), + connErrCh: make(chan error, 1), + } +} + +// testEventHandler is a struct that implements the xdsChannelEventhandler +// interface. It is used to receive events from an xdsChannel, and has multiple +// channels on which it makes these events available to the test. +type testEventHandler struct { + typeCh chan ResourceType // Resource type of an update or resource-does-not-exist error. + updateCh chan map[string]dataAndErrTuple // Resource updates. + mdCh chan xdsresource.UpdateMetadata // Metadata from an update. + nameCh chan string // Name of the non-existent resource. + connErrCh chan error // Connectivity error. + +} + +func (ta *testEventHandler) adsStreamFailure(err error) { + ta.connErrCh <- err +} + +func (ta *testEventHandler) waitForStreamFailure(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ta.connErrCh: + } + return nil +} + +func (ta *testEventHandler) adsResourceUpdate(typ ResourceType, updates map[string]dataAndErrTuple, md xdsresource.UpdateMetadata, onDone func()) { + ta.typeCh <- typ + ta.updateCh <- updates + ta.mdCh <- md + onDone() +} + +// waitForUpdate waits for the next resource update event from the xdsChannel. +// It returns the resource type, the resource updates, and the update metadata. +// If the context is canceled, it returns an error. +func (ta *testEventHandler) waitForUpdate(ctx context.Context) (ResourceType, map[string]dataAndErrTuple, xdsresource.UpdateMetadata, error) { + var typ ResourceType + var updates map[string]dataAndErrTuple + var md xdsresource.UpdateMetadata + + select { + case typ = <-ta.typeCh: + case <-ctx.Done(): + return ResourceType{}, nil, xdsresource.UpdateMetadata{}, ctx.Err() + } + + select { + case updates = <-ta.updateCh: + case <-ctx.Done(): + return ResourceType{}, nil, xdsresource.UpdateMetadata{}, ctx.Err() + } + + select { + case md = <-ta.mdCh: + case <-ctx.Done(): + return ResourceType{}, nil, xdsresource.UpdateMetadata{}, ctx.Err() + } + return typ, updates, md, nil +} + +func (ta *testEventHandler) adsResourceDoesNotExist(typ ResourceType, name string) { + ta.typeCh <- typ + ta.nameCh <- name +} + +// waitForResourceDoesNotExist waits for the next resource-does-not-exist event +// from the xdsChannel. It returns the resource type and the resource name. If +// the context is canceled, it returns an error. +func (ta *testEventHandler) waitForResourceDoesNotExist(ctx context.Context) (ResourceType, string, error) { + var typ ResourceType + var name string + + select { + case typ = <-ta.typeCh: + case <-ctx.Done(): + return ResourceType{}, "", ctx.Err() + } + + select { + case name = <-ta.nameCh: + case <-ctx.Done(): + return ResourceType{}, "", ctx.Err() + } + return typ, name, nil +} + +type panicDecoder struct{} + +func (panicDecoder) Decode(*AnyProto, DecodeOptions) (*DecodeResult, error) { + panic("simulate panic") +} + +// TestDecodeResponse_PanicRecoveryEnabled tests the panic recovery mechanism +// in decodeResponse. It verifies that if XDSRecoverPanicInResourceParsing +// env variable is enabled, panics during unmarshaling are caught and returned +// as errors. +func (s) TestDecodeResponse_PanicRecoveryEnabled(t *testing.T) { + rType := &ResourceType{ + TypeName: "resourceType", + Decoder: panicDecoder{}, + } + resp := response{resources: []*anypb.Any{{Value: []byte("test")}}} + wantErr := "recovered from panic during resource parsing" + + xc := &xdsChannel{} + if _, _, err := xc.decodeResponse(rType, resp); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("decodeResponse() failed with err: %v, want %q", err, wantErr) + } +} + +// TestDecodeResponse_PanicRecoveryDisabled tests the panic recovery mechanism +// in decodeResponse. It verifies that when XDSRecoverPanicInResourceParsing +// env variable is disabled, panics during unmarshaling propagate. +func (s) TestDecodeResponse_PanicRecoveryDisabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSRecoverPanicInResourceParsing, false) + rType := &ResourceType{ + TypeName: "resourceType", + Decoder: panicDecoder{}, + } + resp := response{resources: []*anypb.Any{{Value: []byte("test")}}} + wantErr := "simulate panic" + xc := &xdsChannel{} + + defer func() { + if r := recover(); r == nil || !strings.Contains(fmt.Sprint(r), wantErr) { + t.Fatalf("Expected panic in decodeResponse, got: %v, want: %q", r, wantErr) + } + }() + xc.decodeResponse(rType, resp) +} diff --git a/internal/xds/clients/xdsclient/clientimpl_watchers.go b/internal/xds/clients/xdsclient/clientimpl_watchers.go new file mode 100644 index 0000000..b305b06 --- /dev/null +++ b/internal/xds/clients/xdsclient/clientimpl_watchers.go @@ -0,0 +1,109 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "context" + "fmt" + + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" +) + +// wrappingWatcher is a wrapper around an xdsresource.ResourceWatcher that adds +// the node ID to the error messages reported to the watcher. +type wrappingWatcher struct { + ResourceWatcher + nodeID string +} + +func (w *wrappingWatcher) AmbientError(err error, done func()) { + w.ResourceWatcher.AmbientError(fmt.Errorf("[xDS node id: %v]: %w", w.nodeID, err), done) +} + +func (w *wrappingWatcher) ResourceError(err error, done func()) { + w.ResourceWatcher.ResourceError(fmt.Errorf("[xDS node id: %v]: %w", w.nodeID, err), done) +} + +// WatchResource starts watching the specified resource. +// +// The watch fails to start if: +// - There is no ResourceType implementation for the given typeURL in the +// ResourceTypes field of the Config struct used to create the XDSClient. +// - The provided resourceName contains an authority that is not present in the +// Authorities field. +// +// The returned function cancels the watch and prevents future calls to the +// watcher. +func (c *XDSClient) WatchResource(typeURL, resourceName string, watcher ResourceWatcher) (cancel func()) { + // Return early if the client is already closed. + if c.done.HasFired() { + logger.Warningf("Watch registered for type %q, but client is closed", typeURL) + return func() {} + } + + watcher = &wrappingWatcher{ + ResourceWatcher: watcher, + nodeID: c.config.Node.ID, + } + + rType, ok := c.config.ResourceTypes[typeURL] + if !ok { + logger.Warningf("ResourceType implementation for resource type url %q is not found", rType.TypeURL) + c.serializer.TrySchedule(func(context.Context) { + watcher.ResourceError(fmt.Errorf("no ResourceType implementation found for typeURL %q", rType.TypeURL), func() {}) + }) + return func() {} + } + + n := xdsresource.ParseName(resourceName) + a := c.getAuthorityForResource(n) + if a == nil { + logger.Warningf("Watch registered for name %q of type %q, authority %q is not found", rType.TypeName, resourceName, n.Authority) + c.serializer.TrySchedule(func(context.Context) { + watcher.ResourceError(fmt.Errorf("authority %q not found in the config for resource %q", n.Authority, resourceName), func() {}) + }) + return func() {} + } + // The watchResource method on the authority is invoked with n.String() + // instead of resourceName because n.String() canonicalizes the given name. + // So, two resource names which don't differ in the query string, but only + // differ in the order of context params will result in the same resource + // being watched by the authority. + return a.watchResource(rType, n.String(), watcher) +} + +// Gets the authority for the given resource name. +// +// See examples in this section of the gRFC: +// https://github.com/grpc/proposal/blob/master/A47-xds-federation.md#bootstrap-config-changes +func (c *XDSClient) getAuthorityForResource(name *xdsresource.Name) *authority { + // For new-style resource names, always lookup the authorities map. If the + // name does not specify an authority, we will end up looking for an entry + // in the map with the empty string as the key. + if name.Scheme == xdsresource.FederationScheme { + return c.authorities[name.Authority] + } + + // For old-style resource names, we use the top-level authority if the name + // does not specify an authority. + if name.Authority == "" { + return c.topLevelAuthority + } + return c.authorities[name.Authority] +} diff --git a/internal/xds/clients/xdsclient/helpers_test.go b/internal/xds/clients/xdsclient/helpers_test.go new file mode 100644 index 0000000..b10dbcf --- /dev/null +++ b/internal/xds/clients/xdsclient/helpers_test.go @@ -0,0 +1,233 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/xds/clients/internal/pretty" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. + // listenerResourceTypeName represents the transport agnostic name for the + // listener resource. + listenerResourceTypeName = "ListenerResource" +) + +var ( + // Singleton instantiation of the resource type implementation. + listenerType = ResourceType{ + TypeURL: xdsresource.V3ListenerURL, + TypeName: listenerResourceTypeName, + AllResourcesRequiredInSotW: true, + Decoder: listenerDecoder{}, + } +) + +func unmarshalListenerResource(rProto *anypb.Any) (string, listenerUpdate, error) { + rProto, err := xdsresource.UnwrapResource(rProto) + if err != nil { + return "", listenerUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err) + } + if !xdsresource.IsListenerResource(rProto.GetTypeUrl()) { + return "", listenerUpdate{}, fmt.Errorf("unexpected listener resource type: %q", rProto.GetTypeUrl()) + } + lis := &v3listenerpb.Listener{} + if err := proto.Unmarshal(rProto.GetValue(), lis); err != nil { + return "", listenerUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + + lu, err := processListener(lis) + if err != nil { + return lis.GetName(), listenerUpdate{}, err + } + lu.Raw = rProto.GetValue() + return lis.GetName(), *lu, nil +} + +func processListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) { + if lis.GetApiListener() != nil { + return processClientSideListener(lis) + } + return processServerSideListener(lis) +} + +// processClientSideListener checks if the provided Listener proto meets +// the expected criteria. If so, it returns a non-empty routeConfigName. +func processClientSideListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) { + update := &listenerUpdate{} + + apiLisAny := lis.GetApiListener().GetApiListener() + if !xdsresource.IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) { + return nil, fmt.Errorf("unexpected http connection manager resource type: %q", apiLisAny.GetTypeUrl()) + } + apiLis := &v3httppb.HttpConnectionManager{} + if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil { + return nil, fmt.Errorf("failed to unmarshal api_listener: %v", err) + } + + switch apiLis.RouteSpecifier.(type) { + case *v3httppb.HttpConnectionManager_Rds: + if configsource := apiLis.GetRds().GetConfigSource(); configsource.GetAds() == nil && configsource.GetSelf() == nil { + return nil, fmt.Errorf("LDS's RDS configSource is not ADS or Self: %+v", lis) + } + name := apiLis.GetRds().GetRouteConfigName() + if name == "" { + return nil, fmt.Errorf("empty route_config_name: %+v", lis) + } + update.RouteConfigName = name + case *v3httppb.HttpConnectionManager_RouteConfig: + routeU := apiLis.GetRouteConfig() + if routeU == nil { + return nil, fmt.Errorf("empty inline RDS resp:: %+v", lis) + } + if routeU.Name == "" { + return nil, fmt.Errorf("empty route_config_name in inline RDS resp: %+v", lis) + } + update.RouteConfigName = routeU.Name + case nil: + return nil, fmt.Errorf("no RouteSpecifier: %+v", apiLis) + default: + return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", apiLis.RouteSpecifier) + } + + return update, nil +} + +func processServerSideListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) { + if n := len(lis.ListenerFilters); n != 0 { + return nil, fmt.Errorf("unsupported field 'listener_filters' contains %d entries", n) + } + if lis.GetUseOriginalDst().GetValue() { + return nil, errors.New("unsupported field 'use_original_dst' is present and set to true") + } + addr := lis.GetAddress() + if addr == nil { + return nil, fmt.Errorf("no address field in LDS response: %+v", lis) + } + sockAddr := addr.GetSocketAddress() + if sockAddr == nil { + return nil, fmt.Errorf("no socket_address field in LDS response: %+v", lis) + } + lu := &listenerUpdate{ + InboundListenerCfg: &inboundListenerConfig{ + Address: sockAddr.GetAddress(), + Port: strconv.Itoa(int(sockAddr.GetPortValue())), + }, + } + + return lu, nil +} + +type listenerDecoder struct{} + +// Decode deserializes and validates an xDS resource serialized inside the +// provided `Any` proto, as received from the xDS management server. +func (listenerDecoder) Decode(resource *AnyProto, _ DecodeOptions) (*DecodeResult, error) { + name, listener, err := unmarshalListenerResource(resource.ToAny()) + switch { + case name == "": + // Name is unset only when protobuf deserialization fails. + return nil, err + case err != nil: + // Protobuf deserialization succeeded, but resource validation failed. + return &DecodeResult{Name: name, Resource: &listenerResourceData{Resource: listenerUpdate{}}}, err + } + + return &DecodeResult{Name: name, Resource: &listenerResourceData{Resource: listener}}, nil + +} + +// listenerResourceData wraps the configuration of a Listener resource as +// received from the management server. +// +// Implements the ResourceData interface. +type listenerResourceData struct { + ResourceData + + Resource listenerUpdate +} + +// Equal returns true if other is equal to l. +func (l *listenerResourceData) Equal(other ResourceData) bool { + if l == nil && other == nil { + return true + } + if (l == nil) != (other == nil) { + return false + } + return bytes.Equal(l.Resource.Raw, other.Bytes()) +} + +// ToJSON returns a JSON string representation of the resource data. +func (l *listenerResourceData) ToJSON() string { + return pretty.ToJSON(l.Resource) +} + +// Bytes returns the underlying raw protobuf form of the listener resource. +func (l *listenerResourceData) Bytes() []byte { + return l.Resource.Raw +} + +// ListenerUpdate contains information received in an LDS response, which is of +// interest to the registered LDS watcher. +type listenerUpdate struct { + // RouteConfigName is the route configuration name corresponding to the + // target which is being watched through LDS. + RouteConfigName string + + // InboundListenerCfg contains inbound listener configuration. + InboundListenerCfg *inboundListenerConfig + + // Raw is the resource from the xds response. + Raw []byte +} + +// InboundListenerConfig contains information about the inbound listener, i.e +// the server-side listener. +type inboundListenerConfig struct { + // Address is the local address on which the inbound listener is expected to + // accept incoming connections. + Address string + // Port is the local port on which the inbound listener is expected to + // accept incoming connections. + Port string +} diff --git a/internal/xds/clients/xdsclient/internal/internal.go b/internal/xds/clients/xdsclient/internal/internal.go new file mode 100644 index 0000000..38d2473 --- /dev/null +++ b/internal/xds/clients/xdsclient/internal/internal.go @@ -0,0 +1,32 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package internal contains functionality internal to the xdsclient package. +package internal + +import "time" + +var ( + // StreamBackoff is the stream backoff for xDS client. It can be overridden + // by tests to change the default backoff strategy. + StreamBackoff func(int) time.Duration + + // ResourceWatchStateForTesting gets the watch state for the resource + // identified by the given resource type and resource name. Returns a + // non-nil error if there is no such resource being watched. + ResourceWatchStateForTesting any // func(*xdsclient.XDSClient, xdsclient.ResourceType, string) error +) diff --git a/internal/xds/clients/xdsclient/internal/xdsresource/ads_stream.go b/internal/xds/clients/xdsclient/internal/xdsresource/ads_stream.go new file mode 100644 index 0000000..87fe9ac --- /dev/null +++ b/internal/xds/clients/xdsclient/internal/xdsresource/ads_stream.go @@ -0,0 +1,46 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import "time" + +// WatchState is a enum that describes the watch state of a particular +// resource. +type WatchState int + +const ( + // ResourceWatchStateStarted is the state where a watch for a resource was + // started, but a request asking for that resource is yet to be sent to the + // management server. + ResourceWatchStateStarted WatchState = iota + // ResourceWatchStateRequested is the state when a request has been sent for + // the resource being watched. + ResourceWatchStateRequested + // ResourceWatchStateReceived is the state when a response has been received + // for the resource being watched. + ResourceWatchStateReceived + // ResourceWatchStateTimeout is the state when the watch timer associated + // with the resource expired because no response was received. + ResourceWatchStateTimeout +) + +// ResourceWatchState is the state corresponding to a resource being watched. +type ResourceWatchState struct { + State WatchState // Watch state of the resource. + ExpiryTimer *time.Timer // Timer for the expiry of the watch. +} diff --git a/internal/xds/clients/xdsclient/internal/xdsresource/errors.go b/internal/xds/clients/xdsclient/internal/xdsresource/errors.go new file mode 100644 index 0000000..2bcc2e5 --- /dev/null +++ b/internal/xds/clients/xdsclient/internal/xdsresource/errors.go @@ -0,0 +1,78 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "errors" + "fmt" +) + +// ErrorType is the type of the error that the watcher will receive from the xds +// client. +type ErrorType int + +const ( + // ErrorTypeUnknown indicates the error doesn't have a specific type. It is + // the default value, and is returned if the error is not an xds error. + ErrorTypeUnknown ErrorType = iota + // ErrorTypeConnection indicates a connection error from the gRPC client. + ErrorTypeConnection + // ErrorTypeResourceNotFound indicates a resource is not found from the xds + // response. It's typically returned if the resource is removed in the xds + // server. + ErrorTypeResourceNotFound + // ErrorTypeResourceTypeUnsupported indicates the receipt of a message from + // the management server with resources of an unsupported resource type. + ErrorTypeResourceTypeUnsupported + // ErrTypeStreamFailedAfterRecv indicates an ADS stream error, after + // successful receipt of at least one message from the server. + ErrTypeStreamFailedAfterRecv + // ErrorTypeNACKed indicates that configuration provided by the xDS management + // server was NACKed. + ErrorTypeNACKed +) + +type xdsClientError struct { + t ErrorType + desc string +} + +func (e *xdsClientError) Error() string { + return e.desc +} + +// NewErrorf creates an xDS client error. The callbacks are called with this +// error, to pass additional information about the error. +func NewErrorf(t ErrorType, format string, args ...any) error { + return &xdsClientError{t: t, desc: fmt.Sprintf(format, args...)} +} + +// NewError creates an xDS client error. The callbacks are called with this +// error, to pass additional information about the error. +func NewError(t ErrorType, message string) error { + return NewErrorf(t, "%s", message) +} + +// ErrType returns the error's type. +func ErrType(e error) ErrorType { + var xe *xdsClientError + if ok := errors.As(e, &xe); ok { + return xe.t + } + return ErrorTypeUnknown +} diff --git a/internal/xds/clients/xdsclient/internal/xdsresource/name.go b/internal/xds/clients/xdsclient/internal/xdsresource/name.go new file mode 100644 index 0000000..24200ea --- /dev/null +++ b/internal/xds/clients/xdsclient/internal/xdsresource/name.go @@ -0,0 +1,127 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "net/url" + "sort" + "strings" +) + +// FederationScheme is the scheme of a federation resource name. +const FederationScheme = "xdstp" + +// Name contains the parsed component of an xDS resource name. +// +// An xDS resource name is in the format of +// xdstp://[{authority}]/{resource type}/{id/*}?{context parameters}{#processing directive,*} +// +// See +// https://github.com/cncf/xds/blob/main/proposals/TP1-xds-transport-next.md#uri-based-xds-resource-names +// for details, and examples. +type Name struct { + Scheme string + Authority string + Type string + ID string + + ContextParams map[string]string + + processingDirective string +} + +// ParseName splits the name and returns a struct representation of the Name. +// +// If the name isn't a valid new-style xDS name, field ID is set to the input. +// Note that this is not an error, because we still support the old-style +// resource names (those not starting with "xdstp:"). +// +// The caller can tell if the parsing is successful by checking the returned +// Scheme. +func ParseName(name string) *Name { + if !strings.Contains(name, "://") { + // Only the long form URL, with ://, is valid. + return &Name{ID: name} + } + parsed, err := url.Parse(name) + if err != nil { + return &Name{ID: name} + } + + ret := &Name{ + Scheme: parsed.Scheme, + Authority: parsed.Host, + } + split := strings.SplitN(parsed.Path, "/", 3) + if len(split) < 3 { + // Path is in the format of "/type/id". There must be at least 3 + // segments after splitting. + return &Name{ID: name} + } + ret.Type = split[1] + ret.ID = split[2] + if len(parsed.Query()) != 0 { + ret.ContextParams = make(map[string]string) + for k, vs := range parsed.Query() { + if len(vs) > 0 { + // We only keep one value of each key. Behavior for multiple values + // is undefined. + ret.ContextParams[k] = vs[0] + } + } + } + // TODO: processing directive (the part comes after "#" in the URL, stored + // in parsed.RawFragment) is kept but not processed. Add support for that + // when it's needed. + ret.processingDirective = parsed.RawFragment + return ret +} + +// String returns a canonicalized string of name. The context parameters are +// sorted by the keys. +func (n *Name) String() string { + if n.Scheme == "" { + return n.ID + } + + // Sort and build query. + keys := make([]string, 0, len(n.ContextParams)) + for k := range n.ContextParams { + keys = append(keys, k) + } + sort.Strings(keys) + var pairs []string + for _, k := range keys { + pairs = append(pairs, strings.Join([]string{k, n.ContextParams[k]}, "=")) + } + rawQuery := strings.Join(pairs, "&") + + path := n.Type + if n.ID != "" { + path = "/" + path + "/" + n.ID + } + + tempURL := &url.URL{ + Scheme: n.Scheme, + Host: n.Authority, + Path: path, + RawQuery: rawQuery, + RawFragment: n.processingDirective, + } + return tempURL.String() +} diff --git a/internal/xds/clients/xdsclient/internal/xdsresource/type.go b/internal/xds/clients/xdsclient/internal/xdsresource/type.go new file mode 100644 index 0000000..ea4d854 --- /dev/null +++ b/internal/xds/clients/xdsclient/internal/xdsresource/type.go @@ -0,0 +1,101 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "time" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +// UpdateMetadata contains the metadata for each update, including timestamp, +// raw message, and so on. +type UpdateMetadata struct { + // Status is the status of this resource, e.g. ACKed, NACKed, or + // Not_exist(removed). + Status ServiceStatus + // Version is the version of the xds response. Note that this is the version + // of the resource in use (previous ACKed). If a response is NACKed, the + // NACKed version is in ErrState. + Version string + // Timestamp is when the response is received. + Timestamp time.Time + // ErrState is set when the update is NACKed. + ErrState *UpdateErrorMetadata +} + +// IsListenerResource returns true if the provider URL corresponds to an xDS +// Listener resource. +func IsListenerResource(url string) bool { + return url == V3ListenerURL +} + +// IsHTTPConnManagerResource returns true if the provider URL corresponds to an xDS +// HTTPConnManager resource. +func IsHTTPConnManagerResource(url string) bool { + return url == V3HTTPConnManagerURL +} + +// UnwrapResource unwraps and returns the inner resource if it's in a resource +// wrapper. The original resource is returned if it's not wrapped. +func UnwrapResource(r *anypb.Any) (*anypb.Any, error) { + url := r.GetTypeUrl() + if url != V3ResourceWrapperURL { + // Not wrapped. + return r, nil + } + inner := &v3discoverypb.Resource{} + if err := proto.Unmarshal(r.GetValue(), inner); err != nil { + return nil, err + } + return inner.Resource, nil +} + +// ServiceStatus is the status of the update. +type ServiceStatus int + +const ( + // ServiceStatusUnknown is the default state, before a watch is started for + // the resource. + ServiceStatusUnknown ServiceStatus = iota + // ServiceStatusRequested is when the watch is started, but before and + // response is received. + ServiceStatusRequested + // ServiceStatusNotExist is when the resource doesn't exist in + // state-of-the-world responses (e.g. LDS and CDS), which means the resource + // is removed by the management server. + ServiceStatusNotExist // Resource is removed in the server, in LDS/CDS. + // ServiceStatusACKed is when the resource is ACKed. + ServiceStatusACKed + // ServiceStatusNACKed is when the resource is NACKed. + ServiceStatusNACKed +) + +// UpdateErrorMetadata is part of UpdateMetadata. It contains the error state +// when a response is NACKed. +type UpdateErrorMetadata struct { + // Version is the version of the NACKed response. + Version string + // Err contains why the response was NACKed. + Err error + // Timestamp is when the NACKed response was received. + Timestamp time.Time +} diff --git a/internal/xds/clients/xdsclient/internal/xdsresource/version.go b/internal/xds/clients/xdsclient/internal/xdsresource/version.go new file mode 100644 index 0000000..60f47e6 --- /dev/null +++ b/internal/xds/clients/xdsclient/internal/xdsresource/version.go @@ -0,0 +1,31 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xdsresource defines constants to distinguish between supported xDS +// API versions. +package xdsresource + +// Resource URLs. We need to be able to accept either version of the resource +// regardless of the version of the transport protocol in use. +const ( + googleapiPrefix = "type.googleapis.com/" + + V3ListenerURL = googleapiPrefix + "envoy.config.listener.v3.Listener" + V3HTTPConnManagerURL = googleapiPrefix + "envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager" + V3ResourceWrapperURL = googleapiPrefix + "envoy.service.discovery.v3.Resource" +) diff --git a/internal/xds/clients/xdsclient/logging.go b/internal/xds/clients/xdsclient/logging.go new file mode 100644 index 0000000..c7ba21c --- /dev/null +++ b/internal/xds/clients/xdsclient/logging.go @@ -0,0 +1,36 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +var logger = grpclog.Component("xds") + +func prefixLogger(p *XDSClient) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, clientPrefix(p)) +} + +func clientPrefix(p *XDSClient) string { + return fmt.Sprintf("[xds-client %p] ", p) +} diff --git a/internal/xds/clients/xdsclient/metrics/metrics.go b/internal/xds/clients/xdsclient/metrics/metrics.go new file mode 100644 index 0000000..fa1277c --- /dev/null +++ b/internal/xds/clients/xdsclient/metrics/metrics.go @@ -0,0 +1,64 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package metrics defines all metrics that can be produced by an xDS client. +// All calls to the MetricsRecorder by the xDS client will contain a struct +// from this package passed by pointer. +// +// For definitions of these metrics and their labels, see gRFC A78: +// https://github.com/grpc/proposal/blob/master/A78-grpc-metrics-wrr-xds.md +package metrics + +// ResourceUpdateValid is a metric to report a valid resource update from +// the xDS management server for a given resource type. +type ResourceUpdateValid struct { + ServerURI string + ResourceType string +} + +// ResourceUpdateInvalid is a metric to report an invalid resource update +// from the xDS management server for a given resource type. +type ResourceUpdateInvalid struct { + ServerURI string + ResourceType string +} + +// ServerFailure is a metric to report a server failure of the xDS +// management server. +type ServerFailure struct { + ServerURI string +} + +// XDSClientConnected reports the connectivity state of the ADS stream. +// Per gRFC A78, Value is 1 if connected, and 0 otherwise. +// This metric provides the labels grpc.target and grpc.xds.server. +type XDSClientConnected struct { + ServerURI string + Value int64 +} + +// XDSClientResourceStats reports the current cache states of xDS resources. +// For label definitions, see gRFC A78. +// This metric provides the labels grpc.target, grpc.xds.authority, +// grpc.xds.cache_state, and grpc.xds.resource_type. +type XDSClientResourceStats struct { + Authority string + ResourceType string + CacheState string + Count int64 +} diff --git a/internal/xds/clients/xdsclient/resource_type.go b/internal/xds/clients/xdsclient/resource_type.go new file mode 100644 index 0000000..ea16c21 --- /dev/null +++ b/internal/xds/clients/xdsclient/resource_type.go @@ -0,0 +1,123 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import "google.golang.org/protobuf/types/known/anypb" + +// ResourceType wraps all resource-type specific functionality. Each supported +// resource type needs to provide an implementation of the Decoder. +type ResourceType struct { + // TypeURL is the xDS type URL of this resource type for the v3 xDS + // protocol. This URL is used as the key to look up the corresponding + // ResourceType implementation in the ResourceTypes map provided in the + // Config. + TypeURL string + + // TODO: Revisit if we need TypeURL to be part of the struct because it is + // already a key in config's ResouceTypes map. + + // TypeName is a shorter representation of the TypeURL to identify the + // resource type. It is used for logging/debugging purposes. + TypeName string + + // AllResourcesRequiredInSotW indicates whether this resource type requires + // that all resources be present in every SotW response from the server. If + // true, a response that does not include a previously seen resource will + // be interpreted as a deletion of that resource. + AllResourcesRequiredInSotW bool + + // Decoder is used to deserialize and validate an xDS resource received + // from the xDS management server. + Decoder Decoder +} + +// Decoder wraps the resource-type specific functionality for validation +// and deserialization. +type Decoder interface { + // Decode deserializes and validates an xDS resource as received from the + // xDS management server. + // + // The `resource` parameter may contain a value of the serialized wrapped + // resource (i.e. with the type URL + // `type.googleapis.com/envoy.service.discovery.v3.Resource`). + // Implementations are responsible for unwrapping the underlying resource if + // it is wrapped. + // + // If unmarshalling or validation fails, it returns a non-nil error. + // Otherwise, returns a fully populated DecodeResult. + Decode(resource *AnyProto, options DecodeOptions) (*DecodeResult, error) +} + +// AnyProto contains the type URL and serialized proto data of an xDS resource. +type AnyProto struct { + typeURL string + value []byte +} + +// NewAnyProto creates an AnyProto from an anypb.Any. Must be called with a +// non-nil argument. +func NewAnyProto(a *anypb.Any) *AnyProto { + return &AnyProto{ + typeURL: a.TypeUrl, + value: a.Value, + } +} + +// ToAny converts an AnyProto to an anypb.Any. Never returns nil. +func (a *AnyProto) ToAny() *anypb.Any { + return &anypb.Any{ + TypeUrl: a.typeURL, + Value: a.value, + } +} + +// DecodeOptions wraps the options required by ResourceType implementations for +// decoding configuration received from the xDS management server. +type DecodeOptions struct { + // Config contains the complete configuration passed to the xDS client. + // This contains useful data for resource validation. + Config *Config + + // ServerConfig contains the configuration of the xDS server that provided + // the current resource being decoded. + ServerConfig *ServerConfig +} + +// DecodeResult is the result of a decode operation. +type DecodeResult struct { + // Name is the name of the decoded resource. + Name string + + // Resource contains the configuration associated with the decoded + // resource. + Resource ResourceData +} + +// ResourceData contains the configuration data sent by the xDS management +// server, associated with the resource being watched. Every resource type must +// provide an implementation of this interface to represent the configuration +// received from the xDS management server. +type ResourceData interface { + // Equal returns true if the passed in resource data is equal to that of + // the receiver. + Equal(other ResourceData) bool + + // Bytes returns the underlying raw bytes of the resource proto. + Bytes() []byte +} diff --git a/internal/xds/clients/xdsclient/resource_watcher.go b/internal/xds/clients/xdsclient/resource_watcher.go new file mode 100644 index 0000000..15891d9 --- /dev/null +++ b/internal/xds/clients/xdsclient/resource_watcher.go @@ -0,0 +1,48 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +// ResourceWatcher is notified of the resource updates and errors that are +// received by the xDS client from the management server. +// +// All methods on this interface are guaranteed to be called serially by the xDS +// client. +// +// All methods contain a done parameter which should be called when processing +// of the update has completed. For example, if processing a resource requires +// watching new resources, those watches should be completed before done is +// called, which can happen after the ResourceWatcher method has returned. +// Failure to call done will prevent the xDS client from providing future +// ResourceWatcher notifications. +type ResourceWatcher interface { + // ResourceChanged indicates a new version of the resource is available. + ResourceChanged(resourceData ResourceData, done func()) + + // ResourceError indicates an error occurred while trying to fetch or + // decode the associated resource. The previous version of the resource + // should be considered invalid. + ResourceError(err error, done func()) + + // AmbientError indicates an error occurred after a resource has been + // received that should not modify the use of that resource but may provide + // useful information about the state of the XDSClient for debugging + // purposes. The previous version of the resource should still be + // considered valid. + AmbientError(err error, done func()) +} diff --git a/internal/xds/clients/xdsclient/test/ads_stream_ack_nack_test.go b/internal/xds/clients/xdsclient/test/ads_stream_ack_nack_test.go new file mode 100644 index 0000000..f028299 --- /dev/null +++ b/internal/xds/clients/xdsclient/test/ads_stream_ack_nack_test.go @@ -0,0 +1,671 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/internal/testutils" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +// Creates an xDS client with the given management server address, node ID +// and transport builder. +func createXDSClient(t *testing.T, mgmtServerAddress string, nodeID string, transportBuilder clients.TransportBuilder) *xdsclient.XDSClient { + t.Helper() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServerAddress, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"}, + TransportBuilder: transportBuilder, + ResourceTypes: resourceTypes, + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + t.Cleanup(func() { client.Close() }) + return client +} + +// Tests simple ACK and NACK scenarios on the ADS stream: +// 1. When a good response is received, i.e. once that is expected to be ACKed, +// the test verifies that an ACK is sent matching the version and nonce from +// the response. +// 2. When a subsequent bad response is received, i.e. once is expected to be +// NACKed, the test verifies that a NACK is sent matching the previously +// ACKed version and current nonce from the response. +// 3. When a subsequent good response is received, the test verifies that an +// ACK is sent matching the version and nonce from the current response. +func (s) TestADS_ACK_NACK_Simple(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create an xDS management server listening on a local port. Configure the + // request and response handlers to push on channels that are inspected by + // the test goroutine to verify ACK version and nonce. + streamRequestCh := testutils.NewChannelWithSize(1) + streamResponseCh := testutils.NewChannelWithSize(1) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + streamRequestCh.SendContext(ctx, req) + return nil + }, + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + streamResponseCh.SendContext(ctx, resp) + }, + }) + + // Create a listener resource on the management server. + const listenerName = "listener" + const routeConfigName = "route-config" + nodeID := uuid.New().String() + listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client pointing to the above server. + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs)) + + // Register a watch for a listener resource. + lw := newListenerWatcher() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw) + defer ldsCancel() + + // Verify that the initial discovery request matches expectation. + r, err := streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for the initial discovery request") + } + gotReq := r.(*v3discoverypb.DiscoveryRequest) + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{ + Id: nodeID, + UserAgentName: "user-agent", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + }, + ResourceNames: []string{listenerName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Capture the version and nonce from the response. + r, err = streamResponseCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for a discovery response from the server") + } + gotResp := r.(*v3discoverypb.DiscoveryResponse) + + // Verify that the ACK contains the appropriate version and nonce. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for ACK") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + wantReq.VersionInfo = gotResp.GetVersionInfo() + wantReq.ResponseNonce = gotResp.GetNonce() + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Verify the update received by the watcher. + wantUpdate := listenerUpdateErrTuple{ + update: listenerUpdate{RouteConfigName: routeConfigName}, + } + if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Update the management server with a listener resource that contains an + // empty HTTP connection manager within the apiListener, which will cause + // the resource to be NACKed. + badListener := proto.Clone(listenerResource).(*v3listenerpb.Listener) + badListener.ApiListener.ApiListener = nil + mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{badListener}, + SkipValidation: true, + }) + + r, err = streamResponseCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for a discovery response from the server") + } + gotResp = r.(*v3discoverypb.DiscoveryResponse) + + wantNackErr := xdsresource.NewError(xdsresource.ErrorTypeNACKed, "unexpected http connection manager resource type") + if err := verifyListenerUpdate(ctx, lw.ambientErrCh, listenerUpdateErrTuple{ambientErr: wantNackErr}); err != nil { + t.Fatal(err) + } + + // Verify that the NACK contains the appropriate version, nonce and error. + // We expect the version to not change as this is a NACK. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for NACK") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + if gotNonce, wantNonce := gotReq.GetResponseNonce(), gotResp.GetNonce(); gotNonce != wantNonce { + t.Errorf("Unexpected nonce in discovery request, got: %v, want: %v", gotNonce, wantNonce) + } + if gotErr := gotReq.GetErrorDetail(); gotErr == nil || !strings.Contains(gotErr.GetMessage(), wantNackErr.Error()) { + t.Fatalf("Unexpected error in discovery request, got: %v, want: %v", gotErr.GetMessage(), wantNackErr) + } + + // Update the management server to send a good resource again. + mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource}, + SkipValidation: true, + }) + + // The envoy-go-control-plane management server keeps resending the same + // resource as long as we keep NACK'ing it. So, we will see the bad resource + // sent to us a few times here, before receiving the good resource. + var lastErr error + for { + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for an ACK from the xDS client. Last seen error: %v", lastErr) + } + + r, err = streamResponseCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for a discovery response from the server") + } + gotResp = r.(*v3discoverypb.DiscoveryResponse) + + // Verify that the ACK contains the appropriate version and nonce. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for ACK") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + wantReq.VersionInfo = gotResp.GetVersionInfo() + wantReq.ResponseNonce = gotResp.GetNonce() + wantReq.ErrorDetail = nil + diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()) + if diff == "" { + lastErr = nil + break + } + lastErr = fmt.Errorf("unexpected diff in discovery request, diff (-got, +want):\n%s", diff) + } + + // Verify the update received by the watcher. + for ; ctx.Err() == nil; <-time.After(100 * time.Millisecond) { + if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil { + lastErr = err + continue + } + break + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for listener update. Last seen error: %v", lastErr) + } +} + +// Tests the case where the first response is invalid. The test verifies that +// the NACK contains an empty version string. +func (s) TestADS_NACK_InvalidFirstResponse(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create an xDS management server listening on a local port. Configure the + // request and response handlers to push on channels that are inspected by + // the test goroutine to verify ACK version and nonce. + streamRequestCh := testutils.NewChannelWithSize(1) + streamResponseCh := testutils.NewChannelWithSize(1) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + streamRequestCh.SendContext(ctx, req) + return nil + }, + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + streamResponseCh.SendContext(ctx, resp) + }, + }) + + // Create a listener resource on the management server that is expected to + // be NACKed by the xDS client. + const listenerName = "listener" + const routeConfigName = "route-config" + nodeID := uuid.New().String() + listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName) + listenerResource.ApiListener.ApiListener = nil + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client pointing to the above server. + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs)) + + // Register a watch for a listener resource. + lw := newListenerWatcher() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw) + defer ldsCancel() + + // Verify that the initial discovery request matches expectation. + r, err := streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for the initial discovery request") + } + gotReq := r.(*v3discoverypb.DiscoveryRequest) + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{ + Id: nodeID, + UserAgentName: "user-agent", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + }, + ResourceNames: []string{listenerName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Capture the version and nonce from the response. + r, err = streamResponseCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for the discovery response from client") + } + gotResp := r.(*v3discoverypb.DiscoveryResponse) + + // Verify that the error is propagated to the watcher. + var wantNackErr = xdsresource.NewError(xdsresource.ErrorTypeNACKed, "unexpected http connection manager resource type") + if err := verifyListenerUpdate(ctx, lw.resourceErrCh, listenerUpdateErrTuple{resourceErr: wantNackErr}); err != nil { + t.Fatal(err) + } + + // NACK should contain the appropriate error, nonce, but empty version. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for ACK") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + if gotVersion, wantVersion := gotReq.GetVersionInfo(), ""; gotVersion != wantVersion { + t.Errorf("Unexpected version in discovery request, got: %v, want: %v", gotVersion, wantVersion) + } + if gotNonce, wantNonce := gotReq.GetResponseNonce(), gotResp.GetNonce(); gotNonce != wantNonce { + t.Errorf("Unexpected nonce in discovery request, got: %v, want: %v", gotNonce, wantNonce) + } + if gotErr := gotReq.GetErrorDetail(); gotErr == nil || !strings.Contains(gotErr.GetMessage(), wantNackErr.Error()) { + t.Fatalf("Unexpected error in discovery request, got: %v, want: %v", gotErr.GetMessage(), wantNackErr) + } +} + +// Tests the scenario where the xDS client is no longer interested in a +// resource. The following sequence of events are tested: +// 1. A resource is requested and a good response is received. The test verifies +// that an ACK is sent for this resource. +// 2. The previously requested resource is no longer requested. The test +// verifies that the connection to the management server is closed. +// 3. The same resource is requested again. The test verifies that a new +// request is sent with an empty version string, which corresponds to the +// first request on a new connection. +func (s) TestADS_ACK_NACK_ResourceIsNotRequestedAnymore(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create an xDS management server listening on a local port. Configure the + // request and response handlers to push on channels that are inspected by + // the test goroutine to verify ACK version and nonce. + streamRequestCh := testutils.NewChannelWithSize(1) + streamResponseCh := testutils.NewChannelWithSize(1) + streamCloseCh := testutils.NewChannelWithSize(1) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + streamRequestCh.SendContext(ctx, req) + return nil + }, + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + streamResponseCh.SendContext(ctx, resp) + }, + OnStreamClosed: func(int64, *v3corepb.Node) { + streamCloseCh.SendContext(ctx, struct{}{}) + }, + }) + + // Create a listener resource on the management server. + const listenerName = "listener" + const routeConfigName = "route-config" + nodeID := uuid.New().String() + listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client pointing to the above server. + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs)) + + // Register a watch for a listener resource. + lw := newListenerWatcher() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw) + defer ldsCancel() + + // Verify that the initial discovery request matches expectation. + r, err := streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for the initial discovery request") + } + gotReq := r.(*v3discoverypb.DiscoveryRequest) + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{ + Id: nodeID, + UserAgentName: "user-agent", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + }, + ResourceNames: []string{listenerName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Capture the version and nonce from the response. + r, err = streamResponseCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for the discovery response from client") + } + gotResp := r.(*v3discoverypb.DiscoveryResponse) + + // Verify that the ACK contains the appropriate version and nonce. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for ACK") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + wantACKReq := proto.Clone(wantReq).(*v3discoverypb.DiscoveryRequest) + wantACKReq.VersionInfo = gotResp.GetVersionInfo() + wantACKReq.ResponseNonce = gotResp.GetNonce() + if diff := cmp.Diff(gotReq, wantACKReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Verify the update received by the watcher. + wantUpdate := listenerUpdateErrTuple{ + update: listenerUpdate{RouteConfigName: routeConfigName}, + } + if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Cancel the watch on the listener resource. This should result in the + // existing connection to be management server getting closed. + ldsCancel() + if _, err := streamCloseCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when expecting existing connection to be closed: %v", err) + } + + // There is a race between two events when the last watch on an xdsChannel + // is canceled: + // - an empty discovery request being sent out + // - the ADS stream being closed + // To handle this race, we drain the request channel here so that if an + // empty discovery request was received, it is pulled out of the request + // channel and thereby guaranteeing a clean slate for the next watch + // registered below. + streamRequestCh.Drain() + + // Register a watch for the same listener resource. + lw = newListenerWatcher() + ldsCancel = client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw) + defer ldsCancel() + + // Verify that the discovery request is identical to the first one sent out + // to the management server. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for discovery request") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Verify the update received by the watcher. + if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestADS_NACKError_DuplicateSuppression verifies that when a duplicate invalid +// LDS resource is sent by the server, the client suppresses the duplicate error +// notification to the watcher. +func (s) TestADS_NACKError_DuplicateSuppression(t *testing.T) { + // The first NACK is expected upon receiving the invalid resource. The + // second NACK is a duplicate, sent because the management server resends + // the invalid resource in a loop as long as the resource update has not + // changed to a valid one. The third NACK is waited for to verify that the + // duplicated error is still suppressed as expected. + const wantNACKCount = 3 + + nacksReceivedCh := make(chan struct{}) + nackCount := 0 + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() != xdsresource.V3ListenerURL { + return nil + } + if req.GetErrorDetail() != nil { + nackCount++ + if nackCount == wantNACKCount { + close(nacksReceivedCh) + } + } + return nil + }, + }) + + const listenerName = "listener" + nodeID := uuid.New().String() + + // Create an xDS client pointing to the above server. + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs)) + + watcher := newListenerWatcher() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher) + defer ldsCancel() + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{badListenerResource(t, listenerName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the watcher receives the initial error. + v, err := watcher.resourceErrCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for an endpoints resource error from the management server: %v", err) + } + const wantNackErr = "no RouteSpecifier" + if gotErr := v.(listenerUpdateErrTuple).resourceErr; !strings.Contains(gotErr.Error(), wantNackErr) { + t.Fatalf("Update received with error: %v, want %q", gotErr, wantNackErr) + } + + // Wait for the management server to receive at least wantNACKCount NACKs, + // indicating that the NACK resend loop has run multiple times. + select { + case <-nacksReceivedCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for %d NACKs to be received by the server: %v", wantNACKCount, ctx.Err()) + } + + // Verify that no duplicate error update was written to the error channel, + // proving that subsequent duplicate error updates were suppressed. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if v, err := watcher.resourceErrCh.Receive(sCtx); err == nil { + t.Fatalf("Unexpected duplicate error received by watcher: %v", v) + } +} + +// TestADS_NACKError_DuplicateSuppression_MultipleResources verifies that +// when multiple invalid resources cause errors, and a subsequent update carries +// a duplicate error for only a subset of resources, the client still suppresses +// the duplicate error notification to the watcher of the affected resource(s). +func (s) TestADS_NACKError_DuplicateSuppression_MultipleResources(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + const listenerName1 = "listener-1" + const listenerName2 = "listener-2" + nodeID := uuid.New().String() + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs)) + + watcher1 := newListenerWatcher() + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerName1, watcher1) + defer ldsCancel1() + + watcher2 := newListenerWatcher() + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerName2, watcher2) + defer ldsCancel2() + + // Update 1: Management server sends two invalid listener resources. + resources1 := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + badListenerResource(t, listenerName1), + badListenerResource(t, listenerName2), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if err := mgmtServer.Update(ctx, resources1); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources1, err) + } + + // Verify that watcher1 receives the initial error for listener 1. + v1, err := watcher1.resourceErrCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for listener 1 error from management server: %v", err) + } + const wantNackErr = "no RouteSpecifier" + if gotErr := v1.(listenerUpdateErrTuple).resourceErr; !strings.Contains(gotErr.Error(), wantNackErr) { + t.Fatalf("Update 1 listener 1 received with error: %v, want %q", gotErr, wantNackErr) + } + + // Verify that watcher2 receives the initial error for listener 2. + v2, err := watcher2.resourceErrCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for listener 2 error from management server: %v", err) + } + + if gotErr := v2.(listenerUpdateErrTuple).resourceErr; !strings.Contains(gotErr.Error(), wantNackErr) { + t.Fatalf("Update 1 listener 2 received with error: %v, want %q", gotErr, wantNackErr) + } + + // Update 2: Management server sends listener 1 still invalid, but listener + // 2 now valid. + resources2 := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + badListenerResource(t, listenerName1), + e2e.DefaultClientListener(listenerName2, "route-config-2"), + }, + SkipValidation: true, + } + + if err := mgmtServer.Update(ctx, resources2); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources2, err) + } + + // Wait for watcher2 to receive the valid resource update for listener 2. + if _, err := watcher2.updateCh.Receive(ctx); err != nil { + t.Fatalf("Timeout waiting for listener 2 resource update: %v", err) + } + + // Verify that no duplicate error update was written to watcher 1. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if v, err := watcher1.resourceErrCh.Receive(sCtx); err == nil { + t.Fatalf("Unexpected duplicate error received by watcher 1: %v", v) + } +} diff --git a/internal/xds/clients/xdsclient/test/ads_stream_backoff_test.go b/internal/xds/clients/xdsclient/test/ads_stream_backoff_test.go new file mode 100644 index 0000000..6dc2ac5 --- /dev/null +++ b/internal/xds/clients/xdsclient/test/ads_stream_backoff_test.go @@ -0,0 +1,444 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "errors" + "fmt" + "net" + "testing" + "time" + + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/internal/testutils" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + xdsclientinternal "google.golang.org/grpc/internal/xds/clients/xdsclient/internal" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/protobuf/testing/protocmp" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" +) + +func overrideStreamBackOff(t *testing.T, streamBackOff func(int) time.Duration) { + originalStreamBackoff := xdsclientinternal.StreamBackoff + xdsclientinternal.StreamBackoff = streamBackOff + t.Cleanup(func() { xdsclientinternal.StreamBackoff = originalStreamBackoff }) +} + +// Creates an xDS client with the given management server address, nodeID and backoff function. +func createXDSClientWithBackoff(t *testing.T, mgmtServerAddress string, nodeID string, streamBackoff func(int) time.Duration) *xdsclient.XDSClient { + t.Helper() + overrideStreamBackOff(t, streamBackoff) + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + return createXDSClient(t, mgmtServerAddress, nodeID, grpctransport.NewBuilder(configs)) +} + +// Tests the case where the management server returns an error in the ADS +// streaming RPC. Verifies that the ADS stream is restarted after a backoff +// period, and that the previously requested resources are re-requested on the +// new stream. +func (s) TestADS_BackoffAfterStreamFailure(t *testing.T) { + // Channels used for verifying different events in the test. + streamCloseCh := make(chan struct{}, 1) // ADS stream is closed. + ldsResourcesCh := make(chan []string, 1) // Listener resource names in the discovery request. + backoffCh := make(chan struct{}, 1) // Backoff after stream failure. + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create an xDS management server that returns RPC errors. + streamErr := errors.New("ADS stream error") + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + // Push the requested resource names on to a channel. + if req.GetTypeUrl() == version.V3ListenerURL { + t.Logf("Received LDS request for resources: %v", req.GetResourceNames()) + select { + case ldsResourcesCh <- req.GetResourceNames(): + case <-ctx.Done(): + } + } + // Return an error everytime a request is sent on the stream. This + // should cause the transport to backoff before attempting to + // recreate the stream. + return streamErr + }, + // Push on a channel whenever the stream is closed. + OnStreamClosed: func(int64, *v3corepb.Node) { + select { + case streamCloseCh <- struct{}{}: + case <-ctx.Done(): + } + }, + }) + + // Override the backoff implementation to push on a channel that is read by + // the test goroutine. + backoffCtx, backoffCancel := context.WithCancel(ctx) + streamBackoff := func(int) time.Duration { + select { + case backoffCh <- struct{}{}: + case <-backoffCtx.Done(): + } + return 0 + } + defer backoffCancel() + + // Create an xDS client with bootstrap pointing to the above server. + nodeID := uuid.New().String() + client := createXDSClientWithBackoff(t, mgmtServer.Address, nodeID, streamBackoff) + + // Register a watch for a listener resource. + const listenerName = "listener" + lw := newListenerWatcher() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw) + defer ldsCancel() + + // Verify that an ADS stream is created and an LDS request with the above + // resource name is sent. + if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil { + t.Fatal(err) + } + + // Verify that the received stream error is reported to the watcher. + if err := verifyListenerResourceError(ctx, lw.resourceErrCh, streamErr.Error(), nodeID); err != nil { + t.Fatal(err) + } + + // Verify that the stream is closed. + select { + case <-streamCloseCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for stream to be closed after an error") + } + + // Verify that the ADS stream backs off before recreating the stream. + select { + case <-backoffCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for ADS stream to backoff after stream failure") + } + + // Verify that the same resource name is re-requested on the new stream. + if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil { + t.Fatal(err) + } + + // To prevent indefinite blocking during xDS client close, which is caused + // by a blocking backoff channel write, cancel the backoff context early + // given that the test is complete. + backoffCancel() + +} + +// Tests the case where a stream breaks because the server goes down. Verifies +// that when the server comes back up, the same resources are re-requested, this +// time with the previously acked version and an empty nonce. +func (s) TestADS_RetriesAfterBrokenStream(t *testing.T) { + // Channels used for verifying different events in the test. + streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) // Discovery request is received. + streamResponseCh := make(chan *v3discoverypb.DiscoveryResponse, 1) // Discovery response is received. + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create an xDS management server listening on a local port. + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + // Push the received request on to a channel for the test goroutine to + // verify that it matches expectations. + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + select { + case streamRequestCh <- req: + case <-ctx.Done(): + } + return nil + }, + // Push the response that the management server is about to send on to a + // channel. The test goroutine to uses this to extract the version and + // nonce, expected on subsequent requests. + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + select { + case streamResponseCh <- resp: + case <-ctx.Done(): + } + }, + }) + + // Create a listener resource on the management server. + const listenerName = "listener" + const routeConfigName = "route-config" + nodeID := uuid.New().String() + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Override the backoff implementation to always return 0, to reduce test + // run time. Instead control when the backoff returns by blocking on a + // channel, that the test closes. + backoffCh := make(chan struct{}) + streamBackoff := func(int) time.Duration { + select { + case backoffCh <- struct{}{}: + case <-ctx.Done(): + } + return 0 + } + + // Create an xDS client pointing to the above server. + client := createXDSClientWithBackoff(t, mgmtServer.Address, nodeID, streamBackoff) + + // Register a watch for a listener resource. + lw := newListenerWatcher() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw) + defer ldsCancel() + + // Verify that the initial discovery request matches expectation. + var gotReq *v3discoverypb.DiscoveryRequest + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery request on the stream") + } + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{ + Id: nodeID, + UserAgentName: "user-agent", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + }, + ResourceNames: []string{listenerName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Capture the version and nonce from the response. + var gotResp *v3discoverypb.DiscoveryResponse + select { + case gotResp = <-streamResponseCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery response on the stream") + } + version := gotResp.GetVersionInfo() + nonce := gotResp.GetNonce() + + // Verify that the ACK contains the appropriate version and nonce. + wantReq.VersionInfo = version + wantReq.ResponseNonce = nonce + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for the discovery request ACK on the stream") + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Verify the update received by the watcher. + wantUpdate := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: routeConfigName}, + } + if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Bring down the management server to simulate a broken stream. + lis.Stop() + + // Verify that the error callback on the watcher is not invoked. + verifyNoListenerUpdate(ctx, lw.updateCh) + + // Wait for backoff to kick in, and unblock the first backoff attempt. + select { + case <-backoffCh: + case <-ctx.Done(): + t.Fatal("Timeout waiting for stream backoff") + } + + // Bring up the management server. The test does not have prcecise control + // over when new streams to the management server will start succeeding. The + // ADS stream implementation will backoff as many times as required before + // it can successfully create a new stream. Therefore, we need to receive on + // the backoffCh as many times as required, and unblock the backoff + // implementation. + lis.Restart() + go func() { + for { + select { + case <-backoffCh: + case <-ctx.Done(): + return + } + } + }() + + // Verify that the transport creates a new stream and sends out a new + // request which contains the previously acked version, but an empty nonce. + wantReq.ResponseNonce = "" + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for the discovery request ACK on the stream") + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } +} + +// Tests the case where a resource is requested before the a valid ADS stream +// exists. Verifies that the a discovery request is sent out for the previously +// requested resource once a valid stream is created. +func (s) TestADS_ResourceRequestedBeforeStreamCreation(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Channels used for verifying different events in the test. + streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) // Discovery request is received. + + // Create an xDS management server listening on a local port. + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + streamErr := errors.New("ADS stream error") + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + + // Return an error everytime a request is sent on the stream. This + // should cause the transport to backoff before attempting to recreate + // the stream. + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + select { + case streamRequestCh <- req: + default: + } + return streamErr + }, + }) + + // Bring down the management server before creating the transport. This + // allows us to test the case where SendRequest() is called when there is no + // stream to the management server. + lis.Stop() + + // Override the backoff implementation to always return 0, to reduce test + // run time. Instead control when the backoff returns by blocking on a + // channel, that the test closes. + backoffCh := make(chan struct{}, 1) + unblockBackoffCh := make(chan struct{}) + streamBackoff := func(int) time.Duration { + select { + case backoffCh <- struct{}{}: + default: + } + <-unblockBackoffCh + return 0 + } + + // Create an xDS client with bootstrap pointing to the above server. + nodeID := uuid.New().String() + client := createXDSClientWithBackoff(t, mgmtServer.Address, nodeID, streamBackoff) + + // Register a watch for a listener resource. + const listenerName = "listener" + lw := newListenerWatcher() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw) + defer ldsCancel() + + // The above watch results in an attempt to create a new stream, which will + // fail, and will result in backoff. Wait for backoff to kick in. + select { + case <-backoffCh: + case <-ctx.Done(): + t.Fatal("Timeout waiting for stream backoff") + } + + // Bring up the connection to the management server, and unblock the backoff + // implementation. + lis.Restart() + close(unblockBackoffCh) + + // Verify that the initial discovery request matches expectation. + var gotReq *v3discoverypb.DiscoveryRequest + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery request on the stream") + } + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{ + Id: nodeID, + UserAgentName: "user-agent", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + }, + ResourceNames: []string{listenerName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } +} + +// waitForResourceNames waits for the wantNames to be received on namesCh. +// Returns a non-nil error if the context expires before that. +func waitForResourceNames(ctx context.Context, t *testing.T, namesCh chan []string, wantNames []string) error { + t.Helper() + + var lastRequestedNames []string + for ; ; <-time.After(defaultTestShortTimeout) { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for resources %v to be requested from the management server. Last requested resources: %v", wantNames, lastRequestedNames) + case gotNames := <-namesCh: + if cmp.Equal(gotNames, wantNames, cmpopts.EquateEmpty(), cmpopts.SortSlices(func(s1, s2 string) bool { return s1 < s2 })) { + return nil + } + lastRequestedNames = gotNames + } + } +} diff --git a/internal/xds/clients/xdsclient/test/ads_stream_flow_control_test.go b/internal/xds/clients/xdsclient/test/ads_stream_flow_control_test.go new file mode 100644 index 0000000..83155a9 --- /dev/null +++ b/internal/xds/clients/xdsclient/test/ads_stream_flow_control_test.go @@ -0,0 +1,625 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "slices" + "sort" + "testing" + "time" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" +) + +// blockingListenerWatcher implements xdsresource.ListenerWatcher. It writes to +// a channel when it receives a callback from the watch. It also makes the +// DoneNotifier passed to the callback available to the test, thereby enabling +// the test to block this watcher for as long as required. +type blockingListenerWatcher struct { + doneNotifierCh chan func() // DoneNotifier passed to the callback. + updateCh chan struct{} // Written to when an update is received. + ambientErrCh chan struct{} // Written to when an ambient error is received. + resourceErrCh chan struct{} // Written to when a resource error is received. +} + +func newBLockingListenerWatcher() *blockingListenerWatcher { + return &blockingListenerWatcher{ + doneNotifierCh: make(chan func(), 1), + updateCh: make(chan struct{}, 1), + ambientErrCh: make(chan struct{}, 1), + resourceErrCh: make(chan struct{}, 1), + } +} + +func (lw *blockingListenerWatcher) ResourceChanged(_ xdsclient.ResourceData, done func()) { + // Notify receipt of the update. + select { + case lw.updateCh <- struct{}{}: + default: + } + + select { + case lw.doneNotifierCh <- done: + default: + } +} + +func (lw *blockingListenerWatcher) ResourceError(_ error, done func()) { + // Notify receipt of an error. + select { + case lw.resourceErrCh <- struct{}{}: + default: + } + + select { + case lw.doneNotifierCh <- done: + default: + } +} + +func (lw *blockingListenerWatcher) AmbientError(_ error, done func()) { + // Notify receipt of an error. + select { + case lw.ambientErrCh <- struct{}{}: + default: + } + + select { + case lw.doneNotifierCh <- done: + default: + } +} + +type transportBuilder struct { + adsStreamCh chan *stream +} + +func (b *transportBuilder) Build(si clients.ServerIdentifier) (clients.Transport, error) { + cc, err := grpc.NewClient(si.ServerURI, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions(grpc.ForceCodec(&byteCodec{}))) + if err != nil { + return nil, err + } + + return &transport{cc: cc, adsStreamCh: b.adsStreamCh}, nil +} + +type transport struct { + cc *grpc.ClientConn + adsStreamCh chan *stream +} + +func (t *transport) NewStream(ctx context.Context, method string) (clients.Stream, error) { + s, err := t.cc.NewStream(ctx, &grpc.StreamDesc{ClientStreams: true, ServerStreams: true}, method) + if err != nil { + return nil, err + } + + stream := &stream{ + stream: s, + recvCh: make(chan struct{}, 1), + } + t.adsStreamCh <- stream + + return stream, nil +} + +func (t *transport) Close() { + t.cc.Close() +} + +type stream struct { + stream grpc.ClientStream + recvCh chan struct{} +} + +func (s *stream) Send(msg []byte) error { + return s.stream.SendMsg(msg) +} + +func (s *stream) Recv() ([]byte, error) { + select { + case s.recvCh <- struct{}{}: + case <-s.stream.Context().Done(): + // Unblock the recv() once the stream is done. + } + + var typedRes []byte + if err := s.stream.RecvMsg(&typedRes); err != nil { + return nil, err + } + return typedRes, nil +} + +type byteCodec struct{} + +func (c *byteCodec) Marshal(v any) ([]byte, error) { + if b, ok := v.([]byte); ok { + return b, nil + } + return nil, fmt.Errorf("transport: message is %T, but must be a []byte", v) +} + +func (c *byteCodec) Unmarshal(data []byte, v any) error { + if b, ok := v.(*[]byte); ok { + *b = data + return nil + } + return fmt.Errorf("transport: target is %T, but must be *[]byte", v) +} + +func (c *byteCodec) Name() string { + return "transport.byteCodec" +} + +// Tests ADS stream level flow control with a single resource. The test does the +// following: +// - Starts a management server and configures a listener resource on it. +// - Creates an xDS client to the above management server, starts a couple of +// listener watchers for the above resource, and verifies that the update +// reaches these watchers. +// - These watchers don't invoke the onDone callback until explicitly +// triggered by the test. This allows the test to verify that the next +// Recv() call on the ADS stream does not happen until both watchers have +// completely processed the update, i.e invoke the onDone callback. +// - Resource is updated on the management server, and the test verifies that +// the update reaches the watchers. +func (s) TestADSFlowControl_ResourceUpdates_SingleResource(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + + // Create an xDS client pointing to the above server with a test transport + // that allow monitoring the underlying stream through adsStreamCh. + adsStreamCh := make(chan *stream, 1) + client := createXDSClient(t, mgmtServer.Address, nodeID, &transportBuilder{adsStreamCh: adsStreamCh}) + + // Configure two watchers for the same listener resource. + const listenerResourceName = "test-listener-resource" + const routeConfigurationName = "test-route-configuration-resource" + watcher1 := newBLockingListenerWatcher() + cancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName, watcher1) + defer cancel1() + watcher2 := newBLockingListenerWatcher() + cancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName, watcher2) + defer cancel2() + + // Wait for the ADS stream to be created. + var adsStream *stream + select { + case adsStream = <-adsStreamCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for ADS stream to be created") + } + + // Configure the listener resource on the management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Ensure that there is a read on the stream. + select { + case <-adsStream.recvCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for ADS stream to be read from") + } + + // Wait for the update to reach the watchers. + select { + case <-watcher1.updateCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for update to reach watcher 1") + } + select { + case <-watcher2.updateCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for update to reach watcher 2") + } + + // Update the listener resource on the management server to point to a new + // route configuration resource. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, "new-route")}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Unblock one watcher. + onDone := <-watcher1.doneNotifierCh + onDone() + + // Wait for a short duration and ensure that there is no read on the stream. + select { + case <-adsStream.recvCh: + t.Fatal("Recv() called on the ADS stream before all watchers have processed the previous update") + case <-time.After(defaultTestShortTimeout): + } + + // Unblock the second watcher. + onDone = <-watcher2.doneNotifierCh + onDone() + + // Ensure that there is a read on the stream, now that the previous update + // has been consumed by all watchers. + select { + case <-adsStream.recvCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream after all watchers have processed the previous update") + } + + // Wait for the new update to reach the watchers. + select { + case <-watcher1.updateCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for update to reach watcher 1") + } + select { + case <-watcher2.updateCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for update to reach watcher 2") + } + + // At this point, the xDS client is shut down (and the associated transport + // is closed) without the watchers invoking their respective onDone + // callbacks. This verifies that the closing a transport that has pending + // watchers does not block. +} + +// Tests ADS stream level flow control with a multiple resources. The test does +// the following: +// - Starts a management server and configures two listener resources on it. +// - Creates an xDS client to the above management server, starts a couple of +// listener watchers for the two resources, and verifies that the update +// reaches these watchers. +// - These watchers don't invoke the onDone callback until explicitly +// triggered by the test. This allows the test to verify that the next +// Recv() call on the ADS stream does not happen until both watchers have +// completely processed the update, i.e invoke the onDone callback. +func (s) TestADSFlowControl_ResourceUpdates_MultipleResources(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server. + const listenerResourceName1 = "test-listener-resource-1" + const listenerResourceName2 = "test-listener-resource-2" + wantResourceNames := []string{listenerResourceName1, listenerResourceName2} + requestCh := make(chan struct{}, 1) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() != version.V3ListenerURL { + return nil + } + gotResourceNames := req.GetResourceNames() + sort.Slice(gotResourceNames, func(i, j int) bool { return req.ResourceNames[i] < req.ResourceNames[j] }) + if slices.Equal(gotResourceNames, wantResourceNames) { + // The two resource names will be part of the initial request + // and also the ACK. Hence, we need to make this write + // non-blocking. + select { + case requestCh <- struct{}{}: + default: + } + } + return nil + }, + }) + + nodeID := uuid.New().String() + + // Create an xDS client pointing to the above server with a test transport + // that allow monitoring the underlying stream through adsStreamCh. + adsStreamCh := make(chan *stream, 1) + client := createXDSClient(t, mgmtServer.Address, nodeID, &transportBuilder{adsStreamCh: adsStreamCh}) + + // Configure two watchers for two different listener resources. + const routeConfigurationName1 = "test-route-configuration-resource-1" + watcher1 := newBLockingListenerWatcher() + cancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName1, watcher1) + defer cancel1() + const routeConfigurationName2 = "test-route-configuration-resource-2" + watcher2 := newBLockingListenerWatcher() + cancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName2, watcher2) + defer cancel2() + + // Wait for the wrapped ADS stream to be created. + var adsStream *stream + select { + case adsStream = <-adsStreamCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for ADS stream to be created") + } + + // Ensure that there is a read on the stream. + select { + case <-adsStream.recvCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for ADS stream to be read from") + } + + // Wait for both resource names to be requested. + select { + case <-requestCh: + case <-ctx.Done(): + t.Fatal("Timed out waiting for both resource names to be requested") + } + + // Configure the listener resources on the management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(listenerResourceName1, routeConfigurationName1), + e2e.DefaultClientListener(listenerResourceName2, routeConfigurationName2), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // At this point, we expect the management server to send both resources in + // the same response. So, both watchers would be notified at the same time, + // and no more Recv() calls should happen until both of them have invoked + // their respective onDone() callbacks. + + // The order of callback invocations among the two watchers is not + // guaranteed. So, we select on both of them and unblock the first watcher + // whose callback is invoked. + var otherWatcherUpdateCh chan struct{} + var otherWatcherDoneCh chan func() + select { + case <-watcher1.updateCh: + onDone := <-watcher1.doneNotifierCh + onDone() + otherWatcherUpdateCh = watcher2.updateCh + otherWatcherDoneCh = watcher2.doneNotifierCh + case <-watcher2.updateCh: + onDone := <-watcher2.doneNotifierCh + onDone() + otherWatcherUpdateCh = watcher1.updateCh + otherWatcherDoneCh = watcher1.doneNotifierCh + case <-ctx.Done(): + t.Fatal("Timed out waiting for update to reach first watchers") + } + + // Wait for a short duration and ensure that there is no read on the stream. + select { + case <-adsStream.recvCh: + t.Fatal("Recv() called on the ADS stream before all watchers have processed the previous update") + case <-time.After(defaultTestShortTimeout): + } + + // Wait for the update on the second watcher and unblock it. + select { + case <-otherWatcherUpdateCh: + onDone := <-otherWatcherDoneCh + onDone() + case <-ctx.Done(): + t.Fatal("Timed out waiting for update to reach second watcher") + } + + // Ensure that there is a read on the stream, now that the previous update + // has been consumed by all watchers. + select { + case <-adsStream.recvCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream after all watchers have processed the previous update") + } +} + +// Test ADS stream flow control with a single resource that is expected to be +// NACKed by the xDS client and the watcher's ResourceError() callback is +// expected to be invoked because resource is not cached. Verifies that no +// further reads are attempted until the error is completely processed by the +// watcher. +func (s) TestADSFlowControl_ResourceErrors(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + + // Create an xDS client pointing to the above server with a test transport + // that allow monitoring the underlying stream through adsStreamCh. + adsStreamCh := make(chan *stream, 1) + client := createXDSClient(t, mgmtServer.Address, nodeID, &transportBuilder{adsStreamCh: adsStreamCh}) + + // Configure a watcher for a listener resource. + const listenerResourceName = "test-listener-resource" + watcher := newBLockingListenerWatcher() + cancel = client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName, watcher) + defer cancel() + + // Wait for the stream to be created. + var adsStream *stream + select { + case adsStream = <-adsStreamCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for ADS stream to be created") + } + + // Configure the management server to return a single listener resource + // which is expected to be NACKed by the client. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{badListenerResource(t, listenerResourceName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Ensure that there is a read on the stream. + select { + case <-adsStream.recvCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for ADS stream to be read from") + } + + // Wait for the resource error to reach the watcher. + select { + case <-watcher.resourceErrCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for error to reach watcher") + } + + // Wait for a short duration and ensure that there is no read on the stream. + select { + case <-adsStream.recvCh: + t.Fatal("Recv() called on the ADS stream before all watchers have processed the previous update") + case <-time.After(defaultTestShortTimeout): + } + + // Unblock one watcher. + onDone := <-watcher.doneNotifierCh + onDone() + + // Ensure that there is a read on the stream, now that the previous error + // has been consumed by the watcher. + select { + case <-adsStream.recvCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream after all watchers have processed the previous update") + } +} + +// Test ADS stream flow control with a single resource that is deleted from the +// management server and therefore the watcher's ResourceError() +// callback is expected to be invoked. Verifies that no further reads are +// attempted until the callback is completely handled by the watcher. +func (s) TestADSFlowControl_ResourceDoesNotExist(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + + // Create an xDS client pointing to the above server with a test transport + // that allow monitoring the underlying stream through adsStreamCh. + adsStreamCh := make(chan *stream, 1) + client := createXDSClient(t, mgmtServer.Address, nodeID, &transportBuilder{adsStreamCh: adsStreamCh}) + + // Configure a watcher for a listener resource. + const listenerResourceName = "test-listener-resource" + const routeConfigurationName = "test-route-configuration-resource" + watcher := newBLockingListenerWatcher() + cancel = client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName, watcher) + defer cancel() + + // Wait for the ADS stream to be created. + var adsStream *stream + select { + case adsStream = <-adsStreamCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for ADS stream to be created") + } + + // Configure the listener resource on the management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Ensure that there is a read on the stream. + select { + case <-adsStream.recvCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream") + } + + // Wait for the update to reach the watcher and unblock it. + select { + case <-watcher.updateCh: + onDone := <-watcher.doneNotifierCh + onDone() + case <-ctx.Done(): + t.Fatalf("Timed out waiting for update to reach watcher 1") + } + + // Ensure that there is a read on the stream. + select { + case <-adsStream.recvCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream") + } + + // Remove the listener resource on the management server. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Wait for the resource not found callback to be invoked. + select { + case <-watcher.resourceErrCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for resource not found callback to be invoked on the watcher") + } + + // Wait for a short duration and ensure that there is no read on the stream. + select { + case <-adsStream.recvCh: + t.Fatal("Recv() called on the ADS stream before all watchers have processed the previous update") + case <-time.After(defaultTestShortTimeout): + } + + // Unblock the watcher. + onDone := <-watcher.doneNotifierCh + onDone() + + // Ensure that there is a read on the stream. + select { + case <-adsStream.recvCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream") + } +} diff --git a/internal/xds/clients/xdsclient/test/ads_stream_restart_test.go b/internal/xds/clients/xdsclient/test/ads_stream_restart_test.go new file mode 100644 index 0000000..b205ac2 --- /dev/null +++ b/internal/xds/clients/xdsclient/test/ads_stream_restart_test.go @@ -0,0 +1,202 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "net" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/internal/testutils" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +// Tests that an ADS stream is restarted after a connection failure. Also +// verifies that if there were any watches registered before the connection +// failed, those resources are re-requested after the stream is restarted. +func (s) TestADS_ResourcesAreRequestedAfterStreamRestart(t *testing.T) { + // Create a restartable listener that can simulate a broken ADS stream. + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server that uses a couple of channels to inform + // the test about the specific LDS and CDS resource names being requested. + ldsResourcesCh := make(chan []string, 2) + streamOpened := make(chan struct{}, 1) + streamClosed := make(chan struct{}, 1) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + t.Logf("Received request for resources: %v of type %s", req.GetResourceNames(), req.GetTypeUrl()) + + // Drain the resource name channels before writing to them to ensure + // that the most recently requested names are made available to the + // test. + if req.GetTypeUrl() == version.V3ListenerURL { + select { + case <-ldsResourcesCh: + default: + } + ldsResourcesCh <- req.GetResourceNames() + } + return nil + }, + OnStreamClosed: func(int64, *v3corepb.Node) { + select { + case streamClosed <- struct{}{}: + default: + } + + }, + OnStreamOpen: func(context.Context, int64, string) error { + select { + case streamOpened <- struct{}{}: + default: + } + return nil + }, + }) + + // Create a listener resource on the management server. + const listenerName = "listener" + const routeConfigName = "route-config" + nodeID := uuid.New().String() + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client pointing to the above server. + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs)) + + // Register a watch for a listener resource. + lw := newListenerWatcher() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw) + defer ldsCancel() + + // Verify that an ADS stream is opened and an LDS request with the above + // resource name is sent. + select { + case <-streamOpened: + case <-ctx.Done(): + t.Fatal("Timeout when waiting for ADS stream to open") + } + if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil { + t.Fatal(err) + } + + // Verify the update received by the watcher. + wantListenerUpdate := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: routeConfigName, + }, + } + if err := verifyListenerUpdate(ctx, lw.updateCh, wantListenerUpdate); err != nil { + t.Fatal(err) + } + + // Create another listener resource on the management server, in addition + // to the existing listener resource. + const listenerName2 = "listener2" + const routeConfigName2 = "route-config2" + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName), e2e.DefaultClientListener(listenerName2, routeConfigName2)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Register a watch for another listener resource, and verify that a LDS request + // with the both listener resource names are sent. + lw2 := newListenerWatcher() + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerName2, lw2) + if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName, listenerName2}); err != nil { + t.Fatal(err) + } + + // Verify the update received by the watcher. + wantListenerUpdate = listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: routeConfigName2, + }, + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantListenerUpdate); err != nil { + t.Fatal(err) + } + + // Cancel the watch for the second listener resource, and verify that an LDS + // request with only first listener resource names is sent. + ldsCancel2() + if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil { + t.Fatal(err) + } + + // Stop the restartable listener and wait for the stream to close. + lis.Stop() + select { + case <-streamClosed: + case <-ctx.Done(): + t.Fatal("Timeout when waiting for ADS stream to close") + } + + // Restart the restartable listener and wait for the stream to open. + lis.Restart() + select { + case <-streamOpened: + case <-ctx.Done(): + t.Fatal("Timeout when waiting for ADS stream to open") + } + + // Verify that the first listener resource is requested again. + if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil { + t.Fatal(err) + } + + // Wait for a short duration and verify that no LDS request is sent, since + // there are no resources being watched. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case names := <-ldsResourcesCh: + t.Fatalf("LDS request sent for resource names %v, when expecting no request", names) + } +} diff --git a/internal/xds/clients/xdsclient/test/ads_stream_watch_test.go b/internal/xds/clients/xdsclient/test/ads_stream_watch_test.go new file mode 100644 index 0000000..40ca8ea --- /dev/null +++ b/internal/xds/clients/xdsclient/test/ads_stream_watch_test.go @@ -0,0 +1,222 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "net" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/internal/testutils" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + xdsclientinternal "google.golang.org/grpc/internal/xds/clients/xdsclient/internal" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" +) + +func waitForResourceWatchState(ctx context.Context, client *xdsclient.XDSClient, resourceName string, wantState xdsresource.WatchState, wantTimer bool) error { + var lastErr error + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + err := verifyResourceWatchState(client, resourceName, wantState, wantTimer) + if err == nil { + break + } + lastErr = err + } + if ctx.Err() != nil { + return fmt.Errorf("timeout when waiting for expected watch state for resource %q: %v", resourceName, lastErr) + } + return nil +} + +func verifyResourceWatchState(client *xdsclient.XDSClient, resourceName string, wantState xdsresource.WatchState, wantTimer bool) error { + resourceWatchStateForTesting := xdsclientinternal.ResourceWatchStateForTesting.(func(*xdsclient.XDSClient, xdsclient.ResourceType, string) (xdsresource.ResourceWatchState, error)) + gotState, err := resourceWatchStateForTesting(client, listenerType, resourceName) + if err != nil { + return fmt.Errorf("failed to get watch state for resource %q: %v", resourceName, err) + } + if gotState.State != wantState { + return fmt.Errorf("watch state for resource %q is %v, want %v", resourceName, gotState.State, wantState) + } + if (gotState.ExpiryTimer != nil) != wantTimer { + return fmt.Errorf("expiry timer for resource %q is %t, want %t", resourceName, gotState.ExpiryTimer != nil, wantTimer) + } + return nil +} + +// Tests the state transitions of the resource specific watch state within the +// ADS stream, specifically when the stream breaks (for both resources that have +// been previously received and for resources that are yet to be received). +func (s) TestADS_WatchState_StreamBreaks(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create an xDS management server with a restartable listener. + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis}) + + // Create an xDS client pointing to the above server. + nodeID := uuid.New().String() + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs)) + + // Create a watch for the first listener resource and verify that the timer + // is running and the watch state is `requested`. + const listenerName1 = "listener1" + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerName1, noopListenerWatcher{}) + defer ldsCancel1() + if err := waitForResourceWatchState(ctx, client, listenerName1, xdsresource.ResourceWatchStateRequested, true); err != nil { + t.Fatal(err) + } + + // Configure the first resource on the management server. This should result + // in the resource being pushed to the xDS client and should result in the + // timer getting stopped and the watch state moving to `received`. + const routeConfigName = "route-config" + listenerResource1 := e2e.DefaultClientListener(listenerName1, routeConfigName) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource1}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := waitForResourceWatchState(ctx, client, listenerName1, xdsresource.ResourceWatchStateReceived, false); err != nil { + t.Fatal(err) + } + + // Create a watch for the second listener resource and verify that the timer + // is running and the watch state is `requested`. + const listenerName2 = "listener2" + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerName2, noopListenerWatcher{}) + defer ldsCancel2() + if err := waitForResourceWatchState(ctx, client, listenerName2, xdsresource.ResourceWatchStateRequested, true); err != nil { + t.Fatal(err) + } + + // Stop the server to break the ADS stream. Since the first resource was + // already received, this should not change anything for it. But for the + // second resource, it should result in the timer getting stopped and the + // watch state moving to `started`. + lis.Stop() + if err := waitForResourceWatchState(ctx, client, listenerName2, xdsresource.ResourceWatchStateStarted, false); err != nil { + t.Fatal(err) + } + if err := verifyResourceWatchState(client, listenerName1, xdsresource.ResourceWatchStateReceived, false); err != nil { + t.Fatal(err) + } + + // Restart the server and verify that the timer is running and the watch + // state is `requested`, for the second resource. For the first resource, + // nothing should change. + lis.Restart() + if err := waitForResourceWatchState(ctx, client, listenerName2, xdsresource.ResourceWatchStateRequested, true); err != nil { + t.Fatal(err) + } + if err := verifyResourceWatchState(client, listenerName1, xdsresource.ResourceWatchStateReceived, false); err != nil { + t.Fatal(err) + } + + // Configure the second resource on the management server. This should result + // in the resource being pushed to the xDS client and should result in the + // timer getting stopped and the watch state moving to `received`. + listenerResource2 := e2e.DefaultClientListener(listenerName2, routeConfigName) + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource1, listenerResource2}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := waitForResourceWatchState(ctx, client, listenerName2, xdsresource.ResourceWatchStateReceived, false); err != nil { + t.Fatal(err) + } +} + +// Tests the behavior of the xDS client when a resource watch timer expires and +// verifies the resource watch state transitions as expected. +func (s) TestADS_WatchState_TimerFires(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create an xDS client with bootstrap pointing to the above server, and a + // short resource expiry timeout. + nodeID := uuid.New().String() + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + t.Cleanup(func() { client.Close() }) + // Create a watch for the first listener resource and verify that the timer + // is running and the watch state is `requested`. + const listenerName = "listener" + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerName, noopListenerWatcher{}) + defer ldsCancel1() + if err := waitForResourceWatchState(ctx, client, listenerName, xdsresource.ResourceWatchStateRequested, true); err != nil { + t.Fatal(err) + } + + // Since the resource is not configured on the management server, the watch + // expiry timer is expected to fire, and the watch state should move to + // `timeout`. + if err := waitForResourceWatchState(ctx, client, listenerName, xdsresource.ResourceWatchStateTimeout, false); err != nil { + t.Fatal(err) + } +} diff --git a/internal/xds/clients/xdsclient/test/authority_test.go b/internal/xds/clients/xdsclient/test/authority_test.go new file mode 100644 index 0000000..9941b98 --- /dev/null +++ b/internal/xds/clients/xdsclient/test/authority_test.go @@ -0,0 +1,334 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "net" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/internal/testutils" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" +) + +const ( + testAuthority1 = "test-authority1" + testAuthority2 = "test-authority2" + testAuthority3 = "test-authority3" +) + +var ( + // These two resources use `testAuthority1`, which contains an empty server + // config and therefore will use the default management server. + authorityTestResourceName11 = buildResourceName(listenerResourceTypeName, testAuthority1, ldsName, nil) + authorityTestResourceName12 = buildResourceName(listenerResourceTypeName, testAuthority1, ldsName+"2", nil) + // This resource uses `testAuthority2`, which contains an empty server + // config and therefore will use the default management server. + authorityTestResourceName2 = buildResourceName(listenerResourceTypeName, testAuthority2, ldsName+"3", nil) + // This resource uses `testAuthority3`, which contains a non-empty server + // config, and therefore will use the non-default management server. + authorityTestResourceName3 = buildResourceName(listenerResourceTypeName, testAuthority3, ldsName+"3", nil) +) + +// setupForAuthorityTests spins up two management servers, one to act as the +// default and the other to act as the non-default. It also creates a +// xDS client configuration with three authorities (the first two pointing to +// the default and the third one pointing to the non-default). +// +// Returns two listeners used by the default and non-default management servers +// respectively, and the xDS client. +func setupForAuthorityTests(ctx context.Context, t *testing.T) (*testutils.ListenerWrapper, *testutils.ListenerWrapper, *xdsclient.XDSClient) { + // Create listener wrappers which notify on to a channel whenever a new + // connection is accepted. We use this to track the number of transports + // used by the xDS client. + lisDefault := testutils.NewListenerWrapper(t, nil) + lisNonDefault := testutils.NewListenerWrapper(t, nil) + + // Start a management server to act as the default authority. + defaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisDefault}) + + // Start a management server to act as the non-default authority. + nonDefaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisNonDefault}) + + // Create a bootstrap configuration with two non-default authorities which + // have empty server configs, and therefore end up using the default server + // config, which points to the above management server. + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + ext := grpctransport.ServerIdentifierExtension{ConfigName: "insecure"} + siDefault := clients.ServerIdentifier{ + ServerURI: defaultAuthorityServer.Address, + Extensions: ext, + } + siNonDefault := clients.ServerIdentifier{ + ServerURI: nonDefaultAuthorityServer.Address, + Extensions: ext, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: siDefault}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + Authorities: map[string]xdsclient.Authority{ + testAuthority1: {XDSServers: []xdsclient.ServerConfig{}}, + testAuthority2: {XDSServers: []xdsclient.ServerConfig{}}, + testAuthority3: {XDSServers: []xdsclient.ServerConfig{{ServerIdentifier: siNonDefault}}}, + }, + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(authorityTestResourceName11, rdsName), + e2e.DefaultClientListener(authorityTestResourceName12, rdsName), + e2e.DefaultClientListener(authorityTestResourceName2, rdsName), + e2e.DefaultClientListener(authorityTestResourceName3, rdsName), + }, + SkipValidation: true, + } + if err := defaultAuthorityServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + return lisDefault, lisNonDefault, client +} + +// Tests the xdsChannel sharing logic among authorities. The test verifies the +// following scenarios: +// - A watch for a resource name with an authority matching an existing watch +// should not result in a new transport being created. +// - A watch for a resource name with different authority name but same +// authority config as an existing watch should not result in a new transport +// being created. +func (s) TestAuthority_XDSChannelSharing(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + lis, _, client := setupForAuthorityTests(ctx, t) + defer client.Close() + + // Verify that no connection is established to the management server at this + // point. A transport is created only when a resource (which belongs to that + // authority) is requested. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } + + // Request the first resource. Verify that a new transport is created. + watcher := noopListenerWatcher{} + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName11, watcher) + defer ldsCancel1() + if _, err := lis.NewConnCh.Receive(ctx); err != nil { + t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err) + } + + // Request the second resource. Verify that no new transport is created. + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName12, watcher) + defer ldsCancel2() + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } + + // Request the third resource. Verify that no new transport is created. + ldsCancel3 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName2, watcher) + defer ldsCancel3() + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } +} + +// Test the xdsChannel close logic. The test verifies that the xDS client +// closes an xdsChannel immediately after the last watch is canceled. +func (s) TestAuthority_XDSChannelClose(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + lis, _, client := setupForAuthorityTests(ctx, t) + defer client.Close() + + // Request the first resource. Verify that a new transport is created. + watcher := noopListenerWatcher{} + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName11, watcher) + val, err := lis.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err) + } + conn := val.(*testutils.ConnWrapper) + + // Request the second resource. Verify that no new transport is created. + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName12, watcher) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } + + // Cancel both watches, and verify that the connection to the management + // server is closed. + ldsCancel1() + ldsCancel2() + if _, err := conn.CloseCh.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for connection to management server to be closed") + } +} + +// Tests the scenario where the primary management server is unavailable at +// startup and the xDS client falls back to the secondary. The test verifies +// that the resource watcher is not notified of the connectivity failure until +// all servers have failed. +func (s) TestAuthority_Fallback(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create primary and secondary management servers with restartable + // listeners. + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + primaryLis := testutils.NewRestartableListener(l) + primaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis}) + l, err = net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + secondaryLis := testutils.NewRestartableListener(l) + secondaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: secondaryLis}) + + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + psi := clients.ServerIdentifier{ + ServerURI: primaryMgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + ssi := clients.ServerIdentifier{ + ServerURI: secondaryMgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + // Create config with the above primary and fallback management servers, + // and an xDS client with that configuration. + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: psi}, {ServerIdentifier: ssi}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + const listenerName = "listener" + const rdsPrimaryName = "rds-primary" + const rdsSecondaryName = "rds-secondary" + + // Create a Cluster resource on the primary. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, rdsPrimaryName)}, + SkipValidation: true, + } + if err := primaryMgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err) + } + + // Create a Cluster resource on the secondary . + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, rdsSecondaryName)}, + SkipValidation: true, + } + if err := secondaryMgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err) + } + + // Stop the primary. + primaryLis.Close() + + // Register a watch. + watcher := newListenerWatcher() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher) + defer ldsCancel() + + // Ensure that the connectivity error callback is not called. Since, this + // is the first watch without cached resource, it checks for resourceErrCh + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if v, err := watcher.resourceErrCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Resource error callback on the watcher with error: %v", v.(error)) + } + + // Ensure that the resource update callback is invoked. + wantUpdate := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsSecondaryName, + }, + } + if err := verifyListenerUpdate(ctx, watcher.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Stop the secondary. + secondaryLis.Close() + + // Ensure that the connectivity error callback is called as ambient error + // since cached resource exist. + if _, err := watcher.ambientErrCh.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for ambient error callback on the watcher") + } +} diff --git a/internal/xds/clients/xdsclient/test/custom_resource_watch_test.go b/internal/xds/clients/xdsclient/test/custom_resource_watch_test.go new file mode 100644 index 0000000..79865e0 --- /dev/null +++ b/internal/xds/clients/xdsclient/test/custom_resource_watch_test.go @@ -0,0 +1,254 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +// customTestResourceType is a fake resource type used for testing. +var customTestResourceType = xdsclient.ResourceType{ + TypeURL: "type.googleapis.com/google.protobuf.StringValue", + TypeName: "CustomResource", + AllResourcesRequiredInSotW: false, + Decoder: customTestDecoder{}, +} + +type customTestDecoder struct{} + +func (customTestDecoder) Decode(resource *xdsclient.AnyProto, _ xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) { + any := resource.ToAny() + if any.GetTypeUrl() != customTestResourceType.TypeURL { + return nil, fmt.Errorf("unexpected resource type: %q", any.GetTypeUrl()) + } + var val wrapperspb.StringValue + if err := proto.Unmarshal(any.GetValue(), &val); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + + // We expect the value to be "name|content". This allows us to extract the + // name which is required by the xDS client to match the watch. + str := val.GetValue() + name, content, found := strings.Cut(str, "|") + if !found { + return nil, fmt.Errorf("invalid format: expected 'name|content', got %q", str) + } + + return &xdsclient.DecodeResult{ + Name: name, + Resource: &customTestResourceData{val: content}, + }, nil +} + +type customTestResourceData struct { + val string +} + +func (c *customTestResourceData) Equal(other xdsclient.ResourceData) bool { + if c == nil && other == nil { + return true + } + if c == nil || other == nil { + return false + } + o, ok := other.(*customTestResourceData) + if !ok { + return false + } + return c.val == o.val +} + +func (c *customTestResourceData) Bytes() []byte { + return []byte(c.val) +} + +// customTestWatcher is a watcher for the custom resource type. +type customTestWatcher struct { + updateCh chan xdsclient.ResourceData + errCh chan error +} + +func newCustomTestWatcher() *customTestWatcher { + return &customTestWatcher{ + updateCh: make(chan xdsclient.ResourceData, 1), + errCh: make(chan error, 1), + } +} + +func (w *customTestWatcher) ResourceChanged(update xdsclient.ResourceData, onDone func()) { + w.updateCh <- update + onDone() +} + +func (w *customTestWatcher) ResourceError(err error, onDone func()) { + w.errCh <- fmt.Errorf("ResourceError: %v", err) + onDone() +} + +func (w *customTestWatcher) AmbientError(err error, onDone func()) { + w.errCh <- fmt.Errorf("AmbientError: %v", err) + onDone() +} + +// Tests that the xDS client can watch a custom resource type that is injected +// via the config. +func (s) TestCustomResourceWatch(t *testing.T) { + const authority = "my-authority" + resourceName := "xdstp://" + authority + "/" + customTestResourceType.TypeURL + "/my-resource" + + tests := []struct { + name string + resourceValue string + wantUpdate string + wantNACK string + }{ + { + name: "valid_resource", + resourceValue: resourceName + "|hello world", + wantUpdate: "hello world", + }, + { + name: "decode_error", + resourceValue: "malformed-value", + wantNACK: "invalid format: expected 'name|content'", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Start a fake xDS management server. + mgmtServer, cleanup, err := fakeserver.StartServer(nil) + if err != nil { + t.Fatalf("Failed to start fake xDS server: %v", err) + } + defer cleanup() + + resourceTypes := map[string]xdsclient.ResourceType{ + customTestResourceType.TypeURL: customTestResourceType, + } + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + nodeID := uuid.New().String() + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + Authorities: map[string]xdsclient.Authority{ + authority: {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + watcher := newCustomTestWatcher() + cancelWatch := client.WatchResource(customTestResourceType.TypeURL, resourceName, watcher) + defer cancelWatch() + + // Wait for the discovery request. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Verify the request type URL. + v, err := mgmtServer.XDSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for a DiscoveryRequest message: %v", err) + } + req := v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest) + if req.GetTypeUrl() != customTestResourceType.TypeURL { + t.Fatalf("DiscoveryRequest TypeUrl = %v, want %v", req.GetTypeUrl(), customTestResourceType.TypeURL) + } + if len(req.GetResourceNames()) != 1 || req.GetResourceNames()[0] != resourceName { + t.Fatalf("DiscoveryRequest ResourceNames = %v, want [%v]", req.GetResourceNames(), resourceName) + } + + // Send a response with the custom resource. + resource, err := anypb.New(&wrapperspb.StringValue{Value: test.resourceValue}) + if err != nil { + t.Fatalf("Failed to marshal resource: %v", err) + } + mgmtServer.XDSResponseChan <- &fakeserver.Response{ + Resp: &v3discoverypb.DiscoveryResponse{ + TypeUrl: customTestResourceType.TypeURL, + VersionInfo: "1", + Resources: []*anypb.Any{resource}, + }, + } + + if test.wantNACK != "" { + // Verify the NACK. + // We expect a new DiscoveryRequest with ErrorDetail set. + v, err = mgmtServer.XDSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for NACK DiscoveryRequest message: %v", err) + } + req = v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest) + if req.GetTypeUrl() != customTestResourceType.TypeURL { + t.Fatalf("DiscoveryRequest TypeUrl = %v, want %v", req.GetTypeUrl(), customTestResourceType.TypeURL) + } + if req.GetErrorDetail() == nil { + t.Fatalf("DiscoveryRequest ErrorDetail is nil, want non-nil") + } + if !strings.Contains(req.GetErrorDetail().GetMessage(), test.wantNACK) { + t.Fatalf("DiscoveryRequest ErrorDetail = %v, want substring %q", req.GetErrorDetail().GetMessage(), test.wantNACK) + } + return + } + + // Verify the update. + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for resource update") + case err := <-watcher.errCh: + t.Fatalf("Received unexpected error: %v", err) + case got := <-watcher.updateCh: + gotData, ok := got.(*customTestResourceData) + if !ok { + t.Fatalf("Received unexpected data type: %T", got) + } + if gotData.val != test.wantUpdate { + t.Fatalf("Received resource value %q, want %q", gotData.val, test.wantUpdate) + } + } + }) + } +} diff --git a/internal/xds/clients/xdsclient/test/dump_test.go b/internal/xds/clients/xdsclient/test/dump_test.go new file mode 100644 index 0000000..6455409 --- /dev/null +++ b/internal/xds/clients/xdsclient/test/dump_test.go @@ -0,0 +1,557 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/internal/pretty" + "google.golang.org/grpc/internal/xds/clients/internal/testutils" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + + v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" +) + +func makeGenericXdsConfig(typeURL, name, version string, status v3adminpb.ClientResourceStatus, config *anypb.Any, failure *v3adminpb.UpdateFailureState) *v3statuspb.ClientConfig_GenericXdsConfig { + return &v3statuspb.ClientConfig_GenericXdsConfig{ + TypeUrl: typeURL, + Name: name, + VersionInfo: version, + ClientStatus: status, + XdsConfig: config, + ErrorState: failure, + } +} + +func checkResourceDump(ctx context.Context, want *v3statuspb.ClientStatusResponse, client *xdsclient.XDSClient) error { + var cmpOpts = cmp.Options{ + protocmp.Transform(), + protocmp.IgnoreFields((*v3statuspb.ClientConfig_GenericXdsConfig)(nil), "last_updated"), + protocmp.IgnoreFields((*v3statuspb.ClientConfig)(nil), "client_scope"), + protocmp.IgnoreFields((*v3adminpb.UpdateFailureState)(nil), "last_update_attempt", "details"), + } + + var lastErr error + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + b, err := client.DumpResources() + if err != nil { + lastErr = err + continue + } + got := &v3statuspb.ClientStatusResponse{} + if err := proto.Unmarshal(b, got); err != nil { + lastErr = err + continue + } + // Sort the client configs based on the `client_scope` field. + slices.SortFunc(got.GetConfig(), func(a, b *v3statuspb.ClientConfig) int { + return strings.Compare(a.ClientScope, b.ClientScope) + }) + // Sort the resource configs based on the type_url and name fields. + for _, cc := range got.GetConfig() { + slices.SortFunc(cc.GetGenericXdsConfigs(), func(a, b *v3statuspb.ClientConfig_GenericXdsConfig) int { + if strings.Compare(a.TypeUrl, b.TypeUrl) == 0 { + return strings.Compare(a.Name, b.Name) + } + return strings.Compare(a.TypeUrl, b.TypeUrl) + }) + } + diff := cmp.Diff(want, got, cmpOpts) + if diff == "" { + return nil + } + lastErr = fmt.Errorf("received unexpected resource dump, diff (-got, +want):\n%s, got: %s\n want:%s", diff, pretty.ToJSON(got), pretty.ToJSON(want)) + } + return fmt.Errorf("timeout when waiting for resource dump to reach expected state: %v", lastErr) +} + +// Tests the scenario where there are multiple xDS clients talking to the same +// management server, and requesting the same set of resources. Verifies that +// under all circumstances, both xDS clients receive the same configuration from +// the server. +func (s) TestDumpResources_ManyToOne(t *testing.T) { + // Initialize the xDS resources to be used in this test. + ldsTargets := []string{"lds.target.good:0000", "lds.target.good:1111"} + rdsTargets := []string{"route-config-0", "route-config-1"} + listeners := make([]*v3listenerpb.Listener, len(ldsTargets)) + listenerAnys := make([]*anypb.Any, len(ldsTargets)) + for i := range ldsTargets { + listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i]) + listenerAnys[i] = testutils.MarshalAny(t, listeners[i]) + } + + // Spin up an xDS management server on a local port. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + // Create two xDS clients with the above config. + client1, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client1.Close() + client2, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client2.Close() + + // Dump resources and expect empty configs. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantNode := &v3corepb.Node{ + Id: nodeID, + UserAgentName: "user-agent", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + } + wantResp := &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, client1); err != nil { + t.Fatal(err) + } + if err := checkResourceDump(ctx, wantResp, client2); err != nil { + t.Fatal(err) + } + + // Register watches, dump resources and expect configs in requested state. + for _, xdsC := range []*xdsclient.XDSClient{client1, client2} { + for _, target := range ldsTargets { + xdsC.WatchResource(xdsresource.V3ListenerURL, target, noopListenerWatcher{}) + } + } + wantConfigs := []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, client1); err != nil { + t.Fatal(err) + } + if err := checkResourceDump(ctx, wantResp, client2); err != nil { + t.Fatal(err) + } + + // Configure the resources on the management server. + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + SkipValidation: true, + }); err != nil { + t.Fatal(err) + } + + // Dump resources and expect ACK configs. + wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[0], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, client1); err != nil { + t.Fatal(err) + } + if err := checkResourceDump(ctx, wantResp, client2); err != nil { + t.Fatal(err) + } + + // Update the first resource of each type in the management server to a + // value which is expected to be NACK'ed by the xDS client. + listeners[0] = func() *v3listenerpb.Listener { + hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{}) + return &v3listenerpb.Listener{ + Name: ldsTargets[0], + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + } + }() + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + SkipValidation: true, + }); err != nil { + t.Fatal(err) + } + + wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, listenerAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, client1); err != nil { + t.Fatal(err) + } + if err := checkResourceDump(ctx, wantResp, client2); err != nil { + t.Fatal(err) + } +} + +// Tests the scenario where there are multiple xDS client talking to different +// management server, and requesting different set of resources. +func (s) TestDumpResources_ManyToMany(t *testing.T) { + // Initialize the xDS resources to be used in this test: + // - The first xDS client watches old style resource names, and thereby + // requests these resources from the top-level xDS server. + // - The second xDS client watches new style resource names with a non-empty + // authority, and thereby requests these resources from the server + // configuration for that authority. + authority := strings.Join(strings.Split(t.Name(), "/"), "") + ldsTargets := []string{ + "lds.target.good:0000", + fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/lds.targer.good:1111", authority), + } + rdsTargets := []string{ + "route-config-0", + fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/route-config-1", authority), + } + listeners := make([]*v3listenerpb.Listener, len(ldsTargets)) + listenerAnys := make([]*anypb.Any, len(ldsTargets)) + for i := range ldsTargets { + listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i]) + listenerAnys[i] = testutils.MarshalAny(t, listeners[i]) + } + + // Start two management servers. + mgmtServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + mgmtServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // The first of the above management servers will be the top-level xDS + // server in the bootstrap configuration, and the second will be the xDS + // server corresponding to the test authority. + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{} + listenerType := listenerType + resourceTypes[xdsresource.V3ListenerURL] = listenerType + si1 := clients.ServerIdentifier{ + ServerURI: mgmtServer1.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + si2 := clients.ServerIdentifier{ + ServerURI: mgmtServer2.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si1}}, + Node: clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + Authorities: map[string]xdsclient.Authority{ + authority: {XDSServers: []xdsclient.ServerConfig{{ServerIdentifier: si2}}}, + }, + } + + // Create two xDS clients with the above config. + client1, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client1.Close() + client2, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client2.Close() + + // Check the resource dump before configuring resources on the management server. + // Dump resources and expect empty configs. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantNode := &v3corepb.Node{ + Id: nodeID, + UserAgentName: "user-agent", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + } + wantResp := &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, client1); err != nil { + t.Fatal(err) + } + if err := checkResourceDump(ctx, wantResp, client2); err != nil { + t.Fatal(err) + } + + // Register watches, the first xDS client watches old style resource names, + // while the second xDS client watches new style resource names. + client1.WatchResource(xdsresource.V3ListenerURL, ldsTargets[0], noopListenerWatcher{}) + client2.WatchResource(xdsresource.V3ListenerURL, ldsTargets[1], noopListenerWatcher{}) + + // Check the resource dump. Both clients should have all resources in + // REQUESTED state. + wantConfigs1 := []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + } + wantConfigs2 := []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs1, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, client1); err != nil { + t.Fatal(err) + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs2, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, client2); err != nil { + t.Fatal(err) + } + + // Configure resources on the first management server. + if err := mgmtServer1.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners[:1], + SkipValidation: true, + }); err != nil { + t.Fatal(err) + } + + // Check the resource dump. One client should have resources in ACKED state, + // while the other should still have resources in REQUESTED state. + wantConfigs1 = []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[0], nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs1, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, client1); err != nil { + t.Fatal(err) + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs2, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, client2); err != nil { + t.Fatal(err) + } + + // Configure resources on the second management server. + if err := mgmtServer2.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners[1:], + SkipValidation: true, + }); err != nil { + t.Fatal(err) + } + + // Check the resource dump. Both clients should have appropriate resources + // in REQUESTED state. + wantConfigs2 = []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs1, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, client1); err != nil { + t.Fatal(err) + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs2, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, client2); err != nil { + t.Fatal(err) + } +} + +// Tests that when a resource update is NACKed by the xDS client, the resource +// dump mentions that the xDS client is using the old good version of the +// resource, but contains the error state for the most recent bad update. +func (s) TestDumpResources_ACK_NACK(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Spin up an xDS management server on a local port. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs)) + + // Start watching a listener resource. + client.WatchResource(xdsresource.V3ListenerURL, ldsName, noopListenerWatcher{}) + + // Configure the management server with a good version of the listener resource. + goodListener := e2e.DefaultClientListener(ldsName, rdsName) + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{goodListener}, + SkipValidation: true, + }); err != nil { + t.Fatal(err) + } + + // Dump resources and expect ACK config with version "1". + wantNode := &v3corepb.Node{ + Id: nodeID, + UserAgentName: "user-agent", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + } + goodListenerAny := testutils.MarshalAny(t, goodListener) + wantConfigs := []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsName, "1", v3adminpb.ClientResourceStatus_ACKED, goodListenerAny, nil), + } + wantResp := &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, client); err != nil { + t.Fatal(err) + } + + // Update the management server with a bad version of the listener resource + // which is expected to be NACKed by the xDS client. + badListener := badListenerResource(t, ldsName) + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{badListener}, + SkipValidation: true, + }); err != nil { + t.Fatal(err) + } + + // Dump resources and verify that the xDS client is using the old good + // version of the resource ("1"), but contains the error state for the most + // recent bad update ("2"). + wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsName, "1", v3adminpb.ClientResourceStatus_NACKED, goodListenerAny, &v3adminpb.UpdateFailureState{VersionInfo: "2"}), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, client); err != nil { + t.Fatal(err) + } +} diff --git a/internal/xds/clients/xdsclient/test/helpers_test.go b/internal/xds/clients/xdsclient/test/helpers_test.go new file mode 100644 index 0000000..2908ec0 --- /dev/null +++ b/internal/xds/clients/xdsclient/test/helpers_test.go @@ -0,0 +1,390 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "bytes" + "context" + "errors" + "fmt" + "strconv" + "strings" + "sync" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/internal/buffer" + "google.golang.org/grpc/internal/xds/clients/internal/pretty" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + "github.com/google/go-cmp/cmp" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestWatchExpiryTimeout = 500 * time.Millisecond + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. + + // ListenerResourceTypeName represents the transport agnostic name for the + // listener resource. + listenerResourceTypeName = "ListenerResource" + + ldsName = "xdsclient-test-lds-resource" + rdsName = "xdsclient-test-rds-resource" + ldsNameNewStyle = "xdstp:///envoy.config.listener.v3.Listener/xdsclient-test-lds-resource" + rdsNameNewStyle = "xdstp:///envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource" +) + +var ( + // Singleton instantiation of the resource type implementation. + listenerType = xdsclient.ResourceType{ + TypeURL: xdsresource.V3ListenerURL, + TypeName: listenerResourceTypeName, + AllResourcesRequiredInSotW: true, + Decoder: listenerDecoder{}, + } +) + +func unmarshalListenerResource(rProto *anypb.Any) (string, listenerUpdate, error) { + rProto, err := xdsresource.UnwrapResource(rProto) + if err != nil { + return "", listenerUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err) + } + if !xdsresource.IsListenerResource(rProto.GetTypeUrl()) { + return "", listenerUpdate{}, fmt.Errorf("unexpected listener resource type: %q", rProto.GetTypeUrl()) + } + lis := &v3listenerpb.Listener{} + if err := proto.Unmarshal(rProto.GetValue(), lis); err != nil { + return "", listenerUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + + lu, err := processListener(lis) + if err != nil { + return lis.GetName(), listenerUpdate{}, err + } + lu.Raw = rProto.GetValue() + return lis.GetName(), *lu, nil +} + +func processListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) { + if lis.GetApiListener() != nil { + return processClientSideListener(lis) + } + return processServerSideListener(lis) +} + +// processClientSideListener checks if the provided Listener proto meets +// the expected criteria. If so, it returns a non-empty routeConfigName. +func processClientSideListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) { + update := &listenerUpdate{} + + apiLisAny := lis.GetApiListener().GetApiListener() + if !xdsresource.IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) { + return nil, fmt.Errorf("unexpected http connection manager resource type: %q", apiLisAny.GetTypeUrl()) + } + apiLis := &v3httppb.HttpConnectionManager{} + if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil { + return nil, fmt.Errorf("failed to unmarshal api_listener: %v", err) + } + + switch apiLis.RouteSpecifier.(type) { + case *v3httppb.HttpConnectionManager_Rds: + if configsource := apiLis.GetRds().GetConfigSource(); configsource.GetAds() == nil && configsource.GetSelf() == nil { + return nil, fmt.Errorf("LDS's RDS configSource is not ADS or Self: %+v", lis) + } + name := apiLis.GetRds().GetRouteConfigName() + if name == "" { + return nil, fmt.Errorf("empty route_config_name: %+v", lis) + } + update.RouteConfigName = name + case *v3httppb.HttpConnectionManager_RouteConfig: + routeU := apiLis.GetRouteConfig() + if routeU == nil { + return nil, fmt.Errorf("empty inline RDS resp:: %+v", lis) + } + if routeU.Name == "" { + return nil, fmt.Errorf("empty route_config_name in inline RDS resp: %+v", lis) + } + update.RouteConfigName = routeU.Name + case nil: + return nil, fmt.Errorf("no RouteSpecifier: %+v", apiLis) + default: + return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", apiLis.RouteSpecifier) + } + + return update, nil +} + +func processServerSideListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) { + if n := len(lis.ListenerFilters); n != 0 { + return nil, fmt.Errorf("unsupported field 'listener_filters' contains %d entries", n) + } + if lis.GetUseOriginalDst().GetValue() { + return nil, errors.New("unsupported field 'use_original_dst' is present and set to true") + } + addr := lis.GetAddress() + if addr == nil { + return nil, fmt.Errorf("no address field in LDS response: %+v", lis) + } + sockAddr := addr.GetSocketAddress() + if sockAddr == nil { + return nil, fmt.Errorf("no socket_address field in LDS response: %+v", lis) + } + lu := &listenerUpdate{ + InboundListenerCfg: &inboundListenerConfig{ + Address: sockAddr.GetAddress(), + Port: strconv.Itoa(int(sockAddr.GetPortValue())), + }, + } + + return lu, nil +} + +type listenerDecoder struct{} + +// Decode deserializes and validates an xDS resource serialized inside the +// provided `Any` proto, as received from the xDS management server. +func (listenerDecoder) Decode(resource *xdsclient.AnyProto, _ xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) { + name, listener, err := unmarshalListenerResource(resource.ToAny()) + switch { + case name == "": + // Name is unset only when protobuf deserialization fails. + return nil, err + case err != nil: + // Protobuf deserialization succeeded, but resource validation failed. + return &xdsclient.DecodeResult{Name: name, Resource: &listenerResourceData{Resource: listenerUpdate{}}}, err + } + + return &xdsclient.DecodeResult{Name: name, Resource: &listenerResourceData{Resource: listener}}, nil + +} + +// listenerResourceData wraps the configuration of a Listener resource as +// received from the management server. +// +// Implements the ResourceData interface. +type listenerResourceData struct { + xdsclient.ResourceData + + Resource listenerUpdate +} + +// Equal returns true if other is equal to l. +func (l *listenerResourceData) Equal(other xdsclient.ResourceData) bool { + if l == nil && other == nil { + return true + } + if (l == nil) != (other == nil) { + return false + } + return bytes.Equal(l.Resource.Raw, other.Bytes()) +} + +// ToJSON returns a JSON string representation of the resource data. +func (l *listenerResourceData) ToJSON() string { + return pretty.ToJSON(l.Resource) +} + +// Bytes returns the underlying raw protobuf form of the listener resource. +func (l *listenerResourceData) Bytes() []byte { + return l.Resource.Raw +} + +// ListenerUpdate contains information received in an LDS response, which is of +// interest to the registered LDS watcher. +type listenerUpdate struct { + // RouteConfigName is the route configuration name corresponding to the + // target which is being watched through LDS. + RouteConfigName string + + // InboundListenerCfg contains inbound listener configuration. + InboundListenerCfg *inboundListenerConfig + + // Raw is the resource from the xds response. + Raw []byte +} + +// InboundListenerConfig contains information about the inbound listener, i.e +// the server-side listener. +type inboundListenerConfig struct { + // Address is the local address on which the inbound listener is expected to + // accept incoming connections. + Address string + // Port is the local port on which the inbound listener is expected to + // accept incoming connections. + Port string +} + +func makeAuthorityName(name string) string { + segs := strings.Split(name, "/") + return strings.Join(segs, "") +} + +func makeNewStyleLDSName(authority string) string { + return fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource", authority) +} + +// buildResourceName returns the resource name in the format of an xdstp:// +// resource. +func buildResourceName(typeName, auth, id string, ctxParams map[string]string) string { + var typS string + switch typeName { + case listenerResourceTypeName: + typS = "envoy.config.listener.v3.Listener" + default: + // If the name doesn't match any of the standard resources fallback + // to the type name. + typS = typeName + } + return (&xdsresource.Name{ + Scheme: "xdstp", + Authority: auth, + Type: typS, + ID: id, + ContextParams: ctxParams, + }).String() +} + +// testMetricsReporter is a MetricsReporter to be used in tests. It sends +// recording events on channels and provides helpers to check if certain events +// have taken place. +type testMetricsReporter struct { + metricsCh *buffer.Unbounded[any] + + mu sync.Mutex + asyncReporters map[clients.AsyncReporter]struct{} +} + +// newTestMetricsReporter returns a new testMetricsReporter. +func newTestMetricsReporter() *testMetricsReporter { + return &testMetricsReporter{ + metricsCh: buffer.NewUnbounded[any](), + asyncReporters: make(map[clients.AsyncReporter]struct{}), + } +} + +// waitForMetric waits for a metric to be recorded and verifies that the +// recorded metrics data matches the expected metricsDataWant. Returns +// an error if failed to wait or received wrong data. +func (r *testMetricsReporter) waitForMetric(ctx context.Context, metricsDataWant any) error { + select { + case got := <-r.metricsCh.Get(): + r.metricsCh.Load() + if diff := cmp.Diff(got, metricsDataWant); diff != "" { + return fmt.Errorf("received unexpected metrics value (-got, +want): %v", diff) + } + return nil + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for metric check: %v", ctx.Err()) + } +} + +// waitForSpecificMetric waits for a specific metric to be recorded, ignoring +// and discarding any other metrics received in the meantime. Returns an error +// if the context expires before the requested metric is received. +// +// This is necessary when multiple distinct metrics (e.g., ServerFailure, +// ResourceUpdateValid, and XDSClientConnected) might be emitted asynchronously +// and concurrently. Using waitForMetric would fail the test if an unrelated +// metric happens to be at the front of the channel. +func (r *testMetricsReporter) waitForSpecificMetric(ctx context.Context, metricsDataWant any) error { + for { + select { + case got := <-r.metricsCh.Get(): + r.metricsCh.Load() + if diff := cmp.Diff(got, metricsDataWant); diff == "" { + return nil + } + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for specific metric: %v", ctx.Err()) + } + } +} + +// Receive waits for a metric to be recorded and returns it. Returns an error +// if the context expires before a metric is received. +func (r *testMetricsReporter) Receive(ctx context.Context) (any, error) { + select { + case got := <-r.metricsCh.Get(): + r.metricsCh.Load() + return got, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// Drain clears all accumulated metrics from the channel. +func (r *testMetricsReporter) Drain() { + for { + select { + case <-r.metricsCh.Get(): + r.metricsCh.Load() + default: + return + } + } +} + +// ReportMetric sends the metrics data to the metricsCh channel. +func (r *testMetricsReporter) ReportMetric(m any) { + r.metricsCh.Put(m) +} + +func (r *testMetricsReporter) RegisterAsyncReporter(reporter clients.AsyncReporter) func() { + r.mu.Lock() + defer r.mu.Unlock() + r.asyncReporters[reporter] = struct{}{} + return sync.OnceFunc(func() { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.asyncReporters, reporter) + }) +} + +// triggerAsyncMetrics invokes the Report method on all registered asynchronous +// metric reporters. This allows tests to explicitly compel the emission of +// async gauge metrics at specific precise points in time for verification. +func (r *testMetricsReporter) triggerAsyncMetrics() { + r.mu.Lock() + defer r.mu.Unlock() + for reporter := range r.asyncReporters { + reporter.Report(r) + } +} + +func (r *testMetricsReporter) numAsyncReporters() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.asyncReporters) +} diff --git a/internal/xds/clients/xdsclient/test/lds_watchers_test.go b/internal/xds/clients/xdsclient/test/lds_watchers_test.go new file mode 100644 index 0000000..ae47839 --- /dev/null +++ b/internal/xds/clients/xdsclient/test/lds_watchers_test.go @@ -0,0 +1,1402 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/internal/syncutil" + "google.golang.org/grpc/internal/xds/clients/internal/testutils" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +type noopListenerWatcher struct{} + +func (noopListenerWatcher) ResourceChanged(_ xdsclient.ResourceData, onDone func()) { + onDone() +} +func (noopListenerWatcher) ResourceError(_ error, onDone func()) { + onDone() +} +func (noopListenerWatcher) AmbientError(_ error, onDone func()) { + onDone() +} + +type listenerUpdateErrTuple struct { + update listenerUpdate + resourceErr error + ambientErr error +} + +type listenerWatcher struct { + updateCh *testutils.Channel // Messages of type listenerUpdate + resourceErrCh *testutils.Channel // Messages of type resource error + ambientErrCh *testutils.Channel // Messages of type ambient error +} + +func newListenerWatcher() *listenerWatcher { + return &listenerWatcher{ + updateCh: testutils.NewChannelWithSize(1), + resourceErrCh: testutils.NewChannelWithSize(1), + ambientErrCh: testutils.NewChannelWithSize(1), + } +} + +func (lw *listenerWatcher) ResourceChanged(update xdsclient.ResourceData, onDone func()) { + lisData, ok := update.(*listenerResourceData) + if !ok { + lw.resourceErrCh.Send(listenerUpdateErrTuple{resourceErr: fmt.Errorf("unexpected resource type: %T", update)}) + onDone() + return + } + select { + case <-lw.updateCh.C: + default: + } + lw.updateCh.Send(listenerUpdateErrTuple{update: lisData.Resource}) + onDone() +} + +func (lw *listenerWatcher) AmbientError(err error, onDone func()) { + // When used with a go-control-plane management server that continuously + // resends resources which are NACKed by the xDS client, using a `Replace()` + // here and in OnResourceDoesNotExist() simplifies tests which will have + // access to the most recently received error. + lw.ambientErrCh.Replace(listenerUpdateErrTuple{ambientErr: err}) + onDone() +} + +func (lw *listenerWatcher) ResourceError(err error, onDone func()) { + lw.resourceErrCh.Replace(listenerUpdateErrTuple{resourceErr: err}) + onDone() +} + +// badListenerResource returns a listener resource for the given name which does +// not contain the `RouteSpecifier` field in the HTTPConnectionManager, and +// hence is expected to be NACKed by the client. +func badListenerResource(t *testing.T, name string) *v3listenerpb.Listener { + hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{}) + return &v3listenerpb.Listener{ + Name: name, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + } +} + +// verifyNoListenerUpdate verifies that no listener update is received on the +// provided update channel, and returns an error if an update is received. +// +// A very short deadline is used while waiting for the update, as this function +// is intended to be used when an update is not expected. +func verifyNoListenerUpdate(ctx context.Context, updateCh *testutils.Channel) error { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if u, err := updateCh.Receive(sCtx); err != context.DeadlineExceeded { + return fmt.Errorf("unexpected ListenerUpdate: %v", u) + } + return nil +} + +// verifyListenerUpdate waits for a listenerUpdateErrTuple from the provided +// updateCh (typically the updateCh, resourceErrCh, or ambientErrCh of +// listenerWatcher) and verifies that it matches the expected wantUpdate tuple. +// +// It performs the following checks: +// - Waits for an item on updateCh until the context deadline. +// - If wantUpdate contains a resourceErr or ambientErr, it compares the +// xdsresource.ErrorType of the received error with the expected error +// type. +// - If wantUpdate contains an update, it compares the received update with +// the expected update, ignoring the Raw field. +// +// Returns an error if the context expires, or if the received tuple does not +// match the expected tuple according to the comparison logic. +func verifyListenerUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate listenerUpdateErrTuple) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for a listener resource from the management server: %v", err) + } + got := u.(listenerUpdateErrTuple) + if wantUpdate.resourceErr != nil { + if gotType, wantType := xdsresource.ErrType(got.resourceErr), xdsresource.ErrType(wantUpdate.resourceErr); gotType != wantType { + return fmt.Errorf("received update with resource error type %v, want %v", gotType, wantType) + } + } + if wantUpdate.ambientErr != nil { + if gotType, wantType := xdsresource.ErrType(got.ambientErr), xdsresource.ErrType(wantUpdate.ambientErr); gotType != wantType { + return fmt.Errorf("received update with ambient error type %v, want %v", gotType, wantType) + } + } + cmpOpts := []cmp.Option{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(listenerUpdate{}, "Raw"), + } + if diff := cmp.Diff(wantUpdate.update, got.update, cmpOpts...); diff != "" { + return fmt.Errorf("received unexpected diff in the listener resource update: (-want, got):\n%s", diff) + } + return nil +} + +func verifyListenerResourceError(ctx context.Context, updateCh *testutils.Channel, wantErr, wantNodeID string) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for a listener error from the management server: %v", err) + } + gotErr := u.(listenerUpdateErrTuple).resourceErr + return verifyListenerError(ctx, gotErr, wantErr, wantNodeID) +} + +func verifyListenerError(_ context.Context, gotErr error, wantErr, wantNodeID string) error { + if gotErr == nil || !strings.Contains(gotErr.Error(), wantErr) { + return fmt.Errorf("update received with error: %v, want %q", gotErr, wantErr) + } + if !strings.Contains(gotErr.Error(), wantNodeID) { + return fmt.Errorf("update received with error: %v, want error with node ID: %q", gotErr, wantNodeID) + } + return nil +} + +func verifyAmbientErrorType(ctx context.Context, updateCh *testutils.Channel, wantErrType xdsresource.ErrorType, wantNodeID string) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for a listener error from the management server: %v", err) + } + gotErr := u.(listenerUpdateErrTuple).ambientErr + return verifyErrorType(gotErr, wantErrType, wantNodeID) +} + +func verifyResourceErrorType(ctx context.Context, updateCh *testutils.Channel, wantErrType xdsresource.ErrorType, wantNodeID string) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for a listener error from the management server: %v", err) + } + gotErr := u.(listenerUpdateErrTuple).resourceErr + return verifyErrorType(gotErr, wantErrType, wantNodeID) +} + +func verifyErrorType(gotErr error, wantErrType xdsresource.ErrorType, wantNodeID string) error { + if got, want := xdsresource.ErrType(gotErr), wantErrType; got != want { + return fmt.Errorf("update received with error %v of type: %v, want %v", gotErr, got, want) + } + if !strings.Contains(gotErr.Error(), wantNodeID) { + return fmt.Errorf("update received with error: %v, want error with node ID: %q", gotErr, wantNodeID) + } + return nil +} + +// TestLDSWatch covers the case where a single watcher exists for a single +// listener resource. The test verifies the following scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of the watch callback. +// 2. An update from the management server containing a resource *not* being +// watched should not result in the invocation of the watch callback. +// 3. After the watch is cancelled, an update from the management server +// containing the resource that was being watched should not result in the +// invocation of the watch callback. +// +// The test is run for old and new style names. +func (s) TestLDSWatch(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3listenerpb.Listener // The resource being watched. + updatedWatchedResource *v3listenerpb.Listener // The watched resource after an update. + notWatchedResource *v3listenerpb.Listener // A resource which is not being watched. + wantUpdate listenerUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: ldsName, + watchedResource: e2e.DefaultClientListener(ldsName, rdsName), + updatedWatchedResource: e2e.DefaultClientListener(ldsName, "new-rds-resource"), + notWatchedResource: e2e.DefaultClientListener("unsubscribed-lds-resource", rdsName), + wantUpdate: listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsName, + }, + }, + }, + { + desc: "new style resource", + resourceName: ldsNameNewStyle, + watchedResource: e2e.DefaultClientListener(ldsNameNewStyle, rdsNameNewStyle), + updatedWatchedResource: e2e.DefaultClientListener(ldsNameNewStyle, "new-rds-resource"), + notWatchedResource: e2e.DefaultClientListener("unsubscribed-lds-resource", rdsNameNewStyle), + wantUpdate: listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsNameNewStyle, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + lw := newListenerWatcher() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, test.resourceName, lw) + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyListenerUpdate(ctx, lw.updateCh, test.wantUpdate); err != nil { + t.Fatal(err) + } + + // Configure the management server to return an additional listener + // resource, one that we are not interested in. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.watchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoListenerUpdate(ctx, lw.updateCh); err != nil { + t.Fatal(err) + } + + // Cancel the watch and update the resource corresponding to the original + // watch. Ensure that the cancelled watch callback is not invoked. + ldsCancel() + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.updatedWatchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoListenerUpdate(ctx, lw.updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestLDSWatch_TwoWatchesForSameResourceName covers the case where two watchers +// exist for a single listener resource. The test verifies the following +// scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of both watch callbacks. +// 2. After one of the watches is cancelled, a redundant update from the +// management server should not result in the invocation of either of the +// watch callbacks. +// 3. An update from the management server containing the resource being +// watched should result in the invocation of the un-cancelled watch +// callback. +// +// The test is run for old and new style names. +func (s) TestLDSWatch_TwoWatchesForSameResourceName(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3listenerpb.Listener // The resource being watched. + updatedWatchedResource *v3listenerpb.Listener // The watched resource after an update. + wantUpdateV1 listenerUpdateErrTuple + wantUpdateV2 listenerUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: ldsName, + watchedResource: e2e.DefaultClientListener(ldsName, rdsName), + updatedWatchedResource: e2e.DefaultClientListener(ldsName, "new-rds-resource"), + wantUpdateV1: listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsName, + }, + }, + wantUpdateV2: listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: "new-rds-resource", + }, + }, + }, + { + desc: "new style resource", + resourceName: ldsNameNewStyle, + watchedResource: e2e.DefaultClientListener(ldsNameNewStyle, rdsNameNewStyle), + updatedWatchedResource: e2e.DefaultClientListener(ldsNameNewStyle, "new-rds-resource"), + wantUpdateV1: listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsNameNewStyle, + }, + }, + wantUpdateV2: listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: "new-rds-resource", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Register two watches for the same listener resource and have the + // callbacks push the received updates on to a channel. + lw1 := newListenerWatcher() + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, test.resourceName, lw1) + defer ldsCancel1() + lw2 := newListenerWatcher() + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, test.resourceName, lw2) + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered watches for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyListenerUpdate(ctx, lw1.updateCh, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + + // Cancel the second watch and force the management server to push a + // redundant update for the resource being watched. Neither of the + // two watch callbacks should be invoked. + ldsCancel2() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoListenerUpdate(ctx, lw1.updateCh); err != nil { + t.Fatal(err) + } + if err := verifyNoListenerUpdate(ctx, lw2.updateCh); err != nil { + t.Fatal(err) + } + + // Update to the resource being watched. The un-cancelled callback + // should be invoked while the cancelled one should not be. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.updatedWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, test.wantUpdateV2); err != nil { + t.Fatal(err) + } + if err := verifyNoListenerUpdate(ctx, lw2.updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestLDSWatch_ThreeWatchesForDifferentResourceNames covers the case with three +// watchers (two watchers for one resource, and the third watcher for another +// resource), exist across two listener resources. The test verifies that an +// update from the management server containing both resources results in the +// invocation of all watch callbacks. +// +// The test is run with both old and new style names. +func (s) TestLDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + Authorities: map[string]xdsclient.Authority{ + authority: {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Register two watches for the same listener resource and have the + // callbacks push the received updates on to a channel. + lw1 := newListenerWatcher() + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw1) + defer ldsCancel1() + lw2 := newListenerWatcher() + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw2) + defer ldsCancel2() + + // Register the third watch for a different listener resource. + ldsNameNewStyle := makeNewStyleLDSName(authority) + lw3 := newListenerWatcher() + ldsCancel3 := client.WatchResource(xdsresource.V3ListenerURL, ldsNameNewStyle, lw3) + defer ldsCancel3() + + // Configure the management server to return two listener resources, + // corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(ldsName, rdsName), + e2e.DefaultClientListener(ldsNameNewStyle, rdsName), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the all watchers. The two + // resources returned differ only in the resource name. Therefore the + // expected update is the same for all the watchers. + wantUpdate := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsName, + }, + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, lw3.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_ResourceCaching covers the case where a watch is registered for +// a resource which is already present in the cache. The test verifies that the +// watch callback is invoked with the contents from the cache, instead of a +// request being sent to the management server. +func (s) TestLDSWatch_ResourceCaching(t *testing.T) { + firstRequestReceived := false + firstAckReceived := syncutil.NewEvent() + secondRequestReceived := syncutil.NewEvent() + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + // The first request has an empty version string. + if !firstRequestReceived && req.GetVersionInfo() == "" { + firstRequestReceived = true + return nil + } + // The first ack has a non-empty version string. + if !firstAckReceived.HasFired() && req.GetVersionInfo() != "" { + firstAckReceived.Fire() + return nil + } + // Any requests after the first request and ack, are not expected. + secondRequestReceived.Fire() + return nil + }, + }) + + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + lw1 := newListenerWatcher() + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw1) + defer ldsCancel1() + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsName, + }, + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + t.Fatal("timeout when waiting for receipt of ACK at the management server") + case <-firstAckReceived.Done(): + } + + // Register another watch for the same resource. This should get the update + // from the cache. + lw2 := newListenerWatcher() + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw2) + defer ldsCancel2() + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + // No request should get sent out as part of this watch. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-secondRequestReceived.Done(): + t.Fatal("xdsClient sent out request instead of using update from cache") + } +} + +// TestLDSWatch_ExpiryTimerFiresBeforeResponse tests the case where the client +// does not receive an LDS response for the request that it sends. The test +// verifies that the watch callback is invoked with an error once the +// watchExpiryTimer fires. +func (s) TestLDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Override the default watch expiry timeout. + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + } + + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Register a watch for a resource which is expected to fail with an error + // after the watch expiry timer fires. + lw := newListenerWatcher() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw) + defer ldsCancel() + + // Wait for the watch expiry timer to fire. + <-time.After(defaultTestWatchExpiryTimeout) + + // Verify that an empty update with the expected resource error is + // received. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantErr := xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, "") + if err := verifyListenerUpdate(ctx, lw.resourceErrCh, listenerUpdateErrTuple{resourceErr: wantErr}); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_ValidResponseCancelsExpiryTimerBehavior tests the case where the +// client receives a valid LDS response for the request that it sends. The test +// verifies that the behavior associated with the expiry timer (i.e, callback +// invocation with error) does not take place. +func (s) TestLDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Override the default watch expiry timeout. + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + lw := newListenerWatcher() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw) + defer ldsCancel() + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsName, + }, + } + if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Wait for the watch expiry timer to fire, and verify that the callback is + // not invoked. + <-time.After(defaultTestWatchExpiryTimeout) + if err := verifyNoListenerUpdate(ctx, lw.updateCh); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_ResourceRemoved covers the cases where a resource being watched +// is removed from the management server. The test verifies the following +// scenarios: +// 1. Removing a resource should trigger the watch callback with a resource +// removed error. It should not trigger the watch callback for an unrelated +// resource. +// 2. An update to another resource should result in the invocation of the watch +// callback associated with that resource. It should not result in the +// invocation of the watch callback associated with the deleted resource. +// +// The test is run with both old and new style names. +func (s) TestLDSWatch_ResourceRemoved(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + Authorities: map[string]xdsclient.Authority{ + authority: {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Register two watches for two listener resources and have the + // callbacks push the received updates on to a channel. + resourceName1 := ldsName + lw1 := newListenerWatcher() + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, resourceName1, lw1) + defer ldsCancel1() + + resourceName2 := makeNewStyleLDSName(authority) + lw2 := newListenerWatcher() + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, resourceName2, lw2) + defer ldsCancel2() + + // Configure the management server to return two listener resources, + // corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(resourceName1, rdsName), + e2e.DefaultClientListener(resourceName2, rdsName), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for both watchers. The two + // resources returned differ only in the resource name. Therefore the + // expected update is the same for both watchers. + wantUpdate := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsName, + }, + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Remove the first listener resource on the management server. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(resourceName2, rdsName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // The first watcher should receive a resource error for resource removal, + // while the second watcher should not see an update. + if err := verifyListenerUpdate(ctx, lw1.resourceErrCh, listenerUpdateErrTuple{ + resourceErr: xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, ""), + }); err != nil { + t.Fatal(err) + } + if err := verifyNoListenerUpdate(ctx, lw2.updateCh); err != nil { + t.Fatal(err) + } + + // Update the second listener resource on the management server. The first + // watcher should not see an update, while the second watcher should. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(resourceName2, "new-rds-resource")}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoListenerUpdate(ctx, lw1.updateCh); err != nil { + t.Fatal(err) + } + wantUpdate = listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: "new-rds-resource", + }, + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_NewWatcherForRemovedResource covers the case where a new +// watcher registers for a resource that has been removed. The test verifies +// the following scenarios: +// 1. When a resource is deleted by the management server, any active +// watchers of that resource should be notified with a "resource removed" +// error through their watch callback. +// 2. If a new watcher attempts to register for a resource that has already +// been deleted, its watch callback should be immediately invoked with a +// "resource removed" error. +func (s) TestLDSWatch_NewWatcherForRemovedResource(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Register watch for the listener resource and have the + // callbacks push the received updates on to a channel. + lw1 := newListenerWatcher() + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw1) + defer ldsCancel1() + + // Configure the management server to return listener resource, + // corresponding to the registered watch. + resource := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resource); err != nil { + t.Fatalf("Failed to update management server with resource: %v, err: %v", resource, err) + } + + // Verify the contents of the received update for existing watch. + wantUpdate := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsName, + }, + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Remove the listener resource on the management server. + resource = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resource); err != nil { + t.Fatalf("Failed to update management server with resource: %v, err: %v", resource, err) + } + + // The existing watcher should receive a resource error for resource + // removal. + updateError := listenerUpdateErrTuple{resourceErr: xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, "")} + if err := verifyListenerUpdate(ctx, lw1.resourceErrCh, updateError); err != nil { + t.Fatal(err) + } + + // New watchers attempting to register for a deleted resource should also + // receive a "resource removed" error. + lw2 := newListenerWatcher() + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw2) + defer ldsCancel2() + if err := verifyListenerUpdate(ctx, lw2.resourceErrCh, updateError); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_NACKError covers the case where an update from the management +// server is NACKed by the xdsclient. The test verifies that the error is +// propagated to the existing watcher. After NACK, if a new watcher registers +// for the resource, error is propagated to the new watcher as well. +func (s) TestLDSWatch_NACKError(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + lw := newListenerWatcher() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw) + defer ldsCancel() + + // Configure the management server to return a single listener resource + // which is expected to be NACKed by the client. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{badListenerResource(t, ldsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the existing watcher. + // Since the resource is not cached, it should be received as resource + // error. + if err := verifyResourceErrorType(ctx, lw.resourceErrCh, xdsresource.ErrorTypeNACKed, nodeID); err != nil { + t.Fatal(err) + } + + // Verify that the expected error is propagated to the new watcher as well. + // Since the resource is not cached, it should be received as resource + // error. + lw2 := newListenerWatcher() + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw2) + defer ldsCancel2() + if err := verifyResourceErrorType(ctx, lw2.resourceErrCh, xdsresource.ErrorTypeNACKed, nodeID); err != nil { + t.Fatal(err) + } +} + +// Tests the scenario where a watch registered for a resource results in a good +// update followed by a bad update. This results in the resource cache +// containing both the old good update and the latest NACK error. The test +// verifies that a when a new watch is registered for the same resource, the new +// watcher receives the good update followed by the NACK error. +func (s) TestLDSWatch_ResourceCaching_NACKError(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + lw1 := newListenerWatcher() + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw1) + defer ldsCancel1() + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), 1000*defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsName, + }, + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Configure the management server to return a single listener resource + // which is expected to be NACKed by the client. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{badListenerResource(t, ldsName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the existing watcher. + // Since the resource is cached, it should be received as ambient error. + if err := verifyAmbientErrorType(ctx, lw1.ambientErrCh, xdsresource.ErrorTypeNACKed, nodeID); err != nil { + t.Fatal(err) + } + + // Register another watch for the same resource. This should get the update + // and error from the cache. + lw2 := newListenerWatcher() + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw2) + defer ldsCancel2() + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + // Verify that the expected error is propagated to the existing watcher. + // Since the resource is cached, it should be received as ambient error. + if err := verifyAmbientErrorType(ctx, lw2.ambientErrCh, xdsresource.ErrorTypeNACKed, nodeID); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_PartialValid covers the case where a response from the +// management server contains both valid and invalid resources and is expected +// to be NACKed by the xdsclient. The test verifies that watchers corresponding +// to the valid resource receive the update, while watchers corresponding to the +// invalid resource receive an error. +func (s) TestLDSWatch_PartialValid(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + Authorities: map[string]xdsclient.Authority{ + authority: {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Register two watches for listener resources. The first watch is expected + // to receive an error because the received resource is NACKed. The second + // watch is expected to get a good update. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + badResourceName := ldsName + lw1 := newListenerWatcher() + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, badResourceName, lw1) + defer ldsCancel1() + goodResourceName := makeNewStyleLDSName(authority) + lw2 := newListenerWatcher() + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, goodResourceName, lw2) + defer ldsCancel2() + + // Configure the management server with two listener resources. One of these + // is a bad resource causing the update to be NACKed. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + badListenerResource(t, badResourceName), + e2e.DefaultClientListener(goodResourceName, rdsName), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher which + // requested for the bad resource. + // Verify that the expected error is propagated to the existing watcher. + // Since the resource is not cached, it should be received as resource + // error. + if err := verifyResourceErrorType(ctx, lw1.resourceErrCh, xdsresource.ErrorTypeNACKed, nodeID); err != nil { + t.Fatal(err) + } + + // Verify that the watcher watching the good resource receives a good + // update. + wantUpdate := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsName, + }, + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_PartialResponse covers the case where a response from the +// management server does not contain all requested resources. LDS responses are +// supposed to contain all requested resources, and the absence of one usually +// indicates that the management server does not know about it. In cases where +// the server has never responded with this resource before, the xDS client is +// expected to wait for the watch timeout to expire before concluding that the +// resource does not exist on the server +func (s) TestLDSWatch_PartialResponse(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + Authorities: map[string]xdsclient.Authority{ + authority: {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Register two watches for two listener resources and have the + // callbacks push the received updates on to a channel. + resourceName1 := ldsName + lw1 := newListenerWatcher() + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, resourceName1, lw1) + defer ldsCancel1() + + resourceName2 := makeNewStyleLDSName(authority) + lw2 := newListenerWatcher() + ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, resourceName2, lw2) + defer ldsCancel2() + + // Configure the management server to return only one of the two listener + // resources, corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(resourceName1, rdsName), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for first watcher. + wantUpdate1 := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsName, + }, + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate1); err != nil { + t.Fatal(err) + } + + // Verify that the second watcher does not get an update with an error. + if err := verifyNoListenerUpdate(ctx, lw2.updateCh); err != nil { + t.Fatal(err) + } + + // Configure the management server to return two listener resources, + // corresponding to the registered watches. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(resourceName1, rdsName), + e2e.DefaultClientListener(resourceName2, rdsName), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the second watcher. + wantUpdate2 := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsName, + }, + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate2); err != nil { + t.Fatal(err) + } + + // Verify that the first watcher gets no update, as the first resource did + // not change. + if err := verifyNoListenerUpdate(ctx, lw1.updateCh); err != nil { + t.Fatal(err) + } +} diff --git a/internal/xds/clients/xdsclient/test/metrics_test.go b/internal/xds/clients/xdsclient/test/metrics_test.go new file mode 100644 index 0000000..6df2b1b --- /dev/null +++ b/internal/xds/clients/xdsclient/test/metrics_test.go @@ -0,0 +1,911 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "errors" + "net" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + "google.golang.org/grpc/internal/xds/clients/xdsclient/metrics" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +// TestResourceUpdateMetrics configures an xDS client, and a management server +// to send valid and invalid LDS updates, and verifies that the expected metrics +// for both good and bad updates are emitted. +func (s) TestResourceUpdateMetrics(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + tmr := newTestMetricsReporter() + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l}) + const listenerResourceName = "test-listener-resource" + const routeConfigurationName = "test-route-configuration-resource" + nodeID := uuid.New().String() + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + MetricsReporter: tmr, + } + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Watch the valid listener configured on the management server. This should + // cause a resource update valid metric to emit eventually. + client.WatchResource(listenerType.TypeURL, listenerResourceName, noopListenerWatcher{}) + if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil { + t.Fatal(err) + } + + // Update management server with a bad update. This should cause a resource + // update invalid metric to emit eventually. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)}, + SkipValidation: true, + } + resources.Listeners[0].ApiListener = nil + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateInvalid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil { + t.Fatal(err) + } + + // Resource update valid metric should have not emitted. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if err := tmr.waitForMetric(sCtx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err == nil { + t.Fatal("tmr.WaitForInt64Count(ctx, mdWant) succeeded when expected to timeout.") + } +} + +// TestServerFailureMetrics_BeforeResponseRecv configures an xDS client, and a +// management server. It then register a watcher and stops the management +// server before sending a resource update, and verifies that the expected +// metric for server failure is emitted. +func (s) TestServerFailureMetrics_BeforeResponseRecv(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + tmr := newTestMetricsReporter() + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + + lis := testutils.NewRestartableListener(l) + streamOpened := make(chan struct{}, 1) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + OnStreamOpen: func(context.Context, int64, string) error { + select { + case streamOpened <- struct{}{}: + default: + } + return nil + }, + }) + + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + MetricsReporter: tmr, + } + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + const listenerResourceName = "test-listener-resource" + + // Watch for the listener on the above management server. + client.WatchResource(listenerType.TypeURL, listenerResourceName, noopListenerWatcher{}) + // Verify that an ADS stream is opened and an LDS request with the above + // resource name is sent. + select { + case <-streamOpened: + case <-ctx.Done(): + t.Fatal("Timeout when waiting for ADS stream to open") + } + + // Close the listener and ensure that the ADS stream breaks. This should + // cause a server failure metric to emit eventually. + lis.Stop() + + // Restart to prevent the attempt to create a new ADS stream after back off. + lis.Restart() + + if err := tmr.waitForMetric(ctx, &metrics.ServerFailure{ServerURI: mgmtServer.Address}); err != nil { + t.Fatal(err) + } +} + +// TestServerFailureMetrics_AfterResponseRecv configures an xDS client and a +// management server to send a valid LDS update, and verifies that the +// successful update metric is emitted. When the client ACKs the update, the +// server returns an error, breaking the stream. The test then verifies that the +// server failure metric is not emitted, because the ADS stream was closed after +// a response was received on the stream. Finally, the test waits for the client +// to establish a new stream and verifies that the client emits a metric after +// receiving a successful update. +func (s) TestServerFailureMetrics_AfterResponseRecv(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + tmr := newTestMetricsReporter() + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + streamCreationQuota := make(chan struct{}, 1) + streamCreationQuota <- struct{}{} + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + OnStreamOpen: func(context.Context, int64, string) error { + // The following select block is used to block stream creation after + // the first stream has failed, but while we are waiting to verify + // that the failure metric is not reported. + select { + case <-streamCreationQuota: + case <-ctx.Done(): + } + return nil + }, + OnStreamRequest: func(streamID int64, req *v3discoverypb.DiscoveryRequest) error { + // We only want the ACK on the first stream to return an error + // (leading to stream closure), without effecting subsequent stream + // attempts. + if streamID == 1 && req.GetVersionInfo() != "" { + return errors.New("test configured error") + } + return nil + }}, + ) + const listenerResourceName = "test-listener-resource" + const routeConfigurationName = "test-route-configuration-resource" + nodeID := uuid.New().String() + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + MetricsReporter: tmr, + } + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Watch the valid listener configured on the management server. This should + // cause a resource update valid metric to emit eventually. + client.WatchResource(listenerType.TypeURL, listenerResourceName, noopListenerWatcher{}) + if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil { + t.Fatal(err) + } + + // When the client sends an ACK, the management server would reply with an + // error, breaking the stream. + // Server failure should still have no recording point. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + failureMetric := &metrics.ServerFailure{ServerURI: mgmtServer.Address} + if err := tmr.waitForMetric(sCtx, failureMetric); err == nil { + t.Fatalf("tmr.waitForMetric(%v) succeeded when expected to timeout.", failureMetric) + } else if sCtx.Err() == nil { + t.Fatalf("tmr.WaitForInt64Count(%v) = %v, want context deadline exceeded", failureMetric, err) + } + // Unblock stream creation and verify that an update is received + // successfully. + close(streamCreationQuota) + if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil { + t.Fatal(err) + } +} + +// TestConnectedMetric verifies the "grpc.xds_client.connected" metric state +// transitions. It begins by ensuring no metrics are reported before connection +// is attempted. Then it establishes a connection by watching a valid resource +// and verifies the connected state pulses to 1. Finally, it stops the +// management server and verifies the state drops back to 0. +func (s) TestConnectedMetric(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + tmr := newTestMetricsReporter() + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l}) + nodeID := uuid.New().String() + + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ + ServerIdentifier: clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + }, + }}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{ + "insecure": {Credentials: insecure.NewBundle()}, + }), + ResourceTypes: map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}, + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + MetricsReporter: tmr, + } + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + tmr.triggerAsyncMetrics() + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if err := tmr.waitForSpecificMetric(sCtx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address}); err == nil { + t.Fatal("XDSClientConnected metric reported before any watch was started") + } + + const listenerName = "test-listener-resource" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, "route-config")}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server: %v", err) + } + client.WatchResource(listenerType.TypeURL, listenerName, noopListenerWatcher{}) + + // Wait for the update to ensure we are connected. + if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil { + t.Fatal(err) + } + + // Now trigger async metrics. + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 1}); err != nil { + t.Fatal(err) + } + + mgmtServer.Stop() + + // Wait for the synchronous server failure metric to confirm disconnect. + if err := tmr.waitForSpecificMetric(ctx, &metrics.ServerFailure{ServerURI: mgmtServer.Address}); err != nil { + t.Fatal(err) + } + + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 0}); err != nil { + t.Fatal(err) + } + + // Verify async metric reporters are unregistered when client closes. + client.Close() + + if count := tmr.numAsyncReporters(); count != 0 { + t.Fatalf("Async reporter not unregistered after client close, count: %d", count) + } + + // Drain the channel of any leftover metrics from previous pulses. + tmr.Drain() + + tmr.triggerAsyncMetrics() + // No metrics should be reported now because there are no reporters. + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := tmr.Receive(sCtx); err == nil { + t.Fatal("Metrics reported after all reporters were unregistered") + } +} + +// TestResourceMetrics verifies that the xDS client correctly tracks resource +// states (acked, nacked_but_cached) in "grpc.xds_client.resources" metric. It +// watches a resource, pushes a valid update from the management server, and +// asserts that the resource transitions to 'acked' state. Then it pushes an +// invalid update and asserts that the resource transitions to +// 'nacked_but_cached' state. +func (s) TestResourceMetrics(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + tmr := newTestMetricsReporter() + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l}) + nodeID := uuid.New().String() + + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ + ServerIdentifier: clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + }, + }}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{ + "insecure": {Credentials: insecure.NewBundle()}, + }), + ResourceTypes: map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}, + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + MetricsReporter: tmr, + } + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + const listenerName = "test-listener-resource" + const routeConfigName = "test-route-configuration-resource" + + client.WatchResource(listenerType.TypeURL, listenerName, noopListenerWatcher{}) + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server: %v", err) + } + + if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil { + t.Fatal(err) + } + + // Trigger async metrics. + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{ + Authority: "#old", // Default value for old-style non-xdstp as per gRFC A78 + ResourceType: "ListenerResource", + CacheState: "acked", + Count: 1, + }); err != nil { + t.Fatal(err) + } + + resources.Listeners[0].ApiListener = nil + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server: %v", err) + } + + // Wait for Invalid update. + if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateInvalid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil { + t.Fatal(err) + } + + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{ + Authority: "#old", + ResourceType: "ListenerResource", + CacheState: "nacked_but_cached", + Count: 1, + }); err != nil { + t.Fatal(err) + } +} + +// TestResourceMetrics_Extended verifies complex resource state transitions +// (requested, does_not_exist, nacked) in "grpc.xds_client.resources" metric +// across multiple resources. It watches several resources, pushes a partial +// update (some valid, some invalid) to transition them to 'acked' and 'nacked' +// states, then management server removes a resource and asserts that active +// watchers transition back to 'requested' while omitted resources transition +// to 'does_not_exist'. +func (s) TestResourceMetrics_Extended(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + tmr := newTestMetricsReporter() + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l}) + nodeID := uuid.New().String() + + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ + ServerIdentifier: clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + }, + }}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{ + "insecure": {Credentials: insecure.NewBundle()}, + }), + ResourceTypes: map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}, + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + MetricsReporter: tmr, + } + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + resRequested1 := "res-requested-1" + resRequested2 := "res-requested-2" + resNacked1 := "res-nacked-1" + resNacked2 := "res-nacked-2" + resRemoved := "res-not-exist" + + resList := []string{resRequested1, resRequested2, resNacked1, resNacked2, resRemoved} + for _, res := range resList { + client.WatchResource(listenerType.TypeURL, res, noopListenerWatcher{}) + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(resNacked1, "route-config"), + e2e.DefaultClientListener(resNacked2, "route-config"), + e2e.DefaultClientListener(resRemoved, "route-config"), + }, + SkipValidation: true, + } + resources.Listeners[0].ApiListener = nil + resources.Listeners[1].ApiListener = nil + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server: %v", err) + } + + // Wait for the resource update to be accepted. + if err := tmr.waitForSpecificMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil { + t.Fatal(err) + } + // Push an empty update to remove all resources from the management server. + resourcesEmpty := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resourcesEmpty); err != nil { + t.Fatalf("Failed to update management server: %v", err) + } + + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{ + Authority: "#old", + ResourceType: "ListenerResource", + CacheState: "requested", + Count: 2, + }); err != nil { + t.Fatalf("Failed to verify requested count: %v", err) + } + + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{ + Authority: "#old", + ResourceType: "ListenerResource", + CacheState: "nacked", + Count: 2, + }); err != nil { + t.Fatalf("Failed to verify nacked count: %v", err) + } + + // Wait for the xDS client to process the empty update and trigger + // ResourceNotFound for omitted resources immediately. + lw := newListenerWatcher() + client.WatchResource(listenerType.TypeURL, resRemoved, lw) + + // Verify that resources missing from the authoritative update transition to + // the does_not_exist state. + if err := verifyResourceErrorType(ctx, lw.resourceErrCh, xdsresource.ErrorTypeResourceNotFound, ""); err != nil { + t.Fatal(err) + } + + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{ + Authority: "#old", + ResourceType: "ListenerResource", + CacheState: "does_not_exist", + Count: 1, + }); err != nil { + t.Fatalf("Failed to verify does_not_exist count: %v", err) + } +} + +// TestConnectedMetric_Reconnection verifies "grpc.xds_client.connected" metric +// accuracy during flaky network conditions. It uses a restartable listener and +// a dial interceptor to pause connection attempts. It asserts that the +// connected state pulses to 1 when a stream succeeds, remains 1 immediately +// after failures before retries, transitions to 0 after subsequent failures, +// and returns to 1 only after successful stream recreation. +func (s) TestConnectedMetric_Reconnection(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + + sendResponse := make(chan struct{}) + streamOpened := make(chan struct{}, 10) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + OnStreamOpen: func(ctx context.Context, _ int64, _ string) error { + select { + case streamOpened <- struct{}{}: + case <-ctx.Done(): + } + return nil + }, + OnStreamResponse: func(ctx context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, _ *v3discoverypb.DiscoveryResponse) { + select { + case <-sendResponse: + case <-ctx.Done(): + } + }, + }) + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + // streamAttemptCh is used by the interceptor to notify the test that a + // stream is attempting to open. + streamAttemptCh := testutils.NewChannel() + // commandCh is used by the test to tell the interceptor whether to block or pass. + commandCh := testutils.NewChannel() + // blocked is used by the interceptor to signal the test that it has + // successfully blocked the stream creation attempt. + blocked := make(chan struct{}) + // unblock is used by the test to release the interceptor and allow the + // stream creation to proceed. + unblock := make(chan struct{}) + + // customGRPCNewClient overrides the gRPC client creation to inject a stream + // interceptor that can block ADS stream creation attempts for testing + // reconnection behavior. + customGRPCNewClient := func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + interceptor := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, cops ...grpc.CallOption) (grpc.ClientStream, error) { + streamAttemptCh.Send(struct{}{}) // Notify test + + cmd, err := commandCh.Receive(ctx) // Wait for command + if err != nil { + return nil, err + } + + if cmd.(string) == "block" { + select { + case blocked <- struct{}{}: + case <-ctx.Done(): + } + // Pause stream creation until released by the test. + <-unblock + } + return streamer(ctx, desc, cc, method, cops...) + } + opts = append(opts, grpc.WithStreamInterceptor(interceptor)) + return grpc.NewClient(target, opts...) + } + + tmr := newTestMetricsReporter() + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle(), GRPCNewClient: customGRPCNewClient}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + MetricsReporter: tmr, + } + + waitForStreamSuccess := func() { + for { + select { + case <-streamOpened: + return + case <-streamAttemptCh.C: + commandCh.Send("pass") + case <-ctx.Done(): + t.Fatalf("Timeout waiting for stream success") + } + } + } + + // Stop listener to force initial connection failure. + lis.Stop() + + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + const listenerName = "test-listener-resource" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, "route-config")}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server: %v", err) + } + + // Initiate watch to trigger connection attempt. + client.WatchResource(listenerType.TypeURL, listenerName, noopListenerWatcher{}) + + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 0}); err != nil { + t.Fatalf("XDSClientConnected check failed at start - got: %v, want 0", err) + } + + // Verify state transitions to connected (Value: 1) when stream succeeds. + lis.Restart() + + waitForStreamSuccess() + + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 1}); err != nil { + t.Fatalf("XDSClientConnected check failed after 1st NewStream - got: %v, want 1", err) + } + + // Push resource update to confirm the connection is functional. + close(sendResponse) + if err := tmr.waitForSpecificMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil { + t.Fatal(err) + } + + // Verify state remains connected (Value: 1) immediately after failure before + // NewStream retry. + lis.Stop() + + if _, err := streamAttemptCh.Receive(ctx); err != nil { + t.Fatalf("Timeout waiting for 2nd stream attempt (block): %v", err) + } + commandCh.Send("block") + + <-blocked + + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 1}); err != nil { + t.Fatalf("XDSClientConnected check failed while NewStream is blocked - got: %v, want 1", err) + } + + // Verify state transitions to disconnected (Value: 0) after NewStream attempt + // fails. + close(unblock) + + if err := tmr.waitForSpecificMetric(ctx, &metrics.ServerFailure{ServerURI: mgmtServer.Address}); err != nil { + t.Fatal(err) + } + + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 0}); err != nil { + t.Fatalf("XDSClientConnected check failed after NewStream failure - got: %v, want 0", err) + } + + sendResponse = make(chan struct{}) + + // Verify state remains disconnected (Value: 0) while waiting for the first + // response on the new stream. + lis.Restart() + + // Clear channels to ensure fresh readings. + for len(streamOpened) > 0 { + <-streamOpened + } + + waitForStreamSuccess() + + tmr.Drain() + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 0}); err != nil { + t.Fatalf("XDSClientConnected check failed after successful NewStream but before response - got: %v, want 0", err) + } + + // Verify state returns to connected (Value: 1) after receiving the first + // response on the new stream. + close(sendResponse) + + if err := tmr.waitForSpecificMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil { + t.Fatal(err) + } + + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 1}); err != nil { + t.Fatalf("XDSClientConnected check failed after response- got: %v, want 1", err) + } +} + +// TestResourceMetrics_AuthorityOldStyle verifies that the xDS client correctly +// falls back to '#old' in "grpc.xds_client.resources" metrics for legacy +// authorities. It watches a resource using default authority and asserts that +// the metric is reported under '#old' instead of an empty string. +func (s) TestResourceMetrics_AuthorityOldStyle(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + nodeID := uuid.New().String() + + serverCfg := xdsclient.ServerConfig{ + ServerIdentifier: clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + }, + } + + tmr := newTestMetricsReporter() + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{serverCfg}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{ + "insecure": {Credentials: insecure.NewBundle()}, + }), + ResourceTypes: map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}, + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{serverCfg}}, + }, + MetricsReporter: tmr, + } + + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const listenerName = "test-listener" + + client.WatchResource(listenerType.TypeURL, listenerName, newListenerWatcher()) + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, "route-config")}, + SkipValidation: true, + } + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server: %v", err) + } + + if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil { + t.Fatal(err) + } + + // Verify that the client substitutes empty authority with '#old' in metrics. + tmr.triggerAsyncMetrics() + if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{ + Authority: "#old", + ResourceType: "ListenerResource", + CacheState: "acked", + Count: 1, + }); err != nil { + t.Fatalf("Failed to observe grpc.xds.authority '#old' metric substitution: %v", err) + } +} diff --git a/internal/xds/clients/xdsclient/test/misc_watchers_test.go b/internal/xds/clients/xdsclient/test/misc_watchers_test.go new file mode 100644 index 0000000..e611c22 --- /dev/null +++ b/internal/xds/clients/xdsclient/test/misc_watchers_test.go @@ -0,0 +1,529 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "net" + "strings" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/internal/testutils" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e" + "google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + "google.golang.org/protobuf/types/known/anypb" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +// testLDSWatcher is a test watcher that registers two watches corresponding to +// the names passed in at creation time on a valid update. +type testLDSWatcher struct { + client *xdsclient.XDSClient + name1, name2 string + lw1, lw2 *listenerWatcher + cancel1, cancel2 func() + updateCh *testutils.Channel +} + +func newTestLDSWatcher(client *xdsclient.XDSClient, name1, name2 string) *testLDSWatcher { + return &testLDSWatcher{ + client: client, + name1: name1, + name2: name2, + lw1: newListenerWatcher(), + lw2: newListenerWatcher(), + updateCh: testutils.NewChannelWithSize(1), + } +} + +func (lw *testLDSWatcher) ResourceChanged(update xdsclient.ResourceData, onDone func()) { + lisData, ok := update.(*listenerResourceData) + if !ok { + lw.updateCh.Send(listenerUpdateErrTuple{resourceErr: fmt.Errorf("unexpected resource type: %T", update)}) + onDone() + return + } + lw.updateCh.Send(listenerUpdateErrTuple{update: lisData.Resource}) + + lw.cancel1 = lw.client.WatchResource(xdsresource.V3ListenerURL, lw.name1, lw.lw1) + lw.cancel2 = lw.client.WatchResource(xdsresource.V3ListenerURL, lw.name2, lw.lw2) + onDone() +} + +func (lw *testLDSWatcher) AmbientError(err error, onDone func()) { + // When used with a go-control-plane management server that continuously + // resends resources which are NACKed by the xDS client, using a `Replace()` + // here and in OnResourceDoesNotExist() simplifies tests which will have + // access to the most recently received error. + lw.updateCh.Replace(listenerUpdateErrTuple{ambientErr: err}) + onDone() +} + +func (lw *testLDSWatcher) ResourceError(_ error, onDone func()) { + lw.updateCh.Replace(listenerUpdateErrTuple{resourceErr: xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, "Listener not found in received response")}) + onDone() +} + +func (lw *testLDSWatcher) cancel() { + lw.cancel1() + lw.cancel2() +} + +// TestWatchCallAnotherWatch tests the scenario where a watch is registered for +// a resource, and more watches are registered from the first watch's callback. +// The test verifies that this scenario does not lead to a deadlock. +func (s) TestWatchCallAnotherWatch(t *testing.T) { + // Start an xDS management server and set the option to allow it to respond + // to requests which only specify a subset of the configured resources. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + Authorities: map[string]xdsclient.Authority{ + authority: {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + // Configure the management server to return two listener resources, + // corresponding to the registered watches. + ldsNameNewStyle := makeNewStyleLDSName(authority) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(ldsName, rdsName), + e2e.DefaultClientListener(ldsNameNewStyle, rdsNameNewStyle), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Create a listener watcher that registers two more watches from + // the OnUpdate callback: + // - one for the same resource name as this watch, which would be + // satisfied from xdsClient cache + // - the other for a different resource name, which would be + // satisfied from the server + lw := newTestLDSWatcher(client, ldsName, ldsNameNewStyle) + defer lw.cancel() + ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw) + defer ldsCancel() + + // Verify the contents of the received update for the all watchers. + // Verify the contents of the received update for the all watchers. The two + // resources returned differ only in the resource name. Therefore the + // expected update is the same for all the watchers. + wantUpdate12 := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsName, + }, + } + // Verify the contents of the received update for the all watchers. The two + // resources returned differ only in the resource name. Therefore the + // expected update is the same for all the watchers. + wantUpdate3 := listenerUpdateErrTuple{ + update: listenerUpdate{ + RouteConfigName: rdsNameNewStyle, + }, + } + if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate12); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, lw.lw1.updateCh, wantUpdate12); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, lw.lw2.updateCh, wantUpdate3); err != nil { + t.Fatal(err) + } +} + +// TestNodeProtoSentOnlyInFirstRequest verifies that a non-empty node proto gets +// sent only on the first discovery request message on the ADS stream. +// +// It also verifies the same behavior holds after a stream restart. +func (s) TestNodeProtoSentOnlyInFirstRequest(t *testing.T) { + // Create a restartable listener which can close existing connections. + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + lis := testutils.NewRestartableListener(l) + + // Start a fake xDS management server with the above restartable listener. + // + // We are unable to use the go-control-plane server here, because it caches + // the node proto received in the first request message and adds it to + // subsequent requests before invoking the OnStreamRequest() callback. + // Therefore we cannot verify what is sent by the xDS client. + mgmtServer, cleanup, err := fakeserver.StartServer(lis) + if err != nil { + t.Fatalf("Failed to start fake xDS server: %v", err) + } + defer cleanup() + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + const ( + serviceName = "my-service-client-side-xds" + routeConfigName = "route-" + serviceName + clusterName = "cluster-" + serviceName + serviceName2 = "my-service-client-side-xds-2" + ) + + // Register a watch for the Listener resource. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + watcher := newListenerWatcher() + ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, serviceName, watcher) + defer ldsCancel1() + + // Ensure the watch results in a discovery request with an empty node proto. + if err := readDiscoveryResponseAndCheckForNonEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil { + t.Fatal(err) + } + + // Configure a listener resource on the fake xDS server. + lisAny, err := anypb.New(e2e.DefaultClientListener(serviceName, routeConfigName)) + if err != nil { + t.Fatalf("Failed to marshal listener resource into an Any proto: %v", err) + } + mgmtServer.XDSResponseChan <- &fakeserver.Response{ + Resp: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + Resources: []*anypb.Any{lisAny}, + }, + } + + // The xDS client is expected to ACK the Listener resource. The discovery + // request corresponding to the ACK must contain a nil node proto. + if err := readDiscoveryResponseAndCheckForEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil { + t.Fatal(err) + } + + // Register a watch for another Listener resource. + ldscancel2 := client.WatchResource(xdsresource.V3ListenerURL, serviceName2, watcher) + defer ldscancel2() + + // Ensure the watch results in a discovery request with an empty node proto. + if err := readDiscoveryResponseAndCheckForEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil { + t.Fatal(err) + } + + // Configure the other listener resource on the fake xDS server. + lisAny2, err := anypb.New(e2e.DefaultClientListener(serviceName2, routeConfigName)) + + if err != nil { + t.Fatalf("Failed to marshal route configuration resource into an Any proto: %v", err) + } + + mgmtServer.XDSResponseChan <- &fakeserver.Response{ + Resp: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + Resources: []*anypb.Any{lisAny2}, + }, + } + + // Ensure the discovery request for the ACK contains an empty node proto. + if err := readDiscoveryResponseAndCheckForEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil { + t.Fatal(err) + } + + // Stop the management server and expect the error callback to be invoked. + lis.Stop() + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for the connection error to be propagated to the watcher") + case <-watcher.ambientErrCh.C: + } + // Restart the management server. + lis.Restart() + + // The xDS client is expected to re-request previously requested resources. + // Here, we expect 1 DiscoveryRequest messages with both the listener resources. + // The message should contain a non-nil node proto (since its the first + // request after restart). + // + // And since we don't push any responses on the response channel of the fake + // server, we do not expect any ACKs here. + if err := readDiscoveryResponseAndCheckForNonEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil { + t.Fatal(err) + } +} + +// readDiscoveryResponseAndCheckForEmptyNodeProto reads a discovery request +// message out of the provided reqCh. It returns an error if it fails to read a +// message before the context deadline expires, or if the read message contains +// a non-empty node proto. +func readDiscoveryResponseAndCheckForEmptyNodeProto(ctx context.Context, reqCh *testutils.Channel) error { + v, err := reqCh.Receive(ctx) + if err != nil { + return fmt.Errorf("Timeout when waiting for a DiscoveryRequest message") + } + req := v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest) + if node := req.GetNode(); node != nil { + return fmt.Errorf("Node proto received in DiscoveryRequest message is %v, want empty node proto", node) + } + return nil +} + +// readDiscoveryResponseAndCheckForNonEmptyNodeProto reads a discovery request +// message out of the provided reqCh. It returns an error if it fails to read a +// message before the context deadline expires, or if the read message contains +// an empty node proto. +func readDiscoveryResponseAndCheckForNonEmptyNodeProto(ctx context.Context, reqCh *testutils.Channel) error { + v, err := reqCh.Receive(ctx) + if err != nil { + return fmt.Errorf("Timeout when waiting for a DiscoveryRequest message") + } + req := v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest) + if node := req.GetNode(); node == nil { + return fmt.Errorf("Empty node proto received in DiscoveryRequest message, want non-empty node proto") + } + return nil +} + +// Tests that the errors returned by the xDS client when watching a resource +// contain the node ID that was used to create the client. This test covers two +// scenarios: +// +// 1. When a watch is registered for an already registered resource type, but +// a new watch is registered with a type url which is not present in +// provided resource types. +// 2. When a watch is registered for a resource name whose authority is not +// found in the xDS client config. +func (s) TestWatchErrorsContainNodeID(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: grpctransport.NewBuilder(configs), + ResourceTypes: resourceTypes, + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + t.Run("Right_Wrong_ResourceType_Implementations", func(t *testing.T) { + const listenerName = "listener-name" + watcher := newListenerWatcher() + client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher) + + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-watcher.updateCh.C: + t.Fatal("Unexpected resource update") + case <-watcher.resourceErrCh.C: + t.Fatal("Unexpected resource error") + } + + client.WatchResource("non-existent-type-url", listenerName, watcher) + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for error callback to be invoked") + case u, ok := <-watcher.resourceErrCh.C: + if !ok { + t.Fatalf("got no update, wanted listener resource error from the management server") + } + gotErr := u.(listenerUpdateErrTuple).resourceErr + if !strings.Contains(gotErr.Error(), nodeID) { + t.Fatalf("Unexpected error: %v, want error with node ID: %q", err, nodeID) + } + } + }) + + t.Run("Missing_Authority", func(t *testing.T) { + const listenerName = "xdstp://nonexistant-authority/envoy.config.listener.v3.Listener/listener-name" + watcher := newListenerWatcher() + client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher) + + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for error callback to be invoked") + case u, ok := <-watcher.resourceErrCh.C: + if !ok { + t.Fatalf("got no update, wanted listener resource error from the management server") + } + gotErr := u.(listenerUpdateErrTuple).resourceErr + if !strings.Contains(gotErr.Error(), nodeID) { + t.Fatalf("Unexpected error: %v, want error with node ID: %q", err, nodeID) + } + } + }) +} + +// erroringTransportBuilder is a transport builder which always returns a nil +// transport along with an error. +type erroringTransportBuilder struct{} + +func (*erroringTransportBuilder) Build(_ clients.ServerIdentifier) (clients.Transport, error) { + return nil, fmt.Errorf("failed to create transport") +} + +// Tests that the errors returned by the xDS client when watching a resource +// contain the node ID when channel creation to the management server fails. +func (s) TestWatchErrorsContainNodeID_ChannelCreationFailure(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + + resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType} + si := clients.ServerIdentifier{ + ServerURI: mgmtServer.Address, + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}, + } + + xdsClientConfig := xdsclient.Config{ + Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}}, + Node: clients.Node{ID: nodeID}, + TransportBuilder: &erroringTransportBuilder{}, + ResourceTypes: resourceTypes, + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + Authorities: map[string]xdsclient.Authority{ + "": {XDSServers: []xdsclient.ServerConfig{}}, + }, + } + + // Create an xDS client with the above config. + client, err := xdsclient.New(xdsClientConfig) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const listenerName = "listener-name" + watcher := newListenerWatcher() + client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher) + + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for error callback to be invoked") + case u, ok := <-watcher.resourceErrCh.C: + if !ok { + t.Fatalf("got no update, wanted listener resource error from the management server") + } + gotErr := u.(listenerUpdateErrTuple).resourceErr + if !strings.Contains(gotErr.Error(), nodeID) { + t.Fatalf("Unexpected error: %v, want error with node ID: %q", err, nodeID) + } + } +} diff --git a/internal/xds/clients/xdsclient/xdsclient.go b/internal/xds/clients/xdsclient/xdsclient.go new file mode 100644 index 0000000..ad82e7f --- /dev/null +++ b/internal/xds/clients/xdsclient/xdsclient.go @@ -0,0 +1,516 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xdsclient provides an xDS (* Discovery Service) client. +// +// It allows applications to: +// - Create xDS client instances with in-memory configurations. +// - Register watches for named resources. +// - Receive resources via an ADS (Aggregated Discovery Service) stream. +// - Register watches for named resources (e.g. listeners, routes, or +// clusters). +// +// This enables applications to dynamically discover and configure resources +// such as listeners, routes, clusters, and endpoints from an xDS management +// server. +package xdsclient + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/xds/clients" + clientsinternal "google.golang.org/grpc/internal/xds/clients/internal" + "google.golang.org/grpc/internal/xds/clients/internal/backoff" + "google.golang.org/grpc/internal/xds/clients/internal/syncutil" + xdsclientinternal "google.golang.org/grpc/internal/xds/clients/xdsclient/internal" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" + "google.golang.org/grpc/internal/xds/clients/xdsclient/metrics" + "google.golang.org/protobuf/proto" + + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" +) + +const ( + defaultWatchExpiryTimeout = 15 * time.Second + name = "xds-client" +) + +var ( + defaultExponentialBackoff = backoff.DefaultExponential.Backoff +) + +func init() { + xdsclientinternal.StreamBackoff = defaultExponentialBackoff + xdsclientinternal.ResourceWatchStateForTesting = resourceWatchStateForTesting +} + +// XDSClient is a client which queries a set of discovery APIs (collectively +// termed as xDS) on a remote management server, to discover +// various dynamic resources. +type XDSClient struct { + // The following fields are initialized at creation time and are read-only + // after that, and therefore can be accessed without a mutex. + done *syncutil.Event // Fired when the client is closed. + topLevelAuthority *authority // The top-level authority, used only for old-style names without an authority. + authorities map[string]*authority // Map from authority names in config to authority struct. + config *Config // Complete xDS client configuration. + watchExpiryTimeout time.Duration // Expiry timeout for ADS watch. + backoff func(int) time.Duration // Backoff for ADS and LRS stream failures. + transportBuilder clients.TransportBuilder // Builder to create transports to xDS server. + resourceTypes map[string]ResourceType // Registry of resource types, for parsing incoming ADS responses. + serializer *syncutil.CallbackSerializer // Serializer for invoking resource watcher callbacks. + serializerClose func() // Function to close the serializer. + logger *grpclog.PrefixLogger + target string + metricsReporter clients.MetricsReporter + + // The XDSClient owns a bunch of channels to individual xDS servers + // specified in the xDS client configuration. Authorities acquire references + // to these channels based on server configs within the authority config. + // The XDSClient maintains a list of interested authorities for each of + // these channels, and forwards updates from the channels to each of these + // authorities. + // + // Once all references to a channel are dropped, the channel is closed. + channelsMu sync.Mutex + xdsActiveChannels map[ServerConfig]*channelState // Map from server config to in-use xdsChannels. + + metricsCleanup func() +} + +// New returns a new xDS Client configured with the provided config. +func New(config Config) (*XDSClient, error) { + switch { + case config.ResourceTypes == nil: + return nil, errors.New("xdsclient: resource types map is nil") + case config.TransportBuilder == nil: + return nil, errors.New("xdsclient: transport builder is nil") + case config.Authorities == nil && config.Servers == nil: + return nil, errors.New("xdsclient: no servers or authorities specified") + } + if config.WatchExpiryTimeout == 0 { + config.WatchExpiryTimeout = defaultWatchExpiryTimeout + } + client, err := newClient(&config, name) + if err != nil { + return nil, err + } + // Register this client instance as an Async Reporter. + if client.metricsReporter != nil { + reporter := &xdsClientMetricReporter{c: client} + client.metricsCleanup = client.metricsReporter.RegisterAsyncReporter(reporter) + } + return client, nil +} + +// newClient returns a new XDSClient with the given config. +func newClient(config *Config, target string) (*XDSClient, error) { + ctx, cancel := context.WithCancel(context.Background()) + c := &XDSClient{ + target: target, + done: syncutil.NewEvent(), + authorities: make(map[string]*authority), + config: config, + watchExpiryTimeout: config.WatchExpiryTimeout, + backoff: xdsclientinternal.StreamBackoff, + serializer: syncutil.NewCallbackSerializer(ctx), + serializerClose: cancel, + transportBuilder: config.TransportBuilder, + resourceTypes: config.ResourceTypes, + xdsActiveChannels: make(map[ServerConfig]*channelState), + metricsReporter: config.MetricsReporter, + } + + for name, cfg := range config.Authorities { + // If server configs are specified in the authorities map, use that. + // Else, use the top-level server configs. + serverCfg := config.Servers + if len(cfg.XDSServers) >= 1 { + serverCfg = cfg.XDSServers + } + c.authorities[name] = newAuthority(authorityBuildOptions{ + serverConfigs: serverCfg, + name: name, + serializer: c.serializer, + getChannelForADS: c.getChannelForADS, + logPrefix: clientPrefix(c), + target: target, + metricsReporter: c.metricsReporter, + }) + } + c.topLevelAuthority = newAuthority(authorityBuildOptions{ + serverConfigs: config.Servers, + name: "", + serializer: c.serializer, + getChannelForADS: c.getChannelForADS, + logPrefix: clientPrefix(c), + target: target, + metricsReporter: c.metricsReporter, + }) + c.logger = prefixLogger(c) + + return c, nil +} + +// Close closes the xDS client and releases all resources. +func (c *XDSClient) Close() { + if c.done.HasFired() { + return + } + if c.metricsCleanup != nil { + c.metricsCleanup() + } + c.done.Fire() + + c.topLevelAuthority.close() + for _, a := range c.authorities { + a.close() + } + + // Channel close cannot be invoked with the lock held, because it can race + // with stream failure happening at the same time. The latter will callback + // into the XDSClient and will attempt to grab the lock. This will result + // in a deadlock. So instead, we release the lock and wait for all active + // channels to be closed. + var channelsToClose []*xdsChannel + c.channelsMu.Lock() + for _, cs := range c.xdsActiveChannels { + channelsToClose = append(channelsToClose, cs.channel) + } + c.xdsActiveChannels = nil + c.channelsMu.Unlock() + for _, c := range channelsToClose { + c.close() + } + + c.serializerClose() + <-c.serializer.Done() + + c.logger.Infof("Shutdown") +} + +// getChannelForADS returns an xdsChannel for the given server configuration. +// +// If an xdsChannel exists for the given server configuration, it is returned. +// Else a new one is created. It also ensures that the calling authority is +// added to the set of interested authorities for the returned channel. +// +// It returns the xdsChannel and a function to release the calling authority's +// reference on the channel. The caller must call the cancel function when it is +// no longer interested in this channel. +// +// A non-nil error is returned if an xdsChannel was not created. +func (c *XDSClient) getChannelForADS(serverConfig *ServerConfig, callingAuthority *authority) (*xdsChannel, func(), error) { + if c.done.HasFired() { + return nil, nil, errors.New("xds: the xDS client is closed") + } + + initLocked := func(s *channelState) { + if c.logger.V(2) { + c.logger.Infof("Adding authority %q to the set of interested authorities for channel [%p]", callingAuthority.name, s.channel) + } + s.interestedAuthorities[callingAuthority] = true + } + deInitLocked := func(s *channelState) { + if c.logger.V(2) { + c.logger.Infof("Removing authority %q from the set of interested authorities for channel [%p]", callingAuthority.name, s.channel) + } + delete(s.interestedAuthorities, callingAuthority) + } + + return c.getOrCreateChannel(serverConfig, initLocked, deInitLocked) +} + +// getOrCreateChannel returns an xdsChannel for the given server configuration. +// +// If an active xdsChannel exists for the given server configuration, it is +// returned. If an idle xdsChannel exists for the given server configuration, it +// is revived from the idle cache and returned. Else a new one is created. +// +// The initLocked function runs some initialization logic before the channel is +// returned. This includes adding the calling authority to the set of interested +// authorities for the channel or incrementing the count of the number of LRS +// calls on the channel. +// +// The deInitLocked function runs some cleanup logic when the returned cleanup +// function is called. This involves removing the calling authority from the set +// of interested authorities for the channel or decrementing the count of the +// number of LRS calls on the channel. +// +// Both initLocked and deInitLocked are called with the c.channelsMu held. +// +// Returns the xdsChannel and a cleanup function to be invoked when the channel +// is no longer required. A non-nil error is returned if an xdsChannel was not +// created. +func (c *XDSClient) getOrCreateChannel(serverConfig *ServerConfig, initLocked, deInitLocked func(*channelState)) (*xdsChannel, func(), error) { + c.channelsMu.Lock() + defer c.channelsMu.Unlock() + + if c.logger.V(2) { + c.logger.Infof("Received request for a reference to an xdsChannel for server config %q", serverConfig) + } + + // Use an existing channel, if one exists for this server config. + if st, ok := c.xdsActiveChannels[*serverConfig]; ok { + if c.logger.V(2) { + c.logger.Infof("Reusing an existing xdsChannel for server config %q", serverConfig) + } + initLocked(st) + return st.channel, c.releaseChannel(serverConfig, st, deInitLocked), nil + } + + if c.logger.V(2) { + c.logger.Infof("Creating a new xdsChannel for server config %q", serverConfig) + } + + // Create a new transport and create a new xdsChannel, and add it to the + // map of xdsChannels. + tr, err := c.transportBuilder.Build(serverConfig.ServerIdentifier) + if err != nil { + return nil, func() {}, fmt.Errorf("xds: failed to create transport for server config %v: %v", serverConfig, err) + } + state := &channelState{ + parent: c, + serverConfig: serverConfig, + interestedAuthorities: make(map[*authority]bool), + } + channel, err := newXDSChannel(xdsChannelOpts{ + transport: tr, + serverConfig: serverConfig, + clientConfig: c.config, + eventHandler: state, + backoff: c.backoff, + watchExpiryTimeout: c.watchExpiryTimeout, + logPrefix: clientPrefix(c), + }) + if err != nil { + return nil, func() {}, fmt.Errorf("xds: failed to create a new channel for server config %v: %v", serverConfig, err) + } + state.channel = channel + c.xdsActiveChannels[*serverConfig] = state + initLocked(state) + return state.channel, c.releaseChannel(serverConfig, state, deInitLocked), nil +} + +// releaseChannel is a function that is called when a reference to an xdsChannel +// needs to be released. It handles closing channels with no active references. +// +// The function takes the following parameters: +// - serverConfig: the server configuration for the xdsChannel +// - state: the state of the xdsChannel +// - deInitLocked: a function that performs any necessary cleanup for the xdsChannel +// +// The function returns another function that can be called to release the +// reference to the xdsChannel. This returned function is idempotent, meaning +// it can be called multiple times without any additional effect. +func (c *XDSClient) releaseChannel(serverConfig *ServerConfig, state *channelState, deInitLocked func(*channelState)) func() { + return sync.OnceFunc(func() { + c.channelsMu.Lock() + + if c.logger.V(2) { + c.logger.Infof("Received request to release a reference to an xdsChannel for server config %+v", serverConfig) + } + deInitLocked(state) + + // The channel has active users. Do nothing and return. + if len(state.interestedAuthorities) != 0 { + if c.logger.V(2) { + c.logger.Infof("xdsChannel %p has other active references", state.channel) + } + c.channelsMu.Unlock() + return + } + + delete(c.xdsActiveChannels, *serverConfig) + if c.logger.V(2) { + c.logger.Infof("Closing xdsChannel [%p] for server config %s", state.channel, serverConfig) + } + channelToClose := state.channel + c.channelsMu.Unlock() + + channelToClose.close() + }) +} + +// DumpResources returns the status and contents of all xDS resources being +// watched by the xDS client. +func (c *XDSClient) DumpResources() ([]byte, error) { + retCfg := c.topLevelAuthority.dumpResources() + for _, a := range c.authorities { + retCfg = append(retCfg, a.dumpResources()...) + } + + nodeProto := clientsinternal.NodeProto(c.config.Node) + nodeProto.ClientFeatures = []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper} + resp := &v3statuspb.ClientStatusResponse{} + resp.Config = append(resp.Config, &v3statuspb.ClientConfig{ + Node: nodeProto, + GenericXdsConfigs: retCfg, + }) + return proto.Marshal(resp) +} + +// channelState represents the state of an xDS channel. It tracks the number of +// LRS references, the authorities interested in the channel, and the server +// configuration used for the channel. +// +// It receives callbacks for events on the underlying ADS stream and invokes +// corresponding callbacks on interested authorities. +type channelState struct { + parent *XDSClient + serverConfig *ServerConfig + + // Access to the following fields should be protected by the parent's + // channelsMu. + channel *xdsChannel + interestedAuthorities map[*authority]bool +} + +func (cs *channelState) adsStreamFailure(err error) { + if cs.parent.done.HasFired() { + return + } + + if xdsresource.ErrType(err) != xdsresource.ErrTypeStreamFailedAfterRecv && cs.parent.metricsReporter != nil { + cs.parent.metricsReporter.ReportMetric(&metrics.ServerFailure{ + ServerURI: cs.serverConfig.ServerIdentifier.ServerURI, + }) + } + + cs.parent.channelsMu.Lock() + defer cs.parent.channelsMu.Unlock() + for authority := range cs.interestedAuthorities { + authority.adsStreamFailure(cs.serverConfig, err) + } +} + +func (cs *channelState) adsResourceUpdate(typ ResourceType, updates map[string]dataAndErrTuple, md xdsresource.UpdateMetadata, onDone func()) { + if cs.parent.done.HasFired() { + return + } + + cs.parent.channelsMu.Lock() + defer cs.parent.channelsMu.Unlock() + + if len(cs.interestedAuthorities) == 0 { + onDone() + return + } + + authorityCnt := new(atomic.Int64) + authorityCnt.Add(int64(len(cs.interestedAuthorities))) + done := func() { + if authorityCnt.Add(-1) == 0 { + onDone() + } + } + for authority := range cs.interestedAuthorities { + authority.adsResourceUpdate(cs.serverConfig, typ, updates, md, done) + } +} + +func (cs *channelState) adsResourceDoesNotExist(typ ResourceType, resourceName string) { + if cs.parent.done.HasFired() { + return + } + + cs.parent.channelsMu.Lock() + defer cs.parent.channelsMu.Unlock() + for authority := range cs.interestedAuthorities { + authority.adsResourceDoesNotExist(typ, resourceName) + } +} + +func resourceWatchStateForTesting(c *XDSClient, rType ResourceType, resourceName string) (xdsresource.ResourceWatchState, error) { + n := xdsresource.ParseName(resourceName) + a := c.getAuthorityForResource(n) + if a == nil { + return xdsresource.ResourceWatchState{}, fmt.Errorf("unable to find authority for resource name %q", resourceName) + } + return a.resourceWatchStateForTesting(rType, resourceName) + +} + +// xdsClientMetricReporter is a wrapper around XDSClient used solely for +// reporting metrics. We create this separate type to implement the +// clients.AsyncReporter interface, preventing its Report method from +// becoming part of the public XDSClient API. This is especially important +// because the AsyncReporter interface is experimental, and we want to +// avoid coupling experimental changes to the stable XDSClient API. +type xdsClientMetricReporter struct { + c *XDSClient +} + +// Report implements clients.AsyncReporter. +// This is the entry point invoked by the metrics system during a scrape. +func (r *xdsClientMetricReporter) Report(rec clients.AsyncMetricsRecorder) error { + r.c.reportConnectedState(rec) + r.c.reportResourceStats(rec) + return nil +} + +// reportConnectedState handles the "grpc.xds_client.connected" metric. +func (c *XDSClient) reportConnectedState(rec clients.AsyncMetricsRecorder) { + c.channelsMu.Lock() + defer c.channelsMu.Unlock() + + for _, cs := range c.xdsActiveChannels { + val := int64(0) + if cs.channel.ads.isStreamEstablished() { + val = 1 + } + + rec.ReportMetric(&metrics.XDSClientConnected{ + ServerURI: cs.serverConfig.ServerIdentifier.ServerURI, + Value: val, + }) + } +} + +// reportResourceStats handles the "grpc.xds_client.resources" metric. +func (c *XDSClient) reportResourceStats(rec clients.AsyncMetricsRecorder) { + reportForAuthority := func(a *authority) { + stats := a.resourceStats() + for resourceType, stateCounts := range stats { + for cacheState, count := range stateCounts { + if count > 0 { + authorityName := a.name + if authorityName == "" { + authorityName = "#old" + } + rec.ReportMetric(&metrics.XDSClientResourceStats{ + Authority: authorityName, + ResourceType: resourceType, + CacheState: cacheState, + Count: int64(count), + }) + } + } + } + } + reportForAuthority(c.topLevelAuthority) + for _, a := range c.authorities { + reportForAuthority(a) + } +} diff --git a/internal/xds/clients/xdsclient/xdsclient_test.go b/internal/xds/clients/xdsclient/xdsclient_test.go new file mode 100644 index 0000000..d166438 --- /dev/null +++ b/internal/xds/clients/xdsclient/xdsclient_test.go @@ -0,0 +1,128 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "strings" + "testing" + + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource" +) + +func (s) TestXDSClient_New(t *testing.T) { + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + + tests := []struct { + name string + config Config + wantErr string + }{ + { + name: "nil resource types", + config: Config{ + Node: clients.Node{ID: "node-id"}, + }, + wantErr: "resource types map is nil", + }, + { + name: "nil transport builder", + config: Config{ + Node: clients.Node{ID: "node-id"}, + ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType}, + }, + wantErr: "transport builder is nil", + }, + { + name: "no servers or authorities", + config: Config{ + Node: clients.Node{ID: "node-id"}, + ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType}, + TransportBuilder: grpctransport.NewBuilder(configs), + }, + wantErr: "no servers or authorities specified", + }, + { + name: "success with servers", + config: Config{ + Node: clients.Node{ID: "node-id"}, + ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType}, + TransportBuilder: grpctransport.NewBuilder(configs), + Servers: []ServerConfig{{ServerIdentifier: clients.ServerIdentifier{ServerURI: "dummy-server"}}}, + }, + wantErr: "", + }, + { + name: "success with servers and empty nodeID", + config: Config{ + Node: clients.Node{ID: ""}, + ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType}, + TransportBuilder: grpctransport.NewBuilder(configs), + Servers: []ServerConfig{{ServerIdentifier: clients.ServerIdentifier{ServerURI: "dummy-server"}}}, + }, + wantErr: "", + }, + { + name: "success with authorities", + config: Config{ + Node: clients.Node{ID: "node-id"}, + ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType}, + TransportBuilder: grpctransport.NewBuilder(configs), + Authorities: map[string]Authority{"authority-name": {XDSServers: []ServerConfig{{ServerIdentifier: clients.ServerIdentifier{ServerURI: "dummy-server"}}}}}, + }, + wantErr: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, err := New(tt.config) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("New(%+v) failed: %v", tt.config, err) + } + } else { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("New(%+v) returned error %v, want error %q", tt.config, err, tt.wantErr) + } + } + if c != nil { + c.Close() + } + }) + } +} + +func (s) TestXDSClient_Close(t *testing.T) { + configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}} + config := Config{ + Node: clients.Node{ID: "node-id"}, + ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType}, + TransportBuilder: grpctransport.NewBuilder(configs), + Servers: []ServerConfig{{ServerIdentifier: clients.ServerIdentifier{ServerURI: "dummy-server"}}}, + } + c, err := New(config) + if err != nil { + t.Fatalf("New(%+v) failed: %v", config, err) + } + c.Close() + // Calling close again should not panic. + c.Close() +} diff --git a/internal/xds/clients/xdsclient/xdsconfig.go b/internal/xds/clients/xdsclient/xdsconfig.go new file mode 100644 index 0000000..14d64c1 --- /dev/null +++ b/internal/xds/clients/xdsclient/xdsconfig.go @@ -0,0 +1,112 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "time" + + "google.golang.org/grpc/internal/xds/clients" +) + +// ServerFeature indicates the features that will be supported by an xDS server. +type ServerFeature uint64 + +const ( + // ServerFeatureIgnoreResourceDeletion indicates that the server supports a + // feature where the xDS client can ignore resource deletions from this server, + // as described in gRFC A53. + ServerFeatureIgnoreResourceDeletion ServerFeature = 1 << iota + // ServerFeatureTrustedXDSServer returns true if this server is trusted, + // and gRPC should accept security-config-affecting fields from the server + // as described in gRFC A81. + ServerFeatureTrustedXDSServer +) + +// Config is used to configure an xDS client. After one has been passed to the +// xDS client's New function, no part of it may be modified. A Config may be +// reused; the xdsclient package will also not modify it. +type Config struct { + // Servers specifies a list of xDS management servers to connect to. The + // order of the servers in this list reflects the order of preference of + // the data returned by those servers. The xDS client uses the first + // available server from the list. + // + // See gRFC A71 for more details on fallback behavior when the primary + // xDS server is unavailable. + // + // gRFC A71: https://github.com/grpc/proposal/blob/master/A71-xds-fallback.md + Servers []ServerConfig + + // Authorities defines the configuration for each xDS authority. Federated resources + // will be fetched from the servers specified by the corresponding Authority. + Authorities map[string]Authority + + // Node is the identity of the xDS client connecting to the xDS + // management server. + Node clients.Node + + // TransportBuilder is used to create connections to xDS management servers. + TransportBuilder clients.TransportBuilder + + // ResourceTypes is a map from resource type URLs to resource type + // implementations. Each resource type URL uniquely identifies a specific + // kind of xDS resource, and the corresponding resource type implementation + // provides logic for parsing, validating, and processing resources of that + // type. + // + // For example: "type.googleapis.com/envoy.config.listener.v3.Listener" + ResourceTypes map[string]ResourceType + + // MetricsReporter is used to report registered metrics. If unset, no + // metrics will be reported. + MetricsReporter clients.MetricsReporter + + // WatchExpiryTimeout is the duration after which a resource watch expires + // if the requested resource is not received from the management server. + // Most users will not need to set this. If zero, a default value of 15 + // seconds is used as specified here: + // envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#knowing-when-a-requested-resource-does-not-exist + WatchExpiryTimeout time.Duration +} + +// ServerConfig contains configuration for an xDS management server. +type ServerConfig struct { + ServerIdentifier clients.ServerIdentifier + ServerFeature ServerFeature // ServerFeature stores a bitmap of supported features. +} + +// Authority contains configuration for an xDS control plane authority. +// +// See: https://www.envoyproxy.io/docs/envoy/latest/xds/core/v3/resource_locator.proto#xds-core-v3-resourcelocator +type Authority struct { + // XDSServers contains the list of server configurations for this authority. + // + // See Config.Servers for more details. + XDSServers []ServerConfig +} + +func isServerConfigEqual(a, b *ServerConfig) bool { + return a.ServerIdentifier == b.ServerIdentifier && a.ServerFeature == b.ServerFeature +} + +// SupportsServerFeature returns true if the server configuration indicates that +// the server supports the given feature. +func (s *ServerConfig) SupportsServerFeature(feature ServerFeature) bool { + return s.ServerFeature&feature != 0 +} diff --git a/internal/xds/clusterspecifier/cluster_specifier.go b/internal/xds/clusterspecifier/cluster_specifier.go new file mode 100644 index 0000000..f81e2ca --- /dev/null +++ b/internal/xds/clusterspecifier/cluster_specifier.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package clusterspecifier contains the ClusterSpecifier interface and a registry for +// storing and retrieving their implementations. +package clusterspecifier + +import ( + "google.golang.org/protobuf/proto" +) + +// BalancerConfig is the Go Native JSON representation of a balancer +// configuration. +type BalancerConfig []map[string]any + +// ClusterSpecifier defines the parsing functionality of a Cluster Specifier. +type ClusterSpecifier interface { + // TypeURLs are the proto message types supported by this + // ClusterSpecifierPlugin. A ClusterSpecifierPlugin will be registered by + // each of its supported message types. + TypeURLs() []string + // ParseClusterSpecifierConfig parses the provided configuration + // proto.Message from the top level RDS configuration. The resulting + // BalancerConfig will be used as configuration for a child LB Policy of the + // Cluster Manager LB Policy. A nil BalancerConfig is invalid. + ParseClusterSpecifierConfig(proto.Message) (BalancerConfig, error) +} + +var ( + // m is a map from scheme to filter. + m = make(map[string]ClusterSpecifier) +) + +// Register registers the ClusterSpecifierPlugin to the ClusterSpecifier map. +// cs.TypeURLs() will be used as the types for this ClusterSpecifierPlugin. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple cluster specifier +// plugins are registered with the same type URL, the one registered last will +// take effect. +func Register(cs ClusterSpecifier) { + for _, u := range cs.TypeURLs() { + m[u] = cs + } +} + +// Get returns the ClusterSpecifier registered with typeURL. +// +// If no cluster specifier is registered with typeURL, nil will be returned. +func Get(typeURL string) ClusterSpecifier { + return m[typeURL] +} + +// UnregisterForTesting unregisters the ClusterSpecifier for testing purposes. +func UnregisterForTesting(typeURL string) { + delete(m, typeURL) +} diff --git a/internal/xds/clusterspecifier/rls/rls.go b/internal/xds/clusterspecifier/rls/rls.go new file mode 100644 index 0000000..7ac38f0 --- /dev/null +++ b/internal/xds/clusterspecifier/rls/rls.go @@ -0,0 +1,95 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package rls implements the RLS cluster specifier plugin. +package rls + +import ( + "encoding/json" + "fmt" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/internal" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/grpc/internal/xds/clusterspecifier" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +func init() { + clusterspecifier.Register(rls{}) +} + +type rls struct{} + +func (rls) TypeURLs() []string { + return []string{"type.googleapis.com/grpc.lookup.v1.RouteLookupClusterSpecifier"} +} + +// lbConfigJSON is the RLS LB Policies configuration in JSON format. +// RouteLookupConfig will be a raw JSON string from the passed in proto +// configuration, and the other fields will be hardcoded. +type lbConfigJSON struct { + RouteLookupConfig json.RawMessage `json:"routeLookupConfig"` + ChildPolicy []map[string]json.RawMessage `json:"childPolicy"` + ChildPolicyConfigTargetFieldName string `json:"childPolicyConfigTargetFieldName"` +} + +func (rls) ParseClusterSpecifierConfig(cfg proto.Message) (clusterspecifier.BalancerConfig, error) { + if cfg == nil { + return nil, fmt.Errorf("rls_csp: nil configuration message provided") + } + m, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("rls_csp: error parsing config %v: unknown type %T", cfg, cfg) + } + rlcs := new(rlspb.RouteLookupClusterSpecifier) + + if err := m.UnmarshalTo(rlcs); err != nil { + return nil, fmt.Errorf("rls_csp: error parsing config %v: %v", cfg, err) + } + rlcJSON, err := protojson.Marshal(rlcs.GetRouteLookupConfig()) + if err != nil { + return nil, fmt.Errorf("rls_csp: error marshaling route lookup config: %v: %v", rlcs.GetRouteLookupConfig(), err) + } + lbCfgJSON := &lbConfigJSON{ + RouteLookupConfig: rlcJSON, // "JSON form of RouteLookupClusterSpecifier.config" - RLS in xDS Design Doc + ChildPolicy: []map[string]json.RawMessage{ + { + "cds_experimental": json.RawMessage(`{"isDynamic":true}`), + }, + }, + ChildPolicyConfigTargetFieldName: "cluster", + } + + rawJSON, err := json.Marshal(lbCfgJSON) + if err != nil { + return nil, fmt.Errorf("rls_csp: error marshaling load balancing config %v: %v", lbCfgJSON, err) + } + + rlsBB := balancer.Get(internal.RLSLoadBalancingPolicyName) + if rlsBB == nil { + return nil, fmt.Errorf("RLS LB policy not registered") + } + if _, err = rlsBB.(balancer.ConfigParser).ParseConfig(rawJSON); err != nil { + return nil, fmt.Errorf("rls_csp: validation error from rls lb policy parsing: %v", err) + } + + return clusterspecifier.BalancerConfig{{internal.RLSLoadBalancingPolicyName: lbCfgJSON}}, nil +} diff --git a/internal/xds/clusterspecifier/rls/rls_test.go b/internal/xds/clusterspecifier/rls/rls_test.go new file mode 100644 index 0000000..60e949a --- /dev/null +++ b/internal/xds/clusterspecifier/rls/rls_test.go @@ -0,0 +1,162 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "encoding/json" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/grpctest" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/clusterspecifier" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" + + _ "google.golang.org/grpc/balancer/rls" // Register the RLS LB policy. + _ "google.golang.org/grpc/internal/xds/balancer/cdsbalancer" // Register the CDS LB policy. +) + +func init() { + clusterspecifier.Register(rls{}) +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestParseClusterSpecifierConfig tests the parsing functionality of the RLS +// Cluster Specifier Plugin. +func (s) TestParseClusterSpecifierConfig(t *testing.T) { + tests := []struct { + name string + rlcs proto.Message + wantConfig clusterspecifier.BalancerConfig + wantErr bool + }{ + { + + // This will error because the required_match field is set in grpc key builder + name: "invalid-rls-cluster-specifier", + rlcs: testutils.MarshalAny(t, &rlspb.RouteLookupClusterSpecifier{ + RouteLookupConfig: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + { + Service: "service", + Method: "method", + }, + }, + Headers: []*rlspb.NameMatcher{ + { + Key: "k1", + RequiredMatch: true, + Names: []string{"v1"}, + }, + }, + }, + }, + }, + }), + wantErr: true, + }, + { + name: "valid-rls-cluster-specifier", + rlcs: testutils.MarshalAny(t, &rlspb.RouteLookupClusterSpecifier{ + RouteLookupConfig: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + { + Service: "service", + Method: "method", + }, + }, + Headers: []*rlspb.NameMatcher{ + { + Key: "k1", + Names: []string{"v1"}, + }, + }, + }, + }, + LookupService: "target", + LookupServiceTimeout: &durationpb.Duration{Seconds: 100}, + MaxAge: &durationpb.Duration{Seconds: 60}, + StaleAge: &durationpb.Duration{Seconds: 50}, + CacheSizeBytes: 1000, + DefaultTarget: "passthrough:///default", + }, + }), + wantConfig: configWithoutTransformationsWant, + }, + } + for _, test := range tests { + cs := clusterspecifier.Get("type.googleapis.com/grpc.lookup.v1.RouteLookupClusterSpecifier") + if cs == nil { + t.Fatal("Error getting cluster specifier") + } + lbCfg, err := cs.ParseClusterSpecifierConfig(test.rlcs) + + if (err != nil) != test.wantErr { + t.Fatalf("ParseClusterSpecifierConfig(%+v) returned err: %v, wantErr: %v", test.rlcs, err, test.wantErr) + } + if test.wantErr { // Successfully received an error. + continue + } + // Marshal and then unmarshal into any to get rid of nondeterministic + // protojson Marshaling. + lbCfgJSON, err := json.Marshal(lbCfg) + if err != nil { + t.Fatalf("json.Marshal(%+v) returned err %v", lbCfg, err) + } + var got any + if err := json.Unmarshal(lbCfgJSON, &got); err != nil { + t.Fatalf("json.Unmarshal(%+v) returned err %v", lbCfgJSON, err) + } + wantCfgJSON, err := json.Marshal(test.wantConfig) + if err != nil { + t.Fatalf("json.Marshal(%+v) returned err %v", test.wantConfig, err) + } + var want any + if err := json.Unmarshal(wantCfgJSON, &want); err != nil { + t.Fatalf("json.Unmarshal(%+v) returned err %v", lbCfgJSON, err) + } + if diff := cmp.Diff(want, got, cmpopts.EquateEmpty()); diff != "" { + t.Fatalf("ParseClusterSpecifierConfig(%+v) returned expected, diff (-want +got) %v", test.rlcs, diff) + } + } +} + +var configWithoutTransformationsWant = clusterspecifier.BalancerConfig{{"rls_experimental": &lbConfigJSON{ + RouteLookupConfig: []byte(`{"grpcKeybuilders":[{"names":[{"service":"service","method":"method"}],"headers":[{"key":"k1","names":["v1"]}]}],"lookupService":"target","lookupServiceTimeout":"100s","maxAge":"60s","staleAge":"50s","cacheSizeBytes":"1000","defaultTarget":"passthrough:///default"}`), + ChildPolicy: []map[string]json.RawMessage{ + { + "cds_experimental": []byte(`{"isDynamic":true}`), + }, + }, + ChildPolicyConfigTargetFieldName: "cluster", +}}} diff --git a/internal/xds/credentials/access_token.go b/internal/xds/credentials/access_token.go new file mode 100644 index 0000000..cf7c9a7 --- /dev/null +++ b/internal/xds/credentials/access_token.go @@ -0,0 +1,71 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials + +import ( + "context" + "fmt" + + "google.golang.org/grpc/credentials" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + accesstokenpb "github.com/envoyproxy/go-control-plane/envoy/extensions/grpc_service/call_credentials/access_token/v3" +) + +const accessTokenCredsTypeURL = "type.googleapis.com/envoy.extensions.grpc_service.call_credentials.access_token.v3.AccessTokenCredentials" + +func init() { + RegisterCallCredsBuilder(accessTokenCredsTypeURL, func(config *anypb.Any) (credentials.PerRPCCredentials, func(), error) { + var accessToken accesstokenpb.AccessTokenCredentials + if err := anypb.UnmarshalTo(config, &accessToken, proto.UnmarshalOptions{}); err != nil { + return nil, nil, fmt.Errorf("credentials: failed to unmarshal AccessTokenCredentials: %v", err) + } + if accessToken.GetToken() == "" { + return nil, nil, fmt.Errorf("credentials: access token must be non-empty") + } + // These credentials hold no resources; the no-op cleanup satisfies + // the registry contract. + return &accessTokenCallCreds{token: accessToken.GetToken()}, func() {}, nil + }) +} + +// accessTokenCallCreds implements credentials.PerRPCCredentials by attaching +// a static bearer token to each RPC (gRFC A102). The credentials require +// transport security: the token is only ever sent on connections that provide +// privacy and integrity, and RPCs on weaker connections fail. +type accessTokenCallCreds struct { + token string +} + +// GetRequestMetadata returns the token as an authorization header. It fails +// if the connection does not provide privacy and integrity. +func (c *accessTokenCallCreds) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) { + ri, _ := credentials.RequestInfoFromContext(ctx) + if err := credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil { + return nil, fmt.Errorf("credentials: unable to transfer access token PerRPCCredentials: %v", err) + } + return map[string]string{"authorization": "Bearer " + c.token}, nil +} + +// RequireTransportSecurity indicates whether the credentials require +// transport security. +func (c *accessTokenCallCreds) RequireTransportSecurity() bool { + return true +} diff --git a/internal/xds/credentials/access_token_test.go b/internal/xds/credentials/access_token_test.go new file mode 100644 index 0000000..342fc65 --- /dev/null +++ b/internal/xds/credentials/access_token_test.go @@ -0,0 +1,126 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials_test + +import ( + "context" + "strings" + "testing" + + "google.golang.org/grpc/credentials" + xdscreds "google.golang.org/grpc/internal/xds/credentials" + "google.golang.org/protobuf/types/known/anypb" + + accesstokenpb "github.com/envoyproxy/go-control-plane/envoy/extensions/grpc_service/call_credentials/access_token/v3" +) + +const accessTokenCredsTypeURL = "type.googleapis.com/envoy.extensions.grpc_service.call_credentials.access_token.v3.AccessTokenCredentials" + +func accessTokenConfig(t *testing.T, token string) *anypb.Any { + t.Helper() + a, err := anypb.New(&accesstokenpb.AccessTokenCredentials{Token: token}) + if err != nil { + t.Fatalf("Failed to marshal AccessTokenCredentials: %v", err) + } + return a +} + +// Tests that building access token call credentials fails on malformed +// configs and on an empty token. +func (s) TestAccessTokenCredsBuild_Errors(t *testing.T) { + tests := []struct { + name string + config *anypb.Any + wantErr string + }{ + { + name: "unmarshal_failure", + config: &anypb.Any{TypeUrl: accessTokenCredsTypeURL, Value: []byte{0xff}}, + wantErr: "failed to unmarshal AccessTokenCredentials", + }, + { + name: "empty_token", + config: accessTokenConfig(t, ""), + wantErr: "access token must be non-empty", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + callCreds, _, err := xdscreds.GetCallCredsBuilder(accessTokenCredsTypeURL)(tt.config) + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("Build returned error %v, want error containing %q", err, tt.wantErr) + } + if callCreds != nil { + t.Errorf("Build returned non-nil call credentials alongside error") + } + }) + } +} + +// Tests that the token is attached as a bearer authorization header on +// connections providing privacy and integrity, and that an error is returned +// on weaker connections. +func (s) TestAccessTokenCredsGetRequestMetadata(t *testing.T) { + callCreds, cleanup, err := xdscreds.GetCallCredsBuilder(accessTokenCredsTypeURL)(accessTokenConfig(t, "test-token")) + if err != nil { + t.Fatalf("Build failed: %v", err) + } + defer cleanup() + + if !callCreds.RequireTransportSecurity() { + t.Error("RequireTransportSecurity() = false, want true") + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // The token must be attached on a connection with privacy and integrity. + secureCtx := credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &testAuthInfo{secLevel: credentials.PrivacyAndIntegrity}, + }) + md, err := callCreds.GetRequestMetadata(secureCtx) + if err != nil { + t.Fatalf("GetRequestMetadata() on a secure connection failed: %v", err) + } + if got, want := md["authorization"], "Bearer test-token"; got != want { + t.Fatalf("GetRequestMetadata() on a secure connection returned authorization header %q, want %q", got, want) + } + + // RPCs on a connection that does not provide privacy and integrity must + // fail. + insecureCtx := credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{ + AuthInfo: &testAuthInfo{secLevel: credentials.NoSecurity}, + }) + if _, err := callCreds.GetRequestMetadata(insecureCtx); err == nil { + t.Fatal("GetRequestMetadata() on an insecure connection succeeded, want error") + } +} + +// testAuthInfo implements credentials.AuthInfo for testing. +type testAuthInfo struct { + secLevel credentials.SecurityLevel +} + +func (t *testAuthInfo) AuthType() string { + return "test" +} + +func (t *testAuthInfo) GetCommonAuthInfo() credentials.CommonAuthInfo { + return credentials.CommonAuthInfo{SecurityLevel: t.secLevel} +} diff --git a/internal/xds/credentials/credentials.go b/internal/xds/credentials/credentials.go new file mode 100644 index 0000000..fb1707b --- /dev/null +++ b/internal/xds/credentials/credentials.go @@ -0,0 +1,217 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package credentials provides the credentials used for xDS-configured side +// channels (gRFC A102): built channel and call credentials paired with the +// identity of the configuration they were built from, and registries of +// credential builders keyed by proto type URL. +package credentials + +import ( + "bytes" + "fmt" + "sync" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + xdscredspb "github.com/envoyproxy/go-control-plane/envoy/extensions/grpc_service/channel_credentials/xds/v3" +) + +const ( + insecureCredsTypeURL = "type.googleapis.com/envoy.extensions.grpc_service.channel_credentials.insecure.v3.InsecureCredentials" + googleDefaultCredsTypeURL = "type.googleapis.com/envoy.extensions.grpc_service.channel_credentials.google_default.v3.GoogleDefaultCredentials" + xdsCredsTypeURL = "type.googleapis.com/envoy.extensions.grpc_service.channel_credentials.xds.v3.XdsCredentials" +) + +func init() { + RegisterChannelCredsBuilder(insecureCredsTypeURL, func(*anypb.Any, CertProviderConfigResolver) (credentials.Bundle, func(), error) { + return insecure.NewBundle(), func() {}, nil + }) + RegisterChannelCredsBuilder(googleDefaultCredsTypeURL, func(*anypb.Any, CertProviderConfigResolver) (credentials.Bundle, func(), error) { + return google.NewDefaultCredentials(), func() {}, nil + }) + // The xds credential resolves to its required fallback credential, whose + // builder is looked up in the registry: a side-channel target is not an + // xDS cluster, so there is no xDS security configuration for it. + RegisterChannelCredsBuilder(xdsCredsTypeURL, func(config *anypb.Any, resolver CertProviderConfigResolver) (credentials.Bundle, func(), error) { + var xdsCfg xdscredspb.XdsCredentials + if err := anypb.UnmarshalTo(config, &xdsCfg, proto.UnmarshalOptions{}); err != nil { + return nil, nil, fmt.Errorf("credentials: failed to unmarshal XdsCredentials: %v", err) + } + fallback := xdsCfg.GetFallbackCredentials() + if fallback == nil { + return nil, nil, fmt.Errorf("credentials: xds credentials missing required fallback credentials") + } + if b := GetChannelCredsBuilder(fallback.GetTypeUrl()); b != nil { + return b(fallback, resolver) + } + return nil, nil, fmt.Errorf("credentials: unsupported fallback credentials type %q in xds credentials", fallback.GetTypeUrl()) + }) +} + +// CertProviderConfigResolver resolves certificate provider instance names to +// their configuration. It is implemented by the bootstrap config, and +// declared here so that credential builders do not depend on the bootstrap +// package. +type CertProviderConfigResolver interface { + CertProviderConfigs() map[string]*certprovider.BuildableConfig +} + +// ChannelCredsBuilder builds a channel credentials bundle from a GrpcService +// channel credentials plugin config. The resolver gives access to the +// certificate provider instances configured in the bootstrap config; builders +// that do not reference them ignore it. The returned function releases the +// resources held by the bundle when it is no longer needed. +type ChannelCredsBuilder func(config *anypb.Any, resolver CertProviderConfigResolver) (credentials.Bundle, func(), error) + +// CallCredsBuilder builds per-RPC credentials from a GrpcService call +// credentials plugin config. The returned function releases the resources +// held by the credentials when they are no longer needed. +type CallCredsBuilder func(config *anypb.Any) (credentials.PerRPCCredentials, func(), error) + +var ( + // channelCredsBuilders is a map from proto type URL to + // ChannelCredsBuilder. + channelCredsBuilders = make(map[string]ChannelCredsBuilder) + // callCredsBuilders is a map from proto type URL to CallCredsBuilder. + callCredsBuilders = make(map[string]CallCredsBuilder) +) + +// RegisterChannelCredsBuilder registers the builder for the given proto type +// URL. Must be called at init time. Not thread safe. +func RegisterChannelCredsBuilder(typeURL string, b ChannelCredsBuilder) { + channelCredsBuilders[typeURL] = b +} + +// GetChannelCredsBuilder returns the builder registered for the given proto +// type URL, or nil if there is none. +func GetChannelCredsBuilder(typeURL string) ChannelCredsBuilder { + return channelCredsBuilders[typeURL] +} + +// RegisterCallCredsBuilder registers the builder for the given proto type +// URL. Must be called at init time. Not thread safe. +func RegisterCallCredsBuilder(typeURL string, b CallCredsBuilder) { + callCredsBuilders[typeURL] = b +} + +// GetCallCredsBuilder returns the builder registered for the given proto type +// URL, or nil if there is none. +func GetCallCredsBuilder(typeURL string) CallCredsBuilder { + return callCredsBuilders[typeURL] +} + +// Identity identifies the configuration a credential was built from. +type Identity struct { + // Type is the bootstrap credential type name for JSON-sourced + // credentials, or the proto type URL for proto-sourced ones. The two + // namespaces cannot collide: proto type URLs contain dots and slashes. + Type string + // Data is the raw JSON configuration, or the proto config value bytes. + Data []byte +} + +// Equal reports whether i and other describe the same configuration. +func (i Identity) Equal(other Identity) bool { + return i.Type == other.Type && bytes.Equal(i.Data, other.Data) +} + +// ChannelCreds pairs a built credentials bundle with the identity of the +// configuration it was built from. +type ChannelCreds struct { + bundle credentials.Bundle + identity Identity + cleanup func() +} + +// NewChannelCreds pairs the given bundle with its identity. cleanup releases +// the resources held by the bundle and is run by Close; it must be nil when +// the bundle is owned by another component (e.g. the bootstrap config), in +// which case Close is a no-op. +func NewChannelCreds(bundle credentials.Bundle, identity Identity, cleanup func()) *ChannelCreds { + if cleanup != nil { + cleanup = sync.OnceFunc(cleanup) + } + return &ChannelCreds{bundle: bundle, identity: identity, cleanup: cleanup} +} + +// Bundle returns the built credentials bundle. +func (c *ChannelCreds) Bundle() credentials.Bundle { + return c.bundle +} + +// Equal reports whether c and other were built from the same configuration. +func (c *ChannelCreds) Equal(other *ChannelCreds) bool { + if c == nil || other == nil { + return c == other + } + return c.identity.Equal(other.identity) +} + +// Close releases the resources held by the bundle, if owned. It is +// idempotent. +func (c *ChannelCreds) Close() { + if c.cleanup != nil { + c.cleanup() + } +} + +// CallCreds pairs built per-RPC credentials with the identity of the +// configuration they were built from. +type CallCreds struct { + creds credentials.PerRPCCredentials + identity Identity + cleanup func() +} + +// NewCallCreds pairs the given per-RPC credentials with their identity. +// cleanup releases the resources held by the credentials and is run by Close; +// it must be nil when the credentials are owned by another component (e.g. +// the bootstrap config), in which case Close is a no-op. +func NewCallCreds(creds credentials.PerRPCCredentials, identity Identity, cleanup func()) *CallCreds { + if cleanup != nil { + cleanup = sync.OnceFunc(cleanup) + } + return &CallCreds{creds: creds, identity: identity, cleanup: cleanup} +} + +// Credentials returns the built per-RPC credentials. +func (c *CallCreds) Credentials() credentials.PerRPCCredentials { + return c.creds +} + +// Equal reports whether c and other were built from the same configuration. +func (c *CallCreds) Equal(other *CallCreds) bool { + if c == nil || other == nil { + return c == other + } + return c.identity.Equal(other.identity) +} + +// Close releases the resources held by the credentials, if owned. It is +// idempotent. +func (c *CallCreds) Close() { + if c.cleanup != nil { + c.cleanup() + } +} diff --git a/internal/xds/credentials/tls.go b/internal/xds/credentials/tls.go new file mode 100644 index 0000000..682eb01 --- /dev/null +++ b/internal/xds/credentials/tls.go @@ -0,0 +1,174 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + tlscredspb "github.com/envoyproxy/go-control-plane/envoy/extensions/grpc_service/channel_credentials/tls/v3" +) + +const tlsCredsTypeURL = "type.googleapis.com/envoy.extensions.grpc_service.channel_credentials.tls.v3.TlsCredentials" + +func init() { + RegisterChannelCredsBuilder(tlsCredsTypeURL, buildTLSCredentials) +} + +// buildTLSCredentials builds TLS channel credentials from a TlsCredentials +// plugin config, whose root and identity certificates are sourced from +// certificate provider instances configured in the bootstrap config. Unknown +// instance names and provider build failures are errors, resulting in the +// resource being NACKed, the same way CommonTlsContext instances are handled +// (gRFC A29). The returned cleanup closes the certificate providers. +func buildTLSCredentials(config *anypb.Any, resolver CertProviderConfigResolver) (credentials.Bundle, func(), error) { + var tlsCfg tlscredspb.TlsCredentials + if err := anypb.UnmarshalTo(config, &tlsCfg, proto.UnmarshalOptions{}); err != nil { + return nil, nil, fmt.Errorf("credentials: failed to unmarshal TlsCredentials: %v", err) + } + if resolver == nil { + return nil, nil, fmt.Errorf("credentials: no bootstrap configuration available to resolve certificate provider instances") + } + + rootInstanceName := tlsCfg.GetRootCertificateProvider().GetInstanceName() + if rootInstanceName == "" { + return nil, nil, fmt.Errorf("credentials: tls credentials must specify root_certificate_provider with an instance_name") + } + rootCfg, err := certProviderConfig(resolver, rootInstanceName, "root") + if err != nil { + return nil, nil, err + } + rootProvider, err := rootCfg.Build(certprovider.BuildOptions{ + CertName: tlsCfg.GetRootCertificateProvider().GetCertificateName(), + WantRoot: true, + }) + if err != nil { + return nil, nil, fmt.Errorf("credentials: failed to build root certificate provider: %v", err) + } + + b := &tlsBundle{rootProvider: rootProvider} + // The identity certificate provider is optional, and is configured only + // for mTLS. When the field is set, it must name a provider instance. + if identity := tlsCfg.GetIdentityCertificateProvider(); identity != nil { + identityInstanceName := identity.GetInstanceName() + if identityInstanceName == "" { + rootProvider.Close() + return nil, nil, fmt.Errorf("credentials: tls credentials identity_certificate_provider must specify an instance_name") + } + identityCfg, err := certProviderConfig(resolver, identityInstanceName, "identity") + if err != nil { + rootProvider.Close() + return nil, nil, err + } + identityProvider, err := identityCfg.Build(certprovider.BuildOptions{ + CertName: identity.GetCertificateName(), + WantIdentity: true, + }) + if err != nil { + rootProvider.Close() + return nil, nil, fmt.Errorf("credentials: failed to build identity certificate provider: %v", err) + } + b.identityProvider = identityProvider + } + return b, b.close, nil +} + +// certProviderConfig looks up the certificate provider instance with the +// given name via the resolver. kind names the certificate the provider is +// used for (root or identity), for error messages. +func certProviderConfig(resolver CertProviderConfigResolver, instanceName, kind string) (*certprovider.BuildableConfig, error) { + cfg, ok := resolver.CertProviderConfigs()[instanceName] + if !ok { + return nil, fmt.Errorf("credentials: tls credentials %s certificate provider: instance name %q missing in bootstrap configuration", kind, instanceName) + } + return cfg, nil +} + +// tlsBundle is a credentials.Bundle providing client-side TLS transport +// credentials whose server root CA certificates, and optionally client +// identity certificates, come from certificate provider instances. The key +// material is fetched from the providers on every handshake, so certificate +// reloads are picked up. It carries no per-RPC credentials. +type tlsBundle struct { + rootProvider certprovider.Provider + identityProvider certprovider.Provider // nil when no identity certificate is configured +} + +func (b *tlsBundle) TransportCredentials() credentials.TransportCredentials { + return b +} + +func (b *tlsBundle) PerRPCCredentials() credentials.PerRPCCredentials { + return nil +} + +func (b *tlsBundle) NewWithMode(string) (credentials.Bundle, error) { + return nil, fmt.Errorf("credentials: xDS TLS channel credentials only support one mode") +} + +// close closes the certificate providers. Subsequent handshakes fail. +func (b *tlsBundle) close() { + b.rootProvider.Close() + if b.identityProvider != nil { + b.identityProvider.Close() + } +} + +func (b *tlsBundle) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + rootKM, err := b.rootProvider.KeyMaterial(ctx) + if err != nil { + return nil, nil, fmt.Errorf("credentials: failed to get root certificates: %v", err) + } + if rootKM.Roots == nil { + return nil, nil, errors.New("credentials: root certificate provider returned no root certificates") + } + cfg := &tls.Config{RootCAs: rootKM.Roots} + if b.identityProvider != nil { + identityKM, err := b.identityProvider.KeyMaterial(ctx) + if err != nil { + return nil, nil, fmt.Errorf("credentials: failed to get identity certificates: %v", err) + } + cfg.Certificates = identityKM.Certs + } + return credentials.NewTLS(cfg).ClientHandshake(ctx, authority, rawConn) +} + +func (b *tlsBundle) ServerHandshake(net.Conn) (net.Conn, credentials.AuthInfo, error) { + return nil, nil, errors.New("credentials: server handshake is not supported by xDS TLS channel credentials") +} + +func (b *tlsBundle) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{SecurityProtocol: "tls"} +} + +func (b *tlsBundle) Clone() credentials.TransportCredentials { + return &tlsBundle{rootProvider: b.rootProvider, identityProvider: b.identityProvider} +} + +func (b *tlsBundle) OverrideServerName(string) error { + return errors.New("credentials: overriding server name is not supported by xDS TLS channel credentials") +} diff --git a/internal/xds/credentials/tls_test.go b/internal/xds/credentials/tls_test.go new file mode 100644 index 0000000..ebe98bf --- /dev/null +++ b/internal/xds/credentials/tls_test.go @@ -0,0 +1,287 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials_test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "net" + "os" + "strings" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/xds/bootstrap" + xdscreds "google.golang.org/grpc/internal/xds/credentials" + "google.golang.org/grpc/testdata" + "google.golang.org/protobuf/types/known/anypb" + + tlscredspb "github.com/envoyproxy/go-control-plane/envoy/extensions/grpc_service/channel_credentials/tls/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 10 * time.Second + + tlsCredsTypeURL = "type.googleapis.com/envoy.extensions.grpc_service.channel_credentials.tls.v3.TlsCredentials" +) + +// testBootstrapConfig returns a bootstrap config with two certificate +// provider instances: "root-instance" watching the CA certificate that signed +// the test server's certificate, and "identity-instance" watching a client +// certificate and key. +func testBootstrapConfig(t *testing.T) *bootstrap.Config { + t.Helper() + + rootCert := testdata.Path("x509/server_ca_cert.pem") + clientCert := testdata.Path("x509/client1_cert.pem") + clientKey := testdata.Path("x509/client1_key.pem") + + contents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: json.RawMessage(`[{"server_uri": "passthrough:///unused", "channel_creds": [{"type": "insecure"}]}]`), + Node: json.RawMessage(`{"id": "test-node"}`), + CertificateProviders: map[string]json.RawMessage{ + "root-instance": json.RawMessage(fmt.Sprintf(`{ + "plugin_name": "file_watcher", + "config": {"ca_certificate_file": %q} + }`, rootCert)), + "identity-instance": json.RawMessage(fmt.Sprintf(`{ + "plugin_name": "file_watcher", + "config": {"certificate_file": %q, "private_key_file": %q} + }`, clientCert, clientKey)), + }, + }) + if err != nil { + t.Fatalf("NewContentsForTesting() failed: %v", err) + } + cfg, err := bootstrap.NewConfigFromContents(contents) + if err != nil { + t.Fatalf("NewConfigFromContents() failed: %v", err) + } + return cfg +} + +// tlsCredsConfig returns a marshaled TlsCredentials plugin config referencing +// the given provider instance names. An empty identity omits the identity +// certificate provider. +func tlsCredsConfig(t *testing.T, root, identity string) *anypb.Any { + t.Helper() + cfg := &tlscredspb.TlsCredentials{} + if root != "" { + cfg.RootCertificateProvider = &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: root} + } + if identity != "" { + cfg.IdentityCertificateProvider = &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: identity} + } + a, err := anypb.New(cfg) + if err != nil { + t.Fatalf("Failed to marshal TlsCredentials: %v", err) + } + return a +} + +// Tests that building TLS channel credentials validates the certificate +// provider instance names against the bootstrap config. +func (s) TestTLSCredsBuild_Errors(t *testing.T) { + bc := testBootstrapConfig(t) + + // The tlsCredsConfig helper cannot express an identity certificate + // provider with an empty instance name, so build that config directly. + emptyIdentityInstance, err := anypb.New(&tlscredspb.TlsCredentials{ + RootCertificateProvider: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: "root-instance"}, + IdentityCertificateProvider: &v3tlspb.CommonTlsContext_CertificateProviderInstance{}, + }) + if err != nil { + t.Fatalf("Failed to marshal TlsCredentials: %v", err) + } + + tests := []struct { + name string + config *anypb.Any + resolver xdscreds.CertProviderConfigResolver + wantErr string + }{ + { + name: "unmarshal_failure", + config: &anypb.Any{TypeUrl: tlsCredsTypeURL, Value: []byte{0xff}}, + resolver: bc, + wantErr: "failed to unmarshal TlsCredentials", + }, + { + name: "missing_root_certificate_provider", + config: tlsCredsConfig(t, "", "identity-instance"), + resolver: bc, + wantErr: "must specify root_certificate_provider", + }, + { + name: "empty_identity_instance_name", + config: emptyIdentityInstance, + resolver: bc, + wantErr: "identity_certificate_provider must specify an instance_name", + }, + { + name: "unknown_root_instance", + config: tlsCredsConfig(t, "unknown-instance", ""), + resolver: bc, + wantErr: `"unknown-instance" missing in bootstrap`, + }, + { + name: "unknown_identity_instance", + config: tlsCredsConfig(t, "root-instance", "unknown-instance"), + resolver: bc, + wantErr: `"unknown-instance" missing in bootstrap`, + }, + { + name: "nil_resolver", + config: tlsCredsConfig(t, "root-instance", ""), + resolver: nil, + wantErr: "no bootstrap configuration", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, err := xdscreds.GetChannelCredsBuilder(tlsCredsTypeURL)(tt.config, tt.resolver) + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("Build returned error %v, want error containing %q", err, tt.wantErr) + } + }) + } +} + +// startTestTLSServer starts a TLS server that performs one handshake per +// accepted connection. If mTLS is true, the server requires and verifies a +// client certificate. +func startTestTLSServer(t *testing.T, mTLS bool) net.Listener { + t.Helper() + + serverCert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("Failed to load server certificate: %v", err) + } + // gRPC's TLS credentials enforce ALPN, so the server must advertise h2. + cfg := &tls.Config{Certificates: []tls.Certificate{serverCert}, NextProtos: []string{"h2"}} + if mTLS { + pem, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem")) + if err != nil { + t.Fatalf("Failed to read client CA certificate: %v", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + t.Fatal("Failed to parse client CA certificate") + } + cfg.ClientAuth = tls.RequireAndVerifyClientCert + cfg.ClientCAs = pool + } + + lis, err := tls.Listen("tcp", "localhost:0", cfg) + if err != nil { + t.Fatalf("Failed to start test TLS server: %v", err) + } + t.Cleanup(func() { lis.Close() }) + go func() { + for { + conn, err := lis.Accept() + if err != nil { + return + } + go func() { + conn.(*tls.Conn).Handshake() + conn.Close() + }() + } + }() + return lis +} + +// clientHandshake dials the test server and performs a client-side TLS +// handshake with credentials built from the given plugin config. +func clientHandshake(t *testing.T, config *anypb.Any, bc *bootstrap.Config) error { + t.Helper() + + bundle, cleanup, err := xdscreds.GetChannelCredsBuilder(tlsCredsTypeURL)(config, bc) + if err != nil { + t.Fatalf("Build failed: %v", err) + } + defer cleanup() + + mTLS := false + var cfg tlscredspb.TlsCredentials + if err := config.UnmarshalTo(&cfg); err == nil && cfg.GetIdentityCertificateProvider() != nil { + mTLS = true + } + lis := startTestTLSServer(t, mTLS) + + rawConn, err := net.Dial("tcp", lis.Addr().String()) + if err != nil { + t.Fatalf("Failed to dial test server: %v", err) + } + defer rawConn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // The test server certificate is issued for *.test.example.com. + _, _, err = bundle.TransportCredentials().ClientHandshake(ctx, "x.test.example.com", rawConn) + return err +} + +// Tests that TLS channel credentials backed by certificate provider instances +// complete a TLS handshake, with and without an identity certificate. +func (s) TestTLSCredsHandshake(t *testing.T) { + bc := testBootstrapConfig(t) + + if err := clientHandshake(t, tlsCredsConfig(t, "root-instance", ""), bc); err != nil { + t.Fatalf("ClientHandshake() with root-only TLS credentials failed: %v", err) + } + if err := clientHandshake(t, tlsCredsConfig(t, "root-instance", "identity-instance"), bc); err != nil { + t.Fatalf("ClientHandshake() with mTLS credentials failed: %v", err) + } +} + +// Tests that closed TLS channel credentials fail handshakes. +func (s) TestTLSCredsClose(t *testing.T) { + bc := testBootstrapConfig(t) + + bundle, cleanup, err := xdscreds.GetChannelCredsBuilder(tlsCredsTypeURL)(tlsCredsConfig(t, "root-instance", ""), bc) + if err != nil { + t.Fatalf("Build failed: %v", err) + } + cleanup() + + // A handshake after close must fail. + server, client := net.Pipe() + defer server.Close() + defer client.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, _, err := bundle.TransportCredentials().ClientHandshake(ctx, "x.test.example.com", client); err == nil || !strings.Contains(err.Error(), "closed") { + t.Fatalf("ClientHandshake() after close returned error %v, want error containing %q", err, "closed") + } +} diff --git a/internal/xds/credentials/xds_test.go b/internal/xds/credentials/xds_test.go new file mode 100644 index 0000000..c3d7a66 --- /dev/null +++ b/internal/xds/credentials/xds_test.go @@ -0,0 +1,76 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials_test + +import ( + "strings" + "testing" + + xdscreds "google.golang.org/grpc/internal/xds/credentials" + "google.golang.org/protobuf/types/known/anypb" + + xdscredspb "github.com/envoyproxy/go-control-plane/envoy/extensions/grpc_service/channel_credentials/xds/v3" +) + +const xdsCredsTypeURL = "type.googleapis.com/envoy.extensions.grpc_service.channel_credentials.xds.v3.XdsCredentials" + +// Tests that building xds channel credentials fails on malformed configs and +// on missing or unsupported fallback credentials. +func (s) TestXDSCredsBuild_Errors(t *testing.T) { + missingFallback, err := anypb.New(&xdscredspb.XdsCredentials{}) + if err != nil { + t.Fatalf("Failed to marshal XdsCredentials: %v", err) + } + unsupportedFallback, err := anypb.New(&xdscredspb.XdsCredentials{ + FallbackCredentials: &anypb.Any{TypeUrl: "type.googleapis.com/unknown.Credentials"}, + }) + if err != nil { + t.Fatalf("Failed to marshal XdsCredentials: %v", err) + } + + tests := []struct { + name string + config *anypb.Any + wantErr string + }{ + { + name: "unmarshal_failure", + config: &anypb.Any{TypeUrl: xdsCredsTypeURL, Value: []byte{0xff}}, + wantErr: "failed to unmarshal XdsCredentials", + }, + { + name: "missing_fallback_credentials", + config: missingFallback, + wantErr: "missing required fallback credentials", + }, + { + name: "unsupported_fallback_credentials_type", + config: unsupportedFallback, + wantErr: "unsupported fallback credentials type", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, err := xdscreds.GetChannelCredsBuilder(xdsCredsTypeURL)(tt.config, nil) + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("Build returned error %v, want error containing %q", err, tt.wantErr) + } + }) + } +} diff --git a/internal/xds/grpcservice/grpcservice.go b/internal/xds/grpcservice/grpcservice.go new file mode 100644 index 0000000..d5d5299 --- /dev/null +++ b/internal/xds/grpcservice/grpcservice.go @@ -0,0 +1,306 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpcservice parses and validates envoy GrpcService protos into a +// form usable for creating side-channel gRPC connections. +package grpcservice + +import ( + "fmt" + "maps" + "net/url" + "slices" + "strings" + "time" + + "google.golang.org/grpc" + imetadata "google.golang.org/grpc/internal/metadata" + "google.golang.org/grpc/internal/xds/bootstrap" + xdscreds "google.golang.org/grpc/internal/xds/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + "google.golang.org/protobuf/types/known/anypb" +) + +const ( + maxHeaderKeyLen = 16384 + maxHeaderValueLen = 16384 +) + +// Config is the parsed form of a GrpcService proto. It carries the built, +// ready-to-use credentials for the side channel. +type Config struct { + // TargetURI is the gRPC target URI of the side-channel service. + TargetURI string + // Timeout, if non-zero, is the deadline to use for RPCs on the side + // channel. + Timeout time.Duration + // InitialMetadata is the metadata to add to RPCs on the side channel. + InitialMetadata metadata.MD + // ChannelCredentials are the channel credentials to create the side + // channel with, paired with the identity of their source configuration. + ChannelCredentials *xdscreds.ChannelCreds + // CallCredentials are the call credentials to apply to RPCs sent on the + // side channel, paired with the identities of their source + // configurations, preserving order. + CallCredentials []*xdscreds.CallCreds +} + +// Equal reports whether c and other are equal. +func (c *Config) Equal(other *Config) bool { + if c == nil || other == nil { + return c == other + } + targetEqual := c.TargetURI == other.TargetURI + timeoutEqual := c.Timeout == other.Timeout + metadataEqual := maps.EqualFunc(c.InitialMetadata, other.InitialMetadata, slices.Equal) + channelCredsEqual := c.ChannelCredentials.Equal(other.ChannelCredentials) + callCredsEqual := slices.EqualFunc(c.CallCredentials, other.CallCredentials, (*xdscreds.CallCreds).Equal) + return targetEqual && timeoutEqual && metadataEqual && channelCredsEqual && callCredsEqual +} + +// Close releases the credentials owned by the config. It is idempotent, and a +// no-op for credentials owned by another component (e.g. the allowlisted +// credentials owned by the bootstrap config). +func (c *Config) Close() { + c.ChannelCredentials.Close() + for _, cc := range c.CallCredentials { + cc.Close() + } +} + +// Dial creates a channel to the side-channel service, using the channel and +// call credentials from the config along with the provided dial options. +// +// Dial does not take ownership of the config: the caller releases the +// config's credentials via Close when the config is no longer needed, after +// closing any channel created from it. +func (c *Config) Dial(opts ...grpc.DialOption) (*grpc.ClientConn, error) { + // The credentials are appended after the provided options so that they + // cannot be overridden. + dialOpts := make([]grpc.DialOption, 0, len(opts)+len(c.CallCredentials)+1) + dialOpts = append(dialOpts, opts...) + dialOpts = append(dialOpts, grpc.WithCredentialsBundle(c.ChannelCredentials.Bundle())) + for _, cc := range c.CallCredentials { + dialOpts = append(dialOpts, grpc.WithPerRPCCredentials(cc.Credentials())) + } + return grpc.NewClient(c.TargetURI, dialOpts...) +} + +// Parse parses and validates a GrpcService proto into a Config, applying the +// gRFC A102 trust policy. bc is the bootstrap configuration of the xDS client +// that received the resource, and sc is the configuration of the management +// server that sent it; both must be non-nil. +// +// Credentials configured in the proto are honored only when the delivering +// management server is trusted, i.e. configured with the trusted_xds_server +// server feature. For untrusted management servers the target must be present +// in the bootstrap allowed_grpc_services map, and the returned Config carries +// the credentials configured there. +// +// The credentials in the returned Config are built and ready to use. The +// caller owns the Config and releases its credentials via Close when it is no +// longer needed, after closing any channel dialed from it. +func Parse(gs *v3corepb.GrpcService, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (*Config, error) { + googleGrpc := gs.GetGoogleGrpc() + if googleGrpc == nil { + return nil, fmt.Errorf("grpcservice: only google_grpc GrpcService config is supported") + } + + targetURI := googleGrpc.GetTargetUri() + if targetURI == "" { + return nil, fmt.Errorf("grpcservice: target_uri must be non-empty") + } + if err := validateTargetURI(targetURI); err != nil { + return nil, err + } + + cfg := &Config{TargetURI: targetURI} + var err error + if cfg.Timeout, err = parseTimeout(gs); err != nil { + return nil, err + } + if cfg.InitialMetadata, err = parseInitialMetadata(gs.GetInitialMetadata()); err != nil { + return nil, err + } + + // Credentials are built last, so that no error path can drop built + // credentials. + if sc.ServerFeaturesTrustedXDSServer() { + if cfg.ChannelCredentials, err = buildChannelCredentials(googleGrpc.GetChannelCredentialsPlugin(), bc); err != nil { + return nil, err + } + if cfg.CallCredentials, err = buildCallCredentials(googleGrpc.GetCallCredentialsPlugin()); err != nil { + cfg.ChannelCredentials.Close() + return nil, err + } + } else { + svc := bc.AllowedGRPCService(targetURI) + if svc == nil { + return nil, fmt.Errorf("grpcservice: target_uri %q is not present in allowed_grpc_services", targetURI) + } + // The allowlisted credentials are owned by the bootstrap config: + // their pairs carry no cleanup, so Config.Close does not affect + // them. + cfg.ChannelCredentials, cfg.CallCredentials = svc.SideChannelCredentials() + } + return cfg, nil +} + +// buildChannelCredentials builds the first channel credential from the plugin +// list whose proto type has a registered builder. It is an error if none of +// the configured plugins are supported, or if building the supported plugin +// fails. +func buildChannelCredentials(plugins []*anypb.Any, bc *bootstrap.Config) (*xdscreds.ChannelCreds, error) { + for _, p := range plugins { + if p == nil { + continue + } + b := xdscreds.GetChannelCredsBuilder(p.GetTypeUrl()) + if b == nil { + continue + } + bundle, cleanup, err := b(p, bc) + if err != nil { + return nil, fmt.Errorf("grpcservice: failed to build channel credentials %q: %v", p.GetTypeUrl(), err) + } + identity := xdscreds.Identity{Type: p.GetTypeUrl(), Data: p.GetValue()} + return xdscreds.NewChannelCreds(bundle, identity, cleanup), nil + } + return nil, fmt.Errorf("grpcservice: no supported channel credentials found in grpc_service") +} + +// buildCallCredentials builds the call credentials from the plugin list, +// preserving order. Plugins whose proto type has no registered builder are +// skipped; call credentials are optional, so an empty result is not an error. +func buildCallCredentials(plugins []*anypb.Any) (_ []*xdscreds.CallCreds, err error) { + var out []*xdscreds.CallCreds + // Release any credentials built before a mid-iteration failure. + defer func() { + if err != nil { + for _, cc := range out { + cc.Close() + } + } + }() + for _, p := range plugins { + if p == nil { + continue + } + b := xdscreds.GetCallCredsBuilder(p.GetTypeUrl()) + if b == nil { + continue + } + cc, cleanup, err := b(p) + if err != nil { + return nil, fmt.Errorf("grpcservice: failed to build call credentials %q: %v", p.GetTypeUrl(), err) + } + identity := xdscreds.Identity{Type: p.GetTypeUrl(), Data: p.GetValue()} + out = append(out, xdscreds.NewCallCreds(cc, identity, cleanup)) + } + return out, nil +} + +// validateTargetURI verifies that the target URI can be handled by a +// registered resolver. +func validateTargetURI(targetURI string) error { + // Mirror the scheme resolution performed by grpc.NewClient: use the + // target's scheme if it parses and is registered; otherwise fall back + // to the default scheme with the whole target as the endpoint. + if u, err := url.Parse(targetURI); err == nil && resolver.Get(u.Scheme) != nil { + return nil + } + canonicalTarget := resolver.GetDefaultScheme() + ":///" + targetURI + u, err := url.Parse(canonicalTarget) + if err != nil { + return fmt.Errorf("grpcservice: target_uri %q is invalid: %v", targetURI, err) + } + if resolver.Get(u.Scheme) == nil { + return fmt.Errorf("grpcservice: no resolver for default scheme %q", u.Scheme) + } + return nil +} + +// parseTimeout validates and converts the GrpcService timeout. A zero timeout +// (unset) is allowed; any set value must be strictly positive. +func parseTimeout(gs *v3corepb.GrpcService) (time.Duration, error) { + d := gs.GetTimeout() + if d == nil { + return 0, nil + } + if err := d.CheckValid(); err != nil { + return 0, fmt.Errorf("grpcservice: invalid timeout: %v", err) + } + timeout := d.AsDuration() + if timeout <= 0 { + return 0, fmt.Errorf("grpcservice: timeout must be strictly positive, got %v", timeout) + } + return timeout, nil +} + +// parseInitialMetadata validates the HeaderValue protos and returns them as +// metadata.MD, preserving the control-plane order for each key. +func parseInitialMetadata(headers []*v3corepb.HeaderValue) (metadata.MD, error) { + if len(headers) == 0 { + return nil, nil + } + md := metadata.MD{} + for _, h := range headers { + key := h.GetKey() + // raw_value takes precedence over the legacy value field. + val := h.GetValue() + if len(h.GetRawValue()) > 0 { + val = string(h.GetRawValue()) + } + if err := validateHeaderKey(key); err != nil { + return nil, fmt.Errorf("grpcservice: invalid header key %q: %v", key, err) + } + if err := validateHeaderValue(key, val); err != nil { + return nil, fmt.Errorf("grpcservice: invalid value for header key %q: %v", key, err) + } + md.Append(key, val) + } + return md, nil +} + +func validateHeaderKey(key string) error { + switch { + case len(key) > maxHeaderKeyLen: + return fmt.Errorf("header key exceeds maximum allowed length of %d", maxHeaderKeyLen) + case key == "host": + return fmt.Errorf("header key cannot be %q", "host") + case strings.HasPrefix(key, ":"): + // imetadata.ValidateKey ignores pseudo-headers, but gRFC A102 + // requires them to be rejected. + return fmt.Errorf("header key cannot start with %q", ":") + case strings.HasPrefix(key, "grpc-"): + return fmt.Errorf("header key cannot start with %q", "grpc-") + } + return imetadata.ValidateKey(key) +} + +func validateHeaderValue(key, val string) error { + if len(val) > maxHeaderValueLen { + return fmt.Errorf("header value exceeds maximum allowed length of %d", maxHeaderValueLen) + } + // ValidatePair skips value validation for "-bin" keys, which may carry + // arbitrary bytes. + return imetadata.ValidatePair(key, val) +} diff --git a/internal/xds/grpcservice/grpcservice_test.go b/internal/xds/grpcservice/grpcservice_test.go new file mode 100644 index 0000000..ca6408b --- /dev/null +++ b/internal/xds/grpcservice/grpcservice_test.go @@ -0,0 +1,462 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcservice + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/bootstrap" + xdscreds "google.golang.org/grpc/internal/xds/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/testdata" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + accesstokenpb "github.com/envoyproxy/go-control-plane/envoy/extensions/grpc_service/call_credentials/access_token/v3" + tlscredspb "github.com/envoyproxy/go-control-plane/envoy/extensions/grpc_service/channel_credentials/tls/v3" + xdscredspb "github.com/envoyproxy/go-control-plane/envoy/extensions/grpc_service/channel_credentials/xds/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/durationpb" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + target = "dns:///my-service:443" + + insecureCredsTypeURL = "type.googleapis.com/envoy.extensions.grpc_service.channel_credentials.insecure.v3.InsecureCredentials" + + defaultTestTimeout = 10 * time.Second +) + +// bootstrapConfig builds a bootstrap Config whose allowed_grpc_services is set +// to the provided JSON (a map from target URI to allowed service config). +func bootstrapConfig(t *testing.T, allowed string) *bootstrap.Config { + t.Helper() + // The allowed_grpc_services bootstrap field is parsed only when a + // consuming feature is enabled. + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true) + contents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: json.RawMessage(`[{"server_uri":"td.googleapis.com:443","channel_creds":[{"type":"insecure"}]}]`), + Node: json.RawMessage(`{}`), + AllowedGRPCServices: json.RawMessage(allowed), + }) + if err != nil { + t.Fatalf("NewContentsForTesting() failed: %v", err) + } + cfg, err := bootstrap.NewConfigFromContents(contents) + if err != nil { + t.Fatalf("NewConfigFromContents() failed: %v", err) + } + return cfg +} + +// trustedServerConfig returns a server config carrying the trusted_xds_server +// server feature. +func trustedServerConfig(t *testing.T) *bootstrap.ServerConfig { + t.Helper() + sc, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{ + URI: "trusted-server:443", + ServerFeatures: []string{"trusted_xds_server"}, + }) + if err != nil { + t.Fatalf("ServerConfigForTesting() failed: %v", err) + } + return sc +} + +// untrustedServerConfig returns a server config without the +// trusted_xds_server server feature. +func untrustedServerConfig(t *testing.T) *bootstrap.ServerConfig { + t.Helper() + sc, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: "untrusted-server:443"}) + if err != nil { + t.Fatalf("ServerConfigForTesting() failed: %v", err) + } + return sc +} + +func googleGrpcService(target string, channelPlugins, callPlugins []*anypb.Any, timeout *durationpb.Duration) *v3corepb.GrpcService { + return &v3corepb.GrpcService{ + TargetSpecifier: &v3corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &v3corepb.GrpcService_GoogleGrpc{ + TargetUri: target, + ChannelCredentialsPlugin: channelPlugins, + CallCredentialsPlugin: callPlugins, + }, + }, + Timeout: timeout, + } +} + +// protoIdentity returns the identity of a proto plugin config. +func protoIdentity(a *anypb.Any) xdscreds.Identity { + return xdscreds.Identity{Type: a.GetTypeUrl(), Data: a.GetValue()} +} + +func accessTokenPlugin(t *testing.T, token string) *anypb.Any { + t.Helper() + a, err := anypb.New(&accesstokenpb.AccessTokenCredentials{Token: token}) + if err != nil { + t.Fatalf("Failed to marshal AccessTokenCredentials: %v", err) + } + return a +} + +func (s) TestParse(t *testing.T) { + insecurePlugin := &anypb.Any{TypeUrl: insecureCredsTypeURL} + xdsWithInsecureFallback, err := anypb.New(&xdscredspb.XdsCredentials{FallbackCredentials: insecurePlugin}) + if err != nil { + t.Fatalf("Failed to marshal XdsCredentials: %v", err) + } + tokenPlugin := accessTokenPlugin(t, "test-token") + unsupportedCallPlugin := &anypb.Any{TypeUrl: "type.googleapis.com/unsupported.CallCredentials"} + allowedInsecure := `{"dns:///my-service:443":{"channel_creds":[{"type":"insecure"}]}}` + + tests := []struct { + name string + gs *v3corepb.GrpcService + sc *bootstrap.ServerConfig + config *bootstrap.Config + // want carries the expected target and credential identities; + // comparisons use Config.Equal via cmp.Diff. + want *Config + wantErr string + }{ + { + name: "trusted_insecure_channel_creds", + gs: googleGrpcService(target, []*anypb.Any{insecurePlugin}, nil, nil), + sc: trustedServerConfig(t), + config: bootstrapConfig(t, "{}"), + want: &Config{TargetURI: target, ChannelCredentials: xdscreds.NewChannelCreds(nil, protoIdentity(insecurePlugin), nil)}, + }, + { + name: "trusted_xds_creds_resolve_to_fallback", + gs: googleGrpcService(target, []*anypb.Any{xdsWithInsecureFallback}, nil, nil), + sc: trustedServerConfig(t), + config: bootstrapConfig(t, "{}"), + want: &Config{TargetURI: target, ChannelCredentials: xdscreds.NewChannelCreds(nil, protoIdentity(xdsWithInsecureFallback), nil)}, + }, + { + // Call-creds plugins of unsupported types are skipped, while + // supported ones are built in order. + name: "trusted_with_call_creds", + gs: googleGrpcService(target, []*anypb.Any{insecurePlugin}, []*anypb.Any{tokenPlugin, unsupportedCallPlugin}, nil), + sc: trustedServerConfig(t), + config: bootstrapConfig(t, "{}"), + want: &Config{ + TargetURI: target, + ChannelCredentials: xdscreds.NewChannelCreds(nil, protoIdentity(insecurePlugin), nil), + CallCredentials: []*xdscreds.CallCreds{xdscreds.NewCallCreds(nil, protoIdentity(tokenPlugin), nil)}, + }, + }, + { + name: "trusted_empty_call_creds_token", + gs: googleGrpcService(target, []*anypb.Any{insecurePlugin}, []*anypb.Any{accessTokenPlugin(t, "")}, nil), + sc: trustedServerConfig(t), + config: bootstrapConfig(t, "{}"), + wantErr: "access token must be non-empty", + }, + { + name: "trusted_no_supported_channel_creds", + gs: googleGrpcService(target, nil, nil, nil), + sc: trustedServerConfig(t), + config: bootstrapConfig(t, "{}"), + wantErr: "no supported channel credentials", + }, + { + name: "untrusted_allowlisted_uses_allowlist_creds", + gs: googleGrpcService(target, []*anypb.Any{insecurePlugin}, nil, nil), + sc: untrustedServerConfig(t), + config: bootstrapConfig(t, allowedInsecure), + want: &Config{TargetURI: target, ChannelCredentials: xdscreds.NewChannelCreds(nil, xdscreds.Identity{Type: "insecure"}, nil)}, + }, + { + // The proto's call-creds plugins are ignored along with its + // channel-creds plugins: the allowlist's call creds are used. + name: "untrusted_allowlisted_uses_allowlist_call_creds", + gs: googleGrpcService(target, []*anypb.Any{insecurePlugin}, []*anypb.Any{tokenPlugin}, nil), + sc: untrustedServerConfig(t), + config: bootstrapConfig(t, `{"dns:///my-service:443":{"channel_creds":[{"type":"insecure"}],"call_creds":[{"type":"jwt_token_file","config":{"jwt_token_file":"/tokens/token"}}]}}`), + want: &Config{ + TargetURI: target, + ChannelCredentials: xdscreds.NewChannelCreds(nil, xdscreds.Identity{Type: "insecure"}, nil), + CallCredentials: []*xdscreds.CallCreds{xdscreds.NewCallCreds(nil, xdscreds.Identity{ + Type: "jwt_token_file", + // Bootstrap parsing normalizes the configuration JSON + // via json.Indent, and the identity carries the + // normalized bytes. + Data: json.RawMessage("{\n\"jwt_token_file\": \"/tokens/token\"\n}"), + }, nil)}, + }, + }, + { + name: "untrusted_not_allowlisted", + gs: googleGrpcService(target, nil, nil, nil), + sc: untrustedServerConfig(t), + config: bootstrapConfig(t, "{}"), + wantErr: "not present in allowed_grpc_services", + }, + { + name: "missing_google_grpc", + gs: &v3corepb.GrpcService{}, + sc: trustedServerConfig(t), + config: bootstrapConfig(t, "{}"), + wantErr: "only google_grpc", + }, + { + name: "empty_target_uri", + gs: googleGrpcService("", nil, nil, nil), + sc: trustedServerConfig(t), + config: bootstrapConfig(t, "{}"), + wantErr: "target_uri must be non-empty", + }, + { + name: "valid_timeout", + gs: googleGrpcService(target, []*anypb.Any{insecurePlugin}, nil, durationpb.New(10*time.Second)), + sc: trustedServerConfig(t), + config: bootstrapConfig(t, "{}"), + want: &Config{ + TargetURI: target, + Timeout: 10 * time.Second, + ChannelCredentials: xdscreds.NewChannelCreds(nil, protoIdentity(insecurePlugin), nil), + }, + }, + { + name: "zero_timeout_rejected", + gs: googleGrpcService(target, []*anypb.Any{insecurePlugin}, nil, durationpb.New(0)), + sc: trustedServerConfig(t), + config: bootstrapConfig(t, "{}"), + wantErr: "timeout must be strictly positive", + }, + { + name: "initial_metadata", + gs: func() *v3corepb.GrpcService { + gs := googleGrpcService(target, []*anypb.Any{insecurePlugin}, nil, nil) + gs.InitialMetadata = []*v3corepb.HeaderValue{ + {Key: "key-b", Value: "b"}, + // raw_value takes precedence over the legacy value field. + {Key: "key-a", Value: "legacy", RawValue: []byte("raw-a")}, + } + return gs + }(), + sc: trustedServerConfig(t), + config: bootstrapConfig(t, "{}"), + want: &Config{ + TargetURI: target, + InitialMetadata: metadata.MD{"key-b": {"b"}, "key-a": {"raw-a"}}, + ChannelCredentials: xdscreds.NewChannelCreds(nil, protoIdentity(insecurePlugin), nil), + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := Parse(test.gs, test.config, test.sc) + if test.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("Parse() error = %v, want substring %q", err, test.wantErr) + } + return + } + if err != nil { + t.Fatalf("Parse() returned unexpected error: %v", err) + } + // The cmp.Diff below compares credentials by identity only, so it + // cannot tell a built credential from a nil one; verify the + // credentials were built. + if got.ChannelCredentials.Bundle() == nil { + t.Error("Parse() returned channel credentials without a built bundle") + } + for i, cc := range got.CallCredentials { + if cc.Credentials() == nil { + t.Errorf("Parse() call credentials[%d] have no built credentials", i) + } + } + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("Parse() returned unexpected config (-want +got):\n%s", diff) + } + }) + } +} + +// Tests that Dial applies the config's credentials to the created channel: +// call credentials that require transport security combined with insecure +// channel credentials must fail channel creation. +func (s) TestConfigDial(t *testing.T) { + gs := googleGrpcService(target, []*anypb.Any{{TypeUrl: insecureCredsTypeURL}}, []*anypb.Any{accessTokenPlugin(t, "test-token")}, nil) + cfg, err := Parse(gs, bootstrapConfig(t, "{}"), trustedServerConfig(t)) + if err != nil { + t.Fatalf("Parse() returned unexpected error: %v", err) + } + defer cfg.Close() + if _, err := cfg.Dial(); err == nil || !strings.Contains(err.Error(), "transport level security") { + t.Fatalf("Dial() returned error %v, want transport security error", err) + } +} + +// Tests that a Config carrying TLS channel credentials and access token call +// credentials, built from a trusted server's GrpcService proto, can dial a +// TLS-serving backend, and that RPCs on the resulting channel succeed and +// carry the token. +func (s) TestConfigDialTLS(t *testing.T) { + // Start a TLS-serving backend that verifies the access token arrives. + const wantAuthorization = "Bearer test-token" + serverCreds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("Failed to create server TLS credentials: %v", err) + } + backend := stubserver.StartTestService(t, &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, fmt.Errorf("no incoming metadata") + } + if got := md.Get("authorization"); len(got) != 1 || got[0] != wantAuthorization { + return nil, fmt.Errorf("authorization metadata = %v, want [%q]", got, wantAuthorization) + } + return &testpb.Empty{}, nil + }, + }, grpc.Creds(serverCreds)) + defer backend.Stop() + + // Bootstrap configuration with a certificate provider instance watching + // the CA certificate that signed the backend's certificate. + contents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: json.RawMessage(`[{"server_uri":"td.googleapis.com:443","channel_creds":[{"type":"insecure"}]}]`), + Node: json.RawMessage(`{}`), + CertificateProviders: map[string]json.RawMessage{ + "root-instance": json.RawMessage(fmt.Sprintf(`{ + "plugin_name": "file_watcher", + "config": {"ca_certificate_file": %q} + }`, testdata.Path("x509/server_ca_cert.pem"))), + }, + }) + if err != nil { + t.Fatalf("NewContentsForTesting() failed: %v", err) + } + bc, err := bootstrap.NewConfigFromContents(contents) + if err != nil { + t.Fatalf("NewConfigFromContents() failed: %v", err) + } + + tlsPlugin, err := anypb.New(&tlscredspb.TlsCredentials{ + RootCertificateProvider: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: "root-instance"}, + }) + if err != nil { + t.Fatalf("Failed to marshal TlsCredentials: %v", err) + } + gs := googleGrpcService(backend.Address, []*anypb.Any{tlsPlugin}, []*anypb.Any{accessTokenPlugin(t, "test-token")}, nil) + cfg, err := Parse(gs, bc, trustedServerConfig(t)) + if err != nil { + t.Fatalf("Parse() returned unexpected error: %v", err) + } + defer cfg.Close() + + // The backend certificate's SANs cover x.test.example.com. + cc, err := cfg.Dial(grpc.WithAuthority("x.test.example.com")) + if err != nil { + t.Fatalf("Dial() returned unexpected error: %v", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} + +func (s) TestConfigEqual(t *testing.T) { + insecurePlugin := &anypb.Any{TypeUrl: insecureCredsTypeURL} + protoInsecure := xdscreds.NewChannelCreds(nil, protoIdentity(insecurePlugin), nil) + jsonInsecure := xdscreds.NewChannelCreds(nil, xdscreds.Identity{Type: "insecure"}, nil) + + tests := []struct { + name string + a, b *Config + want bool + }{ + { + name: "equal", + a: &Config{TargetURI: target, ChannelCredentials: protoInsecure, Timeout: 1, InitialMetadata: metadata.Pairs("k", "v")}, + b: &Config{TargetURI: target, ChannelCredentials: xdscreds.NewChannelCreds(nil, protoIdentity(insecurePlugin), nil), Timeout: 1, InitialMetadata: metadata.Pairs("k", "v")}, + want: true, + }, + { + name: "different_timeouts", + a: &Config{TargetURI: target, ChannelCredentials: protoInsecure, Timeout: 1}, + b: &Config{TargetURI: target, ChannelCredentials: protoInsecure, Timeout: 2}, + want: false, + }, + { + name: "different_initial_metadata", + a: &Config{TargetURI: target, ChannelCredentials: protoInsecure}, + b: &Config{TargetURI: target, ChannelCredentials: protoInsecure, InitialMetadata: metadata.Pairs("k", "v")}, + want: false, + }, + { + name: "different_targets", + a: &Config{TargetURI: target, ChannelCredentials: protoInsecure}, + b: &Config{TargetURI: "dns:///other:443", ChannelCredentials: protoInsecure}, + want: false, + }, + { + name: "different_identity_flavors", + a: &Config{TargetURI: target, ChannelCredentials: protoInsecure}, + b: &Config{TargetURI: target, ChannelCredentials: jsonInsecure}, + want: false, + }, + { + name: "different_call_creds", + a: &Config{TargetURI: target, ChannelCredentials: protoInsecure}, + b: &Config{TargetURI: target, ChannelCredentials: protoInsecure, CallCredentials: []*xdscreds.CallCreds{ + xdscreds.NewCallCreds(nil, xdscreds.Identity{Type: "access_token"}, nil), + }}, + want: false, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := test.a.Equal(test.b); got != test.want { + t.Errorf("Equal() = %v, want %v", got, test.want) + } + }) + } +} diff --git a/internal/xds/httpfilter/ext_authz/config.go b/internal/xds/httpfilter/ext_authz/config.go new file mode 100644 index 0000000..d53c26d --- /dev/null +++ b/internal/xds/httpfilter/ext_authz/config.go @@ -0,0 +1,79 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package extauthz + +import ( + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/xds/grpcservice" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/matcher" +) + +// config contains the configuration for the external authorization filter. +type config struct { + httpfilter.FilterConfig + // grpcService is the configuration for the external authorization server. + grpcService *grpcservice.Config + // filterEnabled specifies the percentage of requests to be authorized by + // the external authorization server. + filterEnabled fraction + // denyAtDisabled specifies whether to deny requests when external + // authorization is disabled via the filterEnabled configuration. If true, + // requests will be denied with a status based on statusOnError. + denyAtDisable bool + // failureModeAllow specifies the behavior when the call to the external + // authorization server fails. If true, the request will be allowed in such + // cases. If false, the request will be denied with a status based on + // statusOnError. + failureModeAllow bool + // failureModeAllowHeaderAdd specifies whether to add the + // `x-envoy-auth-failure-mode-allowed: true` header to the data plane RPC + // when the call to the external authorization server fails. + failureModeAllowHeaderAdd bool + // statusOnError is the gRPC status code to use when the external + // authorization is disabled or when the call to the external authorization + // server fails. + // + // The default value is PERMISSION_DENIED. + statusOnError codes.Code + // allowedHeaders specifies the headers that are allowed to be sent to the + // external authorization server. If unset, all headers are allowed. + allowedHeaders []matcher.StringMatcher + // disallowedHeaders specifies the headers that will not be sent to the + // external authorization server. This takes precedence over the + // allowedHeaders field. + disallowedHeaders []matcher.StringMatcher + // decoderHeaderMutationRules specifies the rules for what modifications an + // external authorization server may make to headers sent on the data plane + // RPC. + decoderHeaderMutationRules httpfilter.HeaderMutationRules + // includePeerCertificate specifies whether to include the peer certificate + // in the request sent to the external authorization server. + includePeerCertificate bool +} + +// fraction uses a numerator and denominator to specify a fractional value. If +// the denominator specified is less than the numerator, the final fractional +// value is capped at 1. +type fraction struct { + // numerator is the numerator of the fraction. + numerator uint32 + // denominator is the denominator of the fraction. + denominator uint32 +} diff --git a/internal/xds/httpfilter/ext_authz/ext_authz.go b/internal/xds/httpfilter/ext_authz/ext_authz.go new file mode 100644 index 0000000..e4fc4da --- /dev/null +++ b/internal/xds/httpfilter/ext_authz/ext_authz.go @@ -0,0 +1,184 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package extauthz implements the xDS External Authorization HTTP filter. +package extauthz + +import ( + "fmt" + "net/http" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/internal/xds/grpcservice" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3extauthzpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +func init() { + if envconfig.XDSClientExtAuthzEnabled { + httpfilter.Register(builder{}) + } +} + +var ( + // TODO: Parse via grpcservice.Parse with the filter parse options, + // as ext_proc does, when ext_authz is wired up for gRFC A102. + parseGRPCServiceConfig = func(*v3corepb.GrpcService) (*grpcservice.Config, error) { + return nil, fmt.Errorf("parseGRPCServiceConfig not implemented") + } +) + +type builder struct{} + +func (builder) TypeURLs() []string { + return []string{ + "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz", + "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute", + } +} + +func parseFilterEnabled(fp *v3corepb.RuntimeFractionalPercent) (fraction, error) { + if fp == nil { + return fraction{numerator: 100, denominator: 100}, nil + } + fracPercent := fp.GetDefaultValue() + if fracPercent == nil { + return fraction{}, fmt.Errorf("extauthz: missing default_value in filter_enabled") + } + + den := uint32(100) + switch fracPercent.GetDenominator() { + case v3typepb.FractionalPercent_TEN_THOUSAND: + den = 10000 + case v3typepb.FractionalPercent_MILLION: + den = 1000000 + } + + // If the numerator exceeds the denominator, cap the fractional value at 100%. + num := min(fracPercent.GetNumerator(), den) + return fraction{numerator: num, denominator: den}, nil +} + +// grpcStatusCode converts an HTTP status code to a gRPC status code. +func grpcStatusCode(httpStatus int32) codes.Code { + if code, ok := transport.HTTPStatusConvTab[int(httpStatus)]; ok { + return code + } + return codes.Unknown +} + +func (builder) ParseFilterConfig(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + m, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("extauthz: error parsing config %v: unknown type %T, want *anypb.Any", cfg, cfg) + } + msg := new(v3extauthzpb.ExtAuthz) + if err := m.UnmarshalTo(msg); err != nil { + return nil, fmt.Errorf("extauthz: failed to unmarshal config: %v", err) + } + + if msg.GetGrpcService() == nil { + return nil, fmt.Errorf("extauthz: empty grpc_service provided in config %v", cfg) + } + server, err := parseGRPCServiceConfig(msg.GetGrpcService()) + if err != nil { + return nil, fmt.Errorf("extauthz: failed to parse grpc_service: %v", err) + } + + filterEnabled, err := parseFilterEnabled(msg.GetFilterEnabled()) + if err != nil { + return nil, err + } + + var denyAtDisable bool + if denyAtDisableFlag := msg.GetDenyAtDisable(); denyAtDisableFlag != nil { + if denyAtDisableFlag.GetDefaultValue() == nil { + return nil, fmt.Errorf("extauthz: missing default_value in deny_at_disable") + } + denyAtDisable = denyAtDisableFlag.GetDefaultValue().GetValue() + } + + httpStatus := int32(http.StatusForbidden) + if st := msg.GetStatusOnError().GetCode(); st != 0 { + httpStatus = int32(st) + } + statusOnError := grpcStatusCode(httpStatus) + + mutationRules, err := httpfilter.HeaderMutationRulesFromProto(msg.GetDecoderHeaderMutationRules()) + if err != nil { + return nil, err + } + + var allowedHeaders, disallowedHeaders []matcher.StringMatcher + if allowed := msg.GetAllowedHeaders(); allowed != nil { + allowedHeaders, err = httpfilter.ConvertStringMatchers(allowed.GetPatterns()) + if err != nil { + return nil, err + } + } + + if disallowed := msg.GetDisallowedHeaders(); disallowed != nil { + disallowedHeaders, err = httpfilter.ConvertStringMatchers(disallowed.GetPatterns()) + if err != nil { + return nil, err + } + } + + return config{ + grpcService: server, + filterEnabled: filterEnabled, + denyAtDisable: denyAtDisable, + failureModeAllow: msg.GetFailureModeAllow(), + failureModeAllowHeaderAdd: msg.GetFailureModeAllowHeaderAdd(), + statusOnError: statusOnError, + allowedHeaders: allowedHeaders, + disallowedHeaders: disallowedHeaders, + decoderHeaderMutationRules: mutationRules, + includePeerCertificate: msg.GetIncludePeerCertificate(), + }, nil +} + +// ParseFilterConfigOverride parses the provided override configuration. +// +// Note that ExtAuthzPerRoute is unmarshaled to verify its syntax during xDS +// resource validation, no filter configuration object is returned. Per-route +// disabling is supported via the generic FilterConfig wrapper mechanism rather +// than the ExtAuthzPerRoute.disabled field directly. +func (builder) ParseFilterConfigOverride(overrideCfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + m, ok := overrideCfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("extauthz: error parsing override config %v: unknown type %T, want *anypb.Any", overrideCfg, overrideCfg) + } + msg := new(v3extauthzpb.ExtAuthzPerRoute) + if err := m.UnmarshalTo(msg); err != nil { + return nil, fmt.Errorf("extauthz: failed to unmarshal override config %v: %v", overrideCfg, err) + } + return nil, nil +} + +func (builder) IsTerminal() bool { + return false +} diff --git a/internal/xds/httpfilter/ext_authz/ext_authz_test.go b/internal/xds/httpfilter/ext_authz/ext_authz_test.go new file mode 100644 index 0000000..cae658e --- /dev/null +++ b/internal/xds/httpfilter/ext_authz/ext_authz_test.go @@ -0,0 +1,439 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package extauthz + +import ( + "fmt" + "regexp" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/grpcservice" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + mutationpb "github.com/envoyproxy/go-control-plane/envoy/config/common/mutation_rules/v3" + corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3extauthzpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3" + matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// testParseGRPCServiceConfig is a helper function that parses a GrpcService +// proto message into a GRPCServiceConfig. This is a temporary test +// implementation that will be removed once gRFC A102 is implemented. +func testParseGRPCServiceConfig(grpcService *corepb.GrpcService) (*grpcservice.Config, error) { + if grpcService.GetGoogleGrpc() == nil { + return nil, fmt.Errorf("only google_grpc grpc_service is supported") + } + if grpcService.GetGoogleGrpc().GetTargetUri() == "" { + return nil, fmt.Errorf("targetURI must be a non-empty string") + } + return &grpcservice.Config{ + TargetURI: grpcService.GetGoogleGrpc().GetTargetUri(), + }, nil +} + +var cmpOpts = []cmp.Option{ + cmp.AllowUnexported( + config{}, + fraction{}, + ), + cmp.Transformer("RegexpToString", func(r *regexp.Regexp) string { + if r == nil { + return "" + } + return r.String() + }), + cmp.Comparer(func(x, y matcher.StringMatcher) bool { + return x.Equal(y) + }), +} + +// Test verifies that ParseFilterConfig successfully parses valid external +// authorization filter configurations into their internal representation. +func (s) TestParseFilterConfig_Success(t *testing.T) { + origParseGRPCServiceConfig := parseGRPCServiceConfig + defer func() { parseGRPCServiceConfig = origParseGRPCServiceConfig }() + parseGRPCServiceConfig = testParseGRPCServiceConfig + + tests := []struct { + name string + desc string + cfg proto.Message + wantCfg httpfilter.FilterConfig + }{ + { + name: "DefaultConfig", + desc: "verifies default fallback values when only grpc_service is provided", + cfg: testutils.MarshalAny(t, &v3extauthzpb.ExtAuthz{ + Services: &v3extauthzpb.ExtAuthz_GrpcService{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:1234", + }, + }, + }, + }, + }), + wantCfg: config{ + grpcService: &grpcservice.Config{ + TargetURI: "localhost:1234", + }, + filterEnabled: fraction{ + numerator: 100, + denominator: 100, + }, + statusOnError: codes.PermissionDenied, + }, + }, + { + name: "FullConfig", + desc: "verifies all config fields are parsed and mapped correctly when explicitly configured", + cfg: testutils.MarshalAny(t, &v3extauthzpb.ExtAuthz{ + Services: &v3extauthzpb.ExtAuthz_GrpcService{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:5678", + }, + }, + }, + }, + FilterEnabled: &corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 50, + Denominator: v3typepb.FractionalPercent_TEN_THOUSAND, + }, + }, + DenyAtDisable: &corepb.RuntimeFeatureFlag{ + DefaultValue: wrapperspb.Bool(true), + }, + FailureModeAllow: true, + FailureModeAllowHeaderAdd: true, + StatusOnError: &v3typepb.HttpStatus{ + Code: v3typepb.StatusCode_Unauthorized, + }, + DecoderHeaderMutationRules: &mutationpb.HeaderMutationRules{ + AllowExpression: &matcherpb.RegexMatcher{Regex: ".*"}, + DisallowExpression: &matcherpb.RegexMatcher{Regex: "a"}, + }, + AllowedHeaders: &matcherpb.ListStringMatcher{ + Patterns: []*matcherpb.StringMatcher{{ + MatchPattern: &matcherpb.StringMatcher_Exact{Exact: "allow-header"}, + }}, + }, + DisallowedHeaders: &matcherpb.ListStringMatcher{ + Patterns: []*matcherpb.StringMatcher{ + { + MatchPattern: &matcherpb.StringMatcher_Exact{Exact: "disallow-header"}, + }, + }, + }, + IncludePeerCertificate: true, + }), + wantCfg: config{ + grpcService: &grpcservice.Config{ + TargetURI: "localhost:5678", + }, + filterEnabled: fraction{ + numerator: 50, + denominator: 10000, + }, + denyAtDisable: true, + failureModeAllow: true, + failureModeAllowHeaderAdd: true, + statusOnError: codes.Unauthenticated, + decoderHeaderMutationRules: httpfilter.HeaderMutationRules{ + AllowExpr: regexp.MustCompile("^(?:.*)$"), + DisallowExpr: regexp.MustCompile("^(?:a)$"), + }, + allowedHeaders: []matcher.StringMatcher{ + matcher.NewExactStringMatcher("allow-header", false), + }, + disallowedHeaders: []matcher.StringMatcher{ + matcher.NewExactStringMatcher("disallow-header", false), + }, + includePeerCertificate: true, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log(tt.desc) + b := builder{} + got, err := b.ParseFilterConfig(tt.cfg, httpfilter.ParseOptions{}) + if err != nil { + t.Fatalf("ParseFilterConfig() failed with unexpected error: %v", err) + } + if diff := cmp.Diff(tt.wantCfg, got, cmpOpts...); diff != "" { + t.Fatalf("ParseFilterConfig() returned unexpected config (-want, +got):\n%s", diff) + } + }) + } +} + +// Test verifies that ParseFilterConfig returns an error when provided with +// invalid or unsupported configurations. +func (s) TestParseFilterConfig_Failure(t *testing.T) { + origParseGRPCServiceConfig := parseGRPCServiceConfig + defer func() { parseGRPCServiceConfig = origParseGRPCServiceConfig }() + parseGRPCServiceConfig = testParseGRPCServiceConfig + + tests := []struct { + name string + desc string + cfg proto.Message + wantErr string + }{ + { + name: "InvalidConfigType", + desc: "verifies error when input message is not a valid proto message", + cfg: &v3extauthzpb.ExtAuthz{}, + wantErr: "extauthz: error parsing config", + }, + { + name: "Config_Unmarshaling_Failed", + desc: "verifies error when input proto message cannot be unmarshaled into an ExtAuthz message", + cfg: &anypb.Any{ + TypeUrl: "type.googleapis.com/invalid", + Value: []byte("invalid"), + }, + wantErr: "extauthz: failed to unmarshal config", + }, + { + name: "MissingGrpcService", + desc: "verifies error when required grpc_service is missing", + cfg: testutils.MarshalAny(t, &v3extauthzpb.ExtAuthz{}), + wantErr: "extauthz: empty grpc_service provided", + }, + { + name: "UnsupportedGrpcService_EnvoyGrpc", + desc: "verifies error when unsupported envoy_grpc service is used", + cfg: testutils.MarshalAny(t, &v3extauthzpb.ExtAuthz{ + Services: &v3extauthzpb.ExtAuthz_GrpcService{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &corepb.GrpcService_EnvoyGrpc{ + ClusterName: "cluster", + }, + }, + }, + }, + }), + wantErr: "extauthz: failed to parse grpc_service: only google_grpc grpc_service is supported", + }, + { + name: "InvalidServerConfig_EmptyTargetURI", + desc: "verifies error when google_grpc has empty target URI", + cfg: testutils.MarshalAny(t, &v3extauthzpb.ExtAuthz{ + Services: &v3extauthzpb.ExtAuthz_GrpcService{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "", + }, + }, + }, + }, + }), + wantErr: "extauthz: failed to parse grpc_service: targetURI must be a non-empty string", + }, + { + name: "MissingDefaultValueInFilterEnabled", + desc: "verifies error when filter_enabled lacks default value", + cfg: testutils.MarshalAny(t, &v3extauthzpb.ExtAuthz{ + Services: &v3extauthzpb.ExtAuthz_GrpcService{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:1234", + }, + }, + }, + }, + FilterEnabled: &corepb.RuntimeFractionalPercent{}, + }), + wantErr: "extauthz: missing default_value in filter_enabled", + }, + { + name: "MissingDefaultValueInDenyAtDisable", + desc: "verifies error when deny_at_disable lacks default value", + cfg: testutils.MarshalAny(t, &v3extauthzpb.ExtAuthz{ + Services: &v3extauthzpb.ExtAuthz_GrpcService{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:1234", + }, + }, + }, + }, + DenyAtDisable: &corepb.RuntimeFeatureFlag{}, + }), + wantErr: "extauthz: missing default_value in deny_at_disable", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log(tt.desc) + b := builder{} + if _, err := b.ParseFilterConfig(tt.cfg, httpfilter.ParseOptions{}); err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("ParseFilterConfig() returned error = %v, wantErr containing %v", err, tt.wantErr) + } + }) + } +} + +// Test verifies that ParseFilterConfigOverride successfully unmarshals valid +// per-route override configurations. +func (s) TestParseFilterConfigOverride_Success(t *testing.T) { + override := testutils.MarshalAny(t, &v3extauthzpb.ExtAuthzPerRoute{ + Override: &v3extauthzpb.ExtAuthzPerRoute_Disabled{ + Disabled: true, + }, + }) + + b := builder{} + got, err := b.ParseFilterConfigOverride(override, httpfilter.ParseOptions{}) + if err != nil { + t.Fatalf("ParseFilterConfigOverride() failed with unexpected error: %v", err) + } + if got != nil { + t.Fatalf("ParseFilterConfigOverride() = %v, want nil", got) + } +} + +// Test verifies that ParseFilterConfigOverride returns an error when provided +// with an invalid proto message type or a malformed Any message. +func (s) TestParseFilterConfigOverride_Failure(t *testing.T) { + tests := []struct { + name string + desc string + override proto.Message + wantErr string + }{ + { + name: "InvalidOverrideType", + desc: "verifies error when input override is not an Any message", + override: &v3extauthzpb.ExtAuthzPerRoute{}, + wantErr: "extauthz: error parsing override config", + }, + { + name: "Unmarshal_Failed", + desc: "verifies error when Any message unmarshal to ExtAuthzPerRoute fails", + override: &anypb.Any{ + TypeUrl: "type.googleapis.com/invalid", + Value: []byte("invalid"), + }, + wantErr: "extauthz: failed to unmarshal override config", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log(tt.desc) + b := builder{} + if _, err := b.ParseFilterConfigOverride(tt.override, httpfilter.ParseOptions{}); err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("ParseFilterConfigOverride() returned error = %v, wantErr containing %v", err, tt.wantErr) + } + }) + } +} + +// Test verifies that parseFilterEnabled correctly parses the +// RuntimeFractionalPercent configuration into its internal representation. +func (s) TestParseFilterEnabled(t *testing.T) { + tests := []struct { + name string + fp *corepb.RuntimeFractionalPercent + want fraction + }{ + { + name: "NilFraction", + fp: nil, + want: fraction{numerator: 100, denominator: 100}, + }, + { + name: "DenominatorHundred", + fp: &corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 10, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }, + want: fraction{numerator: 10, denominator: 100}, + }, + { + name: "DenominatorMillion", + fp: &corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 5, + Denominator: v3typepb.FractionalPercent_MILLION, + }, + }, + want: fraction{numerator: 5, denominator: 1000000}, + }, + { + name: "DefaultDenominator", + fp: &corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 25, + }, + }, + want: fraction{numerator: 25, denominator: 100}, + }, + { + name: "CappedToHundredPercent", + fp: &corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 200, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }, + want: fraction{numerator: 100, denominator: 100}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseFilterEnabled(tt.fp) + if err != nil { + t.Fatalf("parseFilterEnabled(%v) failed: %v", tt.fp, err) + } + if diff := cmp.Diff(tt.want, got, cmp.AllowUnexported(fraction{})); diff != "" { + t.Fatalf("parseFilterEnabled(%v) returned unexpected fraction (-want, +got):\n%s", tt.fp, diff) + } + }) + } +} diff --git a/internal/xds/httpfilter/extconfig.go b/internal/xds/httpfilter/extconfig.go new file mode 100644 index 0000000..cd68714 --- /dev/null +++ b/internal/xds/httpfilter/extconfig.go @@ -0,0 +1,326 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package httpfilter + +import ( + "encoding/base64" + "fmt" + "regexp" + "strings" + + imetadata "google.golang.org/grpc/internal/metadata" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/metadata" + + v3mutationpb "github.com/envoyproxy/go-control-plane/envoy/config/common/mutation_rules/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" +) + +// maxHeaderSize is the maximum length, in bytes, of a header key or value in a +// mutation received from an external processing server. +const maxHeaderSize = 16384 + +// HeaderMutationRules specifies the rules for what modifications an external +// processing server may make to headers sent on the data plane RPC. +type HeaderMutationRules struct { + // AllowExpr specifies a regular expression that matches the headers that can + // be mutated. + AllowExpr *regexp.Regexp + // DisallowExpr specifies a regular expression that matches the headers that + // cannot be mutated. This overrides the above allowExpr if a header matches + // both. + DisallowExpr *regexp.Regexp + // DisallowAll specifies that no header mutations are allowed. This overrides + // all other settings. + DisallowAll bool + // DisallowIsError specifies whether to return an error if a header mutation + // is disallowed. If true, the data plane RPC will be failed with a grpc + // status code of Unknown. + DisallowIsError bool +} + +// ConvertStringMatchers converts a slice of protobuf StringMatcher messages to +// a slice of matcher.StringMatcher. +func ConvertStringMatchers(patterns []*v3matcherpb.StringMatcher) ([]matcher.StringMatcher, error) { + matchers := make([]matcher.StringMatcher, 0, len(patterns)) + for _, p := range patterns { + sm, err := matcher.StringMatcherFromProto(p) + if err != nil { + return nil, err + } + matchers = append(matchers, sm) + } + return matchers, nil +} + +// HeaderMutationRulesFromProto converts a protobuf HeaderMutationRules proto +// message to a HeaderMutationRules struct. +func HeaderMutationRulesFromProto(mr *v3mutationpb.HeaderMutationRules) (HeaderMutationRules, error) { + var rules HeaderMutationRules + if mr == nil { + return rules, nil + } + if allowExpr := mr.GetAllowExpression(); allowExpr != nil { + re, err := matcher.CompileSafeRegex(allowExpr.GetRegex()) + if err != nil { + return rules, fmt.Errorf("httpfilter: %v", err) + } + rules.AllowExpr = re + } + if disallowExpr := mr.GetDisallowExpression(); disallowExpr != nil { + re, err := matcher.CompileSafeRegex(disallowExpr.GetRegex()) + if err != nil { + return rules, fmt.Errorf("httpfilter: %v", err) + } + rules.DisallowExpr = re + } + rules.DisallowAll = mr.GetDisallowAll().GetValue() + rules.DisallowIsError = mr.GetDisallowIsError().GetValue() + return rules, nil +} + +// ApplyAdditions takes a set of header mutations (for additions and +// modifications) received from an external server and applies them to the +// provided metadata, subject to the rules defined in hmr. +// +// If the DisallowAll field is true, no mutations are performed, and the input +// metadata is returned unmodified. +// +// It iterates through each header mutation, performs validation on the header +// key and value, and checks if the mutation is permitted by the AllowExpr and +// DisallowExpr regular expressions. +// +// A mutation for any of the following headers fails validation and an error is +// returned: +// - Pseudo-headers (keys starting with ':'). +// - The 'host' header. +// - Headers with keys in the reserved 'grpc-' space. +// - Headers with non-lowercase keys. +// - Headers with keys or values exceeding 16384 bytes. +// - Headers with an invalid gRPC header name or header value. +// +// If a mutation is disallowed by the mutation rules and DisallowIsError is +// true, an error is returned. Otherwise, the disallowed mutation is silently +// ignored. +// +// The input metadata must not be nil. +func (hmr *HeaderMutationRules) ApplyAdditions(hvos []*v3corepb.HeaderValueOption, input metadata.MD) error { + if hmr == nil { + hmr = &HeaderMutationRules{} + } + if input == nil { + return fmt.Errorf("input metadata is nil") + } + if hmr.DisallowAll { + return nil + } + + for _, hvo := range hvos { + header := hvo.GetHeader() + key := header.GetKey() + if err := validateHeaderKey(key); err != nil { + return fmt.Errorf("invalid header mutation: %v", err) + } + + value := header.GetValue() + if strings.HasSuffix(key, "-bin") { + value = string(header.GetRawValue()) + } + if len(value) > maxHeaderSize { + return fmt.Errorf("invalid header mutation: value for header key %q exceeds the maximum length of %d bytes", key, maxHeaderSize) + } + // ValidatePair rejects values carrying bytes outside %x20-%x7E. It + // skips the value check for "-bin" keys, whose values the transport + // base64 encodes. + if err := imetadata.ValidatePair(key, value); err != nil { + return fmt.Errorf("invalid header mutation: %v", err) + } + + if !hmr.allow(key) { + if hmr.DisallowIsError { + return fmt.Errorf("header mutation disallowed by headerMutationRules for header key %q", key) + } + continue + } + + // Perform the mutation on output metadata using the append_action + // field from the header value option. + switch hvo.GetAppendAction() { + case v3corepb.HeaderValueOption_APPEND_IF_EXISTS_OR_ADD: + input.Append(key, value) + case v3corepb.HeaderValueOption_ADD_IF_ABSENT: + if input.Get(key) == nil { + input.Set(key, value) + } + case v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD: + input.Set(key, value) + case v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS: + if input.Get(key) != nil { + input.Set(key, value) + } + } + } + return nil +} + +// ApplyRemovals takes a set of headers (for removal) received from an external +// processing server and applies them to the provided metadata, subject to the +// rules defined in hmr. +// +// This method is very similar to ApplyAdditions, except that headers are +// removed here instead of added or mutated as is the case in the latter. See +// ApplyAdditions for more details. +// +// The input metadata must not be nil. +func (hmr *HeaderMutationRules) ApplyRemovals(headersToRemove []string, input metadata.MD) error { + if hmr == nil { + hmr = &HeaderMutationRules{} + } + if input == nil { + return fmt.Errorf("input metadata is nil") + } + if hmr.DisallowAll { + return nil + } + + for _, header := range headersToRemove { + if err := validateHeaderKey(header); err != nil { + return fmt.Errorf("invalid header mutation: %v", err) + } + if !hmr.allow(header) { + if hmr.DisallowIsError { + return fmt.Errorf("header mutation disallowed by headerMutationRules for header %q", header) + } + continue + } + input.Delete(header) + } + return nil +} + +// validateHeaderKey returns a non-nil error if key may not be mutated by an +// external processing server, either because the key is reserved or because it +// is not a valid gRPC header name. +func validateHeaderKey(key string) error { + switch { + case len(key) == 0: + return fmt.Errorf("header key is empty") + case key[0] == ':': + return fmt.Errorf("header key %q is a pseudo-header", key) + case key == "host": + return fmt.Errorf("header key %q is reserved", key) + case strings.HasPrefix(key, "grpc-"): + return fmt.Errorf("header key %q is in the reserved 'grpc-' space", key) + case key != strings.ToLower(key): + return fmt.Errorf("header key %q is not lowercase", key) + case len(key) > maxHeaderSize: + return fmt.Errorf("header key exceeds the maximum length of %d bytes", maxHeaderSize) + } + return imetadata.ValidateKey(key) +} + +func (hmr *HeaderMutationRules) allow(key string) bool { + if hmr.DisallowExpr != nil && hmr.DisallowExpr.MatchString(key) { + return false + } + if hmr.AllowExpr != nil && hmr.AllowExpr.MatchString(key) { + return true + } + if hmr.AllowExpr != nil { + return false + } + return true +} + +// ConstructHeaderMap constructs a HeaderMap from the given metadata and raw +// appended metadata slice, using the following rules: +// - if the header is matched by the disallowed_headers config field, it will +// not be added to the map, otherwise, +// - if the allowed_headers config field is unset or matches the header, the +// header will be added to the map, otherwise, +// - the header will be excluded from the map. +func ConstructHeaderMap(md metadata.MD, added [][]string, allowedHeaders, disallowedHeaders []matcher.StringMatcher) *v3corepb.HeaderMap { + headerMap := &v3corepb.HeaderMap{} + // Process the base metadata map. + for key, values := range md { + if isDisallowedHeader(key, disallowedHeaders) { + continue + } + if isAllowedHeader(key, allowedHeaders) { + for _, value := range values { + headerMap.Headers = append(headerMap.Headers, constructHeader(key, value)) + } + } + } + // Process the raw appended metadata slice. + for _, kvs := range added { + for i := 0; i < len(kvs); i += 2 { + key := strings.ToLower(kvs[i]) + if isDisallowedHeader(key, disallowedHeaders) { + continue + } + if isAllowedHeader(key, allowedHeaders) { + headerMap.Headers = append(headerMap.Headers, constructHeader(key, kvs[i+1])) + } + } + } + if len(headerMap.Headers) == 0 { + return nil + } + return headerMap +} + +func constructHeader(key, value string) *v3corepb.HeaderValue { + rawValue := []byte(value) + if strings.HasSuffix(key, "-bin") { + encoded := make([]byte, base64.StdEncoding.EncodedLen(len(rawValue))) + base64.StdEncoding.Encode(encoded, rawValue) + rawValue = encoded + } + return &v3corepb.HeaderValue{ + Key: key, + RawValue: rawValue, + } +} + +// isDisallowedHeader returns true if the given header key matches any of the +// provided disallowed header matchers. +func isDisallowedHeader(key string, matchers []matcher.StringMatcher) bool { + for _, m := range matchers { + if m.Match(key) { + return true + } + } + return false +} + +// isAllowedHeader returns true if the allowed header matchers list is empty, or +// if the given header key matches any of the provided allowed header matchers. +func isAllowedHeader(key string, matchers []matcher.StringMatcher) bool { + if len(matchers) == 0 { + return true + } + for _, m := range matchers { + if m.Match(key) { + return true + } + } + return false +} diff --git a/internal/xds/httpfilter/extconfig_test.go b/internal/xds/httpfilter/extconfig_test.go new file mode 100644 index 0000000..8b6973e --- /dev/null +++ b/internal/xds/httpfilter/extconfig_test.go @@ -0,0 +1,712 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package httpfilter + +import ( + "regexp" + "sort" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3mutationpb "github.com/envoyproxy/go-control-plane/envoy/config/common/mutation_rules/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestHeaderMutationRulesFromProto_HappyPath(t *testing.T) { + mr := &v3mutationpb.HeaderMutationRules{ + AllowExpression: &v3matcherpb.RegexMatcher{Regex: "^allow$"}, + DisallowExpression: &v3matcherpb.RegexMatcher{Regex: "^disallow$"}, + DisallowAll: wrapperspb.Bool(true), + DisallowIsError: wrapperspb.Bool(false), + } + want := HeaderMutationRules{ + AllowExpr: regexp.MustCompile("^(?:^allow$)$"), + DisallowExpr: regexp.MustCompile("^(?:^disallow$)$"), + DisallowAll: true, + DisallowIsError: false, + } + + got, err := HeaderMutationRulesFromProto(mr) + if err != nil { + t.Fatalf("HeaderMutationRulesFromProto() unexpected error: %v", err) + } + if diff := cmp.Diff(got, want, + cmp.Transformer("RegexpToString", func(r *regexp.Regexp) string { + if r == nil { + return "" + } + return r.String() + }), + ); diff != "" { + t.Fatalf("HeaderMutationRulesFromProto() mismatch (-got +want):\n%s", diff) + } +} + +func (s) TestHeaderMutationRulesFromProto_Errors(t *testing.T) { + tests := []struct { + name string + mutationRules *v3mutationpb.HeaderMutationRules + wantErr string + }{ + { + name: "invalid allow expression", + mutationRules: &v3mutationpb.HeaderMutationRules{ + AllowExpression: &v3matcherpb.RegexMatcher{Regex: "["}, + }, + wantErr: "httpfilter: error parsing regexp", + }, + { + name: "invalid disallow expression", + mutationRules: &v3mutationpb.HeaderMutationRules{ + DisallowExpression: &v3matcherpb.RegexMatcher{Regex: "["}, + }, + wantErr: "httpfilter: error parsing regexp", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := HeaderMutationRulesFromProto(tt.mutationRules) + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("HeaderMutationRulesFromProto() error = %v, wantErr %q", err, tt.wantErr) + } + }) + } +} + +func (s) TestHeaderMutationRules_ApplyAdditons(t *testing.T) { + tests := []struct { + name string + hmr *HeaderMutationRules + hvos []*v3corepb.HeaderValueOption + inputMD metadata.MD + wantMD metadata.MD + wantErr bool + }{ + { + name: "NilReceiver", + hmr: nil, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantMD: metadata.MD{"a": []string{"1"}}, + }, + { + name: "DisallowAll", + hmr: &HeaderMutationRules{DisallowAll: true}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{"x": []string{"y"}}, + wantMD: metadata.MD{"x": []string{"y"}}, + }, + { + name: "DisallowExprMatchAndDisallowIsErrorIsFalse", + hmr: &HeaderMutationRules{DisallowExpr: regexp.MustCompile("^a$")}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantMD: metadata.MD{}, + }, + { + name: "DisallowExprMatchAndDisallowIsErrorIsTrue", + hmr: &HeaderMutationRules{DisallowExpr: regexp.MustCompile("^a$"), DisallowIsError: true}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantErr: true, + }, + { + name: "AllowExprMatch", + hmr: &HeaderMutationRules{AllowExpr: regexp.MustCompile("^a$")}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + {Header: &v3corepb.HeaderValue{Key: "b", Value: "2"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantMD: metadata.MD{"a": []string{"1"}}, + }, + { + name: "AllowExprNoMatch", + hmr: &HeaderMutationRules{AllowExpr: regexp.MustCompile("^a$")}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "b", Value: "2"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantMD: metadata.MD{}, + }, + { + name: "DisallowOverridesAllow", + hmr: &HeaderMutationRules{AllowExpr: regexp.MustCompile("."), DisallowExpr: regexp.MustCompile("^a$")}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + {Header: &v3corepb.HeaderValue{Key: "b", Value: "2"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantMD: metadata.MD{"b": []string{"2"}}, + }, + { + name: "InvalidHeaderKeyEmpty", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyPseudoHeader", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: ":path", Value: "/"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyHost", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "host", Value: "example.com"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyNotLowercase", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "A", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyTooLong", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: strings.Repeat("a", 16385), Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyGRPCPrefix", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "grpc-accept-encoding", Value: "identity"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyCRLF", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "x-user\r\nauthorization", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyWithSpace", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "x user", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantErr: true, + }, + { + name: "InvalidHeaderValueCRLF", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "x-user", Value: "alice\r\nauthorization: Bearer token"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantErr: true, + }, + { + name: "InvalidHeaderValueNonPrintable", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "x-nul", Value: "a\x00b"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantErr: true, + }, + { + name: "InvalidHeaderValueTooLong", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: strings.Repeat("1", 16385)}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantErr: true, + }, + { + name: "InvalidBinaryHeaderValueTooLong", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a-bin", RawValue: make([]byte, 16385)}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantErr: true, + }, + { + name: "BinaryHeaderValue", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a-bin", RawValue: []byte{1, 2, 3}}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantMD: metadata.MD{"a-bin": []string{string([]byte{1, 2, 3})}}, + }, + { + name: "AppendIfExistsOrAdd_Exists", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_APPEND_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{"a": []string{"0"}}, + wantMD: metadata.MD{"a": []string{"0", "1"}}, + }, + { + name: "AppendIfExistsOrAdd_Add", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_APPEND_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantMD: metadata.MD{"a": []string{"1"}}, + }, + { + name: "AddIfAbsent_Absent", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_ADD_IF_ABSENT}, + }, + inputMD: metadata.MD{}, + wantMD: metadata.MD{"a": []string{"1"}}, + }, + { + name: "AddIfAbsent_Present", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_ADD_IF_ABSENT}, + }, + inputMD: metadata.MD{"a": []string{"1"}}, + wantMD: metadata.MD{"a": []string{"1"}}, + }, + { + name: "OverwriteIfExistsOrAdd_Absent", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{}, + wantMD: metadata.MD{"a": []string{"1"}}, + }, + { + name: "OverwriteIfExistsOrAdd_Present", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD}, + }, + inputMD: metadata.MD{"a": []string{"0"}}, + wantMD: metadata.MD{"a": []string{"1"}}, + }, + { + name: "OverwriteIfExists_Absent", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS}, + }, + inputMD: metadata.MD{}, + wantMD: metadata.MD{}, + }, + { + name: "OverwriteIfExists_Present", + hmr: &HeaderMutationRules{}, + hvos: []*v3corepb.HeaderValueOption{ + {Header: &v3corepb.HeaderValue{Key: "a", Value: "1"}, AppendAction: v3corepb.HeaderValueOption_OVERWRITE_IF_EXISTS}, + }, + inputMD: metadata.MD{"a": []string{"0"}}, + wantMD: metadata.MD{"a": []string{"1"}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.hmr.ApplyAdditions(tt.hvos, tt.inputMD); (err != nil) != tt.wantErr { + t.Fatalf("ApplyAdditions() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if diff := cmp.Diff(tt.wantMD, tt.inputMD); diff != "" { + t.Fatalf("ApplyAdditions() returned diff in metadata (-want +got):\n%s", diff) + } + }) + } +} + +func (s) TestHeaderMutationRules_ApplyRemovals(t *testing.T) { + tests := []struct { + name string + hmr *HeaderMutationRules + headersToRemove []string + inputMD metadata.MD + wantMD metadata.MD + wantErr bool + }{ + { + name: "NilReceiver", + hmr: nil, + headersToRemove: []string{"a"}, + inputMD: metadata.MD{"a": []string{"1"}}, + wantMD: metadata.MD{}, + }, + { + name: "DisallowAll", + hmr: &HeaderMutationRules{DisallowAll: true}, + headersToRemove: []string{"a"}, + inputMD: metadata.MD{"a": []string{"1"}}, + wantMD: metadata.MD{"a": []string{"1"}}, + }, + { + name: "DisallowExprMatchAndDisallowIsErrorIsFalse", + hmr: &HeaderMutationRules{DisallowExpr: regexp.MustCompile("^a$")}, + headersToRemove: []string{"a"}, + inputMD: metadata.MD{"a": []string{"1"}}, + wantMD: metadata.MD{"a": []string{"1"}}, + }, + { + name: "DisallowExprMatchAndDisallowIsErrorIsTrue", + hmr: &HeaderMutationRules{DisallowExpr: regexp.MustCompile("^a$"), DisallowIsError: true}, + headersToRemove: []string{"a"}, + inputMD: metadata.MD{"a": []string{"1"}}, + wantErr: true, + }, + { + name: "AllowExprMatch", + hmr: &HeaderMutationRules{AllowExpr: regexp.MustCompile("^a$")}, + headersToRemove: []string{"a", "b"}, + inputMD: metadata.MD{"a": []string{"1"}, "b": []string{"2"}}, + wantMD: metadata.MD{"b": []string{"2"}}, + }, + { + name: "AllowExprNoMatch", + hmr: &HeaderMutationRules{AllowExpr: regexp.MustCompile("^a$")}, + headersToRemove: []string{"b"}, + inputMD: metadata.MD{"a": []string{"1"}, "b": []string{"2"}}, + wantMD: metadata.MD{"a": []string{"1"}, "b": []string{"2"}}, + }, + { + name: "DisallowOverridesAllow", + hmr: &HeaderMutationRules{AllowExpr: regexp.MustCompile("."), DisallowExpr: regexp.MustCompile("^a$")}, + headersToRemove: []string{"a", "b"}, + inputMD: metadata.MD{"a": []string{"1"}, "b": []string{"2"}}, + wantMD: metadata.MD{"a": []string{"1"}}, + }, + { + name: "InvalidHeaderKeyEmpty", + hmr: &HeaderMutationRules{}, + headersToRemove: []string{""}, + inputMD: metadata.MD{"a": []string{"1"}}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyPseudoHeader", + hmr: &HeaderMutationRules{}, + headersToRemove: []string{":path"}, + inputMD: metadata.MD{":path": []string{"/"}}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyHost", + hmr: &HeaderMutationRules{}, + headersToRemove: []string{"host"}, + inputMD: metadata.MD{"host": []string{"example.com"}}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyNotLowercase", + hmr: &HeaderMutationRules{}, + headersToRemove: []string{"A"}, + inputMD: metadata.MD{"A": []string{"1"}}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyTooLong", + hmr: &HeaderMutationRules{}, + headersToRemove: []string{strings.Repeat("a", 16385)}, + inputMD: metadata.MD{strings.Repeat("a", 16385): []string{"1"}}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyGRPCPrefix", + hmr: &HeaderMutationRules{}, + headersToRemove: []string{"grpc-accept-encoding"}, + inputMD: metadata.MD{"grpc-accept-encoding": []string{"gzip"}}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyCRLF", + hmr: &HeaderMutationRules{}, + headersToRemove: []string{"x-user\r\nauthorization"}, + inputMD: metadata.MD{"x-user": []string{"1"}}, + wantErr: true, + }, + { + name: "InvalidHeaderKeyWithSpace", + hmr: &HeaderMutationRules{}, + headersToRemove: []string{"x user"}, + inputMD: metadata.MD{"x user": []string{"1"}}, + wantErr: true, + }, + { + name: "HeaderExists", + hmr: &HeaderMutationRules{}, + headersToRemove: []string{"a"}, + inputMD: metadata.MD{"a": []string{"1"}}, + wantMD: metadata.MD{}, + }, + { + name: "HeaderDoesNotExist", + hmr: &HeaderMutationRules{}, + headersToRemove: []string{"a"}, + inputMD: metadata.MD{"b": []string{"2"}}, + wantMD: metadata.MD{"b": []string{"2"}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.hmr.ApplyRemovals(tt.headersToRemove, tt.inputMD); (err != nil) != tt.wantErr { + t.Fatalf("ApplyRemovals() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if diff := cmp.Diff(tt.wantMD, tt.inputMD); diff != "" { + t.Fatalf("ApplyRemovals() returned diff in metadata (-want +got):\n%s", diff) + } + }) + } +} + +func (s) TestConstructHeaderMap(t *testing.T) { + tests := []struct { + name string + md metadata.MD + added [][]string + allowedHeaders []matcher.StringMatcher + disallowedHeaders []matcher.StringMatcher + wantHeaderMap *v3corepb.HeaderMap + }{ + { + name: "NoHeaders", + md: metadata.MD{}, + wantHeaderMap: nil, + }, + { + name: "NoAllowedHeaders", + md: metadata.MD{ + "a": {"1"}, + "b": {"2"}, + }, + wantHeaderMap: &v3corepb.HeaderMap{ + Headers: []*v3corepb.HeaderValue{ + {Key: "a", RawValue: []byte("1")}, + {Key: "b", RawValue: []byte("2")}, + }, + }, + }, + { + name: "WithMultipleValues_ForSingleHeader", + md: metadata.MD{ + "a": {"1", "11"}, + "b": {"2"}, + }, + wantHeaderMap: &v3corepb.HeaderMap{ + Headers: []*v3corepb.HeaderValue{ + {Key: "a", RawValue: []byte("1")}, + {Key: "a", RawValue: []byte("11")}, + {Key: "b", RawValue: []byte("2")}, + }, + }, + }, + { + name: "WithAllowedHeaders", + md: metadata.MD{ + "a": {"1"}, + "b": {"2"}, + "c": {"3"}, + }, + allowedHeaders: []matcher.StringMatcher{ + matcher.NewExactStringMatcher("a", false), + matcher.NewPrefixStringMatcher("c", false), + }, + wantHeaderMap: &v3corepb.HeaderMap{ + Headers: []*v3corepb.HeaderValue{ + {Key: "a", RawValue: []byte("1")}, + {Key: "c", RawValue: []byte("3")}, + }, + }, + }, + { + name: "WithDisallowedHeaders", + md: metadata.MD{ + "a": {"1"}, + "b": {"2"}, + "c": {"3"}, + }, + disallowedHeaders: []matcher.StringMatcher{ + matcher.NewExactStringMatcher("b", false), + matcher.NewPrefixStringMatcher("c", false), + }, + wantHeaderMap: &v3corepb.HeaderMap{ + Headers: []*v3corepb.HeaderValue{ + {Key: "a", RawValue: []byte("1")}, + }, + }, + }, + { + name: "WithAllowedAndDisallowedHeaders", + md: metadata.MD{ + "a": {"1"}, + "b": {"2"}, + "c": {"3"}, + "d": {"4"}, + }, + allowedHeaders: []matcher.StringMatcher{ + matcher.NewExactStringMatcher("a", false), + matcher.NewPrefixStringMatcher("b", false), + matcher.NewPrefixStringMatcher("c", false), + }, + disallowedHeaders: []matcher.StringMatcher{ + matcher.NewExactStringMatcher("b", false), + matcher.NewPrefixStringMatcher("c", false), + }, + wantHeaderMap: &v3corepb.HeaderMap{ + Headers: []*v3corepb.HeaderValue{ + {Key: "a", RawValue: []byte("1")}, + }, + }, + }, + { + name: "WithAddedHeaders", + added: [][]string{ + {"a", "1", "b", "2"}, + {"c", "3"}, + }, + wantHeaderMap: &v3corepb.HeaderMap{ + Headers: []*v3corepb.HeaderValue{ + {Key: "a", RawValue: []byte("1")}, + {Key: "b", RawValue: []byte("2")}, + {Key: "c", RawValue: []byte("3")}, + }, + }, + }, + { + name: "WithAddedAndBaseHeaders", + md: metadata.MD{ + "a": {"1"}, + }, + added: [][]string{ + {"B", "2"}, // Lowercase check + }, + wantHeaderMap: &v3corepb.HeaderMap{ + Headers: []*v3corepb.HeaderValue{ + {Key: "a", RawValue: []byte("1")}, + {Key: "b", RawValue: []byte("2")}, + }, + }, + }, + { + name: "WithAddedHeadersAndFiltering", + added: [][]string{ + {"a", "1", "b", "2", "c", "3"}, + }, + allowedHeaders: []matcher.StringMatcher{ + matcher.NewExactStringMatcher("a", false), + matcher.NewExactStringMatcher("b", false), + }, + disallowedHeaders: []matcher.StringMatcher{ + matcher.NewExactStringMatcher("b", false), + }, + wantHeaderMap: &v3corepb.HeaderMap{ + Headers: []*v3corepb.HeaderValue{ + {Key: "a", RawValue: []byte("1")}, + }, + }, + }, + { + name: "WithBinaryHeaders", + md: metadata.MD{ + "a-bin": {"\x00\x01\x02"}, + "b": {"1"}, + }, + added: [][]string{ + {"c-bin", "\x03\x04\x05"}, + }, + wantHeaderMap: &v3corepb.HeaderMap{ + Headers: []*v3corepb.HeaderValue{ + {Key: "a-bin", RawValue: []byte("AAEC")}, + {Key: "b", RawValue: []byte("1")}, + {Key: "c-bin", RawValue: []byte("AwQF")}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotHeaderMap := ConstructHeaderMap(tt.md, tt.added, tt.allowedHeaders, tt.disallowedHeaders) + if gotHeaderMap != nil { + sort.Slice(gotHeaderMap.Headers, func(i, j int) bool { + return gotHeaderMap.Headers[i].Key < gotHeaderMap.Headers[j].Key + }) + } + if diff := cmp.Diff(tt.wantHeaderMap, gotHeaderMap, protocmp.Transform()); diff != "" { + t.Fatalf("constructHeaderMap() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/internal/xds/httpfilter/extproc/config.go b/internal/xds/httpfilter/extproc/config.go new file mode 100644 index 0000000..bc5fd42 --- /dev/null +++ b/internal/xds/httpfilter/extproc/config.go @@ -0,0 +1,184 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package extproc + +import ( + "time" + + "google.golang.org/grpc/internal/optional" + "google.golang.org/grpc/internal/xds/grpcservice" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/matcher" + + v3procfilterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3" +) + +// baseConfig contains the configuration for the external processing +// interceptor. +type baseConfig struct { + httpfilter.FilterConfig + + // The following fields can be set either in the filter config or the override + // config. If both are set, the override config will be used. + + // server is the configuration for the external processing server. + server *grpcservice.Config + // processingModes specifies the processing mode for each dataplane event. + processingModes processingModes + // failureModeAllow specifies the behavior when the RPC to the external + // processing server fails. If true, the dataplane RPC will be allowed to + // continue. If false, the data plane RPC will be failed with a grpc status + // code of INTERNAL. + failureModeAllow bool + // Attributes to be sent to the external processing server along with the + // request and response dataplane events. + requestAttributes []string + responseAttributes []string + + // The following fields can only be set in the base config. + + // mutationRules specifies the rules for what modifications an external + // processing server may make to headers/trailers sent to it. + mutationRules httpfilter.HeaderMutationRules + // allowedHeaders specifies the headers that are allowed to be sent to the + // external processing server. If unset, all headers are allowed. + allowedHeaders []matcher.StringMatcher + // disallowedHeaders specifies the headers that will not be sent to the + // external processing server. This overrides the above allowedHeaders if a + // header matches both. + disallowedHeaders []matcher.StringMatcher + // disableImmediateResponse specifies whether to disable immediate responses + // from the external processing server. When true, any immediate response + // received from the server is treated as an error (causing the stream to fail + // with an Internal error, or to be bypassed if failureModeAllow is true). + // When false, the immediate response is honored, and the stream is aborted + // with the specified status code and details provided by the external + // processing server. + disableImmediateResponse bool + // observabilityMode determines if the filter waits for the external + // processing server. If true, events are sent to the server in + // "observation-only" mode; the filter does not wait for a response. If false, + // the filter waits for a response, allowing the server to modify events + // before they reach the dataplane. + observabilityMode bool + // deferredCloseTimeout is the duration the filter waits before closing the + // external processing stream after the dataplane RPC completes. This is only + // applicable when observabilityMode is true; otherwise, it is ignored. The + // default value is 5 seconds. + deferredCloseTimeout time.Duration +} + +// overrideConfig contains the configuration for the external processing +// interceptor used for overriding the base config. If a particular field is +// set, that will be used instead of the base config. The fields are similar to +// base config. +type overrideConfig struct { + httpfilter.FilterConfig + server optional.Optional[*grpcservice.Config] + processingModes optional.Optional[processingModes] + failureModeAllow optional.Optional[bool] + requestAttributes []string + responseAttributes []string +} + +// processingMode defines how headers, trailers, and bodies are handled in +// relation to the external processing server. +type processingMode int + +const ( + // modeSkip indicates that the header/trailer/body should not be sent. + modeSkip processingMode = iota + // modeSend indicates that the header/trailer/body should be sent. + modeSend +) + +type processingModes struct { + requestHeaderMode processingMode + requestBodyMode processingMode + responseHeaderMode processingMode + responseTrailerMode processingMode + responseBodyMode processingMode +} + +// resolveHeaderMode resolves the processing mode for headers based on the +// protobuf enum value. If the mode is not set or set to Default processing +// mode, it returns the provided defaultMode. +func resolveHeaderMode(mode v3procfilterpb.ProcessingMode_HeaderSendMode, defaultMode processingMode) processingMode { + switch mode { + case v3procfilterpb.ProcessingMode_SEND: + return modeSend + case v3procfilterpb.ProcessingMode_SKIP: + return modeSkip + case v3procfilterpb.ProcessingMode_DEFAULT: + return defaultMode + default: + return defaultMode + } +} + +// resolveBodyMode resolves the processing mode for body based on the protobuf +// enum value. If the mode is not set (i.e., default), it returns modeSkip, as +// the default for body is to skip. +func resolveBodyMode(mode v3procfilterpb.ProcessingMode_BodySendMode) processingMode { + switch mode { + case v3procfilterpb.ProcessingMode_GRPC: + return modeSend + case v3procfilterpb.ProcessingMode_NONE: + return modeSkip + default: + return modeSkip + } +} + +// processingModesFromProto converts a protobuf ProcessingMode message +// to a processingModes struct. +func processingModesFromProto(pm *v3procfilterpb.ProcessingMode) processingModes { + return processingModes{ + requestHeaderMode: resolveHeaderMode(pm.GetRequestHeaderMode(), modeSend), + requestBodyMode: resolveBodyMode(pm.GetRequestBodyMode()), + responseHeaderMode: resolveHeaderMode(pm.GetResponseHeaderMode(), modeSend), + responseBodyMode: resolveBodyMode(pm.GetResponseBodyMode()), + responseTrailerMode: resolveHeaderMode(pm.GetResponseTrailerMode(), modeSkip), + } +} + +// newInterceptorConfig creates the interceptor config from the base and +// override filter configs. If a field is set in both the base and override +// configs, the value from the override config will be used. +func newInterceptorConfig(base baseConfig, override overrideConfig) baseConfig { + ic := base + + // Apply overrides if present. + if val, ok := override.server.Value(); ok { + ic.server = val + } + if val, ok := override.failureModeAllow.Value(); ok { + ic.failureModeAllow = val + } + if override.requestAttributes != nil { + ic.requestAttributes = override.requestAttributes + } + if override.responseAttributes != nil { + ic.responseAttributes = override.responseAttributes + } + if val, ok := override.processingModes.Value(); ok { + ic.processingModes = val + } + return ic +} diff --git a/internal/xds/httpfilter/extproc/config_test.go b/internal/xds/httpfilter/extproc/config_test.go new file mode 100644 index 0000000..0a01341 --- /dev/null +++ b/internal/xds/httpfilter/extproc/config_test.go @@ -0,0 +1,980 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package extproc + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/optional" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/bootstrap" + xdscreds "google.golang.org/grpc/internal/xds/credentials" + "google.golang.org/grpc/internal/xds/grpcservice" + "google.golang.org/grpc/internal/xds/httpfilter" + iextproc "google.golang.org/grpc/internal/xds/httpfilter/extproc/internal" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + mutationpb "github.com/envoyproxy/go-control-plane/envoy/config/common/mutation_rules/v3" + corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + fpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3" + matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const testBaseURI = "base-uri" + +// allowlistInsecureCreds carries the identity of the insecure channel +// credentials configured for allowlisted targets by testParseOptions; want +// configs compare against it by identity. It carries a real insecure bundle +// so that configs built from it can be dialed. +var allowlistInsecureCreds = xdscreds.NewChannelCreds(insecure.NewBundle(), xdscreds.Identity{Type: "insecure"}, nil) + +// testPerRPCCreds is a no-op PerRPCCredentials for fixtures whose configs are +// dialed for real. +type testPerRPCCreds struct{} + +func (testPerRPCCreds) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { + return nil, nil +} + +func (testPerRPCCreds) RequireTransportSecurity() bool { return false } + +// testParseOptions returns ParseOptions whose bootstrap +// configuration allowlists the given side-channel targets with insecure +// channel credentials. The returned options carry a ServerConfig without the +// trusted_xds_server feature, so the delivering server is untrusted and +// GrpcService parsing takes the allowed_grpc_services path. +func testParseOptions(t *testing.T, targets ...string) httpfilter.ParseOptions { + t.Helper() + + // The allowed_grpc_services bootstrap field is parsed only when a + // consuming feature is enabled. + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true) + + allowed := make(map[string]json.RawMessage, len(targets)) + for _, target := range targets { + allowed[target] = json.RawMessage(`{"channel_creds": [{"type": "insecure"}]}`) + } + allowedJSON, err := json.Marshal(allowed) + if err != nil { + t.Fatalf("Failed to marshal allowed_grpc_services: %v", err) + } + contents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(`[{"server_uri": "passthrough:///unused", "channel_creds": [{"type": "insecure"}]}]`), + Node: []byte(`{"id": "test-node"}`), + AllowedGRPCServices: allowedJSON, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap contents: %v", err) + } + config, err := bootstrap.NewConfigFromContents(contents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %v", err) + } + untrustedServer, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: "untrusted-server:1234"}) + if err != nil { + t.Fatalf("ServerConfigForTesting() failed: %v", err) + } + return httpfilter.ParseOptions{BootstrapConfig: config, ServerConfig: untrustedServer} +} + +// overrideCreateExtProcChannelWithError overrides the channel creation seam +// to fail channel creation. The override is restored when the test ends. +func overrideCreateExtProcChannelWithError(t *testing.T) { + t.Helper() + origCreateExtProcChannel := iextproc.CreateExtProcChannel + iextproc.CreateExtProcChannel = func(*grpcservice.Config) (grpc.ClientConnInterface, func(), error) { + return nil, nil, fmt.Errorf("dial error") + } + t.Cleanup(func() { iextproc.CreateExtProcChannel = origCreateExtProcChannel }) +} + +// Tests that channel sharing considers the target and the credential +// identities, and ignores the per-RPC timeout and initial metadata. +func (s) TestSharesChannel(t *testing.T) { + insecureCreds := func() *xdscreds.ChannelCreds { + return xdscreds.NewChannelCreds(nil, xdscreds.Identity{Type: "insecure"}, nil) + } + const target = "dns:///proc-server:443" + + tests := []struct { + name string + a, b *grpcservice.Config + want bool + }{ + { + name: "equal_identities_share_despite_timeout_and_metadata", + a: &grpcservice.Config{TargetURI: target, ChannelCredentials: insecureCreds(), Timeout: time.Second}, + b: &grpcservice.Config{TargetURI: target, ChannelCredentials: insecureCreds(), InitialMetadata: metadata.Pairs("k", "v")}, + want: true, + }, + { + name: "different_targets", + a: &grpcservice.Config{TargetURI: target, ChannelCredentials: insecureCreds()}, + b: &grpcservice.Config{TargetURI: "dns:///other:443", ChannelCredentials: insecureCreds()}, + want: false, + }, + { + name: "different_channel_creds", + a: &grpcservice.Config{TargetURI: target, ChannelCredentials: insecureCreds()}, + b: &grpcservice.Config{TargetURI: target, ChannelCredentials: xdscreds.NewChannelCreds(nil, xdscreds.Identity{Type: "other"}, nil)}, + want: false, + }, + { + name: "different_call_creds", + a: &grpcservice.Config{TargetURI: target, ChannelCredentials: insecureCreds()}, + b: &grpcservice.Config{TargetURI: target, ChannelCredentials: insecureCreds(), CallCredentials: []*xdscreds.CallCreds{ + xdscreds.NewCallCreds(nil, xdscreds.Identity{Type: "access_token"}, nil), + }}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := sharesChannel(tt.a, tt.b); got != tt.want { + t.Errorf("sharesChannel() = %v, want %v", got, tt.want) + } + }) + } +} + +var cmpOpts = []cmp.Option{ + cmp.AllowUnexported( + baseConfig{}, + overrideConfig{}, + processingModes{}, + optional.Optional[*grpcservice.Config]{}, + optional.Optional[processingModes]{}, + optional.Optional[bool]{}, + ), + cmp.Transformer("RegexpToString", func(r *regexp.Regexp) string { + if r == nil { + return "" + } + return r.String() + }), +} + +func (s) TestParseFilterConfig_Success(t *testing.T) { + opts := testParseOptions(t, "localhost:1234") + + tests := []struct { + name string + cfg proto.Message + wantCfg httpfilter.FilterConfig + }{ + { + name: "DefaultConfig", + cfg: func() proto.Message { + m, _ := anypb.New(&fpb.ExternalProcessor{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:1234", + }, + }, + }, + ProcessingMode: &fpb.ProcessingMode{}, + }) + return m + }(), + wantCfg: baseConfig{ + server: &grpcservice.Config{TargetURI: "localhost:1234", ChannelCredentials: allowlistInsecureCreds}, + processingModes: processingModes{ + requestHeaderMode: modeSend, + responseHeaderMode: modeSend, + responseTrailerMode: modeSkip, + requestBodyMode: modeSkip, + responseBodyMode: modeSkip, + }, + failureModeAllow: false, + deferredCloseTimeout: defaultDeferredCloseTimeout, + }, + }, + { + name: "ConfigWithGrpcMode", + cfg: func() proto.Message { + m, _ := anypb.New(&fpb.ExternalProcessor{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:1234", + }, + }, + }, + ProcessingMode: &fpb.ProcessingMode{ + RequestBodyMode: fpb.ProcessingMode_GRPC, + ResponseBodyMode: fpb.ProcessingMode_GRPC, + ResponseTrailerMode: fpb.ProcessingMode_SEND, + }, + }) + return m + }(), + wantCfg: baseConfig{ + server: &grpcservice.Config{TargetURI: "localhost:1234", ChannelCredentials: allowlistInsecureCreds}, + processingModes: processingModes{ + requestHeaderMode: modeSend, + responseHeaderMode: modeSend, + responseTrailerMode: modeSend, + requestBodyMode: modeSend, + responseBodyMode: modeSend, + }, + failureModeAllow: false, + deferredCloseTimeout: defaultDeferredCloseTimeout, + }, + }, + { + name: "ConfigWithMutationRules", + cfg: func() proto.Message { + m, _ := anypb.New(&fpb.ExternalProcessor{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:1234", + }, + }, + }, + ProcessingMode: &fpb.ProcessingMode{}, + MutationRules: &mutationpb.HeaderMutationRules{ + AllowExpression: &matcherpb.RegexMatcher{Regex: ".*"}, + DisallowExpression: &matcherpb.RegexMatcher{Regex: "a"}, + }, + }) + return m + }(), + wantCfg: baseConfig{ + server: &grpcservice.Config{TargetURI: "localhost:1234", ChannelCredentials: allowlistInsecureCreds}, + processingModes: processingModes{ + requestHeaderMode: modeSend, + responseHeaderMode: modeSend, + responseTrailerMode: modeSkip, + requestBodyMode: modeSkip, + responseBodyMode: modeSkip, + }, + failureModeAllow: false, + mutationRules: httpfilter.HeaderMutationRules{ + AllowExpr: regexp.MustCompile("^(?:.*)$"), + DisallowExpr: regexp.MustCompile("^(?:a)$"), + }, + deferredCloseTimeout: defaultDeferredCloseTimeout, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := builder{} + got, err := b.ParseFilterConfig(tt.cfg, opts) + if err != nil { + t.Fatalf("ParseFilterConfig() returned unexpected error: %v", err) + } + if diff := cmp.Diff(got, tt.wantCfg, cmpOpts...); diff != "" { + t.Fatalf("ParseFilterConfig() returned unexpected config (-got +want):\n%s", diff) + } + }) + } +} + +func (s) TestParseFilterConfig_Errors(t *testing.T) { + opts := testParseOptions(t, "localhost:1234") + + tests := []struct { + name string + cfg proto.Message + wantErr string + }{ + { + name: "MissingGrpcService", + cfg: func() proto.Message { + m, _ := anypb.New(&fpb.ExternalProcessor{ProcessingMode: &fpb.ProcessingMode{}}) + return m + }(), + wantErr: "extproc: empty grpc_service provided", + }, + { + name: "UnsupportedGrpcService_EnvoyGrpc", + cfg: func() proto.Message { + m, _ := anypb.New(&fpb.ExternalProcessor{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &corepb.GrpcService_EnvoyGrpc{ + ClusterName: "cluster", + }, + }, + }, + ProcessingMode: &fpb.ProcessingMode{}, + }) + return m + }(), + wantErr: "only google_grpc GrpcService config is supported", + }, + { + name: "MissingProcessingMode", + cfg: func() proto.Message { + m, _ := anypb.New(&fpb.ExternalProcessor{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:1234", + }, + }, + }, + }) + return m + }(), + wantErr: "extproc: missing processing_mode", + }, + { + name: "InvalidProcessingMode_RequestBodyStreamed", + cfg: func() proto.Message { + m, _ := anypb.New(&fpb.ExternalProcessor{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:1234", + }, + }, + }, + ProcessingMode: &fpb.ProcessingMode{RequestBodyMode: fpb.ProcessingMode_STREAMED}, + }) + return m + }(), + wantErr: "extproc: invalid request body mode STREAMED", + }, + { + name: "InvalidProcessingMode_ResponseBodyStreamed", + cfg: func() proto.Message { + m, _ := anypb.New(&fpb.ExternalProcessor{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:1234", + }, + }, + }, + ProcessingMode: &fpb.ProcessingMode{ResponseBodyMode: fpb.ProcessingMode_STREAMED}, + }) + return m + }(), + wantErr: "extproc: invalid response body mode STREAMED", + }, + { + name: "InvalidProcessingMode_ResponseBodySendTrailerDefault", + cfg: func() proto.Message { + m, _ := anypb.New(&fpb.ExternalProcessor{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:1234", + }, + }, + }, + ProcessingMode: &fpb.ProcessingMode{ + ResponseBodyMode: fpb.ProcessingMode_GRPC, + ResponseTrailerMode: fpb.ProcessingMode_DEFAULT, + }, + }) + return m + }(), + wantErr: fmt.Sprintf("extproc: invalid response trailer mode DEFAULT: must be %q when response body mode is %q", "SEND", "GRPC"), + }, + { + name: "InvalidProcessingMode_ResponseBodySendTrailerSkip", + cfg: func() proto.Message { + m, _ := anypb.New(&fpb.ExternalProcessor{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:1234", + }, + }, + }, + ProcessingMode: &fpb.ProcessingMode{ + ResponseBodyMode: fpb.ProcessingMode_GRPC, + ResponseTrailerMode: fpb.ProcessingMode_SKIP, + }, + }) + return m + }(), + wantErr: fmt.Sprintf("extproc: invalid response trailer mode SKIP: must be %q when response body mode is %q", "SEND", "GRPC"), + }, + { + name: "InvalidMutationRules", + cfg: func() proto.Message { + m, _ := anypb.New(&fpb.ExternalProcessor{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:1234", + }, + }, + }, + ProcessingMode: &fpb.ProcessingMode{}, + MutationRules: &mutationpb.HeaderMutationRules{ + AllowExpression: &matcherpb.RegexMatcher{Regex: "["}, + }, + }) + return m + }(), + wantErr: "httpfilter: error parsing regexp", + }, + { + name: "InvalidAllowedHeaders_EmptyPrefix", + cfg: func() proto.Message { + m, _ := anypb.New(&fpb.ExternalProcessor{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "localhost:1234", + }, + }, + }, + ProcessingMode: &fpb.ProcessingMode{}, + ForwardRules: &fpb.HeaderForwardingRules{ + AllowedHeaders: &matcherpb.ListStringMatcher{ + Patterns: []*matcherpb.StringMatcher{ + { + MatchPattern: &matcherpb.StringMatcher_Prefix{Prefix: ""}, + }, + }, + }, + }, + }) + return m + }(), + wantErr: "empty prefix is not allowed", + }, + { + name: "InvalidServerConfig_EmptyTargetURI", + cfg: func() proto.Message { + m, _ := anypb.New(&fpb.ExternalProcessor{ + GrpcService: &corepb.GrpcService{ + TargetSpecifier: &corepb.GrpcService_GoogleGrpc_{ + GoogleGrpc: &corepb.GrpcService_GoogleGrpc{ + TargetUri: "", + }, + }, + }, + ProcessingMode: &fpb.ProcessingMode{}, + }) + return m + }(), + wantErr: "target_uri must be non-empty", + }, + { + name: "InvalidConfigType", + cfg: &fpb.ExternalProcessor{}, // Not Any + wantErr: "extproc: error parsing config", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + builder := builder{} + _, err := builder.ParseFilterConfig(tt.cfg, opts) + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("ParseFilterConfig() returned error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func (s) TestParseFilterConfigOverride_Success(t *testing.T) { + tests := []struct { + name string + override proto.Message + wantOverrideCfg httpfilter.FilterConfig + }{ + { + name: "EmptyOverride", + override: func() proto.Message { + m, _ := anypb.New(&fpb.ExtProcPerRoute{}) + return m + }(), + wantOverrideCfg: overrideConfig{}, + }, + { + name: "GrpcProcessingMode", + override: func() proto.Message { + m, _ := anypb.New( + &fpb.ExtProcPerRoute{ + Override: &fpb.ExtProcPerRoute_Overrides{ + Overrides: &fpb.ExtProcOverrides{ + ProcessingMode: &fpb.ProcessingMode{ + RequestBodyMode: fpb.ProcessingMode_GRPC, + ResponseBodyMode: fpb.ProcessingMode_GRPC, + ResponseTrailerMode: fpb.ProcessingMode_SEND, + }, + }, + }, + }) + return m + }(), + wantOverrideCfg: overrideConfig{ + processingModes: optional.New(processingModes{ + requestHeaderMode: modeSend, + responseHeaderMode: modeSend, + responseTrailerMode: modeSend, + requestBodyMode: modeSend, + responseBodyMode: modeSend, + }), + }, + }, + { + name: "FailureModeAllow", + override: func() proto.Message { + m, _ := anypb.New( + &fpb.ExtProcPerRoute{ + Override: &fpb.ExtProcPerRoute_Overrides{ + Overrides: &fpb.ExtProcOverrides{ + FailureModeAllow: wrapperspb.Bool(true), + }, + }, + }) + return m + }(), + wantOverrideCfg: overrideConfig{ + failureModeAllow: optional.New(true), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + builder := builder{} + got, err := builder.ParseFilterConfigOverride(tt.override, httpfilter.ParseOptions{}) + if err != nil { + t.Fatalf("ParseFilterConfigOverride() returned unexpected error: %v", err) + } + if diff := cmp.Diff(got, tt.wantOverrideCfg, cmpOpts...); diff != "" { + t.Fatalf("ParseFilterConfigOverride() returned unexpected config (-got +want):\n%s", diff) + } + }) + } +} + +func (s) TestParseFilterConfigOverride_Errors(t *testing.T) { + tests := []struct { + name string + override proto.Message + wantErr string + }{ + { + name: "ProcessingMode_RequestBodyStreamed", + override: func() proto.Message { + m, _ := anypb.New(&fpb.ExtProcPerRoute{ + Override: &fpb.ExtProcPerRoute_Overrides{ + Overrides: &fpb.ExtProcOverrides{ + ProcessingMode: &fpb.ProcessingMode{ + RequestBodyMode: fpb.ProcessingMode_STREAMED, + }, + }, + }, + }) + return m + }(), + wantErr: "extproc: invalid request body mode STREAMED", + }, + { + name: "ProcessingMode_ResponseBodyStreamed", + override: func() proto.Message { + m, _ := anypb.New(&fpb.ExtProcPerRoute{ + Override: &fpb.ExtProcPerRoute_Overrides{ + Overrides: &fpb.ExtProcOverrides{ + ProcessingMode: &fpb.ProcessingMode{ + ResponseBodyMode: fpb.ProcessingMode_STREAMED, + }, + }, + }, + }) + return m + }(), + wantErr: "extproc: invalid response body mode STREAMED", + }, + { + name: "ProcessingMode_ResponseBodySendTrailerDefault", + override: func() proto.Message { + m, _ := anypb.New(&fpb.ExtProcPerRoute{ + Override: &fpb.ExtProcPerRoute_Overrides{ + Overrides: &fpb.ExtProcOverrides{ + ProcessingMode: &fpb.ProcessingMode{ + ResponseBodyMode: fpb.ProcessingMode_GRPC, + ResponseTrailerMode: fpb.ProcessingMode_DEFAULT, + }, + }, + }, + }) + return m + }(), + wantErr: fmt.Sprintf("extproc: invalid response trailer mode DEFAULT: must be %q when response body mode is %q", "SEND", "GRPC"), + }, + { + name: "ProcessingMode_ResponseBodySendTrailerSkip", + override: func() proto.Message { + m, _ := anypb.New(&fpb.ExtProcPerRoute{ + Override: &fpb.ExtProcPerRoute_Overrides{ + Overrides: &fpb.ExtProcOverrides{ + ProcessingMode: &fpb.ProcessingMode{ + ResponseBodyMode: fpb.ProcessingMode_GRPC, + ResponseTrailerMode: fpb.ProcessingMode_SKIP, + }, + }, + }, + }) + return m + }(), + wantErr: fmt.Sprintf("extproc: invalid response trailer mode SKIP: must be %q when response body mode is %q", "SEND", "GRPC"), + }, + { + name: "InvalidOverrideType", + override: &fpb.ExtProcOverrides{}, // Not Any + wantErr: "extproc: error parsing override", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + builder := builder{} + _, err := builder.ParseFilterConfigOverride(tt.override, httpfilter.ParseOptions{}) + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("ParseFilterConfigOverride() returned error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func (s) TestBuildClientInterceptor_Success(t *testing.T) { + tests := []struct { + name string + cfg httpfilter.FilterConfig + override httpfilter.FilterConfig + wantConfig baseConfig + }{ + { + name: "ConfigUsingOnlyBase", + cfg: baseConfig{ + failureModeAllow: true, + requestAttributes: []string{"attr1"}, + responseAttributes: []string{"attr2"}, + observabilityMode: true, + disableImmediateResponse: true, + deferredCloseTimeout: 10 * time.Second, + processingModes: processingModes{ + requestHeaderMode: modeSend, + responseHeaderMode: modeSkip, + responseTrailerMode: modeSend, + requestBodyMode: modeSend, + responseBodyMode: modeSkip, + }, + server: &grpcservice.Config{ + TargetURI: testBaseURI, + ChannelCredentials: xdscreds.NewChannelCreds(insecure.NewBundle(), xdscreds.Identity{Type: "test-channel-creds"}, nil), + CallCredentials: []*xdscreds.CallCreds{xdscreds.NewCallCreds(testPerRPCCreds{}, xdscreds.Identity{Type: "test-call-creds"}, nil)}, + InitialMetadata: metadata.MD(metadata.Pairs("key1", "value1")), + Timeout: 5 * time.Second, + }, + mutationRules: httpfilter.HeaderMutationRules{ + AllowExpr: regexp.MustCompile("^(?:allow-.*)$"), + DisallowExpr: regexp.MustCompile("^(?:disallow-.*)$"), + DisallowAll: true, + DisallowIsError: true, + }, + allowedHeaders: []matcher.StringMatcher{matcher.NewExactStringMatcher("allow-header", false)}, + }, + wantConfig: baseConfig{ + failureModeAllow: true, + requestAttributes: []string{"attr1"}, + responseAttributes: []string{"attr2"}, + mutationRules: httpfilter.HeaderMutationRules{ + AllowExpr: regexp.MustCompile("^(?:allow-.*)$"), + DisallowExpr: regexp.MustCompile("^(?:disallow-.*)$"), + DisallowAll: true, + DisallowIsError: true, + }, + observabilityMode: true, + disableImmediateResponse: true, + deferredCloseTimeout: 10 * time.Second, + processingModes: processingModes{ + requestHeaderMode: modeSend, + responseHeaderMode: modeSkip, + responseTrailerMode: modeSend, + requestBodyMode: modeSend, + responseBodyMode: modeSkip, + }, + server: &grpcservice.Config{ + TargetURI: testBaseURI, + ChannelCredentials: xdscreds.NewChannelCreds(insecure.NewBundle(), xdscreds.Identity{Type: "test-channel-creds"}, nil), + CallCredentials: []*xdscreds.CallCreds{xdscreds.NewCallCreds(testPerRPCCreds{}, xdscreds.Identity{Type: "test-call-creds"}, nil)}, + InitialMetadata: metadata.MD(metadata.Pairs("key1", "value1")), + Timeout: 5 * time.Second, + }, + allowedHeaders: []matcher.StringMatcher{matcher.NewExactStringMatcher("allow-header", false)}, + }, + }, + { + name: "ConfigUsingBaseAndOverride", + cfg: baseConfig{ + failureModeAllow: false, + requestAttributes: []string{"base-attr1"}, + responseAttributes: []string{"base-attr2"}, + observabilityMode: true, + disableImmediateResponse: true, + processingModes: processingModes{ + requestHeaderMode: modeSend, + responseHeaderMode: modeSkip, + responseTrailerMode: modeSend, + requestBodyMode: modeSend, + responseBodyMode: modeSkip, + }, + server: &grpcservice.Config{ + TargetURI: testBaseURI, + ChannelCredentials: allowlistInsecureCreds, + Timeout: time.Second, + InitialMetadata: metadata.MD(metadata.Pairs("key1", "value1")), + }, + mutationRules: httpfilter.HeaderMutationRules{ + AllowExpr: regexp.MustCompile("^(?:allow-.*)$"), + DisallowExpr: regexp.MustCompile("^(?:disallow-.*)$"), + DisallowAll: true, + DisallowIsError: true, + }, + allowedHeaders: []matcher.StringMatcher{matcher.NewExactStringMatcher("allow-header", false)}, + disallowedHeaders: []matcher.StringMatcher{matcher.NewExactStringMatcher("disallow-header", false)}, + deferredCloseTimeout: 10 * time.Second, + }, + override: overrideConfig{ + failureModeAllow: optional.New(true), + requestAttributes: []string{"override-attr1"}, + responseAttributes: []string{"override-attr2"}, + processingModes: optional.New(processingModes{ + requestHeaderMode: modeSkip, + responseHeaderMode: modeSend, + responseTrailerMode: modeSkip, + requestBodyMode: modeSkip, + responseBodyMode: modeSend, + }), + server: optional.New(&grpcservice.Config{ + TargetURI: "override-uri", + ChannelCredentials: allowlistInsecureCreds, + }), + }, + wantConfig: baseConfig{ + failureModeAllow: true, + requestAttributes: []string{"override-attr1"}, + responseAttributes: []string{"override-attr2"}, + mutationRules: httpfilter.HeaderMutationRules{ + AllowExpr: regexp.MustCompile("^(?:allow-.*)$"), + DisallowExpr: regexp.MustCompile("^(?:disallow-.*)$"), + DisallowAll: true, + DisallowIsError: true, + }, + observabilityMode: true, + disableImmediateResponse: true, + deferredCloseTimeout: 10 * time.Second, + processingModes: processingModes{ + requestHeaderMode: modeSkip, + responseHeaderMode: modeSend, + responseTrailerMode: modeSkip, + requestBodyMode: modeSkip, + responseBodyMode: modeSend, + }, + server: &grpcservice.Config{ + TargetURI: "override-uri", + ChannelCredentials: allowlistInsecureCreds, + }, + allowedHeaders: []matcher.StringMatcher{matcher.NewExactStringMatcher("allow-header", false)}, + disallowedHeaders: []matcher.StringMatcher{matcher.NewExactStringMatcher("disallow-header", false)}, + }, + }, + { + name: "ConfigUsingBaseAndPartialOverride", + cfg: baseConfig{ + failureModeAllow: false, + requestAttributes: []string{"base-attr1"}, + responseAttributes: []string{"base-attr2"}, + observabilityMode: true, + disableImmediateResponse: true, + deferredCloseTimeout: 10 * time.Second, + processingModes: processingModes{ + requestHeaderMode: modeSend, + responseHeaderMode: modeSkip, + responseTrailerMode: modeSend, + requestBodyMode: modeSend, + responseBodyMode: modeSkip, + }, + server: &grpcservice.Config{ + TargetURI: testBaseURI, + ChannelCredentials: allowlistInsecureCreds, + Timeout: time.Second, + InitialMetadata: metadata.MD(metadata.Pairs("key1", "value1")), + }, + mutationRules: httpfilter.HeaderMutationRules{ + AllowExpr: regexp.MustCompile("^(?:allow-.*)$"), + DisallowExpr: regexp.MustCompile("^(?:disallow-.*)$"), + DisallowAll: true, + DisallowIsError: true, + }, + allowedHeaders: []matcher.StringMatcher{matcher.NewExactStringMatcher("allow-header", false)}, + disallowedHeaders: []matcher.StringMatcher{matcher.NewExactStringMatcher("disallow-header", false)}, + }, + override: overrideConfig{ + failureModeAllow: optional.New(true), + }, + wantConfig: baseConfig{ + failureModeAllow: true, + requestAttributes: []string{"base-attr1"}, + responseAttributes: []string{"base-attr2"}, + mutationRules: httpfilter.HeaderMutationRules{ + AllowExpr: regexp.MustCompile("^(?:allow-.*)$"), + DisallowExpr: regexp.MustCompile("^(?:disallow-.*)$"), + DisallowAll: true, + DisallowIsError: true, + }, + observabilityMode: true, + disableImmediateResponse: true, + deferredCloseTimeout: 10 * time.Second, + processingModes: processingModes{ + requestHeaderMode: modeSend, + responseHeaderMode: modeSkip, + responseTrailerMode: modeSend, + requestBodyMode: modeSend, + responseBodyMode: modeSkip, + }, + server: &grpcservice.Config{ + TargetURI: testBaseURI, + ChannelCredentials: allowlistInsecureCreds, + Timeout: time.Second, + InitialMetadata: metadata.MD(metadata.Pairs("key1", "value1")), + }, + allowedHeaders: []matcher.StringMatcher{matcher.NewExactStringMatcher("allow-header", false)}, + disallowedHeaders: []matcher.StringMatcher{matcher.NewExactStringMatcher("disallow-header", false)}, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + builder := builder{} + filter := builder.BuildClientFilter(httpfilter.ClientFilterOptions{}) + defer filter.Close() + + intptr, err := filter.BuildClientInterceptor(tc.cfg, tc.override) + if err != nil { + t.Fatalf("BuildClientInterceptor() returned unexpected error: %v", err) + } + defer intptr.Close() + ic, _ := intptr.(*clientInterceptor) + if diff := cmp.Diff(ic.config, tc.wantConfig, cmpOpts...); diff != "" { + t.Fatalf("Interceptor config returned unexpected diff (-got +want):\n%s", diff) + } + }) + } +} + +func (s) TestBuildClientInterceptor_Failure(t *testing.T) { + // incorrectFilterConfig embeds httpfilter.FilterConfig but is not of type + // baseConfig/overrideConfig, and is used to test incorrect config types being + // passed to BuildClientInterceptor. + type incorrectFilterConfig struct { + httpfilter.FilterConfig + } + + tests := []struct { + name string + cfg httpfilter.FilterConfig + override httpfilter.FilterConfig + wantErr string + }{ + { + name: "NilConfig", + cfg: nil, + wantErr: "extproc: incorrect config type provided", + }, + { + name: "IncorrectConfigType", + cfg: incorrectFilterConfig{}, + wantErr: "extproc: incorrect config type provided", + }, + { + name: "IncorrectOverrideType", + cfg: baseConfig{}, + override: incorrectFilterConfig{}, + wantErr: "extproc: incorrect override config type provided", + }, + { + name: "ChannelCreationFailure", + cfg: baseConfig{ + server: &grpcservice.Config{ + TargetURI: "error-uri", + }, + }, + wantErr: fmt.Sprintf("extproc: failed to create channel to the external processor server %q: dial error", "error-uri"), + }, + { + name: "ChannelCreationFailureInOverride", + cfg: baseConfig{ + server: &grpcservice.Config{ + TargetURI: testBaseURI, + }, + }, + override: overrideConfig{ + server: optional.New(&grpcservice.Config{ + TargetURI: "error-uri", + }), + }, + wantErr: fmt.Sprintf("extproc: failed to create channel to the external processor server %q: dial error", "error-uri"), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + overrideCreateExtProcChannelWithError(t) + builder := builder{} + filter := builder.BuildClientFilter(httpfilter.ClientFilterOptions{}) + defer filter.Close() + + _, err := filter.BuildClientInterceptor(tc.cfg, tc.override) + if err == nil { + t.Fatalf("BuildClientInterceptor() returned nil error, want error %q", tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("BuildClientInterceptor() returned error: %v, want %v", err, tc.wantErr) + } + }) + } +} diff --git a/internal/xds/httpfilter/extproc/ext_proc.go b/internal/xds/httpfilter/extproc/ext_proc.go new file mode 100644 index 0000000..fb17c48 --- /dev/null +++ b/internal/xds/httpfilter/extproc/ext_proc.go @@ -0,0 +1,2240 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package extproc implements the Envoy external processing HTTP filter. +package extproc + +import ( + "context" + "errors" + "fmt" + "io" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/optional" + "google.golang.org/grpc/internal/resolver" + xdscreds "google.golang.org/grpc/internal/xds/credentials" + "google.golang.org/grpc/internal/xds/grpcservice" + "google.golang.org/grpc/internal/xds/httpfilter" + iextproc "google.golang.org/grpc/internal/xds/httpfilter/extproc/internal" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" + + v3procfilterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3" + v3procservicegrpc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + v3procservicepb "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" +) + +func init() { + if envconfig.XDSClientExtProcEnabled { + httpfilter.Register(builder{}) + } + iextproc.RegisterForTesting = func() { + httpfilter.Register(builder{}) + } + iextproc.UnregisterForTesting = func() { + for _, typeURL := range builder.TypeURLs(builder{}) { + httpfilter.UnregisterForTesting(typeURL) + } + } + iextproc.TimeNowFunc = time.Now + iextproc.TimeSinceFunc = time.Since +} + +var metadataFromOutgoingContextRaw = internal.FromOutgoingContextRaw.(func(context.Context) (metadata.MD, [][]string, bool)) + +const defaultDeferredCloseTimeout = 5 * time.Second + +func timeNow() time.Time { + return iextproc.TimeNowFunc() +} + +func timeSince(t time.Time) time.Duration { + return iextproc.TimeSinceFunc(t) +} + +type builder struct{} + +func (builder) TypeURLs() []string { + return []string{ + "type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor", + "type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExtProcPerRoute", + } +} + +// validateBodyProcessingMode ensures that the body processing mode is either +// NONE or GRPC. Also ensures that if response body mode is GRPC then response +// trailer mode must be SEND. +func validateBodyProcessingMode(mode *v3procfilterpb.ProcessingMode) error { + if m := mode.GetRequestBodyMode(); m != v3procfilterpb.ProcessingMode_NONE && m != v3procfilterpb.ProcessingMode_GRPC { + return fmt.Errorf("extproc: invalid request body mode %v: want %q or %q", m, "NONE", "GRPC") + } + if m := mode.GetResponseBodyMode(); m != v3procfilterpb.ProcessingMode_NONE && m != v3procfilterpb.ProcessingMode_GRPC { + return fmt.Errorf("extproc: invalid response body mode %v: want %q or %q", m, "NONE", "GRPC") + } + if mode.GetResponseBodyMode() == v3procfilterpb.ProcessingMode_GRPC && mode.GetResponseTrailerMode() != v3procfilterpb.ProcessingMode_SEND { + return fmt.Errorf("extproc: invalid response trailer mode %v: must be %q when response body mode is %q", mode.GetResponseTrailerMode(), "SEND", "GRPC") + } + return nil +} + +// ParseFilterConfig parses the provided filter configuration. The GrpcService +// identifying the external processor server is validated against the provided +// parse options, as per gRFC A102. +func (builder) ParseFilterConfig(cfg proto.Message, opts httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + m, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("extproc: error parsing config %v: unknown type %T, want *anypb.Any", cfg, cfg) + } + msg := new(v3procfilterpb.ExternalProcessor) + if err := m.UnmarshalTo(msg); err != nil { + return nil, fmt.Errorf("extproc: failed to unmarshal config %v: %v", cfg, err) + } + if msg.GetProcessingMode() == nil { + return nil, fmt.Errorf("extproc: missing processing_mode in config %v", cfg) + } + if err := validateBodyProcessingMode(msg.GetProcessingMode()); err != nil { + return nil, err + } + + mutationRules, err := httpfilter.HeaderMutationRulesFromProto(msg.GetMutationRules()) + if err != nil { + return nil, err + } + + var allowedHeaders, disallowedHeaders []matcher.StringMatcher + if allowed := msg.GetForwardRules().GetAllowedHeaders(); allowed != nil { + allowedHeaders, err = httpfilter.ConvertStringMatchers(allowed.GetPatterns()) + if err != nil { + return nil, err + } + } + + if disallowed := msg.GetForwardRules().GetDisallowedHeaders(); disallowed != nil { + disallowedHeaders, err = httpfilter.ConvertStringMatchers(disallowed.GetPatterns()) + if err != nil { + return nil, err + } + } + + deferredCloseTimeout := defaultDeferredCloseTimeout + if msg.GetDeferredCloseTimeout() != nil { + deferredCloseTimeout = msg.GetDeferredCloseTimeout().AsDuration() + } + + // Parse the GrpcService last, so that no error path can drop the built + // credentials: the caller owns them from here on. + if msg.GetGrpcService() == nil { + return nil, fmt.Errorf("extproc: empty grpc_service provided in config %v", cfg) + } + server, err := grpcservice.Parse(msg.GetGrpcService(), opts.BootstrapConfig, opts.ServerConfig) + if err != nil { + return nil, fmt.Errorf("extproc: failed to parse grpc_service: %v", err) + } + + return baseConfig{ + processingModes: processingModesFromProto(msg.GetProcessingMode()), + requestAttributes: msg.GetRequestAttributes(), + responseAttributes: msg.GetResponseAttributes(), + disableImmediateResponse: msg.GetDisableImmediateResponse(), + observabilityMode: msg.GetObservabilityMode(), + failureModeAllow: msg.GetFailureModeAllow(), + server: server, + mutationRules: mutationRules, + allowedHeaders: allowedHeaders, + disallowedHeaders: disallowedHeaders, + deferredCloseTimeout: deferredCloseTimeout, + }, nil +} + +// ParseFilterConfigOverride parses the provided override filter +// configuration. The GrpcService identifying the external processor server is +// validated against the provided parse options, as per gRFC A102. +func (builder) ParseFilterConfigOverride(ov proto.Message, opts httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + m, ok := ov.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("extproc: error parsing override %v: unknown type %T, want *anypb.Any", ov, ov) + } + msg := new(v3procfilterpb.ExtProcPerRoute) + if err := m.UnmarshalTo(msg); err != nil { + return nil, fmt.Errorf("extproc: failed to unmarshal override %v: %v", ov, err) + } + override := msg.GetOverrides() + + var processingModesOpt optional.Optional[processingModes] + if pm := override.GetProcessingMode(); pm != nil { + if err := validateBodyProcessingMode(pm); err != nil { + return nil, err + } + processingModesOpt = optional.New(processingModesFromProto(pm)) + } + + var failureModeAllowOpt optional.Optional[bool] + if override.GetFailureModeAllow() != nil { + failureModeAllowOpt = optional.New(override.GetFailureModeAllow().GetValue()) + } + + // Parse the GrpcService last, so that no error path can drop the built + // credentials: the caller owns them from here on. + var serverOpt optional.Optional[*grpcservice.Config] + if gs := override.GetGrpcService(); gs != nil { + server, err := grpcservice.Parse(gs, opts.BootstrapConfig, opts.ServerConfig) + if err != nil { + return nil, fmt.Errorf("extproc: failed to parse grpc_service: %v", err) + } + serverOpt = optional.New(server) + } + + return overrideConfig{ + server: serverOpt, + processingModes: processingModesOpt, + failureModeAllow: failureModeAllowOpt, + requestAttributes: override.GetRequestAttributes(), + responseAttributes: override.GetResponseAttributes(), + }, nil +} + +func (builder) IsTerminal() bool { + return false +} + +func (builder) BuildClientFilter(opts httpfilter.ClientFilterOptions) httpfilter.ClientFilter { + return &clientFilter{ + metricsRecorder: opts.MetricsRecorder, + target: opts.Target, + } +} + +var _ httpfilter.ClientFilterBuilder = builder{} + +// procChannelEntry is a shared external processor channel, together with the +// GrpcService config it was created from. The config is used for channel +// sharing comparisons when deciding whether the channel can be reused, and +// its credentials are released when the channel is closed. +type procChannelEntry struct { + server *grpcservice.Config + rc *grpcsync.RefCounted[v3procservicegrpc.ExternalProcessorClient] +} + +type clientFilter struct { + metricsRecorder estats.MetricsRecorder + target string + + mu sync.Mutex + procChannels []*procChannelEntry +} + +func (*clientFilter) Close() {} + +// sharesChannel reports whether two GrpcService configs may share a channel +// to the external processor server: the same target with the same channel +// and call credential identities. Timeout and initial metadata are applied +// per-RPC and do not affect sharing; call credentials do, because Dial +// attaches them to the channel. +func sharesChannel(a, b *grpcservice.Config) bool { + targetEqual := a.TargetURI == b.TargetURI + channelCredsEqual := a.ChannelCredentials.Equal(b.ChannelCredentials) + callCredsEqual := slices.EqualFunc(a.CallCredentials, b.CallCredentials, (*xdscreds.CallCreds).Equal) + return targetEqual && channelCredsEqual && callCredsEqual +} + +// getProcChannel returns an existing refcounted client for a channel-sharing +// compatible GrpcService config if present and its refcount is incremented +// successfully. +func (cf *clientFilter) getProcChannel(server *grpcservice.Config) *grpcsync.RefCounted[v3procservicegrpc.ExternalProcessorClient] { + cf.mu.Lock() + defer cf.mu.Unlock() + if i := slices.IndexFunc(cf.procChannels, func(e *procChannelEntry) bool { + return sharesChannel(e.server, server) && e.rc.TryIncrement() + }); i != -1 { + return cf.procChannels[i].rc + } + return nil +} + +// storeProcChannel stores the created channel entry if no valid channel +// exists for a channel-sharing compatible GrpcService config. If another +// goroutine already stored one while unlocked, it increments the existing +// channel's refcount and returns it. +func (cf *clientFilter) storeProcChannel(entry *procChannelEntry) *grpcsync.RefCounted[v3procservicegrpc.ExternalProcessorClient] { + cf.mu.Lock() + defer cf.mu.Unlock() + if i := slices.IndexFunc(cf.procChannels, func(e *procChannelEntry) bool { + return sharesChannel(e.server, entry.server) && e.rc.TryIncrement() + }); i != -1 { + return cf.procChannels[i].rc + } + cf.procChannels = append(cf.procChannels, entry) + return entry.rc +} + +// removeProcChannel removes the entry from the list. When a channel's +// reference count drops to zero this cleanup runs asynchronously, so a +// replacement entry with an equal config may already have been stored; +// removal by entry pointer ensures only the expiring entry is removed. +func (cf *clientFilter) removeProcChannel(entry *procChannelEntry) { + cf.mu.Lock() + defer cf.mu.Unlock() + cf.procChannels = slices.DeleteFunc(cf.procChannels, func(e *procChannelEntry) bool { return e == entry }) +} + +// getOrCreateExtProcChannel retrieves an existing refcounted external processor +// client for an equal server config and increases its refcount, or creates a +// new one if there is none. +func (cf *clientFilter) getOrCreateExtProcChannel(server *grpcservice.Config) (*grpcsync.RefCounted[v3procservicegrpc.ExternalProcessorClient], error) { + // If a channel for a sharing-compatible GrpcService config is present and + // its refcount is greater than 0, increment the refcount and return the + // channel. + if rc := cf.getProcChannel(server); rc != nil { + return rc, nil + } + + // Create the external processor channel without holding the lock. The + // release function closes the channel. + cc, release, err := iextproc.CreateExtProcChannel(server) + if err != nil { + return nil, fmt.Errorf("extproc: failed to create channel to the external processor server %q: %v", server.TargetURI, err) + } + + client := v3procservicegrpc.NewExternalProcessorClient(cc) + // Create a new refcounted client. The onZero cleanup function removes the + // entry from the list, closes the underlying channel, and releases the + // credentials of the config generation the channel was created from. + entry := &procChannelEntry{server: server} + entry.rc = grpcsync.NewRefCounted(client, func() { + cf.removeProcChannel(entry) + release() + entry.server.Close() + }) + + // Double-check if another goroutine created and stored a channel for an + // equal config while we were unlocked. + if existing := cf.storeProcChannel(entry); existing != entry.rc { + entry.rc.Decrement() + return existing, nil + } + return entry.rc, nil +} + +func (cf *clientFilter) BuildClientInterceptor(base, override httpfilter.FilterConfig) (httpfilter.ClientInterceptor, error) { + b, ok := base.(baseConfig) + if !ok { + return nil, fmt.Errorf("extproc: incorrect config type provided (%T): %v", base, base) + } + + var ov overrideConfig + if override != nil { + ov, ok = override.(overrideConfig) + if !ok { + return nil, fmt.Errorf("extproc: incorrect override config type provided (%T): %v", override, override) + } + } + + config := newInterceptorConfig(b, ov) + + // Create or reuse a refcounted channel to the external processor server. + rc, err := cf.getOrCreateExtProcChannel(config.server) + if err != nil { + return nil, err + } + return &clientInterceptor{ + config: config, + procClient: rc, + metricsRecorder: cf.metricsRecorder, + target: cf.target, + }, nil +} + +type clientInterceptor struct { + config baseConfig + procClient *grpcsync.RefCounted[v3procservicegrpc.ExternalProcessorClient] + metricsRecorder estats.MetricsRecorder + target string +} + +func (i *clientInterceptor) Close() { + i.procClient.Decrement() +} + +// createProcContext creates a new child context for the RPC to the external +// processor server to be able to cancel it independently. This context has a +// deadline of the timeout specified in the config, if present, and contains the +// initial metadata specified in the config. +func createProcContext(ctx context.Context, server *grpcservice.Config) (procCtx context.Context, cancel context.CancelFunc) { + if server.Timeout != 0 { + procCtx, cancel = context.WithTimeout(ctx, server.Timeout) + } else { + procCtx, cancel = context.WithCancel(ctx) + } + procCtx = metadata.NewOutgoingContext(procCtx, server.InitialMetadata) + return procCtx, cancel +} + +func (i *clientInterceptor) NewStream(ctx context.Context, ri resolver.RPCInfo, newStream func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) { + // Increment procClient's refcount so this RPC stream keeps the connection + // open even if the interceptor is closed. Decremented when the Process + // stream finishes (via grpc.OnFinish) or fails to start. + i.procClient.Increment() + outgoingMD, added, _ := metadataFromOutgoingContextRaw(ctx) + // Create a cancelable context to cancel the dataplane stream and close any + // goroutines in case of error. + cancelCtx, cancel := context.WithCancel(ctx) + procCtx, procCancel := createProcContext(cancelCtx, i.config.server) + + // Collect OnFinishCallOption functions from the call options. + var onFinish []func(error) + for _, o := range opts { + if onFinishOpt, ok := o.(grpc.OnFinishCallOption); ok && onFinishOpt.OnFinish != nil { + onFinish = append(onFinish, onFinishOpt.OnFinish) + } + } + + csCommon := &commonStream{ + ctx: cancelCtx, + cancel: cancel, + procCancel: procCancel, + config: i.config, + metricsRecorder: i.metricsRecorder, + target: i.target, + clientHeadersStartTime: timeNow(), + // Construct request attributes once during stream initialization to capture + // the original, unmutated request metadata and RPC info. For streaming + // RPCs, where stream creation and message transmission are separable, + // initializing attributes upfront avoids repeated protobuf construction on + // subsequent SendMsg and RecvMsg calls. + reqAttrs: constructRequestAttributes(ri, outgoingMD, added, i.config.requestAttributes), + onFinish: onFinish, + } + + // In the ClientStream API, outgoing headers are sent immediately when the + // stream is created. Because we need to send or mutate these headers before + // they go over the wire, we must establish the ext_proc stream now rather + // than deferring it. + var err error + procClient := i.procClient.Value() + if csCommon.procStream, err = procClient.Process(procCtx, grpc.OnFinish(func(error) { + i.procClient.Decrement() + })); err != nil { + return csCommon.handleInitError(fmt.Errorf("failed to create a stream to external processor: %w", err), newStream, opts...) + } + + // Observability mode. + if i.config.observabilityMode { + ocs := &observabilityClientStream{ + commonStream: csCommon, + procRecvDone: make(chan struct{}), + } + + // Defer the closing of ext proc stream by the defined deferred close + // timeout to allow the server to read all messages from the proc stream. + onFinishFunc := func(error) { + time.AfterFunc(ocs.config.deferredCloseTimeout, ocs.procCancel) + } + newOpts := append(opts, grpc.OnFinish(onFinishFunc)) + + if ocs.dataplaneStream, err = newStream(ocs.ctx, newOpts...); err != nil { + ocs.cancel() + return nil, ocs.streamError(err) + } + ocs.recordMetric(clientHeadersDurationMetric, timeSince(ocs.clientHeadersStartTime).Seconds()) + // Start background goroutine to receive any messages from the external + // processor server and discard them. + go ocs.discardProcessorResponsesLoop() + + // If the request header processing mode is set to "Send", forward the + // headers to the external processor server. + if i.config.processingModes.requestHeaderMode == modeSend { + if err = ocs.sendToProcessor(ocs.requestHeaders(outgoingMD, added)); err != nil { + return ocs.handleInitError(fmt.Errorf("failed to send client headers to external processor server: %w", err), newStream, opts...) + } + } + + return ocs, nil + } + + // Normal mode. + cs := &clientStream{ + commonStream: csCommon, + procStreamFailed: grpcsync.NewEvent(), + procStreamBypass: grpcsync.NewEvent(), + mutatedReqBuffer: buffer.NewUnbounded[*v3procservicepb.StreamedBodyResponse](), + mutatedRespBuffer: buffer.NewUnbounded[*v3procservicepb.StreamedBodyResponse](), + responseHeadersReady: grpcsync.NewEvent(), + responseTrailerReady: grpcsync.NewEvent(), + dataplaneSetup: make(chan struct{}), + procSendCh: make(chan *v3procservicepb.ProcessingRequest), + requestForwardLoopDoneCh: make(chan struct{}), + procRecvLoopDone: grpcsync.NewEvent(), + downstreamToSidestreamPositiveUpdateCh: make(chan struct{}, 1), + upstreamToSidestreamPositiveUpdateCh: make(chan struct{}, 1), + } + cs.downstreamToSidestreamWindow.Store(iextproc.DefaultFlowControlWindowSize) + cs.upstreamToSidestreamWindow.Store(iextproc.DefaultFlowControlWindowSize) + + // When request header processing is "Send", defer creating the dataplane + // stream until the external processor responds, because gRPC transmits + // outgoing headers immediately upon stream creation and we must apply any + // header mutations first. If header processing is skipped, create the + // dataplane stream immediately. + if i.config.processingModes.requestHeaderMode == modeSend { + if err = cs.procStream.Send(cs.requestHeaders(outgoingMD, added)); err != nil { + if err == io.EOF { + _, err = cs.procStream.Recv() + } + return cs.handleInitError(fmt.Errorf("failed to send client headers to external processor server: %w", err), newStream, opts...) + } + } else { + if err = cs.createDataplaneStream(cs.ctx, newStream, opts); err != nil { + return nil, err + } + } + + // Start a background loop for sending messages to the external processor + // server. Use a single goroutine to ensure no concurrent sends. + go cs.sendToProcServerLoop() + + // Start a background loop to receive messages from the external processor + // server and send them on the dataplane stream in either direction. + go cs.recvFromProcServerLoop(newStream, opts) + + return cs, nil +} + +// commonStream contains state and synchronization primitives shared between the +// normal and observability mode client stream implementations. +type commonStream struct { + // ctx is stored to allow blocking ClientStream interface methods (which do + // not accept context parameters) to respect context cancellation/timeout and + // retrieve ctx.Err() for returning the correct gRPC status error. This + // context is directly derived from the dataplane RPC's context. + ctx context.Context + cancel context.CancelFunc + procCancel context.CancelFunc + config baseConfig + + dataplaneStream grpc.ClientStream // underlying gRPC stream to the backend + procStream v3procservicegrpc.ExternalProcessor_ProcessClient // bidirectional stream to the external processor + procStreamClosed atomic.Bool // ensures the stream closure logic is executed exactly once + procStreamErr atomic.Value // holds the terminal error causing the external processor stream to fail + + reqAttrs map[string]*structpb.Struct // attributes to be sent to proc server with client message + reqAttrsSent atomic.Bool // tracks whether the first client message has been sent to ensure request attributes are sent only once + firstProcReqSent atomic.Bool // tracks whether the first proc req has been sent + trailersOnly bool // tracks whether the backend response is trailers-only + responseHeaderOnce atomic.Bool // ensures response headers are forwarded to the external processor only once + responseTrailerOnce atomic.Bool // ensures response trailers are forwarded to the external processor only once + + metricsRecorder estats.MetricsRecorder + target string + clientHeadersStartTime time.Time // holds the start time of client headers processing for metrics and relative timestamp calculations + + // onFinish stores OnFinishCallOption callbacks to execute if calling the + // delegate (newStream) fails or if the RPC fails before delegating. + onFinish []func(err error) + dataplaneRecvCalled atomic.Bool // tracks if RecvMsg has been called on dataplaneStream by the application +} + +func (cs *commonStream) recordMetric(handle *estats.Float64HistoHandle, duration float64) { + if cs.metricsRecorder == nil { + return + } + handle.Record(cs.metricsRecorder, duration, cs.target) +} + +// handleInitError handles failures during the initialization of the external +// processor stream in NewStream. In deny mode, it returns an internal error. In +// allow mode, it bypasses the external processor and creates and returns the +// dataplane stream directly. +// +// The error passed into handleInitError is a wrapped error and must not be +// returned directly to the caller to avoid exposing it as part of the public +// API. +func (cs *commonStream) handleInitError(err error, newStream func(context.Context, ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) { + cs.procCancel() + + if !errors.Is(err, io.EOF) && !cs.config.failureModeAllow { + cs.cancel() + return nil, status.Errorf(codes.Internal, "extproc: %v", err) + } + cs.recordMetric(clientHeadersDurationMetric, timeSince(cs.clientHeadersStartTime).Seconds()) + if cs.dataplaneStream, err = newStream(cs.ctx, opts...); err != nil { + cs.cancel() + return nil, err + } + cs.recordMetric(clientHeadersDurationMetric, timeSince(cs.clientHeadersStartTime).Seconds()) + return cs.dataplaneStream, nil +} + +// callOnFinish executes all registered OnFinish CallOption callbacks. +func (cs *commonStream) callOnFinish(err error) { + for _, f := range cs.onFinish { + f(err) + } +} + +// cleanupDataplane cancels the dataplane stream's context to immediately tear +// down the active dataplane connection and unblock any pending client I/O. +// +// Additionally, after context cancellation, if the application has not called +// RecvMsg on the dataplane stream, it invokes a dummy RecvMsg on it. In +// gRPC-Go, unary RPCs do not start a background goroutine waiting for context +// expiration to call cs.finish(err). If an application or interceptor layer +// abandons an RPC after NewStream without calling SendMsg, CloseSend, or +// RecvMsg on the dataplane stream, calling RecvMsg on the canceled stream +// forces gRPC-Go's internal clientStream to encounter context.Canceled and +// execute cleanup. This guarantees that all registered OnFinish CallOption +// callbacks are executed and reference counts are decremented so that channels +// do not leak. +// +// For streaming RPCs where RecvMsg has already been called by the application, +// we avoid invoking a dummy RecvMsg because ClientStream.RecvMsg is not safe +// for concurrent calls by multiple goroutines on the same stream. Multiple +// goroutines may enter here concurrently; mutual exclusion is not required +// because cs.cancel() makes subsequent RecvMsg calls after context cancellation +// are idempotent. +func (cs *commonStream) cleanupDataplane() { + cs.cancel() + if cs.dataplaneStream != nil && !cs.dataplaneRecvCalled.Load() { + cs.dataplaneStream.RecvMsg(new(any)) + } +} + +// newProcessingRequest creates a new ProcessingRequest with ObservabilityMode, +// ProtocolConfig, FlowControlInit and Attributes fields initialized. +func (cs *commonStream) newProcessingRequest(isClientMessage bool) *v3procservicepb.ProcessingRequest { + req := &v3procservicepb.ProcessingRequest{ + ObservabilityMode: cs.config.observabilityMode, + } + + if cs.firstProcReqSent.CompareAndSwap(false, true) { + req.ProtocolConfig = &v3procservicepb.ProtocolConfiguration{ + RequestBodyMode: convertBodyMode(cs.config.processingModes.requestBodyMode), + ResponseBodyMode: convertBodyMode(cs.config.processingModes.responseBodyMode), + } + if !cs.config.observabilityMode { + req.FlowControlInit = &v3procservicepb.ProcessingRequest_FlowControlInit{ + InitialWindowDownstreamToSidestream: iextproc.DefaultFlowControlWindowSize, + InitialWindowUpstreamToSidestream: iextproc.DefaultFlowControlWindowSize, + InitialWindowSidestreamToUpstream: iextproc.DefaultFlowControlWindowSize, + InitialWindowSidestreamToDownstream: iextproc.DefaultFlowControlWindowSize, + } + } + } + + if isClientMessage && cs.reqAttrsSent.CompareAndSwap(false, true) { + req.Attributes = cs.reqAttrs + } + return req +} + +// requestHeaders creates a ProcessingRequest for sending request headers. +func (cs *commonStream) requestHeaders(md metadata.MD, added [][]string) *v3procservicepb.ProcessingRequest { + req := cs.newProcessingRequest(true) + req.Request = &v3procservicepb.ProcessingRequest_RequestHeaders{ + RequestHeaders: &v3procservicepb.HttpHeaders{ + Headers: httpfilter.ConstructHeaderMap(md, added, cs.config.allowedHeaders, cs.config.disallowedHeaders), + }, + } + return req +} + +// responseHeaders creates a ProcessingRequest for sending response +// headers. +func (cs *commonStream) responseHeaders(header metadata.MD) *v3procservicepb.ProcessingRequest { + req := cs.newProcessingRequest(false) + req.Request = &v3procservicepb.ProcessingRequest_ResponseHeaders{ResponseHeaders: &v3procservicepb.HttpHeaders{ + Headers: httpfilter.ConstructHeaderMap(header, nil, cs.config.allowedHeaders, cs.config.disallowedHeaders), + EndOfStream: cs.trailersOnly, + }} + return req +} + +// responseTrailers creates a ProcessingRequest for sending response +// trailers. +func (cs *commonStream) responseTrailers(trailers metadata.MD) *v3procservicepb.ProcessingRequest { + req := cs.newProcessingRequest(false) + req.Request = &v3procservicepb.ProcessingRequest_ResponseTrailers{ResponseTrailers: &v3procservicepb.HttpTrailers{ + Trailers: httpfilter.ConstructHeaderMap(trailers, nil, cs.config.allowedHeaders, cs.config.disallowedHeaders), + }} + return req +} + +// halfClose creates a ProcessingRequest indicating end of stream without +// a message body. +func (cs *commonStream) halfClose() *v3procservicepb.ProcessingRequest { + req := cs.newProcessingRequest(true) + req.Request = &v3procservicepb.ProcessingRequest_RequestBody{ + RequestBody: &v3procservicepb.HttpBody{ + EndOfStreamWithoutMessage: true, + EndOfStream: true, + }, + } + return req +} + +// marshalAndCreateBodyReq marshals the message and creates a request or +// response body ProcessingRequest. +func (cs *commonStream) marshalAndCreateBodyReq(m any, isClientMessage bool) (*v3procservicepb.ProcessingRequest, error) { + msg, ok := m.(proto.Message) + if !ok { + return nil, status.Errorf(codes.Internal, "extproc: message does not implement proto.Message") + } + bodyBytes, err := proto.Marshal(msg) + if err != nil { + return nil, status.Errorf(codes.Internal, "extproc: failed to marshal message: %v", err) + } + + req := cs.newProcessingRequest(isClientMessage) + if isClientMessage { + req.Request = &v3procservicepb.ProcessingRequest_RequestBody{ + RequestBody: &v3procservicepb.HttpBody{Body: bodyBytes}, + } + return req, nil + } + req.Request = &v3procservicepb.ProcessingRequest_ResponseBody{ + ResponseBody: &v3procservicepb.HttpBody{Body: bodyBytes}, + } + return req, nil +} + +// observabilityClientStream implements resolver.ClientStream to coordinate +// message exchanges between the application client, the external processor, and +// the backend dataplane in observability mode. +type observabilityClientStream struct { + *commonStream + + procStreamBypass atomic.Bool // set to true when the external processor stream should be bypassed + procRecvDone chan struct{} // closed when discardProcessorResponsesLoop detects stream closure/failure + + responseRecvStarted bool // tracks whether RecvMsg has started processing response messages + responseHeader metadata.MD // stores response headers received from dataplane + responseHeaderErr error // stores response header error received from dataplane + mu sync.Mutex // guards concurrent writes to the external processor stream +} + +// streamError returns the appropriate error when a stream operation fails. +// It prioritizes the external processor stream's terminal error. +func (ocs *observabilityClientStream) streamError(err error) error { + if fatalErr, ok := ocs.procStreamErr.Load().(error); ok { + return fatalErr + } + return err +} + +func (ocs *observabilityClientStream) Header() (metadata.MD, error) { + if fatalErr, ok := ocs.procStreamErr.Load().(error); ok { + return nil, fatalErr + } + return ocs.initiateResponseHeaderProcessing() +} + +func (ocs *observabilityClientStream) Trailer() metadata.MD { + if _, ok := ocs.procStreamErr.Load().(error); ok { + return nil + } + + // If trailer is not yet ready, return nil. This may happen when Trailer() is + // called before the server sends trailers, or before the entire response has + // been received. + if tr := ocs.dataplaneStream.Trailer(); tr != nil { + ocs.initiateResponseTrailerProcessing(tr) + return tr + } + return nil +} + +func (ocs *observabilityClientStream) CloseSend() error { + if fatalErr, ok := ocs.procStreamErr.Load().(error); ok { + return fatalErr + } + startTime := timeNow() + if !ocs.procStreamBypass.Load() && ocs.config.processingModes.requestBodyMode == modeSend { + if err := ocs.sendToProcessor(ocs.halfClose()); err != nil { + return err + } + } + err := ocs.dataplaneStream.CloseSend() + ocs.recordMetric(clientHalfCloseDurationMetric, timeSince(startTime).Seconds()) + return ocs.streamError(err) +} + +func (ocs *observabilityClientStream) Context() context.Context { + return ocs.dataplaneStream.Context() +} + +func (ocs *observabilityClientStream) SendMsg(m any) error { + if fatalErr, ok := ocs.procStreamErr.Load().(error); ok { + return fatalErr + } + if ocs.config.processingModes.requestBodyMode == modeSend && !ocs.procStreamBypass.Load() { + if err := ocs.sendBodyToProcessor(m, true); err != nil { + return err + } + } + err := ocs.dataplaneStream.SendMsg(m) + return ocs.streamError(err) +} + +func (ocs *observabilityClientStream) RecvMsg(m any) error { + if fatalErr, ok := ocs.procStreamErr.Load().(error); ok { + return fatalErr + } + + // If this is the first time RecvMsg is called, initiate response header + // processing. + if !ocs.responseRecvStarted { + ocs.responseRecvStarted = true + if _, err := ocs.initiateResponseHeaderProcessing(); err != nil { + return err + } + } + + if !ocs.dataplaneRecvCalled.Load() { + ocs.dataplaneRecvCalled.Store(true) + } + if err := ocs.dataplaneStream.RecvMsg(m); err != nil { + if trerr := ocs.initiateResponseTrailerProcessing(ocs.dataplaneStream.Trailer()); trerr != nil { + return trerr + } + return ocs.streamError(err) + } + + if ocs.config.processingModes.responseBodyMode == modeSend && !ocs.procStreamBypass.Load() { + if err := ocs.sendBodyToProcessor(m, false); err != nil { + return err + } + } + return ocs.streamError(nil) +} + +// sendBodyToProcessor marshals the given message and forwards it as a request +// or response body ProcessingRequest to the external processor stream. +func (ocs *observabilityClientStream) sendBodyToProcessor(m any, isClientMessage bool) error { + req, err := ocs.marshalAndCreateBodyReq(m, isClientMessage) + if err != nil { + return err + } + return ocs.sendToProcessor(req) +} + +// initiateResponseTrailerProcessing fetches response trailers from the +// dataplane stream and forwards them to the external processor if configured. +// It is guarded by responseTrailerOnce to run at most once. +func (ocs *observabilityClientStream) initiateResponseTrailerProcessing(trailers metadata.MD) error { + if ocs.responseTrailerOnce.Load() || !ocs.responseTrailerOnce.CompareAndSwap(false, true) { + return nil + } + startTime := timeNow() + if ocs.config.processingModes.responseTrailerMode == modeSend && !ocs.procStreamBypass.Load() && !ocs.trailersOnly { + if err := ocs.sendToProcessor(ocs.responseTrailers(trailers)); err != nil { + return err + } + } + ocs.recordMetric(serverTrailersDurationMetric, timeSince(startTime).Seconds()) + + // Half-close the external processor stream immediately so the server can + // shut down gracefully. + if !ocs.procStreamBypass.Load() { + ocs.mu.Lock() + ocs.procStream.CloseSend() + ocs.mu.Unlock() + } + return nil +} + +// initiateResponseHeaderProcessing fetches response headers from the dataplane +// stream, detects if the response is trailers-only, and forwards the response +// headers to the external processor if configured. It is guarded by +// responseHeaderOnce to run at most once. +func (ocs *observabilityClientStream) initiateResponseHeaderProcessing() (metadata.MD, error) { + if ocs.responseHeaderOnce.Load() || !ocs.responseHeaderOnce.CompareAndSwap(false, true) { + return ocs.responseHeader, ocs.streamError(ocs.responseHeaderErr) + } + ocs.responseHeader, ocs.responseHeaderErr = ocs.dataplaneStream.Header() + if ocs.responseHeaderErr != nil { + return nil, ocs.streamError(ocs.responseHeaderErr) + } + startTime := timeNow() + // A trailers-only response returns nil from dataplaneStream.Header(). + procHeader := ocs.responseHeader + if ocs.responseHeader == nil { + procHeader = ocs.dataplaneStream.Trailer() + if len(procHeader) > 0 { + ocs.trailersOnly = true + } + } + + if ocs.config.processingModes.responseHeaderMode == modeSend && !ocs.procStreamBypass.Load() { + if err := ocs.sendToProcessor(ocs.responseHeaders(procHeader)); err != nil { + return nil, err + } + } + ocs.recordMetric(serverHeadersDurationMetric, timeSince(startTime).Seconds()) + return ocs.responseHeader, nil +} + +// sendToProcessor sends a ProcessingRequest to the external processor stream. +// If the write fails, it waits for the receive background loop to process the +// closure and returns the resulting error. +func (ocs *observabilityClientStream) sendToProcessor(req *v3procservicepb.ProcessingRequest) error { + ocs.mu.Lock() + err := ocs.procStream.Send(req) + ocs.mu.Unlock() + if err != nil { + select { + case <-ocs.procRecvDone: + case <-ocs.ctx.Done(): + } + if fatalErr, ok := ocs.procStreamErr.Load().(error); ok { + return fatalErr + } + if ocs.procStreamBypass.Load() { + return nil + } + return err + } + return nil +} + +// discardProcessorResponsesLoop runs in a background goroutine to continuously +// read and discard messages from the external processor server, while also +// monitoring the stream for closure or failure. +// +// Per the Envoy ExtProc protobuf specification for observability mode, +// "External processor should not send back processing response, as any +// responses will be ignored." +func (ocs *observabilityClientStream) discardProcessorResponsesLoop() { + for { + if _, err := ocs.procStream.Recv(); err != nil { + ocs.failObsProcStream(err) + return + } + } +} + +// failObsProcStream handles proc stream closure or failure by updating the +// stream state. If the failure is a non-EOF error and failure mode is deny, it +// stores the error to fail the dataplane RPC. Otherwise, it enables bypass +// mode. +func (ocs *observabilityClientStream) failObsProcStream(err error) { + if !ocs.procStreamClosed.CompareAndSwap(false, true) { + return + } + defer close(ocs.procRecvDone) + ocs.procCancel() + if err != io.EOF && !ocs.config.failureModeAllow { + ocs.procStreamErr.Store(status.Errorf(codes.Internal, "extproc: external processor RPC failed: %v", err)) + ocs.cleanupDataplane() + return + } + ocs.procStreamBypass.Store(true) +} + +// clientStream implements resolver.ClientStream to coordinate bidirectional +// message exchanges between the application client, the external processor, and +// the backend dataplane. +type clientStream struct { + *commonStream + + procStreamFailed *grpcsync.Event // fired when external processor stream has closed and RPC should be failed + procStreamBypass *grpcsync.Event // fired when the external processor stream should be bypassed or drained + + ignoreFailureMode atomic.Bool // tracks whether the failureModeAllow setting should be ignored (e.g. after sending body messages) + discardRequests atomic.Bool // set when ext_proc server signals end_of_stream to stop client sends + mutatedReqBuffer *buffer.Unbounded[*v3procservicepb.StreamedBodyResponse] // buffers mutated request body messages from the ext_proc server + reqForwardingStarted bool // tracks whether request forwarding loop to the dataplane has started + procSendCh chan *v3procservicepb.ProcessingRequest // serializes writes to the external processor stream to ensure thread-safety + + responseHeader metadata.MD // stores headers received from the dataplane stream + responseHeadersReady *grpcsync.Event // signals that response headers are ready for client + responseHeaderSent atomic.Bool // tracks whether response headers have been dispatched to external processor + responseTrailers metadata.MD // stores trailers received from the dataplane stream + responseTrailerReady *grpcsync.Event // signals that response trailers are ready for client + trailerSent atomic.Bool // tracks whether response trailers have been dispatched + trailerErr atomic.Value // holds any terminal status returned upon processing response trailers (e.g. ImmediateResponse) + mutatedRespBuffer *buffer.Unbounded[*v3procservicepb.StreamedBodyResponse] // buffers mutated response body messages from the ext_proc server + responseDrained atomic.Bool // tracks whether all buffered response body messages from the ext_proc server have been drained + dataplaneSetup chan struct{} // closed once dataplaneStream creation attempt is complete + dataplaneCreationErr error // stores the error from the dataplane stream creation attempt + respForwardingStarted bool // tracks whether response forwarding loop to the external processor has started + + requestForwardLoopDoneCh chan struct{} // closed when request forwarding loop finishes draining + procRecvLoopDone *grpcsync.Event // fires when external processor stream receive loop finishes + + // The following start times are accessed concurrently across goroutines + // during the stream lifetime. We store them as atomic.Int64 nanosecond + // offsets from clientHeadersStartTime instead of time.Time because time.Time + // cannot be accessed atomically without a mutex and atomic.Int64 provides + // lock-free atomic operations. + clientHalfCloseStartTime atomic.Int64 + serverHeadersStartTime atomic.Int64 + serverTrailersStartTime atomic.Int64 + + downstreamToSidestreamWindow atomic.Int64 // downstream to sidestream window size remaining + downstreamToSidestreamPositiveUpdateCh chan struct{} // signals the downstream to sidestream window becoming positive + + upstreamToSidestreamWindow atomic.Int64 // upstream to sidestream window size remaining + upstreamToSidestreamPositiveUpdateCh chan struct{} // signals the upstream to sidestream window becoming positive + + pendingRespWindowUpdate atomic.Int64 + pendingReqWindowUpdate atomic.Int64 +} + +// recordDuration records the elapsed time since the event start timestamp +// stored in val to the histogram metric. val holds the start timestamp as a +// nanosecond offset from cs.clientHeadersStartTime. It is a no-op if val has +// not been set (value is 0). +func (cs *clientStream) recordDuration(handle *estats.Float64HistoHandle, val *atomic.Int64) { + if offsetNs := val.Load(); offsetNs != 0 { + startTime := cs.clientHeadersStartTime.Add(time.Duration(offsetNs)) + cs.recordMetric(handle, timeSince(startTime).Seconds()) + } +} + +// fireResponseHeadersReady fires the responseHeadersReady event, and records +// the server_headers_duration metric exactly once upon firing. +func (cs *clientStream) fireResponseHeadersReady() { + if cs.responseHeadersReady.Fire() { + cs.recordDuration(serverHeadersDurationMetric, &cs.serverHeadersStartTime) + } +} + +// fireResponseTrailerReady fires the responseTrailerReady event, and records +// the server_trailers_duration metric exactly once upon firing. +func (cs *clientStream) fireResponseTrailerReady() { + if cs.responseTrailerReady.Fire() { + cs.recordDuration(serverTrailersDurationMetric, &cs.serverTrailersStartTime) + } +} + +// Header returns the response headers received from the backend, potentially +// modified by the external processor if configured. It blocks until header +// processing is complete, or returns nil if the response is trailers-only. +func (cs *clientStream) Header() (metadata.MD, error) { + if err := cs.initiateResponseHeaderProcessing(); err != nil { + return nil, err + } + // Wait for the response headers to be modified. + select { + case <-cs.responseHeadersReady.Done(): + if cs.trailersOnly { + return nil, nil + } + return cs.responseHeader, nil + case <-cs.procStreamFailed.Done(): + return nil, cs.streamError() + case <-cs.ctx.Done(): + return nil, cs.streamError() + } +} + +// Trailer returns the trailer metadata received from the server, potentially +// modified by the external processor if configured. It returns nil or empty map +// immediately if trailers are not yet received from the backend. If backend +// trailers are available, it blocks until any configured trailer processing by +// the external processor completes. +func (cs *clientStream) Trailer() metadata.MD { + // If trailers are already modified and ready, return them immediately. + if cs.responseTrailerReady.HasFired() { + return cs.responseTrailers + } + // Checking if backend stream has trailers ready now. + s, err := cs.waitForDataplaneStream(cs.ctx) + // Since Trailer is not supposed to be a blocking function and should be + // called after CloseRecv or after Recv() returns an error, it should have the + // dataplane stream created. If it does not, then Trailer has been called + // prematurely, which means we need to return nil and not block here. + if err != nil { + return nil + } + // If no backend trailers are ready yet, return nil to preserve standard + // non-blocking behaviour. + if len(s.Trailer()) == 0 { + return nil + } + + cs.initiateResponseTrailerProcessing() + select { + case <-cs.responseTrailerReady.Done(): + return cs.responseTrailers + case <-cs.procStreamFailed.Done(): + return nil + case <-cs.ctx.Done(): + return nil + } +} + +func (cs *clientStream) CloseSend() error { + if cs.discardRequests.Load() { + return nil + } + if cs.clientHalfCloseStartTime.Load() == 0 { + offset := timeSince(cs.clientHeadersStartTime) + cs.clientHalfCloseStartTime.CompareAndSwap(0, int64(offset)) + } + + s, err := cs.bypassProcStreamForClientMsg() + if err != nil { + return err + } + if s != nil { + cs.recordDuration(clientHalfCloseDurationMetric, &cs.clientHalfCloseStartTime) + err = s.CloseSend() + return err + } + // If external processor stream is active, client CloseSend is sent to the + // processor server as request message with `EndOfStreamWithoutMessage` set. + s, err = cs.sendClientReqToProcServer(cs.halfClose()) + if s != nil { + cs.recordDuration(clientHalfCloseDurationMetric, &cs.clientHalfCloseStartTime) + err = s.CloseSend() + return err + } + return err +} + +func (cs *clientStream) Context() context.Context { + s, err := cs.waitForDataplaneStream(cs.ctx) + if err != nil { + return cs.ctx + } + return s.Context() +} + +func (cs *clientStream) RecvMsg(m any) error { + // Initiate response header processing because external processor requires the + // events to be sent in the correct order, i.e. response header before + // response message. And if Header() has not already been called, send the + // response headers to external processor server first. + if err := cs.initiateResponseHeaderProcessing(); err != nil { + return err + } + + // If all the responses from external processor server have been drained or if + // the external processor is bypassed, or if the response body mode is skip, + // then receive directly from the dataplane stream. + if cs.responseDrained.Load() || (cs.procStreamBypass.HasFired() && !cs.respForwardingStarted) || cs.config.processingModes.responseBodyMode == modeSkip { + return cs.recvFromDataplane(m) + } + + msg, ok := m.(proto.Message) + if !ok { + return fmt.Errorf("extproc: response message does not implement proto.Message") + } + + // Start the background receiving loop on the first RecvMsg call to capture + // the type of message to be received. + if !cs.respForwardingStarted { + cs.respForwardingStarted = true + go cs.responseForwardingToProcServerLoop(msg.ProtoReflect().Type()) + } + + // Pull response messages from mutatedRespBuffer. + select { + case streamedResp, ok := <-cs.mutatedRespBuffer.Get(): + cs.mutatedRespBuffer.Load() + if !ok { + // Closed channel implies that all messages from the external processor + // have been received. Start receiving directly from dataplane stream. + cs.responseDrained.Store(true) + return cs.recvFromDataplane(m) + } + // Update the pending window update after pulling from the buffer. If the + // pending update exceeds the threshold of defaultWindowSize/2, send a + // standalone processing request with window update and clear the pending + // update. + if bodySize := int64(len(streamedResp.GetBody())); bodySize > 0 { + if cs.pendingRespWindowUpdate.Add(bodySize) >= iextproc.DefaultFlowControlWindowSize/2 { + if err := cs.sendClientWindowUpdate(); err != nil { + return err + } + } + } + return proto.Unmarshal(streamedResp.GetBody(), msg) + case <-cs.ctx.Done(): + return cs.streamError() + case <-cs.procStreamFailed.Done(): + return cs.streamError() + } +} + +func (cs *clientStream) SendMsg(m any) error { + s, err := cs.bypassProcStreamForClientMsg() + if err != nil { + return err + } + if s != nil { + return s.SendMsg(m) + } + + req, err := cs.marshalAndCreateBodyReq(m, true) + if err != nil { + return err + } + msg := m.(proto.Message) + + // Start request forwarding loop on the first send because we need the message + // type to send the data to the dataplane server. + if !cs.reqForwardingStarted { + cs.reqForwardingStarted = true + go cs.requestForwardingToDataplaneLoop(msg.ProtoReflect().Type()) + } + + // If the ExtProc server has already signaled end_of_stream, discard any + // subsequent client messages. + if cs.discardRequests.Load() { + return nil + } + + s, err = cs.sendClientReqToProcServer(req) + if s != nil { + return s.SendMsg(m) + } + return err +} + +// recvFromDataplane receives directly from the dataplane stream. If the +// dataplane stream returns an error, it initiates and waits for response +// trailers. +func (cs *clientStream) recvFromDataplane(m any) error { + if cs.procStreamFailed.HasFired() { + return cs.procStreamErr.Load().(error) + } + s, err := cs.waitForDataplaneStream(cs.ctx) + if err != nil { + return err + } + if !cs.dataplaneRecvCalled.Load() { + cs.dataplaneRecvCalled.Store(true) + } + if err := s.RecvMsg(m); err != nil { + // If RecvMsg returns error, fail the RPC in case external processor stream + // has failed. Otherwise process the trailers. + if cs.procStreamFailed.HasFired() { + return cs.procStreamErr.Load().(error) + } + cs.initiateResponseTrailerProcessing() + // Wait for trailer processing to be done to make sure we are able to return + // correct error and fail RPC if the trailer processing fails. This needs to + // be done because Trailer() does not return an error. TODO: Verify if the + // wait is acceptable. + return cs.waitForTrailerProcessing(err) + } + return nil +} + +// sendClientReqToProcServer attempts to send the given request for client to +// server msg to the external processor. If the processor stream is bypassed +// while waiting, it blocks until all queued messages are forwarded, and returns +// the dataplane stream so the caller can send the message directly to the +// dataplane stream instead. +func (cs *clientStream) sendClientReqToProcServer(req *v3procservicepb.ProcessingRequest) (grpc.ClientStream, error) { + // Ignoring the failureMode as we will be sending the request to proc server. + if !cs.ignoreFailureMode.Load() { + cs.ignoreFailureMode.Store(true) + } + + // Acquire flow control window for downstream to sidestream only if request + // body is present. + if bodysize := len(req.GetRequestBody().GetBody()); bodysize > 0 { + if !cs.acquireDownstreamToSidestreamWindow(int64(bodysize)) { + return cs.handleClientReqProcStreamFallback() + } + } + + select { + case cs.procSendCh <- req: + return nil, nil + case <-cs.procStreamBypass.Done(): + return cs.handleClientReqProcStreamFallback() + case <-cs.procStreamFailed.Done(): + return nil, cs.streamError() + case <-cs.ctx.Done(): + return nil, cs.streamError() + } +} + +// handleClientReqProcStreamFallback handles processor stream bypass during +// client request processing. If the processor stream is bypassed, it blocks +// until all queued client messages are forwarded, and returns the dataplane +// stream so the caller can send directly to the dataplane stream. Otherwise, it +// returns the stream error. +func (cs *clientStream) handleClientReqProcStreamFallback() (grpc.ClientStream, error) { + if cs.procStreamBypass.HasFired() { + if cs.reqForwardingStarted { + // When the drain is triggered, wait for all the queued requests to be sent + // to dataplane server before forwarding current request directly to + // dataplane server. + if err := cs.waitChannel(cs.requestForwardLoopDoneCh); err != nil { + return nil, err + } + } + return cs.waitForDataplaneStream(cs.ctx) + } + return nil, cs.streamError() +} + +// bypassProcStreamForClientMsg checks if the external processor should be +// bypassed for the current request message or CloseSend. If so, it returns the +// dataplane stream to be used directly. It blocks until all client messages +// received from the proc server are forwarded to ensure correct sequence. It +// also blocks to wait for stream creation if necessary. If the message needs to +// be sent to the external processor server, it returns (nil, nil). +func (cs *clientStream) bypassProcStreamForClientMsg() (grpc.ClientStream, error) { + if cs.procStreamFailed.HasFired() { + return nil, cs.procStreamErr.Load().(error) + } + + bypass := cs.procStreamBypass.HasFired() + if bypass && cs.reqForwardingStarted { + // We started forwarding messages to the proc server and then it + // initiated a drain. We must wait until all messages received from + // the proc server are sent on the dataplane stream, before + // unblocking the application's Send() or CloseSend(), to ensure + // correct message sequencing. + if err := cs.waitChannel(cs.requestForwardLoopDoneCh); err != nil { + return nil, err + } + } + + if bypass || cs.config.processingModes.requestBodyMode == modeSkip { + return cs.waitForDataplaneStream(cs.ctx) + } + + return nil, nil // Do not bypass. +} + +// responseForwardingToProcServerLoop continuously receives response messages +// from the underlying dataplane stream, marshals them, and forwards them to the +// external processor server via procSendCh. +func (cs *clientStream) responseForwardingToProcServerLoop(msgType protoreflect.MessageType) { + defer func() { + // Once the external processor server receive loop finishes, close the + // buffer to signal end of processed responses. + cs.waitChannel(cs.procRecvLoopDone.Done()) + cs.mutatedRespBuffer.Close() + }() + + dataplaneStream, err := cs.waitForDataplaneStream(cs.ctx) + if err != nil { + return + } + + for { + // If the processor stream has closed or the receive loop finishes, we can + // stop receiving messages in the background and Recv should now directly be + // called from the cs.RecvMsg() function. + if cs.procStreamBypass.HasFired() || cs.procRecvLoopDone.HasFired() { + return + } + + newMsg := msgType.New().Interface() + if !cs.dataplaneRecvCalled.Load() { + cs.dataplaneRecvCalled.Store(true) + } + if err := dataplaneStream.RecvMsg(newMsg); err != nil { + cs.initiateResponseTrailerProcessing() + return + } + + req, err := cs.marshalAndCreateBodyReq(newMsg, false) + if err != nil { + cs.failProcStream(err) + return + } + + if !cs.ignoreFailureMode.Load() { + cs.ignoreFailureMode.Store(true) + } + + // Acquire upstream to sidestream window before sending the response body to + // proc server. + if bodySize := int64(len(req.GetResponseBody().GetBody())); bodySize > 0 { + if !cs.acquireUpstreamToSidestreamWindow(bodySize) { + // If proc stream is bypassed before acquiring positive window, wait for + // external processor server receive loop to finish before pushing this + // message on the mutatedRespBuffer to ensure correct order of messages. + if cs.procStreamBypass.HasFired() { + if err := cs.waitChannel(cs.procRecvLoopDone.Done()); err != nil { + return + } + resp := &v3procservicepb.StreamedBodyResponse{Body: req.GetResponseBody().GetBody()} + cs.mutatedRespBuffer.Put(resp) + } + return + } + } + + select { + case cs.procSendCh <- req: + case <-cs.procStreamBypass.Done(): + // If drain is triggered, wait for external processor server receive loop + // to finish before pushing this message on the mutatedRespBuffer + // to ensure correct order of messages. + if err := cs.waitChannel(cs.procRecvLoopDone.Done()); err != nil { + return + } + resp := &v3procservicepb.StreamedBodyResponse{Body: req.GetResponseBody().GetBody()} + cs.mutatedRespBuffer.Put(resp) + return + case <-cs.procStreamFailed.Done(): + return + case <-cs.ctx.Done(): + return + } + } +} + +// requestForwardingToDataplaneLoop continuously consumes mutated request body +// from mutatedReqBuffer (populated by the external processor server), +// unmarshals them, and forwards the final messages to the backend server. +func (cs *clientStream) requestForwardingToDataplaneLoop(msgType protoreflect.MessageType) { + defer close(cs.requestForwardLoopDoneCh) + dataplaneStream, err := cs.waitForDataplaneStream(cs.ctx) + if err != nil { + return + } + for { + select { + case streamedResp, ok := <-cs.mutatedReqBuffer.Get(): + if !ok { + return + } + cs.mutatedReqBuffer.Load() + + // As per gRFC A93, ignore `end_of_stream_without_message` if + // `end_of_stream` is false. + if streamedResp.GetEndOfStream() && streamedResp.GetEndOfStreamWithoutMessage() { + cs.recordDuration(clientHalfCloseDurationMetric, &cs.clientHalfCloseStartTime) + dataplaneStream.CloseSend() + return + } + + newMsg := msgType.New().Interface() + if err := proto.Unmarshal(streamedResp.GetBody(), newMsg); err != nil { + cs.failProcStream(err) + return + } + if err := dataplaneStream.SendMsg(newMsg); err != nil { + cs.cancelStream(err) + return + } + // Update the pending window update after pulling from the buffer. If the + // pending update exceeds the threshold of defaultWindowSize/2, send a + // standalone processing request with window update and clear the pending + // update. + if bodySize := int64(len(streamedResp.GetBody())); bodySize > 0 { + if cs.pendingReqWindowUpdate.Add(bodySize) >= iextproc.DefaultFlowControlWindowSize/2 { + if err := cs.sendClientWindowUpdate(); err != nil { + return + } + } + } + if streamedResp.GetEndOfStream() { + cs.recordDuration(clientHalfCloseDurationMetric, &cs.clientHalfCloseStartTime) + dataplaneStream.CloseSend() + return + } + case <-cs.ctx.Done(): + return + case <-cs.procStreamFailed.Done(): + return + } + } +} + +// recvFromProcServerLoop continuously receives messages from the external +// processor server and redirects them according to the response type. +func (cs *clientStream) recvFromProcServerLoop(newStream func(context.Context, ...grpc.CallOption) (grpc.ClientStream, error), opts []grpc.CallOption) { + defer func() { + cs.procRecvLoopDone.Fire() + // Close mutatedReqBuffer to indicate completion of receiving + // the mutated requests. Do not close mutatedResponseBuffer because we + // might push the message that has been read when drain is triggered. + cs.mutatedReqBuffer.Close() + }() + + // If request header mode is send, the first response should be the mutation + // for request header. Create the dataplane stream using the mutated header. + if cs.config.processingModes.requestHeaderMode == modeSend { + if !cs.processInitialHeaders(cs.ctx, newStream, opts) { + return + } + } + + // If header mode is not send or if we receive the header and create the + // dataplane stream, start receiving from the external processor server. + for { + resp, err := cs.procStream.Recv() + if err != nil { + cs.failProcStream(err) + return + } + if resp.GetRequestDrain() { + // Trigger the drain but continue receiving the drained messages until we + // get io.EOF. + cs.triggerBypass() + } + + if resp.GetImmediateResponse() != nil { + cs.handleImmediateResponse(resp.GetImmediateResponse(), newStream, opts) + return + } + + if windowUpdate := resp.GetServerWindowUpdate(); windowUpdate != nil { + deltaDownstreamToSidestream := windowUpdate.GetWindowIncrementDownstreamToSidestream() + if deltaDownstreamToSidestream != 0 { + // Add the window update to the existing window. + newQuota := cs.downstreamToSidestreamWindow.Add(deltaDownstreamToSidestream) + previousQuota := newQuota - deltaDownstreamToSidestream + // If the window turned from negative to positive, send a signal to the + // `acquireDownstreamToSidestreamWindow` function to wake it up. + if previousQuota <= 0 && newQuota > 0 { + select { + case cs.downstreamToSidestreamPositiveUpdateCh <- struct{}{}: + default: + } + } + } + deltaUpstreamToSidestream := windowUpdate.GetWindowIncrementUpstreamToSidestream() + if deltaUpstreamToSidestream != 0 { + // Add the window update to the existing window. + newQuota := cs.upstreamToSidestreamWindow.Add(deltaUpstreamToSidestream) + previousQuota := newQuota - deltaUpstreamToSidestream + // If the window turned from negative to positive, send a signal to the + // `acquireUpstreamToSidestreamWindow` function to wake it up. + if previousQuota <= 0 && newQuota > 0 { + select { + case cs.upstreamToSidestreamPositiveUpdateCh <- struct{}{}: + default: + } + } + } + } + + switch { + case resp.GetRequestBody() != nil: + if cs.config.processingModes.requestBodyMode == modeSkip { + cs.failProcStream(fmt.Errorf("external processor unexpectedly sent request body when request body processing is disabled")) + return + } + + streamedResp, ok := cs.validateBodyResponse(resp.GetRequestBody()) + if !ok { + return + } + if streamedResp.GetEndOfStream() { + cs.discardRequests.Store(true) + } + cs.mutatedReqBuffer.Put(streamedResp) + + case resp.GetResponseBody() != nil: + if cs.config.processingModes.responseBodyMode == modeSkip { + cs.failProcStream(fmt.Errorf("external processor unexpectedly sent response body when response body processing is disabled")) + return + } + + // If response headers have been sent and mutated response headers have + // not been received before receiving the response body message, fail the + // RPC. + if cs.config.processingModes.responseHeaderMode == modeSend && !cs.responseHeadersReady.HasFired() { + cs.failProcStream(fmt.Errorf("external processor sent response body before sending response headers")) + return + } + + // If mutated response trailers have been received before receiving the + // response body message, fail the RPC. + if cs.config.processingModes.responseTrailerMode == modeSend && cs.responseTrailerReady.HasFired() { + cs.failProcStream(fmt.Errorf("external processor sent response body after response trailers were already processed")) + return + } + + streamedResp, ok := cs.validateBodyResponse(resp.GetResponseBody()) + if !ok { + return + } + if streamedResp.GetEndOfStream() { + cs.failProcStream(fmt.Errorf("external processor unexpectedly set end of stream in response body mutation")) + return + } + cs.mutatedRespBuffer.Put(streamedResp) + + case resp.GetResponseHeaders() != nil: + if cs.config.processingModes.responseHeaderMode == modeSkip { + cs.failProcStream(fmt.Errorf("external processor unexpectedly sent response headers when response header processing is disabled")) + return + } + if !cs.responseHeaderSent.Load() { + cs.failProcStream(fmt.Errorf("external processor sent response headers before response headers were sent to it")) + return + } + if cs.responseHeadersReady.HasFired() { + cs.failProcStream(fmt.Errorf("external processor unexpectedly sent duplicate response headers after response headers were already processed")) + return + } + + header := resp.GetResponseHeaders() + // Check if the status in the header response is CONTINUE; if not, fail + // the stream. + if status := header.GetResponse().GetStatus(); status != v3procservicepb.CommonResponse_CONTINUE { + cs.failProcStream(fmt.Errorf("external processor returned unexpected status %v for response headers, expected %v", status, v3procservicepb.CommonResponse_CONTINUE)) + return + } + if err = cs.applyMutations(header.GetResponse().GetHeaderMutation(), cs.responseHeader); err != nil { + cs.failProcStream(err) + return + } + // Signal that the response header is modified and ready to be sent to the + // client, so that if there is any buffered response body, it can be sent + // after the header. + cs.fireResponseHeadersReady() + + case resp.GetResponseTrailers() != nil: + if cs.config.processingModes.responseTrailerMode == modeSkip { + cs.failProcStream(fmt.Errorf("external processor unexpectedly sent response trailers when response trailer processing is disabled")) + return + } + if !cs.trailerSent.Load() { + cs.failProcStream(fmt.Errorf("external processor sent response trailers before response trailers were sent to it")) + return + } + if cs.responseTrailerReady.HasFired() { + cs.failProcStream(fmt.Errorf("external processor unexpectedly sent duplicate response trailers after response trailers were already processed")) + return + } + trailer := resp.GetResponseTrailers() + if err = cs.applyMutations(trailer.GetHeaderMutation(), cs.responseTrailers); err != nil { + cs.failProcStream(err) + return + } + // Signal that the response trailer is modified and ready to be sent to + // the client. + cs.fireResponseTrailerReady() + } + } +} + +func (cs *clientStream) validateBodyResponse(bodyResp *v3procservicepb.BodyResponse) (*v3procservicepb.StreamedBodyResponse, bool) { + if status := bodyResp.GetResponse().GetStatus(); status != v3procservicepb.CommonResponse_CONTINUE { + cs.failProcStream(fmt.Errorf("external processor returned unexpected status %v for body response, expected %v", status, v3procservicepb.CommonResponse_CONTINUE)) + return nil, false + } + streamedResp := bodyResp.GetResponse().GetBodyMutation().GetStreamedResponse() + if streamedResp == nil { + cs.failProcStream(fmt.Errorf("external processor returned invalid body mutation in body response")) + return nil, false + } + if streamedResp.GetGrpcMessageCompressed() { + cs.failProcStream(fmt.Errorf("external processor returned compressed grpc message which is not supported")) + return nil, false + } + return streamedResp, true +} + +func (cs *clientStream) applyMutations(mutation *v3procservicepb.HeaderMutation, md metadata.MD) error { + if mutation == nil { + return nil + } + if err := cs.config.mutationRules.ApplyAdditions(mutation.GetSetHeaders(), md); err != nil { + return err + } + return cs.config.mutationRules.ApplyRemovals(mutation.GetRemoveHeaders(), md) +} + +// closeProcSend gracefully half-closes the external processor stream across the +// background send goroutine by pushing a nil sentinel onto procSendCh. +func (cs *clientStream) closeProcSend() { + select { + case cs.procSendCh <- nil: + case <-cs.ctx.Done(): + case <-cs.procStreamFailed.Done(): + case <-cs.procStreamBypass.Done(): + } +} + +// sendToProcServer runs as a dedicated background goroutine that serializes all +// outbound messages to the external processor server. It listens on procSendCh +// for messages to forward. It actively monitors stream lifecycle events: +// - Drain/Bypass: If procStreamBypass fires, it initiates a graceful shutdown +// by sending a half-close (CloseSend) to the external processor server and +// returning so that no more messages can be sent to the external processor. +// - Cancellation/Failure: It aborts immediately if the context is canceled or +// the underlying stream fails. +func (cs *clientStream) sendToProcServerLoop() { + for { + select { + case req := <-cs.procSendCh: + if req == nil { + // A nil request is used as a sentinel message to gracefully half-close + // the processor stream from this background thread. This ensures that + // CloseSend is called sequentially after all preceding requests in the + // channel are successfully sent, avoiding out-of-order data races. + cs.procStream.CloseSend() + return + } + if req.GetResponseHeaders() != nil { + cs.responseHeaderSent.Store(true) + } + if req.GetResponseTrailers() != nil { + cs.trailerSent.Store(true) + } + + // Merge any pending window updates into the outgoing request. If the + // request is already a standalone ClientWindowUpdate, we must accumulate + // (add) the newly drained increments to any existing increments rather + // than replacing the struct, ensuring no pending window quota is dropped. + respInc := cs.pendingRespWindowUpdate.Swap(0) + reqInc := cs.pendingReqWindowUpdate.Swap(0) + if respInc > 0 || reqInc > 0 { + if req.ClientWindowUpdate == nil { + req.ClientWindowUpdate = &v3procservicepb.ProcessingRequest_ClientWindowUpdate{} + } + req.ClientWindowUpdate.WindowIncrementSidestreamToDownstream += respInc + req.ClientWindowUpdate.WindowIncrementSidestreamToUpstream += reqInc + } + + if err := cs.procStream.Send(req); err != nil { + if err != io.EOF { + // For non-EOF client-side send errors, fail the stream immediately. + cs.failProcStream(err) + } + // If Send returned io.EOF, the server closed the stream early. Let + // the Recv loop retrieve the actual status error from the server and + // propagate it. + return + } + case <-cs.procStreamBypass.Done(): + cs.procStream.CloseSend() + return + case <-cs.procStreamFailed.Done(): + return + case <-cs.ctx.Done(): + return + } + } +} + +// streamError returns the appropriate error when a stream operation is +// preempted by a stream failure or context cancellation. It prioritizes the +// external processor stream's terminal error over the context error. +func (cs *clientStream) streamError() error { + if cs.procStreamFailed.HasFired() { + return cs.procStreamErr.Load().(error) + } + return status.FromContextError(cs.ctx.Err()).Err() +} + +// waitChannel waits for the provided channel to be closed, while also +// respecting context cancellation and stream failures. +func (cs *clientStream) waitChannel(ch <-chan struct{}) error { + select { + case <-ch: + return nil + case <-cs.ctx.Done(): + return cs.streamError() + case <-cs.procStreamFailed.Done(): + return cs.streamError() + } +} + +// createDataplaneStream initializes the underlying gRPC dataplane stream using +// the provided newStream function. If dataplane stream creation fails, the +// top-level context used by the extproc filter is canceled and thereby all +// spawned goroutines terminate. +func (cs *clientStream) createDataplaneStream(ctx context.Context, newStream func(context.Context, ...grpc.CallOption) (grpc.ClientStream, error), opts []grpc.CallOption) error { + defer close(cs.dataplaneSetup) + cs.dataplaneStream, cs.dataplaneCreationErr = newStream(ctx, opts...) + if cs.dataplaneCreationErr != nil { + // Execute the OnFinish call options if the delegate failed. + cs.callOnFinish(cs.dataplaneCreationErr) + cs.cancel() + return cs.dataplaneCreationErr + } + cs.recordMetric(clientHeadersDurationMetric, timeSince(cs.clientHeadersStartTime).Seconds()) + return nil +} + +// failProcStream handles stream failures, recording errors or bypassing the +// external processor based on failureModeAllow configuration. It returns true +// if the stream is bypassed, and false if the connection is torn down. +func (cs *clientStream) failProcStream(err error) bool { + if !cs.procStreamClosed.CompareAndSwap(false, true) { + return cs.procStreamBypass.HasFired() + } + cs.procCancel() + if err != io.EOF && (cs.ignoreFailureMode.Load() || !cs.config.failureModeAllow) { + // Execute the OnFinish call options if the delegate is never called. + if cs.dataplaneStream == nil { + cs.callOnFinish(err) + } + cs.procStreamErr.Store(status.Errorf(codes.Internal, "extproc: %v", err)) + cs.procStreamFailed.Fire() + cs.cleanupDataplane() + return false + } + cs.triggerBypass() + return true +} + +// cancelStream immediately terminates the stream with the specified error and +// fires the failure event. It does not respect the failure mode. +func (cs *clientStream) cancelStream(err error) { + if !cs.procStreamClosed.CompareAndSwap(false, true) { + return + } + // Execute the OnFinish call options if the delegate is never called. + if cs.dataplaneStream == nil { + cs.callOnFinish(err) + } + cs.procStreamErr.Store(err) + cs.procStreamFailed.Fire() + cs.cleanupDataplane() +} + +// handleHeaderError handles failures that occur during receiving or processing +// the initial request headers. If the failure mode allows it, the external +// processor is bypassed and the direct dataplane stream is created. +func (cs *clientStream) handleHeaderError(err error, newStream func(context.Context, ...grpc.CallOption) (grpc.ClientStream, error), opts []grpc.CallOption) { + if cs.failProcStream(err) { + cs.createDataplaneStream(cs.ctx, newStream, opts) + } +} + +// processInitialHeaders waits for the initial header response from the external +// processor server, applies any requested header mutations to the outgoing +// context, and initializes the data plane stream. It returns true if the +// headers were processed successfully and dataplane stream was created. It +// returns false if the proc stream needs to be bypassed, or failed due to an +// error, or aborted early by a processor-initiated immediate response. +func (cs *clientStream) processInitialHeaders(ctx context.Context, newStream func(context.Context, ...grpc.CallOption) (grpc.ClientStream, error), opts []grpc.CallOption) bool { + resp, err := cs.procStream.Recv() + if err != nil { + cs.handleHeaderError(err, newStream, opts) + return false + } + if resp.GetRequestDrain() { + cs.triggerBypass() + } + if resp.GetImmediateResponse() != nil { + cs.handleImmediateResponse(resp.GetImmediateResponse(), newStream, opts) + return false + } + + header := resp.GetRequestHeaders() + if header == nil { + err := fmt.Errorf("external processor returned an unexpected message type %T, expected request headers response", resp.GetResponse()) + cs.handleHeaderError(err, newStream, opts) + return false + } + // Check if status in header response is CONTINUE; if not, fail the stream. + if status := header.GetResponse().GetStatus(); status != v3procservicepb.CommonResponse_CONTINUE { + cs.handleHeaderError(fmt.Errorf("external processor returned unexpected status %v for request headers, expected %v", status, v3procservicepb.CommonResponse_CONTINUE), newStream, opts) + return false + } + // Mutate the outgoing headers with additions and removals received from the + // external processor. + outgoingMD, _ := metadata.FromOutgoingContext(ctx) + if outgoingMD == nil { + outgoingMD = metadata.MD{} + } + if err = cs.applyMutations(header.GetResponse().GetHeaderMutation(), outgoingMD); err != nil { + cs.handleHeaderError(err, newStream, opts) + return false + } + dataplaneCtx := metadata.NewOutgoingContext(ctx, outgoingMD) + if err = cs.createDataplaneStream(dataplaneCtx, newStream, opts); err != nil { + return false + } + return true +} + +func (cs *clientStream) handleImmediateResponse(imm *v3procservicepb.ImmediateResponse, newStream func(context.Context, ...grpc.CallOption) (grpc.ClientStream, error), opts []grpc.CallOption) { + if cs.config.disableImmediateResponse { + err := fmt.Errorf("external processor sent an immediate response but immediate responses are disabled in configuration") + if cs.dataplaneStream == nil { + cs.handleHeaderError(err, newStream, opts) + } else { + cs.failProcStream(err) + } + return + } + + statusCode := codes.Internal + if imm.GetGrpcStatus() != nil { + statusCode = codes.Code(imm.GetGrpcStatus().GetStatus()) + if statusCode > codes.Code(16) { + statusCode = codes.Unknown + } + } + err := status.Error(statusCode, imm.GetDetails()) + + if cs.trailerSent.Load() { + if mutation := imm.GetHeaders(); mutation != nil { + cs.applyMutations(mutation, cs.responseTrailers) + } + cs.trailerErr.Store(err) + cs.fireResponseTrailerReady() + } else { + cs.cancelStream(err) + } +} + +func (cs *clientStream) triggerBypass() { + if cs.procStreamBypass.Fire() { + cs.fireResponseHeadersReady() + cs.fireResponseTrailerReady() + } +} + +// waitForDataplaneStream waits for the dataplane stream to be created or for +// the context to be done. It also checks if the processor stream has not ended +// abruptly with a non-io.EOF error. +func (cs *clientStream) waitForDataplaneStream(ctx context.Context) (grpc.ClientStream, error) { + select { + case <-cs.dataplaneSetup: + if cs.dataplaneCreationErr != nil { + return nil, cs.dataplaneCreationErr + } + return cs.dataplaneStream, nil + case <-ctx.Done(): + // Prioritize returning the underlying dataplane creation failure or + // external processor error over the context cancellation error. + if cs.dataplaneCreationErr != nil { + return nil, cs.dataplaneCreationErr + } + return nil, cs.streamError() + case <-cs.procStreamFailed.Done(): + return nil, cs.procStreamErr.Load().(error) + } +} + +// initiateResponseHeaderProcessing receives initial response headers exactly +// once, forwards them to the external processor if configured, or unblocks +// readers immediately. +func (cs *clientStream) initiateResponseHeaderProcessing() error { + if cs.procStreamFailed.HasFired() { + return cs.procStreamErr.Load().(error) + } + if cs.responseHeaderOnce.Load() || !cs.responseHeaderOnce.CompareAndSwap(false, true) { + return nil + } + dataplaneStream, err := cs.waitForDataplaneStream(cs.ctx) + if err != nil { + return err + } + header, err := dataplaneStream.Header() + if err != nil { + // If the processor stream failed first and canceled cs.ctx, prefer + // that root error over the secondary dataplaneStream.Header() failure. + if cs.procStreamFailed.HasFired() { + return cs.procStreamErr.Load().(error) + } + return err + } + // Capture the start time for response headers after they have been + // successfully retrieved from the dataplane stream. + cs.serverHeadersStartTime.Store(int64(timeSince(cs.clientHeadersStartTime))) + + if header == nil { + // A trailers-only response returns nil from dataplaneStream.Header(). + header = dataplaneStream.Trailer() + if len(header) > 0 { + cs.trailersOnly = true + // For trailers-only, the trailers are retrieved during the header + // phase. Start timing them now since they will be processed as part + // of the header phase. + cs.serverTrailersStartTime.Store(int64(timeSince(cs.clientHeadersStartTime))) + } + } + cs.responseHeader = header + if cs.config.processingModes.responseHeaderMode != modeSend || cs.procStreamBypass.HasFired() { + // If header does not need to be sent to the external processor, unblock + // the functions waiting on header modifications. + cs.fireResponseHeadersReady() + return nil + } + + select { + case cs.procSendCh <- cs.responseHeaders(header): + return nil + case <-cs.procStreamBypass.Done(): + return nil + case <-cs.procStreamFailed.Done(): + return cs.streamError() + case <-cs.ctx.Done(): + return cs.streamError() + } +} + +func (cs *clientStream) initiateResponseTrailerProcessing() { + if cs.responseTrailerOnce.Load() || !cs.responseTrailerOnce.CompareAndSwap(false, true) { + return + } + if cs.trailersOnly { + // For trailers-only response, the trailers metadata map is processed and + // mutated during the response header phase. Wait for those mutations to + // be received and applied. + select { + case <-cs.responseHeadersReady.Done(): + case <-cs.procStreamFailed.Done(): + case <-cs.ctx.Done(): + } + cs.responseTrailers = cs.responseHeader + cs.fireResponseTrailerReady() + // Gracefully half-close the external processor stream for trailers-only + // responses once header modifications finish. + cs.closeProcSend() + return + } + cs.responseTrailers = cs.dataplaneStream.Trailer() + // Capture the start time for response trailers after they have been + // successfully retrieved from the dataplane stream. + offset := timeSince(cs.clientHeadersStartTime) + cs.serverTrailersStartTime.Store(int64(offset)) + + if cs.config.processingModes.responseTrailerMode == modeSend && !cs.procStreamBypass.HasFired() { + select { + case cs.procSendCh <- cs.commonStream.responseTrailers(cs.responseTrailers): + case <-cs.ctx.Done(): + case <-cs.procStreamFailed.Done(): + case <-cs.procStreamBypass.Done(): + } + } else { + cs.fireResponseTrailerReady() + } + // Gracefully half-close the external processor stream after forwarding response + // trailers to signal that all responses have completely processed. + cs.closeProcSend() +} + +func (cs *clientStream) waitForTrailerProcessing(recvErr error) error { + select { + case <-cs.responseTrailerReady.Done(): + if val := cs.trailerErr.Load(); val != nil { + return val.(error) + } + if cs.procStreamFailed.HasFired() { + return cs.procStreamErr.Load().(error) + } + return recvErr + case <-cs.procStreamBypass.Done(): + return recvErr + case <-cs.procStreamFailed.Done(): + return cs.streamError() + case <-cs.ctx.Done(): + return cs.streamError() + } +} + +// acquireDownstreamToSidestreamWindow checks available flow control window for +// downstream to sidestream. If window is positive, it deducts bodySize and +// returns true immediately. If window is <= 0, it blocks until a positive +// window update is received, or returns false if bypassed, failed, or context +// is done. +func (cs *clientStream) acquireDownstreamToSidestreamWindow(bodySize int64) bool { + // Re-check the window in a loop. A signal on the buffered channel may be + // stale if a previous caller acquired the positive window on the fast path + // without draining the channel. + for { + // If enough window is available, deduct body size and return immediately. + if cs.downstreamToSidestreamWindow.Load() > 0 { + cs.downstreamToSidestreamWindow.Add(-bodySize) + return true + } + // If window is not available, wait for the window to become positive or + // return false if bypassed, failed, or context is done. + select { + case <-cs.downstreamToSidestreamPositiveUpdateCh: + continue + case <-cs.procStreamBypass.Done(): + return false + case <-cs.procStreamFailed.Done(): + return false + case <-cs.ctx.Done(): + return false + } + } +} + +// acquireUpstreamToSidestreamWindow checks available flow control window for +// upstream to sidestream. If window is positive, it deducts bodySize and +// returns true immediately. If window is <= 0, it blocks until a positive +// window update is received, or returns false if bypassed, failed, or context +// is done. +func (cs *clientStream) acquireUpstreamToSidestreamWindow(bodySize int64) bool { + // Re-check the window in a loop. A signal on the buffered channel may be + // stale if a previous caller acquired the positive window on the fast path + // without draining the channel. + for { + if cs.upstreamToSidestreamWindow.Load() > 0 { + cs.upstreamToSidestreamWindow.Add(-bodySize) + return true + } + select { + case <-cs.upstreamToSidestreamPositiveUpdateCh: + continue + case <-cs.procStreamBypass.Done(): + return false + case <-cs.procStreamFailed.Done(): + return false + case <-cs.ctx.Done(): + return false + } + } +} + +// sendClientWindowUpdate pushes a standalone ClientWindowUpdate +// ProcessingRequest to procSendCh to replenish the sidestream to downstream and +// sidestream to upstream flow control windows. +func (cs *clientStream) sendClientWindowUpdate() error { + respInc := cs.pendingRespWindowUpdate.Swap(0) + reqInc := cs.pendingReqWindowUpdate.Swap(0) + if respInc == 0 && reqInc == 0 { + return nil + } + + req := &v3procservicepb.ProcessingRequest{ + ClientWindowUpdate: &v3procservicepb.ProcessingRequest_ClientWindowUpdate{ + WindowIncrementSidestreamToDownstream: respInc, + WindowIncrementSidestreamToUpstream: reqInc, + }, + } + + select { + case cs.procSendCh <- req: + return nil + case <-cs.procStreamBypass.Done(): + return nil + case <-cs.procStreamFailed.Done(): + return cs.streamError() + case <-cs.ctx.Done(): + return cs.streamError() + } +} + +// convertBodyMode converts the body mode from processingMode to +// v3procfilterpb.ProcessingMode_BodySendMode. +func convertBodyMode(mode processingMode) v3procfilterpb.ProcessingMode_BodySendMode { + if mode == modeSend { + return v3procfilterpb.ProcessingMode_GRPC + } + return v3procfilterpb.ProcessingMode_NONE +} + +func getHeader(md metadata.MD, added [][]string, key string) string { + var vals []string + if md != nil { + vals = append(vals, md.Get(key)...) + } + for _, kvs := range added { + for i := 0; i < len(kvs); i += 2 { + if strings.EqualFold(kvs[i], key) { + vals = append(vals, kvs[i+1]) + } + } + } + return strings.Join(vals, ",") +} + +// constructRequestAttributes builds a map of request attributes specified in +// the interceptor config. +func constructRequestAttributes(rpcInfo resolver.RPCInfo, md metadata.MD, added [][]string, requestedAttributes []string) map[string]*structpb.Struct { + if len(requestedAttributes) == 0 { + return nil + } + + reqFields := make(map[string]any) + for _, attr := range requestedAttributes { + switch attr { + case "request.path", "request.url_path": + reqFields[attr] = rpcInfo.Method + case "request.host": + reqFields[attr] = rpcInfo.Authority + case "request.method": + reqFields[attr] = "POST" + case "request.headers": + headers := make(map[string]any) + for k, values := range md { + headers[strings.ToLower(k)] = strings.Join(values, ",") + } + for _, kvs := range added { + for i := 0; i < len(kvs); i += 2 { + k := strings.ToLower(kvs[i]) + v := kvs[i+1] + if prev, ok := headers[k].(string); ok { + headers[k] = prev + "," + v + } else { + headers[k] = v + } + } + } + reqFields[attr] = headers + case "request.referer": + if val := getHeader(md, added, "referer"); val != "" { + reqFields[attr] = val + } + case "request.useragent": + if val := getHeader(md, added, "user-agent"); val != "" { + reqFields[attr] = val + } + case "request.id": + if val := getHeader(md, added, "x-request-id"); val != "" { + reqFields[attr] = val + } + case "request.query": + reqFields[attr] = "" + default: + // Unknown/unsupported attributes are ignored. + } + } + + reqStruct := &structpb.Struct{Fields: make(map[string]*structpb.Value)} + for k, v := range reqFields { + val, err := structpb.NewValue(v) + if err != nil { + // Skip values that cannot be represented in a structpb.Value to + // match Envoy's behavior of encoding as many attributes as possible. + continue + } + reqStruct.Fields[k] = val + } + + if len(reqStruct.Fields) == 0 { + return nil + } + + return map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": reqStruct, + } +} diff --git a/internal/xds/httpfilter/extproc/internal/internal.go b/internal/xds/httpfilter/extproc/internal/internal.go new file mode 100644 index 0000000..d37299f --- /dev/null +++ b/internal/xds/httpfilter/extproc/internal/internal.go @@ -0,0 +1,61 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains functionality internal to the extproc package. +package internal + +import ( + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/internal/xds/grpcservice" +) + +var ( + // CreateExtProcChannel creates the channel to the external processing + // server described by the given config. The returned function closes the + // channel. It is a variable so that tests can intercept channel creation + // and observe its release. + CreateExtProcChannel = func(server *grpcservice.Config) (grpc.ClientConnInterface, func(), error) { + conn, err := server.Dial() + if err != nil { + return nil, nil, err + } + return conn, func() { conn.Close() }, nil + } + + // RegisterForTesting registers the external processor HTTP Filter for testing + // purposes. + RegisterForTesting func() + + // UnregisterForTesting unregisters the external processor HTTP Filter for + // testing purposes. + UnregisterForTesting func() + + // TimeNowFunc returns the current time.Time, and can be overridden for + // testing purposes. + TimeNowFunc func() time.Time + + // TimeSinceFunc returns the time elapsed, and can be overridden for testing + // purposes. + TimeSinceFunc func(t time.Time) time.Duration + + // DefaultFlowControlWindowSize is the initial flow control window size for + // the external processor filter and can be overridden for testing purposes. + DefaultFlowControlWindowSize int64 = 64 * 1024 +) diff --git a/internal/xds/httpfilter/extproc/metrics.go b/internal/xds/httpfilter/extproc/metrics.go new file mode 100644 index 0000000..40dd49e --- /dev/null +++ b/internal/xds/httpfilter/extproc/metrics.go @@ -0,0 +1,54 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package extproc + +import ( + estats "google.golang.org/grpc/experimental/stats" +) + +var ( + clientHeadersDurationMetric = estats.RegisterFloat64Histo(estats.MetricDescriptor{ + Name: "grpc.client_ext_proc.client_headers_duration", + Description: "Time between when the ext_proc filter sees the client's headers and when it allows those headers to continue on to the next filter.", + Unit: "s", + Labels: []string{"grpc.target"}, + Default: false, + }) + clientHalfCloseDurationMetric = estats.RegisterFloat64Histo(estats.MetricDescriptor{ + Name: "grpc.client_ext_proc.client_half_close_duration", + Description: "Time between when the ext_proc filter sees the client's half-close and when it allows that half-close to continue on to the next filter.", + Unit: "s", + Labels: []string{"grpc.target"}, + Default: false, + }) + serverHeadersDurationMetric = estats.RegisterFloat64Histo(estats.MetricDescriptor{ + Name: "grpc.client_ext_proc.server_headers_duration", + Description: "Time between when the ext_proc filter sees the server's headers and when it allows those headers to continue on to the next filter.", + Unit: "s", + Labels: []string{"grpc.target"}, + Default: false, + }) + serverTrailersDurationMetric = estats.RegisterFloat64Histo(estats.MetricDescriptor{ + Name: "grpc.client_ext_proc.server_trailers_duration", + Description: "Time between when the ext_proc filter sees the server's trailers and when it allows those trailers to continue on to the next filter.", + Unit: "s", + Labels: []string{"grpc.target"}, + Default: false, + }) +) diff --git a/internal/xds/httpfilter/fault/fault.go b/internal/xds/httpfilter/fault/fault.go new file mode 100644 index 0000000..3bb7aa5 --- /dev/null +++ b/internal/xds/httpfilter/fault/fault.go @@ -0,0 +1,311 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package fault implements the Envoy Fault Injection HTTP filter. +package fault + +import ( + "context" + "errors" + "fmt" + "io" + rand "math/rand/v2" + "strconv" + "sync/atomic" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + cpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/fault/v3" + fpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/fault/v3" + tpb "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +const headerAbortHTTPStatus = "x-envoy-fault-abort-request" +const headerAbortGRPCStatus = "x-envoy-fault-abort-grpc-request" +const headerAbortPercentage = "x-envoy-fault-abort-request-percentage" + +const headerDelayPercentage = "x-envoy-fault-delay-request-percentage" +const headerDelayDuration = "x-envoy-fault-delay-request" + +var statusMap = map[int]codes.Code{ + 400: codes.Internal, + 401: codes.Unauthenticated, + 403: codes.PermissionDenied, + 404: codes.Unimplemented, + 429: codes.Unavailable, + 502: codes.Unavailable, + 503: codes.Unavailable, + 504: codes.Unavailable, +} + +func init() { + httpfilter.Register(builder{}) +} + +type builder struct { +} + +type config struct { + httpfilter.FilterConfig + config *fpb.HTTPFault +} + +func (builder) TypeURLs() []string { + return []string{"type.googleapis.com/envoy.extensions.filters.http.fault.v3.HTTPFault"} +} + +// Parsing is the same for the base config and the override config. +func parseConfig(cfg proto.Message) (httpfilter.FilterConfig, error) { + if cfg == nil { + return nil, fmt.Errorf("fault: nil configuration message provided") + } + m, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("fault: error parsing config %v: unknown type %T", cfg, cfg) + } + msg := new(fpb.HTTPFault) + if err := m.UnmarshalTo(msg); err != nil { + return nil, fmt.Errorf("fault: error parsing config %v: %v", cfg, err) + } + return config{config: msg}, nil +} + +func (builder) ParseFilterConfig(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return parseConfig(cfg) +} + +func (builder) ParseFilterConfigOverride(override proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return parseConfig(override) +} + +func (builder) IsTerminal() bool { + return false +} + +func (builder) BuildClientFilter(httpfilter.ClientFilterOptions) httpfilter.ClientFilter { + return clientFilter{} +} + +var _ httpfilter.ClientFilterBuilder = builder{} + +type clientFilter struct{} + +func (clientFilter) Close() {} + +func (clientFilter) BuildClientInterceptor(cfg, override httpfilter.FilterConfig) (httpfilter.ClientInterceptor, error) { + if cfg == nil { + return nil, fmt.Errorf("fault: nil config provided") + } + + c, ok := cfg.(config) + if !ok { + return nil, fmt.Errorf("fault: incorrect config type provided (%T): %v", cfg, cfg) + } + + if override != nil { + // override completely replaces the listener configuration; but we + // still validate the listener config type. + c, ok = override.(config) + if !ok { + return nil, fmt.Errorf("fault: incorrect override config type provided (%T): %v", override, override) + } + } + + icfg := c.config + if (icfg.GetMaxActiveFaults() != nil && icfg.GetMaxActiveFaults().GetValue() == 0) || + (icfg.GetDelay() == nil && icfg.GetAbort() == nil) { + return nil, nil + } + return &interceptor{config: icfg}, nil +} + +type interceptor struct { + config *fpb.HTTPFault +} + +var activeFaults uint32 // global active faults; accessed atomically + +func (i *interceptor) NewStream(ctx context.Context, _ iresolver.RPCInfo, newStream func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) { + if maxAF := i.config.GetMaxActiveFaults(); maxAF != nil { + defer atomic.AddUint32(&activeFaults, ^uint32(0)) // decrement counter + if af := atomic.AddUint32(&activeFaults, 1); af > maxAF.GetValue() { + // Would exceed maximum active fault limit. + return newStream(ctx, opts...) + } + } + + if err := injectDelay(ctx, i.config.GetDelay()); err != nil { + return nil, err + } + + if err := injectAbort(ctx, i.config.GetAbort()); err != nil { + if err == errOKStream { + return &okStream{ctx: ctx}, nil + } + return nil, err + } + return newStream(ctx, opts...) +} + +func (i *interceptor) Close() {} + +// For overriding in tests +var randIntn = rand.IntN +var newTimer = time.NewTimer + +func injectDelay(ctx context.Context, delayCfg *cpb.FaultDelay) error { + numerator, denominator := splitPct(delayCfg.GetPercentage()) + var delay time.Duration + switch delayType := delayCfg.GetFaultDelaySecifier().(type) { + case *cpb.FaultDelay_FixedDelay: + delay = delayType.FixedDelay.AsDuration() + case *cpb.FaultDelay_HeaderDelay_: + md, _ := metadata.FromOutgoingContext(ctx) + v := md[headerDelayDuration] + if v == nil { + // No delay configured for this RPC. + return nil + } + ms, ok := parseIntFromMD(v) + if !ok { + // Malformed header; no delay. + return nil + } + delay = time.Duration(ms) * time.Millisecond + if v := md[headerDelayPercentage]; v != nil { + if num, ok := parseIntFromMD(v); ok && num < numerator { + numerator = num + } + } + } + if delay == 0 || randIntn(denominator) >= numerator { + return nil + } + t := newTimer(delay) + select { + case <-t.C: + case <-ctx.Done(): + t.Stop() + return ctx.Err() + } + return nil +} + +func injectAbort(ctx context.Context, abortCfg *fpb.FaultAbort) error { + numerator, denominator := splitPct(abortCfg.GetPercentage()) + code := codes.OK + okCode := false + switch errType := abortCfg.GetErrorType().(type) { + case *fpb.FaultAbort_HttpStatus: + code, okCode = grpcFromHTTP(int(errType.HttpStatus)) + case *fpb.FaultAbort_GrpcStatus: + code, okCode = sanitizeGRPCCode(codes.Code(errType.GrpcStatus)), true + case *fpb.FaultAbort_HeaderAbort_: + md, _ := metadata.FromOutgoingContext(ctx) + if v := md[headerAbortHTTPStatus]; v != nil { + // HTTP status has priority over gRPC status. + if httpStatus, ok := parseIntFromMD(v); ok { + code, okCode = grpcFromHTTP(httpStatus) + } + } else if v := md[headerAbortGRPCStatus]; v != nil { + if grpcStatus, ok := parseIntFromMD(v); ok { + code, okCode = sanitizeGRPCCode(codes.Code(grpcStatus)), true + } + } + if v := md[headerAbortPercentage]; v != nil { + if num, ok := parseIntFromMD(v); ok && num < numerator { + numerator = num + } + } + } + if !okCode || randIntn(denominator) >= numerator { + return nil + } + if code == codes.OK { + return errOKStream + } + return status.Errorf(code, "RPC terminated due to fault injection") +} + +var errOKStream = errors.New("stream terminated early with OK status") + +// parseIntFromMD returns the integer in the last header or nil if parsing +// failed. +func parseIntFromMD(header []string) (int, bool) { + if len(header) == 0 { + return 0, false + } + v, err := strconv.Atoi(header[len(header)-1]) + return v, err == nil +} + +func splitPct(fp *tpb.FractionalPercent) (num int, den int) { + if fp == nil { + return 0, 100 + } + num = int(fp.GetNumerator()) + switch fp.GetDenominator() { + case tpb.FractionalPercent_HUNDRED: + return num, 100 + case tpb.FractionalPercent_TEN_THOUSAND: + return num, 10 * 1000 + case tpb.FractionalPercent_MILLION: + return num, 1000 * 1000 + } + return num, 100 +} + +func grpcFromHTTP(httpStatus int) (codes.Code, bool) { + if httpStatus < 200 || httpStatus >= 600 { + // Malformed; ignore this fault type. + return codes.OK, false + } + if c := statusMap[httpStatus]; c != codes.OK { + // OK = 0/the default for the map. + return c, true + } + // All undefined HTTP status codes convert to Unknown. HTTP status of 200 + // is "success", but gRPC converts to Unknown due to missing grpc status. + return codes.Unknown, true +} + +func sanitizeGRPCCode(c codes.Code) codes.Code { + if c > codes.Code(16) { + return codes.Unknown + } + return c +} + +type okStream struct { + ctx context.Context +} + +func (*okStream) Header() (metadata.MD, error) { return nil, nil } +func (*okStream) Trailer() metadata.MD { return nil } +func (*okStream) CloseSend() error { return nil } +func (o *okStream) Context() context.Context { return o.ctx } +func (*okStream) SendMsg(any) error { return io.EOF } +func (*okStream) RecvMsg(any) error { return io.EOF } diff --git a/internal/xds/httpfilter/fault/fault_test.go b/internal/xds/httpfilter/fault/fault_test.go new file mode 100644 index 0000000..1f77ad3 --- /dev/null +++ b/internal/xds/httpfilter/fault/fault_test.go @@ -0,0 +1,661 @@ +//go:build !386 +// +build !386 + +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xds_test contains e2e tests for xDS use. +package fault + +import ( + "context" + "fmt" + "io" + rand "math/rand/v2" + "net" + "reflect" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + cpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/fault/v3" + fpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/fault/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + tpb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/internal/xds/balancer" // Register the balancers. + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter. + _ "google.golang.org/grpc/internal/xds/resolver" // Register the xds_resolver. +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// clientSetup performs a bunch of steps common to all xDS server tests here: +// - spin up an xDS management server on a local port +// - spin up a gRPC server and register the test service on it +// - create a local TCP listener and start serving on it +// +// Returns the following: +// - the management server: tests use this to configure resources +// - nodeID expected by the management server: this is set in the Node proto +// sent by the xdsClient for queries +// - the port the server is listening on +// - contents of the bootstrap configuration pointing to xDS management +// server +func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, []byte) { + // Spin up a xDS management server on a local port. + nodeID := uuid.New().String() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create a bootstrap config pointing to the above management server with + // the nodeID. + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create a local listener. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + + // Initialize a test gRPC server, assign it to the stub server, and start the test service. + stub := &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + // End RPC after client does a CloseSend. + for { + if _, err := stream.Recv(); err == io.EOF { + return nil + } else if err != nil { + return err + } + } + }, + } + + stubserver.StartTestService(t, stub) + t.Cleanup(stub.S.Stop) + return managementServer, nodeID, uint32(lis.Addr().(*net.TCPAddr).Port), bootstrapContents +} + +func (s) TestFaultInjection_Unary(t *testing.T) { + type subcase struct { + name string + code codes.Code + repeat int + randIn []int // Intn calls per-repeat (not per-subcase) + delays []time.Duration // NewTimer calls per-repeat (not per-subcase) + md metadata.MD + } + testCases := []struct { + name string + cfgs []*fpb.HTTPFault + randOutInc int + want []subcase + }{{ + name: "max faults zero", + cfgs: []*fpb.HTTPFault{{ + MaxActiveFaults: wrapperspb.UInt32(0), + Abort: &fpb.FaultAbort{ + Percentage: &tpb.FractionalPercent{Numerator: 100, Denominator: tpb.FractionalPercent_HUNDRED}, + ErrorType: &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Aborted)}, + }, + }}, + randOutInc: 5, + want: []subcase{{ + code: codes.OK, + repeat: 25, + }}, + }, { + name: "no abort or delay", + cfgs: []*fpb.HTTPFault{{}}, + randOutInc: 5, + want: []subcase{{ + code: codes.OK, + repeat: 25, + }}, + }, { + name: "abort always", + cfgs: []*fpb.HTTPFault{{ + Abort: &fpb.FaultAbort{ + Percentage: &tpb.FractionalPercent{Numerator: 100, Denominator: tpb.FractionalPercent_HUNDRED}, + ErrorType: &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Aborted)}, + }, + }}, + randOutInc: 5, + want: []subcase{{ + code: codes.Aborted, + randIn: []int{100}, + repeat: 25, + }}, + }, { + name: "abort 10%", + cfgs: []*fpb.HTTPFault{{ + Abort: &fpb.FaultAbort{ + Percentage: &tpb.FractionalPercent{Numerator: 100000, Denominator: tpb.FractionalPercent_MILLION}, + ErrorType: &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Aborted)}, + }, + }}, + randOutInc: 50000, + want: []subcase{{ + name: "[0,10]%", + code: codes.Aborted, + randIn: []int{1000000}, + repeat: 2, + }, { + name: "(10,100]%", + code: codes.OK, + randIn: []int{1000000}, + repeat: 18, + }, { + name: "[0,10]% again", + code: codes.Aborted, + randIn: []int{1000000}, + repeat: 2, + }}, + }, { + name: "delay always", + cfgs: []*fpb.HTTPFault{{ + Delay: &cpb.FaultDelay{ + Percentage: &tpb.FractionalPercent{Numerator: 100, Denominator: tpb.FractionalPercent_HUNDRED}, + FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: durationpb.New(time.Second)}, + }, + }}, + randOutInc: 5, + want: []subcase{{ + randIn: []int{100}, + repeat: 25, + delays: []time.Duration{time.Second}, + }}, + }, { + name: "delay 10%", + cfgs: []*fpb.HTTPFault{{ + Delay: &cpb.FaultDelay{ + Percentage: &tpb.FractionalPercent{Numerator: 1000, Denominator: tpb.FractionalPercent_TEN_THOUSAND}, + FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: durationpb.New(time.Second)}, + }, + }}, + randOutInc: 500, + want: []subcase{{ + name: "[0,10]%", + randIn: []int{10000}, + repeat: 2, + delays: []time.Duration{time.Second}, + }, { + name: "(10,100]%", + randIn: []int{10000}, + repeat: 18, + }, { + name: "[0,10]% again", + randIn: []int{10000}, + repeat: 2, + delays: []time.Duration{time.Second}, + }}, + }, { + name: "delay 80%, abort 50%", + cfgs: []*fpb.HTTPFault{{ + Delay: &cpb.FaultDelay{ + Percentage: &tpb.FractionalPercent{Numerator: 80, Denominator: tpb.FractionalPercent_HUNDRED}, + FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: durationpb.New(3 * time.Second)}, + }, + Abort: &fpb.FaultAbort{ + Percentage: &tpb.FractionalPercent{Numerator: 50, Denominator: tpb.FractionalPercent_HUNDRED}, + ErrorType: &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Unimplemented)}, + }, + }}, + randOutInc: 5, + want: []subcase{{ + name: "50% delay and abort", + code: codes.Unimplemented, + randIn: []int{100, 100}, + repeat: 10, + delays: []time.Duration{3 * time.Second}, + }, { + name: "30% delay, no abort", + randIn: []int{100, 100}, + repeat: 6, + delays: []time.Duration{3 * time.Second}, + }, { + name: "20% success", + randIn: []int{100, 100}, + repeat: 4, + }, { + name: "50% delay and abort again", + code: codes.Unimplemented, + randIn: []int{100, 100}, + repeat: 10, + delays: []time.Duration{3 * time.Second}, + }}, + }, { + name: "header abort", + cfgs: []*fpb.HTTPFault{{ + Abort: &fpb.FaultAbort{ + Percentage: &tpb.FractionalPercent{Numerator: 80, Denominator: tpb.FractionalPercent_HUNDRED}, + ErrorType: &fpb.FaultAbort_HeaderAbort_{}, + }, + }}, + randOutInc: 10, + want: []subcase{{ + name: "30% abort; [0,30]%", + md: metadata.MD{ + headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)}, + headerAbortPercentage: []string{"30"}, + }, + code: codes.DataLoss, + randIn: []int{100}, + repeat: 3, + }, { + name: "30% abort; (30,60]%", + md: metadata.MD{ + headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)}, + headerAbortPercentage: []string{"30"}, + }, + randIn: []int{100}, + repeat: 3, + }, { + name: "80% abort; (60,80]%", + md: metadata.MD{ + headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)}, + headerAbortPercentage: []string{"80"}, + }, + code: codes.DataLoss, + randIn: []int{100}, + repeat: 2, + }, { + name: "cannot exceed percentage in filter", + md: metadata.MD{ + headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)}, + headerAbortPercentage: []string{"100"}, + }, + randIn: []int{100}, + repeat: 2, + }, { + name: "HTTP Status 404", + md: metadata.MD{ + headerAbortHTTPStatus: []string{"404"}, + headerAbortPercentage: []string{"100"}, + }, + code: codes.Unimplemented, + randIn: []int{100}, + repeat: 1, + }, { + name: "HTTP Status 429", + md: metadata.MD{ + headerAbortHTTPStatus: []string{"429"}, + headerAbortPercentage: []string{"100"}, + }, + code: codes.Unavailable, + randIn: []int{100}, + repeat: 1, + }, { + name: "HTTP Status 200", + md: metadata.MD{ + headerAbortHTTPStatus: []string{"200"}, + headerAbortPercentage: []string{"100"}, + }, + // No GRPC status, but HTTP Status of 200 translates to Unknown, + // per spec in statuscodes.md. + code: codes.Unknown, + randIn: []int{100}, + repeat: 1, + }, { + name: "gRPC Status OK", + md: metadata.MD{ + headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.OK)}, + headerAbortPercentage: []string{"100"}, + }, + // This should be Unimplemented (mismatched request/response + // count), per spec in statuscodes.md, but grpc-go currently + // returns io.EOF which status.Code() converts to Unknown + code: codes.Unknown, + randIn: []int{100}, + repeat: 1, + }, { + name: "invalid header results in no abort", + md: metadata.MD{ + headerAbortGRPCStatus: []string{"error"}, + headerAbortPercentage: []string{"100"}, + }, + repeat: 1, + }, { + name: "invalid header results in default percentage", + md: metadata.MD{ + headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)}, + headerAbortPercentage: []string{"error"}, + }, + code: codes.DataLoss, + randIn: []int{100}, + repeat: 1, + }}, + }, { + name: "header delay", + cfgs: []*fpb.HTTPFault{{ + Delay: &cpb.FaultDelay{ + Percentage: &tpb.FractionalPercent{Numerator: 80, Denominator: tpb.FractionalPercent_HUNDRED}, + FaultDelaySecifier: &cpb.FaultDelay_HeaderDelay_{}, + }, + }}, + randOutInc: 10, + want: []subcase{{ + name: "30% delay; [0,30]%", + md: metadata.MD{ + headerDelayDuration: []string{"2"}, + headerDelayPercentage: []string{"30"}, + }, + randIn: []int{100}, + delays: []time.Duration{2 * time.Millisecond}, + repeat: 3, + }, { + name: "30% delay; (30, 60]%", + md: metadata.MD{ + headerDelayDuration: []string{"2"}, + headerDelayPercentage: []string{"30"}, + }, + randIn: []int{100}, + repeat: 3, + }, { + name: "invalid header results in no delay", + md: metadata.MD{ + headerDelayDuration: []string{"error"}, + headerDelayPercentage: []string{"80"}, + }, + repeat: 1, + }, { + name: "invalid header results in default percentage", + md: metadata.MD{ + headerDelayDuration: []string{"2"}, + headerDelayPercentage: []string{"error"}, + }, + randIn: []int{100}, + delays: []time.Duration{2 * time.Millisecond}, + repeat: 1, + }, { + name: "invalid header results in default percentage", + md: metadata.MD{ + headerDelayDuration: []string{"2"}, + headerDelayPercentage: []string{"error"}, + }, + randIn: []int{100}, + repeat: 1, + }, { + name: "cannot exceed percentage in filter", + md: metadata.MD{ + headerDelayDuration: []string{"2"}, + headerDelayPercentage: []string{"100"}, + }, + randIn: []int{100}, + repeat: 1, + }}, + }, { + name: "abort then delay filters", + cfgs: []*fpb.HTTPFault{{ + Abort: &fpb.FaultAbort{ + Percentage: &tpb.FractionalPercent{Numerator: 50, Denominator: tpb.FractionalPercent_HUNDRED}, + ErrorType: &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Unimplemented)}, + }, + }, { + Delay: &cpb.FaultDelay{ + Percentage: &tpb.FractionalPercent{Numerator: 80, Denominator: tpb.FractionalPercent_HUNDRED}, + FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: durationpb.New(time.Second)}, + }, + }}, + randOutInc: 10, + want: []subcase{{ + name: "50% delay and abort (abort skips delay)", + code: codes.Unimplemented, + randIn: []int{100}, + repeat: 5, + }, { + name: "30% delay, no abort", + randIn: []int{100, 100}, + repeat: 3, + delays: []time.Duration{time.Second}, + }, { + name: "20% success", + randIn: []int{100, 100}, + repeat: 2, + }}, + }} + + fs, nodeID, port, bc := clientSetup(t) + // Create an xDS resolver with the above bootstrap configuration. + xdsResolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + for tcNum, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + defer func() { randIntn = rand.IntN; newTimer = time.NewTimer }() + var intnCalls []int + var newTimerCalls []time.Duration + randOut := 0 + randIntn = func(n int) int { + intnCalls = append(intnCalls, n) + return randOut % n + } + + newTimer = func(d time.Duration) *time.Timer { + newTimerCalls = append(newTimerCalls, d) + return time.NewTimer(0) + } + + serviceName := fmt.Sprintf("myservice%d", tcNum) + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + hcm := new(v3httppb.HttpConnectionManager) + lis := resources.Listeners[0].GetApiListener().GetApiListener() + err := lis.UnmarshalTo(hcm) + if err != nil { + t.Fatal(err) + } + routerFilter := hcm.HttpFilters[len(hcm.HttpFilters)-1] + + hcm.HttpFilters = nil + for i, cfg := range tc.cfgs { + hcm.HttpFilters = append(hcm.HttpFilters, e2e.HTTPFilter(fmt.Sprintf("fault%d", i), cfg)) + } + hcm.HttpFilters = append(hcm.HttpFilters, routerFilter) + hcmAny := testutils.MarshalAny(t, hcm) + resources.Listeners[0].ApiListener.ApiListener = hcmAny + resources.Listeners[0].FilterChains[0].Filters[0].ConfigType = &v3listenerpb.Filter_TypedConfig{TypedConfig: hcmAny} + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := fs.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and run the test case. + cc, err := grpc.NewClient("xds:///"+serviceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + count := 0 + for _, want := range tc.want { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if want.repeat == 0 { + t.Fatalf("invalid repeat count") + } + for n := 0; n < want.repeat; n++ { + intnCalls = nil + newTimerCalls = nil + ctx = metadata.NewOutgoingContext(ctx, want.md) + _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)) + t.Logf("%v: RPC %d: err: %v, intnCalls: %v, newTimerCalls: %v", want.name, count, err, intnCalls, newTimerCalls) + if status.Code(err) != want.code || !reflect.DeepEqual(intnCalls, want.randIn) || !reflect.DeepEqual(newTimerCalls, want.delays) { + t.Fatalf("WANTED code: %v, intnCalls: %v, newTimerCalls: %v", want.code, want.randIn, want.delays) + } + randOut += tc.randOutInc + count++ + } + } + }) + } +} + +func (s) TestFaultInjection_MaxActiveFaults(t *testing.T) { + fs, nodeID, port, bc := clientSetup(t) + // Create an xDS resolver with the above bootstrap configuration. + xdsResolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: "myservice", + NodeID: nodeID, + Host: "localhost", + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + hcm := new(v3httppb.HttpConnectionManager) + lis := resources.Listeners[0].GetApiListener().GetApiListener() + if err = lis.UnmarshalTo(hcm); err != nil { + t.Fatal(err) + } + + defer func() { newTimer = time.NewTimer }() + timers := make(chan *time.Timer, 2) + newTimer = func(time.Duration) *time.Timer { + t := time.NewTimer(24 * time.Hour) // Will reset to fire. + timers <- t + return t + } + + hcm.HttpFilters = append([]*v3httppb.HttpFilter{ + e2e.HTTPFilter("fault", &fpb.HTTPFault{ + MaxActiveFaults: wrapperspb.UInt32(2), + Delay: &cpb.FaultDelay{ + Percentage: &tpb.FractionalPercent{Numerator: 100, Denominator: tpb.FractionalPercent_HUNDRED}, + FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: durationpb.New(time.Second)}, + }, + })}, + hcm.HttpFilters...) + hcmAny := testutils.MarshalAny(t, hcm) + resources.Listeners[0].ApiListener.ApiListener = hcmAny + resources.Listeners[0].FilterChains[0].Filters[0].ConfigType = &v3listenerpb.Filter_TypedConfig{TypedConfig: hcmAny} + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := fs.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn + cc, err := grpc.NewClient("xds:///myservice", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + streams := make(chan testgrpc.TestService_FullDuplexCallClient, 5) // startStream() is called 5 times + startStream := func() { + str, err := client.FullDuplexCall(ctx) + if err != nil { + t.Error("RPC error:", err) + } + streams <- str + } + endStream := func() { + str := <-streams + str.CloseSend() + if _, err := str.Recv(); err != io.EOF { + t.Error("stream error:", err) + } + } + releaseStream := func() { + timer := <-timers + timer.Reset(0) + } + + // Start three streams; two should delay. + go startStream() + go startStream() + go startStream() + + // End one of the streams. Ensure the others are blocked on creation. + endStream() + + select { + case <-streams: + t.Errorf("unexpected second stream created before delay expires") + case <-time.After(50 * time.Millisecond): + // Wait a short time to ensure no other streams were started yet. + } + + // Start one more; it should not be blocked. + go startStream() + endStream() + + // Expire one stream's delay; it should be created. + releaseStream() + endStream() + + // Another new stream should delay. + go startStream() + select { + case <-streams: + t.Errorf("unexpected second stream created before delay expires") + case <-time.After(50 * time.Millisecond): + // Wait a short time to ensure no other streams were started yet. + } + + // Expire both pending timers and end the two streams. + releaseStream() + releaseStream() + endStream() + endStream() +} diff --git a/internal/xds/httpfilter/gcp_authn/gcp_authn_filter.go b/internal/xds/httpfilter/gcp_authn/gcp_authn_filter.go new file mode 100644 index 0000000..86fc308 --- /dev/null +++ b/internal/xds/httpfilter/gcp_authn/gcp_authn_filter.go @@ -0,0 +1,333 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package gcpauthn implements the GCP Authentication HTTP filter. +package gcpauthn + +import ( + "container/list" + "context" + "fmt" + "strings" + "sync" + + "golang.org/x/sync/singleflight" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/xds/balancer/clustermanager" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + v3gcpauthnpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/gcp_authn/v3" +) + +const defaultCacheSize = 10 // default capacity of the LRU credentials cache. + +func init() { + httpfilter.Register(builder{}) +} + +type builder struct{} + +type config struct { + httpfilter.FilterConfig + cacheSize uint64 +} + +func (builder) TypeURLs() []string { + return []string{"type.googleapis.com/envoy.extensions.filters.http.gcp_authn.v3.GcpAuthnFilterConfig"} +} + +func (builder) ParseFilterConfig(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + m, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("gcpauthn: invalid filter config type %T", cfg) + } + msg := &v3gcpauthnpb.GcpAuthnFilterConfig{} + if err := m.UnmarshalTo(msg); err != nil { + return nil, fmt.Errorf("gcpauthn: failed to unmarshal filter config: %v", err) + } + + cacheSize := uint64(defaultCacheSize) + if cacheSizeConfig := msg.GetCacheConfig().GetCacheSize(); cacheSizeConfig != nil { + if cacheSize = cacheSizeConfig.GetValue(); cacheSize == 0 { + return nil, fmt.Errorf("gcpauthn: cache_config.cache_size must be greater than zero") + } + } + + return config{cacheSize: cacheSize}, nil +} + +// ParseFilterConfigOverride parses the provided override configuration. +// +// Note that we don't support overrides for this filter configuration, +// but still validate it as part of the normal resource validation. +func (b builder) ParseFilterConfigOverride(cfg proto.Message, opts httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return b.ParseFilterConfig(cfg, opts) +} + +func (builder) IsTerminal() bool { + return false +} + +func (builder) BuildClientFilter(opts httpfilter.ClientFilterOptions) httpfilter.ClientFilter { + ctx, cancel := context.WithCancel(context.Background()) + return &clientFilter{ + ctx: ctx, + cancel: cancel, + filterName: opts.FilterName, + } +} + +var _ httpfilter.ClientFilterBuilder = builder{} + +// clientFilter implements the httpfilter.ClientFilter interface. +type clientFilter struct { + // ctx is initialized using context.Background() and is scoped to the + // lifetime of the filter. It is used as the parent context for fetching + // service account identity credentials, ensuring that token fetch requests + // are not terminated by individual RPC's context. + ctx context.Context + + // cancel is the cancellation function for ctx, called when the filter + // is closed. + cancel context.CancelFunc + + // filterName is the name of the HTTP filter instance in the xDS + // configuration. It is used as the key in the cluster metadata to look up + // the audience value for the cluster to which the RPC is destined. + filterName string + + // cache is the LRU cache of PerRPCCredentials instances, keyed by audience + // and is initialized or resized when BuildClientInterceptor is called. + cache *lruCache +} + +// BuildClientInterceptor builds a client interceptor for the GCP +// Authentication filter. +func (cf *clientFilter) BuildClientInterceptor(cfg, _ httpfilter.FilterConfig) (httpfilter.ClientInterceptor, error) { + c, ok := cfg.(config) + if !ok { + return nil, fmt.Errorf("gcpauthn: invalid filter config type %T", cfg) + } + + if cf.cache == nil { + cf.cache = newLRUCache(c.cacheSize) + } else { + cf.cache.resizeCache(c.cacheSize) + } + + return &interceptor{ + ctx: cf.ctx, + filterName: cf.filterName, + cache: cf.cache, + }, nil +} + +// Close closes the client filter. +func (cf *clientFilter) Close() { + cf.cancel() +} + +type interceptor struct { + ctx context.Context + filterName string + cache *lruCache +} + +func (i *interceptor) NewStream(ctx context.Context, _ resolver.RPCInfo, newStream func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) { + clusterName := clustermanager.PickedCluster(ctx) + // The picked cluster name in the context is formatted by the xDS + // resolver with a prefix to distinguish between standard CDS clusters and + // cluster_specifier_plugins: + // - cluster_specifier_plugin: Since the target cluster is dynamically + // resolved later at the load balancing layer, the final cluster name is + // not yet known at this stage, hence we bypass this filter. + // - cluster: Standard CDS routing where the destination cluster is + // known at routing time. We strip the prefix to get the raw cluster name, + // which is used to look up the cluster configuration in the CDS response. + if strings.HasPrefix(clusterName, "cluster_specifier_plugin:") { + return newStream(ctx, opts...) + } + clusterName = strings.TrimPrefix(clusterName, "cluster:") + + cfg := xdsresource.XDSConfigFromContext(ctx) + if cfg == nil { + return nil, status.Errorf(codes.Unavailable, "gcpauthn: xDS config not found in context") + } + + clusterResult, ok := cfg.Clusters[clusterName] + if !ok { + return nil, status.Errorf(codes.Unavailable, "gcpauthn: cluster %q not found in xDS config", clusterName) + } + + if clusterResult.Err != nil { + return nil, status.Errorf(codes.Unavailable, "gcpauthn: cluster config for %q is invalid or missing: %v", clusterName, clusterResult.Err) + } + + val, ok := clusterResult.Config.Cluster.Metadata[i.filterName] + if !ok { + return newStream(ctx, opts...) + } + + audienceMetadata, ok := val.(xdsresource.AudienceMetadataValue) + if !ok { + return nil, status.Errorf(codes.Unavailable, "gcpauthn: cluster metadata for key %q is not of type AudienceMetadataValue, got %T", i.filterName, val) + } + + creds, err := i.cache.getOrCreate(i.ctx, audienceMetadata.Audience) + if err != nil { + return nil, status.Errorf(codes.Unavailable, "gcpauthn: failed to create credentials: %v", err) + } + + // We pass the credentials via a PerRPCCredentials call option rather than + // directly attaching the token here. Since this filter runs before load + // balancer has selected a connection, it cannot check if the connection is + // secure. Passing it as a call option defers token retrieval and injection + // to the credentials package, which can verify transport security after + // the connection has been established. + opts = append(opts, grpc.PerRPCCredentials(creds)) + + return newStream(ctx, opts...) +} + +func (i *interceptor) Close() {} + +// cacheEntry represents a cached PerRPCCredentials instance and its position +// in the LRU list. +type cacheEntry struct { + creds credentials.PerRPCCredentials + elem *list.Element +} + +// lruCache is a thread-safe LRU cache that stores PerRPCCredentials instances +// by their target audience string. +type lruCache struct { + // The following fields are protected by mu. + mu sync.Mutex + + // cacheSize is the maximum capacity of the lruList. + cacheSize uint64 + + // lruList is a doubly linked list tracking access order. The front of the + // list holds the most recently used entry, and the back holds the least + // recently used entry. Elements in the list store values of type + // string (representing the target audience). + lruList *list.List + + // cache maps audience keys to their corresponding cacheEntry pointers, + // allowing O(1) lookups and updates in the LRU list. + cache map[string]*cacheEntry + + // sf is used to deduplicate credential creation for the same audience. + sf singleflight.Group +} + +// newLRUCache instantiates a new lruCache with the specified capacity. +func newLRUCache(size uint64) *lruCache { + return &lruCache{ + cacheSize: size, + lruList: list.New(), + cache: make(map[string]*cacheEntry), + } +} + +// resizeCache dynamically updates the capacity of the LRU cache, +// immediately evicting Least Recently Used entries if the new size is +// smaller than the current cache size. +func (c *lruCache) resizeCache(newCacheSize uint64) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.cacheSize == newCacheSize { + return + } + + c.cacheSize = newCacheSize + for uint64(len(c.cache)) > c.cacheSize { + c.removeOldestLocked() + } +} + +// getOrCreate retrieves or constructs the PerRPCCredentials for a specified +// audience. If the audience is not found in the cache, it creates new +// credentials using the configured creator, adds it to the cache, and evicts +// the least recently used entry if the cache size is at capacity. +func (c *lruCache) getOrCreate(ctx context.Context, audience string) (credentials.PerRPCCredentials, error) { + if creds, ok := c.getExisting(audience); ok { + return creds, nil + } + + val, err, _ := c.sf.Do(audience, func() (any, error) { + // Double-check cache inside singleflight callback in case another call + // completed the initial check before the current call updates the cache. + if creds, ok := c.getExisting(audience); ok { + return creds, nil + } + + creds, err := google.NewServiceAccountIdentityCredentials(ctx, audience) + if err != nil { + return nil, err + } + + c.mu.Lock() + defer c.mu.Unlock() + + if uint64(len(c.cache)) >= c.cacheSize { + c.removeOldestLocked() + } + c.cache[audience] = &cacheEntry{ + creds: creds, + elem: c.lruList.PushFront(audience), + } + return creds, nil + }) + if err != nil { + return nil, err + } + return val.(credentials.PerRPCCredentials), nil +} + +// getExisting retrieves the PerRPCCredentials for a specified audience if it +// already exists in the cache and updates the access order in the LRU list. +// +// The boolean return value indicates if an entry was found in the cache. +func (c *lruCache) getExisting(audience string) (credentials.PerRPCCredentials, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + if entry, ok := c.cache[audience]; ok { + c.lruList.MoveToFront(entry.elem) + return entry.creds, ok + } + return nil, false +} + +// removeOldestLocked evicts the least recently used entry from the cache. +// It must be called with mu locked. +func (c *lruCache) removeOldestLocked() { + oldest := c.lruList.Back() + delete(c.cache, oldest.Value.(string)) + c.lruList.Remove(oldest) +} diff --git a/internal/xds/httpfilter/gcp_authn/gcp_authn_filter_e2e_test.go b/internal/xds/httpfilter/gcp_authn/gcp_authn_filter_e2e_test.go new file mode 100644 index 0000000..c3ca731 --- /dev/null +++ b/internal/xds/httpfilter/gcp_authn/gcp_authn_filter_e2e_test.go @@ -0,0 +1,952 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package gcpauthn_test + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3gcpauthnpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/gcp_authn/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + gceMetadataHostEnvVar = "GCE_METADATA_HOST" + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 100 * time.Millisecond +) + +func setupGCPAuthnTest(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.GCPAuthenticationFilterEnabled, true) + cleanup, err := xdsresource.RegisterMetadataConverterForTesting(version.V3AudienceURL) + if err != nil { + t.Fatalf("RegisterMetadataConverterForTesting(%q) failed: %v", version.V3AudienceURL, err) + } + t.Cleanup(cleanup) +} + +// Test verifies the basic end-to-end flow. It ensures that the gcp_authn +// filter successfully fetches a token from the stub metadata server and +// attaches it to the outgoing gRPC request metadata. Than verify that +// subsequent RPC calls with same audience reuse the same token. +func (s) TestGCPAuthnFilter_SuccessCase(t *testing.T) { + setupGCPAuthnTest(t) + const tokenValue = "token" + + // Starts a local HTTP server and sets GCE_METADATA_HOST to spoof the + // GCE metadata server and redirect token fetch requests to it. + var requestCount atomic.Int32 + metadataServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requestCount.Add(1) + w.Write([]byte(tokenValue)) + })) + defer metadataServer.Close() + t.Setenv(gceMetadataHostEnvVar, strings.TrimPrefix(metadataServer.URL, "http://")) + + // Setup management server and resolver. + mgmtServer, nodeID, _, resolverBuilder := setup.ManagementServerAndResolver(t) + + // Start a test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, _ := metadata.FromIncomingContext(ctx) + // Verify that the token is attached to the RPC + if !strings.Contains(md.Get("authorization")[0], "Bearer "+tokenValue) { + return nil, fmt.Errorf("Expected token not found in metadata: %v", md) + } + return &testpb.Empty{}, nil + }, + } + stubserver.StartTestService(t, backend, grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.NoClientCert))) + defer backend.Stop() + + const ( + testServiceName = "service-name" + clusterName = "cluster_A" + endpointName = "endpoint_A" + routeConfigName = "route-service-name" + filterName = "com.google.grpc.gcp_authn" + ) + + // Configure resources on the management server. + listener := e2e.DefaultClientListener(testServiceName, routeConfigName) + hcm := new(v3httppb.HttpConnectionManager) + lis := listener.GetApiListener().GetApiListener() + if err := lis.UnmarshalTo(hcm); err != nil { + t.Fatal(err) + } + hcm.HttpFilters = append([]*v3httppb.HttpFilter{ + { + Name: filterName, + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3gcpauthnpb.GcpAuthnFilterConfig{ + CacheConfig: &v3gcpauthnpb.TokenCacheConfig{ + CacheSize: &wrapperspb.UInt64Value{Value: 10}, + }, + }), + }, + }, + }, hcm.HttpFilters...) + listener.ApiListener.ApiListener = testutils.MarshalAny(t, hcm) + + cluster := e2e.DefaultCluster(clusterName, endpointName, e2e.SecurityLevelTLS) + cluster.Metadata = &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + filterName: testutils.MarshalAny(t, &v3gcpauthnpb.Audience{ + Url: "https://example.com", + }), + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, testServiceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointName, "localhost", []uint32{testutils.ParsePort(t, backend.Address)})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a gRPC client using the xDS resolver. + clientCreds, err := xds.NewClientCredentials(xds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("Failed to create client credentials: %v", err) + } + cc, err := grpc.NewClient("xds:///"+testServiceName, grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + + } + // Verify request count is 1. + if count := requestCount.Load(); count != 1 { + t.Fatalf("Unexpected request to metadata server, got %d want 1", count) + } + + const numCalls = 3 + for i := 0; i < numCalls; i++ { + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + } + + // Verify request count is 1. This ensures that the token fetched for + // the first RPC was reused by the subsequent RPCs. + if count := requestCount.Load(); count != 1 { + t.Fatalf("Unexpected request to metadata server, got %d want 1", count) + } +} + +// Test verifies that the filter correctly handles concurrent RPC calls +// (thundering herd problem) by fetching the token only once and reusing +// it across all subsequent RPCs with the same audience. +func (s) TestGCPAuthnFilter_ConcurrentRequests(t *testing.T) { + setupGCPAuthnTest(t) + const tokenValue = "token" + + // Starts a local HTTP server and sets GCE_METADATA_HOST to spoof the + // GCE metadata server and redirect token fetch requests to it. + var requestCount atomic.Int32 + metadataServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requestCount.Add(1) + w.Write([]byte(tokenValue)) + })) + defer metadataServer.Close() + t.Setenv(gceMetadataHostEnvVar, strings.TrimPrefix(metadataServer.URL, "http://")) + + // Setup management server and resolver. + mgmtServer, nodeID, _, resolverBuilder := setup.ManagementServerAndResolver(t) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start a test backend. + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, _ := metadata.FromIncomingContext(ctx) + // Verify that the token is attached to the RPC + if !strings.Contains(md.Get("authorization")[0], "Bearer "+tokenValue) { + return nil, fmt.Errorf("Expected token not found in metadata: %v", md) + } + return &testpb.Empty{}, nil + }, + } + stubserver.StartTestService(t, backend, grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.NoClientCert))) + defer backend.Stop() + + const ( + testServiceName = "service-name" + clusterName = "cluster_A" + endpointName = "endpoint_A" + routeConfigName = "route-service-name" + filterName = "com.google.grpc.gcp_authn" + ) + + // Configure resources on the management server. + listener := e2e.DefaultClientListener(testServiceName, routeConfigName) + hcm := new(v3httppb.HttpConnectionManager) + lis := listener.GetApiListener().GetApiListener() + if err := lis.UnmarshalTo(hcm); err != nil { + t.Fatal(err) + } + hcm.HttpFilters = append([]*v3httppb.HttpFilter{ + { + Name: filterName, + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3gcpauthnpb.GcpAuthnFilterConfig{ + CacheConfig: &v3gcpauthnpb.TokenCacheConfig{ + CacheSize: &wrapperspb.UInt64Value{Value: 10}, + }, + }), + }, + }, + }, hcm.HttpFilters...) + listener.ApiListener.ApiListener = testutils.MarshalAny(t, hcm) + + cluster := e2e.DefaultCluster(clusterName, endpointName, e2e.SecurityLevelTLS) + cluster.Metadata = &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + filterName: testutils.MarshalAny(t, &v3gcpauthnpb.Audience{ + Url: "https://example.com", + }), + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, testServiceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointName, "localhost", []uint32{testutils.ParsePort(t, backend.Address)})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a gRPC client using the xDS resolver. + clientCreds, err := xds.NewClientCredentials(xds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("Failed to create client credentials: %v", err) + } + cc, err := grpc.NewClient("xds:///"+testServiceName, grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + + const numCalls = 100 + errs := make([]error, numCalls) + var wg sync.WaitGroup + for i := 0; i < numCalls; i++ { + wg.Go(func() { + _, errs[i] = client.EmptyCall(ctx, &testpb.Empty{}) + }) + } + wg.Wait() + + for _, err := range errs { + if err != nil { + t.Errorf("EmptyCall() failed: %v", err) + } + } + + // Verify request count is 1. This ensures that only one token fetch call was + // made to the metadata server and all 100 concurrent RPC calls reused the + // same token. + if count := requestCount.Load(); count != 1 { + t.Fatalf("Unexpected request to metadata server, got %d want 1", count) + } +} + +// Test verifies that the filter refuses to attach credentials when the target +// cluster is configured without transport security. +func (s) TestGCPAuthnFilter_InsecureTransport(t *testing.T) { + setupGCPAuthnTest(t) + + // Starts a local HTTP server and sets GCE_METADATA_HOST to spoof the + // GCE metadata server and redirect token fetch requests to it. + metadataServer := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + defer metadataServer.Close() + t.Setenv(gceMetadataHostEnvVar, strings.TrimPrefix(metadataServer.URL, "http://")) + + // Setup management server and resolver. + mgmtServer, nodeID, _, resolverBuilder := setup.ManagementServerAndResolver(t) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start a test backend. + backend := &stubserver.StubServer{} + stubserver.StartTestService(t, backend) + defer backend.Stop() + + const ( + testServiceName = "service-name" + clusterName = "cluster_A" + endpointName = "endpoint_A" + filterName = "com.google.grpc.gcp_authn" + routeConfigName = "route-service-name" + ) + + // Configure resources on the management server. + listener := e2e.DefaultClientListener(testServiceName, routeConfigName) + hcm := new(v3httppb.HttpConnectionManager) + lis := listener.GetApiListener().GetApiListener() + if err := lis.UnmarshalTo(hcm); err != nil { + t.Fatal(err) + } + hcm.HttpFilters = append([]*v3httppb.HttpFilter{ + { + Name: filterName, + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3gcpauthnpb.GcpAuthnFilterConfig{ + CacheConfig: &v3gcpauthnpb.TokenCacheConfig{ + CacheSize: &wrapperspb.UInt64Value{Value: 10}, + }, + }), + }, + }, + }, hcm.HttpFilters...) + listener.ApiListener.ApiListener = testutils.MarshalAny(t, hcm) + + // SecurityLevelNone causes SecurityCfg to be nil + cluster := e2e.DefaultCluster(clusterName, endpointName, e2e.SecurityLevelNone) + cluster.Metadata = &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + filterName: testutils.MarshalAny(t, &v3gcpauthnpb.Audience{ + Url: "https://example.com", + }), + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, testServiceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointName, "localhost", []uint32{testutils.ParsePort(t, backend.Address)})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a gRPC client using the xDS resolver. + cc, err := grpc.NewClient("xds:///"+testServiceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + const wantErr = "cannot send secure credentials on an insecure connection" + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall() failed with error: %v; want error containing: %v", err, wantErr) + } +} + +// Test verifies that the credential cache of the gcp_authn filter correctly +// handles cache resizing across xDS updates. It performs RPC calls to 3 +// different clusters to fill a cache of size 3. Then it updates the xDS +// configuration to reduce the cache size to 1. Finally, it verifies that only +// the most recently used element survives in the cache, and making calls to +// the other two results in cache misses (forcing a new token fetch). +func (s) TestGCPAuthnFilter_CacheSharingConfigUpdate(t *testing.T) { + setupGCPAuthnTest(t) + + // Starts a local HTTP server and sets GCE_METADATA_HOST to spoof the + // GCE metadata server and redirect token fetch requests to it. + var requestCount atomic.Int32 + metadataServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requestCount.Add(1) + w.Write([]byte("token")) + })) + defer metadataServer.Close() + t.Setenv(gceMetadataHostEnvVar, strings.TrimPrefix(metadataServer.URL, "http://")) + + // Setup management server and resolver. + mgmtServer, nodeID, _, resolverBuilder := setup.ManagementServerAndResolver(t) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start a test backend. + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + stubserver.StartTestService(t, backend, grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.NoClientCert))) + defer backend.Stop() + + const ( + testServiceName = "service-name" + filterName = "com.google.grpc.gcp_authn" + ) + + // Configure resources on the management server. + listener := &v3listenerpb.Listener{ + Name: testServiceName, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{testServiceName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}, + Headers: []*v3routepb.HeaderMatcher{{ + Name: "match-id", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "a"}, + }}, + }, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: "A"}, + }}, + }, + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}, + Headers: []*v3routepb.HeaderMatcher{{ + Name: "match-id", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "b"}, + }}, + }, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: "B"}, + }}, + }, + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}, + Headers: []*v3routepb.HeaderMatcher{{ + Name: "match-id", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "c"}, + }}, + }, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: "C"}, + }}, + }, + }, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + { + Name: filterName, + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3gcpauthnpb.GcpAuthnFilterConfig{ + CacheConfig: &v3gcpauthnpb.TokenCacheConfig{ + CacheSize: &wrapperspb.UInt64Value{Value: 10}, + }, + }), + }, + }, + e2e.RouterHTTPFilter, + }, + }), + }, + } + + clusterA := e2e.DefaultCluster("A", "endpoint_A", e2e.SecurityLevelTLS) + clusterA.Metadata = &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + filterName: testutils.MarshalAny(t, &v3gcpauthnpb.Audience{Url: "url-1"}), + }, + } + + clusterB := e2e.DefaultCluster("B", "endpoint_B", e2e.SecurityLevelTLS) + clusterB.Metadata = &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + filterName: testutils.MarshalAny(t, &v3gcpauthnpb.Audience{Url: "url-2"}), + }, + } + + clusterC := e2e.DefaultCluster("C", "endpoint_C", e2e.SecurityLevelTLS) + clusterC.Metadata = &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + filterName: testutils.MarshalAny(t, &v3gcpauthnpb.Audience{Url: "url-3"}), + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Clusters: []*v3clusterpb.Cluster{clusterA, clusterB, clusterC}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint("endpoint_A", "localhost", []uint32{testutils.ParsePort(t, backend.Address)}), + e2e.DefaultEndpoint("endpoint_B", "localhost", []uint32{testutils.ParsePort(t, backend.Address)}), + e2e.DefaultEndpoint("endpoint_C", "localhost", []uint32{testutils.ParsePort(t, backend.Address)}), + }, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a gRPC client using the xDS resolver. + clientCreds, err := xds.NewClientCredentials(xds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("Failed to create client credentials: %v", err) + } + cc, err := grpc.NewClient("xds:///"+testServiceName, grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + makeCall := func(ctx context.Context, rpcName string) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall(%s) failed: %v", rpcName, err) + } + } + + // Make initial calls to all 3 clusters to populate the cache with 3 entries. + // The access order is A, then B, then C (leaving C as the most + // recently used). + ctxA := metadata.AppendToOutgoingContext(ctx, "match-id", "a") + ctxB := metadata.AppendToOutgoingContext(ctx, "match-id", "b") + ctxC := metadata.AppendToOutgoingContext(ctx, "match-id", "c") + + makeCall(ctxA, "A") + makeCall(ctxB, "B") + makeCall(ctxC, "C") + + // Verify request count is 3! + if count := requestCount.Load(); count != 3 { + t.Fatalf("Unexpected requests to metadata server, got %d want 3", count) + } + + // Make another call to cluster C and verify that it did not increase the + // request count. + makeCall(ctxC, "C") + if count := requestCount.Load(); count != 3 { + t.Fatalf("Unexpected requests to metadata server, got %d want 3", count) + } + + // Update cache config to size 1! + hcm := &v3httppb.HttpConnectionManager{} + if err := anypb.UnmarshalTo(listener.ApiListener.ApiListener, hcm, proto.UnmarshalOptions{}); err != nil { + t.Fatalf("Failed to unmarshal HCM: %v", err) + } + hcm.HttpFilters[0].ConfigType = &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3gcpauthnpb.GcpAuthnFilterConfig{ + CacheConfig: &v3gcpauthnpb.TokenCacheConfig{ + CacheSize: &wrapperspb.UInt64Value{Value: 1}, + }, + }), + } + + // Add a new route matching "match-id": "d" pointing to Cluster D as canary. + routeD := &v3routepb.Route{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}, + Headers: []*v3routepb.HeaderMatcher{{ + Name: "match-id", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "d"}, + }}, + }, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: "D"}, + }}, + } + hcm.GetRouteConfig().GetVirtualHosts()[0].Routes = append(hcm.GetRouteConfig().GetVirtualHosts()[0].Routes, routeD) + resources.Listeners[0].ApiListener.ApiListener = testutils.MarshalAny(t, hcm) + resources.Clusters = append(resources.Clusters, e2e.DefaultCluster("D", "endpoint_D", e2e.SecurityLevelTLS)) + resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint("endpoint_D", "localhost", []uint32{testutils.ParsePort(t, backend.Address)})) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Active Probing: Wait for the new route "d" to become live. Because + // it points to Cluster D with no metadata, a successful call here + // won't trigger a new fetch. This guarantees that the resource update + // is propagated. + ctxD := metadata.AppendToOutgoingContext(ctx, "match-id", "d") + for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) { + if _, err = client.EmptyCall(ctxD, &testpb.Empty{}); err == nil { + break + } + } + if ctx.Err() != nil { + t.Fatal("Timeout waiting for xDS update to propagate") + } + + // Make calls for all 3 clusters again. Resizing the cache from 3 to 1 + // in the previous step leaves only the token for Cluster C in the + // cache (as it was the most recently used). Therefore, calls to + // clusters A and B will miss the cache and trigger new metadata + // server fetches, while C will hit. + makeCall(ctxC, "C on second pass") + if count := requestCount.Load(); count != 3 { + t.Fatalf("Unexpected requests to metadata server, got %d want 3", count) + } + + makeCall(ctxB, "B on second pass") + makeCall(ctxA, "A on second pass") + + // Verify that the total requests to the metadata server is exactly 5 + // This proves that token for cluster C was successfully cached in the + // second pass, while A and B were not. + if count := requestCount.Load(); count != 5 { + t.Fatalf("Unexpected requests to metadata server, got %d want 5", count) + } +} + +// Test verifies the scenario where two RPCs are made: the first with a +// short context which triggers a token fetch but times out while waiting +// for the metadata server to respond and the second RPC with a long context, +// which blocks waiting for the first token fetch to finish, and then succeeds +// by using the token fetched by the first. +func (s) TestGCPAuthnFilter_ConcurrentRPCWithShortAndLongContext(t *testing.T) { + setupGCPAuthnTest(t) + var once sync.Once + const tokenValue = "token" + requestStarted := make(chan struct{}) + proceedCh := make(chan struct{}) + + // Starts a local HTTP server and sets GCE_METADATA_HOST to spoof the + // GCE metadata server and redirect token fetch requests to it. + var requestCount atomic.Int32 + metadataServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // Close requestStarted channel to signal that a request has started. + once.Do(func() { + close(requestStarted) + }) + requestCount.Add(1) + // Block until signaled to proceed. + <-proceedCh + w.Write([]byte(tokenValue)) + })) + defer metadataServer.Close() + t.Setenv(gceMetadataHostEnvVar, strings.TrimPrefix(metadataServer.URL, "http://")) + + // Setup management server and resolver. + mgmtServer, nodeID, _, resolverBuilder := setup.ManagementServerAndResolver(t) + + // Start a test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, _ := metadata.FromIncomingContext(ctx) + // Verify that the token is attached to the RPC + if !strings.Contains(md.Get("authorization")[0], "Bearer "+tokenValue) { + return nil, fmt.Errorf("Expected token not found in metadata: %v", md) + } + return &testpb.Empty{}, nil + }, + } + stubserver.StartTestService(t, backend, grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.NoClientCert))) + defer backend.Stop() + + const ( + testServiceName = "service-name" + clusterName = "cluster_A" + endpointName = "endpoint_A" + routeConfigName = "route-service-name" + filterName = "com.google.grpc.gcp_authn" + ) + + // Configure resources on the management server. + listener := e2e.DefaultClientListener(testServiceName, routeConfigName) + hcm := new(v3httppb.HttpConnectionManager) + lis := listener.GetApiListener().GetApiListener() + if err := lis.UnmarshalTo(hcm); err != nil { + t.Fatal(err) + } + hcm.HttpFilters = append([]*v3httppb.HttpFilter{ + { + Name: filterName, + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3gcpauthnpb.GcpAuthnFilterConfig{ + CacheConfig: &v3gcpauthnpb.TokenCacheConfig{ + CacheSize: &wrapperspb.UInt64Value{Value: 10}, + }, + }), + }, + }, + }, hcm.HttpFilters...) + listener.ApiListener.ApiListener = testutils.MarshalAny(t, hcm) + + cluster := e2e.DefaultCluster(clusterName, endpointName, e2e.SecurityLevelTLS) + cluster.Metadata = &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + filterName: testutils.MarshalAny(t, &v3gcpauthnpb.Audience{ + Url: "https://example.com", + }), + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, testServiceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointName, "localhost", []uint32{testutils.ParsePort(t, backend.Address)})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a gRPC client using the xDS resolver. + clientCreds, err := xds.NewClientCredentials(xds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("Failed to create client credentials: %v", err) + } + cc, err := grpc.NewClient("xds:///"+testServiceName, grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + cc.Connect() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Create a short context for the first RPC call. + shortCtx, shortBufCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer shortBufCancel() + + errCh1, errCh2 := make(chan error, 1), make(chan error, 1) + + // First RPC call with short context (triggers token fetch request + // to metadata server). + go func() { + _, err := client.EmptyCall(shortCtx, &testpb.Empty{}) + errCh1 <- err + }() + + // Wait for the first RPC to hit the local metadata server. + select { + case <-requestStarted: + case <-ctx.Done(): + t.Fatal("Timeout while waiting for token fetch to start in metadata server") + } + + // Second RPC call with long context. + go func() { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + errCh2 <- err + }() + + // The first RPC should fail with context deadline exceeded because its + // context timeout is short and the metadata server is blocked. + select { + case err := <-errCh1: + if err == nil || status.Code(err) != codes.Internal { + t.Fatalf("First RPC failed unexpectedly with error code %v, want %v", status.Code(err), codes.Internal) + } + case <-ctx.Done(): + t.Fatal("Timeout waiting for first RPC to fail") + } + + // Now allow the metadata server to complete and return token. + close(proceedCh) + + // Verify that the second RPC successfully completes. + select { + case err := <-errCh2: + if err != nil { + t.Fatalf("Second RPC failed unexpectedly with: %v, want success", err) + } + case <-ctx.Done(): + t.Fatal("Timeout while waiting for second RPC to succeed") + } + + // Verify request count is 1. This ensures that the token fetched for + // the first RPC was reused for the second RPC. + if count := requestCount.Load(); count != 1 { + t.Fatalf("Unexpected request to metadata server, got %d want 1", count) + } +} + +// Test verifies that when an RPC is made with user-specified call options +// (e.g., grpc.Header), the filter successfully appends its PerRPCCredentials +// option without overriding, corrupting or interfering with the existing ones. +func (s) TestGCPAuthnFilter_PreservesUserCallOptions(t *testing.T) { + setupGCPAuthnTest(t) + const tokenValue = "token" + + // Starts a local HTTP server and sets GCE_METADATA_HOST to spoof the + // GCE metadata server and redirect token fetch requests to it. + metadataServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte(tokenValue)) + })) + defer metadataServer.Close() + t.Setenv(gceMetadataHostEnvVar, strings.TrimPrefix(metadataServer.URL, "http://")) + + // Setup management server and resolver. + mgmtServer, nodeID, _, resolverBuilder := setup.ManagementServerAndResolver(t) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start a test backend. + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + // Set a custom header back to the client to verify the original + // CallOption works. + if err := grpc.SetHeader(ctx, metadata.Pairs("custom-response-header", "custom-val")); err != nil { + t.Errorf("Failed to set response header: %v", err) + } + + md, _ := metadata.FromIncomingContext(ctx) + // Verify that the token is attached to the RPC + if !strings.Contains(md.Get("authorization")[0], "Bearer "+tokenValue) { + return nil, fmt.Errorf("Expected token not found in metadata: %v", md) + } + return &testpb.Empty{}, nil + }, + } + stubserver.StartTestService(t, backend, grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.NoClientCert))) + defer backend.Stop() + + const ( + testServiceName = "service-name" + clusterName = "cluster_A" + endpointName = "endpoint_A" + routeConfigName = "route-service-name" + filterName = "com.google.grpc.gcp_authn" + ) + + // Configure resources on the management server. + listener := e2e.DefaultClientListener(testServiceName, routeConfigName) + hcm := new(v3httppb.HttpConnectionManager) + lis := listener.GetApiListener().GetApiListener() + if err := lis.UnmarshalTo(hcm); err != nil { + t.Fatal(err) + } + hcm.HttpFilters = append([]*v3httppb.HttpFilter{ + { + Name: filterName, + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3gcpauthnpb.GcpAuthnFilterConfig{ + CacheConfig: &v3gcpauthnpb.TokenCacheConfig{ + CacheSize: &wrapperspb.UInt64Value{Value: 10}, + }, + }), + }, + }, + }, hcm.HttpFilters...) + listener.ApiListener.ApiListener = testutils.MarshalAny(t, hcm) + + cluster := e2e.DefaultCluster(clusterName, endpointName, e2e.SecurityLevelTLS) + cluster.Metadata = &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + filterName: testutils.MarshalAny(t, &v3gcpauthnpb.Audience{ + Url: "https://example.com", + }), + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, testServiceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointName, "localhost", []uint32{testutils.ParsePort(t, backend.Address)})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a gRPC client using the xDS resolver. + clientCreds, err := xds.NewClientCredentials(xds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("Failed to create client credentials: %v", err) + } + cc, err := grpc.NewClient("xds:///"+testServiceName, grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + var header metadata.MD + + // RPC call with a custom CallOption. + if _, err = client.EmptyCall(ctx, &testpb.Empty{}, grpc.Header(&header)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Verify that the response header option was executed successfully, + // proving the first entry was preserved. + val := header.Get("custom-response-header") + if len(val) == 0 || val[0] != "custom-val" { + t.Fatalf("Unexpected response header, got: %v want custom-val", val) + } +} diff --git a/internal/xds/httpfilter/gcp_authn/gcp_authn_filter_test.go b/internal/xds/httpfilter/gcp_authn/gcp_authn_filter_test.go new file mode 100644 index 0000000..f564c8d --- /dev/null +++ b/internal/xds/httpfilter/gcp_authn/gcp_authn_filter_test.go @@ -0,0 +1,295 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package gcpauthn + +import ( + "context" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/balancer/clustermanager" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3gcpauthnpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/gcp_authn/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// Test verifies that the filter successfully decodes the raw protobuf +// configuration into the internal config struct. +func (s) TestParseFilterConfig(t *testing.T) { + testCases := []struct { + name string + config proto.Message + wantCfg config + wantErr string + }{ + { + name: "valid_config_with_cache_size", + config: testutils.MarshalAny(t, &v3gcpauthnpb.GcpAuthnFilterConfig{ + CacheConfig: &v3gcpauthnpb.TokenCacheConfig{ + CacheSize: &wrapperspb.UInt64Value{Value: 50}, + }, + }), + wantCfg: config{cacheSize: 50}, + }, + { + name: "config_with_no_cache_size", + config: testutils.MarshalAny(t, &v3gcpauthnpb.GcpAuthnFilterConfig{ + CacheConfig: &v3gcpauthnpb.TokenCacheConfig{}, + }), + wantCfg: config{cacheSize: defaultCacheSize}, + }, + { + name: "config_with_zero_cache_size", + config: testutils.MarshalAny(t, &v3gcpauthnpb.GcpAuthnFilterConfig{ + CacheConfig: &v3gcpauthnpb.TokenCacheConfig{ + CacheSize: &wrapperspb.UInt64Value{Value: 0}, + }, + }), + wantErr: "cache_config.cache_size must be greater than zero", + }, + { + name: "invalid_message_type", + config: &v3gcpauthnpb.GcpAuthnFilterConfig{}, + wantErr: "gcpauthn: invalid filter config type", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotCfg, err := builder{}.ParseFilterConfig(tc.config, httpfilter.ParseOptions{}) + if err != nil { + if tc.wantErr == "" || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("ParseFilterConfig() failed with error = %v; want error %q", err, tc.wantErr) + } + return + } + if diff := cmp.Diff(gotCfg, tc.wantCfg, cmp.AllowUnexported(config{})); diff != "" { + t.Fatalf("ParseFilterConfig() returned unexpected config (-got +want):\n%s", diff) + } + }) + } +} + +// Test verifies that the filter correctly creates the stream interceptor +// from the parsed config, and correctly handles cache initialization. +func (s) TestBuildClientInterceptor(t *testing.T) { + type dummyConfig struct { + httpfilter.FilterConfig + } + tests := []struct { + name string + cfg httpfilter.FilterConfig + wantErr string + wantCacheSize uint64 + }{ + { + name: "invalid_config_type", + cfg: dummyConfig{}, + wantErr: "invalid filter config type", + }, + { + name: "cache_initialization", + cfg: config{cacheSize: 5}, + wantCacheSize: 5, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + const filterName = "com.google.grpc.gcp_authn" + clientFilter := builder{}.BuildClientFilter(httpfilter.ClientFilterOptions{FilterName: filterName}) + defer clientFilter.Close() + + i, err := clientFilter.BuildClientInterceptor(tc.cfg, nil) + if err != nil { + if tc.wantErr == "" || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("BuildClientInterceptor() failed with error = %v, want error %q", err, tc.wantErr) + } + return + } + + interceptor := i.(*interceptor) + if interceptor.filterName != filterName { + t.Fatalf("BuildClientInterceptor() returned interceptor with filtername = %q, want %q", interceptor.filterName, filterName) + } + if interceptor.cache == nil { + t.Fatalf("BuildClientInterceptor() returned interceptor with nil cache") + } + if interceptor.cache.cacheSize != tc.wantCacheSize { + t.Fatalf("BuildClientInterceptor() returned interceptor with cacheSize = %d, want %d", interceptor.cache.cacheSize, tc.wantCacheSize) + } + }) + } +} + +// Test verifies that consecutive calls to BuildClientInterceptor with +// different cache sizes dynamically resize the shared credentials cache. +func (s) TestBuildClientInterceptor_CacheResizing(t *testing.T) { + tests := []struct { + name string + cfgs []httpfilter.FilterConfig + wantCacheSizes []uint64 + }{ + { + name: "cache_resize_smaller", + cfgs: []httpfilter.FilterConfig{config{cacheSize: 5}, config{cacheSize: 3}}, + wantCacheSizes: []uint64{5, 3}, + }, + { + name: "cache_resize_larger", + cfgs: []httpfilter.FilterConfig{config{cacheSize: 5}, config{cacheSize: 15}}, + wantCacheSizes: []uint64{5, 15}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + const filterName = "com.google.grpc.gcp_authn" + clientFilter := builder{}.BuildClientFilter(httpfilter.ClientFilterOptions{FilterName: filterName}) + defer clientFilter.Close() + + for i, cfg := range tc.cfgs { + intptr, err := clientFilter.BuildClientInterceptor(cfg, nil) + if err != nil { + t.Fatalf("BuildClientInterceptor(%v) returned unexpected error: %v", cfg, err) + } + interceptor := intptr.(*interceptor) + if interceptor.filterName != filterName { + t.Fatalf("BuildClientInterceptor() returned interceptor with filtername = %q, want %q", interceptor.filterName, filterName) + } + if interceptor.cache == nil { + t.Fatalf("BuildClientInterceptor() returned interceptor with nil cache") + } + if interceptor.cache.cacheSize != tc.wantCacheSizes[i] { + t.Fatalf("BuildClientInterceptor() returned interceptor with cacheSize = %d, want %d", interceptor.cache.cacheSize, tc.wantCacheSizes[i]) + } + } + }) + } +} + +// Test verifies that interceptor.NewStream returns the expected errors under +// various invalid states, such as a missing XDS config, a cluster not defined +// in the CDS, or a malformed metadata type. +func (s) TestInterceptor_NewStream_Errors(t *testing.T) { + builder := httpfilter.Get("type.googleapis.com/envoy.extensions.filters.http.gcp_authn.v3.GcpAuthnFilterConfig") + cfg := testutils.MarshalAny(t, &v3gcpauthnpb.GcpAuthnFilterConfig{ + CacheConfig: &v3gcpauthnpb.TokenCacheConfig{ + CacheSize: &wrapperspb.UInt64Value{Value: 10}, + }, + }) + filterConfig, err := builder.ParseFilterConfig(cfg, httpfilter.ParseOptions{}) + if err != nil { + t.Fatalf("Failed to parse filter config: %v", err) + } + clientFilterBuilder, ok := builder.(httpfilter.ClientFilterBuilder) + if !ok { + t.Fatalf("Filter Builder does not implement ClientFilterBuilder") + } + clientFilter := clientFilterBuilder.BuildClientFilter(httpfilter.ClientFilterOptions{FilterName: "com.google.grpc.gcp_authn"}).(*clientFilter) + interceptor, err := clientFilter.BuildClientInterceptor(filterConfig, nil) + if err != nil { + t.Fatalf("Failed to build client interceptor: %v", err) + } + + validConfig := &xdsresource.XDSConfig{ + Clusters: map[string]*xdsresource.ClusterResult{ + "cluster1": { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + Metadata: map[string]any{ + "com.google.grpc.gcp_authn": xdsresource.AudienceMetadataValue{Audience: "https://example.com"}, + }, + }, + }, + }, + "cluster_wrong_type": { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + Metadata: map[string]any{ + "com.google.grpc.gcp_authn": "not-audience-metadata", + }, + }, + }, + }, + "cluster_empty_audience": { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + Metadata: map[string]any{ + "com.google.grpc.gcp_authn": xdsresource.AudienceMetadataValue{Audience: ""}, + }, + }, + }, + }, + }, + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tests := []struct { + name string + ctx context.Context + wantErr string + }{ + { + name: "missing_xds_config", + ctx: clustermanager.SetPickedCluster(ctx, "cluster1"), + wantErr: "xDS config not found in context", + }, + { + name: "cluster_not_found_in_CDS", + ctx: clustermanager.SetPickedCluster(xdsresource.NewContextWithXDSConfig(ctx, validConfig), "cluster_not_found"), + wantErr: "not found in xDS config", + }, + { + name: "wrong_metadata_type", + ctx: clustermanager.SetPickedCluster(xdsresource.NewContextWithXDSConfig(ctx, validConfig), "cluster_wrong_type"), + wantErr: "not of type AudienceMetadataValue", + }, + { + name: "empty_audience", + ctx: clustermanager.SetPickedCluster(xdsresource.NewContextWithXDSConfig(ctx, validConfig), "cluster_empty_audience"), + wantErr: "failed to create credentials", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + newStream := func(context.Context, ...grpc.CallOption) (grpc.ClientStream, error) { + return nil, nil + } + if _, err := interceptor.NewStream(tc.ctx, resolver.RPCInfo{}, newStream); err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("NewStream() returned unexpected results, got %q , want error containing %q", err, tc.wantErr) + } + }) + } +} diff --git a/internal/xds/httpfilter/httpfilter.go b/internal/xds/httpfilter/httpfilter.go new file mode 100644 index 0000000..0a0f259 --- /dev/null +++ b/internal/xds/httpfilter/httpfilter.go @@ -0,0 +1,204 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package httpfilter contains interface definitions for xDS-based HTTP filters +// and a registry for filter builders. +package httpfilter + +import ( + "context" + + "google.golang.org/grpc" + estats "google.golang.org/grpc/experimental/stats" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/protobuf/proto" +) + +// FilterConfig represents an opaque data structure holding configuration for a +// filter. Embed this interface to implement it. +type FilterConfig interface { + isFilterConfig() +} + +// ParseOptions contains additional information passed to the HTTP filter's +// config parsing methods. +type ParseOptions struct { + // BootstrapConfig contains the complete bootstrap configuration used by the + // xDS client that received the resource being parsed. + BootstrapConfig *bootstrap.Config + // ServerConfig contains the configuration of the xDS management server that + // sent the resource being parsed. + ServerConfig *bootstrap.ServerConfig +} + +// DisabledFilterConfig represents a disabled filter override. It implements the +// FilterConfig interface and can be returned by ParseFilterConfigOverride to +// indicate that the filter should be disabled. It is not used as a config for +// any filter, and is only used as a marker in the override configuration. For +// more information, see +// envoyproxy.io/docs/envoy/latest/intro/arch_overview/http/http_filters#route-based-filter-chain +type DisabledFilterConfig struct{} + +func (DisabledFilterConfig) isFilterConfig() {} + +// Builder defines the parsing functionality of an HTTP filter. A Builder may +// optionally implement either ClientFilterBuilder or ServerFilterBuilder or +// both, indicating it is capable of working on the client side or server side +// or both, respectively. +type Builder interface { + // TypeURLs are the proto message types supported by this filter. A filter + // will be registered by each of its supported message types. + TypeURLs() []string + // ParseFilterConfig parses the provided configuration proto.Message from + // the LDS configuration of this filter. This may be an anypb.Any, a + // udpa.type.v1.TypedStruct, or an xds.type.v3.TypedStruct for filters that + // do not accept a custom type. The resulting FilterConfig will later be + // passed to Build. + ParseFilterConfig(proto.Message, ParseOptions) (FilterConfig, error) + // ParseFilterConfigOverride parses the provided override configuration + // proto.Message from the RDS override configuration of this filter. This + // may be an anypb.Any, a udpa.type.v1.TypedStruct, or an + // xds.type.v3.TypedStruct for filters that do not accept a custom type. + // The resulting FilterConfig will later be passed to Build. + ParseFilterConfigOverride(proto.Message, ParseOptions) (FilterConfig, error) + // IsTerminal returns whether this Filter is terminal or not (i.e. it must + // be last filter in the filter chain). + IsTerminal() bool +} + +// ClientInterceptor is an interceptor for gRPC client streams. +type ClientInterceptor interface { + // NewStream creates a ClientStream for an RPC. + // + // Implementations may delegate stream creation to the provided newStream + // function, passing the provided CallOption slice along with any new + // CallOption instances they wish to add. To intercept or override stream + // behavior, implementations may wrap the ClientStream returned by the + // delegate. + // + // Note: RPCInfo.Context is currently unused and will be nil. + NewStream(ctx context.Context, ri iresolver.RPCInfo, newStream func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) + + // Close closes the interceptor. No new RPCs will be dispatched to this + // interceptor, but ongoing calls to NewStream are allowed to complete. + Close() +} + +// ClientFilterOptions contains options for building a client filter. +type ClientFilterOptions struct { + FilterName string // FilterName is the filter name from the xDS configuration. + MetricsRecorder estats.MetricsRecorder // MetricsRecorder is the metrics recorder to capture metrics for the filter. + Target string // Target is the target string of the channel. +} + +// ClientFilterBuilder is an optional interface that a Builder can implement to +// indicate its capability to build client-side filters. +type ClientFilterBuilder interface { + // BuildClientFilter constructs a ClientFilter. + BuildClientFilter(opts ClientFilterOptions) ClientFilter +} + +// ClientFilter represents the actual filter implementation on the client side. +// Implementations are free to maintain internal state when required, and share +// it across interceptors. Filter instances are retained by the resolver as long +// as they are present in the LDS configuration. +type ClientFilter interface { + // BuildClientInterceptor uses the given FilterConfigs to produce an HTTP + // filter interceptor for clients. config will always be non-nil, but + // override may be nil if no override config exists for the filter. + // + // It is valid for this method to return a nil Interceptor and a nil error. + // In this case, the RPC will not be intercepted by this filter. + BuildClientInterceptor(config, override FilterConfig) (ClientInterceptor, error) + + // Close is called when the filter is no longer needed. + Close() +} + +// ServerInterceptor is an interceptor for incoming RPC's on gRPC server side. +type ServerInterceptor interface { + // InterceptRPC intercepts an incoming RPC on the server side. + // + // On success, implementations must return either the original ServerStream + // or a wrapped ServerStream, with a nil error. + // + // Returning a non-nil error will terminate the RPC with that error. + // Implementations are expected to return an error created using the status + // package; otherwise, the RPC will fail with an UNKNOWN status code. + // + // Implementations should never return (nil, nil). + InterceptRPC(ss grpc.ServerStream) (grpc.ServerStream, error) + + // Close closes the interceptor. No new RPCs will be dispatched to this + // interceptor, but ongoing calls to InterceptRPC are allowed to complete. + Close() +} + +// ServerFilterBuilder is an optional interface that a Builder can implement to +// indicate its capability to build server-side filters. +type ServerFilterBuilder interface { + // BuildServerFilter constructs a ServerFilter. + BuildServerFilter() ServerFilter +} + +// ServerFilter represents the actual filter implementation on the server side. +// Implementations are free to maintain internal state when required, and share +// it across interceptors. Filter instances are retained by the server as long +// as they are present in any of the filter chains in the LDS configuration. +type ServerFilter interface { + // BuildServerInterceptor uses the given FilterConfigs to produce + // an HTTP filter interceptor for servers. config will always be non-nil, + // but override may be nil if no override config exists for the filter. + // + // It is valid for this method to return a nil Interceptor and a nil error. + // In this case, the RPC will not be intercepted by this filter. + BuildServerInterceptor(config, override FilterConfig) (ServerInterceptor, error) + + // Close is called when the filter is no longer needed. + Close() +} + +var ( + // registeredBuilders is a map from scheme to filter builder. + registeredBuilders = make(map[string]Builder) +) + +// Register registers the HTTP Filter Builder with the registry. b.TypeURLs() +// will be used as the types for this filter. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple filters are +// registered with the same type URL, the one registered last will take effect. +func Register(b Builder) { + for _, u := range b.TypeURLs() { + registeredBuilders[u] = b + } +} + +// UnregisterForTesting unregisters the HTTP Filter Builder for testing purposes. +func UnregisterForTesting(typeURL string) { + delete(registeredBuilders, typeURL) +} + +// Get returns the HTTP Filter Builder registered with typeURL. +// +// If no filter builder is register with typeURL, nil will be returned. +func Get(typeURL string) Builder { + return registeredBuilders[typeURL] +} diff --git a/internal/xds/httpfilter/rbac/rbac.go b/internal/xds/httpfilter/rbac/rbac.go new file mode 100644 index 0000000..761eb3b --- /dev/null +++ b/internal/xds/httpfilter/rbac/rbac.go @@ -0,0 +1,279 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package rbac implements the Envoy RBAC HTTP filter. +package rbac + +import ( + "errors" + "fmt" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/rbac" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + rpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" +) + +func init() { + httpfilter.Register(builder{}) +} + +type builder struct { +} + +type config struct { + httpfilter.FilterConfig + chainEngine *rbac.ChainEngine +} + +func (builder) TypeURLs() []string { + return []string{ + "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC", + "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute", + } +} + +// Parsing is the same for the base config and the override config. +func parseConfig(rbacCfg *rpb.RBAC) (httpfilter.FilterConfig, error) { + // All the validation logic described in A41. + for _, policy := range rbacCfg.GetRules().GetPolicies() { + // "Policy.condition and Policy.checked_condition must cause a + // validation failure if present." - A41 + if policy.Condition != nil { + return nil, errors.New("rbac: Policy.condition is present") + } + if policy.CheckedCondition != nil { + return nil, errors.New("rbac: policy.CheckedCondition is present") + } + + // "It is also a validation failure if Permission or Principal has a + // header matcher for a grpc- prefixed header name or :scheme." - A41. + // + // "Envoy aliases :authority and Host in its header map implementation, + // so they should be treated equivalent for the RBAC matchers; there must + // be no behavior change depending on which of the two header names is + // used in the RBAC policy." - A41. Any header matcher with value "host" + // is rewritten to ":authority", as that is what grpc-go shifts both + // headers to in the transport layer. + // + // Both rules apply to header matchers nested inside and/or/not rules, so + // the whole permission and principal trees are walked. + for _, principal := range policy.GetPrincipals() { + if err := normalizePrincipalHeaders(principal); err != nil { + return nil, err + } + } + for _, permission := range policy.GetPermissions() { + if err := normalizePermissionHeaders(permission); err != nil { + return nil, err + } + } + } + + // Two cases where this HTTP Filter is a no op: + // "If absent, no enforcing RBAC policy will be applied" - RBAC + // Documentation for Rules field. + // "At this time, if the RBAC.action is Action.LOG then the policy will be + // completely ignored, as if RBAC was not configured." - A41 + if rbacCfg.Rules == nil || rbacCfg.GetRules().GetAction() == v3rbacpb.RBAC_LOG { + return config{}, nil + } + + // TODO(gregorycooke) - change the call chain to here so we have the filter + // name to input here instead of an empty string. It will come from here: + // https://github.com/grpc/grpc-go/blob/eff0942e95d93112921414aee758e619ec86f26f/xds/internal/xdsclient/xdsresource/unmarshal_lds.go#L199 + ce, err := rbac.NewChainEngine([]*v3rbacpb.RBAC{rbacCfg.GetRules()}, "") + if err != nil { + // "At this time, if the RBAC.action is Action.LOG then the policy will be + // completely ignored, as if RBAC was not configured." - A41 + if rbacCfg.GetRules().GetAction() != v3rbacpb.RBAC_LOG { + return nil, fmt.Errorf("rbac: error constructing matching engine: %v", err) + } + } + + return config{chainEngine: ce}, nil +} + +// normalizePermissionHeaders applies the A41 header-name rules to every header +// matcher reachable from permission, including those nested inside and/or/not +// rules. +func normalizePermissionHeaders(permission *v3rbacpb.Permission) error { + switch p := permission.GetRule().(type) { + case *v3rbacpb.Permission_Header: + return normalizeHeaderMatcher(p.Header) + case *v3rbacpb.Permission_AndRules: + for _, rule := range p.AndRules.GetRules() { + if err := normalizePermissionHeaders(rule); err != nil { + return err + } + } + case *v3rbacpb.Permission_OrRules: + for _, rule := range p.OrRules.GetRules() { + if err := normalizePermissionHeaders(rule); err != nil { + return err + } + } + case *v3rbacpb.Permission_NotRule: + return normalizePermissionHeaders(p.NotRule) + } + return nil +} + +// normalizePrincipalHeaders applies the A41 header-name rules to every header +// matcher reachable from principal, including those nested inside and/or/not +// ids. +func normalizePrincipalHeaders(principal *v3rbacpb.Principal) error { + switch p := principal.GetIdentifier().(type) { + case *v3rbacpb.Principal_Header: + return normalizeHeaderMatcher(p.Header) + case *v3rbacpb.Principal_AndIds: + for _, id := range p.AndIds.GetIds() { + if err := normalizePrincipalHeaders(id); err != nil { + return err + } + } + case *v3rbacpb.Principal_OrIds: + for _, id := range p.OrIds.GetIds() { + if err := normalizePrincipalHeaders(id); err != nil { + return err + } + } + case *v3rbacpb.Principal_NotId: + return normalizePrincipalHeaders(p.NotId) + } + return nil +} + +// normalizeHeaderMatcher lowercases the name of a header matcher, rejects the +// names that A41 forbids (:scheme or a grpc- prefixed name) and rewrites a +// "host" matcher to ":authority". +func normalizeHeaderMatcher(header *v3routepb.HeaderMatcher) error { + // The keys of the metadata the matchers run against are always lowercase, + // so a name that contains an uppercase character matches no header at all + // and the rule using it never fires. Lowercase the name, as Envoy and + // grpc-java do, both to make it match and to keep the checks below from + // being evaded by the case of the name. + name := header.GetName() + lowerName := strings.ToLower(name) + if lowerName != name { + header.Name = lowerName + } + if lowerName == ":scheme" { + return fmt.Errorf("rbac: header matcher for %q is %q", name, ":scheme") + } + if strings.HasPrefix(lowerName, "grpc-") { + return fmt.Errorf("rbac: header matcher for %q starts with %q", name, "grpc-") + } + if lowerName == "host" { + header.Name = ":authority" + } + return nil +} + +func (builder) ParseFilterConfig(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + if cfg == nil { + return nil, fmt.Errorf("rbac: nil configuration message provided") + } + m, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("rbac: error parsing config %v: unknown type %T", cfg, cfg) + } + msg := new(rpb.RBAC) + if err := m.UnmarshalTo(msg); err != nil { + return nil, fmt.Errorf("rbac: error parsing config %v: %v", cfg, err) + } + return parseConfig(msg) +} + +func (builder) ParseFilterConfigOverride(override proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + if override == nil { + return nil, fmt.Errorf("rbac: nil configuration message provided") + } + m, ok := override.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("rbac: error parsing override config %v: unknown type %T", override, override) + } + msg := new(rpb.RBACPerRoute) + if err := m.UnmarshalTo(msg); err != nil { + return nil, fmt.Errorf("rbac: error parsing override config %v: %v", override, err) + } + return parseConfig(msg.Rbac) +} + +func (builder) IsTerminal() bool { + return false +} + +func (builder) BuildServerFilter() httpfilter.ServerFilter { + return serverFilter{} +} + +var _ httpfilter.ServerFilterBuilder = builder{} + +type serverFilter struct{} + +func (serverFilter) Close() {} + +func (serverFilter) BuildServerInterceptor(cfg httpfilter.FilterConfig, override httpfilter.FilterConfig) (httpfilter.ServerInterceptor, error) { + if cfg == nil { + return nil, fmt.Errorf("rbac: nil config provided") + } + + c, ok := cfg.(config) + if !ok { + return nil, fmt.Errorf("rbac: incorrect config type provided (%T): %v", cfg, cfg) + } + + if override != nil { + // override completely replaces the listener configuration; but we + // still validate the listener config type. + c, ok = override.(config) + if !ok { + return nil, fmt.Errorf("rbac: incorrect override config type provided (%T): %v", override, override) + } + } + + // RBAC HTTP Filter is a no op from one of these two cases: + // "If absent, no enforcing RBAC policy will be applied" - RBAC + // Documentation for Rules field. + // "At this time, if the RBAC.action is Action.LOG then the policy will be + // completely ignored, as if RBAC was not configured." - A41 + if c.chainEngine == nil { + return nil, nil + } + return &interceptor{chainEngine: c.chainEngine}, nil +} + +type interceptor struct { + chainEngine *rbac.ChainEngine +} + +func (i *interceptor) InterceptRPC(ss grpc.ServerStream) (grpc.ServerStream, error) { + if err := i.chainEngine.IsAuthorized(ss.Context()); err != nil { + return nil, err + } + return ss, nil +} + +func (i *interceptor) Close() {} diff --git a/internal/xds/httpfilter/rbac/rbac_test.go b/internal/xds/httpfilter/rbac/rbac_test.go new file mode 100644 index 0000000..04d1022 --- /dev/null +++ b/internal/xds/httpfilter/rbac/rbac_test.go @@ -0,0 +1,185 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rbac + +import ( + "testing" + + "google.golang.org/grpc/internal/grpctest" + + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + rpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func headerMatcher(name string) *v3routepb.HeaderMatcher { + return &v3routepb.HeaderMatcher{ + Name: name, + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PresentMatch{PresentMatch: true}, + } +} + +func headerPermission(name string) *v3rbacpb.Permission { + return &v3rbacpb.Permission{Rule: &v3rbacpb.Permission_Header{Header: headerMatcher(name)}} +} + +func headerPrincipal(name string) *v3rbacpb.Principal { + return &v3rbacpb.Principal{Identifier: &v3rbacpb.Principal_Header{Header: headerMatcher(name)}} +} + +func rbacConfig(perm *v3rbacpb.Permission, principal *v3rbacpb.Principal) *rpb.RBAC { + return &rpb.RBAC{Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "test-policy": { + Permissions: []*v3rbacpb.Permission{perm}, + Principals: []*v3rbacpb.Principal{principal}, + }, + }, + }} +} + +// TestNestedHeaderMatcherValidation checks that a header matcher for :scheme or +// a grpc- prefixed name is rejected even when it is nested inside an and/or/not +// rule, as A41 requires. +func (s) TestNestedHeaderMatcherValidation(t *testing.T) { + anyPermission := &v3rbacpb.Permission{Rule: &v3rbacpb.Permission_Any{Any: true}} + anyPrincipal := &v3rbacpb.Principal{Identifier: &v3rbacpb.Principal_Any{Any: true}} + + tests := []struct { + name string + cfg *rpb.RBAC + }{ + { + name: "permission and_rules :scheme", + cfg: rbacConfig(&v3rbacpb.Permission{Rule: &v3rbacpb.Permission_AndRules{AndRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{headerPermission(":scheme")}, + }}}, anyPrincipal), + }, + { + name: "permission not_rule grpc- prefix", + cfg: rbacConfig(&v3rbacpb.Permission{Rule: &v3rbacpb.Permission_NotRule{NotRule: headerPermission("grpc-timeout")}}, anyPrincipal), + }, + { + name: "principal or_ids :scheme", + cfg: rbacConfig(anyPermission, &v3rbacpb.Principal{Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{headerPrincipal(":scheme")}, + }}}), + }, + { + name: "principal not_id grpc- prefix", + cfg: rbacConfig(anyPermission, &v3rbacpb.Principal{Identifier: &v3rbacpb.Principal_NotId{NotId: headerPrincipal("grpc-encoding")}}), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := parseConfig(test.cfg); err == nil { + t.Fatalf("parseConfig() succeeded; want error rejecting a nested :scheme/grpc- header matcher") + } + }) + } +} + +// TestNestedHostHeaderAliasing checks that a "host" header matcher nested inside +// an and/or/not rule is rewritten to ":authority", so it behaves the same as a +// top-level host matcher (A41 host/:authority equivalence). +func (s) TestNestedHostHeaderAliasing(t *testing.T) { + perm := &v3rbacpb.Permission{Rule: &v3rbacpb.Permission_NotRule{NotRule: headerPermission("host")}} + principal := &v3rbacpb.Principal{Identifier: &v3rbacpb.Principal_AndIds{AndIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{headerPrincipal("host")}, + }}} + + if _, err := parseConfig(rbacConfig(perm, principal)); err != nil { + t.Fatalf("parseConfig() failed: %v", err) + } + + gotPerm := perm.GetRule().(*v3rbacpb.Permission_NotRule).NotRule.GetRule().(*v3rbacpb.Permission_Header).Header.GetName() + if gotPerm != ":authority" { + t.Errorf("Nested permission host matcher name = %q, want %q", gotPerm, ":authority") + } + gotPrincipal := principal.GetIdentifier().(*v3rbacpb.Principal_AndIds).AndIds.GetIds()[0].GetIdentifier().(*v3rbacpb.Principal_Header).Header.GetName() + if gotPrincipal != ":authority" { + t.Errorf("Nested principal host matcher name = %q, want %q", gotPrincipal, ":authority") + } +} + +// TestHeaderMatcherNameIsLowercased checks that a header matcher name is +// lowercased before it reaches the matching engine. gRPC lowercases every +// metadata key, so a name that carries an uppercase character matches no +// header at all, and a DENY policy using one fails open. +func (s) TestHeaderMatcherNameIsLowercased(t *testing.T) { + tests := []struct { + name string + wantName string + }{ + {name: "X-Role", wantName: "x-role"}, + {name: "USER-AGENT", wantName: "user-agent"}, + // The host to :authority alias must not depend on the case either. + {name: "Host", wantName: ":authority"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // The permission matcher is nested and the principal matcher is at + // the top level, so both the recursive walk and the top level are + // covered. + permHeader, principalHeader := headerMatcher(test.name), headerMatcher(test.name) + perm := &v3rbacpb.Permission{Rule: &v3rbacpb.Permission_NotRule{ + NotRule: &v3rbacpb.Permission{Rule: &v3rbacpb.Permission_Header{Header: permHeader}}, + }} + principal := &v3rbacpb.Principal{Identifier: &v3rbacpb.Principal_Header{Header: principalHeader}} + + if _, err := parseConfig(rbacConfig(perm, principal)); err != nil { + t.Fatalf("parseConfig() failed: %v", err) + } + if got := permHeader.GetName(); got != test.wantName { + t.Errorf("Permission header matcher name = %q, want %q", got, test.wantName) + } + if got := principalHeader.GetName(); got != test.wantName { + t.Errorf("Principal header matcher name = %q, want %q", got, test.wantName) + } + }) + } +} + +// TestHeaderMatcherValidationIsCaseInsensitive checks that the A41 rejection of +// :scheme and grpc- prefixed header matchers cannot be evaded by spelling the +// name in another case. +func (s) TestHeaderMatcherValidationIsCaseInsensitive(t *testing.T) { + anyPermission := &v3rbacpb.Permission{Rule: &v3rbacpb.Permission_Any{Any: true}} + anyPrincipal := &v3rbacpb.Principal{Identifier: &v3rbacpb.Principal_Any{Any: true}} + + for _, name := range []string{":Scheme", "Grpc-Timeout", "GRPC-STATUS"} { + t.Run(name, func(t *testing.T) { + if _, err := parseConfig(rbacConfig(headerPermission(name), anyPrincipal)); err == nil { + t.Errorf("parseConfig() succeeded for permission header matcher %q; want error rejecting a :scheme/grpc- header matcher", name) + } + if _, err := parseConfig(rbacConfig(anyPermission, headerPrincipal(name))); err == nil { + t.Errorf("parseConfig() succeeded for principal header matcher %q; want error rejecting a :scheme/grpc- header matcher", name) + } + }) + } +} diff --git a/internal/xds/httpfilter/router/router.go b/internal/xds/httpfilter/router/router.go new file mode 100644 index 0000000..4864197 --- /dev/null +++ b/internal/xds/httpfilter/router/router.go @@ -0,0 +1,122 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package router implements the Envoy Router HTTP filter. +package router + +import ( + "fmt" + + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + pb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" +) + +// TypeURL is the message type for the Router configuration. +const TypeURL = "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + +func init() { + httpfilter.Register(builder{}) +} + +// IsRouterFilter returns true iff b is a Router filter builder. +func IsRouterFilter(b httpfilter.Builder) bool { + _, ok := b.(builder) + return ok +} + +type builder struct { +} + +func (builder) TypeURLs() []string { return []string{TypeURL} } + +func (builder) ParseFilterConfig(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + // The gRPC router filter does not currently use any fields from the + // config. Verify type only. + if cfg == nil { + return nil, fmt.Errorf("router: nil configuration message provided") + } + m, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("router: error parsing config %v: unknown type %T", cfg, cfg) + } + msg := new(pb.Router) + if err := m.UnmarshalTo(msg); err != nil { + return nil, fmt.Errorf("router: error parsing config %v: %v", cfg, err) + } + return config{}, nil +} + +func (builder) ParseFilterConfigOverride(override proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + if override != nil { + return nil, fmt.Errorf("router: unexpected config override specified: %v", override) + } + return config{}, nil +} + +func (builder) IsTerminal() bool { + return true +} + +func (builder) BuildClientFilter(httpfilter.ClientFilterOptions) httpfilter.ClientFilter { + return filter{} +} + +func (builder) BuildServerFilter() httpfilter.ServerFilter { + return filter{} +} + +var _ httpfilter.ClientFilterBuilder = builder{} +var _ httpfilter.ServerFilterBuilder = builder{} + +type filter struct{} + +func (filter) Close() {} + +func (filter) BuildClientInterceptor(cfg, override httpfilter.FilterConfig) (httpfilter.ClientInterceptor, error) { + if _, ok := cfg.(config); !ok { + return nil, fmt.Errorf("router: incorrect config type provided (%T): %v", cfg, cfg) + } + if override != nil { + return nil, fmt.Errorf("router: unexpected override configuration specified: %v", override) + } + // The gRPC router is implemented within the xds resolver's config + // selector, not as a separate plugin. So we return a nil HTTPFilter, + // which will not be invoked. + return nil, nil +} + +func (filter) BuildServerInterceptor(cfg, override httpfilter.FilterConfig) (httpfilter.ServerInterceptor, error) { + if _, ok := cfg.(config); !ok { + return nil, fmt.Errorf("router: incorrect config type provided (%T): %v", cfg, cfg) + } + if override != nil { + return nil, fmt.Errorf("router: unexpected override configuration specified: %v", override) + } + // The gRPC router is currently unimplemented on the server side. So we + // return a nil HTTPFilter, which will not be invoked. + return nil, nil +} + +// The gRPC router filter does not currently support any configuration. Verify +// type only. +type config struct { + httpfilter.FilterConfig +} diff --git a/internal/xds/matcher/matcher_header.go b/internal/xds/matcher/matcher_header.go new file mode 100644 index 0000000..efda7f8 --- /dev/null +++ b/internal/xds/matcher/matcher_header.go @@ -0,0 +1,273 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package matcher + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "google.golang.org/grpc/metadata" +) + +// HeaderMatcher is an interface for header matchers. These are +// documented in (EnvoyProxy link here?). These matchers will match on different +// aspects of HTTP header name/value pairs. +type HeaderMatcher interface { + Match(metadata.MD) bool + String() string +} + +// valueFromMD retrieves metadata from context. If there are +// multiple values, the values are concatenated with "," (comma and no space). +// +// All header matchers only match against the comma-concatenated string. +func valueFromMD(md metadata.MD, key string) (string, bool) { + vs, ok := md[key] + if !ok { + return "", false + } + return strings.Join(vs, ","), true +} + +// HeaderExactMatcher matches on an exact match of the value of the header. +type HeaderExactMatcher struct { + key string + exact string + invert bool +} + +// NewHeaderExactMatcher returns a new HeaderExactMatcher. +func NewHeaderExactMatcher(key, exact string, invert bool) *HeaderExactMatcher { + return &HeaderExactMatcher{key: key, exact: exact, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderExactMatcher. +func (hem *HeaderExactMatcher) Match(md metadata.MD) bool { + v, ok := valueFromMD(md, hem.key) + if !ok { + return false + } + return (v == hem.exact) != hem.invert +} + +func (hem *HeaderExactMatcher) String() string { + return fmt.Sprintf("headerExact:%v:%v", hem.key, hem.exact) +} + +// HeaderRegexMatcher matches on whether the entire request header value matches +// the regex. +type HeaderRegexMatcher struct { + key string + re *regexp.Regexp + invert bool +} + +// NewHeaderRegexMatcher returns a new HeaderRegexMatcher. +func NewHeaderRegexMatcher(key string, re *regexp.Regexp, invert bool) *HeaderRegexMatcher { + return &HeaderRegexMatcher{key: key, re: re, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderRegexMatcher. +func (hrm *HeaderRegexMatcher) Match(md metadata.MD) bool { + v, ok := valueFromMD(md, hrm.key) + if !ok { + return false + } + return hrm.re.MatchString(v) != hrm.invert +} + +func (hrm *HeaderRegexMatcher) String() string { + return fmt.Sprintf("headerRegex:%v:%v", hrm.key, hrm.re.String()) +} + +// HeaderRangeMatcher matches on whether the request header value is within the +// range. The header value must be an integer in base 10 notation. +type HeaderRangeMatcher struct { + key string + start, end int64 // represents [start, end). + invert bool +} + +// NewHeaderRangeMatcher returns a new HeaderRangeMatcher. +func NewHeaderRangeMatcher(key string, start, end int64, invert bool) *HeaderRangeMatcher { + return &HeaderRangeMatcher{key: key, start: start, end: end, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderRangeMatcher. +func (hrm *HeaderRangeMatcher) Match(md metadata.MD) bool { + v, ok := valueFromMD(md, hrm.key) + if !ok { + return false + } + if i, err := strconv.ParseInt(v, 10, 64); err == nil && i >= hrm.start && i < hrm.end { + return !hrm.invert + } + return hrm.invert +} + +func (hrm *HeaderRangeMatcher) String() string { + return fmt.Sprintf("headerRange:%v:[%d,%d)", hrm.key, hrm.start, hrm.end) +} + +// HeaderPresentMatcher will match based on whether the header is present in the +// whole request. +type HeaderPresentMatcher struct { + key string + present bool +} + +// NewHeaderPresentMatcher returns a new HeaderPresentMatcher. +func NewHeaderPresentMatcher(key string, present bool, invert bool) *HeaderPresentMatcher { + if invert { + present = !present + } + return &HeaderPresentMatcher{key: key, present: present} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderPresentMatcher. +func (hpm *HeaderPresentMatcher) Match(md metadata.MD) bool { + vs, ok := valueFromMD(md, hpm.key) + present := ok && len(vs) > 0 // TODO: Are we sure we need this len(vs) > 0? + return present == hpm.present +} + +func (hpm *HeaderPresentMatcher) String() string { + return fmt.Sprintf("headerPresent:%v:%v", hpm.key, hpm.present) +} + +// HeaderPrefixMatcher matches on whether the prefix of the header value matches +// the prefix passed into this struct. +type HeaderPrefixMatcher struct { + key string + prefix string + invert bool +} + +// NewHeaderPrefixMatcher returns a new HeaderPrefixMatcher. +func NewHeaderPrefixMatcher(key string, prefix string, invert bool) *HeaderPrefixMatcher { + return &HeaderPrefixMatcher{key: key, prefix: prefix, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderPrefixMatcher. +func (hpm *HeaderPrefixMatcher) Match(md metadata.MD) bool { + v, ok := valueFromMD(md, hpm.key) + if !ok { + return false + } + return strings.HasPrefix(v, hpm.prefix) != hpm.invert +} + +func (hpm *HeaderPrefixMatcher) String() string { + return fmt.Sprintf("headerPrefix:%v:%v", hpm.key, hpm.prefix) +} + +// HeaderSuffixMatcher matches on whether the suffix of the header value matches +// the suffix passed into this struct. +type HeaderSuffixMatcher struct { + key string + suffix string + invert bool +} + +// NewHeaderSuffixMatcher returns a new HeaderSuffixMatcher. +func NewHeaderSuffixMatcher(key string, suffix string, invert bool) *HeaderSuffixMatcher { + return &HeaderSuffixMatcher{key: key, suffix: suffix, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderSuffixMatcher. +func (hsm *HeaderSuffixMatcher) Match(md metadata.MD) bool { + v, ok := valueFromMD(md, hsm.key) + if !ok { + return false + } + return strings.HasSuffix(v, hsm.suffix) != hsm.invert +} + +func (hsm *HeaderSuffixMatcher) String() string { + return fmt.Sprintf("headerSuffix:%v:%v", hsm.key, hsm.suffix) +} + +// HeaderContainsMatcher matches on whether the header value contains the +// value passed into this struct. +type HeaderContainsMatcher struct { + key string + contains string + invert bool +} + +// NewHeaderContainsMatcher returns a new HeaderContainsMatcher. key is the HTTP +// Header key to match on, and contains is the value that the header should +// contain for a successful match. An empty contains string does not +// work, use HeaderPresentMatcher in that case. +func NewHeaderContainsMatcher(key string, contains string, invert bool) *HeaderContainsMatcher { + return &HeaderContainsMatcher{key: key, contains: contains, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderContainsMatcher. +func (hcm *HeaderContainsMatcher) Match(md metadata.MD) bool { + v, ok := valueFromMD(md, hcm.key) + if !ok { + return false + } + return strings.Contains(v, hcm.contains) != hcm.invert +} + +func (hcm *HeaderContainsMatcher) String() string { + return fmt.Sprintf("headerContains:%v%v", hcm.key, hcm.contains) +} + +// HeaderStringMatcher matches on whether the header value matches against the +// StringMatcher specified. +type HeaderStringMatcher struct { + key string + stringMatcher StringMatcher + invert bool +} + +// NewHeaderStringMatcher returns a new HeaderStringMatcher. +func NewHeaderStringMatcher(key string, sm StringMatcher, invert bool) *HeaderStringMatcher { + return &HeaderStringMatcher{ + key: key, + stringMatcher: sm, + invert: invert, + } +} + +// Match returns whether the passed in HTTP Headers match according to the +// specified StringMatcher. +func (hsm *HeaderStringMatcher) Match(md metadata.MD) bool { + v, ok := valueFromMD(md, hsm.key) + if !ok { + return false + } + return hsm.stringMatcher.Match(v) != hsm.invert +} + +func (hsm *HeaderStringMatcher) String() string { + return fmt.Sprintf("headerString:%v:%v", hsm.key, hsm.stringMatcher) +} diff --git a/internal/xds/matcher/matcher_header_test.go b/internal/xds/matcher/matcher_header_test.go new file mode 100644 index 0000000..9a20cf1 --- /dev/null +++ b/internal/xds/matcher/matcher_header_test.go @@ -0,0 +1,549 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package matcher + +import ( + "regexp" + "testing" + + "google.golang.org/grpc/metadata" +) + +func TestHeaderExactMatcherMatch(t *testing.T) { + tests := []struct { + name string + key, exact string + md metadata.MD + want bool + invert bool + }{ + { + name: "one value one match", + key: "th", + exact: "tv", + md: metadata.Pairs("th", "tv"), + want: true, + }, + { + name: "two value one match", + key: "th", + exact: "tv", + md: metadata.Pairs("th", "abc", "th", "tv"), + // Doesn't match comma-concatenated string. + want: false, + }, + { + name: "two value match concatenated", + key: "th", + exact: "abc,tv", + md: metadata.Pairs("th", "abc", "th", "tv"), + want: true, + }, + { + name: "not match", + key: "th", + exact: "tv", + md: metadata.Pairs("th", "abc"), + want: false, + }, + { + name: "invert header not present", + key: "th", + exact: "tv", + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + exact: "tv", + md: metadata.Pairs("th", "tv"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + exact: "tv", + md: metadata.Pairs("th", "tvv"), + want: true, + invert: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hem := NewHeaderExactMatcher(tt.key, tt.exact, tt.invert) + if got := hem.Match(tt.md); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHeaderRegexMatcherMatch(t *testing.T) { + tests := []struct { + name string + key, regexStr string + md metadata.MD + want bool + invert bool + }{ + { + name: "one value one match", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs("th", "tttvv"), + want: true, + }, + { + name: "two value one match", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs("th", "abc", "th", "tttvv"), + want: false, + }, + { + name: "two value match concatenated", + key: "th", + regexStr: "^[abc]*,t+v*$", + md: metadata.Pairs("th", "abc", "th", "tttvv"), + want: true, + }, + { + name: "no match", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs("th", "abc"), + want: false, + }, + { + name: "no match because only part of value matches with regex", + key: "header", + regexStr: "^a+$", + md: metadata.Pairs("header", "ab"), + want: false, + }, + { + name: "match because full value matches with regex", + key: "header", + regexStr: "^a+$", + md: metadata.Pairs("header", "aa"), + want: true, + }, + { + name: "invert header not present", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs("th", "tttvv"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs("th", "abc"), + want: true, + invert: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hrm := NewHeaderRegexMatcher(tt.key, regexp.MustCompile(tt.regexStr), tt.invert) + if got := hrm.Match(tt.md); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHeaderRangeMatcherMatch(t *testing.T) { + tests := []struct { + name string + key string + start, end int64 + md metadata.MD + want bool + invert bool + }{ + { + name: "match", + key: "th", + start: 1, end: 10, + md: metadata.Pairs("th", "5"), + want: true, + }, + { + name: "equal to start", + key: "th", + start: 1, end: 10, + md: metadata.Pairs("th", "1"), + want: true, + }, + { + name: "equal to end", + key: "th", + start: 1, end: 10, + md: metadata.Pairs("th", "10"), + want: false, + }, + { + name: "negative", + key: "th", + start: -10, end: 10, + md: metadata.Pairs("th", "-5"), + want: true, + }, + { + name: "invert header not present", + key: "th", + start: 1, end: 10, + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + start: 1, end: 10, + md: metadata.Pairs("th", "5"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + start: 1, end: 9, + md: metadata.Pairs("th", "10"), + want: true, + invert: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hrm := NewHeaderRangeMatcher(tt.key, tt.start, tt.end, tt.invert) + if got := hrm.Match(tt.md); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHeaderPresentMatcherMatch(t *testing.T) { + tests := []struct { + name string + key string + present bool + md metadata.MD + want bool + invert bool + }{ + { + name: "want present is present", + key: "th", + present: true, + md: metadata.Pairs("th", "tv"), + want: true, + }, + { + name: "want present not present", + key: "th", + present: true, + md: metadata.Pairs("abc", "tv"), + want: false, + }, + { + name: "want not present is present", + key: "th", + present: false, + md: metadata.Pairs("th", "tv"), + want: false, + }, + { + name: "want not present is not present", + key: "th", + present: false, + md: metadata.Pairs("abc", "tv"), + want: true, + }, + { + name: "invert header not present", + key: "th", + present: true, + md: metadata.Pairs(":method", "GET"), + want: true, + invert: true, + }, + { + name: "invert header match", + key: "th", + present: true, + md: metadata.Pairs("th", "tv"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + present: true, + md: metadata.Pairs(":method", "GET"), + want: true, + invert: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hpm := NewHeaderPresentMatcher(tt.key, tt.present, tt.invert) + if got := hpm.Match(tt.md); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHeaderPrefixMatcherMatch(t *testing.T) { + tests := []struct { + name string + key, prefix string + md metadata.MD + want bool + invert bool + }{ + { + name: "one value one match", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "tv123"), + want: true, + }, + { + name: "two value one match", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "abc", "th", "tv123"), + want: false, + }, + { + name: "two value match concatenated", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "tv123", "th", "abc"), + want: true, + }, + { + name: "not match", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "abc"), + want: false, + }, + { + name: "invert header not present", + key: "th", + prefix: "tv", + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "tv123"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "abc"), + want: true, + invert: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hpm := NewHeaderPrefixMatcher(tt.key, tt.prefix, tt.invert) + if got := hpm.Match(tt.md); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHeaderSuffixMatcherMatch(t *testing.T) { + tests := []struct { + name string + key, suffix string + md metadata.MD + want bool + invert bool + }{ + { + name: "one value one match", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "123tv"), + want: true, + }, + { + name: "two value one match", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "123tv", "th", "abc"), + want: false, + }, + { + name: "two value match concatenated", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "abc", "th", "123tv"), + want: true, + }, + { + name: "not match", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "abc"), + want: false, + }, + { + name: "invert header not present", + key: "th", + suffix: "tv", + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "123tv"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "abc"), + want: true, + invert: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hsm := NewHeaderSuffixMatcher(tt.key, tt.suffix, tt.invert) + if got := hsm.Match(tt.md); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHeaderStringMatch(t *testing.T) { + tests := []struct { + name string + key string + sm StringMatcher + invert bool + md metadata.MD + want bool + }{ + { + name: "should-match", + key: "th", + sm: StringMatcher{ + exactMatch: newStringP("tv"), + }, + invert: false, + md: metadata.Pairs("th", "tv"), + want: true, + }, + { + name: "not match", + key: "th", + sm: StringMatcher{ + containsMatch: newStringP("tv"), + }, + invert: false, + md: metadata.Pairs("th", "not-match"), + want: false, + }, + { + name: "invert string match", + key: "th", + sm: StringMatcher{ + containsMatch: newStringP("tv"), + }, + invert: true, + md: metadata.Pairs("th", "not-match"), + want: true, + }, + { + name: "header missing", + key: "th", + sm: StringMatcher{ + containsMatch: newStringP("tv"), + }, + invert: false, + md: metadata.Pairs("not-specified-key", "not-match"), + want: false, + }, + { + name: "header missing invert true", + key: "th", + sm: StringMatcher{ + containsMatch: newStringP("tv"), + }, + invert: true, + md: metadata.Pairs("not-specified-key", "not-match"), + want: false, + }, + { + name: "header empty string invert", + key: "th", + sm: StringMatcher{ + containsMatch: newStringP("tv"), + }, + invert: true, + md: metadata.Pairs("th", ""), + want: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + hsm := NewHeaderStringMatcher(test.key, test.sm, test.invert) + if got := hsm.Match(test.md); got != test.want { + t.Errorf("match() = %v, want %v", got, test.want) + } + }) + } +} diff --git a/internal/xds/matcher/string_matcher.go b/internal/xds/matcher/string_matcher.go new file mode 100644 index 0000000..edcc36c --- /dev/null +++ b/internal/xds/matcher/string_matcher.go @@ -0,0 +1,229 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package matcher contains types that need to be shared between code under +// google.golang.org/grpc/xds/... and the rest of gRPC. +package matcher + +import ( + "errors" + "fmt" + "regexp" + "strings" + + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" +) + +// StringMatcher contains match criteria for matching a string, and is an +// internal representation of the `StringMatcher` proto defined at +// https://github.com/envoyproxy/envoy/blob/main/api/envoy/type/matcher/v3/string.proto. +type StringMatcher struct { + // Since these match fields are part of a `oneof` in the corresponding xDS + // proto, only one of them is expected to be set. + exactMatch *string + prefixMatch *string + suffixMatch *string + regexMatch *regexp.Regexp + containsMatch *string + // If true, indicates the exact/prefix/suffix/contains matching should be + // case insensitive. This has no effect on the regex match. + ignoreCase bool +} + +// Match returns true if input matches the criteria in the given StringMatcher. +func (sm StringMatcher) Match(input string) bool { + switch { + case sm.exactMatch != nil: + if sm.ignoreCase { + input = strings.ToLower(input) + } + return input == *sm.exactMatch + case sm.prefixMatch != nil: + if sm.ignoreCase { + input = strings.ToLower(input) + } + return strings.HasPrefix(input, *sm.prefixMatch) + case sm.suffixMatch != nil: + if sm.ignoreCase { + input = strings.ToLower(input) + } + return strings.HasSuffix(input, *sm.suffixMatch) + case sm.containsMatch != nil: + if sm.ignoreCase { + input = strings.ToLower(input) + } + return strings.Contains(input, *sm.containsMatch) + case sm.regexMatch != nil: + return sm.regexMatch.MatchString(input) + } + return false +} + +// newStrPtr allocates a new string that holds the value of input and returns a +// pointer to it. ignoreCase controls if a lower case version of input is used. +func newStrPtr(input *string, ignoreCase bool) *string { + if input == nil { + return nil + } + + s := new(string) + if ignoreCase { + *s = strings.ToLower(*input) + } else { + *s = *input + } + return s +} + +// StringMatcherFromProto is a helper function to create a StringMatcher from +// the corresponding StringMatcher proto. +// +// Returns a non-nil error if matcherProto is invalid. +func StringMatcherFromProto(matcherProto *v3matcherpb.StringMatcher) (StringMatcher, error) { + if matcherProto == nil { + return StringMatcher{}, errors.New("input StringMatcher proto is nil") + } + + matcher := StringMatcher{ignoreCase: matcherProto.GetIgnoreCase()} + switch mt := matcherProto.GetMatchPattern().(type) { + case *v3matcherpb.StringMatcher_Exact: + matcher.exactMatch = newStrPtr(&mt.Exact, matcher.ignoreCase) + case *v3matcherpb.StringMatcher_Prefix: + if matcherProto.GetPrefix() == "" { + return StringMatcher{}, errors.New("empty prefix is not allowed in StringMatcher") + } + matcher.prefixMatch = newStrPtr(&mt.Prefix, matcher.ignoreCase) + case *v3matcherpb.StringMatcher_Suffix: + if matcherProto.GetSuffix() == "" { + return StringMatcher{}, errors.New("empty suffix is not allowed in StringMatcher") + } + matcher.suffixMatch = newStrPtr(&mt.Suffix, matcher.ignoreCase) + case *v3matcherpb.StringMatcher_SafeRegex: + regex := matcherProto.GetSafeRegex().GetRegex() + re, err := CompileSafeRegex(regex) + if err != nil { + return StringMatcher{}, fmt.Errorf("safe_regex matcher %q is invalid", regex) + } + matcher = NewRegexStringMatcher(re) + case *v3matcherpb.StringMatcher_Contains: + if matcherProto.GetContains() == "" { + return StringMatcher{}, errors.New("empty contains is not allowed in StringMatcher") + } + matcher.containsMatch = newStrPtr(&mt.Contains, matcher.ignoreCase) + default: + return StringMatcher{}, fmt.Errorf("unrecognized string matcher: %+v", matcherProto) + } + return matcher, nil +} + +// NewExactStringMatcher creates a string matcher that requires the input string +// to exactly match the pattern specified here. The match will be case +// insensitive if ignore_case is true. +func NewExactStringMatcher(pattern string, ignoreCase bool) StringMatcher { + return StringMatcher{ + exactMatch: newStrPtr(&pattern, ignoreCase), + ignoreCase: ignoreCase, + } +} + +// NewPrefixStringMatcher creates a string matcher that requires the input +// string to contain the prefix specified here. The match will be case +// insensitive if ignore_case is true. +func NewPrefixStringMatcher(prefix string, ignoreCase bool) StringMatcher { + return StringMatcher{ + prefixMatch: newStrPtr(&prefix, ignoreCase), + ignoreCase: ignoreCase, + } +} + +// NewSuffixStringMatcher creates a string matcher that requires the input +// string to contain the suffix specified here. The match will be case +// insensitive if ignore_case is true. +func NewSuffixStringMatcher(suffix string, ignoreCase bool) StringMatcher { + return StringMatcher{ + suffixMatch: newStrPtr(&suffix, ignoreCase), + ignoreCase: ignoreCase, + } +} + +// NewContainsStringMatcher creates a string matcher that requires the input +// string to contain the pattern specified here. The match will be case +// insensitive if ignore_case is true. +func NewContainsStringMatcher(pattern string, ignoreCase bool) StringMatcher { + return StringMatcher{ + containsMatch: newStrPtr(&pattern, ignoreCase), + ignoreCase: ignoreCase, + } +} + +// NewRegexStringMatcher creates a string matcher that requires the input string +// to match the regular expression specified here. +func NewRegexStringMatcher(regex *regexp.Regexp) StringMatcher { + return StringMatcher{ + regexMatch: regex, + } +} + +// ExactMatch returns the value of the configured exact match or an empty string +// if exact match criteria was not specified. +func (sm StringMatcher) ExactMatch() string { + if sm.exactMatch != nil { + return *sm.exactMatch + } + return "" +} + +// Equal returns true if other and sm are equivalent to each other. +func (sm StringMatcher) Equal(other StringMatcher) bool { + if sm.ignoreCase != other.ignoreCase { + return false + } + + if (sm.exactMatch != nil) != (other.exactMatch != nil) || + (sm.prefixMatch != nil) != (other.prefixMatch != nil) || + (sm.suffixMatch != nil) != (other.suffixMatch != nil) || + (sm.regexMatch != nil) != (other.regexMatch != nil) || + (sm.containsMatch != nil) != (other.containsMatch != nil) { + return false + } + + switch { + case sm.exactMatch != nil: + return *sm.exactMatch == *other.exactMatch + case sm.prefixMatch != nil: + return *sm.prefixMatch == *other.prefixMatch + case sm.suffixMatch != nil: + return *sm.suffixMatch == *other.suffixMatch + case sm.regexMatch != nil: + return sm.regexMatch.String() == other.regexMatch.String() + case sm.containsMatch != nil: + return *sm.containsMatch == *other.containsMatch + } + return true +} + +// CompileSafeRegex attempts to compile the provided pattern as a regular +// expression. It first compiles the unanchored pattern to catch syntax errors +// and then compiles and returns the explicitly anchored pattern to guarantee +// full-string matching. +func CompileSafeRegex(pattern string) (*regexp.Regexp, error) { + if _, err := regexp.Compile(pattern); err != nil { + return nil, err + } + return regexp.Compile(fmt.Sprintf("^(?:%s)$", pattern)) +} diff --git a/internal/xds/matcher/string_matcher_test.go b/internal/xds/matcher/string_matcher_test.go new file mode 100644 index 0000000..9f9afc9 --- /dev/null +++ b/internal/xds/matcher/string_matcher_test.go @@ -0,0 +1,349 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package matcher + +import ( + "regexp" + "testing" + + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + "github.com/google/go-cmp/cmp" +) + +func TestStringMatcherFromProto(t *testing.T) { + tests := []struct { + desc string + inputProto *v3matcherpb.StringMatcher + wantMatcher StringMatcher + wantErr bool + }{ + { + desc: "nil proto", + wantErr: true, + }, + { + desc: "empty prefix", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: ""}, + }, + wantErr: true, + }, + { + desc: "empty suffix", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: ""}, + }, + wantErr: true, + }, + { + desc: "empty contains", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: ""}, + }, + wantErr: true, + }, + { + desc: "invalid regex", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{ + SafeRegex: &v3matcherpb.RegexMatcher{Regex: "??"}, + }, + }, + wantErr: true, + }, + { + desc: "happy case exact", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "exact"}, + }, + wantMatcher: StringMatcher{exactMatch: newStringP("exact")}, + }, + { + desc: "happy case exact ignore case", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "EXACT"}, + IgnoreCase: true, + }, + wantMatcher: StringMatcher{ + exactMatch: newStringP("exact"), + ignoreCase: true, + }, + }, + { + desc: "happy case prefix", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "prefix"}, + }, + wantMatcher: StringMatcher{prefixMatch: newStringP("prefix")}, + }, + { + desc: "happy case prefix ignore case", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "PREFIX"}, + IgnoreCase: true, + }, + wantMatcher: StringMatcher{ + prefixMatch: newStringP("prefix"), + ignoreCase: true, + }, + }, + { + desc: "happy case suffix", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "suffix"}, + }, + wantMatcher: StringMatcher{suffixMatch: newStringP("suffix")}, + }, + { + desc: "happy case suffix ignore case", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "SUFFIX"}, + IgnoreCase: true, + }, + wantMatcher: StringMatcher{ + suffixMatch: newStringP("suffix"), + ignoreCase: true, + }, + }, + { + desc: "happy case regex", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{ + SafeRegex: &v3matcherpb.RegexMatcher{Regex: "good?regex?"}, + }, + }, + wantMatcher: StringMatcher{regexMatch: regexp.MustCompile("^(?:good?regex?)$")}, + }, + { + desc: "regex with ignore case", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{ + SafeRegex: &v3matcherpb.RegexMatcher{Regex: "good?regex?"}, + }, + IgnoreCase: true, + }, + wantMatcher: StringMatcher{ + regexMatch: regexp.MustCompile("^(?:good?regex?)$"), + }, + }, + { + desc: "happy case contains", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "contains"}, + }, + wantMatcher: StringMatcher{containsMatch: newStringP("contains")}, + }, + { + desc: "happy case contains ignore case", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "CONTAINS"}, + IgnoreCase: true, + }, + wantMatcher: StringMatcher{ + containsMatch: newStringP("contains"), + ignoreCase: true, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + gotMatcher, err := StringMatcherFromProto(test.inputProto) + if (err != nil) != test.wantErr { + t.Fatalf("StringMatcherFromProto(%+v) returned err: %v, wantErr: %v", test.inputProto, err, test.wantErr) + } + if diff := cmp.Diff(gotMatcher, test.wantMatcher, cmp.AllowUnexported(regexp.Regexp{})); diff != "" { + t.Fatalf("StringMatcherFromProto(%+v) returned unexpected diff (-got, +want):\n%s", test.inputProto, diff) + } + }) + } +} + +func TestMatch(t *testing.T) { + var ( + exactMatcher = NewExactStringMatcher("exact", false) + exactMatcherIgnoreCase = NewExactStringMatcher("exact", true) + prefixMatcher = NewPrefixStringMatcher("prefix", false) + prefixMatcherIgnoreCase = NewPrefixStringMatcher("prefix", true) + suffixMatcher = NewSuffixStringMatcher("suffix", false) + suffixMatcherIgnoreCase = NewSuffixStringMatcher("suffix", true) + containsMatcher = NewContainsStringMatcher("contains", false) + containsMatcherIgnoreCase = NewContainsStringMatcher("contains", true) + regexMatcher = NewRegexStringMatcher(regexp.MustCompile("^(?:good?regex?)$")) + ) + + tests := []struct { + desc string + matcher StringMatcher + input string + wantMatch bool + }{ + { + desc: "exact match success", + matcher: exactMatcher, + input: "exact", + wantMatch: true, + }, + { + desc: "exact match failure", + matcher: exactMatcher, + input: "not-exact", + }, + { + desc: "exact match success with ignore case", + matcher: exactMatcherIgnoreCase, + input: "EXACT", + wantMatch: true, + }, + { + desc: "exact match failure with ignore case", + matcher: exactMatcherIgnoreCase, + input: "not-exact", + }, + { + desc: "prefix match success", + matcher: prefixMatcher, + input: "prefixIsHere", + wantMatch: true, + }, + { + desc: "prefix match failure", + matcher: prefixMatcher, + input: "not-prefix", + }, + { + desc: "prefix match success with ignore case", + matcher: prefixMatcherIgnoreCase, + input: "PREFIXisHere", + wantMatch: true, + }, + { + desc: "prefix match failure with ignore case", + matcher: prefixMatcherIgnoreCase, + input: "not-PREFIX", + }, + { + desc: "suffix match success", + matcher: suffixMatcher, + input: "hereIsThesuffix", + wantMatch: true, + }, + { + desc: "suffix match failure", + matcher: suffixMatcher, + input: "suffix-is-not-here", + }, + { + desc: "suffix match success with ignore case", + matcher: suffixMatcherIgnoreCase, + input: "hereIsTheSuFFix", + wantMatch: true, + }, + { + desc: "suffix match failure with ignore case", + matcher: suffixMatcherIgnoreCase, + input: "SUFFIX-is-not-here", + }, + { + desc: "regex match success", + matcher: regexMatcher, + input: "goodregex", + wantMatch: true, + }, + { + desc: "regex match failure because only part match", + matcher: regexMatcher, + input: "goodregexa", + wantMatch: false, + }, + { + desc: "regex match failure", + matcher: regexMatcher, + input: "regex-is-not-here", + }, + { + desc: "contains match success", + matcher: containsMatcher, + input: "IScontainsHERE", + wantMatch: true, + }, + { + desc: "contains match failure", + matcher: containsMatcher, + input: "con-tains-is-not-here", + }, + { + desc: "contains match success with ignore case", + matcher: containsMatcherIgnoreCase, + input: "isCONTAINShere", + wantMatch: true, + }, + { + desc: "contains match failure with ignore case", + matcher: containsMatcherIgnoreCase, + input: "CON-TAINS-is-not-here", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if gotMatch := test.matcher.Match(test.input); gotMatch != test.wantMatch { + t.Errorf("StringMatcher.Match(%s) returned %v, want %v", test.input, gotMatch, test.wantMatch) + } + }) + } +} + +func newStringP(s string) *string { + return &s +} + +func TestCompileSafeRegex(t *testing.T) { + tests := []struct { + desc string + pattern string + wantRegex string + wantErr bool + }{ + { + desc: "ValidPattern", + pattern: "abc", + wantRegex: "^(?:abc)$", + }, + { + desc: "UnmatchedParenthesis", + pattern: "abc(", + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + re, err := CompileSafeRegex(test.pattern) + if (err != nil) != test.wantErr { + t.Fatalf("CompileSafeRegex(%q) returned error %q, wantErr=%v", test.pattern, err, test.wantErr) + } + if err == nil { + if re.String() != test.wantRegex { + t.Fatalf("CompileSafeRegex(%q) = %q, want %q", test.pattern, re.String(), test.wantRegex) + } + } + }) + } +} diff --git a/internal/xds/rbac/converter.go b/internal/xds/rbac/converter.go new file mode 100644 index 0000000..fb59995 --- /dev/null +++ b/internal/xds/rbac/converter.go @@ -0,0 +1,101 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rbac + +import ( + "encoding/json" + "fmt" + "strings" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3auditloggersstreampb "github.com/envoyproxy/go-control-plane/envoy/extensions/rbac/audit_loggers/stream/v3" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/authz/audit/stdout" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" +) + +func buildLogger(loggerConfig *v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig) (audit.Logger, error) { + if loggerConfig.GetAuditLogger().GetTypedConfig() == nil { + return nil, fmt.Errorf("missing required field: TypedConfig") + } + customConfig, loggerName, err := getCustomConfig(loggerConfig.AuditLogger.TypedConfig) + if err != nil { + return nil, err + } + if loggerName == "" { + return nil, fmt.Errorf("field TypedConfig.TypeURL cannot be an empty string") + } + factory := audit.GetLoggerBuilder(loggerName) + if factory == nil { + if loggerConfig.IsOptional { + return nil, nil + } + return nil, fmt.Errorf("no builder registered for %v", loggerName) + } + auditLoggerConfig, err := factory.ParseLoggerConfig(customConfig) + if err != nil { + return nil, fmt.Errorf("custom config could not be parsed by registered factory. error: %v", err) + } + auditLogger := factory.Build(auditLoggerConfig) + return auditLogger, nil +} + +func getCustomConfig(config *anypb.Any) (json.RawMessage, string, error) { + c, err := config.UnmarshalNew() + if err != nil { + return nil, "", err + } + switch m := c.(type) { + case *v1xdsudpatypepb.TypedStruct: + return convertCustomConfig(m.TypeUrl, m.Value) + case *v3xdsxdstypepb.TypedStruct: + return convertCustomConfig(m.TypeUrl, m.Value) + case *v3auditloggersstreampb.StdoutAuditLog: + return convertStdoutConfig(m) + } + return nil, "", fmt.Errorf("custom config not implemented for type [%v]", config.GetTypeUrl()) +} + +func convertStdoutConfig(config *v3auditloggersstreampb.StdoutAuditLog) (json.RawMessage, string, error) { + json, err := protojson.Marshal(config) + return json, stdout.Name, err +} + +func convertCustomConfig(typeURL string, s *structpb.Struct) (json.RawMessage, string, error) { + // The gRPC policy name will be the "type name" part of the value of the + // type_url field in the TypedStruct. We get this by using the part after + // the last / character. Can assume a valid type_url from the control plane. + urls := strings.Split(typeURL, "/") + if len(urls) == 0 { + return nil, "", fmt.Errorf("error converting custom audit logger %v for %v: typeURL must have a url-like format with the typeName being the value after the last /", typeURL, s) + } + name := urls[len(urls)-1] + + rawJSON := []byte("{}") + var err error + if s != nil { + rawJSON, err = json.Marshal(s) + if err != nil { + return nil, "", fmt.Errorf("error converting custom audit logger %v for %v: %v", typeURL, s, err) + } + } + return rawJSON, name, nil +} diff --git a/internal/xds/rbac/converter_test.go b/internal/xds/rbac/converter_test.go new file mode 100644 index 0000000..b920ed5 --- /dev/null +++ b/internal/xds/rbac/converter_test.go @@ -0,0 +1,170 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rbac + +import ( + "reflect" + "strings" + "testing" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3auditloggersstreampb "github.com/envoyproxy/go-control-plane/envoy/extensions/rbac/audit_loggers/stream/v3" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/authz/audit/stdout" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/protobuf/types/known/anypb" +) + +func (s) TestBuildLoggerErrors(t *testing.T) { + tests := []struct { + name string + loggerConfig *v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig + expectedLogger audit.Logger + expectedError string + }{ + { + name: "nil typed config", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + TypedConfig: nil, + }, + }, + expectedError: "missing required field: TypedConfig", + }, + { + name: "Unsupported Type", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: testutils.MarshalAny(t, &v3rbacpb.RBAC_AuditLoggingOptions{}), + }, + }, + expectedError: "custom config not implemented for type ", + }, + { + name: "Empty name", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, ""), + }, + }, + expectedError: "field TypedConfig.TypeURL cannot be an empty string", + }, + { + name: "No registered logger", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "UnregisteredLogger", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "UnregisteredLogger"), + }, + IsOptional: false, + }, + expectedError: "no builder registered for UnregisteredLogger", + }, + { + name: "fail to parse custom config", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerCustomConfig", + TypedConfig: createUDPATypedStruct(t, map[string]any{"abc": "BADVALUE", "xyz": "123"}, "fail to parse custom config_TestAuditLoggerCustomConfig")}, + IsOptional: false, + }, + expectedError: "custom config could not be parsed", + }, + { + name: "no registered logger but optional passes", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "UnregisteredLogger", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "no registered logger but optional passes_UnregisteredLogger"), + }, + IsOptional: true, + }, + expectedLogger: nil, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + b := TestAuditLoggerCustomConfigBuilder{testName: test.name} + audit.RegisterLoggerBuilder(&b) + logger, err := buildLogger(test.loggerConfig) + if err != nil && !strings.HasPrefix(err.Error(), test.expectedError) { + t.Fatalf("expected error: %v. got error: %v", test.expectedError, err) + } + if logger != test.expectedLogger { + t.Fatalf("expected logger: %v. got logger: %v", test.expectedLogger, logger) + } + + }) + } +} + +func (s) TestBuildLoggerKnownTypes(t *testing.T) { + tests := []struct { + name string + loggerConfig *v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig + expectedType reflect.Type + }{ + { + name: "stdout logger", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: stdout.Name, + TypedConfig: createStdoutPb(t), + }, + IsOptional: false, + }, + expectedType: reflect.TypeOf(audit.GetLoggerBuilder(stdout.Name).Build(nil)), + }, + { + name: "stdout logger with generic TypedConfig", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: stdout.Name, + TypedConfig: createXDSTypedStruct(t, map[string]any{}, stdout.Name), + }, + IsOptional: false, + }, + expectedType: reflect.TypeOf(audit.GetLoggerBuilder(stdout.Name).Build(nil)), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + logger, err := buildLogger(test.loggerConfig) + if err != nil { + t.Fatalf("expected success. got error: %v", err) + } + loggerType := reflect.TypeOf(logger) + if test.expectedType != loggerType { + t.Fatalf("logger not of expected type. want: %v got: %v", test.expectedType, loggerType) + } + }) + } +} + +// Builds stdout config for audit logger proto. +func createStdoutPb(t *testing.T) *anypb.Any { + t.Helper() + pb := &v3auditloggersstreampb.StdoutAuditLog{} + customConfig, err := anypb.New(pb) + if err != nil { + t.Fatalf("createStdoutPb failed during anypb.New: %v", err) + } + return customConfig +} diff --git a/internal/xds/rbac/matchers.go b/internal/xds/rbac/matchers.go new file mode 100644 index 0000000..f5b064d --- /dev/null +++ b/internal/xds/rbac/matchers.go @@ -0,0 +1,505 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rbac + +import ( + "errors" + "fmt" + "net" + "net/netip" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3route_componentspb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + internalmatcher "google.golang.org/grpc/internal/xds/matcher" +) + +// matcher is an interface that takes data about incoming RPC's and returns +// whether it matches with whatever matcher implements this interface. +type matcher interface { + match(data *rpcData) bool +} + +// policyMatcher helps determine whether an incoming RPC call matches a policy. +// A policy is a logical role (e.g. Service Admin), which is comprised of +// permissions and principals. A principal is an identity (or identities) for a +// downstream subject which are assigned the policy (role), and a permission is +// an action(s) that a principal(s) can take. A policy matches if both a +// permission and a principal match, which will be determined by the child or +// permissions and principal matchers. policyMatcher implements the matcher +// interface. +type policyMatcher struct { + permissions *orMatcher + principals *orMatcher +} + +func newPolicyMatcher(policy *v3rbacpb.Policy) (*policyMatcher, error) { + permissions, err := matchersFromPermissions(policy.Permissions) + if err != nil { + return nil, err + } + principals, err := matchersFromPrincipals(policy.Principals) + if err != nil { + return nil, err + } + return &policyMatcher{ + permissions: &orMatcher{matchers: permissions}, + principals: &orMatcher{matchers: principals}, + }, nil +} + +func (pm *policyMatcher) match(data *rpcData) bool { + // A policy matches if and only if at least one of its permissions match the + // action taking place AND at least one if its principals match the + // downstream peer. + return pm.permissions.match(data) && pm.principals.match(data) +} + +// matchersFromPermissions takes a list of permissions (can also be +// a single permission, e.g. from a not matcher which is logically !permission) +// and returns a list of matchers which correspond to that permission. This will +// be called in many instances throughout the initial construction of the RBAC +// engine from the AND and OR matchers and also from the NOT matcher. +func matchersFromPermissions(permissions []*v3rbacpb.Permission) ([]matcher, error) { + var matchers []matcher + for _, permission := range permissions { + switch p := permission.GetRule().(type) { + case *v3rbacpb.Permission_AndRules: + mList, err := matchersFromPermissions(permission.GetAndRules().Rules) + if err != nil { + return nil, err + } + matchers = append(matchers, &andMatcher{matchers: mList}) + case *v3rbacpb.Permission_OrRules: + mList, err := matchersFromPermissions(permission.GetOrRules().Rules) + if err != nil { + return nil, err + } + matchers = append(matchers, &orMatcher{matchers: mList}) + case *v3rbacpb.Permission_Any: + matchers = append(matchers, &alwaysMatcher{}) + case *v3rbacpb.Permission_Header: + m, err := newHeaderMatcher(permission.GetHeader()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Permission_UrlPath: + m, err := newURLPathMatcher(permission.GetUrlPath()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Permission_DestinationIp: + // Due to this being on server side, the destination IP is the local + // IP. + m, err := newLocalIPMatcher(permission.GetDestinationIp()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Permission_DestinationPort: + matchers = append(matchers, newPortMatcher(permission.GetDestinationPort())) + case *v3rbacpb.Permission_NotRule: + mList, err := matchersFromPermissions([]*v3rbacpb.Permission{{Rule: permission.GetNotRule().Rule}}) + if err != nil { + return nil, err + } + if len(mList) != 1 { + return nil, fmt.Errorf("NotRule must contain exactly one rule") + } + matchers = append(matchers, ¬Matcher{matcherToNot: mList[0]}) + case *v3rbacpb.Permission_Metadata: + if permission.GetMetadata().GetInvert() { + matchers = append(matchers, &alwaysMatcher{}) + } else { + matchers = append(matchers, &neverMatcher{}) + } + case *v3rbacpb.Permission_RequestedServerName: + m, err := newRequestedServerNameMatcher(permission.GetRequestedServerName()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + default: + return nil, fmt.Errorf("unsupported permission rule type: %T", p) + } + } + return matchers, nil +} + +func matchersFromPrincipals(principals []*v3rbacpb.Principal) ([]matcher, error) { + var matchers []matcher + for _, principal := range principals { + switch p := principal.GetIdentifier().(type) { + case *v3rbacpb.Principal_AndIds: + mList, err := matchersFromPrincipals(principal.GetAndIds().Ids) + if err != nil { + return nil, err + } + matchers = append(matchers, &andMatcher{matchers: mList}) + case *v3rbacpb.Principal_OrIds: + mList, err := matchersFromPrincipals(principal.GetOrIds().Ids) + if err != nil { + return nil, err + } + matchers = append(matchers, &orMatcher{matchers: mList}) + case *v3rbacpb.Principal_Any: + matchers = append(matchers, &alwaysMatcher{}) + case *v3rbacpb.Principal_Authenticated_: + authenticatedMatcher, err := newAuthenticatedMatcher(principal.GetAuthenticated()) + if err != nil { + return nil, err + } + matchers = append(matchers, authenticatedMatcher) + case *v3rbacpb.Principal_DirectRemoteIp: + m, err := newRemoteIPMatcher(principal.GetDirectRemoteIp()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Principal_Header: + // Do we need an error here? + m, err := newHeaderMatcher(principal.GetHeader()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Principal_UrlPath: + m, err := newURLPathMatcher(principal.GetUrlPath()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Principal_Metadata: + if principal.GetMetadata().GetInvert() { + matchers = append(matchers, &alwaysMatcher{}) + } else { + matchers = append(matchers, &neverMatcher{}) + } + case *v3rbacpb.Principal_NotId: + mList, err := matchersFromPrincipals([]*v3rbacpb.Principal{{Identifier: principal.GetNotId().Identifier}}) + if err != nil { + return nil, err + } + if len(mList) != 1 { + return nil, fmt.Errorf("NotId must contain exactly one identifier") + } + matchers = append(matchers, ¬Matcher{matcherToNot: mList[0]}) + case *v3rbacpb.Principal_SourceIp: + // The source ip principal identifier is deprecated, but gRPC RBAC + // treats it as equivalent to direct_remote_ip as per A41. + m, err := newRemoteIPMatcher(principal.GetSourceIp()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Principal_RemoteIp: + // RBAC in gRPC treats direct_remote_ip and remote_ip as logically + // equivalent, as per A41. + m, err := newRemoteIPMatcher(principal.GetRemoteIp()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + default: + return nil, fmt.Errorf("unsupported principal identifier type: %T", p) + } + } + return matchers, nil +} + +// orMatcher is a matcher where it successfully matches if one of it's +// children successfully match. It also logically represents a principal or +// permission, but can also be it's own entity further down the tree of +// matchers. orMatcher implements the matcher interface. +type orMatcher struct { + matchers []matcher +} + +func (om *orMatcher) match(data *rpcData) bool { + // Range through child matchers and pass in data about incoming RPC, and + // only one child matcher has to match to be logically successful. + for _, m := range om.matchers { + if m.match(data) { + return true + } + } + return false +} + +// andMatcher is a matcher that is successful if every child matcher +// matches. andMatcher implements the matcher interface. +type andMatcher struct { + matchers []matcher +} + +func (am *andMatcher) match(data *rpcData) bool { + for _, m := range am.matchers { + if !m.match(data) { + return false + } + } + return true +} + +// alwaysMatcher is a matcher that will always match. This logically +// represents an any rule for a permission or a principal. alwaysMatcher +// implements the matcher interface. +type alwaysMatcher struct { +} + +func (am *alwaysMatcher) match(*rpcData) bool { + return true +} + +// neverMatcher is a matcher that will never match. This logically represents a +// permission or principal that is unsupported in gRPC. neverMatcher implements +// the matcher interface. +type neverMatcher struct { +} + +func (nm *neverMatcher) match(*rpcData) bool { + return false +} + +// notMatcher is a matcher that nots an underlying matcher. notMatcher +// implements the matcher interface. +type notMatcher struct { + matcherToNot matcher +} + +func (nm *notMatcher) match(data *rpcData) bool { + return !nm.matcherToNot.match(data) +} + +// headerMatcher is a matcher that matches on incoming HTTP Headers present +// in the incoming RPC. headerMatcher implements the matcher interface. +type headerMatcher struct { + matcher internalmatcher.HeaderMatcher +} + +func newHeaderMatcher(headerMatcherConfig *v3route_componentspb.HeaderMatcher) (*headerMatcher, error) { + var m internalmatcher.HeaderMatcher + switch headerMatcherConfig.HeaderMatchSpecifier.(type) { + case *v3route_componentspb.HeaderMatcher_ExactMatch: + m = internalmatcher.NewHeaderExactMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetExactMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_SafeRegexMatch: + regex, err := internalmatcher.CompileSafeRegex(headerMatcherConfig.GetSafeRegexMatch().GetRegex()) + if err != nil { + return nil, err + } + m = internalmatcher.NewHeaderRegexMatcher(headerMatcherConfig.Name, regex, headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_RangeMatch: + m = internalmatcher.NewHeaderRangeMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetRangeMatch().Start, headerMatcherConfig.GetRangeMatch().End, headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_PresentMatch: + m = internalmatcher.NewHeaderPresentMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetPresentMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_PrefixMatch: + m = internalmatcher.NewHeaderPrefixMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetPrefixMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_SuffixMatch: + m = internalmatcher.NewHeaderSuffixMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetSuffixMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_ContainsMatch: + m = internalmatcher.NewHeaderContainsMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetContainsMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_StringMatch: + sm, err := internalmatcher.StringMatcherFromProto(headerMatcherConfig.GetStringMatch()) + if err != nil { + return nil, fmt.Errorf("invalid string matcher %+v: %v", headerMatcherConfig.GetStringMatch(), err) + } + m = internalmatcher.NewHeaderStringMatcher(headerMatcherConfig.Name, sm, headerMatcherConfig.InvertMatch) + default: + return nil, errors.New("unknown header matcher type") + } + return &headerMatcher{matcher: m}, nil +} + +func (hm *headerMatcher) match(data *rpcData) bool { + return hm.matcher.Match(data.md) +} + +// urlPathMatcher matches on the URL Path of the incoming RPC. In gRPC, this +// logically maps to the full method name the RPC is calling on the server side. +// urlPathMatcher implements the matcher interface. +type urlPathMatcher struct { + stringMatcher internalmatcher.StringMatcher +} + +func newURLPathMatcher(pathMatcher *v3matcherpb.PathMatcher) (*urlPathMatcher, error) { + stringMatcher, err := internalmatcher.StringMatcherFromProto(pathMatcher.GetPath()) + if err != nil { + return nil, err + } + return &urlPathMatcher{stringMatcher: stringMatcher}, nil +} + +func (upm *urlPathMatcher) match(data *rpcData) bool { + return upm.stringMatcher.Match(data.fullMethod) +} + +// remoteIPMatcher and localIPMatcher both are matchers that match against +// a CIDR Range. Two different matchers are needed as the remote and destination +// ip addresses come from different parts of the data about incoming RPC's +// passed in. Matching a CIDR Range means to determine whether the IP Address +// falls within the CIDR Range or not. They both implement the matcher +// interface. +type remoteIPMatcher struct { + // ipNet represents the CidrRange that this matcher was configured with. + // This is what will remote and destination IP's will be matched against. + ipNet netip.Prefix +} + +func newRemoteIPMatcher(cidrRange *v3corepb.CidrRange) (*remoteIPMatcher, error) { + // Convert configuration to a cidrRangeString, as Go standard library has + // methods that parse cidr string. + cidrRangeString := fmt.Sprintf("%s/%d", cidrRange.GetAddressPrefix(), cidrRange.GetPrefixLen().GetValue()) + ipNet, err := netip.ParsePrefix(cidrRangeString) + if err != nil { + return nil, err + } + return &remoteIPMatcher{ipNet: ipNet.Masked()}, nil +} + +func (sim *remoteIPMatcher) match(data *rpcData) bool { + host, _, err := net.SplitHostPort(data.peerInfo.Addr.String()) + if err != nil { + // Fallback for addresses without a port. + host = data.peerInfo.Addr.String() + } + ip, err := netip.ParseAddr(host) + if err != nil { + return false + } + return sim.ipNet.Contains(ip) +} + +type localIPMatcher struct { + ipNet netip.Prefix +} + +func newLocalIPMatcher(cidrRange *v3corepb.CidrRange) (*localIPMatcher, error) { + cidrRangeString := fmt.Sprintf("%s/%d", cidrRange.GetAddressPrefix(), cidrRange.GetPrefixLen().GetValue()) + ipNet, err := netip.ParsePrefix(cidrRangeString) + if err != nil { + return nil, err + } + return &localIPMatcher{ipNet: ipNet.Masked()}, nil +} + +func (dim *localIPMatcher) match(data *rpcData) bool { + host, _, err := net.SplitHostPort(data.localAddr.String()) + if err != nil { + // Fallback for addresses without a port. + host = data.localAddr.String() + } + ip, err := netip.ParseAddr(host) + if err != nil { + return false + } + return dim.ipNet.Contains(ip) +} + +// portMatcher matches on whether the destination port of the RPC matches the +// destination port this matcher was instantiated with. portMatcher +// implements the matcher interface. +type portMatcher struct { + destinationPort uint32 +} + +func newPortMatcher(destinationPort uint32) *portMatcher { + return &portMatcher{destinationPort: destinationPort} +} + +func (pm *portMatcher) match(data *rpcData) bool { + return data.destinationPort == pm.destinationPort +} + +// requestedServerNameMatcher matches on if the given string matcher matches +// on "", as per A41-xds-rbac.md. requestedServerNameMatcher implements +// the matcher interface. +type requestedServerNameMatcher struct { + stringMatcher internalmatcher.StringMatcher +} + +func newRequestedServerNameMatcher(stringMatcherProto *v3matcherpb.StringMatcher) (*requestedServerNameMatcher, error) { + stringMatcher, err := internalmatcher.StringMatcherFromProto(stringMatcherProto) + if err != nil { + return nil, err + } + return &requestedServerNameMatcher{stringMatcher: stringMatcher}, nil +} + +func (r *requestedServerNameMatcher) match(*rpcData) bool { + return r.stringMatcher.Match("") +} + +// authenticatedMatcher matches on the name of the Principal. If set, the URI +// SAN or DNS SAN in that order is used from the certificate, otherwise the +// subject field is used. If unset, it applies to any user that is +// authenticated. authenticatedMatcher implements the matcher interface. +type authenticatedMatcher struct { + stringMatcher *internalmatcher.StringMatcher +} + +func newAuthenticatedMatcher(authenticatedMatcherConfig *v3rbacpb.Principal_Authenticated) (*authenticatedMatcher, error) { + // Represents this line in the RBAC documentation = "If unset, it applies to + // any user that is authenticated" (see package-level comments). + if authenticatedMatcherConfig.PrincipalName == nil { + return &authenticatedMatcher{}, nil + } + stringMatcher, err := internalmatcher.StringMatcherFromProto(authenticatedMatcherConfig.PrincipalName) + if err != nil { + return nil, err + } + return &authenticatedMatcher{stringMatcher: &stringMatcher}, nil +} + +func (am *authenticatedMatcher) match(data *rpcData) bool { + if data.authType != "tls" { + // Connection is not authenticated. + return false + } + if am.stringMatcher == nil { + // Allows any authenticated user. + return true + } + // "If there is no client certificate (thus no SAN nor Subject), check if "" + // (empty string) matches. If it matches, the principal_name is said to + // match" - A41 + if len(data.certs) == 0 { + return am.stringMatcher.Match("") + } + cert := data.certs[0] + // Use the first non-empty identity source in priority order: + // URI SANs, then DNS SANs, then Subject. + if len(cert.URIs) > 0 { + for _, uriSAN := range cert.URIs { + if am.stringMatcher.Match(uriSAN.String()) { + return true + } + } + return false + } + if len(cert.DNSNames) > 0 { + for _, dnsSAN := range cert.DNSNames { + if am.stringMatcher.Match(dnsSAN) { + return true + } + } + return false + } + return am.stringMatcher.Match(cert.Subject.String()) +} diff --git a/internal/xds/rbac/rbac_engine.go b/internal/xds/rbac/rbac_engine.go new file mode 100644 index 0000000..d0c7373 --- /dev/null +++ b/internal/xds/rbac/rbac_engine.go @@ -0,0 +1,319 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package rbac provides service-level and method-level access control for a +// service. See +// https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/rbac/v3/rbac.proto#role-based-access-control-rbac +// for documentation. +package rbac + +import ( + "context" + "crypto/x509" + "errors" + "fmt" + "net" + "strconv" + + "google.golang.org/grpc" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" +) + +var logger = grpclog.Component("rbac") + +var getConnection = transport.GetConnection + +// ChainEngine represents a chain of RBAC Engines, used to make authorization +// decisions on incoming RPCs. +type ChainEngine struct { + chainedEngines []*engine +} + +// NewChainEngine returns a chain of RBAC engines, used to make authorization +// decisions on incoming RPCs. Returns a non-nil error for invalid policies. +func NewChainEngine(policies []*v3rbacpb.RBAC, policyName string) (*ChainEngine, error) { + engines := make([]*engine, 0, len(policies)) + for _, policy := range policies { + engine, err := newEngine(policy, policyName) + if err != nil { + return nil, err + } + engines = append(engines, engine) + } + return &ChainEngine{chainedEngines: engines}, nil +} + +func (cre *ChainEngine) logRequestDetails(rpcData *rpcData) { + if logger.V(2) { + logger.Infof("checking request: url path=%s", rpcData.fullMethod) + if len(rpcData.certs) > 0 { + cert := rpcData.certs[0] + logger.Infof("uri sans=%q, dns sans=%q, subject=%v", cert.URIs, cert.DNSNames, cert.Subject) + } + } +} + +// IsAuthorized determines if an incoming RPC is authorized based on the chain of RBAC +// engines and their associated actions. +// +// Errors returned by this function are compatible with the status package. +func (cre *ChainEngine) IsAuthorized(ctx context.Context) error { + // This conversion step (i.e. pulling things out of ctx) can be done once, + // and then be used for the whole chain of RBAC Engines. + rpcData, err := newRPCData(ctx) + if err != nil { + logger.Errorf("newRPCData: %v", err) + return status.Errorf(codes.Internal, "gRPC RBAC: %v", err) + } + for _, engine := range cre.chainedEngines { + matchingPolicyName, ok := engine.findMatchingPolicy(rpcData) + if logger.V(2) && ok { + logger.Infof("incoming RPC matched to policy %v in engine with action %v", matchingPolicyName, engine.action) + } + + switch { + case engine.action == v3rbacpb.RBAC_ALLOW && !ok: + cre.logRequestDetails(rpcData) + engine.doAuditLogging(rpcData, matchingPolicyName, false) + return status.Errorf(codes.PermissionDenied, "incoming RPC did not match an allow policy") + case engine.action == v3rbacpb.RBAC_DENY && ok: + cre.logRequestDetails(rpcData) + engine.doAuditLogging(rpcData, matchingPolicyName, false) + return status.Errorf(codes.PermissionDenied, "incoming RPC matched a deny policy %q", matchingPolicyName) + } + // Every policy in the engine list must be queried. Thus, iterate to the + // next policy. + engine.doAuditLogging(rpcData, matchingPolicyName, true) + } + // If the incoming RPC gets through all of the engines successfully (i.e. + // doesn't not match an allow or match a deny engine), the RPC is authorized + // to proceed. + return nil +} + +// engine is used for matching incoming RPCs to policies. +type engine struct { + // TODO(gtcooke94) - differentiate between `policyName`, `policies`, and `rules` + policyName string + policies map[string]*policyMatcher + // action must be ALLOW or DENY. + action v3rbacpb.RBAC_Action + auditLoggers []audit.Logger + auditCondition v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition +} + +// newEngine creates an RBAC Engine based on the contents of a policy. Returns a +// non-nil error if the policy is invalid. +func newEngine(config *v3rbacpb.RBAC, policyName string) (*engine, error) { + a := config.GetAction() + if a != v3rbacpb.RBAC_ALLOW && a != v3rbacpb.RBAC_DENY { + return nil, fmt.Errorf("unsupported action %s", config.Action) + } + + policies := make(map[string]*policyMatcher, len(config.GetPolicies())) + for name, policy := range config.GetPolicies() { + matcher, err := newPolicyMatcher(policy) + if err != nil { + return nil, err + } + policies[name] = matcher + } + + auditLoggers, auditCondition, err := parseAuditOptions(config.GetAuditLoggingOptions()) + if err != nil { + return nil, err + } + return &engine{ + policyName: policyName, + policies: policies, + action: a, + auditLoggers: auditLoggers, + auditCondition: auditCondition, + }, nil +} + +func parseAuditOptions(opts *v3rbacpb.RBAC_AuditLoggingOptions) ([]audit.Logger, v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition, error) { + if opts == nil { + return nil, v3rbacpb.RBAC_AuditLoggingOptions_NONE, nil + } + var auditLoggers []audit.Logger + for _, logger := range opts.LoggerConfigs { + auditLogger, err := buildLogger(logger) + if err != nil { + return nil, v3rbacpb.RBAC_AuditLoggingOptions_NONE, err + } + if auditLogger == nil { + // This occurs when the audit logger is not registered but also + // marked optional. + continue + } + auditLoggers = append(auditLoggers, auditLogger) + } + return auditLoggers, opts.GetAuditCondition(), nil + +} + +// findMatchingPolicy determines if an incoming RPC matches a policy. On a +// successful match, it returns the name of the matching policy and a true bool +// to specify that there was a matching policy found. It returns false in +// the case of not finding a matching policy. +func (e *engine) findMatchingPolicy(rpcData *rpcData) (string, bool) { + for policy, matcher := range e.policies { + if matcher.match(rpcData) { + return policy, true + } + } + return "", false +} + +// newRPCData takes an incoming context (should be a context representing state +// needed for server RPC Call with metadata, peer info (used for source ip/port +// and TLS information) and connection (used for destination ip/port) piped into +// it) and the method name of the Service being called server side and populates +// an rpcData struct ready to be passed to the RBAC Engine to find a matching +// policy. +func newRPCData(ctx context.Context) (*rpcData, error) { + // The caller should populate all of these fields (i.e. for empty headers, + // pipe an empty md into context). + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, errors.New("missing metadata in incoming context") + } + // ":method can be hard-coded to POST if unavailable" - A41 + md[":method"] = []string{"POST"} + // "If the transport exposes TE in Metadata, then RBAC must special-case the + // header to treat it as not present." - A41 + delete(md, "TE") + + pi, ok := peer.FromContext(ctx) + if !ok { + return nil, errors.New("missing peer info in incoming context") + } + + // The methodName will be available in the passed in ctx from a unary or streaming + // interceptor, as grpc.Server pipes in a transport stream which contains the methodName + // into contexts available in both unary or streaming interceptors. + mn, ok := grpc.Method(ctx) + if !ok { + return nil, errors.New("missing method in incoming context") + } + // gRPC-Go strips :path from the headers given to the application, but RBAC should be + // able to match against it. + md[":path"] = []string{mn} + + // The connection is needed in order to find the destination address and + // port of the incoming RPC Call. + conn := getConnection(ctx) + if conn == nil { + return nil, errors.New("missing connection in incoming context") + } + _, dPort, err := net.SplitHostPort(conn.LocalAddr().String()) + if err != nil { + return nil, fmt.Errorf("error parsing local address: %v", err) + } + dp, err := strconv.ParseUint(dPort, 10, 32) + if err != nil { + return nil, fmt.Errorf("error parsing local address: %v", err) + } + + var authType string + var peerCertificates []*x509.Certificate + if tlsInfo, ok := pi.AuthInfo.(credentials.TLSInfo); ok { + authType = pi.AuthInfo.AuthType() + peerCertificates = tlsInfo.State.PeerCertificates + } + + return &rpcData{ + md: md, + peerInfo: pi, + fullMethod: mn, + destinationPort: uint32(dp), + localAddr: conn.LocalAddr(), + authType: authType, + certs: peerCertificates, + }, nil +} + +// rpcData wraps data pulled from an incoming RPC that the RBAC engine needs to +// find a matching policy. +type rpcData struct { + // md is the HTTP Headers that are present in the incoming RPC. + md metadata.MD + // peerInfo is information about the downstream peer. + peerInfo *peer.Peer + // fullMethod is the method name being called on the upstream service. + fullMethod string + // destinationPort is the port that the RPC is being sent to on the + // server. + destinationPort uint32 + // localAddr is the address that the RPC is being sent to. + localAddr net.Addr + // authType is the type of authentication e.g. "tls". + authType string + // certs are the certificates presented by the peer during a TLS + // handshake. + certs []*x509.Certificate +} + +func (e *engine) doAuditLogging(rpcData *rpcData, rule string, authorized bool) { + // In the RBAC world, we need to have a SPIFFE ID as the principal for this + // to be meaningful + principal := "" + if rpcData.peerInfo != nil { + // If AuthType = tls, then we can cast AuthInfo to TLSInfo. + if tlsInfo, ok := rpcData.peerInfo.AuthInfo.(credentials.TLSInfo); ok { + if tlsInfo.SPIFFEID != nil { + principal = tlsInfo.SPIFFEID.String() + } + } + } + + // TODO(gtcooke94) check if we need to log before creating the event + event := &audit.Event{ + FullMethodName: rpcData.fullMethod, + Principal: principal, + PolicyName: e.policyName, + MatchedRule: rule, + Authorized: authorized, + } + for _, logger := range e.auditLoggers { + switch e.auditCondition { + case v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY: + if !authorized { + logger.Log(event) + } + case v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW: + if authorized { + logger.Log(event) + } + case v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY_AND_ALLOW: + logger.Log(event) + } + } +} + +// This is used when converting a custom config from raw JSON to a TypedStruct. +// The TypeURL of the TypeStruct will be "grpc.authz.audit_logging/". +const typeURLPrefix = "grpc.authz.audit_logging/" diff --git a/internal/xds/rbac/rbac_engine_test.go b/internal/xds/rbac/rbac_engine_test.go new file mode 100644 index 0000000..f5a3b26 --- /dev/null +++ b/internal/xds/rbac/rbac_engine_test.go @@ -0,0 +1,2521 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rbac + +import ( + "context" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "fmt" + "net" + "net/url" + "reflect" + "testing" + "time" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type addr struct { + ipAddress string +} + +func (addr) Network() string { return "" } +func (a *addr) String() string { return a.ipAddress } + +// TestNewChainEngine tests the construction of the ChainEngine. Due to some +// types of RBAC configuration being logically wrong and returning an error +// rather than successfully constructing the RBAC Engine, this test tests both +// RBAC Configurations deemed successful and also RBAC Configurations that will +// raise errors. +func (s) TestNewChainEngine(t *testing.T) { + tests := []struct { + name string + policies []*v3rbacpb.RBAC + wantErr bool + policyName string + }{ + { + name: "SuccessCaseAnyMatchSingular", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "SuccessCaseAnyMatchMultiple", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "SuccessCaseSimplePolicySingular", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationPort{DestinationPort: 8080}}, + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + // SuccessCaseSimplePolicyTwoPolicies tests the construction of the + // chained engines in the case where there are two policies in a list, + // one with an allow policy and one with a deny policy. A situation + // where two policies (allow and deny) is a very common use case for + // this API, and should successfully build. + { + name: "SuccessCaseSimplePolicyTwoPolicies", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationPort{DestinationPort: 8080}}, + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationPort{DestinationPort: 8080}}, + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "SuccessCaseEnvoyExampleSingular", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "service-admin": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "cluster.local/ns/default/sa/admin"}}}}}, + {Identifier: &v3rbacpb.Principal_Authenticated_{Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "cluster.local/ns/default/sa/superuser"}}}}}, + }, + }, + "product-viewer": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_AndRules{AndRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "GET"}}}}, + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "/products"}}}}}}, + {Rule: &v3rbacpb.Permission_OrRules{OrRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationPort{DestinationPort: 80}}, + {Rule: &v3rbacpb.Permission_DestinationPort{DestinationPort: 443}}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "SourceIpMatcherSuccessSingular", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + }, + }, + }, + { + name: "SourceIpMatcherFailureSingular", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "not a correct address", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "DestinationIpMatcherSuccess", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "certain-destination-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationIp{DestinationIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "DestinationIpMatcherFailure", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "certain-destination-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationIp{DestinationIp: &v3corepb.CidrRange{AddressPrefix: "not a correct address", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "RemoteIpMatcherUnsetPrefixLen", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0"}}}, + }, + }, + }, + }, + }, + }, + { + name: "DestinationIpMatcherUnsetPrefixLen", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "certain-destination-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationIp{DestinationIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0"}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "MatcherToNotPolicy", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "not-secret-content": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_NotRule{NotRule: &v3rbacpb.Permission{Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "/secret-content"}}}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "MatcherToNotPrincipal", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "not-from-certain-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_NotId{NotId: &v3rbacpb.Principal{Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}}}, + }, + }, + }, + }, + }, + }, + // PrincipalProductViewer tests the construction of a chained engine + // with a policy that allows any downstream to send a GET request on a + // certain path. + { + name: "PrincipalProductViewer", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "product-viewer": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + { + Identifier: &v3rbacpb.Principal_AndIds{AndIds: &v3rbacpb.Principal_Set{Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "GET"}}}}, + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "/books"}}}}}}, + {Identifier: &v3rbacpb.Principal_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "/cars"}}}}}}, + }, + }}}, + }}}, + }, + }, + }, + }, + }, + }, + }, + // Certain Headers tests the construction of a chained engine with a + // policy that allows any downstream to send an HTTP request with + // certain headers. + { + name: "CertainHeaders", + policies: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-headers": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + { + Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "GET"}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{SafeRegexMatch: &v3matcherpb.RegexMatcher{Regex: "GET"}}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_RangeMatch{RangeMatch: &v3typepb.Int64Range{ + Start: 0, + End: 64, + }}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PresentMatch{PresentMatch: true}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: "GET"}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SuffixMatch{SuffixMatch: "GET"}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ContainsMatch{ContainsMatch: "GET"}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ContainsMatch{ContainsMatch: "GET"}}}}, + }}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "LogAction", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_LOG, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "ActionNotSpecified", + policies: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "SimpleAuditLogger", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "SimpleAuditLogger_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + { + name: "AuditLoggerCustomConfig", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerCustomConfig", + TypedConfig: createUDPATypedStruct(t, map[string]any{"abc": 123, "xyz": "123"}, "AuditLoggerCustomConfig_TestAuditLoggerCustomConfig")}, + IsOptional: false, + }, + }, + }, + }, + }, + policyName: "test_policy", + }, + { + name: "AuditLoggerCustomConfigXdsTypedStruct", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerCustomConfig", + TypedConfig: createXDSTypedStruct(t, map[string]any{"abc": 123, "xyz": "123"}, "AuditLoggerCustomConfigXdsTypedStruct_TestAuditLoggerCustomConfig")}, + IsOptional: false, + }, + }, + }, + }, + }, + policyName: "test_policy", + }, + { + name: "Missing Optional AuditLogger doesn't fail", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "UnsupportedLogger", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "Missing Optional AuditLogger doesn't fail_UnsupportedLogger")}, + IsOptional: true, + }, + }, + }, + }, + }, + }, + { + name: "Missing Non-Optional AuditLogger fails", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "UnsupportedLogger", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "Missing Non-Optional AuditLogger fails_UnsupportedLogger")}, + IsOptional: false, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "Cannot_parse_missing_CustomConfig", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerCustomConfig", + }, + IsOptional: false, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "Cannot_parse_bad_CustomConfig", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerCustomConfig", + TypedConfig: createUDPATypedStruct(t, map[string]any{"abc": "BADVALUE", "xyz": "123"}, "Cannot_parse_bad_CustomConfig_TestAuditLoggerCustomConfig")}, + IsOptional: false, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "Cannot_parse_missing_typedConfig_name", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerCustomConfig", + TypedConfig: createUDPATypedStruct(t, map[string]any{"abc": 123, "xyz": "123"}, "")}, + IsOptional: false, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "Unsupported_field_in_NotRule", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + { + Rule: &v3rbacpb.Permission_NotRule{ + NotRule: &v3rbacpb.Permission{ + Rule: &v3rbacpb.Permission_SourcedMetadata{ + SourcedMetadata: &v3rbacpb.SourcedMetadata{}, + }, + }, + }, + }, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "supported_and_unsupported_fields_in_NotRule", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + { + Rule: &v3rbacpb.Permission_NotRule{ + NotRule: &v3rbacpb.Permission{ + Rule: &v3rbacpb.Permission_AndRules{ + AndRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + { + Rule: &v3rbacpb.Permission_UrlPath{ + UrlPath: &v3matcherpb.PathMatcher{ + Rule: &v3matcherpb.PathMatcher_Path{ + Path: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "allowed"}, + }, + }, + }, + }, + }, + { + Rule: &v3rbacpb.Permission_SourcedMetadata{ + SourcedMetadata: &v3rbacpb.SourcedMetadata{}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "Unsupported_field_in_NotId", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + { + Identifier: &v3rbacpb.Principal_NotId{ + NotId: &v3rbacpb.Principal{ + Identifier: &v3rbacpb.Principal_SourcedMetadata{ + SourcedMetadata: &v3rbacpb.SourcedMetadata{}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "supported_and_unsupported_fields_in_NotId", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + { + Identifier: &v3rbacpb.Principal_NotId{ + NotId: &v3rbacpb.Principal{ + Identifier: &v3rbacpb.Principal_AndIds{ + AndIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + { + Identifier: &v3rbacpb.Principal_DirectRemoteIp{ + DirectRemoteIp: &v3corepb.CidrRange{ + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}, + }, + }, + }, + { + Identifier: &v3rbacpb.Principal_SourcedMetadata{ + SourcedMetadata: &v3rbacpb.SourcedMetadata{}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "PermissionMetadata", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "metadata-permission": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Metadata{Metadata: &v3matcherpb.MetadataMatcher{}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "PrincipalMetadata", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "metadata-principal": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Metadata{Metadata: &v3matcherpb.MetadataMatcher{}}}, + }, + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + b := TestAuditLoggerBufferBuilder{testName: test.name} + audit.RegisterLoggerBuilder(&b) + b2 := TestAuditLoggerCustomConfigBuilder{testName: test.name} + audit.RegisterLoggerBuilder(&b2) + if _, err := NewChainEngine(test.policies, test.policyName); (err != nil) != test.wantErr { + t.Fatalf("NewChainEngine(%+v) returned err: %v, wantErr: %v", test.policies, err, test.wantErr) + } + }) + } +} + +type rbacQuery struct { + rpcData *rpcData + wantStatusCode codes.Code + wantAuditEvents []*audit.Event +} + +// TestChainEngine tests the chain of RBAC Engines by configuring the chain of +// engines in a certain way in different scenarios. After configuring the chain +// of engines in a certain way, this test pings the chain of engines with +// different types of data representing incoming RPC's (piped into a context), +// and verifies that it works as expected. +func (s) TestChainEngine(t *testing.T) { + defer func(gc func(ctx context.Context) net.Conn) { + getConnection = gc + }(getConnection) + tests := []struct { + name string + rbacConfigs []*v3rbacpb.RBAC + rbacQueries []rbacQuery + policyName string + }{ + { + name: "PermissionMetadata", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "metadata-permission": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Metadata{Metadata: &v3matcherpb.MetadataMatcher{Invert: false}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + { + name: "PermissionMetadataInvert", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "metadata-permission-invert": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Metadata{Metadata: &v3matcherpb.MetadataMatcher{Invert: true}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + }, + }, + { + name: "PrincipalMetadata", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "metadata-principal": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Metadata{Metadata: &v3matcherpb.MetadataMatcher{Invert: false}}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + { + name: "PrincipalMetadataInvert", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "metadata-principal-invert": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Metadata{Metadata: &v3matcherpb.MetadataMatcher{Invert: true}}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + }, + }, + { + name: "RequestedServerNameNotMatching", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "requested-server-name": { + Permissions: []*v3rbacpb.Permission{ + { + Rule: &v3rbacpb.Permission_RequestedServerName{ + RequestedServerName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "foo"}, + }, + }, + }, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + { + name: "RequestedServerNameMatching", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "requested-server-name": { + Permissions: []*v3rbacpb.Permission{ + { + Rule: &v3rbacpb.Permission_RequestedServerName{ + RequestedServerName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }, + }, + }, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + }, + }, + // SuccessCaseAnyMatch tests a single RBAC Engine instantiated with + // a config with a policy with any rules for both permissions and + // principals, meaning that any data about incoming RPC's that the RBAC + // Engine is queried with should match that policy. + { + name: "SuccessCaseAnyMatch", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + { + rpcData: &rpcData{ + fullMethod: "some method", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + }, + }, + // SuccessCaseSimplePolicy is a test that tests a single policy + // that only allows an rpc to proceed if the rpc is calling with a certain + // path. + { + name: "SuccessCaseSimplePolicy", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC should match with the local host fan policy. Thus, + // this RPC should be allowed to proceed. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.OK, + }, + + // This RPC shouldn't match with the local host fan policy. Thus, + // this rpc shouldn't be allowed to proceed. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + // SuccessCaseEnvoyExample is a test based on the example provided + // in the EnvoyProxy docs. The RBAC Config contains two policies, + // service admin and product viewer, that provides an example of a real + // RBAC Config that might be configured for a given for a given backend + // service. + { + name: "SuccessCaseEnvoyExample", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "service-admin": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "//cluster.local/ns/default/sa/admin"}}}}}, + {Identifier: &v3rbacpb.Principal_Authenticated_{Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "//cluster.local/ns/default/sa/superuser"}}}}}, + }, + }, + "product-viewer": { + Permissions: []*v3rbacpb.Permission{ + { + Rule: &v3rbacpb.Permission_AndRules{AndRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "GET"}}}}, + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "/products"}}}}}}, + }, + }, + }, + }, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This incoming RPC Call should match with the service admin + // policy. + { + rpcData: &rpcData{ + fullMethod: "some method", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + AuthInfo: credentials.TLSInfo{ + State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{ + { + URIs: []*url.URL{ + { + Host: "cluster.local", + Path: "/ns/default/sa/admin", + }, + }, + }, + }, + }, + }, + }, + }, + wantStatusCode: codes.OK, + }, + // These incoming RPC calls should not match any policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + { + rpcData: &rpcData{ + fullMethod: "get-product-list", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + AuthInfo: credentials.TLSInfo{ + State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{ + { + Subject: pkix.Name{ + CommonName: "localhost", + }, + }, + }, + }, + }, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + { + name: "NotMatcher", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "not-secret-content": { + Permissions: []*v3rbacpb.Permission{ + { + Rule: &v3rbacpb.Permission_NotRule{ + NotRule: &v3rbacpb.Permission{Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "/secret-content"}}}}}}, + }, + }, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This incoming RPC Call should match with the not-secret-content policy. + { + rpcData: &rpcData{ + fullMethod: "/regular-content", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This incoming RPC Call shouldn't match with the not-secret-content-policy. + { + rpcData: &rpcData{ + fullMethod: "/secret-content", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + { + name: "DirectRemoteIpMatcher", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-direct-remote-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This incoming RPC Call should match with the certain-direct-remote-ip policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This incoming RPC Call shouldn't match with the certain-direct-remote-ip policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + // This test tests a RBAC policy configured with a remote-ip policy. + // This should be logically equivalent to configuring a Engine with a + // direct-remote-ip policy, as per A41 - "allow equating RBAC's + // direct_remote_ip and remote_ip." + { + name: "RemoteIpMatcher", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-remote-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_RemoteIp{RemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This incoming RPC Call should match with the certain-remote-ip policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This incoming RPC Call shouldn't match with the certain-remote-ip policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + // This test tests a RBAC policy configured with a source-ip policy. + // This should be logically equivalent to configuring a Engine with a + // direct-remote-ip or remote-ip policy, as per A41 - "allow equating + // RBAC's direct_remote_ip and remote_ip." + { + name: "SourceIpMatcher", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_SourceIp{SourceIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This incoming RPC Call should match with the certain-source-ip policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This incoming RPC Call shouldn't match with the certain-source-ip policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + { + name: "DestinationIpMatcher", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-destination-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationIp{DestinationIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This incoming RPC Call shouldn't match with the + // certain-destination-ip policy, as the test listens on local + // host. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + // AllowAndDenyPolicy tests a policy with an allow (on path) and + // deny (on port) policy chained together. This represents how a user + // configured interceptor would use this, and also is a potential + // configuration for a dynamic xds interceptor. + { + name: "AllowAndDenyPolicy", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + Action: v3rbacpb.RBAC_ALLOW, + }, + { + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + Action: v3rbacpb.RBAC_DENY, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC should match with the allow policy, and shouldn't + // match with the deny and thus should be allowed to proceed. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This RPC should match with both the allow policy and deny policy + // and thus shouldn't be allowed to proceed as matched with deny. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + // This RPC shouldn't match with either policy, and thus + // shouldn't be allowed to proceed as didn't match with allow. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + // This RPC shouldn't match with allow, match with deny, and + // thus shouldn't be allowed to proceed. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + // This test verifies that when there are no SANs or Subject + // distinguished name in incoming RPCs, authenticated matchers + // match against the empty string. + { + name: "default-matching-no-credentials", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "service-admin": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}}}}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This incoming RPC should match the service admin policy. No + // authentication info is provided, so the authenticated matcher + // evaluates the empty string. + { + rpcData: &rpcData{ + fullMethod: "some method", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + AuthInfo: credentials.TLSInfo{ + State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{ + { + URIs: []*url.URL{ + { + Host: "cluster.local", + Path: "/ns/default/sa/admin", + }, + }, + }, + }, + }, + }, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + { + rpcData: &rpcData{ + fullMethod: "some method", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + AuthInfo: credentials.TLSInfo{ + State: tls.ConnectionState{}, + }, + }, + }, + wantStatusCode: codes.OK, + }, + }, + }, + // This test verifies that SafeRegex header matching is implicitly + // anchored at both ends (full string matching). + { + name: "SafeRegexHeaderMatcherImplicitAnchoring", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "safe-regex-policy": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + { + Identifier: &v3rbacpb.Principal_Header{ + Header: &v3routepb.HeaderMatcher{ + Name: "foo", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{ + SafeRegexMatch: &v3matcherpb.RegexMatcher{ + Regex: "abc", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC has header "foo: abc", which matches "abc" exactly, so it should be allowed (OK). + { + rpcData: &rpcData{ + md: metadata.MD{ + "foo": []string{"abc"}, + }, + fullMethod: "some method", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This RPC has header "foo: 123abc456", which contains "abc" but is not an exact match. + // Since safe regex is implicitly anchored, it should not match and return PermissionDenied. + { + rpcData: &rpcData{ + md: metadata.MD{ + "foo": []string{"123abc456"}, + }, + fullMethod: "some method", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + // This test verifies that SafeRegex header matching with wildcard matching + // (e.g. .*abc.*) matches substrings as expected. + { + name: "SafeRegexHeaderMatcherWildcard", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "safe-regex-wildcard-policy": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + { + Identifier: &v3rbacpb.Principal_Header{ + Header: &v3routepb.HeaderMatcher{ + Name: "foo", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{ + SafeRegexMatch: &v3matcherpb.RegexMatcher{ + Regex: ".*abc.*", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC has header "foo: 123abc456", which matches the wildcard safe regex, so it should be allowed (OK). + { + rpcData: &rpcData{ + md: metadata.MD{ + "foo": []string{"123abc456"}, + }, + fullMethod: "some method", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This RPC has header "foo: xyz", which does not contain "abc", so it should return PermissionDenied. + { + rpcData: &rpcData{ + md: metadata.MD{ + "foo": []string{"xyz"}, + }, + fullMethod: "some method", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + // AllowAndDenyPolicy tests a policy with an allow (on path) and + // deny (on port) policy chained together. This represents how a user + // configured interceptor would use this, and also is a potential + // configuration for a dynamic xds interceptor. Further, it tests that + // the audit logger works properly in each scenario. + { + name: "AuditLoggingAllowAndDenyPolicy_ON_ALLOW", + policyName: "test_policy", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + Action: v3rbacpb.RBAC_DENY, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_ON_ALLOW_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + Action: v3rbacpb.RBAC_ALLOW, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_ON_ALLOW_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC should match with the allow policy, and shouldn't + // match with the deny and thus should be allowed to proceed. + { + rpcData: &rpcData{ + fullMethod: "", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + AuthInfo: credentials.TLSInfo{ + State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{ + { + URIs: []*url.URL{ + { + Scheme: "spiffe", + Host: "cluster.local", + Path: "/ns/default/sa/admin", + }, + }, + }, + }, + }, + SPIFFEID: &url.URL{ + Scheme: "spiffe", + Host: "cluster.local", + Path: "/ns/default/sa/admin", + }, + }, + }, + }, + wantStatusCode: codes.OK, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "", + Principal: "spiffe://cluster.local/ns/default/sa/admin", + PolicyName: "test_policy", + MatchedRule: "certain-source-ip", + Authorized: true, + }, + }, + }, + // This RPC should match with both the allow policy and deny policy + // and thus shouldn't be allowed to proceed as matched with deny. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + // This RPC shouldn't match with either policy, and thus + // shouldn't be allowed to proceed as didn't match with allow. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + // This RPC shouldn't match with allow, match with deny, and + // thus shouldn't be allowed to proceed. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + { + name: "AuditLoggingAllowAndDenyPolicy_ON_DENY", + policyName: "test_policy", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + Action: v3rbacpb.RBAC_DENY, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_ON_DENY_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + Action: v3rbacpb.RBAC_ALLOW, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_ON_DENY_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC should match with the allow policy, and shouldn't + // match with the deny and thus should be allowed to proceed. + // Audit logging matches with nothing. + { + rpcData: &rpcData{ + fullMethod: "", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This RPC should match with both the allow policy and deny policy + // and thus shouldn't be allowed to proceed as matched with deny. + // Audit logging matches with deny and short circuits. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + AuthInfo: credentials.TLSInfo{ + State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{ + { + URIs: []*url.URL{ + { + Host: "cluster.local", + Path: "/ns/default/sa/admin", + }, + }, + }, + }, + }, + }, + }, + }, + wantStatusCode: codes.PermissionDenied, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "localhost-fan-page", + PolicyName: "test_policy", + MatchedRule: "localhost-fan", + Authorized: false, + }, + }, + }, + // This RPC shouldn't match with either policy, and thus + // shouldn't be allowed to proceed as didn't match with allow. + // Audit logging matches with the allow policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "", + PolicyName: "test_policy", + MatchedRule: "", + Authorized: false, + }, + }, + }, + // This RPC shouldn't match with allow, match with deny, and + // thus shouldn't be allowed to proceed. + // Audit logging will have the deny logged. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "localhost-fan-page", + PolicyName: "test_policy", + MatchedRule: "localhost-fan", + Authorized: false, + }, + }, + }, + }, + }, + { + name: "AuditLoggingAllowAndDenyPolicy_NONE", + policyName: "test_policy", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + Action: v3rbacpb.RBAC_DENY, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_NONE_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + Action: v3rbacpb.RBAC_ALLOW, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_NONE_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC should match with the allow policy, and shouldn't + // match with the deny and thus should be allowed to proceed. + // Audit logging is NONE. + { + rpcData: &rpcData{ + fullMethod: "", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This RPC should match with both the allow policy and deny policy + // and thus shouldn't be allowed to proceed as matched with deny. + // Audit logging is NONE. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + // This RPC shouldn't match with either policy, and thus + // shouldn't be allowed to proceed as didn't match with allow. + // Audit logging is NONE. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + // This RPC shouldn't match with allow, match with deny, and + // thus shouldn't be allowed to proceed. + // Audit logging is NONE. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + { + name: "AuditLoggingAllowAndDenyPolicy_ON_DENY_AND_ALLOW", + policyName: "test_policy", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + Action: v3rbacpb.RBAC_DENY, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_ON_DENY_AND_ALLOW_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + Action: v3rbacpb.RBAC_ALLOW, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY_AND_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_ON_DENY_AND_ALLOW_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC should match with the allow policy, and shouldn't + // match with the deny and thus should be allowed to proceed. + // Audit logging matches with nothing. + { + rpcData: &rpcData{ + fullMethod: "", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "", + PolicyName: "test_policy", + MatchedRule: "certain-source-ip", + Authorized: true, + }, + }, + }, + // This RPC should match with both the allow policy and deny policy + // and thus shouldn't be allowed to proceed as matched with deny. + // Audit logging matches with deny and short circuits. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "localhost-fan-page", + PolicyName: "test_policy", + MatchedRule: "localhost-fan", + Authorized: false, + }, + }, + }, + // This RPC shouldn't match with either policy, and thus + // shouldn't be allowed to proceed as didn't match with allow. + // Audit logging matches with the allow policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0:8080"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "", + PolicyName: "test_policy", + MatchedRule: "", + Authorized: false, + }, + }, + }, + // This RPC shouldn't match with allow, match with deny, and + // thus shouldn't be allowed to proceed. + // Audit logging will have the deny logged. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "localhost-fan-page", + PolicyName: "test_policy", + MatchedRule: "localhost-fan", + Authorized: false, + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + b := TestAuditLoggerBufferBuilder{testName: test.name} + audit.RegisterLoggerBuilder(&b) + b2 := TestAuditLoggerCustomConfigBuilder{testName: test.name} + audit.RegisterLoggerBuilder(&b2) + + // Instantiate the chainedRBACEngine with different configurations that are + // interesting to test and to query. + cre, err := NewChainEngine(test.rbacConfigs, test.policyName) + if err != nil { + t.Fatalf("Error constructing RBAC Engine: %v", err) + } + // Query the created chain of RBAC Engines with different args to see + // if the chain of RBAC Engines configured as such works as intended. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for _, data := range test.rbacQueries { + func() { + // Construct the context with three data points that have enough + // information to represent incoming RPC's. This will be how a + // user uses this API. A user will have to put MD, PeerInfo, and + // the connection the RPC is sent on in the context. + ctx = metadata.NewIncomingContext(ctx, data.rpcData.md) + // Make a TCP connection with a certain destination port. The + // address/port of this connection will be used to populate the + // destination ip/port in RPCData struct. This represents what + // the user of ChainEngine will have to place into context, + // as this is only way to get destination ip and port. + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error listening: %v", err) + } + defer lis.Close() + connCh := make(chan net.Conn, 1) + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("Error accepting connection: %v", err) + return + } + connCh <- conn + }() + _, err = net.Dial("tcp", lis.Addr().String()) + if err != nil { + t.Fatalf("Error dialing: %v", err) + } + conn := <-connCh + defer conn.Close() + getConnection = func(context.Context) net.Conn { + return conn + } + ctx = peer.NewContext(ctx, data.rpcData.peerInfo) + stream := &ServerTransportStreamWithMethod{ + method: data.rpcData.fullMethod, + } + + ctx = grpc.NewContextWithServerTransportStream(ctx, stream) + err = cre.IsAuthorized(ctx) + if gotCode := status.Code(err); gotCode != data.wantStatusCode { + t.Fatalf("IsAuthorized(%+v, %+v) returned (%+v), want(%+v)", ctx, data.rpcData.fullMethod, gotCode, data.wantStatusCode) + } + if !reflect.DeepEqual(b.auditEvents, data.wantAuditEvents) { + t.Fatalf("Unexpected audit event for query:%v", data) + } + + // This builder's auditEvents can be shared for several queries, make sure it's empty. + b.auditEvents = nil + }() + } + }) + } +} + +type ServerTransportStreamWithMethod struct { + method string +} + +func (sts *ServerTransportStreamWithMethod) Method() string { + return sts.method +} + +func (sts *ServerTransportStreamWithMethod) SetHeader(metadata.MD) error { + return nil +} + +func (sts *ServerTransportStreamWithMethod) SendHeader(metadata.MD) error { + return nil +} + +func (sts *ServerTransportStreamWithMethod) SetTrailer(metadata.MD) error { + return nil +} + +// An audit logger that will log to the auditEvents slice. +type TestAuditLoggerBuffer struct { + auditEvents *[]*audit.Event +} + +func (logger *TestAuditLoggerBuffer) Log(e *audit.Event) { + *(logger.auditEvents) = append(*(logger.auditEvents), e) +} + +// Builds TestAuditLoggerBuffer. +type TestAuditLoggerBufferBuilder struct { + auditEvents []*audit.Event + testName string +} + +// The required config for TestAuditLoggerBuffer. +type TestAuditLoggerBufferConfig struct { + audit.LoggerConfig +} + +func (b *TestAuditLoggerBufferBuilder) ParseLoggerConfig(json.RawMessage) (config audit.LoggerConfig, err error) { + return TestAuditLoggerBufferConfig{}, nil +} + +func (b *TestAuditLoggerBufferBuilder) Build(audit.LoggerConfig) audit.Logger { + return &TestAuditLoggerBuffer{auditEvents: &b.auditEvents} +} + +func (b *TestAuditLoggerBufferBuilder) Name() string { + return b.testName + "_TestAuditLoggerBuffer" +} + +// An audit logger to test using a custom config. +type TestAuditLoggerCustomConfig struct{} + +func (logger *TestAuditLoggerCustomConfig) Log(*audit.Event) {} + +// Build TestAuditLoggerCustomConfig. This builds a TestAuditLoggerCustomConfig +// logger that uses a custom config. +type TestAuditLoggerCustomConfigBuilder struct { + testName string +} + +// The custom config for the TestAuditLoggerCustomConfig logger. +type TestAuditLoggerCustomConfigConfig struct { + audit.LoggerConfig + Abc int + Xyz string +} + +// Parses TestAuditLoggerCustomConfigConfig. Hard-coded to match with it's test +// case above. +func (b TestAuditLoggerCustomConfigBuilder) ParseLoggerConfig(configJSON json.RawMessage) (audit.LoggerConfig, error) { + c := TestAuditLoggerCustomConfigConfig{} + err := json.Unmarshal(configJSON, &c) + if err != nil { + return nil, fmt.Errorf("could not parse custom config: %v", err) + } + return c, nil +} + +func (b *TestAuditLoggerCustomConfigBuilder) Build(audit.LoggerConfig) audit.Logger { + return &TestAuditLoggerCustomConfig{} +} + +func (b *TestAuditLoggerCustomConfigBuilder) Name() string { + return b.testName + "_TestAuditLoggerCustomConfig" +} + +// Builds custom configs for audit logger RBAC protos. +func createUDPATypedStruct(t *testing.T, in map[string]any, name string) *anypb.Any { + t.Helper() + pb, err := structpb.NewStruct(in) + if err != nil { + t.Fatalf("createUDPATypedStructFailed during structpb.NewStruct: %v", err) + } + typedURL := "" + if name != "" { + typedURL = typeURLPrefix + name + } + typedStruct := &v1xdsudpatypepb.TypedStruct{ + TypeUrl: typedURL, + Value: pb, + } + customConfig, err := anypb.New(typedStruct) + if err != nil { + t.Fatalf("createUDPATypedStructFailed during anypb.New: %v", err) + } + return customConfig +} + +// Builds custom configs for audit logger RBAC protos. +func createXDSTypedStruct(t *testing.T, in map[string]any, name string) *anypb.Any { + t.Helper() + pb, err := structpb.NewStruct(in) + if err != nil { + t.Fatalf("createXDSTypedStructFailed during structpb.NewStruct: %v", err) + } + typedStruct := &v3xdsxdstypepb.TypedStruct{ + TypeUrl: typeURLPrefix + name, + Value: pb, + } + customConfig, err := anypb.New(typedStruct) + if err != nil { + t.Fatalf("createXDSTypedStructFailed during anypb.New: %v", err) + } + return customConfig +} + +func TestAuthenticatedMatcherIdentitySourcePrecedence(t *testing.T) { + tests := []struct { + name string + uris []string + dnsNames []string + subjectCN string + matcherContains string + wantMatch bool + }{ + { + name: "uriSANPresentAndMatches", + uris: []string{"spiffe://corp.example/svc/legitimate"}, + matcherContains: "spiffe://corp.example/svc/legitimate", + wantMatch: true, + }, + { + name: "uriSANPresentDoesNotFallbackToDNS", + uris: []string{"spiffe://corp.example/svc/other"}, + dnsNames: []string{"spiffe://corp.example/svc/legitimate"}, + matcherContains: "spiffe://corp.example/svc/legitimate", + wantMatch: false, + }, + { + name: "uriSANPresentDoesNotFallbackToSubject", + uris: []string{"spiffe://corp.example/svc/other"}, + subjectCN: "spiffe://corp.example/svc/legitimate", + matcherContains: "spiffe://corp.example/svc/legitimate", + wantMatch: false, + }, + { + name: "dnsSANPresentAndMatchesWhenNoURI", + dnsNames: []string{"svc.legitimate.internal"}, + matcherContains: "svc.legitimate.internal", + wantMatch: true, + }, + { + name: "dnsSANPresentDoesNotFallbackToSubject", + dnsNames: []string{"svc.other.internal"}, + subjectCN: "svc.legitimate.internal", + matcherContains: "svc.legitimate.internal", + wantMatch: false, + }, + { + name: "subjectUsedWhenNoURINorDNS", + subjectCN: "svc.legitimate.internal", + matcherContains: "svc.legitimate.internal", + wantMatch: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var uriSANs []*url.URL + for _, u := range tt.uris { + parsed, err := url.Parse(u) + if err != nil { + t.Fatalf("url.Parse(%q) failed: %v", u, err) + } + uriSANs = append(uriSANs, parsed) + } + cert := &x509.Certificate{ + URIs: uriSANs, + DNSNames: tt.dnsNames, + Subject: pkix.Name{ + CommonName: tt.subjectCN, + }, + } + + amConfig := &v3rbacpb.Principal_Authenticated{ + PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Contains{ + Contains: tt.matcherContains, + }, + }, + } + matcher, err := newAuthenticatedMatcher(amConfig) + if err != nil { + t.Fatalf("failed to create matcher: %v", err) + } + + rpcData := &rpcData{ + authType: "tls", + certs: []*x509.Certificate{cert}, + } + + if got := matcher.match(rpcData); got != tt.wantMatch { + t.Errorf("match() = %v, want %v", got, tt.wantMatch) + } + }) + } +} diff --git a/internal/xds/resolver/cluster_specifier_plugin_test.go b/internal/xds/resolver/cluster_specifier_plugin_test.go new file mode 100644 index 0000000..df3ff2e --- /dev/null +++ b/internal/xds/resolver/cluster_specifier_plugin_test.go @@ -0,0 +1,479 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver_test + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/balancer/clustermanager" + "google.golang.org/grpc/internal/xds/clusterspecifier" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" +) + +func init() { + balancer.Register(cspBalancerBuilder{}) + clusterspecifier.Register(testClusterSpecifierPlugin{}) +} + +// cspBalancerBuilder is a no-op LB policy which is referenced by the +// testClusterSpecifierPlugin. +type cspBalancerBuilder struct{} + +func (cspBalancerBuilder) Build(balancer.ClientConn, balancer.BuildOptions) balancer.Balancer { + return nil +} + +func (cspBalancerBuilder) Name() string { + return "csp_experimental" +} + +type cspBalancerConfig struct { + serviceconfig.LoadBalancingConfig + ArbitraryField string `json:"arbitrary_field"` +} + +func (cspBalancerBuilder) ParseConfig(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + cfg := &cspBalancerConfig{} + if err := json.Unmarshal(lbCfg, cfg); err != nil { + return nil, err + } + return cfg, nil + +} + +// testClusterSpecifierPlugin is a test cluster specifier plugin which returns +// an LB policy configuration specifying the cspBalancer. +type testClusterSpecifierPlugin struct { +} + +func (testClusterSpecifierPlugin) TypeURLs() []string { + // The config for this plugin contains a wrapperspb.StringValue, and since + // we marshal that proto as an Any proto, the type URL on the latter gets + // set to "type.googleapis.com/google.protobuf.StringValue". If we wanted a + // more descriptive type URL for this test plugin, we would have to define a + // proto package with a message for the configuration. That would be + // overkill for a test. Therefore, this seems to be an acceptable tradeoff. + return []string{"type.googleapis.com/google.protobuf.StringValue"} +} + +func (testClusterSpecifierPlugin) ParseClusterSpecifierConfig(cfg proto.Message) (clusterspecifier.BalancerConfig, error) { + if cfg == nil { + return nil, fmt.Errorf("testClusterSpecifierPlugin: nil configuration message provided") + } + anyp, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("testClusterSpecifierPlugin: error parsing config %v: got type %T, want *anypb.Any", cfg, cfg) + } + lbCfg := new(wrapperspb.StringValue) + if err := anypb.UnmarshalTo(anyp, lbCfg, proto.UnmarshalOptions{}); err != nil { + return nil, fmt.Errorf("testClusterSpecifierPlugin: error parsing config %v: %v", cfg, err) + } + return []map[string]any{{"csp_experimental": cspBalancerConfig{ArbitraryField: lbCfg.GetValue()}}}, nil +} + +// TestResolverClusterSpecifierPlugin tests the case where a route configuration +// containing cluster specifier plugins is sent by the management server. The +// test verifies that the service config output by the resolver contains the LB +// policy specified by the cluster specifier plugin, and the config selector +// returns the cluster associated with the cluster specifier plugin. +// +// The test also verifies that a change in the cluster specifier plugin config +// result in appropriate change in the service config pushed by the resolver. +func (s) TestResolverClusterSpecifierPlugin(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + // Configure resources on the management server. + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)} + routes := []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: defaultTestRouteConfigName, + ListenerName: defaultTestServiceName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeClusterSpecifierPlugin, + ClusterSpecifierPluginName: "cspA", + ClusterSpecifierPluginConfig: testutils.MarshalAny(t, &wrapperspb.StringValue{Value: "anything"}), + })} + configureResources(ctx, t, mgmtServer, nodeID, listeners, routes, nil, nil) + + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + // Wait for an update from the resolver, and verify the service config. + wantSC := ` + { + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster_specifier_plugin:cspA": { + "childPolicy": [ + { + "csp_experimental": { + "arbitrary_field": "anything" + } + } + ] + } + } + } + } + ] + }` + cs := verifyUpdateFromResolver(ctx, t, stateCh, wantSC) + res, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + gotCluster := clustermanager.PickedCluster(res.Context) + wantCluster := "cluster_specifier_plugin:cspA" + if gotCluster != wantCluster { + t.Fatalf("config selector returned cluster: %v, want: %v", gotCluster, wantCluster) + } + + // Change the cluster specifier plugin configuration. + routes = []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: defaultTestRouteConfigName, + ListenerName: defaultTestServiceName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeClusterSpecifierPlugin, + ClusterSpecifierPluginName: "cspA", + ClusterSpecifierPluginConfig: testutils.MarshalAny(t, &wrapperspb.StringValue{Value: "changed"}), + })} + configureResources(ctx, t, mgmtServer, nodeID, listeners, routes, nil, nil) + + // Wait for an update from the resolver, and verify the service config. + wantSC = ` + { + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster_specifier_plugin:cspA": { + "childPolicy": [ + { + "csp_experimental": { + "arbitrary_field": "changed" + } + } + ] + } + } + } + } + ] + }` + verifyUpdateFromResolver(ctx, t, stateCh, wantSC) +} + +// TestXDSResolverDelayedOnCommittedCSP tests that cluster specifier plugins and +// their corresponding configurations remain in service config if RPCs are in +// flight. +func (s) TestXDSResolverDelayedOnCommittedCSP(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + // Configure resources on the management server. + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)} + routes := []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: defaultTestRouteConfigName, + ListenerName: defaultTestServiceName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeClusterSpecifierPlugin, + ClusterSpecifierPluginName: "cspA", + ClusterSpecifierPluginConfig: testutils.MarshalAny(t, &wrapperspb.StringValue{Value: "anythingA"}), + })} + configureResources(ctx, t, mgmtServer, nodeID, listeners, routes, nil, nil) + + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + // Wait for an update from the resolver, and verify the service config. + wantSC := ` + { + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster_specifier_plugin:cspA": { + "childPolicy": [ + { + "csp_experimental": { + "arbitrary_field": "anythingA" + } + } + ] + } + } + } + } + ] + }` + cs := verifyUpdateFromResolver(ctx, t, stateCh, wantSC) + + resOld, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + gotCluster := clustermanager.PickedCluster(resOld.Context) + wantCluster := "cluster_specifier_plugin:cspA" + if gotCluster != wantCluster { + t.Fatalf("config selector returned cluster: %v, want: %v", gotCluster, wantCluster) + } + + // Delay resOld.OnCommitted(). As long as there are pending RPCs to removed + // clusters, they still appear in the service config. + + // Change the cluster specifier plugin configuration. + routes = []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: defaultTestRouteConfigName, + ListenerName: defaultTestServiceName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeClusterSpecifierPlugin, + ClusterSpecifierPluginName: "cspB", + ClusterSpecifierPluginConfig: testutils.MarshalAny(t, &wrapperspb.StringValue{Value: "anythingB"}), + })} + configureResources(ctx, t, mgmtServer, nodeID, listeners, routes, nil, nil) + + // Wait for an update from the resolver, and verify the service config. + wantSC = ` + { + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster_specifier_plugin:cspA": { + "childPolicy": [ + { + "csp_experimental": { + "arbitrary_field": "anythingA" + } + } + ] + }, + "cluster_specifier_plugin:cspB": { + "childPolicy": [ + { + "csp_experimental": { + "arbitrary_field": "anythingB" + } + } + ] + } + } + } + } + ] + }` + cs = verifyUpdateFromResolver(ctx, t, stateCh, wantSC) + + // Perform an RPC and ensure that it is routed to the new cluster. + resNew, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + gotCluster = clustermanager.PickedCluster(resNew.Context) + wantCluster = "cluster_specifier_plugin:cspB" + if gotCluster != wantCluster { + t.Fatalf("config selector returned cluster: %v, want: %v", gotCluster, wantCluster) + } + + // Invoke resOld.OnCommitted; should lead to a service config update that deletes + // cspA. + resOld.OnCommitted() + + wantSC = ` + { + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster_specifier_plugin:cspB": { + "childPolicy": [ + { + "csp_experimental": { + "arbitrary_field": "anythingB" + } + } + ] + } + } + } + } + ] + }` + verifyUpdateFromResolver(ctx, t, stateCh, wantSC) +} + +// TestResolverClusterSpecifierPlugin_WithFilters tests the case where a route +// configuration containing cluster specifier plugins is sent by the management +// server, and HTTP filters are configured. The test verifies that the +// interceptor chain is built for routes matching cluster specifier plugins. +func (s) TestResolverClusterSpecifierPlugin_WithFilters(t *testing.T) { + // Register custom httpFilter builders for the test. + testFilterTypeURL1 := "test-filter-type-url-1" + uuid.New().String() + testFilterTypeURL2 := "test-filter-type-url-2" + uuid.New().String() + newStreamChan := testutils.NewChannelWithSize(2) + fb1 := &testHTTPFilterWithRPCMetadata{ + logger: t, + typeURL: testFilterTypeURL1, + newStreamChan: newStreamChan, + } + fb2 := &testHTTPFilterWithRPCMetadata{ + logger: t, + typeURL: testFilterTypeURL2, + newStreamChan: newStreamChan, + } + httpfilter.Register(fb1) + httpfilter.Register(fb2) + defer httpfilter.UnregisterForTesting(fb1.typeURL) + defer httpfilter.UnregisterForTesting(fb2.typeURL) + + // Spin up an xDS management server. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + // Configure resources on the management server. + // We need a listener with the filter, and a route with ClusterSpecifierPlugin. + listeners := []*v3listenerpb.Listener{{ + Name: defaultTestServiceName, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: defaultTestRouteConfigName, + }}, + HttpFilters: []*v3httppb.HttpFilter{ + newHTTPFilter(t, "test-filter-1", testFilterTypeURL1, "filter-path-1", ""), + newHTTPFilter(t, "test-filter-2", testFilterTypeURL2, "filter-path-2", ""), + e2e.RouterHTTPFilter, + }, + }), + }, + }} + + routes := []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: defaultTestRouteConfigName, + ListenerName: defaultTestServiceName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeClusterSpecifierPlugin, + ClusterSpecifierPluginName: "cspA", + ClusterSpecifierPluginConfig: testutils.MarshalAny(t, &wrapperspb.StringValue{Value: "anything"}), + })} + // Override the configuration for "test-filter-1" in the route. + routes[0].VirtualHosts[0].Routes[0].TypedPerFilterConfig = map[string]*anypb.Any{ + "test-filter-1": newHTTPFilter(t, "test-filter-1", testFilterTypeURL1, "override-path-1", "").GetTypedConfig(), + } + configureResources(ctx, t, mgmtServer, nodeID, listeners, routes, nil, nil) + + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + // Wait for an update from the resolver, and verify the service config. + wantSC := ` + { + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster_specifier_plugin:cspA": { + "childPolicy": [ + { + "csp_experimental": { + "arbitrary_field": "anything" + } + } + ] + } + } + } + } + ] + }` + cs := verifyUpdateFromResolver(ctx, t, stateCh, wantSC) + res, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + // Verify that the interceptor is not nil. + if res.Interceptor == nil { + t.Fatal("RPCInfo does not contain interceptors list") + } + newStream := func(context.Context, ...grpc.CallOption) (grpc.ClientStream, error) { + return nil, nil + } + + if interceptor, ok := res.Interceptor.(httpfilter.ClientInterceptor); ok { + if _, err = interceptor.NewStream(ctx, iresolver.RPCInfo{Method: "/service/method", Context: ctx}, newStream); err != nil { + t.Fatalf("NewStream() failed with error: %v", err) + } + } else { + t.Fatalf("res.Interceptor is type %T, want httpfilter.ClientInterceptor", res.Interceptor) + } + + // Verify that first filter receives the config. + cfg, err := newStreamChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for first filter to receive config: %v", err) + } + ofc := cfg.(overallFilterConfig) + if ofc.BasePath != "filter-path-1" { + t.Fatalf("Unexpected base path for first filter, got: %q, want: %q", ofc.BasePath, "filter-path-1") + } + if ofc.OverridePath != "override-path-1" { + t.Fatalf("Unexpected override path for first filter, got: %q, want: %q", ofc.OverridePath, "override-path-1") + } + + // Verify that second filter receives the base path. + cfg, err = newStreamChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for second filter to receive config: %v", err) + } + ofc = cfg.(overallFilterConfig) + if ofc.BasePath != "filter-path-2" { + t.Fatalf("Unexpected base path for second filter, got: %q, want: %q", ofc.BasePath, "filter-path-2") + } + if ofc.OverridePath != "" { + t.Fatalf("Unexpected override path for second filter, got: %q, want: %q", ofc.OverridePath, "") + } +} diff --git a/internal/xds/resolver/helpers_test.go b/internal/xds/resolver/helpers_test.go new file mode 100644 index 0000000..f168b66 --- /dev/null +++ b/internal/xds/resolver/helpers_test.go @@ -0,0 +1,288 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver_test + +import ( + "context" + "fmt" + "net/url" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + xdsresolver "google.golang.org/grpc/internal/xds/resolver" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 100 * time.Microsecond + + defaultTestServiceName = "service-name" + defaultTestRouteConfigName = "route-config-name" + defaultTestClusterName = "cluster-name" + defaultTestEndpointName = "endpoint-name" + defaultTestHostname = "test-host" +) + +var defaultTestPort = []uint32{8080} + +// wantServiceConfig returns a JSON representation of a service config with +// xds_cluster_manager_experimental LB policy with a child policy of +// cds_experimental for the provided cluster name. +func wantServiceConfig(clusterName string) string { + return fmt.Sprintf(`{ + "loadBalancingConfig": [{ + "xds_cluster_manager_experimental": { + "children": { + "cluster:%s": { + "childPolicy": [{ + "cds_experimental": { + "cluster": "%s" + } + }] + } + } + } + }] + }`, clusterName, clusterName) +} + +// buildResolverForTarget builds an xDS resolver for the given target. If +// the bootstrap contents are provided, it build the xDS resolver using them +// otherwise, it uses the default xDS resolver. +// +// It returns the following: +// - a channel to read updates from the resolver +// - a channel to read errors from the resolver +// - the newly created xDS resolver +func buildResolverForTarget(t *testing.T, target resolver.Target, bootstrapContents []byte) (chan resolver.State, chan error, resolver.Resolver) { + t.Helper() + + var builder resolver.Builder + if bootstrapContents != nil { + // Create an xDS resolver with the provided bootstrap configuration. + var err error + builder, err = internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } else { + builder = resolver.Get(xdsresolver.Scheme) + if builder == nil { + t.Fatalf("Scheme %q is not registered", xdsresolver.Scheme) + } + } + + stateCh := make(chan resolver.State, 1) + updateStateF := func(s resolver.State) error { + stateCh <- s + return nil + } + errCh := make(chan error, 1) + reportErrorF := func(err error) { + select { + case errCh <- err: + default: + } + } + tcc := &testutils.ResolverClientConn{Logger: t, UpdateStateF: updateStateF, ReportErrorF: reportErrorF} + r, err := builder.Build(target, tcc, resolver.BuildOptions{ + Authority: url.PathEscape(target.Endpoint()), + }) + if err != nil { + t.Fatalf("Failed to build xDS resolver for target %q: %v", target, err) + } + t.Cleanup(r.Close) + return stateCh, errCh, r +} + +// verifyUpdateFromResolver waits for the resolver to push an update to the fake +// resolver.ClientConn and verifies that update matches the provided service +// config. +// +// Tests that want to skip verifying the contents of the service config can pass +// an empty string. +// +// Returns the config selector from the state update pushed by the resolver. +// Tests that don't need the config selector can ignore the return value. +func verifyUpdateFromResolver(ctx context.Context, t *testing.T, stateCh chan resolver.State, wantSC string) iresolver.ConfigSelector { + t.Helper() + + var state resolver.State + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for an update from the resolver: %v", ctx.Err()) + case state = <-stateCh: + if err := state.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", state.ServiceConfig.Err) + } + if wantSC == "" { + break + } + wantSCParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(wantSC) + if !internal.EqualServiceConfigForTesting(state.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, state.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + } + cs := iresolver.GetConfigSelector(state) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + return cs +} + +// verifyNoUpdateFromResolver verifies that no update is pushed on stateCh. +// Calls t.Fatal() if an update is received before defaultTestShortTimeout +// expires. +func verifyNoUpdateFromResolver(ctx context.Context, t *testing.T, stateCh chan resolver.State) { + t.Helper() + + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case u := <-stateCh: + t.Fatalf("Received update from resolver %v when none expected", u) + } +} + +func verifyResolverError(gotErr error, wantCode codes.Code, wantErr, wantNodeID string) error { + if gotErr == nil { + return fmt.Errorf("got nil error from resolver, want error with code %v", wantCode) + } + if !strings.Contains(gotErr.Error(), wantErr) { + return fmt.Errorf("got error from resolver %q, want %q", gotErr, wantErr) + } + if gotCode := status.Code(gotErr); gotCode != wantCode { + return fmt.Errorf("got error from resolver with code %v, want %v", gotCode, wantCode) + } + if !strings.Contains(gotErr.Error(), wantNodeID) { + return fmt.Errorf("got error from resolver %q, want nodeID %q", gotErr, wantNodeID) + } + return nil +} + +// Spins up an xDS management server and sets up an xDS bootstrap configuration +// file that points to it. +// +// Returns the following: +// - A reference to the xDS management server +// - A channel to read requested Listener resource names +// - A channel to read requested RouteConfiguration resource names +// - Contents of the bootstrap configuration pointing to xDS management +// server +func setupManagementServerForTest(t *testing.T, nodeID string) (*e2e.ManagementServer, chan []string, chan []string, []byte) { + t.Helper() + + listenerResourceNamesCh := make(chan []string, 1) + routeConfigResourceNamesCh := make(chan []string, 1) + + // Setup the management server to push the requested listener and route + // configuration resource names on to separate channels for the test to + // inspect. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + switch req.GetTypeUrl() { + case version.V3ListenerURL: + select { + case <-listenerResourceNamesCh: + default: + } + select { + case listenerResourceNamesCh <- req.GetResourceNames(): + default: + } + case version.V3RouteConfigURL: + select { + case <-routeConfigResourceNamesCh: + default: + } + select { + case routeConfigResourceNamesCh <- req.GetResourceNames(): + default: + } + } + return nil + }, + AllowResourceSubset: true, + }) + + // Create a bootstrap configuration specifying the above management server. + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + return mgmtServer, listenerResourceNamesCh, routeConfigResourceNamesCh, bootstrapContents +} + +// Updates all resources on the given management server. +func configureResources(ctx context.Context, t *testing.T, mgmtServer *e2e.ManagementServer, nodeID string, listeners []*v3listenerpb.Listener, routes []*v3routepb.RouteConfiguration, clusters []*v3clusterpb.Cluster, endpoints []*v3endpointpb.ClusterLoadAssignment) { + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } +} + +// waitForResourceNames waits for the wantNames to be pushed on to namesCh. +// Fails the test by calling t.Fatal if the context expires before that. +func waitForResourceNames(ctx context.Context, t *testing.T, namesCh chan []string, wantNames []string) { + t.Helper() + + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + select { + case <-ctx.Done(): + case gotNames := <-namesCh: + if cmp.Equal(gotNames, wantNames, cmpopts.EquateEmpty(), cmpopts.SortSlices(func(a, b string) bool { return a < b })) { + return + } + t.Logf("Received resource names %v, want %v", gotNames, wantNames) + } + } + t.Fatalf("Timeout waiting for resource to be requested from the management server") +} diff --git a/internal/xds/resolver/internal/internal.go b/internal/xds/resolver/internal/internal.go new file mode 100644 index 0000000..f22caff --- /dev/null +++ b/internal/xds/resolver/internal/internal.go @@ -0,0 +1,30 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains functionality internal to the xDS resolver. +package internal + +// The following variables are overridden in tests. +var ( + // NewWRR is the function used to create a new weighted round robin + // implementation. + NewWRR any // func() wrr.WRR + + // NewXDSClient is the function used to create a new xDS client. + NewXDSClient any // func(string, estats.MetricsRecorder) (xdsclient.XDSClient, func(), error) +) diff --git a/internal/xds/resolver/logging.go b/internal/xds/resolver/logging.go new file mode 100644 index 0000000..746b85a --- /dev/null +++ b/internal/xds/resolver/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[xds-resolver %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *xdsResolver) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/internal/xds/resolver/serviceconfig.go b/internal/xds/resolver/serviceconfig.go new file mode 100644 index 0000000..bdbaea6 --- /dev/null +++ b/internal/xds/resolver/serviceconfig.go @@ -0,0 +1,381 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "encoding/json" + "fmt" + "math/bits" + rand "math/rand/v2" + "strings" + "sync" + "time" + + xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpcutil" + iresolver "google.golang.org/grpc/internal/resolver" + iringhash "google.golang.org/grpc/internal/ringhash" + "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/wrr" + "google.golang.org/grpc/internal/xds/balancer/clusterimpl" + "google.golang.org/grpc/internal/xds/balancer/clustermanager" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +const ( + cdsName = "cds_experimental" + xdsClusterManagerName = "xds_cluster_manager_experimental" + clusterPrefix = "cluster:" + clusterSpecifierPluginPrefix = "cluster_specifier_plugin:" +) + +type serviceConfig struct { + LoadBalancingConfig balancerConfig `json:"loadBalancingConfig"` +} + +type balancerConfig []map[string]any + +func newBalancerConfig(name string, config any) balancerConfig { + return []map[string]any{{name: config}} +} + +type cdsBalancerConfig struct { + Cluster string `json:"cluster"` +} + +type xdsChildConfig struct { + ChildPolicy balancerConfig `json:"childPolicy"` +} + +type xdsClusterManagerConfig struct { + Children map[string]xdsChildConfig `json:"children"` +} + +// serviceConfigJSON produces a service config in JSON format that contains LB +// policy config for the "xds_cluster_manager" LB policy, with entries in the +// children map for all active clusters. +func serviceConfigJSON(activeClusters map[string]*clusterInfo, activePlugins map[string]*clusterInfo) []byte { + // Generate children (all entries in activeClusters). + children := make(map[string]xdsChildConfig) + for cluster, ci := range activeClusters { + children[cluster] = ci.cfg + } + for plugin, ci := range activePlugins { + children[plugin] = ci.cfg + } + + sc := serviceConfig{ + LoadBalancingConfig: newBalancerConfig( + xdsClusterManagerName, xdsClusterManagerConfig{Children: children}, + ), + } + + // This is not expected to fail as we have constructed the service config by + // hand right above, and therefore ok to panic. + bs, err := json.Marshal(sc) + if err != nil { + panic(fmt.Sprintf("failed to marshal service config %+v: %v", sc, err)) + } + return bs +} + +type virtualHost struct { + // retry policy present in virtual host + retryConfig *xdsresource.RetryConfig +} + +// routeCluster holds information about a cluster as referenced by a route. +type routeCluster struct { + name string // Name of the cluster. + interceptor httpfilter.ClientInterceptor // HTTP filters to run for RPCs matching this route. +} + +type route struct { + m *xdsresource.CompositeMatcher // converted from route matchers + actionType xdsresource.RouteActionType // holds route action type + clusters wrr.WRR // holds *routeCluster entries + routeClusters []*grpcsync.RefCounted[*routeCluster] // Route clusters belonging to this route + maxStreamDuration time.Duration + retryConfig *xdsresource.RetryConfig + hashPolicies []*xdsresource.HashPolicy + autoHostRewrite bool +} + +func (r route) String() string { + return fmt.Sprintf("%s -> { clusters: %v, maxStreamDuration: %v }", r.m.String(), r.clusters, r.maxStreamDuration) +} + +// stoppableConfigSelector extends the iresolver.ConfigSelector interface with a +// stop() method. This makes it possible to swap the current config selector +// with an erroring config selector when the LDS or RDS resource is not found on +// the management server. +type stoppableConfigSelector interface { + iresolver.ConfigSelector + stop() +} + +// erroringConfigSelector always returns an error, with the xDS node ID included +// in the error message. It is used to swap out the current config selector +// when the LDS or RDS resource is not found on the management server. +type erroringConfigSelector struct { + err error +} + +func newErroringConfigSelector(err error, xdsNodeID string) *erroringConfigSelector { + return &erroringConfigSelector{err: annotateErrorWithNodeID(status.Error(codes.Unavailable, err.Error()), xdsNodeID)} +} + +func (cs *erroringConfigSelector) SelectConfig(iresolver.RPCInfo) (*iresolver.RPCConfig, error) { + return nil, cs.err +} +func (cs *erroringConfigSelector) stop() {} + +type configSelector struct { + channelID uint64 // Static hash when hash policy is HashPolicyTypeChannelID + xdsNodeID string // xDS node ID, for annotating errors. + sendNewServiceConfig func() // Function to send a new service config to gRPC. + + // Configuration received from the xDS management server. + virtualHost virtualHost + routes []route + clusters map[string]*clusterInfo + plugins map[string]*clusterInfo + httpFilterConfig []xdsresource.HTTPFilter + xdsConfig *xdsresource.XDSConfig +} + +var errNoMatchedRouteFound = status.Errorf(codes.Unavailable, "no matched route was found") +var errUnsupportedClientRouteAction = status.Errorf(codes.Unavailable, "matched route does not have a supported route action type") + +// annotateErrorWithNodeID annotates the given error with the provided xDS node +// ID. This is used by the real config selector when it runs into errors, and +// also by the erroring config selector. +func annotateErrorWithNodeID(err error, nodeID string) error { + return fmt.Errorf("[xDS node id: %s]: %w", nodeID, err) +} + +func (cs *configSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RPCConfig, error) { + var rt *route + md, _ := metadata.FromOutgoingContext(rpcInfo.Context) + if extraMD, ok := grpcutil.ExtraMetadata(rpcInfo.Context); ok { + md = metadata.Join(md, extraMD) + // Remove all binary headers. They are hard to match with. May need + // to add back if asked by users. + for k := range md { + if strings.HasSuffix(k, "-bin") { + delete(md, k) + } + } + } + // Loop through routes in order and select first match. + for _, r := range cs.routes { + if r.m.Match(rpcInfo.Method, md) { + rt = &r + break + } + } + + if rt == nil || rt.clusters == nil { + return nil, annotateErrorWithNodeID(errNoMatchedRouteFound, cs.xdsNodeID) + } + + if rt.actionType != xdsresource.RouteActionRoute { + return nil, annotateErrorWithNodeID(errUnsupportedClientRouteAction, cs.xdsNodeID) + } + + rc, ok := rt.clusters.Next().(*grpcsync.RefCounted[*routeCluster]) + if !ok { + return nil, annotateErrorWithNodeID(status.Errorf(codes.Internal, "error retrieving cluster for match: %v (%T)", rc, rc), cs.xdsNodeID) + } + cluster := rc.Value() + lbCtx := clustermanager.SetPickedCluster(rpcInfo.Context, cluster.name) + lbCtx = xdsresource.NewContextWithXDSConfig(lbCtx, cs.xdsConfig) + lbCtx = iringhash.SetXDSRequestHash(lbCtx, cs.generateHash(rpcInfo, rt.hashPolicies)) + if rt.autoHostRewrite { + lbCtx = clusterimpl.EnableAutoHostRewrite(lbCtx) + } + + config := &iresolver.RPCConfig{ + Context: lbCtx, + Interceptor: cluster.interceptor, + } + // Add a ref to the selected cluster to keep the interceptors alive until RPC + // is committed. + rc.Increment() + if info, ok := cs.clusters[cluster.name]; ok { + // Add a ref to the selected cluster, as this RPC needs this + // cluster until it is committed. + info.refCount.Add(1) + config.OnCommitted = sync.OnceFunc(func() { + if v := info.refCount.Add(-1); v == 0 { + // We call unsubscribe rather than sendNewServiceConfig to + // prevent redundant updates. If the reference count in the + // dependency manager drops to zero, it will automatically + // trigger a service config update with this cluster + // removed. Calling unsubscribe allows the dependency + // manager to handle the update flow once and for all. + info.unsubscribe() + } + // Decrement the refcount of the route cluster and close the interceptor + // if refcount goes to zero. + rc.Decrement() + }) + } else if info, ok := cs.plugins[cluster.name]; ok { + // Add a ref to the selected plugin, as this RPC needs this + // plugin until it is committed. + info.refCount.Add(1) + config.OnCommitted = sync.OnceFunc(func() { + if v := info.refCount.Add(-1); v == 0 { + // This entry will be removed from activePlugins when + // producing a new service config update. + cs.sendNewServiceConfig() + } + // Decrement the refcount of the route cluster and close the interceptor + // if refcount goes to zero. + rc.Decrement() + }) + } else { + // This should be unreachable because all route clusters are normalized + // into cs.clusters or cs.plugins during config selector creation. + panic(fmt.Sprintf("matched cluster %q not found in ConfigSelector", cluster.name)) + } + + if rt.maxStreamDuration != 0 { + config.MethodConfig.Timeout = &rt.maxStreamDuration + } + if rt.retryConfig != nil { + config.MethodConfig.RetryPolicy = retryConfigToPolicy(rt.retryConfig) + } else if cs.virtualHost.retryConfig != nil { + config.MethodConfig.RetryPolicy = retryConfigToPolicy(cs.virtualHost.retryConfig) + } + + return config, nil +} + +func retryConfigToPolicy(config *xdsresource.RetryConfig) *serviceconfig.RetryPolicy { + return &serviceconfig.RetryPolicy{ + MaxAttempts: int(config.NumRetries) + 1, + InitialBackoff: config.RetryBackoff.BaseInterval, + MaxBackoff: config.RetryBackoff.MaxInterval, + BackoffMultiplier: 2, + RetryableStatusCodes: config.RetryOn, + } +} + +func (cs *configSelector) generateHash(rpcInfo iresolver.RPCInfo, hashPolicies []*xdsresource.HashPolicy) uint64 { + var hash uint64 + var generatedHash bool + var md, emd metadata.MD + var mdRead bool + for _, policy := range hashPolicies { + var policyHash uint64 + var generatedPolicyHash bool + switch policy.HashPolicyType { + case xdsresource.HashPolicyTypeHeader: + if strings.HasSuffix(policy.HeaderName, "-bin") { + continue + } + if !mdRead { + md, _ = metadata.FromOutgoingContext(rpcInfo.Context) + emd, _ = grpcutil.ExtraMetadata(rpcInfo.Context) + mdRead = true + } + values := emd.Get(policy.HeaderName) + if len(values) == 0 { + // Extra metadata (e.g. the "content-type" header) takes + // precedence over the user's metadata. + values = md.Get(policy.HeaderName) + if len(values) == 0 { + // If the header isn't present at all, this policy is a no-op. + continue + } + } + joinedValues := strings.Join(values, ",") + if policy.Regex != nil { + joinedValues = policy.Regex.ReplaceAllString(joinedValues, policy.RegexSubstitution) + } + policyHash = xxhash.Sum64String(joinedValues) + generatedHash = true + generatedPolicyHash = true + case xdsresource.HashPolicyTypeChannelID: + // Use the static channel ID as the hash for this policy. + policyHash = cs.channelID + generatedHash = true + generatedPolicyHash = true + } + + // Deterministically combine the hash policies. Rotating prevents + // duplicate hash policies from cancelling each other out and preserves + // the 64 bits of entropy. + if generatedPolicyHash { + hash = bits.RotateLeft64(hash, 1) + hash = hash ^ policyHash + } + + // If terminal policy and a hash has already been generated, ignore the + // rest of the policies and use that hash already generated. + if policy.Terminal && generatedHash { + break + } + } + + if generatedHash { + return hash + } + // If no generated hash return a random long. In the grand scheme of things + // this logically will map to choosing a random backend to route request to. + return rand.Uint64() +} + +// stop decrements refs of all clusters referenced by this config selector. +func (cs *configSelector) stop() { + // The resolver's old configSelector may be nil. Handle that here. + if cs == nil { + return + } + + // Decrement the refcount of all the route clusters associated with this + // config selector and close the interceptors of the route cluster if it's + // refcount goes to zero. + for _, r := range cs.routes { + for _, rc := range r.routeClusters { + rc.Decrement() + } + } + + // If any reference counts drop to zero, a service config update is required + // to remove the clusters. Since the old config selector is stopped + // after a new one is active, we must trigger a subsequent update to delete + // the now-unused clusters. + for _, ci := range cs.clusters { + if v := ci.refCount.Add(-1); v == 0 { + ci.unsubscribe() + } + } + for _, ci := range cs.plugins { + if v := ci.refCount.Add(-1); v == 0 { + cs.sendNewServiceConfig() + } + } +} diff --git a/internal/xds/resolver/serviceconfig_test.go b/internal/xds/resolver/serviceconfig_test.go new file mode 100644 index 0000000..4fa25d6 --- /dev/null +++ b/internal/xds/resolver/serviceconfig_test.go @@ -0,0 +1,187 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "context" + "regexp" + "testing" + "time" + + xxhash "github.com/cespare/xxhash/v2" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/grpcutil" + iresolver "google.golang.org/grpc/internal/resolver" + _ "google.golang.org/grpc/internal/xds/balancer/cdsbalancer" // To parse LB config + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/metadata" +) + +var defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestPruneActiveClusters(t *testing.T) { + newClusterInfo := func(ref int32, unsubscribe func()) *clusterInfo { + ci := &clusterInfo{unsubscribe: unsubscribe} + ci.refCount.Store(ref) + return ci + } + r := &xdsResolver{ + activeClusters: map[string]*clusterInfo{ + "zero": newClusterInfo(0, func() {}), + "one": newClusterInfo(1, func() {}), + "two": newClusterInfo(2, func() {}), + "anotherzero": newClusterInfo(0, func() {}), + }, + activePlugins: map[string]*clusterInfo{ + "zero": newClusterInfo(0, nil), + "one": newClusterInfo(1, nil), + "two": newClusterInfo(2, nil), + "anotherzero": newClusterInfo(0, nil), + }, + } + wantActiveClusters := map[string]int32{ + "one": 1, + "two": 2, + } + wantActivePlugins := map[string]int32{ + "one": 1, + "two": 2, + } + r.pruneActiveClustersAndPlugins() + + getRefCounts := func(m map[string]*clusterInfo) map[string]int32 { + res := make(map[string]int32) + for k, v := range m { + res[k] = v.refCount.Load() + } + return res + } + + if d := cmp.Diff(getRefCounts(r.activeClusters), wantActiveClusters); d != "" { + t.Fatalf("r.activeClusters refCounts mismatch (-got +want):\n%s", d) + } + if d := cmp.Diff(getRefCounts(r.activePlugins), wantActivePlugins); d != "" { + t.Fatalf("r.activePlugins refCounts mismatch (-got +want):\n%s", d) + } +} + +func (s) TestGenerateRequestHash(t *testing.T) { + const channelID = 12378921 + cs := &configSelector{channelID: channelID} + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + tests := []struct { + name string + hashPolicies []*xdsresource.HashPolicy + requestHashWant uint64 + rpcInfo iresolver.RPCInfo + }{ + // TestGenerateRequestHashHeaders tests generating request hashes for + // hash policies that specify to hash headers. + { + name: "test-generate-request-hash-headers", + hashPolicies: []*xdsresource.HashPolicy{{ + HashPolicyType: xdsresource.HashPolicyTypeHeader, + HeaderName: ":path", + Regex: func() *regexp.Regexp { return regexp.MustCompile("/products") }(), // Will replace /products with /new-products, to test find and replace functionality. + RegexSubstitution: "/new-products", + }}, + requestHashWant: xxhash.Sum64String("/new-products"), + rpcInfo: iresolver.RPCInfo{ + Context: metadata.NewOutgoingContext(ctx, metadata.Pairs(":path", "/products")), + Method: "/some-method", + }, + }, + // TestGenerateHashChannelID tests generating request hashes for hash + // policies that specify to hash something that uniquely identifies the + // ClientConn (the pointer). + { + name: "test-generate-request-hash-channel-id", + hashPolicies: []*xdsresource.HashPolicy{{ + HashPolicyType: xdsresource.HashPolicyTypeChannelID, + }}, + requestHashWant: channelID, + rpcInfo: iresolver.RPCInfo{}, + }, + // TestGenerateRequestHashEmptyString tests generating request hashes + // for hash policies that specify to hash headers and replace empty + // strings in the headers. + { + name: "test-generate-request-hash-empty-string", + hashPolicies: []*xdsresource.HashPolicy{{ + HashPolicyType: xdsresource.HashPolicyTypeHeader, + HeaderName: ":path", + Regex: func() *regexp.Regexp { return regexp.MustCompile("") }(), + RegexSubstitution: "e", + }}, + requestHashWant: xxhash.Sum64String("eaebece"), + rpcInfo: iresolver.RPCInfo{ + Context: metadata.NewOutgoingContext(ctx, metadata.Pairs(":path", "abc")), + Method: "/some-method", + }, + }, + // Tests that bin headers are skipped. + { + name: "skip-bin", + hashPolicies: []*xdsresource.HashPolicy{{ + HashPolicyType: xdsresource.HashPolicyTypeHeader, + HeaderName: "something-bin", + }, { + HashPolicyType: xdsresource.HashPolicyTypeChannelID, + }}, + requestHashWant: channelID, + rpcInfo: iresolver.RPCInfo{ + Context: metadata.NewOutgoingContext(ctx, metadata.Pairs("something-bin", "xyz")), + }, + }, + // Tests that extra metadata takes precedence over the user's metadata. + { + name: "extra-metadata", + hashPolicies: []*xdsresource.HashPolicy{{ + HashPolicyType: xdsresource.HashPolicyTypeHeader, + HeaderName: "content-type", + }}, + requestHashWant: xxhash.Sum64String("grpc value"), + rpcInfo: iresolver.RPCInfo{ + Context: grpcutil.WithExtraMetadata( + metadata.NewOutgoingContext(ctx, metadata.Pairs("content-type", "user value")), + metadata.Pairs("content-type", "grpc value"), + ), + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + requestHashGot := cs.generateHash(test.rpcInfo, test.hashPolicies) + if requestHashGot != test.requestHashWant { + t.Fatalf("requestHashGot = %v, requestHashWant = %v", requestHashGot, test.requestHashWant) + } + }) + } +} diff --git a/internal/xds/resolver/watch_service_test.go b/internal/xds/resolver/watch_service_test.go new file mode 100644 index 0000000..2077059 --- /dev/null +++ b/internal/xds/resolver/watch_service_test.go @@ -0,0 +1,187 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver_test + +import ( + "context" + "testing" + + "github.com/envoyproxy/go-control-plane/pkg/wellknown" + "github.com/google/uuid" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/resolver" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" +) + +// Tests the case where the listener resource starts pointing to a new route +// configuration resource after the xDS resolver has successfully resolved the +// service name and pushed an update on the channel. The test verifies that the +// resolver stops requesting the old route configuration resource and requests +// the new resource, and once successfully resolved, verifies that the update +// from the resolver matches expected service config. +func (s) TestServiceWatch_ListenerPointsToNewRouteConfiguration(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, lisCh, routeCfgCh, bc := setupManagementServerForTest(t, nodeID) + + // Configure resources on the management server. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: defaultTestServiceName, + NodeID: nodeID, + Host: defaultTestHostname, + Port: defaultTestPort[0], + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + // Verify initial update from the resolver. + waitForResourceNames(ctx, t, lisCh, []string{defaultTestServiceName}) + waitForResourceNames(ctx, t, routeCfgCh, []string{resources.Routes[0].Name}) + verifyUpdateFromResolver(ctx, t, stateCh, wantServiceConfig(resources.Clusters[0].Name)) + + // Update the listener resource to point to a new route configuration name. + // The old route configuration resource is left unchanged to prevent a race: + // if it were removed immediately, the resolver might encounter a + // "resource-not-found" error for the old route before the listener update + // successfully transitions the client to the new route. + newTestRouteConfigName := defaultTestRouteConfigName + "-new" + resources.Listeners = []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, newTestRouteConfigName)} + resources.SkipValidation = true + mgmtServer.Update(ctx, resources) + + // Verify that the new route configuration resource is requested. + waitForResourceNames(ctx, t, routeCfgCh, []string{newTestRouteConfigName}) + + // Update the old route configuration resource by adding a new route. + resources.Routes[0].VirtualHosts[0].Routes = append(resources.Routes[0].VirtualHosts[0].Routes, &v3routepb.Route{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/foo/bar"}, + CaseSensitive: &wrapperspb.BoolValue{Value: false}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: "some-random-cluster"}, + }, + }, + }) + mgmtServer.Update(ctx, resources) + + // Wait for no update from the resolver since the listener resource no + // longer points to the old route resource and new route resource hasn't + // been sent yet. + verifyNoUpdateFromResolver(ctx, t, stateCh) + + // Update the management server with the new route configuration resource. + resources.Routes = append(resources.Routes, e2e.DefaultRouteConfig(newTestRouteConfigName, defaultTestServiceName, resources.Clusters[0].Name)) + configureResources(ctx, t, mgmtServer, nodeID, resources.Listeners, resources.Routes, nil, nil) + + // Ensure update from the resolver. + verifyUpdateFromResolver(ctx, t, stateCh, wantServiceConfig(resources.Clusters[0].Name)) +} + +// Tests the case where the listener resource changes to contain an inline route +// configuration and changes back to having a route configuration resource name. +// Verifies that the expected xDS resource names are requested by the resolver +// and that the update from the resolver matches expected service config. +func (s) TestServiceWatch_ListenerPointsToInlineRouteConfiguration(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, lisCh, routeCfgCh, bc := setupManagementServerForTest(t, nodeID) + + // Configure resources on the management server. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: defaultTestServiceName, + NodeID: nodeID, + Host: defaultTestHostname, + Port: defaultTestPort[0], + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + // Verify initial update from the resolver. + waitForResourceNames(ctx, t, lisCh, []string{defaultTestServiceName}) + waitForResourceNames(ctx, t, routeCfgCh, []string{resources.Routes[0].Name}) + verifyUpdateFromResolver(ctx, t, stateCh, wantServiceConfig(resources.Clusters[0].Name)) + + // Update listener to contain an inline route configuration. + hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: resources.Routes[0].Name, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{defaultTestServiceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: resources.Clusters[0].Name}, + }, + }, + }}, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, + }) + resources.Listeners = []*v3listenerpb.Listener{{ + Name: defaultTestServiceName, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + }} + configureResources(ctx, t, mgmtServer, nodeID, resources.Listeners, nil, nil, nil) + + // Verify that the old route configuration is not requested anymore. + waitForResourceNames(ctx, t, routeCfgCh, []string{}) + verifyUpdateFromResolver(ctx, t, stateCh, wantServiceConfig(resources.Clusters[0].Name)) + + // Update listener back to contain a route configuration name. + resources.Listeners = []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, resources.Routes[0].Name)} + configureResources(ctx, t, mgmtServer, nodeID, resources.Listeners, resources.Routes, nil, nil) + + // Verify that that route configuration resource is requested. + waitForResourceNames(ctx, t, routeCfgCh, []string{resources.Routes[0].Name}) + + // Verify that appropriate SC is pushed on the channel. + verifyUpdateFromResolver(ctx, t, stateCh, wantServiceConfig(resources.Clusters[0].Name)) +} diff --git a/internal/xds/resolver/xds_http_filters_test.go b/internal/xds/resolver/xds_http_filters_test.go new file mode 100644 index 0000000..a5cb85f --- /dev/null +++ b/internal/xds/resolver/xds_http_filters_test.go @@ -0,0 +1,1462 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver_test + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/httpfilter" + rinternal "google.golang.org/grpc/internal/xds/resolver/internal" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/xds" // Register all required xDS components +) + +const ( + filterCfgPathFieldName = "path" + filterCfgErrorFieldName = "new_stream_error" + filterCfgMetadataKey = "test-filter-config" +) + +// testFilterCfg is the internal representation of the filter config proto. It +// is returned by filter's config parsing methods. +type testFilterCfg struct { + httpfilter.FilterConfig + path string + newStreamErr string +} + +// filterConfigFromProto parses filter config specified as a v3.TypedStruct into +// a testFilterCfg. +func filterConfigFromProto(cfg proto.Message) (httpfilter.FilterConfig, error) { + ts, ok := cfg.(*v3xdsxdstypepb.TypedStruct) + if !ok { + return nil, fmt.Errorf("unsupported filter config type: %T, want %T", cfg, &v3xdsxdstypepb.TypedStruct{}) + } + + if ts.GetValue() == nil { + return testFilterCfg{}, nil + } + ret := testFilterCfg{} + if v := ts.GetValue().GetFields()[filterCfgPathFieldName]; v != nil { + ret.path = v.GetStringValue() + } + if v := ts.GetValue().GetFields()[filterCfgErrorFieldName]; v != nil { + ret.newStreamErr = v.GetStringValue() + } + return ret, nil +} + +type logger interface { + Logf(format string, args ...any) +} + +// testHTTPFilterWithRPCMetadata is a HTTP filter used for testing purposes. +// +// This filter is used to verify that the xDS resolver and filter stack +// correctly propagate filter configuration (both base and override) to RPCs. It +// does this by injecting the config paths from its base and override configs as +// JSON-encoded metadata into outgoing RPCs. The metadata can then be observed +// by the backend, allowing tests to assert that the correct filter +// configuration was applied for each RPC. +type testHTTPFilterWithRPCMetadata struct { + logger logger + typeURL string + newStreamChan *testutils.Channel // If set, filter config is written to this field from NewStream() +} + +func (fb *testHTTPFilterWithRPCMetadata) TypeURLs() []string { return []string{fb.typeURL} } + +func (*testHTTPFilterWithRPCMetadata) ParseFilterConfig(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return filterConfigFromProto(cfg) +} + +func (*testHTTPFilterWithRPCMetadata) ParseFilterConfigOverride(override proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return filterConfigFromProto(override) +} + +func (*testHTTPFilterWithRPCMetadata) IsTerminal() bool { return false } + +func (fb *testHTTPFilterWithRPCMetadata) BuildClientFilter(httpfilter.ClientFilterOptions) httpfilter.ClientFilter { + return fb +} + +func (fb *testHTTPFilterWithRPCMetadata) Close() {} + +// ClientFilterBuilder is an optional interface for filters to implement. This +// compile time check ensures the test filter implements it. +var _ httpfilter.ClientFilterBuilder = &testHTTPFilterWithRPCMetadata{} + +func (fb *testHTTPFilterWithRPCMetadata) BuildClientInterceptor(config, override httpfilter.FilterConfig) (httpfilter.ClientInterceptor, error) { + fb.logger.Logf("BuildClientInterceptor called with config: %+v, override: %+v", config, override) + + if config == nil { + return nil, fmt.Errorf("unexpected missing config") + } + + baseCfg := config.(testFilterCfg) + basePath := baseCfg.path + newStreamErr := baseCfg.newStreamErr + + var overridePath string + if override != nil { + overrideCfg := override.(testFilterCfg) + overridePath = overrideCfg.path + if overrideCfg.newStreamErr != "" { + newStreamErr = overrideCfg.newStreamErr + } + } + + return &testFilterInterceptor{ + logger: fb.logger, + cfg: overallFilterConfig{ + BasePath: basePath, + OverridePath: overridePath, + Error: newStreamErr, + }, + newStreamChan: fb.newStreamChan, + }, nil +} + +// overallFilterConfig is a JSON representation of the filter config. +// It is sent as RPC metadata and written to a channel for test verification. +type overallFilterConfig struct { + BasePath string `json:"base_path,omitempty"` + OverridePath string `json:"override_path,omitempty"` + Error string `json:"error,omitempty"` +} + +// testFilterInterceptor is a client interceptor that injects RPC metadata +// corresponding to its filter config. +type testFilterInterceptor struct { + logger logger + cfg overallFilterConfig + newStreamChan *testutils.Channel // If set, filter config is written to this field from NewStream() +} + +func (fi *testFilterInterceptor) NewStream(ctx context.Context, _ iresolver.RPCInfo, newStream func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) { + // Write the config to the channel, if set. This allows tests to verify that + // the filter was invoked at RPC time. This is useful for tests where the + // RPC is expected to fail, and therefore the RPC metadata cannot be + // observed from the backend. + if fi.newStreamChan != nil { + fi.newStreamChan.Send(fi.cfg) + } + + if fi.cfg.Error != "" { + return nil, status.Error(codes.Unavailable, fi.cfg.Error) + } + + // Marshal the filter config to JSON and inject it as metadata. + bytes, err := json.Marshal(fi.cfg) + if err != nil { + return nil, fmt.Errorf("failed to marshal filter config: %w", err) + } + cfg := string(bytes) + fi.logger.Logf("Injecting filter config metadata: %v", cfg) + + return newStream(metadata.AppendToOutgoingContext(ctx, filterCfgMetadataKey, cfg), opts...) +} + +func (fi *testFilterInterceptor) Close() {} + +func newHTTPFilter(t *testing.T, name, typeURL, path, err string) *v3httppb.HttpFilter { + return &v3httppb.HttpFilter{ + Name: name, + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: typeURL, + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: path}}, + filterCfgErrorFieldName: {Kind: &structpb.Value_StringValue{StringValue: err}}, + }, + }, + }), + }, + } +} + +// newStubServer returns a stub server that sends any filter config metadata +// received as part of incoming RPCs to the provided channel. +func newStubServer(metadataCh chan []string) *stubserver.StubServer { + return &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Error(codes.InvalidArgument, "missing metadata") + } + select { + case metadataCh <- md.Get(filterCfgMetadataKey): + case <-ctx.Done(): + return nil, ctx.Err() + } + return &testpb.Empty{}, nil + }, + UnaryCallF: func(ctx context.Context, req *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Error(codes.InvalidArgument, "missing metadata") + } + select { + case metadataCh <- md.Get(filterCfgMetadataKey): + case <-ctx.Done(): + return nil, ctx.Err() + } + return &testpb.SimpleResponse{Payload: req.GetPayload()}, nil + }, + } +} + +// Tests HTTP filters with the xDS resolver. The test exercises various levels +// of filter config overrides (base, virtual host-level, route-level and +// cluster-level), and verifies that the correct config is applied for each RPC. +func (s) TestXDSResolverHTTPFilters_AllOverrides(t *testing.T) { + // Override default WRR with a deterministic test version. + origNewWRR := rinternal.NewWRR + rinternal.NewWRR = testutils.NewTestWRR + defer func() { rinternal.NewWRR = origNewWRR }() + + // Register a custom httpFilter builder for the test. + testFilterName := t.Name() + fb := &testHTTPFilterWithRPCMetadata{logger: t, typeURL: testFilterName} + httpfilter.Register(fb) + defer httpfilter.UnregisterForTesting(fb.typeURL) + + // Spin up an xDS management server + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer mgmtServer.Stop() + + // Create an xDS resolver with bootstrap configuration pointing to the above + // management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a couple of test backends. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + const chBufSize = 4 // Expecting 4 metadata entries (2 RPCs, each with 2 filters). + metadataCh := make(chan []string, chBufSize) + backend1 := stubserver.StartTestService(t, newStubServer(metadataCh)) + defer backend1.Stop() + backend2 := stubserver.StartTestService(t, newStubServer(metadataCh)) + defer backend2.Stop() + + // Configure resources on the management server. + // + // The route configuration contains two routes, matching two different RPCs. + // The route for the UnaryCall RPC does not contain any cluster-level or + // route-level per-filter config overrides. A virtual host-level per-filter + // config override exists and it should apply for RPCs matching this route. + // + // The route for the EmptyCall RPC contains a route-level per-filter config + // override that should apply for RPCs routed to cluster "A" since it does + // not have any cluster-level overrides. For RPCs matching cluster "B" + // though, a cluster-level per-filter config override should take + // precedence. + const testServiceName = "service-name" + const routeConfigName = "route-config" + listener := &v3listenerpb.Listener{ + Name: testServiceName, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{testServiceName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "A", Weight: wrapperspb.UInt32(1)}, + {Name: "B", Weight: wrapperspb.UInt32(1)}, + }, + }, + }, + }, + }, + }, + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: "A", + Weight: wrapperspb.UInt32(1), + }, + { + Name: "B", + Weight: wrapperspb.UInt32(1), + TypedPerFilterConfig: map[string]*anypb.Any{ + "foo": testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: testFilterName, + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: "foo4"}}, + }, + }, + }), + "bar": testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: testFilterName, + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: "bar4"}}, + }, + }, + }), + }, + }, + }, + }, + }, + }, + }, + TypedPerFilterConfig: map[string]*anypb.Any{ + "foo": testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: testFilterName, + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: "foo3"}}, + }, + }, + }), + "bar": testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: testFilterName, + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: "bar3"}}, + }, + }, + }), + }, + }, + }, + TypedPerFilterConfig: map[string]*anypb.Any{ + "foo": testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: testFilterName, + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: "foo2"}}, + }, + }, + }), + "bar": testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: testFilterName, + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: "bar2"}}, + }, + }, + }), + }, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + newHTTPFilter(t, "foo", testFilterName, "foo1", ""), + newHTTPFilter(t, "bar", testFilterName, "bar1", ""), + e2e.RouterHTTPFilter, + }, + }), + }, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster("A", "endpoint_A", e2e.SecurityLevelNone), + e2e.DefaultCluster("B", "endpoint_B", e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint("endpoint_A", "localhost", []uint32{testutils.ParsePort(t, backend1.Address)}), + e2e.DefaultEndpoint("endpoint_B", "localhost", []uint32{testutils.ParsePort(t, backend2.Address)}), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a gRPC client using the xDS resolver. + cc, err := grpc.NewClient("xds:///"+testServiceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + // Helper to make an RPC twice and collect filter configs from metadata. We + // make the RPC two times to ensure that we hit both clusters (because of + // the deterministic WRR). The returned filter configs are in the order in + // which the RPCs were made. + collectFilterConfigs := func(rpc func() error) []overallFilterConfig { + t.Helper() + var gotFilterCfgs []overallFilterConfig + for i := 0; i < 2; i++ { + if err := rpc(); err != nil { + t.Fatalf("Unexpected RPC error: %v", err) + } + select { + case cfg := <-metadataCh: + if len(cfg) != 2 { + t.Fatalf("Unexpected number of filter config metadata, got: %d, want: 2", len(cfg)) + } + for _, c := range cfg { + var ofc overallFilterConfig + if err := json.Unmarshal([]byte(c), &ofc); err != nil { + t.Fatalf("Failed to unmarshal filter config JSON %q: %v", c, err) + } + gotFilterCfgs = append(gotFilterCfgs, ofc) + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for metadata from backend") + } + } + return gotFilterCfgs + } + + // Test base filter config (UnaryCall). Because of the deterministic WRR, we + // know the expected order of clusters for the two RPCs. + wantFilterCfgs := []overallFilterConfig{ + {BasePath: "foo1", OverridePath: "foo2"}, // Routed to cluster A + {BasePath: "bar1", OverridePath: "bar2"}, // Routed to cluster A + {BasePath: "foo1", OverridePath: "foo2"}, // Routed to cluster B + {BasePath: "bar1", OverridePath: "bar2"}, // Routed to cluster B + } + client := testgrpc.NewTestServiceClient(cc) + gotFilterCfgs := collectFilterConfigs(func() error { + _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}) + return err + }) + if diff := cmp.Diff(wantFilterCfgs, gotFilterCfgs); diff != "" { + t.Fatalf("Unexpected filter configs (-want +got):\n%s", diff) + } + + // Test per-route and per-cluster overrides (EmptyCall). + wantFilterCfgs = []overallFilterConfig{ + {BasePath: "foo1", OverridePath: "foo3"}, // Routed to cluster A + {BasePath: "bar1", OverridePath: "bar3"}, // Routed to cluster A + {BasePath: "foo1", OverridePath: "foo4"}, // Routed to cluster B + {BasePath: "bar1", OverridePath: "bar4"}, // Routed to cluster B + } + gotFilterCfgs = collectFilterConfigs(func() error { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + return err + }) + if diff := cmp.Diff(wantFilterCfgs, gotFilterCfgs); diff != "" { + t.Fatalf("Unexpected filter configs (-want +got):\n%s", diff) + } +} + +// Tests that if a filter returns an error from its NewStream method, the RPC +// fails with that error. It also verifies that subsequent filters in the chain +// are not run. +func (s) TestXDSResolverHTTPFilters_NewStreamError(t *testing.T) { + // Register a custom httpFilter builder for the test and use a channel to + // get notified when the interceptor is invoked. + testFilterName := t.Name() + fb := &testHTTPFilterWithRPCMetadata{ + logger: t, + typeURL: testFilterName, + newStreamChan: testutils.NewChannelWithSize(3), // We have three filters. + } + httpfilter.Register(fb) + defer httpfilter.UnregisterForTesting(fb.typeURL) + + // Spin up an xDS management server + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer mgmtServer.Stop() + + // Create an xDS resolver with bootstrap configuration pointing to the above + // management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a test backend, but we expect the filter to fail the RPC before it + // ever gets to the backend. The test is designed to fail if the RPC + // *succeeds* (i.e., if the backend is reached). A large channel buffer is + // used to prevent blocking in the unexpected case where the filter fails to + // reject the RPC. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + metadataCh := make(chan []string, 10) + backend := stubserver.StartTestService(t, newStubServer(metadataCh)) + defer backend.Stop() + + // Configure resources on the management server. + // + // The route configuration contains two routes, matching two different RPCs. + // The route for the UnaryCall RPC does not contain any cluster-level or + // route-level per-filter config overrides. A virtual host-level per-filter + // config override exists and it should apply for RPCs matching this route. + // + // The route for the EmptyCall RPC contains a route-level per-filter config + // override that should apply for RPCs routed to cluster "A" since it does + // not have any cluster-level overrides. For RPCs matching cluster "B" + // though, a cluster-level per-filter config override should take + // precedence. + const testServiceName = "service-name" + const routeConfigName = "route-config" + listener := &v3listenerpb.Listener{ + Name: testServiceName, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{testServiceName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "A", Weight: wrapperspb.UInt32(1)}, + }, + }, + }, + }, + }, + }, + }, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + newHTTPFilter(t, "foo-good", testFilterName, "foo-good", ""), + newHTTPFilter(t, "foo-failing", testFilterName, "foo-failing", "filter interceptor error"), + newHTTPFilter(t, "bar-good", testFilterName, "bar-good", ""), + e2e.RouterHTTPFilter, + }, + }), + }, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster("A", "endpoint_A", e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint("endpoint_A", "localhost", []uint32{testutils.ParsePort(t, backend.Address)})}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a gRPC client using the xDS resolver. + cc, err := grpc.NewClient("xds:///"+testServiceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Fatalf("EmptyCall() RPC succeeded when expected to fail") + } + if got, want := status.Code(err), codes.Unavailable; got != want { + t.Fatalf("EmptyCall() RPC error code, got: %v, want: %v", got, want) + } + if got, want := err.Error(), "filter interceptor error"; !strings.Contains(got, want) { + t.Fatalf("Unexpected RPC error, got: %v, want: %v", err, "rpc error: code = Unavailable desc = filter interceptor error") + } + + // Verify that the first good filter was invoked + cfg, err := fb.newStreamChan.Receive(ctx) + if err != nil { + t.Fatal("Timeout waiting for first filter to be invoked") + } + ofc := cfg.(overallFilterConfig) + wantCfg := overallFilterConfig{BasePath: "foo-good"} + if diff := cmp.Diff(wantCfg, ofc); diff != "" { + t.Fatalf("Unexpected first filter config (-want +got):\n%s", diff) + } + + // Verify that the failing filter was invoked too. + cfg, err = fb.newStreamChan.Receive(ctx) + if err != nil { + t.Fatal("Timeout waiting for second filter to be invoked") + } + ofc = cfg.(overallFilterConfig) + wantCfg = overallFilterConfig{BasePath: "foo-failing", Error: "filter interceptor error"} + if diff := cmp.Diff(wantCfg, ofc); diff != "" { + t.Fatalf("Unexpected second filter config (-want +got):\n%s", diff) + } + + // Verify that the last good filter was not invoked. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err = fb.newStreamChan.Receive(sCtx); err == nil { + t.Fatal("Last filter was invoked when expected not to be") + } +} + +// trackingHTTPFilterBuilder is a test filter that allows counting the number of +// times a filter instance or an interceptor instance is built or closed. +type trackingHTTPFilterBuilder struct { + httpfilter.Builder + filtersCreated *atomic.Int32 + filtersDestroyed *atomic.Int32 + interceptorsCreated *atomic.Int32 + interceptorsDestroyed *atomic.Int32 + typeURL string + pathCh chan string +} + +func (t *trackingHTTPFilterBuilder) IsTerminal() bool { return false } + +func (t *trackingHTTPFilterBuilder) TypeURLs() []string { return []string{t.typeURL} } + +func (*trackingHTTPFilterBuilder) ParseFilterConfig(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return filterConfigFromProto(cfg) +} + +func (*trackingHTTPFilterBuilder) ParseFilterConfigOverride(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return filterConfigFromProto(cfg) +} + +func (t *trackingHTTPFilterBuilder) BuildClientFilter(httpfilter.ClientFilterOptions) httpfilter.ClientFilter { + t.filtersCreated.Add(1) + return t +} + +func (t *trackingHTTPFilterBuilder) Close() { + t.filtersDestroyed.Add(1) +} + +var _ httpfilter.ClientFilterBuilder = &trackingHTTPFilterBuilder{} + +func (t *trackingHTTPFilterBuilder) BuildClientInterceptor(config, _ httpfilter.FilterConfig) (httpfilter.ClientInterceptor, error) { + t.interceptorsCreated.Add(1) + + if config == nil { + return nil, fmt.Errorf("unexpected missing config") + } + baseCfg := config.(testFilterCfg) + + interceptor := &trackingInterceptor{ + pathCh: t.pathCh, + basePath: baseCfg.path, + parent: t, + } + return interceptor, nil +} + +type trackingInterceptor struct { + parent *trackingHTTPFilterBuilder + pathCh chan string + basePath string +} + +func (i *trackingInterceptor) NewStream(ctx context.Context, _ iresolver.RPCInfo, newStream func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) { + i.pathCh <- i.basePath + return newStream(ctx, opts...) +} + +func (i *trackingInterceptor) Close() { + i.parent.interceptorsDestroyed.Add(1) +} + +// Tests that a single filter instance is created for a given filter +// configuration, and that this instance is retained across config updates. +// Verifies that a new interceptor is built for each config update, and that the +// old interceptor is closed. +func (s) TestXDSResolverHTTPFilters_FilterInstanceRetained(t *testing.T) { + // Register a custom httpFilter builder for the test. + var filtersCreated, filtersDestroyed, interceptorsCreated, interceptorsDestroyed atomic.Int32 + pathCh := make(chan string, 1) + testFilterTypeURL := t.Name() + fb := &trackingHTTPFilterBuilder{ + filtersCreated: &filtersCreated, + filtersDestroyed: &filtersDestroyed, + interceptorsCreated: &interceptorsCreated, + interceptorsDestroyed: &interceptorsDestroyed, + typeURL: testFilterTypeURL, + pathCh: pathCh, + } + httpfilter.Register(fb) + defer httpfilter.UnregisterForTesting(fb.typeURL) + + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer mgmtServer.Stop() + + // Create an xDS resolver with bootstrap configuration pointing to the above + // management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + + // Configure resources on the management server. + const testServiceName = "service-name" + const routeConfigName = "route-config" + listener := &v3listenerpb.Listener{ + Name: testServiceName, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{testServiceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: "A"}, + }, + }, + }}, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + newHTTPFilter(t, "tracker", testFilterTypeURL, "initial-path", ""), + e2e.RouterHTTPFilter, + }, + }), + }, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster("A", "endpoint_A", e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint("endpoint_A", "localhost", []uint32{testutils.ParsePort(t, backend.Address)})}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a gRPC client using the xDS resolver. + cc, err := grpc.NewClient("xds:///"+testServiceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and verify that one filter and one interceptor are created. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + select { + case cfg := <-pathCh: + if got, want := cfg, "initial-path"; got != want { + t.Fatalf("Unexpected config sent to filter, got: %q, want: %q", got, want) + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for interceptor to be invoked") + } + if got, want := filtersCreated.Load(), int32(1); got != want { + t.Fatalf("Created %d filter instances, want: %d", got, want) + } + if got, want := interceptorsCreated.Load(), int32(1); got != want { + t.Fatalf("Created %d interceptor instances, want: %d", got, want) + } + + // Update the filter config in the listener resource. + listener.ApiListener.ApiListener = testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{testServiceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: "A"}, + }, + }, + }}, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + newHTTPFilter(t, "tracker", testFilterTypeURL, "final-path", ""), + e2e.RouterHTTPFilter, + }, + }) + resources.Listeners = []*v3listenerpb.Listener{listener} + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the updated config to be applied on the gRPC client. +WaitForUpdatedConfig: + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + select { + case cfg := <-pathCh: + if got, want := cfg, "final-path"; got == want { + break WaitForUpdatedConfig + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for interceptor get updated config") + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for updated config to be applied: %v", ctx.Err()) + } + + // Verify the filter instance is retained, while the interceptor instances + // are replaced with the updated config. The new ConfigSelector is pushed to + // the channel before the old one is stopped. So, the previous loop waiting + // for the updated config to be applied does not guarantee that the old + // interceptor is stopped. We need to wait for a short duration after the + // updated config is applied to ensure that the old interceptor is stopped + // and the new one is created. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if got, want := filtersCreated.Load(), int32(1); got != want { + continue + } + if got, want := interceptorsCreated.Load(), int32(2); got != want { + continue + } + if got, want := interceptorsDestroyed.Load(), int32(1); got != want { + continue + } + break + } + if ctx.Err() != nil { + t.Errorf("Destroyed %d interceptor instances, want: %d", filtersCreated.Load(), 1) + t.Errorf("Created %d interceptor instances, want: %d", interceptorsCreated.Load(), 2) + t.Errorf("Destroyed %d interceptor instances, want: %d", interceptorsDestroyed.Load(), 1) + t.Fatal("Timeout when waiting for interceptos instances to be replaced") + } + + // Close the client and verify that the filter and interceptor are closed. + cc.Close() + if got, want := filtersDestroyed.Load(), int32(1); got != want { + t.Fatalf("Destroyed %d filter instances, want: %d", got, want) + } + if got, want := interceptorsDestroyed.Load(), int32(2); got != want { + t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want) + } +} + +// TestXDSResolverHTTPFilters_MultiLevelOverride tests that filter overrides at +// the Virtual Host, Route, and Cluster levels are handled correctly. It +// verifies that the filter receives configuration and the disable information +// from the most specific override available. If no override is present, the +// filter should receive the disable information and configuration from the base +// config. +func (s) TestXDSResolverHTTPFilters_MultiLevelOverride(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true) + + // Register two custom httpFilter builders for the test. + newStreamChan1 := testutils.NewChannel() + fb1 := &testHTTPFilterWithRPCMetadata{ + logger: t, + typeURL: "filter1", + newStreamChan: newStreamChan1, + } + httpfilter.Register(fb1) + defer httpfilter.UnregisterForTesting(fb1.typeURL) + + newStreamChan2 := testutils.NewChannel() + fb2 := &testHTTPFilterWithRPCMetadata{ + logger: t, + typeURL: "filter2", + newStreamChan: newStreamChan2, + } + httpfilter.Register(fb2) + defer httpfilter.UnregisterForTesting(fb2.typeURL) + + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + defer mgmtServer.Stop() + + // Create an xDS resolver with bootstrap configuration pointing to the above + // management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + + // Configure resources on the management server. + const testServiceName = "service-name" + const routeConfigName = "route-config" + + // Filter 1: path="filter1-path" + baseFilter1 := newHTTPFilter(t, "filter1", "filter1", "filter1-path", "") + baseFilter1.Disabled = true // Disabled in base. + + // Filter 2: path="filter2-path" + baseFilter2 := newHTTPFilter(t, "filter2", "filter2", "filter2-path", "") + + // Virtual host override for filter 1 that enables it with path = + // "vhost-value". + vhostOverride := &v3routepb.FilterConfig{ + Disabled: false, + Config: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: "filter1", + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: "vhost-value"}}, + }, + }, + }), + } + + // Route override for filter 1 that enables it with path = "route-value". + routeOverride := &v3routepb.FilterConfig{ + Disabled: false, + Config: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: "filter1", + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: "route-value"}}, + }, + }, + }), + } + + // Cluster override for filter 1 that enables it with path = "cluster-value". + clusterOverride := &v3routepb.FilterConfig{ + Disabled: false, + Config: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: "filter1", + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: "cluster-value"}}, + }, + }, + }), + } + + listener := &v3listenerpb.Listener{ + Name: testServiceName, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{testServiceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: "A", + Weight: wrapperspb.UInt32(1), + TypedPerFilterConfig: map[string]*anypb.Any{"filter1": testutils.MarshalAny(t, clusterOverride)}, + }, + }, + }, + }, + }}, + TypedPerFilterConfig: map[string]*anypb.Any{"filter1": testutils.MarshalAny(t, routeOverride)}, + }}, + TypedPerFilterConfig: map[string]*anypb.Any{"filter1": testutils.MarshalAny(t, vhostOverride)}, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + baseFilter1, + baseFilter2, + e2e.RouterHTTPFilter, + }, + }), + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster("A", "endpoint_A", e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint("endpoint_A", "localhost", []uint32{testutils.ParsePort(t, backend.Address)})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a gRPC client using the xDS resolver. + cc, err := grpc.NewClient("xds:///"+testServiceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC and verify that the filter was invoked with the combined config. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("UnaryCall() failed: %v", err) + } + + // Verify that filter1 receives BasePath="filter1-path" and + // OverridePath="cluster-value" because the cluster override has the highest + // precedence. This ensures the filter is enabled by the cluster override, + // even if it is disabled in the base config. + val1, err := newStreamChan1.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for filter1 to be invoked: %v", err) + } + cfg1 := val1.(overallFilterConfig) + if got, want := cfg1.BasePath, "filter1-path"; got != want { + t.Errorf("Unexpected BasePath for filter1, got: %q, want: %q", got, want) + } + if got, want := cfg1.OverridePath, "cluster-value"; got != want { + t.Errorf("Unexpected OverridePath for filter1, got: %q, want: %q", got, want) + } + + // Verify that filter2 was also invoked and has base filter config and no + // override path. + val2, err := newStreamChan2.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for filter2 to be invoked: %v", err) + } + cfg2 := val2.(overallFilterConfig) + if got, want := cfg2.BasePath, "filter2-path"; got != want { + t.Errorf("Unexpected BasePath for filter2, got: %q, want: %q", got, want) + } + if got, want := cfg2.OverridePath, ""; got != want { + t.Errorf("Unexpected OverridePath for filter2, got: %q, want: %q", got, want) + } +} + +// TestXDSResolverHTTPFilters_DisabledInOverride tests various combinations of +// base filter configurations and overrides at Virtual Host, Route, and Cluster +// levels that disable the filter. It verifies that the filter is not created +// when disabled. +func (s) TestXDSResolverHTTPFilters_DisabledInOverride(t *testing.T) { + type overrideInfo struct { + disabled bool + path string + } + + tests := []struct { + name string + baseDisabled bool + vhost *overrideInfo + route *overrideInfo + cluster *overrideInfo + }{ + { + name: "BaseDisabled", + baseDisabled: true, + }, + { + name: "VHostDisables", + baseDisabled: false, + vhost: &overrideInfo{disabled: true}, + }, + { + name: "RouteDisables", + baseDisabled: false, + vhost: &overrideInfo{disabled: false, path: "vhost-value"}, + route: &overrideInfo{disabled: true}, + }, + { + name: "ClusterDisables", + baseDisabled: false, + vhost: &overrideInfo{disabled: false, path: "vhost-value"}, + route: &overrideInfo{disabled: false, path: "route-value"}, + cluster: &overrideInfo{disabled: true}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true) + + // Register a custom httpFilter builder for the test. + var filtersCreated, interceptorsCreated atomic.Int32 + testFilterTypeURL := t.Name() + fb := &trackingHTTPFilterBuilder{ + filtersCreated: &filtersCreated, + interceptorsCreated: &interceptorsCreated, + typeURL: testFilterTypeURL, + } + httpfilter.Register(fb) + defer httpfilter.UnregisterForTesting(fb.typeURL) + + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + defer mgmtServer.Stop() + + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + + const testServiceName = "service-name" + const routeConfigName = "route-config" + + baseFilter := newHTTPFilter(t, "test-filter", testFilterTypeURL, "base-path", "") + baseFilter.Disabled = tc.baseDisabled + + makeTypedConfig := func(info *overrideInfo) map[string]*anypb.Any { + if info == nil { + return nil + } + cfg := &v3routepb.FilterConfig{Disabled: info.disabled} + cfg.Config = testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: testFilterTypeURL, + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: info.path}}, + }, + }, + }) + return map[string]*anypb.Any{"test-filter": testutils.MarshalAny(t, cfg)} + } + + vhostTypedConfig := makeTypedConfig(tc.vhost) + routeTypedConfig := makeTypedConfig(tc.route) + clusterTypedConfig := makeTypedConfig(tc.cluster) + + listener := &v3listenerpb.Listener{ + Name: testServiceName, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{testServiceName}, + TypedPerFilterConfig: vhostTypedConfig, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: "A", + Weight: wrapperspb.UInt32(1), + TypedPerFilterConfig: clusterTypedConfig, + }, + }, + }, + }, + }}, + TypedPerFilterConfig: routeTypedConfig, + }}, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + baseFilter, + e2e.RouterHTTPFilter, + }, + }), + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster("A", "endpoint_A", e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint("endpoint_A", "localhost", []uint32{testutils.ParsePort(t, backend.Address)})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient("xds:///"+testServiceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("UnaryCall() failed: %v", err) + } + + if got, want := filtersCreated.Load(), int32(0); got != want { + t.Fatalf("Created %d filter instances, want: %d", got, want) + } + if got, want := interceptorsCreated.Load(), int32(0); got != want { + t.Fatalf("Created %d interceptor instances, want: %d", got, want) + } + }) + } +} + +// TestXDSResolverHTTPFilters_EnabledInOverride tests various combinations of +// base filter configurations and overrides at Virtual Host, Route, and Cluster +// levels that enable the filter. It verifies that the filter is created and +// receives the correct configuration. +func (s) TestXDSResolverHTTPFilters_EnabledInOverride(t *testing.T) { + const basePath = "base-path" + type overrideInfo struct { + disabled bool + path string + } + + tests := []struct { + name string + baseDisabled bool + vhost *overrideInfo + route *overrideInfo + cluster *overrideInfo + wantOverridePath string + }{ + { + name: "BaseEnabled", + baseDisabled: false, + }, + { + name: "VHostEnables", + baseDisabled: true, + vhost: &overrideInfo{disabled: false, path: "vhost-value"}, + wantOverridePath: "vhost-value", + }, + { + name: "RouteEnables", + baseDisabled: true, + vhost: &overrideInfo{disabled: true}, + route: &overrideInfo{disabled: false, path: "route-value"}, + wantOverridePath: "route-value", + }, + { + name: "ClusterEnables", + baseDisabled: true, + vhost: &overrideInfo{disabled: true}, + route: &overrideInfo{disabled: true}, + cluster: &overrideInfo{disabled: false, path: "cluster-value"}, + wantOverridePath: "cluster-value", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true) + + // Register a custom httpFilter builder for the test. + newStreamChan := testutils.NewChannel() + testFilterTypeURL := t.Name() + fb := &testHTTPFilterWithRPCMetadata{ + logger: t, + typeURL: testFilterTypeURL, + newStreamChan: newStreamChan, + } + httpfilter.Register(fb) + defer httpfilter.UnregisterForTesting(fb.typeURL) + + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + defer mgmtServer.Stop() + + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + + const testServiceName = "service-name" + const routeConfigName = "route-config" + + baseFilter := newHTTPFilter(t, "test-filter", testFilterTypeURL, basePath, "") + baseFilter.Disabled = tc.baseDisabled + + makeTypedConfig := func(info *overrideInfo) map[string]*anypb.Any { + if info == nil { + return nil + } + cfg := &v3routepb.FilterConfig{Disabled: info.disabled} + cfg.Config = testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: testFilterTypeURL, + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: info.path}}, + }, + }, + }) + return map[string]*anypb.Any{"test-filter": testutils.MarshalAny(t, cfg)} + } + + vhostTypedConfig := makeTypedConfig(tc.vhost) + routeTypedConfig := makeTypedConfig(tc.route) + clusterTypedConfig := makeTypedConfig(tc.cluster) + + listener := &v3listenerpb.Listener{ + Name: testServiceName, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{testServiceName}, + TypedPerFilterConfig: vhostTypedConfig, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: "A", + Weight: wrapperspb.UInt32(1), + TypedPerFilterConfig: clusterTypedConfig, + }, + }, + }, + }, + }}, + TypedPerFilterConfig: routeTypedConfig, + }}, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + baseFilter, + e2e.RouterHTTPFilter, + }, + }), + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster("A", "endpoint_A", e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint("endpoint_A", "localhost", []uint32{testutils.ParsePort(t, backend.Address)})}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient("xds:///"+testServiceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("UnaryCall() failed: %v", err) + } + + val, err := newStreamChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for interceptor to be invoked: %v", err) + } + cfg := val.(overallFilterConfig) + if tc.wantOverridePath != "" { + if got, want := cfg.OverridePath, tc.wantOverridePath; got != want { + t.Fatalf("Unexpected OverridePath, got: %q, want: %q", got, want) + } + return + } + if got, want := cfg.BasePath, basePath; got != want { + t.Fatalf("Unexpected BasePath, got: %q, want: %q", got, want) + } + }) + } +} diff --git a/internal/xds/resolver/xds_resolver.go b/internal/xds/resolver/xds_resolver.go new file mode 100644 index 0000000..36c7132 --- /dev/null +++ b/internal/xds/resolver/xds_resolver.go @@ -0,0 +1,731 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package resolver implements the xds resolver, that does LDS and RDS to find +// the cluster to use. +package resolver + +import ( + "context" + "fmt" + rand "math/rand/v2" + "slices" + "strings" + "sync/atomic" + + "google.golang.org/grpc" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/wrr" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/httpfilter" + rinternal "google.golang.org/grpc/internal/xds/resolver/internal" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/internal/xds/xdsdepmgr" + "google.golang.org/grpc/resolver" +) + +// Scheme is the xDS resolver's scheme. +// +// TODO(easwars): Rename this package as xdsresolver so that this is accessed as +// xdsresolver.Scheme +const Scheme = "xds" + +// newBuilderWithConfigForTesting creates a new xds resolver builder using a +// specific xds bootstrap config, so tests can use multiple xDS clients in +// different ClientConns at the same time. The builder creates a new pool with +// the provided config and a new xDS client in that pool. +func newBuilderWithConfigForTesting(config []byte) (resolver.Builder, error) { + return &xdsResolverBuilder{ + newXDSClient: func(name string, mr estats.MetricsRecorder) (xdsclient.XDSClient, func(), error) { + config, err := bootstrap.NewConfigFromContents(config) + if err != nil { + return nil, nil, err + } + pool := xdsclient.NewPool(config) + return pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: name, + MetricsRecorder: mr, + }) + }, + }, nil +} + +// newBuilderWithPoolForTesting creates a new xds resolver builder using the +// specific xds client pool, so that tests have complete control over the exact +// specific xds client pool being used. +func newBuilderWithPoolForTesting(pool *xdsclient.Pool) (resolver.Builder, error) { + return &xdsResolverBuilder{ + newXDSClient: func(name string, mr estats.MetricsRecorder) (xdsclient.XDSClient, func(), error) { + return pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: name, + MetricsRecorder: mr, + }) + }, + }, nil +} + +// newBuilderWithClientForTesting creates a new xds resolver builder using the +// specific xDS client, so that tests have complete control over the exact +// specific xDS client being used. +func newBuilderWithClientForTesting(client xdsclient.XDSClient) (resolver.Builder, error) { + return &xdsResolverBuilder{ + newXDSClient: func(string, estats.MetricsRecorder) (xdsclient.XDSClient, func(), error) { + // Returning an empty close func here means that the responsibility + // of closing the client lies with the caller. + return client, func() {}, nil + }, + }, nil +} + +func init() { + resolver.Register(&xdsResolverBuilder{}) + internal.NewXDSResolverWithConfigForTesting = newBuilderWithConfigForTesting + internal.NewXDSResolverWithPoolForTesting = newBuilderWithPoolForTesting + internal.NewXDSResolverWithClientForTesting = newBuilderWithClientForTesting + + rinternal.NewWRR = wrr.NewRandom + rinternal.NewXDSClient = xdsclient.DefaultPool.NewClient +} + +type xdsResolverBuilder struct { + newXDSClient func(string, estats.MetricsRecorder) (xdsclient.XDSClient, func(), error) +} + +// Build helps implement the resolver.Builder interface. +// +// The xds bootstrap process is performed (and a new xDS client is built) every +// time an xds resolver is built. +func (b *xdsResolverBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (_ resolver.Resolver, retErr error) { + // Initialize the xDS client. + newXDSClient := rinternal.NewXDSClient.(func(string, estats.MetricsRecorder) (xdsclient.XDSClient, func(), error)) + if b.newXDSClient != nil { + newXDSClient = b.newXDSClient + } + client, xdsClientClose, err := newXDSClient(target.String(), opts.MetricsRecorder) + if err != nil { + return nil, fmt.Errorf("xds: failed to create xds-client: %v", err) + } + + template, err := sanityChecksOnBootstrapConfig(target, client) + if err != nil { + xdsClientClose() + return nil, err + } + ldsResourceName := bootstrap.PopulateResourceTemplate(template, target.Endpoint()) + + ctx, cancel := context.WithCancel(context.Background()) + r := &xdsResolver{ + cc: cc, + xdsClient: client, + xdsClientClose: xdsClientClose, + activeClusters: make(map[string]*clusterInfo), + activePlugins: make(map[string]*clusterInfo), + httpFilters: make(map[clientFilterKey]httpfilter.ClientFilter), + channelID: rand.Uint64(), + ldsResourceName: ldsResourceName, + target: target.String(), + metricsRecorder: opts.MetricsRecorder, + + // serializer used to synchronize the following: + // - updates from the dependency manager. This could lead to generation + // of new service config if resolution is complete. + // - completion of an RPC to a removed cluster causing the associated + // ref count to become zero, resulting in generation of new service + // config. + // - stopping of a config selector that results in generation of new + // service config. + serializer: grpcsync.NewCallbackSerializer(ctx), + serializerCancel: cancel, + } + r.logger = prefixLogger(r) + r.logger.Infof("Creating resolver for target: %+v", target) + + dmSet := make(chan struct{}) + // Schedule a callback that blocks until r.dm is set i.e xdsdepmgr.New() + // returns. This acts as a gatekeeper: even if dependency manager sends the + // updates before the xdsdepmgr.New() has a chance to return, they will be + // queued behind this blocker and processed only after initialization is + // complete. + r.serializer.TrySchedule(func(ctx context.Context) { + select { + case <-dmSet: + case <-ctx.Done(): + } + }) + r.dm = xdsdepmgr.New(r.ldsResourceName, opts.Authority, r.xdsClient, r) + close(dmSet) + return r, nil +} + +// Performs the following sanity checks: +// - Verifies that the bootstrap configuration is not empty. +// - Verifies that if xDS credentials are specified by the user, the +// bootstrap configuration contains certificate providers. +// - Verifies that if the provided dial target contains an authority, the +// bootstrap configuration contains server config for that authority. +// +// Returns the listener resource name template to use. If any of the above +// validations fail, a non-nil error is returned. +func sanityChecksOnBootstrapConfig(target resolver.Target, client xdsclient.XDSClient) (string, error) { + bootstrapConfig := client.BootstrapConfig() + if bootstrapConfig == nil { + // This is never expected to happen after a successful xDS client + // creation. Defensive programming. + return "", fmt.Errorf("xds: bootstrap configuration is empty") + } + + // Find the client listener template to use from the bootstrap config: + // - If authority is not set in the target, use the top level template + // - If authority is set, use the template from the authority map. + template := bootstrapConfig.ClientDefaultListenerResourceNameTemplate() + if authority := target.URL.Host; authority != "" { + authorities := bootstrapConfig.Authorities() + if authorities == nil { + return "", fmt.Errorf("xds: authority %q specified in dial target %q is not found in the bootstrap file", authority, target) + } + a := authorities[authority] + if a == nil { + return "", fmt.Errorf("xds: authority %q specified in dial target %q is not found in the bootstrap file", authority, target) + } + if a.ClientListenerResourceNameTemplate != "" { + // This check will never be false, because + // ClientListenerResourceNameTemplate is required to start with + // xdstp://, and has a default value (not an empty string) if unset. + template = a.ClientListenerResourceNameTemplate + } + } + return template, nil +} + +// Name helps implement the resolver.Builder interface. +func (*xdsResolverBuilder) Scheme() string { + return Scheme +} + +// xdsResolver implements the resolver.Resolver interface. +// +// It manages the dependency manager which in turn manages all xDS resource +// watches. It receives the xDS resource config and passes them to ClientConn. +type xdsResolver struct { + // The following fields are initialized at creation time and are read-only + // after that. + cc resolver.ClientConn + logger *grpclog.PrefixLogger + ldsResourceName string + dm *xdsdepmgr.DependencyManager + xdsClient xdsclient.XDSClient + xdsClientClose func() + channelID uint64 // Unique random ID for the channel owning this resolver. + target string + metricsRecorder estats.MetricsRecorder + + // All methods on the xdsResolver type except for the ones invoked by gRPC, + // i.e ResolveNow() and Close(), are guaranteed to execute in the context of + // this serializer's callback. We use the serializer because these shared + // states are accessed by each RPC when it is committed, and so + // serializer is preffered over a mutex. + serializer *grpcsync.CallbackSerializer + serializerCancel context.CancelFunc + + // The following fields are accessed only from within the serializer + // callbacks. + xdsConfig *xdsresource.XDSConfig + // activeClusters is a map from cluster name to information about the + // weighted cluster that includes a reference count and load balancing + // configuration. These counts are used only by the resolver. The current + // configSelector holds one reference, and each ongoing RPC holds an + // additional reference. When the count hits zero, the resolver removes the + // cluster from this map and calls unsubscribe. This signals the dependency + // manager to stop the xDS watch once its own reference count reaches zero. + activeClusters map[string]*clusterInfo + // activePlugins is a map from cluster specifier plugin name to information + // about the cluster specifier plugin that includes a ref count and load + // balancing configuration. These counts are used only by the resolver. The + // current configSelector holds one reference, and each ongoing RPC holds an + // additional reference. When the count hits zero, the resolver removes the + // plugin name from this map. + activePlugins map[string]*clusterInfo + curConfigSelector stoppableConfigSelector + // httpFilters is a map from client filter key to client filter instance. It + // lives here so that the resolver can reuse filter instances across config + // updates when the same filter is specified, and to be able to clean up + // filter instances that are no longer used. + httpFilters map[clientFilterKey]httpfilter.ClientFilter +} + +// ResolveNow calls RequestDNSReresolution on the dependency manager. +func (r *xdsResolver) ResolveNow(opts resolver.ResolveNowOptions) { + if r.dm != nil { + r.dm.RequestDNSReresolution(opts) + } +} + +func (r *xdsResolver) Close() { + // Cancel the context passed to the serializer and wait for any scheduled + // callbacks to complete. Canceling the context ensures that no new + // callbacks will be scheduled. + r.serializerCancel() + <-r.serializer.Done() + + if r.dm != nil { + r.dm.Close() + } + if r.curConfigSelector != nil { + r.curConfigSelector.stop() + } + for _, cf := range r.httpFilters { + cf.Close() + } + // Release the xDS client only after the filters are torn down: filters + // may hold side channels whose credentials are owned by the client's + // bootstrap config and are released when the last reference to the + // client is dropped. + if r.xdsClientClose != nil { + r.xdsClientClose() + } + // Drop field references so the resolver isn't retained across ClientConn + // close via any cross-object retention chain (e.g. ClientConn holding the + // ServiceConfig that captures the configSelector, shared xDS-client + // watcher metadata, channelz trace entries). Each nil individually looks + // harmless; the combination breaks the multi-path retention observed when + // callers repeatedly Close and redial DirectPath channels. + r.cc = nil + r.curConfigSelector = nil + r.dm = nil + r.xdsClient = nil + r.xdsClientClose = nil + r.httpFilters = nil + r.activeClusters = nil + r.activePlugins = nil + r.xdsConfig = nil + r.logger.Infof("Shutdown") +} + +// Update is called when there is a new xDS config available from the dependency +// manager and does the following: +// - creates a new config selector (this involves incrementing references to +// clusters owned by this config selector). +// - stops the old config selector (this involves decrementing references to +// clusters owned by this config selector). +// - prunes active clusters and pushes a new service config to the channel. +// - updates the current config selector used by the resolver. +func (r *xdsResolver) Update(config *xdsresource.XDSConfig) { + r.serializer.TrySchedule(func(context.Context) { + r.xdsConfig = config + cs, err := r.newConfigSelector() + if err != nil { + r.onResourceError(err) + return + } + r.sendNewServiceConfig(cs) + + if r.curConfigSelector != nil { + r.curConfigSelector.stop() + } + r.curConfigSelector = cs + }) +} + +func (r *xdsResolver) Error(err error) { + r.serializer.TrySchedule(func(context.Context) { + r.onResourceError(err) + }) +} + +// sendNewServiceConfig prunes active clusters, generates a new service config +// based on the current set of active clusters, and sends an update to the +// channel with that service config and the provided config selector. It is safe +// to ignore the error from `cc.UpdateState` because the clientconn uses the +// given configSelector even if it returns an error. +// +// Only executed in the context of a serializer callback. +func (r *xdsResolver) sendNewServiceConfig(cs stoppableConfigSelector) { + // Delete entries from r.activeClusters with zero references; + // otherwise serviceConfigJSON will generate a config including + // them. + r.pruneActiveClustersAndPlugins() + + if errCS, ok := cs.(*erroringConfigSelector); ok { + // Send an empty config, which picks pick-first, with no address, and + // puts the ClientConn into transient failure. + // + // This call to UpdateState is expected to return ErrBadResolverState + // since pick_first doesn't like an update with no addresses. + r.cc.UpdateState(resolver.State{ServiceConfig: r.cc.ParseServiceConfig("{}")}) + + // Send a resolver error to pick_first so that RPCs will fail with a + // more meaningful error, as opposed to one that says that pick_first + // received no addresses. + r.cc.ReportError(errCS.err) + return + } + + sc := serviceConfigJSON(r.activeClusters, r.activePlugins) + if r.logger.V(2) { + r.logger.Infof("For Listener resource %q and RouteConfiguration resource %q, generated service config: %s", r.ldsResourceName, r.xdsConfig.Listener.APIListener.RouteConfigName, string(sc)) + } + + // Send the update to the ClientConn. + state := iresolver.SetConfigSelector(resolver.State{ + ServiceConfig: r.cc.ParseServiceConfig(string(sc)), + }, cs) + state = xdsresource.SetXDSConfig(state, r.xdsConfig) + state = xdsdepmgr.SetXDSClusterSubscriber(state, r.dm) + r.cc.UpdateState(xdsclient.SetClient(state, r.xdsClient)) +} + +// newConfigSelector creates a new config selector using the most recently +// received listener and route config updates. May add entries to +// r.activeClusters for previously-unseen clusters. +// +// Only executed in the context of a serializer callback. +func (r *xdsResolver) newConfigSelector() (_ *configSelector, err error) { + cs := &configSelector{ + channelID: r.channelID, + xdsNodeID: r.xdsClient.BootstrapConfig().Node().GetId(), + sendNewServiceConfig: func() { + r.serializer.TrySchedule(func(context.Context) { + r.sendNewServiceConfig(r.curConfigSelector) + }) + }, + virtualHost: virtualHost{ + retryConfig: r.xdsConfig.VirtualHost.RetryConfig, + }, + routes: make([]route, len(r.xdsConfig.VirtualHost.Routes)), + clusters: make(map[string]*clusterInfo), + plugins: make(map[string]*clusterInfo), + httpFilterConfig: r.xdsConfig.Listener.APIListener.HTTPFilters, + xdsConfig: r.xdsConfig, + } + + defer func() { + if err != nil { + // Stop the config selector if an error occurs during construction + // to ensure that interceptors that were created successfully before + // the error are cleaned up. + cs.stop() + } + }() + + for i, rt := range r.xdsConfig.VirtualHost.Routes { + clusters := rinternal.NewWRR.(func() wrr.WRR)() + interceptors := []httpfilter.ClientInterceptor{} + // TODO: Carve out the common logic between the ClusterSpecifierPlugin + // and WeightedClusters. + if rt.ClusterSpecifierPlugin != "" { + clusterName := clusterSpecifierPluginPrefix + rt.ClusterSpecifierPlugin + interceptor, err := r.newInterceptor(r.xdsConfig.Listener.APIListener.HTTPFilters, nil, rt.HTTPFilterConfigOverride, r.xdsConfig.VirtualHost.HTTPFilterConfigOverride) + if err != nil { + // Clean up any interceptors that were successfully built + // for the current route before this error occurred. Note + // that this is not handled by the call to cs.stop() in the + // deferred function. + for _, i := range interceptors { + i.Close() + } + return nil, err + } + routeCluster := &routeCluster{ + name: clusterName, + interceptor: interceptor, + } + rc := grpcsync.NewRefCounted(routeCluster, func() { interceptor.Close() }) + cs.routes[i].routeClusters = append(cs.routes[i].routeClusters, rc) + clusters.Add(rc, 1) + ci := r.addOrGetActiveClusterInfo(clusterName, "") + ci.cfg = xdsChildConfig{ChildPolicy: balancerConfig(r.xdsConfig.RouteConfig.ClusterSpecifierPlugins[rt.ClusterSpecifierPlugin])} + cs.plugins[clusterName] = ci + } else { + for _, wc := range rt.WeightedClusters { + clusterName := clusterPrefix + wc.Name + interceptor, err := r.newInterceptor(r.xdsConfig.Listener.APIListener.HTTPFilters, wc.HTTPFilterConfigOverride, rt.HTTPFilterConfigOverride, r.xdsConfig.VirtualHost.HTTPFilterConfigOverride) + if err != nil { + // Clean up any interceptors that were successfully built + // for the current route before this error occurred. Note + // that this is not handled by the call to cs.stop() in the + // deferred function. + for _, i := range interceptors { + i.Close() + } + return nil, err + } + routeCluster := &routeCluster{ + name: clusterName, + interceptor: interceptor, + } + rc := grpcsync.NewRefCounted(routeCluster, func() { interceptor.Close() }) + cs.routes[i].routeClusters = append(cs.routes[i].routeClusters, rc) + clusters.Add(rc, int64(wc.Weight)) + ci := r.addOrGetActiveClusterInfo(clusterName, wc.Name) + ci.cfg = xdsChildConfig{ChildPolicy: newBalancerConfig(cdsName, cdsBalancerConfig{Cluster: wc.Name})} + cs.clusters[clusterName] = ci + } + } + cs.routes[i].clusters = clusters + cs.routes[i].m = xdsresource.RouteToMatcher(rt) + cs.routes[i].actionType = rt.ActionType + if rt.MaxStreamDuration == nil { + cs.routes[i].maxStreamDuration = r.xdsConfig.Listener.APIListener.MaxStreamDuration + } else { + cs.routes[i].maxStreamDuration = *rt.MaxStreamDuration + } + + cs.routes[i].retryConfig = rt.RetryConfig + cs.routes[i].hashPolicies = rt.HashPolicies + cs.routes[i].autoHostRewrite = rt.AutoHostRewrite + } + + // Account for this config selector's clusters. Do this after no further + // errors may occur. Note: cs.clusters are pointers to entries in + // activeClusters. + for _, ci := range cs.clusters { + ci.refCount.Add(1) + } + for _, ci := range cs.plugins { + ci.refCount.Add(1) + } + + // Cleanup filter instances that are no longer specified in the current + // listener resource. + filtersInNewConfig := make(map[clientFilterKey]bool) + for _, filter := range r.xdsConfig.Listener.APIListener.HTTPFilters { + filtersInNewConfig[newClientFilterKey(&filter)] = true + } + for key, cf := range r.httpFilters { + if _, ok := filtersInNewConfig[key]; ok { + continue + } + cf.Close() + delete(r.httpFilters, key) + } + + return cs, nil +} + +// pruneActiveClustersAndPlugins removes entries from activeClusters and +// activePlugins that have a reference count of zero. For clusters, it also +// invokes the unsubscribe function to signal the dependency manager to stop the +// xDS watch. Because cluster specifier plugins do not have their own watches, +// they are simply removed from the map without an unsubscribe call. +// +// Only executed in the context of a serializer callback. +func (r *xdsResolver) pruneActiveClustersAndPlugins() { + for cluster, ci := range r.activeClusters { + if ci.refCount.Load() == 0 { + ci.unsubscribe() + delete(r.activeClusters, cluster) + } + } + for cluster, ci := range r.activePlugins { + if ci.refCount.Load() == 0 { + delete(r.activePlugins, cluster) + } + } +} + +// addOrGetActiveClusterInfo returns the clusterInfo for the provided key, +// creating it if it does not exist. It accepts the following parameters: +// - key: Formatted as "cluster:" or "cluster_specifier_plugin:", +// this is the lookup key for the activeClusters or activePlugins maps. +// - name: The actual xDS resource name used to initiate a CDS watch. +// If empty (e.g., for plugins), no resource watch is triggered. +// +// This function manages entry creation and xDS subscriptions but does not +// increment the reference count of the returned clusterInfo. +func (r *xdsResolver) addOrGetActiveClusterInfo(key string, name string) *clusterInfo { + if name == "" { + ci, ok := r.activePlugins[key] + if !ok { + ci = &clusterInfo{} + r.activePlugins[key] = ci + } + return ci + } + ci, ok := r.activeClusters[key] + if !ok { + ci = &clusterInfo{unsubscribe: r.dm.SubscribeToCluster(name)} + r.activeClusters[key] = ci + } + return ci +} + +type clusterInfo struct { + // refCount is the number of references to this cluster. + refCount atomic.Int32 + // cfg is the child configuration for this cluster, containing either the + // csp config or the cds cluster config. + cfg xdsChildConfig + // unsubscribe is the function to call to unsubscribe from this cluster's + // CDS resource. It is populated only for clusters in activeClusters and not + // for cluster specifier plugins. When invoked, it decrements the reference + // count in the dependency manager; once that count reaches zero, the + // underlying CDS watch is terminated. Plugins do not have associated + // watches and therefore do not require an unsubscribe function. + unsubscribe func() +} + +// Contains common functionality to be executed when resources of either type +// are removed. +// +// Only executed in the context of a serializer callback. +func (r *xdsResolver) onResourceError(err error) { + // We cannot remove clusters from the service config that have ongoing RPCs. + // Instead, what we can do is to send an erroring config selector + // along with normal service config. This will ensure that new RPCs will + // fail, and once the active RPCs complete, the reference counts on the + // clusters will come down to zero. At that point, we will send an empty + // service config with no addresses. This results in the pick-first + // LB policy being configured on the channel, and since there are no + // address, pick-first will put the channel in TRANSIENT_FAILURE. + cs := newErroringConfigSelector(err, r.xdsClient.BootstrapConfig().Node().GetId()) + r.sendNewServiceConfig(cs) + + // Stop and dereference the active config selector, if one exists. + if r.curConfigSelector != nil { + r.curConfigSelector.stop() + } + r.curConfigSelector = cs +} + +// newInterceptor builds a chain of client interceptors for the given filters +// and override configuration. The cluster override has the highest priority, +// followed by the route override, and finally the virtual host override. +// +// Only executed in the context of a serializer callback. +func (r *xdsResolver) newInterceptor(filters []xdsresource.HTTPFilter, clusterOverride, routeOverride, virtualHostOverride map[string]httpfilter.FilterConfig) (_ httpfilter.ClientInterceptor, err error) { + interceptors := make([]httpfilter.ClientInterceptor, 0, len(filters)) + defer func() { + // Clean up any interceptors that were successfully built before the + // error occurred, to avoid leaking resources. + if err != nil { + for _, i := range interceptors { + i.Close() + } + } + }() + for _, filter := range filters { + override := clusterOverride[filter.Name] + if override == nil { + override = routeOverride[filter.Name] + } + if override == nil { + override = virtualHostOverride[filter.Name] + } + + // Determine the effective disabled state of the filter. The base + // configuration's disabled state is used unless an override is present. + // If an override is present, the filter is disabled if the override is + // a DisabledFilterConfig. + disabled := filter.Disabled + if override != nil { + _, disabled = override.(httpfilter.DisabledFilterConfig) + } + + if disabled { + if r.logger.V(2) { + r.logger.Infof("Filter %q has been disabled.", filter.Name) + } + continue + } + + builder, ok := filter.Filter.(httpfilter.ClientFilterBuilder) + if !ok { + // Should not happen if it passed xdsClient validation. + return nil, fmt.Errorf("filter %q does not support use in client", filter.Name) + } + + clientFilter := r.getOrCreateClientFilter(builder, newClientFilterKey(&filter)) + i, err := clientFilter.BuildClientInterceptor(filter.Config, override) + if err != nil { + return nil, fmt.Errorf("failed to build client interceptor for filter %q: %v", filter.Name, err) + } + if i != nil { + interceptors = append(interceptors, i) + } + } + + return &interceptorList{interceptors: interceptors}, nil +} + +// interceptorList is a client interceptor that contains a list of client +// interceptors to execute in order. +type interceptorList struct { + interceptors []httpfilter.ClientInterceptor +} + +func (il *interceptorList) NewStream(ctx context.Context, ri iresolver.RPCInfo, newStream func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) { + for idx := len(il.interceptors) - 1; idx >= 0; idx-- { + ns := newStream + i := il.interceptors[idx] + newStream = func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error) { + return i.NewStream(ctx, ri, ns, opts...) + } + } + return newStream(ctx, opts...) +} + +func (il *interceptorList) Close() { + for _, i := range il.interceptors { + i.Close() + } +} + +// getOrCreateClientFilter retrieves an existing client filter from the +// httpFilters map or creates a new one if it doesn't exist. It uses the filter +// builder to create a new client filter and stores it in the httpFilters map +// for future use. +// +// Only executed in the context of a serializer callback. +func (r *xdsResolver) getOrCreateClientFilter(builder httpfilter.ClientFilterBuilder, key clientFilterKey) httpfilter.ClientFilter { + clientFilter, ok := r.httpFilters[key] + if ok { + return clientFilter + } + + cf := builder.BuildClientFilter(httpfilter.ClientFilterOptions{ + FilterName: key.name, + MetricsRecorder: r.metricsRecorder, + Target: r.target, + }) + r.httpFilters[key] = cf + return cf +} + +// newClientFilterKey generates a key for the given filter using the filter name +// and type URLs. This is used for storing ClientFilters in a map. +func newClientFilterKey(f *xdsresource.HTTPFilter) clientFilterKey { + typeURLs := slices.Clone(f.Filter.TypeURLs()) + slices.Sort(typeURLs) + return clientFilterKey{ + name: f.Name, + typeURLs: strings.Join(typeURLs, ":"), + } +} + +func (f *clientFilterKey) String() string { + return f.name + ":" + f.typeURLs +} + +type clientFilterKey struct { + name string + typeURLs string +} diff --git a/internal/xds/resolver/xds_resolver_test.go b/internal/xds/resolver/xds_resolver_test.go new file mode 100644 index 0000000..b8ed304 --- /dev/null +++ b/internal/xds/resolver/xds_resolver_test.go @@ -0,0 +1,1511 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver_test + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strings" + "sync" + "testing" + "time" + + xxhash "github.com/cespare/xxhash/v2" + "github.com/envoyproxy/go-control-plane/pkg/wellknown" + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc/codes" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + iresolver "google.golang.org/grpc/internal/resolver" + iringhash "google.golang.org/grpc/internal/ringhash" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/balancer/clusterimpl" + "google.golang.org/grpc/internal/xds/balancer/clustermanager" + "google.golang.org/grpc/internal/xds/bootstrap" + serverFeature "google.golang.org/grpc/internal/xds/clients/xdsclient" + rinternal "google.golang.org/grpc/internal/xds/resolver/internal" + xdstestutils "google.golang.org/grpc/internal/xds/test/e2e" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + + _ "google.golang.org/grpc/internal/xds/balancer/cdsbalancer" // Register the cds LB policy + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter +) + +// Tests the case where xDS client creation is expected to fail because the +// bootstrap configuration for the xDS client pool is not specified. The test +// verifies that xDS resolver build fails as well. +func (s) TestResolverBuilder_ClientCreationFails_NoBootstrap(t *testing.T) { + // Build an xDS resolver specifying nil for bootstrap configuration for the + // xDS client pool. + pool := xdsclient.NewPool(nil) + var xdsResolver resolver.Builder + if newResolver := internal.NewXDSResolverWithPoolForTesting; newResolver != nil { + var err error + xdsResolver, err = newResolver.(func(*xdsclient.Pool) (resolver.Builder, error))(pool) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } + + target := resolver.Target{URL: *testutils.MustParseURL("xds:///target")} + if _, err := xdsResolver.Build(target, nil, resolver.BuildOptions{}); err == nil { + t.Fatalf("xds Resolver Build(%v) succeeded when expected to fail, because there is no bootstrap configuration for the xDS client pool", pool) + } +} + +// Tests the case where the specified dial target contains an authority that is +// not specified in the bootstrap file. Verifies that the resolver.Build method +// fails with the expected error string. +func (s) TestResolverBuilder_AuthorityNotDefinedInBootstrap(t *testing.T) { + contents := e2e.DefaultBootstrapContents(t, "node-id", "dummy-management-server") + + // Create an xDS resolver with the above bootstrap configuration. + xdsResolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(contents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + target := resolver.Target{URL: *testutils.MustParseURL("xds://non-existing-authority/target")} + const wantErr = `authority "non-existing-authority" specified in dial target "xds://non-existing-authority/target" is not found in the bootstrap file` + r, err := xdsResolver.Build(target, &testutils.ResolverClientConn{Logger: t}, resolver.BuildOptions{}) + if r != nil { + r.Close() + } + if err == nil { + t.Fatalf("xds Resolver Build(%v) succeeded for target with authority not specified in bootstrap", target) + } + if !strings.Contains(err.Error(), wantErr) { + t.Fatalf("xds Resolver Build(%v) returned err: %v, wantErr: %v", target, err, wantErr) + } +} + +// Test builds an xDS resolver and verifies that the resource name specified in +// the discovery request matches expectations. +func (s) TestResolverResourceName(t *testing.T) { + tests := []struct { + name string + listenerResourceNameTemplate string + extraAuthority string + dialTarget string + wantResourceNames []string + }{ + { + name: "default %s old style", + listenerResourceNameTemplate: "%s", + dialTarget: "xds:///target", + wantResourceNames: []string{"target"}, + }, + { + name: "old style no percent encoding", + listenerResourceNameTemplate: "/path/to/%s", + dialTarget: "xds:///target", + wantResourceNames: []string{"/path/to/target"}, + }, + { + name: "new style with %s", + listenerResourceNameTemplate: "xdstp://authority.com/%s", + dialTarget: "xds:///0.0.0.0:8080", + wantResourceNames: []string{"xdstp://authority.com/0.0.0.0:8080"}, + }, + { + name: "new style percent encoding", + listenerResourceNameTemplate: "xdstp://authority.com/%s", + dialTarget: "xds:///[::1]:8080", + wantResourceNames: []string{"xdstp://authority.com/%5B::1%5D:8080"}, + }, + { + name: "new style different authority", + listenerResourceNameTemplate: "xdstp://authority.com/%s", + extraAuthority: "test-authority", + dialTarget: "xds://test-authority/target", + wantResourceNames: []string{"xdstp://test-authority/envoy.config.listener.v3.Listener/target"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, lisCh, _, _ := setupManagementServerForTest(t, nodeID) + + // Create a bootstrap configuration with test options. + opts := bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + ClientDefaultListenerResourceNameTemplate: tt.listenerResourceNameTemplate, + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + } + if tt.extraAuthority != "" { + // In this test, we really don't care about having multiple + // management servers. All we need to verify is whether the + // resource name matches expectation. + opts.Authorities = map[string]json.RawMessage{ + tt.extraAuthority: []byte(fmt.Sprintf(`{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }`, mgmtServer.Address)), + } + } + contents, err := bootstrap.NewContentsForTesting(opts) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL(tt.dialTarget)}, contents) + waitForResourceNames(ctx, t, lisCh, tt.wantResourceNames) + }) + } +} + +// Tests the case where a service update from the underlying xDS client is +// received after the resolver is closed, and verifies that the update is not +// propagated to the ClientConn. +func (s) TestResolverWatchCallbackAfterClose(t *testing.T) { + // Setup the management server that synchronizes with the test goroutine + // using two channels. The management server signals the test goroutine when + // it receives a discovery request for a route configuration resource. And + // the test goroutine signals the management server when the resolver is + // closed. + routeConfigResourceNamesCh := make(chan []string, 1) + waitForResolverCloseCh := make(chan struct{}) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3RouteConfigURL { + select { + case <-routeConfigResourceNamesCh: + default: + } + select { + case routeConfigResourceNamesCh <- req.GetResourceNames(): + default: + } + <-waitForResolverCloseCh + } + return nil + }, + }) + + // Create a bootstrap configuration specifying the above management server. + nodeID := uuid.New().String() + contents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Configure resources on the management server. + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)} + routes := []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(defaultTestRouteConfigName, defaultTestServiceName, defaultTestClusterName)} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + configureResources(ctx, t, mgmtServer, nodeID, listeners, routes, nil, nil) + + // Wait for a discovery request for a route configuration resource. + stateCh, _, r := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, contents) + waitForResourceNames(ctx, t, routeConfigResourceNamesCh, []string{defaultTestRouteConfigName}) + + // Close the resolver and unblock the management server. + r.Close() + close(waitForResolverCloseCh) + + // Verify that the update from the management server is not propagated to + // the ClientConn. The xDS resolver, once closed, is expected to drop + // updates from the xDS client. + verifyNoUpdateFromResolver(ctx, t, stateCh) +} + +// Tests that the xDS resolver's Close method closes the xDS client. +func (s) TestResolverCloseClosesXDSClient(t *testing.T) { + // Override xDS client creation to use bootstrap configuration pointing to a + // dummy management server. Also close a channel when the returned xDS + // client is closed. + origNewClient := rinternal.NewXDSClient + closeCh := make(chan struct{}) + rinternal.NewXDSClient = func(string, estats.MetricsRecorder) (xdsclient.XDSClient, func(), error) { + bc := e2e.DefaultBootstrapContents(t, uuid.New().String(), "dummy-management-server-address") + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + c, cancel, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestTimeout, + }) + return c, sync.OnceFunc(func() { + close(closeCh) + cancel() + }), err + } + defer func() { rinternal.NewXDSClient = origNewClient }() + + _, _, r := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///my-service-client-side-xds")}, nil) + r.Close() + + select { + case <-closeCh: + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout when waiting for xDS client to be closed") + } +} + +// Tests the case where there is no virtual host in the route configuration +// matches the dataplane authority. Verifies that the resolver returns the +// correct error. +func (s) TestNoMatchingVirtualHost(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + // Configure route resource with no virtual host so that it does not match the authority. + listener := e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName) + route := e2e.DefaultRouteConfig(defaultTestRouteConfigName, defaultTestServiceName, defaultTestClusterName) + route.VirtualHosts = nil + configureResources(ctx, t, mgmtServer, nodeID, []*v3listenerpb.Listener{listener}, []*v3routepb.RouteConfiguration{route}, nil, nil) + + // Build the resolver inline (duplicating buildResolverForTarget internals) + // to avoid issues with blocked channel writes when NACKs occur. + target := resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)} + + // Create an xDS resolver with the provided bootstrap configuration. + builder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + errCh := testutils.NewChannel() + tcc := &testutils.ResolverClientConn{Logger: t, ReportErrorF: func(err error) { errCh.Replace(err) }} + r, err := builder.Build(target, tcc, resolver.BuildOptions{ + Authority: url.PathEscape(target.Endpoint()), + }) + if err != nil { + t.Fatalf("Failed to build xDS resolver for target %q: %v", target, err) + } + defer r.Close() + + // Wait for and verify the error update from the resolver. + // Since the resource is not cached, it should be received as resource error. + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for error to be propagated to the ClientConn") + case gotErr := <-errCh.C: + if gotErr == nil { + t.Fatalf("got nil error from resolver, want error containing 'could not find VirtualHost'") + } + errStr := fmt.Sprint(gotErr) + if !strings.Contains(errStr, fmt.Sprintf("could not find VirtualHost for %q", defaultTestServiceName)) { + t.Fatalf("got error from resolver %q, want error containing 'could not find VirtualHost for %q'", errStr, defaultTestServiceName) + } + if !strings.Contains(errStr, nodeID) { + t.Fatalf("got error from resolver %q, want nodeID %q", errStr, nodeID) + } + } +} + +// Tests the case where a resource, not present in cache, returned by the +// management server is NACKed by the xDS client, which then returns an update +// containing a resource error to the resolver. It tests the case where the +// resolver gets an error update without any previous good update. The test +// also verifies that these are propagated to the ClientConn. +func (s) TestResolverBadServiceUpdate_NACKedWithoutCache(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + // Configure a listener resource that is expected to be NACKed because it + // does not contain the `RouteSpecifier` field in the HTTPConnectionManager. + hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, + }) + lis := &v3listenerpb.Listener{ + Name: defaultTestServiceName, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + } + configureResources(ctx, t, mgmtServer, nodeID, []*v3listenerpb.Listener{lis}, nil, nil, nil) + + // Build the resolver inline (duplicating buildResolverForTarget internals) + // to avoid issues with blocked channel writes when NACKs occur. + target := resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)} + + // Create an xDS resolver with the provided bootstrap configuration. + builder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + errCh := testutils.NewChannel() + tcc := &testutils.ResolverClientConn{Logger: t, ReportErrorF: func(err error) { errCh.Replace(err) }} + r, err := builder.Build(target, tcc, resolver.BuildOptions{ + Authority: url.PathEscape(target.Endpoint()), + }) + if err != nil { + t.Fatalf("Failed to build xDS resolver for target %q: %v", target, err) + } + defer r.Close() + + // Wait for and verify the error update from the resolver. + // Since the resource is not cached, it should be received as resource error. + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for error to be propagated to the ClientConn") + case gotErr := <-errCh.C: + if gotErr == nil { + t.Fatalf("got nil error from resolver, want error containing 'no RouteSpecifier'") + } + errStr := fmt.Sprint(gotErr) + if !strings.Contains(errStr, "no RouteSpecifier") { + t.Fatalf("got error from resolver %q, want error containing 'no RouteSpecifier'", errStr) + } + if !strings.Contains(errStr, nodeID) { + t.Fatalf("got error from resolver %q, want nodeID %q", errStr, nodeID) + } + } +} + +// Tests the case where a resource, present in cache, returned by the management +// server is NACKed by the xDS client, which then returns an update containing +// an ambient error to the resolver. It tests the case where the resolver gets a +// good update first, and an error after the good update. The test verifies that +// the RPCs succeeds as expected after receiving good update as well as ambient +// error. +func (s) TestResolverBadServiceUpdate_NACKedWithCache(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + // Configure all resources on the management server. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + }) + mgmtServer.Update(ctx, resources) + + // Expect a good update from the resolver. + cs := verifyUpdateFromResolver(ctx, t, stateCh, wantServiceConfig(resources.Clusters[0].Name)) + + // "Make an RPC" by invoking the config selector. + _, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + // Configure a listener resource that is expected to be NACKed because it + // does not contain the `RouteSpecifier` field in the HTTPConnectionManager. + hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, + }) + lis := &v3listenerpb.Listener{ + Name: defaultTestServiceName, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + } + + // Since the resource is cached, it should be received as an ambient error + // and so the RPCs should continue passing. + configureResources(ctx, t, mgmtServer, nodeID, []*v3listenerpb.Listener{lis}, nil, nil, nil) + + // "Make an RPC" by invoking the config selector which should succeed by + // continuing to use the previously cached resource. + _, err = cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } +} + +// TestResolverGoodServiceUpdate tests the case where the resource returned by +// the management server is ACKed by the xDS client, which then returns a good +// service update to the resolver. The test verifies that the service config +// returned by the resolver matches expectations, and that the config selector +// returned by the resolver picks clusters based on the route configuration +// received from the management server. +func (s) TestResolverGoodServiceUpdate(t *testing.T) { + for _, tt := range []struct { + name string + routeConfig *v3routepb.RouteConfiguration + clusterConfig []*v3clusterpb.Cluster + endpointConfig []*v3endpointpb.ClusterLoadAssignment + wantServiceConfig string + wantClusters map[string]bool + }{ + { + name: "single cluster", + routeConfig: e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: defaultTestRouteConfigName, + ListenerName: defaultTestServiceName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeCluster, + ClusterName: defaultTestClusterName, + }), + clusterConfig: []*v3clusterpb.Cluster{e2e.DefaultCluster(defaultTestClusterName, defaultTestEndpointName, e2e.SecurityLevelNone)}, + endpointConfig: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(defaultTestEndpointName, defaultTestHostname, defaultTestPort)}, + wantServiceConfig: wantServiceConfig(defaultTestClusterName), + wantClusters: map[string]bool{fmt.Sprintf("cluster:%s", defaultTestClusterName): true}, + }, + { + name: "two clusters", + routeConfig: e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: defaultTestRouteConfigName, + ListenerName: defaultTestServiceName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeWeightedCluster, + WeightedClusters: map[string]int{"cluster_1": 75, "cluster_2": 25}, + }), + clusterConfig: []*v3clusterpb.Cluster{ + e2e.DefaultCluster("cluster_1", "endpoint_1", e2e.SecurityLevelNone), + e2e.DefaultCluster("cluster_2", "endpoint_2", e2e.SecurityLevelNone)}, + endpointConfig: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint("endpoint_1", defaultTestHostname, defaultTestPort), + e2e.DefaultEndpoint("endpoint_2", defaultTestHostname, defaultTestPort)}, + // This update contains the cluster from the previous update as well + // as this update, as the previous config selector still references + // the old cluster when the new one is pushed. + wantServiceConfig: `{ + "loadBalancingConfig": [{ + "xds_cluster_manager_experimental": { + "children": { + "cluster:cluster_1": { + "childPolicy": [{ + "cds_experimental": { + "cluster": "cluster_1" + } + }] + }, + "cluster:cluster_2": { + "childPolicy": [{ + "cds_experimental": { + "cluster": "cluster_2" + } + }] + } + } + } + }]}`, + wantClusters: map[string]bool{"cluster:cluster_1": true, "cluster:cluster_2": true}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + // Configure the management server with a good listener resource and a + // route configuration resource, as specified by the test case. + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)} + routes := []*v3routepb.RouteConfiguration{tt.routeConfig} + configureResources(ctx, t, mgmtServer, nodeID, listeners, routes, tt.clusterConfig, tt.endpointConfig) + + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + // Read the update pushed by the resolver to the ClientConn. + cs := verifyUpdateFromResolver(ctx, t, stateCh, tt.wantServiceConfig) + + pickedClusters := make(map[string]bool) + // Odds of picking 75% cluster 100 times in a row: 1 in 3E-13. And + // with the random number generator stubbed out, we can rely on this + // to be 100% reproducible. + for i := 0; i < 100; i++ { + res, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + cluster := clustermanager.PickedCluster(res.Context) + pickedClusters[cluster] = true + res.OnCommitted() + } + if !cmp.Equal(pickedClusters, tt.wantClusters) { + t.Errorf("Picked clusters: %v; want: %v", pickedClusters, tt.wantClusters) + } + }) + } +} + +// Tests a case where a resolver receives a RouteConfig update with a HashPolicy +// specifying to generate a hash. The configSelector generated should +// successfully generate a Hash. +func (s) TestResolverRequestHash(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + // Configure the management server with a good listener resource and a + // route configuration resource that specifies a hash policy. + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)} + routes := []*v3routepb.RouteConfiguration{{ + Name: defaultTestRouteConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{defaultTestServiceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: defaultTestClusterName, + Weight: &wrapperspb.UInt32Value{Value: 100}, + }, + }, + }}, + HashPolicy: []*v3routepb.RouteAction_HashPolicy{{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: ":path", + }, + }, + Terminal: true, + }}, + }}, + }}, + }}, + }} + cluster := []*v3clusterpb.Cluster{e2e.DefaultCluster(defaultTestClusterName, defaultTestEndpointName, e2e.SecurityLevelNone)} + endpoints := []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(defaultTestEndpointName, defaultTestHostname, defaultTestPort)} + configureResources(ctx, t, mgmtServer, nodeID, listeners, routes, cluster, endpoints) + + // Build the resolver and read the config selector out of it. + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + cs := verifyUpdateFromResolver(ctx, t, stateCh, "") + + // Selecting a config when there was a hash policy specified in the route + // that will be selected should put a request hash in the config's context. + res, err := cs.SelectConfig(iresolver.RPCInfo{ + Context: metadata.NewOutgoingContext(ctx, metadata.Pairs(":path", "/products")), + Method: "/service/method", + }) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + wantHash := xxhash.Sum64String("/products") + gotHash, ok := iringhash.XDSRequestHash(res.Context) + if !ok { + t.Fatalf("Got no request hash, want: %v", wantHash) + } + if gotHash != wantHash { + t.Fatalf("Got request hash: %v, want: %v", gotHash, wantHash) + } +} + +// Tests the case where resources are removed from the management server, +// causing it to send an empty update to the xDS client, which returns a +// resource-not-found error to the xDS resolver. +func (s) TestResolverRemovedWithRPCs(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + // Configure resources on the management server. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: defaultTestServiceName, + NodeID: nodeID, + Host: defaultTestHostname, + Port: defaultTestPort[0], + SecLevel: e2e.SecurityLevelNone, + }) + mgmtServer.Update(ctx, resources) + + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + // Read the update pushed by the resolver to the ClientConn. + cs := verifyUpdateFromResolver(ctx, t, stateCh, wantServiceConfig(resources.Clusters[0].Name)) + + res, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + // Delete the listener resource on the management server. This should result + // in a resource-not-found error from the xDS client. + oldListeners := resources.Listeners + resources.Listeners = nil + resources.SkipValidation = true + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Even though the RPC started earlier is still in progress, the xDS resolver will + // produce an empty service config. + var state resolver.State + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for an update from the resolver: %v", ctx.Err()) + case state = <-stateCh: + if err := state.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", state.ServiceConfig.Err) + } + wantSCParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)("{}") + if !internal.EqualServiceConfigForTesting(state.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, state.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + } + + // "Finish the RPC"; this could cause a panic if the resolver doesn't + // handle it correctly. + res.OnCommitted() + + // Add the resources back. + resources.Listeners = oldListeners + mgmtServer.Update(ctx, resources) + + // The resolver should send a service config with the cluster name. + cs = verifyUpdateFromResolver(ctx, t, stateCh, wantServiceConfig(resources.Clusters[0].Name)) + + // Verify that RPCs pass again. + res, err = cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + res.OnCommitted() +} + +// Tests the case where resources returned by the management server are removed. +// The test verifies that the resolver pushes the expected config selector and +// service config in this case. +func (s) TestResolverRemovedResource(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + // Configure resources on the management server. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: defaultTestServiceName, + NodeID: nodeID, + Host: defaultTestHostname, + Port: defaultTestPort[0], + SecLevel: e2e.SecurityLevelNone, + }) + mgmtServer.Update(ctx, resources) + stateCh, errCh, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + // Read the update pushed by the resolver to the ClientConn. + cs := verifyUpdateFromResolver(ctx, t, stateCh, wantServiceConfig(resources.Clusters[0].Name)) + + // "Make an RPC" by invoking the config selector. + res, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + // "Finish the RPC"; this could cause a panic if the resolver doesn't + // handle it correctly. + res.OnCommitted() + + // Delete the listener resource on the management server, resulting in a + // resource-not-found error from the xDS client. + resources.Listeners = nil + resources.SkipValidation = true + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // An empty ServiceConfig update should have been sent. + var state resolver.State + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for an update from the resolver: %v", ctx.Err()) + case state = <-stateCh: + if err := state.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", state.ServiceConfig.Err) + } + wantSCParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)("{}") + if !internal.EqualServiceConfigForTesting(state.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, state.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + } + + // The xDS resolver is expected to report an error to the channel. + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for an error from the resolver: %v", ctx.Err()) + case err := <-errCh: + if err := verifyResolverError(err, codes.Unavailable, "has been removed", nodeID); err != nil { + t.Fatal(err) + } + } +} + +// Tests the case where the resolver receives max stream duration as part of the +// listener and route configuration resources. The test verifies that the RPC +// timeout returned by the config selector matches expectations. A non-nil max +// stream duration (this includes an explicit zero value) in a matching route +// overrides the value specified in the listener resource. +func (s) TestResolverMaxStreamDuration(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + // Configure the management server with a listener resource that specifies a + // max stream duration as part of its HTTP connection manager. Also + // configure a route configuration resource, which has multiple routes with + // different values of max stream duration. + hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: defaultTestRouteConfigName, + }}, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{ + MaxStreamDuration: durationpb.New(1 * time.Second), + }, + }) + listeners := []*v3listenerpb.Listener{{ + Name: defaultTestServiceName, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + }} + routes := []*v3routepb.RouteConfiguration{{ + Name: defaultTestRouteConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{defaultTestServiceName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/foo"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: "A", + Weight: &wrapperspb.UInt32Value{Value: 100}, + }, + }}, + }, + MaxStreamDuration: &v3routepb.RouteAction_MaxStreamDuration{ + MaxStreamDuration: durationpb.New(5 * time.Second), + }, + }}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/bar"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: "B", + Weight: &wrapperspb.UInt32Value{Value: 100}, + }, + }}, + }, + MaxStreamDuration: &v3routepb.RouteAction_MaxStreamDuration{ + MaxStreamDuration: durationpb.New(0 * time.Second), + }, + }}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: "C", + Weight: &wrapperspb.UInt32Value{Value: 100}, + }, + }}, + }, + }}, + }, + }, + }}, + }} + cluster := []*v3clusterpb.Cluster{ + e2e.DefaultCluster("A", "endpoint_A", e2e.SecurityLevelNone), + e2e.DefaultCluster("B", "endpoint_B", e2e.SecurityLevelNone), + e2e.DefaultCluster("C", "endpoint_C", e2e.SecurityLevelNone), + } + endpoints := []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint("endpoint_A", defaultTestHostname, defaultTestPort), + e2e.DefaultEndpoint("endpoint_B", defaultTestHostname, defaultTestPort), + e2e.DefaultEndpoint("endpoint_C", defaultTestHostname, defaultTestPort), + } + configureResources(ctx, t, mgmtServer, nodeID, listeners, routes, cluster, endpoints) + + // Read the update pushed by the resolver to the ClientConn. + cs := verifyUpdateFromResolver(ctx, t, stateCh, "") + + testCases := []struct { + name string + method string + want *time.Duration + }{{ + name: "RDS setting", + method: "/foo/method", + want: newDurationP(5 * time.Second), + }, { + name: "explicit zero in RDS; ignore LDS", + method: "/bar/method", + want: nil, + }, { + name: "no config in RDS; fallback to LDS", + method: "/baz/method", + want: newDurationP(time.Second), + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + req := iresolver.RPCInfo{ + Method: tc.method, + Context: ctx, + } + res, err := cs.SelectConfig(req) + if err != nil { + t.Errorf("cs.SelectConfig(%v): %v", req, err) + return + } + res.OnCommitted() + got := res.MethodConfig.Timeout + if !cmp.Equal(got, tc.want) { + t.Errorf("For method %q: res.MethodConfig.Timeout = %v; want %v", tc.method, got, tc.want) + } + }) + } +} + +// Tests the case where two LDS updates with the same RDS name to watch are +// received without an RDS in between. Those LDS updates shouldn't trigger a +// service config update. +func (s) TestResolverMultipleLDSUpdates(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + // Configure the management server with a listener resource, but no route + // configuration resource. + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)} + configureResources(ctx, t, mgmtServer, nodeID, listeners, nil, nil, nil) + + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + // Ensure there is no update from the resolver. + verifyNoUpdateFromResolver(ctx, t, stateCh) + + // Configure the management server with a listener resource that points to + // the same route configuration resource but has different values for max + // stream duration field. There is still no route configuration resource on + // the management server. + hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: defaultTestRouteConfigName, + }}, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{ + MaxStreamDuration: durationpb.New(1 * time.Second), + }, + }) + listeners = []*v3listenerpb.Listener{{ + Name: defaultTestServiceName, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + }} + configureResources(ctx, t, mgmtServer, nodeID, listeners, nil, nil, nil) + + // Ensure that there is no update from the resolver. + verifyNoUpdateFromResolver(ctx, t, stateCh) +} + +// TestResolverWRR tests the case where the route configuration returned by the +// management server contains a set of weighted clusters. The test performs a +// bunch of RPCs using the cluster specifier returned by the resolver, and +// verifies the cluster distribution. +func (s) TestResolverWRR(t *testing.T) { + origNewWRR := rinternal.NewWRR + rinternal.NewWRR = testutils.NewTestWRR + defer func() { rinternal.NewWRR = origNewWRR }() + + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + // Configure resources on the management server. + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)} + routes := []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: defaultTestRouteConfigName, + ListenerName: defaultTestServiceName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeWeightedCluster, + WeightedClusters: map[string]int{"A": 75, "B": 25}, + })} + clusters := []*v3clusterpb.Cluster{ + e2e.DefaultCluster("A", "endpoint_A", e2e.SecurityLevelNone), + e2e.DefaultCluster("B", "endpoint_B", e2e.SecurityLevelNone), + } + endpoints := []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint("endpoint_A", defaultTestHostname, defaultTestPort), + e2e.DefaultEndpoint("endpoint_B", defaultTestHostname, defaultTestPort), + } + configureResources(ctx, t, mgmtServer, nodeID, listeners, routes, clusters, endpoints) + + // Read the update pushed by the resolver to the ClientConn. + cs := verifyUpdateFromResolver(ctx, t, stateCh, "") + + // Make RPCs to verify WRR behavior in the cluster specifier. + picks := map[string]int{} + for i := 0; i < 100; i++ { + res, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + picks[clustermanager.PickedCluster(res.Context)]++ + res.OnCommitted() + } + want := map[string]int{"cluster:A": 75, "cluster:B": 25} + if !cmp.Equal(picks, want) { + t.Errorf("Picked clusters: %v; want: %v", picks, want) + } +} + +func (s) TestConfigSelector_FailureCases(t *testing.T) { + const methodName = "1" + + tests := []struct { + name string + listener *v3listenerpb.Listener + wantErr string + }{ + { + name: "route type RouteActionUnsupported invalid for client", + listener: &v3listenerpb.Listener{ + Name: defaultTestServiceName, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: defaultTestRouteConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{defaultTestServiceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: methodName}, + }, + Action: &v3routepb.Route_FilterAction{}, + }}, + }}, + }}, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + }), + }, + }, + wantErr: "matched route does not have a supported route action type", + }, + { + name: "route type RouteActionNonForwardingAction invalid for client", + listener: &v3listenerpb.Listener{ + Name: defaultTestServiceName, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: defaultTestRouteConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{defaultTestServiceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: methodName}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}, + }}, + }}, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + }), + }, + }, + wantErr: "matched route does not have a supported route action type", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Spin up an xDS management server. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + // Build an xDS resolver. + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + // Update the management server with a listener resource that + // contains inline route configuration. + configureResources(ctx, t, mgmtServer, nodeID, []*v3listenerpb.Listener{test.listener}, nil, nil, nil) + + // Ensure that the resolver pushes a state update to the channel. + cs := verifyUpdateFromResolver(ctx, t, stateCh, "") + + // Ensure that it returns the expected error. + _, err := cs.SelectConfig(iresolver.RPCInfo{Method: methodName, Context: ctx}) + if err := verifyResolverError(err, codes.Unavailable, test.wantErr, nodeID); err != nil { + t.Fatal(err) + } + }) + } +} + +func newDurationP(d time.Duration) *time.Duration { + return &d +} + +// TestResolver_AutoHostRewrite verifies the propagation of the AutoHostRewrite +// field from the xDS resolver. +// +// Per gRFC A81, this feature should only be active if two conditions met: +// 1. The environment variable (XDSAuthorityRewrite) is enabled. +// 2. The xDS server is marked as "trusted_xds_server" in the bootstrap config. +func (s) TestResolver_AutoHostRewrite(t *testing.T) { + for _, tt := range []struct { + name string + autoHostRewrite bool + envconfig bool + serverfeature serverFeature.ServerFeature + wantAutoHostRewrite bool + }{ + { + name: "EnvVarDisabled_NonTrustedServer_AutoHostRewriteOff", + autoHostRewrite: false, + envconfig: false, + wantAutoHostRewrite: false, + }, + { + name: "EnvVarDisabled_NonTrustedServer_AutoHostRewriteOn", + autoHostRewrite: true, + envconfig: false, + wantAutoHostRewrite: false, + }, + { + name: "EnvVarDisabled_TrustedServer_AutoHostRewriteOff", + autoHostRewrite: false, + envconfig: false, + serverfeature: serverFeature.ServerFeatureTrustedXDSServer, + wantAutoHostRewrite: false, + }, + { + name: "EnvVarDisabled_TrustedServer_AutoHostRewriteOn", + autoHostRewrite: true, + envconfig: false, + serverfeature: serverFeature.ServerFeatureTrustedXDSServer, + wantAutoHostRewrite: false, + }, + { + name: "EnvVarEnabled_NonTrustedServer_AutoHostRewriteOff", + autoHostRewrite: false, + envconfig: true, + wantAutoHostRewrite: false, + }, + { + name: "EnvVarEnabled_NonTrustedServer_AutoHostRewriteOn", + autoHostRewrite: true, + envconfig: true, + wantAutoHostRewrite: false, + }, + { + name: "EnvVarEnabled_TrustedServer_AutoHostRewriteOff", + autoHostRewrite: false, + envconfig: true, + serverfeature: serverFeature.ServerFeatureTrustedXDSServer, + wantAutoHostRewrite: false, + }, + { + name: "EnvVarEnabled_TrustedServer_AutoHostRewriteOn", + autoHostRewrite: true, + envconfig: true, + serverfeature: serverFeature.ServerFeatureTrustedXDSServer, + wantAutoHostRewrite: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSAuthorityRewrite, tt.envconfig) + + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer mgmtServer.Stop() + + // Configure the management server with a good listener resource and a + // route configuration resource, as specified by the test case. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)}, + Routes: []*v3routepb.RouteConfiguration{{ + Name: defaultTestRouteConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{defaultTestServiceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: defaultTestClusterName, + Weight: &wrapperspb.UInt32Value{Value: 100}, + }, + }, + }}, + HostRewriteSpecifier: &v3routepb.RouteAction_AutoHostRewrite{ + AutoHostRewrite: &wrapperspb.BoolValue{Value: tt.autoHostRewrite}, + }, + }}, + }}, + }}, + }}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(defaultTestClusterName, defaultTestEndpointName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(defaultTestEndpointName, "localhost", []uint32{8080})}, + } + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + trustedXdsServer := "[]" + if tt.serverfeature == serverFeature.ServerFeatureTrustedXDSServer { + trustedXdsServer = `["trusted_xds_server"]` + } + + opts := bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}], + "server_features": %s + }]`, mgmtServer.Address, trustedXdsServer)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + } + + contents, err := bootstrap.NewContentsForTesting(opts) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Build the resolver and read the config selector out of it. + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, contents) + cs := verifyUpdateFromResolver(ctx, t, stateCh, "") + + res, err := cs.SelectConfig(iresolver.RPCInfo{ + Context: ctx, + Method: "/service/method", + }) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + gotAutoHostRewrite := clusterimpl.AutoHostRewriteEnabledForTesting(res.Context) + if gotAutoHostRewrite != tt.wantAutoHostRewrite { + t.Fatalf("Got autoHostRewrite: %v, want: %v", gotAutoHostRewrite, tt.wantAutoHostRewrite) + } + }) + } +} + +// TestResolverKeepWatchOpen_ActiveRPCs tests that the dependency manager keeps +// a cluster watch open when there are active RPCs using that cluster, even if +// the cluster is no longer referenced by the current route configuration. +func (s) TestResolverKeepWatchOpen_ActiveRPCs(t *testing.T) { + clusterA := "cluster-A" + clusterB := "cluster-B" + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cdsResourceRequestedCh := make(chan []string, 2) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() != version.V3ClusterURL { + return nil + } + if len(req.GetResourceNames()) > 0 { + select { + case cdsResourceRequestedCh <- req.GetResourceNames(): + case <-ctx.Done(): + } + } + return nil + }, + AllowResourceSubset: true, + }) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Configure initial resources: Route -> ClusterA. + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)} + routes := []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(defaultTestRouteConfigName, defaultTestServiceName, clusterA)} + clusters := []*v3clusterpb.Cluster{ + e2e.DefaultCluster(clusterA, "endpoint-A", e2e.SecurityLevelNone), + e2e.DefaultCluster(clusterB, "endpoint-B", e2e.SecurityLevelNone), + } + endpoints := []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint("endpoint-A", "localhost", []uint32{8080}), + e2e.DefaultEndpoint("endpoint-B", "localhost", []uint32{8081}), + } + + configureResources(ctx, t, mgmtServer, nodeID, listeners, routes, clusters, endpoints) + + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + + cs := verifyUpdateFromResolver(ctx, t, stateCh, wantServiceConfig(clusterA)) + + // Start RPC (Ref Counts ClusterA). + res, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + // Switch Configuration to ClusterB. + routes[0] = e2e.DefaultRouteConfig(defaultTestRouteConfigName, defaultTestServiceName, clusterB) + configureResources(ctx, t, mgmtServer, nodeID, listeners, routes, clusters, endpoints) + + // Resolver should request BOTH A (due to active RPC) and B (due to new + // config). + wantNames := []string{clusterA, clusterB} + waitForResourceNames(ctx, t, cdsResourceRequestedCh, wantNames) + + // Verify Service Config has both clusters. + const wantServiceRaw = `{ + "loadBalancingConfig": [{ + "xds_cluster_manager_experimental": { + "children": { + "cluster:cluster-A": { + "childPolicy": [{"cds_experimental": {"cluster": "cluster-A"}}] + }, + "cluster:cluster-B": { + "childPolicy": [{"cds_experimental": {"cluster": "cluster-B"}}] + } + } + } + }] + }` + verifyUpdateFromResolver(ctx, t, stateCh, wantServiceRaw) + + // Finish RPC (Drops Ref to ClusterA). + res.OnCommitted() + + // ONLY cluster B should be requested now that there are no references to + // cluster A. + wantNames = []string{clusterB} + waitForResourceNames(ctx, t, cdsResourceRequestedCh, wantNames) + + // ServiceConfig update should also contain only cluster B. + cs = verifyUpdateFromResolver(ctx, t, stateCh, wantServiceConfig(clusterB)) + + // Verify that RPCs pass again. + res, err = cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + res.OnCommitted() +} + +// TestResolver_XDSConfigInRPCContext verifies that the xDS resolver's config +// selector places the complete XDSConfig into the RPC context during config +// selection, making it available to HTTP filters. +func (s) TestResolver_XDSConfigInRPCContext(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + defer mgmtServer.Stop() + + // Configure xDS resources on the management server. + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)} + routes := []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(defaultTestRouteConfigName, defaultTestServiceName, defaultTestClusterName)} + cluster := []*v3clusterpb.Cluster{e2e.DefaultCluster(defaultTestClusterName, defaultTestEndpointName, e2e.SecurityLevelNone)} + endpoints := []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(defaultTestEndpointName, defaultTestHostname, defaultTestPort)} + configureResources(ctx, t, mgmtServer, nodeID, listeners, routes, cluster, endpoints) + + // Create default bootstrap config. + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Build the resolver and read the config selector out of it. + stateCh, _, _ := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)}, bc) + cs := verifyUpdateFromResolver(ctx, t, stateCh, "") + + // Selecting a config should put the XDSConfig in the config's context. + res, err := cs.SelectConfig(iresolver.RPCInfo{ + Context: ctx, + Method: "/service/method", + }) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + wantXDSConfig := xdstestutils.MakeXDSConfig(defaultTestServiceName, defaultTestRouteConfigName, defaultTestClusterName, defaultTestEndpointName, "test-host:8080") + + // Verify the XDSConfig. + gotConfig := xdsresource.XDSConfigFromContext(res.Context) + if err := xdstestutils.CompareXDSConfig(gotConfig, wantXDSConfig); err != nil { + t.Fatal(err) + } +} + +// Tests the case where the resolver receives a server-side listener resource. +// The resolver should report the mismatch error, put the channel in TRANSIENT_FAILURE +// (using an empty service config update), and recover when a valid client-side +// listener update is subsequently received. +func (s) TestResolverServerSideListenerReceivedOnClient(t *testing.T) { + // Spin up an xDS management server for the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer, _, _, bc := setupManagementServerForTest(t, nodeID) + + // Configure a server-side listener resource on the management server. + // We use the client-side dial target as the resource name, but return a + // server-side listener resource (meaning it lacks ApiListener). + lis := e2e.DefaultServerListener("localhost", 8080, e2e.SecurityLevelNone, defaultTestRouteConfigName) + lis.Name = defaultTestServiceName + configureResources(ctx, t, mgmtServer, nodeID, []*v3listenerpb.Listener{lis}, nil, nil, nil) + + // Build the resolver. + target := resolver.Target{URL: *testutils.MustParseURL("xds:///" + defaultTestServiceName)} + stateCh, errCh, _ := buildResolverForTarget(t, target, bc) + + // Wait for and verify the error update from the resolver. + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for error to be propagated to the ClientConn") + case gotErr := <-errCh: + const wantErr = "does not contain API listener configuration" + if gotErr == nil { + t.Fatalf("Got nil error from resolver, want error containing '%s'", wantErr) + } + errStr := fmt.Sprint(gotErr) + if !strings.Contains(errStr, wantErr) { + t.Fatalf("Got error from resolver %q, want error containing '%s'", errStr, wantErr) + } + if !strings.Contains(errStr, nodeID) { + t.Fatalf("Got error from resolver %q, want nodeID %q", errStr, nodeID) + } + } + + // Verify that the resolver pushed an empty service config update (putting the channel in TF). + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for state update containing empty service config") + case gotState := <-stateCh: + state := gotState + if err := state.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", state.ServiceConfig.Err) + } + wantSCParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)("{}") + if !internal.EqualServiceConfigForTesting(state.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, state.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + } + + // Configure valid client-side listener and route config resources. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: defaultTestServiceName, + NodeID: nodeID, + Host: defaultTestHostname, + Port: defaultTestPort[0], + SecLevel: e2e.SecurityLevelNone, + }) + mgmtServer.Update(ctx, resources) + + // The resolver should recover and send a service config with the cluster name. + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for state update after recovery") + case gotState := <-stateCh: + state := gotState + if err := state.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", state.ServiceConfig.Err) + } + wantSCParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(wantServiceConfig(resources.Clusters[0].Name)) + if !internal.EqualServiceConfigForTesting(state.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, state.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + cs := iresolver.GetConfigSelector(state) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + } +} diff --git a/internal/xds/server/conn_wrapper.go b/internal/xds/server/conn_wrapper.go new file mode 100644 index 0000000..104e6f8 --- /dev/null +++ b/internal/xds/server/conn_wrapper.go @@ -0,0 +1,203 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package server + +import ( + "fmt" + "net" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal/credentials/xds" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/transport" +) + +// connWrapper is a thin wrapper around a net.Conn returned by Accept(). It +// provides the following additional functionality: +// 1. A way to retrieve the configured deadline. This is required by the +// ServerHandshake() method of the xdsCredentials when it attempts to read +// key material from the certificate providers. +// 2. Implements the XDSHandshakeInfo() method used by the xdsCredentials to +// retrieve the configured certificate providers. +// 3. xDS filter_chain configuration determines security configuration. +// 4. Dynamically reads routing configuration in UsableRouteConfiguration(), called +// to process incoming RPC's. (LDS + RDS configuration). +type connWrapper struct { + net.Conn + + // The specific filter chain picked for handling this connection. + filterChain *filterChain + + // A reference to the listenerWrapper on which this connection was accepted. + parent *listenerWrapper + + // The connection deadline as configured by the grpc.Server on the rawConn + // that is returned by a call to Accept(). This is set to the connection + // timeout value configured by the user (or to a default value) before + // initiating the transport credential handshake, and set to zero after + // completing the HTTP2 handshake. + deadlineMu sync.Mutex + deadline time.Time + + mu sync.Mutex + st transport.ServerTransport + draining bool + closed bool + hi *grpcsync.RefCounted[*xds.HandshakeInfo] + + // The virtual hosts with matchable routes and instantiated HTTP Filters per + // route, or an error. + urc *atomic.Pointer[usableRouteConfiguration] +} + +// SetDeadline makes a copy of the passed in deadline and forwards the call to +// the underlying rawConn. +func (c *connWrapper) SetDeadline(t time.Time) error { + c.deadlineMu.Lock() + c.deadline = t + c.deadlineMu.Unlock() + return c.Conn.SetDeadline(t) +} + +// GetDeadline returns the configured deadline. This will be invoked by the +// ServerHandshake() method of the XdsCredentials, which needs a deadline to +// pass to the certificate provider. +func (c *connWrapper) GetDeadline() time.Time { + c.deadlineMu.Lock() + t := c.deadline + c.deadlineMu.Unlock() + return t +} + +// XDSHandshakeInfo returns a HandshakeInfo with appropriate security +// configuration for this connection. This method is invoked by the +// ServerHandshake() method of the XdsCredentials. +func (c *connWrapper) XDSHandshakeInfo() (*grpcsync.RefCounted[*xds.HandshakeInfo], error) { + if c.filterChain.securityCfg == nil { + // If the security config is empty, this means that the control plane + // did not provide any security configuration and therefore we should + // return nil here so that the xdsCreds can use the configured fallback + // credentials. + return nil, nil + } + + cpc := c.parent.xdsC.BootstrapConfig().CertProviderConfigs() + // Identity provider name is mandatory on the server-side, and this is + // enforced when the resource is received at the XDSClient layer. + secCfg := c.filterChain.securityCfg + ip, err := buildProviderFunc(cpc, secCfg.IdentityInstanceName, secCfg.IdentityCertName, true, false) + if err != nil { + return nil, err + } + // Root provider name is optional and required only when doing mTLS. + var rp certprovider.Provider + if instance, cert := secCfg.RootInstanceName, secCfg.RootCertName; instance != "" { + rp, err = buildProviderFunc(cpc, instance, cert, false, true) + if err != nil { + ip.Close() + return nil, err + } + } + + // Note that this method is invoked when a connection is accepted, and the + // xdsCredentials are doing a handshake on it. This can only ever be called + // once per connection. So, we do not need to worry about decrementing the + // reference count for the existing HandshakeInfo, which will always be nil. + // The reference count will be decremented when the connection is closed. + c.mu.Lock() + defer c.mu.Unlock() + if c.closed { + if rp != nil { + rp.Close() + } + ip.Close() + return nil, fmt.Errorf("xds: connection closed") + } + c.hi = xds.NewHandshakeInfo(rp, ip, nil, "", secCfg.RequireClientCert, false, false) + return c.hi, nil +} + +// PassServerTransport drains the passed in ServerTransport if draining is set, +// or persists it to be drained once drained is called. +func (c *connWrapper) PassServerTransport(st transport.ServerTransport) { + c.mu.Lock() + defer c.mu.Unlock() + if c.draining { + st.Drain("draining") + } else { + c.st = st + } +} + +// Drain drains the associated ServerTransport, or sets draining to true so it +// will be drained after it is created. +func (c *connWrapper) Drain() { + c.mu.Lock() + defer c.mu.Unlock() + if c.st == nil { + c.draining = true + } else { + c.st.Drain("draining") + } +} + +// Close closes the providers and the underlying connection. +func (c *connWrapper) Close() error { + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return nil + } + c.closed = true + hi := c.hi + c.mu.Unlock() + + if hi != nil { + hi.Decrement() + } + + c.parent.removeConn(c) + return c.Conn.Close() +} + +func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) { + cfg, ok := configs[instanceName] + if !ok { + // Defensive programming. If a resource received from the management + // server contains a certificate provider instance name that is not + // found in the bootstrap, the resource is NACKed by the xDS client. + return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName) + } + provider, err := cfg.Build(certprovider.BuildOptions{ + CertName: certName, + WantIdentity: wantIdentity, + WantRoot: wantRoot, + }) + if err != nil { + // This error is not expected since the bootstrap process parses the + // config and makes sure that it is acceptable to the plugin. Still, it + // is possible that the plugin parses the config successfully, but its + // Build() method errors out. + return nil, fmt.Errorf("failed to get security plugin instance (%+v): %v", cfg, err) + } + return provider, nil +} diff --git a/internal/xds/server/filter_chain_manager.go b/internal/xds/server/filter_chain_manager.go new file mode 100644 index 0000000..88c648e --- /dev/null +++ b/internal/xds/server/filter_chain_manager.go @@ -0,0 +1,607 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package server + +import ( + "errors" + "fmt" + "net/netip" + "slices" + "strings" + "sync/atomic" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/status" +) + +const ( + // An unspecified destination or source prefix should be considered a less + // specific match than a wildcard prefix, `0.0.0.0/0` or `::/0`. Also, an + // unspecified prefix should match most v4 and v6 addresses compared to the + // wildcard prefixes which match only a specific network (v4 or v6). + // + // We use these constants when looking up the most specific prefix match. A + // wildcard prefix will match 0 bits, and to make sure that a wildcard + // prefix is considered a more specific match than an unspecified prefix, we + // use a value of -1 for the latter. + noPrefixMatch = -2 + unspecifiedPrefixMatch = -1 +) + +// filterChainManager contains the match criteria specified through filter +// chains in a single Listener resource. +// +// For the match criteria and the filter chains, we need to use package local +// structs that are very similar to the xdsresource structs. This is because the +// xdsresource structs are meant to contain only configuration and not runtime +// state. Here, we need to store runtime state such as the usable route +// configuration. +type filterChainManager struct { + dstPrefixes []*destPrefixEntry // Linear lookup list. + defaultFilterChain *filterChain // Default filter chain, if specified. + filterChains []*filterChain // Filter chains managed by this filter chain manager. + routeConfigNames map[string]bool // Route configuration names which need to be dynamically queried. +} + +func newFilterChainManager(filterChainConfigs *xdsresource.NetworkFilterChainMap, defFilterChainConfig *xdsresource.NetworkFilterChainConfig) *filterChainManager { + fcMgr := &filterChainManager{routeConfigNames: make(map[string]bool)} + + if filterChainConfigs != nil { + for _, entry := range filterChainConfigs.DstPrefixes { + dstEntry := &destPrefixEntry{prefix: entry.Prefix} + + for i, srcPrefixes := range entry.SourceTypeArr { + if len(srcPrefixes.Entries) == 0 { + continue + } + stDest := &sourcePrefixes{} + dstEntry.srcTypeArr[i] = stDest + for _, srcEntryConfig := range srcPrefixes.Entries { + srcEntry := &sourcePrefixEntry{ + prefix: srcEntryConfig.Prefix, + srcPortMap: make(map[int]*filterChain, len(srcEntryConfig.PortMap)), + } + stDest.srcPrefixes = append(stDest.srcPrefixes, srcEntry) + + for port, fcConfig := range srcEntryConfig.PortMap { + fc := fcMgr.filterChainFromConfig(&fcConfig) + if fc.routeConfigName != "" { + fcMgr.routeConfigNames[fc.routeConfigName] = true + } + srcEntry.srcPortMap[port] = fc + fcMgr.filterChains = append(fcMgr.filterChains, fc) + } + } + } + fcMgr.dstPrefixes = append(fcMgr.dstPrefixes, dstEntry) + } + } + + if defFilterChainConfig != nil && !defFilterChainConfig.IsEmpty() { + fc := fcMgr.filterChainFromConfig(defFilterChainConfig) + if fc.routeConfigName != "" { + fcMgr.routeConfigNames[fc.routeConfigName] = true + } + fcMgr.defaultFilterChain = fc + fcMgr.filterChains = append(fcMgr.filterChains, fc) + } + + return fcMgr +} + +func (fcm *filterChainManager) filterChainFromConfig(config *xdsresource.NetworkFilterChainConfig) *filterChain { + fc := &filterChain{ + securityCfg: config.SecurityCfg, + routeConfigName: config.HTTPConnMgr.RouteConfigName, + inlineRouteConfig: config.HTTPConnMgr.InlineRouteConfig, + httpFilters: config.HTTPConnMgr.HTTPFilters, + usableRouteConfiguration: &atomic.Pointer[usableRouteConfiguration]{}, // Active state + } + fc.usableRouteConfiguration.Store(&usableRouteConfiguration{}) + return fc +} + +func (fcm *filterChainManager) stop() { + for _, fc := range fcm.filterChains { + fc.usableRouteConfiguration.Load().stop() + fc.stop() + } +} + +// destPrefixEntry contains a destination prefix entry and associated source +// type matchers. +type destPrefixEntry struct { + prefix netip.Prefix + srcTypeArr sourceTypesArray +} + +// An array for the fixed number of source types that we have. +type sourceTypesArray [3]*sourcePrefixes + +// sourceType specifies the connection source IP match type. +type sourceType int + +const ( + sourceTypeAny sourceType = iota // matches connection attempts from any source + sourceTypeSameOrLoopback // matches connection attempts from the same host + sourceTypeExternal // matches connection attempts from a different host +) + +// sourcePrefixes contains a list of source prefix entries. +type sourcePrefixes struct { + srcPrefixes []*sourcePrefixEntry +} + +// sourcePrefixEntry contains a source prefix entry and associated source port +// matchers. +type sourcePrefixEntry struct { + prefix netip.Prefix + srcPortMap map[int]*filterChain +} + +// filterChain captures information from within a FilterChain message in a +// Listener resource. This struct contains the active state of a filter chain, +// which includes the usable route configuration. +type filterChain struct { + // The following fields are set at initialization time, and are not + // updated afterwards. + securityCfg *xdsresource.SecurityConfig + httpFilters []xdsresource.HTTPFilter + routeConfigName string + inlineRouteConfig *xdsresource.RouteConfigUpdate + + // Server filters with reference counts. Is updated when a usable route + // configuration is set, and when the filter chain is stopped. Both these + // operations always happen with the listener wrapper's lock held. + serverFilters []httpfilter.ServerFilter + + // The usable route configuration, is passed to the connWrapper, and is used + // to route incoming RPCs. Therefore, this needs to be accessed atomically. + usableRouteConfiguration *atomic.Pointer[usableRouteConfiguration] +} + +// usableRouteConfiguration contains a matchable route configuration, with +// instantiated HTTP Filters per route. +type usableRouteConfiguration struct { + vhs []virtualHostWithInterceptors + err error + nodeID string // For logging purposes. Populated by the listener wrapper. +} + +func (rc *usableRouteConfiguration) stop() { + for _, vh := range rc.vhs { + for _, r := range vh.routes { + if r.interceptor != nil { + r.interceptor.Close() + } + } + } +} + +// virtualHostWithInterceptors captures information present in a VirtualHost +// update, and also contains routes with instantiated HTTP Filters. +type virtualHostWithInterceptors struct { + domains []string + routes []routeWithInterceptors +} + +// routeWithInterceptors captures information in a Route, and contains +// a usable matcher and also instantiated HTTP Filters. +type routeWithInterceptors struct { + matcher *xdsresource.CompositeMatcher + actionType xdsresource.RouteActionType + interceptor httpfilter.ServerInterceptor +} + +type lookupParams struct { + isUnspecifiedListener bool // Whether the server is listening on a wildcard address. + dstAddr netip.Addr // dstAddr is the local address of an incoming connection. + srcAddr netip.Addr // srcAddr is the remote address of an incoming connection. + srcPort int // srcPort is the remote port of an incoming connection. +} + +// lookup returns the most specific matching filter chain to be used for an +// incoming connection on the server side. Returns a non-nil error if no +// matching filter chain could be found. +func (fcm *filterChainManager) lookup(params lookupParams) (*filterChain, error) { + dstPrefixes := filterByDestinationPrefixes(fcm.dstPrefixes, params.isUnspecifiedListener, params.dstAddr) + if len(dstPrefixes) == 0 { + if fcm.defaultFilterChain != nil { + return fcm.defaultFilterChain, nil + } + return nil, fmt.Errorf("no matching filter chain based on destination prefix match for %+v", params) + } + + srcType := sourceTypeExternal + if params.srcAddr == params.dstAddr || params.srcAddr.IsLoopback() { + srcType = sourceTypeSameOrLoopback + } + srcPrefixes := filterBySourceType(dstPrefixes, srcType) + if len(srcPrefixes) == 0 { + if fcm.defaultFilterChain != nil { + return fcm.defaultFilterChain, nil + } + return nil, fmt.Errorf("no matching filter chain based on source type match for %+v", params) + } + srcPrefixEntry, err := filterBySourcePrefixes(srcPrefixes, params.srcAddr) + if err != nil { + return nil, err + } + if fc := filterBySourcePorts(srcPrefixEntry, params.srcPort); fc != nil { + return fc, nil + } + if fcm.defaultFilterChain != nil { + return fcm.defaultFilterChain, nil + } + return nil, fmt.Errorf("no matching filter chain after all match criteria for %+v", params) +} + +// filterByDestinationPrefixes is the first stage of the filter chain +// matching algorithm. It takes the complete set of configured filter chain +// matchers and returns the most specific matchers based on the destination +// prefix match criteria (the prefixes which match the most number of bits). +func filterByDestinationPrefixes(dstPrefixes []*destPrefixEntry, isUnspecified bool, dstAddr netip.Addr) []*destPrefixEntry { + if !isUnspecified { + // Destination prefix matchers are considered only when the listener is + // bound to the wildcard address. + return dstPrefixes + } + + var matchingDstPrefixes []*destPrefixEntry + maxSubnetMatch := noPrefixMatch + for _, entry := range dstPrefixes { + if entry.prefix.IsValid() && !entry.prefix.Contains(dstAddr) { + // Skip prefixes which don't match. + continue + } + // For unspecified prefixes, since we do not store a real netip.Prefix + // inside prefix, we do not perform a match. Instead we simply set + // the matchSize to -1, which is less than the matchSize (0) for a + // wildcard prefix. + matchSize := unspecifiedPrefixMatch + if entry.prefix.IsValid() { + matchSize = entry.prefix.Bits() + } + if matchSize < maxSubnetMatch { + continue + } + if matchSize > maxSubnetMatch { + maxSubnetMatch = matchSize + matchingDstPrefixes = make([]*destPrefixEntry, 0, 1) + } + matchingDstPrefixes = append(matchingDstPrefixes, entry) + } + return matchingDstPrefixes +} + +// filterBySourceType is the second stage of the matching algorithm. It +// trims the filter chains based on the most specific source type match. +// +// srcType is one of sourceTypeSameOrLoopback or sourceTypeExternal. +func filterBySourceType(dstPrefixes []*destPrefixEntry, srcType sourceType) []*sourcePrefixes { + var ( + srcPrefixes []*sourcePrefixes + bestSrcTypeMatch sourceType + ) + for _, entry := range dstPrefixes { + match := srcType + srcPrefix := entry.srcTypeArr[srcType] + if srcPrefix == nil { + match = sourceTypeAny + srcPrefix = entry.srcTypeArr[sourceTypeAny] + } + if match < bestSrcTypeMatch { + continue + } + if match > bestSrcTypeMatch { + bestSrcTypeMatch = match + srcPrefixes = make([]*sourcePrefixes, 0) + } + if srcPrefix != nil { + // The source type array always has 3 entries, but these could be + // nil if the appropriate source type match was not specified. + srcPrefixes = append(srcPrefixes, srcPrefix) + } + } + return srcPrefixes +} + +// filterBySourcePrefixes is the third stage of the filter chain matching +// algorithm. It trims the filter chains based on the source prefix. At most one +// filter chain with the most specific match progress to the next stage. +func filterBySourcePrefixes(srcPrefixes []*sourcePrefixes, srcAddr netip.Addr) (*sourcePrefixEntry, error) { + var matchingSrcPrefixes []*sourcePrefixEntry + maxSubnetMatch := noPrefixMatch + for _, sp := range srcPrefixes { + for _, entry := range sp.srcPrefixes { + if entry.prefix.IsValid() && !entry.prefix.Contains(srcAddr) { + // Skip prefixes which don't match. + continue + } + // For unspecified prefixes, since we do not store a real netip.Prefix + // inside prefix, we do not perform a match. Instead we simply set + // the matchSize to -1, which is less than the matchSize (0) for a + // wildcard prefix. + matchSize := unspecifiedPrefixMatch + if entry.prefix.IsValid() { + matchSize = entry.prefix.Bits() + } + if matchSize < maxSubnetMatch { + continue + } + if matchSize > maxSubnetMatch { + maxSubnetMatch = matchSize + matchingSrcPrefixes = make([]*sourcePrefixEntry, 0, 1) + } + matchingSrcPrefixes = append(matchingSrcPrefixes, entry) + } + } + if len(matchingSrcPrefixes) == 0 { + // Finding no match is not an error condition. The caller will end up + // using the default filter chain if one was configured. + return nil, nil + } + if len(matchingSrcPrefixes) != 1 { + // We expect at most a single matching source prefix entry at this + // point. If we have multiple entries here, and some of their source + // port matchers had wildcard entries, we could be left with more than + // one matching filter chain and hence would have been flagged as an + // invalid configuration at config validation time. + return nil, errors.New("multiple matching filter chains") + } + return matchingSrcPrefixes[0], nil +} + +// filterBySourcePorts is the last stage of the filter chain matching +// algorithm. It trims the filter chains based on the source ports. +func filterBySourcePorts(spe *sourcePrefixEntry, srcPort int) *filterChain { + if spe == nil { + return nil + } + // A match could be a wildcard match (this happens when the match + // criteria does not specify source ports) or a specific port match (this + // happens when the match criteria specifies a set of ports and the source + // port of the incoming connection matches one of the specified ports). The + // latter is considered to be a more specific match. + if fc := spe.srcPortMap[srcPort]; fc != nil { + return fc + } + if fc := spe.srcPortMap[0]; fc != nil { + return fc + } + return nil +} + +func (fc *filterChain) stop() { + for _, sf := range fc.serverFilters { + sf.Close() + } +} + +// serverFilterProvider is used to get a ServerFilter. +// +// This functionality is provided by the listener wrapper, which maintains a map +// of reference counted ServerFilters keyed by {filter_name + type_urls} for all +// filters across all filter chains, and ensures that the same filter instance +// is reused across resource updates. This allows filter instances to retain +// state across resource updates. +type serverFilterProvider func(filter xdsresource.HTTPFilter) (httpfilter.ServerFilter, error) + +// updateUsableRouteConfiguration takes Route Configuration and converts it +// into matchable route configuration, with instantiated HTTP Filters per route. +func (fc *filterChain) updateUsableRouteConfiguration(config *xdsresource.RouteConfigUpdate, updateErr error, provider serverFilterProvider, nodeID string) { + if updateErr != nil { + urc := &usableRouteConfiguration{err: updateErr, nodeID: nodeID} + fc.applyConfiguration(urc, nil) + return + } + + var serverFilters []httpfilter.ServerFilter + vhs := make([]virtualHostWithInterceptors, 0, len(config.VirtualHosts)) + for _, vh := range config.VirtualHosts { + vhwi, sfs, err := fc.convertVirtualHost(vh, provider) + if err != nil { + for _, sf := range serverFilters { + sf.Close() + } + // Close interceptors from successfully converted virtual hosts. + for _, v := range vhs { + for _, r := range v.routes { + if r.interceptor != nil { + r.interceptor.Close() + r.interceptor = nil + } + } + } + // Non nil if (lds + rds) fails, shouldn't happen since validated by + // xDS Client, treat as L7 error but shouldn't happen. + urc := &usableRouteConfiguration{err: fmt.Errorf("virtual host construction: %v", err), nodeID: nodeID} + fc.applyConfiguration(urc, nil) + return + } + vhs = append(vhs, vhwi) + serverFilters = append(serverFilters, sfs...) + } + + urc := &usableRouteConfiguration{vhs: vhs, nodeID: nodeID} + fc.applyConfiguration(urc, serverFilters) +} + +func (fc *filterChain) applyConfiguration(urc *usableRouteConfiguration, serverFilters []httpfilter.ServerFilter) { + // Swap in the new configuration first so new RPCs use it immediately. + oldURC := fc.usableRouteConfiguration.Swap(urc) + oldFilters := fc.serverFilters + fc.serverFilters = serverFilters + + // Stop the old interceptors before releasing the filters they might depend on. + if oldURC != nil { + oldURC.stop() + } + + // Release references to old server filters. + for _, sf := range oldFilters { + sf.Close() + } +} + +func (fc *filterChain) convertVirtualHost(virtualHost *xdsresource.VirtualHost, provider serverFilterProvider) (_ virtualHostWithInterceptors, _ []httpfilter.ServerFilter, err error) { + var serverFilters []httpfilter.ServerFilter + defer func() { + if err != nil { + for _, sf := range serverFilters { + sf.Close() + } + } + }() + + rs := make([]routeWithInterceptors, len(virtualHost.Routes)) + for i, r := range virtualHost.Routes { + rs[i].actionType = r.ActionType + rs[i].matcher = xdsresource.RouteToMatcher(r) + interceptor, sfs, err := fc.newInterceptor(r.HTTPFilterConfigOverride, virtualHost.HTTPFilterConfigOverride, provider) + if err != nil { + // Close interceptors from successfully converted routes. + for _, route := range rs { + if route.interceptor != nil { + route.interceptor.Close() + route.interceptor = nil + } + } + return virtualHostWithInterceptors{}, nil, err + } + serverFilters = append(serverFilters, sfs...) + rs[i].interceptor = interceptor + } + return virtualHostWithInterceptors{domains: virtualHost.Domains, routes: rs}, serverFilters, nil +} + +// statusErrWithNodeID returns an error produced by the status package with the +// specified code and message, and includes the xDS node ID. +func (rc *usableRouteConfiguration) statusErrWithNodeID(c codes.Code, msg string, args ...any) error { + return status.Error(c, fmt.Sprintf("[xDS node id: %v]: %s", rc.nodeID, fmt.Sprintf(msg, args...))) +} + +func (fc *filterChain) newInterceptor(routeOverride, virtualHostOverride map[string]httpfilter.FilterConfig, provider serverFilterProvider) (_ httpfilter.ServerInterceptor, _ []httpfilter.ServerFilter, err error) { + serverFilters := []httpfilter.ServerFilter{} + interceptors := make([]httpfilter.ServerInterceptor, 0, len(fc.httpFilters)) + defer func() { + if err != nil { + for _, sf := range serverFilters { + sf.Close() + } + for _, i := range interceptors { + i.Close() + } + } + }() + + for _, filter := range fc.httpFilters { + // Route is highest priority on server side, as there is no concept + // of an upstream cluster on server side. + override := routeOverride[filter.Name] + if override == nil { + // Virtual Host is second priority. + override = virtualHostOverride[filter.Name] + } + + disabled := filter.Disabled + if override != nil { + _, disabled = override.(httpfilter.DisabledFilterConfig) + } + if disabled { + continue + } + + serverFilter, err := provider(filter) + if err != nil { + return nil, nil, err + } + serverFilters = append(serverFilters, serverFilter) + i, err := serverFilter.BuildServerInterceptor(filter.Config, override) + if err != nil { + return nil, nil, fmt.Errorf("error constructing filter: %v", err) + } + if i != nil { + interceptors = append(interceptors, i) + } + } + + return &interceptorList{interceptors: interceptors}, serverFilters, nil +} + +type interceptorList struct { + interceptors []httpfilter.ServerInterceptor +} + +func (il *interceptorList) InterceptRPC(ss grpc.ServerStream) (grpc.ServerStream, error) { + var err error + for _, i := range il.interceptors { + ss, err = i.InterceptRPC(ss) + if err != nil { + return nil, err + } + } + return ss, nil +} + +func (il *interceptorList) Close() { + for _, i := range il.interceptors { + i.Close() + } +} + +// refCountedServerFilter wraps a ServerFilter along with a reference count. +// This is used to manage server filters that are shared across filter chains +// within a filter chain manager. +type refCountedServerFilter struct { + httpfilter.ServerFilter + refCnt atomic.Int32 +} + +func (rsf *refCountedServerFilter) incRef() { + rsf.refCnt.Add(1) +} + +func (rsf *refCountedServerFilter) Close() { + if rsf.refCnt.Add(-1) == 0 { + rsf.ServerFilter.Close() + } +} + +// newServerFilterKey generates a key for the given filter using the filter name +// and type URLs. This is used for storing ServerFilters in a map. +func newServerFilterKey(f *xdsresource.HTTPFilter) serverFilterKey { + typeURLs := slices.Clone(f.Filter.TypeURLs()) + slices.Sort(typeURLs) + return serverFilterKey{ + name: f.Name, + typeURLs: strings.Join(typeURLs, ":"), + } +} + +type serverFilterKey struct { + name string + typeURLs string +} + +func (f *serverFilterKey) String() string { + return f.name + ":" + f.typeURLs +} diff --git a/internal/xds/server/filter_chain_manager_test.go b/internal/xds/server/filter_chain_manager_test.go new file mode 100644 index 0000000..e7dda34 --- /dev/null +++ b/internal/xds/server/filter_chain_manager_test.go @@ -0,0 +1,796 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package server + +import ( + "errors" + "fmt" + "net/netip" + "strings" + "sync/atomic" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/httpfilter/router" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" +) + +const ( + topLevel = "top level" + vhLevel = "virtual host level" + rLevel = "route level" +) + +func init() { + certprovider.Register(&testCertProviderBuilder{}) +} + +type testCertProviderBuilder struct{} + +func (b *testCertProviderBuilder) ParseConfig(any) (*certprovider.BuildableConfig, error) { + return certprovider.NewBuildableConfig("test_cert_provider", nil, func(certprovider.BuildOptions) certprovider.Provider { + return nil + }), nil +} + +func (b *testCertProviderBuilder) Name() string { + return "test_cert_provider" +} + +// newFilterChainManagerForTesting creates a filterChainManager using the +// newFilterChainManager() function. It parses the provided listener resource +// using the xdsresource package. +func newFilterChainManagerForTesting(t *testing.T, lis *v3listenerpb.Listener) *filterChainManager { + t.Helper() + if lis.Address == nil { + lis.Address = &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "0.0.0.0", + PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: 80}, + }, + }, + } + } + if lis.Name == "" { + lis.Name = "default_listener" + } + lAny, err := anypb.New(lis) + if err != nil { + t.Fatalf("anypb.New() failed: %v", err) + } + bc, err := bootstrap.NewConfigFromContents([]byte(`{ + "xds_servers": [ + { + "server_uri": "ipv4:///127.0.0.1:443", + "channel_creds": [ + { + "type": "insecure" + } + ] + } + ], + "certificate_providers": { + "default": { + "plugin_name": "test_cert_provider" + }, + "instance1": { + "plugin_name": "test_cert_provider" + }, + "unspecified-dest-and-source-prefix": { + "plugin_name": "test_cert_provider" + }, + "wildcard-prefixes-v4": { + "plugin_name": "test_cert_provider" + }, + "wildcard-source-prefix-v6": { + "plugin_name": "test_cert_provider" + }, + "specific-destination-prefix-unspecified-source-type": { + "plugin_name": "test_cert_provider" + }, + "specific-destination-prefix-specific-source-type": { + "plugin_name": "test_cert_provider" + }, + "specific-destination-prefix-specific-source-type-specific-source-prefix": { + "plugin_name": "test_cert_provider" + }, + "specific-destination-prefix-specific-source-type-specific-source-prefix-specific-source-port": { + "plugin_name": "test_cert_provider" + } + } + }`)) + if err != nil { + t.Fatalf("bootstrap.NewConfigFromContents() failed: %v", err) + } + decoder := xdsresource.NewListenerResourceTypeDecoder(bc, nil) + res, err := decoder.Decode(xdsclient.NewAnyProto(lAny), xdsclient.DecodeOptions{}) + if err != nil { + t.Fatalf("decoder.Decode() failed: %v", err) + } + lrd, ok := res.Resource.(*xdsresource.ListenerResourceData) + if !ok { + t.Fatalf("Resource type mismatch: %T", res.Resource) + } + upd := lrd.Resource + if upd.TCPListener == nil { + t.Fatalf("Decoded listener is not TCP or failed validation") + } + return newFilterChainManager(&upd.TCPListener.FilterChains, &upd.TCPListener.DefaultFilterChain) +} + +func emptyValidNetworkFilters(t *testing.T) []*v3listenerpb.Filter { + return []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"lds.target.good:3333"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + }), + }, + }, + } +} + +var inlineRouteConfig = &xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{{ + Domains: []string{"lds.target.good:3333"}, + Routes: []*xdsresource.Route{{Prefix: newStringP("/"), ActionType: xdsresource.RouteActionNonForwardingAction}}, + }}, +} + +func newStringP(s string) *string { + return &s +} + +func makeRouterFilterList(t *testing.T) []xdsresource.HTTPFilter { + routerBuilder := httpfilter.Get(router.TypeURL) + routerConfig, _ := routerBuilder.ParseFilterConfig(testutils.MarshalAny(t, &v3routerpb.Router{}), httpfilter.ParseOptions{}) + return []xdsresource.HTTPFilter{{ + Name: "router", + Filter: routerBuilder, + Config: routerConfig, + }} +} + +func (s) TestLookup_Failures(t *testing.T) { + tests := []struct { + desc string + lis *v3listenerpb.Listener + params lookupParams + wantErr string + }{ + { + desc: "no_destination_prefix_match", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}}, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + params: lookupParams{ + isUnspecifiedListener: true, + dstAddr: netip.AddrFrom4([4]byte([]byte{10, 1, 1, 1})), + }, + wantErr: "no matching filter chain based on destination prefix match", + }, + { + desc: "no_source_type_match", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + params: lookupParams{ + isUnspecifiedListener: true, + dstAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 100, 1})), + srcAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 100, 2})), + }, + wantErr: "no matching filter chain based on source type match", + }, + { + desc: "no_source_prefix_match", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 24)}, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + params: lookupParams{ + isUnspecifiedListener: true, + dstAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 100, 1})), + srcAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 100, 1})), + }, + wantErr: "no matching filter chain after all match criteria", + }, + { + desc: "multiple_matching_filter_chains", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3}}, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + SourcePorts: []uint32{1}, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + params: lookupParams{ + // IsUnspecified is not set. This means that the destination + // prefix matchers will be ignored. + dstAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 100, 1})), + srcAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 100, 1})), + srcPort: 1, + }, + wantErr: "multiple matching filter chains", + }, + { + desc: "no_default_filter_chain", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3}}, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + params: lookupParams{ + isUnspecifiedListener: true, + dstAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 100, 1})), + srcAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 100, 1})), + srcPort: 80, + }, + wantErr: "no matching filter chain after all match criteria", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + fci := newFilterChainManagerForTesting(t, test.lis) + if _, err := fci.lookup(test.params); err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("filterChainManager.lookup() failed with %q want %q", err, test.wantErr) + } + }) + } +} + +func (s) TestLookup_Successes(t *testing.T) { + lisWithDefaultChain := &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}}, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: "instance1"}, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + // A default filter chain with an empty transport socket. + DefaultFilterChain: &v3listenerpb.FilterChain{ + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: "default"}, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + } + lisWithoutDefaultChain := &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: transportSocketWithInstanceName(t, "unspecified-dest-and-source-prefix"), + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("0.0.0.0", 0)}, + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("0.0.0.0", 0)}, + }, + TransportSocket: transportSocketWithInstanceName(t, "wildcard-prefixes-v4"), + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("::", 0)}, + }, + TransportSocket: transportSocketWithInstanceName(t, "wildcard-source-prefix-v6"), + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}}, + TransportSocket: transportSocketWithInstanceName(t, "specific-destination-prefix-unspecified-source-type"), + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 24)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + }, + TransportSocket: transportSocketWithInstanceName(t, "specific-destination-prefix-specific-source-type"), + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 24)}, + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.92.1", 24)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + }, + TransportSocket: transportSocketWithInstanceName(t, "specific-destination-prefix-specific-source-type-specific-source-prefix"), + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 24)}, + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.92.1", 24)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + SourcePorts: []uint32{80}, + }, + TransportSocket: transportSocketWithInstanceName(t, "specific-destination-prefix-specific-source-type-specific-source-prefix-specific-source-port"), + Filters: emptyValidNetworkFilters(t), + }, + }, + } + + tests := []struct { + desc string + lis *v3listenerpb.Listener + params lookupParams + wantFC *filterChain + }{ + { + desc: "default_filter_chain", + lis: lisWithDefaultChain, + params: lookupParams{ + isUnspecifiedListener: true, + dstAddr: netip.AddrFrom4([4]byte([]byte{10, 1, 1, 1})), + }, + wantFC: &filterChain{ + securityCfg: &xdsresource.SecurityConfig{IdentityInstanceName: "default"}, + inlineRouteConfig: inlineRouteConfig, + httpFilters: makeRouterFilterList(t), + }, + }, + { + desc: "unspecified_destination_match", + lis: lisWithoutDefaultChain, + params: lookupParams{ + isUnspecifiedListener: true, + dstAddr: netip.MustParseAddr("2001:68::db8"), + srcAddr: netip.AddrFrom4([4]byte([]byte{10, 1, 1, 1})), + srcPort: 1, + }, + wantFC: &filterChain{ + securityCfg: &xdsresource.SecurityConfig{IdentityInstanceName: "unspecified-dest-and-source-prefix"}, + inlineRouteConfig: inlineRouteConfig, + httpFilters: makeRouterFilterList(t), + }, + }, + { + desc: "wildcard_destination_match_v4", + lis: lisWithoutDefaultChain, + params: lookupParams{ + isUnspecifiedListener: true, + dstAddr: netip.AddrFrom4([4]byte([]byte{10, 1, 1, 1})), + srcAddr: netip.AddrFrom4([4]byte([]byte{10, 1, 1, 1})), + srcPort: 1, + }, + wantFC: &filterChain{ + securityCfg: &xdsresource.SecurityConfig{IdentityInstanceName: "wildcard-prefixes-v4"}, + inlineRouteConfig: inlineRouteConfig, + httpFilters: makeRouterFilterList(t), + }, + }, + { + desc: "wildcard_source_match_v6", + lis: lisWithoutDefaultChain, + params: lookupParams{ + isUnspecifiedListener: true, + dstAddr: netip.MustParseAddr("2001:68::1"), + srcAddr: netip.MustParseAddr("2001:68::2"), + srcPort: 1, + }, + wantFC: &filterChain{ + securityCfg: &xdsresource.SecurityConfig{IdentityInstanceName: "wildcard-source-prefix-v6"}, + inlineRouteConfig: inlineRouteConfig, + httpFilters: makeRouterFilterList(t), + }, + }, + { + desc: "specific_destination_and_wildcard_source_type_match", + lis: lisWithoutDefaultChain, + params: lookupParams{ + isUnspecifiedListener: true, + dstAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 100, 1})), + srcAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 100, 1})), + srcPort: 80, + }, + wantFC: &filterChain{ + securityCfg: &xdsresource.SecurityConfig{IdentityInstanceName: "specific-destination-prefix-unspecified-source-type"}, + inlineRouteConfig: inlineRouteConfig, + httpFilters: makeRouterFilterList(t), + }, + }, + { + desc: "specific_destination_and_source_type_match", + lis: lisWithoutDefaultChain, + params: lookupParams{ + isUnspecifiedListener: true, + dstAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 1, 1})), + srcAddr: netip.AddrFrom4([4]byte([]byte{10, 1, 1, 1})), + srcPort: 80, + }, + wantFC: &filterChain{ + securityCfg: &xdsresource.SecurityConfig{IdentityInstanceName: "specific-destination-prefix-specific-source-type"}, + inlineRouteConfig: inlineRouteConfig, + httpFilters: makeRouterFilterList(t), + }, + }, + { + desc: "specific_destination_source_type_and_source_prefix", + lis: lisWithoutDefaultChain, + params: lookupParams{ + isUnspecifiedListener: true, + dstAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 1, 1})), + srcAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 92, 100})), + srcPort: 70, + }, + wantFC: &filterChain{ + securityCfg: &xdsresource.SecurityConfig{IdentityInstanceName: "specific-destination-prefix-specific-source-type-specific-source-prefix"}, + inlineRouteConfig: inlineRouteConfig, + httpFilters: makeRouterFilterList(t), + }, + }, + { + desc: "specific_destination_source_type_source_prefix_and_source_port", + lis: lisWithoutDefaultChain, + params: lookupParams{ + isUnspecifiedListener: true, + dstAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 1, 1})), + srcAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 92, 100})), + srcPort: 80, + }, + wantFC: &filterChain{ + securityCfg: &xdsresource.SecurityConfig{IdentityInstanceName: "specific-destination-prefix-specific-source-type-specific-source-prefix-specific-source-port"}, + inlineRouteConfig: inlineRouteConfig, + httpFilters: makeRouterFilterList(t), + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + fci := newFilterChainManagerForTesting(t, test.lis) + gotFC, err := fci.lookup(test.params) + if err != nil { + t.Fatalf("FilterChainManager.Lookup(%v) failed: %v", test.params, err) + } + if diff := cmp.Diff(gotFC, test.wantFC, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(filterChain{}, "usableRouteConfiguration"), cmp.AllowUnexported(filterChainManager{}, destPrefixEntry{}, sourcePrefixes{}, sourcePrefixEntry{}, filterChain{}, usableRouteConfiguration{})); diff != "" { + t.Fatalf("FilterChainManager.Lookup(%v) mismatch (-got +want):\n%s", test.params, diff) + } + }) + } +} + +type filterCfg struct { + httpfilter.FilterConfig + // Level is what differentiates top level filters ("top level") vs. second + // level ("virtual host level"), and third level ("route level"). + level string +} + +type filterBuilder struct { + httpfilter.Builder +} + +func (fb *filterBuilder) TypeURLs() []string { return []string{"custom.server.filter"} } + +func (fb *filterBuilder) BuildServerFilter() httpfilter.ServerFilter { + return fb +} + +func (fb *filterBuilder) Close() {} + +var _ httpfilter.ServerFilterBuilder = &filterBuilder{} + +func (fb *filterBuilder) BuildServerInterceptor(config httpfilter.FilterConfig, override httpfilter.FilterConfig) (httpfilter.ServerInterceptor, error) { + var level string + level = config.(filterCfg).level + + if override != nil { + level = override.(filterCfg).level + } + return &serverInterceptor{level: level}, nil +} + +type serverInterceptor struct { + level string +} + +func (si *serverInterceptor) InterceptRPC(grpc.ServerStream) (grpc.ServerStream, error) { + return nil, errors.New(si.level) +} + +func (si *serverInterceptor) Close() {} + +func (s) TestHTTPFilterInstantiation(t *testing.T) { + tests := []struct { + name string + filters []xdsresource.HTTPFilter + routeConfig xdsresource.RouteConfigUpdate + // A list of strings which will be built from iterating through the + // filters ["top level", "vh level", "route level", "route level"...] + // wantErrs is the list of error strings that will be constructed from + // the deterministic iteration through the vh list and route list. The + // error string will be determined by the level of config that the + // filter builder receives (i.e. top level, vs. virtual host level vs. + // route level). + wantErrs []string + }{ + { + name: "one http filter no overrides", + filters: []xdsresource.HTTPFilter{ + {Name: "server-interceptor", Filter: &filterBuilder{}, Config: filterCfg{level: topLevel}}, + }, + routeConfig: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{"target"}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("1"), + }, + }, + }, + }}, + wantErrs: []string{topLevel}, + }, + { + name: "one http filter vh override", + filters: []xdsresource.HTTPFilter{ + {Name: "server-interceptor", Filter: &filterBuilder{}, Config: filterCfg{level: topLevel}}, + }, + routeConfig: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{"target"}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("1"), + }, + }, + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{ + "server-interceptor": filterCfg{level: vhLevel}, + }, + }, + }}, + wantErrs: []string{vhLevel}, + }, + { + name: "one http filter route override", + filters: []xdsresource.HTTPFilter{ + {Name: "server-interceptor", Filter: &filterBuilder{}, Config: filterCfg{level: topLevel}}, + }, + routeConfig: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{"target"}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("1"), + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{ + "server-interceptor": filterCfg{level: rLevel}, + }, + }, + }, + }, + }}, + wantErrs: []string{rLevel}, + }, + { + name: "three routes with different overrides", + filters: []xdsresource.HTTPFilter{ + {Name: "server-interceptor", Filter: &filterBuilder{}, Config: filterCfg{level: topLevel}}, + }, + routeConfig: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{"no overrides"}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("1"), + }}, + }, + { + Domains: []string{"virtual host override"}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("1"), + }}, + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{ + "server-interceptor": filterCfg{level: vhLevel}, + }, + }, + { + Domains: []string{"route and virtual host override"}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("1"), + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{ + "server-interceptor": filterCfg{level: rLevel}, + }, + }}, + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{ + "server-interceptor": filterCfg{level: vhLevel}, + }, + }, + }, + }, + wantErrs: []string{topLevel, vhLevel, rLevel}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fc := filterChain{ + httpFilters: test.filters, + usableRouteConfiguration: &atomic.Pointer[usableRouteConfiguration]{}, + } + fc.usableRouteConfiguration.Store(&usableRouteConfiguration{}) + + filters := make(map[serverFilterKey]*refCountedServerFilter) + provider := func(filter xdsresource.HTTPFilter) (httpfilter.ServerFilter, error) { + builder, ok := filter.Filter.(httpfilter.ServerFilterBuilder) + if !ok { + return nil, fmt.Errorf("filter %q does not support use in server", filter.Name) + } + return getOrCreateServerFilterWithMap(filters, builder, newServerFilterKey(&filter)), nil + } + fc.updateUsableRouteConfiguration(&test.routeConfig, nil, provider, "node-id") + urc := fc.usableRouteConfiguration.Load() + if urc.err != nil { + t.Fatalf("Error constructing usable route configuration: %v", urc.err) + } + // Build out list of errors by iterating through the virtual hosts and routes, + // and running the filters in route configurations. + var errs []string + for _, vh := range urc.vhs { + for _, r := range vh.routes { + _, err := r.interceptor.InterceptRPC(nil) + errs = append(errs, err.Error()) + } + } + if !cmp.Equal(errs, test.wantErrs) { + t.Fatalf("List of errors %v, want %v", errs, test.wantErrs) + } + }) + } +} + +func transportSocketWithInstanceName(t *testing.T, name string) *v3corepb.TransportSocket { + return &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: name}, + }, + }), + }, + } +} + +func cidrRangeFromAddressAndPrefixLen(address string, prefixLen int) *v3corepb.CidrRange { + return &v3corepb.CidrRange{ + AddressPrefix: address, + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(prefixLen), + }, + } +} + +func (s) TestLookup_DroppedChainFallback(t *testing.T) { + lis := &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + // This chain has server names, so it should be dropped. + Name: "test-filter-chain", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.1", 32)}, + ServerNames: []string{"foo"}, + }, + Filters: emptyValidNetworkFilters(t), + }, + { + // This chain matches destination and has no unsupported fields. + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.0", 16)}, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + } + params := lookupParams{ + isUnspecifiedListener: true, + dstAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 100, 1})), + srcAddr: netip.AddrFrom4([4]byte([]byte{192, 168, 100, 1})), + srcPort: 80, + } + + fci := newFilterChainManagerForTesting(t, lis) + fc, err := fci.lookup(params) + if err != nil { + t.Fatalf("filterChainManager.lookup(%v) failed: %v", params, err) + } + if fc == nil { + t.Fatalf("filterChainManager.lookup(%v) returned nil", params) + } +} diff --git a/internal/xds/server/listener_wrapper.go b/internal/xds/server/listener_wrapper.go new file mode 100644 index 0000000..40e9ea6 --- /dev/null +++ b/internal/xds/server/listener_wrapper.go @@ -0,0 +1,521 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package server contains internal server-side functionality used by the public +// facing xds package. +package server + +import ( + "fmt" + "net" + "net/netip" + "sync" + "time" + + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + internalbackoff "google.golang.org/grpc/internal/backoff" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" +) + +var ( + logger = grpclog.Component("xds") + + // Backoff strategy for temporary errors received from Accept(). If this + // needs to be configurable, we can inject it through ListenerWrapperParams. + bs = internalbackoff.Exponential{Config: backoff.Config{ + BaseDelay: 5 * time.Millisecond, + Multiplier: 2.0, + MaxDelay: 1 * time.Second, + }} + backoffFunc = bs.Backoff +) + +// ServingModeCallback is the callback that users can register to get notified +// about the server's serving mode changes. The callback is invoked with the +// address of the listener and its new mode. The err parameter is set to a +// non-nil error if the server has transitioned into not-serving mode. +type ServingModeCallback func(addr net.Addr, mode connectivity.ServingMode, err error) + +// XDSClient wraps the methods on the XDSClient which are required by +// the listenerWrapper. +type XDSClient interface { + WatchResource(typeURL, resourceName string, watcher xdsclient.ResourceWatcher) (cancel func()) + BootstrapConfig() *bootstrap.Config +} + +// ListenerWrapperParams wraps parameters required to create a listenerWrapper. +type ListenerWrapperParams struct { + // Listener is the net.Listener passed by the user that is to be wrapped. + Listener net.Listener + // ListenerResourceName is the xDS Listener resource to request. + ListenerResourceName string + // XDSClient provides the functionality from the XDSClient required here. + XDSClient XDSClient + // ModeCallback is the callback to invoke when the serving mode changes. + ModeCallback ServingModeCallback +} + +// NewListenerWrapper creates a new listenerWrapper with params. It returns a +// net.Listener. It starts in state not serving, which triggers Accept() + +// Close() on any incoming connections. +// +// Only TCP listeners are supported. +func NewListenerWrapper(params ListenerWrapperParams) net.Listener { + lw := &listenerWrapper{ + Listener: params.Listener, + name: params.ListenerResourceName, + xdsC: params.XDSClient, + xdsNodeID: params.XDSClient.BootstrapConfig().Node().GetId(), + modeCallback: params.ModeCallback, + isUnspecifiedAddr: params.Listener.Addr().(*net.TCPAddr).IP.IsUnspecified(), + conns: make(map[*connWrapper]bool), + mode: connectivity.ServingModeNotServing, + closed: grpcsync.NewEvent(), + httpFilters: make(map[serverFilterKey]*refCountedServerFilter), + } + lw.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[xds-server-listener %p] ", lw)) + + // Serve() verifies that Addr() returns a valid TCPAddr. So, it is safe to + // ignore the error from SplitHostPort(). + lisAddr := lw.Listener.Addr().String() + lw.addr, lw.port, _ = net.SplitHostPort(lisAddr) + + lw.rdsHandler = newRDSHandler(lw.handleRDSUpdate, lw.xdsC, lw.logger) + lw.cancelWatch = xdsresource.WatchListener(lw.xdsC, lw.name, &ldsWatcher{ + parent: lw, + logger: lw.logger, + name: lw.name, + }) + return lw +} + +// listenerWrapper wraps the net.Listener associated with the listening address +// passed to Serve(). It also contains all other state associated with this +// particular invocation of Serve(). +type listenerWrapper struct { + // The following fields are initialized at creation and are immutable + // afterwards, so do not need synchronization. + net.Listener + logger *internalgrpclog.PrefixLogger + name string + xdsC XDSClient + xdsNodeID string + cancelWatch func() + modeCallback ServingModeCallback + isUnspecifiedAddr bool // True if listener is bound to IP_ANY address. + addr, port string // Listening address and port, validates socket address in LDS responses. + closed *grpcsync.Event + + // mu guards access to the below fields. + mu sync.Mutex + mode connectivity.ServingMode // Current serving mode. + activeFilterChainManager *filterChainManager // Filter chain manager currently serving. + httpFilters map[serverFilterKey]*refCountedServerFilter // HTTP filter instances keyed by {filter_name + type_urls}. + conns map[*connWrapper]bool // conns accepted with configuration from activeFilterChainManager. + + // These fields are read/written to in the context of xDS updates, which are + // guaranteed to be emitted synchronously from the xDS Client. Thus, they do + // not need further synchronization. + pendingFilterChainManager *filterChainManager // Will become active when all route configurations are ready. + rdsHandler *rdsHandler // Handles RDS watches and updates for this listener. +} + +// maybeUpdateFilterChains swaps in the pending filter chain manager to the +// active one if the pending filter chain manager is present. If a swap occurs, +// it also drains (gracefully stops) any connections that were accepted on the +// old active filter chain manager, and puts this listener in state SERVING. +// Must be called within an xDS Client Callback. +func (l *listenerWrapper) maybeUpdateFilterChains() { + if l.pendingFilterChainManager == nil { + // Nothing to update, return early. + return + } + + l.mu.Lock() + // "Updates to a Listener cause all older connections on that Listener to be + // gracefully shut down with a grace period of 10 minutes for long-lived + // RPC's, such that clients will reconnect and have the updated + // configuration apply." - A36 + connsToClose := l.conns + l.conns = make(map[*connWrapper]bool) + + // Swap in the new filter chain manager before draining connections. + prevActive := l.activeFilterChainManager + l.activeFilterChainManager = l.pendingFilterChainManager + l.pendingFilterChainManager = nil + l.instantiateFilterChainRoutingConfigurationsLocked() + if prevActive != nil { + prevActive.stop() + } + + // Build a set of all keys currently in use. + activeKeys := make(map[serverFilterKey]struct{}) + for _, fc := range l.activeFilterChainManager.filterChains { + for _, f := range fc.httpFilters { + activeKeys[newServerFilterKey(&f)] = struct{}{} + } + } + + // Iterate through the existing filters and delete those not in the active + // set. + for key, filter := range l.httpFilters { + if _, active := activeKeys[key]; !active { + if filter.refCnt.Load() != 0 { + l.logger.Errorf("HTTP filter with key %s is not used by any active filter chain but has non-zero reference count: %d. This indicates a bug in the filter reference counting logic.", key, filter.refCnt.Load()) + } + delete(l.httpFilters, key) + } + } + l.switchModeLocked(connectivity.ServingModeServing, nil) + l.mu.Unlock() + + go func() { + for conn := range connsToClose { + conn.Drain() + } + }() +} + +// handleRDSUpdate rebuilds any routing configuration server side for any filter +// chains that point to this RDS, and potentially makes pending lds +// configuration to swap to be active. +func (l *listenerWrapper) handleRDSUpdate(routeName string, rcu rdsWatcherUpdate) { + // Update any filter chains that point to this route configuration. + l.mu.Lock() + if l.activeFilterChainManager != nil { + for _, fc := range l.activeFilterChainManager.filterChains { + if fc.routeConfigName != routeName { + continue + } + if rcu.err != nil && rcu.data == nil { // Either NACK before update, or resource not found triggers this conditional. + fc.updateUsableRouteConfiguration(nil, rcu.err, l.getOrCreateServerFilterLocked, l.xdsNodeID) + continue + } + fc.updateUsableRouteConfiguration(rcu.data, nil, l.getOrCreateServerFilterLocked, l.xdsNodeID) + } + } + l.mu.Unlock() + + if l.rdsHandler.determineRouteConfigurationReady() { + l.maybeUpdateFilterChains() + } +} + +// instantiateFilterChainRoutingConfigurationsLocked instantiates all of the +// routing configuration for the newly active filter chains. For any inline +// route configurations, uses that, otherwise uses cached rdsHandler updates. +// +// Must be called with l.mu held. +func (l *listenerWrapper) instantiateFilterChainRoutingConfigurationsLocked() { + for _, fc := range l.activeFilterChainManager.filterChains { + if fc.inlineRouteConfig != nil { + fc.updateUsableRouteConfiguration(fc.inlineRouteConfig, nil, l.getOrCreateServerFilterLocked, l.xdsNodeID) + continue + } // Inline configuration constructed once here, will remain for lifetime of filter chain. + rcu := l.rdsHandler.updates[fc.routeConfigName] + if rcu.err != nil && rcu.data == nil { + fc.updateUsableRouteConfiguration(nil, rcu.err, l.getOrCreateServerFilterLocked, l.xdsNodeID) + continue + } + fc.updateUsableRouteConfiguration(rcu.data, nil, l.getOrCreateServerFilterLocked, l.xdsNodeID) + } +} + +// Accept blocks on an Accept() on the underlying listener, and wraps the +// returned net.connWrapper with the configured certificate providers. +func (l *listenerWrapper) Accept() (net.Conn, error) { + var retries int + for { + conn, err := l.Listener.Accept() + if err != nil { + // Temporary() method is implemented by certain error types returned + // from the net package, and it is useful for us to not shutdown the + // server in these conditions. The listen queue being full is one + // such case. + if ne, ok := err.(interface{ Temporary() bool }); !ok || !ne.Temporary() { + return nil, err + } + retries++ + timer := time.NewTimer(backoffFunc(retries)) + select { + case <-timer.C: + case <-l.closed.Done(): + timer.Stop() + // Continuing here will cause us to call Accept() again + // which will return a non-temporary error. + continue + } + continue + } + // Reset retries after a successful Accept(). + retries = 0 + + // Since the net.Conn represents an incoming connection, the source and + // destination address can be retrieved from the local address and + // remote address of the net.Conn respectively. + destAddr, ok1 := conn.LocalAddr().(*net.TCPAddr) + srcAddr, ok2 := conn.RemoteAddr().(*net.TCPAddr) + if !ok1 || !ok2 { + // If the incoming connection is not a TCP connection, which is + // really unexpected since we check whether the provided listener is + // a TCP listener in Serve(), we return an error which would cause + // us to stop serving. + return nil, fmt.Errorf("received connection with non-TCP address (local: %T, remote %T)", conn.LocalAddr(), conn.RemoteAddr()) + } + + l.mu.Lock() + if l.mode == connectivity.ServingModeNotServing { + // Close connections as soon as we accept them when we are in + // "not-serving" mode. Since we accept a net.Listener from the user + // in Serve(), we cannot close the listener when we move to + // "not-serving". Closing the connection immediately upon accepting + // is one of the other ways to implement the "not-serving" mode as + // outlined in gRFC A36. + l.mu.Unlock() + conn.Close() + continue + } + + destIP, _ := netip.AddrFromSlice(destAddr.IP) + srcIP, _ := netip.AddrFromSlice(srcAddr.IP) + fc, err := l.activeFilterChainManager.lookup(lookupParams{ + isUnspecifiedListener: l.isUnspecifiedAddr, + dstAddr: destIP.Unmap(), + srcAddr: srcIP.Unmap(), + srcPort: srcAddr.Port, + }) + if err != nil { + l.mu.Unlock() + // When a matching filter chain is not found, we close the + // connection right away, but do not return an error back to + // `grpc.Serve()` from where this Accept() was invoked. Returning an + // error to `grpc.Serve()` causes the server to shutdown. If we want + // to avoid the server from shutting down, we would need to return + // an error type which implements the `Temporary() bool` method, + // which is invoked by `grpc.Serve()` to see if the returned error + // represents a temporary condition. In the case of a temporary + // error, `grpc.Serve()` method sleeps for a small duration and + // therefore ends up blocking all connection attempts during that + // time frame, which is also not ideal for an error like this. + l.logger.Warningf("Connection from %s to %s failed to find any matching filter chain", conn.RemoteAddr(), conn.LocalAddr()) + conn.Close() + continue + } + cw := &connWrapper{Conn: conn, filterChain: fc, parent: l, urc: fc.usableRouteConfiguration} + l.conns[cw] = true + l.mu.Unlock() + return cw, nil + } +} + +func (l *listenerWrapper) removeConn(conn *connWrapper) { + l.mu.Lock() + defer l.mu.Unlock() + delete(l.conns, conn) +} + +// Close closes the underlying listener. It also cancels the xDS watch +// registered in Serve() and closes any certificate provider instances created +// based on security configuration received in the LDS response. +func (l *listenerWrapper) Close() error { + l.closed.Fire() + l.Listener.Close() + if l.cancelWatch != nil { + l.cancelWatch() + } + l.rdsHandler.close() + l.mu.Lock() + if l.activeFilterChainManager != nil { + l.activeFilterChainManager.stop() + } + if l.pendingFilterChainManager != nil { + l.pendingFilterChainManager.stop() + } + l.mu.Unlock() + return nil +} + +// switchModeLocked switches the current mode of the listener wrapper. It also +// gracefully closes any connections if the listener wrapper transitions into +// not serving. If the serving mode has changed, it invokes the registered mode +// change callback. +func (l *listenerWrapper) switchModeLocked(newMode connectivity.ServingMode, err error) { + if l.mode == newMode && l.mode == connectivity.ServingModeServing { + // Redundant updates are suppressed only when we are SERVING and the new + // mode is also SERVING. In the other case (where we are NOT_SERVING and the + // new mode is also NOT_SERVING), the update is not suppressed as: + // 1. the error may have change + // 2. it provides a timestamp of the last backoff attempt + return + } + l.mode = newMode + if l.mode == connectivity.ServingModeNotServing { + connsToClose := l.conns + l.conns = make(map[*connWrapper]bool) + go func() { + for conn := range connsToClose { + conn.Drain() + } + }() + } + // The XdsServer API will allow applications to register a "serving state" + // callback to be invoked when the server begins serving and when the + // server encounters errors that force it to be "not serving". If "not + // serving", the callback must be provided error information, for + // debugging use by developers - A36. + if l.modeCallback != nil { + l.modeCallback(l.Listener.Addr(), newMode, err) + } +} + +// getOrCreateServerFilterLocked retrieves an existing server filter from the +// httpFilters map or creates a new one if it doesn't exist. It uses the filter +// builder to create a new server filter and stores it in the httpFilters map +// for future use. +// +// Must be called with l.mu held. +func (l *listenerWrapper) getOrCreateServerFilterLocked(filter xdsresource.HTTPFilter) (httpfilter.ServerFilter, error) { + builder, ok := filter.Filter.(httpfilter.ServerFilterBuilder) + if !ok { + return nil, fmt.Errorf("filter %q does not support use in server", filter.Name) + } + return getOrCreateServerFilterWithMap(l.httpFilters, builder, newServerFilterKey(&filter)), nil +} + +// This functionality is put in a separate function to allow for testing with a +// custom map. +func getOrCreateServerFilterWithMap(httpFilters map[serverFilterKey]*refCountedServerFilter, builder httpfilter.ServerFilterBuilder, key serverFilterKey) httpfilter.ServerFilter { + serverFilter, ok := httpFilters[key] + if ok { + serverFilter.incRef() + return serverFilter + } + + sf := builder.BuildServerFilter() + serverFilter = &refCountedServerFilter{ServerFilter: sf} + httpFilters[key] = serverFilter + serverFilter.incRef() + return serverFilter +} + +// ldsWatcher implements the xdsresource.ListenerWatcher interface and is +// passed to the WatchListener API. +type ldsWatcher struct { + parent *listenerWrapper + logger *internalgrpclog.PrefixLogger + name string +} + +func (lw *ldsWatcher) ResourceChanged(update *xdsresource.ListenerUpdate, onDone func()) { + defer onDone() + if lw.parent.closed.HasFired() { + lw.logger.Warningf("Resource %q received update: %#v after listener was closed", lw.name, update) + return + } + if lw.logger.V(2) { + lw.logger.Infof("LDS watch for resource %q received update: %#v", lw.name, update) + } + l := lw.parent + ilc := update.TCPListener + + // A server-side listener update must contain inbound listener configuration + // (TCPListener). If it is nil, it indicates that a client-side listener (API + // Listener) resource was received instead. In this case, we transition the + // server to non-serving mode. + // + // Also, make sure that the socket address on the received Listener resource + // matches the address of the net.Listener passed to us by the user. This + // check is done here instead of at the XDSClient layer because of the + // following couple of reasons: + // - XDSClient cannot know the listening address of every listener in the + // system, and hence cannot perform this check. + // - this is a very context-dependent check and only the server has the + // appropriate context to perform this check. + // + // What this means is that the XDSClient has ACKed a resource which can push + // the server into a "not serving" mode. This is not ideal, but this is + // what we have decided to do. + // + // A port value of 0 in the xDS Listener matches any listening port, but the + // listener address itself must still match exactly. + if ilc == nil || ilc.Address != l.addr || (ilc.Port != "0" && ilc.Port != l.port) { + // TODO(purnesh42h): Are there any other cases where this can be + // treated as an ambient error? + l.mu.Lock() + var err error + if ilc == nil { + err = fmt.Errorf("[xDS node id: %v]: %w", l.xdsNodeID, fmt.Errorf("received client-side listener resource %q on server-side", lw.name)) + } else { + err = fmt.Errorf("[xDS node id: %v]: %w", l.xdsNodeID, fmt.Errorf("address (%s:%s) in Listener update does not match listening address: (%s:%s)", ilc.Address, ilc.Port, l.addr, l.port)) + } + l.switchModeLocked(connectivity.ServingModeNotServing, err) + l.mu.Unlock() + return + } + + fcm := newFilterChainManager(&ilc.FilterChains, &ilc.DefaultFilterChain) + l.pendingFilterChainManager = fcm + l.rdsHandler.updateRouteNamesToWatch(fcm.routeConfigNames) + + if l.rdsHandler.determineRouteConfigurationReady() { + l.maybeUpdateFilterChains() + } +} + +func (lw *ldsWatcher) ResourceError(err error, onDone func()) { + defer onDone() + if lw.parent.closed.HasFired() { + lw.logger.Warningf("Resource %q received resource error: %v after listener was closed", lw.name, err) + return + } + if lw.logger.V(2) { + lw.logger.Infof("LDS watch for resource %q reported resource error: %v", lw.name, err) + } + + l := lw.parent + l.mu.Lock() + defer l.mu.Unlock() + l.switchModeLocked(connectivity.ServingModeNotServing, err) + if l.activeFilterChainManager != nil { + l.activeFilterChainManager.stop() + l.activeFilterChainManager = nil + } + if l.pendingFilterChainManager != nil { + l.pendingFilterChainManager.stop() + l.pendingFilterChainManager = nil + } + l.rdsHandler.updateRouteNamesToWatch(make(map[string]bool)) +} + +func (lw *ldsWatcher) AmbientError(err error, onDone func()) { + defer onDone() + if lw.parent.closed.HasFired() { + lw.logger.Warningf("Resource %q received ambient error: %v after listener was closed", lw.name, err) + return + } + if lw.logger.V(2) { + lw.logger.Infof("LDS watch for resource %q reported ambient error: %v", lw.name, err) + } +} diff --git a/internal/xds/server/listener_wrapper_test.go b/internal/xds/server/listener_wrapper_test.go new file mode 100644 index 0000000..12578f3 --- /dev/null +++ b/internal/xds/server/listener_wrapper_test.go @@ -0,0 +1,148 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package server + +import ( + "errors" + "sync/atomic" + "testing" + + "github.com/google/go-cmp/cmp" +) + +// Test verifies that an error received from RDS before any valid update is +// received is correctly propagated to the filter chains. +func (s) TestHandleRDSUpdate_ErrorBeforeUpdate(t *testing.T) { + const routeName = "route-name" + nodeID := "node-id" + rdsErr := errors.New("rds error") + + // Create a filter chain that points to routeName. + fc := &filterChain{ + routeConfigName: routeName, + usableRouteConfiguration: &atomic.Pointer[usableRouteConfiguration]{}, + } + fc.usableRouteConfiguration.Store(&usableRouteConfiguration{}) // Initial state is an empty configuration with no error. + + // Create a listener wrapper with the above filter chain. This simulates the + // state of the server after LDS has been processed, but before RDS has + // returned any response for the route configuration. + lw := &listenerWrapper{ + xdsNodeID: nodeID, + activeFilterChainManager: &filterChainManager{ + filterChains: []*filterChain{fc}, + }, + rdsHandler: &rdsHandler{ + updates: make(map[string]rdsWatcherUpdate), + cancels: make(map[string]func()), + }, + } + + // Simulate an RDS update with an error for the route configuration. + lw.handleRDSUpdate(routeName, rdsWatcherUpdate{err: rdsErr}) + + // Verify that the error is propagated to the filter chain. + gotURC := fc.usableRouteConfiguration.Load() + wantURC := &usableRouteConfiguration{err: rdsErr, nodeID: nodeID} + if diff := cmp.Diff(gotURC, wantURC, cmp.AllowUnexported(usableRouteConfiguration{}), cmp.Comparer(func(x, y error) bool { + if x == nil || y == nil { + return x == y + } + return x.Error() == y.Error() + })); diff != "" { + t.Fatalf("Unexpected usableRouteConfiguration (-got, +want):\n%s", diff) + } +} + +// Test verifies that a successful RDS update correctly populates the nodeID. +func (s) TestHandleRDSUpdate_Success(t *testing.T) { + const routeName = "route-name" + nodeID := "node-id" + + // Create a filter chain that points to routeName. + fc := &filterChain{ + routeConfigName: routeName, + usableRouteConfiguration: &atomic.Pointer[usableRouteConfiguration]{}, + } + fc.usableRouteConfiguration.Store(&usableRouteConfiguration{}) + + lw := &listenerWrapper{ + xdsNodeID: nodeID, + activeFilterChainManager: &filterChainManager{ + filterChains: []*filterChain{fc}, + }, + rdsHandler: &rdsHandler{ + updates: make(map[string]rdsWatcherUpdate), + cancels: make(map[string]func()), + }, + } + + // Simulate an RDS update with a valid route configuration for the routeName. + lw.handleRDSUpdate(routeName, rdsWatcherUpdate{data: inlineRouteConfig}) + + // Verify that the nodeID is populated in the resulting configuration. + gotURC := fc.usableRouteConfiguration.Load() + if gotURC.nodeID != nodeID { + t.Fatalf("gotURC.nodeID = %q, want %q", gotURC.nodeID, nodeID) + } +} + +// Test verifies that an error received from RDS *after* a valid update is +// received is correctly propagated, and does not result in the filter chain +// keeping the "stale" successful configuration. +func (s) TestHandleRDSUpdate_ErrorAfterSuccess(t *testing.T) { + const routeName = "route-name" + nodeID := "node-id" + rdsErr := errors.New("subsequent rds error") + + fc := &filterChain{ + routeConfigName: routeName, + usableRouteConfiguration: &atomic.Pointer[usableRouteConfiguration]{}, + } + fc.usableRouteConfiguration.Store(&usableRouteConfiguration{}) + + // Create a listener wrapper with the above filter chain. This simulates the + // state of the server after LDS has been processed, but before RDS has + // returned any response for the route configuration. + lw := &listenerWrapper{ + xdsNodeID: nodeID, + activeFilterChainManager: &filterChainManager{ + filterChains: []*filterChain{fc}, + }, + rdsHandler: &rdsHandler{ + updates: make(map[string]rdsWatcherUpdate), + cancels: make(map[string]func()), + }, + } + + // Simulate an RDS update with a valid route configuration for the routeName. + lw.handleRDSUpdate(routeName, rdsWatcherUpdate{data: inlineRouteConfig}) + if err := fc.usableRouteConfiguration.Load().err; err != nil { + t.Fatalf("Initial update failed: %v", err) + } + + // Simulate an RDS update with an error for the route configuration. + lw.handleRDSUpdate(routeName, rdsWatcherUpdate{err: rdsErr}) + + // Verify that the error is propagated. + gotURC := fc.usableRouteConfiguration.Load() + if !errors.Is(gotURC.err, rdsErr) { + t.Fatalf("urc.err = %v, want %v", gotURC.err, rdsErr) + } +} diff --git a/internal/xds/server/rds_handler.go b/internal/xds/server/rds_handler.go new file mode 100644 index 0000000..53ebef5 --- /dev/null +++ b/internal/xds/server/rds_handler.go @@ -0,0 +1,188 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package server + +import ( + "sync" + + igrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" +) + +// rdsHandler handles any RDS queries that need to be started for a given server +// side listeners Filter Chains (i.e. not inline). It persists rdsWatcher +// updates for later use and also determines whether all the rdsWatcher updates +// needed have been received or not. +type rdsHandler struct { + xdsC XDSClient + xdsNodeID string + logger *igrpclog.PrefixLogger + + callback func(string, rdsWatcherUpdate) + + // updates is a map from routeName to rdsWatcher update, including + // RouteConfiguration resources and any errors received. If not written in + // this map, no RouteConfiguration or error for that route name yet. If + // update set in value, use that as valid route configuration, otherwise + // treat as an error case and fail at L7 level. + updates map[string]rdsWatcherUpdate + + mu sync.Mutex + cancels map[string]func() +} + +// newRDSHandler creates a new rdsHandler to watch for RouteConfiguration +// resources. listenerWrapper updates the list of route names to watch by +// calling updateRouteNamesToWatch() upon receipt of new Listener configuration. +func newRDSHandler(cb func(string, rdsWatcherUpdate), xdsC XDSClient, logger *igrpclog.PrefixLogger) *rdsHandler { + r := &rdsHandler{ + xdsC: xdsC, + logger: logger, + callback: cb, + updates: make(map[string]rdsWatcherUpdate), + cancels: make(map[string]func()), + } + r.xdsNodeID = xdsC.BootstrapConfig().Node().GetId() + return r +} + +// updateRouteNamesToWatch handles a list of route names to watch for a given +// server side listener (if a filter chain specifies dynamic +// RouteConfiguration). This function handles all the logic with respect to any +// routes that may have been added or deleted as compared to what was previously +// present. Must be called within an xDS Client callback. +func (rh *rdsHandler) updateRouteNamesToWatch(routeNamesToWatch map[string]bool) { + rh.mu.Lock() + defer rh.mu.Unlock() + // Add and start watches for any new routes in routeNamesToWatch. + for routeName := range routeNamesToWatch { + if _, ok := rh.cancels[routeName]; !ok { + // The xDS client keeps a reference to the watcher until the cancel + // func is invoked. So, we don't need to keep a reference for fear + // of it being garbage collected. + w := &rdsWatcher{parent: rh, routeName: routeName} + cancel := xdsresource.WatchRouteConfig(rh.xdsC, routeName, w) + // Set bit on cancel function to eat any RouteConfiguration calls + // for this watcher after it has been canceled. + rh.cancels[routeName] = func() { + w.mu.Lock() + w.canceled = true + w.mu.Unlock() + cancel() + } + } + } + + // Delete and cancel watches for any routes from persisted routeNamesToWatch + // that are no longer present. + for routeName := range rh.cancels { + if _, ok := routeNamesToWatch[routeName]; !ok { + rh.cancels[routeName]() + delete(rh.cancels, routeName) + delete(rh.updates, routeName) + } + } +} + +// determines if all dynamic RouteConfiguration needed has received +// configuration or update. Must be called from an xDS Client Callback. +func (rh *rdsHandler) determineRouteConfigurationReady() bool { + // Safe to read cancels because only written to in other parts of xDS Client + // Callbacks, which are sync. + return len(rh.updates) == len(rh.cancels) +} + +// close() is meant to be called by wrapped listener when the wrapped listener +// is closed, and it cleans up resources by canceling all the active RDS +// watches. +func (rh *rdsHandler) close() { + rh.mu.Lock() + defer rh.mu.Unlock() + for _, cancel := range rh.cancels { + cancel() + } +} + +type rdsWatcherUpdate struct { + data *xdsresource.RouteConfigUpdate + err error +} + +// rdsWatcher implements the xdsresource.RouteConfigWatcher interface and is +// passed to the WatchRouteConfig API. +type rdsWatcher struct { + parent *rdsHandler + logger *igrpclog.PrefixLogger + routeName string + + mu sync.Mutex + canceled bool // eats callbacks if true +} + +func (rw *rdsWatcher) ResourceChanged(update *xdsresource.RouteConfigUpdate, onDone func()) { + defer onDone() + rw.mu.Lock() + if rw.canceled { + rw.mu.Unlock() + return + } + rw.mu.Unlock() + if rw.logger.V(2) { + rw.logger.Infof("RDS watch for resource %q received update: %#v", rw.routeName, update) + } + + routeName := rw.routeName + rwu := rdsWatcherUpdate{data: update} + rw.parent.updates[routeName] = rwu + rw.parent.callback(routeName, rwu) +} + +func (rw *rdsWatcher) ResourceError(err error, onDone func()) { + defer onDone() + rw.mu.Lock() + if rw.canceled { + rw.mu.Unlock() + return + } + rw.mu.Unlock() + if rw.logger.V(2) { + rw.logger.Infof("RDS watch for resource %q reported resource error", rw.routeName) + } + + routeName := rw.routeName + rwu := rdsWatcherUpdate{err: err} + rw.parent.updates[routeName] = rwu + rw.parent.callback(routeName, rwu) +} + +func (rw *rdsWatcher) AmbientError(err error, onDone func()) { + defer onDone() + rw.mu.Lock() + if rw.canceled { + rw.mu.Unlock() + return + } + rw.mu.Unlock() + if rw.logger.V(2) { + rw.logger.Infof("RDS watch for resource %q reported ambient error: %v", rw.routeName, err) + } + routeName := rw.routeName + rwu := rw.parent.updates[routeName] + rw.parent.callback(routeName, rwu) +} diff --git a/internal/xds/server/rds_handler_test.go b/internal/xds/server/rds_handler_test.go new file mode 100644 index 0000000..429ccaa --- /dev/null +++ b/internal/xds/server/rds_handler_test.go @@ -0,0 +1,272 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package server + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) + +const ( + listenerName = "listener" + clusterName = "cluster" + + route1 = "route1" + route2 = "route2" + route3 = "route3" + route4 = "route4" +) + +// xdsSetupForTests performs the following setup actions: +// - spins up an xDS management server +// - creates an xDS client with a bootstrap configuration pointing to the above +// management server +// +// Returns the following: +// - a reference to the management server +// - nodeID to use when pushing resources to the management server +// - a channel to read lds resource names received by the management server +// - a channel to read rds resource names received by the management server +// - an xDS client to pass to the rdsHandler under test +func xdsSetupForTests(t *testing.T) (*e2e.ManagementServer, string, chan []string, chan []string, xdsclient.XDSClient) { + t.Helper() + + ldsNamesCh := make(chan []string, 1) + rdsNamesCh := make(chan []string, 1) + + // Setup the management server to push the requested route configuration + // resource names on to a channel for the test to inspect. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + switch req.GetTypeUrl() { + case version.V3ListenerURL: // Waits on the listener, and route config below... + select { + case <-ldsNamesCh: + default: + } + select { + case ldsNamesCh <- req.GetResourceNames(): + default: + } + case version.V3RouteConfigURL: // waits on route config names here... + select { + case <-rdsNamesCh: + default: + } + select { + case rdsNamesCh <- req.GetResourceNames(): + default: + } + default: + return fmt.Errorf("unexpected resources %v of type %q requested", req.GetResourceNames(), req.GetTypeUrl()) + } + return nil + }, + AllowResourceSubset: true, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + xdsC, cancel, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(cancel) + + return mgmtServer, nodeID, ldsNamesCh, rdsNamesCh, xdsC +} + +// Waits for the wantNames to be pushed on to namesCh. Fails the test by calling +// t.Fatal if the context expires before that. +func waitForResourceNames(ctx context.Context, t *testing.T, namesCh chan []string, wantNames []string) { + t.Helper() + + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + select { + case <-ctx.Done(): + case gotNames := <-namesCh: + if cmp.Equal(gotNames, wantNames, cmpopts.EquateEmpty(), cmpopts.SortSlices(func(s1, s2 string) bool { return s1 < s2 })) { + return + } + t.Logf("Received resource names %v, want %v", gotNames, wantNames) + } + } + t.Fatalf("Timeout waiting for resource to be requested from the management server") +} + +func routeConfigResourceForName(name string) *v3routepb.RouteConfiguration { + return e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: name, + ListenerName: listenerName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeCluster, + ClusterName: clusterName, + }) +} + +type testCallbackVerify struct { + ch chan callbackStruct +} + +type callbackStruct struct { + routeName string + rwu rdsWatcherUpdate +} + +func (tcv *testCallbackVerify) testCallback(routeName string, rwu rdsWatcherUpdate) { + tcv.ch <- callbackStruct{routeName: routeName, rwu: rwu} +} + +func verifyRouteName(ctx context.Context, t *testing.T, ch chan callbackStruct, want callbackStruct) { + t.Helper() + select { + case got := <-ch: + if diff := cmp.Diff(got.routeName, want.routeName); diff != "" { + t.Fatalf("unexpected update received (-got, +want):%v, want: %v", got, want) + } + case <-ctx.Done(): + t.Fatalf("timeout waiting for callback") + } +} + +// TestRDSHandler tests the RDS Handler. It first configures the rds handler to +// watch route 1 and 2. Before receiving both RDS updates, it should not be +// ready, but after receiving both, it should be ready. It then tells the rds +// handler to watch route 1 2 and 3. It should not be ready until it receives +// route3 from the management server. It then configures the rds handler to +// watch route 1 and 3. It should immediately be ready. It then configures the +// rds handler to watch route 1 and 4. It should not be ready until it receives +// an rds update for route 4. +func (s) TestRDSHandler(t *testing.T) { + mgmtServer, nodeID, _, rdsNamesCh, xdsC := xdsSetupForTests(t) + + ch := make(chan callbackStruct, 1) + tcv := &testCallbackVerify{ch: ch} + rh := newRDSHandler(tcv.testCallback, xdsC, nil) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Configure the management server with a single route config resource. + routeResource1 := routeConfigResourceForName(route1) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{routeResource1}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + rh.updateRouteNamesToWatch(map[string]bool{route1: true, route2: true}) + waitForResourceNames(ctx, t, rdsNamesCh, []string{route1, route2}) + verifyRouteName(ctx, t, ch, callbackStruct{routeName: route1}) + + // The rds handler update should not be ready. + if got := rh.determineRouteConfigurationReady(); got != false { + t.Fatalf("rh.determineRouteConfigurationReady: %v, want: false", false) + } + + // Configure the management server both route config resources. + routeResource2 := routeConfigResourceForName(route2) + resources.Routes = []*v3routepb.RouteConfiguration{routeResource1, routeResource2} + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + verifyRouteName(ctx, t, ch, callbackStruct{routeName: route2}) + + if got := rh.determineRouteConfigurationReady(); got != true { + t.Fatalf("rh.determineRouteConfigurationReady: %v, want: true", got) + } + + rh.updateRouteNamesToWatch(map[string]bool{route1: true, route2: true, route3: true}) + waitForResourceNames(ctx, t, rdsNamesCh, []string{route1, route2, route3}) + if got := rh.determineRouteConfigurationReady(); got != false { + t.Fatalf("rh.determineRouteConfigurationReady: %v, want: false", got) + } + + // Configure the management server with route config resources. + routeResource3 := routeConfigResourceForName(route3) + resources.Routes = []*v3routepb.RouteConfiguration{routeResource1, routeResource2, routeResource3} + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + verifyRouteName(ctx, t, ch, callbackStruct{routeName: route3}) + + if got := rh.determineRouteConfigurationReady(); got != true { + t.Fatalf("rh.determineRouteConfigurationReady: %v, want: true", got) + } + // Update to route 1 and route 2. Should immediately go ready. + rh.updateRouteNamesToWatch(map[string]bool{route1: true, route3: true}) + if got := rh.determineRouteConfigurationReady(); got != true { + t.Fatalf("rh.determineRouteConfigurationReady: %v, want: true", got) + } + + // Update to route 1 and route 4. No route 4, so should not be ready. + rh.updateRouteNamesToWatch(map[string]bool{route1: true, route4: true}) + waitForResourceNames(ctx, t, rdsNamesCh, []string{route1, route4}) + if got := rh.determineRouteConfigurationReady(); got != false { + t.Fatalf("rh.determineRouteConfigurationReady: %v, want: false", got) + } + routeResource4 := routeConfigResourceForName(route4) + resources.Routes = []*v3routepb.RouteConfiguration{routeResource1, routeResource2, routeResource3, routeResource4} + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + verifyRouteName(ctx, t, ch, callbackStruct{routeName: route4}) + if got := rh.determineRouteConfigurationReady(); got != true { + t.Fatalf("rh.determineRouteConfigurationReady: %v, want: true", got) + } +} diff --git a/internal/xds/server/routing.go b/internal/xds/server/routing.go new file mode 100644 index 0000000..36f4af1 --- /dev/null +++ b/internal/xds/server/routing.go @@ -0,0 +1,192 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package server + +import ( + "errors" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/metadata" +) + +// RouteAndProcess routes the incoming RPC to a configured route in the route +// table and also processes the RPC by running the incoming RPC through any HTTP +// Filters configured. +func RouteAndProcess(ss grpc.ServerStream) (grpc.ServerStream, error) { + ctx := ss.Context() + conn := transport.GetConnection(ctx) + cw, ok := conn.(*connWrapper) + if !ok { + return nil, errors.New("missing virtual hosts in incoming context") + } + + rc := cw.urc.Load() + // Error out at routing l7 level with a status code UNAVAILABLE, represents + // an nack before usable route configuration or resource not found for RDS + // or error combining LDS + RDS (Shouldn't happen). + if rc.err != nil { + if logger.V(2) { + logger.Infof("RPC on connection with xDS Configuration error: %v", rc.err) + } + return nil, rc.statusErrWithNodeID(codes.Unavailable, "error from xDS configuration for matched route configuration: %v", rc.err) + } + + mn, ok := grpc.Method(ctx) + if !ok { + return nil, rc.statusErrWithNodeID(codes.Internal, "missing method name in incoming context") + } + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, rc.statusErrWithNodeID(codes.Internal, "missing metadata in incoming context") + } + // A41 added logic to the core grpc implementation to guarantee that once the + // RPC gets to this point, there will be a single, unambiguous authority + // present in the header map. But add a defensive check to ensure authority + // header is present. + authority := md.Get(":authority") + if len(authority) == 0 { + return nil, rc.statusErrWithNodeID(codes.Internal, "no :authority header present") + } + vh := findBestMatchingVirtualHostServer(authority[0], rc.vhs) + if vh == nil { + return nil, rc.statusErrWithNodeID(codes.Unavailable, "the incoming RPC did not match a configured Virtual Host") + } + + var rwi *routeWithInterceptors + for _, r := range vh.routes { + if r.matcher.Match(mn, md) { + // "NonForwardingAction is expected for all Routes used on + // server-side; a route with an inappropriate action causes RPCs + // matching that route to fail with UNAVAILABLE." - A36 + if r.actionType != xdsresource.RouteActionNonForwardingAction { + return nil, rc.statusErrWithNodeID(codes.Unavailable, "the incoming RPC matched to a route that was not of action type non forwarding") + } + rwi = &r + break + } + } + if rwi == nil { + return nil, rc.statusErrWithNodeID(codes.Unavailable, "the incoming RPC did not match a configured Route") + } + if rwi.interceptor != nil { + return rwi.interceptor.InterceptRPC(ss) + } + return ss, nil +} + +// findBestMatchingVirtualHostServer returns the virtual host whose domains field best +// matches host +// +// The domains field support 4 different matching pattern types: +// +// - Exact match +// - Suffix match (e.g. “*ABC”) +// - Prefix match (e.g. “ABC*) +// - Universal match (e.g. “*”) +// +// The best match is defined as: +// - A match is better if it’s matching pattern type is better. +// * Exact match > suffix match > prefix match > universal match. +// +// - If two matches are of the same pattern type, the longer match is +// better. +// * This is to compare the length of the matching pattern, e.g. “*ABCDE” > +// “*ABC” +func findBestMatchingVirtualHostServer(authority string, vHosts []virtualHostWithInterceptors) *virtualHostWithInterceptors { + var ( + matchVh *virtualHostWithInterceptors + matchType = domainMatchTypeInvalid + matchLen int + ) + for _, vh := range vHosts { + for _, domain := range vh.domains { + typ, matched := match(domain, authority) + if typ == domainMatchTypeInvalid { + // The rds response is invalid. + return nil + } + if matchType.betterThan(typ) || matchType == typ && matchLen >= len(domain) || !matched { + // The previous match has better type, or the previous match has + // better length, or this domain isn't a match. + continue + } + matchVh = &vh + matchType = typ + matchLen = len(domain) + } + } + return matchVh +} + +type domainMatchType int + +const ( + domainMatchTypeInvalid domainMatchType = iota + domainMatchTypeUniversal + domainMatchTypePrefix + domainMatchTypeSuffix + domainMatchTypeExact +) + +// Exact > Suffix > Prefix > Universal > Invalid. +func (t domainMatchType) betterThan(b domainMatchType) bool { + return t > b +} + +func matchTypeForDomain(d string) domainMatchType { + if d == "" { + return domainMatchTypeInvalid + } + if d == "*" { + return domainMatchTypeUniversal + } + if strings.HasPrefix(d, "*") { + return domainMatchTypeSuffix + } + if strings.HasSuffix(d, "*") { + return domainMatchTypePrefix + } + if strings.Contains(d, "*") { + return domainMatchTypeInvalid + } + return domainMatchTypeExact +} + +func match(domain, host string) (domainMatchType, bool) { + switch typ := matchTypeForDomain(domain); typ { + case domainMatchTypeInvalid: + return typ, false + case domainMatchTypeUniversal: + return typ, true + case domainMatchTypePrefix: + // abc.* + return typ, strings.HasPrefix(host, strings.TrimSuffix(domain, "*")) + case domainMatchTypeSuffix: + // *.123 + return typ, strings.HasSuffix(host, strings.TrimPrefix(domain, "*")) + case domainMatchTypeExact: + return typ, domain == host + default: + return domainMatchTypeInvalid, false + } +} diff --git a/internal/xds/server/routing_test.go b/internal/xds/server/routing_test.go new file mode 100644 index 0000000..880471b --- /dev/null +++ b/internal/xds/server/routing_test.go @@ -0,0 +1,155 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package server + +import ( + "context" + "strings" + "sync/atomic" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +func (s) TestMatchTypeForDomain(t *testing.T) { + tests := []struct { + d string + want domainMatchType + }{ + {d: "", want: domainMatchTypeInvalid}, + {d: "*", want: domainMatchTypeUniversal}, + {d: "bar.*", want: domainMatchTypePrefix}, + {d: "*.abc.com", want: domainMatchTypeSuffix}, + {d: "foo.bar.com", want: domainMatchTypeExact}, + {d: "foo.*.com", want: domainMatchTypeInvalid}, + } + for _, tt := range tests { + if got := matchTypeForDomain(tt.d); got != tt.want { + t.Errorf("matchTypeForDomain(%q) = %v, want %v", tt.d, got, tt.want) + } + } +} + +func (s) TestMatch(t *testing.T) { + tests := []struct { + name string + domain string + host string + wantTyp domainMatchType + wantMatched bool + }{ + {name: "invalid-empty", domain: "", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false}, + {name: "invalid", domain: "a.*.b", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false}, + {name: "universal", domain: "*", host: "abc.com", wantTyp: domainMatchTypeUniversal, wantMatched: true}, + {name: "prefix-match", domain: "abc.*", host: "abc.123", wantTyp: domainMatchTypePrefix, wantMatched: true}, + {name: "prefix-no-match", domain: "abc.*", host: "abcd.123", wantTyp: domainMatchTypePrefix, wantMatched: false}, + {name: "suffix-match", domain: "*.123", host: "abc.123", wantTyp: domainMatchTypeSuffix, wantMatched: true}, + {name: "suffix-no-match", domain: "*.123", host: "abc.1234", wantTyp: domainMatchTypeSuffix, wantMatched: false}, + {name: "exact-match", domain: "foo.bar", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: true}, + {name: "exact-no-match", domain: "foo.bar.com", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotTyp, gotMatched := match(tt.domain, tt.host); gotTyp != tt.wantTyp || gotMatched != tt.wantMatched { + t.Errorf("match() = %v, %v, want %v, %v", gotTyp, gotMatched, tt.wantTyp, tt.wantMatched) + } + }) + } +} + +func (s) TestFindBestMatchingVirtualHost(t *testing.T) { + var ( + oneExactMatch = virtualHostWithInterceptors{domains: []string{"foo.bar.com"}} + oneSuffixMatch = virtualHostWithInterceptors{domains: []string{"*.bar.com"}} + onePrefixMatch = virtualHostWithInterceptors{domains: []string{"foo.bar.*"}} + oneUniversalMatch = virtualHostWithInterceptors{domains: []string{"*"}} + longExactMatch = virtualHostWithInterceptors{domains: []string{"v2.foo.bar.com"}} + multipleMatch = virtualHostWithInterceptors{domains: []string{"pi.foo.bar.com", "314.*", "*.159"}} + vhs = []virtualHostWithInterceptors{oneExactMatch, oneSuffixMatch, onePrefixMatch, oneUniversalMatch, longExactMatch, multipleMatch} + ) + + tests := []struct { + name string + host string + vHosts []virtualHostWithInterceptors + want *virtualHostWithInterceptors + }{ + {name: "exact-match", host: "foo.bar.com", vHosts: vhs, want: &oneExactMatch}, + {name: "suffix-match", host: "123.bar.com", vHosts: vhs, want: &oneSuffixMatch}, + {name: "prefix-match", host: "foo.bar.org", vHosts: vhs, want: &onePrefixMatch}, + {name: "universal-match", host: "abc.123", vHosts: vhs, want: &oneUniversalMatch}, + {name: "long-exact-match", host: "v2.foo.bar.com", vHosts: vhs, want: &longExactMatch}, + // Matches suffix "*.bar.com" and exact "pi.foo.bar.com". Takes exact. + {name: "multiple-match-exact", host: "pi.foo.bar.com", vHosts: vhs, want: &multipleMatch}, + // Matches suffix "*.159" and prefix "foo.bar.*". Takes suffix. + {name: "multiple-match-suffix", host: "foo.bar.159", vHosts: vhs, want: &multipleMatch}, + // Matches suffix "*.bar.com" and prefix "314.*". Takes suffix. + {name: "multiple-match-prefix", host: "314.bar.com", vHosts: vhs, want: &oneSuffixMatch}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := findBestMatchingVirtualHostServer(tt.host, tt.vHosts); !cmp.Equal(got, tt.want, cmp.AllowUnexported(virtualHostWithInterceptors{}, routeWithInterceptors{})) { + t.Errorf("findBestMatchingVirtualHostServer() = %v, want %v", got, tt.want) + } + }) + } +} + +type testServerTransportStream struct { + method string +} + +func (s *testServerTransportStream) Method() string { return s.method } +func (s *testServerTransportStream) SetHeader(metadata.MD) error { return nil } +func (s *testServerTransportStream) SendHeader(metadata.MD) error { return nil } +func (s *testServerTransportStream) SetTrailer(metadata.MD) error { return nil } + +type testServerStream struct { + grpc.ServerStream + ctx context.Context +} + +func (s *testServerStream) Context() context.Context { + return s.ctx +} + +func (s) TestRouteAndProcess_MissingAuthority(t *testing.T) { + var ptr atomic.Pointer[usableRouteConfiguration] + ptr.Store(&usableRouteConfiguration{}) + cw := &connWrapper{urc: &ptr} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = transport.SetConnection(ctx, cw) + ctx = grpc.NewContextWithServerTransportStream(ctx, &testServerTransportStream{method: "/test.Service/Method"}) + ctx = metadata.NewIncomingContext(ctx, metadata.MD{}) + + ss := &testServerStream{ctx: ctx} + _, err := RouteAndProcess(ss) + if status.Code(err) != codes.Internal { + t.Fatalf("RouteAndProcess() returned error code %v, want %v", status.Code(err), codes.Internal) + } + if !strings.Contains(err.Error(), "no :authority header present") { + t.Fatalf("RouteAndProcess() returned error message %q, want %q", err.Error(), "no :authority header present") + } +} diff --git a/internal/xds/server/server_options.go b/internal/xds/server/server_options.go new file mode 100644 index 0000000..e9e638c --- /dev/null +++ b/internal/xds/server/server_options.go @@ -0,0 +1,69 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package server + +import ( + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/internal/xds/xdsclient" +) + +// Options contains options used by an xDS-enabled gRPC server. +// +// This type is internal so that the public xds package can expose server +// options without also owning their application and storage. +type Options struct { + ModeCallback ServingModeCallback + ClientPoolForTesting *xdsclient.Pool + OverrideListenerResourceName func(net.Addr) string +} + +type serverOption struct { + grpc.EmptyServerOption + apply func(*Options) +} + +// NewServerOption returns a grpc.ServerOption which applies f to the internal +// xDS server options. +func NewServerOption(f func(*Options)) grpc.ServerOption { + return &serverOption{apply: f} +} + +// ApplyServerOptions applies all internal xDS server options in opts to so. +func ApplyServerOptions(opts []grpc.ServerOption, so *Options) { + for _, opt := range opts { + if o, ok := opt.(*serverOption); ok { + o.apply(so) + } + } +} + +// OverrideListenerResourceName returns a server option that overrides the LDS +// resource name selected for an xDS server listener. The supplied function is +// called by Serve with the address returned by the listener's Addr method, and +// its return value is used as-is as the LDS listener resource name. +// +// The function is called once for each Serve invocation that gets past listener +// validation and the server-stopped check. If Serve is called concurrently, the +// function may be called concurrently and must be safe for concurrent use. +func OverrideListenerResourceName(f func(net.Addr) string) grpc.ServerOption { + return NewServerOption(func(o *Options) { + o.OverrideListenerResourceName = f + }) +} diff --git a/internal/xds/test/e2e/README.md b/internal/xds/test/e2e/README.md new file mode 100644 index 0000000..33cffa0 --- /dev/null +++ b/internal/xds/test/e2e/README.md @@ -0,0 +1,19 @@ +Build client and server binaries. + +```sh +go build -o ./binaries/client ../../../../interop/xds/client/ +go build -o ./binaries/server ../../../../interop/xds/server/ +``` + +Run the test + +```sh +go test . -v +``` + +The client/server paths are flags + +```sh +go test . -v -client=$HOME/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-client +``` +Note that grpc logs are only turned on for Go. diff --git a/internal/xds/test/e2e/controlplane.go b/internal/xds/test/e2e/controlplane.go new file mode 100644 index 0000000..aae80a4 --- /dev/null +++ b/internal/xds/test/e2e/controlplane.go @@ -0,0 +1,45 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/internal/testutils/xds/e2e" +) + +type controlPlane struct { + server *e2e.ManagementServer + nodeID string + bootstrapContent string +} + +func newControlPlane(t *testing.T) (*controlPlane, error) { + // Spin up an xDS management server on a local port. + server := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, server.Address) + + return &controlPlane{ + server: server, + nodeID: nodeID, + bootstrapContent: string(bootstrapContents), + }, nil +} diff --git a/internal/xds/test/e2e/e2e.go b/internal/xds/test/e2e/e2e.go new file mode 100644 index 0000000..2869a93 --- /dev/null +++ b/internal/xds/test/e2e/e2e.go @@ -0,0 +1,173 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package e2e implements xds e2e tests using go-control-plane. +package e2e + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + + "google.golang.org/grpc" + channelzgrpc "google.golang.org/grpc/channelz/grpc_channelz_v1" + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" + "google.golang.org/grpc/credentials/insecure" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func cmd(path string, logger io.Writer, args []string, env []string) *exec.Cmd { + cmd := exec.Command(path, args...) + cmd.Env = append(os.Environ(), env...) + cmd.Stdout = logger + cmd.Stderr = logger + return cmd +} + +const ( + clientStatsPort = 60363 // TODO: make this different per-test, only needed for parallel tests. +) + +type client struct { + cmd *exec.Cmd + + target string + statsCC *grpc.ClientConn +} + +// newClient create a client with the given target and bootstrap content. +func newClient(target, binaryPath, bootstrap string, logger io.Writer, flags ...string) (*client, error) { + cmd := cmd( + binaryPath, + logger, + append([]string{ + "--server=" + target, + "--print_response=true", + "--qps=100", + fmt.Sprintf("--stats_port=%d", clientStatsPort), + }, flags...), // Append any flags from caller. + []string{ + "GRPC_GO_LOG_VERBOSITY_LEVEL=99", + "GRPC_GO_LOG_SEVERITY_LEVEL=info", + "GRPC_XDS_BOOTSTRAP_CONFIG=" + bootstrap, // The bootstrap content doesn't need to be quoted. + }, + ) + cmd.Start() + + cc, err := grpc.NewClient(fmt.Sprintf("localhost:%d", clientStatsPort), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true))) + if err != nil { + return nil, err + } + return &client{ + cmd: cmd, + target: target, + statsCC: cc, + }, nil +} + +func (c *client) clientStats(ctx context.Context) (*testpb.LoadBalancerStatsResponse, error) { + ccc := testgrpc.NewLoadBalancerStatsServiceClient(c.statsCC) + return ccc.GetClientStats(ctx, &testpb.LoadBalancerStatsRequest{ + NumRpcs: 100, + TimeoutSec: 10, + }) +} + +func (c *client) configRPCs(ctx context.Context, req *testpb.ClientConfigureRequest) error { + ccc := testgrpc.NewXdsUpdateClientConfigureServiceClient(c.statsCC) + _, err := ccc.Configure(ctx, req) + return err +} + +func (c *client) channelzSubChannels(ctx context.Context) ([]*channelzpb.Subchannel, error) { + ccc := channelzgrpc.NewChannelzClient(c.statsCC) + r, err := ccc.GetTopChannels(ctx, &channelzpb.GetTopChannelsRequest{}) + if err != nil { + return nil, err + } + + var ret []*channelzpb.Subchannel + for _, cc := range r.Channel { + if cc.Data.Target != c.target { + continue + } + for _, sc := range cc.SubchannelRef { + rr, err := ccc.GetSubchannel(ctx, &channelzpb.GetSubchannelRequest{SubchannelId: sc.SubchannelId}) + if err != nil { + return nil, err + } + ret = append(ret, rr.Subchannel) + } + } + return ret, nil +} + +func (c *client) stop() { + c.cmd.Process.Kill() + c.cmd.Wait() +} + +const ( + serverPort = 50051 // TODO: make this different per-test, only needed for parallel tests. +) + +type server struct { + cmd *exec.Cmd + port int +} + +// newServer creates multiple servers with the given bootstrap content. +// +// Each server gets a different hostname, in the format of +// -. +func newServers(hostnamePrefix, binaryPath, bootstrap string, logger io.Writer, count int) (_ []*server, err error) { + var ret []*server + defer func() { + if err != nil { + for _, s := range ret { + s.stop() + } + } + }() + for i := 0; i < count; i++ { + port := serverPort + i + cmd := cmd( + binaryPath, + logger, + []string{ + fmt.Sprintf("--port=%d", port), + fmt.Sprintf("--host_name_override=%s-%d", hostnamePrefix, i), + }, + []string{ + "GRPC_GO_LOG_VERBOSITY_LEVEL=99", + "GRPC_GO_LOG_SEVERITY_LEVEL=info", + "GRPC_XDS_BOOTSTRAP_CONFIG=" + bootstrap, // The bootstrap content doesn't need to be quoted., + }, + ) + cmd.Start() + ret = append(ret, &server{cmd: cmd, port: port}) + } + return ret, nil +} + +func (s *server) stop() { + s.cmd.Process.Kill() + s.cmd.Wait() +} diff --git a/internal/xds/test/e2e/e2e_test.go b/internal/xds/test/e2e/e2e_test.go new file mode 100644 index 0000000..dbbf6d0 --- /dev/null +++ b/internal/xds/test/e2e/e2e_test.go @@ -0,0 +1,258 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "bytes" + "context" + "flag" + "fmt" + "os" + "strconv" + "testing" + "time" + + "google.golang.org/grpc/internal/testutils/xds/e2e" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + clientPath = flag.String("client", "./binaries/client", "The interop client") + serverPath = flag.String("server", "./binaries/server", "The interop server") +) + +type testOpts struct { + testName string + backendCount int + clientFlags []string +} + +func setup(t *testing.T, opts testOpts) (*controlPlane, *client, []*server) { + t.Helper() + if _, err := os.Stat(*clientPath); os.IsNotExist(err) { + t.Skip("skipped because client is not found") + } + if _, err := os.Stat(*serverPath); os.IsNotExist(err) { + t.Skip("skipped because server is not found") + } + backendCount := 1 + if opts.backendCount != 0 { + backendCount = opts.backendCount + } + + cp, err := newControlPlane(t) + if err != nil { + t.Fatalf("failed to start control-plane: %v", err) + } + + var clientLog bytes.Buffer + c, err := newClient(fmt.Sprintf("xds:///%s", opts.testName), *clientPath, cp.bootstrapContent, &clientLog, opts.clientFlags...) + if err != nil { + t.Fatalf("failed to start client: %v", err) + } + t.Cleanup(c.stop) + + var serverLog bytes.Buffer + servers, err := newServers(opts.testName, *serverPath, cp.bootstrapContent, &serverLog, backendCount) + if err != nil { + t.Fatalf("failed to start server: %v", err) + } + t.Cleanup(func() { + for _, s := range servers { + s.stop() + } + }) + t.Cleanup(func() { + // TODO: find a better way to print the log. They are long, and hide the failure. + t.Logf("\n----- client logs -----\n%v", clientLog.String()) + t.Logf("\n----- server logs -----\n%v", serverLog.String()) + }) + return cp, c, servers +} + +func TestPingPong(t *testing.T) { + const testName = "pingpong" + cp, c, _ := setup(t, testOpts{testName: testName}) + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: testName, + NodeID: cp.nodeID, + Host: "localhost", + Port: serverPort, + SecLevel: e2e.SecurityLevelNone, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := cp.server.Update(ctx, resources); err != nil { + t.Fatalf("failed to update control plane resources: %v", err) + } + + st, err := c.clientStats(ctx) + if err != nil { + t.Fatalf("failed to get client stats: %v", err) + } + if st.NumFailures != 0 { + t.Fatalf("Got %v failures: %+v", st.NumFailures, st) + } +} + +// TestAffinity covers the affinity tests with ringhash policy. +// - client is configured to use ringhash, with 3 backends +// - all RPCs will hash a specific metadata header +// - verify that +// - all RPCs with the same metadata value are sent to the same backend +// - only one backend is Ready +// +// - send more RPCs with different metadata values until a new backend is picked, and verify that +// - only two backends are in Ready +func TestAffinity(t *testing.T) { + const ( + testName = "affinity" + backendCount = 3 + testMDKey = "xds_md" + testMDValue = "unary_yranu" + ) + cp, c, servers := setup(t, testOpts{ + testName: testName, + backendCount: backendCount, + clientFlags: []string{"--rpc=EmptyCall", fmt.Sprintf("--metadata=EmptyCall:%s:%s", testMDKey, testMDValue)}, + }) + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: testName, + NodeID: cp.nodeID, + Host: "localhost", + Port: serverPort, + SecLevel: e2e.SecurityLevelNone, + }) + + // Update EDS to multiple backends. + var ports []uint32 + for _, s := range servers { + ports = append(ports, uint32(s.port)) + } + edsMsg := resources.Endpoints[0] + resources.Endpoints[0] = e2e.DefaultEndpoint( + edsMsg.ClusterName, + "localhost", + ports, + ) + + // Update CDS lbpolicy to ringhash. + cdsMsg := resources.Clusters[0] + cdsMsg.LbPolicy = v3clusterpb.Cluster_RING_HASH + + // Update RDS to hash the header. + rdsMsg := resources.Routes[0] + rdsMsg.VirtualHosts[0].Routes[0].Action = &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: cdsMsg.Name}, + HashPolicy: []*v3routepb.RouteAction_HashPolicy{{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: testMDKey, + }, + }, + }}, + }} + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := cp.server.Update(ctx, resources); err != nil { + t.Fatalf("failed to update control plane resources: %v", err) + } + + // Note: We can skip CSDS check because there's no long delay as in TD. + // + // The client stats check doesn't race with the xds resource update because + // there's only one version of xds resource, updated at the beginning of the + // test. So there's no need to retry the stats call. + // + // In the future, we may add tests that update xds in the middle. Then we + // either need to retry clientStats(), or make a CSDS check before so the + // result is stable. + + st, err := c.clientStats(ctx) + if err != nil { + t.Fatalf("failed to get client stats: %v", err) + } + if st.NumFailures != 0 { + t.Fatalf("Got %v failures: %+v", st.NumFailures, st) + } + if len(st.RpcsByPeer) != 1 { + t.Fatalf("more than 1 backends got traffic: %v, want 1", st.RpcsByPeer) + } + + // Call channelz to verify that only one subchannel is in state Ready. + scs, err := c.channelzSubChannels(ctx) + if err != nil { + t.Fatalf("failed to fetch channelz: %v", err) + } + verifySubConnStates(t, scs, map[channelzpb.ChannelConnectivityState_State]int{ + channelzpb.ChannelConnectivityState_READY: 1, + channelzpb.ChannelConnectivityState_IDLE: 2, + }) + + // Send Unary call with different metadata value with integers starting from + // 0. Stop when a second peer is picked. + var ( + diffPeerPicked bool + mdValue int + ) + for !diffPeerPicked { + if err := c.configRPCs(ctx, &testpb.ClientConfigureRequest{ + Types: []testpb.ClientConfigureRequest_RpcType{ + testpb.ClientConfigureRequest_EMPTY_CALL, + testpb.ClientConfigureRequest_UNARY_CALL, + }, + Metadata: []*testpb.ClientConfigureRequest_Metadata{ + {Type: testpb.ClientConfigureRequest_EMPTY_CALL, Key: testMDKey, Value: testMDValue}, + {Type: testpb.ClientConfigureRequest_UNARY_CALL, Key: testMDKey, Value: strconv.Itoa(mdValue)}, + }, + }); err != nil { + t.Fatalf("failed to configure RPC: %v", err) + } + + st, err := c.clientStats(ctx) + if err != nil { + t.Fatalf("failed to get client stats: %v", err) + } + if st.NumFailures != 0 { + t.Fatalf("Got %v failures: %+v", st.NumFailures, st) + } + if len(st.RpcsByPeer) == 2 { + break + } + + mdValue++ + } + + // Call channelz to verify that only one subchannel is in state Ready. + scs2, err := c.channelzSubChannels(ctx) + if err != nil { + t.Fatalf("failed to fetch channelz: %v", err) + } + verifySubConnStates(t, scs2, map[channelzpb.ChannelConnectivityState_State]int{ + channelzpb.ChannelConnectivityState_READY: 2, + channelzpb.ChannelConnectivityState_IDLE: 1, + }) +} diff --git a/internal/xds/test/e2e/e2e_utils.go b/internal/xds/test/e2e/e2e_utils.go new file mode 100644 index 0000000..6ac92de --- /dev/null +++ b/internal/xds/test/e2e/e2e_utils.go @@ -0,0 +1,179 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "context" + "fmt" + "regexp" + "strings" + "testing" + + "github.com/cespare/xxhash/v2" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/resolver" +) + +func verifySubConnStates(t *testing.T, scs []*channelzpb.Subchannel, want map[channelzpb.ChannelConnectivityState_State]int) { + t.Helper() + var scStatsCount = map[channelzpb.ChannelConnectivityState_State]int{} + for _, sc := range scs { + scStatsCount[sc.Data.State.State]++ + } + if diff := cmp.Diff(scStatsCount, want); diff != "" { + t.Fatalf("got unexpected number of subchannels in state Ready, %v, scs: %v", diff, scs) + } +} + +// This function determines the stable, canonical order for any two +// resolver.Endpoint structs. +func lessEndpoint(a, b resolver.Endpoint) bool { + return getHash(a) < getHash(b) +} + +func getHash(e resolver.Endpoint) uint64 { + h := xxhash.New() + + // We iterate through all addresses to ensure the hash represents + // the full endpoint identity. + for _, addr := range e.Addresses { + h.Write([]byte(addr.Addr)) + h.Write([]byte(addr.ServerName)) + } + + return h.Sum64() +} + +// VerifyXDSConfig waits for an update on the xdsCh channel or an error on the +// errCh channel, and verifies that the received XDSConfig matches the expected +// XDS configuration. +func VerifyXDSConfig(ctx context.Context, xdsCh chan *xdsresource.XDSConfig, errCh chan error, want *xdsresource.XDSConfig) error { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for update from dependency manager") + case update := <-xdsCh: + if err := CompareXDSConfig(update, want); err != nil { + return err + } + case err := <-errCh: + return fmt.Errorf("received unexpected error from dependency manager: %v", err) + } + return nil +} + +// CompareXDSConfig compares two XDSConfig structs and returns an error if they +// do not match. +func CompareXDSConfig(gotXDSConfig, wantXDSConfig *xdsresource.XDSConfig) error { + cmpOpts := []cmp.Option{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(xdsresource.HTTPFilter{}, "Filter", "Config"), + cmpopts.IgnoreFields(xdsresource.ListenerUpdate{}, "Raw"), + cmpopts.IgnoreFields(xdsresource.RouteConfigUpdate{}, "Raw"), + cmpopts.IgnoreFields(xdsresource.ClusterUpdate{}, "Raw", "LBPolicy", "TelemetryLabels"), + cmpopts.IgnoreFields(xdsresource.EndpointsUpdate{}, "Raw"), + // Used for EndpointConfig.ResolutionNote and ClusterResult.Err fields. + cmp.Transformer("ErrorsToString", func(in error) string { + if in == nil { + return "" // Treat nil as an empty string + } + s := in.Error() + + // Replace all sequences of whitespace (including newlines and + // tabs) with a single standard space. + s = regexp.MustCompile(`\s+`).ReplaceAllString(s, " ") + + // Trim any leading/trailing space that might be left over and + // return error as string. + return strings.TrimSpace(s) + }), + cmpopts.SortSlices(lessEndpoint), + } + if diff := cmp.Diff(gotXDSConfig, wantXDSConfig, cmpOpts...); diff != "" { + return fmt.Errorf("received unexpected update from dependency manager. Diff (-got +want):\n%v", diff) + } + return nil +} + +// MakeXDSConfig creates a mock XDSConfig with a standard structure containing +// a Listener, RouteConfig, VirtualHost, and a single EDS Cluster, populated +// with the provided parameters. +func MakeXDSConfig(servicename, routeConfigName, clusterName, edsServiceName, addr string) *xdsresource.XDSConfig { + prefix := "/" + return &xdsresource.XDSConfig{ + Listener: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: routeConfigName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + RouteConfig: &xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{servicename}, + Routes: []*xdsresource.Route{{ + Prefix: &prefix, + WeightedClusters: []xdsresource.WeightedCluster{{Name: clusterName, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute, + }}, + }, + }, + }, + VirtualHost: &xdsresource.VirtualHost{ + Domains: []string{servicename}, + Routes: []*xdsresource.Route{{ + Prefix: &prefix, + WeightedClusters: []xdsresource.WeightedCluster{{Name: clusterName, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute}, + }, + }, + Clusters: map[string]*xdsresource.ClusterResult{ + clusterName: { + Config: xdsresource.ClusterConfig{Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: clusterName, + EDSServiceName: edsServiceName, + }, + EndpointConfig: &xdsresource.EndpointConfig{ + EDSUpdate: &xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + {ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: addr}}}, + HealthStatus: xdsresource.EndpointHealthStatusUnknown, + Weight: 1, + }, + }, + Weight: 1, + }, + }, + }, + }, + }, + }, + }, + } +} diff --git a/internal/xds/test/e2e/run.sh b/internal/xds/test/e2e/run.sh new file mode 100755 index 0000000..d4f3833 --- /dev/null +++ b/internal/xds/test/e2e/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +mkdir binaries +go build -o ./binaries/client ../../../../interop/xds/client/ +go build -o ./binaries/server ../../../../interop/xds/server/ +go test . diff --git a/internal/xds/testutils/balancer_test.go b/internal/xds/testutils/balancer_test.go new file mode 100644 index 0000000..73e0d1c --- /dev/null +++ b/internal/xds/testutils/balancer_test.go @@ -0,0 +1,147 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "testing" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/internal/testutils" +) + +func TestIsRoundRobin(t *testing.T) { + var ( + sc1 = &testutils.TestSubConn{} + sc2 = &testutils.TestSubConn{} + sc3 = &testutils.TestSubConn{} + ) + + testCases := []struct { + desc string + want []balancer.SubConn + got []balancer.SubConn + pass bool + }{ + { + desc: "0 element", + want: []balancer.SubConn{}, + got: []balancer.SubConn{}, + pass: true, + }, + { + desc: "1 element RR", + want: []balancer.SubConn{sc1}, + got: []balancer.SubConn{sc1, sc1, sc1, sc1}, + pass: true, + }, + { + desc: "1 element not RR", + want: []balancer.SubConn{sc1}, + got: []balancer.SubConn{sc1, sc2, sc1}, + pass: false, + }, + { + desc: "2 elements RR", + want: []balancer.SubConn{sc1, sc2}, + got: []balancer.SubConn{sc1, sc2, sc1, sc2, sc1, sc2}, + pass: true, + }, + { + desc: "2 elements RR different order from want", + want: []balancer.SubConn{sc2, sc1}, + got: []balancer.SubConn{sc1, sc2, sc1, sc2, sc1, sc2}, + pass: true, + }, + { + desc: "2 elements RR not RR, mistake in first iter", + want: []balancer.SubConn{sc1, sc2}, + got: []balancer.SubConn{sc1, sc1, sc1, sc2, sc1, sc2}, + pass: false, + }, + { + desc: "2 elements RR not RR, mistake in second iter", + want: []balancer.SubConn{sc1, sc2}, + got: []balancer.SubConn{sc1, sc2, sc1, sc1, sc1, sc2}, + pass: false, + }, + { + desc: "2 elements weighted RR", + want: []balancer.SubConn{sc1, sc1, sc2}, + got: []balancer.SubConn{sc1, sc1, sc2, sc1, sc1, sc2}, + pass: true, + }, + { + desc: "2 elements weighted RR different order", + want: []balancer.SubConn{sc1, sc1, sc2}, + got: []balancer.SubConn{sc1, sc2, sc1, sc1, sc2, sc1}, + pass: true, + }, + + { + desc: "3 elements RR", + want: []balancer.SubConn{sc1, sc2, sc3}, + got: []balancer.SubConn{sc1, sc2, sc3, sc1, sc2, sc3, sc1, sc2, sc3}, + pass: true, + }, + { + desc: "3 elements RR different order", + want: []balancer.SubConn{sc1, sc2, sc3}, + got: []balancer.SubConn{sc3, sc2, sc1, sc3, sc2, sc1}, + pass: true, + }, + { + desc: "3 elements weighted RR", + want: []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc3}, + got: []balancer.SubConn{sc1, sc2, sc3, sc1, sc2, sc1, sc1, sc2, sc3, sc1, sc2, sc1}, + pass: true, + }, + { + desc: "3 elements weighted RR not RR, mistake in first iter", + want: []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc3}, + got: []balancer.SubConn{sc1, sc2, sc1, sc1, sc2, sc1, sc1, sc2, sc3, sc1, sc2, sc1}, + pass: false, + }, + { + desc: "3 elements weighted RR not RR, mistake in second iter", + want: []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc3}, + got: []balancer.SubConn{sc1, sc2, sc3, sc1, sc2, sc1, sc1, sc1, sc3, sc1, sc2, sc1}, + pass: false, + }, + } + for _, tC := range testCases { + t.Run(tC.desc, func(t *testing.T) { + err := testutils.IsRoundRobin(tC.want, (&testClosure{r: tC.got}).next) + if err == nil != tC.pass { + t.Errorf("want pass %v, want %v, got err %v", tC.pass, tC.want, err) + } + }) + } +} + +// testClosure is a test util for TestIsRoundRobin. +type testClosure struct { + r []balancer.SubConn + i int +} + +func (tc *testClosure) next() balancer.SubConn { + ret := tc.r[tc.i] + tc.i = (tc.i + 1) % len(tc.r) + return ret +} diff --git a/internal/xds/testutils/fakeclient/client.go b/internal/xds/testutils/fakeclient/client.go new file mode 100644 index 0000000..fa46513 --- /dev/null +++ b/internal/xds/testutils/fakeclient/client.go @@ -0,0 +1,147 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package fakeclient provides a fake implementation of an xDS client. +package fakeclient + +import ( + "context" + + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/lrsclient" + "google.golang.org/grpc/internal/xds/xdsclient" +) + +// Client is a fake implementation of an xds client. It exposes a bunch of +// channels to signal the occurrence of various events. +type Client struct { + // Embed XDSClient so this fake client implements the interface, but it's + // never set (it's always nil). This may cause nil panic since not all the + // methods are implemented. + xdsclient.XDSClient + + name string + loadReportCh *testutils.Channel + lrsCancelCh *testutils.Channel + loadStore *lrsclient.LoadStore + bootstrapCfg *bootstrap.Config +} + +// ReportLoadArgs wraps the arguments passed to ReportLoad. +type ReportLoadArgs struct { + // Server is the name of the server to which the load is reported. + Server *bootstrap.ServerConfig +} + +type transportBuilder struct { +} + +func (*transportBuilder) Build(clients.ServerIdentifier) (clients.Transport, error) { + return &transport{}, nil +} + +type transport struct { +} + +func (*transport) NewStream(context.Context, string) (clients.Stream, error) { + return &stream{}, nil +} + +func (*transport) Close() { +} + +type stream struct { +} + +func (*stream) Send([]byte) error { + return nil +} + +func (*stream) Recv() ([]byte, error) { + return nil, nil + +} + +// ReportLoad starts reporting load about clusterName to server. +func (xdsC *Client) ReportLoad(server *bootstrap.ServerConfig) (loadStore *lrsclient.LoadStore, cancel func(context.Context)) { + lrsClient, _ := lrsclient.New(lrsclient.Config{Node: clients.Node{ID: "fake-node-id"}, TransportBuilder: &transportBuilder{}}) + xdsC.loadStore, _ = lrsClient.ReportLoad(clients.ServerIdentifier{ServerURI: server.ServerURI()}) + + xdsC.loadReportCh.Send(ReportLoadArgs{Server: server}) + + return xdsC.loadStore, func(ctx context.Context) { + xdsC.loadStore.Stop(ctx) + xdsC.lrsCancelCh.Send(nil) + } +} + +// WaitForCancelReportLoad waits for a load report to be cancelled and returns +// context.DeadlineExceeded otherwise. +func (xdsC *Client) WaitForCancelReportLoad(ctx context.Context) error { + _, err := xdsC.lrsCancelCh.Receive(ctx) + return err +} + +// LoadStore returns the underlying load data store. +func (xdsC *Client) LoadStore() *lrsclient.LoadStore { + return xdsC.loadStore +} + +// WaitForReportLoad waits for ReportLoad to be invoked on this client and +// returns the arguments passed to it. +func (xdsC *Client) WaitForReportLoad(ctx context.Context) (ReportLoadArgs, error) { + val, err := xdsC.loadReportCh.Receive(ctx) + if err != nil { + return ReportLoadArgs{}, err + } + return val.(ReportLoadArgs), nil +} + +// BootstrapConfig returns the bootstrap config. +func (xdsC *Client) BootstrapConfig() *bootstrap.Config { + return xdsC.bootstrapCfg +} + +// SetBootstrapConfig updates the bootstrap config. +func (xdsC *Client) SetBootstrapConfig(cfg *bootstrap.Config) { + xdsC.bootstrapCfg = cfg +} + +// Name returns the name of the xds client. +func (xdsC *Client) Name() string { + return xdsC.name +} + +// NewClient returns a new fake xds client. +func NewClient() *Client { + return NewClientWithName("") +} + +// NewClientWithName returns a new fake xds client with the provided name. This +// is used in cases where multiple clients are created in the tests and we need +// to make sure the client is created for the expected balancer name. +func NewClientWithName(name string) *Client { + return &Client{ + name: name, + loadReportCh: testutils.NewChannel(), + lrsCancelCh: testutils.NewChannel(), + bootstrapCfg: &bootstrap.Config{}, + } +} diff --git a/internal/xds/testutils/resource_watcher.go b/internal/xds/testutils/resource_watcher.go new file mode 100644 index 0000000..9feb8d2 --- /dev/null +++ b/internal/xds/testutils/resource_watcher.go @@ -0,0 +1,83 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + +// TestResourceWatcher implements the xdsresource.ResourceWatcher interface, +// used to receive updates on watches registered with the xDS client, when using +// the resource-type agnostic WatchResource API. +// +// Tests can use the channels provided by this type to get access to updates and +// errors sent by the xDS client. +type TestResourceWatcher struct { + // UpdateCh is the channel on which xDS client updates are delivered. + UpdateCh chan *xdsresource.ResourceData + // AmbientErrorCh is the channel on which ambient errors from the xDS + // client are delivered. + AmbientErrorCh chan error + // ResourceErrorCh is the channel on which resource errors from the xDS + // client are delivered. + ResourceErrorCh chan struct{} +} + +// ResourceChanged is invoked by the xDS client to report the latest update. +func (w *TestResourceWatcher) ResourceChanged(data xdsresource.ResourceData, onDone func()) { + defer onDone() + select { + case <-w.UpdateCh: + default: + } + w.UpdateCh <- &data + +} + +// ResourceError is invoked by the xDS client to report the latest error to +// stop watching the resource. +func (w *TestResourceWatcher) ResourceError(err error, onDone func()) { + defer onDone() + select { + case <-w.ResourceErrorCh: + case <-w.AmbientErrorCh: + default: + } + w.AmbientErrorCh <- err + w.ResourceErrorCh <- struct{}{} +} + +// AmbientError is invoked by the xDS client to report the latest ambient +// error. +func (w *TestResourceWatcher) AmbientError(err error, onDone func()) { + defer onDone() + select { + case <-w.AmbientErrorCh: + default: + } + w.AmbientErrorCh <- err +} + +// NewTestResourceWatcher returns a TestResourceWatcher to watch for resources +// via the xDS client. +func NewTestResourceWatcher() *TestResourceWatcher { + return &TestResourceWatcher{ + UpdateCh: make(chan *xdsresource.ResourceData, 1), + AmbientErrorCh: make(chan error, 1), + ResourceErrorCh: make(chan struct{}, 1), + } +} diff --git a/internal/xds/testutils/testutils.go b/internal/xds/testutils/testutils.go new file mode 100644 index 0000000..d806dec --- /dev/null +++ b/internal/xds/testutils/testutils.go @@ -0,0 +1,51 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package testutils provides utility types, for use in xds tests. +package testutils + +import ( + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" +) + +// BuildResourceName returns the resource name in the format of an xdstp:// +// resource. +func BuildResourceName(typeName, auth, id string, ctxParams map[string]string) string { + var typS string + switch typeName { + case xdsresource.ListenerResourceTypeName: + typS = version.V3ListenerType + case xdsresource.RouteConfigTypeName: + typS = version.V3RouteConfigType + case xdsresource.ClusterResourceTypeName: + typS = version.V3ClusterType + case xdsresource.EndpointsResourceTypeName: + typS = version.V3EndpointsType + default: + // If the name doesn't match any of the standard resources fallback + // to the type name. + typS = typeName + } + return (&xdsresource.Name{ + Scheme: "xdstp", + Authority: auth, + Type: typS, + ID: id, + ContextParams: ctxParams, + }).String() +} diff --git a/internal/xds/xds.go b/internal/xds/xds.go new file mode 100644 index 0000000..b05d866 --- /dev/null +++ b/internal/xds/xds.go @@ -0,0 +1,123 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package xds contains functions, structs, and utilities for working with +// handshake cluster names, as well as shared components used by xds balancers +// and resolvers. It is separated from the top-level /internal package to +// avoid circular dependencies. +package xds + +import ( + "fmt" + + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/resolver" +) + +// handshakeClusterNameKey is the type used as the key to store cluster name in +// the Attributes field of resolver.Address. +type handshakeClusterNameKey struct{} + +// SetXDSHandshakeClusterName returns a copy of addr in which the Attributes field +// is updated with the cluster name. +func SetXDSHandshakeClusterName(addr resolver.Address, clusterName string) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(handshakeClusterNameKey{}, clusterName) + return addr +} + +// GetXDSHandshakeClusterName returns cluster name stored in attr. +func GetXDSHandshakeClusterName(attr *attributes.Attributes) (string, bool) { + v := attr.Value(handshakeClusterNameKey{}) + name, ok := v.(string) + return name, ok +} + +// addressToTelemetryLabels prepares a telemetry label map from resolver +// address attributes. +func addressToTelemetryLabels(addr resolver.Address) map[string]string { + cluster, _ := GetXDSHandshakeClusterName(addr.Attributes) + locality := LocalityString(GetLocalityID(addr)) + return map[string]string{ + "grpc.lb.locality": locality, + "grpc.lb.backend_service": cluster, + } +} + +// LocalityString generates a string representation of clients.Locality in the +// format specified in gRFC A76. +func LocalityString(l clients.Locality) string { + return fmt.Sprintf("{region=%q, zone=%q, sub_zone=%q}", l.Region, l.Zone, l.SubZone) +} + +// IsLocalityEqual allows the values to be compared by Attributes.Equal. +func IsLocalityEqual(l clients.Locality, o any) bool { + ol, ok := o.(clients.Locality) + if !ok { + return false + } + return l.Region == ol.Region && l.Zone == ol.Zone && l.SubZone == ol.SubZone +} + +// LocalityFromString converts a string representation of clients.locality as +// specified in gRFC A76, into a LocalityID struct. +func LocalityFromString(s string) (ret clients.Locality, _ error) { + _, err := fmt.Sscanf(s, "{region=%q, zone=%q, sub_zone=%q}", &ret.Region, &ret.Zone, &ret.SubZone) + if err != nil { + return clients.Locality{}, fmt.Errorf("%s is not a well formatted locality ID, error: %v", s, err) + } + return ret, nil +} + +type localityKeyType string + +const localityKey = localityKeyType("grpc.xds.internal.address.locality") + +// GetLocalityID returns the locality ID of addr. +func GetLocalityID(addr resolver.Address) clients.Locality { + path, _ := addr.BalancerAttributes.Value(localityKey).(clients.Locality) + return path +} + +// SetLocalityID sets locality ID in addr to l. +func SetLocalityID(addr resolver.Address, l clients.Locality) resolver.Address { + addr.BalancerAttributes = addr.BalancerAttributes.WithValue(localityKey, l) + return addr +} + +// SetLocalityIDInEndpoint sets locality ID in endpoint to l. +func SetLocalityIDInEndpoint(endpoint resolver.Endpoint, l clients.Locality) resolver.Endpoint { + endpoint.Attributes = endpoint.Attributes.WithValue(localityKey, l) + return endpoint +} + +// LocalityIDFromEndpoint returns the locality ID of ep. +func LocalityIDFromEndpoint(ep resolver.Endpoint) clients.Locality { + path, _ := ep.Attributes.Value(localityKey).(clients.Locality) + return path +} + +// UnknownCSMLabels are TelemetryLabels emitted from CDS if CSM Telemetry Label +// data is not present in the CDS Resource. +var UnknownCSMLabels = map[string]string{ + "csm.service_name": "unknown", + "csm.service_namespace_name": "unknown", +} + +func init() { + internal.AddressToTelemetryLabels = addressToTelemetryLabels +} diff --git a/internal/xds/xds_test.go b/internal/xds/xds_test.go new file mode 100644 index 0000000..659be12 --- /dev/null +++ b/internal/xds/xds_test.go @@ -0,0 +1,115 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xds + +import ( + "reflect" + "strings" + "testing" + "unicode" + + corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/xds/clients" +) + +const ignorePrefix = "XXX_" + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func ignore(name string) bool { + if !unicode.IsUpper([]rune(name)[0]) { + return true + } + return strings.HasPrefix(name, ignorePrefix) +} + +// A reflection based test to make sure internal.Locality contains all the +// fields (expect for XXX_) from the proto message. +func (s) TestLocalityMatchProtoMessage(t *testing.T) { + want1 := make(map[string]string) + for ty, i := reflect.TypeOf(clients.Locality{}), 0; i < ty.NumField(); i++ { + f := ty.Field(i) + if ignore(f.Name) { + continue + } + want1[f.Name] = f.Type.Name() + } + + want2 := make(map[string]string) + for ty, i := reflect.TypeOf(corepb.Locality{}), 0; i < ty.NumField(); i++ { + f := ty.Field(i) + if ignore(f.Name) { + continue + } + want2[f.Name] = f.Type.Name() + } + + if diff := cmp.Diff(want1, want2); diff != "" { + t.Fatalf("internal type and proto message have different fields: (-got +want):\n%+v", diff) + } +} + +func TestLocalityToAndFromString(t *testing.T) { + tests := []struct { + name string + localityID clients.Locality + str string + wantErr bool + }{ + { + name: "3 fields", + localityID: clients.Locality{Region: "r:r", Zone: "z#z", SubZone: "s^s"}, + str: `{region="r:r", zone="z#z", sub_zone="s^s"}`, + }, + { + name: "2 fields", + localityID: clients.Locality{Region: "r:r", Zone: "z#z"}, + str: `{region="r:r", zone="z#z", sub_zone=""}`, + }, + { + name: "1 field", + localityID: clients.Locality{Region: "r:r"}, + str: `{region="r:r", zone="", sub_zone=""}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotStr := LocalityString(tt.localityID) + if gotStr != tt.str { + t.Errorf("%#v.String() = %q, want %q", tt.localityID, gotStr, tt.str) + } + + gotID, err := LocalityFromString(tt.str) + if (err != nil) != tt.wantErr { + t.Errorf("clients.LocalityFromString(%q) error = %v, wantErr %v", tt.str, err, tt.wantErr) + return + } + if diff := cmp.Diff(gotID, tt.localityID); diff != "" { + t.Errorf("clients.LocalityFromString() got = %v, want %v, diff: %s", gotID, tt.localityID, diff) + } + }) + } +} diff --git a/internal/xds/xdsclient/attributes.go b/internal/xds/xdsclient/attributes.go new file mode 100644 index 0000000..9076a76 --- /dev/null +++ b/internal/xds/xdsclient/attributes.go @@ -0,0 +1,36 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import "google.golang.org/grpc/resolver" + +type clientKeyType string + +const clientKey = clientKeyType("grpc.xds.internal.client.Client") + +// FromResolverState returns the Client from state, or nil if not present. +func FromResolverState(state resolver.State) XDSClient { + cs, _ := state.Attributes.Value(clientKey).(XDSClient) + return cs +} + +// SetClient sets c in state and returns the new state. +func SetClient(state resolver.State, c XDSClient) resolver.State { + state.Attributes = state.Attributes.WithValue(clientKey, c) + return state +} diff --git a/internal/xds/xdsclient/client.go b/internal/xds/xdsclient/client.go new file mode 100644 index 0000000..d05382d --- /dev/null +++ b/internal/xds/xdsclient/client.go @@ -0,0 +1,62 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xdsclient implements a full fledged gRPC client for the xDS API used +// by the xds resolver and balancer implementations. +package xdsclient + +import ( + "context" + + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients/lrsclient" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" +) + +// XDSClient is a full fledged gRPC client which queries a set of discovery APIs +// (collectively termed as xDS) on a remote management server, to discover +// various dynamic resources. +type XDSClient interface { + // WatchResource uses xDS to discover the resource associated with the + // provided resource name. The resource type implementation determines how + // xDS responses are are deserialized and validated, as received from the + // xDS management server. Upon receipt of a response from the management + // server, an appropriate callback on the watcher is invoked. + // + // Most callers will not have a need to use this API directly. They will + // instead use a resource-type-specific wrapper API provided by the relevant + // resource type implementation. + // + // + // During a race (e.g. an xDS response is received while the user is calling + // cancel()), there's a small window where the callback can be called after + // the watcher is canceled. Callers need to handle this case. + WatchResource(typeURL, resourceName string, watcher xdsclient.ResourceWatcher) (cancel func()) + + ReportLoad(*bootstrap.ServerConfig) (*lrsclient.LoadStore, func(context.Context)) + + BootstrapConfig() *bootstrap.Config +} + +// DumpResources returns the status and contents of all xDS resources. It uses +// xDS clients from the default pool. +func DumpResources() *v3statuspb.ClientStatusResponse { + return DefaultPool.DumpResources() +} diff --git a/internal/xds/xdsclient/client_refcounted_test.go b/internal/xds/xdsclient/client_refcounted_test.go new file mode 100644 index 0000000..fe44e6e --- /dev/null +++ b/internal/xds/xdsclient/client_refcounted_test.go @@ -0,0 +1,277 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "context" + "sync" + "testing" + + "github.com/google/uuid" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" +) + +// Tests that multiple calls to New() with the same name returns the same +// client. Also verifies that only when all references to the newly created +// client are released, the underlying client is closed. +func (s) TestClientNew_Single(t *testing.T) { + // Create a bootstrap configuration, place it in a file in the temp + // directory, and set the bootstrap env vars to point to it. + nodeID := uuid.New().String() + contents := e2e.DefaultBootstrapContents(t, nodeID, "non-existent-server-address") + config, err := bootstrap.NewConfigFromContents(contents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", contents, err) + } + pool := NewPool(config) + + // Override the client creation hook to get notified. + origClientImplCreateHook := xdsClientImplCreateHook + clientImplCreateCh := testutils.NewChannel() + xdsClientImplCreateHook = func(name string) { + clientImplCreateCh.Replace(name) + } + defer func() { xdsClientImplCreateHook = origClientImplCreateHook }() + + // Override the client close hook to get notified. + origClientImplCloseHook := xdsClientImplCloseHook + clientImplCloseCh := testutils.NewChannel() + xdsClientImplCloseHook = func(name string) { + clientImplCloseCh.Replace(name) + } + defer func() { xdsClientImplCloseHook = origClientImplCloseHook }() + + // The first call to New() should create a new client. + _, closeFunc, err := pool.NewClient(t.Name(), &estats.UnimplementedMetricsRecorder{}) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := clientImplCreateCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for xDS client to be created: %v", err) + } + + // Calling New() again should not create new client implementations. + const count = 9 + closeFuncs := make([]func(), count) + for i := 0; i < count; i++ { + func() { + _, closeFuncs[i], err = pool.NewClient(t.Name(), &estats.UnimplementedMetricsRecorder{}) + if err != nil { + t.Fatalf("%d-th call to New() failed with error: %v", i, err) + } + + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := clientImplCreateCh.Receive(sCtx); err == nil { + t.Fatalf("%d-th call to New() created a new client", i) + } + }() + } + + // Call Close() multiple times on each of the clients created in the above + // for loop. Close() calls are idempotent, and the underlying client + // implementation will not be closed until we release the first reference we + // acquired above, via the first call to New(). + for i := 0; i < count; i++ { + func() { + closeFuncs[i]() + closeFuncs[i]() + + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := clientImplCloseCh.Receive(sCtx); err == nil { + t.Fatal("Client implementation closed before all references are released") + } + }() + } + + // Call the last Close(). The underlying implementation should be closed. + closeFunc() + if _, err := clientImplCloseCh.Receive(ctx); err != nil { + t.Fatalf("Timeout waiting for client implementation to be closed: %v", err) + } + + // Calling New() again, after the previous Client was actually closed, + // should create a new one. + _, closeFunc, err = pool.NewClient(t.Name(), &estats.UnimplementedMetricsRecorder{}) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer closeFunc() + if _, err := clientImplCreateCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for xDS client to be created: %v", err) + } +} + +// Tests the scenario where there are multiple calls to New() with different +// names. Verifies that reference counts are tracked correctly for each client +// and that only when all references are released for a client, it is closed. +func (s) TestClientNew_Multiple(t *testing.T) { + // Create a bootstrap configuration, place it in a file in the temp + // directory, and set the bootstrap env vars to point to it. + nodeID := uuid.New().String() + contents := e2e.DefaultBootstrapContents(t, nodeID, "non-existent-server-address") + config, err := bootstrap.NewConfigFromContents(contents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", contents, err) + } + pool := NewPool(config) + + // Override the client creation hook to get notified. + origClientImplCreateHook := xdsClientImplCreateHook + clientImplCreateCh := testutils.NewChannel() + xdsClientImplCreateHook = func(name string) { + clientImplCreateCh.Replace(name) + } + defer func() { xdsClientImplCreateHook = origClientImplCreateHook }() + + // Override the client close hook to get notified. + origClientImplCloseHook := xdsClientImplCloseHook + clientImplCloseCh := testutils.NewChannel() + xdsClientImplCloseHook = func(name string) { + clientImplCloseCh.Replace(name) + } + defer func() { xdsClientImplCloseHook = origClientImplCloseHook }() + + // Create two xDS clients. + client1Name := t.Name() + "-1" + _, closeFunc1, err := pool.NewClient(client1Name, &estats.UnimplementedMetricsRecorder{}) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + name, err := clientImplCreateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for xDS client to be created: %v", err) + } + if name.(string) != client1Name { + t.Fatalf("xDS client created for name %q, want %q", name.(string), client1Name) + } + + client2Name := t.Name() + "-2" + _, closeFunc2, err := pool.NewClient(client2Name, &estats.UnimplementedMetricsRecorder{}) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + name, err = clientImplCreateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for xDS client to be created: %v", err) + } + if name.(string) != client2Name { + t.Fatalf("xDS client created for name %q, want %q", name.(string), client1Name) + } + + // Create N more references to each of these clients. + const count = 9 + closeFuncs1 := make([]func(), count) + closeFuncs2 := make([]func(), count) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; i < count; i++ { + var err error + _, closeFuncs1[i], err = pool.NewClient(client1Name, &estats.UnimplementedMetricsRecorder{}) + if err != nil { + t.Errorf("%d-th call to New() failed with error: %v", i, err) + } + } + }() + go func() { + defer wg.Done() + for i := 0; i < count; i++ { + var err error + _, closeFuncs2[i], err = pool.NewClient(client2Name, &estats.UnimplementedMetricsRecorder{}) + if err != nil { + t.Errorf("%d-th call to New() failed with error: %v", i, err) + } + } + }() + wg.Wait() + if t.Failed() { + t.FailNow() + } + + // Ensure that none of the create hooks are called. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := clientImplCreateCh.Receive(sCtx); err == nil { + t.Fatalf("New xDS client created when expected to reuse an existing one") + } + + // The close function returned by New() is idempotent and calling it + // multiple times should not decrement the reference count multiple times. + for i := 0; i < count; i++ { + closeFuncs1[i]() + closeFuncs1[i]() + } + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := clientImplCloseCh.Receive(sCtx); err == nil { + t.Fatal("Client implementation closed before all references are released") + } + + // Release the last reference and verify that the client is closed + // completely. + closeFunc1() + name, err = clientImplCloseCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for xDS client to be closed completely") + } + if name.(string) != client1Name { + t.Fatalf("xDS client closed for name %q, want %q", name.(string), client1Name) + } + + // Ensure that the close hook is not called for the second client. + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := clientImplCloseCh.Receive(sCtx); err == nil { + t.Fatal("Client implementation closed before all references are released") + } + + // The close function returned by New() is idempotent and calling it + // multiple times should not decrement the reference count multiple times. + for i := 0; i < count; i++ { + closeFuncs2[i]() + closeFuncs2[i]() + } + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := clientImplCloseCh.Receive(sCtx); err == nil { + t.Fatal("Client implementation closed before all references are released") + } + + // Release the last reference and verify that the client is closed + // completely. + closeFunc2() + name, err = clientImplCloseCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for xDS client to be closed completely") + } + if name.(string) != client2Name { + t.Fatalf("xDS client closed for name %q, want %q", name.(string), client2Name) + } +} diff --git a/internal/xds/xdsclient/client_test.go b/internal/xds/xdsclient/client_test.go new file mode 100644 index 0000000..f326888 --- /dev/null +++ b/internal/xds/xdsclient/client_test.go @@ -0,0 +1,40 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestWatchExpiryTimeout = 100 * time.Millisecond + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) diff --git a/internal/xds/xdsclient/clientimpl.go b/internal/xds/xdsclient/clientimpl.go new file mode 100644 index 0000000..23d6227 --- /dev/null +++ b/internal/xds/xdsclient/clientimpl.go @@ -0,0 +1,328 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "fmt" + "sync/atomic" + "time" + + "google.golang.org/grpc" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/xds/bootstrap" + + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/lrsclient" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/clients/xdsclient/metrics" + xdsbootstrap "google.golang.org/grpc/xds/bootstrap" +) + +const ( + // NameForServer represents the value to be passed as name when creating an xDS + // client from xDS-enabled gRPC servers. This is a well-known dedicated key + // value, and is defined in gRFC A71. + NameForServer = "#server" + + defaultWatchExpiryTimeout = 15 * time.Second +) + +var ( + // The following functions are no-ops in the actual code, but can be + // overridden in tests to give them visibility into certain events. + xdsClientImplCreateHook = func(string) {} + xdsClientImplCloseHook = func(string) {} + + defaultExponentialBackoff = backoff.DefaultExponential.Backoff + + xdsClientResourceUpdatesValidMetric = estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "grpc.xds_client.resource_updates_valid", + Description: "A counter of resources received that were considered valid. The counter will be incremented even for resources that have not changed.", + Unit: "{resource}", + Labels: []string{"grpc.target", "grpc.xds.server", "grpc.xds.resource_type"}, + Default: false, + }) + xdsClientResourceUpdatesInvalidMetric = estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "grpc.xds_client.resource_updates_invalid", + Description: "A counter of resources received that were considered invalid.", + Unit: "{resource}", + Labels: []string{"grpc.target", "grpc.xds.server", "grpc.xds.resource_type"}, + Default: false, + }) + xdsClientServerFailureMetric = estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "grpc.xds_client.server_failure", + Description: "A counter of xDS servers going from healthy to unhealthy. A server goes unhealthy when we have a connectivity failure or when the ADS stream fails without seeing a response message, as per gRFC A57.", + Unit: "{failure}", + Labels: []string{"grpc.target", "grpc.xds.server"}, + Default: false, + }) + xdsClientConnectedMetric = estats.RegisterInt64AsyncGauge(estats.MetricDescriptor{ + Name: "grpc.xds_client.connected", + Description: "A metric that is 1 if the xDS Client has a working ADS stream to the server, 0 otherwise.", + Unit: "{connected}", + Type: estats.MetricTypeIntAsyncGauge, + Labels: []string{"grpc.target", "grpc.xds.server"}, + }) + xdsClientResourcesMetric = estats.RegisterInt64AsyncGauge(estats.MetricDescriptor{ + Name: "grpc.xds_client.resources", + Description: "Counts of xDS resources.", + Unit: "{resource}", + Type: estats.MetricTypeIntAsyncGauge, + Labels: []string{"grpc.target", "grpc.xds.authority", "grpc.xds.cache_state", "grpc.xds.resource_type"}, + }) +) + +// clientImpl embed xdsclient.XDSClient and implement internal XDSClient +// interface with ref counting so that it can be shared by the xds resolver and +// balancer implementations, across multiple ClientConns and Servers. +type clientImpl struct { + *xdsclient.XDSClient // TODO: #8313 - get rid of embedding, if possible. + + // The following fields are initialized at creation time and are read-only + // after that. + xdsClientConfig xdsclient.Config + bootstrapConfig *bootstrap.Config + logger *grpclog.PrefixLogger + target string + lrsClient *lrsclient.LRSClient + + // Accessed atomically + refCount int32 +} + +// metricsReporter implements the clients.MetricsReporter interface and uses an +// underlying stats.MetricsRecorderList to record metrics. +type metricsReporter struct { + recorder estats.MetricsRecorder + target string +} + +// ReportMetric implements the clients.MetricsReporter interface. +// It receives metric data, determines the appropriate metric based on the type +// of the data, and records it using the embedded MetricsRecorderList. +func (mr *metricsReporter) ReportMetric(metric any) { + if mr.recorder == nil { + return + } + + switch m := metric.(type) { + case *metrics.ResourceUpdateValid: + xdsClientResourceUpdatesValidMetric.Record(mr.recorder, 1, mr.target, m.ServerURI, m.ResourceType) + case *metrics.ResourceUpdateInvalid: + xdsClientResourceUpdatesInvalidMetric.Record(mr.recorder, 1, mr.target, m.ServerURI, m.ResourceType) + case *metrics.ServerFailure: + xdsClientServerFailureMetric.Record(mr.recorder, 1, mr.target, m.ServerURI) + } +} + +func newClientImpl(config *bootstrap.Config, metricsRecorder estats.MetricsRecorder, target string, watchExpiryTimeout time.Duration) (*clientImpl, error) { + gConfig, err := BuildXDSClientConfig(config, metricsRecorder, target, watchExpiryTimeout) + if err != nil { + return nil, err + } + client, err := xdsclient.New(gConfig) + if err != nil { + return nil, err + } + lrsC, err := lrsclient.New(lrsclient.Config{ + Node: gConfig.Node, + TransportBuilder: gConfig.TransportBuilder, + }) + if err != nil { + return nil, err + } + c := &clientImpl{ + XDSClient: client, + xdsClientConfig: gConfig, + bootstrapConfig: config, + target: target, + refCount: 1, + lrsClient: lrsC, + } + c.logger = prefixLogger(c) + return c, nil +} + +// BootstrapConfig returns the configuration read from the bootstrap file. +// Callers must treat the return value as read-only. +func (c *clientImpl) BootstrapConfig() *bootstrap.Config { + return c.bootstrapConfig +} + +func (c *clientImpl) incrRef() int32 { + return atomic.AddInt32(&c.refCount, 1) +} + +func (c *clientImpl) decrRef() int32 { + return atomic.AddInt32(&c.refCount, -1) +} + +func buildServerConfigs(bootstrapSC []*bootstrap.ServerConfig, grpcTransportConfigs map[string]grpctransport.Config, gServerCfgMap map[xdsclient.ServerConfig]*bootstrap.ServerConfig) ([]xdsclient.ServerConfig, error) { + var gServerCfg []xdsclient.ServerConfig + for _, sc := range bootstrapSC { + if err := populateGRPCTransportConfigsFromServerConfig(sc, grpcTransportConfigs); err != nil { + return nil, err + } + var serverFeatures xdsclient.ServerFeature + if sc.ServerFeaturesIgnoreResourceDeletion() { + serverFeatures = serverFeatures | xdsclient.ServerFeatureIgnoreResourceDeletion + } + if sc.ServerFeaturesTrustedXDSServer() { + serverFeatures = serverFeatures | xdsclient.ServerFeatureTrustedXDSServer + } + gsc := xdsclient.ServerConfig{ + ServerIdentifier: clients.ServerIdentifier{ + ServerURI: sc.ServerURI(), + Extensions: grpctransport.ServerIdentifierExtension{ConfigName: sc.SelectedChannelCreds().Type}, + }, + ServerFeature: serverFeatures, + } + gServerCfg = append(gServerCfg, gsc) + gServerCfgMap[gsc] = sc + } + return gServerCfg, nil +} + +// BuildXDSClientConfig builds the xdsclient.Config from the bootstrap.Config. +// +// This function is exported for fuzz testing purposes only. +func BuildXDSClientConfig(config *bootstrap.Config, metricsRecorder estats.MetricsRecorder, target string, watchExpiryTimeout time.Duration) (xdsclient.Config, error) { + grpcTransportConfigs := make(map[string]grpctransport.Config) + gServerCfgMap := make(map[xdsclient.ServerConfig]*bootstrap.ServerConfig) + + gAuthorities := make(map[string]xdsclient.Authority) + for name, cfg := range config.Authorities() { + // If server configs are specified in the authorities map, use that. + // Else, use the top-level server configs. + serverCfg := config.XDSServers() + if len(cfg.XDSServers) >= 1 { + serverCfg = cfg.XDSServers + } + gsc, err := buildServerConfigs(serverCfg, grpcTransportConfigs, gServerCfgMap) + if err != nil { + return xdsclient.Config{}, err + } + gAuthorities[name] = xdsclient.Authority{XDSServers: gsc} + } + + gServerCfgs, err := buildServerConfigs(config.XDSServers(), grpcTransportConfigs, gServerCfgMap) + if err != nil { + return xdsclient.Config{}, err + } + + node := config.Node() + gNode := clients.Node{ + ID: node.GetId(), + Cluster: node.GetCluster(), + Metadata: node.Metadata, + UserAgentName: node.UserAgentName, + UserAgentVersion: node.GetUserAgentVersion(), + } + if node.Locality != nil { + gNode.Locality = clients.Locality{ + Region: node.Locality.Region, + Zone: node.Locality.Zone, + SubZone: node.Locality.SubZone, + } + } + + return xdsclient.Config{ + Authorities: gAuthorities, + Servers: gServerCfgs, + Node: gNode, + TransportBuilder: grpctransport.NewBuilder(grpcTransportConfigs), + ResourceTypes: supportedResourceTypes(config, gServerCfgMap), + MetricsReporter: &metricsReporter{recorder: metricsRecorder, target: target}, + WatchExpiryTimeout: watchExpiryTimeout, + }, nil +} + +// populateGRPCTransportConfigsFromServerConfig iterates through the channel +// credentials of the provided server configuration, builds credential bundles, +// and populates the grpctransport.Config map. +func populateGRPCTransportConfigsFromServerConfig(sc *bootstrap.ServerConfig, grpcTransportConfigs map[string]grpctransport.Config) error { + for _, cc := range sc.ChannelCreds() { + c := xdsbootstrap.GetChannelCredentials(cc.Type) + if c == nil { + continue + } + bundle, _, err := c.Build(cc.Config) + if err != nil { + return fmt.Errorf("xds: failed to build credentials bundle from bootstrap for %q: %v", cc.Type, err) + } + grpcTransportConfigs[cc.Type] = grpctransport.Config{ + Credentials: bundle, + GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + opts = append(opts, sc.DialOptions()...) + return grpc.NewClient(target, opts...) + }, + } + } + return nil +} + +// RegisterAsyncReporter adapts the generic clients.AsyncReporter to the +// estats.AsyncMetricReporter interface and registers it. +func (mr *metricsReporter) RegisterAsyncReporter(reporter clients.AsyncReporter) func() { + if mr.recorder == nil || reporter == nil { + return func() {} + } + + // Define which metrics we intend to report for OTel registration. + descriptors := []estats.AsyncMetric{ + xdsClientConnectedMetric, + xdsClientResourcesMetric, + } + + // Create the callback wrapper. + // This function is invoked by the stats system during a scrape. + cbWrapper := func(rec estats.AsyncMetricsRecorder) error { + wrapper := &asyncMetricsRecorderAdapter{ + target: mr.target, + delegate: rec, + } + reporter.Report(wrapper) + return nil + } + + // Register with the underlying gRPC stats recorder. + return mr.recorder.RegisterAsyncReporter(estats.AsyncMetricReporterFunc(cbWrapper), descriptors...) +} + +// asyncMetricsRecorderAdapter adapts estats.AsyncMetricsRecorder to clients.AsyncMetricsRecorder. +type asyncMetricsRecorderAdapter struct { + target string + delegate estats.AsyncMetricsRecorder +} + +func (a *asyncMetricsRecorderAdapter) ReportMetric(metric any) { + switch m := metric.(type) { + case *metrics.XDSClientConnected: + // Record: grpc.xds_client.connected + // Labels: grpc.target, grpc.xds.server + a.delegate.RecordInt64AsyncGauge(xdsClientConnectedMetric, m.Value, a.target, m.ServerURI) + case *metrics.XDSClientResourceStats: + // Record: grpc.xds_client.resources + // Labels: grpc.target, grpc.xds.authority, grpc.xds.cache_state, grpc.xds.resource_type + a.delegate.RecordInt64AsyncGauge(xdsClientResourcesMetric, m.Count, a.target, m.Authority, m.CacheState, m.ResourceType) + } +} diff --git a/internal/xds/xdsclient/clientimpl_loadreport.go b/internal/xds/xdsclient/clientimpl_loadreport.go new file mode 100644 index 0000000..023f302 --- /dev/null +++ b/internal/xds/xdsclient/clientimpl_loadreport.go @@ -0,0 +1,49 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsclient + +import ( + "context" + "sync" + + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/lrsclient" +) + +// ReportLoad starts a load reporting stream to the given server. All load +// reports to the same server share the LRS stream. +// +// It returns a lrsclient.LoadStore for the user to report loads. +func (c *clientImpl) ReportLoad(server *bootstrap.ServerConfig) (*lrsclient.LoadStore, func(context.Context)) { + load, err := c.lrsClient.ReportLoad(clients.ServerIdentifier{ + ServerURI: server.ServerURI(), + Extensions: grpctransport.ServerIdentifierExtension{ + ConfigName: server.SelectedChannelCreds().Type, + }, + }) + if err != nil { + c.logger.Warningf("Failed to create a load store to the management server to report load: %v", server, err) + return nil, func(context.Context) {} + } + var loadStop sync.Once + return load, func(ctx context.Context) { + loadStop.Do(func() { load.Stop(ctx) }) + } +} diff --git a/internal/xds/xdsclient/clientimpl_test.go b/internal/xds/xdsclient/clientimpl_test.go new file mode 100644 index 0000000..3603798 --- /dev/null +++ b/internal/xds/xdsclient/clientimpl_test.go @@ -0,0 +1,363 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "encoding/json" + "fmt" + "reflect" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/stats" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/clients/grpctransport" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/protobuf/testing/protocmp" +) + +const ( + testXDSServerURL = "xds.example.com:8080" + testXDSServerURL2 = "xds.example.com:8081" + testNodeID = "test-node-id" + testClusterName = "test-cluster" + testUserAgentName = "test-ua-name" + testUserAgentVer = "test-ua-ver" + testLocalityRegion = "test-region" + testLocalityZone = "test-zone" + testLocalitySubZone = "test-sub-zone" + testTargetName = "test-target" +) + +var ( + testMetadataJSON, _ = json.Marshal(map[string]any{"foo": "bar", "baz": float64(1)}) +) + +func (s) TestBuildXDSClientConfig_Success(t *testing.T) { + tests := []struct { + name string + bootstrapContents []byte + wantXDSClientConfig func(bootstrapCfg *bootstrap.Config) xdsclient.Config + }{ + { + name: "without authorities", + bootstrapContents: []byte(fmt.Sprintf(`{ + "xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "insecure"}]}], + "node": { + "id": "%s", "cluster": "%s", "metadata": %s, + "locality": {"region": "%s", "zone": "%s", "sub_zone": "%s"}, + "user_agent_name": "%s", "user_agent_version": "%s" + } + }`, testXDSServerURL, testNodeID, testClusterName, testMetadataJSON, testLocalityRegion, testLocalityZone, testLocalitySubZone, testUserAgentName, testUserAgentVer)), + wantXDSClientConfig: func(c *bootstrap.Config) xdsclient.Config { + node, serverCfg := c.Node(), c.XDSServers()[0] + expectedServer := xdsclient.ServerConfig{ServerIdentifier: clients.ServerIdentifier{ServerURI: serverCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}} + gServerCfgMap := map[xdsclient.ServerConfig]*bootstrap.ServerConfig{expectedServer: serverCfg} + return xdsclient.Config{ + Servers: []xdsclient.ServerConfig{expectedServer}, + Node: clients.Node{ID: node.GetId(), Cluster: node.GetCluster(), Metadata: node.Metadata, Locality: clients.Locality{Region: node.Locality.Region, Zone: node.Locality.Zone, SubZone: node.Locality.SubZone}, UserAgentName: node.UserAgentName, UserAgentVersion: node.GetUserAgentVersion()}, + Authorities: map[string]xdsclient.Authority{}, + ResourceTypes: map[string]xdsclient.ResourceType{ + version.V3ListenerURL: {TypeURL: version.V3ListenerURL, TypeName: xdsresource.ListenerResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewListenerResourceTypeDecoder(c, gServerCfgMap)}, + version.V3RouteConfigURL: {TypeURL: version.V3RouteConfigURL, TypeName: xdsresource.RouteConfigTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(c, gServerCfgMap)}, + version.V3ClusterURL: {TypeURL: version.V3ClusterURL, TypeName: xdsresource.ClusterResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewClusterResourceTypeDecoder(c, gServerCfgMap)}, + version.V3EndpointsURL: {TypeURL: version.V3EndpointsURL, TypeName: xdsresource.EndpointsResourceTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewEndpointsResourceTypeDecoder(c)}, + }, + MetricsReporter: &metricsReporter{recorder: stats.NewTestMetricsRecorder(), target: testTargetName}, + TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{ + "insecure": { + Credentials: insecure.NewBundle(), + GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + opts = append(opts, serverCfg.DialOptions()...) + return grpc.NewClient(target, opts...) + }}, + }), + } + }, + }, + { + name: "with authorities", + bootstrapContents: []byte(fmt.Sprintf(`{ + "xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "insecure"}]}], + "node": {"id": "%s"}, + "authorities": { + "auth1": {}, + "auth2": {"xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "insecure"}]}]} + } + }`, testXDSServerURL, testNodeID, testXDSServerURL2)), + wantXDSClientConfig: func(c *bootstrap.Config) xdsclient.Config { + node := c.Node() + topLevelSCfg, auth2SCfg := c.XDSServers()[0], c.Authorities()["auth2"].XDSServers[0] + expTopLevelS := xdsclient.ServerConfig{ServerIdentifier: clients.ServerIdentifier{ServerURI: topLevelSCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}} + expAuth2S := xdsclient.ServerConfig{ServerIdentifier: clients.ServerIdentifier{ServerURI: auth2SCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}} + gServerCfgMap := map[xdsclient.ServerConfig]*bootstrap.ServerConfig{expTopLevelS: topLevelSCfg, expAuth2S: auth2SCfg} + return xdsclient.Config{ + Servers: []xdsclient.ServerConfig{expTopLevelS}, + Node: clients.Node{ID: node.GetId(), Cluster: node.GetCluster(), Metadata: node.Metadata, UserAgentName: node.UserAgentName, UserAgentVersion: node.GetUserAgentVersion()}, + Authorities: map[string]xdsclient.Authority{"auth1": {XDSServers: []xdsclient.ServerConfig{expTopLevelS}}, "auth2": {XDSServers: []xdsclient.ServerConfig{expAuth2S}}}, + ResourceTypes: map[string]xdsclient.ResourceType{ + version.V3ListenerURL: {TypeURL: version.V3ListenerURL, TypeName: xdsresource.ListenerResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewListenerResourceTypeDecoder(c, gServerCfgMap)}, + version.V3RouteConfigURL: {TypeURL: version.V3RouteConfigURL, TypeName: xdsresource.RouteConfigTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(c, gServerCfgMap)}, + version.V3ClusterURL: {TypeURL: version.V3ClusterURL, TypeName: xdsresource.ClusterResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewClusterResourceTypeDecoder(c, gServerCfgMap)}, + version.V3EndpointsURL: {TypeURL: version.V3EndpointsURL, TypeName: xdsresource.EndpointsResourceTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewEndpointsResourceTypeDecoder(c)}, + }, + MetricsReporter: &metricsReporter{recorder: stats.NewTestMetricsRecorder(), target: testTargetName}, + TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{ + "insecure": { + Credentials: insecure.NewBundle(), + GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + opts = append(opts, topLevelSCfg.DialOptions()...) + return grpc.NewClient(target, opts...) + }}, + }), + } + }, + }, + { + name: "server features with ignore_resource_deletion", + bootstrapContents: []byte(fmt.Sprintf(`{ + "xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "insecure"}], "server_features": ["ignore_resource_deletion"]}], + "node": {"id": "%s"} + }`, testXDSServerURL, testNodeID)), + wantXDSClientConfig: func(c *bootstrap.Config) xdsclient.Config { + node, serverCfg := c.Node(), c.XDSServers()[0] + serverFeature := xdsclient.ServerFeatureIgnoreResourceDeletion + expectedServer := xdsclient.ServerConfig{ + ServerIdentifier: clients.ServerIdentifier{ServerURI: serverCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}, + ServerFeature: serverFeature} + gServerCfgMap := map[xdsclient.ServerConfig]*bootstrap.ServerConfig{expectedServer: serverCfg} + return xdsclient.Config{ + Servers: []xdsclient.ServerConfig{expectedServer}, + Node: clients.Node{ID: node.GetId(), Cluster: node.GetCluster(), Metadata: node.Metadata, UserAgentName: node.UserAgentName, UserAgentVersion: node.GetUserAgentVersion()}, + Authorities: map[string]xdsclient.Authority{}, + ResourceTypes: map[string]xdsclient.ResourceType{ + version.V3ListenerURL: {TypeURL: version.V3ListenerURL, TypeName: xdsresource.ListenerResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewListenerResourceTypeDecoder(c, gServerCfgMap)}, + version.V3RouteConfigURL: {TypeURL: version.V3RouteConfigURL, TypeName: xdsresource.RouteConfigTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(c, gServerCfgMap)}, + version.V3ClusterURL: {TypeURL: version.V3ClusterURL, TypeName: xdsresource.ClusterResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewClusterResourceTypeDecoder(c, gServerCfgMap)}, + version.V3EndpointsURL: {TypeURL: version.V3EndpointsURL, TypeName: xdsresource.EndpointsResourceTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewEndpointsResourceTypeDecoder(c)}, + }, + MetricsReporter: &metricsReporter{recorder: stats.NewTestMetricsRecorder(), target: testTargetName}, + TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{ + "insecure": { + Credentials: insecure.NewBundle(), + GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + opts = append(opts, serverCfg.DialOptions()...) + return grpc.NewClient(target, opts...) + }}, + }), + } + }, + }, + { + name: "server features with trusted_xds_server", + bootstrapContents: []byte(fmt.Sprintf(`{ + "xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "insecure"}], "server_features": ["trusted_xds_server"]}], + "node": {"id": "%s"} + }`, testXDSServerURL, testNodeID)), + wantXDSClientConfig: func(c *bootstrap.Config) xdsclient.Config { + node, serverCfg := c.Node(), c.XDSServers()[0] + serverFeature := xdsclient.ServerFeatureTrustedXDSServer + expectedServer := xdsclient.ServerConfig{ServerIdentifier: clients.ServerIdentifier{ServerURI: serverCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}, + ServerFeature: serverFeature} + gServerCfgMap := map[xdsclient.ServerConfig]*bootstrap.ServerConfig{expectedServer: serverCfg} + return xdsclient.Config{ + Servers: []xdsclient.ServerConfig{expectedServer}, + Node: clients.Node{ID: node.GetId(), Cluster: node.GetCluster(), Metadata: node.Metadata, UserAgentName: node.UserAgentName, UserAgentVersion: node.GetUserAgentVersion()}, + Authorities: map[string]xdsclient.Authority{}, + ResourceTypes: map[string]xdsclient.ResourceType{ + version.V3ListenerURL: {TypeURL: version.V3ListenerURL, TypeName: xdsresource.ListenerResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewListenerResourceTypeDecoder(c, gServerCfgMap)}, + version.V3RouteConfigURL: {TypeURL: version.V3RouteConfigURL, TypeName: xdsresource.RouteConfigTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(c, gServerCfgMap)}, + version.V3ClusterURL: {TypeURL: version.V3ClusterURL, TypeName: xdsresource.ClusterResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewClusterResourceTypeDecoder(c, gServerCfgMap)}, + version.V3EndpointsURL: {TypeURL: version.V3EndpointsURL, TypeName: xdsresource.EndpointsResourceTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewEndpointsResourceTypeDecoder(c)}, + }, + MetricsReporter: &metricsReporter{recorder: stats.NewTestMetricsRecorder(), target: testTargetName}, + TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{ + "insecure": { + Credentials: insecure.NewBundle(), + GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + opts = append(opts, serverCfg.DialOptions()...) + return grpc.NewClient(target, opts...) + }}, + }), + } + }, + }, + { + name: "server features with ignore_resource_deletion and trusted_xds_server", + bootstrapContents: []byte(fmt.Sprintf(`{ + "xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "insecure"}], "server_features": ["ignore_resource_deletion", "trusted_xds_server"]}], + "node": {"id": "%s"} + }`, testXDSServerURL, testNodeID)), + wantXDSClientConfig: func(c *bootstrap.Config) xdsclient.Config { + node, serverCfg := c.Node(), c.XDSServers()[0] + serverFeature := xdsclient.ServerFeatureTrustedXDSServer | xdsclient.ServerFeatureIgnoreResourceDeletion + expectedServer := xdsclient.ServerConfig{ServerIdentifier: clients.ServerIdentifier{ServerURI: serverCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}, + ServerFeature: serverFeature} + gServerCfgMap := map[xdsclient.ServerConfig]*bootstrap.ServerConfig{expectedServer: serverCfg} + return xdsclient.Config{ + Servers: []xdsclient.ServerConfig{expectedServer}, + Node: clients.Node{ID: node.GetId(), Cluster: node.GetCluster(), Metadata: node.Metadata, UserAgentName: node.UserAgentName, UserAgentVersion: node.GetUserAgentVersion()}, + Authorities: map[string]xdsclient.Authority{}, + ResourceTypes: map[string]xdsclient.ResourceType{ + version.V3ListenerURL: {TypeURL: version.V3ListenerURL, TypeName: xdsresource.ListenerResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewListenerResourceTypeDecoder(c, gServerCfgMap)}, + version.V3RouteConfigURL: {TypeURL: version.V3RouteConfigURL, TypeName: xdsresource.RouteConfigTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(c, gServerCfgMap)}, + version.V3ClusterURL: {TypeURL: version.V3ClusterURL, TypeName: xdsresource.ClusterResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewClusterResourceTypeDecoder(c, gServerCfgMap)}, + version.V3EndpointsURL: {TypeURL: version.V3EndpointsURL, TypeName: xdsresource.EndpointsResourceTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewEndpointsResourceTypeDecoder(c)}, + }, + MetricsReporter: &metricsReporter{recorder: stats.NewTestMetricsRecorder(), target: testTargetName}, + TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{ + "insecure": { + Credentials: insecure.NewBundle(), + GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + opts = append(opts, serverCfg.DialOptions()...) + return grpc.NewClient(target, opts...) + }}, + }), + } + }, + }, + { + name: "channel creds - unknown type skipped", + bootstrapContents: []byte(fmt.Sprintf(`{ + "xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "unknown-type"}, {"type": "insecure"}]}], + "node": {"id": "%s"} + }`, testXDSServerURL, testNodeID)), // "insecure" is selected + wantXDSClientConfig: func(c *bootstrap.Config) xdsclient.Config { + node, serverCfg := c.Node(), c.XDSServers()[0] // SelectedCreds will be "insecure" + expectedServer := xdsclient.ServerConfig{ServerIdentifier: clients.ServerIdentifier{ServerURI: serverCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}} + gServerCfgMap := map[xdsclient.ServerConfig]*bootstrap.ServerConfig{expectedServer: serverCfg} + return xdsclient.Config{ + Servers: []xdsclient.ServerConfig{expectedServer}, + Node: clients.Node{ID: node.GetId(), Cluster: node.GetCluster(), Metadata: node.Metadata, UserAgentName: node.UserAgentName, UserAgentVersion: node.GetUserAgentVersion()}, + Authorities: map[string]xdsclient.Authority{}, + ResourceTypes: map[string]xdsclient.ResourceType{ + version.V3ListenerURL: {TypeURL: version.V3ListenerURL, TypeName: xdsresource.ListenerResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewListenerResourceTypeDecoder(c, gServerCfgMap)}, + version.V3RouteConfigURL: {TypeURL: version.V3RouteConfigURL, TypeName: xdsresource.RouteConfigTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(c, gServerCfgMap)}, + version.V3ClusterURL: {TypeURL: version.V3ClusterURL, TypeName: xdsresource.ClusterResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewClusterResourceTypeDecoder(c, gServerCfgMap)}, + version.V3EndpointsURL: {TypeURL: version.V3EndpointsURL, TypeName: xdsresource.EndpointsResourceTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewEndpointsResourceTypeDecoder(c)}, + }, + MetricsReporter: &metricsReporter{recorder: stats.NewTestMetricsRecorder(), target: testTargetName}, + TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{ + "insecure": { + Credentials: insecure.NewBundle(), + GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + opts = append(opts, serverCfg.DialOptions()...) + return grpc.NewClient(target, opts...) + }}, + }), + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + bootstrapConfig, err := bootstrap.NewConfigFromContents(tt.bootstrapContents) + if err != nil { + t.Fatalf("Failed to create bootstrap config: %v", err) + } + gotCfg, err := BuildXDSClientConfig(bootstrapConfig, stats.NewTestMetricsRecorder(), testTargetName, 0) + if err != nil { + t.Fatalf("Failed to build XDSClientConfig: %v", err) + } + + wantCfg := tt.wantXDSClientConfig(bootstrapConfig) + + unexportedTypeOpts := cmpopts.IgnoreUnexported(clients.Node{}, grpctransport.Builder{}) + ignoreTypeOpts := cmpopts.IgnoreTypes(sync.Mutex{}) + resourceTypeCmpOpts := cmp.Comparer(func(a, b xdsclient.ResourceType) bool { + return a.TypeURL == b.TypeURL && a.TypeName == b.TypeName && a.AllResourcesRequiredInSotW == b.AllResourcesRequiredInSotW && reflect.TypeOf(a.Decoder) == reflect.TypeOf(b.Decoder) + }) + metricsReporterCmpOpts := cmp.Comparer(func(a, b clients.MetricsReporter) bool { + if (a == nil) != (b == nil) { + return false + } + if a == nil { // Both are nil + return true + } + // Both are non-nil, compare type and target. + aConcrete, aOK := a.(*metricsReporter) + bConcrete, bOK := b.(*metricsReporter) + if !(aOK && bOK && aConcrete.target == bConcrete.target) { + return false + } + // Compare recorder by type. + if (aConcrete.recorder == nil) != (bConcrete.recorder == nil) { + return false + } + // If both are nil, recorder check passes. If both non-nil, check types. + return aConcrete.recorder == nil || reflect.TypeOf(aConcrete.recorder) == reflect.TypeOf(bConcrete.recorder) + }) + transportBuilderCmpOpts := cmp.Comparer(func(a, b grpctransport.Config) bool { + // Compare Credentials by type + credsEqual := true + if (a.Credentials == nil) != (b.Credentials == nil) { + credsEqual = false + } else if a.Credentials != nil && reflect.TypeOf(a.Credentials) != reflect.TypeOf(b.Credentials) { + credsEqual = false + } + // Compare GRPCNewClient by nil-ness + newClientEqual := (a.GRPCNewClient == nil) == (b.GRPCNewClient == nil) + return credsEqual && newClientEqual + }) + + if diff := cmp.Diff(wantCfg, gotCfg, protocmp.Transform(), unexportedTypeOpts, ignoreTypeOpts, resourceTypeCmpOpts, metricsReporterCmpOpts, transportBuilderCmpOpts); diff != "" { + t.Errorf("buildXDSClientConfig() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func (s) TestServerConfigCallCredsIntegration(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSBootstrapCallCredsEnabled, true) + // Test server config with both channel and call credentials. + serverConfigJSON := `{ + "server_uri": "xds-server:443", + "channel_creds": [{"type": "tls", "config": {}}], + "call_creds": [ + { + "type": "jwt_token_file", + "config": {"jwt_token_file": "/token.jwt"} + } + ] + }` + + var sc bootstrap.ServerConfig + if err := sc.UnmarshalJSON([]byte(serverConfigJSON)); err != nil { + t.Fatalf("Failed to unmarshal server config: %v", err) + } + // Verify call credentials are processed. + callCreds := sc.CallCredsConfigs() + if len(callCreds) != 1 { + t.Errorf("Got %d call credential configs, want 1", len(callCreds)) + } + dialOpts := sc.DialOptions() + if len(dialOpts) != 1 { + t.Errorf("Got %d dial options, want 1", len(dialOpts)) + } +} diff --git a/internal/xds/xdsclient/internal/internal.go b/internal/xds/xdsclient/internal/internal.go new file mode 100644 index 0000000..cdbb86d --- /dev/null +++ b/internal/xds/xdsclient/internal/internal.go @@ -0,0 +1,33 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package internal contains functionality internal to the xdsclient package. +package internal + +// The following vars can be overridden by tests. +var ( + // GRPCNewClient returns a new gRPC Client. + GRPCNewClient any // func(string, ...grpc.DialOption) (*grpc.ClientConn, error) + + // NewADSStream returns a new ADS stream. + NewADSStream any // func(context.Context, *grpc.ClientConn) (v3adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesClient, error) + + // ResourceWatchStateForTesting gets the watch state for the resource + // identified by the given resource type and resource name. Returns a + // non-nil error if there is no such resource being watched. + ResourceWatchStateForTesting any // func(xdsclient.XDSClient, xdsresource.Type, string) error +) diff --git a/internal/xds/xdsclient/logging.go b/internal/xds/xdsclient/logging.go new file mode 100644 index 0000000..00b6392 --- /dev/null +++ b/internal/xds/xdsclient/logging.go @@ -0,0 +1,36 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +var logger = grpclog.Component("xds") + +func prefixLogger(p *clientImpl) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, clientPrefix(p)) +} + +func clientPrefix(p *clientImpl) string { + return fmt.Sprintf("[xds-client %p] ", p) +} diff --git a/internal/xds/xdsclient/metrics_test.go b/internal/xds/xdsclient/metrics_test.go new file mode 100644 index 0000000..9b3541d --- /dev/null +++ b/internal/xds/xdsclient/metrics_test.go @@ -0,0 +1,361 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/stats" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter. +) + +type noopListenerWatcher struct{} + +func (noopListenerWatcher) ResourceChanged(_ *xdsresource.ListenerUpdate, onDone func()) { + onDone() +} + +func (noopListenerWatcher) ResourceError(_ error, onDone func()) { + onDone() +} + +func (noopListenerWatcher) AmbientError(_ error, onDone func()) { + onDone() +} + +// TestResourceUpdateMetrics configures an xDS client, and a management server +// to send valid and invalid LDS updates, and verifies that the expected metrics +// for both good and bad updates are emitted. +func (s) TestResourceUpdateMetrics(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + tmr := stats.NewTestMetricsRecorder() + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l}) + const listenerResourceName = "test-listener-resource" + const routeConfigurationName = "test-route-configuration-resource" + nodeID := uuid.New().String() + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + "authority": []byte("{}"), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := NewPool(config) + client, close, err := pool.NewClientForTesting(OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + MetricsRecorder: tmr, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Watch the valid listener configured on the management server. This should + // cause a resource updates valid count to emit eventually. + xdsresource.WatchListener(client, listenerResourceName, noopListenerWatcher{}) + mdWant := stats.MetricsData{ + Handle: xdsClientResourceUpdatesValidMetric.Descriptor(), + IntIncr: 1, + LabelKeys: []string{"grpc.target", "grpc.xds.server", "grpc.xds.resource_type"}, + LabelVals: []string{"Test/ResourceUpdateMetrics", mgmtServer.Address, "ListenerResource"}, + } + if err := tmr.WaitForInt64Count(ctx, mdWant); err != nil { + t.Fatal(err.Error()) + } + // Invalid should have no recording point. + if got, _ := tmr.Metric("grpc.xds_client.resource_updates_invalid"); got != 0 { + t.Fatalf("Unexpected data for metric \"grpc.xds_client.resource_updates_invalid\", got: %v, want: %v", got, 0) + } + + // Update management server with a bad update. Eventually, tmr should + // receive an invalid count received metric. The successful metric should + // stay the same. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)}, + SkipValidation: true, + } + resources.Listeners[0].ApiListener = nil + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + mdWant = stats.MetricsData{ + Handle: xdsClientResourceUpdatesInvalidMetric.Descriptor(), + IntIncr: 1, + LabelKeys: []string{"grpc.target", "grpc.xds.server", "grpc.xds.resource_type"}, + LabelVals: []string{"Test/ResourceUpdateMetrics", mgmtServer.Address, "ListenerResource"}, + } + if err := tmr.WaitForInt64Count(ctx, mdWant); err != nil { + t.Fatal(err.Error()) + } + // Valid should stay the same at 1. + if got, _ := tmr.Metric("grpc.xds_client.resource_updates_valid"); got != 1 { + t.Fatalf("Unexpected data for metric \"grpc.xds_client.resource_updates_invalid\", got: %v, want: %v", got, 1) + } +} + +// TestServerFailureMetrics_BeforeResponseRecv configures an xDS client, and a +// management server. It then register a watcher and stops the management +// server before sending a resource update, and verifies that the expected +// metrics for server failure are emitted. +func (s) TestServerFailureMetrics_BeforeResponseRecv(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + tmr := stats.NewTestMetricsRecorder() + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + streamOpened := make(chan struct{}, 1) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + OnStreamOpen: func(context.Context, int64, string) error { + select { + case streamOpened <- struct{}{}: + default: + } + return nil + }, + }) + + nodeID := uuid.New().String() + + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + "authority": []byte("{}"), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := NewPool(config) + client, close, err := pool.NewClientForTesting(OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + MetricsRecorder: tmr, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + const listenerResourceName = "test-listener-resource" + + // Watch for the listener on the above management server. + xdsresource.WatchListener(client, listenerResourceName, noopListenerWatcher{}) + // Verify that an ADS stream is opened and an LDS request with the above + // resource name is sent. + select { + case <-streamOpened: + case <-ctx.Done(): + t.Fatal("Timeout when waiting for ADS stream to open") + } + + // Close the listener and ensure that the ADS stream breaks. This should + // cause a server failure count to emit eventually. + lis.Stop() + + // Restart to prevent the attempt to create a new ADS stream after back off. + lis.Restart() + + mdWant := stats.MetricsData{ + Handle: xdsClientServerFailureMetric.Descriptor(), + IntIncr: 1, + LabelKeys: []string{"grpc.target", "grpc.xds.server"}, + LabelVals: []string{"Test/ServerFailureMetrics_BeforeResponseRecv", mgmtServer.Address}, + } + if err := tmr.WaitForInt64Count(ctx, mdWant); err != nil { + t.Fatal(err.Error()) + } +} + +// TestServerFailureMetrics_AfterResponseRecv configures an xDS client and a +// management server to send a valid LDS update, and verifies that the +// successful update metric is emitted. When the client ACKs the update, the +// server returns an error, breaking the stream. The test then verifies that the +// server failure metric is not emitted, because the ADS stream was closed after +// a response was received on the stream. Finally, the test waits for the client +// to establish a new stream and verifies that the client emits a metric after +// receiving a successful update. +func (s) TestServerFailureMetrics_AfterResponseRecv(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + tmr := stats.NewTestMetricsRecorder() + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + streamCreationQuota := make(chan struct{}, 1) + streamCreationQuota <- struct{}{} + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + OnStreamOpen: func(context.Context, int64, string) error { + // The following select block is used to block stream creation after + // the first stream has failed, but while we are waiting to verify + // that the failure metric is not reported. + select { + case <-streamCreationQuota: + case <-ctx.Done(): + } + return nil + }, + OnStreamRequest: func(streamID int64, req *v3discoverypb.DiscoveryRequest) error { + // We only want the ACK on the first stream to return an error + // (leading to stream closure), without effecting subsequent stream + // attempts. + if streamID == 1 && req.GetVersionInfo() != "" { + return errors.New("test configured error") + } + return nil + }}, + ) + const listenerResourceName = "test-listener-resource" + const routeConfigurationName = "test-route-configuration-resource" + nodeID := uuid.New().String() + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + "authority": []byte("{}"), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := NewPool(config) + client, closePool, err := pool.NewClientForTesting(OptionsForTesting{ + Name: t.Name(), + MetricsRecorder: tmr, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer closePool() + + // Watch the valid listener configured on the management server. This should + // cause a resource updates valid count to emit eventually. + xdsresource.WatchListener(client, listenerResourceName, noopListenerWatcher{}) + mdSuccess := stats.MetricsData{ + Handle: xdsClientResourceUpdatesValidMetric.Descriptor(), + IntIncr: 1, + LabelKeys: []string{"grpc.target", "grpc.xds.server", "grpc.xds.resource_type"}, + LabelVals: []string{"Test/ServerFailureMetrics_AfterResponseRecv", mgmtServer.Address, "ListenerResource"}, + } + if err := tmr.WaitForInt64Count(ctx, mdSuccess); err != nil { + t.Fatal(err.Error()) + } + + // When the client sends an ACK, the management server would reply with an + // error, breaking the stream. + mdFailure := stats.MetricsData{ + Handle: xdsClientServerFailureMetric.Descriptor(), + IntIncr: 1, + LabelKeys: []string{"grpc.target", "grpc.xds.server"}, + LabelVals: []string{"Test/ServerFailureMetrics_AfterResponseRecv", mgmtServer.Address}, + } + + // Server failure should still have no recording point. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if err := tmr.WaitForInt64Count(sCtx, mdFailure); err == nil { + t.Fatalf("tmr.WaitForInt64Count(%v) succeeded when expected to timeout.", mdFailure) + } else if sCtx.Err() == nil { + t.Fatalf("tmr.WaitForInt64Count(%v) = %v, want context deadline exceeded", mdFailure, err) + } + + // Unblock stream creation and verify that an update is received + // successfully. + close(streamCreationQuota) + if err := tmr.WaitForInt64Count(ctx, mdSuccess); err != nil { + t.Fatal(err.Error()) + } +} diff --git a/internal/xds/xdsclient/pool.go b/internal/xds/xdsclient/pool.go new file mode 100644 index 0000000..cb34a29 --- /dev/null +++ b/internal/xds/xdsclient/pool.go @@ -0,0 +1,320 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "fmt" + "sync" + "time" + + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal/envconfig" + istats "google.golang.org/grpc/internal/stats" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/protobuf/proto" +) + +var ( + // DefaultPool is the default pool for xDS clients. It is created at init + // time and reads bootstrap configuration from env vars to create the xDS + // client. + DefaultPool = &Pool{ + clients: make(map[string]*clientImpl), + getConfiguration: sync.OnceValues(bootstrap.GetConfiguration), + } +) + +// Pool represents a pool of xDS clients that share the same bootstrap +// configuration. +type Pool struct { + // Note that mu should ideally only have to guard clients. But here, we need + // it to guard config as well since SetFallbackBootstrapConfig writes to + // config. + mu sync.Mutex + clients map[string]*clientImpl + fallbackConfig *bootstrap.Config // TODO(i/8661): remove fallbackConfig. + // getConfiguration is a sync.OnceValues that attempts to read the bootstrap + // configuration from environment variables once. + getConfiguration func() (*bootstrap.Config, error) +} + +// OptionsForTesting contains options to configure xDS client creation for +// testing purposes only. +type OptionsForTesting struct { + // Name is a unique name for this xDS client. + Name string + + // WatchExpiryTimeout is the timeout for xDS resource watch expiry. If + // unspecified, uses the default value used in non-test code. + WatchExpiryTimeout time.Duration + + // StreamBackoffAfterFailure is the backoff function used to determine the + // backoff duration after stream failures. + // If unspecified, uses the default value used in non-test code. + StreamBackoffAfterFailure func(int) time.Duration + + // MetricsRecorder is the metrics recorder the xDS Client will use. If + // unspecified, uses a no-op MetricsRecorder. + MetricsRecorder estats.MetricsRecorder + + // Config is the xDS bootstrap configuration that will be used to initialize + // the client. If unset, the client will use the config provided by env + // variables. + Config *bootstrap.Config +} + +// NewPool creates a new xDS client pool with the given bootstrap config. +// +// If a nil bootstrap config is passed and SetFallbackBootstrapConfig is not +// called before a call to NewClient, the latter will fail. i.e. if there is an +// attempt to create an xDS client from the pool without specifying bootstrap +// configuration (either at pool creation time or by setting the fallback +// bootstrap configuration), xDS client creation will fail. +func NewPool(config *bootstrap.Config) *Pool { + return &Pool{ + clients: make(map[string]*clientImpl), + getConfiguration: func() (*bootstrap.Config, error) { + return config, nil + }, + } +} + +// NewClientWithConfig returns an xDS client with the given name from the pool. If the +// client doesn't already exist, it creates a new xDS client and adds it to the +// pool. +// +// The second return value represents a close function which the caller is +// expected to invoke once they are done using the client. It is safe for the +// caller to invoke this close function multiple times. +func (p *Pool) NewClientWithConfig(name string, metricsRecorder estats.MetricsRecorder, config *bootstrap.Config) (XDSClient, func(), error) { + return p.newRefCounted(name, metricsRecorder, defaultWatchExpiryTimeout, config) +} + +// NewClient returns an xDS client with the given name from the pool. If the +// client doesn't already exist, it creates a new xDS client and adds it to the +// pool. +// +// The second return value represents a close function which the caller is +// expected to invoke once they are done using the client. It is safe for the +// caller to invoke this close function multiple times. +func (p *Pool) NewClient(name string, metricsRecorder estats.MetricsRecorder) (XDSClient, func(), error) { + return p.newRefCounted(name, metricsRecorder, defaultWatchExpiryTimeout, nil) +} + +// NewClientForTesting returns an xDS client configured with the provided +// options from the pool. If the client doesn't already exist, it creates a new +// xDS client and adds it to the pool. +// +// The second return value represents a close function which the caller is +// expected to invoke once they are done using the client. It is safe for the +// caller to invoke this close function multiple times. +// +// # Testing Only +// +// This function should ONLY be used for testing purposes. +func (p *Pool) NewClientForTesting(opts OptionsForTesting) (XDSClient, func(), error) { + if opts.Name == "" { + return nil, nil, fmt.Errorf("xds: opts.Name field must be non-empty") + } + if opts.WatchExpiryTimeout == 0 { + opts.WatchExpiryTimeout = defaultWatchExpiryTimeout + } + if opts.StreamBackoffAfterFailure == nil { + opts.StreamBackoffAfterFailure = defaultExponentialBackoff + } + if opts.MetricsRecorder == nil { + opts.MetricsRecorder = istats.NewMetricsRecorderList(nil) + } + c, cancel, err := p.newRefCounted(opts.Name, opts.MetricsRecorder, opts.WatchExpiryTimeout, opts.Config) + if err != nil { + return nil, nil, err + } + return c, cancel, nil +} + +// GetClientForTesting returns an xDS client created earlier using the given +// name from the pool. If the client with the given name doesn't already exist, +// it returns an error. +// +// The second return value represents a close function which the caller is +// expected to invoke once they are done using the client. It is safe for the +// caller to invoke this close function multiple times. +// +// # Testing Only +// +// This function should ONLY be used for testing purposes. +func (p *Pool) GetClientForTesting(name string) (XDSClient, func(), error) { + p.mu.Lock() + defer p.mu.Unlock() + + c, ok := p.clients[name] + if !ok { + return nil, nil, fmt.Errorf("xds:: xDS client with name %q not found", name) + } + c.incrRef() + return c, sync.OnceFunc(func() { p.clientRefCountedClose(name) }), nil +} + +// SetFallbackBootstrapConfig is used to specify a bootstrap configuration +// that will be used as a fallback when the bootstrap environment variables +// are not defined. +// TODO(i/8661): remove SetFallbackBootstrapConfig function. +func (p *Pool) SetFallbackBootstrapConfig(config *bootstrap.Config) { + p.mu.Lock() + defer p.mu.Unlock() + p.fallbackConfig = config +} + +// DumpResources returns the status and contents of all xDS resources. +func (p *Pool) DumpResources() *v3statuspb.ClientStatusResponse { + p.mu.Lock() + defer p.mu.Unlock() + + resp := &v3statuspb.ClientStatusResponse{} + for key, client := range p.clients { + b, err := client.DumpResources() + if err != nil { + return nil + } + r := &v3statuspb.ClientStatusResponse{} + if err := proto.Unmarshal(b, r); err != nil { + return nil + } + cfg := r.Config[0] + cfg.ClientScope = key + resp.Config = append(resp.Config, cfg) + } + return resp +} + +// BootstrapConfigForTesting returns the bootstrap configuration used by the +// pool. The caller should not mutate the returned config. +// +// To be used only for testing purposes. +func (p *Pool) BootstrapConfigForTesting() *bootstrap.Config { + p.mu.Lock() + defer p.mu.Unlock() + cfg, _ := p.getConfiguration() + if cfg != nil { + return cfg + } + return p.fallbackConfig +} + +// UnsetBootstrapConfigForTesting unsets the bootstrap configuration used by +// the pool. +// +// To be used only for testing purposes. +func (p *Pool) UnsetBootstrapConfigForTesting() { + p.mu.Lock() + defer p.mu.Unlock() + p.fallbackConfig = nil + p.getConfiguration = sync.OnceValues(bootstrap.GetConfiguration) +} + +func (p *Pool) clientRefCountedClose(name string) { + p.mu.Lock() + client, ok := p.clients[name] + if !ok { + logger.Errorf("Attempt to close a non-existent xDS client with name %s", name) + p.mu.Unlock() + return + } + if client.decrRef() != 0 { + p.mu.Unlock() + return + } + delete(p.clients, name) + + for _, s := range client.bootstrapConfig.XDSServers() { + for _, f := range s.Cleanups() { + f() + } + } + for _, a := range client.bootstrapConfig.Authorities() { + for _, s := range a.XDSServers { + for _, f := range s.Cleanups() { + f() + } + } + } + // The allowed-services credentials are owned by the (pool-shared) + // bootstrap config and are released when the last reference to the + // client is dropped, following the same lifecycle as the xDS server + // credentials released above. + for _, svc := range client.bootstrapConfig.AllowedGRPCServices() { + for _, f := range svc.Cleanups() { + f() + } + } + p.mu.Unlock() + + // This attempts to close the transport to the management server and could + // theoretically call back into the xdsclient package again and deadlock. + // Hence, this needs to be called without holding the lock. + client.Close() + + xdsClientImplCloseHook(name) +} + +// newRefCounted creates a new reference counted xDS client implementation for +// name, if one does not exist already. If an xDS client for the given name +// exists, it gets a reference to it and returns it. +func (p *Pool) newRefCounted(name string, metricsRecorder estats.MetricsRecorder, watchExpiryTimeout time.Duration, bConfig *bootstrap.Config) (*clientImpl, func(), error) { + p.mu.Lock() + defer p.mu.Unlock() + + if c := p.clients[name]; c != nil { + c.incrRef() + return c, sync.OnceFunc(func() { p.clientRefCountedClose(name) }), nil + } + + config := bConfig + if config == nil { + var err error + config, err = p.getConfiguration() + if err != nil { + return nil, nil, fmt.Errorf("xds: failed to read xDS bootstrap config from env vars: %v", err) + } + if config == nil { + // If the environment variables are not set, then fallback bootstrap + // configuration should be set before attempting to create an xDS client, + // else xDS client creation will fail. + config = p.fallbackConfig + } + } + + if config == nil { + return nil, nil, fmt.Errorf("failed to read xDS bootstrap config from env vars: bootstrap environment variables (%q or %q) not defined and fallback config not set", envconfig.XDSBootstrapFileNameEnv, envconfig.XDSBootstrapFileContentEnv) + } + + c, err := newClientImpl(config, metricsRecorder, name, watchExpiryTimeout) + if err != nil { + return nil, nil, err + } + if logger.V(2) { + c.logger.Infof("Created client with name %q and bootstrap configuration:\n %s", name, config) + } + p.clients[name] = c + xdsClientImplCreateHook(name) + + logger.Infof("xDS node ID: %s", config.Node().GetId()) + return c, sync.OnceFunc(func() { p.clientRefCountedClose(name) }), nil +} diff --git a/internal/xds/xdsclient/pool/pool_ext_test.go b/internal/xds/xdsclient/pool/pool_ext_test.go new file mode 100644 index 0000000..83f8a99 --- /dev/null +++ b/internal/xds/xdsclient/pool/pool_ext_test.go @@ -0,0 +1,250 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package pool_ext_test + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + _ "google.golang.org/grpc/xds" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + estats "google.golang.org/grpc/experimental/stats" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestDefaultPool_LazyLoadBootstrapConfig verifies that the DefaultPool +// lazily loads the bootstrap configuration from environment variables when +// an xDS client is created for the first time. +// +// If tries to create the new client in DefaultPool at the start of test when +// none of the env vars are set. This should fail. +// +// Then it sets the env var XDSBootstrapFileName and retry creating a client +// in DefaultPool. This should succeed. +func (s) TestDefaultPool_LazyLoadBootstrapConfig(t *testing.T) { + _, closeFunc, err := xdsclient.DefaultPool.NewClient(t.Name(), &estats.UnimplementedMetricsRecorder{}) + if err == nil { + t.Fatalf("xdsclient.DefaultPool.NewClient() succeeded without setting bootstrap config env vars, want failure") + } + + bs, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, "non-existent-management-server")), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, uuid.New().String())), + CertificateProviders: map[string]json.RawMessage{ + "cert-provider-instance": json.RawMessage("{}"), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + testutils.CreateBootstrapFileForTesting(t, bs) + if cfg := xdsclient.DefaultPool.BootstrapConfigForTesting(); cfg != nil { + t.Fatalf("DefaultPool.BootstrapConfigForTesting() = %v, want nil", cfg) + } + + // The pool will attempt to read the env vars only once. Reset the pool's + // state to make it re-read the env vars during next client creation. + xdsclient.DefaultPool.UnsetBootstrapConfigForTesting() + + _, closeFunc, err = xdsclient.DefaultPool.NewClient(t.Name(), &estats.UnimplementedMetricsRecorder{}) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer func() { + closeFunc() + xdsclient.DefaultPool.UnsetBootstrapConfigForTesting() + }() + + if xdsclient.DefaultPool.BootstrapConfigForTesting() == nil { + t.Fatalf("DefaultPool.BootstrapConfigForTesting() = nil, want non-nil") + } +} + +// TestNestedXDSChannel tests a scenario where xDS is used to resolve the +// address of the management server, resulting in the creation of two nested xDS +// client channels. The server URI of the first xDS management server uses +// the xDS scheme. To resolve its address, the client creates a second xDS +// channel to a different management server. After retrieving the EDS resource +// for the first management server, the client connects to it to obtain the +// address of the test stub server. The test verifies that an RPC to the test +// stub server succeeds. It also helps detect potential deadlocks during the +// creation and teardown of the two nested xDS channels. +func (s) TestNestedXDSChannel(t *testing.T) { + lis1, err := testutils.LocalTCPListener() + if err != nil { + t.Fatal(err) + } + lis2, err := testutils.LocalTCPListener() + if err != nil { + t.Fatal(err) + } + + const managementServerServiceName = "management_server" + const managementServerAuthority = "management_server_authority" + managementServerLDSName := fmt.Sprintf( + "xdstp://%s/envoy.config.listener.v3.Listener/%s", + managementServerAuthority, managementServerServiceName, + ) + managementServerRDSName := fmt.Sprintf( + "xdstp://%s/envoy.config.route.v3.RouteConfiguration/route-%s", + managementServerAuthority, managementServerServiceName, + ) + + managementServerCDSName := fmt.Sprintf( + "xdstp://%s/envoy.config.cluster.v3.Cluster/cluster-%s", + managementServerAuthority, managementServerServiceName, + ) + + managementServerEDSName := fmt.Sprintf( + "xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/endpoints-%s", + managementServerAuthority, managementServerServiceName, + ) + + const serviceName = "my-service-client-side-xds" + const rdsName = "route-" + serviceName + const cdsName = "cluster-" + serviceName + const edsName = "endpoints-" + serviceName + + // Start a management server that stores resources related to the test + // server. + managementServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis1}) + + // Start a management server that stores resources related to + // managementServer1. + managementServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis2}) + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Create a bootstrap configuration where the xDS scheme is used to connect + // to the default management server. A non-default authority is also + // specified which has resources for resolving the address of the default + // management server. + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: fmt.Appendf(nil, `[{ + "server_uri": "xds://%s/%s", + "channel_creds": [{"type": "insecure"}] + }]`, managementServerAuthority, managementServerServiceName), + Node: fmt.Appendf(nil, `{"id": "%s"}`, nodeID), + Authorities: map[string]json.RawMessage{ + managementServerAuthority: fmt.Appendf(nil, `{ + "xds_servers": [{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]}`, managementServer1.Address), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %v", err) + } + xdsclient.DefaultPool.SetFallbackBootstrapConfig(config) + defer func() { xdsclient.DefaultPool.UnsetBootstrapConfigForTesting() }() + + // Update the management server that holds resources for resolving the real + // management server. + managementServerResources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(managementServerLDSName, managementServerRDSName)}, + Routes: []*v3routepb.RouteConfiguration{ + e2e.DefaultRouteConfig(managementServerRDSName, managementServerServiceName, managementServerCDSName), + }, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(managementServerCDSName, managementServerEDSName, e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(managementServerEDSName, "localhost", []uint32{testutils.ParsePort(t, managementServer2.Address)}), + }, + SkipValidation: true, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if err := managementServer1.Update(ctx, managementServerResources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", managementServerResources, err) + } + + // Update the management server that resolves the address of the test stub + // server. + resouces := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{ + e2e.DefaultRouteConfig(rdsName, serviceName, cdsName), + }, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(edsName, "localhost", []uint32{testutils.ParsePort(t, server.Address)}), + }, + SkipValidation: true, + } + + if err := managementServer2.Update(ctx, resouces); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resouces, err) + } + + cc, err := grpc.NewClient("xds:///"+serviceName, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} diff --git a/internal/xds/xdsclient/requests_counter.go b/internal/xds/xdsclient/requests_counter.go new file mode 100644 index 0000000..08f38bb --- /dev/null +++ b/internal/xds/xdsclient/requests_counter.go @@ -0,0 +1,83 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "fmt" + "sync" + "sync/atomic" +) + +type clusterNameAndServiceName struct { + clusterName, edsServiceName string +} + +type clusterRequestsCounter struct { + mu sync.Mutex + clusters map[clusterNameAndServiceName]*ClusterRequestsCounter +} + +var src = &clusterRequestsCounter{ + clusters: make(map[clusterNameAndServiceName]*ClusterRequestsCounter), +} + +// ClusterRequestsCounter is used to track the total inflight requests for a +// service with the provided name. +type ClusterRequestsCounter struct { + ClusterName string + EDSServiceName string + numRequests uint32 +} + +// GetClusterRequestsCounter returns the ClusterRequestsCounter with the +// provided serviceName. If one does not exist, it creates it. +func GetClusterRequestsCounter(clusterName, edsServiceName string) *ClusterRequestsCounter { + src.mu.Lock() + defer src.mu.Unlock() + k := clusterNameAndServiceName{ + clusterName: clusterName, + edsServiceName: edsServiceName, + } + c, ok := src.clusters[k] + if !ok { + c = &ClusterRequestsCounter{ClusterName: clusterName} + src.clusters[k] = c + } + return c +} + +// StartRequest starts a request for a cluster, incrementing its number of +// requests by 1. Returns an error if the max number of requests is exceeded. +func (c *ClusterRequestsCounter) StartRequest(max uint32) error { + // Note that during race, the limits could be exceeded. This is allowed: + // "Since the implementation is eventually consistent, races between threads + // may allow limits to be potentially exceeded." + // https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/circuit_breaking#arch-overview-circuit-break. + if atomic.LoadUint32(&c.numRequests) >= max { + return fmt.Errorf("max requests %v exceeded on service %v", max, c.ClusterName) + } + atomic.AddUint32(&c.numRequests, 1) + return nil +} + +// EndRequest ends a request for a service, decrementing its number of requests +// by 1. +func (c *ClusterRequestsCounter) EndRequest() { + atomic.AddUint32(&c.numRequests, ^uint32(0)) +} diff --git a/internal/xds/xdsclient/requests_counter_test.go b/internal/xds/xdsclient/requests_counter_test.go new file mode 100644 index 0000000..e2eeea7 --- /dev/null +++ b/internal/xds/xdsclient/requests_counter_test.go @@ -0,0 +1,162 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "sync" + "sync/atomic" + "testing" +) + +const testService = "test-service-name" + +type counterTest struct { + name string + maxRequests uint32 + numRequests uint32 + expectedSuccesses uint32 + expectedErrors uint32 +} + +var tests = []counterTest{ + { + name: "does-not-exceed-max-requests", + maxRequests: 1024, + numRequests: 1024, + expectedSuccesses: 1024, + expectedErrors: 0, + }, + { + name: "exceeds-max-requests", + maxRequests: 32, + numRequests: 64, + expectedSuccesses: 32, + expectedErrors: 32, + }, +} + +func resetClusterRequestsCounter() { + src = &clusterRequestsCounter{ + clusters: make(map[clusterNameAndServiceName]*ClusterRequestsCounter), + } +} + +func testCounter(t *testing.T, test counterTest) { + requestsStarted := make(chan struct{}) + requestsSent := sync.WaitGroup{} + requestsSent.Add(int(test.numRequests)) + requestsDone := sync.WaitGroup{} + requestsDone.Add(int(test.numRequests)) + var lastError atomic.Value + var successes, errors uint32 + for i := 0; i < int(test.numRequests); i++ { + go func() { + counter := GetClusterRequestsCounter(test.name, testService) + defer requestsDone.Done() + err := counter.StartRequest(test.maxRequests) + if err == nil { + atomic.AddUint32(&successes, 1) + } else { + atomic.AddUint32(&errors, 1) + lastError.Store(err) + } + requestsSent.Done() + if err == nil { + <-requestsStarted + counter.EndRequest() + } + }() + } + requestsSent.Wait() + close(requestsStarted) + requestsDone.Wait() + loadedError := lastError.Load() + if test.expectedErrors > 0 && loadedError == nil { + t.Error("no error when error expected") + } + if test.expectedErrors == 0 && loadedError != nil { + t.Errorf("error starting request: %v", loadedError.(error)) + } + // We allow the limits to be exceeded during races. + // + // But we should never over-limit, so this test fails if there are less + // successes than expected. + if successes < test.expectedSuccesses || errors > test.expectedErrors { + t.Errorf("unexpected number of (successes, errors), expected (%v, %v), encountered (%v, %v)", test.expectedSuccesses, test.expectedErrors, successes, errors) + } +} + +func (s) TestRequestsCounter(t *testing.T) { + defer resetClusterRequestsCounter() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testCounter(t, test) + }) + } +} + +func (s) TestGetClusterRequestsCounter(t *testing.T) { + defer resetClusterRequestsCounter() + for _, test := range tests { + counterA := GetClusterRequestsCounter(test.name, testService) + counterB := GetClusterRequestsCounter(test.name, testService) + if counterA != counterB { + t.Errorf("counter %v %v != counter %v %v", counterA, *counterA, counterB, *counterB) + } + } +} + +func startRequests(t *testing.T, n uint32, max uint32, counter *ClusterRequestsCounter) { + for i := uint32(0); i < n; i++ { + if err := counter.StartRequest(max); err != nil { + t.Fatalf("error starting initial request: %v", err) + } + } +} + +func (s) TestSetMaxRequestsIncreased(t *testing.T) { + defer resetClusterRequestsCounter() + const clusterName string = "set-max-requests-increased" + var initialMax uint32 = 16 + + counter := GetClusterRequestsCounter(clusterName, testService) + startRequests(t, initialMax, initialMax, counter) + if err := counter.StartRequest(initialMax); err == nil { + t.Fatal("unexpected success on start request after max met") + } + + newMax := initialMax + 1 + if err := counter.StartRequest(newMax); err != nil { + t.Fatalf("unexpected error on start request after max increased: %v", err) + } +} + +func (s) TestSetMaxRequestsDecreased(t *testing.T) { + defer resetClusterRequestsCounter() + const clusterName string = "set-max-requests-decreased" + var initialMax uint32 = 16 + + counter := GetClusterRequestsCounter(clusterName, testService) + startRequests(t, initialMax-1, initialMax, counter) + + newMax := initialMax - 1 + if err := counter.StartRequest(newMax); err == nil { + t.Fatalf("unexpected success on start request after max decreased: %v", err) + } +} diff --git a/internal/xds/xdsclient/resource_types.go b/internal/xds/xdsclient/resource_types.go new file mode 100644 index 0000000..547e2a4 --- /dev/null +++ b/internal/xds/xdsclient/resource_types.go @@ -0,0 +1,54 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsclient + +import ( + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" +) + +func supportedResourceTypes(config *bootstrap.Config, gServerCfgMap map[xdsclient.ServerConfig]*bootstrap.ServerConfig) map[string]xdsclient.ResourceType { + return map[string]xdsclient.ResourceType{ + version.V3ListenerURL: { + TypeURL: version.V3ListenerURL, + TypeName: xdsresource.ListenerResourceTypeName, + AllResourcesRequiredInSotW: true, + Decoder: xdsresource.NewListenerResourceTypeDecoder(config, gServerCfgMap), + }, + version.V3RouteConfigURL: { + TypeURL: version.V3RouteConfigURL, + TypeName: xdsresource.RouteConfigTypeName, + AllResourcesRequiredInSotW: false, + Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(config, gServerCfgMap), + }, + version.V3ClusterURL: { + TypeURL: version.V3ClusterURL, + TypeName: xdsresource.ClusterResourceTypeName, + AllResourcesRequiredInSotW: true, + Decoder: xdsresource.NewClusterResourceTypeDecoder(config, gServerCfgMap), + }, + version.V3EndpointsURL: { + TypeURL: version.V3EndpointsURL, + TypeName: xdsresource.EndpointsResourceTypeName, + AllResourcesRequiredInSotW: false, + Decoder: xdsresource.NewEndpointsResourceTypeDecoder(config), + }, + } +} diff --git a/internal/xds/xdsclient/tests/ads_stream_ack_nack_test.go b/internal/xds/xdsclient/tests/ads_stream_ack_nack_test.go new file mode 100644 index 0000000..cb6e307 --- /dev/null +++ b/internal/xds/xdsclient/tests/ads_stream_ack_nack_test.go @@ -0,0 +1,512 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +// Creates an xDS client with the given bootstrap contents. +func createXDSClient(t *testing.T, bootstrapContents []byte) xdsclient.XDSClient { + t.Helper() + + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + t.Cleanup(close) + return client +} + +// Tests simple ACK and NACK scenarios on the ADS stream: +// 1. When a good response is received, i.e. once that is expected to be ACKed, +// the test verifies that an ACK is sent matching the version and nonce from +// the response. +// 2. When a subsequent bad response is received, i.e. once is expected to be +// NACKed, the test verifies that a NACK is sent matching the previously +// ACKed version and current nonce from the response. +// 3. When a subsequent good response is received, the test verifies that an +// ACK is sent matching the version and nonce from the current response. +func (s) TestADS_ACK_NACK_Simple(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create an xDS management server listening on a local port. Configure the + // request and response handlers to push on channels that are inspected by + // the test goroutine to verify ACK version and nonce. + streamRequestCh := testutils.NewChannel() + streamResponseCh := testutils.NewChannel() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + streamRequestCh.SendContext(ctx, req) + return nil + }, + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + streamResponseCh.SendContext(ctx, resp) + }, + }) + + // Create a listener resource on the management server. + const listenerName = "listener" + const routeConfigName = "route-config" + nodeID := uuid.New().String() + listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client with bootstrap pointing to the above server. + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + client := createXDSClient(t, bc) + + // Register a watch for a listener resource. + lw := newListenerWatcher() + ldsCancel := xdsresource.WatchListener(client, listenerName, lw) + defer ldsCancel() + + // Verify that the initial discovery request matches expectation. + r, err := streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for the initial discovery request") + } + gotReq := r.(*v3discoverypb.DiscoveryRequest) + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + }, + ResourceNames: []string{listenerName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Capture the version and nonce from the response. + r, err = streamResponseCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for a discovery response from the server") + } + gotResp := r.(*v3discoverypb.DiscoveryResponse) + + // Verify that the ACK contains the appropriate version and nonce. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for ACK") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + wantReq.VersionInfo = gotResp.GetVersionInfo() + wantReq.ResponseNonce = gotResp.GetNonce() + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Verify the update received by the watcher. + wantUpdate := listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: routeConfigName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Update the management server with a listener resource that contains an + // empty HTTP connection manager within the apiListener, which will cause + // the resource to be NACKed. + badListener := proto.Clone(listenerResource).(*v3listenerpb.Listener) + badListener.ApiListener.ApiListener = nil + mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{badListener}, + SkipValidation: true, + }) + + r, err = streamResponseCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for a discovery response from the server") + } + gotResp = r.(*v3discoverypb.DiscoveryResponse) + + wantNackErr := xdsresource.NewError(xdsresource.ErrorTypeNACKed, "unexpected http connection manager resource type") + if err := verifyListenerUpdate(ctx, lw.updateCh, listenerUpdateErrTuple{err: wantNackErr}); err != nil { + t.Fatal(err) + } + + // Verify that the NACK contains the appropriate version, nonce and error. + // We expect the version to not change as this is a NACK. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for NACK") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + if gotNonce, wantNonce := gotReq.GetResponseNonce(), gotResp.GetNonce(); gotNonce != wantNonce { + t.Errorf("Unexpected nonce in discovery request, got: %v, want: %v", gotNonce, wantNonce) + } + if gotErr := gotReq.GetErrorDetail(); gotErr == nil || !strings.Contains(gotErr.GetMessage(), wantNackErr.Error()) { + t.Fatalf("Unexpected error in discovery request, got: %v, want: %v", gotErr.GetMessage(), wantNackErr) + } + + // Update the management server to send a good resource again. + mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource}, + SkipValidation: true, + }) + + // The envoy-go-control-plane management server keeps resending the same + // resource as long as we keep NACK'ing it. So, we will see the bad resource + // sent to us a few times here, before receiving the good resource. + var lastErr error + for { + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for an ACK from the xDS client. Last seen error: %v", lastErr) + } + + r, err = streamResponseCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for a discovery response from the server") + } + gotResp = r.(*v3discoverypb.DiscoveryResponse) + + // Verify that the ACK contains the appropriate version and nonce. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for ACK") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + wantReq.VersionInfo = gotResp.GetVersionInfo() + wantReq.ResponseNonce = gotResp.GetNonce() + wantReq.ErrorDetail = nil + diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()) + if diff == "" { + lastErr = nil + break + } + lastErr = fmt.Errorf("unexpected diff in discovery request, diff (-got, +want):\n%s", diff) + } + + // Verify the update received by the watcher. + for ; ctx.Err() == nil; <-time.After(100 * time.Millisecond) { + if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil { + lastErr = err + continue + } + break + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for listener update. Last seen error: %v", lastErr) + } +} + +// Tests the case where the first response is invalid. The test verifies that +// the NACK contains an empty version string. +func (s) TestADS_NACK_InvalidFirstResponse(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create an xDS management server listening on a local port. Configure the + // request and response handlers to push on channels that are inspected by + // the test goroutine to verify ACK version and nonce. + streamRequestCh := testutils.NewChannel() + streamResponseCh := testutils.NewChannel() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + streamRequestCh.SendContext(ctx, req) + return nil + }, + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + streamResponseCh.SendContext(ctx, resp) + }, + }) + + // Create a listener resource on the management server that is expected to + // be NACKed by the xDS client. + const listenerName = "listener" + const routeConfigName = "route-config" + nodeID := uuid.New().String() + listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName) + listenerResource.ApiListener.ApiListener = nil + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client with bootstrap pointing to the above server. + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + client := createXDSClient(t, bc) + + // Register a watch for a listener resource. + lw := newListenerWatcher() + ldsCancel := xdsresource.WatchListener(client, listenerName, lw) + defer ldsCancel() + + // Verify that the initial discovery request matches expectation. + r, err := streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for the initial discovery request") + } + gotReq := r.(*v3discoverypb.DiscoveryRequest) + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + }, + ResourceNames: []string{listenerName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Capture the version and nonce from the response. + r, err = streamResponseCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for the discovery response from client") + } + gotResp := r.(*v3discoverypb.DiscoveryResponse) + + // Verify that the error is propagated to the watcher. + var wantNackErr = xdsresource.NewError(xdsresource.ErrorTypeNACKed, "unexpected http connection manager resource type") + if err := verifyListenerUpdate(ctx, lw.updateCh, listenerUpdateErrTuple{err: wantNackErr}); err != nil { + t.Fatal(err) + } + + // NACK should contain the appropriate error, nonce, but empty version. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for ACK") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + if gotVersion, wantVersion := gotReq.GetVersionInfo(), ""; gotVersion != wantVersion { + t.Errorf("Unexpected version in discovery request, got: %v, want: %v", gotVersion, wantVersion) + } + if gotNonce, wantNonce := gotReq.GetResponseNonce(), gotResp.GetNonce(); gotNonce != wantNonce { + t.Errorf("Unexpected nonce in discovery request, got: %v, want: %v", gotNonce, wantNonce) + } + if gotErr := gotReq.GetErrorDetail(); gotErr == nil || !strings.Contains(gotErr.GetMessage(), wantNackErr.Error()) { + t.Fatalf("Unexpected error in discovery request, got: %v, want: %v", gotErr.GetMessage(), wantNackErr) + } +} + +// Tests the scenario where the xDS client is no longer interested in a +// resource. The following sequence of events are tested: +// 1. A resource is requested and a good response is received. The test verifies +// that an ACK is sent for this resource. +// 2. The previously requested resource is no longer requested. The test +// verifies that the connection to the management server is closed. +// 3. The same resource is requested again. The test verifies that a new +// request is sent with an empty version string, which corresponds to the +// first request on a new connection. +func (s) TestADS_ACK_NACK_ResourceIsNotRequestedAnymore(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create an xDS management server listening on a local port. Configure the + // request and response handlers to push on channels that are inspected by + // the test goroutine to verify ACK version and nonce. + streamRequestCh := testutils.NewChannel() + streamResponseCh := testutils.NewChannel() + streamCloseCh := testutils.NewChannel() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + streamRequestCh.SendContext(ctx, req) + return nil + }, + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + streamResponseCh.SendContext(ctx, resp) + }, + OnStreamClosed: func(int64, *v3corepb.Node) { + streamCloseCh.SendContext(ctx, struct{}{}) + }, + }) + + // Create a listener resource on the management server. + const listenerName = "listener" + const routeConfigName = "route-config" + nodeID := uuid.New().String() + listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client with bootstrap pointing to the above server. + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a listener resource. + lw := newListenerWatcher() + ldsCancel := xdsresource.WatchListener(client, listenerName, lw) + defer ldsCancel() + + // Verify that the initial discovery request matches expectation. + r, err := streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for the initial discovery request") + } + gotReq := r.(*v3discoverypb.DiscoveryRequest) + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + }, + ResourceNames: []string{listenerName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Capture the version and nonce from the response. + r, err = streamResponseCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for the discovery response from client") + } + gotResp := r.(*v3discoverypb.DiscoveryResponse) + + // Verify that the ACK contains the appropriate version and nonce. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for ACK") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + wantACKReq := proto.Clone(wantReq).(*v3discoverypb.DiscoveryRequest) + wantACKReq.VersionInfo = gotResp.GetVersionInfo() + wantACKReq.ResponseNonce = gotResp.GetNonce() + if diff := cmp.Diff(gotReq, wantACKReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Verify the update received by the watcher. + wantUpdate := listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: routeConfigName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Cancel the watch on the listener resource. This should result in the + // existing connection to be management server getting closed. + ldsCancel() + if _, err := streamCloseCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when expecting existing connection to be closed: %v", err) + } + + // There is a race between two events when the last watch on an xdsChannel + // is canceled: + // - an empty discovery request being sent out + // - the ADS stream being closed + // To handle this race, we drain the request channel here so that if an + // empty discovery request was received, it is pulled out of the request + // channel and thereby guaranteeing a clean slate for the next watch + // registered below. + streamRequestCh.Drain() + + // Register a watch for the same listener resource. + lw = newListenerWatcher() + ldsCancel = xdsresource.WatchListener(client, listenerName, lw) + defer ldsCancel() + + // Verify that the discovery request is identical to the first one sent out + // to the management server. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for discovery request") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Verify the update received by the watcher. + if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} diff --git a/internal/xds/xdsclient/tests/ads_stream_restart_test.go b/internal/xds/xdsclient/tests/ads_stream_restart_test.go new file mode 100644 index 0000000..6a4cfa4 --- /dev/null +++ b/internal/xds/xdsclient/tests/ads_stream_restart_test.go @@ -0,0 +1,196 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/protobuf/testing/protocmp" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +// Tests that an ADS stream is restarted after a connection failure. Also +// verifies that if there were any watches registered before the connection +// failed, those resources are re-requested after the stream is restarted. +func (s) TestADS_ResourcesAreRequestedAfterStreamRestart(t *testing.T) { + // Create a restartable listener that can simulate a broken ADS stream. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server that uses a couple of channels to inform + // the test about the request and response messages being exchanged. + streamRequestCh := testutils.NewChannel() + streamResponseCh := testutils.NewChannel() + streamOpened := testutils.NewChannel() + streamClosed := testutils.NewChannel() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + t.Logf("Received request for resources: %v of type %s", req.GetResourceNames(), req.GetTypeUrl()) + streamRequestCh.SendContext(ctx, req) + return nil + }, + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + streamResponseCh.SendContext(ctx, resp) + }, + OnStreamClosed: func(int64, *v3corepb.Node) { + streamClosed.SendContext(ctx, nil) + }, + OnStreamOpen: func(context.Context, int64, string) error { + streamOpened.SendContext(ctx, nil) + return nil + }, + }) + + // Create a listener resource on the management server. + const listenerName = "listener" + const routeConfigName = "route-config" + nodeID := uuid.New().String() + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create bootstrap configuration pointing to the above management server. + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a listener resource. + lw := newListenerWatcher() + ldsCancel := xdsresource.WatchListener(client, listenerName, lw) + defer ldsCancel() + + // Verify that an ADS stream is opened and an LDS request with the above + // resource name is sent. + if _, err = streamOpened.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for ADS stream to open") + } + + // Verify that the initial discovery request matches expectation. + r, err := streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for the initial discovery request") + } + gotReq := r.(*v3discoverypb.DiscoveryRequest) + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + }, + ResourceNames: []string{listenerName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Capture the version and nonce from the response. + r, err = streamResponseCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for a discovery response from the server") + } + gotResp := r.(*v3discoverypb.DiscoveryResponse) + + // Verify that the ACK contains the appropriate version and nonce. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for ACK") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + wantReq.VersionInfo = gotResp.GetVersionInfo() + wantReq.ResponseNonce = gotResp.GetNonce() + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Verify the update received by the watcher. + wantListenerUpdate := listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: routeConfigName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + if err := verifyListenerUpdate(ctx, lw.updateCh, wantListenerUpdate); err != nil { + t.Fatal(err) + } + + // Stop the restartable listener and wait for the stream to close. + lis.Stop() + if _, err = streamClosed.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for ADS stream to close") + } + + // Restart the restartable listener and wait for the stream to open. + lis.Restart() + if _, err = streamOpened.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for ADS stream to open") + } + + // Verify that the listener resource is requested again. + r, err = streamRequestCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for the initial discovery request") + } + gotReq = r.(*v3discoverypb.DiscoveryRequest) + wantReq.ResponseNonce = "" + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } +} diff --git a/internal/xds/xdsclient/tests/authority_test.go b/internal/xds/xdsclient/tests/authority_test.go new file mode 100644 index 0000000..2f821b6 --- /dev/null +++ b/internal/xds/xdsclient/tests/authority_test.go @@ -0,0 +1,375 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + xdstestutils "google.golang.org/grpc/internal/xds/testutils" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" +) + +const ( + testAuthority1 = "test-authority1" + testAuthority2 = "test-authority2" + testAuthority3 = "test-authority3" +) + +var ( + // These two resources use `testAuthority1`, which contains an empty server + // config in the bootstrap file, and therefore will use the default + // management server. + authorityTestResourceName11 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority1, cdsName+"1", nil) + authorityTestResourceName12 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority1, cdsName+"2", nil) + // This resource uses `testAuthority2`, which contains an empty server + // config in the bootstrap file, and therefore will use the default + // management server. + authorityTestResourceName2 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority2, cdsName+"3", nil) + // This resource uses `testAuthority3`, which contains a non-empty server + // config in the bootstrap file, and therefore will use the non-default + // management server. + authorityTestResourceName3 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority3, cdsName+"3", nil) +) + +// setupForAuthorityTests spins up two management servers, one to act as the +// default and the other to act as the non-default. It also generates a +// bootstrap configuration with three authorities (the first two pointing to the +// default and the third one pointing to the non-default). +// +// Returns two listeners used by the default and non-default management servers +// respectively, and the xDS client and its close function. +func setupForAuthorityTests(ctx context.Context, t *testing.T) (*testutils.ListenerWrapper, *testutils.ListenerWrapper, xdsclient.XDSClient, func()) { + // Create listener wrappers which notify on to a channel whenever a new + // connection is accepted. We use this to track the number of transports + // used by the xDS client. + lisDefault := testutils.NewListenerWrapper(t, nil) + lisNonDefault := testutils.NewListenerWrapper(t, nil) + + // Start a management server to act as the default authority. + defaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisDefault}) + + // Start a management server to act as the non-default authority. + nonDefaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisNonDefault}) + + // Create a bootstrap configuration with two non-default authorities which + // have empty server configs, and therefore end up using the default server + // config, which points to the above management server. + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, defaultAuthorityServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + testAuthority1: []byte(`{}`), + testAuthority2: []byte(`{}`), + testAuthority3: []byte(fmt.Sprintf(`{ + "xds_servers": [{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]}`, nonDefaultAuthorityServer.Address)), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(authorityTestResourceName11, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(authorityTestResourceName12, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(authorityTestResourceName2, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(authorityTestResourceName3, edsName, e2e.SecurityLevelNone), + }, + SkipValidation: true, + } + if err := defaultAuthorityServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + return lisDefault, lisNonDefault, client, close +} + +// Tests the xdsChannel sharing logic among authorities. The test verifies the +// following scenarios: +// - A watch for a resource name with an authority matching an existing watch +// should not result in a new transport being created. +// - A watch for a resource name with different authority name but same +// authority config as an existing watch should not result in a new transport +// being created. +func (s) TestAuthority_XDSChannelSharing(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + lis, _, client, close := setupForAuthorityTests(ctx, t) + defer close() + + // Verify that no connection is established to the management server at this + // point. A transport is created only when a resource (which belongs to that + // authority) is requested. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } + + // Request the first resource. Verify that a new transport is created. + watcher := noopClusterWatcher{} + cdsCancel1 := xdsresource.WatchCluster(client, authorityTestResourceName11, watcher) + defer cdsCancel1() + if _, err := lis.NewConnCh.Receive(ctx); err != nil { + t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err) + } + + // Request the second resource. Verify that no new transport is created. + cdsCancel2 := xdsresource.WatchCluster(client, authorityTestResourceName12, watcher) + defer cdsCancel2() + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } + + // Request the third resource. Verify that no new transport is created. + cdsCancel3 := xdsresource.WatchCluster(client, authorityTestResourceName2, watcher) + defer cdsCancel3() + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } +} + +// Test the xdsChannel close logic. The test verifies that the xDS client +// closes an xdsChannel immediately after the last watch is canceled. +func (s) TestAuthority_XDSChannelClose(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + lis, _, client, close := setupForAuthorityTests(ctx, t) + defer close() + + // Request the first resource. Verify that a new transport is created. + watcher := noopClusterWatcher{} + cdsCancel1 := xdsresource.WatchCluster(client, authorityTestResourceName11, watcher) + val, err := lis.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err) + } + conn := val.(*testutils.ConnWrapper) + + // Request the second resource. Verify that no new transport is created. + cdsCancel2 := xdsresource.WatchCluster(client, authorityTestResourceName12, watcher) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } + + // Cancel both watches, and verify that the connection to the management + // server is closed. + cdsCancel1() + cdsCancel2() + if _, err := conn.CloseCh.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for connection to management server to be closed") + } +} + +// Tests the scenario where the primary management server is unavailable at +// startup and the xDS client falls back to the secondary. The test verifies +// that the resource watcher is not notifified of the connectivity failure until +// all servers have failed. +func (s) TestAuthority_Fallback(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create primary and secondary management servers with restartable + // listeners. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + primaryLis := testutils.NewRestartableListener(l) + primaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis}) + l, err = testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + secondaryLis := testutils.NewRestartableListener(l) + secondaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: secondaryLis}) + + // Create bootstrap configuration with the above primary and fallback + // management servers, and an xDS client with that configuration. + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(` + [ + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }, + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + } + ]`, primaryMgmtServer.Address, secondaryMgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + xdsC, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{Name: t.Name()}) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + const clusterName = "cluster" + const edsPrimaryName = "eds-primary" + const edsSecondaryName = "eds-secondary" + + // Create a Cluster resource on the primary. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(clusterName, edsPrimaryName, e2e.SecurityLevelNone), + }, + SkipValidation: true, + } + if err := primaryMgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err) + } + + // Create a Cluster resource on the secondary . + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(clusterName, edsSecondaryName, e2e.SecurityLevelNone), + }, + SkipValidation: true, + } + if err := secondaryMgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err) + } + + // Stop the primary. + primaryLis.Close() + + // Register a watch. + watcher := newClusterWatcherV2() + cdsCancel := xdsresource.WatchCluster(xdsC, clusterName, watcher) + defer cdsCancel() + + // Ensure that the connectivity error callback is not called. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if v, err := watcher.ambientErrCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Error callback on the watcher with error: %v", v.(error)) + } + + // Ensure that the resource update callback is invoked. + v, err := watcher.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("Error when waiting for a resource update callback: %v", err) + } + gotUpdate := v.(*xdsresource.ClusterUpdate) + wantUpdate := xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: edsSecondaryName, + } + cmpOpts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.ClusterUpdate{}, "Raw", "LBPolicy", "TelemetryLabels")} + if diff := cmp.Diff(wantUpdate, *gotUpdate, cmpOpts...); diff != "" { + t.Fatalf("Diff in the cluster resource update: (-want, got):\n%s", diff) + } + + // Stop the secondary. + secondaryLis.Close() + + // Ensure that the connectivity error callback is called. + if _, err := watcher.ambientErrCh.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for error callback on the watcher") + } +} + +// TODO: Get rid of the clusterWatcher type in cds_watchers_test.go and use this +// one instead. Also, rename this to clusterWatcher as part of that refactor. +type clusterWatcherV2 struct { + updateCh *testutils.Channel // Messages of type xdsresource.ClusterUpdate + ambientErrCh *testutils.Channel // Messages of type ambient error + resourceErrCh *testutils.Channel // Messages of type resource error +} + +func newClusterWatcherV2() *clusterWatcherV2 { + return &clusterWatcherV2{ + updateCh: testutils.NewChannel(), + ambientErrCh: testutils.NewChannel(), + resourceErrCh: testutils.NewChannel(), + } +} + +func (cw *clusterWatcherV2) ResourceChanged(update *xdsresource.ClusterUpdate, onDone func()) { + cw.updateCh.Send(update) + onDone() +} + +func (cw *clusterWatcherV2) AmbientError(err error, onDone func()) { + // When used with a go-control-plane management server that continuously + // resends resources which are NACKed by the xDS client, using a `Replace()` + // here simplifies tests that want access to the most recently received + // error. + cw.ambientErrCh.Replace(err) + onDone() +} + +func (cw *clusterWatcherV2) ResourceError(err error, onDone func()) { + // When used with a go-control-plane management server that continuously + // resends resources which are NACKed by the xDS client, using a `Replace()` + // here simplifies tests that want access to the most recently received + // error. + cw.resourceErrCh.Replace(err) + onDone() +} diff --git a/internal/xds/xdsclient/tests/cds_watchers_test.go b/internal/xds/xdsclient/tests/cds_watchers_test.go new file mode 100644 index 0000000..e8f60bf --- /dev/null +++ b/internal/xds/xdsclient/tests/cds_watchers_test.go @@ -0,0 +1,1141 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +type noopClusterWatcher struct{} + +func (noopClusterWatcher) ResourceChanged(_ *xdsresource.ClusterUpdate, onDone func()) { + onDone() +} +func (noopClusterWatcher) ResourceError(_ error, onDone func()) { + onDone() +} +func (noopClusterWatcher) AmbientError(_ error, onDone func()) { + onDone() +} + +type clusterUpdateErrTuple struct { + update xdsresource.ClusterUpdate + err error +} + +type clusterWatcher struct { + updateCh *testutils.Channel +} + +func newClusterWatcher() *clusterWatcher { + return &clusterWatcher{updateCh: testutils.NewChannel()} +} + +func (cw *clusterWatcher) ResourceChanged(update *xdsresource.ClusterUpdate, onDone func()) { + cw.updateCh.Send(clusterUpdateErrTuple{update: *update}) + onDone() +} + +func (cw *clusterWatcher) ResourceError(err error, onDone func()) { + // When used with a go-control-plane management server that continuously + // resends resources which are NACKed by the xDS client, using a `Replace()` + // here and in AmbientError() simplifies tests which will have + // access to the most recently received error. + cw.updateCh.Replace(clusterUpdateErrTuple{err: err}) + onDone() +} + +func (cw *clusterWatcher) AmbientError(err error, onDone func()) { + cw.updateCh.Replace(clusterUpdateErrTuple{err: err}) + onDone() +} + +// badClusterResource returns a cluster resource for the given name which +// contains a config_source_specifier for the `lrs_server` field which is not +// set to `self`, and hence is expected to be NACKed by the client. +func badClusterResource(clusterName, edsServiceName string, secLevel e2e.SecurityLevel) *v3clusterpb.Cluster { + cluster := e2e.DefaultCluster(clusterName, edsServiceName, secLevel) + cluster.LrsServer = &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{}} + return cluster +} + +// xdsClient is expected to produce an error containing this string when an +// update is received containing a cluster created using `badClusterResource`. +const wantClusterNACKErr = "unsupported config_source_specifier" + +// verifyClusterUpdate waits for an update to be received on the provided update +// channel and verifies that it matches the expected update. +// +// Returns an error if no update is received before the context deadline expires +// or the received update does not match the expected one. +func verifyClusterUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate clusterUpdateErrTuple) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for a cluster resource from the management server: %v", err) + } + got := u.(clusterUpdateErrTuple) + if wantUpdate.err != nil { + if got.err == nil || !strings.Contains(got.err.Error(), wantUpdate.err.Error()) { + return fmt.Errorf("update received with error: %v, want %q", got.err, wantUpdate.err) + } + } + cmpOpts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.ClusterUpdate{}, "Raw", "LBPolicy", "TelemetryLabels")} + if diff := cmp.Diff(wantUpdate.update, got.update, cmpOpts...); diff != "" { + return fmt.Errorf("received unexpected diff in the cluster resource update: (-want, got):\n%s", diff) + } + return nil +} + +// verifyNoClusterUpdate verifies that no cluster update is received on the +// provided update channel, and returns an error if an update is received. +// +// A very short deadline is used while waiting for the update, as this function +// is intended to be used when an update is not expected. +func verifyNoClusterUpdate(ctx context.Context, updateCh *testutils.Channel) error { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if u, err := updateCh.Receive(sCtx); err != context.DeadlineExceeded { + return fmt.Errorf("received unexpected ClusterUpdate when expecting none: %s", pretty.ToJSON(u)) + } + return nil +} + +// TestCDSWatch covers the case where a single watcher exists for a single +// cluster resource. The test verifies the following scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of the watch callback. +// 2. An update from the management server containing a resource *not* being +// watched should not result in the invocation of the watch callback. +// 3. After the watch is cancelled, an update from the management server +// containing the resource that was being watched should not result in the +// invocation of the watch callback. +// +// The test is run for old and new style names. +func (s) TestCDSWatch(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3clusterpb.Cluster // The resource being watched. + updatedWatchedResource *v3clusterpb.Cluster // The watched resource after an update. + notWatchedResource *v3clusterpb.Cluster // A resource which is not being watched. + wantUpdate clusterUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: cdsName, + watchedResource: e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone), + updatedWatchedResource: e2e.DefaultCluster(cdsName, "new-eds-resource", e2e.SecurityLevelNone), + notWatchedResource: e2e.DefaultCluster("unsubscribed-cds-resource", edsName, e2e.SecurityLevelNone), + wantUpdate: clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: cdsName, + EDSServiceName: edsName, + }, + }, + }, + { + desc: "new style resource", + resourceName: cdsNameNewStyle, + watchedResource: e2e.DefaultCluster(cdsNameNewStyle, edsNameNewStyle, e2e.SecurityLevelNone), + updatedWatchedResource: e2e.DefaultCluster(cdsNameNewStyle, "new-eds-resource", e2e.SecurityLevelNone), + notWatchedResource: e2e.DefaultCluster("unsubscribed-cds-resource", edsNameNewStyle, e2e.SecurityLevelNone), + wantUpdate: clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: cdsNameNewStyle, + EDSServiceName: edsNameNewStyle, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a cluster resource and have the watch + // callback push the received update on to a channel. + cw := newClusterWatcher() + cdsCancel := xdsresource.WatchCluster(client, test.resourceName, cw) + + // Configure the management server to return a single cluster + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyClusterUpdate(ctx, cw.updateCh, test.wantUpdate); err != nil { + t.Fatal(err) + } + + // Configure the management server to return an additional cluster + // resource, one that we are not interested in. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{test.watchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoClusterUpdate(ctx, cw.updateCh); err != nil { + t.Fatal(err) + } + + // Cancel the watch and update the resource corresponding to the original + // watch. Ensure that the cancelled watch callback is not invoked. + cdsCancel() + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{test.updatedWatchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoClusterUpdate(ctx, cw.updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestCDSWatch_TwoWatchesForSameResourceName covers the case where two watchers +// exist for a single cluster resource. The test verifies the following +// scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of both watch callbacks. +// 2. After one of the watches is cancelled, a redundant update from the +// management server should not result in the invocation of either of the +// watch callbacks. +// 3. A new update from the management server containing the resource being +// watched should result in the invocation of the un-cancelled watch +// callback. +// +// The test is run for old and new style names. +func (s) TestCDSWatch_TwoWatchesForSameResourceName(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3clusterpb.Cluster // The resource being watched. + updatedWatchedResource *v3clusterpb.Cluster // The watched resource after an update. + wantUpdateV1 clusterUpdateErrTuple + wantUpdateV2 clusterUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: cdsName, + watchedResource: e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone), + updatedWatchedResource: e2e.DefaultCluster(cdsName, "new-eds-resource", e2e.SecurityLevelNone), + wantUpdateV1: clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: cdsName, + EDSServiceName: edsName, + }, + }, + wantUpdateV2: clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: cdsName, + EDSServiceName: "new-eds-resource", + }, + }, + }, + { + desc: "new style resource", + resourceName: cdsNameNewStyle, + watchedResource: e2e.DefaultCluster(cdsNameNewStyle, edsNameNewStyle, e2e.SecurityLevelNone), + updatedWatchedResource: e2e.DefaultCluster(cdsNameNewStyle, "new-eds-resource", e2e.SecurityLevelNone), + wantUpdateV1: clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: cdsNameNewStyle, + EDSServiceName: edsNameNewStyle, + }, + }, + wantUpdateV2: clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: cdsNameNewStyle, + EDSServiceName: "new-eds-resource", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same cluster resource and have the + // callbacks push the received updates on to a channel. + cw1 := newClusterWatcher() + cdsCancel1 := xdsresource.WatchCluster(client, test.resourceName, cw1) + defer cdsCancel1() + cw2 := newClusterWatcher() + cdsCancel2 := xdsresource.WatchCluster(client, test.resourceName, cw2) + + // Configure the management server to return a single cluster + // resource, corresponding to the one we registered watches for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyClusterUpdate(ctx, cw1.updateCh, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + if err := verifyClusterUpdate(ctx, cw2.updateCh, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + + // Cancel the second watch and force the management server to push a + // redundant update for the resource being watched. Neither of the + // two watch callbacks should be invoked. + cdsCancel2() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoClusterUpdate(ctx, cw1.updateCh); err != nil { + t.Fatal(err) + } + if err := verifyNoClusterUpdate(ctx, cw2.updateCh); err != nil { + t.Fatal(err) + } + + // Update to the resource being watched. The un-cancelled callback + // should be invoked while the cancelled one should not be. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{test.updatedWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyClusterUpdate(ctx, cw1.updateCh, test.wantUpdateV2); err != nil { + t.Fatal(err) + } + if err := verifyNoClusterUpdate(ctx, cw2.updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestCDSWatch_ThreeWatchesForDifferentResourceNames covers the case where +// three watchers (two watchers for one resource, and the third watcher for +// another resource) exist across two cluster resources (one with an old style +// name and one with a new style name). The test verifies that an update from +// the management server containing both resources results in the invocation of +// all watch callbacks. +func (s) TestCDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same cluster resource and have the + // callbacks push the received updates on to a channel. + cw1 := newClusterWatcher() + cdsCancel1 := xdsresource.WatchCluster(client, cdsName, cw1) + defer cdsCancel1() + cw2 := newClusterWatcher() + cdsCancel2 := xdsresource.WatchCluster(client, cdsName, cw2) + defer cdsCancel2() + + // Register the third watch for a different cluster resource, and push the + // received updates onto a channel. + cdsNameNewStyle := makeNewStyleCDSName(authority) + cw3 := newClusterWatcher() + cdsCancel3 := xdsresource.WatchCluster(client, cdsNameNewStyle, cw3) + defer cdsCancel3() + + // Configure the management server to return two cluster resources, + // corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(cdsNameNewStyle, edsNameNewStyle, e2e.SecurityLevelNone), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the all watchers. + wantUpdate12 := clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: cdsName, + EDSServiceName: edsName, + }, + } + wantUpdate3 := clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: cdsNameNewStyle, + EDSServiceName: edsNameNewStyle, + }, + } + if err := verifyClusterUpdate(ctx, cw1.updateCh, wantUpdate12); err != nil { + t.Fatal(err) + } + if err := verifyClusterUpdate(ctx, cw2.updateCh, wantUpdate12); err != nil { + t.Fatal(err) + } + if err := verifyClusterUpdate(ctx, cw3.updateCh, wantUpdate3); err != nil { + t.Fatal(err) + } +} + +// TestCDSWatch_ResourceCaching covers the case where a watch is registered for +// a resource which is already present in the cache. The test verifies that the +// watch callback is invoked with the contents from the cache, instead of a +// request being sent to the management server. +func (s) TestCDSWatch_ResourceCaching(t *testing.T) { + firstRequestReceived := false + firstAckReceived := grpcsync.NewEvent() + secondRequestReceived := grpcsync.NewEvent() + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + // The first request has an empty version string. + if !firstRequestReceived && req.GetVersionInfo() == "" { + firstRequestReceived = true + return nil + } + // The first ack has a non-empty version string. + if !firstAckReceived.HasFired() && req.GetVersionInfo() != "" { + firstAckReceived.Fire() + return nil + } + // Any requests after the first request and ack, are not expected. + secondRequestReceived.Fire() + return nil + }, + }) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a cluster resource and have the watch + // callback push the received update on to a channel. + cw1 := newClusterWatcher() + cdsCancel1 := xdsresource.WatchCluster(client, cdsName, cw1) + defer cdsCancel1() + + // Configure the management server to return a single cluster + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: cdsName, + EDSServiceName: edsName, + }, + } + if err := verifyClusterUpdate(ctx, cw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + t.Fatal("timeout when waiting for receipt of ACK at the management server") + case <-firstAckReceived.Done(): + } + + // Register another watch for the same resource. This should get the update + // from the cache. + cw2 := newClusterWatcher() + cdsCancel2 := xdsresource.WatchCluster(client, cdsName, cw2) + defer cdsCancel2() + if err := verifyClusterUpdate(ctx, cw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + // No request should get sent out as part of this watch. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-secondRequestReceived.Done(): + t.Fatal("xdsClient sent out request instead of using update from cache") + } +} + +// TestCDSWatch_ExpiryTimerFiresBeforeResponse tests the case where the client +// does not receive an CDS response for the request that it sends. The test +// verifies that the watch callback is invoked with an error once the +// watchExpiryTimer fires. +func (s) TestCDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Register a watch for a resource which is expected to be invoked with an + // error after the watch expiry timer fires. + cw := newClusterWatcher() + cdsCancel := xdsresource.WatchCluster(client, cdsName, cw) + defer cdsCancel() + + // Wait for the watch expiry timer to fire. + <-time.After(defaultTestWatchExpiryTimeout) + + // Verify that an empty update with the expected error is received. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantErr := xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, "") + if err := verifyClusterUpdate(ctx, cw.updateCh, clusterUpdateErrTuple{err: wantErr}); err != nil { + t.Fatal(err) + } +} + +// TestCDSWatch_ValidResponseCancelsExpiryTimerBehavior tests the case where the +// client receives a valid LDS response for the request that it sends. The test +// verifies that the behavior associated with the expiry timer (i.e, callback +// invocation with error) does not take place. +func (s) TestCDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Register a watch for a cluster resource and have the watch + // callback push the received update on to a channel. + cw := newClusterWatcher() + cdsCancel := xdsresource.WatchCluster(client, cdsName, cw) + defer cdsCancel() + + // Configure the management server to return a single cluster resource, + // corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: cdsName, + EDSServiceName: edsName, + }, + } + if err := verifyClusterUpdate(ctx, cw.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Wait for the watch expiry timer to fire, and verify that the callback is + // not invoked. + <-time.After(defaultTestWatchExpiryTimeout) + if err := verifyNoClusterUpdate(ctx, cw.updateCh); err != nil { + t.Fatal(err) + } +} + +// TestCDSWatch_ResourceRemoved covers the cases where two watchers exists for +// two different resources (one with an old style name and one with a new style +// name). One of these resources being watched is removed from the management +// server. The test verifies the following scenarios: +// 1. Removing a resource should trigger the watch callback associated with that +// resource with a resource removed error. It should not trigger the watch +// callback for an unrelated resource. +// 2. An update to other resource should result in the invocation of the watch +// callback associated with that resource. It should not result in the +// invocation of the watch callback associated with the deleted resource. +func (s) TestCDSWatch_ResourceRemoved(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for two cluster resources and have the + // callbacks push the received updates on to a channel. + resourceName1 := cdsName + cw1 := newClusterWatcher() + cdsCancel1 := xdsresource.WatchCluster(client, resourceName1, cw1) + defer cdsCancel1() + + resourceName2 := makeNewStyleCDSName(authority) + cw2 := newClusterWatcher() + cdsCancel2 := xdsresource.WatchCluster(client, resourceName2, cw2) + defer cdsCancel2() + + // Configure the management server to return two cluster resources, + // corresponding to the registered watches. + edsNameNewStyle := makeNewStyleEDSName(authority) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(resourceName1, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(resourceName2, edsNameNewStyle, e2e.SecurityLevelNone), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for both watchers. + wantUpdate1 := clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: resourceName1, + EDSServiceName: edsName, + }, + } + wantUpdate2 := clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: resourceName2, + EDSServiceName: edsNameNewStyle, + }, + } + if err := verifyClusterUpdate(ctx, cw1.updateCh, wantUpdate1); err != nil { + t.Fatal(err) + } + if err := verifyClusterUpdate(ctx, cw2.updateCh, wantUpdate2); err != nil { + t.Fatal(err) + } + + // Remove the first cluster resource on the management server. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(resourceName2, edsNameNewStyle, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // The first watcher should receive a resource removed error, while the + // second watcher should not receive an update. + if err := verifyClusterUpdate(ctx, cw1.updateCh, clusterUpdateErrTuple{err: xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, "")}); err != nil { + t.Fatal(err) + } + if err := verifyNoClusterUpdate(ctx, cw2.updateCh); err != nil { + t.Fatal(err) + } + + // Update the second cluster resource on the management server. The first + // watcher should not receive an update, while the second watcher should. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(resourceName2, "new-eds-resource", e2e.SecurityLevelNone)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoClusterUpdate(ctx, cw1.updateCh); err != nil { + t.Fatal(err) + } + wantUpdate := clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: resourceName2, + EDSServiceName: "new-eds-resource", + }, + } + if err := verifyClusterUpdate(ctx, cw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestCDSWatch_NACKError covers the case where an update from the management +// server is NACK'ed by the xdsclient. The test verifies that the error is +// propagated to the watcher. +func (s) TestCDSWatch_NACKError(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a cluster resource and have the watch + // callback push the received update on to a channel. + cw := newClusterWatcher() + cdsCancel := xdsresource.WatchCluster(client, cdsName, cw) + defer cdsCancel() + + // Configure the management server to return a single cluster resource + // which is expected to be NACK'ed by the client. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{badClusterResource(cdsName, edsName, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher. + u, err := cw.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for a cluster resource from the management server: %v", err) + } + gotErr := u.(clusterUpdateErrTuple).err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantClusterNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantClusterNACKErr) + } +} + +// TestCDSWatch_PartialValid covers the case where a response from the +// management server contains both valid and invalid resources and is expected +// to be NACK'ed by the xdsclient. The test verifies that watchers corresponding +// to the valid resource receive the update, while watchers corresponding to the +// invalid resource receive an error. +func (s) TestCDSWatch_PartialValid(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for cluster resources. The first watch is expected + // to receive an error because the received resource is NACK'ed. The second + // watch is expected to get a good update. + badResourceName := cdsName + cw1 := newClusterWatcher() + cdsCancel1 := xdsresource.WatchCluster(client, badResourceName, cw1) + defer cdsCancel1() + goodResourceName := makeNewStyleCDSName(authority) + cw2 := newClusterWatcher() + cdsCancel2 := xdsresource.WatchCluster(client, goodResourceName, cw2) + defer cdsCancel2() + + // Configure the management server with two cluster resources. One of these + // is a bad resource causing the update to be NACKed. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + badClusterResource(badResourceName, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(goodResourceName, edsName, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher which is + // watching the bad resource. + u, err := cw1.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for a cluster resource from the management server: %v", err) + } + gotErr := u.(clusterUpdateErrTuple).err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantClusterNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantClusterNACKErr) + } + + // Verify that the watcher watching the good resource receives a good + // update. + wantUpdate := clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: goodResourceName, + EDSServiceName: edsName, + }, + } + if err := verifyClusterUpdate(ctx, cw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestCDSWatch_PartialResponse covers the case where a response from the +// management server does not contain all requested resources. CDS responses are +// supposed to contain all requested resources, and the absence of one usually +// indicates that the management server does not know about it. In cases where +// the server has never responded with this resource before, the xDS client is +// expected to wait for the watch timeout to expire before concluding that the +// resource does not exist on the server +func (s) TestCDSWatch_PartialResponse(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for two cluster resources and have the + // callbacks push the received updates on to a channel. + resourceName1 := cdsName + cw1 := newClusterWatcher() + cdsCancel1 := xdsresource.WatchCluster(client, resourceName1, cw1) + defer cdsCancel1() + resourceName2 := makeNewStyleCDSName(authority) + cw2 := newClusterWatcher() + cdsCancel2 := xdsresource.WatchCluster(client, resourceName2, cw2) + defer cdsCancel2() + + // Configure the management server to return only one of the two cluster + // resources, corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(resourceName1, edsName, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for first watcher. + wantUpdate1 := clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: resourceName1, + EDSServiceName: edsName, + }, + } + if err := verifyClusterUpdate(ctx, cw1.updateCh, wantUpdate1); err != nil { + t.Fatal(err) + } + + // Verify that the second watcher does not get an update with an error. + if err := verifyNoClusterUpdate(ctx, cw2.updateCh); err != nil { + t.Fatal(err) + } + + // Configure the management server to return two cluster resources, + // corresponding to the registered watches. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(resourceName1, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(resourceName2, edsNameNewStyle, e2e.SecurityLevelNone), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the second watcher. + wantUpdate2 := clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: resourceName2, + EDSServiceName: edsNameNewStyle, + }, + } + if err := verifyClusterUpdate(ctx, cw2.updateCh, wantUpdate2); err != nil { + t.Fatal(err) + } + + // Verify that the first watcher gets no update, as the first resource did + // not change. + if err := verifyNoClusterUpdate(ctx, cw1.updateCh); err != nil { + t.Fatal(err) + } +} diff --git a/internal/xds/xdsclient/tests/client_custom_dialopts_test.go b/internal/xds/xdsclient/tests/client_custom_dialopts_test.go new file mode 100644 index 0000000..fd70f85 --- /dev/null +++ b/internal/xds/xdsclient/tests/client_custom_dialopts_test.go @@ -0,0 +1,165 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + internalbootstrap "google.golang.org/grpc/internal/xds/bootstrap" + xci "google.golang.org/grpc/internal/xds/xdsclient/internal" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/xds/bootstrap" +) + +// nopDialOption is a no-op grpc.DialOption with a name. +type nopDialOption struct { + grpc.EmptyDialOption + name string +} + +// testCredsBundle implements `credentials.Bundle` and `extraDialOptions`. +type testCredsBundle struct { + credentials.Bundle + testDialOptNames []string +} + +func (t *testCredsBundle) DialOptions() []grpc.DialOption { + var opts []grpc.DialOption + for _, name := range t.testDialOptNames { + opts = append(opts, &nopDialOption{name: name}) + } + return opts +} + +type testCredsBuilder struct { + testDialOptNames []string +} + +func (t *testCredsBuilder) Build(json.RawMessage) (credentials.Bundle, func(), error) { + return &testCredsBundle{ + Bundle: insecure.NewBundle(), + testDialOptNames: t.testDialOptNames, + }, func() {}, nil +} + +func (t *testCredsBuilder) Name() string { + return "test_dialer_creds" +} + +func (s) TestClientCustomDialOptsFromCredentialsBundle(t *testing.T) { + // Create and register the credentials bundle builder. + credsBuilder := &testCredsBuilder{ + testDialOptNames: []string{"opt1", "opt2", "opt3"}, + } + bootstrap.RegisterChannelCredentials(credsBuilder) + + // Start an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc, err := internalbootstrap.NewContentsForTesting(internalbootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{ + "type": %q, + "config": {"mgmt_server_address": %q} + }] + }]`, mgmtServer.Address, credsBuilder.Name(), mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS resolver with the above bootstrap configuration. + var resolverBuilder resolver.Builder + if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { + resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } + + // Spin up a test backend. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure client side xDS resources on the management server. + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Intercept a grpc.NewClient call from the xds client to validate DialOptions. + xci.GRPCNewClient = func(target string, opts ...grpc.DialOption) (conn *grpc.ClientConn, err error) { + got := map[string]int{} + for _, opt := range opts { + if mo, ok := opt.(*nopDialOption); ok { + got[mo.name]++ + } + } + want := map[string]int{} + for _, name := range credsBuilder.testDialOptNames { + want[name]++ + } + if !cmp.Equal(got, want) { + t.Errorf("grpc.NewClient() was called with unexpected DialOptions: got %v, want %v", got, want) + } + return grpc.NewClient(target, opts...) + } + defer func() { xci.GRPCNewClient = grpc.NewClient }() + + // Create a ClientConn and make a successful RPC. The insecure transport + // credentials passed into the gRPC.NewClient is the credentials for the + // data plane communication with the test backend. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + cc.Close() +} diff --git a/internal/xds/xdsclient/tests/dump_test.go b/internal/xds/xdsclient/tests/dump_test.go new file mode 100644 index 0000000..11a43c1 --- /dev/null +++ b/internal/xds/xdsclient/tests/dump_test.go @@ -0,0 +1,575 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + + v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + "github.com/envoyproxy/go-control-plane/pkg/wellknown" +) + +func makeGenericXdsConfig(typeURL, name, version string, status v3adminpb.ClientResourceStatus, config *anypb.Any, failure *v3adminpb.UpdateFailureState) *v3statuspb.ClientConfig_GenericXdsConfig { + return &v3statuspb.ClientConfig_GenericXdsConfig{ + TypeUrl: typeURL, + Name: name, + VersionInfo: version, + ClientStatus: status, + XdsConfig: config, + ErrorState: failure, + } +} + +func checkResourceDump(ctx context.Context, want *v3statuspb.ClientStatusResponse, pool *xdsclient.Pool) error { + cmpOpts := cmp.Options{ + protocmp.Transform(), + protocmp.IgnoreFields((*v3statuspb.ClientConfig_GenericXdsConfig)(nil), "last_updated"), + protocmp.IgnoreFields((*v3adminpb.UpdateFailureState)(nil), "last_update_attempt", "details"), + } + + var lastErr error + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + got := pool.DumpResources() + // Sort the client configs based on the `client_scope` field. + slices.SortFunc(got.GetConfig(), func(a, b *v3statuspb.ClientConfig) int { + return strings.Compare(a.ClientScope, b.ClientScope) + }) + // Sort the resource configs based on the type_url and name fields. + for _, cc := range got.GetConfig() { + slices.SortFunc(cc.GetGenericXdsConfigs(), func(a, b *v3statuspb.ClientConfig_GenericXdsConfig) int { + if strings.Compare(a.TypeUrl, b.TypeUrl) == 0 { + return strings.Compare(a.Name, b.Name) + } + return strings.Compare(a.TypeUrl, b.TypeUrl) + }) + } + diff := cmp.Diff(want, got, cmpOpts) + if diff == "" { + return nil + } + lastErr = fmt.Errorf("received unexpected resource dump, diff (-want, +got):\n%s, got: %s\n want:%s", diff, pretty.ToJSON(got), pretty.ToJSON(want)) + } + return fmt.Errorf("timeout when waiting for resource dump to reach expected state: %v", lastErr) +} + +// Tests the scenario where there are multiple xDS clients talking to the same +// management server, and requesting the same set of resources. Verifies that +// under all circumstances, both xDS clients receive the same configuration from +// the server. +func (s) TestDumpResources_ManyToOne(t *testing.T) { + // Initialize the xDS resources to be used in this test. + ldsTargets := []string{"lds.target.good:0000", "lds.target.good:1111"} + rdsTargets := []string{"route-config-0", "route-config-1"} + cdsTargets := []string{"cluster-0", "cluster-1"} + edsTargets := []string{"endpoints-0", "endpoints-1"} + listeners := make([]*v3listenerpb.Listener, len(ldsTargets)) + listenerAnys := make([]*anypb.Any, len(ldsTargets)) + for i := range ldsTargets { + listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i]) + listenerAnys[i] = testutils.MarshalAny(t, listeners[i]) + } + routes := make([]*v3routepb.RouteConfiguration, len(rdsTargets)) + routeAnys := make([]*anypb.Any, len(rdsTargets)) + for i := range rdsTargets { + routes[i] = e2e.DefaultRouteConfig(rdsTargets[i], ldsTargets[i], cdsTargets[i]) + routeAnys[i] = testutils.MarshalAny(t, routes[i]) + } + clusters := make([]*v3clusterpb.Cluster, len(cdsTargets)) + clusterAnys := make([]*anypb.Any, len(cdsTargets)) + for i := range cdsTargets { + clusters[i] = e2e.DefaultCluster(cdsTargets[i], edsTargets[i], e2e.SecurityLevelNone) + clusterAnys[i] = testutils.MarshalAny(t, clusters[i]) + } + endpoints := make([]*v3endpointpb.ClusterLoadAssignment, len(edsTargets)) + endpointAnys := make([]*anypb.Any, len(edsTargets)) + ips := []string{"0.0.0.0", "1.1.1.1"} + ports := []uint32{123, 456} + for i := range edsTargets { + endpoints[i] = e2e.DefaultEndpoint(edsTargets[i], ips[i], ports[i:i+1]) + endpointAnys[i] = testutils.MarshalAny(t, endpoints[i]) + } + + // Spin up an xDS management server on a local port. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + // Create two xDS clients with the above bootstrap contents. + client1Name := t.Name() + "-1" + client1, close1, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: client1Name, + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close1() + client2Name := t.Name() + "-2" + client2, close2, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: client2Name, + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close2() + + // Dump resources and expect empty configs. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantNode := &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + } + wantResp := &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + ClientScope: client1Name, + }, + { + Node: wantNode, + ClientScope: client2Name, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, pool); err != nil { + t.Fatal(err) + } + + // Register watches, dump resources and expect configs in requested state. + for _, xdsC := range []xdsclient.XDSClient{client1, client2} { + for _, target := range ldsTargets { + xdsresource.WatchListener(xdsC, target, noopListenerWatcher{}) + } + for _, target := range rdsTargets { + xdsresource.WatchRouteConfig(xdsC, target, noopRouteConfigWatcher{}) + } + for _, target := range cdsTargets { + xdsresource.WatchCluster(xdsC, target, noopClusterWatcher{}) + } + for _, target := range edsTargets { + xdsresource.WatchEndpoints(xdsC, target, noopEndpointsWatcher{}) + } + } + wantConfigs := []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + ClientScope: client1Name, + }, + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + ClientScope: client2Name, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, pool); err != nil { + t.Fatal(err) + } + + // Configure the resources on the management server. + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + }); err != nil { + t.Fatal(err) + } + + // Dump resources and expect ACK configs. + wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[0], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[1], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[0], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[1], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[0], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[0], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[1], nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + ClientScope: client1Name, + }, + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + ClientScope: client2Name, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, pool); err != nil { + t.Fatal(err) + } + + // Update the first resource of each type in the management server to a + // value which is expected to be NACK'ed by the xDS client. + listeners[0] = func() *v3listenerpb.Listener { + hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, + }) + return &v3listenerpb.Listener{ + Name: ldsTargets[0], + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + } + }() + routes[0].VirtualHosts = []*v3routepb.VirtualHost{{Routes: []*v3routepb.Route{{}}}} + clusters[0].ClusterDiscoveryType = &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC} + endpoints[0].Endpoints = []*v3endpointpb.LocalityLbEndpoints{{}} + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + SkipValidation: true, + }); err != nil { + t.Fatal(err) + } + + wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, clusterAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}), + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[1], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, endpointAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[1], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, listenerAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, routeAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, routeAnys[1], nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + ClientScope: client1Name, + }, + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + ClientScope: client2Name, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, pool); err != nil { + t.Fatal(err) + } +} + +// Tests the scenario where there are multiple xDS client talking to different +// management server, and requesting different set of resources. +func (s) TestDumpResources_ManyToMany(t *testing.T) { + // Initialize the xDS resources to be used in this test: + // - The first xDS client watches old style resource names, and thereby + // requests these resources from the top-level xDS server. + // - The second xDS client watches new style resource names with a non-empty + // authority, and thereby requests these resources from the server + // configuration for that authority. + authority := strings.Join(strings.Split(t.Name(), "/"), "") + ldsTargets := []string{ + "lds.target.good:0000", + fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/lds.targer.good:1111", authority), + } + rdsTargets := []string{ + "route-config-0", + fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/route-config-1", authority), + } + cdsTargets := []string{ + "cluster-0", + fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/cluster-1", authority), + } + edsTargets := []string{ + "endpoints-0", + fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/endpoints-1", authority), + } + listeners := make([]*v3listenerpb.Listener, len(ldsTargets)) + listenerAnys := make([]*anypb.Any, len(ldsTargets)) + for i := range ldsTargets { + listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i]) + listenerAnys[i] = testutils.MarshalAny(t, listeners[i]) + } + routes := make([]*v3routepb.RouteConfiguration, len(rdsTargets)) + routeAnys := make([]*anypb.Any, len(rdsTargets)) + for i := range rdsTargets { + routes[i] = e2e.DefaultRouteConfig(rdsTargets[i], ldsTargets[i], cdsTargets[i]) + routeAnys[i] = testutils.MarshalAny(t, routes[i]) + } + clusters := make([]*v3clusterpb.Cluster, len(cdsTargets)) + clusterAnys := make([]*anypb.Any, len(cdsTargets)) + for i := range cdsTargets { + clusters[i] = e2e.DefaultCluster(cdsTargets[i], edsTargets[i], e2e.SecurityLevelNone) + clusterAnys[i] = testutils.MarshalAny(t, clusters[i]) + } + endpoints := make([]*v3endpointpb.ClusterLoadAssignment, len(edsTargets)) + endpointAnys := make([]*anypb.Any, len(edsTargets)) + ips := []string{"0.0.0.0", "1.1.1.1"} + ports := []uint32{123, 456} + for i := range edsTargets { + endpoints[i] = e2e.DefaultEndpoint(edsTargets[i], ips[i], ports[i:i+1]) + endpointAnys[i] = testutils.MarshalAny(t, endpoints[i]) + } + + // Start two management servers. + mgmtServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + mgmtServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // The first of the above management servers will be the top-level xDS + // server in the bootstrap configuration, and the second will be the xDS + // server corresponding to the test authority. + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer1.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + authority: []byte(fmt.Sprintf(`{ + "xds_servers": [{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]}`, mgmtServer2.Address)), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + + // Create two xDS clients with the above bootstrap contents. + client1Name := t.Name() + "-1" + client1, close1, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: client1Name, + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close1() + client2Name := t.Name() + "-2" + client2, close2, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: client2Name, + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close2() + + // Check the resource dump before configuring resources on the management server. + // Dump resources and expect empty configs. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantNode := &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + } + wantResp := &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + ClientScope: client1Name, + }, + { + Node: wantNode, + ClientScope: client2Name, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, pool); err != nil { + t.Fatal(err) + } + + // Register watches, the first xDS client watches old style resource names, + // while the second xDS client watches new style resource names. + xdsresource.WatchListener(client1, ldsTargets[0], noopListenerWatcher{}) + xdsresource.WatchRouteConfig(client1, rdsTargets[0], noopRouteConfigWatcher{}) + xdsresource.WatchCluster(client1, cdsTargets[0], noopClusterWatcher{}) + xdsresource.WatchEndpoints(client1, edsTargets[0], noopEndpointsWatcher{}) + xdsresource.WatchListener(client2, ldsTargets[1], noopListenerWatcher{}) + xdsresource.WatchRouteConfig(client2, rdsTargets[1], noopRouteConfigWatcher{}) + xdsresource.WatchCluster(client2, cdsTargets[1], noopClusterWatcher{}) + xdsresource.WatchEndpoints(client2, edsTargets[1], noopEndpointsWatcher{}) + + // Check the resource dump. Both clients should have all resources in + // REQUESTED state. + wantConfigs1 := []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + } + wantConfigs2 := []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs1, + ClientScope: client1Name, + }, + { + Node: wantNode, + GenericXdsConfigs: wantConfigs2, + ClientScope: client2Name, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, pool); err != nil { + t.Fatal(err) + } + + // Configure resources on the first management server. + if err := mgmtServer1.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners[:1], + Routes: routes[:1], + Clusters: clusters[:1], + Endpoints: endpoints[:1], + }); err != nil { + t.Fatal(err) + } + + // Check the resource dump. One client should have resources in ACKED state, + // while the other should still have resources in REQUESTED state. + wantConfigs1 = []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[0], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[0], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[0], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[0], nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs1, + ClientScope: client1Name, + }, + { + Node: wantNode, + GenericXdsConfigs: wantConfigs2, + ClientScope: client2Name, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, pool); err != nil { + t.Fatal(err) + } + + // Configure resources on the second management server. + if err := mgmtServer2.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners[1:], + Routes: routes[1:], + Clusters: clusters[1:], + Endpoints: endpoints[1:], + }); err != nil { + t.Fatal(err) + } + + // Check the resource dump. Both clients should have appropriate resources + // in REQUESTED state. + wantConfigs2 = []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[1], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[1], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[1], nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs1, + ClientScope: client1Name, + }, + { + Node: wantNode, + GenericXdsConfigs: wantConfigs2, + ClientScope: client2Name, + }, + }, + } + if err := checkResourceDump(ctx, wantResp, pool); err != nil { + t.Fatal(err) + } +} diff --git a/internal/xds/xdsclient/tests/eds_watchers_test.go b/internal/xds/xdsclient/tests/eds_watchers_test.go new file mode 100644 index 0000000..1192c20 --- /dev/null +++ b/internal/xds/xdsclient/tests/eds_watchers_test.go @@ -0,0 +1,1045 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/resolver" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +const ( + edsHost1 = "1.foo.bar.com" + edsHost2 = "2.foo.bar.com" + edsHost3 = "3.foo.bar.com" + edsPort1 = 1 + edsPort2 = 2 + edsPort3 = 3 +) + +type noopEndpointsWatcher struct{} + +func (noopEndpointsWatcher) ResourceChanged(_ *xdsresource.EndpointsUpdate, onDone func()) { + onDone() +} +func (noopEndpointsWatcher) ResourceError(_ error, onDone func()) { + onDone() +} +func (noopEndpointsWatcher) AmbientError(_ error, onDone func()) { + onDone() +} + +type endpointsUpdateErrTuple struct { + update xdsresource.EndpointsUpdate + err error +} + +type endpointsWatcher struct { + updateCh *testutils.Channel +} + +func newEndpointsWatcher() *endpointsWatcher { + return &endpointsWatcher{updateCh: testutils.NewChannel()} +} + +func (ew *endpointsWatcher) ResourceChanged(update *xdsresource.EndpointsUpdate, onDone func()) { + ew.updateCh.Send(endpointsUpdateErrTuple{update: *update}) + onDone() +} + +func (ew *endpointsWatcher) ResourceError(err error, onDone func()) { + // When used with a go-control-plane management server that continuously + // resends resources which are NACKed by the xDS client, using a `Replace()` + // here and in AmbientError() simplifies tests which will have + // access to the most recently received error. + ew.updateCh.Replace(endpointsUpdateErrTuple{err: err}) + onDone() +} + +func (ew *endpointsWatcher) AmbientError(err error, onDone func()) { + ew.updateCh.Replace(endpointsUpdateErrTuple{err: err}) + onDone() +} + +// badEndpointsResource returns a endpoints resource for the given +// edsServiceName which contains an endpoint with a load_balancing weight of +// `0`. This is expected to be NACK'ed by the xDS client. +func badEndpointsResource(edsServiceName string, host string, ports []uint32) *v3endpointpb.ClusterLoadAssignment { + e := e2e.DefaultEndpoint(edsServiceName, host, ports) + e.Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} + return e +} + +// xdsClient is expected to produce an error containing this string when an +// update is received containing an endpoints resource created using +// `badEndpointsResource`. +const wantEndpointsNACKErr = "EDS response contains an endpoint with zero weight" + +// verifyEndpointsUpdate waits for an update to be received on the provided +// update channel and verifies that it matches the expected update. +// +// Returns an error if no update is received before the context deadline expires +// or the received update does not match the expected one. +func verifyEndpointsUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate endpointsUpdateErrTuple) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for a endpoints resource from the management server: %v", err) + } + got := u.(endpointsUpdateErrTuple) + if wantUpdate.err != nil { + if got.err == nil || !strings.Contains(got.err.Error(), wantUpdate.err.Error()) { + return fmt.Errorf("update received with error: %v, want %q", got.err, wantUpdate.err) + } + } + cmpOpts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.EndpointsUpdate{}, "Raw")} + if diff := cmp.Diff(wantUpdate.update, got.update, cmpOpts...); diff != "" { + return fmt.Errorf("received unexpected diff in the endpoints resource update: (-want, got):\n%s", diff) + } + return nil +} + +// verifyNoEndpointsUpdate verifies that no endpoints update is received on the +// provided update channel, and returns an error if an update is received. +// +// A very short deadline is used while waiting for the update, as this function +// is intended to be used when an update is not expected. +func verifyNoEndpointsUpdate(ctx context.Context, updateCh *testutils.Channel) error { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if u, err := updateCh.Receive(sCtx); err != context.DeadlineExceeded { + return fmt.Errorf("unexpected EndpointsUpdate: %v", u) + } + return nil +} + +// TestEDSWatch covers the case where a single endpoint exists for a single +// endpoints resource. The test verifies the following scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of the watch callback. +// 2. An update from the management server containing a resource *not* being +// watched should not result in the invocation of the watch callback. +// 3. After the watch is cancelled, an update from the management server +// containing the resource that was being watched should not result in the +// invocation of the watch callback. +// +// The test is run for old and new style names. +func (s) TestEDSWatch(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3endpointpb.ClusterLoadAssignment // The resource being watched. + updatedWatchedResource *v3endpointpb.ClusterLoadAssignment // The watched resource after an update. + notWatchedResource *v3endpointpb.ClusterLoadAssignment // A resource which is not being watched. + wantUpdate endpointsUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: edsName, + watchedResource: e2e.DefaultEndpoint(edsName, edsHost1, []uint32{edsPort1}), + updatedWatchedResource: e2e.DefaultEndpoint(edsName, edsHost2, []uint32{edsPort2}), + notWatchedResource: e2e.DefaultEndpoint("unsubscribed-eds-resource", edsHost3, []uint32{edsPort3}), + wantUpdate: endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: fmt.Sprintf("%s:%d", edsHost1, edsPort1)}}, + }, + Weight: 1, + }}, + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + }, + { + desc: "new style resource", + resourceName: edsNameNewStyle, + watchedResource: e2e.DefaultEndpoint(edsNameNewStyle, edsHost1, []uint32{edsPort1}), + updatedWatchedResource: e2e.DefaultEndpoint(edsNameNewStyle, edsHost2, []uint32{edsPort2}), + notWatchedResource: e2e.DefaultEndpoint("unsubscribed-eds-resource", edsHost3, []uint32{edsPort3}), + wantUpdate: endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: fmt.Sprintf("%s:%d", edsHost1, edsPort1)}}, + }, + Weight: 1, + }}, + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a endpoint resource and have the watch + // callback push the received update on to a channel. + ew := newEndpointsWatcher() + edsCancel := xdsresource.WatchEndpoints(client, test.resourceName, ew) + + // Configure the management server to return a single endpoint + // resource, corresponding to the one being watched. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyEndpointsUpdate(ctx, ew.updateCh, test.wantUpdate); err != nil { + t.Fatal(err) + } + + // Configure the management server to return an additional endpoint + // resource, one that we are not interested in. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{test.watchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoEndpointsUpdate(ctx, ew.updateCh); err != nil { + t.Fatal(err) + } + + // Cancel the watch and update the resource corresponding to the original + // watch. Ensure that the cancelled watch callback is not invoked. + edsCancel() + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{test.updatedWatchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoEndpointsUpdate(ctx, ew.updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestEDSWatch_TwoWatchesForSameResourceName covers the case where two watchers +// exist for a single endpoint resource. The test verifies the following +// scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of both watch callbacks. +// 2. After one of the watches is cancelled, a redundant update from the +// management server should not result in the invocation of either of the +// watch callbacks. +// 3. An update from the management server containing the resource being +// watched should result in the invocation of the un-cancelled watch +// callback. +// +// The test is run for old and new style names. +func (s) TestEDSWatch_TwoWatchesForSameResourceName(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3endpointpb.ClusterLoadAssignment // The resource being watched. + updatedWatchedResource *v3endpointpb.ClusterLoadAssignment // The watched resource after an update. + wantUpdateV1 endpointsUpdateErrTuple + wantUpdateV2 endpointsUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: edsName, + watchedResource: e2e.DefaultEndpoint(edsName, edsHost1, []uint32{edsPort1}), + updatedWatchedResource: e2e.DefaultEndpoint(edsName, edsHost2, []uint32{edsPort2}), + wantUpdateV1: endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: fmt.Sprintf("%s:%d", edsHost1, edsPort1)}}, + }, + Weight: 1, + }}, + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + wantUpdateV2: endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: fmt.Sprintf("%s:%d", edsHost2, edsPort2)}}, + }, + Weight: 1, + }}, + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + }, + { + desc: "new style resource", + resourceName: edsNameNewStyle, + watchedResource: e2e.DefaultEndpoint(edsNameNewStyle, edsHost1, []uint32{edsPort1}), + updatedWatchedResource: e2e.DefaultEndpoint(edsNameNewStyle, edsHost2, []uint32{edsPort2}), + wantUpdateV1: endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: fmt.Sprintf("%s:%d", edsHost1, edsPort1)}}, + }, + Weight: 1, + }}, + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + wantUpdateV2: endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: fmt.Sprintf("%s:%d", edsHost2, edsPort2)}}, + }, + Weight: 1, + }}, + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same endpoint resource and have the + // callbacks push the received updates on to a channel. + ew1 := newEndpointsWatcher() + edsCancel1 := xdsresource.WatchEndpoints(client, test.resourceName, ew1) + defer edsCancel1() + ew2 := newEndpointsWatcher() + edsCancel2 := xdsresource.WatchEndpoints(client, test.resourceName, ew2) + + // Configure the management server to return a single endpoint + // resource, corresponding to the one being watched. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyEndpointsUpdate(ctx, ew1.updateCh, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + if err := verifyEndpointsUpdate(ctx, ew2.updateCh, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + + // Cancel the second watch and force the management server to push a + // redundant update for the resource being watched. Neither of the + // two watch callbacks should be invoked. + edsCancel2() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoEndpointsUpdate(ctx, ew1.updateCh); err != nil { + t.Fatal(err) + } + if err := verifyNoEndpointsUpdate(ctx, ew2.updateCh); err != nil { + t.Fatal(err) + } + + // Update to the resource being watched. The un-cancelled callback + // should be invoked while the cancelled one should not be. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{test.updatedWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyEndpointsUpdate(ctx, ew1.updateCh, test.wantUpdateV2); err != nil { + t.Fatal(err) + } + if err := verifyNoEndpointsUpdate(ctx, ew2.updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestEDSWatch_ThreeWatchesForDifferentResourceNames covers the case with three +// watchers (two watchers for one resource, and the third watcher for another +// resource), exist across two endpoint configuration resources. The test verifies +// that an update from the management server containing both resources results +// in the invocation of all watch callbacks. +// +// The test is run with both old and new style names. +func (s) TestEDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same endpoint resource and have the + // callbacks push the received updates on to a channel. + ew1 := newEndpointsWatcher() + edsCancel1 := xdsresource.WatchEndpoints(client, edsName, ew1) + defer edsCancel1() + ew2 := newEndpointsWatcher() + edsCancel2 := xdsresource.WatchEndpoints(client, edsName, ew2) + defer edsCancel2() + + // Register the third watch for a different endpoint resource. + edsNameNewStyle := makeNewStyleEDSName(authority) + ew3 := newEndpointsWatcher() + edsCancel3 := xdsresource.WatchEndpoints(client, edsNameNewStyle, ew3) + defer edsCancel3() + + // Configure the management server to return two endpoint resources, + // corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(edsName, edsHost1, []uint32{edsPort1}), + e2e.DefaultEndpoint(edsNameNewStyle, edsHost1, []uint32{edsPort1}), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the all watchers. The two + // resources returned differ only in the resource name. Therefore the + // expected update is the same for all the watchers. + wantUpdate := endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: fmt.Sprintf("%s:%d", edsHost1, edsPort1)}}, + }, + Weight: 1, + }}, + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + } + if err := verifyEndpointsUpdate(ctx, ew1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyEndpointsUpdate(ctx, ew2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyEndpointsUpdate(ctx, ew3.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestEDSWatch_ResourceCaching covers the case where a watch is registered for +// a resource which is already present in the cache. The test verifies that the +// watch callback is invoked with the contents from the cache, instead of a +// request being sent to the management server. +func (s) TestEDSWatch_ResourceCaching(t *testing.T) { + firstRequestReceived := false + firstAckReceived := grpcsync.NewEvent() + secondRequestReceived := grpcsync.NewEvent() + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + // The first request has an empty version string. + if !firstRequestReceived && req.GetVersionInfo() == "" { + firstRequestReceived = true + return nil + } + // The first ack has a non-empty version string. + if !firstAckReceived.HasFired() && req.GetVersionInfo() != "" { + firstAckReceived.Fire() + return nil + } + // Any requests after the first request and ack, are not expected. + secondRequestReceived.Fire() + return nil + }, + }) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for an endpoint resource and have the watch callback + // push the received update on to a channel. + ew1 := newEndpointsWatcher() + edsCancel1 := xdsresource.WatchEndpoints(client, edsName, ew1) + defer edsCancel1() + + // Configure the management server to return a single endpoint resource, + // corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsName, edsHost1, []uint32{edsPort1})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: fmt.Sprintf("%s:%d", edsHost1, edsPort1)}}, + }, + Weight: 1, + }}, + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + } + if err := verifyEndpointsUpdate(ctx, ew1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + t.Fatal("timeout when waiting for receipt of ACK at the management server") + case <-firstAckReceived.Done(): + } + + // Register another watch for the same resource. This should get the update + // from the cache. + ew2 := newEndpointsWatcher() + edsCancel2 := xdsresource.WatchEndpoints(client, edsName, ew2) + defer edsCancel2() + if err := verifyEndpointsUpdate(ctx, ew2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // No request should get sent out as part of this watch. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-secondRequestReceived.Done(): + t.Fatal("xdsClient sent out request instead of using update from cache") + } +} + +// TestEDSWatch_ExpiryTimerFiresBeforeResponse tests the case where the client +// does not receive an EDS response for the request that it sends. The test +// verifies that the watch callback is invoked with an error once the +// watchExpiryTimer fires. +func (s) TestEDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Register a watch for a resource which is expected to fail with an error + // after the watch expiry timer fires. + ew := newEndpointsWatcher() + edsCancel := xdsresource.WatchEndpoints(client, edsName, ew) + defer edsCancel() + + // Wait for the watch expiry timer to fire. + <-time.After(defaultTestWatchExpiryTimeout) + + // Verify that an empty update with the expected error is received. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantErr := xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, "") + if err := verifyEndpointsUpdate(ctx, ew.updateCh, endpointsUpdateErrTuple{err: wantErr}); err != nil { + t.Fatal(err) + } +} + +// TestEDSWatch_ValidResponseCancelsExpiryTimerBehavior tests the case where the +// client receives a valid EDS response for the request that it sends. The test +// verifies that the behavior associated with the expiry timer (i.e, callback +// invocation with error) does not take place. +func (s) TestEDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client talking to the above management server. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Register a watch for an endpoint resource and have the watch callback + // push the received update on to a channel. + ew := newEndpointsWatcher() + edsCancel := xdsresource.WatchEndpoints(client, edsName, ew) + defer edsCancel() + + // Configure the management server to return a single endpoint resource, + // corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsName, edsHost1, []uint32{edsPort1})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: fmt.Sprintf("%s:%d", edsHost1, edsPort1)}}, + }, + Weight: 1, + }}, + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + } + if err := verifyEndpointsUpdate(ctx, ew.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Wait for the watch expiry timer to fire, and verify that the callback is + // not invoked. + <-time.After(defaultTestWatchExpiryTimeout) + if err := verifyNoEndpointsUpdate(ctx, ew.updateCh); err != nil { + t.Fatal(err) + } +} + +// TestEDSWatch_NACKError covers the case where an update from the management +// server is NACK'ed by the xdsclient. The test verifies that the error is +// propagated to the watcher. +func (s) TestEDSWatch_NACKError(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a route configuration resource and have the watch + // callback push the received update on to a channel. + ew := newEndpointsWatcher() + edsCancel := xdsresource.WatchEndpoints(client, edsName, ew) + defer edsCancel() + + // Configure the management server to return a single route configuration + // resource which is expected to be NACKed by the client. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{badEndpointsResource(edsName, edsHost1, []uint32{edsPort1})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher. + u, err := ew.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for an endpoints resource from the management server: %v", err) + } + gotErr := u.(endpointsUpdateErrTuple).err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantEndpointsNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantEndpointsNACKErr) + } +} + +// TestEDSWatch_PartialValid covers the case where a response from the +// management server contains both valid and invalid resources and is expected +// to be NACK'ed by the xdsclient. The test verifies that watchers corresponding +// to the valid resource receive the update, while watchers corresponding to the +// invalid resource receive an error. +func (s) TestEDSWatch_PartialValid(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for two endpoint resources. The first watch is + // expected to receive an error because the received resource is NACKed. + // The second watch is expected to get a good update. + badResourceName := edsName + ew1 := newEndpointsWatcher() + edsCancel1 := xdsresource.WatchEndpoints(client, badResourceName, ew1) + defer edsCancel1() + goodResourceName := makeNewStyleEDSName(authority) + ew2 := newEndpointsWatcher() + edsCancel2 := xdsresource.WatchEndpoints(client, goodResourceName, ew2) + defer edsCancel2() + + // Configure the management server to return two endpoints resources, + // corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + badEndpointsResource(badResourceName, edsHost1, []uint32{edsPort1}), + e2e.DefaultEndpoint(goodResourceName, edsHost1, []uint32{edsPort1}), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher which + // requested for the bad resource. + u, err := ew1.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for an endpoints resource from the management server: %v", err) + } + gotErr := u.(endpointsUpdateErrTuple).err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantEndpointsNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantEndpointsNACKErr) + } + + // Verify that the watcher watching the good resource receives an update. + wantUpdate := endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: fmt.Sprintf("%s:%d", edsHost1, edsPort1)}}, + }, + Weight: 1, + }}, + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + } + if err := verifyEndpointsUpdate(ctx, ew2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} diff --git a/internal/xds/xdsclient/tests/fallback/fallback_test.go b/internal/xds/xdsclient/tests/fallback/fallback_test.go new file mode 100644 index 0000000..5c96ea2 --- /dev/null +++ b/internal/xds/xdsclient/tests/fallback/fallback_test.go @@ -0,0 +1,931 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_fallback_test + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter. + _ "google.golang.org/grpc/xds" // To ensure internal.NewXDSResolverWithConfigForTesting is set. +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// Give the fallback tests additional time to complete because they need to +// first identify failed connections before establishing new ones. +const defaultFallbackTestTimeout = 20 * time.Second +const defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. + +func waitForRPCsToReachBackend(ctx context.Context, client testgrpc.TestServiceClient, backend string) error { + var lastErr error + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + var peer peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + lastErr = err + continue + } + // Veirfy the peer when the RPC succeeds. + if peer.Addr.String() == backend { + break + } + } + if ctx.Err() != nil { + return fmt.Errorf("timeout when waiting for RPCs to reach expected backend. Last error: %v", lastErr) + } + return nil +} + +// Tests fallback on startup where the xDS client is unable to establish a +// connection to the primary server. The test verifies that the xDS client falls +// back to the secondary server, and when the primary comes back up, it reverts +// to it. The test also verifies that when all requested resources are cached +// from the primary, fallback is not triggered when the connection goes down. +func (s) TestFallback_OnStartup(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout) + defer cancel() + + // Create two listeners for the two management servers. The test can + // start/stop these listeners and can also get notified when the listener + // receives a connection request. + primaryWrappedLis := testutils.NewListenerWrapper(t, nil) + primaryLis := testutils.NewRestartableListener(primaryWrappedLis) + fallbackWrappedLis := testutils.NewListenerWrapper(t, nil) + fallbackLis := testutils.NewRestartableListener(fallbackWrappedLis) + + // Start two management servers, primary and fallback, with the above + // listeners. + primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis}) + fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis}) + + // Start two test service backends. + backend1 := stubserver.StartTestService(t, nil) + defer backend1.Stop() + backend2 := stubserver.StartTestService(t, nil) + defer backend2.Stop() + + // Configure xDS resource on the primary management server, with a cluster + // resource that contains an endpoint for backend1. + nodeID := uuid.New().String() + const serviceName = "my-service-fallback-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, backend1.Address), + SecLevel: e2e.SecurityLevelNone, + }) + if err := primaryManagementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Configure xDS resource on the secondary management server, with a cluster + // resource that contains an endpoint for backend2. Only the listener + // resource has the same name on both servers. + fallbackRouteConfigName := "fallback-route-" + serviceName + fallbackClusterName := "fallback-cluster-" + serviceName + fallbackEndpointsName := "fallback-endpoints-" + serviceName + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})}, + } + if err := fallbackManagementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Shut both management servers down before starting the gRPC client to + // trigger fallback on startup. + primaryLis.Stop() + fallbackLis.Stop() + + // Generate bootstrap configuration with the above two servers. + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[ + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }, + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, primaryManagementServer.Address, fallbackManagementServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + }) + if err != nil { + t.Fatalf("Failed to create bootstrap file: %v", err) + } + + // Create an xDS client with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + + // Get the xDS resolver to use the above xDS client. + resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error)) + resolver, err := resolverBuilder(pool) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a gRPC client that uses the above xDS resolver. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + cc.Connect() + + // Ensure that a connection is attempted to the primary. + if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil { + t.Fatalf("Failure when waiting for a connection to be opened to the primary management server: %v", err) + } + + // Ensure that a connection is attempted to the fallback. + if _, err := fallbackWrappedLis.NewConnCh.Receive(ctx); err != nil { + t.Fatalf("Failure when waiting for a connection to be opened to the primary management server: %v", err) + } + + // Make an RPC with a shortish deadline and expect it to fail. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}, grpc.WaitForReady(true)); err == nil || status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() = %v, want DeadlineExceeded", err) + } + + // Start the fallback server. Ensure that an RPC can succeed, and that it + // reaches backend2. + fallbackLis.Restart() + if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil { + t.Fatal(err) + } + + // Start the primary server. It can take a while before the xDS client + // notices this, since the ADS stream implementation uses a backoff before + // retrying the stream. + primaryLis.Restart() + + // Wait for the connection to the secondary to be closed and ensure that an + // RPC can succeed, and that it reaches backend1. + c, err := fallbackWrappedLis.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failure when retrieving the most recent connection to the fallback management server: %v", err) + } + conn := c.(*testutils.ConnWrapper) + if _, err := conn.CloseCh.Receive(ctx); err != nil { + t.Fatalf("Connection to fallback server not closed once primary becomes ready: %v", err) + } + if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil { + t.Fatal(err) + } + + // Stop the primary servers. Since all xDS resources were received from the + // primary (and RPCs were succeeding to the clusters returned by the + // primary), we will not trigger fallback. + primaryLis.Stop() + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := fallbackWrappedLis.NewConnCh.Receive(sCtx); err == nil { + t.Fatalf("Fallback attempted when not expected to. There are no uncached resources from the primary server at this point.") + } + + // Ensure that RPCs still succeed, and that they use the configuration + // received from the primary. + if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil { + t.Fatal(err) + } +} + +// Tests fallback when the primary management server fails during an update. +func (s) TestFallback_MidUpdate(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout) + defer cancel() + + // Create two listeners for the two management servers. The test can + // start/stop these listeners and can also get notified when the listener + // receives a connection request. + primaryWrappedLis := testutils.NewListenerWrapper(t, nil) + primaryLis := testutils.NewRestartableListener(primaryWrappedLis) + fallbackWrappedLis := testutils.NewListenerWrapper(t, nil) + fallbackLis := testutils.NewRestartableListener(fallbackWrappedLis) + + // This boolean helps with triggering fallback mid update. When this boolean + // is set and the below defined cluster resource is requested, the primary + // management server shuts down the connection, forcing the client to + // fallback to the secondary server. + var closeConnOnMidUpdateClusterResource atomic.Bool + const ( + serviceName = "my-service-fallback-xds" + routeConfigName = "route-" + serviceName + clusterName = "cluster-" + serviceName + endpointsName = "endpoints-" + serviceName + midUpdateRouteConfigName = "mid-update-route-" + serviceName + midUpdateClusterName = "mid-update-cluster-" + serviceName + midUpdateEndpointsName = "mid-update-endpoints-" + serviceName + ) + + // Start two management servers, primary and fallback, with the above + // listeners. + primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: primaryLis, + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if closeConnOnMidUpdateClusterResource.Load() == false { + return nil + } + if req.GetTypeUrl() != version.V3ClusterURL { + return nil + } + for _, name := range req.GetResourceNames() { + if name == midUpdateClusterName { + primaryLis.Stop() + return fmt.Errorf("closing ADS stream because %q resource was requested", midUpdateClusterName) + } + } + return nil + }, + AllowResourceSubset: true, + }) + fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis}) + + // Start three test service backends. + backend1 := stubserver.StartTestService(t, nil) + defer backend1.Stop() + backend2 := stubserver.StartTestService(t, nil) + defer backend2.Stop() + backend3 := stubserver.StartTestService(t, nil) + defer backend3.Stop() + + // Configure xDS resource on the primary management server, with a cluster + // resource that contains an endpoint for backend1. + nodeID := uuid.New().String() + primaryResources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, "localhost", []uint32{testutils.ParsePort(t, backend1.Address)})}, + } + if err := primaryManagementServer.Update(ctx, primaryResources); err != nil { + t.Fatal(err) + } + + // Configure xDS resource on the secondary management server, with a cluster + // resource that contains an endpoint for backend2. Only the listener + // resource has the same name on both servers. + const ( + fallbackRouteConfigName = "fallback-route-" + serviceName + fallbackClusterName = "fallback-cluster-" + serviceName + fallbackEndpointsName = "fallback-endpoints-" + serviceName + ) + fallbackResources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})}, + } + if err := fallbackManagementServer.Update(ctx, fallbackResources); err != nil { + t.Fatal(err) + } + + // Generate bootstrap configuration with the above two servers. + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[ + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }, + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, primaryManagementServer.Address, fallbackManagementServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + }) + if err != nil { + t.Fatalf("Failed to create bootstrap file: %v", err) + } + + // Create an xDS client with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + + // Get the xDS resolver to use the above xDS client. + resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error)) + resolver, err := resolverBuilder(pool) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a gRPC client that uses the above xDS resolver. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + cc.Connect() + + // Ensure that RPCs reach the cluster specified by the primary server and + // that no connection is attempted to the fallback server. + client := testgrpc.NewTestServiceClient(cc) + if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil { + t.Fatal(err) + } + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := fallbackWrappedLis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Connection attempt made to fallback server when none expected: %v", err) + } + + // Instruct the primary server to close the connection if below defined + // cluster resource is requested. + closeConnOnMidUpdateClusterResource.Store(true) + + // Update the listener resource on the primary server to point to a new + // route configuration that points to a new cluster that points to a new + // endpoints resource that contains backend3. + primaryResources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, midUpdateRouteConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(midUpdateRouteConfigName, serviceName, midUpdateClusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(midUpdateClusterName, midUpdateEndpointsName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(midUpdateEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend3.Address)})}, + } + if err := primaryManagementServer.Update(ctx, primaryResources); err != nil { + t.Fatal(err) + } + + // Ensure that a connection is attempted to the fallback (because both + // conditions mentioned for fallback in A71 are satisfied: connectivity + // failure and a watcher for an uncached resource), and that RPCs are + // routed to the cluster returned by the fallback server. + c, err := fallbackWrappedLis.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failure when waiting for a connection to be opened to the fallback management server: %v", err) + } + fallbackConn := c.(*testutils.ConnWrapper) + if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil { + t.Fatal(err) + } + + // Set the primary management server to not close the connection anymore if + // the mid-update cluster resource is requested, and get it to start serving + // again. + closeConnOnMidUpdateClusterResource.Store(false) + primaryLis.Restart() + + // A new snapshot, with the same resources, is pushed to the management + // server to get it to respond for already requested resource names. + if err := primaryManagementServer.Update(ctx, primaryResources); err != nil { + t.Fatal(err) + } + + // Ensure that RPCs reach the backend pointed to by the new cluster. + if err := waitForRPCsToReachBackend(ctx, client, backend3.Address); err != nil { + t.Fatal(err) + } + + // Wait for the connection to the secondary to be closed since we have + // reverted back to the primary. + if _, err := fallbackConn.CloseCh.Receive(ctx); err != nil { + t.Fatalf("Connection to fallback server not closed once primary becomes ready: %v", err) + } +} + +// Tests fallback when the primary management server fails during startup. +func (s) TestFallback_MidStartup(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout) + defer cancel() + + // Create two listeners for the two management servers. The test can + // start/stop these listeners and can also get notified when the listener + // receives a connection request. + primaryWrappedLis := testutils.NewListenerWrapper(t, nil) + primaryLis := testutils.NewRestartableListener(primaryWrappedLis) + fallbackWrappedLis := testutils.NewListenerWrapper(t, nil) + fallbackLis := testutils.NewRestartableListener(fallbackWrappedLis) + + // This boolean helps with triggering fallback during startup. When this + // boolean is set and a cluster resource is requested, the primary + // management server shuts down the connection, forcing the client to + // fallback to the secondary server. + var closeConnOnClusterResource atomic.Bool + closeConnOnClusterResource.Store(true) + + // Start two management servers, primary and fallback, with the above + // listeners. + primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: primaryLis, + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if closeConnOnClusterResource.Load() == false { + return nil + } + if req.GetTypeUrl() != version.V3ClusterURL { + return nil + } + primaryLis.Stop() + return fmt.Errorf("closing ADS stream because cluster resource was requested") + }, + AllowResourceSubset: true, + }) + fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis}) + + // Start two test service backends. + backend1 := stubserver.StartTestService(t, nil) + defer backend1.Stop() + backend2 := stubserver.StartTestService(t, nil) + defer backend2.Stop() + + // Configure xDS resource on the primary management server, with a cluster + // resource that contains an endpoint for backend1. + nodeID := uuid.New().String() + const serviceName = "my-service-fallback-xds" + primaryResources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, backend1.Address), + SecLevel: e2e.SecurityLevelNone, + }) + if err := primaryManagementServer.Update(ctx, primaryResources); err != nil { + t.Fatal(err) + } + + // Configure xDS resource on the secondary management server, with a cluster + // resource that contains an endpoint for backend2. Only the listener + // resource has the same name on both servers. + fallbackRouteConfigName := "fallback-route-" + serviceName + fallbackClusterName := "fallback-cluster-" + serviceName + fallbackEndpointsName := "fallback-endpoints-" + serviceName + fallbackResources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})}, + } + if err := fallbackManagementServer.Update(ctx, fallbackResources); err != nil { + t.Fatal(err) + } + + // Generate bootstrap configuration with the above two servers. + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[ + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }, + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, primaryManagementServer.Address, fallbackManagementServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + }) + if err != nil { + t.Fatalf("Failed to create bootstrap file: %v", err) + } + + // Create an xDS client with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + + // Get the xDS resolver to use the above xDS client. + resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error)) + resolver, err := resolverBuilder(pool) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a gRPC client that uses the above xDS resolver. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + cc.Connect() + + // Ensure that a connection is attempted to the primary. + if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil { + t.Fatalf("Failure when waiting for a connection to be opened to the primary management server: %v", err) + } + + // Ensure that a connection is attempted to the fallback. + c, err := fallbackWrappedLis.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failure when waiting for a connection to be opened to the secondary management server: %v", err) + } + fallbackConn := c.(*testutils.ConnWrapper) + + // Ensure that RPCs are routed to the cluster returned by the fallback + // management server. + client := testgrpc.NewTestServiceClient(cc) + if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil { + t.Fatal(err) + } + + // Get the primary management server to no longer close the connection when + // the cluster resource is requested. + closeConnOnClusterResource.Store(false) + primaryLis.Restart() + + // A new snapshot, with the same resources, is pushed to the management + // server to get it to respond for already requested resource names. + if err := primaryManagementServer.Update(ctx, primaryResources); err != nil { + t.Fatal(err) + } + + // Ensure that RPCs are routed to the cluster returned by the primary + // management server. + if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil { + t.Fatal(err) + } + + // Wait for the connection to the secondary to be closed since we have + // reverted back to the primary. + if _, err := fallbackConn.CloseCh.Receive(ctx); err != nil { + t.Fatalf("Connection to fallback server not closed once primary becomes ready: %v", err) + } +} + +// Tests that RPCs succeed at startup when the primary management server is +// down, but the secondary is available. +func (s) TestFallback_OnStartup_RPCSuccess(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout) + defer cancel() + + // Create two listeners for the two management servers. The test can + // start/stop these listeners. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + primaryLis := testutils.NewRestartableListener(l) + l, err = testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + fallbackLis := testutils.NewRestartableListener(l) + + // Start two management servers, primary and fallback, with the above + // listeners. + primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis}) + fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis}) + + // Start two test service backends. + backend1 := stubserver.StartTestService(t, nil) + defer backend1.Stop() + backend2 := stubserver.StartTestService(t, nil) + defer backend2.Stop() + + // Configure xDS resource on the primary management server, with a cluster + // resource that contains an endpoint for backend1. + nodeID := uuid.New().String() + const serviceName = "my-service-fallback-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, backend1.Address), + SecLevel: e2e.SecurityLevelNone, + }) + if err := primaryManagementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Configure xDS resource on the secondary management server, with a cluster + // resource that contains an endpoint for backend2. Only the listener + // resource has the same name on both servers. + fallbackRouteConfigName := "fallback-route-" + serviceName + fallbackClusterName := "fallback-cluster-" + serviceName + fallbackEndpointsName := "fallback-endpoints-" + serviceName + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})}, + } + if err := fallbackManagementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Shutdown the primary management server before starting the gRPC client to + // trigger fallback on startup. + primaryLis.Stop() + + // Generate bootstrap configuration with the above two servers. + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[ + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }, + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, primaryManagementServer.Address, fallbackManagementServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + }) + if err != nil { + t.Fatalf("Failed to create bootstrap file: %v", err) + } + + // Create an xDS client with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + + // Get the xDS resolver to use the above xDS client. + resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error)) + resolver, err := resolverBuilder(pool) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a gRPC client that uses the above xDS resolver. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + + // Make an RPC (without the `wait_for_ready` call option) and expect it to + // succeed since the fallback management server is up and running. + client := testgrpc.NewTestServiceClient(cc) + var peer peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if got, want := peer.Addr.String(), backend2.Address; got != want { + t.Fatalf("Unexpected peer address: got %q, want %q", got, want) + } + + // Start the primary server. It can take a while before the xDS client + // notices this, since the ADS stream implementation uses a backoff before + // retrying the stream. + primaryLis.Restart() + if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil { + t.Fatal(err) + } +} + +// Test verifies that when the primary management server is unavailable, the +// system attempts to connect to the first fallback server, and if that is also +// down, to the second fallback server. It also ensures that the system switches +// back to the first fallback server once it becomes available again, and +// eventually returns to the primary server when it comes back online, closing +// connections to the fallback servers accordingly. +func (s) TestFallback_ThreeServerPromotion(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout) + defer cancel() + + // Create three listener wrappers for three management servers. + primaryWrappedLis := testutils.NewListenerWrapper(t, nil) + primaryLis := testutils.NewRestartableListener(primaryWrappedLis) + + secondaryWrappedLis := testutils.NewListenerWrapper(t, nil) + secondaryLis := testutils.NewRestartableListener(secondaryWrappedLis) + + tertiaryWrappedLis := testutils.NewListenerWrapper(t, nil) + tertiaryLis := testutils.NewRestartableListener(tertiaryWrappedLis) + + // Start the three management servers. + primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis}) + secondaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: secondaryLis}) + tertiaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: tertiaryLis}) + + // Start three test service backends. + backend1 := stubserver.StartTestService(t, nil) + backend1Port := testutils.ParsePort(t, backend1.Address) + defer backend1.Stop() + backend2 := stubserver.StartTestService(t, nil) + backend2Port := testutils.ParsePort(t, backend2.Address) + defer backend2.Stop() + backend3 := stubserver.StartTestService(t, nil) + backend3Port := testutils.ParsePort(t, backend3.Address) + defer backend3.Stop() + + nodeID := uuid.New().String() + const ( + serviceName = "my-service-fallback-xds" + primaryRouteConfigName = "primary-route-" + serviceName + secondaryRouteConfigName = "secondary-route-" + serviceName + tertiaryRouteConfigName = "tertiary-route-" + serviceName + primaryClusterName = "primary-cluster-" + serviceName + secondaryClusterName = "secondary-cluster-" + serviceName + tertiaryClusterName = "tertiary-cluster-" + serviceName + primaryEndpointsName = "primary-endpoints-" + serviceName + secondaryEndpointsName = "secondary-endpoints-" + serviceName + tertiaryEndpointsName = "tertiary-endpoints-" + serviceName + ) + + // Configure partial resources on the primary and secondary + // management servers. + primaryManagementServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, primaryRouteConfigName)}, + }) + secondaryManagementServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, secondaryRouteConfigName)}, + }) + + // Configure full resources on tertiary management server. + updateOpts := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, tertiaryRouteConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(tertiaryRouteConfigName, serviceName, tertiaryClusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(tertiaryClusterName, tertiaryEndpointsName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(tertiaryEndpointsName, "localhost", []uint32{backend3Port})}, + } + tertiaryManagementServer.Update(ctx, updateOpts) + + // Create bootstrap configuration for all three management servers. + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[ + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }, + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }, + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + } + ]`, primaryManagementServer.Address, secondaryManagementServer.Address, tertiaryManagementServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + }) + if err != nil { + t.Fatalf("Failed to create bootstrap file: %v", err) + } + + // Create an xDS client with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %v", err) + } + pool := xdsclient.NewPool(config) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + + // Get the xDS resolver to use the above xDS client. + resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error)) + resolver, err := resolverBuilder(pool) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Start a gRPC client that uses the above xDS resolver. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("Failed to create gRPC client: %v", err) + } + defer cc.Close() + cc.Connect() + client := testgrpc.NewTestServiceClient(cc) + + // Verify that connection attempts were made to primaryWrappedLis and + // secondaryWrappedLis, before using tertiaryWrappedLis to make + // successful RPCs to backend3. + if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for connection to primary: %v", err) + } + + // Stop primary, client should connect to secondary. + primaryLis.Stop() + if _, err := secondaryWrappedLis.NewConnCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for connection to secondary after primary stopped: %v", err) + } + + // Stop secondary, client should connect to tertiary. + secondaryLis.Stop() + tertiaryConn, err := tertiaryWrappedLis.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for connection to tertiary after secondary stopped: %v", err) + } + + // Tertiary has all resources, RPCs should succeed to backend3. + if err := waitForRPCsToReachBackend(ctx, client, backend3.Address); err != nil { + t.Fatal(err) + } + + // Secondary1 becomes available, RPCs go to backend2. + secondaryLis.Restart() + updateOpts = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, secondaryRouteConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(secondaryRouteConfigName, serviceName, secondaryClusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(secondaryClusterName, secondaryEndpointsName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(secondaryEndpointsName, "localhost", []uint32{backend2Port})}, + } + secondaryManagementServer.Update(ctx, updateOpts) + + secondaryConn, err := secondaryWrappedLis.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for new connection to secondary: %v", err) + } + if _, err := tertiaryConn.(*testutils.ConnWrapper).CloseCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for connection to the tertiary to be closed after promotion to secondary: %v", err) + } + if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil { + t.Fatal(err) + } + + // Primary becomes available, RPCs go to backend1. + primaryLis.Restart() + updateOpts = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, primaryRouteConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(primaryRouteConfigName, serviceName, primaryClusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(primaryClusterName, primaryEndpointsName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(primaryEndpointsName, "localhost", []uint32{backend1Port})}, + } + primaryManagementServer.Update(ctx, updateOpts) + + if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for new connection to primary: %v", err) + } + if _, err := secondaryConn.(*testutils.ConnWrapper).CloseCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for connection to the secondary to be closed after promotion to primary: %v", err) + } + if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil { + t.Fatal(err) + } +} diff --git a/internal/xds/xdsclient/tests/federation_watchers_test.go b/internal/xds/xdsclient/tests/federation_watchers_test.go new file mode 100644 index 0000000..996a6b6 --- /dev/null +++ b/internal/xds/xdsclient/tests/federation_watchers_test.go @@ -0,0 +1,324 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsclient_test + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/resolver" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" +) + +const testNonDefaultAuthority = "non-default-authority" + +// setupForFederationWatchersTest spins up two management servers, one for the +// default (empty) authority and another for a non-default authority. +// +// Returns the management server associated with the non-default authority, the +// nodeID to use, and the xDS client. +func setupForFederationWatchersTest(t *testing.T) (*e2e.ManagementServer, string, xdsclient.XDSClient) { + // Start a management server as the default authority. + serverDefaultAuthority := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Start another management server as the other authority. + serverNonDefaultAuthority := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, serverDefaultAuthority.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + testNonDefaultAuthority: []byte(fmt.Sprintf(`{ + "xds_servers": [{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]}`, serverNonDefaultAuthority.Address)), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + t.Cleanup(close) + return serverNonDefaultAuthority, nodeID, client +} + +// TestFederation_ListenerResourceContextParamOrder covers the case of watching +// a Listener resource with the new style resource name and context parameters. +// The test registers watches for two resources which differ only in the order +// of context parameters in their URI. The server is configured to respond with +// a single resource with canonicalized context parameters. The test verifies +// that both watchers are notified. +func (s) TestFederation_ListenerResourceContextParamOrder(t *testing.T) { + serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t) + + var ( + // Two resource names only differ in context parameter order. + resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource?a=1&b=2", testNonDefaultAuthority) + resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource?b=2&a=1", testNonDefaultAuthority) + ) + + // Register two watches for listener resources with the same query string, + // but context parameters in different order. + lw1 := newListenerWatcher() + ldsCancel1 := xdsresource.WatchListener(client, resourceName1, lw1) + defer ldsCancel1() + lw2 := newListenerWatcher() + ldsCancel2 := xdsresource.WatchListener(client, resourceName2, lw2) + defer ldsCancel2() + + // Configure the management server for the non-default authority to return a + // single listener resource, corresponding to the watches registered above. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(resourceName1, "rds-resource")}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + wantUpdate := listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: "rds-resource", + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + // Verify the contents of the received update. + if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestFederation_RouteConfigResourceContextParamOrder covers the case of +// watching a RouteConfiguration resource with the new style resource name and +// context parameters. The test registers watches for two resources which +// differ only in the order of context parameters in their URI. The server is +// configured to respond with a single resource with canonicalized context +// parameters. The test verifies that both watchers are notified. +func (s) TestFederation_RouteConfigResourceContextParamOrder(t *testing.T) { + serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t) + + var ( + // Two resource names only differ in context parameter order. + resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource?a=1&b=2", testNonDefaultAuthority) + resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource?b=2&a=1", testNonDefaultAuthority) + ) + + // Register two watches for route configuration resources with the same + // query string, but context parameters in different order. + rw1 := newRouteConfigWatcher() + rdsCancel1 := xdsresource.WatchRouteConfig(client, resourceName1, rw1) + defer rdsCancel1() + rw2 := newRouteConfigWatcher() + rdsCancel2 := xdsresource.WatchRouteConfig(client, resourceName2, rw2) + defer rdsCancel2() + + // Configure the management server for the non-default authority to return a + // single route config resource, corresponding to the watches registered. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(resourceName1, "listener-resource", "cluster-resource")}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + wantUpdate := routeConfigUpdateErrTuple{ + update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{"listener-resource"}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: []xdsresource.WeightedCluster{{Name: "cluster-resource", Weight: 100}}, + }, + }, + }, + }, + }, + } + // Verify the contents of the received update. + if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestFederation_ClusterResourceContextParamOrder covers the case of watching a +// Cluster resource with the new style resource name and context parameters. +// The test registers watches for two resources which differ only in the order +// of context parameters in their URI. The server is configured to respond with +// a single resource with canonicalized context parameters. The test verifies +// that both watchers are notified. +func (s) TestFederation_ClusterResourceContextParamOrder(t *testing.T) { + serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t) + + var ( + // Two resource names only differ in context parameter order. + resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource?a=1&b=2", testNonDefaultAuthority) + resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource?b=2&a=1", testNonDefaultAuthority) + ) + + // Register two watches for cluster resources with the same query string, + // but context parameters in different order. + cw1 := newClusterWatcher() + cdsCancel1 := xdsresource.WatchCluster(client, resourceName1, cw1) + defer cdsCancel1() + cw2 := newClusterWatcher() + cdsCancel2 := xdsresource.WatchCluster(client, resourceName2, cw2) + defer cdsCancel2() + + // Configure the management server for the non-default authority to return a + // single cluster resource, corresponding to the watches registered. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(resourceName1, "eds-service-name", e2e.SecurityLevelNone)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + wantUpdate := clusterUpdateErrTuple{ + update: xdsresource.ClusterUpdate{ + ClusterName: "xdstp://non-default-authority/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource?a=1&b=2", + EDSServiceName: "eds-service-name", + }, + } + // Verify the contents of the received update. + if err := verifyClusterUpdate(ctx, cw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyClusterUpdate(ctx, cw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestFederation_EndpointsResourceContextParamOrder covers the case of watching +// an Endpoints resource with the new style resource name and context parameters. +// The test registers watches for two resources which differ only in the order +// of context parameters in their URI. The server is configured to respond with +// a single resource with canonicalized context parameters. The test verifies +// that both watchers are notified. +func (s) TestFederation_EndpointsResourceContextParamOrder(t *testing.T) { + serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t) + + var ( + // Two resource names only differ in context parameter order. + resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource?a=1&b=2", testNonDefaultAuthority) + resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource?b=2&a=1", testNonDefaultAuthority) + ) + + // Register two watches for endpoint resources with the same query string, + // but context parameters in different order. + ew1 := newEndpointsWatcher() + edsCancel1 := xdsresource.WatchEndpoints(client, resourceName1, ew1) + defer edsCancel1() + ew2 := newEndpointsWatcher() + edsCancel2 := xdsresource.WatchEndpoints(client, resourceName2, ew2) + defer edsCancel2() + + // Configure the management server for the non-default authority to return a + // single endpoints resource, corresponding to the watches registered. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(resourceName1, "localhost", []uint32{666})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + wantUpdate := endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "localhost:666"}}, + }, + Weight: 1, + }}, + Weight: 1, + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + }, + }, + }, + } + // Verify the contents of the received update. + if err := verifyEndpointsUpdate(ctx, ew1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyEndpointsUpdate(ctx, ew2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +func newStringP(s string) *string { + return &s +} diff --git a/internal/xds/xdsclient/tests/helpers_test.go b/internal/xds/xdsclient/tests/helpers_test.go new file mode 100644 index 0000000..ab060d5 --- /dev/null +++ b/internal/xds/xdsclient/tests/helpers_test.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestWatchExpiryTimeout = 500 * time.Millisecond + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. + + ldsName = "xdsclient-test-lds-resource" + rdsName = "xdsclient-test-rds-resource" + cdsName = "xdsclient-test-cds-resource" + edsName = "xdsclient-test-eds-resource" + ldsNameNewStyle = "xdstp:///envoy.config.listener.v3.Listener/xdsclient-test-lds-resource" + rdsNameNewStyle = "xdstp:///envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource" + cdsNameNewStyle = "xdstp:///envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource" + edsNameNewStyle = "xdstp:///envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource" +) + +func makeAuthorityName(name string) string { + segs := strings.Split(name, "/") + return strings.Join(segs, "") +} + +func makeNewStyleLDSName(authority string) string { + return fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource", authority) +} + +func makeNewStyleRDSName(authority string) string { + return fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource", authority) +} + +func makeNewStyleCDSName(authority string) string { + return fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource", authority) +} + +func makeNewStyleEDSName(authority string) string { + return fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource", authority) +} diff --git a/internal/xds/xdsclient/tests/lds_watchers_test.go b/internal/xds/xdsclient/tests/lds_watchers_test.go new file mode 100644 index 0000000..fc17380 --- /dev/null +++ b/internal/xds/xdsclient/tests/lds_watchers_test.go @@ -0,0 +1,1397 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/envoyproxy/go-control-plane/pkg/wellknown" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter. + _ "google.golang.org/grpc/xds" // To ensure internal.NewXDSResolverWithConfigForTesting is set. +) + +type noopListenerWatcher struct{} + +func (noopListenerWatcher) ResourceChanged(_ *xdsresource.ListenerUpdate, onDone func()) { + onDone() +} +func (noopListenerWatcher) ResourceError(_ error, onDone func()) { + onDone() +} +func (noopListenerWatcher) AmbientError(_ error, onDone func()) { + onDone() +} + +type listenerUpdateErrTuple struct { + update *xdsresource.ListenerUpdate + err error +} + +type listenerWatcher struct { + updateCh *testutils.Channel +} + +func newListenerWatcher() *listenerWatcher { + return &listenerWatcher{updateCh: testutils.NewChannel()} +} + +func (lw *listenerWatcher) ResourceChanged(update *xdsresource.ListenerUpdate, onDone func()) { + lw.updateCh.Send(listenerUpdateErrTuple{update: update}) + onDone() +} + +func (lw *listenerWatcher) ResourceError(err error, onDone func()) { + // When used with a go-control-plane management server that continuously + // resends resources which are NACKed by the xDS client, using a `Replace()` + // here and in AmbientError() simplifies tests which will have + // access to the most recently received error. + lw.updateCh.Replace(listenerUpdateErrTuple{err: err}) + onDone() +} + +func (lw *listenerWatcher) AmbientError(err error, onDone func()) { + lw.updateCh.Replace(listenerUpdateErrTuple{err: err}) + onDone() +} + +type listenerWatcherMultiple struct { + updateCh *testutils.Channel +} + +// TODO: delete this once `newListenerWatcher` is modified to handle multiple +// updates (https://github.com/grpc/grpc-go/issues/7864). +func newListenerWatcherMultiple(size int) *listenerWatcherMultiple { + return &listenerWatcherMultiple{updateCh: testutils.NewChannelWithSize(size)} +} + +func (lw *listenerWatcherMultiple) ResourceChanged(update *xdsresource.ListenerUpdate, onDone func()) { + lw.updateCh.Send(listenerUpdateErrTuple{update: update}) + onDone() +} + +func (lw *listenerWatcherMultiple) ResourceError(err error, onDone func()) { + lw.updateCh.Send(listenerUpdateErrTuple{err: err}) + onDone() +} + +func (lw *listenerWatcherMultiple) AmbientError(err error, onDone func()) { + lw.updateCh.Send(listenerUpdateErrTuple{err: err}) + onDone() +} + +// badListenerResource returns a listener resource for the given name which does +// not contain the `RouteSpecifier` field in the HTTPConnectionManager, and +// hence is expected to be NACKed by the client. +func badListenerResource(t *testing.T, name string) *v3listenerpb.Listener { + hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, + }) + return &v3listenerpb.Listener{ + Name: name, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + } +} + +// verifyNoListenerUpdate verifies that no listener update is received on the +// provided update channel, and returns an error if an update is received. +// +// A very short deadline is used while waiting for the update, as this function +// is intended to be used when an update is not expected. +func verifyNoListenerUpdate(ctx context.Context, updateCh *testutils.Channel) error { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if u, err := updateCh.Receive(sCtx); err != context.DeadlineExceeded { + return fmt.Errorf("unexpected ListenerUpdate: %v", u) + } + return nil +} + +// verifyListenerUpdate waits for an update to be received on the provided +// update channel and verifies that it matches the expected update. +// +// Returns an error if no update is received before the context deadline expires +// or the received update does not match the expected one. +func verifyListenerUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate listenerUpdateErrTuple) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for a listener resource from the management server: %v", err) + } + got := u.(listenerUpdateErrTuple) + if wantUpdate.err != nil { + if got.err == nil || !strings.Contains(got.err.Error(), wantUpdate.err.Error()) { + return fmt.Errorf("update received with error: %v, want %q", got.err, wantUpdate.err) + } + } + cmpOpts := []cmp.Option{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(xdsresource.HTTPFilter{}, "Filter", "Config"), + cmpopts.IgnoreFields(xdsresource.ListenerUpdate{}, "Raw"), + } + if diff := cmp.Diff(wantUpdate.update, got.update, cmpOpts...); diff != "" { + return fmt.Errorf("received unexpected diff in the listener resource update: (-want, got):\n%s", diff) + } + return nil +} + +func verifyErrorType(ctx context.Context, updateCh *testutils.Channel, wantErrType xdsresource.ErrorType, wantNodeID string) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for a listener error from the management server: %v", err) + } + gotErr := u.(listenerUpdateErrTuple).err + if got, want := xdsresource.ErrType(gotErr), wantErrType; got != want { + return fmt.Errorf("update received with error %v of type: %v, want %v", gotErr, got, want) + } + if !strings.Contains(gotErr.Error(), wantNodeID) { + return fmt.Errorf("update received with error: %v, want error with node ID: %q", gotErr, wantNodeID) + } + return nil +} + +// TestLDSWatch covers the case where a single watcher exists for a single +// listener resource. The test verifies the following scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of the watch callback. +// 2. An update from the management server containing a resource *not* being +// watched should not result in the invocation of the watch callback. +// 3. After the watch is cancelled, an update from the management server +// containing the resource that was being watched should not result in the +// invocation of the watch callback. +// +// The test is run for old and new style names. +func (s) TestLDSWatch(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3listenerpb.Listener // The resource being watched. + updatedWatchedResource *v3listenerpb.Listener // The watched resource after an update. + notWatchedResource *v3listenerpb.Listener // A resource which is not being watched. + wantUpdate listenerUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: ldsName, + watchedResource: e2e.DefaultClientListener(ldsName, rdsName), + updatedWatchedResource: e2e.DefaultClientListener(ldsName, "new-rds-resource"), + notWatchedResource: e2e.DefaultClientListener("unsubscribed-lds-resource", rdsName), + wantUpdate: listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + }, + }, + { + desc: "new style resource", + resourceName: ldsNameNewStyle, + watchedResource: e2e.DefaultClientListener(ldsNameNewStyle, rdsNameNewStyle), + updatedWatchedResource: e2e.DefaultClientListener(ldsNameNewStyle, "new-rds-resource"), + notWatchedResource: e2e.DefaultClientListener("unsubscribed-lds-resource", rdsNameNewStyle), + wantUpdate: listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: rdsNameNewStyle, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + lw := newListenerWatcher() + ldsCancel := xdsresource.WatchListener(client, test.resourceName, lw) + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyListenerUpdate(ctx, lw.updateCh, test.wantUpdate); err != nil { + t.Fatal(err) + } + + // Configure the management server to return an additional listener + // resource, one that we are not interested in. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.watchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoListenerUpdate(ctx, lw.updateCh); err != nil { + t.Fatal(err) + } + + // Cancel the watch and update the resource corresponding to the original + // watch. Ensure that the cancelled watch callback is not invoked. + ldsCancel() + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.updatedWatchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoListenerUpdate(ctx, lw.updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestLDSWatch_TwoWatchesForSameResourceName covers the case where two watchers +// exist for a single listener resource. The test verifies the following +// scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of both watch callbacks. +// 2. After one of the watches is cancelled, a redundant update from the +// management server should not result in the invocation of either of the +// watch callbacks. +// 3. An update from the management server containing the resource being +// watched should result in the invocation of the un-cancelled watch +// callback. +// +// The test is run for old and new style names. +func (s) TestLDSWatch_TwoWatchesForSameResourceName(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3listenerpb.Listener // The resource being watched. + updatedWatchedResource *v3listenerpb.Listener // The watched resource after an update. + wantUpdateV1 listenerUpdateErrTuple + wantUpdateV2 listenerUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: ldsName, + watchedResource: e2e.DefaultClientListener(ldsName, rdsName), + updatedWatchedResource: e2e.DefaultClientListener(ldsName, "new-rds-resource"), + wantUpdateV1: listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + }, + wantUpdateV2: listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: "new-rds-resource", + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + }, + }, + { + desc: "new style resource", + resourceName: ldsNameNewStyle, + watchedResource: e2e.DefaultClientListener(ldsNameNewStyle, rdsNameNewStyle), + updatedWatchedResource: e2e.DefaultClientListener(ldsNameNewStyle, "new-rds-resource"), + wantUpdateV1: listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: rdsNameNewStyle, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + }, + wantUpdateV2: listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: "new-rds-resource", + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same listener resource and have the + // callbacks push the received updates on to a channel. + lw1 := newListenerWatcher() + ldsCancel1 := xdsresource.WatchListener(client, test.resourceName, lw1) + defer ldsCancel1() + lw2 := newListenerWatcher() + ldsCancel2 := xdsresource.WatchListener(client, test.resourceName, lw2) + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered watches for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyListenerUpdate(ctx, lw1.updateCh, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + + // Cancel the second watch and force the management server to push a + // redundant update for the resource being watched. Neither of the + // two watch callbacks should be invoked. + ldsCancel2() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoListenerUpdate(ctx, lw1.updateCh); err != nil { + t.Fatal(err) + } + if err := verifyNoListenerUpdate(ctx, lw2.updateCh); err != nil { + t.Fatal(err) + } + + // Update to the resource being watched. The un-cancelled callback + // should be invoked while the cancelled one should not be. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.updatedWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, test.wantUpdateV2); err != nil { + t.Fatal(err) + } + if err := verifyNoListenerUpdate(ctx, lw2.updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestLDSWatch_ThreeWatchesForDifferentResourceNames covers the case with three +// watchers (two watchers for one resource, and the third watcher for another +// resource), exist across two listener resources. The test verifies that an +// update from the management server containing both resources results in the +// invocation of all watch callbacks. +// +// The test is run with both old and new style names. +func (s) TestLDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same listener resource and have the + // callbacks push the received updates on to a channel. + lw1 := newListenerWatcher() + ldsCancel1 := xdsresource.WatchListener(client, ldsName, lw1) + defer ldsCancel1() + lw2 := newListenerWatcher() + ldsCancel2 := xdsresource.WatchListener(client, ldsName, lw2) + defer ldsCancel2() + + // Register the third watch for a different listener resource. + ldsNameNewStyle := makeNewStyleLDSName(authority) + lw3 := newListenerWatcher() + ldsCancel3 := xdsresource.WatchListener(client, ldsNameNewStyle, lw3) + defer ldsCancel3() + + // Configure the management server to return two listener resources, + // corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(ldsName, rdsName), + e2e.DefaultClientListener(ldsNameNewStyle, rdsName), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the all watchers. The two + // resources returned differ only in the resource name. Therefore the + // expected update is the same for all the watchers. + wantUpdate := listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, lw3.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_ResourceCaching covers the case where a watch is registered for +// a resource which is already present in the cache. The test verifies that the +// watch callback is invoked with the contents from the cache, instead of a +// request being sent to the management server. +func (s) TestLDSWatch_ResourceCaching(t *testing.T) { + firstRequestReceived := false + firstAckReceived := grpcsync.NewEvent() + secondRequestReceived := grpcsync.NewEvent() + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + // The first request has an empty version string. + if !firstRequestReceived && req.GetVersionInfo() == "" { + firstRequestReceived = true + return nil + } + // The first ack has a non-empty version string. + if !firstAckReceived.HasFired() && req.GetVersionInfo() != "" { + firstAckReceived.Fire() + return nil + } + // Any requests after the first request and ack, are not expected. + secondRequestReceived.Fire() + return nil + }, + }) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + lw1 := newListenerWatcher() + ldsCancel1 := xdsresource.WatchListener(client, ldsName, lw1) + defer ldsCancel1() + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + t.Fatal("timeout when waiting for receipt of ACK at the management server") + case <-firstAckReceived.Done(): + } + + // Register another watch for the same resource. This should get the update + // from the cache. + lw2 := newListenerWatcher() + ldsCancel2 := xdsresource.WatchListener(client, ldsName, lw2) + defer ldsCancel2() + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + // No request should get sent out as part of this watch. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-secondRequestReceived.Done(): + t.Fatal("xdsClient sent out request instead of using update from cache") + } +} + +// TestLDSWatch_ExpiryTimerFiresBeforeResponse tests the case where the client +// does not receive an LDS response for the request that it sends. The test +// verifies that the watch callback is invoked with an error once the +// watchExpiryTimer fires. +func (s) TestLDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client talking to the above management server. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Register a watch for a resource which is expected to fail with an error + // after the watch expiry timer fires. + lw := newListenerWatcher() + ldsCancel := xdsresource.WatchListener(client, ldsName, lw) + defer ldsCancel() + + // Wait for the watch expiry timer to fire. + <-time.After(defaultTestWatchExpiryTimeout) + + // Verify that an empty update with the expected error is received. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantErr := xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, "") + if err := verifyListenerUpdate(ctx, lw.updateCh, listenerUpdateErrTuple{err: wantErr}); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_ValidResponseCancelsExpiryTimerBehavior tests the case where the +// client receives a valid LDS response for the request that it sends. The test +// verifies that the behavior associated with the expiry timer (i.e, callback +// invocation with error) does not take place. +func (s) TestLDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client talking to the above management server. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + lw := newListenerWatcher() + ldsCancel := xdsresource.WatchListener(client, ldsName, lw) + defer ldsCancel() + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Wait for the watch expiry timer to fire, and verify that the callback is + // not invoked. + <-time.After(defaultTestWatchExpiryTimeout) + if err := verifyNoListenerUpdate(ctx, lw.updateCh); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_ResourceRemoved covers the cases where a resource being watched +// is removed from the management server. The test verifies the following +// scenarios: +// 1. Removing a resource should trigger the watch callback with a resource +// removed error. It should not trigger the watch callback for an unrelated +// resource. +// 2. An update to another resource should result in the invocation of the watch +// callback associated with that resource. It should not result in the +// invocation of the watch callback associated with the deleted resource. +// +// The test is run with both old and new style names. +func (s) TestLDSWatch_ResourceRemoved(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for two listener resources and have the + // callbacks push the received updates on to a channel. + resourceName1 := ldsName + lw1 := newListenerWatcher() + ldsCancel1 := xdsresource.WatchListener(client, resourceName1, lw1) + defer ldsCancel1() + + resourceName2 := makeNewStyleLDSName(authority) + lw2 := newListenerWatcher() + ldsCancel2 := xdsresource.WatchListener(client, resourceName2, lw2) + defer ldsCancel2() + + // Configure the management server to return two listener resources, + // corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(resourceName1, rdsName), + e2e.DefaultClientListener(resourceName2, rdsName), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for both watchers. The two + // resources returned differ only in the resource name. Therefore the + // expected update is the same for both watchers. + wantUpdate := listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Remove the first listener resource on the management server. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(resourceName2, rdsName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // The first watcher should receive a resource removed error, while the + // second watcher should not see an update. + if err := verifyListenerUpdate(ctx, lw1.updateCh, listenerUpdateErrTuple{ + err: xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, ""), + }); err != nil { + t.Fatal(err) + } + if err := verifyNoListenerUpdate(ctx, lw2.updateCh); err != nil { + t.Fatal(err) + } + + // Update the second listener resource on the management server. The first + // watcher should not see an update, while the second watcher should. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(resourceName2, "new-rds-resource")}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoListenerUpdate(ctx, lw1.updateCh); err != nil { + t.Fatal(err) + } + wantUpdate = listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: "new-rds-resource", + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_NewWatcherForRemovedResource covers the case where a new +// watcher registers for a resource that has been removed. The test verifies +// the following scenarios: +// 1. When a resource is deleted by the management server, any active +// watchers of that resource should be notified with a "resource removed" +// error through their watch callback. +// 2. If a new watcher attempts to register for a resource that has already +// been deleted, its watch callback should be immediately invoked with a +// "resource removed" error. +func (s) TestLDSWatch_NewWatcherForRemovedResource(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register watch for the listener resource and have the + // callbacks push the received updates on to a channel. + lw1 := newListenerWatcher() + ldsCancel1 := xdsresource.WatchListener(client, ldsName, lw1) + defer ldsCancel1() + + // Configure the management server to return listener resource, + // corresponding to the registered watch. + resource := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resource); err != nil { + t.Fatalf("Failed to update management server with resource: %v, err: %v", resource, err) + } + + // Verify the contents of the received update for existing watch. + wantUpdate := listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Remove the listener resource on the management server. + resource = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resource); err != nil { + t.Fatalf("Failed to update management server with resource: %v, err: %v", resource, err) + } + + // The existing watcher should receive a resource removed error. + updateError := listenerUpdateErrTuple{err: xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, "")} + if err := verifyListenerUpdate(ctx, lw1.updateCh, updateError); err != nil { + t.Fatal(err) + } + + // New watchers attempting to register for a deleted resource should also + // receive a "resource removed" error. + lw2 := newListenerWatcher() + ldsCancel2 := xdsresource.WatchListener(client, ldsName, lw2) + defer ldsCancel2() + if err := verifyListenerUpdate(ctx, lw2.updateCh, updateError); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_NACKError covers the case where an update from the management +// server is NACKed by the xdsclient. The test verifies that the error is +// propagated to the existing watcher. After NACK, if a new watcher registers +// for the resource, error is propagated to the new watcher as well. +func (s) TestLDSWatch_NACKError(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + lw := newListenerWatcher() + ldsCancel := xdsresource.WatchListener(client, ldsName, lw) + defer ldsCancel() + + // Configure the management server to return a single listener resource + // which is expected to be NACKed by the client. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{badListenerResource(t, ldsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the existing watcher. + if err := verifyErrorType(ctx, lw.updateCh, xdsresource.ErrorTypeUnknown, nodeID); err != nil { + t.Fatal(err) + } + + // Verify that the expected error is propagated to the new watcher as well. + lw2 := newListenerWatcher() + ldsCancel2 := xdsresource.WatchListener(client, ldsName, lw2) + defer ldsCancel2() + if err := verifyErrorType(ctx, lw2.updateCh, xdsresource.ErrorTypeUnknown, nodeID); err != nil { + t.Fatal(err) + } +} + +// Tests the scenario where a watch registered for a resource results in a good +// update followed by a bad update. This results in the resource cache +// containing both the old good update and the latest NACK error. The test +// verifies that a when a new watch is registered for the same resource, the new +// watcher receives the good update followed by the NACK error. +func (s) TestLDSWatch_ResourceCaching_NACKError(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + lw1 := newListenerWatcher() + ldsCancel1 := xdsresource.WatchListener(client, ldsName, lw1) + defer ldsCancel1() + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), 1000*defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Configure the management server to return a single listener resource + // which is expected to be NACKed by the client. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{badListenerResource(t, ldsName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the existing watcher. + if err := verifyErrorType(ctx, lw1.updateCh, xdsresource.ErrorTypeUnknown, nodeID); err != nil { + t.Fatal(err) + } + + // Register another watch for the same resource. This should get the update + // and error from the cache. + lw2 := newListenerWatcherMultiple(2) + ldsCancel2 := xdsresource.WatchListener(client, ldsName, lw2) + defer ldsCancel2() + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + // Verify that the expected error is propagated to the existing watcher. + if err := verifyErrorType(ctx, lw2.updateCh, xdsresource.ErrorTypeUnknown, nodeID); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_PartialValid covers the case where a response from the +// management server contains both valid and invalid resources and is expected +// to be NACKed by the xdsclient. The test verifies that watchers corresponding +// to the valid resource receive the update, while watchers corresponding to the +// invalid resource receive an error. +func (s) TestLDSWatch_PartialValid(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for listener resources. The first watch is expected + // to receive an error because the received resource is NACKed. The second + // watch is expected to get a good update. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + badResourceName := ldsName + lw1 := newListenerWatcher() + ldsCancel1 := xdsresource.WatchListener(client, badResourceName, lw1) + defer ldsCancel1() + goodResourceName := makeNewStyleLDSName(authority) + lw2 := newListenerWatcher() + ldsCancel2 := xdsresource.WatchListener(client, goodResourceName, lw2) + defer ldsCancel2() + + // Configure the management server with two listener resources. One of these + // is a bad resource causing the update to be NACKed. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + badListenerResource(t, badResourceName), + e2e.DefaultClientListener(goodResourceName, rdsName), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher which + // requested for the bad resource. + // Verify that the expected error is propagated to the existing watcher. + if err := verifyErrorType(ctx, lw1.updateCh, xdsresource.ErrorTypeUnknown, nodeID); err != nil { + t.Fatal(err) + } + + // Verify that the watcher watching the good resource receives a good + // update. + wantUpdate := listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_PartialResponse covers the case where a response from the +// management server does not contain all requested resources. LDS responses are +// supposed to contain all requested resources, and the absence of one usually +// indicates that the management server does not know about it. In cases where +// the server has never responded with this resource before, the xDS client is +// expected to wait for the watch timeout to expire before concluding that the +// resource does not exist on the server +func (s) TestLDSWatch_PartialResponse(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for two listener resources and have the + // callbacks push the received updates on to a channel. + resourceName1 := ldsName + lw1 := newListenerWatcher() + ldsCancel1 := xdsresource.WatchListener(client, resourceName1, lw1) + defer ldsCancel1() + + resourceName2 := makeNewStyleLDSName(authority) + lw2 := newListenerWatcher() + ldsCancel2 := xdsresource.WatchListener(client, resourceName2, lw2) + defer ldsCancel2() + + // Configure the management server to return only one of the two listener + // resources, corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(resourceName1, rdsName), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for first watcher. + wantUpdate1 := listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate1); err != nil { + t.Fatal(err) + } + + // Verify that the second watcher does not get an update with an error. + if err := verifyNoListenerUpdate(ctx, lw2.updateCh); err != nil { + t.Fatal(err) + } + + // Configure the management server to return two listener resources, + // corresponding to the registered watches. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(resourceName1, rdsName), + e2e.DefaultClientListener(resourceName2, rdsName), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the second watcher. + wantUpdate2 := listenerUpdateErrTuple{ + update: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + } + if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate2); err != nil { + t.Fatal(err) + } + + // Verify that the first watcher gets no update, as the first resource did + // not change. + if err := verifyNoListenerUpdate(ctx, lw1.updateCh); err != nil { + t.Fatal(err) + } +} diff --git a/internal/xds/xdsclient/tests/loadreport_test.go b/internal/xds/xdsclient/tests/loadreport_test.go new file mode 100644 index 0000000..f43931c --- /dev/null +++ b/internal/xds/xdsclient/tests/loadreport_test.go @@ -0,0 +1,566 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "encoding/json" + "fmt" + "net" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/fakeserver" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/testing/protocmp" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/protobuf/types/known/durationpb" +) + +const ( + testKey1 = "test-key1" + testKey2 = "test-key2" +) + +var ( + testLocality1 = clients.Locality{Region: "test-region1"} + testLocality2 = clients.Locality{Region: "test-region2"} + toleranceCmpOpt = cmpopts.EquateApprox(0, 1e-5) + ignoreOrderCmpOpt = protocmp.FilterField(&v3endpointpb.ClusterStats{}, "upstream_locality_stats", + cmpopts.SortSlices(func(a, b protocmp.Message) bool { + return a.String() < b.String() + }), + ) +) + +type wrappedListener struct { + net.Listener + newConnChan *testutils.Channel // Connection attempts are pushed here. +} + +func (wl *wrappedListener) Accept() (net.Conn, error) { + c, err := wl.Listener.Accept() + if err != nil { + return nil, err + } + wl.newConnChan.Send(struct{}{}) + return c, err +} + +// Tests a load reporting scenario where the xDS client is reporting loads to +// multiple servers. Verifies the following: +// - calling the load reporting API with different server configuration +// results in connections being created to those corresponding servers +// - the same load.Store is not returned when the load reporting API called +// with different server configurations +// - canceling the load reporting from the client results in the LRS stream +// being canceled on the server +func (s) TestReportLoad_ConnectionCreation(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create two management servers that also serve LRS. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create a local TCP listener: %v", err) + } + newConnChan1 := testutils.NewChannel() + lis1 := &wrappedListener{ + Listener: l, + newConnChan: newConnChan1, + } + mgmtServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis1, + SupportLoadReportingService: true, + }) + l, err = testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create a local TCP listener: %v", err) + } + newConnChan2 := testutils.NewChannel() + lis2 := &wrappedListener{ + Listener: l, + newConnChan: newConnChan2, + } + mgmtServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis2, + SupportLoadReportingService: true, + }) + + // Create an xDS client with a bootstrap configuration that contains both of + // the above two servers. The authority name is immaterial here since load + // reporting is per-server and not per-authority. + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer1.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + "test-authority": []byte(fmt.Sprintf(`{ + "xds_servers": [{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]}`, mgmtServer2.Address)), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + client := createXDSClient(t, bc) + + serverCfg1, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: mgmtServer1.Address}) + if err != nil { + t.Fatalf("Failed to create server config for testing: %v", err) + } + // Call the load reporting API to report load to the first management + // server, and ensure that a connection to the server is created. + store1, lrsCancel1 := client.ReportLoad(serverCfg1) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + defer lrsCancel1(sCtx) + if _, err := newConnChan1.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for a connection to the first management server, after starting load reporting") + } + if _, err := mgmtServer1.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for LRS stream to be created") + } + + serverCfg2, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: mgmtServer2.Address}) + if err != nil { + t.Fatalf("Failed to create server config for testing: %v", err) + } + // Call the load reporting API to report load to the second management + // server, and ensure that a connection to the server is created. + store2, lrsCancel2 := client.ReportLoad(serverCfg2) + sCtx2, sCancel2 := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel2() + defer lrsCancel2(sCtx2) + if _, err := newConnChan2.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for a connection to the second management server, after starting load reporting") + } + if _, err := mgmtServer2.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for LRS stream to be created") + } + + if store1 == store2 { + t.Fatalf("Got same store for different servers, want different") + } + + // Push some loads on the received store. + store2.ReporterForCluster("cluster", "eds").CallDropped("test") + + // Ensure the initial load reporting request is received at the server. + lrsServer := mgmtServer2.LRSServer + req, err := lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for initial LRS request: %v", err) + } + gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) + nodeProto := &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"}, + } + wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto} + if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff) + } + + // Send a response from the server with a small deadline. + lrsServer.LRSResponseChan <- &fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + SendAllClusters: true, + LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms + }, + } + + // Ensure that loads are seen on the server. + req, err = lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for LRS request with loads: %v", err) + } + gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if l := len(gotLoad); l != 1 { + t.Fatalf("Received load for %d clusters, want 1", l) + } + + // This field is set by the client to indicate the actual time elapsed since + // the last report was sent. We cannot deterministically compare this, and + // we cannot use the cmpopts.IgnoreFields() option on proto structs, since + // we already use the protocmp.Transform() which marshals the struct into + // another message. Hence setting this field to nil is the best option here. + gotLoad[0].LoadReportInterval = nil + wantLoad := &v3endpointpb.ClusterStats{ + ClusterName: "cluster", + ClusterServiceName: "eds", + TotalDroppedRequests: 1, + DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}}, + } + if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" { + t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff) + } + + // Cancel this load reporting stream, server should see error canceled. + sCtx2, sCancel2 = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel2() + lrsCancel2(sCtx2) + + // Server should receive a stream canceled error. There may be additional + // load reports from the client in the channel. + for { + if ctx.Err() != nil { + t.Fatal("Timeout when waiting for the LRS stream to be canceled on the server") + } + u, err := lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + continue + } + // Ignore load reports sent before the stream was cancelled. + if u.(*fakeserver.Request).Err == nil { + continue + } + if status.Code(u.(*fakeserver.Request).Err) != codes.Canceled { + t.Fatalf("Unexpected LRS request: %v, want error canceled", u) + } + break + } +} + +// Tests a load reporting scenario where the load reporting API is called +// multiple times for the same server. The test verifies the following: +// - calling the load reporting API the second time for the same server +// configuration does not create a new LRS stream +// - the LRS stream is closed *only* after all the API calls invoke their +// cancel functions +// - creating new streams after the previous one was closed works +func (s) TestReportLoad_StreamCreation(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create a management server that serves LRS. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true}) + + // Create an xDS client with bootstrap pointing to the above server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + client := createXDSClient(t, bc) + + // Call the load reporting API, and ensure that an LRS stream is created. + serverConfig, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: mgmtServer.Address}) + if err != nil { + t.Fatalf("Failed to create server config for testing: %v", err) + } + store1, cancel1 := client.ReportLoad(serverConfig) + lrsServer := mgmtServer.LRSServer + if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err) + } + + // Push some loads on the received store. + store1.ReporterForCluster("cluster1", "eds1").CallDropped("test") + store1.ReporterForCluster("cluster1", "eds1").CallStarted(testLocality1) + store1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, testKey1, 3.14) + store1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, testKey1, 2.718) + store1.ReporterForCluster("cluster1", "eds1").CallFinished(testLocality1, nil) + store1.ReporterForCluster("cluster1", "eds1").CallStarted(testLocality2) + store1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality2, testKey2, 1.618) + store1.ReporterForCluster("cluster1", "eds1").CallFinished(testLocality2, nil) + + // Ensure the initial load reporting request is received at the server. + req, err := lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for initial LRS request: %v", err) + } + gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) + nodeProto := &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"}, + } + wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto} + if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff) + } + + // Send a response from the server with a small deadline. + lrsServer.LRSResponseChan <- &fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + SendAllClusters: true, + LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms + }, + } + + // Ensure that loads are seen on the server. + req, err = lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for LRS request with loads") + } + gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if l := len(gotLoad); l != 1 { + t.Fatalf("Received load for %d clusters, want 1", l) + } + + // This field is set by the client to indicate the actual time elapsed since + // the last report was sent. We cannot deterministically compare this, and + // we cannot use the cmpopts.IgnoreFields() option on proto structs, since + // we already use the protocmp.Transform() which marshals the struct into + // another message. Hence setting this field to nil is the best option here. + gotLoad[0].LoadReportInterval = nil + wantLoad := &v3endpointpb.ClusterStats{ + ClusterName: "cluster1", + ClusterServiceName: "eds1", + TotalDroppedRequests: 1, + DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}}, + UpstreamLocalityStats: []*v3endpointpb.UpstreamLocalityStats{ + { + Locality: &v3corepb.Locality{Region: "test-region1"}, + LoadMetricStats: []*v3endpointpb.EndpointLoadMetricStats{ + // TotalMetricValue is the aggregation of 3.14 + 2.718 = 5.858 + {MetricName: testKey1, NumRequestsFinishedWithMetric: 2, TotalMetricValue: 5.858}}, + TotalSuccessfulRequests: 1, + TotalIssuedRequests: 1, + }, + { + Locality: &v3corepb.Locality{Region: "test-region2"}, + LoadMetricStats: []*v3endpointpb.EndpointLoadMetricStats{ + {MetricName: testKey2, NumRequestsFinishedWithMetric: 1, TotalMetricValue: 1.618}}, + TotalSuccessfulRequests: 1, + TotalIssuedRequests: 1, + }, + }, + } + if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" { + t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff) + } + + // Make another call to the load reporting API, and ensure that a new LRS + // stream is not created. + store2, cancel2 := client.ReportLoad(serverConfig) + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := lrsServer.LRSStreamOpenChan.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("New LRS stream created when expected to use an existing one") + } + + // Push more loads. + store2.ReporterForCluster("cluster2", "eds2").CallDropped("test") + + // Ensure that loads are seen on the server. We need a loop here because + // there could have been some requests from the client in the time between + // us reading the first request and now. Those would have been queued in the + // request channel that we read out of. + for { + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for new loads to be seen on the server") + } + + req, err = lrsServer.LRSRequestChan.Receive(ctx) + if err != nil { + continue + } + gotLoad = req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if l := len(gotLoad); l != 1 { + continue + } + gotLoad[0].LoadReportInterval = nil + wantLoad := &v3endpointpb.ClusterStats{ + ClusterName: "cluster2", + ClusterServiceName: "eds2", + TotalDroppedRequests: 1, + DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}}, + } + if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform()); diff != "" { + t.Logf("Unexpected diff in LRS request (-got, +want):\n%s", diff) + continue + } + break + } + + // Cancel the first load reporting call, and ensure that the stream does not + // close (because we have another call open). + sCtx1, sCancel1 := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel1() + cancel1(sCtx1) + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := lrsServer.LRSStreamCloseChan.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("LRS stream closed when expected to stay open") + } + + // Cancel the second load reporting call, and ensure the stream is closed. + sCtx2, sCancel2 := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel2() + cancel2(sCtx2) + if _, err := lrsServer.LRSStreamCloseChan.Receive(ctx); err != nil { + t.Fatal("Timeout waiting for LRS stream to close") + } + + // Calling the load reporting API again should result in the creation of a + // new LRS stream. This ensures that creating and closing multiple streams + // works smoothly. + _, cancel3 := client.ReportLoad(serverConfig) + if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err) + } + sCtx3, sCancel3 := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel3() + cancel3(sCtx3) +} + +// TestConcurrentReportLoad verifies that the client can safely handle concurrent +// requests to initiate load reporting streams. It launches multiple goroutines +// that all call client.ReportLoad simultaneously. We don't verify anything on +// the server here, since that is done in other tests. This is just to ensure +// that concurrent ReportLoad() calls work. +func (s) TestConcurrentReportLoad(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true}) + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + client := createXDSClient(t, bc) + + serverConfig, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: mgmtServer.Address}) + if err != nil { + t.Fatalf("Failed to create server config for testing: %v", err) + } + + // Call ReportLoad() concurrently from multiple go routines. + const numGoroutines = 10 + cancelStores := make([]func(context.Context), numGoroutines) + var wg sync.WaitGroup + wg.Add(numGoroutines) + for i := range numGoroutines { + go func(idx int) { + defer wg.Done() + _, cancelStores[idx] = client.ReportLoad(serverConfig) + }(i) + } + wg.Wait() + + // Cancel all the load reporting streams. The last call to cancel is + // expected to block until a final load report with pending loads is sent. + // But the stream is currently blocked on a recv() call waiting for the LRS + // server to send the initial laod reporting response, which it never does. + // Hence we use a context with short timeout here. + for i, cancel := range cancelStores { + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + cancel(sCtx) + t.Logf("Cancelled LRS stream %d", i) + } +} + +// TestConcurrentChannels verifies that we can create multiple gRPC channels +// concurrently with a shared XDSClient, each of which will create a new LRS +// stream without any race. +func (s) TestConcurrentChannels(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true, SupportLoadReportingService: true}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + if internal.NewXDSResolverWithPoolForTesting == nil { + t.Fatalf("internal.NewXDSResolverWithConfigForTesting is nil") + } + + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + + resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error)) + xdsResolver, err := resolverBuilder(pool) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure the management server with resources that enable LRS. + const serviceName = "my-service-e2e-lrs-test" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + var wg sync.WaitGroup + const ( + numGoroutines = 10 + numRPCs = 10 + ) + for range numGoroutines { + wg.Add(1) + go func() { + defer wg.Done() + for range numRPCs { + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Errorf("grpc.NewClient() failed: %v", err) + return + } + defer cc.Close() + + testClient := testgrpc.NewTestServiceClient(cc) + if _, err := testClient.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("EmptyCall() failed: %v", err) + return + } + } + }() + } + wg.Wait() +} diff --git a/internal/xds/xdsclient/tests/rds_watchers_test.go b/internal/xds/xdsclient/tests/rds_watchers_test.go new file mode 100644 index 0000000..c5b7478 --- /dev/null +++ b/internal/xds/xdsclient/tests/rds_watchers_test.go @@ -0,0 +1,996 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +type noopRouteConfigWatcher struct{} + +func (noopRouteConfigWatcher) ResourceChanged(_ *xdsresource.RouteConfigUpdate, onDone func()) { + onDone() +} +func (noopRouteConfigWatcher) ResourceError(_ error, onDone func()) { + onDone() +} +func (noopRouteConfigWatcher) AmbientError(_ error, onDone func()) { + onDone() +} + +type routeConfigUpdateErrTuple struct { + update xdsresource.RouteConfigUpdate + err error +} + +type routeConfigWatcher struct { + updateCh *testutils.Channel +} + +func newRouteConfigWatcher() *routeConfigWatcher { + return &routeConfigWatcher{updateCh: testutils.NewChannel()} +} + +func (rw *routeConfigWatcher) ResourceChanged(update *xdsresource.RouteConfigUpdate, onDone func()) { + rw.updateCh.Send(routeConfigUpdateErrTuple{update: *update}) + onDone() +} + +func (rw *routeConfigWatcher) ResourceError(err error, onDone func()) { + // When used with a go-control-plane management server that continuously + // resends resources which are NACKed by the xDS client, using a `Replace()` + // here and in AmbientError() simplifies tests which will have + // access to the most recently received error. + rw.updateCh.Replace(routeConfigUpdateErrTuple{err: err}) + onDone() +} + +func (rw *routeConfigWatcher) AmbientError(err error, onDone func()) { + rw.updateCh.Replace(routeConfigUpdateErrTuple{err: err}) + onDone() +} + +// badRouteConfigResource returns a RouteConfiguration resource for the given +// routeName which contains a retry config with num_retries set to `0`. This is +// expected to be NACK'ed by the xDS client. +func badRouteConfigResource(routeName, ldsTarget, clusterName string) *v3routepb.RouteConfiguration { + return &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }}}}, + RetryPolicy: &v3routepb.RetryPolicy{ + NumRetries: &wrapperspb.UInt32Value{Value: 0}, + }, + }}, + } +} + +// xdsClient is expected to produce an error containing this string when an +// update is received containing a route configuration resource created using +// `badRouteConfigResource`. +const wantRouteConfigNACKErr = "received route is invalid: retry_policy.num_retries = 0; must be >= 1" + +// verifyRouteConfigUpdate waits for an update to be received on the provided +// update channel and verifies that it matches the expected update. +// +// Returns an error if no update is received before the context deadline expires +// or the received update does not match the expected one. +func verifyRouteConfigUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate routeConfigUpdateErrTuple) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for a route configuration resource from the management server: %v", err) + } + got := u.(routeConfigUpdateErrTuple) + if wantUpdate.err != nil { + if got.err == nil || !strings.Contains(got.err.Error(), wantUpdate.err.Error()) { + return fmt.Errorf("update received with error: %v, want %q", got.err, wantUpdate.err) + } + } + cmpOpts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.RouteConfigUpdate{}, "Raw")} + if diff := cmp.Diff(wantUpdate.update, got.update, cmpOpts...); diff != "" { + return fmt.Errorf("received unexpected diff in the route configuration resource update: (-want, got):\n%s", diff) + } + return nil +} + +// verifyNoRouteConfigUpdate verifies that no route configuration update is +// received on the provided update channel, and returns an error if an update is +// received. +// +// A very short deadline is used while waiting for the update, as this function +// is intended to be used when an update is not expected. +func verifyNoRouteConfigUpdate(ctx context.Context, updateCh *testutils.Channel) error { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if u, err := updateCh.Receive(sCtx); err != context.DeadlineExceeded { + return fmt.Errorf("unexpected RouteConfigUpdate: %v", u) + } + return nil +} + +// TestRDSWatch covers the case where a single watcher exists for a single route +// configuration resource. The test verifies the following scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of the watch callback. +// 2. An update from the management server containing a resource *not* being +// watched should not result in the invocation of the watch callback. +// 3. After the watch is cancelled, an update from the management server +// containing the resource that was being watched should not result in the +// invocation of the watch callback. +// +// The test is run for old and new style names. +func (s) TestRDSWatch(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3routepb.RouteConfiguration // The resource being watched. + updatedWatchedResource *v3routepb.RouteConfiguration // The watched resource after an update. + notWatchedResource *v3routepb.RouteConfiguration // A resource which is not being watched. + wantUpdate routeConfigUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: rdsName, + watchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, cdsName), + updatedWatchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, "new-cds-resource"), + notWatchedResource: e2e.DefaultRouteConfig("unsubscribed-rds-resource", ldsName, cdsName), + wantUpdate: routeConfigUpdateErrTuple{ + update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}}, + }, + }, + }, + }, + }, + }, + }, + { + desc: "new style resource", + resourceName: rdsNameNewStyle, + watchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, cdsNameNewStyle), + updatedWatchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, "new-cds-resource"), + notWatchedResource: e2e.DefaultRouteConfig("unsubscribed-rds-resource", ldsNameNewStyle, cdsNameNewStyle), + wantUpdate: routeConfigUpdateErrTuple{ + update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsNameNewStyle}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsNameNewStyle, Weight: 100}}, + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a route configuration resource and have the + // watch callback push the received update on to a channel. + rw := newRouteConfigWatcher() + rdsCancel := xdsresource.WatchRouteConfig(client, test.resourceName, rw) + + // Configure the management server to return a single route + // configuration resource, corresponding to the one being watched. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyRouteConfigUpdate(ctx, rw.updateCh, test.wantUpdate); err != nil { + t.Fatal(err) + } + + // Configure the management server to return an additional route + // configuration resource, one that we are not interested in. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{test.watchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoRouteConfigUpdate(ctx, rw.updateCh); err != nil { + t.Fatal(err) + } + + // Cancel the watch and update the resource corresponding to the original + // watch. Ensure that the cancelled watch callback is not invoked. + rdsCancel() + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{test.updatedWatchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoRouteConfigUpdate(ctx, rw.updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestRDSWatch_TwoWatchesForSameResourceName covers the case where two watchers +// exist for a single route configuration resource. The test verifies the +// following scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of both watch callbacks. +// 2. After one of the watches is cancelled, a redundant update from the +// management server should not result in the invocation of either of the +// watch callbacks. +// 3. An update from the management server containing the resource being +// watched should result in the invocation of the un-cancelled watch +// callback. +// +// The test is run for old and new style names. +func (s) TestRDSWatch_TwoWatchesForSameResourceName(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3routepb.RouteConfiguration // The resource being watched. + updatedWatchedResource *v3routepb.RouteConfiguration // The watched resource after an update. + wantUpdateV1 routeConfigUpdateErrTuple + wantUpdateV2 routeConfigUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: rdsName, + watchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, cdsName), + updatedWatchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, "new-cds-resource"), + wantUpdateV1: routeConfigUpdateErrTuple{ + update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}}, + }, + }, + }, + }, + }, + }, + wantUpdateV2: routeConfigUpdateErrTuple{ + update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: []xdsresource.WeightedCluster{{Name: "new-cds-resource", Weight: 100}}, + }, + }, + }, + }, + }, + }, + }, + { + desc: "new style resource", + resourceName: rdsNameNewStyle, + watchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, cdsNameNewStyle), + updatedWatchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, "new-cds-resource"), + wantUpdateV1: routeConfigUpdateErrTuple{ + update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsNameNewStyle}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsNameNewStyle, Weight: 100}}, + }, + }, + }, + }, + }, + }, + wantUpdateV2: routeConfigUpdateErrTuple{ + update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsNameNewStyle}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: []xdsresource.WeightedCluster{{Name: "new-cds-resource", Weight: 100}}, + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp resource names used in this test do not specify an + // authority. These will end up looking up an entry with the + // empty key in the authorities map. Having an entry with an + // empty key and empty configuration, results in these + // resources also using the top-level configuration. + "": []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same route configuration resource + // and have the callbacks push the received updates on to a channel. + rw1 := newRouteConfigWatcher() + rdsCancel1 := xdsresource.WatchRouteConfig(client, test.resourceName, rw1) + defer rdsCancel1() + rw2 := newRouteConfigWatcher() + rdsCancel2 := xdsresource.WatchRouteConfig(client, test.resourceName, rw2) + + // Configure the management server to return a single route + // configuration resource, corresponding to the one being watched. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + + // Cancel the second watch and force the management server to push a + // redundant update for the resource being watched. Neither of the + // two watch callbacks should be invoked. + rdsCancel2() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoRouteConfigUpdate(ctx, rw1.updateCh); err != nil { + t.Fatal(err) + } + if err := verifyNoRouteConfigUpdate(ctx, rw2.updateCh); err != nil { + t.Fatal(err) + } + + // Update to the resource being watched. The un-cancelled callback + // should be invoked while the cancelled one should not be. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{test.updatedWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, test.wantUpdateV2); err != nil { + t.Fatal(err) + } + if err := verifyNoRouteConfigUpdate(ctx, rw2.updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestRDSWatch_ThreeWatchesForDifferentResourceNames covers the case with three +// watchers (two watchers for one resource, and the third watcher for another +// resource), exist across two route configuration resources. The test verifies +// that an update from the management server containing both resources results +// in the invocation of all watch callbacks. +// +// The test is run with both old and new style names. +func (s) TestRDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same route configuration resource + // and have the callbacks push the received updates on to a channel. + rw1 := newRouteConfigWatcher() + rdsCancel1 := xdsresource.WatchRouteConfig(client, rdsName, rw1) + defer rdsCancel1() + rw2 := newRouteConfigWatcher() + rdsCancel2 := xdsresource.WatchRouteConfig(client, rdsName, rw2) + defer rdsCancel2() + + // Register the third watch for a different route configuration resource. + rdsNameNewStyle := makeNewStyleRDSName(authority) + rw3 := newRouteConfigWatcher() + rdsCancel3 := xdsresource.WatchRouteConfig(client, rdsNameNewStyle, rw3) + defer rdsCancel3() + + // Configure the management server to return two route configuration + // resources, corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{ + e2e.DefaultRouteConfig(rdsName, ldsName, cdsName), + e2e.DefaultRouteConfig(rdsNameNewStyle, ldsName, cdsName), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the all watchers. The two + // resources returned differ only in the resource name. Therefore the + // expected update is the same for all the watchers. + wantUpdate := routeConfigUpdateErrTuple{ + update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}}, + }, + }, + }, + }, + }, + } + if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyRouteConfigUpdate(ctx, rw3.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestRDSWatch_ResourceCaching covers the case where a watch is registered for +// a resource which is already present in the cache. The test verifies that the +// watch callback is invoked with the contents from the cache, instead of a +// request being sent to the management server. +func (s) TestRDSWatch_ResourceCaching(t *testing.T) { + firstRequestReceived := false + firstAckReceived := grpcsync.NewEvent() + secondRequestReceived := grpcsync.NewEvent() + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + // The first request has an empty version string. + if !firstRequestReceived && req.GetVersionInfo() == "" { + firstRequestReceived = true + return nil + } + // The first ack has a non-empty version string. + if !firstAckReceived.HasFired() && req.GetVersionInfo() != "" { + firstAckReceived.Fire() + return nil + } + // Any requests after the first request and ack, are not expected. + secondRequestReceived.Fire() + return nil + }, + }) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a route configuration resource and have the watch + // callback push the received update on to a channel. + rw1 := newRouteConfigWatcher() + rdsCancel1 := xdsresource.WatchRouteConfig(client, rdsName, rw1) + defer rdsCancel1() + + // Configure the management server to return a single route configuration + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, cdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := routeConfigUpdateErrTuple{ + update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}}, + }, + }, + }, + }, + }, + } + if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + t.Fatal("timeout when waiting for receipt of ACK at the management server") + case <-firstAckReceived.Done(): + } + + // Register another watch for the same resource. This should get the update + // from the cache. + rw2 := newRouteConfigWatcher() + rdsCancel2 := xdsresource.WatchRouteConfig(client, rdsName, rw2) + defer rdsCancel2() + if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + // No request should get sent out as part of this watch. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-secondRequestReceived.Done(): + t.Fatal("xdsClient sent out request instead of using update from cache") + } +} + +// TestRDSWatch_ExpiryTimerFiresBeforeResponse tests the case where the client +// does not receive an RDS response for the request that it sends. The test +// verifies that the watch callback is invoked with an error once the +// watchExpiryTimer fires. +func (s) TestRDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client talking to the above management server. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Register a watch for a resource which is expected to fail with an error + // after the watch expiry timer fires. + rw := newRouteConfigWatcher() + rdsCancel := xdsresource.WatchRouteConfig(client, rdsName, rw) + defer rdsCancel() + + // Wait for the watch expiry timer to fire. + <-time.After(defaultTestWatchExpiryTimeout) + + // Verify that an empty update with the expected error is received. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantErr := xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, "") + if err := verifyRouteConfigUpdate(ctx, rw.updateCh, routeConfigUpdateErrTuple{err: wantErr}); err != nil { + t.Fatal(err) + } +} + +// TestRDSWatch_ValidResponseCancelsExpiryTimerBehavior tests the case where the +// client receives a valid RDS response for the request that it sends. The test +// verifies that the behavior associated with the expiry timer (i.e, callback +// invocation with error) does not take place. +func (s) TestRDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client talking to the above management server. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Register a watch for a route configuration resource and have the watch + // callback push the received update on to a channel. + rw := newRouteConfigWatcher() + rdsCancel := xdsresource.WatchRouteConfig(client, rdsName, rw) + defer rdsCancel() + + // Configure the management server to return a single route configuration + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, cdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := routeConfigUpdateErrTuple{ + update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}}, + }, + }, + }, + }, + }, + } + if err := verifyRouteConfigUpdate(ctx, rw.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Wait for the watch expiry timer to fire, and verify that the callback is + // not invoked. + <-time.After(defaultTestWatchExpiryTimeout) + if err := verifyNoRouteConfigUpdate(ctx, rw.updateCh); err != nil { + t.Fatal(err) + } +} + +// TestRDSWatch_NACKError covers the case where an update from the management +// server is NACK'ed by the xdsclient. The test verifies that the error is +// propagated to the watcher. +func (s) TestRDSWatch_NACKError(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a route configuration resource and have the watch + // callback push the received update on to a channel. + rw := newRouteConfigWatcher() + rdsCancel := xdsresource.WatchRouteConfig(client, rdsName, rw) + defer rdsCancel() + + // Configure the management server to return a single route configuration + // resource which is expected to be NACKed by the client. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{badRouteConfigResource(rdsName, ldsName, cdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher. + u, err := rw.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for a route configuration resource from the management server: %v", err) + } + gotErr := u.(routeConfigUpdateErrTuple).err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantRouteConfigNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantRouteConfigNACKErr) + } +} + +// TestRDSWatch_PartialValid covers the case where a response from the +// management server contains both valid and invalid resources and is expected +// to be NACK'ed by the xdsclient. The test verifies that watchers corresponding +// to the valid resource receive the update, while watchers corresponding to the +// invalid resource receive an error. +func (s) TestRDSWatch_PartialValid(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + authority := makeAuthorityName(t.Name()) + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + Authorities: map[string]json.RawMessage{ + // Xdstp style resource names used in this test use a slash removed + // version of t.Name as their authority, and the empty config + // results in the top-level xds server configuration being used for + // this authority. + authority: []byte(`{}`), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for route configuration resources. The first watch + // is expected to receive an error because the received resource is NACKed. + // The second watch is expected to get a good update. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + badResourceName := rdsName + rw1 := newRouteConfigWatcher() + rdsCancel1 := xdsresource.WatchRouteConfig(client, badResourceName, rw1) + defer rdsCancel1() + goodResourceName := makeNewStyleRDSName(authority) + rw2 := newRouteConfigWatcher() + rdsCancel2 := xdsresource.WatchRouteConfig(client, goodResourceName, rw2) + defer rdsCancel2() + + // Configure the management server to return two route configuration + // resources, corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{ + badRouteConfigResource(badResourceName, ldsName, cdsName), + e2e.DefaultRouteConfig(goodResourceName, ldsName, cdsName), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher which + // requested for the bad resource. + u, err := rw1.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for a route configuration resource from the management server: %v", err) + } + gotErr := u.(routeConfigUpdateErrTuple).err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantRouteConfigNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantRouteConfigNACKErr) + } + + // Verify that the watcher watching the good resource receives a good + // update. + wantUpdate := routeConfigUpdateErrTuple{ + update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}}, + }, + }, + }, + }, + }, + } + if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} diff --git a/internal/xds/xdsclient/tests/resource_update_test.go b/internal/xds/xdsclient/tests/resource_update_test.go new file mode 100644 index 0000000..9416c61 --- /dev/null +++ b/internal/xds/xdsclient/tests/resource_update_test.go @@ -0,0 +1,1259 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/fakeserver" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/resolver" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter. +) + +// startFakeManagementServer starts a fake xDS management server and registers a +// cleanup function to close the fake server. +func startFakeManagementServer(t *testing.T) *fakeserver.Server { + t.Helper() + fs, cleanup, err := fakeserver.StartServer(nil) + if err != nil { + t.Fatalf("Failed to start fake xDS server: %v", err) + } + t.Logf("Started xDS management server on %s", fs.Address) + t.Cleanup(cleanup) + return fs +} + +func compareUpdateMetadata(ctx context.Context, dumpFunc func() *v3statuspb.ClientStatusResponse, want []*v3statuspb.ClientConfig_GenericXdsConfig) error { + var cmpOpts = cmp.Options{ + cmp.Transformer("sort", func(in []*v3statuspb.ClientConfig_GenericXdsConfig) []*v3statuspb.ClientConfig_GenericXdsConfig { + out := append([]*v3statuspb.ClientConfig_GenericXdsConfig(nil), in...) + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + if a == nil { + return true + } + if b == nil { + return false + } + if strings.Compare(a.TypeUrl, b.TypeUrl) == 0 { + return strings.Compare(a.Name, b.Name) < 0 + } + return strings.Compare(a.TypeUrl, b.TypeUrl) < 0 + }) + return out + }), + protocmp.Transform(), + protocmp.IgnoreFields((*v3statuspb.ClientConfig_GenericXdsConfig)(nil), "last_updated"), + protocmp.IgnoreFields((*v3adminpb.UpdateFailureState)(nil), "last_update_attempt", "details"), + } + + var lastErr error + for ; ctx.Err() == nil; <-time.After(100 * time.Millisecond) { + var got []*v3statuspb.ClientConfig_GenericXdsConfig + for _, cfg := range dumpFunc().GetConfig() { + got = append(got, cfg.GetGenericXdsConfigs()...) + } + diff := cmp.Diff(want, got, cmpOpts) + if diff == "" { + return nil + } + lastErr = fmt.Errorf("unexpected diff in metadata, diff (-want +got):\n%s\n want: %+v\n got: %+v", diff, want, got) + } + return fmt.Errorf("timeout when waiting for expected update metadata: %v", lastErr) +} + +// TestHandleListenerResponseFromManagementServer covers different scenarios +// involving receipt of an LDS response from the management server. The test +// verifies that the internal state of the xDS client (parsed resource and +// metadata) matches expectations. +func (s) TestHandleListenerResponseFromManagementServer(t *testing.T) { + const ( + resourceName1 = "resource-name-1" + resourceName2 = "resource-name-2" + ) + var ( + emptyRouterFilter = e2e.RouterHTTPFilter + apiListener = &v3listenerpb.ApiListener{ + ApiListener: func() *anypb.Any { + return testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-configuration-name", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }) + }(), + } + resource1 = &v3listenerpb.Listener{ + Name: resourceName1, + ApiListener: apiListener, + } + resource2 = &v3listenerpb.Listener{ + Name: resourceName2, + ApiListener: apiListener, + } + ) + + tests := []struct { + desc string + resourceName string + managementServerResponse *v3discoverypb.DiscoveryResponse + wantUpdate *xdsresource.ListenerUpdate + wantErr string + wantGenericXDSConfig []*v3statuspb.ClientConfig_GenericXdsConfig + }{ + { + desc: "badly-marshaled-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + Resources: []*anypb.Any{{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Value: []byte{1, 2, 3, 4}, + }}, + }, + wantErr: fmt.Sprintf("xds: resource %q of type %q has been removed", resourceName1, "ListenerResource"), + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_DOES_NOT_EXIST, + }, + }, + }, + { + desc: "empty-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + }, + wantErr: fmt.Sprintf("xds: resource %q of type %q has been removed", resourceName1, "ListenerResource"), + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_DOES_NOT_EXIST, + }, + }, + }, + { + desc: "unexpected-type-in-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, &v3routepb.RouteConfiguration{})}, + }, + wantErr: fmt.Sprintf("xds: resource %q of type %q has been removed", resourceName1, "ListenerResource"), + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_DOES_NOT_EXIST, + }, + }, + }, + { + desc: "one-bad-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: resourceName1, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{}), + }}), + }, + }, + wantErr: "no RouteSpecifier", + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_NACKED, + ErrorState: &v3adminpb.UpdateFailureState{ + VersionInfo: "1", + }, + }, + }, + }, + { + desc: "one-good-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, resource1)}, + }, + wantUpdate: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: "route-configuration-name", + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_ACKED, + VersionInfo: "1", + XdsConfig: testutils.MarshalAny(t, resource1), + }, + }, + }, + { + desc: "two-resources-when-we-requested-one", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, resource1), testutils.MarshalAny(t, resource2)}, + }, + wantUpdate: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: "route-configuration-name", + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_ACKED, + VersionInfo: "1", + XdsConfig: testutils.MarshalAny(t, resource1), + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // Create a fake xDS management server listening on a local port, + // and set it up with the response to send. + mgmtServer := startFakeManagementServer(t) + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Register a watch, and push the results on to a channel. + lw := newListenerWatcher() + cancel := xdsresource.WatchListener(client, test.resourceName, lw) + defer cancel() + t.Logf("Registered a watch for Listener %q", test.resourceName) + + // Wait for the discovery request to be sent out. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := mgmtServer.XDSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for discovery request at the management server: %v", ctx) + } + wantReq := &fakeserver.Request{Req: &v3discoverypb.DiscoveryRequest{ + Node: &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + ClientFeatures: []string{ + "envoy.lb.does_not_support_overprovisioning", + "xds.config.resource-in-sotw", + }, + }, + ResourceNames: []string{test.resourceName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + }} + gotReq := val.(*fakeserver.Request) + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), protocmp.IgnoreFields(&v3corepb.Node{}, "user_agent_version")); diff != "" { + t.Fatalf("Discovery request received with unexpected diff (-got +want):\n%s\n got: %+v, want: %+v", diff, gotReq, wantReq) + } + t.Logf("Discovery request received at management server") + + // Configure the fake management server with a response. + mgmtServer.XDSResponseChan <- &fakeserver.Response{Resp: test.managementServerResponse} + + // Wait for an update from the xDS client and compare with expected + // update. + val, err = lw.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for watch callback to invoked after response from management server: %v", err) + } + gotUpdate := val.(listenerUpdateErrTuple).update + gotErr := val.(listenerUpdateErrTuple).err + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + cmpOpts := []cmp.Option{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(xdsresource.HTTPFilter{}, "Filter", "Config"), + cmpopts.IgnoreFields(xdsresource.ListenerUpdate{}, "Raw"), + } + if diff := cmp.Diff(test.wantUpdate, gotUpdate, cmpOpts...); diff != "" { + t.Fatalf("Unexpected diff in metadata, diff (-want +got):\n%s", diff) + } + if err := compareUpdateMetadata(ctx, pool.DumpResources, test.wantGenericXDSConfig); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestHandleRouteConfigResponseFromManagementServer covers different scenarios +// involving receipt of an RDS response from the management server. The test +// verifies that the internal state of the xDS client (parsed resource and +// metadata) matches expectations. +func (s) TestHandleRouteConfigResponseFromManagementServer(t *testing.T) { + const ( + resourceName1 = "resource-name-1" + resourceName2 = "resource-name-2" + ) + var ( + virtualHosts = []*v3routepb.VirtualHost{ + { + Domains: []string{"lds-target-name"}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: "cluster-name"}, + }, + }, + }, + }, + }, + } + resource1 = &v3routepb.RouteConfiguration{ + Name: resourceName1, + VirtualHosts: virtualHosts, + } + resource2 = &v3routepb.RouteConfiguration{ + Name: resourceName2, + VirtualHosts: virtualHosts, + } + ) + + tests := []struct { + desc string + resourceName string + managementServerResponse *v3discoverypb.DiscoveryResponse + wantUpdate xdsresource.RouteConfigUpdate + wantErr string + wantGenericXDSConfig []*v3statuspb.ClientConfig_GenericXdsConfig + }{ + // The first three tests involve scenarios where the response fails + // protobuf deserialization (because it contains an invalid data or type + // in the anypb.Any) or the requested resource is not present in the + // response. In either case, no resource update makes its way to the + // top-level xDS client. An RDS response without a requested resource + // does not mean that the resource does not exist in the server. It + // could be part of a future update. Therefore, the only failure mode + // for this resource is for the watch to timeout. + { + desc: "badly-marshaled-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + Resources: []*anypb.Any{{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Value: []byte{1, 2, 3, 4}, + }}, + }, + wantErr: fmt.Sprintf("xds: resource %q of type %q has been removed", resourceName1, "RouteConfigResource"), + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_DOES_NOT_EXIST, + }, + }, + }, + { + desc: "empty-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + }, + wantErr: fmt.Sprintf("xds: resource %q of type %q has been removed", resourceName1, "RouteConfigResource"), + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_DOES_NOT_EXIST, + }, + }, + }, + { + desc: "unexpected-type-in-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, &v3clusterpb.Cluster{})}, + }, + wantErr: fmt.Sprintf("xds: resource %q of type %q has been removed", resourceName1, "RouteConfigResource"), + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_DOES_NOT_EXIST, + }, + }, + }, + { + desc: "one-bad-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, &v3routepb.RouteConfiguration{ + Name: resourceName1, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"lds-resource-name"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: "cluster-resource-name"}, + }}}}, + RetryPolicy: &v3routepb.RetryPolicy{ + NumRetries: &wrapperspb.UInt32Value{Value: 0}, + }, + }}, + })}, + }, + wantErr: "received route is invalid: retry_policy.num_retries = 0; must be >= 1", + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_NACKED, + ErrorState: &v3adminpb.UpdateFailureState{ + VersionInfo: "1", + }, + }, + }, + }, + { + desc: "one-good-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, resource1)}, + }, + wantUpdate: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{"lds-target-name"}, + Routes: []*xdsresource.Route{{Prefix: newStringP(""), + WeightedClusters: []xdsresource.WeightedCluster{{Name: "cluster-name", Weight: 1}}, + ActionType: xdsresource.RouteActionRoute}}, + }, + }, + }, + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_ACKED, + VersionInfo: "1", + XdsConfig: testutils.MarshalAny(t, resource1), + }, + }, + }, + { + desc: "two-resources-when-we-requested-one", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, resource1), testutils.MarshalAny(t, resource2)}, + }, + wantUpdate: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{"lds-target-name"}, + Routes: []*xdsresource.Route{{Prefix: newStringP(""), + WeightedClusters: []xdsresource.WeightedCluster{{Name: "cluster-name", Weight: 1}}, + ActionType: xdsresource.RouteActionRoute}}, + }, + }, + }, + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_ACKED, + VersionInfo: "1", + XdsConfig: testutils.MarshalAny(t, resource1), + }, + }, + }, + } + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // Create a fake xDS management server listening on a local port, + // and set it up with the response to send. + mgmtServer := startFakeManagementServer(t) + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Register a watch, and push the results on to a channel. + rw := newRouteConfigWatcher() + cancel := xdsresource.WatchRouteConfig(client, test.resourceName, rw) + defer cancel() + t.Logf("Registered a watch for Route Configuration %q", test.resourceName) + + // Wait for the discovery request to be sent out. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := mgmtServer.XDSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for discovery request at the management server: %v", ctx) + } + wantReq := &fakeserver.Request{Req: &v3discoverypb.DiscoveryRequest{ + Node: &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + ClientFeatures: []string{ + "envoy.lb.does_not_support_overprovisioning", + "xds.config.resource-in-sotw", + }, + }, + ResourceNames: []string{test.resourceName}, + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + }} + gotReq := val.(*fakeserver.Request) + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), protocmp.IgnoreFields(&v3corepb.Node{}, "user_agent_version")); diff != "" { + t.Fatalf("Discovery request received with unexpected diff (-got +want):\n%s\n got: %+v, want: %+v", diff, gotReq, wantReq) + } + t.Logf("Discovery request received at management server") + + // Configure the fake management server with a response. + mgmtServer.XDSResponseChan <- &fakeserver.Response{Resp: test.managementServerResponse} + + // Wait for an update from the xDS client and compare with expected + // update. + val, err = rw.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for watch callback to invoked after response from management server: %v", err) + } + gotUpdate := val.(routeConfigUpdateErrTuple).update + gotErr := val.(routeConfigUpdateErrTuple).err + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + cmpOpts := []cmp.Option{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(xdsresource.RouteConfigUpdate{}, "Raw"), + } + if diff := cmp.Diff(test.wantUpdate, gotUpdate, cmpOpts...); diff != "" { + t.Fatalf("Unexpected diff in metadata, diff (-want +got):\n%s", diff) + } + if err := compareUpdateMetadata(ctx, pool.DumpResources, test.wantGenericXDSConfig); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestHandleClusterResponseFromManagementServer covers different scenarios +// involving receipt of a CDS response from the management server. The test +// verifies that the internal state of the xDS client (parsed resource and +// metadata) matches expectations. +func (s) TestHandleClusterResponseFromManagementServer(t *testing.T) { + const ( + resourceName1 = "resource-name-1" + resourceName2 = "resource-name-2" + ) + resource1 := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: resourceName1, + ServiceName: "eds-service-name", + EnableLRS: true, + }) + resource2 := proto.Clone(resource1).(*v3clusterpb.Cluster) + resource2.Name = resourceName2 + + tests := []struct { + desc string + resourceName string + managementServerResponse *v3discoverypb.DiscoveryResponse + wantUpdate xdsresource.ClusterUpdate + wantErr string + wantGenericXDSConfig []*v3statuspb.ClientConfig_GenericXdsConfig + }{ + { + desc: "badly-marshaled-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + VersionInfo: "1", + Resources: []*anypb.Any{{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + Value: []byte{1, 2, 3, 4}, + }}, + }, + wantErr: fmt.Sprintf("xds: resource %q of type %q has been removed", resourceName1, "ClusterResource"), + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_DOES_NOT_EXIST, + }, + }, + }, + { + desc: "empty-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + VersionInfo: "1", + }, + wantErr: fmt.Sprintf("xds: resource %q of type %q has been removed", resourceName1, "ClusterResource"), + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_DOES_NOT_EXIST, + }, + }, + }, + { + desc: "unexpected-type-in-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, &v3endpointpb.ClusterLoadAssignment{})}, + }, + wantErr: fmt.Sprintf("xds: resource %q of type %q has been removed", resourceName1, "ClusterResource"), + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_DOES_NOT_EXIST, + }, + }, + }, + { + desc: "one-bad-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, &v3clusterpb.Cluster{ + Name: resourceName1, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: "eds-service-name", + }, + LbPolicy: v3clusterpb.Cluster_MAGLEV, + })}, + }, + wantErr: "unexpected lbPolicy MAGLEV", + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_NACKED, + ErrorState: &v3adminpb.UpdateFailureState{ + VersionInfo: "1", + }, + }, + }, + }, + { + desc: "one-good-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, resource1)}, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: "resource-name-1", + EDSServiceName: "eds-service-name", + }, + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_ACKED, + VersionInfo: "1", + XdsConfig: testutils.MarshalAny(t, resource1), + }, + }, + }, + { + desc: "two-resources-when-we-requested-one", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, resource1), testutils.MarshalAny(t, resource2)}, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: "resource-name-1", + EDSServiceName: "eds-service-name", + }, + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_ACKED, + VersionInfo: "1", + XdsConfig: testutils.MarshalAny(t, resource1), + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // Create a fake xDS management server listening on a local port, + // and set it up with the response to send. + mgmtServer := startFakeManagementServer(t) + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Register a watch, and push the results on to a channel. + cw := newClusterWatcher() + cancel := xdsresource.WatchCluster(client, test.resourceName, cw) + defer cancel() + t.Logf("Registered a watch for Cluster %q", test.resourceName) + + // Wait for the discovery request to be sent out. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := mgmtServer.XDSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for discovery request at the management server: %v", ctx) + } + wantReq := &fakeserver.Request{Req: &v3discoverypb.DiscoveryRequest{ + Node: &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + ClientFeatures: []string{ + "envoy.lb.does_not_support_overprovisioning", + "xds.config.resource-in-sotw", + }, + }, + ResourceNames: []string{test.resourceName}, + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + }} + gotReq := val.(*fakeserver.Request) + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), protocmp.IgnoreFields(&v3corepb.Node{}, "user_agent_version")); diff != "" { + t.Fatalf("Discovery request received with unexpected diff (-got +want):\n%s\n got: %+v, want: %+v", diff, gotReq, wantReq) + } + t.Logf("Discovery request received at management server") + + // Configure the fake management server with a response. + mgmtServer.XDSResponseChan <- &fakeserver.Response{Resp: test.managementServerResponse} + + // Wait for an update from the xDS client and compare with expected + // update. + val, err = cw.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for watch callback to invoked after response from management server: %v", err) + } + gotUpdate := val.(clusterUpdateErrTuple).update + gotErr := val.(clusterUpdateErrTuple).err + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + + // For tests expected to succeed, we expect an LRS server config in + // the update from the xDS client, because the LRS bit is turned on + // in the cluster resource. We *cannot* set the LRS server config in + // the test table because we do not have the address of the xDS + // server at that point, hence we do it here before verifying the + // received update. + if test.wantErr == "" { + serverCfg, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: fmt.Sprintf("passthrough:///%s", mgmtServer.Address), ServerFeatures: []string{"trusted_xds_server"}}) + if err != nil { + t.Fatalf("Failed to create server config for testing: %v", err) + } + test.wantUpdate.LRSServerConfig = serverCfg + } + cmpOpts := []cmp.Option{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(xdsresource.ClusterUpdate{}, "Raw", "LBPolicy", "TelemetryLabels"), + } + if diff := cmp.Diff(test.wantUpdate, gotUpdate, cmpOpts...); diff != "" { + t.Fatalf("Unexpected diff in metadata, diff (-want +got):\n%s", diff) + } + if err := compareUpdateMetadata(ctx, pool.DumpResources, test.wantGenericXDSConfig); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestHandleEndpointsResponseFromManagementServer covers different scenarios +// involving receipt of a CDS response from the management server. The test +// verifies that the internal state of the xDS client (parsed resource and +// metadata) matches expectations. +func (s) TestHandleEndpointsResponseFromManagementServer(t *testing.T) { + const ( + resourceName1 = "resource-name-1" + resourceName2 = "resource-name-2" + ) + resource1 := &v3endpointpb.ClusterLoadAssignment{ + ClusterName: resourceName1, + Endpoints: []*v3endpointpb.LocalityLbEndpoints{ + { + Locality: &v3corepb.Locality{SubZone: "locality-1"}, + LbEndpoints: []*v3endpointpb.LbEndpoint{ + { + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Protocol: v3corepb.SocketAddress_TCP, + Address: "addr1", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: uint32(314), + }, + }, + }, + }, + }, + }, + }, + }, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + Priority: 1, + }, + { + Locality: &v3corepb.Locality{SubZone: "locality-2"}, + LbEndpoints: []*v3endpointpb.LbEndpoint{ + { + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Protocol: v3corepb.SocketAddress_TCP, + Address: "addr2", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: uint32(159), + }, + }, + }, + }, + }, + }, + }, + }, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + Priority: 0, + }, + }, + } + resource2 := proto.Clone(resource1).(*v3endpointpb.ClusterLoadAssignment) + resource2.ClusterName = resourceName2 + + tests := []struct { + desc string + resourceName string + managementServerResponse *v3discoverypb.DiscoveryResponse + wantUpdate xdsresource.EndpointsUpdate + wantErr string + wantGenericXDSConfig []*v3statuspb.ClientConfig_GenericXdsConfig + }{ + // The first three tests involve scenarios where the response fails + // protobuf deserialization (because it contains an invalid data or type + // in the anypb.Any) or the requested resource is not present in the + // response. In either case, no resource update makes its way to the + // top-level xDS client. An EDS response without a requested resource + // does not mean that the resource does not exist in the server. It + // could be part of a future update. Therefore, the only failure mode + // for this resource is for the watch to timeout. + { + desc: "badly-marshaled-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + VersionInfo: "1", + Resources: []*anypb.Any{{ + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + Value: []byte{1, 2, 3, 4}, + }}, + }, + wantErr: fmt.Sprintf("xds: resource %q of type %q has been removed", resourceName1, "EndpointsResource"), + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_DOES_NOT_EXIST, + }, + }, + }, + { + desc: "empty-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + VersionInfo: "1", + }, + wantErr: fmt.Sprintf("xds: resource %q of type %q has been removed", resourceName1, "EndpointsResource"), + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_DOES_NOT_EXIST, + }, + }, + }, + { + desc: "unexpected-type-in-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, &v3listenerpb.Listener{})}, + }, + wantErr: fmt.Sprintf("xds: resource %q of type %q has been removed", resourceName1, "EndpointsResource"), + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_DOES_NOT_EXIST, + }, + }, + }, + { + desc: "one-bad-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, &v3endpointpb.ClusterLoadAssignment{ + ClusterName: resourceName1, + Endpoints: []*v3endpointpb.LocalityLbEndpoints{ + { + Locality: &v3corepb.Locality{SubZone: "locality-1"}, + LbEndpoints: []*v3endpointpb.LbEndpoint{ + { + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Protocol: v3corepb.SocketAddress_TCP, + Address: "addr1", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: uint32(314), + }, + }, + }, + }, + }, + }, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 0}, + }, + }, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + Priority: 1, + }, + }, + }), + }, + }, + wantErr: "EDS response contains an endpoint with zero weight", + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_NACKED, + ErrorState: &v3adminpb.UpdateFailureState{ + VersionInfo: "1", + }, + }, + }, + }, + { + desc: "one-good-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, resource1)}, + }, + wantUpdate: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "addr1:314"}}, + }, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 1, + Weight: 1, + }, + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "addr2:159"}}, + }, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-2"}, + Priority: 0, + Weight: 1, + }, + }, + }, + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_ACKED, + VersionInfo: "1", + XdsConfig: testutils.MarshalAny(t, resource1), + }, + }, + }, + { + desc: "two-resources-when-we-requested-one", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(t, resource1), testutils.MarshalAny(t, resource2)}, + }, + wantUpdate: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "addr1:314"}}, + }, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 1, + Weight: 1, + }, + { + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{ + Addresses: []resolver.Address{{Addr: "addr2:159"}}, + }, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-2"}, + Priority: 0, + Weight: 1, + }, + }, + }, + wantGenericXDSConfig: []*v3statuspb.ClientConfig_GenericXdsConfig{ + { + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + Name: resourceName1, + ClientStatus: v3adminpb.ClientResourceStatus_ACKED, + VersionInfo: "1", + XdsConfig: testutils.MarshalAny(t, resource1), + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // Create a fake xDS management server listening on a local port, + // and set it up with the response to send. + mgmtServer := startFakeManagementServer(t) + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + pool := xdsclient.NewPool(config) + client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + defer close() + + // Register a watch, and push the results on to a channel. + ew := newEndpointsWatcher() + cancel := xdsresource.WatchEndpoints(client, test.resourceName, ew) + defer cancel() + t.Logf("Registered a watch for Endpoint %q", test.resourceName) + + // Wait for the discovery request to be sent out. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := mgmtServer.XDSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for discovery request at the management server: %v", ctx) + } + wantReq := &fakeserver.Request{Req: &v3discoverypb.DiscoveryRequest{ + Node: &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + ClientFeatures: []string{ + "envoy.lb.does_not_support_overprovisioning", + "xds.config.resource-in-sotw", + }, + }, + ResourceNames: []string{test.resourceName}, + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + }} + gotReq := val.(*fakeserver.Request) + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), protocmp.IgnoreFields(&v3corepb.Node{}, "user_agent_version")); diff != "" { + t.Fatalf("Discovery request received with unexpected diff (-got +want):\n%s\n got: %+v, want: %+v", diff, gotReq, wantReq) + } + t.Logf("Discovery request received at management server") + + // Configure the fake management server with a response. + mgmtServer.XDSResponseChan <- &fakeserver.Response{Resp: test.managementServerResponse} + + // Wait for an update from the xDS client and compare with expected + // update. + val, err = ew.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for watch callback to invoked after response from management server: %v", err) + } + gotUpdate := val.(endpointsUpdateErrTuple).update + gotErr := val.(endpointsUpdateErrTuple).err + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + cmpOpts := []cmp.Option{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(xdsresource.EndpointsUpdate{}, "Raw"), + } + if diff := cmp.Diff(test.wantUpdate, gotUpdate, cmpOpts...); diff != "" { + t.Fatalf("Unexpected diff in metadata, diff (-want +got):\n%s", diff) + } + if err := compareUpdateMetadata(ctx, pool.DumpResources, test.wantGenericXDSConfig); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/xds/xdsclient/xdsclient_test.go b/internal/xds/xdsclient/xdsclient_test.go new file mode 100644 index 0000000..d7bb926 --- /dev/null +++ b/internal/xds/xdsclient/xdsclient_test.go @@ -0,0 +1,33 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "testing" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} diff --git a/internal/xds/xdsclient/xdslbregistry/converter/converter.go b/internal/xds/xdsclient/xdslbregistry/converter/converter.go new file mode 100644 index 0000000..11fcce2 --- /dev/null +++ b/internal/xds/xdsclient/xdslbregistry/converter/converter.go @@ -0,0 +1,252 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package converter provides converters to convert proto load balancing +// configuration, defined by the xDS API spec, to JSON load balancing +// configuration. These converters are registered by proto type in a registry, +// which gets pulled from based off proto type passed in. +package converter + +import ( + "encoding/json" + "fmt" + "strings" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/leastrequest" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/balancer/ringhash" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/balancer/weightedroundrobin" + iringhash "google.golang.org/grpc/internal/ringhash" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/xds/balancer/wrrlocality" + "google.golang.org/grpc/internal/xds/xdsclient/xdslbregistry" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3clientsideweightedroundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3" + v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3" + v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3" + v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3" + v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3" +) + +func init() { + xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.client_side_weighted_round_robin.v3.ClientSideWeightedRoundRobin", convertWeightedRoundRobinProtoToServiceConfig) + xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.ring_hash.v3.RingHash", convertRingHashProtoToServiceConfig) + xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.pick_first.v3.PickFirst", convertPickFirstProtoToServiceConfig) + xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.round_robin.v3.RoundRobin", convertRoundRobinProtoToServiceConfig) + xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.wrr_locality.v3.WrrLocality", convertWRRLocalityProtoToServiceConfig) + xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest", convertLeastRequestProtoToServiceConfig) + xdslbregistry.Register("type.googleapis.com/udpa.type.v1.TypedStruct", convertV1TypedStructToServiceConfig) + xdslbregistry.Register("type.googleapis.com/xds.type.v3.TypedStruct", convertV3TypedStructToServiceConfig) +} + +const ( + defaultRingHashMinSize = 1024 + defaultRingHashMaxSize = 8 * 1024 * 1024 // 8M + defaultLeastRequestChoiceCount = 2 +) + +func convertRingHashProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { + rhProto := &v3ringhashpb.RingHash{} + if err := proto.Unmarshal(rawProto, rhProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + if rhProto.GetHashFunction() != v3ringhashpb.RingHash_XX_HASH { + return nil, fmt.Errorf("unsupported ring_hash hash function %v", rhProto.GetHashFunction()) + } + + var minSize, maxSize uint64 = defaultRingHashMinSize, defaultRingHashMaxSize + if min := rhProto.GetMinimumRingSize(); min != nil { + minSize = min.GetValue() + } + if max := rhProto.GetMaximumRingSize(); max != nil { + maxSize = max.GetValue() + } + + rhCfg := &iringhash.LBConfig{ + MinRingSize: minSize, + MaxRingSize: maxSize, + } + + rhCfgJSON, err := json.Marshal(rhCfg) + if err != nil { + return nil, fmt.Errorf("error marshaling JSON for type %T: %v", rhCfg, err) + } + return makeBalancerConfigJSON(ringhash.Name, rhCfgJSON), nil +} + +type pfConfig struct { + ShuffleAddressList bool `json:"shuffleAddressList"` +} + +func convertPickFirstProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { + pfProto := &v3pickfirstpb.PickFirst{} + if err := proto.Unmarshal(rawProto, pfProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + + pfCfg := &pfConfig{ShuffleAddressList: pfProto.GetShuffleAddressList()} + js, err := json.Marshal(pfCfg) + if err != nil { + return nil, fmt.Errorf("error marshaling JSON for type %T: %v", pfCfg, err) + } + return makeBalancerConfigJSON(pickfirst.Name, js), nil +} + +func convertRoundRobinProtoToServiceConfig([]byte, int) (json.RawMessage, error) { + return makeBalancerConfigJSON(roundrobin.Name, json.RawMessage("{}")), nil +} + +type wrrLocalityLBConfig struct { + ChildPolicy json.RawMessage `json:"childPolicy,omitempty"` +} + +func convertWRRLocalityProtoToServiceConfig(rawProto []byte, depth int) (json.RawMessage, error) { + wrrlProto := &v3wrrlocalitypb.WrrLocality{} + if err := proto.Unmarshal(rawProto, wrrlProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + epJSON, err := xdslbregistry.ConvertToServiceConfig(wrrlProto.GetEndpointPickingPolicy(), depth+1) + if err != nil { + return nil, fmt.Errorf("error converting endpoint picking policy: %v for %+v", err, wrrlProto) + } + wrrLCfg := wrrLocalityLBConfig{ + ChildPolicy: epJSON, + } + + lbCfgJSON, err := json.Marshal(wrrLCfg) + if err != nil { + return nil, fmt.Errorf("error marshaling JSON for type %T: %v", wrrLCfg, err) + } + return makeBalancerConfigJSON(wrrlocality.Name, lbCfgJSON), nil +} + +func convertWeightedRoundRobinProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { + cswrrProto := &v3clientsideweightedroundrobinpb.ClientSideWeightedRoundRobin{} + if err := proto.Unmarshal(rawProto, cswrrProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + wrrLBCfg := &wrrLBConfig{} + // Only set fields if specified in proto. If not set, ParseConfig of the WRR + // will populate the config with defaults. + if enableOOBLoadReportCfg := cswrrProto.GetEnableOobLoadReport(); enableOOBLoadReportCfg != nil { + wrrLBCfg.EnableOOBLoadReport = enableOOBLoadReportCfg.GetValue() + } + if oobReportingPeriodCfg := cswrrProto.GetOobReportingPeriod(); oobReportingPeriodCfg != nil { + wrrLBCfg.OOBReportingPeriod = internalserviceconfig.Duration(oobReportingPeriodCfg.AsDuration()) + } + if blackoutPeriodCfg := cswrrProto.GetBlackoutPeriod(); blackoutPeriodCfg != nil { + wrrLBCfg.BlackoutPeriod = internalserviceconfig.Duration(blackoutPeriodCfg.AsDuration()) + } + if weightExpirationPeriodCfg := cswrrProto.GetWeightExpirationPeriod(); weightExpirationPeriodCfg != nil { + wrrLBCfg.WeightExpirationPeriod = internalserviceconfig.Duration(weightExpirationPeriodCfg.AsDuration()) + } + if weightUpdatePeriodCfg := cswrrProto.GetWeightUpdatePeriod(); weightUpdatePeriodCfg != nil { + wrrLBCfg.WeightUpdatePeriod = internalserviceconfig.Duration(weightUpdatePeriodCfg.AsDuration()) + } + if errorUtilizationPenaltyCfg := cswrrProto.GetErrorUtilizationPenalty(); errorUtilizationPenaltyCfg != nil { + wrrLBCfg.ErrorUtilizationPenalty = float64(errorUtilizationPenaltyCfg.GetValue()) + } + + lbCfgJSON, err := json.Marshal(wrrLBCfg) + if err != nil { + return nil, fmt.Errorf("error marshaling JSON for type %T: %v", wrrLBCfg, err) + } + return makeBalancerConfigJSON(weightedroundrobin.Name, lbCfgJSON), nil +} + +func convertLeastRequestProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { + lrProto := &v3leastrequestpb.LeastRequest{} + if err := proto.Unmarshal(rawProto, lrProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + // "The configuration for the Least Request LB policy is the + // least_request_lb_config field. The field is optional; if not present, + // defaults will be assumed for all of its values." - A48 + choiceCount := uint32(defaultLeastRequestChoiceCount) + if cc := lrProto.GetChoiceCount(); cc != nil { + choiceCount = cc.GetValue() + } + lrCfg := &leastrequest.LBConfig{ChoiceCount: choiceCount} + js, err := json.Marshal(lrCfg) + if err != nil { + return nil, fmt.Errorf("error marshaling JSON for type %T: %v", lrCfg, err) + } + return makeBalancerConfigJSON(leastrequest.Name, js), nil +} + +func convertV1TypedStructToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { + tsProto := &v1xdsudpatypepb.TypedStruct{} + if err := proto.Unmarshal(rawProto, tsProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + return convertCustomPolicy(tsProto.GetTypeUrl(), tsProto.GetValue()) +} + +func convertV3TypedStructToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { + tsProto := &v3xdsxdstypepb.TypedStruct{} + if err := proto.Unmarshal(rawProto, tsProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + return convertCustomPolicy(tsProto.GetTypeUrl(), tsProto.GetValue()) +} + +// convertCustomPolicy attempts to prepare json configuration for a custom lb +// proto, which specifies the gRPC balancer type and configuration. Returns the +// converted json and an error which should cause caller to error if error +// converting. If both json and error returned are nil, it means the gRPC +// Balancer registry does not contain that balancer type, and the caller should +// continue to the next policy. +func convertCustomPolicy(typeURL string, s *structpb.Struct) (json.RawMessage, error) { + // The gRPC policy name will be the "type name" part of the value of the + // type_url field in the TypedStruct. We get this by using the part after + // the last / character. Can assume a valid type_url from the control plane. + pos := strings.LastIndex(typeURL, "/") + name := typeURL[pos+1:] + + if balancer.Get(name) == nil { + return nil, nil + } + + rawJSON, err := json.Marshal(s) + if err != nil { + return nil, fmt.Errorf("error converting custom lb policy %v: %v for %+v", err, typeURL, s) + } + + // The Struct contained in the TypedStruct will be returned as-is as the + // configuration JSON object. + return makeBalancerConfigJSON(name, rawJSON), nil +} + +type wrrLBConfig struct { + EnableOOBLoadReport bool `json:"enableOobLoadReport,omitempty"` + OOBReportingPeriod internalserviceconfig.Duration `json:"oobReportingPeriod,omitempty"` + BlackoutPeriod internalserviceconfig.Duration `json:"blackoutPeriod,omitempty"` + WeightExpirationPeriod internalserviceconfig.Duration `json:"weightExpirationPeriod,omitempty"` + WeightUpdatePeriod internalserviceconfig.Duration `json:"weightUpdatePeriod,omitempty"` + ErrorUtilizationPenalty float64 `json:"errorUtilizationPenalty,omitempty"` +} + +func makeBalancerConfigJSON(name string, value json.RawMessage) []byte { + return []byte(fmt.Sprintf(`[{%q: %s}]`, name, value)) +} diff --git a/internal/xds/xdsclient/xdslbregistry/xdslbregistry.go b/internal/xds/xdsclient/xdslbregistry/xdslbregistry.go new file mode 100644 index 0000000..88ee9fa --- /dev/null +++ b/internal/xds/xdsclient/xdslbregistry/xdslbregistry.go @@ -0,0 +1,82 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xdslbregistry provides a registry of converters that convert proto +// from load balancing configuration, defined by the xDS API spec, to JSON load +// balancing configuration. +package xdslbregistry + +import ( + "encoding/json" + "fmt" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" +) + +var ( + // m is a map from proto type to Converter. + m = make(map[string]Converter) +) + +// Register registers the converter to the map keyed on a proto type. Must be +// called at init time. Not thread safe. +func Register(protoType string, c Converter) { + m[protoType] = c +} + +// SetRegistry sets the xDS LB registry. Must be called at init time. Not thread +// safe. +func SetRegistry(registry map[string]Converter) { + m = registry +} + +// Converter converts raw proto bytes into the internal Go JSON representation +// of the proto passed. Returns the json message, and an error. If both +// returned are nil, it represents continuing to the next proto. +type Converter func([]byte, int) (json.RawMessage, error) + +// ConvertToServiceConfig converts a proto Load Balancing Policy configuration +// into a json string. Returns an error if: +// - no supported policy found +// - there is more than 16 layers of recursion in the configuration +// - a failure occurs when converting the policy +func ConvertToServiceConfig(lbPolicy *v3clusterpb.LoadBalancingPolicy, depth int) (json.RawMessage, error) { + // "Configurations that require more than 16 levels of recursion are + // considered invalid and should result in a NACK response." - A51 + if depth > 15 { + return nil, fmt.Errorf("lb policy %v exceeds max depth supported: 16 layers", lbPolicy) + } + + // "This function iterate over the list of policy messages in + // LoadBalancingPolicy, attempting to convert each one to gRPC form, + // stopping at the first supported policy." - A52 + for _, policy := range lbPolicy.GetPolicies() { + converter := m[policy.GetTypedExtensionConfig().GetTypedConfig().GetTypeUrl()] + // "Any entry not in the above list is unsupported and will be skipped." + // - A52 + if converter == nil { + continue + } + json, err := converter(policy.GetTypedExtensionConfig().GetTypedConfig().GetValue(), depth) + if json == nil && err == nil { + continue + } + return json, err + } + return nil, fmt.Errorf("no supported policy found in policy list +%v", lbPolicy) +} diff --git a/internal/xds/xdsclient/xdslbregistry/xdslbregistry_test.go b/internal/xds/xdsclient/xdslbregistry/xdslbregistry_test.go new file mode 100644 index 0000000..0fbf0e7 --- /dev/null +++ b/internal/xds/xdsclient/xdslbregistry/xdslbregistry_test.go @@ -0,0 +1,459 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xdslbregistry_test contains test cases for the xDS LB Policy Registry. +package xdslbregistry_test + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + _ "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/pretty" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/balancer/wrrlocality" + "google.golang.org/grpc/internal/xds/xdsclient/xdslbregistry" + _ "google.golang.org/grpc/xds" // Register the xDS LB Registry Converters. + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3clientsideweightedroundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3" + v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3" + v3maglevpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/maglev/v3" + v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3" + v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3" + v3roundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/round_robin/v3" + v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func wrrLocalityBalancerConfig(childPolicy *internalserviceconfig.BalancerConfig) *internalserviceconfig.BalancerConfig { + return &internalserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: childPolicy, + }, + } +} + +func (s) TestConvertToServiceConfigSuccess(t *testing.T) { + const customLBPolicyName = "myorg.MyCustomLeastRequestPolicy" + stub.Register(customLBPolicyName, stub.BalancerFuncs{}) + + tests := []struct { + name string + policy *v3clusterpb.LoadBalancingPolicy + wantConfig string // JSON config + lrEnabled bool + }{ + { + name: "ring_hash", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_XX_HASH, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }), + }, + }, + }, + }, + wantConfig: `[{"ring_hash_experimental": { "minRingSize": 10, "maxRingSize": 100 }}]`, + }, + { + name: "least_request", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3leastrequestpb.LeastRequest{ + ChoiceCount: wrapperspb.UInt32(3), + }), + }, + }, + }, + }, + wantConfig: `[{"least_request_experimental": { "choiceCount": 3 }}]`, + lrEnabled: true, + }, + { + name: "pick_first_shuffle", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3pickfirstpb.PickFirst{ + ShuffleAddressList: true, + }), + }, + }, + }, + }, + wantConfig: `[{"pick_first": { "shuffleAddressList": true }}]`, + }, + { + name: "pick_first", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3pickfirstpb.PickFirst{}), + }, + }, + }, + }, + wantConfig: `[{"pick_first": { "shuffleAddressList": false }}]`, + }, + { + name: "round_robin", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3roundrobinpb.RoundRobin{}), + }, + }, + }, + }, + wantConfig: `[{"round_robin": {}}]`, + }, + { + name: "weighted_round_robin", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3clientsideweightedroundrobinpb.ClientSideWeightedRoundRobin{}), + }, + }, + }, + }, + wantConfig: `[{"weighted_round_robin": {}}]`, + }, + { + name: "weighted_round_robin_populated", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3clientsideweightedroundrobinpb.ClientSideWeightedRoundRobin{ + EnableOobLoadReport: wrapperspb.Bool(true), + OobReportingPeriod: durationpb.New(10 * time.Second), + BlackoutPeriod: durationpb.New(5 * time.Second), + WeightExpirationPeriod: durationpb.New(3 * time.Minute), + WeightUpdatePeriod: durationpb.New(1 * time.Second), + ErrorUtilizationPenalty: wrapperspb.Float(1.5), + }), + }, + }, + }, + }, + wantConfig: `[{"weighted_round_robin": { + "enableOobLoadReport": true, + "oobReportingPeriod": "10s", + "blackoutPeriod": "5s", + "weightExpirationPeriod": "180s", + "weightUpdatePeriod": "1s", + "errorUtilizationPenalty": 1.5 + }}]`, + }, + { + name: "round_robin_ring_hash_use_first_supported", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3roundrobinpb.RoundRobin{}), + }, + }, + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_XX_HASH, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }), + }, + }, + }, + }, + wantConfig: `[{"round_robin": {}}]`, + }, + { + name: "pf_rr_use_pick_first", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3pickfirstpb.PickFirst{ + ShuffleAddressList: true, + }), + }, + }, + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3roundrobinpb.RoundRobin{}), + }, + }, + }, + }, + wantConfig: `[{"pick_first": { "shuffleAddressList": true }}]`, + }, + { + name: "custom_lb_type_v3_struct", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + // The type not registered in gRPC Policy registry. + // Should fallback to next policy in list. + TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/myorg.ThisTypeDoesNotExist", + Value: &structpb.Struct{}, + }), + }, + }, + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/myorg.MyCustomLeastRequestPolicy", + Value: &structpb.Struct{}, + }), + }, + }, + }, + }, + wantConfig: `[{"myorg.MyCustomLeastRequestPolicy": {}}]`, + }, + { + name: "custom_lb_type_v1_struct", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v1xdsudpatypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/myorg.MyCustomLeastRequestPolicy", + Value: &structpb.Struct{}, + }), + }, + }, + }, + }, + wantConfig: `[{"myorg.MyCustomLeastRequestPolicy": {}}]`, + }, + { + name: "wrr_locality_child_round_robin", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: wrrLocalityAny(t, &v3roundrobinpb.RoundRobin{}), + }, + }, + }, + }, + wantConfig: `[{"xds_wrr_locality_experimental": { "childPolicy": [{"round_robin": {}}] }}]`, + }, + { + name: "wrr_locality_child_custom_lb_type_v3_struct", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: wrrLocalityAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/myorg.MyCustomLeastRequestPolicy", + Value: &structpb.Struct{}, + }), + }, + }, + }, + }, + wantConfig: `[{"xds_wrr_locality_experimental": { "childPolicy": [{"myorg.MyCustomLeastRequestPolicy": {}}] }}]`, + }, + { + name: "on-the-boundary-of-recursive-limit", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: wrrLocalityAny(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, &v3roundrobinpb.RoundRobin{}))))))))))))))), + }, + }, + }, + }, + wantConfig: jsonMarshal(t, wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(&internalserviceconfig.BalancerConfig{ + Name: "round_robin", + })))))))))))))))), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + rawJSON, err := xdslbregistry.ConvertToServiceConfig(test.policy, 0) + if err != nil { + t.Fatalf("ConvertToServiceConfig(%s) failed: %v", pretty.ToJSON(test.policy), err) + } + // got and want must be unmarshalled since JSON strings shouldn't + // generally be directly compared. + var got []map[string]any + if err := json.Unmarshal(rawJSON, &got); err != nil { + t.Fatalf("Error unmarshalling rawJSON (%q): %v", rawJSON, err) + } + var want []map[string]any + if err := json.Unmarshal(json.RawMessage(test.wantConfig), &want); err != nil { + t.Fatalf("Error unmarshalling wantConfig (%q): %v", test.wantConfig, err) + } + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("ConvertToServiceConfig() got unexpected output, diff (-got +want): %v", diff) + } + }) + } +} + +func jsonMarshal(t *testing.T, x any) string { + t.Helper() + js, err := json.Marshal(x) + if err != nil { + t.Fatalf("Error marshalling to JSON (%+v): %v", x, err) + } + return string(js) +} + +// TestConvertToServiceConfigFailure tests failure cases of the xDS LB registry +// of converting proto configuration to JSON configuration. +func (s) TestConvertToServiceConfigFailure(t *testing.T) { + tests := []struct { + name string + policy *v3clusterpb.LoadBalancingPolicy + wantErr string + }{ + { + name: "not xx_hash function", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_MURMUR_HASH_2, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }), + }, + }, + }, + }, + wantErr: "unsupported ring_hash hash function", + }, + { + name: "no-supported-policy", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + // The type not registered in gRPC Policy registry. + TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/myorg.ThisTypeDoesNotExist", + Value: &structpb.Struct{}, + }), + }, + }, + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + // Maglev is not yet supported by gRPC. + TypedConfig: testutils.MarshalAny(t, &v3maglevpb.Maglev{}), + }, + }, + }, + }, + wantErr: "no supported policy found in policy list", + }, + { + name: "exceeds-boundary-of-recursive-limit-by-1", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: wrrLocalityAny(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, &v3roundrobinpb.RoundRobin{})))))))))))))))), + }, + }, + }, + }, + wantErr: "exceeds max depth", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, gotErr := xdslbregistry.ConvertToServiceConfig(test.policy, 0) + // Test the error substring to test the different root causes of + // errors. This is more brittle over time, but it's important to + // test the root cause of the errors emitted from the + // ConvertToServiceConfig function call. Also, this package owns the + // error strings so breakages won't come unexpectedly. + if gotErr == nil || !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("ConvertToServiceConfig() = %v, wantErr %v", gotErr, test.wantErr) + } + }) + } +} + +// wrrLocality is a helper that takes a proto message and returns a +// WrrLocalityProto with the proto message marshaled into a proto.Any as a +// child. +func wrrLocality(t *testing.T, m proto.Message) *v3wrrlocalitypb.WrrLocality { + return &v3wrrlocalitypb.WrrLocality{ + EndpointPickingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, m), + }, + }, + }, + }, + } +} + +// wrrLocalityAny takes a proto message and returns a wrr locality proto +// marshaled as an any with an any child set to the marshaled proto message. +func wrrLocalityAny(t *testing.T, m proto.Message) *anypb.Any { + return testutils.MarshalAny(t, wrrLocality(t, m)) +} diff --git a/internal/xds/xdsclient/xdsresource/cluster_resource_type.go b/internal/xds/xdsclient/xdsresource/cluster_resource_type.go new file mode 100644 index 0000000..af4e785 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/cluster_resource_type.go @@ -0,0 +1,140 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "bytes" + "fmt" + + "google.golang.org/grpc/internal/xds/bootstrap" + xdsclient "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" +) + +const ( + // ClusterResourceTypeName represents the transport agnostic name for the + // cluster resource. + ClusterResourceTypeName = "ClusterResource" +) + +// clusterResourceDecoder is an implementation of the xdsclient.Decoder +// interface for listener resources. +type clusterResourceDecoder struct { + bootstrapConfig *bootstrap.Config + serverConfigs map[xdsclient.ServerConfig]*bootstrap.ServerConfig +} + +func (d *clusterResourceDecoder) Decode(resource *xdsclient.AnyProto, opts xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) { + serverCfg, ok := d.serverConfigs[*opts.ServerConfig] + if !ok { + return nil, fmt.Errorf("no server config found for {%+v}", opts.ServerConfig) + } + name, cluster, err := unmarshalClusterResource(resource.ToAny(), serverCfg) + if name == "" { + // Name is unset only when protobuf deserialization fails. + return nil, err + } + if err != nil { + // Protobuf deserialization succeeded, but resource validation failed. + return &xdsclient.DecodeResult{ + Name: name, + Resource: &ClusterResourceData{Resource: ClusterUpdate{}}, + }, err + } + + // Perform extra validation here. + if err := securityConfigValidator(d.bootstrapConfig, cluster.SecurityCfg); err != nil { + return &xdsclient.DecodeResult{ + Name: name, + Resource: &ClusterResourceData{Resource: ClusterUpdate{}}, + }, err + } + + return &xdsclient.DecodeResult{ + Name: name, + Resource: &ClusterResourceData{Resource: cluster}, + }, nil +} + +// ClusterResourceData wraps the configuration of a Cluster resource as received +// from the management server. +type ClusterResourceData struct { + Resource ClusterUpdate +} + +// Equal returns true if other is equal to c. +func (c *ClusterResourceData) Equal(other xdsclient.ResourceData) bool { + if other == nil { + return false + } + return bytes.Equal(c.Bytes(), other.Bytes()) +} + +// Bytes returns the protobuf serialized bytes of the cluster resource proto. +func (c *ClusterResourceData) Bytes() []byte { + return c.Resource.Raw.GetValue() +} + +// ClusterWatcher wraps the callbacks to be invoked for different events +// corresponding to the cluster resource being watched. gRFC A88 contains an +// exhaustive list of what method is invoked under what conditions. +type ClusterWatcher interface { + // ResourceChanged indicates a new version of the resource is available. + ResourceChanged(resource *ClusterUpdate, done func()) + + // ResourceError indicates an error occurred while trying to fetch or + // decode the associated resource. The previous version of the resource + // should be considered invalid. + ResourceError(err error, done func()) + + // AmbientError indicates an error occurred after a resource has been + // received that should not modify the use of that resource but may provide + // useful information about the state of the XDSClient for debugging + // purposes. The previous version of the resource should still be + // considered valid. + AmbientError(err error, done func()) +} + +type delegatingClusterWatcher struct { + watcher ClusterWatcher +} + +func (d *delegatingClusterWatcher) ResourceChanged(data xdsclient.ResourceData, onDone func()) { + c := data.(*ClusterResourceData) + d.watcher.ResourceChanged(&c.Resource, onDone) +} + +func (d *delegatingClusterWatcher) ResourceError(err error, onDone func()) { + d.watcher.ResourceError(err, onDone) +} + +func (d *delegatingClusterWatcher) AmbientError(err error, onDone func()) { + d.watcher.AmbientError(err, onDone) +} + +// WatchCluster uses xDS to discover the configuration associated with the +// provided cluster resource name. +func WatchCluster(p Producer, name string, w ClusterWatcher) (cancel func()) { + return p.WatchResource(version.V3ClusterURL, name, &delegatingClusterWatcher{watcher: w}) +} + +// NewClusterResourceTypeDecoder returns a xdsclient.Decoder that wraps +// the xdsresource.clusterType. +func NewClusterResourceTypeDecoder(bc *bootstrap.Config, gServerCfgMap map[xdsclient.ServerConfig]*bootstrap.ServerConfig) xdsclient.Decoder { + return &clusterResourceDecoder{bootstrapConfig: bc, serverConfigs: gServerCfgMap} +} diff --git a/internal/xds/xdsclient/xdsresource/endpoints_resource_type.go b/internal/xds/xdsclient/xdsresource/endpoints_resource_type.go new file mode 100644 index 0000000..cf72c49 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/endpoints_resource_type.go @@ -0,0 +1,126 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "bytes" + + "google.golang.org/grpc/internal/xds/bootstrap" + xdsclient "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" +) + +const ( + // EndpointsResourceTypeName represents the transport agnostic name for the + // endpoint resource. + EndpointsResourceTypeName = "EndpointsResource" +) + +// endpointsResourceDecoder is an implementation of the xdsclient.Decoder +// interface for endpoints resources. +type endpointsResourceDecoder struct { + bootstrapConfig *bootstrap.Config +} + +func (d *endpointsResourceDecoder) Decode(resource *xdsclient.AnyProto, _ xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) { + name, endpoints, err := unmarshalEndpointsResource(resource.ToAny()) + if name == "" { + // Name is unset only when protobuf deserialization fails. + return nil, err + } + if err != nil { + // Protobuf deserialization succeeded, but resource validation failed. + return &xdsclient.DecodeResult{ + Name: name, + Resource: &ListenerResourceData{Resource: ListenerUpdate{}}, + }, err + } + + return &xdsclient.DecodeResult{ + Name: name, + Resource: &EndpointsResourceData{Resource: endpoints}, + }, nil +} + +// EndpointsResourceData is an implementation of the xdsclient.ResourceData +// interface for endpoints resources. +type EndpointsResourceData struct { + Resource EndpointsUpdate +} + +// Equal returns true if other is equal to e. +func (e *EndpointsResourceData) Equal(other xdsclient.ResourceData) bool { + if other == nil { + return false + } + return bytes.Equal(e.Bytes(), other.Bytes()) +} + +// Bytes returns the protobuf serialized bytes of the listener resource proto. +func (e *EndpointsResourceData) Bytes() []byte { + return e.Resource.Raw.GetValue() +} + +// EndpointsWatcher wraps the callbacks to be invoked for different +// events corresponding to the endpoints resource being watched. gRFC A88 +// contains an exhaustive list of what method is invoked under what conditions. +type EndpointsWatcher interface { + // ResourceChanged indicates a new version of the resource is available. + ResourceChanged(resource *EndpointsUpdate, done func()) + + // ResourceError indicates an error occurred while trying to fetch or + // decode the associated resource. The previous version of the resource + // should be considered invalid. + ResourceError(err error, done func()) + + // AmbientError indicates an error occurred after a resource has been + // received that should not modify the use of that resource but may provide + // useful information about the state of the XDSClient for debugging + // purposes. The previous version of the resource should still be + // considered valid. + AmbientError(err error, done func()) +} + +type delegatingEndpointsWatcher struct { + watcher EndpointsWatcher +} + +func (d *delegatingEndpointsWatcher) ResourceChanged(data xdsclient.ResourceData, onDone func()) { + e := data.(*EndpointsResourceData) + d.watcher.ResourceChanged(&e.Resource, onDone) +} + +func (d *delegatingEndpointsWatcher) ResourceError(err error, onDone func()) { + d.watcher.ResourceError(err, onDone) +} + +func (d *delegatingEndpointsWatcher) AmbientError(err error, onDone func()) { + d.watcher.AmbientError(err, onDone) +} + +// WatchEndpoints uses xDS to discover the configuration associated with the +// provided endpoints resource name. +func WatchEndpoints(p Producer, name string, w EndpointsWatcher) (cancel func()) { + return p.WatchResource(version.V3EndpointsURL, name, &delegatingEndpointsWatcher{watcher: w}) +} + +// NewEndpointsResourceTypeDecoder returns a xdsclient.Decoder that wraps +// the xdsresource.endpointsType. +func NewEndpointsResourceTypeDecoder(bc *bootstrap.Config) xdsclient.Decoder { + return &endpointsResourceDecoder{bootstrapConfig: bc} +} diff --git a/internal/xds/xdsclient/xdsresource/errors.go b/internal/xds/xdsclient/xdsresource/errors.go new file mode 100644 index 0000000..f90d30b --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/errors.go @@ -0,0 +1,79 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsresource + +import ( + "errors" + "fmt" +) + +// ErrorType is the type of the error that the watcher will receive from the xds +// client. +type ErrorType int + +const ( + // ErrorTypeUnknown indicates the error doesn't have a specific type. It is + // the default value, and is returned if the error is not an xds error. + ErrorTypeUnknown ErrorType = iota + // ErrorTypeConnection indicates a connection error from the gRPC client. + ErrorTypeConnection + // ErrorTypeResourceNotFound indicates a resource is not found from the xds + // response. It's typically returned if the resource is removed in the xds + // server. + ErrorTypeResourceNotFound + // ErrorTypeResourceTypeUnsupported indicates the receipt of a message from + // the management server with resources of an unsupported resource type. + ErrorTypeResourceTypeUnsupported + // ErrTypeStreamFailedAfterRecv indicates an ADS stream error, after + // successful receipt of at least one message from the server. + ErrTypeStreamFailedAfterRecv + // ErrorTypeNACKed indicates that configuration provided by the xDS management + // server was NACKed. + ErrorTypeNACKed +) + +type xdsClientError struct { + t ErrorType + desc string +} + +func (e *xdsClientError) Error() string { + return e.desc +} + +// NewErrorf creates an xDS client error. The callbacks are called with this +// error, to pass additional information about the error. +func NewErrorf(t ErrorType, format string, args ...any) error { + return &xdsClientError{t: t, desc: fmt.Sprintf(format, args...)} +} + +// NewError creates an xDS client error. The callbacks are called with this +// error, to pass additional information about the error. +func NewError(t ErrorType, message string) error { + return NewErrorf(t, "%s", message) +} + +// ErrType returns the error's type. +func ErrType(err error) ErrorType { + var xe *xdsClientError + if errors.As(err, &xe) { + return xe.t + } + return ErrorTypeUnknown +} diff --git a/internal/xds/xdsclient/xdsresource/filter_chain.go b/internal/xds/xdsclient/xdsresource/filter_chain.go new file mode 100644 index 0000000..fdf87d3 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/filter_chain.go @@ -0,0 +1,180 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + "net/netip" + + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" +) + +// NetworkFilterChainMap contains the match configuration for network filter +// chains on the server side. It is a multi-level map structure to facilitate +// efficient matching of incoming connections based on destination IP, source +// {type, IP and port}. +type NetworkFilterChainMap struct { + // DstPrefixes is the list of destination prefix entries to match on. + DstPrefixes []DestinationPrefixEntry +} + +// DestinationPrefixEntry contains a destination prefix entry and the associated +// source type matchers. +type DestinationPrefixEntry struct { + // Prefix is the destination IP prefix. + Prefix netip.Prefix + // SourceTypeArr contains the source type matchers. The supported source + // types and their associated indices in the array are: + // - 0: Any: matches connection attempts from any source. + // - 1: SameOrLoopback: matches connection attempts from the same host. + // - 2: External: matches connection attempts from a different host. + SourceTypeArr [3]SourcePrefixes +} + +// SourcePrefixes contains a list of source prefix entries to match on. +type SourcePrefixes struct { + // Entries is the list of source prefix entries. + Entries []SourcePrefixEntry +} + +// SourcePrefixEntry contains a source prefix entry and the associated source +// port matchers. +type SourcePrefixEntry struct { + // Prefix is the source IP prefix. + Prefix netip.Prefix + // PortMap contains the matchers for source ports. + PortMap map[int]NetworkFilterChainConfig +} + +// NetworkFilterChainConfig contains the configuration for a network filter +// chain on the server side. The only support network filter is the HTTP +// connection manager. +type NetworkFilterChainConfig struct { + // SecurityCfg contains transport socket security configuration. + SecurityCfg *SecurityConfig + // HTTPConnMgr contains the HTTP connection manager configuration. + HTTPConnMgr *HTTPConnectionManagerConfig +} + +// IsEmpty returns true if the NetworkFilterChainConfig contains no +// configuration. +func (n NetworkFilterChainConfig) IsEmpty() bool { + return n.SecurityCfg == nil && n.HTTPConnMgr == nil +} + +func processNetworkFilters(filters []*v3listenerpb.Filter, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (*HTTPConnectionManagerConfig, error) { + hcmConfig := &HTTPConnectionManagerConfig{} + seenNames := make(map[string]bool, len(filters)) + seenHCM := false + for _, filter := range filters { + name := filter.GetName() + if name == "" { + return nil, fmt.Errorf("network filters {%+v} is missing name field in filter: {%+v}", filters, filter) + } + if seenNames[name] { + return nil, fmt.Errorf("network filters {%+v} has duplicate filter name %q", filters, name) + } + seenNames[name] = true + + // Network filters have a oneof field named `config_type` where we + // only support `TypedConfig` variant. + switch typ := filter.GetConfigType().(type) { + case *v3listenerpb.Filter_TypedConfig: + // The typed_config field has an `anypb.Any` proto which could + // directly contain the serialized bytes of the actual filter + // configuration, or it could be encoded as a `TypedStruct`. + // TODO: Add support for `TypedStruct`. + tc := filter.GetTypedConfig() + + // The only network filter that we currently support is the v3 + // HttpConnectionManager. So, we can directly check the type_url + // and unmarshal the config. + // TODO: Implement a registry of supported network filters (like + // we have for HTTP filters), when we have to support network + // filters other than HttpConnectionManager. + if tc.GetTypeUrl() != version.V3HTTPConnManagerURL { + return nil, fmt.Errorf("network filters {%+v} has unsupported network filter %q in filter {%+v}", filters, tc.GetTypeUrl(), filter) + } + hcm := &v3httppb.HttpConnectionManager{} + if err := tc.UnmarshalTo(hcm); err != nil { + return nil, fmt.Errorf("network filters {%+v} failed unmarshalling of network filter {%+v}: %v", filters, filter, err) + } + // "Any filters after HttpConnectionManager should be ignored during + // connection processing but still be considered for validity. + // HTTPConnectionManager must have valid http_filters." - A36 + filters, err := processHTTPFilters(hcm.GetHttpFilters(), true, bc, sc) + if err != nil { + return nil, fmt.Errorf("network filters {%+v} had invalid server side HTTP Filters {%+v}: %v", filters, hcm.GetHttpFilters(), err) + } + if !seenHCM { + // Validate for RBAC in only the HCM that will be used, since this isn't a logical validation failure, + // it's simply a validation to support RBAC HTTP Filter. + // "HttpConnectionManager.xff_num_trusted_hops must be unset or zero and + // HttpConnectionManager.original_ip_detection_extensions must be empty. If + // either field has an incorrect value, the Listener must be NACKed." - A41 + if hcm.XffNumTrustedHops != 0 { + return nil, fmt.Errorf("xff_num_trusted_hops must be unset or zero %+v", hcm) + } + if len(hcm.OriginalIpDetectionExtensions) != 0 { + return nil, fmt.Errorf("original_ip_detection_extensions must be empty %+v", hcm) + } + + // TODO: Implement terminal filter logic, as per A36. + hcmConfig.HTTPFilters = filters + seenHCM = true + switch hcm.RouteSpecifier.(type) { + case *v3httppb.HttpConnectionManager_Rds: + if hcm.GetRds().GetConfigSource().GetAds() == nil { + return nil, fmt.Errorf("ConfigSource is not ADS: %+v", hcm) + } + name := hcm.GetRds().GetRouteConfigName() + if name == "" { + return nil, fmt.Errorf("empty route_config_name: %+v", hcm) + } + hcmConfig.RouteConfigName = name + case *v3httppb.HttpConnectionManager_RouteConfig: + // "RouteConfiguration validation logic inherits all + // previous validations made for client-side usage as RDS + // does not distinguish between client-side and + // server-side." - A36 + // Can specify v3 here, as will never get to this function + // if v2. + routeU, err := generateRDSUpdateFromRouteConfiguration(hcm.GetRouteConfig(), bc, sc) + if err != nil { + return nil, fmt.Errorf("failed to parse inline RDS resp: %v", err) + } + hcmConfig.InlineRouteConfig = &routeU + case nil: + return nil, fmt.Errorf("no RouteSpecifier: %+v", hcm) + default: + return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", hcm.RouteSpecifier) + } + } + default: + return nil, fmt.Errorf("network filters {%+v} has unsupported config_type %T in filter %s", filters, typ, filter.GetName()) + } + } + if !seenHCM { + return nil, fmt.Errorf("network filters {%+v} missing HttpConnectionManager filter", filters) + } + return hcmConfig, nil +} diff --git a/internal/xds/xdsclient/xdsresource/filter_chain_test.go b/internal/xds/xdsclient/xdsresource/filter_chain_test.go new file mode 100644 index 0000000..e799ade --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/filter_chain_test.go @@ -0,0 +1,2568 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + "net/netip" + "strings" + "testing" + + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +var cmpOptsIgnoreRawProto = cmp.Options{ + cmpopts.EquateEmpty(), + cmpopts.EquateComparable(netip.Addr{}, netip.Prefix{}), + cmpopts.IgnoreFields(ListenerUpdate{}, "Raw"), + cmpopts.IgnoreFields(RouteConfigUpdate{}, "Raw"), + protocmp.Transform(), +} + +// Tests cases where the filter chain match criteria contains unsupported +// fields, that result in the chain being dropped. +func (s) TestUnmarshalListener_ServerSide_DroppedFilterChains(t *testing.T) { + tests := []struct { + desc string + resource *anypb.Any + lis *v3listenerpb.Listener + wantErr string + }{ + { + desc: "unsupported destination port field", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{DestinationPort: &wrapperspb.UInt32Value{Value: 666}}, + Name: "test-filter-chain", + }, + }, + }, + wantErr: `Dropping filter chain "test-filter-chain" since it contains unsupported destination_port match field`, + }, + { + desc: "unsupported server names field", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ServerNames: []string{"example-server"}}, + Name: "test-filter-chain", + }, + }, + }, + wantErr: `Dropping filter chain "test-filter-chain" since it contains unsupported server_names match field`, + }, + { + desc: "unsupported transport protocol field", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{TransportProtocol: "tls"}, + Name: "test-filter-chain", + }, + }, + }, + wantErr: `Dropping filter chain "test-filter-chain" since it contains unsupported value for transport_protocol match field`, + }, + { + desc: "unsupported application protocol field", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ApplicationProtocols: []string{"h2"}}, + Name: "test-filter-chain", + }, + }, + }, + wantErr: `Dropping filter chain "test-filter-chain" since it contains unsupported application_protocols match field`, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + grpctest.ExpectWarning(test.wantErr) + resource := listenerProtoToAny(t, test.lis) + if _, _, err := unmarshalListenerResource(resource, nil, nil); err == nil { + t.Errorf("unmarshalListenerResource(%s) succeeded when expected to fail", pretty.ToJSON(resource)) + } + }) + } +} + +// Tests cases where the filter chain match criteria contains invalid +// information, that result in unmarshaling failure. +func (s) TestUnmarshalListener_ServerSide_FilterChains_FailureCases(t *testing.T) { + tests := []struct { + desc string + lis *v3listenerpb.Listener + wantErr string + }{ + { + desc: "bad dest address prefix", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{{AddressPrefix: "a.b.c.d"}}}, + Name: "test-filter-chain", + }, + }, + }, + wantErr: `failed to parse destination prefix ranges: invalid address: "a.b.c.d"`, + }, + { + desc: "bad dest prefix length", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.1.1.0", 50)}}, + Name: "test-filter-chain", + }, + }, + }, + wantErr: `failed to parse destination prefix ranges: length 50 is invalid for "10.1.1.0" (max 32)`, + }, + { + desc: "bad source address prefix", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePrefixRanges: []*v3corepb.CidrRange{{AddressPrefix: "a.b.c.d"}}}, + Name: "test-filter-chain", + }, + }, + }, + wantErr: `failed to parse source prefix ranges: invalid address: "a.b.c.d"`, + }, + { + desc: "bad source prefix length", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.1.1.0", 50)}}, + Name: "test-filter-chain", + }, + }, + }, + wantErr: `failed to parse source prefix ranges: length 50 is invalid for "10.1.1.0" (max 32)`, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + resource := listenerProtoToAny(t, test.lis) + if _, _, err := unmarshalListenerResource(resource, nil, nil); err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Errorf("unmarshalListenerResource(%s) failed with error: %v, want: %v", pretty.ToJSON(resource), err, test.wantErr) + } + }) + } +} + +// Tests cases where there are multiple filter chains and they have overlapping +// match rules. +func (s) TestUnmarshalListener_ServerSide_OverlappingMatchingRules(t *testing.T) { + tests := []struct { + desc string + lis *v3listenerpb.Listener + }{ + { + desc: "matching destination prefixes with no other matchers", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16), cidrRangeFromAddressAndPrefixLen("10.0.0.0", 0)}, + }, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.2.2", 16)}, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + }, + { + desc: "matching source type", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_ANY}, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK}, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_EXTERNAL}, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_EXTERNAL}, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + }, + { + desc: "matching source prefixes", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16), cidrRangeFromAddressAndPrefixLen("10.0.0.0", 0)}, + }, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.2.2", 16)}, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + }, + { + desc: "matching source ports", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3, 4, 5}}, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{}, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{5, 6, 7}}, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + }, + } + + const wantErr = "multiple filter chains with overlapping matching rules are defined" + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + resource := listenerProtoToAny(t, test.lis) + if _, _, err := unmarshalListenerResource(resource, nil, nil); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Errorf("unmarshalListenerResource(%s) failed with error: %v, want: %v", pretty.ToJSON(resource), err, wantErr) + } + }) + } +} + +// Tests cases where the security configuration in the filter chain is invalid. +func (s) TestUnmarshalListener_ServerSide_BadSecurityConfig(t *testing.T) { + tests := []struct { + desc string + lis *v3listenerpb.Listener + wantErr string + enableSystemRootCertsFlag bool + }{ + { + desc: "no filter chains", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + }, + wantErr: "no supported filter chains and no default filter chain", + }, + { + desc: "unexpected transport socket name", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{Name: "unsupported-transport-socket-name"}, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + wantErr: "transport_socket field has unexpected name", + }, + { + desc: "unexpected transport socket URL", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{}), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + wantErr: fmt.Sprintf("transport_socket missing typed_config or wrong type_url: \"%s\"", testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{}).TypeUrl), + }, + { + desc: "badly marshaled transport socket", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3DownstreamTLSContextURL, + Value: []byte{1, 2, 3, 4}, + }, + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + wantErr: "failed to unmarshal DownstreamTlsContext in LDS response", + }, + { + desc: "missing CommonTlsContext", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{}), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + wantErr: "DownstreamTlsContext in LDS response does not contain a CommonTlsContext", + }, + { + desc: "require_sni-set-to-true-in-downstreamTlsContext", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireSni: &wrapperspb.BoolValue{Value: true}, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + wantErr: "require_sni field set to true in DownstreamTlsContext message", + }, + { + desc: "unsupported-ocsp_staple_policy-in-downstreamTlsContext", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + OcspStaplePolicy: v3tlspb.DownstreamTlsContext_STRICT_STAPLING, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + wantErr: "ocsp_staple_policy field set to unsupported value in DownstreamTlsContext message", + }, + { + desc: "unsupported validation context in transport socket", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{ + ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{ + Name: "foo-sds-secret", + }, + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + wantErr: "validation context contains unexpected type", + }, + { + desc: "unsupported match_subject_alt_names field in transport socket", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{ + ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{ + Name: "foo-sds-secret", + }, + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + wantErr: "validation context contains unexpected type", + }, + { + desc: "no root certificate provider with require_client_cert", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + wantErr: "security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set", + }, + { + desc: "no_identity_certificate_provider", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{}, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + wantErr: "security configuration on the server-side does not contain identity certificate provider instance name", + }, + { + desc: "system root certificate field set on server", + enableSystemRootCertsFlag: true, + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + SystemRootCerts: &v3tlspb.CertificateValidationContext_SystemRootCerts{}, + }, + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + wantErr: "expected field ca_certificate_provider_instance is missing and unexpected field system_root_certs is set", + }, + { + desc: "system root certificate field set on server, env var disabled", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + SystemRootCerts: &v3tlspb.CertificateValidationContext_SystemRootCerts{}, + }, + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + }, + wantErr: "expected field ca_certificate_provider_instance is missing", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSystemRootCertsEnabled, test.enableSystemRootCertsFlag) + resource := listenerProtoToAny(t, test.lis) + if _, _, err := unmarshalListenerResource(resource, nil, nil); err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Errorf("unmarshalListenerResource(%s) failed with error: %v, want: %v", pretty.ToJSON(resource), err, test.wantErr) + } + }) + } +} + +// Tests cases where the route configuration specified in the HTTP Connection +// Manager within the filter chain is valid. +func (s) TestUnmarshalListener_ServerSide_GoodRouteUpdate(t *testing.T) { + tests := []struct { + name string + lis *v3listenerpb.Listener + wantListener ListenerUpdate + }{ + { + name: "one_route_config_name", + lis: &v3listenerpb.Listener{ + Name: "listerner-1", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-1", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-1", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + RouteConfigName: "route-1", + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + RouteConfigName: "route-1", + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + { + name: "inline_route_config", + lis: &v3listenerpb.Listener{ + Name: "listerner-1", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + { + name: "two_route_config_names", + lis: &v3listenerpb.Listener{ + Name: "listerner-1", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-1", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-2", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + RouteConfigName: "route-1", + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + RouteConfigName: "route-2", + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resource := listenerProtoToAny(t, test.lis) + _, gotListener, err := unmarshalListenerResource(resource, nil, nil) + if err != nil { + t.Fatalf("unmarshalListenerResource(%s) failed with error: %v", pretty.ToJSON(resource), err) + } + if diff := cmp.Diff(test.wantListener, gotListener, cmpOptsIgnoreRawProto); diff != "" { + t.Errorf("unmarshalListenerResource(%s), got unexpected update, diff (-want, +got):\n%s", pretty.ToJSON(resource), diff) + } + }) + } +} + +// Tests cases where the route configuration specified in the HTTP Connection +// Manager within the filter chain is invalid. +func (s) TestUnmarshalListener_ServerSide_BadRouteUpdate(t *testing.T) { + tests := []struct { + name string + lis *v3listenerpb.Listener + wantErr string + }{ + { + name: "missing-route-specifier", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + wantErr: "no RouteSpecifier", + }, + { + name: "not-ads", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + RouteConfigName: "route-1", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + RouteConfigName: "route-1", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + wantErr: "ConfigSource is not ADS", + }, + { + name: "unsupported-route-specifier", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{}, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{}, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + wantErr: "unsupported type", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resource := listenerProtoToAny(t, test.lis) + if _, _, err := unmarshalListenerResource(resource, nil, nil); err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Errorf("unmarshalListenerResource(%s) failed with error: %v, want: %v", pretty.ToJSON(resource), err, test.wantErr) + } + }) + } +} + +// Tests cases where the HTTP Filters in the filter chain are invalid. +func (s) TestUnmarshalListener_ServerSide_BadHTTPFilters(t *testing.T) { + tests := []struct { + name string + lis *v3listenerpb.Listener + wantErr string + }{ + { + name: "client side HTTP filter", + lis: &v3listenerpb.Listener{ + Name: "grpc/server?xds.resource.listening_address=0.0.0.0:9999", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + { + Name: "clientOnlyCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: clientOnlyCustomFilterConfig}, + }, + }, + }), + }, + }, + }, + }, + }, + }, + wantErr: "invalid server side HTTP Filters", + }, + { + name: "one valid then one invalid HTTP filter", + lis: &v3listenerpb.Listener{ + Name: "grpc/server?xds.resource.listening_address=0.0.0.0:9999", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + { + Name: "clientOnlyCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: clientOnlyCustomFilterConfig}, + }, + }, + }), + }, + }, + }, + }, + }, + }, + wantErr: "invalid server side HTTP Filters", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resource := listenerProtoToAny(t, test.lis) + if _, _, err := unmarshalListenerResource(resource, nil, nil); err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Errorf("unmarshalListenerResource(%s) failed with error: %v, want: %v", pretty.ToJSON(resource), err, test.wantErr) + } + }) + } +} + +// Tests cases where the HTTP Filters in the filter chain are valid. +func (s) TestUnmarshalListener_ServerSide_GoodHTTPFilters(t *testing.T) { + tests := []struct { + name string + lis *v3listenerpb.Listener + wantListener ListenerUpdate + }{ + { + name: "singular valid http filter", + lis: &v3listenerpb.Listener{ + Name: "listener-1", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: []HTTPFilter{ + { + Name: "serverOnlyCustomFilter", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + makeRouterFilter(t), + }, + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: []HTTPFilter{ + { + Name: "serverOnlyCustomFilter", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + makeRouterFilter(t), + }, + }, + }, + }, + }, + }, + { + name: "two valid http filters", + lis: &v3listenerpb.Listener{ + Name: "listener-1", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + validServerSideHTTPFilter2, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + validServerSideHTTPFilter2, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: []HTTPFilter{ + { + Name: "serverOnlyCustomFilter", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + { + Name: "serverOnlyCustomFilter2", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + makeRouterFilter(t), + }, + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: []HTTPFilter{ + { + Name: "serverOnlyCustomFilter", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + { + Name: "serverOnlyCustomFilter2", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + makeRouterFilter(t), + }, + }, + }, + }, + }, + }, + // In the case of two HTTP Connection Manager's being present, the + // second HTTP Connection Manager should be validated, but ignored. + { + name: "two hcms", + lis: &v3listenerpb.Listener{ + Name: "listener-1", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + validServerSideHTTPFilter2, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + { + Name: "hcm2", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + validServerSideHTTPFilter2, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + { + Name: "hcm2", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: []HTTPFilter{ + { + Name: "serverOnlyCustomFilter", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + { + Name: "serverOnlyCustomFilter2", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + makeRouterFilter(t), + }, + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: []HTTPFilter{ + { + Name: "serverOnlyCustomFilter", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + { + Name: "serverOnlyCustomFilter2", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + makeRouterFilter(t), + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resource := listenerProtoToAny(t, test.lis) + _, gotListener, err := unmarshalListenerResource(resource, nil, nil) + if err != nil { + t.Fatalf("unmarshalListenerResource(%s) failed with error: %v", pretty.ToJSON(resource), err) + } + if diff := cmp.Diff(test.wantListener, gotListener, cmpOptsIgnoreRawProto); diff != "" { + t.Errorf("unmarshalListenerResource(%s), got unexpected update, diff (-want, +got):\n%s", pretty.ToJSON(resource), diff) + } + }) + } +} + +// Tests cases where the security configuration in the filter chain contains +// valid data. +func (s) TestUnmarshalListener_ServerSide_GoodSecurityConfig(t *testing.T) { + tests := []struct { + desc string + lis *v3listenerpb.Listener + wantListener ListenerUpdate + enableSystemRootCertsFlag bool + }{ + { + desc: "empty transport socket", + lis: &v3listenerpb.Listener{ + Name: "listerner-1", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters(t), + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: emptyValidNetworkFilters(t), + }, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + { + desc: "no validation context", + lis: &v3listenerpb.Listener{ + Name: "listerner-1", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "defaultIdentityPluginInstance", + CertificateName: "defaultIdentityCertName", + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + SecurityCfg: &SecurityConfig{ + IdentityInstanceName: "identityPluginInstance", + IdentityCertName: "identityCertName", + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + SecurityCfg: &SecurityConfig{ + IdentityInstanceName: "defaultIdentityPluginInstance", + IdentityCertName: "defaultIdentityCertName", + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + { + desc: "validation context with certificate provider", + lis: &v3listenerpb.Listener{ + Name: "listerner-1", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Name: "default-filter-chain-1", + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "defaultIdentityPluginInstance", + CertificateName: "defaultIdentityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "defaultRootPluginInstance", + CertificateName: "defaultRootCertName", + }, + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + SecurityCfg: &SecurityConfig{ + RootInstanceName: "rootPluginInstance", + RootCertName: "rootCertName", + IdentityInstanceName: "identityPluginInstance", + IdentityCertName: "identityCertName", + RequireClientCert: true, + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + SecurityCfg: &SecurityConfig{ + RootInstanceName: "defaultRootPluginInstance", + RootCertName: "defaultRootCertName", + IdentityInstanceName: "defaultIdentityPluginInstance", + IdentityCertName: "defaultIdentityCertName", + RequireClientCert: true, + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + { + desc: "validation context with certificate provider and system root certs", + enableSystemRootCertsFlag: true, + lis: &v3listenerpb.Listener{ + Name: "listerner-1", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + // SystemRootCerts will be ignored + // when + // CaCertificateProviderInstance is + // set. + SystemRootCerts: &v3tlspb.CertificateValidationContext_SystemRootCerts{}, + }, + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Name: "default-filter-chain-1", + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "defaultIdentityPluginInstance", + CertificateName: "defaultIdentityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "defaultRootPluginInstance", + CertificateName: "defaultRootCertName", + }, + // SystemRootCerts will be ignored + // when + // CaCertificateProviderInstance is + // set. + SystemRootCerts: &v3tlspb.CertificateValidationContext_SystemRootCerts{}, + }, + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + SecurityCfg: &SecurityConfig{ + RootInstanceName: "rootPluginInstance", + RootCertName: "rootCertName", + IdentityInstanceName: "identityPluginInstance", + IdentityCertName: "identityCertName", + RequireClientCert: true, + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + SecurityCfg: &SecurityConfig{ + RootInstanceName: "defaultRootPluginInstance", + RootCertName: "defaultRootCertName", + IdentityInstanceName: "defaultIdentityPluginInstance", + IdentityCertName: "defaultIdentityCertName", + RequireClientCert: true, + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSystemRootCertsEnabled, test.enableSystemRootCertsFlag) + resource := listenerProtoToAny(t, test.lis) + _, gotListener, err := unmarshalListenerResource(resource, nil, nil) + if err != nil { + t.Fatalf("unmarshalListenerResource(%s) failed with error: %v", pretty.ToJSON(resource), err) + } + if diff := cmp.Diff(test.wantListener, gotListener, cmpOptsIgnoreRawProto); diff != "" { + t.Errorf("unmarshalListenerResource(%s), got unexpected update, diff (-want, +got):\n%s", pretty.ToJSON(resource), diff) + } + }) + } +} + +// TestNewFilterChainImpl_Success_UnsupportedMatchFields verifies cases where +// there are multiple filter chains, and one of them is valid while the other +// contains unsupported match fields. These configurations should lead to +// success at config validation time and the filter chains which contains +// unsupported match fields will be skipped at lookup time. +func (s) TestUnmarshalListener_ServerSide_Success_UnsupportedMatchFields(t *testing.T) { + tests := []struct { + desc string + lis *v3listenerpb.Listener + wantListener ListenerUpdate + }{ + { + desc: "unsupported destination port", + lis: &v3listenerpb.Listener{ + Name: "listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "good-chain", + Filters: emptyValidNetworkFilters(t), + }, + { + Name: "unsupported-destination-port", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + DestinationPort: &wrapperspb.UInt32Value{Value: 666}, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters(t)}, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + { + desc: "unsupported server names", + lis: &v3listenerpb.Listener{ + Name: "listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "good-chain", + Filters: emptyValidNetworkFilters(t), + }, + { + Name: "unsupported-server-names", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + ServerNames: []string{"example-server"}, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters(t)}, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + { + desc: "unsupported transport protocol", + lis: &v3listenerpb.Listener{ + Name: "listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "good-chain", + Filters: emptyValidNetworkFilters(t), + }, + { + Name: "unsupported-transport-protocol", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + TransportProtocol: "tls", + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters(t)}, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + { + desc: "unsupported application protocol", + lis: &v3listenerpb.Listener{ + Name: "listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "good-chain", + Filters: emptyValidNetworkFilters(t), + }, + { + Name: "unsupported-application-protocol", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + ApplicationProtocols: []string{"h2"}, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters(t)}, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + resource := listenerProtoToAny(t, test.lis) + _, gotListener, err := unmarshalListenerResource(resource, nil, nil) + if err != nil { + t.Fatalf("unmarshalListenerResource(%s) failed with error: %v", pretty.ToJSON(resource), err) + } + if diff := cmp.Diff(test.wantListener, gotListener, cmpOptsIgnoreRawProto); diff != "" { + t.Errorf("unmarshalListenerResource(%s), got unexpected update, diff (-want, +got):\n%s", pretty.ToJSON(resource), diff) + } + }) + } +} + +// TestNewFilterChainImpl_Success_AllCombinations verifies different +// combinations of the supported match criteria. +func (s) TestUnmarshalListener_ServerSide_Success_AllCombinations(t *testing.T) { + tests := []struct { + desc string + lis *v3listenerpb.Listener + wantListener ListenerUpdate + }{ + { + desc: "multiple destination prefixes", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + // Unspecified destination prefix. + FilterChainMatch: &v3listenerpb.FilterChainMatch{}, + Filters: emptyValidNetworkFilters(t), + }, + { + // v4 wildcard destination prefix. + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("0.0.0.0", 0)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + }, + Filters: emptyValidNetworkFilters(t), + }, + { + // v6 wildcard destination prefix. + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("::", 0)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + }, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}}, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 8)}}, + Filters: emptyValidNetworkFilters(t), + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters(t)}, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{ + { + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }, + { + Prefix: prefixFromCIDR("0.0.0.0/0"), + SourceTypeArr: [3]SourcePrefixes{ + {}, + {}, + { + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }, + }, + }, + { + Prefix: prefixFromCIDR("::/0"), + SourceTypeArr: [3]SourcePrefixes{ + {}, + {}, + { + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }, + }, + }, + { + Prefix: prefixFromCIDR("192.168.1.1/16"), + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }, + { + Prefix: prefixFromCIDR("10.0.0.0/8"), + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }, + }, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + { + desc: "multiple source types", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK}, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters(t)}, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{ + { + SourceTypeArr: [3]SourcePrefixes{ + {}, + { + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }, + {}, + }, + }, + { + Prefix: prefixFromCIDR("192.168.1.1/16"), + SourceTypeArr: [3]SourcePrefixes{ + {}, + {}, + { + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }, + }, + }, + }, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + { + desc: "multiple source prefixes", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 8)}}, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters(t)}, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{ + { + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + Prefix: prefixFromCIDR("10.0.0.0/8"), + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }, + { + Prefix: prefixFromCIDR("192.168.1.1/16"), + SourceTypeArr: [3]SourcePrefixes{ + { + Entries: []SourcePrefixEntry{ + { + Prefix: prefixFromCIDR("192.168.1.1/16"), + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + { + desc: "multiple source ports", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3}}, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + SourcePorts: []uint32{1, 2, 3}, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters(t)}, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{ + { + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 1: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + 2: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + 3: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }, + { + Prefix: prefixFromCIDR("192.168.1.1/16"), + SourceTypeArr: [3]SourcePrefixes{ + {}, + {}, + { + Entries: []SourcePrefixEntry{ + { + Prefix: prefixFromCIDR("192.168.1.1/16"), + PortMap: map[int]NetworkFilterChainConfig{ + 1: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + 2: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + 3: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + { + desc: "some chains have unsupported fields", + lis: &v3listenerpb.Listener{ + Name: "test-listener", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{}, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}}, + Filters: emptyValidNetworkFilters(t), + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 8)}, + TransportProtocol: "raw_buffer", + }, + Filters: emptyValidNetworkFilters(t), + }, + { + // This chain will be dropped in favor of the above + // filter chain because they both have the same + // destination prefix, but this one has an empty + // transport protocol while the above chain has the more + // preferred "raw_buffer". + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 8)}, + TransportProtocol: "", + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 16)}, + }, + Filters: emptyValidNetworkFilters(t), + }, + { + // This chain will be dropped for unsupported server + // names. + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.1", 32)}, + ServerNames: []string{"foo", "bar"}, + }, + Filters: emptyValidNetworkFilters(t), + }, + { + // This chain will be dropped for unsupported transport + // protocol. + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.2", 32)}, + TransportProtocol: "not-raw-buffer", + }, + Filters: emptyValidNetworkFilters(t), + }, + { + // This chain will be dropped for unsupported + // application protocol. + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.3", 32)}, + ApplicationProtocols: []string{"h2"}, + }, + Filters: emptyValidNetworkFilters(t), + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters(t)}, + }, + wantListener: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{ + { + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }, + { + Prefix: prefixFromCIDR("192.168.1.1/16"), + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }, + { + Prefix: prefixFromCIDR("10.0.0.0/8"), + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }, + }, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + resource := listenerProtoToAny(t, test.lis) + _, gotListener, err := unmarshalListenerResource(resource, nil, nil) + if err != nil { + t.Fatalf("unmarshalListenerResource(%s) failed with error: %v", pretty.ToJSON(resource), err) + } + if diff := cmp.Diff(test.wantListener, gotListener, cmpOptsIgnoreRawProto); diff != "" { + t.Errorf("unmarshalListenerResource(%s), got unexpected update, diff (-want, +got):\n%s", pretty.ToJSON(resource), diff) + } + }) + } +} + +func listenerProtoToAny(t *testing.T, lis *v3listenerpb.Listener) *anypb.Any { + t.Helper() + mLis, err := proto.Marshal(lis) + if err != nil { + t.Fatalf("Failed to marshal listener proto %+v: %v", lis, err) + } + return &anypb.Any{ + TypeUrl: version.V3ListenerURL, + Value: mLis, + } +} + +func prefixFromCIDR(cidr string) netip.Prefix { + return netip.MustParsePrefix(cidr).Masked() +} + +func cidrRangeFromAddressAndPrefixLen(address string, len int) *v3corepb.CidrRange { + return &v3corepb.CidrRange{ + AddressPrefix: address, + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(len), + }, + } +} diff --git a/internal/xds/xdsclient/xdsresource/listener_resource_type.go b/internal/xds/xdsclient/xdsresource/listener_resource_type.go new file mode 100644 index 0000000..b694ba7 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/listener_resource_type.go @@ -0,0 +1,185 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "bytes" + "fmt" + + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" +) + +// ListenerResourceTypeName is a human friendly name for the listener resource. +const ListenerResourceTypeName = "ListenerResource" + +// listenerResourceDecoder is an implementation of the xdsclient.Decoder +// interface for listener resources. +type listenerResourceDecoder struct { + bootstrapConfig *bootstrap.Config + serverConfigs map[xdsclient.ServerConfig]*bootstrap.ServerConfig +} + +func (d *listenerResourceDecoder) Decode(resource *xdsclient.AnyProto, opts xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) { + var serverCfg *bootstrap.ServerConfig + if opts.ServerConfig != nil { + serverCfg = d.serverConfigs[*opts.ServerConfig] + } + name, listener, err := unmarshalListenerResource(resource.ToAny(), d.bootstrapConfig, serverCfg) + if name == "" { + // Name is unset only when protobuf deserialization fails. + return nil, err + } + if err != nil { + // Protobuf deserialization succeeded, but resource validation failed. + return &xdsclient.DecodeResult{ + Name: name, + Resource: &ListenerResourceData{Resource: ListenerUpdate{}}, + }, err + } + + // Perform extra validation here. + if err := listenerValidator(d.bootstrapConfig, listener); err != nil { + return &xdsclient.DecodeResult{ + Name: name, + Resource: &ListenerResourceData{Resource: ListenerUpdate{}}, + }, err + } + + return &xdsclient.DecodeResult{ + Name: name, + Resource: &ListenerResourceData{Resource: listener}, + }, nil +} + +func securityConfigValidator(bc *bootstrap.Config, sc *SecurityConfig) error { + if sc == nil { + return nil + } + if sc.IdentityInstanceName != "" { + if _, ok := bc.CertProviderConfigs()[sc.IdentityInstanceName]; !ok { + return fmt.Errorf("identity certificate provider instance name %q missing in bootstrap configuration", sc.IdentityInstanceName) + } + } + if sc.RootInstanceName != "" { + if _, ok := bc.CertProviderConfigs()[sc.RootInstanceName]; !ok { + return fmt.Errorf("root certificate provider instance name %q missing in bootstrap configuration", sc.RootInstanceName) + } + } + return nil +} + +func listenerValidator(bc *bootstrap.Config, lis ListenerUpdate) error { + // Validate Filter Chains. + validateFC := func(fc *NetworkFilterChainConfig) error { + if fc == nil { + return nil + } + return securityConfigValidator(bc, fc.SecurityCfg) + } + + if lis.TCPListener == nil { + return nil + } + if err := validateFC(&lis.TCPListener.DefaultFilterChain); err != nil { + return err + } + for _, dst := range lis.TCPListener.FilterChains.DstPrefixes { + for _, srcType := range dst.SourceTypeArr { + if len(srcType.Entries) == 0 { + continue + } + for _, src := range srcType.Entries { + for _, fc := range src.PortMap { + if err := validateFC(&fc); err != nil { + return err + } + } + } + } + } + return nil +} + +// ListenerResourceData is an implementation of the xdsclient.ResourceData +// interface for listener resources. +type ListenerResourceData struct { + Resource ListenerUpdate +} + +// Equal returns true if other is equal to l. +func (l *ListenerResourceData) Equal(other xdsclient.ResourceData) bool { + if other == nil { + return false + } + return bytes.Equal(l.Bytes(), other.Bytes()) +} + +// Bytes returns the protobuf serialized bytes of the listener resource proto. +func (l *ListenerResourceData) Bytes() []byte { + return l.Resource.Raw.GetValue() +} + +// ListenerWatcher wraps the callbacks to be invoked for different +// events corresponding to the listener resource being watched. gRFC A88 +// contains an exhaustive list of what method is invoked under what conditions. +type ListenerWatcher interface { + // ResourceChanged indicates a new version of the resource is available. + ResourceChanged(resource *ListenerUpdate, done func()) + + // ResourceError indicates an error occurred while trying to fetch or + // decode the associated resource. The previous version of the resource + // should be considered invalid. + ResourceError(err error, done func()) + + // AmbientError indicates an error occurred after a resource has been + // received that should not modify the use of that resource but may provide + // useful information about the state of the XDSClient for debugging + // purposes. The previous version of the resource should still be + // considered valid. + AmbientError(err error, done func()) +} + +type delegatingListenerWatcher struct { + watcher ListenerWatcher +} + +func (d *delegatingListenerWatcher) ResourceChanged(data xdsclient.ResourceData, onDone func()) { + l := data.(*ListenerResourceData) + d.watcher.ResourceChanged(&l.Resource, onDone) +} +func (d *delegatingListenerWatcher) ResourceError(err error, onDone func()) { + d.watcher.ResourceError(err, onDone) +} + +func (d *delegatingListenerWatcher) AmbientError(err error, onDone func()) { + d.watcher.AmbientError(err, onDone) +} + +// WatchListener uses xDS to discover the configuration associated with the +// provided listener resource name. +func WatchListener(p Producer, name string, w ListenerWatcher) (cancel func()) { + return p.WatchResource(version.V3ListenerURL, name, &delegatingListenerWatcher{watcher: w}) +} + +// NewListenerResourceTypeDecoder returns a xdsclient.Decoder that wraps +// the xdsresource.listenerType. +func NewListenerResourceTypeDecoder(bc *bootstrap.Config, gServerCfgMap map[xdsclient.ServerConfig]*bootstrap.ServerConfig) xdsclient.Decoder { + return &listenerResourceDecoder{bootstrapConfig: bc, serverConfigs: gServerCfgMap} +} diff --git a/internal/xds/xdsclient/xdsresource/logging.go b/internal/xds/xdsclient/xdsresource/logging.go new file mode 100644 index 0000000..62bcb01 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/logging.go @@ -0,0 +1,28 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsresource + +import ( + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[xds-resource] " + +var logger = internalgrpclog.NewPrefixLogger(grpclog.Component("xds"), prefix) diff --git a/internal/xds/xdsclient/xdsresource/matcher.go b/internal/xds/xdsclient/xdsresource/matcher.go new file mode 100644 index 0000000..a1c15c2 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/matcher.go @@ -0,0 +1,231 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + rand "math/rand/v2" + "strings" + + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/metadata" +) + +// RouteToMatcher converts a route to a Matcher to match incoming RPC's against. +// +// Only expected to be called on a Route that passed validation checks by the +// xDS client. +func RouteToMatcher(r *Route) *CompositeMatcher { + var pm pathMatcher + switch { + case r.Regex != nil: + pm = newPathRegexMatcher(r.Regex) + case r.Path != nil: + pm = newPathExactMatcher(*r.Path, r.CaseInsensitive) + case r.Prefix != nil: + pm = newPathPrefixMatcher(*r.Prefix, r.CaseInsensitive) + default: + panic("illegal route: missing path_matcher") + } + + headerMatchers := make([]matcher.HeaderMatcher, 0, len(r.Headers)) + for _, h := range r.Headers { + var matcherT matcher.HeaderMatcher + invert := h.InvertMatch != nil && *h.InvertMatch + switch { + case h.RegexMatch != nil: + matcherT = matcher.NewHeaderRegexMatcher(h.Name, h.RegexMatch, invert) + case h.RangeMatch != nil: + matcherT = matcher.NewHeaderRangeMatcher(h.Name, h.RangeMatch.Start, h.RangeMatch.End, invert) + case h.PresentMatch != nil: + matcherT = matcher.NewHeaderPresentMatcher(h.Name, *h.PresentMatch, invert) + case h.StringMatch != nil: + matcherT = matcher.NewHeaderStringMatcher(h.Name, *h.StringMatch, invert) + default: + panic("illegal route: missing header_match_specifier") + } + headerMatchers = append(headerMatchers, matcherT) + } + + var fractionMatcher *fractionMatcher + if r.Fraction != nil { + fractionMatcher = newFractionMatcher(*r.Fraction) + } + return newCompositeMatcher(pm, headerMatchers, fractionMatcher) +} + +// CompositeMatcher is a matcher that holds onto many matchers and aggregates +// the matching results. +type CompositeMatcher struct { + pm pathMatcher + hms []matcher.HeaderMatcher + fm *fractionMatcher +} + +func newCompositeMatcher(pm pathMatcher, hms []matcher.HeaderMatcher, fm *fractionMatcher) *CompositeMatcher { + return &CompositeMatcher{pm: pm, hms: hms, fm: fm} +} + +// Match returns true if all matchers return true. +func (a *CompositeMatcher) Match(method string, md metadata.MD) bool { + if a.pm != nil && !a.pm.match(method) { + return false + } + + for _, m := range a.hms { + if !m.Match(md) { + return false + } + } + + if a.fm != nil && !a.fm.match() { + return false + } + return true +} + +func (a *CompositeMatcher) String() string { + var ret string + if a.pm != nil { + ret += a.pm.String() + } + for _, m := range a.hms { + ret += m.String() + } + if a.fm != nil { + ret += a.fm.String() + } + return ret +} + +type fractionMatcher struct { + fraction int64 // real fraction is fraction/1,000,000. +} + +func newFractionMatcher(fraction uint32) *fractionMatcher { + return &fractionMatcher{fraction: int64(fraction)} +} + +// RandInt64n overwrites rand for control in tests. +var RandInt64n = rand.Int64N + +func (fm *fractionMatcher) match() bool { + t := RandInt64n(1000000) + return t <= fm.fraction +} + +func (fm *fractionMatcher) String() string { + return fmt.Sprintf("fraction:%v", fm.fraction) +} + +type domainMatchType int + +const ( + domainMatchTypeInvalid domainMatchType = iota + domainMatchTypeUniversal + domainMatchTypePrefix + domainMatchTypeSuffix + domainMatchTypeExact +) + +// Exact > Suffix > Prefix > Universal > Invalid. +func (t domainMatchType) betterThan(b domainMatchType) bool { + return t > b +} + +func matchTypeForDomain(d string) domainMatchType { + if d == "" { + return domainMatchTypeInvalid + } + if d == "*" { + return domainMatchTypeUniversal + } + if strings.HasPrefix(d, "*") { + return domainMatchTypeSuffix + } + if strings.HasSuffix(d, "*") { + return domainMatchTypePrefix + } + if strings.Contains(d, "*") { + return domainMatchTypeInvalid + } + return domainMatchTypeExact +} + +func match(domain, host string) (domainMatchType, bool) { + switch typ := matchTypeForDomain(domain); typ { + case domainMatchTypeInvalid: + return typ, false + case domainMatchTypeUniversal: + return typ, true + case domainMatchTypePrefix: + // abc.* + return typ, strings.HasPrefix(host, strings.TrimSuffix(domain, "*")) + case domainMatchTypeSuffix: + // *.123 + return typ, strings.HasSuffix(host, strings.TrimPrefix(domain, "*")) + case domainMatchTypeExact: + return typ, domain == host + default: + return domainMatchTypeInvalid, false + } +} + +// FindBestMatchingVirtualHost returns the virtual host whose domains field best +// matches host +// +// The domains field support 4 different matching pattern types: +// +// - Exact match +// - Suffix match (e.g. “*ABC”) +// - Prefix match (e.g. “ABC*) +// - Universal match (e.g. “*”) +// +// The best match is defined as: +// - A match is better if it’s matching pattern type is better. +// * Exact match > suffix match > prefix match > universal match. +// +// - If two matches are of the same pattern type, the longer match is +// better. +// * This is to compare the length of the matching pattern, e.g. “*ABCDE” > +// “*ABC” +func FindBestMatchingVirtualHost(host string, vHosts []*VirtualHost) *VirtualHost { // Maybe move this crap to client + var ( + matchVh *VirtualHost + matchType = domainMatchTypeInvalid + matchLen int + ) + for _, vh := range vHosts { + for _, domain := range vh.Domains { + typ, matched := match(domain, host) + if typ == domainMatchTypeInvalid { + // The rds response is invalid. + return nil + } + if matchType.betterThan(typ) || matchType == typ && matchLen >= len(domain) || !matched { + // The previous match has better type, or the previous match has + // better length, or this domain isn't a match. + continue + } + matchVh = vh + matchType = typ + matchLen = len(domain) + } + } + return matchVh +} diff --git a/internal/xds/xdsclient/xdsresource/matcher_path.go b/internal/xds/xdsclient/xdsresource/matcher_path.go new file mode 100644 index 0000000..8627ed3 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/matcher_path.go @@ -0,0 +1,100 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "regexp" + "strings" +) + +type pathMatcher interface { + match(path string) bool + String() string +} + +type pathExactMatcher struct { + // fullPath is all upper case if caseInsensitive is true. + fullPath string + caseInsensitive bool +} + +func newPathExactMatcher(p string, caseInsensitive bool) *pathExactMatcher { + ret := &pathExactMatcher{ + fullPath: p, + caseInsensitive: caseInsensitive, + } + if caseInsensitive { + ret.fullPath = strings.ToUpper(p) + } + return ret +} + +func (pem *pathExactMatcher) match(path string) bool { + if pem.caseInsensitive { + return pem.fullPath == strings.ToUpper(path) + } + return pem.fullPath == path +} + +func (pem *pathExactMatcher) String() string { + return "pathExact:" + pem.fullPath +} + +type pathPrefixMatcher struct { + // prefix is all upper case if caseInsensitive is true. + prefix string + caseInsensitive bool +} + +func newPathPrefixMatcher(p string, caseInsensitive bool) *pathPrefixMatcher { + ret := &pathPrefixMatcher{ + prefix: p, + caseInsensitive: caseInsensitive, + } + if caseInsensitive { + ret.prefix = strings.ToUpper(p) + } + return ret +} + +func (ppm *pathPrefixMatcher) match(path string) bool { + if ppm.caseInsensitive { + return strings.HasPrefix(strings.ToUpper(path), ppm.prefix) + } + return strings.HasPrefix(path, ppm.prefix) +} + +func (ppm *pathPrefixMatcher) String() string { + return "pathPrefix:" + ppm.prefix +} + +type pathRegexMatcher struct { + re *regexp.Regexp +} + +func newPathRegexMatcher(re *regexp.Regexp) *pathRegexMatcher { + return &pathRegexMatcher{re: re} +} + +func (prm *pathRegexMatcher) match(path string) bool { + return prm.re.MatchString(path) +} + +func (prm *pathRegexMatcher) String() string { + return "pathRegex:" + prm.re.String() +} diff --git a/internal/xds/xdsclient/xdsresource/matcher_path_test.go b/internal/xds/xdsclient/xdsresource/matcher_path_test.go new file mode 100644 index 0000000..507cf15 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/matcher_path_test.go @@ -0,0 +1,93 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "regexp" + "testing" +) + +func (s) TestPathFullMatcherMatch(t *testing.T) { + tests := []struct { + name string + fullPath string + caseInsensitive bool + path string + want bool + }{ + {name: "match", fullPath: "/s/m", path: "/s/m", want: true}, + {name: "case insensitive match", fullPath: "/s/m", caseInsensitive: true, path: "/S/m", want: true}, + {name: "case insensitive match 2", fullPath: "/s/M", caseInsensitive: true, path: "/S/m", want: true}, + {name: "not match", fullPath: "/s/m", path: "/a/b", want: false}, + {name: "case insensitive not match", fullPath: "/s/m", caseInsensitive: true, path: "/a/b", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fpm := newPathExactMatcher(tt.fullPath, tt.caseInsensitive) + if got := fpm.match(tt.path); got != tt.want { + t.Errorf("{%q}.match(%q) = %v, want %v", tt.fullPath, tt.path, got, tt.want) + } + }) + } +} + +func (s) TestPathPrefixMatcherMatch(t *testing.T) { + tests := []struct { + name string + prefix string + caseInsensitive bool + path string + want bool + }{ + {name: "match", prefix: "/s/", path: "/s/m", want: true}, + {name: "case insensitive match", prefix: "/s/", caseInsensitive: true, path: "/S/m", want: true}, + {name: "case insensitive match 2", prefix: "/S/", caseInsensitive: true, path: "/s/m", want: true}, + {name: "not match", prefix: "/s/", path: "/a/b", want: false}, + {name: "case insensitive not match", prefix: "/s/", caseInsensitive: true, path: "/a/b", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fpm := newPathPrefixMatcher(tt.prefix, tt.caseInsensitive) + if got := fpm.match(tt.path); got != tt.want { + t.Errorf("{%q}.match(%q) = %v, want %v", tt.prefix, tt.path, got, tt.want) + } + }) + } +} + +func (s) TestPathRegexMatcherMatch(t *testing.T) { + tests := []struct { + name string + regexPath string + path string + want bool + }{ + {name: "match", regexPath: "^/s+/m.*$", path: "/sss/me", want: true}, + {name: "not match", regexPath: "^/s+/m*$", path: "/sss/b", want: false}, + {name: "no match because only part of path matches with regex", regexPath: "^a+$", path: "ab", want: false}, + {name: "match because full path matches with regex", regexPath: "^a+$", path: "aa", want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fpm := newPathRegexMatcher(regexp.MustCompile(tt.regexPath)) + if got := fpm.match(tt.path); got != tt.want { + t.Errorf("{%q}.match(%q) = %v, want %v", tt.regexPath, tt.path, got, tt.want) + } + }) + } +} diff --git a/internal/xds/xdsclient/xdsresource/matcher_test.go b/internal/xds/xdsclient/xdsresource/matcher_test.go new file mode 100644 index 0000000..8a8d863 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/matcher_test.go @@ -0,0 +1,244 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "context" + rand "math/rand/v2" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpcutil" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/proto" +) + +func (s) TestAndMatcherMatch(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + tests := []struct { + name string + pm pathMatcher + hm matcher.HeaderMatcher + info iresolver.RPCInfo + want bool + }{ + { + name: "both_match", + pm: newPathExactMatcher("/a/b", false), + hm: matcher.NewHeaderExactMatcher("th", "tv", false), + info: iresolver.RPCInfo{ + Method: "/a/b", + Context: metadata.NewOutgoingContext(ctx, metadata.Pairs("th", "tv")), + }, + want: true, + }, + { + name: "both_match_with_path_case_insensitive", + pm: newPathExactMatcher("/A/B", true), + hm: matcher.NewHeaderExactMatcher("th", "tv", false), + info: iresolver.RPCInfo{ + Method: "/a/b", + Context: metadata.NewOutgoingContext(ctx, metadata.Pairs("th", "tv")), + }, + want: true, + }, + { + name: "only_one_match", + pm: newPathExactMatcher("/a/b", false), + hm: matcher.NewHeaderExactMatcher("th", "tv", false), + info: iresolver.RPCInfo{ + Method: "/z/y", + Context: metadata.NewOutgoingContext(ctx, metadata.Pairs("th", "tv")), + }, + want: false, + }, + { + name: "both_not match", + pm: newPathExactMatcher("/z/y", false), + hm: matcher.NewHeaderExactMatcher("th", "abc", false), + info: iresolver.RPCInfo{ + Method: "/a/b", + Context: metadata.NewOutgoingContext(ctx, metadata.Pairs("th", "tv")), + }, + want: false, + }, + { + name: "fake_header", + pm: newPathPrefixMatcher("/", false), + hm: matcher.NewHeaderExactMatcher("content-type", "fake", false), + info: iresolver.RPCInfo{ + Method: "/a/b", + Context: grpcutil.WithExtraMetadata(ctx, metadata.Pairs( + "content-type", "fake", + )), + }, + want: true, + }, + { + name: "binary_header", + pm: newPathPrefixMatcher("/", false), + hm: matcher.NewHeaderPresentMatcher("t-bin", true, false), + info: iresolver.RPCInfo{ + Method: "/a/b", + Context: grpcutil.WithExtraMetadata( + metadata.NewOutgoingContext(ctx, metadata.Pairs("t-bin", "123")), metadata.Pairs( + "content-type", "fake", + )), + }, + // Shouldn't match binary header, even though it's in metadata. + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := newCompositeMatcher(tt.pm, []matcher.HeaderMatcher{tt.hm}, nil) + md, _ := metadata.FromOutgoingContext(tt.info.Context) + if extraMD, ok := grpcutil.ExtraMetadata(tt.info.Context); ok { + md = metadata.Join(md, extraMD) + for k := range md { + if strings.HasSuffix(k, "-bin") { + delete(md, k) + } + } + } + if got := a.Match(tt.info.Method, md); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func (s) TestFractionMatcherMatch(t *testing.T) { + const fraction = 500000 + fm := newFractionMatcher(fraction) + defer func() { + RandInt64n = rand.Int64N + }() + + // rand > fraction, should return false. + RandInt64n = func(int64) int64 { + return fraction + 1 + } + if matched := fm.match(); matched { + t.Errorf("match() = %v, want not match", matched) + } + + // rand == fraction, should return true. + RandInt64n = func(int64) int64 { + return fraction + } + if matched := fm.match(); !matched { + t.Errorf("match() = %v, want match", matched) + } + + // rand < fraction, should return true. + RandInt64n = func(int64) int64 { + return fraction - 1 + } + if matched := fm.match(); !matched { + t.Errorf("match() = %v, want match", matched) + } +} + +func (s) TestMatchTypeForDomain(t *testing.T) { + tests := []struct { + d string + want domainMatchType + }{ + {d: "", want: domainMatchTypeInvalid}, + {d: "*", want: domainMatchTypeUniversal}, + {d: "bar.*", want: domainMatchTypePrefix}, + {d: "*.abc.com", want: domainMatchTypeSuffix}, + {d: "foo.bar.com", want: domainMatchTypeExact}, + {d: "foo.*.com", want: domainMatchTypeInvalid}, + } + for _, tt := range tests { + if got := matchTypeForDomain(tt.d); got != tt.want { + t.Errorf("matchTypeForDomain(%q) = %v, want %v", tt.d, got, tt.want) + } + } +} + +func (s) TestMatch(t *testing.T) { + tests := []struct { + name string + domain string + host string + wantTyp domainMatchType + wantMatched bool + }{ + {name: "invalid-empty", domain: "", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false}, + {name: "invalid", domain: "a.*.b", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false}, + {name: "universal", domain: "*", host: "abc.com", wantTyp: domainMatchTypeUniversal, wantMatched: true}, + {name: "prefix-match", domain: "abc.*", host: "abc.123", wantTyp: domainMatchTypePrefix, wantMatched: true}, + {name: "prefix-no-match", domain: "abc.*", host: "abcd.123", wantTyp: domainMatchTypePrefix, wantMatched: false}, + {name: "suffix-match", domain: "*.123", host: "abc.123", wantTyp: domainMatchTypeSuffix, wantMatched: true}, + {name: "suffix-no-match", domain: "*.123", host: "abc.1234", wantTyp: domainMatchTypeSuffix, wantMatched: false}, + {name: "exact-match", domain: "foo.bar", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: true}, + {name: "exact-no-match", domain: "foo.bar.com", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotTyp, gotMatched := match(tt.domain, tt.host); gotTyp != tt.wantTyp || gotMatched != tt.wantMatched { + t.Errorf("match() = %v, %v, want %v, %v", gotTyp, gotMatched, tt.wantTyp, tt.wantMatched) + } + }) + } +} + +func (s) TestFindBestMatchingVirtualHost(t *testing.T) { + var ( + oneExactMatch = &VirtualHost{Domains: []string{"foo.bar.com"}} + oneSuffixMatch = &VirtualHost{Domains: []string{"*.bar.com"}} + onePrefixMatch = &VirtualHost{Domains: []string{"foo.bar.*"}} + oneUniversalMatch = &VirtualHost{Domains: []string{"*"}} + longExactMatch = &VirtualHost{Domains: []string{"v2.foo.bar.com"}} + multipleMatch = &VirtualHost{Domains: []string{"pi.foo.bar.com", "314.*", "*.159"}} + vhs = []*VirtualHost{oneExactMatch, oneSuffixMatch, onePrefixMatch, oneUniversalMatch, longExactMatch, multipleMatch} + ) + + tests := []struct { + name string + host string + vHosts []*VirtualHost + want *VirtualHost + }{ + {name: "exact-match", host: "foo.bar.com", vHosts: vhs, want: oneExactMatch}, + {name: "suffix-match", host: "123.bar.com", vHosts: vhs, want: oneSuffixMatch}, + {name: "prefix-match", host: "foo.bar.org", vHosts: vhs, want: onePrefixMatch}, + {name: "universal-match", host: "abc.123", vHosts: vhs, want: oneUniversalMatch}, + {name: "long-exact-match", host: "v2.foo.bar.com", vHosts: vhs, want: longExactMatch}, + // Matches suffix "*.bar.com" and exact "pi.foo.bar.com". Takes exact. + {name: "multiple-match-exact", host: "pi.foo.bar.com", vHosts: vhs, want: multipleMatch}, + // Matches suffix "*.159" and prefix "foo.bar.*". Takes suffix. + {name: "multiple-match-suffix", host: "foo.bar.159", vHosts: vhs, want: multipleMatch}, + // Matches suffix "*.bar.com" and prefix "314.*". Takes suffix. + {name: "multiple-match-prefix", host: "314.bar.com", vHosts: vhs, want: oneSuffixMatch}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := FindBestMatchingVirtualHost(tt.host, tt.vHosts); !cmp.Equal(got, tt.want, cmp.Comparer(proto.Equal)) { + t.Errorf("FindBestMatchingxdsclient.VirtualHost() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/xds/xdsclient/xdsresource/metadata.go b/internal/xds/xdsclient/xdsresource/metadata.go new file mode 100644 index 0000000..bde5eb7 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/metadata.go @@ -0,0 +1,164 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + "net/netip" + + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/protobuf/types/known/anypb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3gcpauthnpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/gcp_authn/v3" +) + +func init() { + if envconfig.XDSHTTPConnectEnabled { + registerMetadataConverter(version.V3AddressURL, proxyAddressConverter{}) + } + if envconfig.GCPAuthenticationFilterEnabled { + registerMetadataConverter(version.V3AudienceURL, audienceConverter{}) + } +} + +var ( + // metadataRegistry is a map from proto type to metadataConverter. + metadataRegistry = make(map[string]metadataConverter) +) + +// metadataConverter converts xds metadata entries in +// Metadata.typed_filter_metadata into an internal form with the fields relevant +// to gRPC. +type metadataConverter interface { + // convert parses the Any proto into a concrete struct. + convert(*anypb.Any) (any, error) +} + +// registerMetadataConverter registers the converter to the map keyed on a proto +// type_url. Must be called at init time. Not thread safe. +func registerMetadataConverter(protoType string, c metadataConverter) { + metadataRegistry[protoType] = c +} + +// metadataConverterForType retrieves a converter based on key given. +func metadataConverterForType(typeURL string) metadataConverter { + return metadataRegistry[typeURL] +} + +// RegisterMetadataConverterForTesting registers the converter for testing +// purposes and returns a cleanup function to restore the registry to its +// previous state. +func RegisterMetadataConverterForTesting(typeURL string) (func(), error) { + var conv metadataConverter + switch typeURL { + case version.V3AddressURL: + conv = proxyAddressConverter{} + case version.V3AudienceURL: + conv = audienceConverter{} + default: + return nil, fmt.Errorf("unknown typeURL for testing: %s", typeURL) + } + curConverter, found := metadataRegistry[typeURL] + registerMetadataConverter(typeURL, conv) + return func() { + if found { + metadataRegistry[typeURL] = curConverter + return + } + delete(metadataRegistry, typeURL) + }, nil +} + +// UnregisterMetadataConverterForTesting unregisters the converter for testing +// purposes and returns a cleanup function to restore the registry to its +// previous state. +func UnregisterMetadataConverterForTesting(typeURL string) func() { + curConverter, found := metadataRegistry[typeURL] + delete(metadataRegistry, typeURL) + return func() { + if found { + metadataRegistry[typeURL] = curConverter + } + } +} + +// StructMetadataValue stores the values in a google.protobuf.Struct from +// FilterMetadata. +type StructMetadataValue struct { + // Data stores the parsed JSON representation of a google.protobuf.Struct. + Data map[string]any +} + +// ProxyAddressMetadataValue holds the address parsed from the +// envoy.config.core.v3.Address proto message, as specified in gRFC A86. +type ProxyAddressMetadataValue struct { + // Address stores the proxy address configured (A86). It will be in the form + // of host:port. It has to be either IPv6 or IPv4. + Address string +} + +// proxyAddressConvertor implements the metadataConverter interface to handle +// the conversion of envoy.config.core.v3.Address protobuf messages into an +// internal representation. +type proxyAddressConverter struct{} + +func (proxyAddressConverter) convert(anyProto *anypb.Any) (any, error) { + addressProto := &v3corepb.Address{} + if err := anyProto.UnmarshalTo(addressProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource from Any proto: %v", err) + } + socketaddress := addressProto.GetSocketAddress() + if socketaddress == nil { + return nil, fmt.Errorf("no socket_address field in metadata") + } + if _, err := netip.ParseAddr(socketaddress.GetAddress()); err != nil { + return nil, fmt.Errorf("address field is not a valid IPv4 or IPv6 address: %q", socketaddress.GetAddress()) + } + portvalue := socketaddress.GetPortValue() + if portvalue == 0 { + return nil, fmt.Errorf("port value not set in socket_address") + } + return ProxyAddressMetadataValue{Address: parseAddress(socketaddress)}, nil +} + +// AudienceMetadataValue holds the audience parsed from the +// envoy.extensions.filters.http.gcp_authn.v3.Audience proto message, as +// specified in gRFC A83. +type AudienceMetadataValue struct { + // Audience is the URL of the receiving service that performs token + // authentication. + Audience string +} + +// audienceConverter implements the metadataConverter interface to +// handle the conversion of envoy.extensions.filters.http.gcp_authn.v3.Audience +// protobuf messages into an internal representation. +type audienceConverter struct{} + +func (audienceConverter) convert(anyProto *anypb.Any) (any, error) { + audienceProto := &v3gcpauthnpb.Audience{} + if err := anyProto.UnmarshalTo(audienceProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal the envoy.extensions.filters.http.gcp_authn.v3.Audience resource from Any proto: %v", err) + } + if audienceProto.GetUrl() == "" { + return nil, fmt.Errorf("empty url field in audience metadata") + } + return AudienceMetadataValue{Audience: audienceProto.GetUrl()}, nil +} diff --git a/internal/xds/xdsclient/xdsresource/metadata_test.go b/internal/xds/xdsclient/xdsresource/metadata_test.go new file mode 100644 index 0000000..58fba08 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/metadata_test.go @@ -0,0 +1,259 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package xdsresource + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/protobuf/types/known/anypb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3gcpauthnpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/gcp_authn/v3" +) + +func (s) TestProxyAddressConverterSuccess(t *testing.T) { + cleanup, err := RegisterMetadataConverterForTesting(version.V3AddressURL) + if err != nil { + t.Fatalf("RegisterMetadataConverterForTesting(%q) failed: %v", version.V3AddressURL, err) + } + t.Cleanup(cleanup) + converter := metadataConverterForType(version.V3AddressURL) + if converter == nil { + t.Fatalf("Converter for %q not found in registry", version.V3AddressURL) + } + tests := []struct { + name string + addr *v3corepb.Address + want ProxyAddressMetadataValue + }{ + { + name: "valid_IPv4_address_and_port", + addr: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "192.168.1.1", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 8080, + }, + }, + }, + }, + want: ProxyAddressMetadataValue{ + Address: "192.168.1.1:8080", + }, + }, + { + name: "valid_full_IPv6_address_and_port", + addr: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "2001:0db8:85a3:0000:0000:8a2e:0370:7334", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 9090, + }, + }, + }, + }, + want: ProxyAddressMetadataValue{ + Address: "[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9090", + }, + }, + { + name: "valid_shortened_IPv6_address", + addr: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "2001:db8::1", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 9090, + }, + }, + }, + }, + want: ProxyAddressMetadataValue{ + Address: "[2001:db8::1]:9090", + }, + }, + { + name: "valid_link-local_IPv6_address", + addr: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "fe80::1ff:fe23:4567:890a", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 8888, + }, + }, + }, + }, + want: ProxyAddressMetadataValue{ + Address: "[fe80::1ff:fe23:4567:890a]:8888", + }, + }, + { + name: "valid_IPv4-mapped_IPv6_address", + addr: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "::ffff:192.0.2.128", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 1234, + }, + }, + }, + }, + want: ProxyAddressMetadataValue{ + Address: "[::ffff:192.0.2.128]:1234", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + anyProto := testutils.MarshalAny(t, tt.addr) + got, err := converter.convert(anyProto) + if err != nil { + t.Fatalf("convert() failed with error: %v", err) + } + if diff := cmp.Diff(tt.want, got, cmp.AllowUnexported(ProxyAddressMetadataValue{})); diff != "" { + t.Errorf("convert() returned unexpected value (-want +got):\n%s", diff) + } + }) + } +} + +func (s) TestProxyAddressConverterFailure(t *testing.T) { + cleanup, err := RegisterMetadataConverterForTesting(version.V3AddressURL) + if err != nil { + t.Fatalf("RegisterMetadataConverterForTesting(%q) failed: %v", version.V3AddressURL, err) + } + t.Cleanup(cleanup) + converter := metadataConverterForType(version.V3AddressURL) + if converter == nil { + t.Fatalf("Converter for %q not found in registry", version.V3AddressURL) + } + tests := []struct { + name string + addr *v3corepb.Address + wantErr string + }{ + { + name: "invalid_address", + addr: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "invalid-ip", + }, + }, + }, + wantErr: "address field is not a valid IPv4 or IPv6 address: \"invalid-ip\"", + }, + { + name: "missing_socket_address", + addr: &v3corepb.Address{ + // No SocketAddress field set. + }, + wantErr: "no socket_address field in metadata", + }, + { + name: "address_is_not_a_socket_address", + addr: &v3corepb.Address{ + Address: &v3corepb.Address_EnvoyInternalAddress{ + EnvoyInternalAddress: &v3corepb.EnvoyInternalAddress{ + AddressNameSpecifier: &v3corepb.EnvoyInternalAddress_ServerListenerName{ + ServerListenerName: "some-internal-listener", + }, + }, + }, + }, + wantErr: "no socket_address field in metadata", + }, + { + name: "port_value_not_set", + addr: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "127.0.0.1", + PortSpecifier: nil, + }, + }, + }, + wantErr: "port value not set in socket_address", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + anyProto := testutils.MarshalAny(t, tt.addr) + _, err := converter.convert(anyProto) + if err == nil || err.Error() != tt.wantErr { + t.Errorf("convert() got error = %v, wantErr = %q", err, tt.wantErr) + } + }) + } +} + +func (s) TestAudienceConverterSuccess(t *testing.T) { + cleanup, err := RegisterMetadataConverterForTesting(version.V3AudienceURL) + if err != nil { + t.Fatalf("RegisterMetadataConverterForTesting(%q) failed: %v", version.V3AudienceURL, err) + } + t.Cleanup(cleanup) + converter := metadataConverterForType(version.V3AudienceURL) + if converter == nil { + t.Fatalf("Converter for %q not found in registry", version.V3AudienceURL) + } + tests := []struct { + name string + audience *anypb.Any + want AudienceMetadataValue + wantErr string + }{ + { + name: "valid_audience", + audience: testutils.MarshalAny(t, &v3gcpauthnpb.Audience{Url: "https://example.com"}), + want: AudienceMetadataValue{Audience: "https://example.com"}, + }, + { + name: "empty_audience", + audience: testutils.MarshalAny(t, &v3gcpauthnpb.Audience{Url: ""}), + want: AudienceMetadataValue{Audience: ""}, + wantErr: "empty url field in audience metadata", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := converter.convert(tt.audience) + if tt.wantErr != "" { + if err == nil || err.Error() != tt.wantErr { + t.Errorf("convert() got error = %v, wantErr = %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("convert() failed with error: %v", err) + } + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("convert(%s) returned unexpected diff (-want +got):\n%s", tt.audience.GetTypeUrl(), diff) + } + }) + } +} diff --git a/internal/xds/xdsclient/xdsresource/name.go b/internal/xds/xdsclient/xdsresource/name.go new file mode 100644 index 0000000..24200ea --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/name.go @@ -0,0 +1,127 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "net/url" + "sort" + "strings" +) + +// FederationScheme is the scheme of a federation resource name. +const FederationScheme = "xdstp" + +// Name contains the parsed component of an xDS resource name. +// +// An xDS resource name is in the format of +// xdstp://[{authority}]/{resource type}/{id/*}?{context parameters}{#processing directive,*} +// +// See +// https://github.com/cncf/xds/blob/main/proposals/TP1-xds-transport-next.md#uri-based-xds-resource-names +// for details, and examples. +type Name struct { + Scheme string + Authority string + Type string + ID string + + ContextParams map[string]string + + processingDirective string +} + +// ParseName splits the name and returns a struct representation of the Name. +// +// If the name isn't a valid new-style xDS name, field ID is set to the input. +// Note that this is not an error, because we still support the old-style +// resource names (those not starting with "xdstp:"). +// +// The caller can tell if the parsing is successful by checking the returned +// Scheme. +func ParseName(name string) *Name { + if !strings.Contains(name, "://") { + // Only the long form URL, with ://, is valid. + return &Name{ID: name} + } + parsed, err := url.Parse(name) + if err != nil { + return &Name{ID: name} + } + + ret := &Name{ + Scheme: parsed.Scheme, + Authority: parsed.Host, + } + split := strings.SplitN(parsed.Path, "/", 3) + if len(split) < 3 { + // Path is in the format of "/type/id". There must be at least 3 + // segments after splitting. + return &Name{ID: name} + } + ret.Type = split[1] + ret.ID = split[2] + if len(parsed.Query()) != 0 { + ret.ContextParams = make(map[string]string) + for k, vs := range parsed.Query() { + if len(vs) > 0 { + // We only keep one value of each key. Behavior for multiple values + // is undefined. + ret.ContextParams[k] = vs[0] + } + } + } + // TODO: processing directive (the part comes after "#" in the URL, stored + // in parsed.RawFragment) is kept but not processed. Add support for that + // when it's needed. + ret.processingDirective = parsed.RawFragment + return ret +} + +// String returns a canonicalized string of name. The context parameters are +// sorted by the keys. +func (n *Name) String() string { + if n.Scheme == "" { + return n.ID + } + + // Sort and build query. + keys := make([]string, 0, len(n.ContextParams)) + for k := range n.ContextParams { + keys = append(keys, k) + } + sort.Strings(keys) + var pairs []string + for _, k := range keys { + pairs = append(pairs, strings.Join([]string{k, n.ContextParams[k]}, "=")) + } + rawQuery := strings.Join(pairs, "&") + + path := n.Type + if n.ID != "" { + path = "/" + path + "/" + n.ID + } + + tempURL := &url.URL{ + Scheme: n.Scheme, + Host: n.Authority, + Path: path, + RawQuery: rawQuery, + RawFragment: n.processingDirective, + } + return tempURL.String() +} diff --git a/internal/xds/xdsclient/xdsresource/name_test.go b/internal/xds/xdsclient/xdsresource/name_test.go new file mode 100644 index 0000000..61deb19 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/name_test.go @@ -0,0 +1,103 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +func TestParseName(t *testing.T) { + tests := []struct { + name string + in string + want *Name + wantStr string + }{ + { + name: "old style name", + in: "test-resource", + want: &Name{ID: "test-resource"}, + wantStr: "test-resource", + }, + { + name: "invalid not url", + in: "a:/b/c", + want: &Name{ID: "a:/b/c"}, + wantStr: "a:/b/c", + }, + { + name: "invalid no resource type", + in: "xdstp://auth/id", + want: &Name{ID: "xdstp://auth/id"}, + wantStr: "xdstp://auth/id", + }, + { + name: "valid with no authority", + in: "xdstp:///type/id", + want: &Name{Scheme: "xdstp", Authority: "", Type: "type", ID: "id"}, + wantStr: "xdstp:///type/id", + }, + { + name: "valid no ctx params", + in: "xdstp://auth/type/id", + want: &Name{Scheme: "xdstp", Authority: "auth", Type: "type", ID: "id"}, + wantStr: "xdstp://auth/type/id", + }, + { + name: "valid with ctx params", + in: "xdstp://auth/type/id?a=1&b=2", + want: &Name{Scheme: "xdstp", Authority: "auth", Type: "type", ID: "id", ContextParams: map[string]string{"a": "1", "b": "2"}}, + wantStr: "xdstp://auth/type/id?a=1&b=2", + }, + { + name: "valid with ctx params sorted by keys", + in: "xdstp://auth/type/id?b=2&a=1", + want: &Name{Scheme: "xdstp", Authority: "auth", Type: "type", ID: "id", ContextParams: map[string]string{"a": "1", "b": "2"}}, + wantStr: "xdstp://auth/type/id?a=1&b=2", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ParseName(tt.in) + if !cmp.Equal(got, tt.want, cmpopts.IgnoreFields(Name{}, "processingDirective")) { + t.Errorf("ParseName() = %#v, want %#v", got, tt.want) + } + if gotStr := got.String(); gotStr != tt.wantStr { + t.Errorf("Name.String() = %s, want %s", gotStr, tt.wantStr) + } + }) + } +} + +// TestNameStringCtxParamsOrder covers the case that if two names differ only in +// context parameter __order__, the parsed name.String() has the same value. +func TestNameStringCtxParamsOrder(t *testing.T) { + const ( + a = "xdstp://auth/type/id?a=1&b=2" + b = "xdstp://auth/type/id?b=2&a=1" + ) + aParsed := ParseName(a).String() + bParsed := ParseName(b).String() + + if aParsed != bParsed { + t.Fatalf("aParsed.String() = %q, bParsed.String() = %q, want them to be the same", aParsed, bParsed) + } +} diff --git a/internal/xds/xdsclient/xdsresource/resource_type.go b/internal/xds/xdsclient/xdsresource/resource_type.go new file mode 100644 index 0000000..05f0d58 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/resource_type.go @@ -0,0 +1,139 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package xdsresource implements the xDS data model layer. +// +// Provides resource-type specific functionality to unmarshal xDS protos into +// internal data structures that contain only fields gRPC is interested in. +// These internal data structures are passed to components in the xDS stack +// (resolver/balancers/server) that have expressed interest in receiving +// updates to specific resources. +package xdsresource + +import ( + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/protobuf/types/known/anypb" +) + +// Producer contains a single method to discover resource configuration from a +// remote management server using xDS APIs. +// +// The xdsclient package provides a concrete implementation of this interface. +type Producer interface { + // WatchResource uses xDS to discover the resource associated with the + // provided resource name. The resource type implementation determines how + // xDS responses are are deserialized and validated, as received from the + // xDS management server. Upon receipt of a response from the management + // server, an appropriate callback on the watcher is invoked. + WatchResource(typeURL, resourceName string, watcher xdsclient.ResourceWatcher) (cancel func()) +} + +// ResourceWatcher is notified of the resource updates and errors that are +// received by the xDS client from the management server. +// +// All methods contain a done parameter which should be called when processing +// of the update has completed. For example, if processing a resource requires +// watching new resources, registration of those new watchers should be +// completed before done is called, which can happen after the ResourceWatcher +// method has returned. Failure to call done will prevent the xDS client from +// providing future ResourceWatcher notifications. +type ResourceWatcher interface { + // ResourceChanged indicates a new version of the resource is available. + ResourceChanged(resourceData ResourceData, done func()) + + // ResourceError indicates an error occurred while trying to fetch or + // decode the associated resource. The previous version of the resource + // should be considered invalid. + ResourceError(err error, done func()) + + // AmbientError indicates an error occurred after a resource has been + // received that should not modify the use of that resource but may provide + // useful information about the state of the XDSClient for debugging + // purposes. The previous version of the resource should still be + // considered valid. + AmbientError(err error, done func()) +} + +// TODO: Once the implementation is complete, rename this interface as +// ResourceType and get rid of the existing ResourceType enum. + +// Type wraps all resource-type specific functionality. Each supported resource +// type will provide an implementation of this interface. +type Type interface { + // TypeURL is the xDS type URL of this resource type for v3 transport. + TypeURL() string + + // TypeName identifies resources in a transport protocol agnostic way. This + // can be used for logging/debugging purposes, as well in cases where the + // resource type name is to be uniquely identified but the actual + // functionality provided by the resource type is not required. + // + // TODO: once Type is renamed to ResourceType, rename TypeName to + // ResourceTypeName. + TypeName() string + + // AllResourcesRequiredInSotW indicates whether this resource type requires + // that all resources be present in every SotW response from the server. If + // true, a response that does not include a previously seen resource will be + // interpreted as a deletion of that resource. + AllResourcesRequiredInSotW() bool + + // Decode deserializes and validates an xDS resource serialized inside the + // provided `Any` proto, as received from the xDS management server. + // + // If protobuf deserialization fails or resource validation fails, + // returns a non-nil error. Otherwise, returns a fully populated + // DecodeResult. + Decode(*DecodeOptions, *anypb.Any) (*DecodeResult, error) +} + +// ResourceData contains the configuration data sent by the xDS management +// server, associated with the resource being watched. Every resource type must +// provide an implementation of this interface to represent the configuration +// received from the xDS management server. +type ResourceData interface { + // RawEqual returns true if the passed in resource data is equal to that of + // the receiver, based on the underlying raw protobuf message. + RawEqual(ResourceData) bool + + // ToJSON returns a JSON string representation of the resource data. + ToJSON() string + + Raw() *anypb.Any +} + +// DecodeOptions wraps the options required by ResourceType implementation for +// decoding configuration received from the xDS management server. +type DecodeOptions struct { + // BootstrapConfig contains the complete bootstrap configuration passed to + // the xDS client. This contains useful data for resource validation. + BootstrapConfig *bootstrap.Config + // ServerConfig contains the server config (from the above bootstrap + // configuration) of the xDS server from which the current resource, for + // which Decode() is being invoked, was received. + ServerConfig *bootstrap.ServerConfig +} + +// DecodeResult is the result of a decode operation. +type DecodeResult struct { + // Name is the name of the resource being watched. + Name string + // Resource contains the configuration associated with the resource being + // watched. + Resource ResourceData +} diff --git a/internal/xds/xdsclient/xdsresource/route_config_resource_type.go b/internal/xds/xdsclient/xdsresource/route_config_resource_type.go new file mode 100644 index 0000000..2b392b8 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/route_config_resource_type.go @@ -0,0 +1,132 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "bytes" + + "google.golang.org/grpc/internal/xds/bootstrap" + xdsclient "google.golang.org/grpc/internal/xds/clients/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" +) + +const ( + // RouteConfigTypeName represents the transport agnostic name for the + // route config resource. + RouteConfigTypeName = "RouteConfigResource" +) + +// routeConfigResourceDecoder is an implementation of the xdsclient.Decoder +// interface for route configuration resources. +type routeConfigResourceDecoder struct { + bootstrapConfig *bootstrap.Config + serverConfigs map[xdsclient.ServerConfig]*bootstrap.ServerConfig +} + +func (d *routeConfigResourceDecoder) Decode(resource *xdsclient.AnyProto, opts xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) { + var serverCfg *bootstrap.ServerConfig + if opts.ServerConfig != nil { + serverCfg = d.serverConfigs[*opts.ServerConfig] + } + name, rc, err := unmarshalRouteConfigResource(resource.ToAny(), d.bootstrapConfig, serverCfg) + if name == "" { + // Name is unset only when protobuf deserialization fails. + return nil, err + } + if err != nil { + // Protobuf deserialization succeeded, but resource validation failed. + return &xdsclient.DecodeResult{ + Name: name, + Resource: &RouteConfigResourceData{Resource: RouteConfigUpdate{}}, + }, err + } + + return &xdsclient.DecodeResult{ + Name: name, + Resource: &RouteConfigResourceData{Resource: rc}, + }, nil +} + +// RouteConfigResourceData is an implementation of the xdsclient.ResourceData +// interface for route configuration resources. +type RouteConfigResourceData struct { + Resource RouteConfigUpdate +} + +// Equal returns true if other is equal to er. +func (r *RouteConfigResourceData) Equal(other xdsclient.ResourceData) bool { + if other == nil { + return false + } + return bytes.Equal(r.Bytes(), other.Bytes()) +} + +// Bytes returns the protobuf serialized bytes of the route config resource proto. +func (r *RouteConfigResourceData) Bytes() []byte { + return r.Resource.Raw.GetValue() +} + +// RouteConfigWatcher wraps the callbacks to be invoked for different +// events corresponding to the route configuration resource being watched. gRFC +// A88 contains an exhaustive list of what method is invoked under what +// conditions. +type RouteConfigWatcher interface { + // ResourceChanged indicates a new version of the resource is available. + ResourceChanged(resource *RouteConfigUpdate, done func()) + + // ResourceError indicates an error occurred while trying to fetch or + // decode the associated resource. The previous version of the resource + // should be considered invalid. + ResourceError(err error, done func()) + + // AmbientError indicates an error occurred after a resource has been + // received that should not modify the use of that resource but may provide + // useful information about the state of the XDSClient for debugging + // purposes. The previous version of the resource should still be + // considered valid. + AmbientError(err error, done func()) +} + +type delegatingRouteConfigWatcher struct { + watcher RouteConfigWatcher +} + +func (d *delegatingRouteConfigWatcher) ResourceChanged(data xdsclient.ResourceData, onDone func()) { + rc := data.(*RouteConfigResourceData) + d.watcher.ResourceChanged(&rc.Resource, onDone) +} + +func (d *delegatingRouteConfigWatcher) ResourceError(err error, onDone func()) { + d.watcher.ResourceError(err, onDone) +} + +func (d *delegatingRouteConfigWatcher) AmbientError(err error, onDone func()) { + d.watcher.AmbientError(err, onDone) +} + +// WatchRouteConfig uses xDS to discover the configuration associated with the +// provided route configuration resource name. +func WatchRouteConfig(p Producer, name string, w RouteConfigWatcher) (cancel func()) { + return p.WatchResource(version.V3RouteConfigURL, name, &delegatingRouteConfigWatcher{watcher: w}) +} + +// NewRouteConfigResourceTypeDecoder returns a xdsclient.Decoder that wraps +// the xdsresource.routeConfigType. +func NewRouteConfigResourceTypeDecoder(bc *bootstrap.Config, gServerCfgMap map[xdsclient.ServerConfig]*bootstrap.ServerConfig) xdsclient.Decoder { + return &routeConfigResourceDecoder{bootstrapConfig: bc, serverConfigs: gServerCfgMap} +} diff --git a/internal/xds/xdsclient/xdsresource/test_utils_test.go b/internal/xds/xdsclient/xdsresource/test_utils_test.go new file mode 100644 index 0000000..716558d --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/test_utils_test.go @@ -0,0 +1,124 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "net/netip" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/httpfilter/router" + "google.golang.org/protobuf/testing/protocmp" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const defaultTestTimeout = 10 * time.Second + +var ( + cmpOpts = cmp.Options{ + cmpopts.EquateEmpty(), + cmpopts.EquateComparable(netip.Addr{}, netip.Prefix{}), + cmp.FilterValues(func(_, _ error) bool { return true }, cmpopts.EquateErrors()), + cmp.Comparer(func(_, _ time.Time) bool { return true }), + protocmp.Transform(), + } + routeConfig = &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"lds.target.good:3333"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}} + inlineRouteConfig = &RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{{ + Domains: []string{"lds.target.good:3333"}, + Routes: []*Route{{Prefix: newStringP("/"), ActionType: RouteActionNonForwardingAction}}, + }}} + + validServerSideHTTPFilter1 = &v3httppb.HttpFilter{ + Name: "serverOnlyCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig}, + } + validServerSideHTTPFilter2 = &v3httppb.HttpFilter{ + Name: "serverOnlyCustomFilter2", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig}, + } + emptyRouterFilter = e2e.RouterHTTPFilter + localSocketAddress = &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "0.0.0.0", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 9999, + }, + }, + }, + } +) + +func makeRouterFilter(t *testing.T) HTTPFilter { + t.Helper() + routerBuilder := httpfilter.Get(router.TypeURL) + routerConfig, err := routerBuilder.ParseFilterConfig(testutils.MarshalAny(t, &v3routerpb.Router{}), httpfilter.ParseOptions{}) + if err != nil { + t.Fatalf("Failed to parse Router filter configuration: %v", err) + } + return HTTPFilter{Name: "router", Filter: routerBuilder, Config: routerConfig} +} + +func makeRouterFilterList(t *testing.T) []HTTPFilter { + return []HTTPFilter{makeRouterFilter(t)} +} + +func emptyValidNetworkFilters(t *testing.T) []*v3listenerpb.Filter { + return []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + } +} diff --git a/internal/xds/xdsclient/xdsresource/tests/unmarshal_cds_test.go b/internal/xds/xdsclient/xdsresource/tests/unmarshal_cds_test.go new file mode 100644 index 0000000..c25ddf6 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/tests/unmarshal_cds_test.go @@ -0,0 +1,575 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package tests_test contains test cases for unmarshalling of CDS resources. +package tests_test + +import ( + "encoding/json" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/balancer/leastrequest" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + iringhash "google.golang.org/grpc/internal/ringhash" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + xdsinternal "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/internal/xds/balancer/wrrlocality" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" + v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3" + v3roundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/round_robin/v3" + v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3" + + _ "google.golang.org/grpc/balancer/roundrobin" // To register round_robin load balancer. + _ "google.golang.org/grpc/xds" // Register the xDS LB Registry Converters. +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + clusterName = "clusterName" + serviceName = "service" +) + +func wrrLocality(t *testing.T, m proto.Message) *v3wrrlocalitypb.WrrLocality { + return &v3wrrlocalitypb.WrrLocality{ + EndpointPickingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, m), + }, + }, + }, + }, + } +} + +func wrrLocalityAny(t *testing.T, m proto.Message) *anypb.Any { + return testutils.MarshalAny(t, wrrLocality(t, m)) +} + +type customLBConfig struct { + serviceconfig.LoadBalancingConfig +} + +// We have this test in a separate test package in order to not take a +// dependency on the internal xDS balancer packages within the xDS Client. +func (s) TestValidateCluster_Success(t *testing.T) { + const customLBPolicyName = "myorg.MyCustomLeastRequestPolicy" + stub.Register(customLBPolicyName, stub.BalancerFuncs{ + ParseConfig: func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return customLBConfig{}, nil + }, + }) + serverCfg, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: "test-server"}) + if err != nil { + t.Fatalf("Failed to create server config for testing: %v", err) + } + + tests := []struct { + name string + cluster *v3clusterpb.Cluster + serverCfg *bootstrap.ServerConfig + wantUpdate xdsresource.ClusterUpdate + wantLBConfig *iserviceconfig.BalancerConfig + }{ + { + name: "happy-case-logical-dns", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_LOGICAL_DNS}, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LoadAssignment: &v3endpointpb.ClusterLoadAssignment{ + Endpoints: []*v3endpointpb.LocalityLbEndpoints{{ + LbEndpoints: []*v3endpointpb.LbEndpoint{{ + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "dns_host", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 8080, + }, + }, + }, + }, + }, + }, + }}, + }}, + }, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + ClusterType: xdsresource.ClusterTypeLogicalDNS, + DNSHostName: "dns_host:8080", + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happy-case-aggregate-v3", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_ClusterType{ + ClusterType: &v3clusterpb.Cluster_CustomClusterType{ + Name: "envoy.clusters.aggregate", + TypedConfig: testutils.MarshalAny(t, &v3aggregateclusterpb.ClusterConfig{ + Clusters: []string{"a", "b", "c"}, + }), + }, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + ClusterType: xdsresource.ClusterTypeAggregate, + PrioritizedClusterNames: []string{"a", "b", "c"}, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happy-case-no-service-name-no-lrs", + cluster: e2e.DefaultCluster(clusterName, "", e2e.SecurityLevelNone), + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happy-case-no-lrs", + cluster: e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone), + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happiest-case-with-lrs", + cluster: e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: serviceName, + EnableLRS: true, + }), + serverCfg: serverCfg, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + LRSServerConfig: serverCfg, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happiest-case-with-circuitbreakers", + cluster: func() *v3clusterpb.Cluster { + c := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: serviceName, + EnableLRS: true, + }) + c.CircuitBreakers = &v3clusterpb.CircuitBreakers{ + Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{ + { + Priority: v3corepb.RoutingPriority_DEFAULT, + MaxRequests: wrapperspb.UInt32(512), + }, + { + Priority: v3corepb.RoutingPriority_HIGH, + MaxRequests: nil, + }, + }, + } + return c + }(), + serverCfg: serverCfg, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + LRSServerConfig: serverCfg, + MaxRequests: func() *uint32 { i := uint32(512); return &i }(), + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happiest-case-with-ring-hash-lb-policy-with-default-config", + cluster: func() *v3clusterpb.Cluster { + c := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone) + c.LbPolicy = v3clusterpb.Cluster_RING_HASH + return c + }(), + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: "ring_hash_experimental", + Config: &iringhash.LBConfig{ + MinRingSize: 1024, + MaxRingSize: 4096, + }, + }, + }, + { + name: "happiest-case-with-least-request-lb-policy-with-default-config", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: "least_request_experimental", + Config: &leastrequest.LBConfig{ + ChoiceCount: 2, + }, + }, + }, + { + name: "happiest-case-with-ring-hash-lb-policy-with-none-default-config", + cluster: func() *v3clusterpb.Cluster { + c := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone) + c.LbPolicy = v3clusterpb.Cluster_RING_HASH + c.LbConfig = &v3clusterpb.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &v3clusterpb.Cluster_RingHashLbConfig{ + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }, + } + return c + }(), + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: "ring_hash_experimental", + Config: &iringhash.LBConfig{ + MinRingSize: 10, + MaxRingSize: 100, + }, + }, + }, + { + name: "happiest-case-with-least-request-lb-policy-with-none-default-config", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST, + LbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig_{ + LeastRequestLbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig{ + ChoiceCount: wrapperspb.UInt32(3), + }, + }, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: "least_request_experimental", + Config: &leastrequest.LBConfig{ + ChoiceCount: 3, + }, + }, + }, + { + name: "happiest-case-with-ring-hash-lb-policy-configured-through-LoadBalancingPolicy", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_XX_HASH, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }), + }, + }, + }, + }, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: "ring_hash_experimental", + Config: &iringhash.LBConfig{ + MinRingSize: 10, + MaxRingSize: 100, + }, + }, + }, + { + name: "happiest-case-with-wrrlocality-rr-child-configured-through-LoadBalancingPolicy", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: wrrLocalityAny(t, &v3roundrobinpb.RoundRobin{}), + }, + }, + }, + }, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happiest-case-with-custom-lb-configured-through-LoadBalancingPolicy", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: wrrLocalityAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/myorg.MyCustomLeastRequestPolicy", + Value: &structpb.Struct{}, + }), + }, + }, + }, + }, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "myorg.MyCustomLeastRequestPolicy", + Config: customLBConfig{}, + }, + }, + }, + }, + { + name: "load-balancing-policy-takes-precedence-over-lb-policy-and-enum", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_RING_HASH, + LbConfig: &v3clusterpb.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &v3clusterpb.Cluster_RingHashLbConfig{ + MinimumRingSize: wrapperspb.UInt64(20), + MaximumRingSize: wrapperspb.UInt64(200), + }, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_XX_HASH, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }), + }, + }, + }, + }, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: "ring_hash_experimental", + Config: &iringhash.LBConfig{ + MinRingSize: 10, + MaxRingSize: 100, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + update, err := xdsresource.ValidateClusterAndConstructClusterUpdateForTesting(test.cluster, test.serverCfg) + if err != nil { + t.Errorf("validateClusterAndConstructClusterUpdate(%+v) failed: %v", test.cluster, err) + } + // Ignore the raw JSON string into the cluster update. JSON bytes + // are nondeterministic (whitespace etc.) so we cannot reliably + // compare JSON bytes in a test. Thus, marshal into a Balancer + // Config struct and compare on that. Only need to test this JSON + // emission here, as this covers the possible output space. + if diff := cmp.Diff(update, test.wantUpdate, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.ClusterUpdate{}, "LBPolicy")); diff != "" { + t.Errorf("validateClusterAndConstructClusterUpdate(%+v) got diff: %v (-got, +want)", test.cluster, diff) + } + bc := &iserviceconfig.BalancerConfig{} + if err := json.Unmarshal(update.LBPolicy, bc); err != nil { + t.Fatalf("failed to unmarshal JSON: %v", err) + } + if diff := cmp.Diff(bc, test.wantLBConfig); diff != "" { + t.Fatalf("update.LBConfig got unexpected output, diff (-got +want): %v", diff) + } + }) + } +} diff --git a/internal/xds/xdsclient/xdsresource/type.go b/internal/xds/xdsclient/xdsresource/type.go new file mode 100644 index 0000000..7f85c1d --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/type.go @@ -0,0 +1,135 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "time" + + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +// UpdateValidatorFunc performs validations on update structs using +// context/logic available at the xdsClient layer. Since these validation are +// performed on internal update structs, they can be shared between different +// API clients. +type UpdateValidatorFunc func(any) error + +// UpdateMetadata contains the metadata for each update, including timestamp, +// raw message, and so on. +type UpdateMetadata struct { + // Status is the status of this resource, e.g. ACKed, NACKed, or + // Not_exist(removed). + Status ServiceStatus + // Version is the version of the xds response. Note that this is the version + // of the resource in use (previous ACKed). If a response is NACKed, the + // NACKed version is in ErrState. + Version string + // Timestamp is when the response is received. + Timestamp time.Time + // ErrState is set when the update is NACKed. + ErrState *UpdateErrorMetadata +} + +// IsListenerResource returns true if the provider URL corresponds to an xDS +// Listener resource. +func IsListenerResource(url string) bool { + return url == version.V3ListenerURL +} + +// IsHTTPConnManagerResource returns true if the provider URL corresponds to an xDS +// HTTPConnManager resource. +func IsHTTPConnManagerResource(url string) bool { + return url == version.V3HTTPConnManagerURL +} + +// IsRouteConfigResource returns true if the provider URL corresponds to an xDS +// RouteConfig resource. +func IsRouteConfigResource(url string) bool { + return url == version.V3RouteConfigURL +} + +// IsClusterResource returns true if the provider URL corresponds to an xDS +// Cluster resource. +func IsClusterResource(url string) bool { + return url == version.V3ClusterURL +} + +// IsEndpointsResource returns true if the provider URL corresponds to an xDS +// Endpoints resource. +func IsEndpointsResource(url string) bool { + return url == version.V3EndpointsURL +} + +// UnwrapResource unwraps and returns the inner resource if it's in a resource +// wrapper. The original resource is returned if it's not wrapped. +func UnwrapResource(r *anypb.Any) (*anypb.Any, error) { + url := r.GetTypeUrl() + if url != version.V3ResourceWrapperURL { + // Not wrapped. + return r, nil + } + inner := &v3discoverypb.Resource{} + if err := proto.Unmarshal(r.GetValue(), inner); err != nil { + return nil, err + } + return inner.Resource, nil +} + +// ServiceStatus is the status of the update. +type ServiceStatus int + +const ( + // ServiceStatusUnknown is the default state, before a watch is started for + // the resource. + ServiceStatusUnknown ServiceStatus = iota + // ServiceStatusRequested is when the watch is started, but before and + // response is received. + ServiceStatusRequested + // ServiceStatusNotExist is when the resource doesn't exist in + // state-of-the-world responses (e.g. LDS and CDS), which means the resource + // is removed by the management server. + ServiceStatusNotExist // Resource is removed in the server, in LDS/CDS. + // ServiceStatusACKed is when the resource is ACKed. + ServiceStatusACKed + // ServiceStatusNACKed is when the resource is NACKed. + ServiceStatusNACKed +) + +// UpdateErrorMetadata is part of UpdateMetadata. It contains the error state +// when a response is NACKed. +type UpdateErrorMetadata struct { + // Version is the version of the NACKed response. + Version string + // Err contains why the response was NACKed. + Err error + // Timestamp is when the NACKed response was received. + Timestamp time.Time +} + +// UpdateWithMD contains the raw message of the update and the metadata, +// including version, raw message, timestamp. +// +// This is to be used for config dump and CSDS, not directly by users (like +// resolvers/balancers). +type UpdateWithMD struct { + MD UpdateMetadata + Raw *anypb.Any +} diff --git a/internal/xds/xdsclient/xdsresource/type_cds.go b/internal/xds/xdsclient/xdsresource/type_cds.go new file mode 100644 index 0000000..6c35905 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/type_cds.go @@ -0,0 +1,228 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "encoding/json" + + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/protobuf/types/known/anypb" +) + +// ClusterType is the type of cluster from a received CDS response. +type ClusterType int + +const ( + // ClusterTypeEDS represents the EDS cluster type, which will delegate endpoint + // discovery to the management server. + ClusterTypeEDS ClusterType = iota + // ClusterTypeLogicalDNS represents the Logical DNS cluster type, which essentially + // maps to the gRPC behavior of using the DNS resolver with pick_first LB policy. + ClusterTypeLogicalDNS + // ClusterTypeAggregate represents the Aggregate Cluster type, which provides a + // prioritized list of clusters to use. It is used for failover between clusters + // with a different configuration. + ClusterTypeAggregate +) + +// ClusterUpdate contains information from a received CDS response, which is of +// interest to the registered CDS watcher. +type ClusterUpdate struct { + ClusterType ClusterType + // ClusterName is the clusterName being watched for through CDS. + ClusterName string + // EDSServiceName is an optional name for EDS. If it's not set, the balancer + // should watch ClusterName for the EDS resources. + EDSServiceName string + // LRSServerConfig contains configuration about the xDS server that sent + // this cluster resource. This is also the server where load reports are to + // be sent, for this cluster. + LRSServerConfig *bootstrap.ServerConfig + // SecurityCfg contains security configuration sent by the control plane. + SecurityCfg *SecurityConfig + // MaxRequests for circuit breaking, if any (otherwise nil). + MaxRequests *uint32 + // DNSHostName is used only for cluster type DNS. It's the DNS name to + // resolve in "host:port" form + DNSHostName string + // PrioritizedClusterNames is used only for cluster type aggregate. It represents + // a prioritized list of cluster names. + PrioritizedClusterNames []string + + // LBPolicy represents the locality and endpoint picking policy in JSON, + // which will be the child policy of xds_cluster_impl. + LBPolicy json.RawMessage + + // OutlierDetection is the outlier detection configuration for this cluster. + // If nil, it means this cluster does not use the outlier detection feature. + OutlierDetection json.RawMessage + + // Raw is the resource from the xds response. + Raw *anypb.Any + // TelemetryLabels are the string valued metadata of filter_metadata type + // "com.google.csm.telemetry_labels" with keys "service_name" or + // "service_namespace". + TelemetryLabels map[string]string + + // LRSReportEndpointMetrics specifies the subset of ORCA metrics that + // should be propagated to the LRS server. + LRSReportEndpointMetrics *LRSReportEndpointMetricsConfig + + // Metadata contains the metadata from the cluster resource. + Metadata map[string]any + + // IsHTTP11ProxyEnabled is true if the cluster transport socket is wrapped + // in Http11ProxyUpstreamTransport. + IsHTTP11ProxyEnabled bool +} + +// LRSReportEndpointMetricsConfig holds the configuration for propagating ORCA +// metrics to the LRS server. +type LRSReportEndpointMetricsConfig struct { + CPUUtilization bool + MemUtilization bool + ApplicationUtilization bool + // NamedMetricsAll specifies whether all named metrics should be propagated. + // If true, NamedMetrics is ignored. + NamedMetricsAll bool + // NamedMetrics specifies the set of named metrics to propagate when + // NamedMetricsAll is false. + NamedMetrics map[string]struct{} +} + +// Equal returns whether the two LRSReportEndpointMetricsConfig configurations +// are identical. +func (bmp *LRSReportEndpointMetricsConfig) Equal(other *LRSReportEndpointMetricsConfig) bool { + switch { + case bmp == nil && other == nil: + return true + case (bmp != nil) != (other != nil): + return false + } + if bmp.CPUUtilization != other.CPUUtilization || + bmp.MemUtilization != other.MemUtilization || + bmp.ApplicationUtilization != other.ApplicationUtilization || + bmp.NamedMetricsAll != other.NamedMetricsAll { + return false + } + if len(bmp.NamedMetrics) != len(other.NamedMetrics) { + return false + } + for k := range bmp.NamedMetrics { + if _, ok := other.NamedMetrics[k]; !ok { + return false + } + } + return true +} + +// SecurityConfig contains the security configuration received as part of the +// Cluster resource on the client-side, and as part of the Listener resource on +// the server-side. +type SecurityConfig struct { + // RootInstanceName identifies the certProvider plugin to be used to fetch + // root certificates. This instance name will be resolved to the plugin name + // and its associated configuration from the certificate_providers field of + // the bootstrap file. + RootInstanceName string + // RootCertName is the certificate name to be passed to the plugin (looked + // up from the bootstrap file) while fetching root certificates. + RootCertName string + // IdentityInstanceName identifies the certProvider plugin to be used to + // fetch identity certificates. This instance name will be resolved to the + // plugin name and its associated configuration from the + // certificate_providers field of the bootstrap file. + IdentityInstanceName string + // IdentityCertName is the certificate name to be passed to the plugin + // (looked up from the bootstrap file) while fetching identity certificates. + IdentityCertName string + // SubjectAltNameMatchers is an optional list of match criteria for SANs + // specified on the peer certificate. Used only on the client-side. + // + // Some intricacies: + // - If this field is empty, then any peer certificate is accepted. + // - If the peer certificate contains a wildcard DNS SAN, and an `exact` + // matcher is configured, a wildcard DNS match is performed instead of a + // regular string comparison. + SubjectAltNameMatchers []matcher.StringMatcher + // RequireClientCert indicates if the server handshake process expects the + // client to present a certificate. Set to true when performing mTLS. Used + // only on the server-side. + RequireClientCert bool + // UseSystemRootCerts indicates that the client should use system root + // certificates to validate the server certificate. This field is mutually + // exclusive with RootCertName and RootInstanceName. Validation performed + // after unmarshalling xDS resources ensures that this field is set only + // when both RootCertName and RootInstanceName are empty. + UseSystemRootCerts bool + // SNI is the string to be used as the Server Name when creating TLS + // configurations for the handshake. An empty string for SNI value will be + // treated as SNI not specified. + SNI string + // UseAutoHostSNI indicates whether to set the ServerName for the TLS handshake + // configuration to the hostname (if available). The host is the DNS + // hostname for DNS clusters, or Endpoint.hostname for EDS clusters. The + // port will not be included in the SNI value. + UseAutoHostSNI bool + // AutoSNISANValidation indicates whether to replace any Subject Alternative + // Name (SAN) matchers with a validation for a DNS SAN matching the SNI + // value sent. This validation uses the SNI being set in the TLS + // configuration, regardless of how the SNI is determined. + AutoSNISANValidation bool +} + +// Equal returns true if sc is equal to other. +func (sc *SecurityConfig) Equal(other *SecurityConfig) bool { + switch { + case sc == nil && other == nil: + return true + case (sc != nil) != (other != nil): + return false + } + switch { + case sc.RootInstanceName != other.RootInstanceName: + return false + case sc.RootCertName != other.RootCertName: + return false + case sc.IdentityInstanceName != other.IdentityInstanceName: + return false + case sc.IdentityCertName != other.IdentityCertName: + return false + case sc.RequireClientCert != other.RequireClientCert: + return false + case sc.UseSystemRootCerts != other.UseSystemRootCerts: + return false + case sc.SNI != other.SNI: + return false + case sc.UseAutoHostSNI != other.UseAutoHostSNI: + return false + case sc.AutoSNISANValidation != other.AutoSNISANValidation: + return false + default: + if len(sc.SubjectAltNameMatchers) != len(other.SubjectAltNameMatchers) { + return false + } + for i := 0; i < len(sc.SubjectAltNameMatchers); i++ { + if !sc.SubjectAltNameMatchers[i].Equal(other.SubjectAltNameMatchers[i]) { + return false + } + } + } + return true +} diff --git a/internal/xds/xdsclient/xdsresource/type_eds.go b/internal/xds/xdsclient/xdsresource/type_eds.go new file mode 100644 index 0000000..937d2cf --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/type_eds.go @@ -0,0 +1,78 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/resolver" + "google.golang.org/protobuf/types/known/anypb" +) + +// OverloadDropConfig contains the config to drop overloads. +type OverloadDropConfig struct { + Category string + Numerator uint32 + Denominator uint32 +} + +// EndpointHealthStatus represents the health status of an endpoint. +type EndpointHealthStatus int32 + +const ( + // EndpointHealthStatusUnknown represents HealthStatus UNKNOWN. + EndpointHealthStatusUnknown EndpointHealthStatus = iota + // EndpointHealthStatusHealthy represents HealthStatus HEALTHY. + EndpointHealthStatusHealthy + // EndpointHealthStatusUnhealthy represents HealthStatus UNHEALTHY. + EndpointHealthStatusUnhealthy + // EndpointHealthStatusDraining represents HealthStatus DRAINING. + EndpointHealthStatusDraining + // EndpointHealthStatusTimeout represents HealthStatus TIMEOUT. + EndpointHealthStatusTimeout + // EndpointHealthStatusDegraded represents HealthStatus DEGRADED. + EndpointHealthStatusDegraded +) + +// Endpoint contains information of an endpoint. +type Endpoint struct { + ResolverEndpoint resolver.Endpoint + HealthStatus EndpointHealthStatus + Weight uint32 + Metadata map[string]any +} + +// Locality contains information of a locality. +type Locality struct { + Endpoints []Endpoint + ID clients.Locality + Priority uint32 + Weight uint32 + Metadata map[string]any +} + +// EndpointsUpdate contains an EDS update. +type EndpointsUpdate struct { + Drops []OverloadDropConfig + // Localities in the EDS response with `load_balancing_weight` field not set + // or explicitly set to 0 are ignored while parsing the resource, and + // therefore do not show up here. + Localities []Locality + + // Raw is the resource from the xds response. + Raw *anypb.Any +} diff --git a/internal/xds/xdsclient/xdsresource/type_lds.go b/internal/xds/xdsclient/xdsresource/type_lds.go new file mode 100644 index 0000000..a03fd2d --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/type_lds.go @@ -0,0 +1,92 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "time" + + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/protobuf/types/known/anypb" +) + +// ListenerUpdate contains information received in an LDS response, which is of +// interest to the registered LDS watcher. +type ListenerUpdate struct { + // APIListener contains the HTTP connection manager configuration. + APIListener *HTTPConnectionManagerConfig + // TCPListener contains inbound listener configuration. + TCPListener *InboundListenerConfig + + // Raw is the resource from the xds response. + Raw *anypb.Any +} + +// HTTPConnectionManagerConfig contains the HTTP connection manager configuration. +type HTTPConnectionManagerConfig struct { + // RouteConfigName is the route configuration name corresponding to the + // target which is being watched through LDS. + // + // Exactly one of RouteConfigName and InlineRouteConfig is set. + RouteConfigName string + // InlineRouteConfig is the inline route configuration (RDS response) + // returned inside LDS. + // + // Exactly one of RouteConfigName and InlineRouteConfig is set. + InlineRouteConfig *RouteConfigUpdate + + // MaxStreamDuration contains the HTTP connection manager's + // common_http_protocol_options.max_stream_duration field, or zero if + // unset. + MaxStreamDuration time.Duration + // HTTPFilters is a list of HTTP filters (name, config) from the LDS + // response. + HTTPFilters []HTTPFilter +} + +// HTTPFilter represents one HTTP filter from an LDS response's HTTP connection +// manager field. +type HTTPFilter struct { + // Name is an arbitrary name of the filter. Used for applying override + // settings in virtual host / route / weighted cluster configuration (not + // yet supported). + Name string + // Filter is the HTTP filter found in the registry for the config type. + Filter httpfilter.Builder + // Config contains the filter's configuration + Config httpfilter.FilterConfig + // Disabled specifies if the filter is disabled. For more information, see + // envoyproxy.io/docs/envoy/latest/intro/arch_overview/http/http_filters#route-based-filter-chain + Disabled bool +} + +// InboundListenerConfig contains information about the inbound listener, i.e +// the server-side listener. +type InboundListenerConfig struct { + // Address is the local address on which the inbound listener is expected to + // accept incoming connections. + Address string + // Port is the local port on which the inbound listener is expected to + // accept incoming connections. + Port string + + // DefaultFilterChain is the default filter chain to use if no other filter + // chain matches. + DefaultFilterChain NetworkFilterChainConfig + // FilterChains contains the filter chains associated with this listener. + FilterChains NetworkFilterChainMap +} diff --git a/internal/xds/xdsclient/xdsresource/type_rds.go b/internal/xds/xdsclient/xdsresource/type_rds.go new file mode 100644 index 0000000..1cddfa5 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/type_rds.go @@ -0,0 +1,184 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "regexp" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/xds/clusterspecifier" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/protobuf/types/known/anypb" +) + +// RouteConfigUpdate contains information received in an RDS response, which is +// of interest to the registered RDS watcher. +type RouteConfigUpdate struct { + VirtualHosts []*VirtualHost + // ClusterSpecifierPlugins are the LB Configurations for any + // ClusterSpecifierPlugins referenced by the Route Table. + ClusterSpecifierPlugins map[string]clusterspecifier.BalancerConfig + // Raw is the resource from the xds response. + Raw *anypb.Any +} + +// VirtualHost contains the routes for a list of Domains. +// +// Note that the domains in this slice can be a wildcard, not an exact string. +// The consumer of this struct needs to find the best match for its hostname. +type VirtualHost struct { + Domains []string + // Routes contains a list of routes, each containing matchers and + // corresponding action. + Routes []*Route + // HTTPFilterConfigOverride contains any HTTP filter config overrides for + // the virtual host which may be present. An individual filter's override + // may be unused if the matching Route contains an override for that + // filter. + HTTPFilterConfigOverride map[string]httpfilter.FilterConfig + RetryConfig *RetryConfig +} + +// RetryConfig contains all retry-related configuration in either a VirtualHost +// or Route. +type RetryConfig struct { + // RetryOn is a set of status codes on which to retry. Only Canceled, + // DeadlineExceeded, Internal, ResourceExhausted, and Unavailable are + // supported; any other values will be omitted. + RetryOn map[codes.Code]bool + NumRetries uint32 // maximum number of retry attempts + RetryBackoff RetryBackoff // retry backoff policy +} + +// RetryBackoff describes the backoff policy for retries. +type RetryBackoff struct { + BaseInterval time.Duration // initial backoff duration between attempts + MaxInterval time.Duration // maximum backoff duration +} + +// HashPolicyType specifies the type of HashPolicy from a received RDS Response. +type HashPolicyType int + +const ( + // HashPolicyTypeHeader specifies to hash a Header in the incoming request. + HashPolicyTypeHeader HashPolicyType = iota + // HashPolicyTypeChannelID specifies to hash a unique Identifier of the + // Channel. This is a 64-bit random int computed at initialization time. + HashPolicyTypeChannelID +) + +// HashPolicy specifies the HashPolicy if the upstream cluster uses a hashing +// load balancer. +type HashPolicy struct { + HashPolicyType HashPolicyType + Terminal bool + // Fields used for type HEADER. + HeaderName string + Regex *regexp.Regexp + RegexSubstitution string +} + +// RouteActionType is the action of the route from a received RDS response. +type RouteActionType int + +const ( + // RouteActionUnsupported are routing types currently unsupported by grpc. + // According to A36, "A Route with an inappropriate action causes RPCs + // matching that route to fail." + RouteActionUnsupported RouteActionType = iota + // RouteActionRoute is the expected route type on the client side. Route + // represents routing a request to some upstream cluster. On the client + // side, if an RPC matches to a route that is not RouteActionRoute, the RPC + // will fail according to A36. + RouteActionRoute + // RouteActionNonForwardingAction is the expected route type on the server + // side. NonForwardingAction represents when a route will generate a + // response directly, without forwarding to an upstream host. + RouteActionNonForwardingAction +) + +// Route is both a specification of how to match a request as well as an +// indication of the action to take upon match. +type Route struct { + Path *string + Prefix *string + Regex *regexp.Regexp + // Indicates if prefix/path matching should be case insensitive. The default + // is false (case sensitive). + CaseInsensitive bool + Headers []*HeaderMatcher + Fraction *uint32 + + HashPolicies []*HashPolicy + + // If the matchers above indicate a match, the below configuration is used. + // If MaxStreamDuration is nil, it indicates neither of the route action's + // max_stream_duration fields (grpc_timeout_header_max nor + // max_stream_duration) were set. In this case, the ListenerUpdate's + // MaxStreamDuration field should be used. If MaxStreamDuration is set to + // an explicit zero duration, the application's deadline should be used. + MaxStreamDuration *time.Duration + // HTTPFilterConfigOverride contains any HTTP filter config overrides for + // the route which may be present. An individual filter's override may be + // unused if the matching WeightedCluster contains an override for that + // filter. + HTTPFilterConfigOverride map[string]httpfilter.FilterConfig + RetryConfig *RetryConfig + + ActionType RouteActionType + + // Only one of the following fields (WeightedClusters or + // ClusterSpecifierPlugin) will be set for a route. + WeightedClusters []WeightedCluster + // ClusterSpecifierPlugin is the name of the Cluster Specifier Plugin that + // this Route is linked to, if specified by xDS. + ClusterSpecifierPlugin string + // AutoHostRewrite indicates that the ":authority" header can be rewritten + // to the hostname of the upstream endpoint. + AutoHostRewrite bool +} + +// WeightedCluster contains settings for an xds ActionType.WeightedCluster. +type WeightedCluster struct { + // Name is the name of the cluster. + Name string + // Weight is the relative weight of the cluster. It will never be zero. + Weight uint32 + // HTTPFilterConfigOverride contains any HTTP filter config overrides for + // the weighted cluster which may be present. + HTTPFilterConfigOverride map[string]httpfilter.FilterConfig +} + +// HeaderMatcher represents header matchers. Exact, prefix, suffix and contains +// matches are represented as a StringMatch. +type HeaderMatcher struct { + Name string + InvertMatch *bool + RegexMatch *regexp.Regexp + RangeMatch *Int64Range + PresentMatch *bool + StringMatch *matcher.StringMatcher +} + +// Int64Range is a range for header range match. +type Int64Range struct { + Start int64 + End int64 +} diff --git a/internal/xds/xdsclient/xdsresource/unmarshal_cds.go b/internal/xds/xdsclient/xdsresource/unmarshal_cds.go new file mode 100644 index 0000000..ea6d001 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/unmarshal_cds.go @@ -0,0 +1,805 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "encoding/json" + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" + v3http11proxypb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/http_11_proxy/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/pretty" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/internal/xds/xdsclient/xdslbregistry" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" +) + +// ValidateClusterAndConstructClusterUpdateForTesting exports the +// validateClusterAndConstructClusterUpdate function for testing purposes. +var ValidateClusterAndConstructClusterUpdateForTesting = validateClusterAndConstructClusterUpdate + +const ( + maxSNILength = 255 + // TransportSocket proto message has a `name` field which is expected to be set + // to this value by the management server. + transportSocketName = "envoy.transport_sockets.tls" +) + +func unmarshalClusterResource(r *anypb.Any, serverCfg *bootstrap.ServerConfig) (string, ClusterUpdate, error) { + r, err := UnwrapResource(r) + if err != nil { + return "", ClusterUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err) + } + + if !IsClusterResource(r.GetTypeUrl()) { + return "", ClusterUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl()) + } + + cluster := &v3clusterpb.Cluster{} + if err := proto.Unmarshal(r.GetValue(), cluster); err != nil { + return "", ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + + if cluster.GetName() == "" { + return "", ClusterUpdate{}, fmt.Errorf("empty resource name in Cluster resource") + } + cu, err := validateClusterAndConstructClusterUpdate(cluster, serverCfg) + if err != nil { + return cluster.GetName(), ClusterUpdate{}, err + } + cu.Raw = r + + return cluster.GetName(), cu, nil +} + +const ( + defaultRingHashMinSize = 1024 + defaultRingHashMaxSize = 8 * 1024 * 1024 // 8M + ringHashSizeUpperBound = 8 * 1024 * 1024 // 8M + + defaultLeastRequestChoiceCount = 2 +) + +func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster, serverCfg *bootstrap.ServerConfig) (ClusterUpdate, error) { + telemetryLabels := make(map[string]string) + if fmd := cluster.GetMetadata().GetFilterMetadata(); fmd != nil { + if val, ok := fmd["com.google.csm.telemetry_labels"]; ok { + if fields := val.GetFields(); fields != nil { + if val, ok := fields["service_name"]; ok { + if _, ok := val.GetKind().(*structpb.Value_StringValue); ok { + telemetryLabels["csm.service_name"] = val.GetStringValue() + } + } + if val, ok := fields["service_namespace"]; ok { + if _, ok := val.GetKind().(*structpb.Value_StringValue); ok { + telemetryLabels["csm.service_namespace_name"] = val.GetStringValue() + } + } + } + } + } + // "The values for the service labels csm.service_name and + // csm.service_namespace_name come from xDS, “unknown” if not present." - + // CSM Design. + if _, ok := telemetryLabels["csm.service_name"]; !ok { + telemetryLabels["csm.service_name"] = "unknown" + } + if _, ok := telemetryLabels["csm.service_namespace_name"]; !ok { + telemetryLabels["csm.service_namespace_name"] = "unknown" + } + + var lbPolicy json.RawMessage + var err error + switch cluster.GetLbPolicy() { + case v3clusterpb.Cluster_ROUND_ROBIN: + lbPolicy = []byte(`[{"xds_wrr_locality_experimental": {"childPolicy": [{"round_robin": {}}]}}]`) + case v3clusterpb.Cluster_RING_HASH: + rhc := cluster.GetRingHashLbConfig() + if rhc.GetHashFunction() != v3clusterpb.Cluster_RingHashLbConfig_XX_HASH { + return ClusterUpdate{}, fmt.Errorf("unsupported ring_hash hash function %v in response: %+v", rhc.GetHashFunction(), cluster) + } + // Minimum defaults to 1024 entries, and limited to 8M entries Maximum + // defaults to 8M entries, and limited to 8M entries + var minSize, maxSize uint64 = defaultRingHashMinSize, defaultRingHashMaxSize + if min := rhc.GetMinimumRingSize(); min != nil { + minSize = min.GetValue() + } + if max := rhc.GetMaximumRingSize(); max != nil { + maxSize = max.GetValue() + } + + rhLBCfg := []byte(fmt.Sprintf("{\"minRingSize\": %d, \"maxRingSize\": %d}", minSize, maxSize)) + lbPolicy = []byte(fmt.Sprintf(`[{"ring_hash_experimental": %s}]`, rhLBCfg)) + case v3clusterpb.Cluster_LEAST_REQUEST: + // "The configuration for the Least Request LB policy is the + // least_request_lb_config field. The field is optional; if not present, + // defaults will be assumed for all of its values." - A48 + lr := cluster.GetLeastRequestLbConfig() + var choiceCount uint32 = defaultLeastRequestChoiceCount + if cc := lr.GetChoiceCount(); cc != nil { + choiceCount = cc.GetValue() + } + // "If choice_count < 2, the config will be rejected." - A48 + if choiceCount < 2 { + return ClusterUpdate{}, fmt.Errorf("Cluster_LeastRequestLbConfig.ChoiceCount must be >= 2, got: %v", choiceCount) + } + + lrLBCfg := []byte(fmt.Sprintf("{\"choiceCount\": %d}", choiceCount)) + lbPolicy = []byte(fmt.Sprintf(`[{"least_request_experimental": %s}]`, lrLBCfg)) + default: + return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster) + } + // Process security configuration received from the control plane iff the + // corresponding environment variable is set. + var sc *SecurityConfig + var isHTTP11ProxyEnabled bool + if sc, isHTTP11ProxyEnabled, err = securityConfigFromCluster(cluster); err != nil { + return ClusterUpdate{}, err + } + + // Process outlier detection received from the control plane iff the + // corresponding environment variable is set. + var od json.RawMessage + if od, err = outlierConfigFromCluster(cluster); err != nil { + return ClusterUpdate{}, err + } + + if cluster.GetLoadBalancingPolicy() != nil { + lbPolicy, err = xdslbregistry.ConvertToServiceConfig(cluster.GetLoadBalancingPolicy(), 0) + if err != nil { + return ClusterUpdate{}, fmt.Errorf("error converting LoadBalancingPolicy %v in response: %+v: %v", cluster.GetLoadBalancingPolicy(), cluster, err) + } + // "It will be the responsibility of the XdsClient to validate the + // converted configuration. It will do this by having the gRPC LB policy + // registry parse the configuration." - A52 + bc := &iserviceconfig.BalancerConfig{} + if err := json.Unmarshal(lbPolicy, bc); err != nil { + return ClusterUpdate{}, fmt.Errorf("JSON generated from xDS LB policy registry: %s is invalid: %v", pretty.FormatJSON(lbPolicy), err) + } + } + var lrsReportEndpointMetrics *LRSReportEndpointMetricsConfig + if envconfig.XDSORCAToLRSPropEnabled && len(cluster.GetLrsReportEndpointMetrics()) > 0 { + lrsReportEndpointMetrics = &LRSReportEndpointMetricsConfig{ + NamedMetrics: make(map[string]struct{}), + } + for _, m := range cluster.GetLrsReportEndpointMetrics() { + switch m { + case "cpu_utilization": + lrsReportEndpointMetrics.CPUUtilization = true + case "mem_utilization": + lrsReportEndpointMetrics.MemUtilization = true + case "application_utilization": + lrsReportEndpointMetrics.ApplicationUtilization = true + case "named_metrics.*": + // If "named_metrics.*" is present, it takes precedence over any specific + // "named_metrics.foo" fields. Per gRFC A85, specific named metrics are ignored + // if NamedMetricsAll is true. We clear the map to save memory. + lrsReportEndpointMetrics.NamedMetricsAll = true + lrsReportEndpointMetrics.NamedMetrics = nil + default: + if lrsReportEndpointMetrics.NamedMetricsAll { + continue + } + if name, found := strings.CutPrefix(m, "named_metrics."); found && name != "" { + lrsReportEndpointMetrics.NamedMetrics[name] = struct{}{} + } + } + } + } + + var metadata map[string]any + if envconfig.GCPAuthenticationFilterEnabled { + var err error + if metadata, err = validateAndConstructMetadata(cluster.GetMetadata()); err != nil { + return ClusterUpdate{}, err + } + } + + ret := ClusterUpdate{ + ClusterName: cluster.GetName(), + SecurityCfg: sc, + MaxRequests: circuitBreakersFromCluster(cluster), + LBPolicy: lbPolicy, + OutlierDetection: od, + TelemetryLabels: telemetryLabels, + LRSReportEndpointMetrics: lrsReportEndpointMetrics, + Metadata: metadata, + IsHTTP11ProxyEnabled: isHTTP11ProxyEnabled, + } + + if lrs := cluster.GetLrsServer(); lrs != nil { + if lrs.GetSelf() == nil { + return ClusterUpdate{}, fmt.Errorf("unsupported config_source_specifier %T in lrs_server field", lrs.ConfigSourceSpecifier) + } + ret.LRSServerConfig = serverCfg + } + + // Validate and set cluster type from the response. + switch { + case cluster.GetType() == v3clusterpb.Cluster_EDS: + if configsource := cluster.GetEdsClusterConfig().GetEdsConfig(); configsource.GetAds() == nil && configsource.GetSelf() == nil { + return ClusterUpdate{}, fmt.Errorf("CDS's EDS config source is not ADS or Self: %+v", cluster) + } + ret.ClusterType = ClusterTypeEDS + ret.EDSServiceName = cluster.GetEdsClusterConfig().GetServiceName() + if strings.HasPrefix(ret.ClusterName, "xdstp:") && ret.EDSServiceName == "" { + return ClusterUpdate{}, fmt.Errorf("CDS's EDS service name is not set with a new-style cluster name: %+v", cluster) + } + return ret, nil + case cluster.GetType() == v3clusterpb.Cluster_LOGICAL_DNS: + ret.ClusterType = ClusterTypeLogicalDNS + dnsHN, err := dnsHostNameFromCluster(cluster) + if err != nil { + return ClusterUpdate{}, err + } + ret.DNSHostName = dnsHN + return ret, nil + case cluster.GetClusterType() != nil && cluster.GetClusterType().Name == "envoy.clusters.aggregate": + clusters := &v3aggregateclusterpb.ClusterConfig{} + if err := proto.Unmarshal(cluster.GetClusterType().GetTypedConfig().GetValue(), clusters); err != nil { + return ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + if len(clusters.Clusters) == 0 { + return ClusterUpdate{}, fmt.Errorf("xds: aggregate cluster has empty clusters field in response: %+v", cluster) + } + ret.ClusterType = ClusterTypeAggregate + ret.PrioritizedClusterNames = clusters.Clusters + return ret, nil + default: + return ClusterUpdate{}, fmt.Errorf("unsupported cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster) + } +} + +// dnsHostNameFromCluster extracts the DNS host name from the cluster's load +// assignment. +// +// There should be exactly one locality, with one endpoint, whose address +// contains the address and port. +func dnsHostNameFromCluster(cluster *v3clusterpb.Cluster) (string, error) { + loadAssignment := cluster.GetLoadAssignment() + if loadAssignment == nil { + return "", fmt.Errorf("load_assignment not present for LOGICAL_DNS cluster") + } + if len(loadAssignment.GetEndpoints()) != 1 { + return "", fmt.Errorf("load_assignment for LOGICAL_DNS cluster must have exactly one locality, got: %+v", loadAssignment) + } + endpoints := loadAssignment.GetEndpoints()[0].GetLbEndpoints() + if len(endpoints) != 1 { + return "", fmt.Errorf("locality for LOGICAL_DNS cluster must have exactly one endpoint, got: %+v", endpoints) + } + endpoint := endpoints[0].GetEndpoint() + if endpoint == nil { + return "", fmt.Errorf("endpoint for LOGICAL_DNS cluster not set") + } + socketAddr := endpoint.GetAddress().GetSocketAddress() + if socketAddr == nil { + return "", fmt.Errorf("socket address for endpoint for LOGICAL_DNS cluster not set") + } + if socketAddr.GetResolverName() != "" { + return "", fmt.Errorf("socket address for endpoint for LOGICAL_DNS cluster not set has unexpected custom resolver name: %v", socketAddr.GetResolverName()) + } + host := socketAddr.GetAddress() + if host == "" { + return "", fmt.Errorf("host for endpoint for LOGICAL_DNS cluster not set") + } + port := socketAddr.GetPortValue() + if port == 0 { + return "", fmt.Errorf("port for endpoint for LOGICAL_DNS cluster not set") + } + return net.JoinHostPort(host, strconv.Itoa(int(port))), nil +} + +// securityConfigFromCluster extracts the relevant security configuration from +// the received Cluster resource. +func securityConfigFromCluster(cluster *v3clusterpb.Cluster) (*SecurityConfig, bool, error) { + if tsm := cluster.GetTransportSocketMatches(); len(tsm) != 0 { + return nil, false, fmt.Errorf("unsupported transport_socket_matches field is non-empty: %+v", tsm) + } + // The Cluster resource contains a `transport_socket` field, which contains + // a oneof `typed_config` field of type `protobuf.Any`. The any proto + // contains a marshaled representation of an `UpstreamTlsContext` message. + ts := cluster.GetTransportSocket() + if ts == nil { + return nil, false, nil + } + var isHTTP11ProxyEnabled bool + tc := ts.GetTypedConfig() + typeURL := tc.GetTypeUrl() + + if envconfig.XDSHTTPConnectEnabled && typeURL == version.V3HTTP11ProxyUpstreamTransportURL { + isHTTP11ProxyEnabled = true + http11Proxy := &v3http11proxypb.Http11ProxyUpstreamTransport{} + if err := proto.Unmarshal(tc.GetValue(), http11Proxy); err != nil { + return nil, false, fmt.Errorf("failed to unmarshal Http11ProxyUpstreamTransport in CDS response: %v", err) + } + ts = http11Proxy.GetTransportSocket() + if ts == nil { + return nil, true, nil + } + tc = ts.GetTypedConfig() + typeURL = tc.GetTypeUrl() + } + + if name := ts.GetName(); name != transportSocketName { + return nil, false, fmt.Errorf("transport_socket field has unexpected name: %s", name) + } + if typeURL != version.V3UpstreamTLSContextURL { + return nil, false, fmt.Errorf("transport_socket missing typed_config or wrong type_url: %q", typeURL) + } + upstreamCtx := &v3tlspb.UpstreamTlsContext{} + if err := proto.Unmarshal(tc.GetValue(), upstreamCtx); err != nil { + return nil, false, fmt.Errorf("failed to unmarshal UpstreamTlsContext in CDS response: %v", err) + } + // The following fields from `UpstreamTlsContext` are ignored: + // - allow_renegotiation + // - max_session_keys + if upstreamCtx.GetCommonTlsContext() == nil { + return nil, false, errors.New("UpstreamTlsContext in CDS response does not contain a CommonTlsContext") + } + + sc, err := securityConfigFromCommonTLSContext(upstreamCtx.GetCommonTlsContext(), false) + if err != nil { + return nil, false, err + } + // Set SNI related fields in SecurityConfig from UpstreamTlsContext if + // `GRPC_EXPERIMENTAL_XDS_SNI` is enabled. + if envconfig.XDSSNIEnabled { + sc.SNI = upstreamCtx.GetSni() + if len(sc.SNI) > maxSNILength { + return nil, false, fmt.Errorf("SNI value %q in UpstreamTlsContext in CDS response exceeds max length of %d", sc.SNI, maxSNILength) + } + sc.UseAutoHostSNI = upstreamCtx.GetAutoHostSni() + sc.AutoSNISANValidation = upstreamCtx.GetAutoSniSanValidation() + } + return sc, isHTTP11ProxyEnabled, nil +} + +// common is expected to be not nil. +// The `alpn_protocols` field is ignored. +func securityConfigFromCommonTLSContext(common *v3tlspb.CommonTlsContext, server bool) (*SecurityConfig, error) { + if common.GetTlsParams() != nil { + return nil, fmt.Errorf("unsupported tls_params field in CommonTlsContext message: %+v", common) + } + if common.GetCustomHandshaker() != nil { + return nil, fmt.Errorf("unsupported custom_handshaker field in CommonTlsContext message: %+v", common) + } + + // For now, if we can't get a valid security config from the new fields, we + // fallback to the old deprecated fields. + // TODO: Drop support for deprecated fields. NACK if err != nil here. + sc, err1 := securityConfigFromCommonTLSContextUsingNewFields(common, server) + if sc == nil || sc.Equal(&SecurityConfig{}) { + var err error + sc, err = securityConfigFromCommonTLSContextWithDeprecatedFields(common, server) + if err != nil { + // Retain the validation error from using the new fields. + return nil, errors.Join(err1, fmt.Errorf("failed to parse config using deprecated fields: %v", err)) + } + } + if sc != nil { + // sc == nil is a valid case where the control plane has not sent us any + // security configuration. xDS creds will use fallback creds. + if server { + if sc.IdentityInstanceName == "" { + return nil, errors.New("security configuration on the server-side does not contain identity certificate provider instance name") + } + } else { + if !sc.UseSystemRootCerts && sc.RootInstanceName == "" { + return nil, errors.New("security configuration on the client-side does not contain root certificate provider instance name") + } + } + } + return sc, nil +} + +func securityConfigFromCommonTLSContextWithDeprecatedFields(common *v3tlspb.CommonTlsContext, server bool) (*SecurityConfig, error) { + // The `CommonTlsContext` contains a + // `tls_certificate_certificate_provider_instance` field of type + // `CertificateProviderInstance`, which contains the provider instance name + // and the certificate name to fetch identity certs. + sc := &SecurityConfig{} + if identity := common.GetTlsCertificateCertificateProviderInstance(); identity != nil { + sc.IdentityInstanceName = identity.GetInstanceName() + sc.IdentityCertName = identity.GetCertificateName() + } + + // The `CommonTlsContext` contains a `validation_context_type` field which + // is a oneof. We can get the values that we are interested in from two of + // those possible values: + // - combined validation context: + // - contains a default validation context which holds the list of + // matchers for accepted SANs. + // - contains certificate provider instance configuration + // - certificate provider instance configuration + // - in this case, we do not get a list of accepted SANs. + switch t := common.GetValidationContextType().(type) { + case *v3tlspb.CommonTlsContext_CombinedValidationContext: + combined := common.GetCombinedValidationContext() + var matchers []matcher.StringMatcher + if def := combined.GetDefaultValidationContext(); def != nil { + for _, m := range def.GetMatchSubjectAltNames() { + matcher, err := matcher.StringMatcherFromProto(m) + if err != nil { + return nil, err + } + matchers = append(matchers, matcher) + } + } + if server && len(matchers) != 0 { + return nil, fmt.Errorf("match_subject_alt_names field in validation context is not supported on the server: %v", common) + } + sc.SubjectAltNameMatchers = matchers + if pi := combined.GetValidationContextCertificateProviderInstance(); pi != nil { + sc.RootInstanceName = pi.GetInstanceName() + sc.RootCertName = pi.GetCertificateName() + } + case *v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance: + pi := common.GetValidationContextCertificateProviderInstance() + sc.RootInstanceName = pi.GetInstanceName() + sc.RootCertName = pi.GetCertificateName() + case nil: + // It is valid for the validation context to be nil on the server side. + default: + return nil, fmt.Errorf("validation context contains unexpected type: %T", t) + } + return sc, nil +} + +// gRFC A29 https://github.com/grpc/proposal/blob/master/A29-xds-tls-security.md +// specifies the new way to fetch security configuration and says the following: +// +// Although there are various ways to obtain certificates as per this proto +// (which are supported by Envoy), gRPC supports only one of them and that is +// the `CertificateProviderPluginInstance` proto. +// +// This helper function attempts to fetch security configuration from the +// `CertificateProviderPluginInstance` message, given a CommonTlsContext. +func securityConfigFromCommonTLSContextUsingNewFields(common *v3tlspb.CommonTlsContext, server bool) (*SecurityConfig, error) { + // The `tls_certificate_provider_instance` field of type + // `CertificateProviderPluginInstance` is used to fetch the identity + // certificate provider. + sc := &SecurityConfig{} + identity := common.GetTlsCertificateProviderInstance() + if identity == nil && len(common.GetTlsCertificates()) != 0 { + return nil, fmt.Errorf("expected field tls_certificate_provider_instance is not set, while unsupported field tls_certificates is set in CommonTlsContext message: %+v", common) + } + if identity == nil && common.GetTlsCertificateSdsSecretConfigs() != nil { + return nil, fmt.Errorf("expected field tls_certificate_provider_instance is not set, while unsupported field tls_certificate_sds_secret_configs is set in CommonTlsContext message: %+v", common) + } + sc.IdentityInstanceName = identity.GetInstanceName() + sc.IdentityCertName = identity.GetCertificateName() + + // The `CommonTlsContext` contains a oneof field `validation_context_type`, + // which contains the `CertificateValidationContext` message in one of the + // following ways: + // - `validation_context` field + // - this is directly of type `CertificateValidationContext` + // - `combined_validation_context` field + // - this is of type `CombinedCertificateValidationContext` and contains + // a `default validation context` field of type + // `CertificateValidationContext` + // + // The `CertificateValidationContext` message has the following fields that + // we are interested in: + // - `ca_certificate_provider_instance` + // - this is of type `CertificateProviderPluginInstance` + // - `system_root_certs`: + // - This indicates the usage of system root certs for validation. + // - `match_subject_alt_names` + // - this is a list of string matchers + // + // The `CertificateProviderPluginInstance` message contains two fields + // - instance_name + // - this is the certificate provider instance name to be looked up in + // the bootstrap configuration + // - certificate_name + // - this is an opaque name passed to the certificate provider + var validationCtx *v3tlspb.CertificateValidationContext + switch typ := common.GetValidationContextType().(type) { + case *v3tlspb.CommonTlsContext_ValidationContext: + validationCtx = common.GetValidationContext() + case *v3tlspb.CommonTlsContext_CombinedValidationContext: + validationCtx = common.GetCombinedValidationContext().GetDefaultValidationContext() + case nil: + // It is valid for the validation context to be nil on the server side. + return sc, nil + default: + return nil, fmt.Errorf("validation context contains unexpected type: %T", typ) + } + // If we get here, it means that the `CertificateValidationContext` message + // was found through one of the supported ways. It is an error if the + // validation context is specified, but it does not specify a way to + // validate TLS certificates. Peer TLS certs can be verified in the + // following ways: + // 1. If the ca_certificate_provider_instance field is set, it contains + // information about the certificate provider to be used for the root + // certificates, else + // 2. If the system_root_certs field is set, and the config is for a client, + // use the system default root certs. + useSystemRootCerts := false + if validationCtx.GetCaCertificateProviderInstance() == nil && envconfig.XDSSystemRootCertsEnabled { + if server { + if validationCtx.GetSystemRootCerts() != nil { + // The `system_root_certs` field will not be supported on the + // gRPC server side. If `ca_certificate_provider_instance` is + // unset and `system_root_certs` is set, the LDS resource will + // be NACKed. + // - A82 + return nil, fmt.Errorf("expected field ca_certificate_provider_instance is missing and unexpected field system_root_certs is set for server in CommonTlsContext message: %+v", common) + } + } else { + if validationCtx.GetSystemRootCerts() != nil { + useSystemRootCerts = true + } + } + } + + // The following fields are ignored: + // - trusted_ca + // - watched_directory + // - allow_expired_certificate + // - trust_chain_verification + switch { + case len(validationCtx.GetVerifyCertificateSpki()) != 0: + return nil, fmt.Errorf("unsupported verify_certificate_spki field in CommonTlsContext message: %+v", common) + case len(validationCtx.GetVerifyCertificateHash()) != 0: + return nil, fmt.Errorf("unsupported verify_certificate_hash field in CommonTlsContext message: %+v", common) + case validationCtx.GetRequireSignedCertificateTimestamp().GetValue(): + return nil, fmt.Errorf("unsupported require_signed_certificate_timestamp field in CommonTlsContext message: %+v", common) + case validationCtx.GetCrl() != nil: + return nil, fmt.Errorf("unsupported crl field in CommonTlsContext message: %+v", common) + case validationCtx.GetCustomValidatorConfig() != nil: + return nil, fmt.Errorf("unsupported custom_validator_config field in CommonTlsContext message: %+v", common) + } + + if rootProvider := validationCtx.GetCaCertificateProviderInstance(); rootProvider != nil { + sc.RootInstanceName = rootProvider.GetInstanceName() + sc.RootCertName = rootProvider.GetCertificateName() + } else if useSystemRootCerts { + sc.UseSystemRootCerts = true + } else if !server && envconfig.XDSSystemRootCertsEnabled { + return nil, fmt.Errorf("expected fields ca_certificate_provider_instance and system_root_certs are missing in CommonTlsContext message: %+v", common) + } else { + // Don't mention the system_root_certs field if it was not checked. + return nil, fmt.Errorf("expected field ca_certificate_provider_instance is missing in CommonTlsContext message: %+v", common) + } + + var matchers []matcher.StringMatcher + for _, m := range validationCtx.GetMatchSubjectAltNames() { + matcher, err := matcher.StringMatcherFromProto(m) + if err != nil { + return nil, err + } + matchers = append(matchers, matcher) + } + if server && len(matchers) != 0 { + return nil, fmt.Errorf("match_subject_alt_names field in validation context is not supported on the server: %v", common) + } + sc.SubjectAltNameMatchers = matchers + return sc, nil +} + +// circuitBreakersFromCluster extracts the circuit breakers configuration from +// the received cluster resource. Returns nil if no CircuitBreakers or no +// Thresholds in CircuitBreakers. +func circuitBreakersFromCluster(cluster *v3clusterpb.Cluster) *uint32 { + for _, threshold := range cluster.GetCircuitBreakers().GetThresholds() { + if threshold.GetPriority() != v3corepb.RoutingPriority_DEFAULT { + continue + } + maxRequestsPb := threshold.GetMaxRequests() + if maxRequestsPb == nil { + return nil + } + maxRequests := maxRequestsPb.GetValue() + return &maxRequests + } + return nil +} + +// idurationp takes a time.Duration and converts it to an internal duration, and +// returns a pointer to that internal duration. +func idurationp(d time.Duration) *iserviceconfig.Duration { + id := iserviceconfig.Duration(d) + return &id +} + +func uint32p(i uint32) *uint32 { + return &i +} + +// Helper types to prepare Outlier Detection JSON. Pointer types to distinguish +// between unset and a zero value. +type successRateEjection struct { + StdevFactor *uint32 `json:"stdevFactor,omitempty"` + EnforcementPercentage *uint32 `json:"enforcementPercentage,omitempty"` + MinimumHosts *uint32 `json:"minimumHosts,omitempty"` + RequestVolume *uint32 `json:"requestVolume,omitempty"` +} + +type failurePercentageEjection struct { + Threshold *uint32 `json:"threshold,omitempty"` + EnforcementPercentage *uint32 `json:"enforcementPercentage,omitempty"` + MinimumHosts *uint32 `json:"minimumHosts,omitempty"` + RequestVolume *uint32 `json:"requestVolume,omitempty"` +} + +type odLBConfig struct { + Interval *iserviceconfig.Duration `json:"interval,omitempty"` + BaseEjectionTime *iserviceconfig.Duration `json:"baseEjectionTime,omitempty"` + MaxEjectionTime *iserviceconfig.Duration `json:"maxEjectionTime,omitempty"` + MaxEjectionPercent *uint32 `json:"maxEjectionPercent,omitempty"` + SuccessRateEjection *successRateEjection `json:"successRateEjection,omitempty"` + FailurePercentageEjection *failurePercentageEjection `json:"failurePercentageEjection,omitempty"` +} + +// outlierConfigFromCluster converts the received Outlier Detection +// configuration into JSON configuration for Outlier Detection, taking into +// account xDS Defaults. Returns nil if no OutlierDetection field set in the +// cluster resource. +func outlierConfigFromCluster(cluster *v3clusterpb.Cluster) (json.RawMessage, error) { + od := cluster.GetOutlierDetection() + if od == nil { + return nil, nil + } + + // "The outlier_detection field of the Cluster resource should have its fields + // validated according to the rules for the corresponding LB policy config + // fields in the above "Validation" section. If any of these requirements is + // violated, the Cluster resource should be NACKed." - A50 + // "The google.protobuf.Duration fields interval, base_ejection_time, and + // max_ejection_time must obey the restrictions in the + // google.protobuf.Duration documentation and they must have non-negative + // values." - A50 + var interval *iserviceconfig.Duration + if i := od.GetInterval(); i != nil { + if err := i.CheckValid(); err != nil { + return nil, fmt.Errorf("outlier_detection.interval is invalid with error: %v", err) + } + if interval = idurationp(i.AsDuration()); *interval < 0 { + return nil, fmt.Errorf("outlier_detection.interval = %v; must be a valid duration and >= 0", *interval) + } + } + + var baseEjectionTime *iserviceconfig.Duration + if bet := od.GetBaseEjectionTime(); bet != nil { + if err := bet.CheckValid(); err != nil { + return nil, fmt.Errorf("outlier_detection.base_ejection_time is invalid with error: %v", err) + } + if baseEjectionTime = idurationp(bet.AsDuration()); *baseEjectionTime < 0 { + return nil, fmt.Errorf("outlier_detection.base_ejection_time = %v; must be >= 0", *baseEjectionTime) + } + } + + var maxEjectionTime *iserviceconfig.Duration + if met := od.GetMaxEjectionTime(); met != nil { + if err := met.CheckValid(); err != nil { + return nil, fmt.Errorf("outlier_detection.max_ejection_time is invalid: %v", err) + } + if maxEjectionTime = idurationp(met.AsDuration()); *maxEjectionTime < 0 { + return nil, fmt.Errorf("outlier_detection.max_ejection_time = %v; must be >= 0", *maxEjectionTime) + } + } + + // "The fields max_ejection_percent, enforcing_success_rate, + // failure_percentage_threshold, and enforcing_failure_percentage must have + // values less than or equal to 100. If any of these requirements is + // violated, the Cluster resource should be NACKed." - A50 + var maxEjectionPercent *uint32 + if mep := od.GetMaxEjectionPercent(); mep != nil { + if maxEjectionPercent = uint32p(mep.GetValue()); *maxEjectionPercent > 100 { + return nil, fmt.Errorf("outlier_detection.max_ejection_percent = %v; must be <= 100", *maxEjectionPercent) + } + } + // "if the enforcing_success_rate field is set to 0, the config + // success_rate_ejection field will be null and all success_rate_* fields + // will be ignored." - A50 + var enforcingSuccessRate *uint32 + if esr := od.GetEnforcingSuccessRate(); esr != nil { + if enforcingSuccessRate = uint32p(esr.GetValue()); *enforcingSuccessRate > 100 { + return nil, fmt.Errorf("outlier_detection.enforcing_success_rate = %v; must be <= 100", *enforcingSuccessRate) + } + } + var failurePercentageThreshold *uint32 + if fpt := od.GetFailurePercentageThreshold(); fpt != nil { + if failurePercentageThreshold = uint32p(fpt.GetValue()); *failurePercentageThreshold > 100 { + return nil, fmt.Errorf("outlier_detection.failure_percentage_threshold = %v; must be <= 100", *failurePercentageThreshold) + } + } + // "If the enforcing_failure_percent field is set to 0 or null, the config + // failure_percent_ejection field will be null and all failure_percent_* + // fields will be ignored." - A50 + var enforcingFailurePercentage *uint32 + if efp := od.GetEnforcingFailurePercentage(); efp != nil { + if enforcingFailurePercentage = uint32p(efp.GetValue()); *enforcingFailurePercentage > 100 { + return nil, fmt.Errorf("outlier_detection.enforcing_failure_percentage = %v; must be <= 100", *enforcingFailurePercentage) + } + } + + var successRateStdevFactor *uint32 + if srsf := od.GetSuccessRateStdevFactor(); srsf != nil { + successRateStdevFactor = uint32p(srsf.GetValue()) + } + var successRateMinimumHosts *uint32 + if srmh := od.GetSuccessRateMinimumHosts(); srmh != nil { + successRateMinimumHosts = uint32p(srmh.GetValue()) + } + var successRateRequestVolume *uint32 + if srrv := od.GetSuccessRateRequestVolume(); srrv != nil { + successRateRequestVolume = uint32p(srrv.GetValue()) + } + var failurePercentageMinimumHosts *uint32 + if fpmh := od.GetFailurePercentageMinimumHosts(); fpmh != nil { + failurePercentageMinimumHosts = uint32p(fpmh.GetValue()) + } + var failurePercentageRequestVolume *uint32 + if fprv := od.GetFailurePercentageRequestVolume(); fprv != nil { + failurePercentageRequestVolume = uint32p(fprv.GetValue()) + } + + // "if the enforcing_success_rate field is set to 0, the config + // success_rate_ejection field will be null and all success_rate_* fields + // will be ignored." - A50 + var sre *successRateEjection + if enforcingSuccessRate == nil || *enforcingSuccessRate != 0 { + sre = &successRateEjection{ + StdevFactor: successRateStdevFactor, + EnforcementPercentage: enforcingSuccessRate, + MinimumHosts: successRateMinimumHosts, + RequestVolume: successRateRequestVolume, + } + } + + // "If the enforcing_failure_percent field is set to 0 or null, the config + // failure_percent_ejection field will be null and all failure_percent_* + // fields will be ignored." - A50 + var fpe *failurePercentageEjection + if enforcingFailurePercentage != nil && *enforcingFailurePercentage != 0 { + fpe = &failurePercentageEjection{ + Threshold: failurePercentageThreshold, + EnforcementPercentage: enforcingFailurePercentage, + MinimumHosts: failurePercentageMinimumHosts, + RequestVolume: failurePercentageRequestVolume, + } + } + + odLBCfg := &odLBConfig{ + Interval: interval, + BaseEjectionTime: baseEjectionTime, + MaxEjectionTime: maxEjectionTime, + MaxEjectionPercent: maxEjectionPercent, + SuccessRateEjection: sre, + FailurePercentageEjection: fpe, + } + return json.Marshal(odLBCfg) +} diff --git a/internal/xds/xdsclient/xdsresource/unmarshal_cds_test.go b/internal/xds/xdsclient/xdsresource/unmarshal_cds_test.go new file mode 100644 index 0000000..e5b31b2 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/unmarshal_cds_test.go @@ -0,0 +1,2600 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "encoding/json" + "regexp" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/testutils" + xdsinternal "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" + v3gcpauthnpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/gcp_authn/v3" + v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3" + v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3" + v3http11proxypb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/http_11_proxy/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +const ( + clusterName = "clusterName" + serviceName = "service" +) + +func (s) TestValidateCluster_Failure(t *testing.T) { + tests := []struct { + name string + cluster *v3clusterpb.Cluster + wantErr bool + }{ + { + name: "non-supported-cluster-type-static", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + }, + LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST, + }, + wantErr: true, + }, + { + name: "non-supported-cluster-type-original-dst", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_ORIGINAL_DST}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + }, + LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST, + }, + wantErr: true, + }, + { + name: "no-eds-config", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + }, + wantErr: true, + }, + { + name: "no-ads-config-source", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{}, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + }, + wantErr: true, + }, + { + name: "unsupported-lb-policy", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + }, + LbPolicy: v3clusterpb.Cluster_MAGLEV, + }, + wantErr: true, + }, + { + name: "logical-dns-multiple-localities", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_LOGICAL_DNS}, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LoadAssignment: &v3endpointpb.ClusterLoadAssignment{ + Endpoints: []*v3endpointpb.LocalityLbEndpoints{ + // Invalid if there are more than one locality. + {LbEndpoints: nil}, + {LbEndpoints: nil}, + }, + }, + }, + wantErr: true, + }, + { + name: "ring-hash-hash-function-not-xx-hash", + cluster: &v3clusterpb.Cluster{ + LbPolicy: v3clusterpb.Cluster_RING_HASH, + LbConfig: &v3clusterpb.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &v3clusterpb.Cluster_RingHashLbConfig{ + HashFunction: v3clusterpb.Cluster_RingHashLbConfig_MURMUR_HASH_2, + }, + }, + }, + wantErr: true, + }, + { + name: "least-request-choice-count-less-than-two", + cluster: &v3clusterpb.Cluster{ + LbPolicy: v3clusterpb.Cluster_RING_HASH, + LbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig_{ + LeastRequestLbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig{ + ChoiceCount: wrapperspb.UInt32(1), + }, + }, + }, + wantErr: true, + }, + { + name: "ring-hash-max-bound-greater-than-upper-bound", + cluster: &v3clusterpb.Cluster{ + LbPolicy: v3clusterpb.Cluster_RING_HASH, + LbConfig: &v3clusterpb.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &v3clusterpb.Cluster_RingHashLbConfig{ + MaximumRingSize: wrapperspb.UInt64(ringHashSizeUpperBound + 1), + }, + }, + }, + wantErr: true, + }, + { + name: "ring-hash-max-bound-greater-than-upper-bound-load-balancing-policy", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_XX_HASH, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(ringHashSizeUpperBound + 1), + }), + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "least-request-unsupported-in-converter-since-env-var-unset", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, &v3leastrequestpb.LeastRequest{}), + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "aggregate-nil-clusters", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_ClusterType{ + ClusterType: &v3clusterpb.Cluster_CustomClusterType{ + Name: "envoy.clusters.aggregate", + TypedConfig: testutils.MarshalAny(t, &v3aggregateclusterpb.ClusterConfig{}), + }, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + }, + wantErr: true, + }, + { + name: "aggregate-empty-clusters", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_ClusterType{ + ClusterType: &v3clusterpb.Cluster_CustomClusterType{ + Name: "envoy.clusters.aggregate", + TypedConfig: testutils.MarshalAny(t, &v3aggregateclusterpb.ClusterConfig{ + Clusters: []string{}, + }), + }, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + }, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if update, err := validateClusterAndConstructClusterUpdate(test.cluster, nil); err == nil { + t.Errorf("validateClusterAndConstructClusterUpdate(%+v) = %v, wanted error", test.cluster, update) + } + }) + } +} + +func (s) TestSecurityConfigFromCommonTLSContextUsingNewFields_ErrorCases(t *testing.T) { + tests := []struct { + name string + common *v3tlspb.CommonTlsContext + server bool + wantErr string + enableSystemRootCertsFlag bool + }{ + { + name: "unsupported-tls_certificates-field-for-identity-certs", + common: &v3tlspb.CommonTlsContext{ + TlsCertificates: []*v3tlspb.TlsCertificate{ + {CertificateChain: &v3corepb.DataSource{}}, + }, + }, + wantErr: "unsupported field tls_certificates is set in CommonTlsContext message", + }, + { + name: "unsupported-tls_certificates_sds_secret_configs-field-for-identity-certs", + common: &v3tlspb.CommonTlsContext{ + TlsCertificateSdsSecretConfigs: []*v3tlspb.SdsSecretConfig{ + {Name: "sds-secrets-config"}, + }, + }, + wantErr: "unsupported field tls_certificate_sds_secret_configs is set in CommonTlsContext message", + }, + { + name: "unsupported-sds-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{ + ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{ + Name: "foo-sds-secret", + }, + }, + }, + wantErr: "validation context contains unexpected type", + }, + { + name: "missing-ca_certificate_provider_instance-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{}, + }, + }, + wantErr: "expected field ca_certificate_provider_instance is missing in CommonTlsContext message", + }, + { + name: "unsupported-field-verify_certificate_spki-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + VerifyCertificateSpki: []string{"spki"}, + }, + }, + }, + wantErr: "unsupported verify_certificate_spki field in CommonTlsContext message", + }, + { + name: "unsupported-field-verify_certificate_hash-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + VerifyCertificateHash: []string{"hash"}, + }, + }, + }, + wantErr: "unsupported verify_certificate_hash field in CommonTlsContext message", + }, + { + name: "unsupported-field-require_signed_certificate_timestamp-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + RequireSignedCertificateTimestamp: &wrapperspb.BoolValue{Value: true}, + }, + }, + }, + wantErr: "unsupported require_signed_certificate_timestamp field in CommonTlsContext message", + }, + { + name: "unsupported-field-crl-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + Crl: &v3corepb.DataSource{}, + }, + }, + }, + wantErr: "unsupported crl field in CommonTlsContext message", + }, + { + name: "unsupported-field-custom_validator_config-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + CustomValidatorConfig: &v3corepb.TypedExtensionConfig{}, + }, + }, + }, + wantErr: "unsupported custom_validator_config field in CommonTlsContext message", + }, + { + name: "invalid-match_subject_alt_names-field-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: ""}}, + }, + }, + }, + }, + wantErr: "empty prefix is not allowed in StringMatcher", + }, + { + name: "unsupported-field-matching-subject-alt-names-in-validation-context-of-server", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "sanPrefix"}}, + }, + }, + }, + }, + server: true, + wantErr: "match_subject_alt_names field in validation context is not supported on the server", + }, + { + name: "client-missing-root-cert-provider-and-use-system-certs-fields", + enableSystemRootCertsFlag: true, + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{}, + }, + }, + wantErr: "expected fields ca_certificate_provider_instance and system_root_certs are missing", + }, + { + name: "server-missing-root-cert-provider-and-use-system-certs-fields", + enableSystemRootCertsFlag: true, + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{}, + }, + }, + server: true, + wantErr: "expected field ca_certificate_provider_instance is missing", + }, + { + name: "client-missing-root-cert-provider-and-use-system-certs-fields-env-var-unset", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{}, + }, + }, + wantErr: "expected field ca_certificate_provider_instance is missing", + }, + { + name: "server-missing-root-cert-provider-and-use-system-certs-fields-env-var-unset", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{}, + }, + }, + server: true, + wantErr: "expected field ca_certificate_provider_instance is missing", + }, + { + name: "server-missing-root-cert-provider-and-set-use-system-certs-fields", + enableSystemRootCertsFlag: true, + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + SystemRootCerts: &v3tlspb.CertificateValidationContext_SystemRootCerts{}, + }, + }, + }, + server: true, + wantErr: "expected field ca_certificate_provider_instance is missing and unexpected field system_root_certs is set", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSystemRootCertsEnabled, test.enableSystemRootCertsFlag) + _, err := securityConfigFromCommonTLSContextUsingNewFields(test.common, test.server) + if err == nil { + t.Fatal("securityConfigFromCommonTLSContextUsingNewFields() succeeded when expected to fail") + } + if !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("securityConfigFromCommonTLSContextUsingNewFields() returned err: %v, wantErr: %v", err, test.wantErr) + } + }) + } +} + +func (s) TestValidateClusterWithSecurityConfig(t *testing.T) { + const ( + identityPluginInstance = "identityPluginInstance" + identityCertName = "identityCert" + rootPluginInstance = "rootPluginInstance" + rootCertName = "rootCert" + clusterName = "cluster" + serviceName = "service" + sanExact = "san-exact" + sanPrefix = "san-prefix" + sanSuffix = "san-suffix" + sanRegexBad = "??" + sanRegexGood = "san?regex?" + sanContains = "san-contains" + sniString = "test-sni" + ) + var sanRE = regexp.MustCompile("^(?:san?regex?)$") + + tests := []struct { + name string + cluster *v3clusterpb.Cluster + wantUpdate ClusterUpdate + wantErr bool + enableSystemRootCertsFlag bool + enableSNIFlag bool + }{ + { + name: "transport-socket-matches", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocketMatches: []*v3clusterpb.Cluster_TransportSocketMatch{ + {Name: "transport-socket-match-1"}, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-unsupported-name", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "unsupported-foo", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3UpstreamTLSContextURL, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-unsupported-typeURL", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3HTTPConnManagerURL, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-unsupported-type", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3UpstreamTLSContextURL, + Value: []byte{1, 2, 3, 4}, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-unsupported-tls-params-field", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsParams: &v3tlspb.TlsParameters{}, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-unsupported-custom-handshaker-field", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + CustomHandshaker: &v3corepb.TypedExtensionConfig{}, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-unsupported-validation-context", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{ + ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{ + Name: "foo-sds-secret", + }, + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-without-validation-context", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{}, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "empty-prefix-in-matching-SAN", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: ""}}, + }, + }, + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "empty-suffix-in-matching-SAN", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: ""}}, + }, + }, + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "empty-contains-in-matching-SAN", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: ""}}, + }, + }, + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "invalid-regex-in-matching-SAN", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexBad}}}, + }, + }, + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "invalid-regex-in-matching-SAN-with-new-fields", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexBad}}}, + }, + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "happy-case-with-no-identity-certs-using-deprecated-fields", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "happy-case-with-no-identity-certs-using-new-fields", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "happy-case-with-validation-context-provider-instance-using-deprecated-fields", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: identityPluginInstance, + CertificateName: identityCertName, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + IdentityInstanceName: identityPluginInstance, + IdentityCertName: identityCertName, + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "happy-case-with-validation-context-provider-instance-using-new-fields", + enableSystemRootCertsFlag: true, + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: identityPluginInstance, + CertificateName: identityCertName, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + // SystemRootCerts will be ignored due + // to the presence of + // CaCertificateProviderInstance. + SystemRootCerts: &v3tlspb.CertificateValidationContext_SystemRootCerts{}, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + IdentityInstanceName: identityPluginInstance, + IdentityCertName: identityCertName, + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "happy-case-with-validation-context-provider-instance-using-new-fields-and-system-root-certs", + enableSystemRootCertsFlag: true, + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: identityPluginInstance, + CertificateName: identityCertName, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + SystemRootCerts: &v3tlspb.CertificateValidationContext_SystemRootCerts{}, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + SecurityCfg: &SecurityConfig{ + UseSystemRootCerts: true, + IdentityInstanceName: identityPluginInstance, + IdentityCertName: identityCertName, + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "failure-case-with-validation-context-provider-instance-using-new-fields-and-system-root-certs-env-flag-disabled", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: identityPluginInstance, + CertificateName: identityCertName, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + SystemRootCerts: &v3tlspb.CertificateValidationContext_SystemRootCerts{}, + }, + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "happy-case-with-combined-validation-context-using-deprecated-fields", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: identityPluginInstance, + CertificateName: identityCertName, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + { + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: sanExact}, + IgnoreCase: true, + }, + {MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: sanPrefix}}, + {MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: sanSuffix}}, + {MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexGood}}}, + {MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: sanContains}}, + }, + }, + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + IdentityInstanceName: identityPluginInstance, + IdentityCertName: identityCertName, + SubjectAltNameMatchers: []matcher.StringMatcher{ + matcher.NewExactStringMatcher(sanExact, true), + matcher.NewPrefixStringMatcher(sanPrefix, false), + matcher.NewSuffixStringMatcher(sanSuffix, false), + matcher.NewRegexStringMatcher(sanRE), + matcher.NewContainsStringMatcher(sanContains, false), + }, + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "happy-case-with-combined-validation-context-using-new-fields", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: identityPluginInstance, + CertificateName: identityCertName, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + { + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: sanExact}, + IgnoreCase: true, + }, + {MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: sanPrefix}}, + {MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: sanSuffix}}, + {MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexGood}}}, + {MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: sanContains}}, + }, + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + IdentityInstanceName: identityPluginInstance, + IdentityCertName: identityCertName, + SubjectAltNameMatchers: []matcher.StringMatcher{ + matcher.NewExactStringMatcher(sanExact, true), + matcher.NewPrefixStringMatcher(sanPrefix, false), + matcher.NewSuffixStringMatcher(sanSuffix, false), + matcher.NewRegexStringMatcher(sanRE), + matcher.NewContainsStringMatcher(sanContains, false), + }, + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "happy-case-with-combined-validation-context-using-new-fields-and-system-root-certs", + enableSystemRootCertsFlag: true, + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: identityPluginInstance, + CertificateName: identityCertName, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + { + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: sanExact}, + IgnoreCase: true, + }, + {MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: sanPrefix}}, + {MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: sanSuffix}}, + {MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexGood}}}, + {MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: sanContains}}, + }, + SystemRootCerts: &v3tlspb.CertificateValidationContext_SystemRootCerts{}, + }, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + SecurityCfg: &SecurityConfig{ + UseSystemRootCerts: true, + IdentityInstanceName: identityPluginInstance, + IdentityCertName: identityCertName, + SubjectAltNameMatchers: []matcher.StringMatcher{ + matcher.NewExactStringMatcher(sanExact, true), + matcher.NewPrefixStringMatcher(sanPrefix, false), + matcher.NewSuffixStringMatcher(sanSuffix, false), + matcher.NewRegexStringMatcher(sanRE), + matcher.NewContainsStringMatcher(sanContains, false), + }, + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "happy-case-with-sni-flag-enabled", + enableSNIFlag: true, + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + Sni: sniString, + AutoHostSni: true, + AutoSniSanValidation: true, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + SNI: sniString, + UseAutoHostSNI: true, + AutoSNISANValidation: true, + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "sni-env-variable-disabled-should-not-be-populated-in-update", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + Sni: sniString, + AutoHostSni: true, + AutoSniSanValidation: true, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSystemRootCertsEnabled, test.enableSystemRootCertsFlag) + testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, test.enableSNIFlag) + update, err := validateClusterAndConstructClusterUpdate(test.cluster, nil) + if (err != nil) != test.wantErr { + t.Errorf("validateClusterAndConstructClusterUpdate() returned err %v wantErr %v)", err, test.wantErr) + } + if diff := cmp.Diff(test.wantUpdate, update, cmpopts.EquateEmpty(), cmp.AllowUnexported(regexp.Regexp{}), cmpopts.IgnoreFields(ClusterUpdate{}, "LBPolicy")); diff != "" { + t.Errorf("validateClusterAndConstructClusterUpdate() returned unexpected diff (-want, +got):\n%s", diff) + } + }) + } +} + +func (s) TestUnmarshalCluster(t *testing.T) { + const ( + v3ClusterName = "v3clusterName" + v3Service = "v3Service" + ) + var ( + v3ClusterAny = testutils.MarshalAny(t, &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LrsServer: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + }, + }) + v3ClusterWithEmptyName = testutils.MarshalAny(t, &v3clusterpb.Cluster{ + Name: "", + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LrsServer: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + }, + }) + + v3ClusterAnyWithEDSConfigSourceSelf = testutils.MarshalAny(t, &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{}, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LrsServer: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + }, + }) + + v3ClusterAnyWithTelemetryLabels = testutils.MarshalAny(t, &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LrsServer: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + }, + Metadata: &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "com.google.csm.telemetry_labels": { + Fields: map[string]*structpb.Value{ + "service_name": structpb.NewStringValue("grpc-service"), + "service_namespace": structpb.NewStringValue("grpc-service-namespace"), + }, + }, + }, + }, + }) + v3ClusterAnyWithTelemetryLabelsIgnoreSome = testutils.MarshalAny(t, &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LrsServer: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + }, + Metadata: &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "com.google.csm.telemetry_labels": { + Fields: map[string]*structpb.Value{ + "string-value-should-ignore": structpb.NewStringValue("string-val"), + "float-value-ignore": structpb.NewNumberValue(3), + "bool-value-ignore": structpb.NewBoolValue(false), + "service_name": structpb.NewStringValue("grpc-service"), // shouldn't ignore + "service_namespace": structpb.NewNullValue(), // should ignore - wrong type + }, + }, + "ignore-this-metadata": { // should ignore this filter_metadata + Fields: map[string]*structpb.Value{ + "service_namespace": structpb.NewStringValue("string-val-should-ignore"), + }, + }, + }, + }, + }) + ) + + serverCfg, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: "test-server"}) + if err != nil { + t.Fatalf("Failed to create server config for testing: %v", err) + } + + tests := []struct { + name string + resource *anypb.Any + serverCfg *bootstrap.ServerConfig + wantName string + wantUpdate ClusterUpdate + wantErr bool + }{ + { + name: "non-cluster resource type", + resource: &anypb.Any{TypeUrl: version.V3HTTPConnManagerURL}, + wantErr: true, + }, + { + name: "badly marshaled cluster resource", + resource: &anypb.Any{ + TypeUrl: version.V3ClusterURL, + Value: []byte{1, 2, 3, 4}, + }, + wantErr: true, + }, + { + name: "bad_cluster_resource", + resource: testutils.MarshalAny(t, &v3clusterpb.Cluster{ + Name: "test", + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC}, + }), + wantName: "test", + wantErr: true, + }, + { + name: "cluster_resource_with_non-self_lrs_server_field", + resource: testutils.MarshalAny(t, &v3clusterpb.Cluster{ + Name: "test", + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LrsServer: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + }), + wantName: "test", + wantErr: true, + }, + { + name: "v3_cluster", + resource: v3ClusterAny, + serverCfg: serverCfg, + wantName: v3ClusterName, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, + LRSServerConfig: serverCfg, + Raw: v3ClusterAny, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "cluster_resource_with_empty_name", + resource: v3ClusterWithEmptyName, + serverCfg: serverCfg, + wantName: "", + wantErr: true, + }, + { + name: "v3_cluster_wrapped", + resource: testutils.MarshalAny(t, &v3discoverypb.Resource{Resource: v3ClusterAny}), + serverCfg: serverCfg, + wantName: v3ClusterName, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, + LRSServerConfig: serverCfg, + Raw: v3ClusterAny, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "v3_cluster_with_EDS_config_source_self", + resource: v3ClusterAnyWithEDSConfigSourceSelf, + serverCfg: serverCfg, + wantName: v3ClusterName, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, + LRSServerConfig: serverCfg, + Raw: v3ClusterAnyWithEDSConfigSourceSelf, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "v3_cluster_with_telemetry_case", + resource: v3ClusterAnyWithTelemetryLabels, + serverCfg: serverCfg, + wantName: v3ClusterName, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, + LRSServerConfig: serverCfg, + Raw: v3ClusterAnyWithTelemetryLabels, + TelemetryLabels: map[string]string{ + "csm.service_name": "grpc-service", + "csm.service_namespace_name": "grpc-service-namespace", + }, + }, + }, + { + name: "v3_metadata_ignore_other_types_not_string_and_not_com.google.csm.telemetry_labels", + resource: v3ClusterAnyWithTelemetryLabelsIgnoreSome, + serverCfg: serverCfg, + wantName: v3ClusterName, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, + LRSServerConfig: serverCfg, + Raw: v3ClusterAnyWithTelemetryLabelsIgnoreSome, + TelemetryLabels: map[string]string{ + "csm.service_name": "grpc-service", + "csm.service_namespace_name": "unknown", + }, + }, + }, + { + name: "xdstp_cluster_resource_with_unset_EDS_service_name", + resource: testutils.MarshalAny(t, &v3clusterpb.Cluster{ + Name: "xdstp:foo", + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: "", + }, + }), + wantName: "xdstp:foo", + wantErr: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + name, update, err := unmarshalClusterResource(test.resource, test.serverCfg) + if (err != nil) != test.wantErr { + t.Fatalf("unmarshalClusterResource(%s), got err: %v, wantErr: %v", pretty.ToJSON(test.resource), err, test.wantErr) + } + if name != test.wantName { + t.Errorf("unmarshalClusterResource(%s), got name: %s, want: %s", pretty.ToJSON(test.resource), name, test.wantName) + } + if diff := cmp.Diff(update, test.wantUpdate, cmpOpts, cmpopts.IgnoreFields(ClusterUpdate{}, "LBPolicy")); diff != "" { + t.Errorf("unmarshalClusterResource(%s), got unexpected update, diff (-got +want): %v", pretty.ToJSON(test.resource), diff) + } + }) + } +} + +// enableGCPAuthenticationFilter enables A83 support for the duration of the +// test by: +// 1. Setting GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER env var to true. +// 2. Registering the audience converter (and restore the original on cleanup). +func enableGCPAuthenticationFilter(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.GCPAuthenticationFilterEnabled, true) + cleanup, err := RegisterMetadataConverterForTesting(version.V3AudienceURL) + if err != nil { + t.Fatalf("RegisterMetadataConverterForTesting(%q) failed: %v", version.V3AudienceURL, err) + } + t.Cleanup(cleanup) +} + +// disableGCPAuthenticationFilter disables A83 support for the duration of the +// test by: +// 1. Setting GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER env var to false. +// 2. Unregistering the audience converter for the duration of the test +// (and restoring the original on cleanup). +func disableGCPAuthenticationFilter(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.GCPAuthenticationFilterEnabled, false) + cleanup := UnregisterMetadataConverterForTesting(version.V3AudienceURL) + t.Cleanup(cleanup) +} + +// Tests custom metadata parsing for success cases when the +// GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER env var is set. +func (s) TestValidateClusterAndConstructClusterUpdate_GCP_AUTHENTICATION_FILTER_EnvVarOn(t *testing.T) { + enableGCPAuthenticationFilter(t) + const ( + v3ClusterName = "v3clusterName" + v3Service = "v3Service" + ) + tests := []struct { + name string + cluster *v3clusterpb.Cluster + wantUpdate ClusterUpdate + }{ + { + name: "typed_filter_metadata_in_cluster", + cluster: &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "com.google.grpc.gcp_authn": testutils.MarshalAny(t, &v3gcpauthnpb.Audience{ + Url: "https://example.com", + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, + Metadata: map[string]any{ + "com.google.grpc.gcp_authn": AudienceMetadataValue{ + Audience: "https://example.com", + }, + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "typed_filter_metadata_over_filter_metadata_in_cluster", + cluster: &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "com.google.grpc.gcp_authn": testutils.MarshalAny(t, &v3gcpauthnpb.Audience{ + Url: "https://example.com", + }), + }, + FilterMetadata: map[string]*structpb.Struct{ + "com.google.grpc.gcp_authn": { + Fields: map[string]*structpb.Value{ + "url": structpb.NewStringValue("https://example.com"), + }, + }, + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, + Metadata: map[string]any{ + "com.google.grpc.gcp_authn": AudienceMetadataValue{ + Audience: "https://example.com", + }, + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + name: "both_filter_and_typed_filter_metadata_in_cluster", + cluster: &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "com.google.grpc.gcp_authn": testutils.MarshalAny(t, &v3gcpauthnpb.Audience{ + Url: "https://example.com", + }), + }, + FilterMetadata: map[string]*structpb.Struct{ + "another-test-key": { + Fields: map[string]*structpb.Value{ + "url": structpb.NewStringValue("https://example.com"), + }, + }, + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, + Metadata: map[string]any{ + "com.google.grpc.gcp_authn": AudienceMetadataValue{ + Audience: "https://example.com", + }, + "another-test-key": StructMetadataValue{Data: map[string]any{ + "url": "https://example.com", + }}, + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := validateClusterAndConstructClusterUpdate(tt.cluster, nil) + if err != nil { + t.Fatalf("validateClusterAndConstructClusterUpdate() failed: %v", err) + } + if diff := cmp.Diff(tt.wantUpdate, got, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(ClusterUpdate{}, "LBPolicy")); diff != "" { + t.Errorf("validateClusterAndConstructClusterUpdate() returned unexpected diff (-want +got):\n%s", diff) + } + }) + } +} + +// Tests custom metadata parsing for failure cases when the +// GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER env var is not set. +func (s) TestValidateClusterAndConstructClusterUpdate_GCP_AUTHENTICATION_FILTER_EnvVarOff(t *testing.T) { + disableGCPAuthenticationFilter(t) + const ( + v3ClusterName = "v3ClusterName" + v3Service = "v3Service" + ) + tests := []struct { + name string + cluster *v3clusterpb.Cluster + wantUpdate ClusterUpdate + }{ + { + name: "typed_filter_metadata_in_cluster", + cluster: &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "com.google.grpc.gcp_authn": testutils.MarshalAny(t, &v3gcpauthnpb.Audience{ + Url: "https://example.com", + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := validateClusterAndConstructClusterUpdate(tt.cluster, nil) + if err != nil { + t.Fatalf("validateClusterAndConstructClusterUpdate() failed: %v", err) + } + if diff := cmp.Diff(tt.wantUpdate, got, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(ClusterUpdate{}, "LBPolicy")); diff != "" { + t.Errorf("validateClusterAndConstructClusterUpdate() returned unexpected diff (-want +got):\n%s", diff) + } + }) + } +} + +// Tests custom metadata parsing for converter failure cases when the +// GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER environment variable is set. +func (s) TestValidateClusterAndConstructClusterUpdate_GCP_AUTHENTICATION_FILTER_ConverterFailure(t *testing.T) { + enableGCPAuthenticationFilter(t) + tests := []struct { + name string + cluster *v3clusterpb.Cluster + wantErr string + }{ + { + name: "converter_failure_in_cluster", + cluster: &v3clusterpb.Cluster{ + Name: "v3clusterName", + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: "v3Service", + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "com.google.grpc.gcp_authn": testutils.MarshalAny(t, &v3gcpauthnpb.Audience{ + Url: "", + }), + }, + }, + }, + wantErr: "empty url field in audience metadata", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := validateClusterAndConstructClusterUpdate(tt.cluster, nil); err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("validateClusterAndConstructClusterUpdate() did not return error when expected: %v", err) + } + }) + } +} + +func (s) TestValidateClusterWithOutlierDetection(t *testing.T) { + odToClusterProto := func(od *v3clusterpb.OutlierDetection) *v3clusterpb.Cluster { + // Cluster parsing doesn't fail with respect to fields orthogonal to + // outlier detection. + return &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + OutlierDetection: od, + } + } + + tests := []struct { + name string + cluster *v3clusterpb.Cluster + wantODCfg string + wantErr bool + }{ + { + name: "success-and-failure-null", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{}), + wantODCfg: `{"successRateEjection": {}}`, + }, + { + name: "success-and-failure-zero", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{ + EnforcingSuccessRate: &wrapperspb.UInt32Value{Value: 0}, // Thus doesn't create sre - to focus on fpe + EnforcingFailurePercentage: &wrapperspb.UInt32Value{Value: 0}, + }), + wantODCfg: `{}`, + }, + { + name: "some-fields-set", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{ + Interval: &durationpb.Duration{Seconds: 1}, + MaxEjectionTime: &durationpb.Duration{Seconds: 3}, + EnforcingSuccessRate: &wrapperspb.UInt32Value{Value: 3}, + SuccessRateRequestVolume: &wrapperspb.UInt32Value{Value: 5}, + EnforcingFailurePercentage: &wrapperspb.UInt32Value{Value: 7}, + FailurePercentageRequestVolume: &wrapperspb.UInt32Value{Value: 9}, + }), + wantODCfg: `{ + "interval": "1s", + "maxEjectionTime": "3s", + "successRateEjection": { + "enforcementPercentage": 3, + "requestVolume": 5 + }, + "failurePercentageEjection": { + "enforcementPercentage": 7, + "requestVolume": 9 + } + }`, + }, + { + name: "every-field-set-non-zero", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{ + // all fields set (including ones that will be layered) should + // pick up those too and explicitly all fields, including those + // put in layers, in the JSON generated. + Interval: &durationpb.Duration{Seconds: 1}, + BaseEjectionTime: &durationpb.Duration{Seconds: 2}, + MaxEjectionTime: &durationpb.Duration{Seconds: 3}, + MaxEjectionPercent: &wrapperspb.UInt32Value{Value: 1}, + SuccessRateStdevFactor: &wrapperspb.UInt32Value{Value: 2}, + EnforcingSuccessRate: &wrapperspb.UInt32Value{Value: 3}, + SuccessRateMinimumHosts: &wrapperspb.UInt32Value{Value: 4}, + SuccessRateRequestVolume: &wrapperspb.UInt32Value{Value: 5}, + FailurePercentageThreshold: &wrapperspb.UInt32Value{Value: 6}, + EnforcingFailurePercentage: &wrapperspb.UInt32Value{Value: 7}, + FailurePercentageMinimumHosts: &wrapperspb.UInt32Value{Value: 8}, + FailurePercentageRequestVolume: &wrapperspb.UInt32Value{Value: 9}, + }), + wantODCfg: `{ + "interval": "1s", + "baseEjectionTime": "2s", + "maxEjectionTime": "3s", + "maxEjectionPercent": 1, + "successRateEjection": { + "stdevFactor": 2, + "enforcementPercentage": 3, + "minimumHosts": 4, + "requestVolume": 5 + }, + "failurePercentageEjection": { + "threshold": 6, + "enforcementPercentage": 7, + "minimumHosts": 8, + "requestVolume": 9 + } + }`, + }, + { + name: "interval-is-negative", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{Interval: &durationpb.Duration{Seconds: -10}}), + wantErr: true, + }, + { + name: "interval-overflows", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{Interval: &durationpb.Duration{Seconds: 315576000001}}), + wantErr: true, + }, + { + name: "base-ejection-time-is-negative", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{BaseEjectionTime: &durationpb.Duration{Seconds: -10}}), + wantErr: true, + }, + { + name: "base-ejection-time-overflows", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{BaseEjectionTime: &durationpb.Duration{Seconds: 315576000001}}), + wantErr: true, + }, + { + name: "max-ejection-time-is-negative", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{MaxEjectionTime: &durationpb.Duration{Seconds: -10}}), + wantErr: true, + }, + { + name: "max-ejection-time-overflows", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{MaxEjectionTime: &durationpb.Duration{Seconds: 315576000001}}), + wantErr: true, + }, + { + name: "max-ejection-percent-is-greater-than-100", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{MaxEjectionPercent: &wrapperspb.UInt32Value{Value: 150}}), + wantErr: true, + }, + { + name: "enforcing-success-rate-is-greater-than-100", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{EnforcingSuccessRate: &wrapperspb.UInt32Value{Value: 150}}), + wantErr: true, + }, + { + name: "failure-percentage-threshold-is-greater-than-100", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{FailurePercentageThreshold: &wrapperspb.UInt32Value{Value: 150}}), + wantErr: true, + }, + { + name: "enforcing-failure-percentage-is-greater-than-100", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{EnforcingFailurePercentage: &wrapperspb.UInt32Value{Value: 150}}), + wantErr: true, + }, + // A Outlier Detection proto not present should lead to a nil + // OutlierDetection field in the ClusterUpdate, which is implicitly + // tested in every other test in this file. + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + update, err := validateClusterAndConstructClusterUpdate(test.cluster, nil) + if (err != nil) != test.wantErr { + t.Errorf("validateClusterAndConstructClusterUpdate() returned err %v wantErr %v)", err, test.wantErr) + } + if test.wantErr { + return + } + // got and want must be unmarshalled since JSON strings shouldn't + // generally be directly compared. + var got map[string]any + if err := json.Unmarshal(update.OutlierDetection, &got); err != nil { + t.Fatalf("Error unmarshalling update.OutlierDetection (%q): %v", update.OutlierDetection, err) + } + var want map[string]any + if err := json.Unmarshal(json.RawMessage(test.wantODCfg), &want); err != nil { + t.Fatalf("Error unmarshalling wantODCfg (%q): %v", test.wantODCfg, err) + } + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("cluster.OutlierDetection got unexpected output, diff (-got, +want): %v", diff) + } + }) + } +} + +func (s) TestValidateClusterWithSecurityConfig_SNITooLong(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true) + + cluster := &v3clusterpb.Cluster{ + Name: "cluster", + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: "service", + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCert", + }, + }, + }, + }, + Sni: strings.Repeat("a", 256), + }), + }, + }, + } + + wantErr := "exceeds max length" + if _, err := validateClusterAndConstructClusterUpdate(cluster, nil); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("validateClusterAndConstructClusterUpdate() returned err: %v, want err containing: %s", err, wantErr) + } +} + +// TestValidateCluster_LRSReportEndpointMetrics tests the parsing of CDS +// lrs_report_endpoint_metrics field and verifies it is correctly ignored when +// the feature flag is disabled and parsed correctly when enabled, including +// precedence rules for named_metrics.*. +func (s) TestValidateCluster_LRSReportEndpointMetrics(t *testing.T) { + tests := []struct { + desc string + lrsPropEnabled bool + lrsReportEndpointMetrics []string + wantMetrics *LRSReportEndpointMetricsConfig + }{ + { + desc: "DisabledByEnvVar", + lrsPropEnabled: false, + lrsReportEndpointMetrics: []string{"cpu_utilization", "named_metrics.foo", "named_metrics.*"}, + wantMetrics: nil, + }, + { + desc: "AllValidMetrics", + lrsPropEnabled: true, + lrsReportEndpointMetrics: []string{"cpu_utilization", "mem_utilization", "application_utilization", "named_metrics.foo", "named_metrics.bar"}, + wantMetrics: &LRSReportEndpointMetricsConfig{ + CPUUtilization: true, + MemUtilization: true, + ApplicationUtilization: true, + NamedMetricsAll: false, + NamedMetrics: map[string]struct{}{ + "foo": {}, + "bar": {}, + }, + }, + }, + { + desc: "EnabledByEnvVar", + lrsPropEnabled: true, + lrsReportEndpointMetrics: []string{"named_metrics.*", "named_metrics.foo", "cpu_utilization"}, + wantMetrics: &LRSReportEndpointMetricsConfig{ + CPUUtilization: true, + NamedMetricsAll: true, + NamedMetrics: nil, + }, + }, + { + desc: "IgnoresInvalidMetrics", + lrsPropEnabled: true, + lrsReportEndpointMetrics: []string{"named_metrics.", "invalid_metric", "named_metrics.valid"}, + wantMetrics: &LRSReportEndpointMetricsConfig{ + NamedMetricsAll: false, + NamedMetrics: map[string]struct{}{ + "valid": {}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSORCAToLRSPropEnabled, tt.lrsPropEnabled) + cluster := &v3clusterpb.Cluster{ + Name: "test-cluster", + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: "test-service", + }, + LrsReportEndpointMetrics: tt.lrsReportEndpointMetrics, + } + update, err := validateClusterAndConstructClusterUpdate(cluster, nil) + if err != nil { + t.Fatalf("validateClusterAndConstructClusterUpdate() failed: %v", err) + } + if !update.LRSReportEndpointMetrics.Equal(tt.wantMetrics) { + t.Fatalf("LRSReportEndpointMetrics:\n got %+v\nwant %+v", update.LRSReportEndpointMetrics, tt.wantMetrics) + } + }) + } +} + +// Test verifies the validation and unmarshaling of CDS resources containing +// Http11ProxyUpstreamTransport transport socket wrapper when the env var +// is disabled. +func (s) TestValidateCluster_Http11ProxyUpstreamTransport_EnvVarOff(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSHTTPConnectEnabled, false) + const ( + v3ClusterName = "v3clusterName" + v3Service = "v3Service" + ) + + tests := []struct { + desc string + cluster *v3clusterpb.Cluster + wantErr bool + }{ + { + desc: "HTTP11ProxyUpstreamTransport_with_UpstreamTLSContext", + cluster: &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.http_11_proxy", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{ + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "identity_provider", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "root_provider", + }, + }, + }, + }, + }), + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + desc: "HTTP11ProxyUpstreamTransport_with_plaintext_inner_socket", + cluster: &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{ + TransportSocket: nil, + }), + }, + }, + }, + wantErr: true, + }, + { + desc: "HTTP11ProxyUpstreamTransport_with_unsupported_inner_socket", + cluster: &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3UpstreamTLSContextURL, + Value: []byte{1, 2, 3, 4}, + }, + }, + }, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + if _, err := validateClusterAndConstructClusterUpdate(tt.cluster, nil); err == nil { + t.Fatalf("validateClusterAndConstructClusterUpdate() did not return error when expected") + } + }) + } +} + +// Test verifies the validation and unmarshaling of CDS resources containing +// Http11ProxyUpstreamTransport transport socket wrapper when the env var +// is enabled. +func (s) TestValidateCluster_Http11ProxyUpstreamTransport_EnvVarOn(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSHTTPConnectEnabled, true) + const ( + v3ClusterName = "v3clusterName" + v3Service = "v3Service" + ) + cmpOpts := []cmp.Option{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(ClusterUpdate{}, "Raw", "LBPolicy", "TelemetryLabels"), + } + + tests := []struct { + desc string + cluster *v3clusterpb.Cluster + wantUpdate ClusterUpdate + wantErr bool + }{ + { + desc: "HTTP11ProxyUpstreamTransport_with_UpstreamTLSContext", + cluster: &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.http_11_proxy", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{ + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "identity_provider", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "root_provider", + }, + }, + }, + }, + }), + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, + IsHTTP11ProxyEnabled: true, + SecurityCfg: &SecurityConfig{ + IdentityInstanceName: "identity_provider", + RootInstanceName: "root_provider", + }, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + desc: "HTTP11ProxyUpstreamTransport_with_plaintext_inner_socket", + cluster: &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{ + TransportSocket: nil, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, + IsHTTP11ProxyEnabled: true, + SecurityCfg: nil, + TelemetryLabels: xdsinternal.UnknownCSMLabels, + }, + }, + { + desc: "HTTP11ProxyUpstreamTransport_with_unsupported_inner_socket", + cluster: &v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3UpstreamTLSContextURL, + Value: []byte{1, 2, 3, 4}, + }, + }, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + gotUpdate, err := validateClusterAndConstructClusterUpdate(tt.cluster, nil) + if tt.wantErr { + if err == nil { + t.Fatalf("validateClusterAndConstructClusterUpdate() did not return an error when expected") + } + return + } + if err != nil { + t.Fatalf("validateClusterAndConstructClusterUpdate() returned an error: %v", err) + } + + if diff := cmp.Diff(gotUpdate, tt.wantUpdate, cmpOpts...); diff != "" { + t.Errorf("validateClusterAndConstructClusterUpdate() returned unexpected diff (-got +want):\n%s", diff) + } + }) + } +} diff --git a/internal/xds/xdsclient/xdsresource/unmarshal_eds.go b/internal/xds/xdsclient/xdsresource/unmarshal_eds.go new file mode 100644 index 0000000..3327d67 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/unmarshal_eds.go @@ -0,0 +1,294 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + "math" + "net" + "strconv" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "google.golang.org/grpc/experimental/balancer/hostname" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/pretty" + xdsinternal "google.golang.org/grpc/internal/xds" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/ringhash" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +// Hostname returns the hostname from the BalancerAttributes of the +// given Address. If this attribute is not set, it returns the empty +// string. +func Hostname(addr resolver.Address) string { + ep := resolver.Endpoint{Attributes: addr.BalancerAttributes} + return hostname.FromEndpoint(ep) +} + +func unmarshalEndpointsResource(r *anypb.Any) (string, EndpointsUpdate, error) { + r, err := UnwrapResource(r) + if err != nil { + return "", EndpointsUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err) + } + + if !IsEndpointsResource(r.GetTypeUrl()) { + return "", EndpointsUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl()) + } + + cla := &v3endpointpb.ClusterLoadAssignment{} + if err := proto.Unmarshal(r.GetValue(), cla); err != nil { + return "", EndpointsUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + + if cla.GetClusterName() == "" { + return "", EndpointsUpdate{}, fmt.Errorf("empty resource name in endpoints resource") + } + + u, err := parseEDSRespProto(cla) + if err != nil { + return cla.GetClusterName(), EndpointsUpdate{}, err + } + u.Raw = r + + return cla.GetClusterName(), u, nil +} + +func parseAddress(socketAddress *v3corepb.SocketAddress) string { + return net.JoinHostPort(socketAddress.GetAddress(), strconv.Itoa(int(socketAddress.GetPortValue()))) +} + +func parseDropPolicy(dropPolicy *v3endpointpb.ClusterLoadAssignment_Policy_DropOverload) (OverloadDropConfig, error) { + percentage := dropPolicy.GetDropPercentage() + var ( + numerator = percentage.GetNumerator() + denominator uint32 + ) + switch percentage.GetDenominator() { + case v3typepb.FractionalPercent_HUNDRED: + denominator = 100 + case v3typepb.FractionalPercent_TEN_THOUSAND: + denominator = 10000 + case v3typepb.FractionalPercent_MILLION: + denominator = 1000000 + default: + return OverloadDropConfig{}, fmt.Errorf("EDS response contains a drop policy with unsupported denominator: %v", percentage.GetDenominator()) + } + return OverloadDropConfig{ + Category: dropPolicy.GetCategory(), + Numerator: numerator, + Denominator: denominator, + }, nil +} + +func parseEndpoints(lbEndpoints []*v3endpointpb.LbEndpoint, uniqueEndpointAddrs map[string]bool) ([]Endpoint, error) { + endpoints := make([]Endpoint, 0, len(lbEndpoints)) + var totalWeight uint64 + for _, lbEndpoint := range lbEndpoints { + // If the load_balancing_weight field is specified, it must be set to a + // value of at least 1. If unspecified, each host is presumed to have + // equal weight in a locality. + weight := uint32(1) + if w := lbEndpoint.GetLoadBalancingWeight(); w != nil { + if w.GetValue() == 0 { + return nil, fmt.Errorf("EDS response contains an endpoint with zero weight: %+v", lbEndpoint) + } + weight = w.GetValue() + } + + totalWeight += uint64(weight) + if totalWeight > math.MaxUint32 { + return nil, fmt.Errorf("sum of weights of endpoints in the same locality exceeds maximum value %d", uint64(math.MaxUint32)) + } + + addrs := []string{parseAddress(lbEndpoint.GetEndpoint().GetAddress().GetSocketAddress())} + if envconfig.XDSDualstackEndpointsEnabled { + for _, sa := range lbEndpoint.GetEndpoint().GetAdditionalAddresses() { + addrs = append(addrs, parseAddress(sa.GetAddress().GetSocketAddress())) + } + } + + address := []resolver.Address{} + for _, a := range addrs { + address = append(address, resolver.Address{Addr: a}) + if uniqueEndpointAddrs[a] { + return nil, fmt.Errorf("duplicate endpoint with the same address %s", a) + } + uniqueEndpointAddrs[a] = true + } + + var endpointMetadata map[string]any + var hashKey string + if envconfig.XDSHTTPConnectEnabled || !envconfig.XDSEndpointHashKeyBackwardCompat { + var err error + endpointMetadata, err = validateAndConstructMetadata(lbEndpoint.GetMetadata()) + if err != nil { + return nil, err + } + + // "The xDS resolver, described in A74, will be changed to set the hash_key + // endpoint attribute to the value of LbEndpoint.Metadata envoy.lb hash_key + // field, as described in Envoy's documentation for the ring hash load + // balancer." - A76 + if !envconfig.XDSEndpointHashKeyBackwardCompat { + hashKey = hashKeyFromMetadata(endpointMetadata) + } + } + endpoint := resolver.Endpoint{Addresses: address} + endpoint = hostname.Set(endpoint, lbEndpoint.GetEndpoint().GetHostname()) + endpoint = ringhash.SetHashKey(endpoint, hashKey) + endpoints = append(endpoints, Endpoint{ + ResolverEndpoint: endpoint, + HealthStatus: EndpointHealthStatus(lbEndpoint.GetHealthStatus()), + Weight: weight, + Metadata: endpointMetadata, + }) + } + return endpoints, nil +} + +// hashKey extracts and returns the hash key from the given endpoint metadata. +// If no hash key is found, it returns an empty string. +func hashKeyFromMetadata(metadata map[string]any) string { + envoyLB, ok := metadata["envoy.lb"].(StructMetadataValue) + if ok { + if h, ok := envoyLB.Data["hash_key"].(string); ok { + return h + } + } + return "" +} + +func parseEDSRespProto(m *v3endpointpb.ClusterLoadAssignment) (EndpointsUpdate, error) { + ret := EndpointsUpdate{} + for _, dropPolicy := range m.GetPolicy().GetDropOverloads() { + drop, err := parseDropPolicy(dropPolicy) + if err != nil { + return EndpointsUpdate{}, err + } + ret.Drops = append(ret.Drops, drop) + } + priorities := make(map[uint32]map[string]bool) + sumOfWeights := make(map[uint32]uint64) + uniqueEndpointAddrs := make(map[string]bool) + for _, locality := range m.Endpoints { + l := locality.GetLocality() + if l == nil { + return EndpointsUpdate{}, fmt.Errorf("EDS response contains a locality without ID, locality: %+v", locality) + } + weight := locality.GetLoadBalancingWeight().GetValue() + if weight == 0 { + logger.Warningf("Ignoring locality %s with weight 0", pretty.ToJSON(l)) + continue + } + priority := locality.GetPriority() + sumOfWeights[priority] += uint64(weight) + if sumOfWeights[priority] > math.MaxUint32 { + return EndpointsUpdate{}, fmt.Errorf("sum of weights of localities at the same priority %d exceeded maximal value", priority) + } + localitiesWithPriority := priorities[priority] + if localitiesWithPriority == nil { + localitiesWithPriority = make(map[string]bool) + priorities[priority] = localitiesWithPriority + } + lid := clients.Locality{ + Region: l.Region, + Zone: l.Zone, + SubZone: l.SubZone, + } + lidStr := xdsinternal.LocalityString(lid) + + // "Since an xDS configuration can place a given locality under multiple + // priorities, it is possible to see locality weight attributes with + // different values for the same locality." - A52 + // + // This is handled in the client by emitting the locality weight + // specified for the priority it is specified in. If the same locality + // has a different weight in two priorities, each priority will specify + // a locality with the locality weight specified for that priority, and + // thus the subsequent tree of balancers linked to that priority will + // use that locality weight as well. + if localitiesWithPriority[lidStr] { + return EndpointsUpdate{}, fmt.Errorf("duplicate locality %s with the same priority %v", lidStr, priority) + } + localitiesWithPriority[lidStr] = true + endpoints, err := parseEndpoints(locality.GetLbEndpoints(), uniqueEndpointAddrs) + if err != nil { + return EndpointsUpdate{}, err + } + var localityMetadata map[string]any + if envconfig.XDSHTTPConnectEnabled { + var err error + localityMetadata, err = validateAndConstructMetadata(locality.GetMetadata()) + if err != nil { + return EndpointsUpdate{}, err + } + } + + ret.Localities = append(ret.Localities, Locality{ + ID: lid, + Endpoints: endpoints, + Weight: weight, + Priority: priority, + Metadata: localityMetadata, + }) + } + for i := 0; i < len(priorities); i++ { + if _, ok := priorities[uint32(i)]; !ok { + return EndpointsUpdate{}, fmt.Errorf("priority %v missing (with different priorities %v received)", i, priorities) + } + } + return ret, nil +} + +// validateAndConstructMetadata processes the metadata from the xDS resource +// and returns a map of parsed metadata values. +func validateAndConstructMetadata(metadataProto *v3corepb.Metadata) (map[string]any, error) { + if metadataProto == nil { + return nil, nil + } + metadata := make(map[string]any) + // First go through TypedFilterMetadata. + for key, anyProto := range metadataProto.GetTypedFilterMetadata() { + converter := metadataConverterForType(anyProto.GetTypeUrl()) + // Ignore types we don't have a converter for. + if converter == nil { + continue + } + val, err := converter.convert(anyProto) + if err != nil { + // If the converter fails, nack the whole resource. + return nil, fmt.Errorf("metadata conversion for key %q and type %q failed: %v", key, anyProto.GetTypeUrl(), err) + } + metadata[key] = val + } + + // Process FilterMetadata for any keys not already handled. + for key, structProto := range metadataProto.GetFilterMetadata() { + // Skip keys already added from TypedFilterMetadata. + if metadata[key] != nil { + continue + } + metadata[key] = StructMetadataValue{Data: structProto.AsMap()} + } + return metadata, nil +} diff --git a/internal/xds/xdsclient/xdsresource/unmarshal_eds_test.go b/internal/xds/xdsclient/xdsresource/unmarshal_eds_test.go new file mode 100644 index 0000000..ef85fbb --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/unmarshal_eds_test.go @@ -0,0 +1,1721 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + "math" + "net" + "strconv" + "testing" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/experimental/balancer/hostname" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/clients" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/ringhash" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// enableA86 enables A86 support for the duration of the test by: +// 1. Setting the GRPC_EXPERIMENTAL_XDS_HTTP_CONNECT env var to true. +// 2. Registering the proxy address converter (and restore original +// on cleanup). +func enableA86(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSHTTPConnectEnabled, true) + cleanup, err := RegisterMetadataConverterForTesting(version.V3AddressURL) + if err != nil { + t.Fatalf("RegisterMetadataConverterForTesting(%q) failed: %v", version.V3AddressURL, err) + } + t.Cleanup(cleanup) +} + +// disableA86 disables A86 support for the duration of the test by: +// 1. Setting the GRPC_EXPERIMENTAL_XDS_HTTP_CONNECT env var to false. +// 2. Unregistering the proxy address converter for the duration of the test +// (and restoring the original on cleanup). +func disableA86(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSHTTPConnectEnabled, false) + cleanup := UnregisterMetadataConverterForTesting(version.V3AddressURL) + t.Cleanup(cleanup) +} + +func buildResolverEndpoint(addr []string, host string) resolver.Endpoint { + address := []resolver.Address{} + for _, a := range addr { + address = append(address, resolver.Address{Addr: a}) + } + resolverEndpoint := resolver.Endpoint{Addresses: address} + resolverEndpoint = hostname.Set(resolverEndpoint, host) + return resolverEndpoint +} + +func (s) TestEDSParseRespProto(t *testing.T) { + tests := []struct { + name string + m *v3endpointpb.ClusterLoadAssignment + want EndpointsUpdate + wantErr bool + }{ + { + name: "missing-priority", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 0, []endpointOpts{{addrWithPort: "addr1:314"}}, nil) + clab0.addLocality("locality-2", 1, 2, []endpointOpts{{addrWithPort: "addr2:159"}}, nil) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "missing-locality-ID", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("", 1, 0, []endpointOpts{{addrWithPort: "addr1:314"}}, nil) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "zero-endpoint-weight", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-0", 1, 0, []endpointOpts{{addrWithPort: "addr1:314"}}, &addLocalityOptions{Weight: []uint32{0}}) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "duplicate-locality-in-the-same-priority", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-0", 1, 0, []endpointOpts{{addrWithPort: "addr1:314"}}, nil) + clab0.addLocality("locality-0", 1, 0, []endpointOpts{{addrWithPort: "addr1:314"}}, nil) // Duplicate locality with the same priority. + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "missing locality weight", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints1 := []endpointOpts{{addrWithPort: "addr1:314"}} + locOption1 := &addLocalityOptions{Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_HEALTHY}} + clab0.addLocality("locality-1", 0, 1, endpoints1, locOption1) + endpoints2 := []endpointOpts{{addrWithPort: "addr2:159"}} + locOption2 := &addLocalityOptions{Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_HEALTHY}} + clab0.addLocality("locality-2", 0, 0, endpoints2, locOption2) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + }, + { + name: "max sum of endpoint weights within a locality exceeded", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{ + {addrWithPort: "addr1:314"}, + {addrWithPort: "addr2:159"}, + } + locOption := &addLocalityOptions{ + Weight: []uint32{math.MaxUint32, 1}, + } + clab0.addLocality("locality-1", 1, 0, endpoints, locOption) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "max sum of weights at the same priority exceeded", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 0, []endpointOpts{{addrWithPort: "addr1:314"}}, nil) + clab0.addLocality("locality-2", 4294967295, 1, []endpointOpts{{addrWithPort: "addr2:159"}}, nil) + clab0.addLocality("locality-3", 1, 1, []endpointOpts{{addrWithPort: "addr2:88"}}, nil) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "duplicate endpoint address", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 1, []endpointOpts{{addrWithPort: "addr:997"}}, nil) + clab0.addLocality("locality-2", 1, 0, []endpointOpts{{addrWithPort: "addr:997"}}, nil) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "unsupported-drop-denominator", + m: &v3endpointpb.ClusterLoadAssignment{ + ClusterName: "test", + Policy: &v3endpointpb.ClusterLoadAssignment_Policy{ + DropOverloads: []*v3endpointpb.ClusterLoadAssignment_Policy_DropOverload{{ + Category: "test-drop", + DropPercentage: &v3typepb.FractionalPercent{ + Numerator: 1, + Denominator: v3typepb.FractionalPercent_DenominatorType(7), + }, + }}, + }, + }, + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "good", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints1 := []endpointOpts{{addrWithPort: "addr1:314", hostname: "addr1"}} + locOption1 := &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY}, + Weight: []uint32{271}, + } + clab0.addLocality("locality-1", 1, 1, endpoints1, locOption1) + endpoints2 := []endpointOpts{{addrWithPort: "addr2:159", hostname: "addr2"}} + locOption2 := &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING}, + Weight: []uint32{828}, + } + clab0.addLocality("locality-2", 1, 0, endpoints2, locOption2) + return clab0.Build() + }(), + want: EndpointsUpdate{ + Drops: nil, + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnhealthy, + Weight: 271, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 1, + Weight: 1, + }, + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr2:159"}, "addr2"), + HealthStatus: EndpointHealthStatusDraining, + Weight: 828, + }}, + ID: clients.Locality{SubZone: "locality-2"}, + Priority: 0, + Weight: 1, + }, + }, + }, + wantErr: false, + }, + { + name: "good duplicate locality with different priority", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints1 := []endpointOpts{{addrWithPort: "addr1:314", hostname: "addr1"}} + locOption1 := &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY}, + Weight: []uint32{271}, + } + clab0.addLocality("locality-1", 1, 1, endpoints1, locOption1) + // Same locality name, but with different priority. + endpoints2 := []endpointOpts{{addrWithPort: "addr2:159", hostname: "addr2"}} + locOption2 := &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING}, + Weight: []uint32{828}, + } + clab0.addLocality("locality-1", 1, 0, endpoints2, locOption2) + return clab0.Build() + }(), + want: EndpointsUpdate{ + Drops: nil, + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnhealthy, + Weight: 271, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 1, + Weight: 1, + }, + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr2:159"}, "addr2"), + HealthStatus: EndpointHealthStatusDraining, + Weight: 828, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseEDSRespProto(tt.m) + if (err != nil) != tt.wantErr { + t.Fatalf("parseEDSRespProto() error = %v, wantErr %v", err, tt.wantErr) + } + if d := cmp.Diff(got, tt.want, cmpopts.EquateEmpty()); d != "" { + t.Errorf("parseEDSRespProto() got = %v, want %v, diff: %v", got, tt.want, d) + } + }) + } +} + +func (s) TestEDSParseRespProtoAdditionalAddrs(t *testing.T) { + origDualstackEndpointsEnabled := envconfig.XDSDualstackEndpointsEnabled + defer func() { + envconfig.XDSDualstackEndpointsEnabled = origDualstackEndpointsEnabled + }() + envconfig.XDSDualstackEndpointsEnabled = true + + tests := []struct { + name string + m *v3endpointpb.ClusterLoadAssignment + want EndpointsUpdate + wantErr bool + }{ + { + name: "duplicate primary address in self additional addresses", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 0, []endpointOpts{{addrWithPort: "addr:998", additionalAddrWithPorts: []string{"addr:998"}}}, nil) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "duplicate primary address in other locality additional addresses", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 1, []endpointOpts{{addrWithPort: "addr:997"}}, nil) + clab0.addLocality("locality-2", 1, 0, []endpointOpts{{addrWithPort: "addr:998", additionalAddrWithPorts: []string{"addr:997"}}}, nil) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "duplicate additional address in self additional addresses", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 0, []endpointOpts{{addrWithPort: "addr:998", additionalAddrWithPorts: []string{"addr:999", "addr:999"}}}, nil) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "duplicate additional address in other locality additional addresses", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 1, []endpointOpts{{addrWithPort: "addr:997", additionalAddrWithPorts: []string{"addr:1000"}}}, nil) + clab0.addLocality("locality-2", 1, 0, []endpointOpts{{addrWithPort: "addr:998", additionalAddrWithPorts: []string{"addr:1000"}}}, nil) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "multiple localities", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints1 := []endpointOpts{{addrWithPort: "addr1:997", additionalAddrWithPorts: []string{"addr1:1000"}, hostname: "addr1"}} + locOption1 := &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY}, + Weight: []uint32{271}, + } + clab0.addLocality("locality-1", 1, 1, endpoints1, locOption1) + endpoints2 := []endpointOpts{{addrWithPort: "addr2:998", additionalAddrWithPorts: []string{"addr2:1000"}, hostname: "addr2"}} + locOption2 := &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_HEALTHY}, + Weight: []uint32{828}, + } + clab0.addLocality("locality-2", 1, 0, endpoints2, locOption2) + return clab0.Build() + }(), + want: EndpointsUpdate{ + Drops: nil, + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:997", "addr1:1000"}, "addr1"), + HealthStatus: EndpointHealthStatusUnhealthy, + Weight: 271, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 1, + Weight: 1, + }, + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr2:998", "addr2:1000"}, "addr2"), + HealthStatus: EndpointHealthStatusHealthy, + Weight: 828, + }}, + ID: clients.Locality{SubZone: "locality-2"}, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseEDSRespProto(tt.m) + if (err != nil) != tt.wantErr { + t.Fatalf("parseEDSRespProto() error = %v, wantErr %v", err, tt.wantErr) + } + if d := cmp.Diff(got, tt.want, cmpopts.EquateEmpty()); d != "" { + t.Errorf("parseEDSRespProto() got = %v, want %v, diff: %v", got, tt.want, d) + } + }) + } +} + +func (s) TestUnmarshalEndpointHashKey(t *testing.T) { + baseCLA := &v3endpointpb.ClusterLoadAssignment{ + ClusterName: "test-cluster", + Endpoints: []*v3endpointpb.LocalityLbEndpoints{ + { + Locality: &v3corepb.Locality{Region: "r"}, + LbEndpoints: []*v3endpointpb.LbEndpoint{ + { + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "test-address", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 8080, + }, + }, + }, + }, + }, + }, + }, + }, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + }, + }, + } + + tests := []struct { + name string + metadata *v3corepb.Metadata + wantHashKey string + compatEnvVar bool + }{ + { + name: "no metadata", + metadata: nil, + wantHashKey: "", + }, + { + name: "empty metadata", + metadata: &v3corepb.Metadata{}, + wantHashKey: "", + }, + { + name: "filter metadata without envoy.lb", + metadata: &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "test-filter": {}, + }, + }, + wantHashKey: "", + }, + { + name: "nil envoy.lb", + metadata: &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "envoy.lb": nil, + }, + }, + wantHashKey: "", + }, + { + name: "envoy.lb without hash key", + metadata: &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "envoy.lb": { + Fields: map[string]*structpb.Value{ + "hash_key": { + Kind: &structpb.Value_NumberValue{NumberValue: 123.0}, + }, + }, + }, + }, + }, + wantHashKey: "", + }, + { + name: "envoy.lb with hash key, compat mode off", + metadata: &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "envoy.lb": { + Fields: map[string]*structpb.Value{ + "hash_key": { + Kind: &structpb.Value_StringValue{StringValue: "test-hash-key"}, + }, + }, + }, + }, + }, + wantHashKey: "test-hash-key", + }, + { + name: "envoy.lb with hash key, compat mode on", + metadata: &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "envoy.lb": { + Fields: map[string]*structpb.Value{ + "hash_key": { + Kind: &structpb.Value_StringValue{StringValue: "test-hash-key"}, + }, + }, + }, + }, + }, + wantHashKey: "", + compatEnvVar: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSEndpointHashKeyBackwardCompat, test.compatEnvVar) + + cla := proto.Clone(baseCLA).(*v3endpointpb.ClusterLoadAssignment) + cla.Endpoints[0].LbEndpoints[0].Metadata = test.metadata + marshalledCLA := testutils.MarshalAny(t, cla) + _, update, err := unmarshalEndpointsResource(marshalledCLA) + if err != nil { + t.Fatalf("unmarshalEndpointsResource() got error = %v, want success", err) + } + got := ringhash.HashKey(update.Localities[0].Endpoints[0].ResolverEndpoint) + if got != test.wantHashKey { + t.Errorf("unmarshalEndpointResource() endpoint hash key: got %s, want %s", got, test.wantHashKey) + } + }) + } +} + +func (s) TestUnmarshalEndpoints(t *testing.T) { + enableA86(t) + var v3EndpointsAny = testutils.MarshalAny(t, func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints1 := []endpointOpts{{addrWithPort: "addr1:314", hostname: "addr1"}} + locOption1 := &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY}, + Weight: []uint32{271}, + } + clab0.addLocality("locality-1", 1, 1, endpoints1, locOption1) + endpoints2 := []endpointOpts{{addrWithPort: "addr2:159", hostname: "addr2"}} + locOption2 := &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING}, + Weight: []uint32{828}, + } + clab0.addLocality("locality-2", 1, 0, endpoints2, locOption2) + return clab0.Build() + }()) + + tests := []struct { + name string + resource *anypb.Any + wantName string + wantUpdate EndpointsUpdate + wantErr bool + }{ + { + name: "non-clusterLoadAssignment_resourcetype", + resource: &anypb.Any{TypeUrl: version.V3HTTPConnManagerURL}, + wantErr: true, + }, + { + name: "badly_marshaled_clusterLoadAssignment_resource", + resource: &anypb.Any{ + TypeUrl: version.V3EndpointsURL, + Value: []byte{1, 2, 3, 4}, + }, + wantErr: true, + }, + { + name: "bad_endpoints_resource", + resource: testutils.MarshalAny(t, func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 0, []endpointOpts{{addrWithPort: "addr1:314"}}, nil) + clab0.addLocality("locality-2", 1, 2, []endpointOpts{{addrWithPort: "addr2:159"}}, nil) + return clab0.Build() + }()), + wantName: "test", + wantErr: true, + }, + { + name: "endpoint_resource_with_empty_cluster_name", + resource: testutils.MarshalAny(t, func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("", nil) + clab0.addLocality("locality-1", 1, 0, []endpointOpts{{addrWithPort: "addr1:314"}}, nil) + clab0.addLocality("locality-2", 1, 2, []endpointOpts{{addrWithPort: "addr2:159"}}, nil) + return clab0.Build() + }()), + wantName: "", + wantErr: true, + }, + { + name: "v3_endpoints", + resource: v3EndpointsAny, + wantName: "test", + wantUpdate: EndpointsUpdate{ + Drops: nil, + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnhealthy, + Weight: 271, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 1, + Weight: 1, + }, + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr2:159"}, "addr2"), + HealthStatus: EndpointHealthStatusDraining, + Weight: 828, + }}, + ID: clients.Locality{SubZone: "locality-2"}, + Priority: 0, + Weight: 1, + }, + }, + Raw: v3EndpointsAny, + }, + }, + { + name: "v3_endpoints_wrapped", + resource: testutils.MarshalAny(t, &v3discoverypb.Resource{Resource: v3EndpointsAny}), + wantName: "test", + wantUpdate: EndpointsUpdate{ + Drops: nil, + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnhealthy, + Weight: 271, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 1, + Weight: 1, + }, + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr2:159"}, "addr2"), + HealthStatus: EndpointHealthStatusDraining, + Weight: 828, + }}, + ID: clients.Locality{SubZone: "locality-2"}, + Priority: 0, + Weight: 1, + }, + }, + Raw: v3EndpointsAny, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + name, update, err := unmarshalEndpointsResource(test.resource) + if (err != nil) != test.wantErr { + t.Fatalf("unmarshalEndpointsResource(%s), got err: %v, wantErr: %v", pretty.ToJSON(test.resource), err, test.wantErr) + } + if name != test.wantName { + t.Errorf("unmarshalEndpointsResource(%s), got name: %s, want: %s", pretty.ToJSON(test.resource), name, test.wantName) + } + if diff := cmp.Diff(update, test.wantUpdate, cmpOpts); diff != "" { + t.Errorf("unmarshalEndpointsResource(%s), got unexpected update, diff (-got +want): %v", pretty.ToJSON(test.resource), diff) + } + }) + } +} + +// Tests custom metadata parsing for success cases when the +// GRPC_EXPERIMENTAL_XDS_HTTP_CONNECT environment variable is set. +func (s) TestEDSParseRespProto_HTTP_Connect_CustomMetadata_EnvVarOn(t *testing.T) { + enableA86(t) + tests := []struct { + name string + endpointProto *v3endpointpb.ClusterLoadAssignment + wantEndpoint EndpointsUpdate + }{ + { + name: "typed_filter_metadata_in_endpoint", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{ + addrWithPort: "addr1:314", + metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "test-key": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "1.2.3.4", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 1111, + }}, + }, + }), + }, + }, + hostname: "addr1", + }} + clab0.addLocality("locality-1", 1, 0, endpoints, nil) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + Metadata: map[string]any{ + "test-key": ProxyAddressMetadataValue{ + Address: "1.2.3.4:1111", + }, + }, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + { + name: "filter_metadata_in_endpoint", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{ + addrWithPort: "addr1:314", + metadata: &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "test-key": { + Fields: map[string]*structpb.Value{ + "key": { + Kind: &structpb.Value_NumberValue{NumberValue: 123.0}, + }, + }, + }, + }, + }, + hostname: "addr1", + }} + clab0.addLocality("locality-1", 1, 0, endpoints, nil) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + Metadata: map[string]any{ + "test-key": StructMetadataValue{Data: map[string]any{ + "key": float64(123), + }}, + }, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + { + name: "typed_filter_metadata_in_locality", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{addrWithPort: "addr1:314", hostname: "addr1"}} + locOption := &addLocalityOptions{ + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "test-key": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "1.2.3.4", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 1111, + }}, + }, + }), + }, + }, + } + clab0.addLocality("locality-1", 1, 0, endpoints, locOption) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + Metadata: map[string]any{ + "test-key": ProxyAddressMetadataValue{ + Address: "1.2.3.4:1111", + }, + }, + }, + }, + }, + }, + { + name: "filter_metadata_in_locality", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{addrWithPort: "addr1:314", hostname: "addr1"}} + locOption := &addLocalityOptions{ + Metadata: &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "test-key": { + Fields: map[string]*structpb.Value{ + "key": { + Kind: &structpb.Value_NumberValue{NumberValue: 123.0}, + }, + }, + }, + }, + }, + } + clab0.addLocality("locality-1", 1, 0, endpoints, locOption) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + Metadata: map[string]any{ + "test-key": StructMetadataValue{Data: map[string]any{ + "key": float64(123), + }}, + }, + }, + }, + }, + }, + { + name: "typed_filter_metadata_over_filter_metadata_in_endpoint", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{ + addrWithPort: "addr1:314", + metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "test-key": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "1.2.3.4", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 1111, + }}, + }, + }), + }, + FilterMetadata: map[string]*structpb.Struct{ + "test-key": { + Fields: map[string]*structpb.Value{ + "key": { + Kind: &structpb.Value_NumberValue{NumberValue: 123.0}, + }, + }, + }, + }, + }, + hostname: "addr1", + }} + clab0.addLocality("locality-1", 1, 0, endpoints, nil) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + Metadata: map[string]any{ + "test-key": ProxyAddressMetadataValue{ + Address: "1.2.3.4:1111", + }, + }, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + { + name: "typed_filter_metadata_over_filter_metadata_in_locality", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{addrWithPort: "addr1:314", hostname: "addr1"}} + locOption := &addLocalityOptions{ + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "test-key": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "1.2.3.4", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 1111, + }}, + }, + }), + }, + FilterMetadata: map[string]*structpb.Struct{ + "test-key": { + Fields: map[string]*structpb.Value{ + "key": { + Kind: &structpb.Value_NumberValue{NumberValue: 123.0}, + }, + }, + }, + }, + }, + } + clab0.addLocality("locality-1", 1, 0, endpoints, locOption) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + Metadata: map[string]any{ + "test-key": ProxyAddressMetadataValue{ + Address: "1.2.3.4:1111", + }, + }, + }, + }, + }, + }, + { + name: "both_filter_and_typed_filter_metadata_in_endpoint", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{ + addrWithPort: "addr1:314", + metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "test-key": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "1.2.3.4", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 1111, + }}, + }, + }), + }, + FilterMetadata: map[string]*structpb.Struct{ + "another-test-key": { + Fields: map[string]*structpb.Value{ + "key": { + Kind: &structpb.Value_NumberValue{NumberValue: 123.0}, + }, + }, + }, + }, + }, + hostname: "addr1", + }} + clab0.addLocality("locality-1", 1, 0, endpoints, nil) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + Metadata: map[string]any{ + "test-key": ProxyAddressMetadataValue{ + Address: "1.2.3.4:1111", + }, + "another-test-key": StructMetadataValue{Data: map[string]any{ + "key": float64(123), + }}, + }, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + { + name: "both_filter_and_typed_filter_metadata_in_locality", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{addrWithPort: "addr1:314", hostname: "addr1"}} + locOption := &addLocalityOptions{ + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "test-key": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "1.2.3.4", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 1111, + }}, + }, + }), + }, + FilterMetadata: map[string]*structpb.Struct{ + "another-test-key": { + Fields: map[string]*structpb.Value{ + "key": { + Kind: &structpb.Value_NumberValue{NumberValue: 123.0}, + }, + }, + }, + }, + }, + } + clab0.addLocality("locality-1", 1, 0, endpoints, locOption) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + Metadata: map[string]any{ + "test-key": ProxyAddressMetadataValue{ + Address: "1.2.3.4:1111", + }, + "another-test-key": StructMetadataValue{Data: map[string]any{ + "key": float64(123), + }}, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseEDSRespProto(tt.endpointProto) + if err != nil { + t.Fatalf("parseEDSRespProto() failed: %v", err) + } + if diff := cmp.Diff(tt.wantEndpoint, got, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("parseEDSRespProto() returned unexpected diff (-want +got):\n%s", diff) + } + }) + } +} + +// Tests custom metadata parsing for success cases when the +// GRPC_EXPERIMENTAL_XDS_HTTP_CONNECT environment variable is not set and +// GRPC_XDS_ENDPOINT_HASH_KEY_BACKWARD_COMPAT is set to true (disabling A76). +func (s) TestEDSParseRespProto_HTTP_Connect_CustomMetadata_EnvVarOff(t *testing.T) { + disableA86(t) + testutils.SetEnvConfig(t, &envconfig.XDSEndpointHashKeyBackwardCompat, true) + tests := []struct { + name string + endpointProto *v3endpointpb.ClusterLoadAssignment + wantEndpoint EndpointsUpdate + }{ + { + name: "typed_filter_metadata_in_endpoint", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{ + addrWithPort: "addr1:314", + metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "test-key": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "1.2.3.4", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 1111, + }}, + }, + }), + }, + }, + hostname: "addr1", + }} + clab0.addLocality("locality-1", 1, 0, endpoints, nil) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + { + name: "filter_metadata_in_endpoint", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{ + addrWithPort: "addr1:314", + metadata: &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "test-key": { + Fields: map[string]*structpb.Value{ + "key": { + Kind: &structpb.Value_NumberValue{NumberValue: 123.0}, + }, + }, + }, + }, + }, + hostname: "addr1", + }} + clab0.addLocality("locality-1", 1, 0, endpoints, nil) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + { + name: "typed_filter_metadata_in_locality", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{addrWithPort: "addr1:314", hostname: "addr1"}} + locOption := &addLocalityOptions{ + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "test-key": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "1.2.3.4", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 1111, + }}, + }, + }), + }, + }, + } + clab0.addLocality("locality-1", 1, 0, endpoints, locOption) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + { + name: "filter_metadata_in_locality", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{addrWithPort: "addr1:314", hostname: "addr1"}} + locOption := &addLocalityOptions{ + Metadata: &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "test-key": { + Fields: map[string]*structpb.Value{ + "key": { + Kind: &structpb.Value_NumberValue{NumberValue: 123.0}, + }, + }, + }, + }, + }, + } + clab0.addLocality("locality-1", 1, 0, endpoints, locOption) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + { + name: "both_filter_and_typed_filter_metadata_in_endpoint", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{ + addrWithPort: "addr1:314", + metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "test-key": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "1.2.3.4", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 1111, + }}, + }, + }), + }, + FilterMetadata: map[string]*structpb.Struct{ + "another-test-key": { + Fields: map[string]*structpb.Value{ + "key": { + Kind: &structpb.Value_NumberValue{NumberValue: 123.0}, + }, + }, + }, + }, + }, + hostname: "addr1", + }} + clab0.addLocality("locality-1", 1, 0, endpoints, nil) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + { + name: "both_filter_and_typed_filter_metadata_in_locality", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{addrWithPort: "addr1:314", hostname: "addr1"}} + locOption := &addLocalityOptions{ + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "test-key": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "1.2.3.4", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 1111, + }}, + }, + }), + }, + FilterMetadata: map[string]*structpb.Struct{ + "another-test-key": { + Fields: map[string]*structpb.Value{ + "key": { + Kind: &structpb.Value_NumberValue{NumberValue: 123.0}, + }, + }, + }, + }, + }, + } + clab0.addLocality("locality-1", 1, 0, endpoints, locOption) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + { + name: "converter_failure_is_not_triggerred", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{ + addrWithPort: "addr1:314", + metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "invalid", + }, + }, + }), + }, + }, + hostname: "addr1", + }} + locOption := &addLocalityOptions{ + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "invalid", + }, + }, + }), + }, + }, + } + clab0.addLocality("locality-1", 1, 0, endpoints, locOption) + return clab0.Build() + }(), + wantEndpoint: EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseEDSRespProto(tt.endpointProto) + if err != nil { + t.Fatalf("parseEDSRespProto() failed: %v", err) + } + if diff := cmp.Diff(tt.wantEndpoint, got, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("parseEDSRespProto() returned unexpected diff (-want +got):\n%s", diff) + } + }) + } +} + +// Tests custom metadata parsing for converter failure cases when the +// GRPC_EXPERIMENTAL_XDS_HTTP_CONNECT environment variable is set. +func (s) TestEDSParseRespProto_HTTP_Connect_CustomMetadata_ConverterFailure(t *testing.T) { + enableA86(t) + tests := []struct { + name string + endpointProto *v3endpointpb.ClusterLoadAssignment + }{ + { + name: "converter_failure_in_endpoint", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{ + addrWithPort: "addr1:314", + metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "invalid", + }, + }, + }), + }, + }, + }} + clab0.addLocality("locality-1", 1, 0, endpoints, nil) + return clab0.Build() + }(), + }, + { + name: "converter_failure_in_locality", + endpointProto: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{addrWithPort: "addr1:314"}} + locOption := &addLocalityOptions{ + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "invalid", + }, + }, + }), + }, + }, + } + clab0.addLocality("locality-1", 1, 0, endpoints, locOption) + return clab0.Build() + }(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := parseEDSRespProto(tt.endpointProto); err == nil { + t.Fatalf("parseEDSRespProto() did not return error when expected") + } + }) + } +} + +// Tests metadata parsing when HTTP Connect is enabled but A76 hash key is +// disabled (backward compat mode). This verifies that: +// - Metadata parsing happens (TypedFilterMetadata + FilterMetadata) +// - Hash key is NOT extracted from envoy.lb +func (s) TestEDSParseRespProto_HTTP_Connect_On_HashKeyBackwardCompat_On(t *testing.T) { + enableA86(t) + testutils.SetEnvConfig(t, &envconfig.XDSEndpointHashKeyBackwardCompat, true) + + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{ + addrWithPort: "addr1:314", + metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "1.2.3.4", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 1111, + }, + }, + }, + }), + }, + FilterMetadata: map[string]*structpb.Struct{ + "envoy.lb": { + Fields: map[string]*structpb.Value{ + "hash_key": { + Kind: &structpb.Value_StringValue{StringValue: "test-hash-key"}, + }, + }, + }, + }, + }, + hostname: "addr1", + }} + clab0.addLocality("locality-1", 1, 0, endpoints, nil) + + got, err := parseEDSRespProto(clab0.Build()) + if err != nil { + t.Fatalf("parseEDSRespProto() failed: %v", err) + } + + wantEndpoint := EndpointsUpdate{ + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + ResolverEndpoint: buildResolverEndpoint([]string{"addr1:314"}, "addr1"), + HealthStatus: EndpointHealthStatusUnknown, + Weight: 1, + Metadata: map[string]any{ + "envoy.http11_proxy_transport_socket.proxy_address": ProxyAddressMetadataValue{ + Address: "1.2.3.4:1111", + }, + "envoy.lb": StructMetadataValue{Data: map[string]any{ + "hash_key": "test-hash-key", + }}, + }, + }}, + ID: clients.Locality{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + } + + if diff := cmp.Diff(wantEndpoint, got, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("parseEDSRespProto() returned unexpected diff (-want +got):\n%s", diff) + } + + // Verify hash key is NOT extracted when backward compat is on. + hashKey := ringhash.HashKey(got.Localities[0].Endpoints[0].ResolverEndpoint) + if hashKey != "" { + t.Errorf("Expected empty hash key with backward compat on, got %q", hashKey) + } +} + +// Tests that when A76 is enabled but A86 is disabled, invalid typed metadata +// does not cause a parsing failure, and hash key is still extracted. +func (s) TestEDSParseRespProto_HTTP_Connect_Off_HashKeyBackwardCompat_Off_InvalidTypedMetadata(t *testing.T) { + disableA86(t) + testutils.SetEnvConfig(t, &envconfig.XDSEndpointHashKeyBackwardCompat, false) // A76 on + + clab0 := newClaBuilder("test", nil) + endpoints := []endpointOpts{{ + addrWithPort: "addr1:314", + metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "invalid", // This would fail conversion if processed. + }, + }, + }), + }, + FilterMetadata: map[string]*structpb.Struct{ + "envoy.lb": { + Fields: map[string]*structpb.Value{ + "hash_key": { + Kind: &structpb.Value_StringValue{StringValue: "test-hash-key"}, + }, + }, + }, + }, + }, + hostname: "addr1", + }} + clab0.addLocality("locality-1", 1, 0, endpoints, nil) + + got, err := parseEDSRespProto(clab0.Build()) + if err != nil { + t.Fatalf("parseEDSRespProto() failed unexpectedly with A76 on and A86 off: %v", err) + } + + // Verify hash key is extracted when A76 is on. + hashKey := ringhash.HashKey(got.Localities[0].Endpoints[0].ResolverEndpoint) + if want := "test-hash-key"; hashKey != want { + t.Errorf("Expected hash key %q with A76 on, got %q", want, hashKey) + } +} + +// claBuilder builds a ClusterLoadAssignment, aka EDS +// response. +type claBuilder struct { + v *v3endpointpb.ClusterLoadAssignment +} + +// newClaBuilder creates a claBuilder. +func newClaBuilder(clusterName string, dropPercents []uint32) *claBuilder { + var drops []*v3endpointpb.ClusterLoadAssignment_Policy_DropOverload + for i, d := range dropPercents { + drops = append(drops, &v3endpointpb.ClusterLoadAssignment_Policy_DropOverload{ + Category: fmt.Sprintf("test-drop-%d", i), + DropPercentage: &v3typepb.FractionalPercent{ + Numerator: d, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }) + } + + return &claBuilder{ + v: &v3endpointpb.ClusterLoadAssignment{ + ClusterName: clusterName, + Policy: &v3endpointpb.ClusterLoadAssignment_Policy{ + DropOverloads: drops, + }, + }, + } +} + +// addLocalityOptions contains options when adding locality to the builder. +type addLocalityOptions struct { + Health []v3corepb.HealthStatus + Weight []uint32 + Metadata *v3corepb.Metadata +} + +type endpointOpts struct { + addrWithPort string + additionalAddrWithPorts []string + metadata *v3corepb.Metadata + hostname string +} + +func addressFromStr(addrWithPort string) *v3corepb.Address { + host, portStr, err := net.SplitHostPort(addrWithPort) + if err != nil { + panic("failed to split " + addrWithPort) + } + port, err := strconv.Atoi(portStr) + if err != nil { + panic("failed to atoi " + portStr) + } + + return &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Protocol: v3corepb.SocketAddress_TCP, + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(port)}, + }, + }, + } +} + +// addLocality adds a locality to the builder. +func (clab *claBuilder) addLocality(subzone string, weight uint32, priority uint32, endpoints []endpointOpts, opts *addLocalityOptions) { + var lbEndPoints []*v3endpointpb.LbEndpoint + for i, e := range endpoints { + var additionalAddrs []*v3endpointpb.Endpoint_AdditionalAddress + for _, a := range e.additionalAddrWithPorts { + additionalAddrs = append(additionalAddrs, &v3endpointpb.Endpoint_AdditionalAddress{ + Address: addressFromStr(a), + }) + } + lbe := &v3endpointpb.LbEndpoint{ + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: addressFromStr(e.addrWithPort), + AdditionalAddresses: additionalAddrs, + Hostname: e.hostname, + }, + }, + Metadata: e.metadata, + } + if opts != nil { + if i < len(opts.Health) { + lbe.HealthStatus = opts.Health[i] + } + if i < len(opts.Weight) { + lbe.LoadBalancingWeight = &wrapperspb.UInt32Value{Value: opts.Weight[i]} + } + } + lbEndPoints = append(lbEndPoints, lbe) + } + + var localityID *v3corepb.Locality + if subzone != "" { + localityID = &v3corepb.Locality{ + Region: "", + Zone: "", + SubZone: subzone, + } + } + + locality := &v3endpointpb.LocalityLbEndpoints{ + Locality: localityID, + LbEndpoints: lbEndPoints, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: weight}, + Priority: priority, + } + if opts != nil { + locality.Metadata = opts.Metadata + } + clab.v.Endpoints = append(clab.v.Endpoints, locality) +} + +// Build builds ClusterLoadAssignment. +func (clab *claBuilder) Build() *v3endpointpb.ClusterLoadAssignment { + return clab.v +} diff --git a/internal/xds/xdsclient/xdsresource/unmarshal_lds.go b/internal/xds/xdsclient/xdsresource/unmarshal_lds.go new file mode 100644 index 0000000..9994d7a --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/unmarshal_lds.go @@ -0,0 +1,618 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "errors" + "fmt" + "net/netip" + "strconv" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +func unmarshalListenerResource(r *anypb.Any, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (string, ListenerUpdate, error) { + r, err := UnwrapResource(r) + if err != nil { + return "", ListenerUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err) + } + + if !IsListenerResource(r.GetTypeUrl()) { + return "", ListenerUpdate{}, fmt.Errorf("unexpected listener resource type: %q ", r.GetTypeUrl()) + } + lis := &v3listenerpb.Listener{} + if err := proto.Unmarshal(r.GetValue(), lis); err != nil { + return "", ListenerUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + + if lis.GetName() == "" { + return "", ListenerUpdate{}, fmt.Errorf("empty resource name in listener resource") + } + + lu, err := processListener(lis, bc, sc) + if err != nil { + return lis.GetName(), ListenerUpdate{}, err + } + lu.Raw = r + + return lis.GetName(), *lu, nil +} + +func processListener(lis *v3listenerpb.Listener, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (*ListenerUpdate, error) { + if lis.GetApiListener() != nil { + return processClientSideListener(lis, bc, sc) + } + return processServerSideListener(lis, bc, sc) +} + +// processClientSideListener checks if the provided Listener proto meets +// the expected criteria. If so, it returns a non-empty routeConfigName. +func processClientSideListener(lis *v3listenerpb.Listener, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (*ListenerUpdate, error) { + apiLisAny := lis.GetApiListener().GetApiListener() + if !IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) { + return nil, fmt.Errorf("unexpected http connection manager resource type: %q", apiLisAny.GetTypeUrl()) + } + apiLis := &v3httppb.HttpConnectionManager{} + if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil { + return nil, fmt.Errorf("failed to unmarshal api_listener: %v", err) + } + // "HttpConnectionManager.xff_num_trusted_hops must be unset or zero and + // HttpConnectionManager.original_ip_detection_extensions must be empty. If + // either field has an incorrect value, the Listener must be NACKed." - A41 + if apiLis.XffNumTrustedHops != 0 { + return nil, fmt.Errorf("xff_num_trusted_hops must be unset or zero %+v", apiLis) + } + if len(apiLis.OriginalIpDetectionExtensions) != 0 { + return nil, fmt.Errorf("original_ip_detection_extensions must be empty %+v", apiLis) + } + + hcm := &HTTPConnectionManagerConfig{} + switch apiLis.RouteSpecifier.(type) { + case *v3httppb.HttpConnectionManager_Rds: + if configsource := apiLis.GetRds().GetConfigSource(); configsource.GetAds() == nil && configsource.GetSelf() == nil { + return nil, fmt.Errorf("LDS's RDS configSource is not ADS or Self: %+v", lis) + } + name := apiLis.GetRds().GetRouteConfigName() + if name == "" { + return nil, fmt.Errorf("empty route_config_name: %+v", lis) + } + hcm.RouteConfigName = name + case *v3httppb.HttpConnectionManager_RouteConfig: + routeU, err := generateRDSUpdateFromRouteConfiguration(apiLis.GetRouteConfig(), bc, sc) + if err != nil { + return nil, fmt.Errorf("failed to parse inline RDS resp: %v", err) + } + hcm.InlineRouteConfig = &routeU + case nil: + return nil, fmt.Errorf("no RouteSpecifier: %+v", apiLis) + default: + return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", apiLis.RouteSpecifier) + } + + // The following checks and fields only apply to xDS protocol versions v3+. + + hcm.MaxStreamDuration = apiLis.GetCommonHttpProtocolOptions().GetMaxStreamDuration().AsDuration() + + var err error + if hcm.HTTPFilters, err = processHTTPFilters(apiLis.GetHttpFilters(), false, bc, sc); err != nil { + return nil, err + } + + return &ListenerUpdate{APIListener: hcm}, nil +} + +func unwrapHTTPFilterConfig(config *anypb.Any) (proto.Message, string, error) { + switch { + case config.MessageIs(&v3xdsxdstypepb.TypedStruct{}): + // The real type name is inside the new TypedStruct message. + s := new(v3xdsxdstypepb.TypedStruct) + if err := config.UnmarshalTo(s); err != nil { + return nil, "", fmt.Errorf("error unmarshalling TypedStruct filter config: %v", err) + } + return s, s.GetTypeUrl(), nil + case config.MessageIs(&v1xdsudpatypepb.TypedStruct{}): + // The real type name is inside the old TypedStruct message. + s := new(v1xdsudpatypepb.TypedStruct) + if err := config.UnmarshalTo(s); err != nil { + return nil, "", fmt.Errorf("error unmarshalling TypedStruct filter config: %v", err) + } + return s, s.GetTypeUrl(), nil + default: + return config, config.GetTypeUrl(), nil + } +} + +func validateHTTPFilterConfig(cfg *anypb.Any, lds, optional bool, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (httpfilter.Builder, httpfilter.FilterConfig, error) { + config, typeURL, err := unwrapHTTPFilterConfig(cfg) + if err != nil { + return nil, nil, err + } + filterBuilder := httpfilter.Get(typeURL) + if filterBuilder == nil { + if optional { + return nil, nil, nil + } + return nil, nil, fmt.Errorf("no filter implementation found for %q", typeURL) + } + parseFunc := filterBuilder.ParseFilterConfig + if !lds { + parseFunc = filterBuilder.ParseFilterConfigOverride + } + filterConfig, err := parseFunc(config, httpfilter.ParseOptions{BootstrapConfig: bc, ServerConfig: sc}) + if err != nil { + return nil, nil, fmt.Errorf("error parsing config for filter %q: %v", typeURL, err) + } + return filterBuilder, filterConfig, nil +} + +func processHTTPFilterOverrides(cfgs map[string]*anypb.Any, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (map[string]httpfilter.FilterConfig, error) { + if len(cfgs) == 0 { + return nil, nil + } + m := make(map[string]httpfilter.FilterConfig) + for name, cfg := range cfgs { + optional := false + disabled := false + s := new(v3routepb.FilterConfig) + if cfg.MessageIs(s) { + if err := cfg.UnmarshalTo(s); err != nil { + return nil, fmt.Errorf("filter override %q: error unmarshalling FilterConfig: %v", name, err) + } + cfg = s.GetConfig() + optional = s.GetIsOptional() + if envconfig.XDSClientExtProcEnabled || envconfig.XDSClientExtAuthzEnabled { + disabled = s.GetDisabled() + } + } + + if disabled { + m[name] = httpfilter.DisabledFilterConfig{} + continue + } + + httpFilter, config, err := validateHTTPFilterConfig(cfg, false, optional, bc, sc) + if err != nil { + return nil, fmt.Errorf("filter override %q: %v", name, err) + } + if httpFilter == nil { + // Optional configs are ignored. + continue + } + m[name] = config + } + return m, nil +} + +func processHTTPFilters(filters []*v3httppb.HttpFilter, server bool, bc *bootstrap.Config, sc *bootstrap.ServerConfig) ([]HTTPFilter, error) { + ret := make([]HTTPFilter, 0, len(filters)) + seenNames := make(map[string]bool, len(filters)) + for _, filter := range filters { + name := filter.GetName() + if name == "" { + return nil, errors.New("filter missing name field") + } + if seenNames[name] { + return nil, fmt.Errorf("duplicate filter name %q", name) + } + seenNames[name] = true + + httpFilter, config, err := validateHTTPFilterConfig(filter.GetTypedConfig(), true, filter.GetIsOptional(), bc, sc) + if err != nil { + return nil, err + } + if httpFilter == nil { + // Optional configs are ignored. + continue + } + if server { + if _, ok := httpFilter.(httpfilter.ServerFilterBuilder); !ok { + if filter.GetIsOptional() { + continue + } + return nil, fmt.Errorf("HTTP filter %q not supported server-side", name) + } + } else if _, ok := httpFilter.(httpfilter.ClientFilterBuilder); !ok { + if filter.GetIsOptional() { + continue + } + return nil, fmt.Errorf("HTTP filter %q not supported client-side", name) + } + + disabled := false + if envconfig.XDSClientExtProcEnabled || envconfig.XDSClientExtAuthzEnabled { + disabled = filter.GetDisabled() + } + // Save name/config + ret = append(ret, HTTPFilter{Name: name, Filter: httpFilter, Config: config, Disabled: disabled}) + } + // "Validation will fail if a terminal filter is not the last filter in the + // chain or if a non-terminal filter is the last filter in the chain." - A39 + if len(ret) == 0 { + return nil, fmt.Errorf("http filters list is empty") + } + var i int + for ; i < len(ret)-1; i++ { + if ret[i].Filter.IsTerminal() { + return nil, fmt.Errorf("http filter %q is a terminal filter but it is not last in the filter chain", ret[i].Name) + } + } + if !ret[i].Filter.IsTerminal() { + return nil, fmt.Errorf("http filter %q is not a terminal filter", ret[len(ret)-1].Name) + } + return ret, nil +} + +func processServerSideListener(lis *v3listenerpb.Listener, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (*ListenerUpdate, error) { + if n := len(lis.ListenerFilters); n != 0 { + return nil, fmt.Errorf("unsupported field 'listener_filters' contains %d entries", n) + } + if lis.GetUseOriginalDst().GetValue() { + return nil, errors.New("unsupported field 'use_original_dst' is present and set to true") + } + addr := lis.GetAddress() + if addr == nil { + return nil, fmt.Errorf("no address field in LDS response: %+v", lis) + } + sockAddr := addr.GetSocketAddress() + if sockAddr == nil { + return nil, fmt.Errorf("no socket_address field in LDS response: %+v", lis) + } + lu := &ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: sockAddr.GetAddress(), + Port: strconv.Itoa(int(sockAddr.GetPortValue())), + }, + } + + // Populate the default filter chain. + if dfc := lis.GetDefaultFilterChain(); dfc != nil { + fc, err := filterChainFromProto(dfc, bc, sc) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal default filter chain: %v", err) + } + lu.TCPListener.DefaultFilterChain = fc + } + + // Populated the filter chain map. + fcm, err := buildFilterChainMap(lis.GetFilterChains(), bc, sc) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal filter chains: %v", err) + } + lu.TCPListener.FilterChains = fcm + + // If there are no supported filter chains and no default filter chain, we + // fail here. This will cause the Listener resource to be NACK'ed. + if len(lu.TCPListener.FilterChains.DstPrefixes) == 0 && lu.TCPListener.DefaultFilterChain.IsEmpty() { + return nil, fmt.Errorf("no supported filter chains and no default filter chain") + } + + return lu, nil +} + +func filterChainFromProto(fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (NetworkFilterChainConfig, error) { + var emptyFilterChain NetworkFilterChainConfig + + hcmConfig, err := processNetworkFilters(fc.GetFilters(), bc, sc) + if err != nil { + return emptyFilterChain, err + } + + fcc := NetworkFilterChainConfig{HTTPConnMgr: hcmConfig} + // If the transport_socket field is not specified, it means that the control + // plane has not sent us any security config. This is fine and the server + // will use the fallback credentials configured as part of the + // xdsCredentials. + ts := fc.GetTransportSocket() + if ts == nil { + return fcc, nil + } + if name := ts.GetName(); name != transportSocketName { + return emptyFilterChain, fmt.Errorf("transport_socket field has unexpected name: %s", name) + } + tc := ts.GetTypedConfig() + if typeURL := tc.GetTypeUrl(); typeURL != version.V3DownstreamTLSContextURL { + return emptyFilterChain, fmt.Errorf("transport_socket missing typed_config or wrong type_url: %q", typeURL) + } + downstreamCtx := &v3tlspb.DownstreamTlsContext{} + if err := proto.Unmarshal(tc.GetValue(), downstreamCtx); err != nil { + return emptyFilterChain, fmt.Errorf("failed to unmarshal DownstreamTlsContext in LDS response: %v", err) + } + if downstreamCtx.GetRequireSni().GetValue() { + return emptyFilterChain, fmt.Errorf("require_sni field set to true in DownstreamTlsContext message: %v", downstreamCtx) + } + if downstreamCtx.GetOcspStaplePolicy() != v3tlspb.DownstreamTlsContext_LENIENT_STAPLING { + return emptyFilterChain, fmt.Errorf("ocsp_staple_policy field set to unsupported value in DownstreamTlsContext message: %v", downstreamCtx) + } + if downstreamCtx.GetCommonTlsContext() == nil { + return emptyFilterChain, errors.New("DownstreamTlsContext in LDS response does not contain a CommonTlsContext") + } + secCfg, err := securityConfigFromCommonTLSContext(downstreamCtx.GetCommonTlsContext(), true) + if err != nil { + return emptyFilterChain, err + } + if secCfg != nil { + secCfg.RequireClientCert = downstreamCtx.GetRequireClientCertificate().GetValue() + if secCfg.RequireClientCert && secCfg.RootInstanceName == "" { + return emptyFilterChain, errors.New("security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set") + } + fcc.SecurityCfg = secCfg + } + return fcc, nil +} + +// dstPrefixEntry wraps DestinationPrefixEntry to track build state. +type dstPrefixEntry struct { + entry DestinationPrefixEntry + rawBufferSeen bool +} + +func buildFilterChainMap(fcs []*v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (NetworkFilterChainMap, error) { + dstPrefixEntries := []*dstPrefixEntry{} + for _, fc := range fcs { + fcMatch := fc.GetFilterChainMatch() + if fcMatch.GetDestinationPort().GetValue() != 0 { + // Destination port is the first match criteria and we do not + // support filter chains that contain this match criteria. + logger.Warningf("Dropping filter chain %q since it contains unsupported destination_port match field", fc.GetName()) + continue + } + + var err error + dstPrefixEntries, err = addFilterChainsForDestPrefixes(dstPrefixEntries, fc, bc, sc) + if err != nil { + return NetworkFilterChainMap{}, err + } + } + + entries := []DestinationPrefixEntry{} + for _, bEntry := range dstPrefixEntries { + fcSeen := false + for _, srcPrefixes := range bEntry.entry.SourceTypeArr { + if len(srcPrefixes.Entries) == 0 { + continue + } + for _, srcPrefix := range srcPrefixes.Entries { + for _, fc := range srcPrefix.PortMap { + if !fc.IsEmpty() { + fcSeen = true + } + } + } + } + if fcSeen { + entries = append(entries, bEntry.entry) + } + } + return NetworkFilterChainMap{DstPrefixes: entries}, nil +} + +func addFilterChainsForDestPrefixes(dstPrefixEntries []*dstPrefixEntry, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) ([]*dstPrefixEntry, error) { + ranges := fc.GetFilterChainMatch().GetPrefixRanges() + dstPrefixes, err := parsePrefixRanges(ranges) + if err != nil { + return nil, fmt.Errorf("failed to parse destination prefix ranges: %v", err) + } + + var entry *dstPrefixEntry + if len(dstPrefixes) == 0 { + // Use the unspecified entry when destination prefix is unspecified, and + // set the `prefix` field to nil. + dstPrefixEntries, entry = getOrCreateDestPrefixEntry(dstPrefixEntries, netip.Prefix{}) + if err := addFilterChainsForServerNames(entry, fc, bc, sc); err != nil { + return nil, err + } + return dstPrefixEntries, nil + } + for _, prefix := range dstPrefixes { + dstPrefixEntries, entry = getOrCreateDestPrefixEntry(dstPrefixEntries, prefix) + if err := addFilterChainsForServerNames(entry, fc, bc, sc); err != nil { + return nil, err + } + } + return dstPrefixEntries, nil +} + +// getOrCreateDestPrefixEntry looks for an existing dstPrefixEntry in the +// provided slice with the same destination prefix as the provided prefix. If +// such an entry is found, it is returned. Otherwise, a new entry is created and +// appended to the slice, and the new entry is returned. +func getOrCreateDestPrefixEntry(dstPrefixEntries []*dstPrefixEntry, prefix netip.Prefix) ([]*dstPrefixEntry, *dstPrefixEntry) { + for _, e := range dstPrefixEntries { + if e.entry.Prefix == prefix { + return dstPrefixEntries, e + } + } + entry := &dstPrefixEntry{entry: DestinationPrefixEntry{Prefix: prefix}} + dstPrefixEntries = append(dstPrefixEntries, entry) + return dstPrefixEntries, entry +} + +func addFilterChainsForServerNames(dstEntry *dstPrefixEntry, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error { + // Filter chains specifying server names in their match criteria always fail + // a match at connection time. So, these filter chains can be dropped now. + if len(fc.GetFilterChainMatch().GetServerNames()) != 0 { + logger.Warningf("Dropping filter chain %q since it contains unsupported server_names match field", fc.GetName()) + return nil + } + + return addFilterChainsForTransportProtocols(dstEntry, fc, bc, sc) +} + +func addFilterChainsForTransportProtocols(dstEntry *dstPrefixEntry, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error { + tp := fc.GetFilterChainMatch().GetTransportProtocol() + switch { + case tp != "" && tp != "raw_buffer": + // Only allow filter chains with transport protocol set to empty string + // or "raw_buffer". + logger.Warningf("Dropping filter chain %q since it contains unsupported value for transport_protocol match field", fc.GetName()) + return nil + case tp == "" && dstEntry.rawBufferSeen: + // If we have already seen filter chains with transport protocol set to + // "raw_buffer", we can drop filter chains with transport protocol set + // to empty string, since the former takes precedence. + logger.Warningf("Dropping filter chain %q since it contains empty string for transport_protocol match field, but one with raw_buffer already exists", fc.GetName()) + return nil + case tp != "" && !dstEntry.rawBufferSeen: + // This is the first "raw_buffer" that we are seeing. Set the bit and + // reset the source types array which might contain entries for filter + // chains with transport protocol set to empty string. + dstEntry.rawBufferSeen = true + dstEntry.entry.SourceTypeArr = [3]SourcePrefixes{} + } + return addFilterChainsForApplicationProtocols(dstEntry, fc, bc, sc) +} + +func addFilterChainsForApplicationProtocols(dstEntry *dstPrefixEntry, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error { + if len(fc.GetFilterChainMatch().GetApplicationProtocols()) != 0 { + logger.Warningf("Dropping filter chain %q since it contains unsupported application_protocols match field", fc.GetName()) + return nil + } + return addFilterChainsForSourceType(&dstEntry.entry, fc, bc, sc) +} + +// sourceType specifies the connection source IP match type. +type sourceType int + +const ( + // sourceTypeAny matches connection attempts from any source. + sourceTypeAny sourceType = iota + // sourceTypeSameOrLoopback matches connection attempts from the same host. + sourceTypeSameOrLoopback + // sourceTypeExternal matches connection attempts from a different host. + sourceTypeExternal +) + +func addFilterChainsForSourceType(entry *DestinationPrefixEntry, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error { + var srcType sourceType + switch st := fc.GetFilterChainMatch().GetSourceType(); st { + case v3listenerpb.FilterChainMatch_ANY: + srcType = sourceTypeAny + case v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK: + srcType = sourceTypeSameOrLoopback + case v3listenerpb.FilterChainMatch_EXTERNAL: + srcType = sourceTypeExternal + default: + return fmt.Errorf("unsupported source type: %v", st) + } + + return addFilterChainsForSourcePrefixes(&entry.SourceTypeArr[srcType], fc, bc, sc) +} + +func addFilterChainsForSourcePrefixes(srcPrefixes *SourcePrefixes, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error { + ranges := fc.GetFilterChainMatch().GetSourcePrefixRanges() + prefixes, err := parsePrefixRanges(ranges) + if err != nil { + return fmt.Errorf("failed to parse source prefix ranges: %v", err) + } + + if len(prefixes) == 0 { + return getOrCreateSourcePrefixEntry(srcPrefixes, netip.Prefix{}, fc, bc, sc) + } + for _, prefix := range prefixes { + if err := getOrCreateSourcePrefixEntry(srcPrefixes, prefix, fc, bc, sc); err != nil { + return err + } + } + return nil +} + +func parsePrefixRanges(ranges []*v3corepb.CidrRange) ([]netip.Prefix, error) { + prefixes := make([]netip.Prefix, 0, len(ranges)) + for _, pr := range ranges { + addrStr := pr.GetAddressPrefix() + bits := int(pr.GetPrefixLen().GetValue()) + + addr, err := netip.ParseAddr(addrStr) + if err != nil { + return nil, fmt.Errorf("invalid address: %q", addrStr) + } + prefix := netip.PrefixFrom(addr.Unmap(), bits).Masked() + + if !prefix.IsValid() { + return nil, fmt.Errorf(`length %d is invalid for "%s" (max %d)`, bits, addrStr, addr.BitLen()) + } + + prefixes = append(prefixes, prefix) + } + return prefixes, nil +} + +// getOrCreateSourcePrefixEntry looks for an existing SourcePrefixEntry in the +// provided SourcePrefixes with the same source prefix as the provided prefix. If +// such an entry is found, the provided filter chain is added to the entry and +// nil is returned. Otherwise, a new entry is created and appended to the +// SourcePrefixes, the provided filter chain is added to the new entry, and nil +// is returned. If there are multiple filter chains with overlapping matching +// rules, an error is returned. +func getOrCreateSourcePrefixEntry(srcPrefixes *SourcePrefixes, prefix netip.Prefix, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error { + for i := range srcPrefixes.Entries { + if srcPrefixes.Entries[i].Prefix == prefix { + return addFilterChainsForSourcePorts(&srcPrefixes.Entries[i], fc, bc, sc) + } + } + + // Not found, create a new entry. + srcPrefixes.Entries = append(srcPrefixes.Entries, SourcePrefixEntry{ + Prefix: prefix, + PortMap: make(map[int]NetworkFilterChainConfig), + }) + return addFilterChainsForSourcePorts(&srcPrefixes.Entries[len(srcPrefixes.Entries)-1], fc, bc, sc) +} + +func addFilterChainsForSourcePorts(entry *SourcePrefixEntry, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error { + ports := fc.GetFilterChainMatch().GetSourcePorts() + srcPorts := make([]int, 0, len(ports)) + for _, port := range ports { + srcPorts = append(srcPorts, int(port)) + } + + if len(srcPorts) == 0 { + if !entry.PortMap[0].IsEmpty() { + return errors.New("multiple filter chains with overlapping matching rules are defined") + } + fcc, err := filterChainFromProto(fc, bc, sc) + if err != nil { + return err + } + entry.PortMap[0] = fcc + return nil + } + for _, port := range srcPorts { + if !entry.PortMap[port].IsEmpty() { + return errors.New("multiple filter chains with overlapping matching rules are defined") + } + fcc, err := filterChainFromProto(fc, bc, sc) + if err != nil { + return err + } + entry.PortMap[port] = fcc + } + return nil +} diff --git a/internal/xds/xdsclient/xdsresource/unmarshal_lds_test.go b/internal/xds/xdsclient/xdsresource/unmarshal_lds_test.go new file mode 100644 index 0000000..a5e3999 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/unmarshal_lds_test.go @@ -0,0 +1,2066 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + "strings" + "testing" + "time" + + v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + rpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + + _ "google.golang.org/grpc/internal/xds/httpfilter/rbac" // Register the RBAC HTTP filter. + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter. +) + +func (s) TestUnmarshalListener_ClientSide(t *testing.T) { + const ( + v3LDSTarget = "lds.target.good:3333" + v3RouteConfigName = "v3RouteConfig" + routeName = "routeName" + ) + + var ( + customFilter = &v3httppb.HttpFilter{ + Name: "customFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: customFilterConfig}, + } + disabledCustomFilter = &v3httppb.HttpFilter{ + Name: "customFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: customFilterConfig}, + Disabled: true, + } + oldTypedStructFilter = &v3httppb.HttpFilter{ + Name: "customFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: testutils.MarshalAny(t, customFilterOldTypedStructConfig)}, + } + newTypedStructFilter = &v3httppb.HttpFilter{ + Name: "customFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: testutils.MarshalAny(t, customFilterNewTypedStructConfig)}, + } + customOptionalFilter = &v3httppb.HttpFilter{ + Name: "customFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: customFilterConfig}, + IsOptional: true, + } + customFilter2 = &v3httppb.HttpFilter{ + Name: "customFilter2", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: customFilterConfig}, + } + errFilter = &v3httppb.HttpFilter{ + Name: "errFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: errFilterConfig}, + } + errOptionalFilter = &v3httppb.HttpFilter{ + Name: "errFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: errFilterConfig}, + IsOptional: true, + } + clientOnlyCustomFilter = &v3httppb.HttpFilter{ + Name: "clientOnlyCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: clientOnlyCustomFilterConfig}, + } + serverOnlyCustomFilter = &v3httppb.HttpFilter{ + Name: "serverOnlyCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig}, + } + serverOnlyOptionalCustomFilter = &v3httppb.HttpFilter{ + Name: "serverOnlyOptionalCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig}, + IsOptional: true, + } + unknownFilter = &v3httppb.HttpFilter{ + Name: "unknownFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: unknownFilterConfig}, + } + unknownOptionalFilter = &v3httppb.HttpFilter{ + Name: "unknownFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: unknownFilterConfig}, + IsOptional: true, + } + v3LisWithInlineRoute = testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{v3LDSTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }}}}}}}, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{ + MaxStreamDuration: durationpb.New(time.Second), + }, + }), + }, + }) + v3LisWithFilters = func(fs ...*v3httppb.HttpFilter) *anypb.Any { + fs = append(fs, emptyRouterFilter) + return testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, + &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: v3RouteConfigName, + }, + }, + CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{ + MaxStreamDuration: durationpb.New(time.Second), + }, + HttpFilters: fs, + }), + }, + }) + } + v3LisToTestRBAC = func(xffNumTrustedHops uint32, originalIpDetectionExtensions []*v3corepb.TypedExtensionConfig) *anypb.Any { + return testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, + &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: v3RouteConfigName, + }, + }, + CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{ + MaxStreamDuration: durationpb.New(time.Second), + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + XffNumTrustedHops: xffNumTrustedHops, + OriginalIpDetectionExtensions: originalIpDetectionExtensions, + }), + }, + }) + } + + v3ListenerWithCDSConfigSourceSelf = testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, + &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{}, + }, + RouteConfigName: v3RouteConfigName, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }) + ) + + tests := []struct { + name string + resource *anypb.Any + wantName string + wantUpdate ListenerUpdate + wantErr bool + xdsClientExtProcEnabled bool + }{ + { + name: "non-listener_resource", + resource: &anypb.Any{TypeUrl: version.V3HTTPConnManagerURL}, + wantErr: true, + }, + { + name: "listener_resource_with_empty_name", + resource: &anypb.Any{ + TypeUrl: version.V3ListenerURL, + Value: func() []byte { + lis := &v3listenerpb.Listener{ + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: &anypb.Any{ + TypeUrl: version.V3HTTPConnManagerURL, + Value: []byte{1, 2, 3, 4}, + }, + }, + } + mLis, _ := proto.Marshal(lis) + return mLis + }(), + }, + wantErr: true, + }, + { + name: "badly_marshaled_listener_resource", + resource: &anypb.Any{ + TypeUrl: version.V3ListenerURL, + Value: func() []byte { + lis := &v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: &anypb.Any{ + TypeUrl: version.V3HTTPConnManagerURL, + Value: []byte{1, 2, 3, 4}, + }, + }, + } + mLis, _ := proto.Marshal(lis) + return mLis + }(), + }, + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "wrong_type_in_apiListener", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v2xdspb.Listener{}), + }, + }), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "empty_httpConnMgr_in_apiListener", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{}, + }, + }), + }, + }), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "scopedRoutes_routeConfig_in_apiListener", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{}, + }), + }, + }), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "rds.ConfigSource_in_apiListener_is_Self", + resource: v3ListenerWithCDSConfigSourceSelf, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + HTTPFilters: []HTTPFilter{makeRouterFilter(t)}, + }, + Raw: v3ListenerWithCDSConfigSourceSelf, + }, + }, + { + name: "rds.ConfigSource_in_apiListener_is_not_ADS_or_Self", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Path{ + Path: "/some/path", + }, + }, + RouteConfigName: v3RouteConfigName, + }, + }, + }), + }, + }), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3_with_no_filters", + resource: v3LisWithFilters(), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: makeRouterFilterList(t), + }, + Raw: v3LisWithFilters(), + }, + }, + { + name: "v3_no_terminal_filter", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, + &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: v3RouteConfigName, + }, + }, + CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{ + MaxStreamDuration: durationpb.New(time.Second), + }, + }), + }, + }), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3_with_custom_filter", + resource: v3LisWithFilters(customFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{ + { + Name: "customFilter", + Filter: httpFilter{}, + Config: filterConfig{Cfg: customFilterConfig}, + }, + makeRouterFilter(t), + }, + }, + Raw: v3LisWithFilters(customFilter), + }, + }, + { + name: "v3_with_custom_filter_in_old_typed_struct", + resource: v3LisWithFilters(oldTypedStructFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{ + { + Name: "customFilter", + Filter: httpFilter{}, + Config: filterConfig{Cfg: customFilterOldTypedStructConfig}, + }, + makeRouterFilter(t), + }, + }, + Raw: v3LisWithFilters(oldTypedStructFilter), + }, + }, + { + name: "v3_with_custom_filter_in_new_typed_struct", + resource: v3LisWithFilters(newTypedStructFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{ + { + Name: "customFilter", + Filter: httpFilter{}, + Config: filterConfig{Cfg: customFilterNewTypedStructConfig}, + }, + makeRouterFilter(t), + }, + }, + Raw: v3LisWithFilters(newTypedStructFilter), + }, + }, + { + name: "v3_with_optional_custom_filter", + resource: v3LisWithFilters(customOptionalFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{ + { + Name: "customFilter", + Filter: httpFilter{}, + Config: filterConfig{Cfg: customFilterConfig}, + }, + makeRouterFilter(t), + }, + }, + Raw: v3LisWithFilters(customOptionalFilter), + }, + }, + { + name: "disabled_custom_filter", + resource: v3LisWithFilters(disabledCustomFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{ + { + Name: "customFilter", + Filter: httpFilter{}, + Config: filterConfig{Cfg: customFilterConfig}, + Disabled: true, + }, + makeRouterFilter(t), + }, + }, + Raw: v3LisWithFilters(disabledCustomFilter), + }, + xdsClientExtProcEnabled: true, + }, + { + name: "v3_with_two_filters_with_same_name", + resource: v3LisWithFilters(customFilter, customFilter), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3_with_two_filters_same_type_different_name", + resource: v3LisWithFilters(customFilter, customFilter2), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{{ + Name: "customFilter", + Filter: httpFilter{}, + Config: filterConfig{Cfg: customFilterConfig}, + }, { + Name: "customFilter2", + Filter: httpFilter{}, + Config: filterConfig{Cfg: customFilterConfig}, + }, + makeRouterFilter(t), + }, + }, + Raw: v3LisWithFilters(customFilter, customFilter2), + }, + }, + { + name: "v3_with_server_only_filter", + resource: v3LisWithFilters(serverOnlyCustomFilter), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3_with_optional_server_only_filter", + resource: v3LisWithFilters(serverOnlyOptionalCustomFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: makeRouterFilterList(t), + }, + Raw: v3LisWithFilters(serverOnlyOptionalCustomFilter), + }, + }, + { + name: "v3_with_client_only_filter", + resource: v3LisWithFilters(clientOnlyCustomFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{ + { + Name: "clientOnlyCustomFilter", + Filter: clientOnlyHTTPFilter{}, + Config: filterConfig{Cfg: clientOnlyCustomFilterConfig}, + }, + makeRouterFilter(t)}, + }, + Raw: v3LisWithFilters(clientOnlyCustomFilter), + }, + }, + { + name: "v3_with_err_filter", + resource: v3LisWithFilters(errFilter), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3_with_optional_err_filter", + resource: v3LisWithFilters(errOptionalFilter), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3_with_unknown_filter", + resource: v3LisWithFilters(unknownFilter), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3_with_unknown_filter_optional", + resource: v3LisWithFilters(unknownOptionalFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: makeRouterFilterList(t), + }, + Raw: v3LisWithFilters(unknownOptionalFilter), + }, + }, + { + name: "v3_listener_resource", + resource: v3LisWithFilters(), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: makeRouterFilterList(t), + }, + Raw: v3LisWithFilters(), + }, + }, + { + name: "v3_listener_resource_wrapped", + resource: testutils.MarshalAny(t, &v3discoverypb.Resource{Resource: v3LisWithFilters()}), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: makeRouterFilterList(t), + }, + Raw: v3LisWithFilters(), + }, + }, + // "To allow equating RBAC's direct_remote_ip and + // remote_ip...HttpConnectionManager.xff_num_trusted_hops must be unset + // or zero and HttpConnectionManager.original_ip_detection_extensions + // must be empty." - A41 + { + name: "rbac-allow-equating-direct-remote-ip-and-remote-ip-valid", + resource: v3LisToTestRBAC(0, nil), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{makeRouterFilter(t)}, + }, + Raw: v3LisToTestRBAC(0, nil), + }, + }, + // In order to support xDS Configured RBAC HTTPFilter equating direct + // remote ip and remote ip, xffNumTrustedHops cannot be greater than + // zero. This is because if you can trust a ingress proxy hop when + // determining an origin clients ip address, direct remote ip != remote + // ip. + { + name: "rbac-allow-equating-direct-remote-ip-and-remote-ip-invalid-num-untrusted-hops", + resource: v3LisToTestRBAC(1, nil), + wantName: v3LDSTarget, + wantErr: true, + }, + // In order to support xDS Configured RBAC HTTPFilter equating direct + // remote ip and remote ip, originalIpDetectionExtensions must be empty. + // This is because if you have to ask ip-detection-extension for the + // original ip, direct remote ip might not equal remote ip. + { + name: "rbac-allow-equating-direct-remote-ip-and-remote-ip-invalid-original-ip-detection-extension", + resource: v3LisToTestRBAC(0, []*v3corepb.TypedExtensionConfig{{Name: "something"}}), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3_listener_with_inline_route_configuration", + resource: v3LisWithInlineRoute, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + APIListener: &HTTPConnectionManagerConfig{ + InlineRouteConfig: &RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{{ + Domains: []string{v3LDSTarget}, + Routes: []*Route{{ + Prefix: newStringP("/"), + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + ActionType: RouteActionRoute, + }}, + }}}, + MaxStreamDuration: time.Second, + HTTPFilters: makeRouterFilterList(t), + }, + Raw: v3LisWithInlineRoute, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, test.xdsClientExtProcEnabled) + + name, update, err := unmarshalListenerResource(test.resource, nil, nil) + if (err != nil) != test.wantErr { + t.Errorf("unmarshalListenerResource(%s), got err: %v, wantErr: %v", pretty.ToJSON(test.resource), err, test.wantErr) + } + if name != test.wantName { + t.Errorf("unmarshalListenerResource(%s), got name: %s, want: %s", pretty.ToJSON(test.resource), name, test.wantName) + } + if diff := cmp.Diff(update, test.wantUpdate, cmpOpts); diff != "" { + t.Errorf("unmarshalListenerResource(%s), got unexpected update, diff (-got +want): %v", pretty.ToJSON(test.resource), diff) + } + }) + } +} + +func (s) TestUnmarshalListener_ServerSide(t *testing.T) { + const ( + v3LDSTarget = "grpc/server?xds.resource.listening_address=0.0.0.0:9999" + testVersion = "test-version-lds-server" + ) + + var ( + serverOnlyCustomFilter = &v3httppb.HttpFilter{ + Name: "serverOnlyCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig}, + } + routeConfig = &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"lds.target.good:3333"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}} + inlineRouteConfig = &RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{{ + Domains: []string{"lds.target.good:3333"}, + Routes: []*Route{{Prefix: newStringP("/"), ActionType: RouteActionNonForwardingAction}}, + }}} + emptyValidNetworkFilters = []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + }), + }, + }, + } + listenerEmptyTransportSocket = testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + }, + }, + }) + listenerNoValidationContextDeprecatedFields = testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + }, + }), + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Name: "default-filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "defaultIdentityPluginInstance", + CertificateName: "defaultIdentityCertName", + }, + }, + }), + }, + }, + }, + }) + listenerNoValidationContextNewFields = testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + }, + }), + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Name: "default-filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "defaultIdentityPluginInstance", + CertificateName: "defaultIdentityCertName", + }, + }, + }), + }, + }, + }, + }) + listenerWithValidationContextDeprecatedFields = testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + }, + }, + }), + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Name: "default-filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "defaultIdentityPluginInstance", + CertificateName: "defaultIdentityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "defaultRootPluginInstance", + CertificateName: "defaultRootCertName", + }, + }, + }, + }), + }, + }, + }, + }) + listenerWithValidationContextNewFields = testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + // SystemRootCerts is ignored when + // CaCertificateProviderInstance is + // present. + SystemRootCerts: &v3tlspb.CertificateValidationContext_SystemRootCerts{}, + }, + }, + }, + }), + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Name: "default-filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "defaultIdentityPluginInstance", + CertificateName: "defaultIdentityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "defaultRootPluginInstance", + CertificateName: "defaultRootCertName", + }, + }, + }, + }, + }, + }), + }, + }, + }, + }) + ) + v3LisToTestRBAC := func(xffNumTrustedHops uint32, originalIpDetectionExtensions []*v3corepb.TypedExtensionConfig) *anypb.Any { + return testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + XffNumTrustedHops: xffNumTrustedHops, + OriginalIpDetectionExtensions: originalIpDetectionExtensions, + }), + }, + }, + }, + }, + }, + }) + } + v3LisWithBadRBACConfiguration := func(rbacCfg *v3rbacpb.RBAC) *anypb.Any { + return testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("rbac", rbacCfg), e2e.RouterHTTPFilter}, + }), + }, + }, + }, + }, + }, + }) + } + badRBACCfgRegex := &v3rbacpb.RBAC{ + Rules: &rpb.RBAC{ + Action: rpb.RBAC_ALLOW, + Policies: map[string]*rpb.Policy{ + "bad-regex-value": { + Permissions: []*rpb.Permission{ + {Rule: &rpb.Permission_Any{Any: true}}, + }, + Principals: []*rpb.Principal{ + {Identifier: &rpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{SafeRegexMatch: &v3matcherpb.RegexMatcher{Regex: "["}}}}}, + }, + }, + }, + }, + } + badRBACCfgDestIP := &v3rbacpb.RBAC{ + Rules: &rpb.RBAC{ + Action: rpb.RBAC_ALLOW, + Policies: map[string]*rpb.Policy{ + "certain-destination-ip": { + Permissions: []*rpb.Permission{ + {Rule: &rpb.Permission_DestinationIp{DestinationIp: &v3corepb.CidrRange{AddressPrefix: "not a correct address", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + Principals: []*rpb.Principal{ + {Identifier: &rpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + } + + tests := []struct { + name string + resource *anypb.Any + wantName string + wantUpdate ListenerUpdate + wantErr string + }{ + { + name: "non-empty listener filters", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + ListenerFilters: []*v3listenerpb.ListenerFilter{ + {Name: "listener-filter-1"}, + }, + }), + wantName: v3LDSTarget, + wantErr: "unsupported field 'listener_filters'", + }, + { + name: "use_original_dst is set", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + UseOriginalDst: &wrapperspb.BoolValue{Value: true}, + }), + wantName: v3LDSTarget, + wantErr: "unsupported field 'use_original_dst'", + }, + { + name: "no address field", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{Name: v3LDSTarget}), + wantName: v3LDSTarget, + wantErr: "no address field in LDS response", + }, + { + name: "no socket address field", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: &v3corepb.Address{}, + }), + wantName: v3LDSTarget, + wantErr: "no socket_address field in LDS response", + }, + { + name: "no filter chains and no default filter chain", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{DestinationPort: &wrapperspb.UInt32Value{Value: 666}}, + Filters: emptyValidNetworkFilters, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "no supported filter chains and no default filter chain", + }, + { + name: "missing http connection manager network filter", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "missing HttpConnectionManager filter", + }, + { + name: "missing filter name in http filter", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{}), + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "missing name field in filter", + }, + { + name: "duplicate filter names in http filter", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "duplicate filter name", + }, + { + name: "no terminal filter", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "http filters list is empty", + }, + { + name: "terminal filter not last", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter, serverOnlyCustomFilter}, + }), + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "is a terminal filter but it is not last in the filter chain", + }, + { + name: "last not terminal filter", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{serverOnlyCustomFilter}, + }), + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "is not a terminal filter", + }, + { + name: "unsupported oneof in typed config of http filter", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_ConfigDiscovery{}, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "unsupported config_type", + }, + { + name: "overlapping filter chain match criteria", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3, 4, 5}}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{5, 6, 7}}, + Filters: emptyValidNetworkFilters, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "multiple filter chains with overlapping matching rules are defined", + }, + { + name: "unsupported network filter", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.LocalReplyConfig{}), + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "unsupported network filter", + }, + { + name: "badly marshaled network filter", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3HTTPConnManagerURL, + Value: []byte{1, 2, 3, 4}, + }, + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "failed unmarshalling of network filter", + }, + { + name: "unexpected transport socket name", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "unsupported-transport-socket-name", + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "transport_socket field has unexpected name", + }, + { + name: "unexpected transport socket typedConfig URL", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{}), + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: fmt.Sprintf("transport_socket missing typed_config or wrong type_url: \"%s\"", testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{}).TypeUrl), + }, + { + name: "badly marshaled transport socket", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3DownstreamTLSContextURL, + Value: []byte{1, 2, 3, 4}, + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "failed to unmarshal DownstreamTlsContext in LDS response", + }, + { + name: "missing CommonTlsContext", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{}), + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "DownstreamTlsContext in LDS response does not contain a CommonTlsContext", + }, + { + name: "rbac-allow-equating-direct-remote-ip-and-remote-ip-valid", + resource: v3LisToTestRBAC(0, nil), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }}, + }}, + }}, + }, + }, + Raw: listenerEmptyTransportSocket, + }, + }, + { + name: "rbac-allow-equating-direct-remote-ip-and-remote-ip-invalid-num-untrusted-hops", + resource: v3LisToTestRBAC(1, nil), + wantName: v3LDSTarget, + wantErr: "xff_num_trusted_hops must be unset or zero", + }, + { + name: "rbac-allow-equating-direct-remote-ip-and-remote-ip-invalid-original-ip-detection-extension", + resource: v3LisToTestRBAC(0, []*v3corepb.TypedExtensionConfig{{Name: "something"}}), + wantName: v3LDSTarget, + wantErr: "original_ip_detection_extensions must be empty", + }, + { + name: "rbac-with-invalid-regex", + resource: v3LisWithBadRBACConfiguration(badRBACCfgRegex), + wantName: v3LDSTarget, + wantErr: "error parsing config for filter", + }, + { + name: "rbac-with-invalid-destination-ip-matcher", + resource: v3LisWithBadRBACConfiguration(badRBACCfgDestIP), + wantName: v3LDSTarget, + wantErr: "error parsing config for filter", + }, + { + name: "unsupported validation context in transport socket", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{ + ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{ + Name: "foo-sds-secret", + }, + }, + }, + }), + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "validation context contains unexpected type", + }, + { + name: "empty transport socket", + resource: listenerEmptyTransportSocket, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + HTTPConnMgr: &HTTPConnectionManagerConfig{InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }}, + }}, + }}, + }, + }, + Raw: listenerEmptyTransportSocket, + }, + }, + { + name: "no identity and root certificate providers using deprecated fields", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + }, + }), + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set", + }, + { + name: "no identity and root certificate providers using new fields", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + }, + }), + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set", + }, + { + name: "no identity certificate provider with require_client_cert", + resource: testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{}, + }), + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "security configuration on the server-side does not contain identity certificate provider instance name", + }, + { + name: "happy case with no validation context using deprecated fields", + resource: listenerNoValidationContextDeprecatedFields, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + SecurityCfg: &SecurityConfig{ + IdentityInstanceName: "identityPluginInstance", + IdentityCertName: "identityCertName", + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }}, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + SecurityCfg: &SecurityConfig{ + IdentityInstanceName: "defaultIdentityPluginInstance", + IdentityCertName: "defaultIdentityCertName", + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + Raw: listenerNoValidationContextDeprecatedFields, + }, + }, + { + name: "happy case with no validation context using new fields", + resource: listenerNoValidationContextNewFields, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + SecurityCfg: &SecurityConfig{ + IdentityInstanceName: "identityPluginInstance", + IdentityCertName: "identityCertName", + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + SecurityCfg: &SecurityConfig{ + IdentityInstanceName: "defaultIdentityPluginInstance", + IdentityCertName: "defaultIdentityCertName", + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + Raw: listenerNoValidationContextNewFields, + }, + }, + { + name: "happy case with validation context provider instance with deprecated fields", + resource: listenerWithValidationContextDeprecatedFields, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + SecurityCfg: &SecurityConfig{ + RootInstanceName: "rootPluginInstance", + RootCertName: "rootCertName", + IdentityInstanceName: "identityPluginInstance", + IdentityCertName: "identityCertName", + RequireClientCert: true, + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + }}, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + SecurityCfg: &SecurityConfig{ + RootInstanceName: "defaultRootPluginInstance", + RootCertName: "defaultRootCertName", + IdentityInstanceName: "defaultIdentityPluginInstance", + IdentityCertName: "defaultIdentityCertName", + RequireClientCert: true, + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + Raw: listenerWithValidationContextDeprecatedFields, + }, + }, + { + name: "happy case with validation context provider instance with new fields", + resource: listenerWithValidationContextNewFields, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + TCPListener: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: NetworkFilterChainMap{ + DstPrefixes: []DestinationPrefixEntry{{ + SourceTypeArr: [3]SourcePrefixes{{ + Entries: []SourcePrefixEntry{{ + PortMap: map[int]NetworkFilterChainConfig{ + 0: { + SecurityCfg: &SecurityConfig{ + RootInstanceName: "rootPluginInstance", + RootCertName: "rootCertName", + IdentityInstanceName: "identityPluginInstance", + IdentityCertName: "identityCertName", + RequireClientCert: true, + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }}, + }, + }}, + }}, + }, + DefaultFilterChain: NetworkFilterChainConfig{ + SecurityCfg: &SecurityConfig{ + RootInstanceName: "defaultRootPluginInstance", + RootCertName: "defaultRootCertName", + IdentityInstanceName: "defaultIdentityPluginInstance", + IdentityCertName: "defaultIdentityCertName", + RequireClientCert: true, + }, + HTTPConnMgr: &HTTPConnectionManagerConfig{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: makeRouterFilterList(t), + }, + }, + }, + Raw: listenerWithValidationContextNewFields, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + name, update, err := unmarshalListenerResource(test.resource, nil, nil) + if err != nil && !strings.Contains(err.Error(), test.wantErr) { + t.Errorf("unmarshalListenerResource(%s) = %v wantErr: %q", pretty.ToJSON(test.resource), err, test.wantErr) + } + if name != test.wantName { + t.Errorf("unmarshalListenerResource(%s), got name: %s, want: %s", pretty.ToJSON(test.resource), name, test.wantName) + } + if diff := cmp.Diff(update, test.wantUpdate, cmpOpts); diff != "" { + t.Errorf("unmarshalListenerResource(%s), got unexpected update, diff (-got +want): %v", pretty.ToJSON(test.resource), diff) + } + }) + } +} + +type filterConfig struct { + httpfilter.FilterConfig + Cfg proto.Message + Override proto.Message +} + +// httpFilter allows testing the http filter registry and parsing functionality. +type httpFilter struct { + httpfilter.ClientFilterBuilder + httpfilter.ServerFilterBuilder +} + +func (httpFilter) TypeURLs() []string { return []string{"custom.filter"} } + +func (httpFilter) ParseFilterConfig(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return filterConfig{Cfg: cfg}, nil +} + +func (httpFilter) ParseFilterConfigOverride(override proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return filterConfig{Override: override}, nil +} + +func (httpFilter) IsTerminal() bool { + return false +} + +// errHTTPFilter returns errors no matter what is passed to ParseFilterConfig. +type errHTTPFilter struct { + httpfilter.ClientFilterBuilder +} + +func (errHTTPFilter) TypeURLs() []string { return []string{"err.custom.filter"} } + +func (errHTTPFilter) ParseFilterConfig(proto.Message, httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return nil, fmt.Errorf("error from ParseFilterConfig") +} + +func (errHTTPFilter) ParseFilterConfigOverride(proto.Message, httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return nil, fmt.Errorf("error from ParseFilterConfigOverride") +} + +func (errHTTPFilter) IsTerminal() bool { + return false +} + +func init() { + httpfilter.Register(httpFilter{}) + httpfilter.Register(errHTTPFilter{}) + httpfilter.Register(serverOnlyHTTPFilter{}) + httpfilter.Register(clientOnlyHTTPFilter{}) +} + +// serverOnlyHTTPFilter does not implement ClientFilterBuilder +type serverOnlyHTTPFilter struct { + httpfilter.ServerFilterBuilder +} + +func (serverOnlyHTTPFilter) TypeURLs() []string { return []string{"serverOnly.custom.filter"} } + +func (serverOnlyHTTPFilter) ParseFilterConfig(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return filterConfig{Cfg: cfg}, nil +} + +func (serverOnlyHTTPFilter) ParseFilterConfigOverride(override proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return filterConfig{Override: override}, nil +} + +func (serverOnlyHTTPFilter) IsTerminal() bool { + return false +} + +// clientOnlyHTTPFilter does not implement ServerFilterBuilder +type clientOnlyHTTPFilter struct { + httpfilter.ClientFilterBuilder +} + +func (clientOnlyHTTPFilter) TypeURLs() []string { return []string{"clientOnly.custom.filter"} } + +func (clientOnlyHTTPFilter) ParseFilterConfig(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return filterConfig{Cfg: cfg}, nil +} + +func (clientOnlyHTTPFilter) ParseFilterConfigOverride(override proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return filterConfig{Override: override}, nil +} + +func (clientOnlyHTTPFilter) IsTerminal() bool { + return false +} + +var customFilterConfig = &anypb.Any{ + TypeUrl: "custom.filter", + Value: []byte{1, 2, 3}, +} + +var errFilterConfig = &anypb.Any{ + TypeUrl: "err.custom.filter", + Value: []byte{1, 2, 3}, +} + +var serverOnlyCustomFilterConfig = &anypb.Any{ + TypeUrl: "serverOnly.custom.filter", + Value: []byte{1, 2, 3}, +} + +var clientOnlyCustomFilterConfig = &anypb.Any{ + TypeUrl: "clientOnly.custom.filter", + Value: []byte{1, 2, 3}, +} + +// This custom filter uses the old TypedStruct message from the cncf/udpa repo. +var customFilterOldTypedStructConfig = &v1xdsudpatypepb.TypedStruct{ + TypeUrl: "custom.filter", + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "foo": {Kind: &structpb.Value_StringValue{StringValue: "bar"}}, + }, + }, +} + +// This custom filter uses the new TypedStruct message from the cncf/xds repo. +var customFilterNewTypedStructConfig = &v3xdsxdstypepb.TypedStruct{ + TypeUrl: "custom.filter", + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "foo": {Kind: &structpb.Value_StringValue{StringValue: "bar"}}, + }, + }, +} + +var unknownFilterConfig = &anypb.Any{ + TypeUrl: "unknown.custom.filter", + Value: []byte{1, 2, 3}, +} + +func wrappedOptionalFilter(t *testing.T, name string) *anypb.Any { + return testutils.MarshalAny(t, &v3routepb.FilterConfig{ + IsOptional: true, + Config: &anypb.Any{ + TypeUrl: name, + Value: []byte{1, 2, 3}, + }, + }) +} + +// parseOptsCapturingFilterBuilder is a test HTTP filter that records the +// ParseOptions passed to its parse methods. +type parseOptsCapturingFilterBuilder struct { + httpfilter.ClientFilterBuilder + httpfilter.ServerFilterBuilder + configOpts *httpfilter.ParseOptions + overrideOpts *httpfilter.ParseOptions +} + +func (*parseOptsCapturingFilterBuilder) TypeURLs() []string { + return []string{"opts.capturing.filter"} +} + +func (b *parseOptsCapturingFilterBuilder) ParseFilterConfig(_ proto.Message, opts httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + b.configOpts = &opts + return filterConfig{}, nil +} + +func (b *parseOptsCapturingFilterBuilder) ParseFilterConfigOverride(_ proto.Message, opts httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + b.overrideOpts = &opts + return filterConfig{}, nil +} + +func (*parseOptsCapturingFilterBuilder) IsTerminal() bool { return false } + +// TestHTTPFilterParseOptions verifies that the bootstrap and server +// configuration available at resource-decode time are threaded through to a +// filter's ParseFilterConfig (via LDS http_filters) and +// ParseFilterConfigOverride (via RDS typed_per_filter_config), on both the +// client and server side. +func (s) TestHTTPFilterParseOptions(t *testing.T) { + const filterName = "optsFilter" + const filterTypeURL = "opts.capturing.filter" + + fb := &parseOptsCapturingFilterBuilder{} + httpfilter.Register(fb) + defer httpfilter.UnregisterForTesting(filterTypeURL) + + wantBC := &bootstrap.Config{} + wantSC := &bootstrap.ServerConfig{} + filterCfg := &anypb.Any{TypeUrl: filterTypeURL} + + checkParseOptions := func(t *testing.T) { + t.Helper() + if fb.configOpts == nil { + t.Fatalf("ParseFilterConfig() was not called") + } + if fb.configOpts.BootstrapConfig != wantBC || fb.configOpts.ServerConfig != wantSC { + t.Errorf("ParseFilterConfig() got options %+v, want {BootstrapConfig: %p, ServerConfig: %p}", fb.configOpts, wantBC, wantSC) + } + if fb.overrideOpts == nil { + t.Fatalf("ParseFilterConfigOverride() was not called") + } + if fb.overrideOpts.BootstrapConfig != wantBC || fb.overrideOpts.ServerConfig != wantSC { + t.Errorf("ParseFilterConfigOverride() got options %+v, want {BootstrapConfig: %p, ServerConfig: %p}", fb.overrideOpts, wantBC, wantSC) + } + } + + t.Run("client_side", func(t *testing.T) { + fb.configOpts, fb.overrideOpts = nil, nil + + // A client-side listener whose http_filters reference the + // capturing filter, followed by the terminal router filter. + lis := testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: "lds.target.good:3333", + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-config-name", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + {Name: filterName, ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: filterCfg}}, + e2e.RouterHTTPFilter, + }, + }), + }, + }) + if _, _, err := unmarshalListenerResource(lis, wantBC, wantSC); err != nil { + t.Fatalf("unmarshalListenerResource() failed: %v", err) + } + + // A route configuration whose virtual host carries a + // typed_per_filter_config override for the capturing filter. + rc := &v3routepb.RouteConfiguration{ + Name: "route-config-name", + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"*"}, + TypedPerFilterConfig: map[string]*anypb.Any{filterName: filterCfg}, + }}, + } + if _, err := generateRDSUpdateFromRouteConfiguration(rc, wantBC, wantSC); err != nil { + t.Fatalf("generateRDSUpdateFromRouteConfiguration() failed: %v", err) + } + checkParseOptions(t) + }) + + t.Run("server_side", func(t *testing.T) { + fb.configOpts, fb.overrideOpts = nil, nil + + // A server-side listener with a single filter chain whose + // HttpConnectionManager references the capturing filter in its + // http_filters and carries an inline route configuration with a + // typed_per_filter_config override for the same filter. + routeConfig := &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"lds.target.good:3333"}, + TypedPerFilterConfig: map[string]*anypb.Any{filterName: filterCfg}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_NonForwardingAction{}, + }}, + }}, + } + lis := testutils.MarshalAny(t, &v3listenerpb.Listener{ + Name: "grpc/server?xds.resource.listening_address=0.0.0.0:9999", + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{{ + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{RouteConfig: routeConfig}, + HttpFilters: []*v3httppb.HttpFilter{ + {Name: filterName, ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: filterCfg}}, + e2e.RouterHTTPFilter, + }, + }), + }, + }}, + }}, + }) + if _, _, err := unmarshalListenerResource(lis, wantBC, wantSC); err != nil { + t.Fatalf("unmarshalListenerResource() failed: %v", err) + } + checkParseOptions(t) + }) +} diff --git a/internal/xds/xdsclient/xdsresource/unmarshal_rds.go b/internal/xds/xdsclient/xdsresource/unmarshal_rds.go new file mode 100644 index 0000000..67adeef --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/unmarshal_rds.go @@ -0,0 +1,464 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + "math" + "regexp" + "strings" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clusterspecifier" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +func unmarshalRouteConfigResource(r *anypb.Any, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (string, RouteConfigUpdate, error) { + r, err := UnwrapResource(r) + if err != nil { + return "", RouteConfigUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err) + } + + if !IsRouteConfigResource(r.GetTypeUrl()) { + return "", RouteConfigUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl()) + } + rc := &v3routepb.RouteConfiguration{} + if err := proto.Unmarshal(r.GetValue(), rc); err != nil { + return "", RouteConfigUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + + if rc.GetName() == "" { + return "", RouteConfigUpdate{}, fmt.Errorf("empty resource name in route config resource") + } + + u, err := generateRDSUpdateFromRouteConfiguration(rc, bc, sc) + if err != nil { + return rc.GetName(), RouteConfigUpdate{}, err + } + u.Raw = r + + return rc.GetName(), u, nil +} + +// generateRDSUpdateFromRouteConfiguration checks if the provided +// RouteConfiguration meets the expected criteria. If so, it returns a +// RouteConfigUpdate with nil error. +// +// A RouteConfiguration resource is considered valid when only if it contains a +// VirtualHost whose domain field matches the server name from the URI passed +// to the gRPC channel, and it contains a clusterName or a weighted cluster. +// +// The RouteConfiguration includes a list of virtualHosts, which may have zero +// or more elements. We are interested in the element whose domains field +// matches the server name specified in the "xds:" URI. The only field in the +// VirtualHost proto that we are interested in is the list of routes. We +// only look at the last route in the list (the default route), whose match +// field must be empty and whose route field must be set. Inside that route +// message, the cluster field will contain the clusterName or weighted clusters +// we are looking for. +func generateRDSUpdateFromRouteConfiguration(rc *v3routepb.RouteConfiguration, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (RouteConfigUpdate, error) { + vhs := make([]*VirtualHost, 0, len(rc.GetVirtualHosts())) + csps, err := processClusterSpecifierPlugins(rc.ClusterSpecifierPlugins) + if err != nil { + return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err) + } + // cspNames represents all the cluster specifiers referenced by Route + // Actions - any cluster specifiers not referenced by a Route Action can be + // ignored and not emitted by the xdsclient. + var cspNames = make(map[string]bool) + for _, vh := range rc.GetVirtualHosts() { + routes, cspNs, err := routesProtoToSlice(vh.Routes, csps, bc, sc) + if err != nil { + return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err) + } + for n := range cspNs { + cspNames[n] = true + } + rc, err := generateRetryConfig(vh.GetRetryPolicy()) + if err != nil { + return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err) + } + vhOut := &VirtualHost{ + Domains: vh.GetDomains(), + Routes: routes, + RetryConfig: rc, + } + cfgs, err := processHTTPFilterOverrides(vh.GetTypedPerFilterConfig(), bc, sc) + if err != nil { + return RouteConfigUpdate{}, fmt.Errorf("virtual host %+v: %v", vh, err) + } + vhOut.HTTPFilterConfigOverride = cfgs + vhs = append(vhs, vhOut) + } + + // "For any entry in the RouteConfiguration.cluster_specifier_plugins not + // referenced by an enclosed ActionType's cluster_specifier_plugin, the xDS + // client should not provide it to its consumers." - RLS in xDS Design + for name := range csps { + if !cspNames[name] { + delete(csps, name) + } + } + + return RouteConfigUpdate{VirtualHosts: vhs, ClusterSpecifierPlugins: csps}, nil +} + +func processClusterSpecifierPlugins(csps []*v3routepb.ClusterSpecifierPlugin) (map[string]clusterspecifier.BalancerConfig, error) { + cspCfgs := make(map[string]clusterspecifier.BalancerConfig) + // "The xDS client will inspect all elements of the + // cluster_specifier_plugins field looking up a plugin based on the + // extension.typed_config of each." - RLS in xDS design + for _, csp := range csps { + cs := clusterspecifier.Get(csp.GetExtension().GetTypedConfig().GetTypeUrl()) + if cs == nil { + if csp.GetIsOptional() { + // "If a plugin is not supported but has is_optional set, then + // we will ignore any routes that point to that plugin" + cspCfgs[csp.GetExtension().GetName()] = nil + continue + } + // "If no plugin is registered for it, the resource will be NACKed." + // - RLS in xDS design + return nil, fmt.Errorf("cluster specifier %q of type %q was not found", csp.GetExtension().GetName(), csp.GetExtension().GetTypedConfig().GetTypeUrl()) + } + lbCfg, err := cs.ParseClusterSpecifierConfig(csp.GetExtension().GetTypedConfig()) + if err != nil { + // "If a plugin is found, the value of the typed_config field will + // be passed to it's conversion method, and if an error is + // encountered, the resource will be NACKED." - RLS in xDS design + return nil, fmt.Errorf("error: %q parsing config %q for cluster specifier %q of type %q", err, csp.GetExtension().GetTypedConfig(), csp.GetExtension().GetName(), csp.GetExtension().GetTypedConfig().GetTypeUrl()) + } + // "If all cluster specifiers are valid, the xDS client will store the + // configurations in a map keyed by the name of the extension instance." - + // RLS in xDS Design + cspCfgs[csp.GetExtension().GetName()] = lbCfg + } + return cspCfgs, nil +} + +func generateRetryConfig(rp *v3routepb.RetryPolicy) (*RetryConfig, error) { + if rp == nil { + return nil, nil + } + + cfg := &RetryConfig{RetryOn: make(map[codes.Code]bool)} + for _, s := range strings.Split(rp.GetRetryOn(), ",") { + switch strings.TrimSpace(strings.ToLower(s)) { + case "cancelled": + cfg.RetryOn[codes.Canceled] = true + case "deadline-exceeded": + cfg.RetryOn[codes.DeadlineExceeded] = true + case "internal": + cfg.RetryOn[codes.Internal] = true + case "resource-exhausted": + cfg.RetryOn[codes.ResourceExhausted] = true + case "unavailable": + cfg.RetryOn[codes.Unavailable] = true + } + } + + if rp.NumRetries == nil { + cfg.NumRetries = 1 + } else { + cfg.NumRetries = rp.GetNumRetries().Value + if cfg.NumRetries < 1 { + return nil, fmt.Errorf("retry_policy.num_retries = %v; must be >= 1", cfg.NumRetries) + } + } + + backoff := rp.GetRetryBackOff() + if backoff == nil { + cfg.RetryBackoff.BaseInterval = 25 * time.Millisecond + } else { + cfg.RetryBackoff.BaseInterval = backoff.GetBaseInterval().AsDuration() + if cfg.RetryBackoff.BaseInterval <= 0 { + return nil, fmt.Errorf("retry_policy.base_interval = %v; must be > 0", cfg.RetryBackoff.BaseInterval) + } + } + if max := backoff.GetMaxInterval(); max == nil { + cfg.RetryBackoff.MaxInterval = 10 * cfg.RetryBackoff.BaseInterval + } else { + cfg.RetryBackoff.MaxInterval = max.AsDuration() + if cfg.RetryBackoff.MaxInterval <= 0 { + return nil, fmt.Errorf("retry_policy.max_interval = %v; must be > 0", cfg.RetryBackoff.MaxInterval) + } + } + + if len(cfg.RetryOn) == 0 { + return &RetryConfig{}, nil + } + return cfg, nil +} + +func routesProtoToSlice(routes []*v3routepb.Route, csps map[string]clusterspecifier.BalancerConfig, bc *bootstrap.Config, sc *bootstrap.ServerConfig) ([]*Route, map[string]bool, error) { + var routesRet []*Route + var cspNames = make(map[string]bool) + for _, r := range routes { + match := r.GetMatch() + if match == nil { + return nil, nil, fmt.Errorf("route %+v doesn't have a match", r) + } + + if len(match.GetQueryParameters()) != 0 { + // Ignore route with query parameters. + logger.Warningf("Ignoring route %+v with query parameter matchers", r) + continue + } + + pathSp := match.GetPathSpecifier() + if pathSp == nil { + return nil, nil, fmt.Errorf("route %+v doesn't have a path specifier", r) + } + + var route Route + switch pt := pathSp.(type) { + case *v3routepb.RouteMatch_Prefix: + route.Prefix = &pt.Prefix + case *v3routepb.RouteMatch_Path: + route.Path = &pt.Path + case *v3routepb.RouteMatch_SafeRegex: + regex := pt.SafeRegex.GetRegex() + re, err := matcher.CompileSafeRegex(regex) + if err != nil { + return nil, nil, fmt.Errorf("route %+v contains an invalid regex %q", r, regex) + } + route.Regex = re + default: + return nil, nil, fmt.Errorf("route %+v has an unrecognized path specifier: %+v", r, pt) + } + + if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil { + route.CaseInsensitive = !caseSensitive.Value + } + + for _, h := range match.GetHeaders() { + var header HeaderMatcher + // The deprecated exact/prefix/suffix/contains match fields are + // converted to the equivalent StringMatcher so that + // StringMatcherFromProto owns validation for all of them. + var smProto *v3matcherpb.StringMatcher + switch ht := h.GetHeaderMatchSpecifier().(type) { + case *v3routepb.HeaderMatcher_ExactMatch: + smProto = &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ht.ExactMatch}} + case *v3routepb.HeaderMatcher_SafeRegexMatch: + regex := ht.SafeRegexMatch.GetRegex() + re, err := matcher.CompileSafeRegex(regex) + if err != nil { + return nil, nil, fmt.Errorf("route %+v contains an invalid regex %q", r, regex) + } + header.RegexMatch = re + case *v3routepb.HeaderMatcher_RangeMatch: + header.RangeMatch = &Int64Range{ + Start: ht.RangeMatch.Start, + End: ht.RangeMatch.End, + } + case *v3routepb.HeaderMatcher_PresentMatch: + header.PresentMatch = &ht.PresentMatch + case *v3routepb.HeaderMatcher_PrefixMatch: + smProto = &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: ht.PrefixMatch}} + case *v3routepb.HeaderMatcher_SuffixMatch: + smProto = &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: ht.SuffixMatch}} + case *v3routepb.HeaderMatcher_ContainsMatch: + smProto = &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: ht.ContainsMatch}} + case *v3routepb.HeaderMatcher_StringMatch: + if ht.StringMatch == nil { + return nil, nil, fmt.Errorf("route %+v has an empty string matcher", r) + } + smProto = ht.StringMatch + default: + return nil, nil, fmt.Errorf("route %+v has an unrecognized header matcher: %+v", r, ht) + } + if smProto != nil { + sm, err := matcher.StringMatcherFromProto(smProto) + if err != nil { + return nil, nil, fmt.Errorf("route %+v has an invalid header matcher: %v", r, err) + } + header.StringMatch = &sm + } + // The metadata the matchers run against always has lowercase keys, + // so a name that contains an uppercase character matches no header + // and the route holding it never fires. + header.Name = strings.ToLower(h.GetName()) + invert := h.GetInvertMatch() + header.InvertMatch = &invert + route.Headers = append(route.Headers, &header) + } + + if fr := match.GetRuntimeFraction(); fr != nil { + d := fr.GetDefaultValue() + n := d.GetNumerator() + switch d.GetDenominator() { + case v3typepb.FractionalPercent_HUNDRED: + n *= 10000 + case v3typepb.FractionalPercent_TEN_THOUSAND: + n *= 100 + case v3typepb.FractionalPercent_MILLION: + } + route.Fraction = &n + } + + switch r.GetAction().(type) { + case *v3routepb.Route_Route: + action := r.GetRoute() + + if envconfig.XDSAuthorityRewrite { + if sc != nil && sc.ServerFeaturesTrustedXDSServer() { + route.AutoHostRewrite = action.GetAutoHostRewrite().GetValue() + } + } + + // Hash Policies are only applicable for a Ring Hash LB. + hp, err := hashPoliciesProtoToSlice(action.HashPolicy) + if err != nil { + return nil, nil, err + } + route.HashPolicies = hp + + switch a := action.GetClusterSpecifier().(type) { + case *v3routepb.RouteAction_Cluster: + route.WeightedClusters = append(route.WeightedClusters, WeightedCluster{Name: a.Cluster, Weight: 1}) + case *v3routepb.RouteAction_WeightedClusters: + wcs := a.WeightedClusters + var totalWeight uint64 + for _, c := range wcs.Clusters { + w := c.GetWeight().GetValue() + if w == 0 { + continue + } + totalWeight += uint64(w) + if totalWeight > math.MaxUint32 { + return nil, nil, fmt.Errorf("xds: total weight of clusters exceeds MaxUint32") + } + wc := WeightedCluster{Name: c.GetName(), Weight: w} + cfgs, err := processHTTPFilterOverrides(c.GetTypedPerFilterConfig(), bc, sc) + if err != nil { + return nil, nil, fmt.Errorf("route %+v, action %+v: %v", r, a, err) + } + wc.HTTPFilterConfigOverride = cfgs + route.WeightedClusters = append(route.WeightedClusters, wc) + } + if totalWeight == 0 { + return nil, nil, fmt.Errorf("route %+v, action %+v, has no valid cluster in WeightedCluster action", r, a) + } + case *v3routepb.RouteAction_ClusterSpecifierPlugin: + // gRFC A28 was updated to say the following: + // + // The route’s action field must be route, and its + // cluster_specifier: + // - Can be Cluster + // - Can be Weighted_clusters + // - Can be unset or an unsupported field. The route containing + // this action will be ignored. + // + // This means that if this env var is not set, we should treat + // it as if it we didn't know about the cluster_specifier_plugin + // at all. + if _, ok := csps[a.ClusterSpecifierPlugin]; !ok { + // "When processing RouteActions, if any action includes a + // cluster_specifier_plugin value that is not in + // RouteConfiguration.cluster_specifier_plugins, the + // resource will be NACKed." - RLS in xDS design + return nil, nil, fmt.Errorf("route %+v, action %+v, specifies a cluster specifier plugin %+v that is not in Route Configuration", r, a, a.ClusterSpecifierPlugin) + } + if csps[a.ClusterSpecifierPlugin] == nil { + logger.Warningf("Ignoring route %+v with optional and unsupported cluster specifier plugin %+v", r, a.ClusterSpecifierPlugin) + continue + } + cspNames[a.ClusterSpecifierPlugin] = true + route.ClusterSpecifierPlugin = a.ClusterSpecifierPlugin + default: + logger.Warningf("Ignoring route %+v with unknown ClusterSpecifier %+v", r, a) + continue + } + + msd := action.GetMaxStreamDuration() + // Prefer grpc_timeout_header_max, if set. + dur := msd.GetGrpcTimeoutHeaderMax() + if dur == nil { + dur = msd.GetMaxStreamDuration() + } + if dur != nil { + d := dur.AsDuration() + route.MaxStreamDuration = &d + } + + route.RetryConfig, err = generateRetryConfig(action.GetRetryPolicy()) + if err != nil { + return nil, nil, fmt.Errorf("route %+v, action %+v: %v", r, action, err) + } + + route.ActionType = RouteActionRoute + + case *v3routepb.Route_NonForwardingAction: + // Expected to be used on server side. + route.ActionType = RouteActionNonForwardingAction + default: + route.ActionType = RouteActionUnsupported + } + + cfgs, err := processHTTPFilterOverrides(r.GetTypedPerFilterConfig(), bc, sc) + if err != nil { + return nil, nil, fmt.Errorf("route %+v: %v", r, err) + } + route.HTTPFilterConfigOverride = cfgs + routesRet = append(routesRet, &route) + } + return routesRet, cspNames, nil +} + +func hashPoliciesProtoToSlice(policies []*v3routepb.RouteAction_HashPolicy) ([]*HashPolicy, error) { + var hashPoliciesRet []*HashPolicy + for _, p := range policies { + policy := HashPolicy{Terminal: p.Terminal} + switch p.GetPolicySpecifier().(type) { + case *v3routepb.RouteAction_HashPolicy_Header_: + policy.HashPolicyType = HashPolicyTypeHeader + policy.HeaderName = p.GetHeader().GetHeaderName() + if rr := p.GetHeader().GetRegexRewrite(); rr != nil { + regex := rr.GetPattern().GetRegex() + re, err := regexp.Compile(regex) + if err != nil { + return nil, fmt.Errorf("hash policy %+v contains an invalid regex %q", p, regex) + } + policy.Regex = re + policy.RegexSubstitution = rr.GetSubstitution() + } + case *v3routepb.RouteAction_HashPolicy_FilterState_: + if p.GetFilterState().GetKey() != "io.grpc.channel_id" { + logger.Warningf("Ignoring hash policy %+v with invalid key for filter state policy %q", p, p.GetFilterState().GetKey()) + continue + } + policy.HashPolicyType = HashPolicyTypeChannelID + default: + logger.Warningf("Ignoring unsupported hash policy %T", p.GetPolicySpecifier()) + continue + } + + hashPoliciesRet = append(hashPoliciesRet, &policy) + } + return hashPoliciesRet, nil +} diff --git a/internal/xds/xdsclient/xdsresource/unmarshal_rds_test.go b/internal/xds/xdsclient/xdsresource/unmarshal_rds_test.go new file mode 100644 index 0000000..bdb0200 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/unmarshal_rds_test.go @@ -0,0 +1,1910 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "errors" + "fmt" + "math" + "regexp" + "testing" + "time" + + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clusterspecifier" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + rpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) { + const ( + uninterestingDomain = "uninteresting.domain" + uninterestingClusterName = "uninterestingClusterName" + ldsTarget = "lds.target.good:1111" + routeName = "routeName" + clusterName = "clusterName" + ) + + var ( + goodRouteConfigWithFilterConfigs = func(cfgs map[string]*anypb.Any) *v3routepb.RouteConfiguration { + return &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }, + }}, + TypedPerFilterConfig: cfgs, + }}, + } + } + goodRouteConfigWithClusterSpecifierPlugins = func(csps []*v3routepb.ClusterSpecifierPlugin, cspReferences []string) *v3routepb.RouteConfiguration { + var rs []*v3routepb.Route + + for i, cspReference := range cspReferences { + rs = append(rs, &v3routepb.Route{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: fmt.Sprint(i + 1)}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_ClusterSpecifierPlugin{ClusterSpecifierPlugin: cspReference}, + }, + }, + }) + } + + rc := &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: rs, + }}, + ClusterSpecifierPlugins: csps, + } + + return rc + } + goodRouteConfigWithClusterSpecifierPluginsAndNormalRoute = func(csps []*v3routepb.ClusterSpecifierPlugin, cspReferences []string) *v3routepb.RouteConfiguration { + rs := goodRouteConfigWithClusterSpecifierPlugins(csps, cspReferences) + rs.VirtualHosts[0].Routes = append(rs.VirtualHosts[0].Routes, &v3routepb.Route{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + CaseSensitive: &wrapperspb.BoolValue{Value: false}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }}}) + return rs + } + goodRouteConfigWithUnsupportedClusterSpecifier = &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + CaseSensitive: &wrapperspb.BoolValue{Value: false}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }}, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "|"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_ClusterHeader{}}, + }}, + }, + }, + }, + } + + goodUpdateWithFilterConfigs = func(cfgs map[string]httpfilter.FilterConfig) RouteConfigUpdate { + return RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("/"), + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + ActionType: RouteActionRoute, + }}, + HTTPFilterConfigOverride: cfgs, + }}, + } + } + goodUpdateWithNormalRoute = RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{Prefix: newStringP("/"), + CaseInsensitive: true, + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + }, + } + goodUpdateWithClusterSpecifierPluginA = RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("1"), + ActionType: RouteActionRoute, + ClusterSpecifierPlugin: "cspA", + }}, + }}, + ClusterSpecifierPlugins: map[string]clusterspecifier.BalancerConfig{ + "cspA": nil, + }, + } + clusterSpecifierPlugin = func(name string, config *anypb.Any, isOptional bool) *v3routepb.ClusterSpecifierPlugin { + return &v3routepb.ClusterSpecifierPlugin{ + Extension: &v3corepb.TypedExtensionConfig{ + Name: name, + TypedConfig: config, + }, + IsOptional: isOptional, + } + } + goodRouteConfigWithRetryPolicy = func(vhrp *v3routepb.RetryPolicy, rrp *v3routepb.RetryPolicy) *v3routepb.RouteConfiguration { + return &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + RetryPolicy: rrp, + }, + }, + }}, + RetryPolicy: vhrp, + }}, + } + } + goodUpdateWithRetryPolicy = func(vhrc *RetryConfig, rrc *RetryConfig) RouteConfigUpdate { + return RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("/"), + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + ActionType: RouteActionRoute, + RetryConfig: rrc, + }}, + RetryConfig: vhrc, + }}, + } + } + defaultRetryBackoff = RetryBackoff{BaseInterval: 25 * time.Millisecond, MaxInterval: 250 * time.Millisecond} + ) + + tests := []struct { + name string + rc *v3routepb.RouteConfiguration + wantUpdate RouteConfigUpdate + wantError bool + xdsClientExtProcEnabled bool + }{ + { + name: "default-route-match-field-is-nil", + rc: &v3routepb.RouteConfiguration{ + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }, + }, + }, + }, + }, + }, + }, + wantError: true, + }, + { + name: "default-route-match-field-is-non-nil", + rc: &v3routepb.RouteConfiguration{ + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{}, + Action: &v3routepb.Route_Route{}, + }, + }, + }, + }, + }, + wantError: true, + }, + { + name: "default-route-routeaction-field-is-nil", + rc: &v3routepb.RouteConfiguration{ + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{}}, + }, + }, + }, + wantError: true, + }, + { + name: "default-route-cluster-field-is-empty", + rc: &v3routepb.RouteConfiguration{ + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_ClusterHeader{}, + }, + }, + }, + }, + }, + }, + }, + wantError: true, + }, + { + // default route's match sets case-sensitive to false. + name: "good-route-config-but-with-casesensitive-false", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + CaseSensitive: &wrapperspb.BoolValue{Value: false}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }}}}}}}, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{Prefix: newStringP("/"), + CaseInsensitive: true, + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + }, + }, + }, + { + name: "good-route-config-with-empty-string-route", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{uninterestingDomain}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: uninterestingClusterName}, + }, + }, + }, + }, + }, + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }, + }, + }, + }, + }, + }, + }, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{uninterestingDomain}, + Routes: []*Route{{Prefix: newStringP(""), + WeightedClusters: []WeightedCluster{{Name: uninterestingClusterName, Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + { + Domains: []string{ldsTarget}, + Routes: []*Route{{Prefix: newStringP(""), + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + }, + }, + }, + { + // default route's match is not empty string, but "/". + name: "good-route-config-with-slash-string-route", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }, + }, + }, + }, + }, + }, + }, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{Prefix: newStringP("/"), + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + }, + }, + }, + { + name: "good-route-config-with-weighted_clusters", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}}, + {Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}}, + {Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("/"), + WeightedClusters: []WeightedCluster{ + {Name: "a", Weight: 2}, + {Name: "b", Weight: 3}, + {Name: "c", Weight: 5}, + }, + ActionType: RouteActionRoute, + }}, + }, + }, + }, + }, + { + name: "good-route-config-with-max-stream-duration", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + MaxStreamDuration: &v3routepb.RouteAction_MaxStreamDuration{MaxStreamDuration: durationpb.New(time.Second)}, + }, + }, + }, + }, + }, + }, + }, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("/"), + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + MaxStreamDuration: newDurationP(time.Second), + ActionType: RouteActionRoute, + }}, + }, + }, + }, + }, + { + name: "good-route-config-with-grpc-timeout-header-max", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + MaxStreamDuration: &v3routepb.RouteAction_MaxStreamDuration{GrpcTimeoutHeaderMax: durationpb.New(time.Second)}, + }, + }, + }, + }, + }, + }, + }, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("/"), + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + MaxStreamDuration: newDurationP(time.Second), + ActionType: RouteActionRoute, + }}, + }, + }, + }, + }, + { + name: "good-route-config-with-both-timeouts", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + MaxStreamDuration: &v3routepb.RouteAction_MaxStreamDuration{MaxStreamDuration: durationpb.New(2 * time.Second), GrpcTimeoutHeaderMax: durationpb.New(0)}, + }, + }, + }, + }, + }, + }, + }, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("/"), + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + MaxStreamDuration: newDurationP(0), + ActionType: RouteActionRoute, + }}, + }, + }, + }, + }, + { + name: "good-route-config-with-http-filter-config", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": customFilterConfig}), + wantUpdate: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterConfig}}), + }, + { + name: "good-route-config-with-disabled-http-filter", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": testutils.MarshalAny(t, &v3routepb.FilterConfig{Disabled: true})}), + wantUpdate: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": httpfilter.DisabledFilterConfig{}}), + xdsClientExtProcEnabled: true, + }, + { + name: "good-route-config-with-http-filter-config-in-old-typed-struct", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": testutils.MarshalAny(t, customFilterOldTypedStructConfig)}), + wantUpdate: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterOldTypedStructConfig}}), + }, + { + name: "good-route-config-with-http-filter-config-in-new-typed-struct", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": testutils.MarshalAny(t, customFilterNewTypedStructConfig)}), + wantUpdate: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterNewTypedStructConfig}}), + }, + { + name: "good-route-config-with-optional-http-filter-config", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter(t, "custom.filter")}), + wantUpdate: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterConfig}}), + }, + { + name: "good-route-config-with-http-err-filter-config", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": errFilterConfig}), + wantError: true, + }, + { + name: "good-route-config-with-http-optional-err-filter-config", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter(t, "err.custom.filter")}), + wantError: true, + }, + { + name: "good-route-config-with-http-unknown-filter-config", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": unknownFilterConfig}), + wantError: true, + }, + { + name: "good-route-config-with-http-optional-unknown-filter-config", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter(t, "unknown.custom.filter")}), + wantUpdate: goodUpdateWithFilterConfigs(nil), + }, + { + name: "good-route-config-with-bad-rbac-http-filter-configuration", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"rbac": testutils.MarshalAny(t, &v3rbacpb.RBACPerRoute{Rbac: &v3rbacpb.RBAC{ + Rules: &rpb.RBAC{ + Action: rpb.RBAC_ALLOW, + Policies: map[string]*rpb.Policy{ + "certain-destination-ip": { + Permissions: []*rpb.Permission{ + {Rule: &rpb.Permission_DestinationIp{DestinationIp: &v3corepb.CidrRange{AddressPrefix: "not a correct address", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + Principals: []*rpb.Principal{ + {Identifier: &rpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }})}), + wantError: true, + }, + { + name: "good-route-config-with-retry-policy", + rc: goodRouteConfigWithRetryPolicy( + &v3routepb.RetryPolicy{RetryOn: "cancelled"}, + &v3routepb.RetryPolicy{RetryOn: "deadline-exceeded,unsupported", NumRetries: &wrapperspb.UInt32Value{Value: 2}}), + wantUpdate: goodUpdateWithRetryPolicy( + &RetryConfig{RetryOn: map[codes.Code]bool{codes.Canceled: true}, NumRetries: 1, RetryBackoff: defaultRetryBackoff}, + &RetryConfig{RetryOn: map[codes.Code]bool{codes.DeadlineExceeded: true}, NumRetries: 2, RetryBackoff: defaultRetryBackoff}), + }, + { + name: "good-route-config-with-retry-backoff", + rc: goodRouteConfigWithRetryPolicy( + &v3routepb.RetryPolicy{RetryOn: "internal", RetryBackOff: &v3routepb.RetryPolicy_RetryBackOff{BaseInterval: durationpb.New(10 * time.Millisecond), MaxInterval: durationpb.New(10 * time.Millisecond)}}, + &v3routepb.RetryPolicy{RetryOn: "resource-exhausted", RetryBackOff: &v3routepb.RetryPolicy_RetryBackOff{BaseInterval: durationpb.New(10 * time.Millisecond)}}), + wantUpdate: goodUpdateWithRetryPolicy( + &RetryConfig{RetryOn: map[codes.Code]bool{codes.Internal: true}, NumRetries: 1, RetryBackoff: RetryBackoff{BaseInterval: 10 * time.Millisecond, MaxInterval: 10 * time.Millisecond}}, + &RetryConfig{RetryOn: map[codes.Code]bool{codes.ResourceExhausted: true}, NumRetries: 1, RetryBackoff: RetryBackoff{BaseInterval: 10 * time.Millisecond, MaxInterval: 100 * time.Millisecond}}), + }, + { + name: "bad-retry-policy-0-retries", + rc: goodRouteConfigWithRetryPolicy(&v3routepb.RetryPolicy{RetryOn: "cancelled", NumRetries: &wrapperspb.UInt32Value{Value: 0}}, nil), + wantUpdate: RouteConfigUpdate{}, + wantError: true, + }, + { + name: "bad-retry-policy-0-base-interval", + rc: goodRouteConfigWithRetryPolicy(&v3routepb.RetryPolicy{RetryOn: "cancelled", RetryBackOff: &v3routepb.RetryPolicy_RetryBackOff{BaseInterval: durationpb.New(0)}}, nil), + wantUpdate: RouteConfigUpdate{}, + wantError: true, + }, + { + name: "bad-retry-policy-negative-max-interval", + rc: goodRouteConfigWithRetryPolicy(&v3routepb.RetryPolicy{RetryOn: "cancelled", RetryBackOff: &v3routepb.RetryPolicy_RetryBackOff{MaxInterval: durationpb.New(-time.Second)}}, nil), + wantUpdate: RouteConfigUpdate{}, + wantError: true, + }, + { + name: "bad-retry-policy-negative-max-interval-no-known-retry-on", + rc: goodRouteConfigWithRetryPolicy(&v3routepb.RetryPolicy{RetryOn: "something", RetryBackOff: &v3routepb.RetryPolicy_RetryBackOff{MaxInterval: durationpb.New(-time.Second)}}, nil), + wantUpdate: RouteConfigUpdate{}, + wantError: true, + }, + { + name: "cluster-specifier-declared-which-not-registered", + rc: goodRouteConfigWithClusterSpecifierPlugins([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", configOfClusterSpecifierDoesntExist, false), + }, []string{"cspA"}), + wantError: true, + }, + { + name: "error-in-cluster-specifier-plugin-conversion-method", + rc: goodRouteConfigWithClusterSpecifierPlugins([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", errorClusterSpecifierConfig, false), + }, []string{"cspA"}), + wantError: true, + }, + { + name: "route-action-that-references-undeclared-cluster-specifier-plugin", + rc: goodRouteConfigWithClusterSpecifierPlugins([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", mockClusterSpecifierConfig, false), + }, []string{"cspA", "cspB"}), + wantError: true, + }, + { + name: "emitted-cluster-specifier-plugins", + rc: goodRouteConfigWithClusterSpecifierPlugins([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", mockClusterSpecifierConfig, false), + }, []string{"cspA"}), + wantUpdate: goodUpdateWithClusterSpecifierPluginA, + }, + { + name: "deleted-cluster-specifier-plugins-not-referenced", + rc: goodRouteConfigWithClusterSpecifierPlugins([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", mockClusterSpecifierConfig, false), + clusterSpecifierPlugin("cspB", mockClusterSpecifierConfig, false), + }, []string{"cspA"}), + wantUpdate: goodUpdateWithClusterSpecifierPluginA, + }, + // This tests a scenario where a cluster specifier plugin is not found + // and is optional. Any routes referencing that not found optional + // cluster specifier plugin should be ignored. The config has two + // routes, and only one of them should be present in the update. + { + name: "cluster-specifier-plugin-not-found-and-optional-route-should-ignore", + rc: goodRouteConfigWithClusterSpecifierPluginsAndNormalRoute([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", configOfClusterSpecifierDoesntExist, true), + }, []string{"cspA"}), + wantUpdate: goodUpdateWithNormalRoute, + }, + // This tests a scenario where a route has an unsupported cluster + // specifier. Any routes with an unsupported cluster specifier should be + // ignored. The config has two routes, and only one of them should be + // present in the update. + { + name: "unsupported-cluster-specifier-route-should-ignore", + rc: goodRouteConfigWithUnsupportedClusterSpecifier, + wantUpdate: goodUpdateWithNormalRoute, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, test.xdsClientExtProcEnabled) + + gotUpdate, gotError := generateRDSUpdateFromRouteConfiguration(test.rc, nil, nil) + if (gotError != nil) != test.wantError || + !cmp.Equal(gotUpdate, test.wantUpdate, cmpopts.EquateEmpty(), + cmp.Transformer("FilterConfig", func(fc httpfilter.FilterConfig) string { + return fmt.Sprint(fc) + })) { + t.Errorf("generateRDSUpdateFromRouteConfiguration(%+v, %v) returned unexpected, diff (-want +got):\\n%s", test.rc, ldsTarget, cmp.Diff(test.wantUpdate, gotUpdate, cmpopts.EquateEmpty())) + } + }) + } +} + +func (s) TestGenerateRDSUpdateFromRouteConfigurationWithAutoHostRewrite(t *testing.T) { + const ( + clusterName = "clusterName" + ldsTarget = "lds.target.good:1111" + ) + + tests := []struct { + name string + isTrusted bool // Whether the xDS server is trusted. + envConfigRewrite bool // envconfig.XDSAuthorityRewrite value. + autoHostRewrite bool + wantResult bool + }{ + { + name: "envConfigOn_Trusted", + isTrusted: true, + envConfigRewrite: true, + autoHostRewrite: true, + wantResult: true, + }, + { + name: "envConfigOn_Trusted_AutoHostRewriteFalse", + isTrusted: true, + envConfigRewrite: true, + autoHostRewrite: false, + wantResult: false, + }, + { + name: "envConfigOff_Trusted", + isTrusted: true, + envConfigRewrite: false, + autoHostRewrite: true, + wantResult: false, + }, + { + name: "envConfigOn_Untrusted", + envConfigRewrite: true, + autoHostRewrite: false, + wantResult: false, + }, + { + name: "envConfigOff_Untrusted", + envConfigRewrite: false, + autoHostRewrite: true, + wantResult: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSAuthorityRewrite, test.envConfigRewrite) + + var serverFeatures []string + if test.isTrusted { + serverFeatures = []string{"trusted_xds_server"} + } + sc, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{ + URI: "trafficdirector.googleapis.com:443", + ServerFeatures: serverFeatures, + }) + if err != nil { + t.Fatalf("Failed to create server config for testing: %v", err) + } + + routeConfig := &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + HostRewriteSpecifier: &v3routepb.RouteAction_AutoHostRewrite{AutoHostRewrite: &wrapperspb.BoolValue{Value: test.autoHostRewrite}}, + }, + }, + }}, + }}, + } + + update, err := generateRDSUpdateFromRouteConfiguration(routeConfig, nil, sc) + if err != nil { + t.Errorf("generateRDSUpdateFromRouteConfiguration() failed, got : %v, want: ", err) + } + if len(update.VirtualHosts) == 0 || len(update.VirtualHosts[0].Routes) == 0 { + t.Errorf("Unexpected parsed routes from generateRDSUpdateFromRouteConfiguration(), got : 0, want: 1") + } + + if update.VirtualHosts[0].Routes[0].AutoHostRewrite != test.wantResult { + t.Errorf("AutoHostRewrite = %v, want %v", update.VirtualHosts[0].Routes[0].AutoHostRewrite, test.wantResult) + } + }) + } +} + +var configOfClusterSpecifierDoesntExist = &anypb.Any{ + TypeUrl: "does.not.exist", + Value: []byte{1, 2, 3}, +} + +var mockClusterSpecifierConfig = &anypb.Any{ + TypeUrl: "mock.cluster.specifier.plugin", + Value: []byte{1, 2, 3}, +} + +var errorClusterSpecifierConfig = &anypb.Any{ + TypeUrl: "error.cluster.specifier.plugin", + Value: []byte{1, 2, 3}, +} + +func init() { + clusterspecifier.Register(mockClusterSpecifierPlugin{}) + clusterspecifier.Register(errorClusterSpecifierPlugin{}) +} + +type mockClusterSpecifierPlugin struct { +} + +func (mockClusterSpecifierPlugin) TypeURLs() []string { + return []string{"mock.cluster.specifier.plugin"} +} + +func (mockClusterSpecifierPlugin) ParseClusterSpecifierConfig(proto.Message) (clusterspecifier.BalancerConfig, error) { + return []map[string]any{}, nil +} + +type errorClusterSpecifierPlugin struct{} + +func (errorClusterSpecifierPlugin) TypeURLs() []string { + return []string{"error.cluster.specifier.plugin"} +} + +func (errorClusterSpecifierPlugin) ParseClusterSpecifierConfig(proto.Message) (clusterspecifier.BalancerConfig, error) { + return nil, errors.New("error from cluster specifier conversion function") +} + +func (s) TestUnmarshalRouteConfig(t *testing.T) { + const ( + ldsTarget = "lds.target.good:1111" + uninterestingDomain = "uninteresting.domain" + uninterestingClusterName = "uninterestingClusterName" + v3RouteConfigName = "v3RouteConfig" + v3ClusterName = "v3Cluster" + ) + + var ( + v3VirtualHost = []*v3routepb.VirtualHost{ + { + Domains: []string{uninterestingDomain}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: uninterestingClusterName}, + }, + }, + }, + }, + }, + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: v3ClusterName}, + }, + }, + }, + }, + }, + } + v3RouteConfig = testutils.MarshalAny(t, &v3routepb.RouteConfiguration{ + Name: v3RouteConfigName, + VirtualHosts: v3VirtualHost, + }) + ) + + tests := []struct { + name string + resource *anypb.Any + wantName string + wantUpdate RouteConfigUpdate + wantErr bool + }{ + { + name: "non-routeConfig resource type", + resource: &anypb.Any{TypeUrl: version.V3HTTPConnManagerURL}, + wantErr: true, + }, + { + name: "badly marshaled routeconfig resource", + resource: &anypb.Any{ + TypeUrl: version.V3RouteConfigURL, + Value: []byte{1, 2, 3, 4}, + }, + wantErr: true, + }, + { + name: "v3 routeConfig resource with empty name", + resource: testutils.MarshalAny(t, &v3routepb.RouteConfiguration{ + Name: "", + VirtualHosts: v3VirtualHost, + }), + wantName: "", + wantErr: true, + }, + { + name: "v3 routeConfig resource", + resource: v3RouteConfig, + wantName: v3RouteConfigName, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{uninterestingDomain}, + Routes: []*Route{{Prefix: newStringP(""), + WeightedClusters: []WeightedCluster{{Name: uninterestingClusterName, Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + { + Domains: []string{ldsTarget}, + Routes: []*Route{{Prefix: newStringP(""), + WeightedClusters: []WeightedCluster{{Name: v3ClusterName, Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + }, + Raw: v3RouteConfig, + }, + }, + { + name: "v3 routeConfig resource wrapped", + resource: testutils.MarshalAny(t, &v3discoverypb.Resource{Resource: v3RouteConfig}), + wantName: v3RouteConfigName, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{uninterestingDomain}, + Routes: []*Route{{Prefix: newStringP(""), + WeightedClusters: []WeightedCluster{{Name: uninterestingClusterName, Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + { + Domains: []string{ldsTarget}, + Routes: []*Route{{Prefix: newStringP(""), + WeightedClusters: []WeightedCluster{{Name: v3ClusterName, Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + }, + Raw: v3RouteConfig, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + name, update, err := unmarshalRouteConfigResource(test.resource, nil, nil) + if (err != nil) != test.wantErr { + t.Errorf("unmarshalRouteConfigResource(%s), got err: %v, wantErr: %v", pretty.ToJSON(test.resource), err, test.wantErr) + } + if name != test.wantName { + t.Errorf("unmarshalRouteConfigResource(%s), got name: %s, want: %s", pretty.ToJSON(test.resource), name, test.wantName) + } + if diff := cmp.Diff(update, test.wantUpdate, cmpOpts); diff != "" { + t.Errorf("unmarshalRouteConfigResource(%s), got unexpected update, diff (-got +want): %v", pretty.ToJSON(test.resource), diff) + } + }) + } +} + +func (s) TestRoutesProtoToSlice(t *testing.T) { + sm, _ := matcher.StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "tv"}}) + prefixSM, _ := matcher.StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "tv"}}) + suffixSM, _ := matcher.StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "tv"}}) + containsSM, _ := matcher.StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "tv"}}) + emptyExactSM, _ := matcher.StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}}) + var ( + goodRouteWithFilterConfigs = func(cfgs map[string]*anypb.Any) []*v3routepb.Route { + // Sets per-filter config in cluster "B" and in the route. + return []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + CaseSensitive: &wrapperspb.BoolValue{Value: false}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}, TypedPerFilterConfig: cfgs}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + TypedPerFilterConfig: cfgs, + }} + } + goodUpdateWithFilterConfigs = func(cfgs map[string]httpfilter.FilterConfig) []*Route { + // Sets per-filter config in cluster "B" and in the route. + return []*Route{{ + Prefix: newStringP("/"), + CaseInsensitive: true, + WeightedClusters: []WeightedCluster{ + {Name: "B", Weight: 60, HTTPFilterConfigOverride: cfgs}, + {Name: "A", Weight: 40}, + }, + HTTPFilterConfigOverride: cfgs, + ActionType: RouteActionRoute, + }} + } + ) + + tests := []struct { + name string + routes []*v3routepb.Route + wantRoutes []*Route + wantErr bool + }{ + { + name: "no path", + routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{}, + }}, + wantErr: true, + }, + { + name: "case_sensitive is false", + routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + CaseSensitive: &wrapperspb.BoolValue{Value: false}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + }}, + wantRoutes: []*Route{{ + Prefix: newStringP("/"), + CaseInsensitive: true, + WeightedClusters: []WeightedCluster{ + {Name: "B", Weight: 60}, + {Name: "A", Weight: 40}, + }, + ActionType: RouteActionRoute, + }}, + }, + { + name: "good", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{ + { + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{ + PrefixMatch: "tv", + }, + InvertMatch: true, + }, + }, + RuntimeFraction: &v3corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 1, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + }, + }, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + Headers: []*HeaderMatcher{ + { + Name: "th", + InvertMatch: newBoolP(true), + StringMatch: &prefixSM, + }, + }, + Fraction: newUInt32P(10000), + WeightedClusters: []WeightedCluster{ + {Name: "B", Weight: 60}, + {Name: "A", Weight: 40}, + }, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "good with regex matchers", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: "/a/"}}, + Headers: []*v3routepb.HeaderMatcher{ + { + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{SafeRegexMatch: &v3matcherpb.RegexMatcher{Regex: "tv"}}, + }, + }, + RuntimeFraction: &v3corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 1, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + }, + }, + wantRoutes: []*Route{{ + Regex: func() *regexp.Regexp { return regexp.MustCompile("^(?:/a/)$") }(), + Headers: []*HeaderMatcher{ + { + Name: "th", + InvertMatch: newBoolP(false), + RegexMatch: func() *regexp.Regexp { return regexp.MustCompile("^(?:tv)$") }(), + }, + }, + Fraction: newUInt32P(10000), + WeightedClusters: []WeightedCluster{ + {Name: "B", Weight: 60}, + {Name: "A", Weight: 40}, + }, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "good with string matcher", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: "/a/"}}, + Headers: []*v3routepb.HeaderMatcher{ + { + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_StringMatch{StringMatch: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "tv"}}}, + }, + }, + RuntimeFraction: &v3corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 1, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + }, + }, + wantRoutes: []*Route{{ + Regex: func() *regexp.Regexp { return regexp.MustCompile("^(?:/a/)$") }(), + Headers: []*HeaderMatcher{ + { + Name: "th", + InvertMatch: newBoolP(false), + StringMatch: &sm, + }, + }, + Fraction: newUInt32P(10000), + WeightedClusters: []WeightedCluster{ + {Name: "B", Weight: 60}, + {Name: "A", Weight: 40}, + }, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "query is ignored", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + }, + { + Name: "with_query", + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/b/"}, + QueryParameters: []*v3routepb.QueryParameterMatcher{{Name: "route_will_be_ignored"}}, + }, + }, + }, + // Only one route in the result, because the second one with query + // parameters is ignored. + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + WeightedClusters: []WeightedCluster{ + {Name: "B", Weight: 60}, + {Name: "A", Weight: 40}, + }, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "unrecognized path specifier", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_ConnectMatcher_{}, + }, + }, + }, + wantErr: true, + }, + { + name: "bad regex in path specifier", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: "??"}}, + Headers: []*v3routepb.HeaderMatcher{ + { + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: "tv"}, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }, + }, + }, + wantErr: true, + }, + { + name: "bad regex in header specifier", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{ + { + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{SafeRegexMatch: &v3matcherpb.RegexMatcher{Regex: "??"}}, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }, + }, + }, + wantErr: true, + }, + { + name: "empty exact_match header specifier is accepted", + routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{{ + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: ""}, + }}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }, + }}, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + Headers: []*HeaderMatcher{{ + Name: "th", + InvertMatch: newBoolP(false), + StringMatch: &emptyExactSM, + }}, + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "header specifier with a name that is not lowercase", + routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{{ + Name: "X-Role", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "tv"}, + }}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }, + }}, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + Headers: []*HeaderMatcher{{ + Name: "x-role", + InvertMatch: newBoolP(false), + StringMatch: &sm, + }}, + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "suffix_match header specifier", + routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{{ + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SuffixMatch{SuffixMatch: "tv"}, + }}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }, + }}, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + Headers: []*HeaderMatcher{{ + Name: "th", + InvertMatch: newBoolP(false), + StringMatch: &suffixSM, + }}, + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "contains_match header specifier", + routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{{ + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ContainsMatch{ContainsMatch: "tv"}, + }}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }, + }}, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + Headers: []*HeaderMatcher{{ + Name: "th", + InvertMatch: newBoolP(false), + StringMatch: &containsSM, + }}, + WeightedClusters: []WeightedCluster{{Name: clusterName, Weight: 1}}, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "empty contains_match header specifier", + routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{{ + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ContainsMatch{ContainsMatch: ""}, + }}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }, + }}, + wantErr: true, + }, + { + name: "empty prefix_match header specifier", + routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{{ + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: ""}, + }}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }, + }}, + wantErr: true, + }, + { + name: "empty suffix_match header specifier", + routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{{ + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SuffixMatch{SuffixMatch: ""}, + }}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }, + }}, + wantErr: true, + }, + { + name: "nil string match header specifier", + routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{{ + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_StringMatch{}, + }}, + }, + }}, + wantErr: true, + }, + { + name: "no cluster in weighted clusters action", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{}}}}, + }, + }, + wantErr: true, + }, + { + name: "all 0-weight clusters in weighted clusters action", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 0}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 0}}, + }, + }}}}, + }, + }, + wantErr: true, + }, + { + name: "The sum of all weighted clusters is more than uint32", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: math.MaxUint32}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: math.MaxUint32}}, + }, + }}}}, + }, + }, + wantErr: true, + }, + { + name: "unsupported cluster specifier", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_ClusterSpecifierPlugin{}}}, + }, + }, + wantErr: true, + }, + { + name: "default totalWeight is 100 in weighted clusters action", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + }, + }, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + WeightedClusters: []WeightedCluster{ + {Name: "B", Weight: 60}, + {Name: "A", Weight: 40}, + }, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "default totalWeight is 100 in weighted clusters action", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 30}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 20}}, + }, + }}}}, + }, + }, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + WeightedClusters: []WeightedCluster{ + {Name: "B", Weight: 30}, + {Name: "A", Weight: 20}, + }, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "good-with-channel-id-hash-policy", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{ + { + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{ + PrefixMatch: "tv", + }, + InvertMatch: true, + }, + }, + RuntimeFraction: &v3corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 1, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}, + HashPolicy: []*v3routepb.RouteAction_HashPolicy{ + {PolicySpecifier: &v3routepb.RouteAction_HashPolicy_FilterState_{FilterState: &v3routepb.RouteAction_HashPolicy_FilterState{Key: "io.grpc.channel_id"}}}, + }, + }}, + }, + }, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + Headers: []*HeaderMatcher{ + { + Name: "th", + InvertMatch: newBoolP(true), + StringMatch: &prefixSM, + }, + }, + Fraction: newUInt32P(10000), + WeightedClusters: []WeightedCluster{ + {Name: "B", Weight: 60}, + {Name: "A", Weight: 40}, + }, + HashPolicies: []*HashPolicy{ + {HashPolicyType: HashPolicyTypeChannelID}, + }, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + // This tests that policy.Regex ends up being nil if RegexRewrite is not + // set in xds response. + { + name: "good-with-header-hash-policy-no-regex-specified", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{ + { + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{ + PrefixMatch: "tv", + }, + InvertMatch: true, + }, + }, + RuntimeFraction: &v3corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 1, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}, + HashPolicy: []*v3routepb.RouteAction_HashPolicy{ + {PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{Header: &v3routepb.RouteAction_HashPolicy_Header{HeaderName: ":path"}}}, + }, + }}, + }, + }, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + Headers: []*HeaderMatcher{ + { + Name: "th", + InvertMatch: newBoolP(true), + StringMatch: &prefixSM, + }, + }, + Fraction: newUInt32P(10000), + WeightedClusters: []WeightedCluster{ + {Name: "B", Weight: 60}, + {Name: "A", Weight: 40}, + }, + HashPolicies: []*HashPolicy{ + {HashPolicyType: HashPolicyTypeHeader, + HeaderName: ":path"}, + }, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "with custom HTTP filter config", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": customFilterConfig}), + wantRoutes: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterConfig}}), + }, + { + name: "with custom HTTP filter config in typed struct", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": testutils.MarshalAny(t, customFilterOldTypedStructConfig)}), + wantRoutes: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterOldTypedStructConfig}}), + }, + { + name: "with optional custom HTTP filter config", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter(t, "custom.filter")}), + wantRoutes: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterConfig}}), + }, + { + name: "with erroring custom HTTP filter config", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": errFilterConfig}), + wantErr: true, + }, + { + name: "with optional erroring custom HTTP filter config", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter(t, "err.custom.filter")}), + wantErr: true, + }, + { + name: "with unknown custom HTTP filter config", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": unknownFilterConfig}), + wantErr: true, + }, + { + name: "with optional unknown custom HTTP filter config", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter(t, "unknown.custom.filter")}), + wantRoutes: goodUpdateWithFilterConfigs(nil), + }, + } + + cmpOpts := []cmp.Option{ + cmp.AllowUnexported(Route{}, HeaderMatcher{}, Int64Range{}, regexp.Regexp{}), + cmpopts.EquateEmpty(), + cmp.Transformer("FilterConfig", func(fc httpfilter.FilterConfig) string { + return fmt.Sprint(fc) + }), + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, _, err := routesProtoToSlice(tt.routes, nil, nil, nil) + if (err != nil) != tt.wantErr { + t.Fatalf("routesProtoToSlice() error = %v, wantErr %v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.wantRoutes, cmpOpts...); diff != "" { + t.Fatalf("routesProtoToSlice() returned unexpected diff (-got +want):\n%s", diff) + } + }) + } +} + +func (s) TestHashPoliciesProtoToSlice(t *testing.T) { + tests := []struct { + name string + hashPolicies []*v3routepb.RouteAction_HashPolicy + wantHashPolicies []*HashPolicy + wantErr bool + }{ + // header-hash-policy tests a basic hash policy that specifies to hash a + // certain header. + { + name: "header-hash-policy", + hashPolicies: []*v3routepb.RouteAction_HashPolicy{ + { + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: ":path", + RegexRewrite: &v3matcherpb.RegexMatchAndSubstitute{ + Pattern: &v3matcherpb.RegexMatcher{Regex: "/products"}, + Substitution: "/products", + }, + }, + }, + }, + }, + wantHashPolicies: []*HashPolicy{ + { + HashPolicyType: HashPolicyTypeHeader, + HeaderName: ":path", + Regex: func() *regexp.Regexp { return regexp.MustCompile("/products") }(), + RegexSubstitution: "/products", + }, + }, + }, + // channel-id-hash-policy tests a basic hash policy that specifies to + // hash a unique identifier of the channel. + { + name: "channel-id-hash-policy", + hashPolicies: []*v3routepb.RouteAction_HashPolicy{ + {PolicySpecifier: &v3routepb.RouteAction_HashPolicy_FilterState_{FilterState: &v3routepb.RouteAction_HashPolicy_FilterState{Key: "io.grpc.channel_id"}}}, + }, + wantHashPolicies: []*HashPolicy{ + {HashPolicyType: HashPolicyTypeChannelID}, + }, + }, + // unsupported-filter-state-key tests that an unsupported key in the + // filter state hash policy are treated as a no-op. + { + name: "wrong-filter-state-key", + hashPolicies: []*v3routepb.RouteAction_HashPolicy{ + {PolicySpecifier: &v3routepb.RouteAction_HashPolicy_FilterState_{FilterState: &v3routepb.RouteAction_HashPolicy_FilterState{Key: "unsupported key"}}}, + }, + }, + // no-op-hash-policy tests that hash policies that are not supported by + // grpc are treated as a no-op. + { + name: "no-op-hash-policy", + hashPolicies: []*v3routepb.RouteAction_HashPolicy{ + {PolicySpecifier: &v3routepb.RouteAction_HashPolicy_FilterState_{}}, + }, + }, + // header-and-channel-id-hash-policy test that a list of header and + // channel id hash policies are successfully converted to an internal + // struct. + { + name: "header-and-channel-id-hash-policy", + hashPolicies: []*v3routepb.RouteAction_HashPolicy{ + { + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: ":path", + RegexRewrite: &v3matcherpb.RegexMatchAndSubstitute{ + Pattern: &v3matcherpb.RegexMatcher{Regex: "/products"}, + Substitution: "/products", + }, + }, + }, + }, + { + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_FilterState_{FilterState: &v3routepb.RouteAction_HashPolicy_FilterState{Key: "io.grpc.channel_id"}}, + Terminal: true, + }, + }, + wantHashPolicies: []*HashPolicy{ + { + HashPolicyType: HashPolicyTypeHeader, + HeaderName: ":path", + Regex: func() *regexp.Regexp { return regexp.MustCompile("/products") }(), + RegexSubstitution: "/products", + }, + { + HashPolicyType: HashPolicyTypeChannelID, + Terminal: true, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := hashPoliciesProtoToSlice(tt.hashPolicies) + if (err != nil) != tt.wantErr { + t.Fatalf("hashPoliciesProtoToSlice() error = %v, wantErr %v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.wantHashPolicies, cmp.AllowUnexported(regexp.Regexp{})); diff != "" { + t.Fatalf("hashPoliciesProtoToSlice() returned unexpected diff (-got +want):\n%s", diff) + } + }) + } +} + +func newStringP(s string) *string { + return &s +} + +func newUInt32P(i uint32) *uint32 { + return &i +} + +func newBoolP(b bool) *bool { + return &b +} + +func newDurationP(d time.Duration) *time.Duration { + return &d +} diff --git a/internal/xds/xdsclient/xdsresource/version/version.go b/internal/xds/xdsclient/xdsresource/version/version.go new file mode 100644 index 0000000..b194592 --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/version/version.go @@ -0,0 +1,44 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package version defines constants to distinguish between supported xDS API +// versions. +package version + +// Resource URLs. We need to be able to accept either version of the resource +// regardless of the version of the transport protocol in use. +const ( + googleapiPrefix = "type.googleapis.com/" + + V3ListenerType = "envoy.config.listener.v3.Listener" + V3RouteConfigType = "envoy.config.route.v3.RouteConfiguration" + V3ClusterType = "envoy.config.cluster.v3.Cluster" + V3EndpointsType = "envoy.config.endpoint.v3.ClusterLoadAssignment" + + V3ResourceWrapperURL = googleapiPrefix + "envoy.service.discovery.v3.Resource" + V3ListenerURL = googleapiPrefix + V3ListenerType + V3RouteConfigURL = googleapiPrefix + V3RouteConfigType + V3ClusterURL = googleapiPrefix + V3ClusterType + V3EndpointsURL = googleapiPrefix + V3EndpointsType + V3HTTPConnManagerURL = googleapiPrefix + "envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager" + V3UpstreamTLSContextURL = googleapiPrefix + "envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext" + V3DownstreamTLSContextURL = googleapiPrefix + "envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext" + V3HTTP11ProxyUpstreamTransportURL = googleapiPrefix + "envoy.extensions.transport_sockets.http_11_proxy.v3.Http11ProxyUpstreamTransport" + V3AddressURL = googleapiPrefix + "envoy.config.core.v3.Address" + V3AudienceURL = googleapiPrefix + "envoy.extensions.filters.http.gcp_authn.v3.Audience" +) diff --git a/internal/xds/xdsclient/xdsresource/xdsconfig.go b/internal/xds/xdsclient/xdsresource/xdsconfig.go new file mode 100644 index 0000000..35e5a1d --- /dev/null +++ b/internal/xds/xdsclient/xdsresource/xdsconfig.go @@ -0,0 +1,133 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "context" + + "google.golang.org/grpc/resolver" +) + +// XDSConfig holds the complete gRPC client-side xDS configuration containing +// all necessary resources. +type XDSConfig struct { + // Listener holds the listener configuration. It is guaranteed to be + // non-nil. + Listener *ListenerUpdate + + // RouteConfig holds the route configuration. It will be populated even if + // the route configuration was inlined into the Listener resource. It is + // guaranteed to be non-nil. + RouteConfig *RouteConfigUpdate + + // VirtualHost is selected from the route configuration whose domain field + // offers the best match against the provided dataplane authority. It is + // guaranteed to be non-nil. + VirtualHost *VirtualHost + + // Clusters is a map from cluster name to its configuration. + Clusters map[string]*ClusterResult +} + +// ClusterResult contains a cluster's configuration when a valid resource is +// received from the management server. It contains an error when: +// - an invalid resource is received from the management server and +// a valid resource was not already present or +// - the cluster resource does not exist on the management server +type ClusterResult struct { + Config ClusterConfig + Err error +} + +// ClusterConfig contains configuration for a single cluster. +type ClusterConfig struct { + // Cluster configuration for the cluster. This field is always set to a + // non-nil value. + Cluster *ClusterUpdate + // EndpointConfig contains endpoint configuration for a leaf cluster. This + // field is only set for EDS and LOGICAL_DNS clusters. + EndpointConfig *EndpointConfig + // AggregateConfig contains configuration for an aggregate cluster. This + // field is only set for AGGREGATE clusters. + AggregateConfig *AggregateConfig +} + +// AggregateConfig holds the configuration for an aggregate cluster. +type AggregateConfig struct { + // LeafClusters contains a prioritized list of names of the leaf clusters + // for the cluster. + LeafClusters []string +} + +// EndpointConfig contains configuration corresponding to the endpoints in a +// cluster. Only one of EDSUpdate or DNSEndpoints will be populated based on the +// cluster type. +type EndpointConfig struct { + // Endpoint configurartion for the EDS clusters. + EDSUpdate *EndpointsUpdate + // Endpoint configuration for the LOGICAL_DNS clusters. + DNSEndpoints *DNSUpdate + // ResolutionNote stores error encountered while obtaining endpoints data + // for the cluster. It will contain a nil value when a valid endpoint data is + // received. It contains an error when: + // - an invalid resource is received from the management server or + // - the endpoint resource does not exist on the management server + ResolutionNote error +} + +// DNSUpdate represents the result of a DNS resolution, containing a list of +// discovered endpoints. +type DNSUpdate struct { + // Endpoints is the complete list of endpoints returned by the DNS resolver. + Endpoints []resolver.Endpoint +} + +// xdsConfigkey is the type used as the key to store XDSConfig in the Attributes +// field of resolver.State. +type xdsConfigkey struct{} + +// SetXDSConfig returns a copy of state in which the Attributes field is updated +// with the XDSConfig. +func SetXDSConfig(state resolver.State, config *XDSConfig) resolver.State { + state.Attributes = state.Attributes.WithValue(xdsConfigkey{}, config) + return state +} + +// XDSConfigFromResolverState returns XDSConfig stored as an attribute in the +// resolver state. +func XDSConfigFromResolverState(state resolver.State) *XDSConfig { + if v := state.Attributes.Value(xdsConfigkey{}); v != nil { + return v.(*XDSConfig) + } + return nil +} + +type xdsConfigKey struct{} + +// NewContextWithXDSConfig returns a new context with xDSConfig added to it. +func NewContextWithXDSConfig(ctx context.Context, config *XDSConfig) context.Context { + return context.WithValue(ctx, xdsConfigKey{}, config) +} + +// XDSConfigFromContext returns the XDSConfig stored in ctx. +func XDSConfigFromContext(ctx context.Context) *XDSConfig { + if v := ctx.Value(xdsConfigKey{}); v != nil { + return v.(*XDSConfig) + } + return nil +} diff --git a/internal/xds/xdsdepmgr/xds_dependency_manager.go b/internal/xds/xdsdepmgr/xds_dependency_manager.go new file mode 100644 index 0000000..7296622 --- /dev/null +++ b/internal/xds/xdsdepmgr/xds_dependency_manager.go @@ -0,0 +1,1027 @@ +/* + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package xdsdepmgr provides the implementation of the xDS dependency manager +// that manages all the xDS watches and resources as described in gRFC A74. +package xdsdepmgr + +import ( + "context" + "fmt" + "maps" + "net/url" + "sync" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const prefix = "[xdsdepmgr %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *DependencyManager) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} + +// ConfigWatcher is the interface for consumers of aggregated xDS configuration +// from the DependencyManager. The only consumer of this configuration is +// currently the xDS resolver. +type ConfigWatcher interface { + // Update is invoked when a new, validated xDS configuration is available. + // + // Implementations must treat the received config as read-only and should + // not modify it. + Update(*xdsresource.XDSConfig) + + // Error is invoked when an error is received from the listener or route + // resource watcher. This includes cases where: + // - The listener or route resource watcher reports a resource error. + // - The received listener resource is a socket listener, not an API + // listener. TODO(i/8114): Implement this check. + // - The received route configuration does not contain a virtual host + // matching the channel's default authority. + Error(error) +} + +// xdsResourceState is a generic struct to hold the state of a watched xDS +// resource. +type xdsResourceState[T any, U any] struct { + lastUpdate *T + lastErr error + updateReceived bool + stop func() + extras U // to store any additional state specific to the watcher +} + +func (x *xdsResourceState[T, U]) setLastUpdate(update *T) { + x.lastUpdate = update + x.updateReceived = true + x.lastErr = nil +} + +func (x *xdsResourceState[T, U]) setLastError(err error) { + x.lastErr = err + x.updateReceived = true + x.lastUpdate = nil +} + +type dnsExtras struct { + dnsR resolver.Resolver +} + +type routeExtras struct { + virtualHost *xdsresource.VirtualHost +} + +// DependencyManager registers watches on the xDS client for all required xDS +// resources, resolves dependencies between them, and returns a complete +// configuration to the xDS resolver. +type DependencyManager struct { + // The following fields are initialized at creation time and are read-only + // after that. + logger *internalgrpclog.PrefixLogger + watcher ConfigWatcher + xdsClient xdsclient.XDSClient + ldsResourceName string + dataplaneAuthority string + nodeID string + + // Used to serialize callbacks from DNS resolvers to avoid deadlocks. Since + // the resolver's Build() is called with the dependency manager lock held, + // direct callbacks to ClientConn (which also require that lock) would + // deadlock. + dnsSerializer *grpcsync.CallbackSerializer + dnsSerializerCancel func() + + // All the fields below are protected by mu. + mu sync.Mutex + stopped bool + listenerWatcher *xdsResourceState[xdsresource.ListenerUpdate, struct{}] + rdsResourceName string + routeConfigWatcher *xdsResourceState[xdsresource.RouteConfigUpdate, routeExtras] + clustersFromLastRouteConfig map[string]bool + clusterWatchers map[string]*xdsResourceState[xdsresource.ClusterUpdate, struct{}] + endpointWatchers map[string]*xdsResourceState[xdsresource.EndpointsUpdate, struct{}] + dnsResolvers map[string]*xdsResourceState[xdsresource.DNSUpdate, dnsExtras] + clusterSubscriptions map[string]*clusterRef +} + +// New creates a new DependencyManager. +// +// - listenerName is the name of the Listener resource to request from the +// management server. +// - dataplaneAuthority is used to select the best matching virtual host from +// the route configuration received from the management server. +// - xdsClient is the xDS client to use to register resource watches. +// - watcher is the ConfigWatcher interface that will receive the aggregated +// XDSConfig updates and errors. +func New(listenerName, dataplaneAuthority string, xdsClient xdsclient.XDSClient, watcher ConfigWatcher) *DependencyManager { + ctx, cancel := context.WithCancel(context.Background()) + dm := &DependencyManager{ + ldsResourceName: listenerName, + dataplaneAuthority: dataplaneAuthority, + xdsClient: xdsClient, + watcher: watcher, + nodeID: xdsClient.BootstrapConfig().Node().GetId(), + dnsSerializer: grpcsync.NewCallbackSerializer(ctx), + dnsSerializerCancel: cancel, + clustersFromLastRouteConfig: make(map[string]bool), + endpointWatchers: make(map[string]*xdsResourceState[xdsresource.EndpointsUpdate, struct{}]), + dnsResolvers: make(map[string]*xdsResourceState[xdsresource.DNSUpdate, dnsExtras]), + clusterWatchers: make(map[string]*xdsResourceState[xdsresource.ClusterUpdate, struct{}]), + clusterSubscriptions: make(map[string]*clusterRef), + } + dm.logger = prefixLogger(dm) + + // The dependency manager starts by watching the listener resource and + // discovers other resources as required. For example, the listener resource + // will contain the name of the route configuration resource, which will be + // subsequently watched.œ + dm.listenerWatcher = &xdsResourceState[xdsresource.ListenerUpdate, struct{}]{} + lw := &xdsResourceWatcher[xdsresource.ListenerUpdate]{ + onUpdate: func(update *xdsresource.ListenerUpdate, onDone func()) { + dm.onListenerResourceUpdate(update, onDone) + }, + onError: func(err error, onDone func()) { + dm.onListenerResourceError(err, onDone) + }, + onAmbientError: func(err error, onDone func()) { + dm.onListenerResourceAmbientError(err, onDone) + }, + } + dm.listenerWatcher.stop = xdsresource.WatchListener(dm.xdsClient, listenerName, lw) + return dm +} + +// Close cancels all registered resource watches. +func (m *DependencyManager) Close() { + m.mu.Lock() + defer m.mu.Unlock() + + if m.stopped { + return + } + + m.stopped = true + m.listenerWatcher.stop() + if m.routeConfigWatcher != nil { + m.routeConfigWatcher.stop() + } + for name, cluster := range m.clusterWatchers { + cluster.stop() + delete(m.clusterWatchers, name) + } + + for name, endpoint := range m.endpointWatchers { + endpoint.stop() + delete(m.endpointWatchers, name) + } + + // We cannot wait for the dns serializer to finish here, as the callbacks + // try to grab the dependency manager lock, which is already held here. + m.dnsSerializerCancel() + for name, dnsResolver := range m.dnsResolvers { + dnsResolver.stop() + delete(m.dnsResolvers, name) + } + + // Drop references to externally-owned collaborators so the dependency + // manager itself doesn't retain them once Close returns. All in-flight + // callbacks re-check m.stopped under m.mu before touching these fields. + m.watcher = nil + m.xdsClient = nil +} + +// annotateErrorWithNodeID annotates the given error with the provided xDS node +// ID. +func (m *DependencyManager) annotateErrorWithNodeID(err error) error { + return fmt.Errorf("[xDS node id: %v]: %v", m.nodeID, err) +} + +// maybeSendUpdateLocked verifies that all expected resources have been +// received, and if so, delivers the complete xDS configuration to the watcher. +func (m *DependencyManager) maybeSendUpdateLocked() { + if m.listenerWatcher.lastUpdate == nil || m.routeConfigWatcher == nil || m.routeConfigWatcher.lastUpdate == nil { + return + } + config := &xdsresource.XDSConfig{ + Listener: m.listenerWatcher.lastUpdate, + RouteConfig: m.routeConfigWatcher.lastUpdate, + VirtualHost: m.routeConfigWatcher.extras.virtualHost, + Clusters: make(map[string]*xdsresource.ClusterResult), + } + + edsResourcesSeen := make(map[string]bool) + dnsResourcesSeen := make(map[string]bool) + clusterResourcesSeen := make(map[string]bool) + haveAllResources := true + + // Start watches for all clusters. Wait for all the clusters with static + // reference(from route config) to be resolved before sending the update. + for cluster := range m.clusterSubscriptions { + clusterConfig := make(map[string]*xdsresource.ClusterResult) + ok, leafClusters, err := m.populateClusterConfigLocked(cluster, 0, clusterConfig, edsResourcesSeen, dnsResourcesSeen, clusterResourcesSeen) + if !ok && m.clusterSubscriptions[cluster].staticRefCount > 0 { + haveAllResources = false + } + // If there are no leaf clusters, add that as error. + if ok && len(leafClusters) == 0 { + config.Clusters[cluster] = &xdsresource.ClusterResult{Err: m.annotateErrorWithNodeID(fmt.Errorf("aggregate cluster graph has no leaf clusters"))} + } + if err != nil { + config.Clusters[cluster] = &xdsresource.ClusterResult{Err: err} + } + // Only if all the dependencies for the cluster is resolved, add the + // clusters to the config. This is to ensure we do not send partial + // updates for dynamic clusters. + if ok { + maps.Copy(config.Clusters, clusterConfig) + } + } + + // Cancel resources not seen in the tree. + for name, ep := range m.endpointWatchers { + if _, ok := edsResourcesSeen[name]; !ok { + ep.stop() + delete(m.endpointWatchers, name) + } + } + for name, dr := range m.dnsResolvers { + if _, ok := dnsResourcesSeen[name]; !ok { + dr.stop() + delete(m.dnsResolvers, name) + } + } + for name, cluster := range m.clusterWatchers { + if _, ok := clusterResourcesSeen[name]; !ok { + cluster.stop() + delete(m.clusterWatchers, name) + } + } + if haveAllResources { + m.watcher.Update(config) + } +} + +// populateClusterConfigLocked resolves and populates the +// configuration for the given cluster and its children, including its +// associated endpoint or aggregate children. For aggregate clusters, it +// recursively resolves the configuration for its child clusters. +// +// This function traverses the cluster dependency graph (e.g., from an Aggregate +// cluster down to its leaf clusters and their endpoints/DNS resources) to +// ensure all necessary xDS resources are watched and fully resolved before +// configuration is considered ready. +// +// Parameters: +// +// clusterName: The name of the cluster resource to resolve. +// depth: The current recursion depth. +// clusterConfigs: Map to store the resolved cluster configuration. +// endpointResourcesSeen: Stores which EDS resource names have been encountered. +// dnsResourcesSeen: Stores which DNS resource names have been encountered. +// clustersSeen: Stores which cluster resource names have been encountered. +// +// Returns: +// +// bool: Returns true if the cluster configuration (and all its +// dependencies) is fully resolved (i.e either update or +// error has been received). +// []string: A slice of all "leaf" cluster names discovered in the +// traversal starting from `clusterName`. For +// non-aggregate clusters, this will contain only `clusterName`. +// error: Error that needs to be propagated up the tree (like +// max depth exceeded or an error propagated from a +// child cluster). +func (m *DependencyManager) populateClusterConfigLocked(clusterName string, depth int, clusterConfigs map[string]*xdsresource.ClusterResult, endpointResourcesSeen, dnsResourcesSeen, clustersSeen map[string]bool) (bool, []string, error) { + const aggregateClusterMaxDepth = 16 + clustersSeen[clusterName] = true + + if depth >= aggregateClusterMaxDepth { + err := m.annotateErrorWithNodeID(fmt.Errorf("aggregate cluster graph exceeds max depth (%d)", aggregateClusterMaxDepth)) + clusterConfigs[clusterName] = &xdsresource.ClusterResult{Err: err} + return true, nil, err + } + + // If cluster is already seen in the tree, return. + if _, ok := clusterConfigs[clusterName]; ok { + return true, nil, nil + } + + // If cluster watcher does not exist, create one. + state, ok := m.clusterWatchers[clusterName] + if !ok { + m.clusterWatchers[clusterName] = newClusterWatcher(clusterName, m) + return false, nil, nil + } + + // If a watch exists but no update received yet, return. + if !state.updateReceived { + return false, nil, nil + } + + // If there was a resource error, propagate it up. + if state.lastErr != nil { + return true, nil, state.lastErr + } + + clusterConfigs[clusterName] = &xdsresource.ClusterResult{ + Config: xdsresource.ClusterConfig{ + Cluster: state.lastUpdate, + }, + } + update := state.lastUpdate + + switch update.ClusterType { + case xdsresource.ClusterTypeEDS: + return m.populateEDSClusterLocked(clusterName, update, clusterConfigs, endpointResourcesSeen) + case xdsresource.ClusterTypeLogicalDNS: + return m.populateLogicalDNSClusterLocked(clusterName, update, clusterConfigs, dnsResourcesSeen) + case xdsresource.ClusterTypeAggregate: + return m.populateAggregateClusterLocked(clusterName, update, depth, clusterConfigs, endpointResourcesSeen, dnsResourcesSeen, clustersSeen) + default: + clusterConfigs[clusterName] = &xdsresource.ClusterResult{Err: m.annotateErrorWithNodeID(fmt.Errorf("cluster type %v of cluster %s not supported", update.ClusterType, clusterName))} + return true, nil, nil + } +} + +func (m *DependencyManager) populateEDSClusterLocked(clusterName string, update *xdsresource.ClusterUpdate, clusterConfigs map[string]*xdsresource.ClusterResult, endpointResourcesSeen map[string]bool) (bool, []string, error) { + edsName := clusterName + if update.EDSServiceName != "" { + edsName = update.EDSServiceName + } + endpointResourcesSeen[edsName] = true + + // If endpoint watcher does not exist, create one. + if _, ok := m.endpointWatchers[edsName]; !ok { + m.endpointWatchers[edsName] = newEndpointWatcher(edsName, m) + return false, nil, nil + } + endpointState := m.endpointWatchers[edsName] + + // If the resource does not have any update yet, return. + if !endpointState.updateReceived { + return false, nil, nil + } + + // Store the update and error. + clusterConfigs[clusterName].Config.EndpointConfig = &xdsresource.EndpointConfig{ + EDSUpdate: endpointState.lastUpdate, + ResolutionNote: endpointState.lastErr, + } + return true, []string{clusterName}, nil +} + +func (m *DependencyManager) populateLogicalDNSClusterLocked(clusterName string, update *xdsresource.ClusterUpdate, clusterConfigs map[string]*xdsresource.ClusterResult, dnsResourcesSeen map[string]bool) (bool, []string, error) { + target := update.DNSHostName + dnsResourcesSeen[target] = true + + // If dns resolver does not exist, create one. + if _, ok := m.dnsResolvers[target]; !ok { + state := m.newDNSResolver(target) + if state == nil { + return false, nil, nil + } + m.dnsResolvers[target] = state + return false, nil, nil + } + dnsState := m.dnsResolvers[target] + + // If no update received, return false. + if !dnsState.updateReceived { + return false, nil, nil + } + + clusterConfigs[clusterName].Config.EndpointConfig = &xdsresource.EndpointConfig{ + DNSEndpoints: dnsState.lastUpdate, + ResolutionNote: dnsState.lastErr, + } + return true, []string{clusterName}, nil +} + +func (m *DependencyManager) populateAggregateClusterLocked(clusterName string, update *xdsresource.ClusterUpdate, depth int, clusterConfigs map[string]*xdsresource.ClusterResult, endpointResourcesSeen, dnsResourcesSeen, clustersSeen map[string]bool) (bool, []string, error) { + var leafClusters []string + haveAllResources := true + for _, child := range update.PrioritizedClusterNames { + ok, childLeafClusters, err := m.populateClusterConfigLocked(child, depth+1, clusterConfigs, endpointResourcesSeen, dnsResourcesSeen, clustersSeen) + if !ok { + haveAllResources = false + } + if err != nil { + clusterConfigs[clusterName] = &xdsresource.ClusterResult{Err: err} + return true, leafClusters, err + } + leafClusters = append(leafClusters, childLeafClusters...) + } + if !haveAllResources { + return false, leafClusters, nil + } + if haveAllResources && len(leafClusters) == 0 { + clusterConfigs[clusterName] = &xdsresource.ClusterResult{Err: m.annotateErrorWithNodeID(fmt.Errorf("aggregate cluster graph has no leaf clusters"))} + return true, leafClusters, nil + } + clusterConfigs[clusterName].Config.AggregateConfig = &xdsresource.AggregateConfig{ + LeafClusters: leafClusters, + } + return true, leafClusters, nil +} + +func (m *DependencyManager) applyRouteConfigUpdateLocked(update *xdsresource.RouteConfigUpdate) { + matchVH := xdsresource.FindBestMatchingVirtualHost(m.dataplaneAuthority, update.VirtualHosts) + if matchVH == nil { + err := m.annotateErrorWithNodeID(fmt.Errorf("could not find VirtualHost for %q", m.dataplaneAuthority)) + m.routeConfigWatcher.setLastError(err) + m.watcher.Error(err) + return + } + m.routeConfigWatcher.setLastUpdate(update) + m.routeConfigWatcher.extras.virtualHost = matchVH + + // Get the clusters to be watched from the routes in the virtual host. + // If the ClusterSpecifierPlugin field is set, we ignore it for now as the + // clusters will be determined dynamically for it. + newClusters := make(map[string]bool) + for _, rt := range matchVH.Routes { + for _, cluster := range rt.WeightedClusters { + newClusters[cluster.Name] = true + } + } + + // Add subscriptions for all new clusters. + for cluster := range newClusters { + // If the cluster already has a reference, increase its static + // reference. + if sub, ok := m.clusterSubscriptions[cluster]; ok { + sub.staticRefCount++ + continue + } + // If cluster is not present in subscriptions, add it with static + // ref count as 1. + m.clusterSubscriptions[cluster] = &clusterRef{ + staticRefCount: 1, + } + } + + // Unsubscribe to clusters from last route config. + for cluster := range m.clustersFromLastRouteConfig { + clusterRef, ok := m.clusterSubscriptions[cluster] + if !ok { + // Should not reach here as the cluster was present in last + // route config so should be present in current cluster + // subscriptions. + continue + } + clusterRef.staticRefCount-- + if clusterRef.staticRefCount == 0 && clusterRef.dynamicRefCount == 0 { + delete(m.clusterSubscriptions, cluster) + } + } + m.clustersFromLastRouteConfig = newClusters + + // maybeSendUpdate is called to update the configuration with the new route, + // start watching the newly added clusters and stop watching clusters that + // are not needed anymore. + m.maybeSendUpdateLocked() +} + +func (m *DependencyManager) onListenerResourceUpdate(update *xdsresource.ListenerUpdate, onDone func()) { + // A client-side listener update must contain API listener configuration. If + // it is nil, it indicates that a server-side listener (TCP Listener) resource + // was received instead. We report this error to the resolver watcher so it + // can transition the channel into TRANSIENT_FAILURE. + if update.APIListener == nil { + err := fmt.Errorf("client-side listener resource %q does not contain API listener configuration", m.ldsResourceName) + m.onListenerResourceError(err, onDone) + return + } + + m.mu.Lock() + defer m.mu.Unlock() + + defer onDone() + if m.stopped { + return + } + + if m.logger.V(2) { + m.logger.Infof("Received update for Listener resource %q: %+v", m.ldsResourceName, update) + } + + m.listenerWatcher.setLastUpdate(update) + + if update.APIListener.InlineRouteConfig != nil { + // If there was a previous route config watcher because of a non-inline + // route configuration, cancel it. + m.rdsResourceName = "" + if m.routeConfigWatcher != nil { + m.routeConfigWatcher.stop() + } + m.routeConfigWatcher = &xdsResourceState[xdsresource.RouteConfigUpdate, routeExtras]{stop: func() {}} + m.applyRouteConfigUpdateLocked(update.APIListener.InlineRouteConfig) + return + } + + // We get here only if there was no inline route configuration. If the route + // config name has not changed, send an update with existing route + // configuration and the newly received listener configuration. + if m.rdsResourceName == update.APIListener.RouteConfigName { + m.maybeSendUpdateLocked() + return + } + + // If the route config name has changed, cancel the old watcher and start a + // new one. At this point, since the new route config name has not yet been + // resolved, no update is sent to the channel, and therefore the old route + // configuration (if received) is used until the new one is received. + m.rdsResourceName = update.APIListener.RouteConfigName + if m.routeConfigWatcher != nil { + m.routeConfigWatcher.stop() + } + rw := &xdsResourceWatcher[xdsresource.RouteConfigUpdate]{ + onUpdate: func(update *xdsresource.RouteConfigUpdate, onDone func()) { + m.onRouteConfigResourceUpdate(m.rdsResourceName, update, onDone) + }, + onError: func(err error, onDone func()) { + m.onRouteConfigResourceError(m.rdsResourceName, err, onDone) + }, + onAmbientError: func(err error, onDone func()) { + m.onRouteConfigResourceAmbientError(m.rdsResourceName, err, onDone) + }, + } + if m.routeConfigWatcher != nil { + m.routeConfigWatcher.stop = xdsresource.WatchRouteConfig(m.xdsClient, m.rdsResourceName, rw) + } else { + m.routeConfigWatcher = &xdsResourceState[xdsresource.RouteConfigUpdate, routeExtras]{ + stop: xdsresource.WatchRouteConfig(m.xdsClient, m.rdsResourceName, rw), + } + } +} + +func (m *DependencyManager) onListenerResourceError(err error, onDone func()) { + m.mu.Lock() + defer m.mu.Unlock() + + defer onDone() + if m.stopped { + return + } + + m.logger.Warningf("Received resource error for Listener resource %q: %v", m.ldsResourceName, m.annotateErrorWithNodeID(err)) + + if m.routeConfigWatcher != nil { + m.routeConfigWatcher.stop() + } + m.listenerWatcher.setLastError(err) + m.rdsResourceName = "" + m.routeConfigWatcher = nil + m.watcher.Error(fmt.Errorf("listener resource error: %v", m.annotateErrorWithNodeID(err))) +} + +// onListenerResourceAmbientError handles ambient errors received from the +// listener resource watcher. Since ambient errors do not impact the current +// state of the resource, no change is made to the current configuration and the +// errors are only logged for visibility. +func (m *DependencyManager) onListenerResourceAmbientError(err error, onDone func()) { + m.mu.Lock() + defer m.mu.Unlock() + + defer onDone() + if m.stopped { + return + } + + m.logger.Warningf("Listener resource ambient error: %v", m.annotateErrorWithNodeID(err)) +} + +func (m *DependencyManager) onRouteConfigResourceUpdate(resourceName string, update *xdsresource.RouteConfigUpdate, onDone func()) { + m.mu.Lock() + defer m.mu.Unlock() + + defer onDone() + if m.stopped || m.rdsResourceName != resourceName { + return + } + + if m.logger.V(2) { + m.logger.Infof("Received update for RouteConfiguration resource %q: %+v", resourceName, update) + } + m.applyRouteConfigUpdateLocked(update) +} + +func (m *DependencyManager) onRouteConfigResourceError(resourceName string, err error, onDone func()) { + m.mu.Lock() + defer m.mu.Unlock() + + defer onDone() + if m.stopped || m.rdsResourceName != resourceName { + return + } + m.routeConfigWatcher.setLastError(err) + m.logger.Warningf("Received resource error for RouteConfiguration resource %q: %v", resourceName, m.annotateErrorWithNodeID(err)) + m.watcher.Error(fmt.Errorf("route resource error: %v", m.annotateErrorWithNodeID(err))) +} + +// onRouteResourceAmbientError handles ambient errors received from the route +// resource watcher. Since ambient errors do not impact the current state of the +// resource, no change is made to the current configuration and the errors are +// only logged for visibility. +func (m *DependencyManager) onRouteConfigResourceAmbientError(resourceName string, err error, onDone func()) { + m.mu.Lock() + defer m.mu.Unlock() + + defer onDone() + if m.stopped || m.rdsResourceName != resourceName { + return + } + + m.logger.Warningf("Route resource ambient error %q: %v", resourceName, m.annotateErrorWithNodeID(err)) +} + +func newClusterWatcher(resourceName string, depMgr *DependencyManager) *xdsResourceState[xdsresource.ClusterUpdate, struct{}] { + w := &xdsResourceWatcher[xdsresource.ClusterUpdate]{ + onUpdate: func(u *xdsresource.ClusterUpdate, onDone func()) { + depMgr.onClusterResourceUpdate(resourceName, u, onDone) + }, + onError: func(err error, onDone func()) { + depMgr.onClusterResourceError(resourceName, err, onDone) + }, + onAmbientError: func(err error, onDone func()) { + depMgr.onClusterAmbientError(resourceName, err, onDone) + }, + } + return &xdsResourceState[xdsresource.ClusterUpdate, struct{}]{ + stop: xdsresource.WatchCluster(depMgr.xdsClient, resourceName, w), + } +} + +// Records a successful Cluster resource update, clears any previous error. +func (m *DependencyManager) onClusterResourceUpdate(resourceName string, update *xdsresource.ClusterUpdate, onDone func()) { + m.mu.Lock() + defer m.mu.Unlock() + + defer onDone() + if m.stopped || m.clusterWatchers[resourceName] == nil { + return + } + + if m.logger.V(2) { + m.logger.Infof("Received update for Cluster resource %q: %+v", resourceName, update) + } + m.clusterWatchers[resourceName].setLastUpdate(update) + m.maybeSendUpdateLocked() +} + +// Records a resource error for a Cluster resource, clears the last successful +// update since we want to stop using the resource if we get a resource error. +func (m *DependencyManager) onClusterResourceError(resourceName string, err error, onDone func()) { + m.mu.Lock() + defer m.mu.Unlock() + + defer onDone() + if m.stopped || m.clusterWatchers[resourceName] == nil { + return + } + m.logger.Warningf("Received resource error for Cluster resource %q: %v", resourceName, m.annotateErrorWithNodeID(err)) + m.clusterWatchers[resourceName].setLastError(err) + m.maybeSendUpdateLocked() +} + +// Ambient errors from cluster resource are logged and the last successful +// update is retained because it should continue to be used. +func (m *DependencyManager) onClusterAmbientError(resourceName string, err error, onDone func()) { + m.mu.Lock() + defer m.mu.Unlock() + + defer onDone() + if m.stopped || m.clusterWatchers[resourceName] == nil { + return + } + m.logger.Warningf("Cluster resource ambient error %q: %v", resourceName, m.annotateErrorWithNodeID(err)) +} + +func newEndpointWatcher(resourceName string, depMgr *DependencyManager) *xdsResourceState[xdsresource.EndpointsUpdate, struct{}] { + w := &xdsResourceWatcher[xdsresource.EndpointsUpdate]{ + onUpdate: func(u *xdsresource.EndpointsUpdate, onDone func()) { + depMgr.onEndpointUpdate(resourceName, u, onDone) + }, + onError: func(err error, onDone func()) { + depMgr.onEndpointResourceError(resourceName, err, onDone) + }, + onAmbientError: func(err error, onDone func()) { + depMgr.onEndpointAmbientError(resourceName, err, onDone) + }, + } + return &xdsResourceState[xdsresource.EndpointsUpdate, struct{}]{ + stop: xdsresource.WatchEndpoints(depMgr.xdsClient, resourceName, w), + } +} + +// Records a successful Endpoint resource update, clears any previous error from +// the state. +func (m *DependencyManager) onEndpointUpdate(resourceName string, update *xdsresource.EndpointsUpdate, onDone func()) { + m.mu.Lock() + defer m.mu.Unlock() + + defer onDone() + if m.stopped || m.endpointWatchers[resourceName] == nil { + return + } + + if m.logger.V(2) { + m.logger.Infof("Received update for Endpoint resource %q: %+v", resourceName, update) + } + m.endpointWatchers[resourceName].setLastUpdate(update) + m.maybeSendUpdateLocked() +} + +// Records a resource error and clears the last successful update since the +// endpoints should not be used after getting a resource error. +func (m *DependencyManager) onEndpointResourceError(resourceName string, err error, onDone func()) { + m.mu.Lock() + defer m.mu.Unlock() + + defer onDone() + if m.stopped || m.endpointWatchers[resourceName] == nil { + return + } + m.logger.Warningf("Received resource error for Endpoint resource %q: %v", resourceName, m.annotateErrorWithNodeID(err)) + // Send an empty EndpointsUpdate instead of nil to avoid nil-check handling + // in the CDS balancer. The priority balancer will handle the case of having + // no endpoints and transition the channel to Transient Failure if needed. + m.endpointWatchers[resourceName].lastUpdate = &xdsresource.EndpointsUpdate{} + m.endpointWatchers[resourceName].lastErr = err + m.endpointWatchers[resourceName].updateReceived = true + m.maybeSendUpdateLocked() +} + +// Logs the ambient error and does not update the state, as the last successful +// update for endpoints should continue to be used. +func (m *DependencyManager) onEndpointAmbientError(resourceName string, err error, onDone func()) { + m.mu.Lock() + defer m.mu.Unlock() + defer onDone() + if m.stopped || m.endpointWatchers[resourceName] == nil { + return + } + + m.logger.Warningf("Endpoint resource ambient error %q: %v", resourceName, m.annotateErrorWithNodeID(err)) +} + +// Converts the DNS resolver state to an internal update, handling address-only +// updates by wrapping them into endpoints. It records the update and clears any +// previous error. +func (m *DependencyManager) onDNSUpdate(resourceName string, update *resolver.State) { + m.mu.Lock() + defer m.mu.Unlock() + if m.stopped || m.dnsResolvers[resourceName] == nil { + return + } + + if m.logger.V(2) { + m.logger.Infof("Received update from DNS resolver for resource %q: %+v", resourceName, update) + } + m.dnsResolvers[resourceName].setLastUpdate(&xdsresource.DNSUpdate{Endpoints: update.Endpoints}) + m.maybeSendUpdateLocked() +} + +// Records a DNS resolver error. It clears the last update only if no successful +// update has been received yet, then triggers a dependency update. +// +// If a previous good update was received, the error is logged and the previous +// update is retained for continued use. Errors are suppressed if a resource +// error was already received, as further propagation would have no downstream +// effect. +func (m *DependencyManager) onDNSError(resourceName string, err error) { + m.mu.Lock() + defer m.mu.Unlock() + + if m.stopped || m.dnsResolvers[resourceName] == nil { + return + } + + err = fmt.Errorf("dns resolver error for target %q: %v", resourceName, m.annotateErrorWithNodeID(err)) + m.logger.Warningf("%v", err) + state := m.dnsResolvers[resourceName] + if state.updateReceived { + return + } + + // Send an empty DNSUpdate instead of nil to avoid nil-check handling in the + // CDS balancer. The priority balancer will handle the case of having no + // endpoints and transition the channel to Transient Failure if needed. + state.lastUpdate = &xdsresource.DNSUpdate{} + state.lastErr = err + state.updateReceived = true + m.maybeSendUpdateLocked() +} + +// RequestDNSReresolution calls all the DNS resolver's ResolveNow. +func (m *DependencyManager) RequestDNSReresolution(opt resolver.ResolveNowOptions) { + m.mu.Lock() + defer m.mu.Unlock() + for _, res := range m.dnsResolvers { + if res.extras.dnsR != nil { + res.extras.dnsR.ResolveNow(opt) + } + } +} + +type resolverClientConn struct { + target string + depMgr *DependencyManager +} + +func (rcc *resolverClientConn) UpdateState(state resolver.State) error { + rcc.depMgr.dnsSerializer.TrySchedule(func(context.Context) { + rcc.depMgr.onDNSUpdate(rcc.target, &state) + }) + return nil +} + +func (rcc *resolverClientConn) ReportError(err error) { + rcc.depMgr.dnsSerializer.TrySchedule(func(context.Context) { + rcc.depMgr.onDNSError(rcc.target, err) + }) +} + +func (rcc *resolverClientConn) NewAddress(addresses []resolver.Address) { + rcc.UpdateState(resolver.State{Addresses: addresses}) +} + +func (rcc *resolverClientConn) ParseServiceConfig(string) *serviceconfig.ParseResult { + return &serviceconfig.ParseResult{Err: fmt.Errorf("service config not supported")} +} + +func (m *DependencyManager) newDNSResolver(target string) *xdsResourceState[xdsresource.DNSUpdate, dnsExtras] { + rcc := &resolverClientConn{ + target: target, + depMgr: m, + } + u, err := url.Parse("dns:///" + target) + if err != nil { + err := fmt.Errorf("failed to parse DNS target %q: %v", target, m.annotateErrorWithNodeID(err)) + m.logger.Warningf("%v", err) + rcc.ReportError(err) + return &xdsResourceState[xdsresource.DNSUpdate, dnsExtras]{ + stop: func() {}, + } + } + + r, err := resolver.Get("dns").Build(resolver.Target{URL: *u}, rcc, resolver.BuildOptions{}) + if err != nil { + rcc.ReportError(err) + err := fmt.Errorf("failed to build DNS resolver for target %q: %v", target, m.annotateErrorWithNodeID(err)) + m.logger.Warningf("%v", err) + return &xdsResourceState[xdsresource.DNSUpdate, dnsExtras]{ + stop: func() {}, + } + } + + return &xdsResourceState[xdsresource.DNSUpdate, dnsExtras]{ + extras: dnsExtras{dnsR: r}, + stop: r.Close, + } +} + +// xdsResourceWatcher is a generic implementation of the xdsresource.Watcher +// interface. +type xdsResourceWatcher[T any] struct { + onUpdate func(*T, func()) + onError func(error, func()) + onAmbientError func(error, func()) +} + +func (x *xdsResourceWatcher[T]) ResourceChanged(update *T, onDone func()) { + x.onUpdate(update, onDone) +} + +func (x *xdsResourceWatcher[T]) ResourceError(err error, onDone func()) { + x.onError(err, onDone) +} + +func (x *xdsResourceWatcher[T]) AmbientError(err error, onDone func()) { + x.onAmbientError(err, onDone) +} + +// xdsClusterSubscriberKey is the type used as the key to store the +// ClusterSubscriber interface in the Attributes field of resolver.states. +type xdsClusterSubscriberKey struct{} + +// SetXDSClusterSubscriber returns a copy of state in which the Attributes field +// is updated with the ClusterSubscriber interface. +func SetXDSClusterSubscriber(state resolver.State, subs ClusterSubscriber) resolver.State { + state.Attributes = state.Attributes.WithValue(xdsClusterSubscriberKey{}, subs) + return state +} + +// XDSClusterSubscriberFromResolverState returns ClusterSubscriber interface +// stored as an attribute in the resolver state. +func XDSClusterSubscriberFromResolverState(state resolver.State) ClusterSubscriber { + if v := state.Attributes.Value(xdsClusterSubscriberKey{}); v != nil { + return v.(ClusterSubscriber) + } + return nil +} + +// ClusterSubscriber allows dynamic subscription to clusters. This is useful for +// scenarios where the cluster name was not present in the RouteConfiguration, +// e.g. when the route uses a ClusterSpecifierPlugin. +// +// The xDS resolver will pass this interface to the LB policies as an attribute +// in the resolver update. +type ClusterSubscriber interface { + // SubscribeToCluster creates a dynamic subscription for the named cluster. + // + // The returned cancel function must be called when the subscription is no + // longer needed. It is safe to call cancel multiple times. + SubscribeToCluster(clusterName string) (cancel func()) +} + +// clusterRef represents a reference to a cluster and maintains reference count +// of the number of users of the cluster. +type clusterRef struct { + // Access to these field is protected by DependencyManager's mutex and so + // they don't need to be atomic. + + // staticRefCount comes from cluster being specified in the route + // configuration. + staticRefCount int32 + // dynamicRefCount comes from cluster being referenced by RPCs or being + // dynamically subscribed by the balancers in case of cluster specifier + // plugin being used. + dynamicRefCount int32 +} + +// SubscribeToCluster increments the reference count for the cluster. If the +// cluster is not already being tracked, it is added to the clusterSubscriptions +// map. It returns a function to unsubscribe from the cluster i.e. decrease its +// refcount. This returned function is idempotent, meaning it can be called +// multiple times without any additional effect. Calling Subscribe in a blocking +// manner while handling an update will lead to a deadlock. +func (m *DependencyManager) SubscribeToCluster(name string) func() { + m.mu.Lock() + defer m.mu.Unlock() + + // If the cluster is already being tracked, increment its dynamic refcount + // and return. + subs, ok := m.clusterSubscriptions[name] + if ok { + subs.dynamicRefCount++ + return sync.OnceFunc(func() { m.unsubscribeFromCluster(name) }) + } + + // If the cluster is not being tracked, add it with dynamic refcount as 1. + m.clusterSubscriptions[name] = &clusterRef{ + dynamicRefCount: 1, + } + m.maybeSendUpdateLocked() + return sync.OnceFunc(func() { m.unsubscribeFromCluster(name) }) +} + +// unsubscribeFromCluster decrements the reference count for the cluster. If +// both the reference counts reaches zero, it removes the cluster from the +// clusterSubscriptions map in the DependencyManager. Calling +// unsubscribeFromCluster in a blocking manner while handling an update will +// lead to a deadlock. +func (m *DependencyManager) unsubscribeFromCluster(name string) { + m.mu.Lock() + defer m.mu.Unlock() + // Balancers can invoke the unsubscribe closure returned by + // SubscribeToCluster after Close has already torn down all watchers and + // dropped m.watcher / m.xdsClient. Match the m.stopped guard used by + // every other post-Close callback in this file. + if m.stopped { + return + } + c := m.clusterSubscriptions[name] + c.dynamicRefCount-- + // This should not happen as unsubscribe returned from the + // ClusterSubscription is wrapped in sync.OnceFunc() + if c.dynamicRefCount < 0 { + m.logger.Errorf("Reference count for a cluster dropped below zero") + } + if c.dynamicRefCount == 0 && c.staticRefCount == 0 { + delete(m.clusterSubscriptions, name) + // Since this cluster has no more references, cancel the watch for it. + m.maybeSendUpdateLocked() + } +} diff --git a/internal/xds/xdsdepmgr/xds_dependency_manager_test.go b/internal/xds/xdsdepmgr/xds_dependency_manager_test.go new file mode 100644 index 0000000..dfa60e8 --- /dev/null +++ b/internal/xds/xdsdepmgr/xds_dependency_manager_test.go @@ -0,0 +1,1760 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsdepmgr_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/envoyproxy/go-control-plane/pkg/wellknown" + "github.com/google/uuid" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/clients" + xdstestutils "google.golang.org/grpc/internal/xds/test/e2e" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/internal/xds/xdsdepmgr" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + + defaultTestServiceName = "service-name" + defaultTestRouteConfigName = "route-config-name" + defaultTestClusterName = "cluster-name" + defaultTestEDSServiceName = "eds-service-name" +) + +func newStringP(s string) *string { + return &s +} + +// testWatcher is an implementation of the ConfigWatcher interface that sends +// the updates and errors received from the dependency manager to respective +// channels, for the tests to verify. +type testWatcher struct { + updateCh chan *xdsresource.XDSConfig + errorCh chan error + done chan struct{} +} + +func newTestWatcher() *testWatcher { + return &testWatcher{ + updateCh: make(chan *xdsresource.XDSConfig, 1), + errorCh: make(chan error), + done: make(chan struct{}), + } +} + +// Update sends the received XDSConfig update to the update channel. Does not +// send updates if the done channel is closed. The done channel is closed in the +// cases of errors because management server keeps sending error updates that +// cases multiple updates to be sent from dependency manager causing the update +// channel to be blocked. +func (w *testWatcher) Update(cfg *xdsresource.XDSConfig) { + select { + case <-w.done: + return + case w.updateCh <- cfg: + } +} + +// Error sends the received error to the error channel. +func (w *testWatcher) Error(err error) { + select { + case <-w.done: + return + case w.errorCh <- err: + } +} + +// Closes the testWatcher.done channel which will stop the updates being pushed +// to testWatcher.updateCh. This is the first thing that needs to happen as soon +// as the test ends before anything else closes to avoid deadlocks in tests with +// CDS and EDS errors because, in case of error , management server keeps +// sending multiple error updates. +func (w *testWatcher) close() { + close(w.done) +} + +func verifyError(ctx context.Context, errCh chan error, wantErr, wantNodeID string) error { + select { + case gotErr := <-errCh: + if gotErr == nil { + return fmt.Errorf("got nil error from resolver, want error %q", wantErr) + } + if !strings.Contains(gotErr.Error(), wantErr) { + return fmt.Errorf("got error from resolver %q, want %q", gotErr, wantErr) + } + if !strings.Contains(gotErr.Error(), wantNodeID) { + return fmt.Errorf("got error from resolver %q, want nodeID %q", gotErr, wantNodeID) + } + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for error from dependency manager") + } + return nil +} + +// setupManagementServerAndClient creates a management server, an xds client and +// returns the node ID, management server and xds client. +func setupManagementServerAndClient(t *testing.T, allowResourceSubset bool) (string, *e2e.ManagementServer, xdsclient.XDSClient) { + t.Helper() + nodeID := uuid.New().String() + mgmtServer, bootstrapContents := setupManagementServerForTest(t, nodeID, allowResourceSubset) + xdsClient := createXDSClient(t, bootstrapContents) + return nodeID, mgmtServer, xdsClient +} + +func createXDSClient(t *testing.T, bootstrapContents []byte) xdsclient.XDSClient { + t.Helper() + + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + + pool := xdsclient.NewPool(config) + c, cancel, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{Name: t.Name()}) + if err != nil { + t.Fatalf("Failed to create an xDS client: %v", err) + } + t.Cleanup(cancel) + return c +} + +// Spins up an xDS management server and sets up the xDS bootstrap +// configuration. +// +// Returns the following: +// - A reference to the xDS management server +// - Contents of the bootstrap configuration pointing to xDS management +// server +func setupManagementServerForTest(t *testing.T, nodeID string, allowResourceSubset bool) (*e2e.ManagementServer, []byte) { + t.Helper() + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + AllowResourceSubset: allowResourceSubset, + }) + t.Cleanup(mgmtServer.Stop) + + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + return mgmtServer, bootstrapContents +} + +// makeAggregateClusterResource returns an aggregate cluster resource with the +// given name and list of child names. +func makeAggregateClusterResource(name string, childNames []string) *v3clusterpb.Cluster { + return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: name, + Type: e2e.ClusterTypeAggregate, + ChildNames: childNames, + }) +} + +// makeLogicalDNSClusterResource returns a LOGICAL_DNS cluster resource with the +// given name and given DNS host and port. +func makeLogicalDNSClusterResource(name, dnsHost string, dnsPort uint32) *v3clusterpb.Cluster { + return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: name, + Type: e2e.ClusterTypeLogicalDNS, + DNSHostName: dnsHost, + DNSPort: dnsPort, + }) +} + +// replaceDNSResolver unregisters the DNS resolver and registers a manual +// resolver for the same scheme. This allows the test to fake the DNS resolution +// by supplying the addresses of the test backends. +func replaceDNSResolver(t *testing.T) *manual.Resolver { + t.Helper() + mr := manual.NewBuilderWithScheme("dns") + + dnsResolverBuilder := resolver.Get("dns") + resolver.Register(mr) + + t.Cleanup(func() { resolver.Register(dnsResolverBuilder) }) + return mr +} + +// Tests the happy case where the dependency manager receives all the required +// resources and verifies that Update is called with the correct XDSConfig. +func (s) TestHappyCase(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + wantXdsConfig := xdstestutils.MakeXDSConfig(defaultTestServiceName, resources.Routes[0].Name, resources.Clusters[0].Name, resources.Clusters[0].EdsClusterConfig.ServiceName, "localhost:8080") + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } +} + +// Tests the case where the listener contains an inline route configuration and +// verifies that Update is called with the correct XDSConfig. +func (s) TestInlineRouteConfig(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: e2e.DefaultRouteConfig(defaultTestRouteConfigName, defaultTestServiceName, defaultTestClusterName), + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, // router fields are unused by grpc + }) + listener := &v3listenerpb.Listener{ + Name: defaultTestServiceName, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + } + cluster := e2e.DefaultCluster(defaultTestClusterName, defaultTestEDSServiceName, e2e.SecurityLevelNone) + endpoint := e2e.DefaultEndpoint(defaultTestEDSServiceName, "localhost", []uint32{8080}) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{endpoint}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + wantXdsConfig := xdstestutils.MakeXDSConfig(defaultTestServiceName, defaultTestRouteConfigName, defaultTestClusterName, defaultTestEDSServiceName, "localhost:8080") + wantXdsConfig.Listener.APIListener.InlineRouteConfig = wantXdsConfig.RouteConfig + wantXdsConfig.Listener.APIListener.RouteConfigName = "" + + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } +} + +// Tests the case where dependency manager only receives listener resource but +// does not receive route config resource. Verifies that Update is not called +// since we do not have all resources. +func (s) TestNoRouteConfigResource(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + listener := e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName) + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + SkipValidation: true, + }); err != nil { + t.Fatal(err) + } + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case update := <-watcher.updateCh: + t.Fatalf("Received unexpected update from dependency manager: %+v", update) + case err := <-watcher.errorCh: + t.Fatalf("Received unexpected error from dependency manager: %v", err) + } +} + +// Tests the case where dependency manager receives a listener resource error by +// sending the correct update first and then removing the listener resource. It +// verifies that Error is called with the correct error. +func (s) TestListenerResourceNotFoundError(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Send a correct update first + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + wantXdsConfig := xdstestutils.MakeXDSConfig(defaultTestServiceName, resources.Routes[0].Name, resources.Clusters[0].Name, resources.Clusters[0].EdsClusterConfig.ServiceName, "localhost:8080") + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } + + // Remove listener resource so that we get listener resource error. + resources.Listeners = nil + resources.SkipValidation = true + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + if err := verifyError(ctx, watcher.errorCh, fmt.Sprintf("xds: resource %q of type %q has been removed", defaultTestServiceName, "ListenerResource"), nodeID); err != nil { + t.Fatal(err) + } +} + +// Tests the scenario where the Dependency Manager receives an invalid +// RouteConfiguration from the management server. The test provides a +// malformed resource to trigger a NACK, and verifies that the Dependency +// Manager propagates the resulting error via Error method. +func (s) TestRouteConfigResourceError(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + listener := e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName) + route := e2e.DefaultRouteConfig(defaultTestRouteConfigName, defaultTestServiceName, defaultTestClusterName) + // Remove the Match to make sure the route resource is NACKed by XDSClient + // sending a route resource error to dependency manager. + route.VirtualHosts[0].Routes[0].Match = nil + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Routes: []*v3routepb.RouteConfiguration{route}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + // Defer closing the watcher to prevent a potential hang. The management + // server may send repeated errors, triggering updates that hold the + // dependency manager's mutex. This defer is defined last so it executes + // first (before dm.Close()). If we don't stop the watcher, dm.Close() will + // deadlock waiting for the mutex currently held by the blocking Update + // call. + defer watcher.close() + + if err := verifyError(ctx, watcher.errorCh, "route resource error", nodeID); err != nil { + t.Fatal(err) + } +} + +// Tests the case where a received route configuration update has no virtual +// hosts. Verifies that Error is called with the expected error. +func (s) TestNoVirtualHost(t *testing.T) { + nodeID := uuid.New().String() + mgmtServer, bc := setupManagementServerForTest(t, nodeID, false) + xdsClient := createXDSClient(t, bc) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + // Make the virtual host match nil so that the route config is NACKed. + resources.Routes[0].VirtualHosts = nil + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + if err := verifyError(ctx, watcher.errorCh, "could not find VirtualHost", nodeID); err != nil { + t.Fatal(err) + } +} + +// Tests the case where we already have a cached resource and then we get a +// route resource with no virtual host, which also results in error being sent +// across. +func (s) TestNoVirtualHost_ExistingResource(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + // Verify valid update. + wantXdsConfig := xdstestutils.MakeXDSConfig(defaultTestServiceName, resources.Routes[0].Name, resources.Clusters[0].Name, resources.Endpoints[0].ClusterName, "localhost:8080") + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } + + // 3. Send route update with no virtual host. + resources.Routes[0].VirtualHosts = nil + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // 4. Verify error. + if err := verifyError(ctx, watcher.errorCh, "could not find VirtualHost", nodeID); err != nil { + t.Fatal(err) + } +} + +// Tests the case where we get an ambient error and verify that we correctly log +// a warning for it. To make sure we get an ambient error, we send a correct +// update first, then send an invalid one and then send the valid resource +// again. We send the valid resource again so that we can be sure the ambient +// error reaches the dependency manager since there is no other way to wait for +// it. +func (s) TestAmbientError(t *testing.T) { + // Expect a warning log for the ambient error. + grpctest.ExpectWarning("Listener resource ambient error") + + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Configure a valid resources. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + // Wait for the initial valid update. + wantXdsConfig := xdstestutils.MakeXDSConfig(defaultTestServiceName, resources.Routes[0].Name, resources.Clusters[0].Name, resources.Clusters[0].EdsClusterConfig.ServiceName, "localhost:8080") + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } + + // Configure a listener resource that is expected to be NACKed because it + // does not contain the `RouteSpecifier` field in the HTTPConnectionManager. + // Since a valid one is already cached, this should result in an ambient + // error. + hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, + }) + lis := &v3listenerpb.Listener{ + Name: defaultTestServiceName, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + } + resources.Listeners = []*v3listenerpb.Listener{lis} + resources.SkipValidation = true + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // We expect no call to Error or Update on our watcher. We just wait for a + // short duration to ensure that. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case err := <-watcher.errorCh: + t.Fatalf("Unexpected call to Error %v", err) + case update := <-watcher.updateCh: + t.Fatalf("Unexpected call to Update %+v", update) + case <-sCtx.Done(): + } + + // Send valid resources again. + resources = e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } +} + +// Tests the case where the cluster name changes in the route resource update +// and verify that each time Update is called with correct cluster name. +func (s) TestRouteResourceUpdate(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Configure initial resources + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + // Wait for the first update. + wantXdsConfig := xdstestutils.MakeXDSConfig(defaultTestServiceName, resources.Routes[0].Name, resources.Clusters[0].Name, resources.Clusters[0].EdsClusterConfig.ServiceName, "localhost:8080") + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } + + // Update route to point to a new cluster. + newClusterName := "new-cluster-name" + route2 := e2e.DefaultRouteConfig(resources.Routes[0].Name, defaultTestServiceName, newClusterName) + cluster2 := e2e.DefaultCluster(newClusterName, newClusterName, e2e.SecurityLevelNone) + endpoints2 := e2e.DefaultEndpoint(newClusterName, "localhost", []uint32{8081}) + resources.Routes = []*v3routepb.RouteConfiguration{route2} + resources.Clusters = []*v3clusterpb.Cluster{cluster2} + resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{endpoints2} + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the second update and verify it has the new cluster. + wantXdsConfig.RouteConfig.VirtualHosts[0].Routes[0].WeightedClusters[0].Name = newClusterName + wantXdsConfig.VirtualHost.Routes[0].WeightedClusters[0].Name = newClusterName + wantXdsConfig.Clusters = map[string]*xdsresource.ClusterResult{ + newClusterName: { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterName: newClusterName, + ClusterType: xdsresource.ClusterTypeEDS, + EDSServiceName: newClusterName, + }, + EndpointConfig: &xdsresource.EndpointConfig{ + EDSUpdate: &xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + {ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "localhost:8081"}}}, + HealthStatus: xdsresource.EndpointHealthStatusUnknown, + Weight: 1, + }, + }, + Weight: 1, + }, + }, + }, + }, + }, + }, + } + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } +} + +// Tests the case where the route resource is first sent from the management +// server and the changed to be inline with the listener and then again changed +// to be received from the management server. It verifies that each time Update +// is called with the correct XDSConfig. +func (s) TestRouteResourceChangeToInline(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Initial resources with defaultTestClusterName + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + // Wait for the first update. + wantXdsConfig := xdstestutils.MakeXDSConfig(defaultTestServiceName, resources.Routes[0].Name, resources.Clusters[0].Name, resources.Clusters[0].EdsClusterConfig.ServiceName, "localhost:8080") + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } + + // Update route to point to a new cluster and make it inline with the + // listener. + newClusterName := "new-cluster-name" + hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: e2e.DefaultRouteConfig(defaultTestRouteConfigName, defaultTestServiceName, newClusterName), + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, // router fields are unused by grpc + }) + resources.Listeners[0].ApiListener.ApiListener = hcm + resources.Clusters = []*v3clusterpb.Cluster{e2e.DefaultCluster(newClusterName, defaultTestEDSServiceName, e2e.SecurityLevelNone)} + resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(defaultTestEDSServiceName, "localhost", []uint32{8081})} + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the second update and verify it has the new cluster. + wantInlineXdsConfig := &xdsresource.XDSConfig{ + Listener: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + InlineRouteConfig: &xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: newClusterName, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute, + }}, + }, + }, + }, + }, + }, + RouteConfig: &xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: newClusterName, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute, + }}, + }, + }, + }, + VirtualHost: &xdsresource.VirtualHost{ + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: newClusterName, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute}, + }, + }, + Clusters: map[string]*xdsresource.ClusterResult{ + newClusterName: { + Config: xdsresource.ClusterConfig{Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: newClusterName, + EDSServiceName: defaultTestEDSServiceName, + }, + EndpointConfig: &xdsresource.EndpointConfig{ + EDSUpdate: &xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + {ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "localhost:8081"}}}, + HealthStatus: xdsresource.EndpointHealthStatusUnknown, + Weight: 1, + }, + }, + Weight: 1, + }, + }, + }, + }, + }, + }, + }, + } + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantInlineXdsConfig); err != nil { + t.Fatal(err) + } + + // Change the route resource back to non-inline. + resources = e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } +} + +// Tests the case where the dependency manager receives a cluster resource error +// and verifies that Update is called with XDSConfig containing cluster error. +func (s) TestClusterResourceError(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{}} + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + // Defer closing the watcher to prevent a potential hang. The management + // server may send repeated errors, triggering updates that hold the + // dependency manager's mutex. This defer is defined last so it executes + // first (before dm.Close()). If we don't stop the watcher, dm.Close() will + // deadlock waiting for the mutex currently held by the blocking Update + // call. + defer watcher.close() + + wantXdsConfig := xdstestutils.MakeXDSConfig(defaultTestServiceName, resources.Routes[0].Name, resources.Clusters[0].Name, resources.Clusters[0].EdsClusterConfig.ServiceName, "localhost:8080") + wantXdsConfig.Clusters[resources.Clusters[0].Name] = &xdsresource.ClusterResult{Err: fmt.Errorf("[xDS node id: %v]: %v", nodeID, fmt.Errorf("unsupported config_source_specifier *corev3.ConfigSource_Ads in lrs_server field"))} + + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } +} + +// Tests the case where the dependency manager receives a cluster resource +// ambient error. A valid cluster resource is sent first, then an invalid +// one and then the valid resource again. The valid resource is sent again +// to make sure that the ambient error reaches the dependency manager since +// there is no other way to wait for it. +func (s) TestClusterAmbientError(t *testing.T) { + // Expect a warning log for the ambient error. + grpctest.ExpectWarning("Cluster resource ambient error") + + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + wantXdsConfig := xdstestutils.MakeXDSConfig(defaultTestServiceName, resources.Routes[0].Name, resources.Clusters[0].Name, resources.Clusters[0].EdsClusterConfig.ServiceName, "localhost:8080") + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } + + // Configure a cluster resource that is expected to be NACKed because it + // does not contain the `LrsServer` field. Since a valid one is already + // cached, this should result in an ambient error. + resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{}} + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + select { + case <-time.After(defaultTestShortTimeout): + case update := <-watcher.updateCh: + t.Fatalf("received unexpected update from dependency manager: %v", update) + } + + // Send valid resources again to guarantee we get the cluster ambient error + // before the test ends. + resources = e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } +} + +// Tests the case where a cluster is an aggregate cluster whose children are of +// type EDS and DNS. Verifies that Update is not called when one of the child +// resources is not configured and then verifies that Update is called with +// correct config when all resources are configured. +func (s) TestAggregateCluster(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, true) + + dnsR := replaceDNSResolver(t) + dnsR.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "127.0.0.1:8081"}}}, + {Addresses: []resolver.Address{{Addr: "[::1]:8081"}}}, + }, + }) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + listener := e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName) + route := e2e.DefaultRouteConfig(defaultTestRouteConfigName, defaultTestServiceName, defaultTestClusterName) + aggregateCluster := makeAggregateClusterResource(defaultTestClusterName, []string{"eds-cluster", "dns-cluster"}) + edsCluster := e2e.DefaultCluster("eds-cluster", defaultTestEDSServiceName, e2e.SecurityLevelNone) + endpoint := e2e.DefaultEndpoint(defaultTestEDSServiceName, "localhost", []uint32{8080}) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Routes: []*v3routepb.RouteConfiguration{route}, + Clusters: []*v3clusterpb.Cluster{aggregateCluster, edsCluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{endpoint}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + // Verify that no configuration is pushed to the child policy yet, because + // not all clusters making up the aggregate cluster have been resolved yet. + select { + case <-time.After(defaultTestShortTimeout): + case update := <-watcher.updateCh: + t.Fatalf("received unexpected update from dependency manager: %+v", update) + case err := <-watcher.errorCh: + t.Fatalf("received unexpected error from dependency manager: %v", err) + } + + // Now configure the LogicalDNS cluster in the management server. This + // should result in configuration being pushed down to the child policy. + resources.Clusters = append(resources.Clusters, makeLogicalDNSClusterResource("dns-cluster", "localhost", 8081)) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + wantXdsConfig := &xdsresource.XDSConfig{ + Listener: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: resources.Routes[0].Name, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + RouteConfig: &xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: resources.Clusters[0].Name, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute, + }}, + }, + }, + }, + VirtualHost: &xdsresource.VirtualHost{ + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: resources.Clusters[0].Name, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute}, + }}, + Clusters: map[string]*xdsresource.ClusterResult{ + resources.Clusters[0].Name: { + Config: xdsresource.ClusterConfig{Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: resources.Clusters[0].Name, + PrioritizedClusterNames: []string{"eds-cluster", "dns-cluster"}, + }, + AggregateConfig: &xdsresource.AggregateConfig{LeafClusters: []string{"eds-cluster", "dns-cluster"}}, + }, + }, + "eds-cluster": { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: "eds-cluster", + EDSServiceName: defaultTestEDSServiceName}, + EndpointConfig: &xdsresource.EndpointConfig{ + EDSUpdate: &xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + {ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Endpoints: []xdsresource.Endpoint{ + { + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "localhost:8080"}}}, + HealthStatus: xdsresource.EndpointHealthStatusUnknown, + Weight: 1, + }, + }, + Weight: 1, + }, + }, + }, + }, + }}, + "dns-cluster": { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeLogicalDNS, + ClusterName: "dns-cluster", + DNSHostName: "localhost:8081", + }, + EndpointConfig: &xdsresource.EndpointConfig{ + DNSEndpoints: &xdsresource.DNSUpdate{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "127.0.0.1:8081"}}}, + {Addresses: []resolver.Address{{Addr: "[::1]:8081"}}}, + }, + }, + }, + }, + }, + }, + } + + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } + +} + +// Tests the case where an aggregate cluster has one child whose resource is +// configured with an error. Verifies that the error is correctly received in +// the XDSConfig. +func (s) TestAggregateClusterChildError(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, true) + + watcher := newTestWatcher() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + dnsR := replaceDNSResolver(t) + dnsR.UpdateState(resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "127.0.0.1:8081"}}}, + {Addresses: []resolver.Address{{Addr: "[::1]:8081"}}}, + }, + }) + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(defaultTestRouteConfigName, defaultTestServiceName, defaultTestClusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(defaultTestClusterName, []string{"err-cluster", "good-cluster"}), + e2e.DefaultCluster("err-cluster", defaultTestEDSServiceName, e2e.SecurityLevelNone), + makeLogicalDNSClusterResource("good-cluster", "localhost", 8081), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(defaultTestEDSServiceName, "localhost", []uint32{8080})}, + } + resources.Endpoints[0].Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + // Defer closing the watcher to prevent a potential hang. The management + // server may send repeated errors, triggering updates that hold the + // dependency manager's mutex. This defer is defined last so it executes + // first (before dm.Close()). If we don't stop the watcher, dm.Close() will + // deadlock waiting for the mutex currently held by the blocking Update + // call. + defer watcher.close() + + wantXdsConfig := &xdsresource.XDSConfig{ + Listener: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: defaultTestRouteConfigName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + RouteConfig: &xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{{ + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: defaultTestClusterName, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute, + }}, + }}, + }, + VirtualHost: &xdsresource.VirtualHost{ + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: defaultTestClusterName, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute, + }}, + }, + Clusters: map[string]*xdsresource.ClusterResult{ + defaultTestClusterName: { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: defaultTestClusterName, + PrioritizedClusterNames: []string{"err-cluster", "good-cluster"}, + }, + AggregateConfig: &xdsresource.AggregateConfig{LeafClusters: []string{"err-cluster", "good-cluster"}}, + }, + }, + "err-cluster": { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: "err-cluster", + EDSServiceName: defaultTestEDSServiceName, + }, + EndpointConfig: &xdsresource.EndpointConfig{ + EDSUpdate: &xdsresource.EndpointsUpdate{}, + ResolutionNote: fmt.Errorf("[xDS node id: %v]: %v", nodeID, fmt.Errorf("EDS response contains an endpoint with zero weight: endpoint:{address:{socket_address:{address:%q port_value:%v}}} load_balancing_weight:{}", "localhost", 8080)), + }, + }, + }, + "good-cluster": { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeLogicalDNS, + ClusterName: "good-cluster", + DNSHostName: "localhost:8081", + }, + EndpointConfig: &xdsresource.EndpointConfig{ + DNSEndpoints: &xdsresource.DNSUpdate{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "127.0.0.1:8081"}}}, + {Addresses: []resolver.Address{{Addr: "[::1]:8081"}}}, + }, + }, + }, + }, + }, + }, + } + + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } +} + +// Tests the case where an aggregate cluster has no leaf clusters by creating a +// cyclic dependency where A->B and B->A. Verifies that an error with "no leaf +// clusters found" is received. +func (s) TestAggregateClusterNoLeafCluster(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, true) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const ( + clusterNameA = defaultTestClusterName // cluster name in cds LB policy config + clusterNameB = defaultTestClusterName + "-B" + ) + // Create a cyclic dependency where A->B and B->A. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(defaultTestRouteConfigName, defaultTestServiceName, defaultTestClusterName)}, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterNameA, []string{clusterNameB}), + makeAggregateClusterResource(clusterNameB, []string{clusterNameA}), + }, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + wantXdsConfig := &xdsresource.XDSConfig{ + Listener: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: defaultTestRouteConfigName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + RouteConfig: &xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{{ + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: defaultTestClusterName, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute, + }}, + }}, + }, + VirtualHost: &xdsresource.VirtualHost{ + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: defaultTestClusterName, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute, + }}, + }, + Clusters: map[string]*xdsresource.ClusterResult{ + defaultTestClusterName: { + Err: fmt.Errorf("[xDS node id: %v]: %v", nodeID, fmt.Errorf("aggregate cluster graph has no leaf clusters")), + }, + clusterNameB: { + Err: fmt.Errorf("[xDS node id: %v]: %v", nodeID, fmt.Errorf("aggregate cluster graph has no leaf clusters")), + }, + }, + } + + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } +} + +// Tests the case where nested aggregate clusters exceed the max depth of 16. +// Verify that the error is correctly received in the XDSConfig in all the +// clusters. +func (s) TestAggregateClusterMaxDepth(t *testing.T) { + const clusterDepth = 17 + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, true) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create a graph of aggregate clusters with 18 clusters. + clusters := make([]*v3clusterpb.Cluster, clusterDepth) + for i := 0; i < clusterDepth; i++ { + clusters[i] = makeAggregateClusterResource(fmt.Sprintf("agg-%d", i), []string{fmt.Sprintf("agg-%d", i+1)}) + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(defaultTestServiceName, defaultTestRouteConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(defaultTestRouteConfigName, defaultTestServiceName, "agg-0")}, + Clusters: clusters, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + commonError := fmt.Errorf("[xDS node id: %v]: %v", nodeID, fmt.Errorf("aggregate cluster graph exceeds max depth (%d)", 16)) + + wantXdsConfig := &xdsresource.XDSConfig{ + Listener: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: defaultTestRouteConfigName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + RouteConfig: &xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{{ + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + // The route should point to the first cluster in the chain: + // agg-0 + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: "agg-0", Weight: 100}}, + ActionType: xdsresource.RouteActionRoute, + }}, + }}, + }, + VirtualHost: &xdsresource.VirtualHost{ + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: "agg-0", Weight: 100}}, + ActionType: xdsresource.RouteActionRoute, + }}, + }, + Clusters: map[string]*xdsresource.ClusterResult{}, // Initialize the map + } + + // Populate the Clusters map with all clusters,except the last one, each + // having the common error + for i := 0; i < clusterDepth; i++ { + clusterName := fmt.Sprintf("agg-%d", i) + + // The ClusterResult only needs the Err field set to the common error + wantXdsConfig.Clusters[clusterName] = &xdsresource.ClusterResult{ + Err: commonError, + } + } + + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } +} + +// Tests the case where the dependency manager receives a endpoint resource +// ambient error. A valid endpoint resource is sent first, then an invalid +// one and then the valid resource again. The valid resource is sent again +// to make sure that the ambient error reaches the dependency manager since +// there is no other way to wait for it. +func (s) TestEndpointAmbientError(t *testing.T) { + // Expect a warning log for the ambient error. + grpctest.ExpectWarning("Endpoint resource ambient error") + + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + wantXdsConfig := xdstestutils.MakeXDSConfig(defaultTestServiceName, resources.Routes[0].Name, resources.Clusters[0].Name, resources.Clusters[0].EdsClusterConfig.ServiceName, "localhost:8080") + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } + + // Configure a endpoint resource that is expected to be NACKed because it + // does not contain the `Locality` field. Since a valid one is already + // cached, this should result in an ambient error. + resources = e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + resources.Endpoints[0].Endpoints[0].Locality = nil + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + select { + case <-time.After(defaultTestShortTimeout): + case update := <-watcher.updateCh: + t.Fatalf("received unexpected update from dependency manager: %v", update) + } + + // Send valid resources again to guarantee we get the cluster ambient error + // before the test ends. + resources = e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } +} + +// Tests the scenario where a cluster is removed from route config but still has +// subscriptions. Verifies that it is present in the XDSConfig update. Also +// verifies that it is removed from the XDSConfig update after all the +// references for that cluster are no longer present. +func (s) TestClusterSubscription_Lifecycle(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Initial resources + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + // Verify initial update. + wantXdsConfig := xdstestutils.MakeXDSConfig(defaultTestServiceName, resources.Routes[0].Name, resources.Clusters[0].Name, resources.Clusters[0].EdsClusterConfig.ServiceName, "localhost:8080") + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } + + clusterName := resources.Clusters[0].Name + edsServiceName := resources.Clusters[0].EdsClusterConfig.ServiceName + + // Subscribe twice to the old cluster to test multiple subscriptions and + // verify that the cluster is only removed after all subscriptions are + // removed. + unsubscribe1 := dm.SubscribeToCluster(clusterName) + unsubscribe2 := dm.SubscribeToCluster(clusterName) + + // Update RouteConfig to REMOVE the cluster and point to a new cluster. The + // old cluster should still be present in the update because of the + // remaining subscription. + newClusterName := "new-cluster-name" + newEDSServcie := "new-eds-servcie" + route2 := e2e.DefaultRouteConfig(resources.Routes[0].Name, defaultTestServiceName, newClusterName) + cluster2 := e2e.DefaultCluster(newClusterName, newEDSServcie, e2e.SecurityLevelNone) + endpoint2 := e2e.DefaultEndpoint(newEDSServcie, "localhost", []uint32{8081}) + + resources.Routes = []*v3routepb.RouteConfiguration{route2} + // Keep the old cluster in the management server so it doesn't return a + // resource error if watched. + resources.Clusters = append(resources.Clusters, cluster2) + resources.Endpoints = append(resources.Endpoints, endpoint2) + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify the update to contain BOTH "new-cluster-name" (from route) AND + // "clusterName" (from subscription). + wantXDSConfig := &xdsresource.XDSConfig{ + Listener: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: resources.Routes[0].Name, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + RouteConfig: &xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{{ + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: newClusterName, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute, + }}, + }}, + }, + VirtualHost: &xdsresource.VirtualHost{ + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: newClusterName, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute, + }}, + }, + Clusters: map[string]*xdsresource.ClusterResult{ + clusterName: { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: clusterName, + EDSServiceName: edsServiceName, + }, + EndpointConfig: &xdsresource.EndpointConfig{ + EDSUpdate: &xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{{ + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "localhost:8080"}}}, + HealthStatus: xdsresource.EndpointHealthStatusUnknown, + Weight: 1, + }}, + Weight: 1, + }}, + }, + }, + }, + }, + newClusterName: { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: newClusterName, + EDSServiceName: newEDSServcie, + }, + EndpointConfig: &xdsresource.EndpointConfig{ + EDSUpdate: &xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{{ + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "localhost:8081"}}}, + HealthStatus: xdsresource.EndpointHealthStatusUnknown, + Weight: 1, + }}, + Weight: 1, + }}, + }, + }, + }, + }, + }, + } + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXDSConfig); err != nil { + t.Fatal(err) + } + + // Unsubscribe one reference to the old cluster. + unsubscribe1() + + // Verify that no update is received since there is still one subscription + // to the old cluster remaining. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case update := <-watcher.updateCh: + t.Fatalf("Received unexpected update from dependency manager: %+v", update) + case err := <-watcher.errorCh: + t.Fatalf("Received unexpected error from dependency manager: %v", err) + } + + unsubscribe2() + // Now "clusterName" should be removed. "newClusterName" should remain. + wantXdsConfig = xdstestutils.MakeXDSConfig(defaultTestServiceName, resources.Routes[0].Name, newClusterName, newEDSServcie, "localhost:8081") + + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } +} + +// Tests the scenario where a dynamic subscription is made for a new cluster +// while an existing cluster is already being watched via route configuration. +// It verifies that updates to the existing cluster are still correctly +// propagated even while the dynamic cluster remains unresolved. Finally, it +// verifies that once the missing resource for the dynamic cluster is provided, +// the manager successfully resolves the complete picture and delivers an update +// containing the state for both clusters. +func (s) TestUpdateWithUnresolvedDynamicSubscription(t *testing.T) { + nodeID, mgmtServer, xdsClient := setupManagementServerAndClient(t, false) + + watcher := newTestWatcher() + defer watcher.close() + + ctx, cancel := context.WithTimeout(context.Background(), 50*defaultTestTimeout) + defer cancel() + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + NodeID: nodeID, + DialTarget: defaultTestServiceName, + Host: "localhost", + Port: 8080, + SecLevel: e2e.SecurityLevelNone, + }) + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + dm := xdsdepmgr.New(defaultTestServiceName, defaultTestServiceName, xdsClient, watcher) + defer dm.Close() + + wantXdsConfig := xdstestutils.MakeXDSConfig(defaultTestServiceName, resources.Routes[0].Name, resources.Clusters[0].Name, resources.Clusters[0].EdsClusterConfig.ServiceName, "localhost:8080") + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } + + // Subscribe to cluster present in route config emulating a RPC. + dm.SubscribeToCluster(resources.Clusters[0].Name) + + // Subscribe to new cluster not present in route config emulating + // susbscription for a dynamic resource. + newClusterName := "new-cluster-name" + dm.SubscribeToCluster(newClusterName) + + // Verify that update is received since all static clusters are present. + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } + + // Update EDS for the first cluster only. + endpoint := e2e.DefaultEndpoint(resources.Clusters[0].EdsClusterConfig.ServiceName, "localhost", []uint32{9090}) + resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{endpoint} + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that we get the update even though we have a subscription to + // new-cluster-name which has no resources yet. + wantXdsConfig = xdstestutils.MakeXDSConfig(defaultTestServiceName, resources.Routes[0].Name, resources.Clusters[0].Name, resources.Clusters[0].EdsClusterConfig.ServiceName, "localhost:9090") + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } + + // Add resources for new cluster. + resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(newClusterName, "new-eds-service", e2e.SecurityLevelNone)) + resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint("new-eds-service", "localhost", []uint32{10080})) + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // The dependency manager sends a duplicate update when the cluster resource + // corresponding to the dynamic subscription is received, because all static + // clusters are fully resolved at this point. A subsequent update with both + // clusters will be sent when the EDS resource corresponding to the dynamic + // cluster is also received. + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } + + wantXdsConfig = &xdsresource.XDSConfig{ + Listener: &xdsresource.ListenerUpdate{ + APIListener: &xdsresource.HTTPConnectionManagerConfig{ + RouteConfigName: resources.Routes[0].Name, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + RouteConfig: &xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{{ + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: resources.Clusters[0].Name, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute, + }}, + }}, + }, + VirtualHost: &xdsresource.VirtualHost{ + Domains: []string{defaultTestServiceName}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("/"), + WeightedClusters: []xdsresource.WeightedCluster{{Name: resources.Clusters[0].Name, Weight: 100}}, + ActionType: xdsresource.RouteActionRoute}, + }, + }, + Clusters: map[string]*xdsresource.ClusterResult{ + resources.Clusters[0].Name: { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: resources.Clusters[0].Name, + EDSServiceName: resources.Clusters[0].EdsClusterConfig.ServiceName, + }, + EndpointConfig: &xdsresource.EndpointConfig{ + EDSUpdate: &xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{{ + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "localhost:9090"}}}, + HealthStatus: xdsresource.EndpointHealthStatusUnknown, + Weight: 1, + }}, + Weight: 1, + }}, + }, + }, + }, + }, + newClusterName: { + Config: xdsresource.ClusterConfig{ + Cluster: &xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: newClusterName, + EDSServiceName: "new-eds-service", + }, + EndpointConfig: &xdsresource.EndpointConfig{ + EDSUpdate: &xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{{ + ID: clients.Locality{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Endpoints: []xdsresource.Endpoint{{ + ResolverEndpoint: resolver.Endpoint{Addresses: []resolver.Address{{Addr: "localhost:10080"}}}, + HealthStatus: xdsresource.EndpointHealthStatusUnknown, + Weight: 1, + }}, + Weight: 1, + }}, + }, + }, + }, + }, + }, + } + // Verify both clusters are now present. + if err := xdstestutils.VerifyXDSConfig(ctx, watcher.updateCh, watcher.errorCh, wantXdsConfig); err != nil { + t.Fatal(err) + } +} diff --git a/interop/alts/client/client.go b/interop/alts/client/client.go new file mode 100644 index 0000000..52f465a --- /dev/null +++ b/interop/alts/client/client.go @@ -0,0 +1,69 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// This binary can only run on Google Cloud Platform (GCP). +package main + +import ( + "context" + "flag" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/alts" + "google.golang.org/grpc/grpclog" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + hsAddr = flag.String("alts_handshaker_service_address", "", "ALTS handshaker gRPC service address") + serverAddr = flag.String("server_address", ":8080", "The port on which the server is listening") + + logger = grpclog.Component("interop") +) + +func main() { + flag.Parse() + + opts := alts.DefaultClientOptions() + if *hsAddr != "" { + opts.HandshakerServiceAddress = *hsAddr + } + altsTC := alts.NewClientCreds(opts) + conn, err := grpc.NewClient(*serverAddr, grpc.WithTransportCredentials(altsTC)) + if err != nil { + logger.Fatalf("grpc.NewClient(%q) = %v", *serverAddr, err) + } + defer conn.Close() + grpcClient := testgrpc.NewTestServiceClient(conn) + + // Call the EmptyCall API. + ctx := context.Background() + request := &testpb.Empty{} + // Block until the server is ready. + if _, err := grpcClient.EmptyCall(ctx, request, grpc.WaitForReady(true)); err != nil { + logger.Fatalf("grpc Client: EmptyCall(_, %v) failed: %v", request, err) + } + logger.Info("grpc Client: empty call succeeded") + + // This sleep prevents the connection from being abruptly disconnected + // when running this binary (along with grpc_server) on GCP dev cluster. + time.Sleep(1 * time.Second) +} diff --git a/interop/alts/server/server.go b/interop/alts/server/server.go new file mode 100644 index 0000000..9db6750 --- /dev/null +++ b/interop/alts/server/server.go @@ -0,0 +1,88 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// This binary can only run on Google Cloud Platform (GCP). +package main + +import ( + "context" + "flag" + "net" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/alts" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/interop" + "google.golang.org/grpc/tap" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +const ( + udsAddrPrefix = "unix:" +) + +var ( + hsAddr = flag.String("alts_handshaker_service_address", "", "ALTS handshaker gRPC service address") + serverAddr = flag.String("server_address", ":8080", "The address on which the server is listening. Only two types of addresses are supported, 'host:port' and 'unix:/path'.") + + logger = grpclog.Component("interop") +) + +func main() { + flag.Parse() + + // If the server address starts with `unix:`, then we have a UDS address. + network := "tcp" + address := *serverAddr + if strings.HasPrefix(address, udsAddrPrefix) { + network = "unix" + address = strings.TrimPrefix(address, udsAddrPrefix) + } + lis, err := net.Listen(network, address) + if err != nil { + logger.Fatalf("gRPC Server: failed to start the server at %v: %v", address, err) + } + opts := alts.DefaultServerOptions() + if *hsAddr != "" { + opts.HandshakerServiceAddress = *hsAddr + } + altsTC := alts.NewServerCreds(opts) + grpcServer := grpc.NewServer(grpc.Creds(altsTC), grpc.InTapHandle(authz)) + testgrpc.RegisterTestServiceServer(grpcServer, interop.NewTestServer()) + grpcServer.Serve(lis) +} + +// authz shows how to access client information at the server side to perform +// application-layer authorization checks. +func authz(ctx context.Context, info *tap.Info) (context.Context, error) { + authInfo, err := alts.AuthInfoFromContext(ctx) + if err != nil { + return nil, err + } + // Access all alts.AuthInfo data: + logger.Infof("authInfo.ApplicationProtocol() = %v", authInfo.ApplicationProtocol()) + logger.Infof("authInfo.RecordProtocol() = %v", authInfo.RecordProtocol()) + logger.Infof("authInfo.SecurityLevel() = %v", authInfo.SecurityLevel()) + logger.Infof("authInfo.PeerServiceAccount() = %v", authInfo.PeerServiceAccount()) + logger.Infof("authInfo.LocalServiceAccount() = %v", authInfo.LocalServiceAccount()) + logger.Infof("authInfo.PeerRPCVersions() = %v", authInfo.PeerRPCVersions()) + logger.Infof("info.FullMethodName = %v", info.FullMethodName) + return ctx, nil +} diff --git a/interop/client/client.go b/interop/client/client.go new file mode 100644 index 0000000..42d4208 --- /dev/null +++ b/interop/client/client.go @@ -0,0 +1,408 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an interop client. +// +// See interop test case descriptions [here]. +// +// [here]: https://github.com/grpc/grpc/blob/master/doc/interop-test-descriptions.md +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "flag" + "log" + "net" + "os" + "strconv" + "strings" + "time" + + "golang.org/x/oauth2" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/alts" + "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/oauth" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/interop" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/testdata" + + _ "google.golang.org/grpc/balancer/grpclb" // Register the grpclb load balancing policy. + _ "google.golang.org/grpc/balancer/rls" // Register the RLS load balancing policy. + "google.golang.org/grpc/xds/googledirectpath" // Register xDS resolver required for c2p directpath. + + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +const ( + googleDefaultCredsName = "google_default_credentials" + computeEngineCredsName = "compute_engine_channel_creds" +) + +var ( + caFile = flag.String("ca_file", "", "The file containing the CA root cert file") + useTLS = flag.Bool("use_tls", false, "Connection uses TLS if true") + useALTS = flag.Bool("use_alts", false, "Connection uses ALTS if true (this option can only be used on GCP)") + customCredentialsType = flag.String("custom_credentials_type", "", "Custom creds to use, excluding TLS or ALTS") + altsHSAddr = flag.String("alts_handshaker_service_address", "", "ALTS handshaker gRPC service address") + testCA = flag.Bool("use_test_ca", false, "Whether to replace platform root CAs with test CA as the CA root") + serviceAccountKeyFile = flag.String("service_account_key_file", "", "Path to service account json key file") + oauthScope = flag.String("oauth_scope", "", "The scope for OAuth2 tokens") + defaultServiceAccount = flag.String("default_service_account", "", "Email of GCE default service account") + googleC2PUniverseDomain = flag.String("google_c2p_universe_domain", "", "Universe domain for google-c2p resolve") + serverHost = flag.String("server_host", "localhost", "The server host name") + serverPort = flag.Int("server_port", 10000, "The server port number") + serviceConfigJSON = flag.String("service_config_json", "", "Disables service config lookups and sets the provided string as the default service config.") + soakIterations = flag.Int("soak_iterations", 10, "The number of iterations to use for the two soak tests: rpc_soak and channel_soak") + soakMaxFailures = flag.Int("soak_max_failures", 0, "The number of iterations in soak tests that are allowed to fail (either due to non-OK status code or exceeding the per-iteration max acceptable latency).") + soakPerIterationMaxAcceptableLatencyMs = flag.Int("soak_per_iteration_max_acceptable_latency_ms", 1000, "The number of milliseconds a single iteration in the two soak tests (rpc_soak and channel_soak) should take.") + soakOverallTimeoutSeconds = flag.Int("soak_overall_timeout_seconds", 10, "The overall number of seconds after which a soak test should stop and fail, if the desired number of iterations have not yet completed.") + soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") + soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") + soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") + soakNumThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") + tlsServerName = flag.String("server_host_override", "", "The server name used to verify the hostname returned by TLS handshake if it is not empty. Otherwise, --server_host is used.") + additionalMetadata = flag.String("additional_metadata", "", "Additional metadata to send in each request, as a semicolon-separated list of key:value pairs.") + testCase = flag.String("test_case", "large_unary", + `Configure different test cases. Valid options are: + empty_unary : empty (zero bytes) request and response; + large_unary : single request and (large) response; + client_streaming : request streaming with single response; + server_streaming : single request with response streaming; + ping_pong : full-duplex streaming; + empty_stream : full-duplex streaming with zero message; + timeout_on_sleeping_server: fullduplex streaming on a sleeping server; + compute_engine_creds: large_unary with compute engine auth; + service_account_creds: large_unary with service account auth; + jwt_token_creds: large_unary with jwt token auth; + per_rpc_creds: large_unary with per rpc token; + oauth2_auth_token: large_unary with oauth2 token auth; + google_default_credentials: large_unary with google default credentials + compute_engine_channel_credentials: large_unary with compute engine creds + cancel_after_begin: cancellation after metadata has been sent but before payloads are sent; + cancel_after_first_response: cancellation after receiving 1st message from the server; + status_code_and_message: status code propagated back to client; + special_status_message: Unicode and whitespace is correctly processed in status message; + custom_metadata: server will echo custom metadata; + unimplemented_method: client attempts to call unimplemented method; + unimplemented_service: client attempts to call unimplemented service; + pick_first_unary: all requests are sent to one server despite multiple servers are resolved; + orca_per_rpc: the client verifies ORCA per-RPC metrics are provided; + orca_oob: the client verifies ORCA out-of-band metrics are provided.`) + + logger = grpclog.Component("interop") +) + +type credsMode uint8 + +const ( + credsNone credsMode = iota + credsTLS + credsALTS + credsGoogleDefaultCreds + credsComputeEngineCreds +) + +// Parses the --additional_metadata flag and returns metadata to send on each RPC, +// formatted as per https://pkg.go.dev/google.golang.org/grpc/metadata#Pairs. +// Allow any character but semicolons in values. If the flag is empty, return a nil map. +func parseAdditionalMetadataFlag() []string { + if len(*additionalMetadata) == 0 { + return nil + } + r := *additionalMetadata + addMd := make([]string, 0) + for len(r) > 0 { + i := strings.Index(r, ":") + if i < 0 { + logger.Fatalf("Error parsing --additional_metadata flag: missing colon separator") + } + addMd = append(addMd, r[:i]) // append key + r = r[i+1:] + i = strings.Index(r, ";") + // append value + if i < 0 { + addMd = append(addMd, r) + break + } + addMd = append(addMd, r[:i]) + r = r[i+1:] + } + return addMd +} + +// createSoakTestConfig creates a shared configuration structure for soak tests. +func createBaseSoakConfig(serverAddr string) interop.SoakTestConfig { + return interop.SoakTestConfig{ + RequestSize: *soakRequestSize, + ResponseSize: *soakResponseSize, + PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, + MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, + OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, + ServerAddr: serverAddr, + NumWorkers: *soakNumThreads, + Iterations: *soakIterations, + MaxFailures: *soakMaxFailures, + } +} + +func main() { + flag.Parse() + logger.Infof("Client running with test case %q", *testCase) + var useGDC bool // use google default creds + var useCEC bool // use compute engine creds + if *customCredentialsType != "" { + switch *customCredentialsType { + case googleDefaultCredsName: + useGDC = true + case computeEngineCredsName: + useCEC = true + default: + logger.Fatalf("If set, custom_credentials_type can only be set to one of %v or %v", + googleDefaultCredsName, computeEngineCredsName) + } + } + if (*useTLS && *useALTS) || (*useTLS && useGDC) || (*useALTS && useGDC) || (*useTLS && useCEC) || (*useALTS && useCEC) { + logger.Fatalf("only one of TLS, ALTS, google default creds, or compute engine creds can be used") + } + + ctx := context.Background() + + var credsChosen credsMode + switch { + case *useTLS: + credsChosen = credsTLS + case *useALTS: + credsChosen = credsALTS + case useGDC: + credsChosen = credsGoogleDefaultCreds + case useCEC: + credsChosen = credsComputeEngineCreds + } + + resolver.SetDefaultScheme("dns") + if len(*googleC2PUniverseDomain) > 0 { + if err := googledirectpath.SetUniverseDomain(*googleC2PUniverseDomain); err != nil { + log.Fatalf("googlec2p.SetUniverseDomain(%s) failed: %v", *googleC2PUniverseDomain, err) + } + } + serverAddr := *serverHost + if *serverPort != 0 { + serverAddr = net.JoinHostPort(*serverHost, strconv.Itoa(*serverPort)) + } + var opts []grpc.DialOption + switch credsChosen { + case credsTLS: + var roots *x509.CertPool + if *testCA { + if *caFile == "" { + *caFile = testdata.Path("ca.pem") + } + b, err := os.ReadFile(*caFile) + if err != nil { + logger.Fatalf("Failed to read root certificate file %q: %v", *caFile, err) + } + roots = x509.NewCertPool() + if !roots.AppendCertsFromPEM(b) { + logger.Fatalf("Failed to append certificates: %s", string(b)) + } + } + var creds credentials.TransportCredentials + if *tlsServerName != "" { + creds = credentials.NewClientTLSFromCert(roots, *tlsServerName) + } else { + creds = credentials.NewTLS(&tls.Config{RootCAs: roots}) + } + opts = append(opts, grpc.WithTransportCredentials(creds)) + case credsALTS: + altsOpts := alts.DefaultClientOptions() + if *altsHSAddr != "" { + altsOpts.HandshakerServiceAddress = *altsHSAddr + } + altsTC := alts.NewClientCreds(altsOpts) + opts = append(opts, grpc.WithTransportCredentials(altsTC)) + case credsGoogleDefaultCreds: + opts = append(opts, grpc.WithCredentialsBundle(google.NewDefaultCredentials())) + case credsComputeEngineCreds: + opts = append(opts, grpc.WithCredentialsBundle(google.NewComputeEngineCredentials())) + case credsNone: + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + default: + logger.Fatal("Invalid creds") + } + if credsChosen == credsTLS { + if *testCase == "compute_engine_creds" { + opts = append(opts, grpc.WithPerRPCCredentials(oauth.NewComputeEngine())) + } else if *testCase == "service_account_creds" { + jwtCreds, err := oauth.NewServiceAccountFromFile(*serviceAccountKeyFile, *oauthScope) + if err != nil { + logger.Fatalf("Failed to create JWT credentials: %v", err) + } + opts = append(opts, grpc.WithPerRPCCredentials(jwtCreds)) + } else if *testCase == "jwt_token_creds" { + jwtCreds, err := oauth.NewJWTAccessFromFile(*serviceAccountKeyFile) + if err != nil { + logger.Fatalf("Failed to create JWT credentials: %v", err) + } + opts = append(opts, grpc.WithPerRPCCredentials(jwtCreds)) + } else if *testCase == "oauth2_auth_token" { + opts = append(opts, grpc.WithPerRPCCredentials(oauth.TokenSource{TokenSource: oauth2.StaticTokenSource(interop.GetToken(ctx, *serviceAccountKeyFile, *oauthScope))})) + } + } + if len(*serviceConfigJSON) > 0 { + opts = append(opts, grpc.WithDisableServiceConfig(), grpc.WithDefaultServiceConfig(*serviceConfigJSON)) + } + if addMd := parseAdditionalMetadataFlag(); addMd != nil { + unaryAddMd := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + ctx = metadata.AppendToOutgoingContext(ctx, addMd...) + return invoker(ctx, method, req, reply, cc, opts...) + } + streamingAddMd := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + ctx = metadata.AppendToOutgoingContext(ctx, addMd...) + return streamer(ctx, desc, cc, method, opts...) + } + opts = append(opts, grpc.WithUnaryInterceptor(unaryAddMd), grpc.WithStreamInterceptor(streamingAddMd)) + } + conn, err := grpc.NewClient(serverAddr, opts...) + if err != nil { + logger.Fatalf("grpc.NewClient(%q) = %v", serverAddr, err) + } + defer conn.Close() + tc := testgrpc.NewTestServiceClient(conn) + ctxWithDeadline, cancel := context.WithTimeout(ctx, time.Duration(*soakOverallTimeoutSeconds)*time.Second) + defer cancel() + switch *testCase { + case "empty_unary": + interop.DoEmptyUnaryCall(ctx, tc) + logger.Infoln("EmptyUnaryCall done") + case "large_unary": + interop.DoLargeUnaryCall(ctx, tc) + logger.Infoln("LargeUnaryCall done") + case "client_streaming": + interop.DoClientStreaming(ctx, tc) + logger.Infoln("ClientStreaming done") + case "server_streaming": + interop.DoServerStreaming(ctx, tc) + logger.Infoln("ServerStreaming done") + case "ping_pong": + interop.DoPingPong(ctx, tc) + logger.Infoln("Pingpong done") + case "empty_stream": + interop.DoEmptyStream(ctx, tc) + logger.Infoln("Emptystream done") + case "timeout_on_sleeping_server": + interop.DoTimeoutOnSleepingServer(ctx, tc) + logger.Infoln("TimeoutOnSleepingServer done") + case "compute_engine_creds": + if credsChosen != credsTLS { + logger.Fatalf("TLS credentials need to be set for compute_engine_creds test case.") + } + interop.DoComputeEngineCreds(ctx, tc, *defaultServiceAccount, *oauthScope) + logger.Infoln("ComputeEngineCreds done") + case "service_account_creds": + if credsChosen != credsTLS { + logger.Fatalf("TLS credentials need to be set for service_account_creds test case.") + } + interop.DoServiceAccountCreds(ctx, tc, *serviceAccountKeyFile, *oauthScope) + logger.Infoln("ServiceAccountCreds done") + case "jwt_token_creds": + if credsChosen != credsTLS { + logger.Fatalf("TLS credentials need to be set for jwt_token_creds test case.") + } + interop.DoJWTTokenCreds(ctx, tc, *serviceAccountKeyFile) + logger.Infoln("JWTtokenCreds done") + case "per_rpc_creds": + if credsChosen != credsTLS { + logger.Fatalf("TLS credentials need to be set for per_rpc_creds test case.") + } + interop.DoPerRPCCreds(ctx, tc, *serviceAccountKeyFile, *oauthScope) + logger.Infoln("PerRPCCreds done") + case "oauth2_auth_token": + if credsChosen != credsTLS { + logger.Fatalf("TLS credentials need to be set for oauth2_auth_token test case.") + } + interop.DoOauth2TokenCreds(ctx, tc, *serviceAccountKeyFile, *oauthScope) + logger.Infoln("Oauth2TokenCreds done") + case "google_default_credentials": + if credsChosen != credsGoogleDefaultCreds { + logger.Fatalf("GoogleDefaultCredentials need to be set for google_default_credentials test case.") + } + interop.DoGoogleDefaultCredentials(ctx, tc, *defaultServiceAccount) + logger.Infoln("GoogleDefaultCredentials done") + case "compute_engine_channel_credentials": + if credsChosen != credsComputeEngineCreds { + logger.Fatalf("ComputeEngineCreds need to be set for compute_engine_channel_credentials test case.") + } + interop.DoComputeEngineChannelCredentials(ctx, tc, *defaultServiceAccount) + logger.Infoln("ComputeEngineChannelCredentials done") + case "cancel_after_begin": + interop.DoCancelAfterBegin(ctx, tc) + logger.Infoln("CancelAfterBegin done") + case "cancel_after_first_response": + interop.DoCancelAfterFirstResponse(ctx, tc) + logger.Infoln("CancelAfterFirstResponse done") + case "status_code_and_message": + interop.DoStatusCodeAndMessage(ctx, tc) + logger.Infoln("StatusCodeAndMessage done") + case "special_status_message": + interop.DoSpecialStatusMessage(ctx, tc) + logger.Infoln("SpecialStatusMessage done") + case "custom_metadata": + interop.DoCustomMetadata(ctx, tc) + logger.Infoln("CustomMetadata done") + case "unimplemented_method": + interop.DoUnimplementedMethod(ctx, conn) + logger.Infoln("UnimplementedMethod done") + case "unimplemented_service": + interop.DoUnimplementedService(ctx, testgrpc.NewUnimplementedServiceClient(conn)) + logger.Infoln("UnimplementedService done") + case "pick_first_unary": + interop.DoPickFirstUnary(ctx, tc) + logger.Infoln("PickFirstUnary done") + case "rpc_soak": + rpcSoakConfig := createBaseSoakConfig(serverAddr) + rpcSoakConfig.ChannelForTest = func() (*grpc.ClientConn, func()) { return conn, func() {} } + interop.DoSoakTest(ctxWithDeadline, rpcSoakConfig) + logger.Infoln("RpcSoak done") + case "channel_soak": + channelSoakConfig := createBaseSoakConfig(serverAddr) + channelSoakConfig.ChannelForTest = func() (*grpc.ClientConn, func()) { + cc, err := grpc.NewClient(serverAddr, opts...) + if err != nil { + log.Fatalf("Failed to create shared channel: %v", err) + } + return cc, func() { cc.Close() } + } + interop.DoSoakTest(ctxWithDeadline, channelSoakConfig) + logger.Infoln("ChannelSoak done") + case "orca_per_rpc": + interop.DoORCAPerRPCTest(ctx, tc) + logger.Infoln("ORCAPerRPC done") + case "orca_oob": + interop.DoORCAOOBTest(ctx, tc) + logger.Infoln("ORCAOOB done") + default: + logger.Fatal("Unsupported test case: ", *testCase) + } +} diff --git a/interop/fake_grpclb/fake_grpclb.go b/interop/fake_grpclb/fake_grpclb.go new file mode 100644 index 0000000..00ae00a --- /dev/null +++ b/interop/fake_grpclb/fake_grpclb.go @@ -0,0 +1,83 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// This file is for testing only. Runs a fake grpclb balancer server. +// The name of the service to load balance for and the addresses +// of that service are provided by command line flags. +package main + +import ( + "flag" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/alts" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/testutils/fakegrpclb" + "google.golang.org/grpc/testdata" +) + +var ( + port = flag.Int("port", 10000, "Port to listen on.") + backendAddrs = flag.String("backend_addrs", "", "Comma separated list of backend IP/port addresses.") + useALTS = flag.Bool("use_alts", false, "Listen on ALTS credentials.") + useTLS = flag.Bool("use_tls", false, "Listen on TLS credentials, using a test certificate.") + shortStream = flag.Bool("short_stream", false, "End the balancer stream immediately after sending the first server list.") + serviceName = flag.String("service_name", "UNSET", "Name of the service being load balanced for.") + + logger = grpclog.Component("interop") +) + +func main() { + flag.Parse() + + var opts []grpc.ServerOption + if *useTLS { + certFile := testdata.Path("server1.pem") + keyFile := testdata.Path("server1.key") + creds, err := credentials.NewServerTLSFromFile(certFile, keyFile) + if err != nil { + logger.Fatalf("Failed to generate credentials: %v", err) + } + opts = append(opts, grpc.Creds(creds)) + } else if *useALTS { + altsOpts := alts.DefaultServerOptions() + altsTC := alts.NewServerCreds(altsOpts) + opts = append(opts, grpc.Creds(altsTC)) + } + + rawBackendAddrs := strings.Split(*backendAddrs, ",") + server, err := fakegrpclb.NewServer(fakegrpclb.ServerParams{ + ListenPort: *port, + ServerOptions: opts, + LoadBalancedServiceName: *serviceName, + LoadBalancedServicePort: 443, // TODO: make this configurable? + BackendAddresses: rawBackendAddrs, + ShortStream: *shortStream, + }) + if err != nil { + logger.Fatalf("Failed to create balancer server: %v", err) + } + + // Serve() starts serving and blocks until Stop() is called. We don't need to + // call Stop() here since we want the server to run until we are killed. + if err := server.Serve(); err != nil { + logger.Fatalf("Failed to start balancer server: %v", err) + } +} diff --git a/interop/grpc_testing/benchmark_service.pb.go b/interop/grpc_testing/benchmark_service.pb.go new file mode 100644 index 0000000..53d453a --- /dev/null +++ b/interop/grpc_testing/benchmark_service.pb.go @@ -0,0 +1,97 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/testing/benchmark_service.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var File_grpc_testing_benchmark_service_proto protoreflect.FileDescriptor + +const file_grpc_testing_benchmark_service_proto_rawDesc = "" + + "\n" + + "$grpc/testing/benchmark_service.proto\x12\fgrpc.testing\x1a\x1bgrpc/testing/messages.proto2\xa6\x03\n" + + "\x10BenchmarkService\x12F\n" + + "\tUnaryCall\x12\x1b.grpc.testing.SimpleRequest\x1a\x1c.grpc.testing.SimpleResponse\x12N\n" + + "\rStreamingCall\x12\x1b.grpc.testing.SimpleRequest\x1a\x1c.grpc.testing.SimpleResponse(\x010\x01\x12R\n" + + "\x13StreamingFromClient\x12\x1b.grpc.testing.SimpleRequest\x1a\x1c.grpc.testing.SimpleResponse(\x01\x12R\n" + + "\x13StreamingFromServer\x12\x1b.grpc.testing.SimpleRequest\x1a\x1c.grpc.testing.SimpleResponse0\x01\x12R\n" + + "\x11StreamingBothWays\x12\x1b.grpc.testing.SimpleRequest\x1a\x1c.grpc.testing.SimpleResponse(\x010\x01B*\n" + + "\x0fio.grpc.testingB\x15BenchmarkServiceProtoP\x01b\x06proto3" + +var file_grpc_testing_benchmark_service_proto_goTypes = []any{ + (*SimpleRequest)(nil), // 0: grpc.testing.SimpleRequest + (*SimpleResponse)(nil), // 1: grpc.testing.SimpleResponse +} +var file_grpc_testing_benchmark_service_proto_depIdxs = []int32{ + 0, // 0: grpc.testing.BenchmarkService.UnaryCall:input_type -> grpc.testing.SimpleRequest + 0, // 1: grpc.testing.BenchmarkService.StreamingCall:input_type -> grpc.testing.SimpleRequest + 0, // 2: grpc.testing.BenchmarkService.StreamingFromClient:input_type -> grpc.testing.SimpleRequest + 0, // 3: grpc.testing.BenchmarkService.StreamingFromServer:input_type -> grpc.testing.SimpleRequest + 0, // 4: grpc.testing.BenchmarkService.StreamingBothWays:input_type -> grpc.testing.SimpleRequest + 1, // 5: grpc.testing.BenchmarkService.UnaryCall:output_type -> grpc.testing.SimpleResponse + 1, // 6: grpc.testing.BenchmarkService.StreamingCall:output_type -> grpc.testing.SimpleResponse + 1, // 7: grpc.testing.BenchmarkService.StreamingFromClient:output_type -> grpc.testing.SimpleResponse + 1, // 8: grpc.testing.BenchmarkService.StreamingFromServer:output_type -> grpc.testing.SimpleResponse + 1, // 9: grpc.testing.BenchmarkService.StreamingBothWays:output_type -> grpc.testing.SimpleResponse + 5, // [5:10] is the sub-list for method output_type + 0, // [0:5] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_grpc_testing_benchmark_service_proto_init() } +func file_grpc_testing_benchmark_service_proto_init() { + if File_grpc_testing_benchmark_service_proto != nil { + return + } + file_grpc_testing_messages_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_testing_benchmark_service_proto_rawDesc), len(file_grpc_testing_benchmark_service_proto_rawDesc)), + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_testing_benchmark_service_proto_goTypes, + DependencyIndexes: file_grpc_testing_benchmark_service_proto_depIdxs, + }.Build() + File_grpc_testing_benchmark_service_proto = out.File + file_grpc_testing_benchmark_service_proto_goTypes = nil + file_grpc_testing_benchmark_service_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/benchmark_service_grpc.pb.go b/interop/grpc_testing/benchmark_service_grpc.pb.go new file mode 100644 index 0000000..181789d --- /dev/null +++ b/interop/grpc_testing/benchmark_service_grpc.pb.go @@ -0,0 +1,297 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: grpc/testing/benchmark_service.proto + +package grpc_testing + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + BenchmarkService_UnaryCall_FullMethodName = "/grpc.testing.BenchmarkService/UnaryCall" + BenchmarkService_StreamingCall_FullMethodName = "/grpc.testing.BenchmarkService/StreamingCall" + BenchmarkService_StreamingFromClient_FullMethodName = "/grpc.testing.BenchmarkService/StreamingFromClient" + BenchmarkService_StreamingFromServer_FullMethodName = "/grpc.testing.BenchmarkService/StreamingFromServer" + BenchmarkService_StreamingBothWays_FullMethodName = "/grpc.testing.BenchmarkService/StreamingBothWays" +) + +// BenchmarkServiceClient is the client API for BenchmarkService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type BenchmarkServiceClient interface { + // One request followed by one response. + // The server returns the client payload as-is. + UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) + // Repeated sequence of one request followed by one response. + // Should be called streaming ping-pong + // The server returns the client payload as-is on each response + StreamingCall(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SimpleRequest, SimpleResponse], error) + // Single-sided unbounded streaming from client to server + // The server returns the client payload as-is once the client does WritesDone + StreamingFromClient(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[SimpleRequest, SimpleResponse], error) + // Single-sided unbounded streaming from server to client + // The server repeatedly returns the client payload as-is + StreamingFromServer(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SimpleResponse], error) + // Two-sided unbounded streaming between server to client + // Both sides send the content of their own choice to the other + StreamingBothWays(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SimpleRequest, SimpleResponse], error) +} + +type benchmarkServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewBenchmarkServiceClient(cc grpc.ClientConnInterface) BenchmarkServiceClient { + return &benchmarkServiceClient{cc} +} + +func (c *benchmarkServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SimpleResponse) + err := c.cc.Invoke(ctx, BenchmarkService_UnaryCall_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *benchmarkServiceClient) StreamingCall(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SimpleRequest, SimpleResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[0], BenchmarkService_StreamingCall_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[SimpleRequest, SimpleResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BenchmarkService_StreamingCallClient = grpc.BidiStreamingClient[SimpleRequest, SimpleResponse] + +func (c *benchmarkServiceClient) StreamingFromClient(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[SimpleRequest, SimpleResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[1], BenchmarkService_StreamingFromClient_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[SimpleRequest, SimpleResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BenchmarkService_StreamingFromClientClient = grpc.ClientStreamingClient[SimpleRequest, SimpleResponse] + +func (c *benchmarkServiceClient) StreamingFromServer(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SimpleResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[2], BenchmarkService_StreamingFromServer_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[SimpleRequest, SimpleResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BenchmarkService_StreamingFromServerClient = grpc.ServerStreamingClient[SimpleResponse] + +func (c *benchmarkServiceClient) StreamingBothWays(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SimpleRequest, SimpleResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[3], BenchmarkService_StreamingBothWays_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[SimpleRequest, SimpleResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BenchmarkService_StreamingBothWaysClient = grpc.BidiStreamingClient[SimpleRequest, SimpleResponse] + +// BenchmarkServiceServer is the server API for BenchmarkService service. +// All implementations must embed UnimplementedBenchmarkServiceServer +// for forward compatibility. +type BenchmarkServiceServer interface { + // One request followed by one response. + // The server returns the client payload as-is. + UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) + // Repeated sequence of one request followed by one response. + // Should be called streaming ping-pong + // The server returns the client payload as-is on each response + StreamingCall(grpc.BidiStreamingServer[SimpleRequest, SimpleResponse]) error + // Single-sided unbounded streaming from client to server + // The server returns the client payload as-is once the client does WritesDone + StreamingFromClient(grpc.ClientStreamingServer[SimpleRequest, SimpleResponse]) error + // Single-sided unbounded streaming from server to client + // The server repeatedly returns the client payload as-is + StreamingFromServer(*SimpleRequest, grpc.ServerStreamingServer[SimpleResponse]) error + // Two-sided unbounded streaming between server to client + // Both sides send the content of their own choice to the other + StreamingBothWays(grpc.BidiStreamingServer[SimpleRequest, SimpleResponse]) error + mustEmbedUnimplementedBenchmarkServiceServer() +} + +// UnimplementedBenchmarkServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedBenchmarkServiceServer struct{} + +func (UnimplementedBenchmarkServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) { + return nil, status.Error(codes.Unimplemented, "method UnaryCall not implemented") +} +func (UnimplementedBenchmarkServiceServer) StreamingCall(grpc.BidiStreamingServer[SimpleRequest, SimpleResponse]) error { + return status.Error(codes.Unimplemented, "method StreamingCall not implemented") +} +func (UnimplementedBenchmarkServiceServer) StreamingFromClient(grpc.ClientStreamingServer[SimpleRequest, SimpleResponse]) error { + return status.Error(codes.Unimplemented, "method StreamingFromClient not implemented") +} +func (UnimplementedBenchmarkServiceServer) StreamingFromServer(*SimpleRequest, grpc.ServerStreamingServer[SimpleResponse]) error { + return status.Error(codes.Unimplemented, "method StreamingFromServer not implemented") +} +func (UnimplementedBenchmarkServiceServer) StreamingBothWays(grpc.BidiStreamingServer[SimpleRequest, SimpleResponse]) error { + return status.Error(codes.Unimplemented, "method StreamingBothWays not implemented") +} +func (UnimplementedBenchmarkServiceServer) mustEmbedUnimplementedBenchmarkServiceServer() {} +func (UnimplementedBenchmarkServiceServer) testEmbeddedByValue() {} + +// UnsafeBenchmarkServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to BenchmarkServiceServer will +// result in compilation errors. +type UnsafeBenchmarkServiceServer interface { + mustEmbedUnimplementedBenchmarkServiceServer() +} + +func RegisterBenchmarkServiceServer(s grpc.ServiceRegistrar, srv BenchmarkServiceServer) { + // If the following call panics, it indicates UnimplementedBenchmarkServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&BenchmarkService_ServiceDesc, srv) +} + +func _BenchmarkService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SimpleRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BenchmarkServiceServer).UnaryCall(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BenchmarkService_UnaryCall_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BenchmarkServiceServer).UnaryCall(ctx, req.(*SimpleRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _BenchmarkService_StreamingCall_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(BenchmarkServiceServer).StreamingCall(&grpc.GenericServerStream[SimpleRequest, SimpleResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BenchmarkService_StreamingCallServer = grpc.BidiStreamingServer[SimpleRequest, SimpleResponse] + +func _BenchmarkService_StreamingFromClient_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(BenchmarkServiceServer).StreamingFromClient(&grpc.GenericServerStream[SimpleRequest, SimpleResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BenchmarkService_StreamingFromClientServer = grpc.ClientStreamingServer[SimpleRequest, SimpleResponse] + +func _BenchmarkService_StreamingFromServer_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(SimpleRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(BenchmarkServiceServer).StreamingFromServer(m, &grpc.GenericServerStream[SimpleRequest, SimpleResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BenchmarkService_StreamingFromServerServer = grpc.ServerStreamingServer[SimpleResponse] + +func _BenchmarkService_StreamingBothWays_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(BenchmarkServiceServer).StreamingBothWays(&grpc.GenericServerStream[SimpleRequest, SimpleResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BenchmarkService_StreamingBothWaysServer = grpc.BidiStreamingServer[SimpleRequest, SimpleResponse] + +// BenchmarkService_ServiceDesc is the grpc.ServiceDesc for BenchmarkService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var BenchmarkService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.BenchmarkService", + HandlerType: (*BenchmarkServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "UnaryCall", + Handler: _BenchmarkService_UnaryCall_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "StreamingCall", + Handler: _BenchmarkService_StreamingCall_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "StreamingFromClient", + Handler: _BenchmarkService_StreamingFromClient_Handler, + ClientStreams: true, + }, + { + StreamName: "StreamingFromServer", + Handler: _BenchmarkService_StreamingFromServer_Handler, + ServerStreams: true, + }, + { + StreamName: "StreamingBothWays", + Handler: _BenchmarkService_StreamingBothWays_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/testing/benchmark_service.proto", +} diff --git a/interop/grpc_testing/control.pb.go b/interop/grpc_testing/control.pb.go new file mode 100644 index 0000000..280bfce --- /dev/null +++ b/interop/grpc_testing/control.pb.go @@ -0,0 +1,2069 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/testing/control.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ClientType int32 + +const ( + // Many languages support a basic distinction between using + // sync or async client, and this allows the specification + ClientType_SYNC_CLIENT ClientType = 0 + ClientType_ASYNC_CLIENT ClientType = 1 + ClientType_OTHER_CLIENT ClientType = 2 // used for some language-specific variants + ClientType_CALLBACK_CLIENT ClientType = 3 +) + +// Enum value maps for ClientType. +var ( + ClientType_name = map[int32]string{ + 0: "SYNC_CLIENT", + 1: "ASYNC_CLIENT", + 2: "OTHER_CLIENT", + 3: "CALLBACK_CLIENT", + } + ClientType_value = map[string]int32{ + "SYNC_CLIENT": 0, + "ASYNC_CLIENT": 1, + "OTHER_CLIENT": 2, + "CALLBACK_CLIENT": 3, + } +) + +func (x ClientType) Enum() *ClientType { + p := new(ClientType) + *p = x + return p +} + +func (x ClientType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ClientType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_control_proto_enumTypes[0].Descriptor() +} + +func (ClientType) Type() protoreflect.EnumType { + return &file_grpc_testing_control_proto_enumTypes[0] +} + +func (x ClientType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ClientType.Descriptor instead. +func (ClientType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{0} +} + +type ServerType int32 + +const ( + ServerType_SYNC_SERVER ServerType = 0 + ServerType_ASYNC_SERVER ServerType = 1 + ServerType_ASYNC_GENERIC_SERVER ServerType = 2 + ServerType_OTHER_SERVER ServerType = 3 // used for some language-specific variants + ServerType_CALLBACK_SERVER ServerType = 4 +) + +// Enum value maps for ServerType. +var ( + ServerType_name = map[int32]string{ + 0: "SYNC_SERVER", + 1: "ASYNC_SERVER", + 2: "ASYNC_GENERIC_SERVER", + 3: "OTHER_SERVER", + 4: "CALLBACK_SERVER", + } + ServerType_value = map[string]int32{ + "SYNC_SERVER": 0, + "ASYNC_SERVER": 1, + "ASYNC_GENERIC_SERVER": 2, + "OTHER_SERVER": 3, + "CALLBACK_SERVER": 4, + } +) + +func (x ServerType) Enum() *ServerType { + p := new(ServerType) + *p = x + return p +} + +func (x ServerType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ServerType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_control_proto_enumTypes[1].Descriptor() +} + +func (ServerType) Type() protoreflect.EnumType { + return &file_grpc_testing_control_proto_enumTypes[1] +} + +func (x ServerType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ServerType.Descriptor instead. +func (ServerType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{1} +} + +type RpcType int32 + +const ( + RpcType_UNARY RpcType = 0 + RpcType_STREAMING RpcType = 1 + RpcType_STREAMING_FROM_CLIENT RpcType = 2 + RpcType_STREAMING_FROM_SERVER RpcType = 3 + RpcType_STREAMING_BOTH_WAYS RpcType = 4 +) + +// Enum value maps for RpcType. +var ( + RpcType_name = map[int32]string{ + 0: "UNARY", + 1: "STREAMING", + 2: "STREAMING_FROM_CLIENT", + 3: "STREAMING_FROM_SERVER", + 4: "STREAMING_BOTH_WAYS", + } + RpcType_value = map[string]int32{ + "UNARY": 0, + "STREAMING": 1, + "STREAMING_FROM_CLIENT": 2, + "STREAMING_FROM_SERVER": 3, + "STREAMING_BOTH_WAYS": 4, + } +) + +func (x RpcType) Enum() *RpcType { + p := new(RpcType) + *p = x + return p +} + +func (x RpcType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (RpcType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_control_proto_enumTypes[2].Descriptor() +} + +func (RpcType) Type() protoreflect.EnumType { + return &file_grpc_testing_control_proto_enumTypes[2] +} + +func (x RpcType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use RpcType.Descriptor instead. +func (RpcType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{2} +} + +// Parameters of poisson process distribution, which is a good representation +// of activity coming in from independent identical stationary sources. +type PoissonParams struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The rate of arrivals (a.k.a. lambda parameter of the exp distribution). + OfferedLoad float64 `protobuf:"fixed64,1,opt,name=offered_load,json=offeredLoad,proto3" json:"offered_load,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PoissonParams) Reset() { + *x = PoissonParams{} + mi := &file_grpc_testing_control_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PoissonParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PoissonParams) ProtoMessage() {} + +func (x *PoissonParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PoissonParams.ProtoReflect.Descriptor instead. +func (*PoissonParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{0} +} + +func (x *PoissonParams) GetOfferedLoad() float64 { + if x != nil { + return x.OfferedLoad + } + return 0 +} + +// Once an RPC finishes, immediately start a new one. +// No configuration parameters needed. +type ClosedLoopParams struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClosedLoopParams) Reset() { + *x = ClosedLoopParams{} + mi := &file_grpc_testing_control_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClosedLoopParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClosedLoopParams) ProtoMessage() {} + +func (x *ClosedLoopParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClosedLoopParams.ProtoReflect.Descriptor instead. +func (*ClosedLoopParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{1} +} + +type LoadParams struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Load: + // + // *LoadParams_ClosedLoop + // *LoadParams_Poisson + Load isLoadParams_Load `protobuf_oneof:"load"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoadParams) Reset() { + *x = LoadParams{} + mi := &file_grpc_testing_control_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoadParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadParams) ProtoMessage() {} + +func (x *LoadParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadParams.ProtoReflect.Descriptor instead. +func (*LoadParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{2} +} + +func (x *LoadParams) GetLoad() isLoadParams_Load { + if x != nil { + return x.Load + } + return nil +} + +func (x *LoadParams) GetClosedLoop() *ClosedLoopParams { + if x != nil { + if x, ok := x.Load.(*LoadParams_ClosedLoop); ok { + return x.ClosedLoop + } + } + return nil +} + +func (x *LoadParams) GetPoisson() *PoissonParams { + if x != nil { + if x, ok := x.Load.(*LoadParams_Poisson); ok { + return x.Poisson + } + } + return nil +} + +type isLoadParams_Load interface { + isLoadParams_Load() +} + +type LoadParams_ClosedLoop struct { + ClosedLoop *ClosedLoopParams `protobuf:"bytes,1,opt,name=closed_loop,json=closedLoop,proto3,oneof"` +} + +type LoadParams_Poisson struct { + Poisson *PoissonParams `protobuf:"bytes,2,opt,name=poisson,proto3,oneof"` +} + +func (*LoadParams_ClosedLoop) isLoadParams_Load() {} + +func (*LoadParams_Poisson) isLoadParams_Load() {} + +// presence of SecurityParams implies use of TLS +type SecurityParams struct { + state protoimpl.MessageState `protogen:"open.v1"` + UseTestCa bool `protobuf:"varint,1,opt,name=use_test_ca,json=useTestCa,proto3" json:"use_test_ca,omitempty"` + ServerHostOverride string `protobuf:"bytes,2,opt,name=server_host_override,json=serverHostOverride,proto3" json:"server_host_override,omitempty"` + CredType string `protobuf:"bytes,3,opt,name=cred_type,json=credType,proto3" json:"cred_type,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SecurityParams) Reset() { + *x = SecurityParams{} + mi := &file_grpc_testing_control_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SecurityParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SecurityParams) ProtoMessage() {} + +func (x *SecurityParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SecurityParams.ProtoReflect.Descriptor instead. +func (*SecurityParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{3} +} + +func (x *SecurityParams) GetUseTestCa() bool { + if x != nil { + return x.UseTestCa + } + return false +} + +func (x *SecurityParams) GetServerHostOverride() string { + if x != nil { + return x.ServerHostOverride + } + return "" +} + +func (x *SecurityParams) GetCredType() string { + if x != nil { + return x.CredType + } + return "" +} + +type ChannelArg struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Types that are valid to be assigned to Value: + // + // *ChannelArg_StrValue + // *ChannelArg_IntValue + Value isChannelArg_Value `protobuf_oneof:"value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ChannelArg) Reset() { + *x = ChannelArg{} + mi := &file_grpc_testing_control_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChannelArg) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChannelArg) ProtoMessage() {} + +func (x *ChannelArg) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChannelArg.ProtoReflect.Descriptor instead. +func (*ChannelArg) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{4} +} + +func (x *ChannelArg) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ChannelArg) GetValue() isChannelArg_Value { + if x != nil { + return x.Value + } + return nil +} + +func (x *ChannelArg) GetStrValue() string { + if x != nil { + if x, ok := x.Value.(*ChannelArg_StrValue); ok { + return x.StrValue + } + } + return "" +} + +func (x *ChannelArg) GetIntValue() int32 { + if x != nil { + if x, ok := x.Value.(*ChannelArg_IntValue); ok { + return x.IntValue + } + } + return 0 +} + +type isChannelArg_Value interface { + isChannelArg_Value() +} + +type ChannelArg_StrValue struct { + StrValue string `protobuf:"bytes,2,opt,name=str_value,json=strValue,proto3,oneof"` +} + +type ChannelArg_IntValue struct { + IntValue int32 `protobuf:"varint,3,opt,name=int_value,json=intValue,proto3,oneof"` +} + +func (*ChannelArg_StrValue) isChannelArg_Value() {} + +func (*ChannelArg_IntValue) isChannelArg_Value() {} + +type ClientConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + // List of targets to connect to. At least one target needs to be specified. + ServerTargets []string `protobuf:"bytes,1,rep,name=server_targets,json=serverTargets,proto3" json:"server_targets,omitempty"` + ClientType ClientType `protobuf:"varint,2,opt,name=client_type,json=clientType,proto3,enum=grpc.testing.ClientType" json:"client_type,omitempty"` + SecurityParams *SecurityParams `protobuf:"bytes,3,opt,name=security_params,json=securityParams,proto3" json:"security_params,omitempty"` + // How many concurrent RPCs to start for each channel. + // For synchronous client, use a separate thread for each outstanding RPC. + OutstandingRpcsPerChannel int32 `protobuf:"varint,4,opt,name=outstanding_rpcs_per_channel,json=outstandingRpcsPerChannel,proto3" json:"outstanding_rpcs_per_channel,omitempty"` + // Number of independent client channels to create. + // i-th channel will connect to server_target[i % server_targets.size()] + ClientChannels int32 `protobuf:"varint,5,opt,name=client_channels,json=clientChannels,proto3" json:"client_channels,omitempty"` + // Only for async client. Number of threads to use to start/manage RPCs. + AsyncClientThreads int32 `protobuf:"varint,7,opt,name=async_client_threads,json=asyncClientThreads,proto3" json:"async_client_threads,omitempty"` + RpcType RpcType `protobuf:"varint,8,opt,name=rpc_type,json=rpcType,proto3,enum=grpc.testing.RpcType" json:"rpc_type,omitempty"` + // The requested load for the entire client (aggregated over all the threads). + LoadParams *LoadParams `protobuf:"bytes,10,opt,name=load_params,json=loadParams,proto3" json:"load_params,omitempty"` + PayloadConfig *PayloadConfig `protobuf:"bytes,11,opt,name=payload_config,json=payloadConfig,proto3" json:"payload_config,omitempty"` + HistogramParams *HistogramParams `protobuf:"bytes,12,opt,name=histogram_params,json=histogramParams,proto3" json:"histogram_params,omitempty"` + // Specify the cores we should run the client on, if desired + CoreList []int32 `protobuf:"varint,13,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"` + CoreLimit int32 `protobuf:"varint,14,opt,name=core_limit,json=coreLimit,proto3" json:"core_limit,omitempty"` + // If we use an OTHER_CLIENT client_type, this string gives more detail + OtherClientApi string `protobuf:"bytes,15,opt,name=other_client_api,json=otherClientApi,proto3" json:"other_client_api,omitempty"` + ChannelArgs []*ChannelArg `protobuf:"bytes,16,rep,name=channel_args,json=channelArgs,proto3" json:"channel_args,omitempty"` + // Number of threads that share each completion queue + ThreadsPerCq int32 `protobuf:"varint,17,opt,name=threads_per_cq,json=threadsPerCq,proto3" json:"threads_per_cq,omitempty"` + // Number of messages on a stream before it gets finished/restarted + MessagesPerStream int32 `protobuf:"varint,18,opt,name=messages_per_stream,json=messagesPerStream,proto3" json:"messages_per_stream,omitempty"` + // Use coalescing API when possible. + UseCoalesceApi bool `protobuf:"varint,19,opt,name=use_coalesce_api,json=useCoalesceApi,proto3" json:"use_coalesce_api,omitempty"` + // If 0, disabled. Else, specifies the period between gathering latency + // medians in milliseconds. + MedianLatencyCollectionIntervalMillis int32 `protobuf:"varint,20,opt,name=median_latency_collection_interval_millis,json=medianLatencyCollectionIntervalMillis,proto3" json:"median_latency_collection_interval_millis,omitempty"` + // Number of client processes. 0 indicates no restriction. + ClientProcesses int32 `protobuf:"varint,21,opt,name=client_processes,json=clientProcesses,proto3" json:"client_processes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClientConfig) Reset() { + *x = ClientConfig{} + mi := &file_grpc_testing_control_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClientConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientConfig) ProtoMessage() {} + +func (x *ClientConfig) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientConfig.ProtoReflect.Descriptor instead. +func (*ClientConfig) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{5} +} + +func (x *ClientConfig) GetServerTargets() []string { + if x != nil { + return x.ServerTargets + } + return nil +} + +func (x *ClientConfig) GetClientType() ClientType { + if x != nil { + return x.ClientType + } + return ClientType_SYNC_CLIENT +} + +func (x *ClientConfig) GetSecurityParams() *SecurityParams { + if x != nil { + return x.SecurityParams + } + return nil +} + +func (x *ClientConfig) GetOutstandingRpcsPerChannel() int32 { + if x != nil { + return x.OutstandingRpcsPerChannel + } + return 0 +} + +func (x *ClientConfig) GetClientChannels() int32 { + if x != nil { + return x.ClientChannels + } + return 0 +} + +func (x *ClientConfig) GetAsyncClientThreads() int32 { + if x != nil { + return x.AsyncClientThreads + } + return 0 +} + +func (x *ClientConfig) GetRpcType() RpcType { + if x != nil { + return x.RpcType + } + return RpcType_UNARY +} + +func (x *ClientConfig) GetLoadParams() *LoadParams { + if x != nil { + return x.LoadParams + } + return nil +} + +func (x *ClientConfig) GetPayloadConfig() *PayloadConfig { + if x != nil { + return x.PayloadConfig + } + return nil +} + +func (x *ClientConfig) GetHistogramParams() *HistogramParams { + if x != nil { + return x.HistogramParams + } + return nil +} + +func (x *ClientConfig) GetCoreList() []int32 { + if x != nil { + return x.CoreList + } + return nil +} + +func (x *ClientConfig) GetCoreLimit() int32 { + if x != nil { + return x.CoreLimit + } + return 0 +} + +func (x *ClientConfig) GetOtherClientApi() string { + if x != nil { + return x.OtherClientApi + } + return "" +} + +func (x *ClientConfig) GetChannelArgs() []*ChannelArg { + if x != nil { + return x.ChannelArgs + } + return nil +} + +func (x *ClientConfig) GetThreadsPerCq() int32 { + if x != nil { + return x.ThreadsPerCq + } + return 0 +} + +func (x *ClientConfig) GetMessagesPerStream() int32 { + if x != nil { + return x.MessagesPerStream + } + return 0 +} + +func (x *ClientConfig) GetUseCoalesceApi() bool { + if x != nil { + return x.UseCoalesceApi + } + return false +} + +func (x *ClientConfig) GetMedianLatencyCollectionIntervalMillis() int32 { + if x != nil { + return x.MedianLatencyCollectionIntervalMillis + } + return 0 +} + +func (x *ClientConfig) GetClientProcesses() int32 { + if x != nil { + return x.ClientProcesses + } + return 0 +} + +type ClientStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + Stats *ClientStats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClientStatus) Reset() { + *x = ClientStatus{} + mi := &file_grpc_testing_control_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClientStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientStatus) ProtoMessage() {} + +func (x *ClientStatus) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientStatus.ProtoReflect.Descriptor instead. +func (*ClientStatus) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{6} +} + +func (x *ClientStatus) GetStats() *ClientStats { + if x != nil { + return x.Stats + } + return nil +} + +// Request current stats +type Mark struct { + state protoimpl.MessageState `protogen:"open.v1"` + // if true, the stats will be reset after taking their snapshot. + Reset_ bool `protobuf:"varint,1,opt,name=reset,proto3" json:"reset,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Mark) Reset() { + *x = Mark{} + mi := &file_grpc_testing_control_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Mark) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Mark) ProtoMessage() {} + +func (x *Mark) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Mark.ProtoReflect.Descriptor instead. +func (*Mark) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{7} +} + +func (x *Mark) GetReset_() bool { + if x != nil { + return x.Reset_ + } + return false +} + +type ClientArgs struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Argtype: + // + // *ClientArgs_Setup + // *ClientArgs_Mark + Argtype isClientArgs_Argtype `protobuf_oneof:"argtype"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClientArgs) Reset() { + *x = ClientArgs{} + mi := &file_grpc_testing_control_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClientArgs) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientArgs) ProtoMessage() {} + +func (x *ClientArgs) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientArgs.ProtoReflect.Descriptor instead. +func (*ClientArgs) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{8} +} + +func (x *ClientArgs) GetArgtype() isClientArgs_Argtype { + if x != nil { + return x.Argtype + } + return nil +} + +func (x *ClientArgs) GetSetup() *ClientConfig { + if x != nil { + if x, ok := x.Argtype.(*ClientArgs_Setup); ok { + return x.Setup + } + } + return nil +} + +func (x *ClientArgs) GetMark() *Mark { + if x != nil { + if x, ok := x.Argtype.(*ClientArgs_Mark); ok { + return x.Mark + } + } + return nil +} + +type isClientArgs_Argtype interface { + isClientArgs_Argtype() +} + +type ClientArgs_Setup struct { + Setup *ClientConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"` +} + +type ClientArgs_Mark struct { + Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"` +} + +func (*ClientArgs_Setup) isClientArgs_Argtype() {} + +func (*ClientArgs_Mark) isClientArgs_Argtype() {} + +type ServerConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + ServerType ServerType `protobuf:"varint,1,opt,name=server_type,json=serverType,proto3,enum=grpc.testing.ServerType" json:"server_type,omitempty"` + SecurityParams *SecurityParams `protobuf:"bytes,2,opt,name=security_params,json=securityParams,proto3" json:"security_params,omitempty"` + // Port on which to listen. Zero means pick unused port. + Port int32 `protobuf:"varint,4,opt,name=port,proto3" json:"port,omitempty"` + // Only for async server. Number of threads used to serve the requests. + AsyncServerThreads int32 `protobuf:"varint,7,opt,name=async_server_threads,json=asyncServerThreads,proto3" json:"async_server_threads,omitempty"` + // Specify the number of cores to limit server to, if desired + CoreLimit int32 `protobuf:"varint,8,opt,name=core_limit,json=coreLimit,proto3" json:"core_limit,omitempty"` + // payload config, used in generic server. + // Note this must NOT be used in proto (non-generic) servers. For proto servers, + // 'response sizes' must be configured from the 'response_size' field of the + // 'SimpleRequest' objects in RPC requests. + PayloadConfig *PayloadConfig `protobuf:"bytes,9,opt,name=payload_config,json=payloadConfig,proto3" json:"payload_config,omitempty"` + // Specify the cores we should run the server on, if desired + CoreList []int32 `protobuf:"varint,10,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"` + // If we use an OTHER_SERVER client_type, this string gives more detail + OtherServerApi string `protobuf:"bytes,11,opt,name=other_server_api,json=otherServerApi,proto3" json:"other_server_api,omitempty"` + // Number of threads that share each completion queue + ThreadsPerCq int32 `protobuf:"varint,12,opt,name=threads_per_cq,json=threadsPerCq,proto3" json:"threads_per_cq,omitempty"` + // Buffer pool size (no buffer pool specified if unset) + ResourceQuotaSize int32 `protobuf:"varint,1001,opt,name=resource_quota_size,json=resourceQuotaSize,proto3" json:"resource_quota_size,omitempty"` + ChannelArgs []*ChannelArg `protobuf:"bytes,1002,rep,name=channel_args,json=channelArgs,proto3" json:"channel_args,omitempty"` + // Number of server processes. 0 indicates no restriction. + ServerProcesses int32 `protobuf:"varint,21,opt,name=server_processes,json=serverProcesses,proto3" json:"server_processes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerConfig) Reset() { + *x = ServerConfig{} + mi := &file_grpc_testing_control_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerConfig) ProtoMessage() {} + +func (x *ServerConfig) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerConfig.ProtoReflect.Descriptor instead. +func (*ServerConfig) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{9} +} + +func (x *ServerConfig) GetServerType() ServerType { + if x != nil { + return x.ServerType + } + return ServerType_SYNC_SERVER +} + +func (x *ServerConfig) GetSecurityParams() *SecurityParams { + if x != nil { + return x.SecurityParams + } + return nil +} + +func (x *ServerConfig) GetPort() int32 { + if x != nil { + return x.Port + } + return 0 +} + +func (x *ServerConfig) GetAsyncServerThreads() int32 { + if x != nil { + return x.AsyncServerThreads + } + return 0 +} + +func (x *ServerConfig) GetCoreLimit() int32 { + if x != nil { + return x.CoreLimit + } + return 0 +} + +func (x *ServerConfig) GetPayloadConfig() *PayloadConfig { + if x != nil { + return x.PayloadConfig + } + return nil +} + +func (x *ServerConfig) GetCoreList() []int32 { + if x != nil { + return x.CoreList + } + return nil +} + +func (x *ServerConfig) GetOtherServerApi() string { + if x != nil { + return x.OtherServerApi + } + return "" +} + +func (x *ServerConfig) GetThreadsPerCq() int32 { + if x != nil { + return x.ThreadsPerCq + } + return 0 +} + +func (x *ServerConfig) GetResourceQuotaSize() int32 { + if x != nil { + return x.ResourceQuotaSize + } + return 0 +} + +func (x *ServerConfig) GetChannelArgs() []*ChannelArg { + if x != nil { + return x.ChannelArgs + } + return nil +} + +func (x *ServerConfig) GetServerProcesses() int32 { + if x != nil { + return x.ServerProcesses + } + return 0 +} + +type ServerArgs struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Argtype: + // + // *ServerArgs_Setup + // *ServerArgs_Mark + Argtype isServerArgs_Argtype `protobuf_oneof:"argtype"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerArgs) Reset() { + *x = ServerArgs{} + mi := &file_grpc_testing_control_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerArgs) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerArgs) ProtoMessage() {} + +func (x *ServerArgs) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerArgs.ProtoReflect.Descriptor instead. +func (*ServerArgs) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{10} +} + +func (x *ServerArgs) GetArgtype() isServerArgs_Argtype { + if x != nil { + return x.Argtype + } + return nil +} + +func (x *ServerArgs) GetSetup() *ServerConfig { + if x != nil { + if x, ok := x.Argtype.(*ServerArgs_Setup); ok { + return x.Setup + } + } + return nil +} + +func (x *ServerArgs) GetMark() *Mark { + if x != nil { + if x, ok := x.Argtype.(*ServerArgs_Mark); ok { + return x.Mark + } + } + return nil +} + +type isServerArgs_Argtype interface { + isServerArgs_Argtype() +} + +type ServerArgs_Setup struct { + Setup *ServerConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"` +} + +type ServerArgs_Mark struct { + Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"` +} + +func (*ServerArgs_Setup) isServerArgs_Argtype() {} + +func (*ServerArgs_Mark) isServerArgs_Argtype() {} + +type ServerStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + Stats *ServerStats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"` + // the port bound by the server + Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + // Number of cores available to the server + Cores int32 `protobuf:"varint,3,opt,name=cores,proto3" json:"cores,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerStatus) Reset() { + *x = ServerStatus{} + mi := &file_grpc_testing_control_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerStatus) ProtoMessage() {} + +func (x *ServerStatus) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerStatus.ProtoReflect.Descriptor instead. +func (*ServerStatus) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{11} +} + +func (x *ServerStatus) GetStats() *ServerStats { + if x != nil { + return x.Stats + } + return nil +} + +func (x *ServerStatus) GetPort() int32 { + if x != nil { + return x.Port + } + return 0 +} + +func (x *ServerStatus) GetCores() int32 { + if x != nil { + return x.Cores + } + return 0 +} + +type CoreRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CoreRequest) Reset() { + *x = CoreRequest{} + mi := &file_grpc_testing_control_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CoreRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CoreRequest) ProtoMessage() {} + +func (x *CoreRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CoreRequest.ProtoReflect.Descriptor instead. +func (*CoreRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{12} +} + +type CoreResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Number of cores available on the server + Cores int32 `protobuf:"varint,1,opt,name=cores,proto3" json:"cores,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CoreResponse) Reset() { + *x = CoreResponse{} + mi := &file_grpc_testing_control_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CoreResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CoreResponse) ProtoMessage() {} + +func (x *CoreResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CoreResponse.ProtoReflect.Descriptor instead. +func (*CoreResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{13} +} + +func (x *CoreResponse) GetCores() int32 { + if x != nil { + return x.Cores + } + return 0 +} + +type Void struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Void) Reset() { + *x = Void{} + mi := &file_grpc_testing_control_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Void) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Void) ProtoMessage() {} + +func (x *Void) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Void.ProtoReflect.Descriptor instead. +func (*Void) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{14} +} + +// A single performance scenario: input to qps_json_driver +type Scenario struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Human readable name for this scenario + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Client configuration + ClientConfig *ClientConfig `protobuf:"bytes,2,opt,name=client_config,json=clientConfig,proto3" json:"client_config,omitempty"` + // Number of clients to start for the test + NumClients int32 `protobuf:"varint,3,opt,name=num_clients,json=numClients,proto3" json:"num_clients,omitempty"` + // Server configuration + ServerConfig *ServerConfig `protobuf:"bytes,4,opt,name=server_config,json=serverConfig,proto3" json:"server_config,omitempty"` + // Number of servers to start for the test + NumServers int32 `protobuf:"varint,5,opt,name=num_servers,json=numServers,proto3" json:"num_servers,omitempty"` + // Warmup period, in seconds + WarmupSeconds int32 `protobuf:"varint,6,opt,name=warmup_seconds,json=warmupSeconds,proto3" json:"warmup_seconds,omitempty"` + // Benchmark time, in seconds + BenchmarkSeconds int32 `protobuf:"varint,7,opt,name=benchmark_seconds,json=benchmarkSeconds,proto3" json:"benchmark_seconds,omitempty"` + // Number of workers to spawn locally (usually zero) + SpawnLocalWorkerCount int32 `protobuf:"varint,8,opt,name=spawn_local_worker_count,json=spawnLocalWorkerCount,proto3" json:"spawn_local_worker_count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Scenario) Reset() { + *x = Scenario{} + mi := &file_grpc_testing_control_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Scenario) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Scenario) ProtoMessage() {} + +func (x *Scenario) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Scenario.ProtoReflect.Descriptor instead. +func (*Scenario) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{15} +} + +func (x *Scenario) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Scenario) GetClientConfig() *ClientConfig { + if x != nil { + return x.ClientConfig + } + return nil +} + +func (x *Scenario) GetNumClients() int32 { + if x != nil { + return x.NumClients + } + return 0 +} + +func (x *Scenario) GetServerConfig() *ServerConfig { + if x != nil { + return x.ServerConfig + } + return nil +} + +func (x *Scenario) GetNumServers() int32 { + if x != nil { + return x.NumServers + } + return 0 +} + +func (x *Scenario) GetWarmupSeconds() int32 { + if x != nil { + return x.WarmupSeconds + } + return 0 +} + +func (x *Scenario) GetBenchmarkSeconds() int32 { + if x != nil { + return x.BenchmarkSeconds + } + return 0 +} + +func (x *Scenario) GetSpawnLocalWorkerCount() int32 { + if x != nil { + return x.SpawnLocalWorkerCount + } + return 0 +} + +// A set of scenarios to be run with qps_json_driver +type Scenarios struct { + state protoimpl.MessageState `protogen:"open.v1"` + Scenarios []*Scenario `protobuf:"bytes,1,rep,name=scenarios,proto3" json:"scenarios,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Scenarios) Reset() { + *x = Scenarios{} + mi := &file_grpc_testing_control_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Scenarios) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Scenarios) ProtoMessage() {} + +func (x *Scenarios) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Scenarios.ProtoReflect.Descriptor instead. +func (*Scenarios) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{16} +} + +func (x *Scenarios) GetScenarios() []*Scenario { + if x != nil { + return x.Scenarios + } + return nil +} + +// Basic summary that can be computed from ClientStats and ServerStats +// once the scenario has finished. +type ScenarioResultSummary struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Total number of operations per second over all clients. What is counted as 1 'operation' depends on the benchmark scenarios: + // For unary benchmarks, an operation is processing of a single unary RPC. + // For streaming benchmarks, an operation is processing of a single ping pong of request and response. + Qps float64 `protobuf:"fixed64,1,opt,name=qps,proto3" json:"qps,omitempty"` + // QPS per server core. + QpsPerServerCore float64 `protobuf:"fixed64,2,opt,name=qps_per_server_core,json=qpsPerServerCore,proto3" json:"qps_per_server_core,omitempty"` + // The total server cpu load based on system time across all server processes, expressed as percentage of a single cpu core. + // For example, 85 implies 85% of a cpu core, 125 implies 125% of a cpu core. Since we are accumulating the cpu load across all the server + // processes, the value could > 100 when there are multiple servers or a single server using multiple threads and cores. + // Same explanation for the total client cpu load below. + ServerSystemTime float64 `protobuf:"fixed64,3,opt,name=server_system_time,json=serverSystemTime,proto3" json:"server_system_time,omitempty"` + // The total server cpu load based on user time across all server processes, expressed as percentage of a single cpu core. (85 => 85%, 125 => 125%) + ServerUserTime float64 `protobuf:"fixed64,4,opt,name=server_user_time,json=serverUserTime,proto3" json:"server_user_time,omitempty"` + // The total client cpu load based on system time across all client processes, expressed as percentage of a single cpu core. (85 => 85%, 125 => 125%) + ClientSystemTime float64 `protobuf:"fixed64,5,opt,name=client_system_time,json=clientSystemTime,proto3" json:"client_system_time,omitempty"` + // The total client cpu load based on user time across all client processes, expressed as percentage of a single cpu core. (85 => 85%, 125 => 125%) + ClientUserTime float64 `protobuf:"fixed64,6,opt,name=client_user_time,json=clientUserTime,proto3" json:"client_user_time,omitempty"` + // X% latency percentiles (in nanoseconds) + Latency_50 float64 `protobuf:"fixed64,7,opt,name=latency_50,json=latency50,proto3" json:"latency_50,omitempty"` + Latency_90 float64 `protobuf:"fixed64,8,opt,name=latency_90,json=latency90,proto3" json:"latency_90,omitempty"` + Latency_95 float64 `protobuf:"fixed64,9,opt,name=latency_95,json=latency95,proto3" json:"latency_95,omitempty"` + Latency_99 float64 `protobuf:"fixed64,10,opt,name=latency_99,json=latency99,proto3" json:"latency_99,omitempty"` + Latency_999 float64 `protobuf:"fixed64,11,opt,name=latency_999,json=latency999,proto3" json:"latency_999,omitempty"` + // server cpu usage percentage + ServerCpuUsage float64 `protobuf:"fixed64,12,opt,name=server_cpu_usage,json=serverCpuUsage,proto3" json:"server_cpu_usage,omitempty"` + // Number of requests that succeeded/failed + SuccessfulRequestsPerSecond float64 `protobuf:"fixed64,13,opt,name=successful_requests_per_second,json=successfulRequestsPerSecond,proto3" json:"successful_requests_per_second,omitempty"` + FailedRequestsPerSecond float64 `protobuf:"fixed64,14,opt,name=failed_requests_per_second,json=failedRequestsPerSecond,proto3" json:"failed_requests_per_second,omitempty"` + // Number of polls called inside completion queue per request + ClientPollsPerRequest float64 `protobuf:"fixed64,15,opt,name=client_polls_per_request,json=clientPollsPerRequest,proto3" json:"client_polls_per_request,omitempty"` + ServerPollsPerRequest float64 `protobuf:"fixed64,16,opt,name=server_polls_per_request,json=serverPollsPerRequest,proto3" json:"server_polls_per_request,omitempty"` + // Queries per CPU-sec over all servers or clients + ServerQueriesPerCpuSec float64 `protobuf:"fixed64,17,opt,name=server_queries_per_cpu_sec,json=serverQueriesPerCpuSec,proto3" json:"server_queries_per_cpu_sec,omitempty"` + ClientQueriesPerCpuSec float64 `protobuf:"fixed64,18,opt,name=client_queries_per_cpu_sec,json=clientQueriesPerCpuSec,proto3" json:"client_queries_per_cpu_sec,omitempty"` + // Start and end time for the test scenario + StartTime *timestamppb.Timestamp `protobuf:"bytes,19,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + EndTime *timestamppb.Timestamp `protobuf:"bytes,20,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ScenarioResultSummary) Reset() { + *x = ScenarioResultSummary{} + mi := &file_grpc_testing_control_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ScenarioResultSummary) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ScenarioResultSummary) ProtoMessage() {} + +func (x *ScenarioResultSummary) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ScenarioResultSummary.ProtoReflect.Descriptor instead. +func (*ScenarioResultSummary) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{17} +} + +func (x *ScenarioResultSummary) GetQps() float64 { + if x != nil { + return x.Qps + } + return 0 +} + +func (x *ScenarioResultSummary) GetQpsPerServerCore() float64 { + if x != nil { + return x.QpsPerServerCore + } + return 0 +} + +func (x *ScenarioResultSummary) GetServerSystemTime() float64 { + if x != nil { + return x.ServerSystemTime + } + return 0 +} + +func (x *ScenarioResultSummary) GetServerUserTime() float64 { + if x != nil { + return x.ServerUserTime + } + return 0 +} + +func (x *ScenarioResultSummary) GetClientSystemTime() float64 { + if x != nil { + return x.ClientSystemTime + } + return 0 +} + +func (x *ScenarioResultSummary) GetClientUserTime() float64 { + if x != nil { + return x.ClientUserTime + } + return 0 +} + +func (x *ScenarioResultSummary) GetLatency_50() float64 { + if x != nil { + return x.Latency_50 + } + return 0 +} + +func (x *ScenarioResultSummary) GetLatency_90() float64 { + if x != nil { + return x.Latency_90 + } + return 0 +} + +func (x *ScenarioResultSummary) GetLatency_95() float64 { + if x != nil { + return x.Latency_95 + } + return 0 +} + +func (x *ScenarioResultSummary) GetLatency_99() float64 { + if x != nil { + return x.Latency_99 + } + return 0 +} + +func (x *ScenarioResultSummary) GetLatency_999() float64 { + if x != nil { + return x.Latency_999 + } + return 0 +} + +func (x *ScenarioResultSummary) GetServerCpuUsage() float64 { + if x != nil { + return x.ServerCpuUsage + } + return 0 +} + +func (x *ScenarioResultSummary) GetSuccessfulRequestsPerSecond() float64 { + if x != nil { + return x.SuccessfulRequestsPerSecond + } + return 0 +} + +func (x *ScenarioResultSummary) GetFailedRequestsPerSecond() float64 { + if x != nil { + return x.FailedRequestsPerSecond + } + return 0 +} + +func (x *ScenarioResultSummary) GetClientPollsPerRequest() float64 { + if x != nil { + return x.ClientPollsPerRequest + } + return 0 +} + +func (x *ScenarioResultSummary) GetServerPollsPerRequest() float64 { + if x != nil { + return x.ServerPollsPerRequest + } + return 0 +} + +func (x *ScenarioResultSummary) GetServerQueriesPerCpuSec() float64 { + if x != nil { + return x.ServerQueriesPerCpuSec + } + return 0 +} + +func (x *ScenarioResultSummary) GetClientQueriesPerCpuSec() float64 { + if x != nil { + return x.ClientQueriesPerCpuSec + } + return 0 +} + +func (x *ScenarioResultSummary) GetStartTime() *timestamppb.Timestamp { + if x != nil { + return x.StartTime + } + return nil +} + +func (x *ScenarioResultSummary) GetEndTime() *timestamppb.Timestamp { + if x != nil { + return x.EndTime + } + return nil +} + +// Results of a single benchmark scenario. +type ScenarioResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Inputs used to run the scenario. + Scenario *Scenario `protobuf:"bytes,1,opt,name=scenario,proto3" json:"scenario,omitempty"` + // Histograms from all clients merged into one histogram. + Latencies *HistogramData `protobuf:"bytes,2,opt,name=latencies,proto3" json:"latencies,omitempty"` + // Client stats for each client + ClientStats []*ClientStats `protobuf:"bytes,3,rep,name=client_stats,json=clientStats,proto3" json:"client_stats,omitempty"` + // Server stats for each server + ServerStats []*ServerStats `protobuf:"bytes,4,rep,name=server_stats,json=serverStats,proto3" json:"server_stats,omitempty"` + // Number of cores available to each server + ServerCores []int32 `protobuf:"varint,5,rep,packed,name=server_cores,json=serverCores,proto3" json:"server_cores,omitempty"` + // An after-the-fact computed summary + Summary *ScenarioResultSummary `protobuf:"bytes,6,opt,name=summary,proto3" json:"summary,omitempty"` + // Information on success or failure of each worker + ClientSuccess []bool `protobuf:"varint,7,rep,packed,name=client_success,json=clientSuccess,proto3" json:"client_success,omitempty"` + ServerSuccess []bool `protobuf:"varint,8,rep,packed,name=server_success,json=serverSuccess,proto3" json:"server_success,omitempty"` + // Number of failed requests (one row per status code seen) + RequestResults []*RequestResultCount `protobuf:"bytes,9,rep,name=request_results,json=requestResults,proto3" json:"request_results,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ScenarioResult) Reset() { + *x = ScenarioResult{} + mi := &file_grpc_testing_control_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ScenarioResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ScenarioResult) ProtoMessage() {} + +func (x *ScenarioResult) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ScenarioResult.ProtoReflect.Descriptor instead. +func (*ScenarioResult) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{18} +} + +func (x *ScenarioResult) GetScenario() *Scenario { + if x != nil { + return x.Scenario + } + return nil +} + +func (x *ScenarioResult) GetLatencies() *HistogramData { + if x != nil { + return x.Latencies + } + return nil +} + +func (x *ScenarioResult) GetClientStats() []*ClientStats { + if x != nil { + return x.ClientStats + } + return nil +} + +func (x *ScenarioResult) GetServerStats() []*ServerStats { + if x != nil { + return x.ServerStats + } + return nil +} + +func (x *ScenarioResult) GetServerCores() []int32 { + if x != nil { + return x.ServerCores + } + return nil +} + +func (x *ScenarioResult) GetSummary() *ScenarioResultSummary { + if x != nil { + return x.Summary + } + return nil +} + +func (x *ScenarioResult) GetClientSuccess() []bool { + if x != nil { + return x.ClientSuccess + } + return nil +} + +func (x *ScenarioResult) GetServerSuccess() []bool { + if x != nil { + return x.ServerSuccess + } + return nil +} + +func (x *ScenarioResult) GetRequestResults() []*RequestResultCount { + if x != nil { + return x.RequestResults + } + return nil +} + +var File_grpc_testing_control_proto protoreflect.FileDescriptor + +const file_grpc_testing_control_proto_rawDesc = "" + + "\n" + + "\x1agrpc/testing/control.proto\x12\fgrpc.testing\x1a\x1bgrpc/testing/payloads.proto\x1a\x18grpc/testing/stats.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"2\n" + + "\rPoissonParams\x12!\n" + + "\foffered_load\x18\x01 \x01(\x01R\vofferedLoad\"\x12\n" + + "\x10ClosedLoopParams\"\x90\x01\n" + + "\n" + + "LoadParams\x12A\n" + + "\vclosed_loop\x18\x01 \x01(\v2\x1e.grpc.testing.ClosedLoopParamsH\x00R\n" + + "closedLoop\x127\n" + + "\apoisson\x18\x02 \x01(\v2\x1b.grpc.testing.PoissonParamsH\x00R\apoissonB\x06\n" + + "\x04load\"\x7f\n" + + "\x0eSecurityParams\x12\x1e\n" + + "\vuse_test_ca\x18\x01 \x01(\bR\tuseTestCa\x120\n" + + "\x14server_host_override\x18\x02 \x01(\tR\x12serverHostOverride\x12\x1b\n" + + "\tcred_type\x18\x03 \x01(\tR\bcredType\"g\n" + + "\n" + + "ChannelArg\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n" + + "\tstr_value\x18\x02 \x01(\tH\x00R\bstrValue\x12\x1d\n" + + "\tint_value\x18\x03 \x01(\x05H\x00R\bintValueB\a\n" + + "\x05value\"\xf6\a\n" + + "\fClientConfig\x12%\n" + + "\x0eserver_targets\x18\x01 \x03(\tR\rserverTargets\x129\n" + + "\vclient_type\x18\x02 \x01(\x0e2\x18.grpc.testing.ClientTypeR\n" + + "clientType\x12E\n" + + "\x0fsecurity_params\x18\x03 \x01(\v2\x1c.grpc.testing.SecurityParamsR\x0esecurityParams\x12?\n" + + "\x1coutstanding_rpcs_per_channel\x18\x04 \x01(\x05R\x19outstandingRpcsPerChannel\x12'\n" + + "\x0fclient_channels\x18\x05 \x01(\x05R\x0eclientChannels\x120\n" + + "\x14async_client_threads\x18\a \x01(\x05R\x12asyncClientThreads\x120\n" + + "\brpc_type\x18\b \x01(\x0e2\x15.grpc.testing.RpcTypeR\arpcType\x129\n" + + "\vload_params\x18\n" + + " \x01(\v2\x18.grpc.testing.LoadParamsR\n" + + "loadParams\x12B\n" + + "\x0epayload_config\x18\v \x01(\v2\x1b.grpc.testing.PayloadConfigR\rpayloadConfig\x12H\n" + + "\x10histogram_params\x18\f \x01(\v2\x1d.grpc.testing.HistogramParamsR\x0fhistogramParams\x12\x1b\n" + + "\tcore_list\x18\r \x03(\x05R\bcoreList\x12\x1d\n" + + "\n" + + "core_limit\x18\x0e \x01(\x05R\tcoreLimit\x12(\n" + + "\x10other_client_api\x18\x0f \x01(\tR\x0eotherClientApi\x12;\n" + + "\fchannel_args\x18\x10 \x03(\v2\x18.grpc.testing.ChannelArgR\vchannelArgs\x12$\n" + + "\x0ethreads_per_cq\x18\x11 \x01(\x05R\fthreadsPerCq\x12.\n" + + "\x13messages_per_stream\x18\x12 \x01(\x05R\x11messagesPerStream\x12(\n" + + "\x10use_coalesce_api\x18\x13 \x01(\bR\x0euseCoalesceApi\x12X\n" + + ")median_latency_collection_interval_millis\x18\x14 \x01(\x05R%medianLatencyCollectionIntervalMillis\x12)\n" + + "\x10client_processes\x18\x15 \x01(\x05R\x0fclientProcesses\"?\n" + + "\fClientStatus\x12/\n" + + "\x05stats\x18\x01 \x01(\v2\x19.grpc.testing.ClientStatsR\x05stats\"\x1c\n" + + "\x04Mark\x12\x14\n" + + "\x05reset\x18\x01 \x01(\bR\x05reset\"u\n" + + "\n" + + "ClientArgs\x122\n" + + "\x05setup\x18\x01 \x01(\v2\x1a.grpc.testing.ClientConfigH\x00R\x05setup\x12(\n" + + "\x04mark\x18\x02 \x01(\v2\x12.grpc.testing.MarkH\x00R\x04markB\t\n" + + "\aargtype\"\xc0\x04\n" + + "\fServerConfig\x129\n" + + "\vserver_type\x18\x01 \x01(\x0e2\x18.grpc.testing.ServerTypeR\n" + + "serverType\x12E\n" + + "\x0fsecurity_params\x18\x02 \x01(\v2\x1c.grpc.testing.SecurityParamsR\x0esecurityParams\x12\x12\n" + + "\x04port\x18\x04 \x01(\x05R\x04port\x120\n" + + "\x14async_server_threads\x18\a \x01(\x05R\x12asyncServerThreads\x12\x1d\n" + + "\n" + + "core_limit\x18\b \x01(\x05R\tcoreLimit\x12B\n" + + "\x0epayload_config\x18\t \x01(\v2\x1b.grpc.testing.PayloadConfigR\rpayloadConfig\x12\x1b\n" + + "\tcore_list\x18\n" + + " \x03(\x05R\bcoreList\x12(\n" + + "\x10other_server_api\x18\v \x01(\tR\x0eotherServerApi\x12$\n" + + "\x0ethreads_per_cq\x18\f \x01(\x05R\fthreadsPerCq\x12/\n" + + "\x13resource_quota_size\x18\xe9\a \x01(\x05R\x11resourceQuotaSize\x12<\n" + + "\fchannel_args\x18\xea\a \x03(\v2\x18.grpc.testing.ChannelArgR\vchannelArgs\x12)\n" + + "\x10server_processes\x18\x15 \x01(\x05R\x0fserverProcesses\"u\n" + + "\n" + + "ServerArgs\x122\n" + + "\x05setup\x18\x01 \x01(\v2\x1a.grpc.testing.ServerConfigH\x00R\x05setup\x12(\n" + + "\x04mark\x18\x02 \x01(\v2\x12.grpc.testing.MarkH\x00R\x04markB\t\n" + + "\aargtype\"i\n" + + "\fServerStatus\x12/\n" + + "\x05stats\x18\x01 \x01(\v2\x19.grpc.testing.ServerStatsR\x05stats\x12\x12\n" + + "\x04port\x18\x02 \x01(\x05R\x04port\x12\x14\n" + + "\x05cores\x18\x03 \x01(\x05R\x05cores\"\r\n" + + "\vCoreRequest\"$\n" + + "\fCoreResponse\x12\x14\n" + + "\x05cores\x18\x01 \x01(\x05R\x05cores\"\x06\n" + + "\x04Void\"\xef\x02\n" + + "\bScenario\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12?\n" + + "\rclient_config\x18\x02 \x01(\v2\x1a.grpc.testing.ClientConfigR\fclientConfig\x12\x1f\n" + + "\vnum_clients\x18\x03 \x01(\x05R\n" + + "numClients\x12?\n" + + "\rserver_config\x18\x04 \x01(\v2\x1a.grpc.testing.ServerConfigR\fserverConfig\x12\x1f\n" + + "\vnum_servers\x18\x05 \x01(\x05R\n" + + "numServers\x12%\n" + + "\x0ewarmup_seconds\x18\x06 \x01(\x05R\rwarmupSeconds\x12+\n" + + "\x11benchmark_seconds\x18\a \x01(\x05R\x10benchmarkSeconds\x127\n" + + "\x18spawn_local_worker_count\x18\b \x01(\x05R\x15spawnLocalWorkerCount\"A\n" + + "\tScenarios\x124\n" + + "\tscenarios\x18\x01 \x03(\v2\x16.grpc.testing.ScenarioR\tscenarios\"\xad\a\n" + + "\x15ScenarioResultSummary\x12\x10\n" + + "\x03qps\x18\x01 \x01(\x01R\x03qps\x12-\n" + + "\x13qps_per_server_core\x18\x02 \x01(\x01R\x10qpsPerServerCore\x12,\n" + + "\x12server_system_time\x18\x03 \x01(\x01R\x10serverSystemTime\x12(\n" + + "\x10server_user_time\x18\x04 \x01(\x01R\x0eserverUserTime\x12,\n" + + "\x12client_system_time\x18\x05 \x01(\x01R\x10clientSystemTime\x12(\n" + + "\x10client_user_time\x18\x06 \x01(\x01R\x0eclientUserTime\x12\x1d\n" + + "\n" + + "latency_50\x18\a \x01(\x01R\tlatency50\x12\x1d\n" + + "\n" + + "latency_90\x18\b \x01(\x01R\tlatency90\x12\x1d\n" + + "\n" + + "latency_95\x18\t \x01(\x01R\tlatency95\x12\x1d\n" + + "\n" + + "latency_99\x18\n" + + " \x01(\x01R\tlatency99\x12\x1f\n" + + "\vlatency_999\x18\v \x01(\x01R\n" + + "latency999\x12(\n" + + "\x10server_cpu_usage\x18\f \x01(\x01R\x0eserverCpuUsage\x12C\n" + + "\x1esuccessful_requests_per_second\x18\r \x01(\x01R\x1bsuccessfulRequestsPerSecond\x12;\n" + + "\x1afailed_requests_per_second\x18\x0e \x01(\x01R\x17failedRequestsPerSecond\x127\n" + + "\x18client_polls_per_request\x18\x0f \x01(\x01R\x15clientPollsPerRequest\x127\n" + + "\x18server_polls_per_request\x18\x10 \x01(\x01R\x15serverPollsPerRequest\x12:\n" + + "\x1aserver_queries_per_cpu_sec\x18\x11 \x01(\x01R\x16serverQueriesPerCpuSec\x12:\n" + + "\x1aclient_queries_per_cpu_sec\x18\x12 \x01(\x01R\x16clientQueriesPerCpuSec\x129\n" + + "\n" + + "start_time\x18\x13 \x01(\v2\x1a.google.protobuf.TimestampR\tstartTime\x125\n" + + "\bend_time\x18\x14 \x01(\v2\x1a.google.protobuf.TimestampR\aendTime\"\xf6\x03\n" + + "\x0eScenarioResult\x122\n" + + "\bscenario\x18\x01 \x01(\v2\x16.grpc.testing.ScenarioR\bscenario\x129\n" + + "\tlatencies\x18\x02 \x01(\v2\x1b.grpc.testing.HistogramDataR\tlatencies\x12<\n" + + "\fclient_stats\x18\x03 \x03(\v2\x19.grpc.testing.ClientStatsR\vclientStats\x12<\n" + + "\fserver_stats\x18\x04 \x03(\v2\x19.grpc.testing.ServerStatsR\vserverStats\x12!\n" + + "\fserver_cores\x18\x05 \x03(\x05R\vserverCores\x12=\n" + + "\asummary\x18\x06 \x01(\v2#.grpc.testing.ScenarioResultSummaryR\asummary\x12%\n" + + "\x0eclient_success\x18\a \x03(\bR\rclientSuccess\x12%\n" + + "\x0eserver_success\x18\b \x03(\bR\rserverSuccess\x12I\n" + + "\x0frequest_results\x18\t \x03(\v2 .grpc.testing.RequestResultCountR\x0erequestResults*V\n" + + "\n" + + "ClientType\x12\x0f\n" + + "\vSYNC_CLIENT\x10\x00\x12\x10\n" + + "\fASYNC_CLIENT\x10\x01\x12\x10\n" + + "\fOTHER_CLIENT\x10\x02\x12\x13\n" + + "\x0fCALLBACK_CLIENT\x10\x03*p\n" + + "\n" + + "ServerType\x12\x0f\n" + + "\vSYNC_SERVER\x10\x00\x12\x10\n" + + "\fASYNC_SERVER\x10\x01\x12\x18\n" + + "\x14ASYNC_GENERIC_SERVER\x10\x02\x12\x10\n" + + "\fOTHER_SERVER\x10\x03\x12\x13\n" + + "\x0fCALLBACK_SERVER\x10\x04*r\n" + + "\aRpcType\x12\t\n" + + "\x05UNARY\x10\x00\x12\r\n" + + "\tSTREAMING\x10\x01\x12\x19\n" + + "\x15STREAMING_FROM_CLIENT\x10\x02\x12\x19\n" + + "\x15STREAMING_FROM_SERVER\x10\x03\x12\x17\n" + + "\x13STREAMING_BOTH_WAYS\x10\x04B!\n" + + "\x0fio.grpc.testingB\fControlProtoP\x01b\x06proto3" + +var ( + file_grpc_testing_control_proto_rawDescOnce sync.Once + file_grpc_testing_control_proto_rawDescData []byte +) + +func file_grpc_testing_control_proto_rawDescGZIP() []byte { + file_grpc_testing_control_proto_rawDescOnce.Do(func() { + file_grpc_testing_control_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_testing_control_proto_rawDesc), len(file_grpc_testing_control_proto_rawDesc))) + }) + return file_grpc_testing_control_proto_rawDescData +} + +var file_grpc_testing_control_proto_enumTypes = make([]protoimpl.EnumInfo, 3) +var file_grpc_testing_control_proto_msgTypes = make([]protoimpl.MessageInfo, 19) +var file_grpc_testing_control_proto_goTypes = []any{ + (ClientType)(0), // 0: grpc.testing.ClientType + (ServerType)(0), // 1: grpc.testing.ServerType + (RpcType)(0), // 2: grpc.testing.RpcType + (*PoissonParams)(nil), // 3: grpc.testing.PoissonParams + (*ClosedLoopParams)(nil), // 4: grpc.testing.ClosedLoopParams + (*LoadParams)(nil), // 5: grpc.testing.LoadParams + (*SecurityParams)(nil), // 6: grpc.testing.SecurityParams + (*ChannelArg)(nil), // 7: grpc.testing.ChannelArg + (*ClientConfig)(nil), // 8: grpc.testing.ClientConfig + (*ClientStatus)(nil), // 9: grpc.testing.ClientStatus + (*Mark)(nil), // 10: grpc.testing.Mark + (*ClientArgs)(nil), // 11: grpc.testing.ClientArgs + (*ServerConfig)(nil), // 12: grpc.testing.ServerConfig + (*ServerArgs)(nil), // 13: grpc.testing.ServerArgs + (*ServerStatus)(nil), // 14: grpc.testing.ServerStatus + (*CoreRequest)(nil), // 15: grpc.testing.CoreRequest + (*CoreResponse)(nil), // 16: grpc.testing.CoreResponse + (*Void)(nil), // 17: grpc.testing.Void + (*Scenario)(nil), // 18: grpc.testing.Scenario + (*Scenarios)(nil), // 19: grpc.testing.Scenarios + (*ScenarioResultSummary)(nil), // 20: grpc.testing.ScenarioResultSummary + (*ScenarioResult)(nil), // 21: grpc.testing.ScenarioResult + (*PayloadConfig)(nil), // 22: grpc.testing.PayloadConfig + (*HistogramParams)(nil), // 23: grpc.testing.HistogramParams + (*ClientStats)(nil), // 24: grpc.testing.ClientStats + (*ServerStats)(nil), // 25: grpc.testing.ServerStats + (*timestamppb.Timestamp)(nil), // 26: google.protobuf.Timestamp + (*HistogramData)(nil), // 27: grpc.testing.HistogramData + (*RequestResultCount)(nil), // 28: grpc.testing.RequestResultCount +} +var file_grpc_testing_control_proto_depIdxs = []int32{ + 4, // 0: grpc.testing.LoadParams.closed_loop:type_name -> grpc.testing.ClosedLoopParams + 3, // 1: grpc.testing.LoadParams.poisson:type_name -> grpc.testing.PoissonParams + 0, // 2: grpc.testing.ClientConfig.client_type:type_name -> grpc.testing.ClientType + 6, // 3: grpc.testing.ClientConfig.security_params:type_name -> grpc.testing.SecurityParams + 2, // 4: grpc.testing.ClientConfig.rpc_type:type_name -> grpc.testing.RpcType + 5, // 5: grpc.testing.ClientConfig.load_params:type_name -> grpc.testing.LoadParams + 22, // 6: grpc.testing.ClientConfig.payload_config:type_name -> grpc.testing.PayloadConfig + 23, // 7: grpc.testing.ClientConfig.histogram_params:type_name -> grpc.testing.HistogramParams + 7, // 8: grpc.testing.ClientConfig.channel_args:type_name -> grpc.testing.ChannelArg + 24, // 9: grpc.testing.ClientStatus.stats:type_name -> grpc.testing.ClientStats + 8, // 10: grpc.testing.ClientArgs.setup:type_name -> grpc.testing.ClientConfig + 10, // 11: grpc.testing.ClientArgs.mark:type_name -> grpc.testing.Mark + 1, // 12: grpc.testing.ServerConfig.server_type:type_name -> grpc.testing.ServerType + 6, // 13: grpc.testing.ServerConfig.security_params:type_name -> grpc.testing.SecurityParams + 22, // 14: grpc.testing.ServerConfig.payload_config:type_name -> grpc.testing.PayloadConfig + 7, // 15: grpc.testing.ServerConfig.channel_args:type_name -> grpc.testing.ChannelArg + 12, // 16: grpc.testing.ServerArgs.setup:type_name -> grpc.testing.ServerConfig + 10, // 17: grpc.testing.ServerArgs.mark:type_name -> grpc.testing.Mark + 25, // 18: grpc.testing.ServerStatus.stats:type_name -> grpc.testing.ServerStats + 8, // 19: grpc.testing.Scenario.client_config:type_name -> grpc.testing.ClientConfig + 12, // 20: grpc.testing.Scenario.server_config:type_name -> grpc.testing.ServerConfig + 18, // 21: grpc.testing.Scenarios.scenarios:type_name -> grpc.testing.Scenario + 26, // 22: grpc.testing.ScenarioResultSummary.start_time:type_name -> google.protobuf.Timestamp + 26, // 23: grpc.testing.ScenarioResultSummary.end_time:type_name -> google.protobuf.Timestamp + 18, // 24: grpc.testing.ScenarioResult.scenario:type_name -> grpc.testing.Scenario + 27, // 25: grpc.testing.ScenarioResult.latencies:type_name -> grpc.testing.HistogramData + 24, // 26: grpc.testing.ScenarioResult.client_stats:type_name -> grpc.testing.ClientStats + 25, // 27: grpc.testing.ScenarioResult.server_stats:type_name -> grpc.testing.ServerStats + 20, // 28: grpc.testing.ScenarioResult.summary:type_name -> grpc.testing.ScenarioResultSummary + 28, // 29: grpc.testing.ScenarioResult.request_results:type_name -> grpc.testing.RequestResultCount + 30, // [30:30] is the sub-list for method output_type + 30, // [30:30] is the sub-list for method input_type + 30, // [30:30] is the sub-list for extension type_name + 30, // [30:30] is the sub-list for extension extendee + 0, // [0:30] is the sub-list for field type_name +} + +func init() { file_grpc_testing_control_proto_init() } +func file_grpc_testing_control_proto_init() { + if File_grpc_testing_control_proto != nil { + return + } + file_grpc_testing_payloads_proto_init() + file_grpc_testing_stats_proto_init() + file_grpc_testing_control_proto_msgTypes[2].OneofWrappers = []any{ + (*LoadParams_ClosedLoop)(nil), + (*LoadParams_Poisson)(nil), + } + file_grpc_testing_control_proto_msgTypes[4].OneofWrappers = []any{ + (*ChannelArg_StrValue)(nil), + (*ChannelArg_IntValue)(nil), + } + file_grpc_testing_control_proto_msgTypes[8].OneofWrappers = []any{ + (*ClientArgs_Setup)(nil), + (*ClientArgs_Mark)(nil), + } + file_grpc_testing_control_proto_msgTypes[10].OneofWrappers = []any{ + (*ServerArgs_Setup)(nil), + (*ServerArgs_Mark)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_testing_control_proto_rawDesc), len(file_grpc_testing_control_proto_rawDesc)), + NumEnums: 3, + NumMessages: 19, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_testing_control_proto_goTypes, + DependencyIndexes: file_grpc_testing_control_proto_depIdxs, + EnumInfos: file_grpc_testing_control_proto_enumTypes, + MessageInfos: file_grpc_testing_control_proto_msgTypes, + }.Build() + File_grpc_testing_control_proto = out.File + file_grpc_testing_control_proto_goTypes = nil + file_grpc_testing_control_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/core/stats.pb.go b/interop/grpc_testing/core/stats.pb.go new file mode 100644 index 0000000..35233ec --- /dev/null +++ b/interop/grpc_testing/core/stats.pb.go @@ -0,0 +1,343 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/core/stats.proto + +package core + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Bucket struct { + state protoimpl.MessageState `protogen:"open.v1"` + Start float64 `protobuf:"fixed64,1,opt,name=start,proto3" json:"start,omitempty"` + Count uint64 `protobuf:"varint,2,opt,name=count,proto3" json:"count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Bucket) Reset() { + *x = Bucket{} + mi := &file_grpc_core_stats_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Bucket) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Bucket) ProtoMessage() {} + +func (x *Bucket) ProtoReflect() protoreflect.Message { + mi := &file_grpc_core_stats_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Bucket.ProtoReflect.Descriptor instead. +func (*Bucket) Descriptor() ([]byte, []int) { + return file_grpc_core_stats_proto_rawDescGZIP(), []int{0} +} + +func (x *Bucket) GetStart() float64 { + if x != nil { + return x.Start + } + return 0 +} + +func (x *Bucket) GetCount() uint64 { + if x != nil { + return x.Count + } + return 0 +} + +type Histogram struct { + state protoimpl.MessageState `protogen:"open.v1"` + Buckets []*Bucket `protobuf:"bytes,1,rep,name=buckets,proto3" json:"buckets,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Histogram) Reset() { + *x = Histogram{} + mi := &file_grpc_core_stats_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Histogram) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Histogram) ProtoMessage() {} + +func (x *Histogram) ProtoReflect() protoreflect.Message { + mi := &file_grpc_core_stats_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Histogram.ProtoReflect.Descriptor instead. +func (*Histogram) Descriptor() ([]byte, []int) { + return file_grpc_core_stats_proto_rawDescGZIP(), []int{1} +} + +func (x *Histogram) GetBuckets() []*Bucket { + if x != nil { + return x.Buckets + } + return nil +} + +type Metric struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Types that are valid to be assigned to Value: + // + // *Metric_Count + // *Metric_Histogram + Value isMetric_Value `protobuf_oneof:"value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Metric) Reset() { + *x = Metric{} + mi := &file_grpc_core_stats_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Metric) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Metric) ProtoMessage() {} + +func (x *Metric) ProtoReflect() protoreflect.Message { + mi := &file_grpc_core_stats_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Metric.ProtoReflect.Descriptor instead. +func (*Metric) Descriptor() ([]byte, []int) { + return file_grpc_core_stats_proto_rawDescGZIP(), []int{2} +} + +func (x *Metric) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Metric) GetValue() isMetric_Value { + if x != nil { + return x.Value + } + return nil +} + +func (x *Metric) GetCount() uint64 { + if x != nil { + if x, ok := x.Value.(*Metric_Count); ok { + return x.Count + } + } + return 0 +} + +func (x *Metric) GetHistogram() *Histogram { + if x != nil { + if x, ok := x.Value.(*Metric_Histogram); ok { + return x.Histogram + } + } + return nil +} + +type isMetric_Value interface { + isMetric_Value() +} + +type Metric_Count struct { + Count uint64 `protobuf:"varint,10,opt,name=count,proto3,oneof"` +} + +type Metric_Histogram struct { + Histogram *Histogram `protobuf:"bytes,11,opt,name=histogram,proto3,oneof"` +} + +func (*Metric_Count) isMetric_Value() {} + +func (*Metric_Histogram) isMetric_Value() {} + +type Stats struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metrics []*Metric `protobuf:"bytes,1,rep,name=metrics,proto3" json:"metrics,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Stats) Reset() { + *x = Stats{} + mi := &file_grpc_core_stats_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Stats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Stats) ProtoMessage() {} + +func (x *Stats) ProtoReflect() protoreflect.Message { + mi := &file_grpc_core_stats_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Stats.ProtoReflect.Descriptor instead. +func (*Stats) Descriptor() ([]byte, []int) { + return file_grpc_core_stats_proto_rawDescGZIP(), []int{3} +} + +func (x *Stats) GetMetrics() []*Metric { + if x != nil { + return x.Metrics + } + return nil +} + +var File_grpc_core_stats_proto protoreflect.FileDescriptor + +const file_grpc_core_stats_proto_rawDesc = "" + + "\n" + + "\x15grpc/core/stats.proto\x12\tgrpc.core\"4\n" + + "\x06Bucket\x12\x14\n" + + "\x05start\x18\x01 \x01(\x01R\x05start\x12\x14\n" + + "\x05count\x18\x02 \x01(\x04R\x05count\"8\n" + + "\tHistogram\x12+\n" + + "\abuckets\x18\x01 \x03(\v2\x11.grpc.core.BucketR\abuckets\"s\n" + + "\x06Metric\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + + "\x05count\x18\n" + + " \x01(\x04H\x00R\x05count\x124\n" + + "\thistogram\x18\v \x01(\v2\x14.grpc.core.HistogramH\x00R\thistogramB\a\n" + + "\x05value\"4\n" + + "\x05Stats\x12+\n" + + "\ametrics\x18\x01 \x03(\v2\x11.grpc.core.MetricR\ametricsb\x06proto3" + +var ( + file_grpc_core_stats_proto_rawDescOnce sync.Once + file_grpc_core_stats_proto_rawDescData []byte +) + +func file_grpc_core_stats_proto_rawDescGZIP() []byte { + file_grpc_core_stats_proto_rawDescOnce.Do(func() { + file_grpc_core_stats_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_core_stats_proto_rawDesc), len(file_grpc_core_stats_proto_rawDesc))) + }) + return file_grpc_core_stats_proto_rawDescData +} + +var file_grpc_core_stats_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_grpc_core_stats_proto_goTypes = []any{ + (*Bucket)(nil), // 0: grpc.core.Bucket + (*Histogram)(nil), // 1: grpc.core.Histogram + (*Metric)(nil), // 2: grpc.core.Metric + (*Stats)(nil), // 3: grpc.core.Stats +} +var file_grpc_core_stats_proto_depIdxs = []int32{ + 0, // 0: grpc.core.Histogram.buckets:type_name -> grpc.core.Bucket + 1, // 1: grpc.core.Metric.histogram:type_name -> grpc.core.Histogram + 2, // 2: grpc.core.Stats.metrics:type_name -> grpc.core.Metric + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_grpc_core_stats_proto_init() } +func file_grpc_core_stats_proto_init() { + if File_grpc_core_stats_proto != nil { + return + } + file_grpc_core_stats_proto_msgTypes[2].OneofWrappers = []any{ + (*Metric_Count)(nil), + (*Metric_Histogram)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_core_stats_proto_rawDesc), len(file_grpc_core_stats_proto_rawDesc)), + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_core_stats_proto_goTypes, + DependencyIndexes: file_grpc_core_stats_proto_depIdxs, + MessageInfos: file_grpc_core_stats_proto_msgTypes, + }.Build() + File_grpc_core_stats_proto = out.File + file_grpc_core_stats_proto_goTypes = nil + file_grpc_core_stats_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/empty.pb.go b/interop/grpc_testing/empty.pb.go new file mode 100644 index 0000000..cbe6715 --- /dev/null +++ b/interop/grpc_testing/empty.pb.go @@ -0,0 +1,135 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/testing/empty.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// An empty message that you can re-use to avoid defining duplicated empty +// messages in your project. A typical example is to use it as argument or the +// return value of a service API. For instance: +// +// service Foo { +// rpc Bar (grpc.testing.Empty) returns (grpc.testing.Empty) { }; +// }; +type Empty struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Empty) Reset() { + *x = Empty{} + mi := &file_grpc_testing_empty_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Empty) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Empty) ProtoMessage() {} + +func (x *Empty) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_empty_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Empty.ProtoReflect.Descriptor instead. +func (*Empty) Descriptor() ([]byte, []int) { + return file_grpc_testing_empty_proto_rawDescGZIP(), []int{0} +} + +var File_grpc_testing_empty_proto protoreflect.FileDescriptor + +const file_grpc_testing_empty_proto_rawDesc = "" + + "\n" + + "\x18grpc/testing/empty.proto\x12\fgrpc.testing\"\a\n" + + "\x05EmptyB*\n" + + "\x1bio.grpc.testing.integrationB\vEmptyProtosb\x06proto3" + +var ( + file_grpc_testing_empty_proto_rawDescOnce sync.Once + file_grpc_testing_empty_proto_rawDescData []byte +) + +func file_grpc_testing_empty_proto_rawDescGZIP() []byte { + file_grpc_testing_empty_proto_rawDescOnce.Do(func() { + file_grpc_testing_empty_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_testing_empty_proto_rawDesc), len(file_grpc_testing_empty_proto_rawDesc))) + }) + return file_grpc_testing_empty_proto_rawDescData +} + +var file_grpc_testing_empty_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_grpc_testing_empty_proto_goTypes = []any{ + (*Empty)(nil), // 0: grpc.testing.Empty +} +var file_grpc_testing_empty_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_grpc_testing_empty_proto_init() } +func file_grpc_testing_empty_proto_init() { + if File_grpc_testing_empty_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_testing_empty_proto_rawDesc), len(file_grpc_testing_empty_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_testing_empty_proto_goTypes, + DependencyIndexes: file_grpc_testing_empty_proto_depIdxs, + MessageInfos: file_grpc_testing_empty_proto_msgTypes, + }.Build() + File_grpc_testing_empty_proto = out.File + file_grpc_testing_empty_proto_goTypes = nil + file_grpc_testing_empty_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/messages.pb.go b/interop/grpc_testing/messages.pb.go new file mode 100644 index 0000000..9bd86f1 --- /dev/null +++ b/interop/grpc_testing/messages.pb.go @@ -0,0 +1,2369 @@ +// Copyright 2015-2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Message definitions to be used by integration test service definitions. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/testing/messages.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// The type of payload that should be returned. +type PayloadType int32 + +const ( + // Compressable text format. + PayloadType_COMPRESSABLE PayloadType = 0 +) + +// Enum value maps for PayloadType. +var ( + PayloadType_name = map[int32]string{ + 0: "COMPRESSABLE", + } + PayloadType_value = map[string]int32{ + "COMPRESSABLE": 0, + } +) + +func (x PayloadType) Enum() *PayloadType { + p := new(PayloadType) + *p = x + return p +} + +func (x PayloadType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (PayloadType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_messages_proto_enumTypes[0].Descriptor() +} + +func (PayloadType) Type() protoreflect.EnumType { + return &file_grpc_testing_messages_proto_enumTypes[0] +} + +func (x PayloadType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use PayloadType.Descriptor instead. +func (PayloadType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{0} +} + +// The type of route that a client took to reach a server w.r.t. gRPCLB. +// The server must fill in "fallback" if it detects that the RPC reached +// the server via the "gRPCLB fallback" path, and "backend" if it detects +// that the RPC reached the server via "gRPCLB backend" path (i.e. if it got +// the address of this server from the gRPCLB server BalanceLoad RPC). Exactly +// how this detection is done is context and server dependent. +type GrpclbRouteType int32 + +const ( + // Server didn't detect the route that a client took to reach it. + GrpclbRouteType_GRPCLB_ROUTE_TYPE_UNKNOWN GrpclbRouteType = 0 + // Indicates that a client reached a server via gRPCLB fallback. + GrpclbRouteType_GRPCLB_ROUTE_TYPE_FALLBACK GrpclbRouteType = 1 + // Indicates that a client reached a server as a gRPCLB-given backend. + GrpclbRouteType_GRPCLB_ROUTE_TYPE_BACKEND GrpclbRouteType = 2 +) + +// Enum value maps for GrpclbRouteType. +var ( + GrpclbRouteType_name = map[int32]string{ + 0: "GRPCLB_ROUTE_TYPE_UNKNOWN", + 1: "GRPCLB_ROUTE_TYPE_FALLBACK", + 2: "GRPCLB_ROUTE_TYPE_BACKEND", + } + GrpclbRouteType_value = map[string]int32{ + "GRPCLB_ROUTE_TYPE_UNKNOWN": 0, + "GRPCLB_ROUTE_TYPE_FALLBACK": 1, + "GRPCLB_ROUTE_TYPE_BACKEND": 2, + } +) + +func (x GrpclbRouteType) Enum() *GrpclbRouteType { + p := new(GrpclbRouteType) + *p = x + return p +} + +func (x GrpclbRouteType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GrpclbRouteType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_messages_proto_enumTypes[1].Descriptor() +} + +func (GrpclbRouteType) Type() protoreflect.EnumType { + return &file_grpc_testing_messages_proto_enumTypes[1] +} + +func (x GrpclbRouteType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GrpclbRouteType.Descriptor instead. +func (GrpclbRouteType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{1} +} + +type LoadBalancerStatsResponse_MetadataType int32 + +const ( + LoadBalancerStatsResponse_UNKNOWN LoadBalancerStatsResponse_MetadataType = 0 + LoadBalancerStatsResponse_INITIAL LoadBalancerStatsResponse_MetadataType = 1 + LoadBalancerStatsResponse_TRAILING LoadBalancerStatsResponse_MetadataType = 2 +) + +// Enum value maps for LoadBalancerStatsResponse_MetadataType. +var ( + LoadBalancerStatsResponse_MetadataType_name = map[int32]string{ + 0: "UNKNOWN", + 1: "INITIAL", + 2: "TRAILING", + } + LoadBalancerStatsResponse_MetadataType_value = map[string]int32{ + "UNKNOWN": 0, + "INITIAL": 1, + "TRAILING": 2, + } +) + +func (x LoadBalancerStatsResponse_MetadataType) Enum() *LoadBalancerStatsResponse_MetadataType { + p := new(LoadBalancerStatsResponse_MetadataType) + *p = x + return p +} + +func (x LoadBalancerStatsResponse_MetadataType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (LoadBalancerStatsResponse_MetadataType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_messages_proto_enumTypes[2].Descriptor() +} + +func (LoadBalancerStatsResponse_MetadataType) Type() protoreflect.EnumType { + return &file_grpc_testing_messages_proto_enumTypes[2] +} + +func (x LoadBalancerStatsResponse_MetadataType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use LoadBalancerStatsResponse_MetadataType.Descriptor instead. +func (LoadBalancerStatsResponse_MetadataType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13, 0} +} + +// Type of RPCs to send. +type ClientConfigureRequest_RpcType int32 + +const ( + ClientConfigureRequest_EMPTY_CALL ClientConfigureRequest_RpcType = 0 + ClientConfigureRequest_UNARY_CALL ClientConfigureRequest_RpcType = 1 +) + +// Enum value maps for ClientConfigureRequest_RpcType. +var ( + ClientConfigureRequest_RpcType_name = map[int32]string{ + 0: "EMPTY_CALL", + 1: "UNARY_CALL", + } + ClientConfigureRequest_RpcType_value = map[string]int32{ + "EMPTY_CALL": 0, + "UNARY_CALL": 1, + } +) + +func (x ClientConfigureRequest_RpcType) Enum() *ClientConfigureRequest_RpcType { + p := new(ClientConfigureRequest_RpcType) + *p = x + return p +} + +func (x ClientConfigureRequest_RpcType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ClientConfigureRequest_RpcType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_messages_proto_enumTypes[3].Descriptor() +} + +func (ClientConfigureRequest_RpcType) Type() protoreflect.EnumType { + return &file_grpc_testing_messages_proto_enumTypes[3] +} + +func (x ClientConfigureRequest_RpcType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ClientConfigureRequest_RpcType.Descriptor instead. +func (ClientConfigureRequest_RpcType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{16, 0} +} + +type HookRequest_HookRequestCommand int32 + +const ( + // Default value + HookRequest_UNSPECIFIED HookRequest_HookRequestCommand = 0 + // Start the HTTP endpoint + HookRequest_START HookRequest_HookRequestCommand = 1 + // Stop + HookRequest_STOP HookRequest_HookRequestCommand = 2 + // Return from HTTP GET/POST + HookRequest_RETURN HookRequest_HookRequestCommand = 3 +) + +// Enum value maps for HookRequest_HookRequestCommand. +var ( + HookRequest_HookRequestCommand_name = map[int32]string{ + 0: "UNSPECIFIED", + 1: "START", + 2: "STOP", + 3: "RETURN", + } + HookRequest_HookRequestCommand_value = map[string]int32{ + "UNSPECIFIED": 0, + "START": 1, + "STOP": 2, + "RETURN": 3, + } +) + +func (x HookRequest_HookRequestCommand) Enum() *HookRequest_HookRequestCommand { + p := new(HookRequest_HookRequestCommand) + *p = x + return p +} + +func (x HookRequest_HookRequestCommand) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (HookRequest_HookRequestCommand) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_messages_proto_enumTypes[4].Descriptor() +} + +func (HookRequest_HookRequestCommand) Type() protoreflect.EnumType { + return &file_grpc_testing_messages_proto_enumTypes[4] +} + +func (x HookRequest_HookRequestCommand) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use HookRequest_HookRequestCommand.Descriptor instead. +func (HookRequest_HookRequestCommand) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{21, 0} +} + +// TODO(dgq): Go back to using well-known types once +// https://github.com/grpc/grpc/issues/6980 has been fixed. +// import "google/protobuf/wrappers.proto"; +type BoolValue struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The bool value. + Value bool `protobuf:"varint,1,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BoolValue) Reset() { + *x = BoolValue{} + mi := &file_grpc_testing_messages_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BoolValue) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BoolValue) ProtoMessage() {} + +func (x *BoolValue) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BoolValue.ProtoReflect.Descriptor instead. +func (*BoolValue) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{0} +} + +func (x *BoolValue) GetValue() bool { + if x != nil { + return x.Value + } + return false +} + +// A block of data, to simply increase gRPC message size. +type Payload struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The type of data in body. + Type PayloadType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.PayloadType" json:"type,omitempty"` + // Primary contents of payload. + Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Payload) Reset() { + *x = Payload{} + mi := &file_grpc_testing_messages_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Payload) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Payload) ProtoMessage() {} + +func (x *Payload) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Payload.ProtoReflect.Descriptor instead. +func (*Payload) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{1} +} + +func (x *Payload) GetType() PayloadType { + if x != nil { + return x.Type + } + return PayloadType_COMPRESSABLE +} + +func (x *Payload) GetBody() []byte { + if x != nil { + return x.Body + } + return nil +} + +// A protobuf representation for grpc status. This is used by test +// clients to specify a status that the server should attempt to return. +type EchoStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + Code int32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EchoStatus) Reset() { + *x = EchoStatus{} + mi := &file_grpc_testing_messages_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EchoStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EchoStatus) ProtoMessage() {} + +func (x *EchoStatus) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EchoStatus.ProtoReflect.Descriptor instead. +func (*EchoStatus) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{2} +} + +func (x *EchoStatus) GetCode() int32 { + if x != nil { + return x.Code + } + return 0 +} + +func (x *EchoStatus) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +// Unary request. +type SimpleRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Desired payload type in the response from the server. + // If response_type is RANDOM, server randomly chooses one from other formats. + ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"` + // Desired payload size in the response from the server. + ResponseSize int32 `protobuf:"varint,2,opt,name=response_size,json=responseSize,proto3" json:"response_size,omitempty"` + // Optional input payload sent along with the request. + Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` + // Whether SimpleResponse should include username. + FillUsername bool `protobuf:"varint,4,opt,name=fill_username,json=fillUsername,proto3" json:"fill_username,omitempty"` + // Whether SimpleResponse should include OAuth scope. + FillOauthScope bool `protobuf:"varint,5,opt,name=fill_oauth_scope,json=fillOauthScope,proto3" json:"fill_oauth_scope,omitempty"` + // Whether to request the server to compress the response. This field is + // "nullable" in order to interoperate seamlessly with clients not able to + // implement the full compression tests by introspecting the call to verify + // the response's compression status. + ResponseCompressed *BoolValue `protobuf:"bytes,6,opt,name=response_compressed,json=responseCompressed,proto3" json:"response_compressed,omitempty"` + // Whether server should return a given status + ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"` + // Whether the server should expect this request to be compressed. + ExpectCompressed *BoolValue `protobuf:"bytes,8,opt,name=expect_compressed,json=expectCompressed,proto3" json:"expect_compressed,omitempty"` + // Whether SimpleResponse should include server_id. + FillServerId bool `protobuf:"varint,9,opt,name=fill_server_id,json=fillServerId,proto3" json:"fill_server_id,omitempty"` + // Whether SimpleResponse should include grpclb_route_type. + FillGrpclbRouteType bool `protobuf:"varint,10,opt,name=fill_grpclb_route_type,json=fillGrpclbRouteType,proto3" json:"fill_grpclb_route_type,omitempty"` + // If set the server should record this metrics report data for the current RPC. + OrcaPerQueryReport *TestOrcaReport `protobuf:"bytes,11,opt,name=orca_per_query_report,json=orcaPerQueryReport,proto3" json:"orca_per_query_report,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SimpleRequest) Reset() { + *x = SimpleRequest{} + mi := &file_grpc_testing_messages_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SimpleRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SimpleRequest) ProtoMessage() {} + +func (x *SimpleRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SimpleRequest.ProtoReflect.Descriptor instead. +func (*SimpleRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{3} +} + +func (x *SimpleRequest) GetResponseType() PayloadType { + if x != nil { + return x.ResponseType + } + return PayloadType_COMPRESSABLE +} + +func (x *SimpleRequest) GetResponseSize() int32 { + if x != nil { + return x.ResponseSize + } + return 0 +} + +func (x *SimpleRequest) GetPayload() *Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *SimpleRequest) GetFillUsername() bool { + if x != nil { + return x.FillUsername + } + return false +} + +func (x *SimpleRequest) GetFillOauthScope() bool { + if x != nil { + return x.FillOauthScope + } + return false +} + +func (x *SimpleRequest) GetResponseCompressed() *BoolValue { + if x != nil { + return x.ResponseCompressed + } + return nil +} + +func (x *SimpleRequest) GetResponseStatus() *EchoStatus { + if x != nil { + return x.ResponseStatus + } + return nil +} + +func (x *SimpleRequest) GetExpectCompressed() *BoolValue { + if x != nil { + return x.ExpectCompressed + } + return nil +} + +func (x *SimpleRequest) GetFillServerId() bool { + if x != nil { + return x.FillServerId + } + return false +} + +func (x *SimpleRequest) GetFillGrpclbRouteType() bool { + if x != nil { + return x.FillGrpclbRouteType + } + return false +} + +func (x *SimpleRequest) GetOrcaPerQueryReport() *TestOrcaReport { + if x != nil { + return x.OrcaPerQueryReport + } + return nil +} + +// Unary response, as configured by the request. +type SimpleResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Payload to increase message size. + Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` + // The user the request came from, for verifying authentication was + // successful when the client expected it. + Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"` + // OAuth scope. + OauthScope string `protobuf:"bytes,3,opt,name=oauth_scope,json=oauthScope,proto3" json:"oauth_scope,omitempty"` + // Server ID. This must be unique among different server instances, + // but the same across all RPC's made to a particular server instance. + ServerId string `protobuf:"bytes,4,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"` + // gRPCLB Path. + GrpclbRouteType GrpclbRouteType `protobuf:"varint,5,opt,name=grpclb_route_type,json=grpclbRouteType,proto3,enum=grpc.testing.GrpclbRouteType" json:"grpclb_route_type,omitempty"` + // Server hostname. + Hostname string `protobuf:"bytes,6,opt,name=hostname,proto3" json:"hostname,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SimpleResponse) Reset() { + *x = SimpleResponse{} + mi := &file_grpc_testing_messages_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SimpleResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SimpleResponse) ProtoMessage() {} + +func (x *SimpleResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SimpleResponse.ProtoReflect.Descriptor instead. +func (*SimpleResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{4} +} + +func (x *SimpleResponse) GetPayload() *Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *SimpleResponse) GetUsername() string { + if x != nil { + return x.Username + } + return "" +} + +func (x *SimpleResponse) GetOauthScope() string { + if x != nil { + return x.OauthScope + } + return "" +} + +func (x *SimpleResponse) GetServerId() string { + if x != nil { + return x.ServerId + } + return "" +} + +func (x *SimpleResponse) GetGrpclbRouteType() GrpclbRouteType { + if x != nil { + return x.GrpclbRouteType + } + return GrpclbRouteType_GRPCLB_ROUTE_TYPE_UNKNOWN +} + +func (x *SimpleResponse) GetHostname() string { + if x != nil { + return x.Hostname + } + return "" +} + +// Client-streaming request. +type StreamingInputCallRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Optional input payload sent along with the request. + Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` + // Whether the server should expect this request to be compressed. This field + // is "nullable" in order to interoperate seamlessly with servers not able to + // implement the full compression tests by introspecting the call to verify + // the request's compression status. + ExpectCompressed *BoolValue `protobuf:"bytes,2,opt,name=expect_compressed,json=expectCompressed,proto3" json:"expect_compressed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamingInputCallRequest) Reset() { + *x = StreamingInputCallRequest{} + mi := &file_grpc_testing_messages_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamingInputCallRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamingInputCallRequest) ProtoMessage() {} + +func (x *StreamingInputCallRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamingInputCallRequest.ProtoReflect.Descriptor instead. +func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{5} +} + +func (x *StreamingInputCallRequest) GetPayload() *Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *StreamingInputCallRequest) GetExpectCompressed() *BoolValue { + if x != nil { + return x.ExpectCompressed + } + return nil +} + +// Client-streaming response. +type StreamingInputCallResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Aggregated size of payloads received from the client. + AggregatedPayloadSize int32 `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamingInputCallResponse) Reset() { + *x = StreamingInputCallResponse{} + mi := &file_grpc_testing_messages_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamingInputCallResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamingInputCallResponse) ProtoMessage() {} + +func (x *StreamingInputCallResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamingInputCallResponse.ProtoReflect.Descriptor instead. +func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{6} +} + +func (x *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 { + if x != nil { + return x.AggregatedPayloadSize + } + return 0 +} + +// Configuration for a particular response. +type ResponseParameters struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Desired payload sizes in responses from the server. + Size int32 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"` + // Desired interval between consecutive responses in the response stream in + // microseconds. + IntervalUs int32 `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"` + // Whether to request the server to compress the response. This field is + // "nullable" in order to interoperate seamlessly with clients not able to + // implement the full compression tests by introspecting the call to verify + // the response's compression status. + Compressed *BoolValue `protobuf:"bytes,3,opt,name=compressed,proto3" json:"compressed,omitempty"` + // Whether to request the server to send the requesting peer's socket + // address in the response. + FillPeerSocketAddress *BoolValue `protobuf:"bytes,4,opt,name=fill_peer_socket_address,json=fillPeerSocketAddress,proto3" json:"fill_peer_socket_address,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResponseParameters) Reset() { + *x = ResponseParameters{} + mi := &file_grpc_testing_messages_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResponseParameters) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResponseParameters) ProtoMessage() {} + +func (x *ResponseParameters) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResponseParameters.ProtoReflect.Descriptor instead. +func (*ResponseParameters) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{7} +} + +func (x *ResponseParameters) GetSize() int32 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *ResponseParameters) GetIntervalUs() int32 { + if x != nil { + return x.IntervalUs + } + return 0 +} + +func (x *ResponseParameters) GetCompressed() *BoolValue { + if x != nil { + return x.Compressed + } + return nil +} + +func (x *ResponseParameters) GetFillPeerSocketAddress() *BoolValue { + if x != nil { + return x.FillPeerSocketAddress + } + return nil +} + +// Server-streaming request. +type StreamingOutputCallRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Desired payload type in the response from the server. + // If response_type is RANDOM, the payload from each response in the stream + // might be of different types. This is to simulate a mixed type of payload + // stream. + ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"` + // Configuration for each expected response message. + ResponseParameters []*ResponseParameters `protobuf:"bytes,2,rep,name=response_parameters,json=responseParameters,proto3" json:"response_parameters,omitempty"` + // Optional input payload sent along with the request. + Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` + // Whether server should return a given status + ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"` + // If set the server should update this metrics report data at the OOB server. + OrcaOobReport *TestOrcaReport `protobuf:"bytes,8,opt,name=orca_oob_report,json=orcaOobReport,proto3" json:"orca_oob_report,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamingOutputCallRequest) Reset() { + *x = StreamingOutputCallRequest{} + mi := &file_grpc_testing_messages_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamingOutputCallRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamingOutputCallRequest) ProtoMessage() {} + +func (x *StreamingOutputCallRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamingOutputCallRequest.ProtoReflect.Descriptor instead. +func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{8} +} + +func (x *StreamingOutputCallRequest) GetResponseType() PayloadType { + if x != nil { + return x.ResponseType + } + return PayloadType_COMPRESSABLE +} + +func (x *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters { + if x != nil { + return x.ResponseParameters + } + return nil +} + +func (x *StreamingOutputCallRequest) GetPayload() *Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *StreamingOutputCallRequest) GetResponseStatus() *EchoStatus { + if x != nil { + return x.ResponseStatus + } + return nil +} + +func (x *StreamingOutputCallRequest) GetOrcaOobReport() *TestOrcaReport { + if x != nil { + return x.OrcaOobReport + } + return nil +} + +// Server-streaming response, as configured by the request and parameters. +type StreamingOutputCallResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Payload to increase response size. + Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` + // The peer's socket address if requested. + PeerSocketAddress string `protobuf:"bytes,2,opt,name=peer_socket_address,json=peerSocketAddress,proto3" json:"peer_socket_address,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamingOutputCallResponse) Reset() { + *x = StreamingOutputCallResponse{} + mi := &file_grpc_testing_messages_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamingOutputCallResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamingOutputCallResponse) ProtoMessage() {} + +func (x *StreamingOutputCallResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamingOutputCallResponse.ProtoReflect.Descriptor instead. +func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{9} +} + +func (x *StreamingOutputCallResponse) GetPayload() *Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *StreamingOutputCallResponse) GetPeerSocketAddress() string { + if x != nil { + return x.PeerSocketAddress + } + return "" +} + +// For reconnect interop test only. +// Client tells server what reconnection parameters it used. +type ReconnectParams struct { + state protoimpl.MessageState `protogen:"open.v1"` + MaxReconnectBackoffMs int32 `protobuf:"varint,1,opt,name=max_reconnect_backoff_ms,json=maxReconnectBackoffMs,proto3" json:"max_reconnect_backoff_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReconnectParams) Reset() { + *x = ReconnectParams{} + mi := &file_grpc_testing_messages_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReconnectParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReconnectParams) ProtoMessage() {} + +func (x *ReconnectParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReconnectParams.ProtoReflect.Descriptor instead. +func (*ReconnectParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{10} +} + +func (x *ReconnectParams) GetMaxReconnectBackoffMs() int32 { + if x != nil { + return x.MaxReconnectBackoffMs + } + return 0 +} + +// For reconnect interop test only. +// Server tells client whether its reconnects are following the spec and the +// reconnect backoffs it saw. +type ReconnectInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + Passed bool `protobuf:"varint,1,opt,name=passed,proto3" json:"passed,omitempty"` + BackoffMs []int32 `protobuf:"varint,2,rep,packed,name=backoff_ms,json=backoffMs,proto3" json:"backoff_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReconnectInfo) Reset() { + *x = ReconnectInfo{} + mi := &file_grpc_testing_messages_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReconnectInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReconnectInfo) ProtoMessage() {} + +func (x *ReconnectInfo) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReconnectInfo.ProtoReflect.Descriptor instead. +func (*ReconnectInfo) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{11} +} + +func (x *ReconnectInfo) GetPassed() bool { + if x != nil { + return x.Passed + } + return false +} + +func (x *ReconnectInfo) GetBackoffMs() []int32 { + if x != nil { + return x.BackoffMs + } + return nil +} + +type LoadBalancerStatsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Request stats for the next num_rpcs sent by client. + NumRpcs int32 `protobuf:"varint,1,opt,name=num_rpcs,json=numRpcs,proto3" json:"num_rpcs,omitempty"` + // If num_rpcs have not completed within timeout_sec, return partial results. + TimeoutSec int32 `protobuf:"varint,2,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"` + // Response header + trailer metadata entries we want the values of. + // Matching of the keys is case-insensitive as per rfc7540#section-8.1.2 + // * (asterisk) is a special value that will return all metadata entries + MetadataKeys []string `protobuf:"bytes,3,rep,name=metadata_keys,json=metadataKeys,proto3" json:"metadata_keys,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoadBalancerStatsRequest) Reset() { + *x = LoadBalancerStatsRequest{} + mi := &file_grpc_testing_messages_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoadBalancerStatsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerStatsRequest) ProtoMessage() {} + +func (x *LoadBalancerStatsRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerStatsRequest.ProtoReflect.Descriptor instead. +func (*LoadBalancerStatsRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{12} +} + +func (x *LoadBalancerStatsRequest) GetNumRpcs() int32 { + if x != nil { + return x.NumRpcs + } + return 0 +} + +func (x *LoadBalancerStatsRequest) GetTimeoutSec() int32 { + if x != nil { + return x.TimeoutSec + } + return 0 +} + +func (x *LoadBalancerStatsRequest) GetMetadataKeys() []string { + if x != nil { + return x.MetadataKeys + } + return nil +} + +type LoadBalancerStatsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The number of completed RPCs for each peer. + RpcsByPeer map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` + // The number of RPCs that failed to record a remote peer. + NumFailures int32 `protobuf:"varint,2,opt,name=num_failures,json=numFailures,proto3" json:"num_failures,omitempty"` + RpcsByMethod map[string]*LoadBalancerStatsResponse_RpcsByPeer `protobuf:"bytes,3,rep,name=rpcs_by_method,json=rpcsByMethod,proto3" json:"rpcs_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // All the metadata of all RPCs for each peer. + MetadatasByPeer map[string]*LoadBalancerStatsResponse_MetadataByPeer `protobuf:"bytes,4,rep,name=metadatas_by_peer,json=metadatasByPeer,proto3" json:"metadatas_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoadBalancerStatsResponse) Reset() { + *x = LoadBalancerStatsResponse{} + mi := &file_grpc_testing_messages_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoadBalancerStatsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerStatsResponse) ProtoMessage() {} + +func (x *LoadBalancerStatsResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerStatsResponse.ProtoReflect.Descriptor instead. +func (*LoadBalancerStatsResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13} +} + +func (x *LoadBalancerStatsResponse) GetRpcsByPeer() map[string]int32 { + if x != nil { + return x.RpcsByPeer + } + return nil +} + +func (x *LoadBalancerStatsResponse) GetNumFailures() int32 { + if x != nil { + return x.NumFailures + } + return 0 +} + +func (x *LoadBalancerStatsResponse) GetRpcsByMethod() map[string]*LoadBalancerStatsResponse_RpcsByPeer { + if x != nil { + return x.RpcsByMethod + } + return nil +} + +func (x *LoadBalancerStatsResponse) GetMetadatasByPeer() map[string]*LoadBalancerStatsResponse_MetadataByPeer { + if x != nil { + return x.MetadatasByPeer + } + return nil +} + +// Request for retrieving a test client's accumulated stats. +type LoadBalancerAccumulatedStatsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoadBalancerAccumulatedStatsRequest) Reset() { + *x = LoadBalancerAccumulatedStatsRequest{} + mi := &file_grpc_testing_messages_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoadBalancerAccumulatedStatsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerAccumulatedStatsRequest) ProtoMessage() {} + +func (x *LoadBalancerAccumulatedStatsRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerAccumulatedStatsRequest.ProtoReflect.Descriptor instead. +func (*LoadBalancerAccumulatedStatsRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{14} +} + +// Accumulated stats for RPCs sent by a test client. +type LoadBalancerAccumulatedStatsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The total number of RPCs have ever issued for each type. + // Deprecated: use stats_per_method.rpcs_started instead. + // + // Deprecated: Marked as deprecated in grpc/testing/messages.proto. + NumRpcsStartedByMethod map[string]int32 `protobuf:"bytes,1,rep,name=num_rpcs_started_by_method,json=numRpcsStartedByMethod,proto3" json:"num_rpcs_started_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` + // The total number of RPCs have ever completed successfully for each type. + // Deprecated: use stats_per_method.result instead. + // + // Deprecated: Marked as deprecated in grpc/testing/messages.proto. + NumRpcsSucceededByMethod map[string]int32 `protobuf:"bytes,2,rep,name=num_rpcs_succeeded_by_method,json=numRpcsSucceededByMethod,proto3" json:"num_rpcs_succeeded_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` + // The total number of RPCs have ever failed for each type. + // Deprecated: use stats_per_method.result instead. + // + // Deprecated: Marked as deprecated in grpc/testing/messages.proto. + NumRpcsFailedByMethod map[string]int32 `protobuf:"bytes,3,rep,name=num_rpcs_failed_by_method,json=numRpcsFailedByMethod,proto3" json:"num_rpcs_failed_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` + // Per-method RPC statistics. The key is the RpcType in string form; e.g. + // 'EMPTY_CALL' or 'UNARY_CALL' + StatsPerMethod map[string]*LoadBalancerAccumulatedStatsResponse_MethodStats `protobuf:"bytes,4,rep,name=stats_per_method,json=statsPerMethod,proto3" json:"stats_per_method,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoadBalancerAccumulatedStatsResponse) Reset() { + *x = LoadBalancerAccumulatedStatsResponse{} + mi := &file_grpc_testing_messages_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoadBalancerAccumulatedStatsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerAccumulatedStatsResponse) ProtoMessage() {} + +func (x *LoadBalancerAccumulatedStatsResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerAccumulatedStatsResponse.ProtoReflect.Descriptor instead. +func (*LoadBalancerAccumulatedStatsResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{15} +} + +// Deprecated: Marked as deprecated in grpc/testing/messages.proto. +func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsStartedByMethod() map[string]int32 { + if x != nil { + return x.NumRpcsStartedByMethod + } + return nil +} + +// Deprecated: Marked as deprecated in grpc/testing/messages.proto. +func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsSucceededByMethod() map[string]int32 { + if x != nil { + return x.NumRpcsSucceededByMethod + } + return nil +} + +// Deprecated: Marked as deprecated in grpc/testing/messages.proto. +func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsFailedByMethod() map[string]int32 { + if x != nil { + return x.NumRpcsFailedByMethod + } + return nil +} + +func (x *LoadBalancerAccumulatedStatsResponse) GetStatsPerMethod() map[string]*LoadBalancerAccumulatedStatsResponse_MethodStats { + if x != nil { + return x.StatsPerMethod + } + return nil +} + +// Configurations for a test client. +type ClientConfigureRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The types of RPCs the client sends. + Types []ClientConfigureRequest_RpcType `protobuf:"varint,1,rep,packed,name=types,proto3,enum=grpc.testing.ClientConfigureRequest_RpcType" json:"types,omitempty"` + // The collection of custom metadata to be attached to RPCs sent by the client. + Metadata []*ClientConfigureRequest_Metadata `protobuf:"bytes,2,rep,name=metadata,proto3" json:"metadata,omitempty"` + // The deadline to use, in seconds, for all RPCs. If unset or zero, the + // client will use the default from the command-line. + TimeoutSec int32 `protobuf:"varint,3,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClientConfigureRequest) Reset() { + *x = ClientConfigureRequest{} + mi := &file_grpc_testing_messages_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClientConfigureRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientConfigureRequest) ProtoMessage() {} + +func (x *ClientConfigureRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientConfigureRequest.ProtoReflect.Descriptor instead. +func (*ClientConfigureRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{16} +} + +func (x *ClientConfigureRequest) GetTypes() []ClientConfigureRequest_RpcType { + if x != nil { + return x.Types + } + return nil +} + +func (x *ClientConfigureRequest) GetMetadata() []*ClientConfigureRequest_Metadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *ClientConfigureRequest) GetTimeoutSec() int32 { + if x != nil { + return x.TimeoutSec + } + return 0 +} + +// Response for updating a test client's configuration. +type ClientConfigureResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClientConfigureResponse) Reset() { + *x = ClientConfigureResponse{} + mi := &file_grpc_testing_messages_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClientConfigureResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientConfigureResponse) ProtoMessage() {} + +func (x *ClientConfigureResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientConfigureResponse.ProtoReflect.Descriptor instead. +func (*ClientConfigureResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{17} +} + +type MemorySize struct { + state protoimpl.MessageState `protogen:"open.v1"` + Rss int64 `protobuf:"varint,1,opt,name=rss,proto3" json:"rss,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MemorySize) Reset() { + *x = MemorySize{} + mi := &file_grpc_testing_messages_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MemorySize) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MemorySize) ProtoMessage() {} + +func (x *MemorySize) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MemorySize.ProtoReflect.Descriptor instead. +func (*MemorySize) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{18} +} + +func (x *MemorySize) GetRss() int64 { + if x != nil { + return x.Rss + } + return 0 +} + +// Metrics data the server will update and send to the client. It mirrors orca load report +// https://github.com/cncf/xds/blob/eded343319d09f30032952beda9840bbd3dcf7ac/xds/data/orca/v3/orca_load_report.proto#L15, +// but avoids orca dependency. Used by both per-query and out-of-band reporting tests. +type TestOrcaReport struct { + state protoimpl.MessageState `protogen:"open.v1"` + CpuUtilization float64 `protobuf:"fixed64,1,opt,name=cpu_utilization,json=cpuUtilization,proto3" json:"cpu_utilization,omitempty"` + MemoryUtilization float64 `protobuf:"fixed64,2,opt,name=memory_utilization,json=memoryUtilization,proto3" json:"memory_utilization,omitempty"` + RequestCost map[string]float64 `protobuf:"bytes,3,rep,name=request_cost,json=requestCost,proto3" json:"request_cost,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"fixed64,2,opt,name=value"` + Utilization map[string]float64 `protobuf:"bytes,4,rep,name=utilization,proto3" json:"utilization,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"fixed64,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TestOrcaReport) Reset() { + *x = TestOrcaReport{} + mi := &file_grpc_testing_messages_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TestOrcaReport) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TestOrcaReport) ProtoMessage() {} + +func (x *TestOrcaReport) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TestOrcaReport.ProtoReflect.Descriptor instead. +func (*TestOrcaReport) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{19} +} + +func (x *TestOrcaReport) GetCpuUtilization() float64 { + if x != nil { + return x.CpuUtilization + } + return 0 +} + +func (x *TestOrcaReport) GetMemoryUtilization() float64 { + if x != nil { + return x.MemoryUtilization + } + return 0 +} + +func (x *TestOrcaReport) GetRequestCost() map[string]float64 { + if x != nil { + return x.RequestCost + } + return nil +} + +func (x *TestOrcaReport) GetUtilization() map[string]float64 { + if x != nil { + return x.Utilization + } + return nil +} + +// Status that will be return to callers of the Hook method +type SetReturnStatusRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + GrpcCodeToReturn int32 `protobuf:"varint,1,opt,name=grpc_code_to_return,json=grpcCodeToReturn,proto3" json:"grpc_code_to_return,omitempty"` + GrpcStatusDescription string `protobuf:"bytes,2,opt,name=grpc_status_description,json=grpcStatusDescription,proto3" json:"grpc_status_description,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetReturnStatusRequest) Reset() { + *x = SetReturnStatusRequest{} + mi := &file_grpc_testing_messages_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetReturnStatusRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetReturnStatusRequest) ProtoMessage() {} + +func (x *SetReturnStatusRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetReturnStatusRequest.ProtoReflect.Descriptor instead. +func (*SetReturnStatusRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{20} +} + +func (x *SetReturnStatusRequest) GetGrpcCodeToReturn() int32 { + if x != nil { + return x.GrpcCodeToReturn + } + return 0 +} + +func (x *SetReturnStatusRequest) GetGrpcStatusDescription() string { + if x != nil { + return x.GrpcStatusDescription + } + return "" +} + +type HookRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Command HookRequest_HookRequestCommand `protobuf:"varint,1,opt,name=command,proto3,enum=grpc.testing.HookRequest_HookRequestCommand" json:"command,omitempty"` + GrpcCodeToReturn int32 `protobuf:"varint,2,opt,name=grpc_code_to_return,json=grpcCodeToReturn,proto3" json:"grpc_code_to_return,omitempty"` + GrpcStatusDescription string `protobuf:"bytes,3,opt,name=grpc_status_description,json=grpcStatusDescription,proto3" json:"grpc_status_description,omitempty"` + // Server port to listen to + ServerPort int32 `protobuf:"varint,4,opt,name=server_port,json=serverPort,proto3" json:"server_port,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HookRequest) Reset() { + *x = HookRequest{} + mi := &file_grpc_testing_messages_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HookRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HookRequest) ProtoMessage() {} + +func (x *HookRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HookRequest.ProtoReflect.Descriptor instead. +func (*HookRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{21} +} + +func (x *HookRequest) GetCommand() HookRequest_HookRequestCommand { + if x != nil { + return x.Command + } + return HookRequest_UNSPECIFIED +} + +func (x *HookRequest) GetGrpcCodeToReturn() int32 { + if x != nil { + return x.GrpcCodeToReturn + } + return 0 +} + +func (x *HookRequest) GetGrpcStatusDescription() string { + if x != nil { + return x.GrpcStatusDescription + } + return "" +} + +func (x *HookRequest) GetServerPort() int32 { + if x != nil { + return x.ServerPort + } + return 0 +} + +type HookResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HookResponse) Reset() { + *x = HookResponse{} + mi := &file_grpc_testing_messages_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HookResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HookResponse) ProtoMessage() {} + +func (x *HookResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HookResponse.ProtoReflect.Descriptor instead. +func (*HookResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{22} +} + +type LoadBalancerStatsResponse_MetadataEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Key, exactly as received from the server. Case may be different from what + // was requested in the LoadBalancerStatsRequest) + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // Value, exactly as received from the server. + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + // Metadata type + Type LoadBalancerStatsResponse_MetadataType `protobuf:"varint,3,opt,name=type,proto3,enum=grpc.testing.LoadBalancerStatsResponse_MetadataType" json:"type,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoadBalancerStatsResponse_MetadataEntry) Reset() { + *x = LoadBalancerStatsResponse_MetadataEntry{} + mi := &file_grpc_testing_messages_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoadBalancerStatsResponse_MetadataEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerStatsResponse_MetadataEntry) ProtoMessage() {} + +func (x *LoadBalancerStatsResponse_MetadataEntry) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerStatsResponse_MetadataEntry.ProtoReflect.Descriptor instead. +func (*LoadBalancerStatsResponse_MetadataEntry) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13, 0} +} + +func (x *LoadBalancerStatsResponse_MetadataEntry) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *LoadBalancerStatsResponse_MetadataEntry) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +func (x *LoadBalancerStatsResponse_MetadataEntry) GetType() LoadBalancerStatsResponse_MetadataType { + if x != nil { + return x.Type + } + return LoadBalancerStatsResponse_UNKNOWN +} + +type LoadBalancerStatsResponse_RpcMetadata struct { + state protoimpl.MessageState `protogen:"open.v1"` + // metadata values for each rpc for the keys specified in + // LoadBalancerStatsRequest.metadata_keys. + Metadata []*LoadBalancerStatsResponse_MetadataEntry `protobuf:"bytes,1,rep,name=metadata,proto3" json:"metadata,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoadBalancerStatsResponse_RpcMetadata) Reset() { + *x = LoadBalancerStatsResponse_RpcMetadata{} + mi := &file_grpc_testing_messages_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoadBalancerStatsResponse_RpcMetadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerStatsResponse_RpcMetadata) ProtoMessage() {} + +func (x *LoadBalancerStatsResponse_RpcMetadata) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[24] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerStatsResponse_RpcMetadata.ProtoReflect.Descriptor instead. +func (*LoadBalancerStatsResponse_RpcMetadata) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13, 1} +} + +func (x *LoadBalancerStatsResponse_RpcMetadata) GetMetadata() []*LoadBalancerStatsResponse_MetadataEntry { + if x != nil { + return x.Metadata + } + return nil +} + +type LoadBalancerStatsResponse_MetadataByPeer struct { + state protoimpl.MessageState `protogen:"open.v1"` + // List of RpcMetadata in for each RPC with a given peer + RpcMetadata []*LoadBalancerStatsResponse_RpcMetadata `protobuf:"bytes,1,rep,name=rpc_metadata,json=rpcMetadata,proto3" json:"rpc_metadata,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoadBalancerStatsResponse_MetadataByPeer) Reset() { + *x = LoadBalancerStatsResponse_MetadataByPeer{} + mi := &file_grpc_testing_messages_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoadBalancerStatsResponse_MetadataByPeer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerStatsResponse_MetadataByPeer) ProtoMessage() {} + +func (x *LoadBalancerStatsResponse_MetadataByPeer) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[25] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerStatsResponse_MetadataByPeer.ProtoReflect.Descriptor instead. +func (*LoadBalancerStatsResponse_MetadataByPeer) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13, 2} +} + +func (x *LoadBalancerStatsResponse_MetadataByPeer) GetRpcMetadata() []*LoadBalancerStatsResponse_RpcMetadata { + if x != nil { + return x.RpcMetadata + } + return nil +} + +type LoadBalancerStatsResponse_RpcsByPeer struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The number of completed RPCs for each peer. + RpcsByPeer map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoadBalancerStatsResponse_RpcsByPeer) Reset() { + *x = LoadBalancerStatsResponse_RpcsByPeer{} + mi := &file_grpc_testing_messages_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoadBalancerStatsResponse_RpcsByPeer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerStatsResponse_RpcsByPeer) ProtoMessage() {} + +func (x *LoadBalancerStatsResponse_RpcsByPeer) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[26] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerStatsResponse_RpcsByPeer.ProtoReflect.Descriptor instead. +func (*LoadBalancerStatsResponse_RpcsByPeer) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13, 3} +} + +func (x *LoadBalancerStatsResponse_RpcsByPeer) GetRpcsByPeer() map[string]int32 { + if x != nil { + return x.RpcsByPeer + } + return nil +} + +type LoadBalancerAccumulatedStatsResponse_MethodStats struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The number of RPCs that were started for this method. + RpcsStarted int32 `protobuf:"varint,1,opt,name=rpcs_started,json=rpcsStarted,proto3" json:"rpcs_started,omitempty"` + // The number of RPCs that completed with each status for this method. The + // key is the integral value of a google.rpc.Code; the value is the count. + Result map[int32]int32 `protobuf:"bytes,2,rep,name=result,proto3" json:"result,omitempty" protobuf_key:"varint,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) Reset() { + *x = LoadBalancerAccumulatedStatsResponse_MethodStats{} + mi := &file_grpc_testing_messages_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerAccumulatedStatsResponse_MethodStats) ProtoMessage() {} + +func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[34] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerAccumulatedStatsResponse_MethodStats.ProtoReflect.Descriptor instead. +func (*LoadBalancerAccumulatedStatsResponse_MethodStats) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{15, 3} +} + +func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) GetRpcsStarted() int32 { + if x != nil { + return x.RpcsStarted + } + return 0 +} + +func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) GetResult() map[int32]int32 { + if x != nil { + return x.Result + } + return nil +} + +// Metadata to be attached for the given type of RPCs. +type ClientConfigureRequest_Metadata struct { + state protoimpl.MessageState `protogen:"open.v1"` + Type ClientConfigureRequest_RpcType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.ClientConfigureRequest_RpcType" json:"type,omitempty"` + Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Value string `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClientConfigureRequest_Metadata) Reset() { + *x = ClientConfigureRequest_Metadata{} + mi := &file_grpc_testing_messages_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClientConfigureRequest_Metadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientConfigureRequest_Metadata) ProtoMessage() {} + +func (x *ClientConfigureRequest_Metadata) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[37] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientConfigureRequest_Metadata.ProtoReflect.Descriptor instead. +func (*ClientConfigureRequest_Metadata) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{16, 0} +} + +func (x *ClientConfigureRequest_Metadata) GetType() ClientConfigureRequest_RpcType { + if x != nil { + return x.Type + } + return ClientConfigureRequest_EMPTY_CALL +} + +func (x *ClientConfigureRequest_Metadata) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *ClientConfigureRequest_Metadata) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +var File_grpc_testing_messages_proto protoreflect.FileDescriptor + +const file_grpc_testing_messages_proto_rawDesc = "" + + "\n" + + "\x1bgrpc/testing/messages.proto\x12\fgrpc.testing\"!\n" + + "\tBoolValue\x12\x14\n" + + "\x05value\x18\x01 \x01(\bR\x05value\"L\n" + + "\aPayload\x12-\n" + + "\x04type\x18\x01 \x01(\x0e2\x19.grpc.testing.PayloadTypeR\x04type\x12\x12\n" + + "\x04body\x18\x02 \x01(\fR\x04body\":\n" + + "\n" + + "EchoStatus\x12\x12\n" + + "\x04code\x18\x01 \x01(\x05R\x04code\x12\x18\n" + + "\amessage\x18\x02 \x01(\tR\amessage\"\xf3\x04\n" + + "\rSimpleRequest\x12>\n" + + "\rresponse_type\x18\x01 \x01(\x0e2\x19.grpc.testing.PayloadTypeR\fresponseType\x12#\n" + + "\rresponse_size\x18\x02 \x01(\x05R\fresponseSize\x12/\n" + + "\apayload\x18\x03 \x01(\v2\x15.grpc.testing.PayloadR\apayload\x12#\n" + + "\rfill_username\x18\x04 \x01(\bR\ffillUsername\x12(\n" + + "\x10fill_oauth_scope\x18\x05 \x01(\bR\x0efillOauthScope\x12H\n" + + "\x13response_compressed\x18\x06 \x01(\v2\x17.grpc.testing.BoolValueR\x12responseCompressed\x12A\n" + + "\x0fresponse_status\x18\a \x01(\v2\x18.grpc.testing.EchoStatusR\x0eresponseStatus\x12D\n" + + "\x11expect_compressed\x18\b \x01(\v2\x17.grpc.testing.BoolValueR\x10expectCompressed\x12$\n" + + "\x0efill_server_id\x18\t \x01(\bR\ffillServerId\x123\n" + + "\x16fill_grpclb_route_type\x18\n" + + " \x01(\bR\x13fillGrpclbRouteType\x12O\n" + + "\x15orca_per_query_report\x18\v \x01(\v2\x1c.grpc.testing.TestOrcaReportR\x12orcaPerQueryReport\"\x82\x02\n" + + "\x0eSimpleResponse\x12/\n" + + "\apayload\x18\x01 \x01(\v2\x15.grpc.testing.PayloadR\apayload\x12\x1a\n" + + "\busername\x18\x02 \x01(\tR\busername\x12\x1f\n" + + "\voauth_scope\x18\x03 \x01(\tR\n" + + "oauthScope\x12\x1b\n" + + "\tserver_id\x18\x04 \x01(\tR\bserverId\x12I\n" + + "\x11grpclb_route_type\x18\x05 \x01(\x0e2\x1d.grpc.testing.GrpclbRouteTypeR\x0fgrpclbRouteType\x12\x1a\n" + + "\bhostname\x18\x06 \x01(\tR\bhostname\"\x92\x01\n" + + "\x19StreamingInputCallRequest\x12/\n" + + "\apayload\x18\x01 \x01(\v2\x15.grpc.testing.PayloadR\apayload\x12D\n" + + "\x11expect_compressed\x18\x02 \x01(\v2\x17.grpc.testing.BoolValueR\x10expectCompressed\"T\n" + + "\x1aStreamingInputCallResponse\x126\n" + + "\x17aggregated_payload_size\x18\x01 \x01(\x05R\x15aggregatedPayloadSize\"\xd4\x01\n" + + "\x12ResponseParameters\x12\x12\n" + + "\x04size\x18\x01 \x01(\x05R\x04size\x12\x1f\n" + + "\vinterval_us\x18\x02 \x01(\x05R\n" + + "intervalUs\x127\n" + + "\n" + + "compressed\x18\x03 \x01(\v2\x17.grpc.testing.BoolValueR\n" + + "compressed\x12P\n" + + "\x18fill_peer_socket_address\x18\x04 \x01(\v2\x17.grpc.testing.BoolValueR\x15fillPeerSocketAddress\"\xe9\x02\n" + + "\x1aStreamingOutputCallRequest\x12>\n" + + "\rresponse_type\x18\x01 \x01(\x0e2\x19.grpc.testing.PayloadTypeR\fresponseType\x12Q\n" + + "\x13response_parameters\x18\x02 \x03(\v2 .grpc.testing.ResponseParametersR\x12responseParameters\x12/\n" + + "\apayload\x18\x03 \x01(\v2\x15.grpc.testing.PayloadR\apayload\x12A\n" + + "\x0fresponse_status\x18\a \x01(\v2\x18.grpc.testing.EchoStatusR\x0eresponseStatus\x12D\n" + + "\x0forca_oob_report\x18\b \x01(\v2\x1c.grpc.testing.TestOrcaReportR\rorcaOobReport\"~\n" + + "\x1bStreamingOutputCallResponse\x12/\n" + + "\apayload\x18\x01 \x01(\v2\x15.grpc.testing.PayloadR\apayload\x12.\n" + + "\x13peer_socket_address\x18\x02 \x01(\tR\x11peerSocketAddress\"J\n" + + "\x0fReconnectParams\x127\n" + + "\x18max_reconnect_backoff_ms\x18\x01 \x01(\x05R\x15maxReconnectBackoffMs\"F\n" + + "\rReconnectInfo\x12\x16\n" + + "\x06passed\x18\x01 \x01(\bR\x06passed\x12\x1d\n" + + "\n" + + "backoff_ms\x18\x02 \x03(\x05R\tbackoffMs\"{\n" + + "\x18LoadBalancerStatsRequest\x12\x19\n" + + "\bnum_rpcs\x18\x01 \x01(\x05R\anumRpcs\x12\x1f\n" + + "\vtimeout_sec\x18\x02 \x01(\x05R\n" + + "timeoutSec\x12#\n" + + "\rmetadata_keys\x18\x03 \x03(\tR\fmetadataKeys\"\xd0\t\n" + + "\x19LoadBalancerStatsResponse\x12Y\n" + + "\frpcs_by_peer\x18\x01 \x03(\v27.grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntryR\n" + + "rpcsByPeer\x12!\n" + + "\fnum_failures\x18\x02 \x01(\x05R\vnumFailures\x12_\n" + + "\x0erpcs_by_method\x18\x03 \x03(\v29.grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntryR\frpcsByMethod\x12h\n" + + "\x11metadatas_by_peer\x18\x04 \x03(\v2<.grpc.testing.LoadBalancerStatsResponse.MetadatasByPeerEntryR\x0fmetadatasByPeer\x1a\x81\x01\n" + + "\rMetadataEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value\x12H\n" + + "\x04type\x18\x03 \x01(\x0e24.grpc.testing.LoadBalancerStatsResponse.MetadataTypeR\x04type\x1a`\n" + + "\vRpcMetadata\x12Q\n" + + "\bmetadata\x18\x01 \x03(\v25.grpc.testing.LoadBalancerStatsResponse.MetadataEntryR\bmetadata\x1ah\n" + + "\x0eMetadataByPeer\x12V\n" + + "\frpc_metadata\x18\x01 \x03(\v23.grpc.testing.LoadBalancerStatsResponse.RpcMetadataR\vrpcMetadata\x1a\xb1\x01\n" + + "\n" + + "RpcsByPeer\x12d\n" + + "\frpcs_by_peer\x18\x01 \x03(\v2B.grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntryR\n" + + "rpcsByPeer\x1a=\n" + + "\x0fRpcsByPeerEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\x05R\x05value:\x028\x01\x1a=\n" + + "\x0fRpcsByPeerEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\x05R\x05value:\x028\x01\x1as\n" + + "\x11RpcsByMethodEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12H\n" + + "\x05value\x18\x02 \x01(\v22.grpc.testing.LoadBalancerStatsResponse.RpcsByPeerR\x05value:\x028\x01\x1az\n" + + "\x14MetadatasByPeerEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12L\n" + + "\x05value\x18\x02 \x01(\v26.grpc.testing.LoadBalancerStatsResponse.MetadataByPeerR\x05value:\x028\x01\"6\n" + + "\fMetadataType\x12\v\n" + + "\aUNKNOWN\x10\x00\x12\v\n" + + "\aINITIAL\x10\x01\x12\f\n" + + "\bTRAILING\x10\x02\"%\n" + + "#LoadBalancerAccumulatedStatsRequest\"\x86\t\n" + + "$LoadBalancerAccumulatedStatsResponse\x12\x8e\x01\n" + + "\x1anum_rpcs_started_by_method\x18\x01 \x03(\v2N.grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntryB\x02\x18\x01R\x16numRpcsStartedByMethod\x12\x94\x01\n" + + "\x1cnum_rpcs_succeeded_by_method\x18\x02 \x03(\v2P.grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntryB\x02\x18\x01R\x18numRpcsSucceededByMethod\x12\x8b\x01\n" + + "\x19num_rpcs_failed_by_method\x18\x03 \x03(\v2M.grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntryB\x02\x18\x01R\x15numRpcsFailedByMethod\x12p\n" + + "\x10stats_per_method\x18\x04 \x03(\v2F.grpc.testing.LoadBalancerAccumulatedStatsResponse.StatsPerMethodEntryR\x0estatsPerMethod\x1aI\n" + + "\x1bNumRpcsStartedByMethodEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\x05R\x05value:\x028\x01\x1aK\n" + + "\x1dNumRpcsSucceededByMethodEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\x05R\x05value:\x028\x01\x1aH\n" + + "\x1aNumRpcsFailedByMethodEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\x05R\x05value:\x028\x01\x1a\xcf\x01\n" + + "\vMethodStats\x12!\n" + + "\frpcs_started\x18\x01 \x01(\x05R\vrpcsStarted\x12b\n" + + "\x06result\x18\x02 \x03(\v2J.grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats.ResultEntryR\x06result\x1a9\n" + + "\vResultEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\x05R\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\x05R\x05value:\x028\x01\x1a\x81\x01\n" + + "\x13StatsPerMethodEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12T\n" + + "\x05value\x18\x02 \x01(\v2>.grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStatsR\x05value:\x028\x01\"\xe9\x02\n" + + "\x16ClientConfigureRequest\x12B\n" + + "\x05types\x18\x01 \x03(\x0e2,.grpc.testing.ClientConfigureRequest.RpcTypeR\x05types\x12I\n" + + "\bmetadata\x18\x02 \x03(\v2-.grpc.testing.ClientConfigureRequest.MetadataR\bmetadata\x12\x1f\n" + + "\vtimeout_sec\x18\x03 \x01(\x05R\n" + + "timeoutSec\x1at\n" + + "\bMetadata\x12@\n" + + "\x04type\x18\x01 \x01(\x0e2,.grpc.testing.ClientConfigureRequest.RpcTypeR\x04type\x12\x10\n" + + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x03 \x01(\tR\x05value\")\n" + + "\aRpcType\x12\x0e\n" + + "\n" + + "EMPTY_CALL\x10\x00\x12\x0e\n" + + "\n" + + "UNARY_CALL\x10\x01\"\x19\n" + + "\x17ClientConfigureResponse\"\x1e\n" + + "\n" + + "MemorySize\x12\x10\n" + + "\x03rss\x18\x01 \x01(\x03R\x03rss\"\x8b\x03\n" + + "\x0eTestOrcaReport\x12'\n" + + "\x0fcpu_utilization\x18\x01 \x01(\x01R\x0ecpuUtilization\x12-\n" + + "\x12memory_utilization\x18\x02 \x01(\x01R\x11memoryUtilization\x12P\n" + + "\frequest_cost\x18\x03 \x03(\v2-.grpc.testing.TestOrcaReport.RequestCostEntryR\vrequestCost\x12O\n" + + "\vutilization\x18\x04 \x03(\v2-.grpc.testing.TestOrcaReport.UtilizationEntryR\vutilization\x1a>\n" + + "\x10RequestCostEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\x01R\x05value:\x028\x01\x1a>\n" + + "\x10UtilizationEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\x01R\x05value:\x028\x01\"\x7f\n" + + "\x16SetReturnStatusRequest\x12-\n" + + "\x13grpc_code_to_return\x18\x01 \x01(\x05R\x10grpcCodeToReturn\x126\n" + + "\x17grpc_status_description\x18\x02 \x01(\tR\x15grpcStatusDescription\"\xa5\x02\n" + + "\vHookRequest\x12F\n" + + "\acommand\x18\x01 \x01(\x0e2,.grpc.testing.HookRequest.HookRequestCommandR\acommand\x12-\n" + + "\x13grpc_code_to_return\x18\x02 \x01(\x05R\x10grpcCodeToReturn\x126\n" + + "\x17grpc_status_description\x18\x03 \x01(\tR\x15grpcStatusDescription\x12\x1f\n" + + "\vserver_port\x18\x04 \x01(\x05R\n" + + "serverPort\"F\n" + + "\x12HookRequestCommand\x12\x0f\n" + + "\vUNSPECIFIED\x10\x00\x12\t\n" + + "\x05START\x10\x01\x12\b\n" + + "\x04STOP\x10\x02\x12\n" + + "\n" + + "\x06RETURN\x10\x03\"\x0e\n" + + "\fHookResponse*\x1f\n" + + "\vPayloadType\x12\x10\n" + + "\fCOMPRESSABLE\x10\x00*o\n" + + "\x0fGrpclbRouteType\x12\x1d\n" + + "\x19GRPCLB_ROUTE_TYPE_UNKNOWN\x10\x00\x12\x1e\n" + + "\x1aGRPCLB_ROUTE_TYPE_FALLBACK\x10\x01\x12\x1d\n" + + "\x19GRPCLB_ROUTE_TYPE_BACKEND\x10\x02B\x1d\n" + + "\x1bio.grpc.testing.integrationb\x06proto3" + +var ( + file_grpc_testing_messages_proto_rawDescOnce sync.Once + file_grpc_testing_messages_proto_rawDescData []byte +) + +func file_grpc_testing_messages_proto_rawDescGZIP() []byte { + file_grpc_testing_messages_proto_rawDescOnce.Do(func() { + file_grpc_testing_messages_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_testing_messages_proto_rawDesc), len(file_grpc_testing_messages_proto_rawDesc))) + }) + return file_grpc_testing_messages_proto_rawDescData +} + +var file_grpc_testing_messages_proto_enumTypes = make([]protoimpl.EnumInfo, 5) +var file_grpc_testing_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 40) +var file_grpc_testing_messages_proto_goTypes = []any{ + (PayloadType)(0), // 0: grpc.testing.PayloadType + (GrpclbRouteType)(0), // 1: grpc.testing.GrpclbRouteType + (LoadBalancerStatsResponse_MetadataType)(0), // 2: grpc.testing.LoadBalancerStatsResponse.MetadataType + (ClientConfigureRequest_RpcType)(0), // 3: grpc.testing.ClientConfigureRequest.RpcType + (HookRequest_HookRequestCommand)(0), // 4: grpc.testing.HookRequest.HookRequestCommand + (*BoolValue)(nil), // 5: grpc.testing.BoolValue + (*Payload)(nil), // 6: grpc.testing.Payload + (*EchoStatus)(nil), // 7: grpc.testing.EchoStatus + (*SimpleRequest)(nil), // 8: grpc.testing.SimpleRequest + (*SimpleResponse)(nil), // 9: grpc.testing.SimpleResponse + (*StreamingInputCallRequest)(nil), // 10: grpc.testing.StreamingInputCallRequest + (*StreamingInputCallResponse)(nil), // 11: grpc.testing.StreamingInputCallResponse + (*ResponseParameters)(nil), // 12: grpc.testing.ResponseParameters + (*StreamingOutputCallRequest)(nil), // 13: grpc.testing.StreamingOutputCallRequest + (*StreamingOutputCallResponse)(nil), // 14: grpc.testing.StreamingOutputCallResponse + (*ReconnectParams)(nil), // 15: grpc.testing.ReconnectParams + (*ReconnectInfo)(nil), // 16: grpc.testing.ReconnectInfo + (*LoadBalancerStatsRequest)(nil), // 17: grpc.testing.LoadBalancerStatsRequest + (*LoadBalancerStatsResponse)(nil), // 18: grpc.testing.LoadBalancerStatsResponse + (*LoadBalancerAccumulatedStatsRequest)(nil), // 19: grpc.testing.LoadBalancerAccumulatedStatsRequest + (*LoadBalancerAccumulatedStatsResponse)(nil), // 20: grpc.testing.LoadBalancerAccumulatedStatsResponse + (*ClientConfigureRequest)(nil), // 21: grpc.testing.ClientConfigureRequest + (*ClientConfigureResponse)(nil), // 22: grpc.testing.ClientConfigureResponse + (*MemorySize)(nil), // 23: grpc.testing.MemorySize + (*TestOrcaReport)(nil), // 24: grpc.testing.TestOrcaReport + (*SetReturnStatusRequest)(nil), // 25: grpc.testing.SetReturnStatusRequest + (*HookRequest)(nil), // 26: grpc.testing.HookRequest + (*HookResponse)(nil), // 27: grpc.testing.HookResponse + (*LoadBalancerStatsResponse_MetadataEntry)(nil), // 28: grpc.testing.LoadBalancerStatsResponse.MetadataEntry + (*LoadBalancerStatsResponse_RpcMetadata)(nil), // 29: grpc.testing.LoadBalancerStatsResponse.RpcMetadata + (*LoadBalancerStatsResponse_MetadataByPeer)(nil), // 30: grpc.testing.LoadBalancerStatsResponse.MetadataByPeer + (*LoadBalancerStatsResponse_RpcsByPeer)(nil), // 31: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer + nil, // 32: grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry + nil, // 33: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry + nil, // 34: grpc.testing.LoadBalancerStatsResponse.MetadatasByPeerEntry + nil, // 35: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry + nil, // 36: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntry + nil, // 37: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntry + nil, // 38: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntry + (*LoadBalancerAccumulatedStatsResponse_MethodStats)(nil), // 39: grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats + nil, // 40: grpc.testing.LoadBalancerAccumulatedStatsResponse.StatsPerMethodEntry + nil, // 41: grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats.ResultEntry + (*ClientConfigureRequest_Metadata)(nil), // 42: grpc.testing.ClientConfigureRequest.Metadata + nil, // 43: grpc.testing.TestOrcaReport.RequestCostEntry + nil, // 44: grpc.testing.TestOrcaReport.UtilizationEntry +} +var file_grpc_testing_messages_proto_depIdxs = []int32{ + 0, // 0: grpc.testing.Payload.type:type_name -> grpc.testing.PayloadType + 0, // 1: grpc.testing.SimpleRequest.response_type:type_name -> grpc.testing.PayloadType + 6, // 2: grpc.testing.SimpleRequest.payload:type_name -> grpc.testing.Payload + 5, // 3: grpc.testing.SimpleRequest.response_compressed:type_name -> grpc.testing.BoolValue + 7, // 4: grpc.testing.SimpleRequest.response_status:type_name -> grpc.testing.EchoStatus + 5, // 5: grpc.testing.SimpleRequest.expect_compressed:type_name -> grpc.testing.BoolValue + 24, // 6: grpc.testing.SimpleRequest.orca_per_query_report:type_name -> grpc.testing.TestOrcaReport + 6, // 7: grpc.testing.SimpleResponse.payload:type_name -> grpc.testing.Payload + 1, // 8: grpc.testing.SimpleResponse.grpclb_route_type:type_name -> grpc.testing.GrpclbRouteType + 6, // 9: grpc.testing.StreamingInputCallRequest.payload:type_name -> grpc.testing.Payload + 5, // 10: grpc.testing.StreamingInputCallRequest.expect_compressed:type_name -> grpc.testing.BoolValue + 5, // 11: grpc.testing.ResponseParameters.compressed:type_name -> grpc.testing.BoolValue + 5, // 12: grpc.testing.ResponseParameters.fill_peer_socket_address:type_name -> grpc.testing.BoolValue + 0, // 13: grpc.testing.StreamingOutputCallRequest.response_type:type_name -> grpc.testing.PayloadType + 12, // 14: grpc.testing.StreamingOutputCallRequest.response_parameters:type_name -> grpc.testing.ResponseParameters + 6, // 15: grpc.testing.StreamingOutputCallRequest.payload:type_name -> grpc.testing.Payload + 7, // 16: grpc.testing.StreamingOutputCallRequest.response_status:type_name -> grpc.testing.EchoStatus + 24, // 17: grpc.testing.StreamingOutputCallRequest.orca_oob_report:type_name -> grpc.testing.TestOrcaReport + 6, // 18: grpc.testing.StreamingOutputCallResponse.payload:type_name -> grpc.testing.Payload + 32, // 19: grpc.testing.LoadBalancerStatsResponse.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry + 33, // 20: grpc.testing.LoadBalancerStatsResponse.rpcs_by_method:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry + 34, // 21: grpc.testing.LoadBalancerStatsResponse.metadatas_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.MetadatasByPeerEntry + 36, // 22: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_started_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntry + 37, // 23: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_succeeded_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntry + 38, // 24: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_failed_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntry + 40, // 25: grpc.testing.LoadBalancerAccumulatedStatsResponse.stats_per_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.StatsPerMethodEntry + 3, // 26: grpc.testing.ClientConfigureRequest.types:type_name -> grpc.testing.ClientConfigureRequest.RpcType + 42, // 27: grpc.testing.ClientConfigureRequest.metadata:type_name -> grpc.testing.ClientConfigureRequest.Metadata + 43, // 28: grpc.testing.TestOrcaReport.request_cost:type_name -> grpc.testing.TestOrcaReport.RequestCostEntry + 44, // 29: grpc.testing.TestOrcaReport.utilization:type_name -> grpc.testing.TestOrcaReport.UtilizationEntry + 4, // 30: grpc.testing.HookRequest.command:type_name -> grpc.testing.HookRequest.HookRequestCommand + 2, // 31: grpc.testing.LoadBalancerStatsResponse.MetadataEntry.type:type_name -> grpc.testing.LoadBalancerStatsResponse.MetadataType + 28, // 32: grpc.testing.LoadBalancerStatsResponse.RpcMetadata.metadata:type_name -> grpc.testing.LoadBalancerStatsResponse.MetadataEntry + 29, // 33: grpc.testing.LoadBalancerStatsResponse.MetadataByPeer.rpc_metadata:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcMetadata + 35, // 34: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry + 31, // 35: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry.value:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer + 30, // 36: grpc.testing.LoadBalancerStatsResponse.MetadatasByPeerEntry.value:type_name -> grpc.testing.LoadBalancerStatsResponse.MetadataByPeer + 41, // 37: grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats.result:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats.ResultEntry + 39, // 38: grpc.testing.LoadBalancerAccumulatedStatsResponse.StatsPerMethodEntry.value:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats + 3, // 39: grpc.testing.ClientConfigureRequest.Metadata.type:type_name -> grpc.testing.ClientConfigureRequest.RpcType + 40, // [40:40] is the sub-list for method output_type + 40, // [40:40] is the sub-list for method input_type + 40, // [40:40] is the sub-list for extension type_name + 40, // [40:40] is the sub-list for extension extendee + 0, // [0:40] is the sub-list for field type_name +} + +func init() { file_grpc_testing_messages_proto_init() } +func file_grpc_testing_messages_proto_init() { + if File_grpc_testing_messages_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_testing_messages_proto_rawDesc), len(file_grpc_testing_messages_proto_rawDesc)), + NumEnums: 5, + NumMessages: 40, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_testing_messages_proto_goTypes, + DependencyIndexes: file_grpc_testing_messages_proto_depIdxs, + EnumInfos: file_grpc_testing_messages_proto_enumTypes, + MessageInfos: file_grpc_testing_messages_proto_msgTypes, + }.Build() + File_grpc_testing_messages_proto = out.File + file_grpc_testing_messages_proto_goTypes = nil + file_grpc_testing_messages_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/payloads.pb.go b/interop/grpc_testing/payloads.pb.go new file mode 100644 index 0000000..173b533 --- /dev/null +++ b/interop/grpc_testing/payloads.pb.go @@ -0,0 +1,352 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/testing/payloads.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ByteBufferParams struct { + state protoimpl.MessageState `protogen:"open.v1"` + ReqSize int32 `protobuf:"varint,1,opt,name=req_size,json=reqSize,proto3" json:"req_size,omitempty"` + RespSize int32 `protobuf:"varint,2,opt,name=resp_size,json=respSize,proto3" json:"resp_size,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ByteBufferParams) Reset() { + *x = ByteBufferParams{} + mi := &file_grpc_testing_payloads_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ByteBufferParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ByteBufferParams) ProtoMessage() {} + +func (x *ByteBufferParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_payloads_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ByteBufferParams.ProtoReflect.Descriptor instead. +func (*ByteBufferParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_payloads_proto_rawDescGZIP(), []int{0} +} + +func (x *ByteBufferParams) GetReqSize() int32 { + if x != nil { + return x.ReqSize + } + return 0 +} + +func (x *ByteBufferParams) GetRespSize() int32 { + if x != nil { + return x.RespSize + } + return 0 +} + +type SimpleProtoParams struct { + state protoimpl.MessageState `protogen:"open.v1"` + ReqSize int32 `protobuf:"varint,1,opt,name=req_size,json=reqSize,proto3" json:"req_size,omitempty"` + RespSize int32 `protobuf:"varint,2,opt,name=resp_size,json=respSize,proto3" json:"resp_size,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SimpleProtoParams) Reset() { + *x = SimpleProtoParams{} + mi := &file_grpc_testing_payloads_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SimpleProtoParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SimpleProtoParams) ProtoMessage() {} + +func (x *SimpleProtoParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_payloads_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SimpleProtoParams.ProtoReflect.Descriptor instead. +func (*SimpleProtoParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_payloads_proto_rawDescGZIP(), []int{1} +} + +func (x *SimpleProtoParams) GetReqSize() int32 { + if x != nil { + return x.ReqSize + } + return 0 +} + +func (x *SimpleProtoParams) GetRespSize() int32 { + if x != nil { + return x.RespSize + } + return 0 +} + +type ComplexProtoParams struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ComplexProtoParams) Reset() { + *x = ComplexProtoParams{} + mi := &file_grpc_testing_payloads_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ComplexProtoParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ComplexProtoParams) ProtoMessage() {} + +func (x *ComplexProtoParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_payloads_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ComplexProtoParams.ProtoReflect.Descriptor instead. +func (*ComplexProtoParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_payloads_proto_rawDescGZIP(), []int{2} +} + +type PayloadConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Payload: + // + // *PayloadConfig_BytebufParams + // *PayloadConfig_SimpleParams + // *PayloadConfig_ComplexParams + Payload isPayloadConfig_Payload `protobuf_oneof:"payload"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PayloadConfig) Reset() { + *x = PayloadConfig{} + mi := &file_grpc_testing_payloads_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PayloadConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PayloadConfig) ProtoMessage() {} + +func (x *PayloadConfig) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_payloads_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PayloadConfig.ProtoReflect.Descriptor instead. +func (*PayloadConfig) Descriptor() ([]byte, []int) { + return file_grpc_testing_payloads_proto_rawDescGZIP(), []int{3} +} + +func (x *PayloadConfig) GetPayload() isPayloadConfig_Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *PayloadConfig) GetBytebufParams() *ByteBufferParams { + if x != nil { + if x, ok := x.Payload.(*PayloadConfig_BytebufParams); ok { + return x.BytebufParams + } + } + return nil +} + +func (x *PayloadConfig) GetSimpleParams() *SimpleProtoParams { + if x != nil { + if x, ok := x.Payload.(*PayloadConfig_SimpleParams); ok { + return x.SimpleParams + } + } + return nil +} + +func (x *PayloadConfig) GetComplexParams() *ComplexProtoParams { + if x != nil { + if x, ok := x.Payload.(*PayloadConfig_ComplexParams); ok { + return x.ComplexParams + } + } + return nil +} + +type isPayloadConfig_Payload interface { + isPayloadConfig_Payload() +} + +type PayloadConfig_BytebufParams struct { + BytebufParams *ByteBufferParams `protobuf:"bytes,1,opt,name=bytebuf_params,json=bytebufParams,proto3,oneof"` +} + +type PayloadConfig_SimpleParams struct { + SimpleParams *SimpleProtoParams `protobuf:"bytes,2,opt,name=simple_params,json=simpleParams,proto3,oneof"` +} + +type PayloadConfig_ComplexParams struct { + ComplexParams *ComplexProtoParams `protobuf:"bytes,3,opt,name=complex_params,json=complexParams,proto3,oneof"` +} + +func (*PayloadConfig_BytebufParams) isPayloadConfig_Payload() {} + +func (*PayloadConfig_SimpleParams) isPayloadConfig_Payload() {} + +func (*PayloadConfig_ComplexParams) isPayloadConfig_Payload() {} + +var File_grpc_testing_payloads_proto protoreflect.FileDescriptor + +const file_grpc_testing_payloads_proto_rawDesc = "" + + "\n" + + "\x1bgrpc/testing/payloads.proto\x12\fgrpc.testing\"J\n" + + "\x10ByteBufferParams\x12\x19\n" + + "\breq_size\x18\x01 \x01(\x05R\areqSize\x12\x1b\n" + + "\tresp_size\x18\x02 \x01(\x05R\brespSize\"K\n" + + "\x11SimpleProtoParams\x12\x19\n" + + "\breq_size\x18\x01 \x01(\x05R\areqSize\x12\x1b\n" + + "\tresp_size\x18\x02 \x01(\x05R\brespSize\"\x14\n" + + "\x12ComplexProtoParams\"\xf6\x01\n" + + "\rPayloadConfig\x12G\n" + + "\x0ebytebuf_params\x18\x01 \x01(\v2\x1e.grpc.testing.ByteBufferParamsH\x00R\rbytebufParams\x12F\n" + + "\rsimple_params\x18\x02 \x01(\v2\x1f.grpc.testing.SimpleProtoParamsH\x00R\fsimpleParams\x12I\n" + + "\x0ecomplex_params\x18\x03 \x01(\v2 .grpc.testing.ComplexProtoParamsH\x00R\rcomplexParamsB\t\n" + + "\apayloadB\"\n" + + "\x0fio.grpc.testingB\rPayloadsProtoP\x01b\x06proto3" + +var ( + file_grpc_testing_payloads_proto_rawDescOnce sync.Once + file_grpc_testing_payloads_proto_rawDescData []byte +) + +func file_grpc_testing_payloads_proto_rawDescGZIP() []byte { + file_grpc_testing_payloads_proto_rawDescOnce.Do(func() { + file_grpc_testing_payloads_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_testing_payloads_proto_rawDesc), len(file_grpc_testing_payloads_proto_rawDesc))) + }) + return file_grpc_testing_payloads_proto_rawDescData +} + +var file_grpc_testing_payloads_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_grpc_testing_payloads_proto_goTypes = []any{ + (*ByteBufferParams)(nil), // 0: grpc.testing.ByteBufferParams + (*SimpleProtoParams)(nil), // 1: grpc.testing.SimpleProtoParams + (*ComplexProtoParams)(nil), // 2: grpc.testing.ComplexProtoParams + (*PayloadConfig)(nil), // 3: grpc.testing.PayloadConfig +} +var file_grpc_testing_payloads_proto_depIdxs = []int32{ + 0, // 0: grpc.testing.PayloadConfig.bytebuf_params:type_name -> grpc.testing.ByteBufferParams + 1, // 1: grpc.testing.PayloadConfig.simple_params:type_name -> grpc.testing.SimpleProtoParams + 2, // 2: grpc.testing.PayloadConfig.complex_params:type_name -> grpc.testing.ComplexProtoParams + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_grpc_testing_payloads_proto_init() } +func file_grpc_testing_payloads_proto_init() { + if File_grpc_testing_payloads_proto != nil { + return + } + file_grpc_testing_payloads_proto_msgTypes[3].OneofWrappers = []any{ + (*PayloadConfig_BytebufParams)(nil), + (*PayloadConfig_SimpleParams)(nil), + (*PayloadConfig_ComplexParams)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_testing_payloads_proto_rawDesc), len(file_grpc_testing_payloads_proto_rawDesc)), + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_testing_payloads_proto_goTypes, + DependencyIndexes: file_grpc_testing_payloads_proto_depIdxs, + MessageInfos: file_grpc_testing_payloads_proto_msgTypes, + }.Build() + File_grpc_testing_payloads_proto = out.File + file_grpc_testing_payloads_proto_goTypes = nil + file_grpc_testing_payloads_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/report_qps_scenario_service.pb.go b/interop/grpc_testing/report_qps_scenario_service.pb.go new file mode 100644 index 0000000..d5206b3 --- /dev/null +++ b/interop/grpc_testing/report_qps_scenario_service.pb.go @@ -0,0 +1,85 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/testing/report_qps_scenario_service.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var File_grpc_testing_report_qps_scenario_service_proto protoreflect.FileDescriptor + +const file_grpc_testing_report_qps_scenario_service_proto_rawDesc = "" + + "\n" + + ".grpc/testing/report_qps_scenario_service.proto\x12\fgrpc.testing\x1a\x1agrpc/testing/control.proto2^\n" + + "\x18ReportQpsScenarioService\x12B\n" + + "\x0eReportScenario\x12\x1c.grpc.testing.ScenarioResult\x1a\x12.grpc.testing.VoidB2\n" + + "\x0fio.grpc.testingB\x1dReportQpsScenarioServiceProtoP\x01b\x06proto3" + +var file_grpc_testing_report_qps_scenario_service_proto_goTypes = []any{ + (*ScenarioResult)(nil), // 0: grpc.testing.ScenarioResult + (*Void)(nil), // 1: grpc.testing.Void +} +var file_grpc_testing_report_qps_scenario_service_proto_depIdxs = []int32{ + 0, // 0: grpc.testing.ReportQpsScenarioService.ReportScenario:input_type -> grpc.testing.ScenarioResult + 1, // 1: grpc.testing.ReportQpsScenarioService.ReportScenario:output_type -> grpc.testing.Void + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_grpc_testing_report_qps_scenario_service_proto_init() } +func file_grpc_testing_report_qps_scenario_service_proto_init() { + if File_grpc_testing_report_qps_scenario_service_proto != nil { + return + } + file_grpc_testing_control_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_testing_report_qps_scenario_service_proto_rawDesc), len(file_grpc_testing_report_qps_scenario_service_proto_rawDesc)), + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_testing_report_qps_scenario_service_proto_goTypes, + DependencyIndexes: file_grpc_testing_report_qps_scenario_service_proto_depIdxs, + }.Build() + File_grpc_testing_report_qps_scenario_service_proto = out.File + file_grpc_testing_report_qps_scenario_service_proto_goTypes = nil + file_grpc_testing_report_qps_scenario_service_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go b/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go new file mode 100644 index 0000000..b9245d7 --- /dev/null +++ b/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go @@ -0,0 +1,141 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: grpc/testing/report_qps_scenario_service.proto + +package grpc_testing + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + ReportQpsScenarioService_ReportScenario_FullMethodName = "/grpc.testing.ReportQpsScenarioService/ReportScenario" +) + +// ReportQpsScenarioServiceClient is the client API for ReportQpsScenarioService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ReportQpsScenarioServiceClient interface { + // Report results of a QPS test benchmark scenario. + ReportScenario(ctx context.Context, in *ScenarioResult, opts ...grpc.CallOption) (*Void, error) +} + +type reportQpsScenarioServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewReportQpsScenarioServiceClient(cc grpc.ClientConnInterface) ReportQpsScenarioServiceClient { + return &reportQpsScenarioServiceClient{cc} +} + +func (c *reportQpsScenarioServiceClient) ReportScenario(ctx context.Context, in *ScenarioResult, opts ...grpc.CallOption) (*Void, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Void) + err := c.cc.Invoke(ctx, ReportQpsScenarioService_ReportScenario_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ReportQpsScenarioServiceServer is the server API for ReportQpsScenarioService service. +// All implementations must embed UnimplementedReportQpsScenarioServiceServer +// for forward compatibility. +type ReportQpsScenarioServiceServer interface { + // Report results of a QPS test benchmark scenario. + ReportScenario(context.Context, *ScenarioResult) (*Void, error) + mustEmbedUnimplementedReportQpsScenarioServiceServer() +} + +// UnimplementedReportQpsScenarioServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedReportQpsScenarioServiceServer struct{} + +func (UnimplementedReportQpsScenarioServiceServer) ReportScenario(context.Context, *ScenarioResult) (*Void, error) { + return nil, status.Error(codes.Unimplemented, "method ReportScenario not implemented") +} +func (UnimplementedReportQpsScenarioServiceServer) mustEmbedUnimplementedReportQpsScenarioServiceServer() { +} +func (UnimplementedReportQpsScenarioServiceServer) testEmbeddedByValue() {} + +// UnsafeReportQpsScenarioServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ReportQpsScenarioServiceServer will +// result in compilation errors. +type UnsafeReportQpsScenarioServiceServer interface { + mustEmbedUnimplementedReportQpsScenarioServiceServer() +} + +func RegisterReportQpsScenarioServiceServer(s grpc.ServiceRegistrar, srv ReportQpsScenarioServiceServer) { + // If the following call panics, it indicates UnimplementedReportQpsScenarioServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&ReportQpsScenarioService_ServiceDesc, srv) +} + +func _ReportQpsScenarioService_ReportScenario_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ScenarioResult) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ReportQpsScenarioServiceServer).ReportScenario(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ReportQpsScenarioService_ReportScenario_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ReportQpsScenarioServiceServer).ReportScenario(ctx, req.(*ScenarioResult)) + } + return interceptor(ctx, in, info, handler) +} + +// ReportQpsScenarioService_ServiceDesc is the grpc.ServiceDesc for ReportQpsScenarioService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ReportQpsScenarioService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.ReportQpsScenarioService", + HandlerType: (*ReportQpsScenarioServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "ReportScenario", + Handler: _ReportQpsScenarioService_ReportScenario_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/report_qps_scenario_service.proto", +} diff --git a/interop/grpc_testing/stats.pb.go b/interop/grpc_testing/stats.pb.go new file mode 100644 index 0000000..0d0259c --- /dev/null +++ b/interop/grpc_testing/stats.pb.go @@ -0,0 +1,525 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/testing/stats.proto + +package grpc_testing + +import ( + core "google.golang.org/grpc/interop/grpc_testing/core" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ServerStats struct { + state protoimpl.MessageState `protogen:"open.v1"` + // wall clock time change in seconds since last reset + TimeElapsed float64 `protobuf:"fixed64,1,opt,name=time_elapsed,json=timeElapsed,proto3" json:"time_elapsed,omitempty"` + // change in user time (in seconds) used by the server since last reset + TimeUser float64 `protobuf:"fixed64,2,opt,name=time_user,json=timeUser,proto3" json:"time_user,omitempty"` + // change in server time (in seconds) used by the server process and all + // threads since last reset + TimeSystem float64 `protobuf:"fixed64,3,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"` + // change in total cpu time of the server (data from proc/stat) + TotalCpuTime uint64 `protobuf:"varint,4,opt,name=total_cpu_time,json=totalCpuTime,proto3" json:"total_cpu_time,omitempty"` + // change in idle time of the server (data from proc/stat) + IdleCpuTime uint64 `protobuf:"varint,5,opt,name=idle_cpu_time,json=idleCpuTime,proto3" json:"idle_cpu_time,omitempty"` + // Number of polls called inside completion queue + CqPollCount uint64 `protobuf:"varint,6,opt,name=cq_poll_count,json=cqPollCount,proto3" json:"cq_poll_count,omitempty"` + // Core library stats + CoreStats *core.Stats `protobuf:"bytes,7,opt,name=core_stats,json=coreStats,proto3" json:"core_stats,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerStats) Reset() { + *x = ServerStats{} + mi := &file_grpc_testing_stats_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerStats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerStats) ProtoMessage() {} + +func (x *ServerStats) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_stats_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerStats.ProtoReflect.Descriptor instead. +func (*ServerStats) Descriptor() ([]byte, []int) { + return file_grpc_testing_stats_proto_rawDescGZIP(), []int{0} +} + +func (x *ServerStats) GetTimeElapsed() float64 { + if x != nil { + return x.TimeElapsed + } + return 0 +} + +func (x *ServerStats) GetTimeUser() float64 { + if x != nil { + return x.TimeUser + } + return 0 +} + +func (x *ServerStats) GetTimeSystem() float64 { + if x != nil { + return x.TimeSystem + } + return 0 +} + +func (x *ServerStats) GetTotalCpuTime() uint64 { + if x != nil { + return x.TotalCpuTime + } + return 0 +} + +func (x *ServerStats) GetIdleCpuTime() uint64 { + if x != nil { + return x.IdleCpuTime + } + return 0 +} + +func (x *ServerStats) GetCqPollCount() uint64 { + if x != nil { + return x.CqPollCount + } + return 0 +} + +func (x *ServerStats) GetCoreStats() *core.Stats { + if x != nil { + return x.CoreStats + } + return nil +} + +// Histogram params based on grpc/support/histogram.c +type HistogramParams struct { + state protoimpl.MessageState `protogen:"open.v1"` + Resolution float64 `protobuf:"fixed64,1,opt,name=resolution,proto3" json:"resolution,omitempty"` // first bucket is [0, 1 + resolution) + MaxPossible float64 `protobuf:"fixed64,2,opt,name=max_possible,json=maxPossible,proto3" json:"max_possible,omitempty"` // use enough buckets to allow this value + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HistogramParams) Reset() { + *x = HistogramParams{} + mi := &file_grpc_testing_stats_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HistogramParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HistogramParams) ProtoMessage() {} + +func (x *HistogramParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_stats_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HistogramParams.ProtoReflect.Descriptor instead. +func (*HistogramParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_stats_proto_rawDescGZIP(), []int{1} +} + +func (x *HistogramParams) GetResolution() float64 { + if x != nil { + return x.Resolution + } + return 0 +} + +func (x *HistogramParams) GetMaxPossible() float64 { + if x != nil { + return x.MaxPossible + } + return 0 +} + +// Histogram data based on grpc/support/histogram.c +type HistogramData struct { + state protoimpl.MessageState `protogen:"open.v1"` + Bucket []uint32 `protobuf:"varint,1,rep,packed,name=bucket,proto3" json:"bucket,omitempty"` + MinSeen float64 `protobuf:"fixed64,2,opt,name=min_seen,json=minSeen,proto3" json:"min_seen,omitempty"` + MaxSeen float64 `protobuf:"fixed64,3,opt,name=max_seen,json=maxSeen,proto3" json:"max_seen,omitempty"` + Sum float64 `protobuf:"fixed64,4,opt,name=sum,proto3" json:"sum,omitempty"` + SumOfSquares float64 `protobuf:"fixed64,5,opt,name=sum_of_squares,json=sumOfSquares,proto3" json:"sum_of_squares,omitempty"` + Count float64 `protobuf:"fixed64,6,opt,name=count,proto3" json:"count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HistogramData) Reset() { + *x = HistogramData{} + mi := &file_grpc_testing_stats_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HistogramData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HistogramData) ProtoMessage() {} + +func (x *HistogramData) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_stats_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HistogramData.ProtoReflect.Descriptor instead. +func (*HistogramData) Descriptor() ([]byte, []int) { + return file_grpc_testing_stats_proto_rawDescGZIP(), []int{2} +} + +func (x *HistogramData) GetBucket() []uint32 { + if x != nil { + return x.Bucket + } + return nil +} + +func (x *HistogramData) GetMinSeen() float64 { + if x != nil { + return x.MinSeen + } + return 0 +} + +func (x *HistogramData) GetMaxSeen() float64 { + if x != nil { + return x.MaxSeen + } + return 0 +} + +func (x *HistogramData) GetSum() float64 { + if x != nil { + return x.Sum + } + return 0 +} + +func (x *HistogramData) GetSumOfSquares() float64 { + if x != nil { + return x.SumOfSquares + } + return 0 +} + +func (x *HistogramData) GetCount() float64 { + if x != nil { + return x.Count + } + return 0 +} + +type RequestResultCount struct { + state protoimpl.MessageState `protogen:"open.v1"` + StatusCode int32 `protobuf:"varint,1,opt,name=status_code,json=statusCode,proto3" json:"status_code,omitempty"` + Count int64 `protobuf:"varint,2,opt,name=count,proto3" json:"count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestResultCount) Reset() { + *x = RequestResultCount{} + mi := &file_grpc_testing_stats_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestResultCount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestResultCount) ProtoMessage() {} + +func (x *RequestResultCount) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_stats_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestResultCount.ProtoReflect.Descriptor instead. +func (*RequestResultCount) Descriptor() ([]byte, []int) { + return file_grpc_testing_stats_proto_rawDescGZIP(), []int{3} +} + +func (x *RequestResultCount) GetStatusCode() int32 { + if x != nil { + return x.StatusCode + } + return 0 +} + +func (x *RequestResultCount) GetCount() int64 { + if x != nil { + return x.Count + } + return 0 +} + +type ClientStats struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Latency histogram. Data points are in nanoseconds. + Latencies *HistogramData `protobuf:"bytes,1,opt,name=latencies,proto3" json:"latencies,omitempty"` + // See ServerStats for details. + TimeElapsed float64 `protobuf:"fixed64,2,opt,name=time_elapsed,json=timeElapsed,proto3" json:"time_elapsed,omitempty"` + TimeUser float64 `protobuf:"fixed64,3,opt,name=time_user,json=timeUser,proto3" json:"time_user,omitempty"` + TimeSystem float64 `protobuf:"fixed64,4,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"` + // Number of failed requests (one row per status code seen) + RequestResults []*RequestResultCount `protobuf:"bytes,5,rep,name=request_results,json=requestResults,proto3" json:"request_results,omitempty"` + // Number of polls called inside completion queue + CqPollCount uint64 `protobuf:"varint,6,opt,name=cq_poll_count,json=cqPollCount,proto3" json:"cq_poll_count,omitempty"` + // Core library stats + CoreStats *core.Stats `protobuf:"bytes,7,opt,name=core_stats,json=coreStats,proto3" json:"core_stats,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClientStats) Reset() { + *x = ClientStats{} + mi := &file_grpc_testing_stats_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClientStats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientStats) ProtoMessage() {} + +func (x *ClientStats) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_stats_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientStats.ProtoReflect.Descriptor instead. +func (*ClientStats) Descriptor() ([]byte, []int) { + return file_grpc_testing_stats_proto_rawDescGZIP(), []int{4} +} + +func (x *ClientStats) GetLatencies() *HistogramData { + if x != nil { + return x.Latencies + } + return nil +} + +func (x *ClientStats) GetTimeElapsed() float64 { + if x != nil { + return x.TimeElapsed + } + return 0 +} + +func (x *ClientStats) GetTimeUser() float64 { + if x != nil { + return x.TimeUser + } + return 0 +} + +func (x *ClientStats) GetTimeSystem() float64 { + if x != nil { + return x.TimeSystem + } + return 0 +} + +func (x *ClientStats) GetRequestResults() []*RequestResultCount { + if x != nil { + return x.RequestResults + } + return nil +} + +func (x *ClientStats) GetCqPollCount() uint64 { + if x != nil { + return x.CqPollCount + } + return 0 +} + +func (x *ClientStats) GetCoreStats() *core.Stats { + if x != nil { + return x.CoreStats + } + return nil +} + +var File_grpc_testing_stats_proto protoreflect.FileDescriptor + +const file_grpc_testing_stats_proto_rawDesc = "" + + "\n" + + "\x18grpc/testing/stats.proto\x12\fgrpc.testing\x1a\x15grpc/core/stats.proto\"\x8d\x02\n" + + "\vServerStats\x12!\n" + + "\ftime_elapsed\x18\x01 \x01(\x01R\vtimeElapsed\x12\x1b\n" + + "\ttime_user\x18\x02 \x01(\x01R\btimeUser\x12\x1f\n" + + "\vtime_system\x18\x03 \x01(\x01R\n" + + "timeSystem\x12$\n" + + "\x0etotal_cpu_time\x18\x04 \x01(\x04R\ftotalCpuTime\x12\"\n" + + "\ridle_cpu_time\x18\x05 \x01(\x04R\vidleCpuTime\x12\"\n" + + "\rcq_poll_count\x18\x06 \x01(\x04R\vcqPollCount\x12/\n" + + "\n" + + "core_stats\x18\a \x01(\v2\x10.grpc.core.StatsR\tcoreStats\"T\n" + + "\x0fHistogramParams\x12\x1e\n" + + "\n" + + "resolution\x18\x01 \x01(\x01R\n" + + "resolution\x12!\n" + + "\fmax_possible\x18\x02 \x01(\x01R\vmaxPossible\"\xab\x01\n" + + "\rHistogramData\x12\x16\n" + + "\x06bucket\x18\x01 \x03(\rR\x06bucket\x12\x19\n" + + "\bmin_seen\x18\x02 \x01(\x01R\aminSeen\x12\x19\n" + + "\bmax_seen\x18\x03 \x01(\x01R\amaxSeen\x12\x10\n" + + "\x03sum\x18\x04 \x01(\x01R\x03sum\x12$\n" + + "\x0esum_of_squares\x18\x05 \x01(\x01R\fsumOfSquares\x12\x14\n" + + "\x05count\x18\x06 \x01(\x01R\x05count\"K\n" + + "\x12RequestResultCount\x12\x1f\n" + + "\vstatus_code\x18\x01 \x01(\x05R\n" + + "statusCode\x12\x14\n" + + "\x05count\x18\x02 \x01(\x03R\x05count\"\xc9\x02\n" + + "\vClientStats\x129\n" + + "\tlatencies\x18\x01 \x01(\v2\x1b.grpc.testing.HistogramDataR\tlatencies\x12!\n" + + "\ftime_elapsed\x18\x02 \x01(\x01R\vtimeElapsed\x12\x1b\n" + + "\ttime_user\x18\x03 \x01(\x01R\btimeUser\x12\x1f\n" + + "\vtime_system\x18\x04 \x01(\x01R\n" + + "timeSystem\x12I\n" + + "\x0frequest_results\x18\x05 \x03(\v2 .grpc.testing.RequestResultCountR\x0erequestResults\x12\"\n" + + "\rcq_poll_count\x18\x06 \x01(\x04R\vcqPollCount\x12/\n" + + "\n" + + "core_stats\x18\a \x01(\v2\x10.grpc.core.StatsR\tcoreStatsB\x1f\n" + + "\x0fio.grpc.testingB\n" + + "StatsProtoP\x01b\x06proto3" + +var ( + file_grpc_testing_stats_proto_rawDescOnce sync.Once + file_grpc_testing_stats_proto_rawDescData []byte +) + +func file_grpc_testing_stats_proto_rawDescGZIP() []byte { + file_grpc_testing_stats_proto_rawDescOnce.Do(func() { + file_grpc_testing_stats_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_testing_stats_proto_rawDesc), len(file_grpc_testing_stats_proto_rawDesc))) + }) + return file_grpc_testing_stats_proto_rawDescData +} + +var file_grpc_testing_stats_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_grpc_testing_stats_proto_goTypes = []any{ + (*ServerStats)(nil), // 0: grpc.testing.ServerStats + (*HistogramParams)(nil), // 1: grpc.testing.HistogramParams + (*HistogramData)(nil), // 2: grpc.testing.HistogramData + (*RequestResultCount)(nil), // 3: grpc.testing.RequestResultCount + (*ClientStats)(nil), // 4: grpc.testing.ClientStats + (*core.Stats)(nil), // 5: grpc.core.Stats +} +var file_grpc_testing_stats_proto_depIdxs = []int32{ + 5, // 0: grpc.testing.ServerStats.core_stats:type_name -> grpc.core.Stats + 2, // 1: grpc.testing.ClientStats.latencies:type_name -> grpc.testing.HistogramData + 3, // 2: grpc.testing.ClientStats.request_results:type_name -> grpc.testing.RequestResultCount + 5, // 3: grpc.testing.ClientStats.core_stats:type_name -> grpc.core.Stats + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_grpc_testing_stats_proto_init() } +func file_grpc_testing_stats_proto_init() { + if File_grpc_testing_stats_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_testing_stats_proto_rawDesc), len(file_grpc_testing_stats_proto_rawDesc)), + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_testing_stats_proto_goTypes, + DependencyIndexes: file_grpc_testing_stats_proto_depIdxs, + MessageInfos: file_grpc_testing_stats_proto_msgTypes, + }.Build() + File_grpc_testing_stats_proto = out.File + file_grpc_testing_stats_proto_goTypes = nil + file_grpc_testing_stats_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/test.pb.go b/interop/grpc_testing/test.pb.go new file mode 100644 index 0000000..c021e33 --- /dev/null +++ b/interop/grpc_testing/test.pb.go @@ -0,0 +1,166 @@ +// Copyright 2015-2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/testing/test.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var File_grpc_testing_test_proto protoreflect.FileDescriptor + +const file_grpc_testing_test_proto_rawDesc = "" + + "\n" + + "\x17grpc/testing/test.proto\x12\fgrpc.testing\x1a\x18grpc/testing/empty.proto\x1a\x1bgrpc/testing/messages.proto2\xcb\x05\n" + + "\vTestService\x125\n" + + "\tEmptyCall\x12\x13.grpc.testing.Empty\x1a\x13.grpc.testing.Empty\x12F\n" + + "\tUnaryCall\x12\x1b.grpc.testing.SimpleRequest\x1a\x1c.grpc.testing.SimpleResponse\x12O\n" + + "\x12CacheableUnaryCall\x12\x1b.grpc.testing.SimpleRequest\x1a\x1c.grpc.testing.SimpleResponse\x12l\n" + + "\x13StreamingOutputCall\x12(.grpc.testing.StreamingOutputCallRequest\x1a).grpc.testing.StreamingOutputCallResponse0\x01\x12i\n" + + "\x12StreamingInputCall\x12'.grpc.testing.StreamingInputCallRequest\x1a(.grpc.testing.StreamingInputCallResponse(\x01\x12i\n" + + "\x0eFullDuplexCall\x12(.grpc.testing.StreamingOutputCallRequest\x1a).grpc.testing.StreamingOutputCallResponse(\x010\x01\x12i\n" + + "\x0eHalfDuplexCall\x12(.grpc.testing.StreamingOutputCallRequest\x1a).grpc.testing.StreamingOutputCallResponse(\x010\x01\x12=\n" + + "\x11UnimplementedCall\x12\x13.grpc.testing.Empty\x1a\x13.grpc.testing.Empty2U\n" + + "\x14UnimplementedService\x12=\n" + + "\x11UnimplementedCall\x12\x13.grpc.testing.Empty\x1a\x13.grpc.testing.Empty2\x89\x01\n" + + "\x10ReconnectService\x12;\n" + + "\x05Start\x12\x1d.grpc.testing.ReconnectParams\x1a\x13.grpc.testing.Empty\x128\n" + + "\x04Stop\x12\x13.grpc.testing.Empty\x1a\x1b.grpc.testing.ReconnectInfo2\x86\x02\n" + + "\x18LoadBalancerStatsService\x12c\n" + + "\x0eGetClientStats\x12&.grpc.testing.LoadBalancerStatsRequest\x1a'.grpc.testing.LoadBalancerStatsResponse\"\x00\x12\x84\x01\n" + + "\x19GetClientAccumulatedStats\x121.grpc.testing.LoadBalancerAccumulatedStatsRequest\x1a2.grpc.testing.LoadBalancerAccumulatedStatsResponse\"\x002\xcc\x01\n" + + "\vHookService\x120\n" + + "\x04Hook\x12\x13.grpc.testing.Empty\x1a\x13.grpc.testing.Empty\x12L\n" + + "\x0fSetReturnStatus\x12$.grpc.testing.SetReturnStatusRequest\x1a\x13.grpc.testing.Empty\x12=\n" + + "\x11ClearReturnStatus\x12\x13.grpc.testing.Empty\x1a\x13.grpc.testing.Empty2\xd5\x01\n" + + "\x16XdsUpdateHealthService\x126\n" + + "\n" + + "SetServing\x12\x13.grpc.testing.Empty\x1a\x13.grpc.testing.Empty\x129\n" + + "\rSetNotServing\x12\x13.grpc.testing.Empty\x1a\x13.grpc.testing.Empty\x12H\n" + + "\x0fSendHookRequest\x12\x19.grpc.testing.HookRequest\x1a\x1a.grpc.testing.HookResponse2{\n" + + "\x1fXdsUpdateClientConfigureService\x12X\n" + + "\tConfigure\x12$.grpc.testing.ClientConfigureRequest\x1a%.grpc.testing.ClientConfigureResponseB\x1d\n" + + "\x1bio.grpc.testing.integrationb\x06proto3" + +var file_grpc_testing_test_proto_goTypes = []any{ + (*Empty)(nil), // 0: grpc.testing.Empty + (*SimpleRequest)(nil), // 1: grpc.testing.SimpleRequest + (*StreamingOutputCallRequest)(nil), // 2: grpc.testing.StreamingOutputCallRequest + (*StreamingInputCallRequest)(nil), // 3: grpc.testing.StreamingInputCallRequest + (*ReconnectParams)(nil), // 4: grpc.testing.ReconnectParams + (*LoadBalancerStatsRequest)(nil), // 5: grpc.testing.LoadBalancerStatsRequest + (*LoadBalancerAccumulatedStatsRequest)(nil), // 6: grpc.testing.LoadBalancerAccumulatedStatsRequest + (*SetReturnStatusRequest)(nil), // 7: grpc.testing.SetReturnStatusRequest + (*HookRequest)(nil), // 8: grpc.testing.HookRequest + (*ClientConfigureRequest)(nil), // 9: grpc.testing.ClientConfigureRequest + (*SimpleResponse)(nil), // 10: grpc.testing.SimpleResponse + (*StreamingOutputCallResponse)(nil), // 11: grpc.testing.StreamingOutputCallResponse + (*StreamingInputCallResponse)(nil), // 12: grpc.testing.StreamingInputCallResponse + (*ReconnectInfo)(nil), // 13: grpc.testing.ReconnectInfo + (*LoadBalancerStatsResponse)(nil), // 14: grpc.testing.LoadBalancerStatsResponse + (*LoadBalancerAccumulatedStatsResponse)(nil), // 15: grpc.testing.LoadBalancerAccumulatedStatsResponse + (*HookResponse)(nil), // 16: grpc.testing.HookResponse + (*ClientConfigureResponse)(nil), // 17: grpc.testing.ClientConfigureResponse +} +var file_grpc_testing_test_proto_depIdxs = []int32{ + 0, // 0: grpc.testing.TestService.EmptyCall:input_type -> grpc.testing.Empty + 1, // 1: grpc.testing.TestService.UnaryCall:input_type -> grpc.testing.SimpleRequest + 1, // 2: grpc.testing.TestService.CacheableUnaryCall:input_type -> grpc.testing.SimpleRequest + 2, // 3: grpc.testing.TestService.StreamingOutputCall:input_type -> grpc.testing.StreamingOutputCallRequest + 3, // 4: grpc.testing.TestService.StreamingInputCall:input_type -> grpc.testing.StreamingInputCallRequest + 2, // 5: grpc.testing.TestService.FullDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest + 2, // 6: grpc.testing.TestService.HalfDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest + 0, // 7: grpc.testing.TestService.UnimplementedCall:input_type -> grpc.testing.Empty + 0, // 8: grpc.testing.UnimplementedService.UnimplementedCall:input_type -> grpc.testing.Empty + 4, // 9: grpc.testing.ReconnectService.Start:input_type -> grpc.testing.ReconnectParams + 0, // 10: grpc.testing.ReconnectService.Stop:input_type -> grpc.testing.Empty + 5, // 11: grpc.testing.LoadBalancerStatsService.GetClientStats:input_type -> grpc.testing.LoadBalancerStatsRequest + 6, // 12: grpc.testing.LoadBalancerStatsService.GetClientAccumulatedStats:input_type -> grpc.testing.LoadBalancerAccumulatedStatsRequest + 0, // 13: grpc.testing.HookService.Hook:input_type -> grpc.testing.Empty + 7, // 14: grpc.testing.HookService.SetReturnStatus:input_type -> grpc.testing.SetReturnStatusRequest + 0, // 15: grpc.testing.HookService.ClearReturnStatus:input_type -> grpc.testing.Empty + 0, // 16: grpc.testing.XdsUpdateHealthService.SetServing:input_type -> grpc.testing.Empty + 0, // 17: grpc.testing.XdsUpdateHealthService.SetNotServing:input_type -> grpc.testing.Empty + 8, // 18: grpc.testing.XdsUpdateHealthService.SendHookRequest:input_type -> grpc.testing.HookRequest + 9, // 19: grpc.testing.XdsUpdateClientConfigureService.Configure:input_type -> grpc.testing.ClientConfigureRequest + 0, // 20: grpc.testing.TestService.EmptyCall:output_type -> grpc.testing.Empty + 10, // 21: grpc.testing.TestService.UnaryCall:output_type -> grpc.testing.SimpleResponse + 10, // 22: grpc.testing.TestService.CacheableUnaryCall:output_type -> grpc.testing.SimpleResponse + 11, // 23: grpc.testing.TestService.StreamingOutputCall:output_type -> grpc.testing.StreamingOutputCallResponse + 12, // 24: grpc.testing.TestService.StreamingInputCall:output_type -> grpc.testing.StreamingInputCallResponse + 11, // 25: grpc.testing.TestService.FullDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse + 11, // 26: grpc.testing.TestService.HalfDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse + 0, // 27: grpc.testing.TestService.UnimplementedCall:output_type -> grpc.testing.Empty + 0, // 28: grpc.testing.UnimplementedService.UnimplementedCall:output_type -> grpc.testing.Empty + 0, // 29: grpc.testing.ReconnectService.Start:output_type -> grpc.testing.Empty + 13, // 30: grpc.testing.ReconnectService.Stop:output_type -> grpc.testing.ReconnectInfo + 14, // 31: grpc.testing.LoadBalancerStatsService.GetClientStats:output_type -> grpc.testing.LoadBalancerStatsResponse + 15, // 32: grpc.testing.LoadBalancerStatsService.GetClientAccumulatedStats:output_type -> grpc.testing.LoadBalancerAccumulatedStatsResponse + 0, // 33: grpc.testing.HookService.Hook:output_type -> grpc.testing.Empty + 0, // 34: grpc.testing.HookService.SetReturnStatus:output_type -> grpc.testing.Empty + 0, // 35: grpc.testing.HookService.ClearReturnStatus:output_type -> grpc.testing.Empty + 0, // 36: grpc.testing.XdsUpdateHealthService.SetServing:output_type -> grpc.testing.Empty + 0, // 37: grpc.testing.XdsUpdateHealthService.SetNotServing:output_type -> grpc.testing.Empty + 16, // 38: grpc.testing.XdsUpdateHealthService.SendHookRequest:output_type -> grpc.testing.HookResponse + 17, // 39: grpc.testing.XdsUpdateClientConfigureService.Configure:output_type -> grpc.testing.ClientConfigureResponse + 20, // [20:40] is the sub-list for method output_type + 0, // [0:20] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_grpc_testing_test_proto_init() } +func file_grpc_testing_test_proto_init() { + if File_grpc_testing_test_proto != nil { + return + } + file_grpc_testing_empty_proto_init() + file_grpc_testing_messages_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_testing_test_proto_rawDesc), len(file_grpc_testing_test_proto_rawDesc)), + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 7, + }, + GoTypes: file_grpc_testing_test_proto_goTypes, + DependencyIndexes: file_grpc_testing_test_proto_depIdxs, + }.Build() + File_grpc_testing_test_proto = out.File + file_grpc_testing_test_proto_goTypes = nil + file_grpc_testing_test_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go new file mode 100644 index 0000000..f84d75c --- /dev/null +++ b/interop/grpc_testing/test_grpc.pb.go @@ -0,0 +1,1316 @@ +// Copyright 2015-2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: grpc/testing/test.proto + +package grpc_testing + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + TestService_EmptyCall_FullMethodName = "/grpc.testing.TestService/EmptyCall" + TestService_UnaryCall_FullMethodName = "/grpc.testing.TestService/UnaryCall" + TestService_CacheableUnaryCall_FullMethodName = "/grpc.testing.TestService/CacheableUnaryCall" + TestService_StreamingOutputCall_FullMethodName = "/grpc.testing.TestService/StreamingOutputCall" + TestService_StreamingInputCall_FullMethodName = "/grpc.testing.TestService/StreamingInputCall" + TestService_FullDuplexCall_FullMethodName = "/grpc.testing.TestService/FullDuplexCall" + TestService_HalfDuplexCall_FullMethodName = "/grpc.testing.TestService/HalfDuplexCall" + TestService_UnimplementedCall_FullMethodName = "/grpc.testing.TestService/UnimplementedCall" +) + +// TestServiceClient is the client API for TestService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// A simple service to test the various types of RPCs and experiment with +// performance with various types of payload. +type TestServiceClient interface { + // One empty request followed by one empty response. + EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) + // One request followed by one response. + UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) + // One request followed by one response. Response has cache control + // headers set such that a caching HTTP proxy (such as GFE) can + // satisfy subsequent requests. + CacheableUnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) + // One request followed by a sequence of responses (streamed download). + // The server returns the payload with client desired type and sizes. + StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[StreamingOutputCallResponse], error) + // A sequence of requests followed by one response (streamed upload). + // The server returns the aggregated size of client payload as the result. + StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[StreamingInputCallRequest, StreamingInputCallResponse], error) + // A sequence of requests with each request served by the server immediately. + // As one request could lead to multiple responses, this interface + // demonstrates the idea of full duplexing. + FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[StreamingOutputCallRequest, StreamingOutputCallResponse], error) + // A sequence of requests followed by a sequence of responses. + // The server buffers all the client requests and then serves them in order. A + // stream of responses are returned to the client when the server starts with + // first request. + HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[StreamingOutputCallRequest, StreamingOutputCallResponse], error) + // The test server will not implement this method. It will be used + // to test the behavior when clients call unimplemented methods. + UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) +} + +type testServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient { + return &testServiceClient{cc} +} + +func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Empty) + err := c.cc.Invoke(ctx, TestService_EmptyCall_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SimpleResponse) + err := c.cc.Invoke(ctx, TestService_UnaryCall_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *testServiceClient) CacheableUnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SimpleResponse) + err := c.cc.Invoke(ctx, TestService_CacheableUnaryCall_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[StreamingOutputCallResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[0], TestService_StreamingOutputCall_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[StreamingOutputCallRequest, StreamingOutputCallResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TestService_StreamingOutputCallClient = grpc.ServerStreamingClient[StreamingOutputCallResponse] + +func (c *testServiceClient) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[StreamingInputCallRequest, StreamingInputCallResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[1], TestService_StreamingInputCall_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[StreamingInputCallRequest, StreamingInputCallResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TestService_StreamingInputCallClient = grpc.ClientStreamingClient[StreamingInputCallRequest, StreamingInputCallResponse] + +func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[StreamingOutputCallRequest, StreamingOutputCallResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[2], TestService_FullDuplexCall_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[StreamingOutputCallRequest, StreamingOutputCallResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TestService_FullDuplexCallClient = grpc.BidiStreamingClient[StreamingOutputCallRequest, StreamingOutputCallResponse] + +func (c *testServiceClient) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[StreamingOutputCallRequest, StreamingOutputCallResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[3], TestService_HalfDuplexCall_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[StreamingOutputCallRequest, StreamingOutputCallResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TestService_HalfDuplexCallClient = grpc.BidiStreamingClient[StreamingOutputCallRequest, StreamingOutputCallResponse] + +func (c *testServiceClient) UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Empty) + err := c.cc.Invoke(ctx, TestService_UnimplementedCall_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// TestServiceServer is the server API for TestService service. +// All implementations must embed UnimplementedTestServiceServer +// for forward compatibility. +// +// A simple service to test the various types of RPCs and experiment with +// performance with various types of payload. +type TestServiceServer interface { + // One empty request followed by one empty response. + EmptyCall(context.Context, *Empty) (*Empty, error) + // One request followed by one response. + UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) + // One request followed by one response. Response has cache control + // headers set such that a caching HTTP proxy (such as GFE) can + // satisfy subsequent requests. + CacheableUnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) + // One request followed by a sequence of responses (streamed download). + // The server returns the payload with client desired type and sizes. + StreamingOutputCall(*StreamingOutputCallRequest, grpc.ServerStreamingServer[StreamingOutputCallResponse]) error + // A sequence of requests followed by one response (streamed upload). + // The server returns the aggregated size of client payload as the result. + StreamingInputCall(grpc.ClientStreamingServer[StreamingInputCallRequest, StreamingInputCallResponse]) error + // A sequence of requests with each request served by the server immediately. + // As one request could lead to multiple responses, this interface + // demonstrates the idea of full duplexing. + FullDuplexCall(grpc.BidiStreamingServer[StreamingOutputCallRequest, StreamingOutputCallResponse]) error + // A sequence of requests followed by a sequence of responses. + // The server buffers all the client requests and then serves them in order. A + // stream of responses are returned to the client when the server starts with + // first request. + HalfDuplexCall(grpc.BidiStreamingServer[StreamingOutputCallRequest, StreamingOutputCallResponse]) error + // The test server will not implement this method. It will be used + // to test the behavior when clients call unimplemented methods. + UnimplementedCall(context.Context, *Empty) (*Empty, error) + mustEmbedUnimplementedTestServiceServer() +} + +// UnimplementedTestServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedTestServiceServer struct{} + +func (UnimplementedTestServiceServer) EmptyCall(context.Context, *Empty) (*Empty, error) { + return nil, status.Error(codes.Unimplemented, "method EmptyCall not implemented") +} +func (UnimplementedTestServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) { + return nil, status.Error(codes.Unimplemented, "method UnaryCall not implemented") +} +func (UnimplementedTestServiceServer) CacheableUnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) { + return nil, status.Error(codes.Unimplemented, "method CacheableUnaryCall not implemented") +} +func (UnimplementedTestServiceServer) StreamingOutputCall(*StreamingOutputCallRequest, grpc.ServerStreamingServer[StreamingOutputCallResponse]) error { + return status.Error(codes.Unimplemented, "method StreamingOutputCall not implemented") +} +func (UnimplementedTestServiceServer) StreamingInputCall(grpc.ClientStreamingServer[StreamingInputCallRequest, StreamingInputCallResponse]) error { + return status.Error(codes.Unimplemented, "method StreamingInputCall not implemented") +} +func (UnimplementedTestServiceServer) FullDuplexCall(grpc.BidiStreamingServer[StreamingOutputCallRequest, StreamingOutputCallResponse]) error { + return status.Error(codes.Unimplemented, "method FullDuplexCall not implemented") +} +func (UnimplementedTestServiceServer) HalfDuplexCall(grpc.BidiStreamingServer[StreamingOutputCallRequest, StreamingOutputCallResponse]) error { + return status.Error(codes.Unimplemented, "method HalfDuplexCall not implemented") +} +func (UnimplementedTestServiceServer) UnimplementedCall(context.Context, *Empty) (*Empty, error) { + return nil, status.Error(codes.Unimplemented, "method UnimplementedCall not implemented") +} +func (UnimplementedTestServiceServer) mustEmbedUnimplementedTestServiceServer() {} +func (UnimplementedTestServiceServer) testEmbeddedByValue() {} + +// UnsafeTestServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to TestServiceServer will +// result in compilation errors. +type UnsafeTestServiceServer interface { + mustEmbedUnimplementedTestServiceServer() +} + +func RegisterTestServiceServer(s grpc.ServiceRegistrar, srv TestServiceServer) { + // If the following call panics, it indicates UnimplementedTestServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&TestService_ServiceDesc, srv) +} + +func _TestService_EmptyCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TestServiceServer).EmptyCall(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TestService_EmptyCall_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TestServiceServer).EmptyCall(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SimpleRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TestServiceServer).UnaryCall(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TestService_UnaryCall_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _TestService_CacheableUnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SimpleRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TestServiceServer).CacheableUnaryCall(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TestService_CacheableUnaryCall_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TestServiceServer).CacheableUnaryCall(ctx, req.(*SimpleRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _TestService_StreamingOutputCall_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(StreamingOutputCallRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(TestServiceServer).StreamingOutputCall(m, &grpc.GenericServerStream[StreamingOutputCallRequest, StreamingOutputCallResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TestService_StreamingOutputCallServer = grpc.ServerStreamingServer[StreamingOutputCallResponse] + +func _TestService_StreamingInputCall_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(TestServiceServer).StreamingInputCall(&grpc.GenericServerStream[StreamingInputCallRequest, StreamingInputCallResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TestService_StreamingInputCallServer = grpc.ClientStreamingServer[StreamingInputCallRequest, StreamingInputCallResponse] + +func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(TestServiceServer).FullDuplexCall(&grpc.GenericServerStream[StreamingOutputCallRequest, StreamingOutputCallResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TestService_FullDuplexCallServer = grpc.BidiStreamingServer[StreamingOutputCallRequest, StreamingOutputCallResponse] + +func _TestService_HalfDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(TestServiceServer).HalfDuplexCall(&grpc.GenericServerStream[StreamingOutputCallRequest, StreamingOutputCallResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TestService_HalfDuplexCallServer = grpc.BidiStreamingServer[StreamingOutputCallRequest, StreamingOutputCallResponse] + +func _TestService_UnimplementedCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TestServiceServer).UnimplementedCall(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TestService_UnimplementedCall_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TestServiceServer).UnimplementedCall(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) +} + +// TestService_ServiceDesc is the grpc.ServiceDesc for TestService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var TestService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.TestService", + HandlerType: (*TestServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "EmptyCall", + Handler: _TestService_EmptyCall_Handler, + }, + { + MethodName: "UnaryCall", + Handler: _TestService_UnaryCall_Handler, + }, + { + MethodName: "CacheableUnaryCall", + Handler: _TestService_CacheableUnaryCall_Handler, + }, + { + MethodName: "UnimplementedCall", + Handler: _TestService_UnimplementedCall_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "StreamingOutputCall", + Handler: _TestService_StreamingOutputCall_Handler, + ServerStreams: true, + }, + { + StreamName: "StreamingInputCall", + Handler: _TestService_StreamingInputCall_Handler, + ClientStreams: true, + }, + { + StreamName: "FullDuplexCall", + Handler: _TestService_FullDuplexCall_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "HalfDuplexCall", + Handler: _TestService_HalfDuplexCall_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/testing/test.proto", +} + +const ( + UnimplementedService_UnimplementedCall_FullMethodName = "/grpc.testing.UnimplementedService/UnimplementedCall" +) + +// UnimplementedServiceClient is the client API for UnimplementedService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// A simple service NOT implemented at servers so clients can test for +// that case. +type UnimplementedServiceClient interface { + // A call that no server should implement + UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) +} + +type unimplementedServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewUnimplementedServiceClient(cc grpc.ClientConnInterface) UnimplementedServiceClient { + return &unimplementedServiceClient{cc} +} + +func (c *unimplementedServiceClient) UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Empty) + err := c.cc.Invoke(ctx, UnimplementedService_UnimplementedCall_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// UnimplementedServiceServer is the server API for UnimplementedService service. +// All implementations must embed UnimplementedUnimplementedServiceServer +// for forward compatibility. +// +// A simple service NOT implemented at servers so clients can test for +// that case. +type UnimplementedServiceServer interface { + // A call that no server should implement + UnimplementedCall(context.Context, *Empty) (*Empty, error) + mustEmbedUnimplementedUnimplementedServiceServer() +} + +// UnimplementedUnimplementedServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedUnimplementedServiceServer struct{} + +func (UnimplementedUnimplementedServiceServer) UnimplementedCall(context.Context, *Empty) (*Empty, error) { + return nil, status.Error(codes.Unimplemented, "method UnimplementedCall not implemented") +} +func (UnimplementedUnimplementedServiceServer) mustEmbedUnimplementedUnimplementedServiceServer() {} +func (UnimplementedUnimplementedServiceServer) testEmbeddedByValue() {} + +// UnsafeUnimplementedServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to UnimplementedServiceServer will +// result in compilation errors. +type UnsafeUnimplementedServiceServer interface { + mustEmbedUnimplementedUnimplementedServiceServer() +} + +func RegisterUnimplementedServiceServer(s grpc.ServiceRegistrar, srv UnimplementedServiceServer) { + // If the following call panics, it indicates UnimplementedUnimplementedServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&UnimplementedService_ServiceDesc, srv) +} + +func _UnimplementedService_UnimplementedCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(UnimplementedServiceServer).UnimplementedCall(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: UnimplementedService_UnimplementedCall_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(UnimplementedServiceServer).UnimplementedCall(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) +} + +// UnimplementedService_ServiceDesc is the grpc.ServiceDesc for UnimplementedService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var UnimplementedService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.UnimplementedService", + HandlerType: (*UnimplementedServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "UnimplementedCall", + Handler: _UnimplementedService_UnimplementedCall_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/test.proto", +} + +const ( + ReconnectService_Start_FullMethodName = "/grpc.testing.ReconnectService/Start" + ReconnectService_Stop_FullMethodName = "/grpc.testing.ReconnectService/Stop" +) + +// ReconnectServiceClient is the client API for ReconnectService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// A service used to control reconnect server. +type ReconnectServiceClient interface { + Start(ctx context.Context, in *ReconnectParams, opts ...grpc.CallOption) (*Empty, error) + Stop(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ReconnectInfo, error) +} + +type reconnectServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewReconnectServiceClient(cc grpc.ClientConnInterface) ReconnectServiceClient { + return &reconnectServiceClient{cc} +} + +func (c *reconnectServiceClient) Start(ctx context.Context, in *ReconnectParams, opts ...grpc.CallOption) (*Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Empty) + err := c.cc.Invoke(ctx, ReconnectService_Start_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *reconnectServiceClient) Stop(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ReconnectInfo, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ReconnectInfo) + err := c.cc.Invoke(ctx, ReconnectService_Stop_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ReconnectServiceServer is the server API for ReconnectService service. +// All implementations must embed UnimplementedReconnectServiceServer +// for forward compatibility. +// +// A service used to control reconnect server. +type ReconnectServiceServer interface { + Start(context.Context, *ReconnectParams) (*Empty, error) + Stop(context.Context, *Empty) (*ReconnectInfo, error) + mustEmbedUnimplementedReconnectServiceServer() +} + +// UnimplementedReconnectServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedReconnectServiceServer struct{} + +func (UnimplementedReconnectServiceServer) Start(context.Context, *ReconnectParams) (*Empty, error) { + return nil, status.Error(codes.Unimplemented, "method Start not implemented") +} +func (UnimplementedReconnectServiceServer) Stop(context.Context, *Empty) (*ReconnectInfo, error) { + return nil, status.Error(codes.Unimplemented, "method Stop not implemented") +} +func (UnimplementedReconnectServiceServer) mustEmbedUnimplementedReconnectServiceServer() {} +func (UnimplementedReconnectServiceServer) testEmbeddedByValue() {} + +// UnsafeReconnectServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ReconnectServiceServer will +// result in compilation errors. +type UnsafeReconnectServiceServer interface { + mustEmbedUnimplementedReconnectServiceServer() +} + +func RegisterReconnectServiceServer(s grpc.ServiceRegistrar, srv ReconnectServiceServer) { + // If the following call panics, it indicates UnimplementedReconnectServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&ReconnectService_ServiceDesc, srv) +} + +func _ReconnectService_Start_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReconnectParams) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ReconnectServiceServer).Start(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ReconnectService_Start_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ReconnectServiceServer).Start(ctx, req.(*ReconnectParams)) + } + return interceptor(ctx, in, info, handler) +} + +func _ReconnectService_Stop_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ReconnectServiceServer).Stop(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ReconnectService_Stop_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ReconnectServiceServer).Stop(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) +} + +// ReconnectService_ServiceDesc is the grpc.ServiceDesc for ReconnectService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ReconnectService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.ReconnectService", + HandlerType: (*ReconnectServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Start", + Handler: _ReconnectService_Start_Handler, + }, + { + MethodName: "Stop", + Handler: _ReconnectService_Stop_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/test.proto", +} + +const ( + LoadBalancerStatsService_GetClientStats_FullMethodName = "/grpc.testing.LoadBalancerStatsService/GetClientStats" + LoadBalancerStatsService_GetClientAccumulatedStats_FullMethodName = "/grpc.testing.LoadBalancerStatsService/GetClientAccumulatedStats" +) + +// LoadBalancerStatsServiceClient is the client API for LoadBalancerStatsService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// A service used to obtain stats for verifying LB behavior. +type LoadBalancerStatsServiceClient interface { + // Gets the backend distribution for RPCs sent by a test client. + GetClientStats(ctx context.Context, in *LoadBalancerStatsRequest, opts ...grpc.CallOption) (*LoadBalancerStatsResponse, error) + // Gets the accumulated stats for RPCs sent by a test client. + GetClientAccumulatedStats(ctx context.Context, in *LoadBalancerAccumulatedStatsRequest, opts ...grpc.CallOption) (*LoadBalancerAccumulatedStatsResponse, error) +} + +type loadBalancerStatsServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewLoadBalancerStatsServiceClient(cc grpc.ClientConnInterface) LoadBalancerStatsServiceClient { + return &loadBalancerStatsServiceClient{cc} +} + +func (c *loadBalancerStatsServiceClient) GetClientStats(ctx context.Context, in *LoadBalancerStatsRequest, opts ...grpc.CallOption) (*LoadBalancerStatsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(LoadBalancerStatsResponse) + err := c.cc.Invoke(ctx, LoadBalancerStatsService_GetClientStats_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *loadBalancerStatsServiceClient) GetClientAccumulatedStats(ctx context.Context, in *LoadBalancerAccumulatedStatsRequest, opts ...grpc.CallOption) (*LoadBalancerAccumulatedStatsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(LoadBalancerAccumulatedStatsResponse) + err := c.cc.Invoke(ctx, LoadBalancerStatsService_GetClientAccumulatedStats_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// LoadBalancerStatsServiceServer is the server API for LoadBalancerStatsService service. +// All implementations must embed UnimplementedLoadBalancerStatsServiceServer +// for forward compatibility. +// +// A service used to obtain stats for verifying LB behavior. +type LoadBalancerStatsServiceServer interface { + // Gets the backend distribution for RPCs sent by a test client. + GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) + // Gets the accumulated stats for RPCs sent by a test client. + GetClientAccumulatedStats(context.Context, *LoadBalancerAccumulatedStatsRequest) (*LoadBalancerAccumulatedStatsResponse, error) + mustEmbedUnimplementedLoadBalancerStatsServiceServer() +} + +// UnimplementedLoadBalancerStatsServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedLoadBalancerStatsServiceServer struct{} + +func (UnimplementedLoadBalancerStatsServiceServer) GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetClientStats not implemented") +} +func (UnimplementedLoadBalancerStatsServiceServer) GetClientAccumulatedStats(context.Context, *LoadBalancerAccumulatedStatsRequest) (*LoadBalancerAccumulatedStatsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetClientAccumulatedStats not implemented") +} +func (UnimplementedLoadBalancerStatsServiceServer) mustEmbedUnimplementedLoadBalancerStatsServiceServer() { +} +func (UnimplementedLoadBalancerStatsServiceServer) testEmbeddedByValue() {} + +// UnsafeLoadBalancerStatsServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to LoadBalancerStatsServiceServer will +// result in compilation errors. +type UnsafeLoadBalancerStatsServiceServer interface { + mustEmbedUnimplementedLoadBalancerStatsServiceServer() +} + +func RegisterLoadBalancerStatsServiceServer(s grpc.ServiceRegistrar, srv LoadBalancerStatsServiceServer) { + // If the following call panics, it indicates UnimplementedLoadBalancerStatsServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&LoadBalancerStatsService_ServiceDesc, srv) +} + +func _LoadBalancerStatsService_GetClientStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LoadBalancerStatsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(LoadBalancerStatsServiceServer).GetClientStats(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: LoadBalancerStatsService_GetClientStats_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(LoadBalancerStatsServiceServer).GetClientStats(ctx, req.(*LoadBalancerStatsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _LoadBalancerStatsService_GetClientAccumulatedStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LoadBalancerAccumulatedStatsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(LoadBalancerStatsServiceServer).GetClientAccumulatedStats(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: LoadBalancerStatsService_GetClientAccumulatedStats_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(LoadBalancerStatsServiceServer).GetClientAccumulatedStats(ctx, req.(*LoadBalancerAccumulatedStatsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// LoadBalancerStatsService_ServiceDesc is the grpc.ServiceDesc for LoadBalancerStatsService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var LoadBalancerStatsService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.LoadBalancerStatsService", + HandlerType: (*LoadBalancerStatsServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetClientStats", + Handler: _LoadBalancerStatsService_GetClientStats_Handler, + }, + { + MethodName: "GetClientAccumulatedStats", + Handler: _LoadBalancerStatsService_GetClientAccumulatedStats_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/test.proto", +} + +const ( + HookService_Hook_FullMethodName = "/grpc.testing.HookService/Hook" + HookService_SetReturnStatus_FullMethodName = "/grpc.testing.HookService/SetReturnStatus" + HookService_ClearReturnStatus_FullMethodName = "/grpc.testing.HookService/ClearReturnStatus" +) + +// HookServiceClient is the client API for HookService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Hook service. Used to keep Kubernetes from shutting the pod down. +type HookServiceClient interface { + // Sends a request that will "hang" until the return status is set by a call + // to a SetReturnStatus + Hook(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) + // Sets a return status for pending and upcoming calls to Hook + SetReturnStatus(ctx context.Context, in *SetReturnStatusRequest, opts ...grpc.CallOption) (*Empty, error) + // Clears the return status. Incoming calls to Hook will "hang" + ClearReturnStatus(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) +} + +type hookServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewHookServiceClient(cc grpc.ClientConnInterface) HookServiceClient { + return &hookServiceClient{cc} +} + +func (c *hookServiceClient) Hook(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Empty) + err := c.cc.Invoke(ctx, HookService_Hook_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *hookServiceClient) SetReturnStatus(ctx context.Context, in *SetReturnStatusRequest, opts ...grpc.CallOption) (*Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Empty) + err := c.cc.Invoke(ctx, HookService_SetReturnStatus_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *hookServiceClient) ClearReturnStatus(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Empty) + err := c.cc.Invoke(ctx, HookService_ClearReturnStatus_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// HookServiceServer is the server API for HookService service. +// All implementations must embed UnimplementedHookServiceServer +// for forward compatibility. +// +// Hook service. Used to keep Kubernetes from shutting the pod down. +type HookServiceServer interface { + // Sends a request that will "hang" until the return status is set by a call + // to a SetReturnStatus + Hook(context.Context, *Empty) (*Empty, error) + // Sets a return status for pending and upcoming calls to Hook + SetReturnStatus(context.Context, *SetReturnStatusRequest) (*Empty, error) + // Clears the return status. Incoming calls to Hook will "hang" + ClearReturnStatus(context.Context, *Empty) (*Empty, error) + mustEmbedUnimplementedHookServiceServer() +} + +// UnimplementedHookServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedHookServiceServer struct{} + +func (UnimplementedHookServiceServer) Hook(context.Context, *Empty) (*Empty, error) { + return nil, status.Error(codes.Unimplemented, "method Hook not implemented") +} +func (UnimplementedHookServiceServer) SetReturnStatus(context.Context, *SetReturnStatusRequest) (*Empty, error) { + return nil, status.Error(codes.Unimplemented, "method SetReturnStatus not implemented") +} +func (UnimplementedHookServiceServer) ClearReturnStatus(context.Context, *Empty) (*Empty, error) { + return nil, status.Error(codes.Unimplemented, "method ClearReturnStatus not implemented") +} +func (UnimplementedHookServiceServer) mustEmbedUnimplementedHookServiceServer() {} +func (UnimplementedHookServiceServer) testEmbeddedByValue() {} + +// UnsafeHookServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to HookServiceServer will +// result in compilation errors. +type UnsafeHookServiceServer interface { + mustEmbedUnimplementedHookServiceServer() +} + +func RegisterHookServiceServer(s grpc.ServiceRegistrar, srv HookServiceServer) { + // If the following call panics, it indicates UnimplementedHookServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&HookService_ServiceDesc, srv) +} + +func _HookService_Hook_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HookServiceServer).Hook(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: HookService_Hook_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HookServiceServer).Hook(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _HookService_SetReturnStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetReturnStatusRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HookServiceServer).SetReturnStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: HookService_SetReturnStatus_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HookServiceServer).SetReturnStatus(ctx, req.(*SetReturnStatusRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _HookService_ClearReturnStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HookServiceServer).ClearReturnStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: HookService_ClearReturnStatus_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HookServiceServer).ClearReturnStatus(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) +} + +// HookService_ServiceDesc is the grpc.ServiceDesc for HookService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var HookService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.HookService", + HandlerType: (*HookServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Hook", + Handler: _HookService_Hook_Handler, + }, + { + MethodName: "SetReturnStatus", + Handler: _HookService_SetReturnStatus_Handler, + }, + { + MethodName: "ClearReturnStatus", + Handler: _HookService_ClearReturnStatus_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/test.proto", +} + +const ( + XdsUpdateHealthService_SetServing_FullMethodName = "/grpc.testing.XdsUpdateHealthService/SetServing" + XdsUpdateHealthService_SetNotServing_FullMethodName = "/grpc.testing.XdsUpdateHealthService/SetNotServing" + XdsUpdateHealthService_SendHookRequest_FullMethodName = "/grpc.testing.XdsUpdateHealthService/SendHookRequest" +) + +// XdsUpdateHealthServiceClient is the client API for XdsUpdateHealthService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// A service to remotely control health status of an xDS test server. +type XdsUpdateHealthServiceClient interface { + SetServing(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) + SetNotServing(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) + SendHookRequest(ctx context.Context, in *HookRequest, opts ...grpc.CallOption) (*HookResponse, error) +} + +type xdsUpdateHealthServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewXdsUpdateHealthServiceClient(cc grpc.ClientConnInterface) XdsUpdateHealthServiceClient { + return &xdsUpdateHealthServiceClient{cc} +} + +func (c *xdsUpdateHealthServiceClient) SetServing(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Empty) + err := c.cc.Invoke(ctx, XdsUpdateHealthService_SetServing_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *xdsUpdateHealthServiceClient) SetNotServing(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Empty) + err := c.cc.Invoke(ctx, XdsUpdateHealthService_SetNotServing_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *xdsUpdateHealthServiceClient) SendHookRequest(ctx context.Context, in *HookRequest, opts ...grpc.CallOption) (*HookResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(HookResponse) + err := c.cc.Invoke(ctx, XdsUpdateHealthService_SendHookRequest_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// XdsUpdateHealthServiceServer is the server API for XdsUpdateHealthService service. +// All implementations must embed UnimplementedXdsUpdateHealthServiceServer +// for forward compatibility. +// +// A service to remotely control health status of an xDS test server. +type XdsUpdateHealthServiceServer interface { + SetServing(context.Context, *Empty) (*Empty, error) + SetNotServing(context.Context, *Empty) (*Empty, error) + SendHookRequest(context.Context, *HookRequest) (*HookResponse, error) + mustEmbedUnimplementedXdsUpdateHealthServiceServer() +} + +// UnimplementedXdsUpdateHealthServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedXdsUpdateHealthServiceServer struct{} + +func (UnimplementedXdsUpdateHealthServiceServer) SetServing(context.Context, *Empty) (*Empty, error) { + return nil, status.Error(codes.Unimplemented, "method SetServing not implemented") +} +func (UnimplementedXdsUpdateHealthServiceServer) SetNotServing(context.Context, *Empty) (*Empty, error) { + return nil, status.Error(codes.Unimplemented, "method SetNotServing not implemented") +} +func (UnimplementedXdsUpdateHealthServiceServer) SendHookRequest(context.Context, *HookRequest) (*HookResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SendHookRequest not implemented") +} +func (UnimplementedXdsUpdateHealthServiceServer) mustEmbedUnimplementedXdsUpdateHealthServiceServer() { +} +func (UnimplementedXdsUpdateHealthServiceServer) testEmbeddedByValue() {} + +// UnsafeXdsUpdateHealthServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to XdsUpdateHealthServiceServer will +// result in compilation errors. +type UnsafeXdsUpdateHealthServiceServer interface { + mustEmbedUnimplementedXdsUpdateHealthServiceServer() +} + +func RegisterXdsUpdateHealthServiceServer(s grpc.ServiceRegistrar, srv XdsUpdateHealthServiceServer) { + // If the following call panics, it indicates UnimplementedXdsUpdateHealthServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&XdsUpdateHealthService_ServiceDesc, srv) +} + +func _XdsUpdateHealthService_SetServing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(XdsUpdateHealthServiceServer).SetServing(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: XdsUpdateHealthService_SetServing_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(XdsUpdateHealthServiceServer).SetServing(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _XdsUpdateHealthService_SetNotServing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(XdsUpdateHealthServiceServer).SetNotServing(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: XdsUpdateHealthService_SetNotServing_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(XdsUpdateHealthServiceServer).SetNotServing(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _XdsUpdateHealthService_SendHookRequest_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(HookRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(XdsUpdateHealthServiceServer).SendHookRequest(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: XdsUpdateHealthService_SendHookRequest_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(XdsUpdateHealthServiceServer).SendHookRequest(ctx, req.(*HookRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// XdsUpdateHealthService_ServiceDesc is the grpc.ServiceDesc for XdsUpdateHealthService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var XdsUpdateHealthService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.XdsUpdateHealthService", + HandlerType: (*XdsUpdateHealthServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "SetServing", + Handler: _XdsUpdateHealthService_SetServing_Handler, + }, + { + MethodName: "SetNotServing", + Handler: _XdsUpdateHealthService_SetNotServing_Handler, + }, + { + MethodName: "SendHookRequest", + Handler: _XdsUpdateHealthService_SendHookRequest_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/test.proto", +} + +const ( + XdsUpdateClientConfigureService_Configure_FullMethodName = "/grpc.testing.XdsUpdateClientConfigureService/Configure" +) + +// XdsUpdateClientConfigureServiceClient is the client API for XdsUpdateClientConfigureService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// A service to dynamically update the configuration of an xDS test client. +type XdsUpdateClientConfigureServiceClient interface { + // Update the tes client's configuration. + Configure(ctx context.Context, in *ClientConfigureRequest, opts ...grpc.CallOption) (*ClientConfigureResponse, error) +} + +type xdsUpdateClientConfigureServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewXdsUpdateClientConfigureServiceClient(cc grpc.ClientConnInterface) XdsUpdateClientConfigureServiceClient { + return &xdsUpdateClientConfigureServiceClient{cc} +} + +func (c *xdsUpdateClientConfigureServiceClient) Configure(ctx context.Context, in *ClientConfigureRequest, opts ...grpc.CallOption) (*ClientConfigureResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ClientConfigureResponse) + err := c.cc.Invoke(ctx, XdsUpdateClientConfigureService_Configure_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// XdsUpdateClientConfigureServiceServer is the server API for XdsUpdateClientConfigureService service. +// All implementations must embed UnimplementedXdsUpdateClientConfigureServiceServer +// for forward compatibility. +// +// A service to dynamically update the configuration of an xDS test client. +type XdsUpdateClientConfigureServiceServer interface { + // Update the tes client's configuration. + Configure(context.Context, *ClientConfigureRequest) (*ClientConfigureResponse, error) + mustEmbedUnimplementedXdsUpdateClientConfigureServiceServer() +} + +// UnimplementedXdsUpdateClientConfigureServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedXdsUpdateClientConfigureServiceServer struct{} + +func (UnimplementedXdsUpdateClientConfigureServiceServer) Configure(context.Context, *ClientConfigureRequest) (*ClientConfigureResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Configure not implemented") +} +func (UnimplementedXdsUpdateClientConfigureServiceServer) mustEmbedUnimplementedXdsUpdateClientConfigureServiceServer() { +} +func (UnimplementedXdsUpdateClientConfigureServiceServer) testEmbeddedByValue() {} + +// UnsafeXdsUpdateClientConfigureServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to XdsUpdateClientConfigureServiceServer will +// result in compilation errors. +type UnsafeXdsUpdateClientConfigureServiceServer interface { + mustEmbedUnimplementedXdsUpdateClientConfigureServiceServer() +} + +func RegisterXdsUpdateClientConfigureServiceServer(s grpc.ServiceRegistrar, srv XdsUpdateClientConfigureServiceServer) { + // If the following call panics, it indicates UnimplementedXdsUpdateClientConfigureServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&XdsUpdateClientConfigureService_ServiceDesc, srv) +} + +func _XdsUpdateClientConfigureService_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ClientConfigureRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(XdsUpdateClientConfigureServiceServer).Configure(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: XdsUpdateClientConfigureService_Configure_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(XdsUpdateClientConfigureServiceServer).Configure(ctx, req.(*ClientConfigureRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// XdsUpdateClientConfigureService_ServiceDesc is the grpc.ServiceDesc for XdsUpdateClientConfigureService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var XdsUpdateClientConfigureService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.XdsUpdateClientConfigureService", + HandlerType: (*XdsUpdateClientConfigureServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Configure", + Handler: _XdsUpdateClientConfigureService_Configure_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/test.proto", +} diff --git a/interop/grpc_testing/worker_service.pb.go b/interop/grpc_testing/worker_service.pb.go new file mode 100644 index 0000000..809ecdb --- /dev/null +++ b/interop/grpc_testing/worker_service.pb.go @@ -0,0 +1,100 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/testing/worker_service.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var File_grpc_testing_worker_service_proto protoreflect.FileDescriptor + +const file_grpc_testing_worker_service_proto_rawDesc = "" + + "\n" + + "!grpc/testing/worker_service.proto\x12\fgrpc.testing\x1a\x1agrpc/testing/control.proto2\x97\x02\n" + + "\rWorkerService\x12E\n" + + "\tRunServer\x12\x18.grpc.testing.ServerArgs\x1a\x1a.grpc.testing.ServerStatus(\x010\x01\x12E\n" + + "\tRunClient\x12\x18.grpc.testing.ClientArgs\x1a\x1a.grpc.testing.ClientStatus(\x010\x01\x12B\n" + + "\tCoreCount\x12\x19.grpc.testing.CoreRequest\x1a\x1a.grpc.testing.CoreResponse\x124\n" + + "\n" + + "QuitWorker\x12\x12.grpc.testing.Void\x1a\x12.grpc.testing.VoidB'\n" + + "\x0fio.grpc.testingB\x12WorkerServiceProtoP\x01b\x06proto3" + +var file_grpc_testing_worker_service_proto_goTypes = []any{ + (*ServerArgs)(nil), // 0: grpc.testing.ServerArgs + (*ClientArgs)(nil), // 1: grpc.testing.ClientArgs + (*CoreRequest)(nil), // 2: grpc.testing.CoreRequest + (*Void)(nil), // 3: grpc.testing.Void + (*ServerStatus)(nil), // 4: grpc.testing.ServerStatus + (*ClientStatus)(nil), // 5: grpc.testing.ClientStatus + (*CoreResponse)(nil), // 6: grpc.testing.CoreResponse +} +var file_grpc_testing_worker_service_proto_depIdxs = []int32{ + 0, // 0: grpc.testing.WorkerService.RunServer:input_type -> grpc.testing.ServerArgs + 1, // 1: grpc.testing.WorkerService.RunClient:input_type -> grpc.testing.ClientArgs + 2, // 2: grpc.testing.WorkerService.CoreCount:input_type -> grpc.testing.CoreRequest + 3, // 3: grpc.testing.WorkerService.QuitWorker:input_type -> grpc.testing.Void + 4, // 4: grpc.testing.WorkerService.RunServer:output_type -> grpc.testing.ServerStatus + 5, // 5: grpc.testing.WorkerService.RunClient:output_type -> grpc.testing.ClientStatus + 6, // 6: grpc.testing.WorkerService.CoreCount:output_type -> grpc.testing.CoreResponse + 3, // 7: grpc.testing.WorkerService.QuitWorker:output_type -> grpc.testing.Void + 4, // [4:8] is the sub-list for method output_type + 0, // [0:4] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_grpc_testing_worker_service_proto_init() } +func file_grpc_testing_worker_service_proto_init() { + if File_grpc_testing_worker_service_proto != nil { + return + } + file_grpc_testing_control_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_testing_worker_service_proto_rawDesc), len(file_grpc_testing_worker_service_proto_rawDesc)), + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_testing_worker_service_proto_goTypes, + DependencyIndexes: file_grpc_testing_worker_service_proto_depIdxs, + }.Build() + File_grpc_testing_worker_service_proto = out.File + file_grpc_testing_worker_service_proto_goTypes = nil + file_grpc_testing_worker_service_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/worker_service_grpc.pb.go b/interop/grpc_testing/worker_service_grpc.pb.go new file mode 100644 index 0000000..a8908cc --- /dev/null +++ b/interop/grpc_testing/worker_service_grpc.pb.go @@ -0,0 +1,269 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: grpc/testing/worker_service.proto + +package grpc_testing + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + WorkerService_RunServer_FullMethodName = "/grpc.testing.WorkerService/RunServer" + WorkerService_RunClient_FullMethodName = "/grpc.testing.WorkerService/RunClient" + WorkerService_CoreCount_FullMethodName = "/grpc.testing.WorkerService/CoreCount" + WorkerService_QuitWorker_FullMethodName = "/grpc.testing.WorkerService/QuitWorker" +) + +// WorkerServiceClient is the client API for WorkerService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type WorkerServiceClient interface { + // Start server with specified workload. + // First request sent specifies the ServerConfig followed by ServerStatus + // response. After that, a "Mark" can be sent anytime to request the latest + // stats. Closing the stream will initiate shutdown of the test server + // and once the shutdown has finished, the OK status is sent to terminate + // this RPC. + RunServer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ServerArgs, ServerStatus], error) + // Start client with specified workload. + // First request sent specifies the ClientConfig followed by ClientStatus + // response. After that, a "Mark" can be sent anytime to request the latest + // stats. Closing the stream will initiate shutdown of the test client + // and once the shutdown has finished, the OK status is sent to terminate + // this RPC. + RunClient(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ClientArgs, ClientStatus], error) + // Just return the core count - unary call + CoreCount(ctx context.Context, in *CoreRequest, opts ...grpc.CallOption) (*CoreResponse, error) + // Quit this worker + QuitWorker(ctx context.Context, in *Void, opts ...grpc.CallOption) (*Void, error) +} + +type workerServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewWorkerServiceClient(cc grpc.ClientConnInterface) WorkerServiceClient { + return &workerServiceClient{cc} +} + +func (c *workerServiceClient) RunServer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ServerArgs, ServerStatus], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &WorkerService_ServiceDesc.Streams[0], WorkerService_RunServer_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[ServerArgs, ServerStatus]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type WorkerService_RunServerClient = grpc.BidiStreamingClient[ServerArgs, ServerStatus] + +func (c *workerServiceClient) RunClient(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ClientArgs, ClientStatus], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &WorkerService_ServiceDesc.Streams[1], WorkerService_RunClient_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[ClientArgs, ClientStatus]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type WorkerService_RunClientClient = grpc.BidiStreamingClient[ClientArgs, ClientStatus] + +func (c *workerServiceClient) CoreCount(ctx context.Context, in *CoreRequest, opts ...grpc.CallOption) (*CoreResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CoreResponse) + err := c.cc.Invoke(ctx, WorkerService_CoreCount_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *workerServiceClient) QuitWorker(ctx context.Context, in *Void, opts ...grpc.CallOption) (*Void, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Void) + err := c.cc.Invoke(ctx, WorkerService_QuitWorker_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// WorkerServiceServer is the server API for WorkerService service. +// All implementations must embed UnimplementedWorkerServiceServer +// for forward compatibility. +type WorkerServiceServer interface { + // Start server with specified workload. + // First request sent specifies the ServerConfig followed by ServerStatus + // response. After that, a "Mark" can be sent anytime to request the latest + // stats. Closing the stream will initiate shutdown of the test server + // and once the shutdown has finished, the OK status is sent to terminate + // this RPC. + RunServer(grpc.BidiStreamingServer[ServerArgs, ServerStatus]) error + // Start client with specified workload. + // First request sent specifies the ClientConfig followed by ClientStatus + // response. After that, a "Mark" can be sent anytime to request the latest + // stats. Closing the stream will initiate shutdown of the test client + // and once the shutdown has finished, the OK status is sent to terminate + // this RPC. + RunClient(grpc.BidiStreamingServer[ClientArgs, ClientStatus]) error + // Just return the core count - unary call + CoreCount(context.Context, *CoreRequest) (*CoreResponse, error) + // Quit this worker + QuitWorker(context.Context, *Void) (*Void, error) + mustEmbedUnimplementedWorkerServiceServer() +} + +// UnimplementedWorkerServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedWorkerServiceServer struct{} + +func (UnimplementedWorkerServiceServer) RunServer(grpc.BidiStreamingServer[ServerArgs, ServerStatus]) error { + return status.Error(codes.Unimplemented, "method RunServer not implemented") +} +func (UnimplementedWorkerServiceServer) RunClient(grpc.BidiStreamingServer[ClientArgs, ClientStatus]) error { + return status.Error(codes.Unimplemented, "method RunClient not implemented") +} +func (UnimplementedWorkerServiceServer) CoreCount(context.Context, *CoreRequest) (*CoreResponse, error) { + return nil, status.Error(codes.Unimplemented, "method CoreCount not implemented") +} +func (UnimplementedWorkerServiceServer) QuitWorker(context.Context, *Void) (*Void, error) { + return nil, status.Error(codes.Unimplemented, "method QuitWorker not implemented") +} +func (UnimplementedWorkerServiceServer) mustEmbedUnimplementedWorkerServiceServer() {} +func (UnimplementedWorkerServiceServer) testEmbeddedByValue() {} + +// UnsafeWorkerServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to WorkerServiceServer will +// result in compilation errors. +type UnsafeWorkerServiceServer interface { + mustEmbedUnimplementedWorkerServiceServer() +} + +func RegisterWorkerServiceServer(s grpc.ServiceRegistrar, srv WorkerServiceServer) { + // If the following call panics, it indicates UnimplementedWorkerServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&WorkerService_ServiceDesc, srv) +} + +func _WorkerService_RunServer_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(WorkerServiceServer).RunServer(&grpc.GenericServerStream[ServerArgs, ServerStatus]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type WorkerService_RunServerServer = grpc.BidiStreamingServer[ServerArgs, ServerStatus] + +func _WorkerService_RunClient_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(WorkerServiceServer).RunClient(&grpc.GenericServerStream[ClientArgs, ClientStatus]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type WorkerService_RunClientServer = grpc.BidiStreamingServer[ClientArgs, ClientStatus] + +func _WorkerService_CoreCount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CoreRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerServiceServer).CoreCount(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: WorkerService_CoreCount_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerServiceServer).CoreCount(ctx, req.(*CoreRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _WorkerService_QuitWorker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Void) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerServiceServer).QuitWorker(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: WorkerService_QuitWorker_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerServiceServer).QuitWorker(ctx, req.(*Void)) + } + return interceptor(ctx, in, info, handler) +} + +// WorkerService_ServiceDesc is the grpc.ServiceDesc for WorkerService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var WorkerService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.WorkerService", + HandlerType: (*WorkerServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "CoreCount", + Handler: _WorkerService_CoreCount_Handler, + }, + { + MethodName: "QuitWorker", + Handler: _WorkerService_QuitWorker_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "RunServer", + Handler: _WorkerService_RunServer_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "RunClient", + Handler: _WorkerService_RunClient_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/testing/worker_service.proto", +} diff --git a/interop/grpclb_fallback/client_linux.go b/interop/grpclb_fallback/client_linux.go new file mode 100644 index 0000000..47c16ad --- /dev/null +++ b/interop/grpclb_fallback/client_linux.go @@ -0,0 +1,192 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary grpclb_fallback is an interop test client for grpclb fallback. +package main + +import ( + "context" + "flag" + "log" + "net" + "os" + "os/exec" + "syscall" + "time" + + "golang.org/x/sys/unix" + "google.golang.org/grpc" + _ "google.golang.org/grpc/balancer/grpclb" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/alts" + "google.golang.org/grpc/credentials/google" + _ "google.golang.org/grpc/xds/googledirectpath" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + customCredentialsType = flag.String("custom_credentials_type", "", "Client creds to use") + serverURI = flag.String("server_uri", "dns:///staging-grpc-directpath-fallback-test.googleapis.com:443", "The server host name") + induceFallbackCmd = flag.String("induce_fallback_cmd", "", "Command to induce fallback e.g. by making certain addresses unroutable") + fallbackDeadlineSeconds = flag.Int("fallback_deadline_seconds", 1, "How long to wait for fallback to happen after induce_fallback_cmd") + testCase = flag.String("test_case", "", + `Configure different test cases. Valid options are: + fallback_before_startup : LB/backend connections fail before RPC's have been made; + fallback_after_startup : LB/backend connections fail after RPC's have been made;`) + infoLog = log.New(os.Stderr, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile) + errorLog = log.New(os.Stderr, "ERROR: ", log.Ldate|log.Ltime|log.Lshortfile) +) + +func doRPCAndGetPath(client testgrpc.TestServiceClient, timeout time.Duration) testpb.GrpclbRouteType { + infoLog.Printf("doRPCAndGetPath timeout:%v\n", timeout) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + req := &testpb.SimpleRequest{ + FillGrpclbRouteType: true, + } + reply, err := client.UnaryCall(ctx, req) + if err != nil { + infoLog.Printf("doRPCAndGetPath error:%v\n", err) + return testpb.GrpclbRouteType_GRPCLB_ROUTE_TYPE_UNKNOWN + } + g := reply.GetGrpclbRouteType() + infoLog.Printf("doRPCAndGetPath got grpclb route type: %v\n", g) + if g != testpb.GrpclbRouteType_GRPCLB_ROUTE_TYPE_FALLBACK && g != testpb.GrpclbRouteType_GRPCLB_ROUTE_TYPE_BACKEND { + errorLog.Fatalf("Expected grpclb route type to be either backend or fallback; got: %d", g) + } + return g +} + +func dialTCPUserTimeout(ctx context.Context, addr string) (net.Conn, error) { + control := func(_, _ string, c syscall.RawConn) error { + var syscallErr error + controlErr := c.Control(func(fd uintptr) { + syscallErr = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, 20000) + }) + if syscallErr != nil { + errorLog.Fatalf("syscall error setting sockopt TCP_USER_TIMEOUT: %v", syscallErr) + } + if controlErr != nil { + errorLog.Fatalf("control error setting sockopt TCP_USER_TIMEOUT: %v", syscallErr) + } + return nil + } + d := &net.Dialer{ + Control: control, + } + return d.DialContext(ctx, "tcp", addr) +} + +func createTestConn() *grpc.ClientConn { + opts := []grpc.DialOption{ + grpc.WithContextDialer(dialTCPUserTimeout), + } + switch *customCredentialsType { + case "tls": + creds := credentials.NewClientTLSFromCert(nil, "") + opts = append(opts, grpc.WithTransportCredentials(creds)) + case "alts": + creds := alts.NewClientCreds(alts.DefaultClientOptions()) + opts = append(opts, grpc.WithTransportCredentials(creds)) + case "google_default_credentials": + opts = append(opts, grpc.WithCredentialsBundle(google.NewDefaultCredentials())) + case "compute_engine_channel_creds": + opts = append(opts, grpc.WithCredentialsBundle(google.NewComputeEngineCredentials())) + default: + errorLog.Fatalf("Invalid --custom_credentials_type:%v", *customCredentialsType) + } + conn, err := grpc.NewClient(*serverURI, opts...) + if err != nil { + errorLog.Fatalf("grpc.NewClient(%q) = %v", *serverURI, err) + } + return conn +} + +func runCmd(command string) { + infoLog.Printf("Running cmd:|%v|\n", command) + if err := exec.Command("bash", "-c", command).Run(); err != nil { + errorLog.Fatalf("error running cmd:|%v| : %v", command, err) + } +} + +func waitForFallbackAndDoRPCs(client testgrpc.TestServiceClient, fallbackDeadline time.Time) { + fallbackRetryCount := 0 + fellBack := false + for time.Now().Before(fallbackDeadline) { + g := doRPCAndGetPath(client, 20*time.Second) + if g == testpb.GrpclbRouteType_GRPCLB_ROUTE_TYPE_FALLBACK { + infoLog.Println("Made one successful RPC to a fallback. Now expect the same for the rest.") + fellBack = true + break + } else if g == testpb.GrpclbRouteType_GRPCLB_ROUTE_TYPE_BACKEND { + errorLog.Fatalf("Got RPC type backend. This suggests an error in test implementation") + } else { + infoLog.Println("Retryable RPC failure on iteration:", fallbackRetryCount) + } + fallbackRetryCount++ + } + if !fellBack { + infoLog.Fatalf("Didn't fall back before deadline: %v\n", fallbackDeadline) + } + for i := 0; i < 30; i++ { + if g := doRPCAndGetPath(client, 20*time.Second); g != testpb.GrpclbRouteType_GRPCLB_ROUTE_TYPE_FALLBACK { + errorLog.Fatalf("Expected RPC to take grpclb route type FALLBACK. Got: %v", g) + } + time.Sleep(time.Second) + } +} + +func doFallbackBeforeStartup() { + runCmd(*induceFallbackCmd) + fallbackDeadline := time.Now().Add(time.Duration(*fallbackDeadlineSeconds) * time.Second) + conn := createTestConn() + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + waitForFallbackAndDoRPCs(client, fallbackDeadline) +} + +func doFallbackAfterStartup() { + conn := createTestConn() + defer conn.Close() + client := testgrpc.NewTestServiceClient(conn) + if g := doRPCAndGetPath(client, 20*time.Second); g != testpb.GrpclbRouteType_GRPCLB_ROUTE_TYPE_BACKEND { + errorLog.Fatalf("Expected RPC to take grpclb route type BACKEND. Got: %v", g) + } + runCmd(*induceFallbackCmd) + fallbackDeadline := time.Now().Add(time.Duration(*fallbackDeadlineSeconds) * time.Second) + waitForFallbackAndDoRPCs(client, fallbackDeadline) +} + +func main() { + flag.Parse() + if len(*induceFallbackCmd) == 0 { + errorLog.Fatalf("--induce_fallback_cmd unset") + } + switch *testCase { + case "fallback_before_startup": + doFallbackBeforeStartup() + log.Printf("FallbackBeforeStartup done!\n") + case "fallback_after_startup": + doFallbackAfterStartup() + log.Printf("FallbackAfterStartup done!\n") + default: + errorLog.Fatalf("Unsupported test case: %v", *testCase) + } +} diff --git a/interop/http2/negative_http2_client.go b/interop/http2/negative_http2_client.go new file mode 100644 index 0000000..35c7b54 --- /dev/null +++ b/interop/http2/negative_http2_client.go @@ -0,0 +1,165 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary http2 is used to test http2 error edge cases like GOAWAYs and +// RST_STREAMs +// +// Documentation: +// https://github.com/grpc/grpc/blob/master/doc/negative-http2-interop-test-descriptions.md +package main + +import ( + "context" + "flag" + "net" + "strconv" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/interop" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + serverHost = flag.String("server_host", "localhost", "The server host name") + serverPort = flag.Int("server_port", 8080, "The server port number") + testCase = flag.String("test_case", "goaway", + `Configure different test cases. Valid options are: + goaway : client sends two requests, the server will send a goaway in between; + rst_after_header : server will send rst_stream after it sends headers; + rst_during_data : server will send rst_stream while sending data; + rst_after_data : server will send rst_stream after sending data; + ping : server will send pings between each http2 frame; + max_streams : server will ensure that the max_concurrent_streams limit is upheld;`) + largeReqSize = 271828 + largeRespSize = 314159 + + logger = grpclog.Component("interop") +) + +func largeSimpleRequest() *testpb.SimpleRequest { + pl := interop.ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) + return &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(largeRespSize), + Payload: pl, + } +} + +// sends two unary calls. The server asserts that the calls use different connections. +func goaway(ctx context.Context, tc testgrpc.TestServiceClient) { + interop.DoLargeUnaryCall(ctx, tc) + // sleep to ensure that the client has time to recv the GOAWAY. + // TODO(ncteisen): make this less hacky. + time.Sleep(1 * time.Second) + interop.DoLargeUnaryCall(ctx, tc) +} + +func rstAfterHeader(tc testgrpc.TestServiceClient) { + req := largeSimpleRequest() + reply, err := tc.UnaryCall(context.Background(), req) + if reply != nil { + logger.Fatal("Client received reply despite server sending rst stream after header") + } + if status.Code(err) != codes.Internal { + logger.Fatalf("%v.UnaryCall() = _, %v, want _, %v", tc, status.Code(err), codes.Internal) + } +} + +func rstDuringData(tc testgrpc.TestServiceClient) { + req := largeSimpleRequest() + reply, err := tc.UnaryCall(context.Background(), req) + if reply != nil { + logger.Fatal("Client received reply despite server sending rst stream during data") + } + if status.Code(err) != codes.Unknown { + logger.Fatalf("%v.UnaryCall() = _, %v, want _, %v", tc, status.Code(err), codes.Unknown) + } +} + +func rstAfterData(tc testgrpc.TestServiceClient) { + req := largeSimpleRequest() + reply, err := tc.UnaryCall(context.Background(), req) + if reply != nil { + logger.Fatal("Client received reply despite server sending rst stream after data") + } + if status.Code(err) != codes.Internal { + logger.Fatalf("%v.UnaryCall() = _, %v, want _, %v", tc, status.Code(err), codes.Internal) + } +} + +func ping(ctx context.Context, tc testgrpc.TestServiceClient) { + // The server will assert that every ping it sends was ACK-ed by the client. + interop.DoLargeUnaryCall(ctx, tc) +} + +func maxStreams(ctx context.Context, tc testgrpc.TestServiceClient) { + interop.DoLargeUnaryCall(ctx, tc) + var wg sync.WaitGroup + for i := 0; i < 15; i++ { + wg.Add(1) + go func() { + defer wg.Done() + interop.DoLargeUnaryCall(ctx, tc) + }() + } + wg.Wait() +} + +func main() { + flag.Parse() + serverAddr := net.JoinHostPort(*serverHost, strconv.Itoa(*serverPort)) + var opts []grpc.DialOption + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + conn, err := grpc.NewClient(serverAddr, opts...) + if err != nil { + logger.Fatalf("grpc.NewClient(%q) = %v", serverAddr, err) + } + defer conn.Close() + tc := testgrpc.NewTestServiceClient(conn) + ctx := context.Background() + switch *testCase { + case "goaway": + goaway(ctx, tc) + logger.Infoln("goaway done") + case "rst_after_header": + rstAfterHeader(tc) + logger.Infoln("rst_after_header done") + case "rst_during_data": + rstDuringData(tc) + logger.Infoln("rst_during_data done") + case "rst_after_data": + rstAfterData(tc) + logger.Infoln("rst_after_data done") + case "ping": + ping(ctx, tc) + logger.Infoln("ping done") + case "max_streams": + maxStreams(ctx, tc) + logger.Infoln("max_streams done") + default: + logger.Fatal("Unsupported test case: ", *testCase) + } +} diff --git a/interop/interop_test.sh b/interop/interop_test.sh new file mode 100755 index 0000000..5240c26 --- /dev/null +++ b/interop/interop_test.sh @@ -0,0 +1,140 @@ +#!/bin/bash +# +# Copyright 2019 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e +x + +export TMPDIR=$(mktemp -d) +trap "rm -rf ${TMPDIR}" EXIT + +clean () { + for i in {1..10}; do + jobs -p | xargs -n1 pkill -P + # A simple "wait" just hangs sometimes. Running `jobs` seems to help. + sleep 1 + if jobs | read; then + return + fi + done + echo "$(tput setaf 1) clean failed to kill tests $(tput sgr 0)" + jobs + pstree + exit 1 +} + +fail () { + echo "$(tput setaf 1) $(date): $1 $(tput sgr 0)" + clean + exit 1 +} + +pass () { + echo "$(tput setaf 2) $(date): $1 $(tput sgr 0)" +} + +withTimeout () { + timer=$1 + shift + + # Run command in the background. + cmd=$(printf '%q ' "$@") + eval "$cmd" & + wpid=$! + # Kill after $timer seconds. + sleep $timer && kill $wpid & + kpid=$! + # Wait for the background thread. + wait $wpid + res=$? + # Kill the killer pid in case it's still running. + kill $kpid || true + wait $kpid || true + return $res +} + +# Don't run some tests that need a special environment: +# "google_default_credentials" +# "compute_engine_channel_credentials" +# "compute_engine_creds" +# "service_account_creds" +# "jwt_token_creds" +# "oauth2_auth_token" +# "per_rpc_creds" +# "pick_first_unary" + +CASES=( + "empty_unary" + "large_unary" + "client_streaming" + "server_streaming" + "ping_pong" + "empty_stream" + "timeout_on_sleeping_server" + "cancel_after_begin" + "cancel_after_first_response" + "status_code_and_message" + "special_status_message" + "custom_metadata" + "unimplemented_method" + "unimplemented_service" + "orca_per_rpc" + "orca_oob" + "rpc_soak" + "channel_soak" +) + +# Build server +echo "$(tput setaf 4) $(date): building server $(tput sgr 0)" +if ! go build -o /dev/null ./interop/server; then + fail "failed to build server" +else + pass "successfully built server" +fi + +# Build client +echo "$(tput setaf 4) $(date): building client $(tput sgr 0)" +if ! go build -o /dev/null ./interop/client; then + fail "failed to build client" +else + pass "successfully built client" +fi + +# Start server +SERVER_LOG="$(mktemp)" +GRPC_GO_LOG_SEVERITY_LEVEL=info go run ./interop/server --use_tls &> $SERVER_LOG & + +for case in ${CASES[@]}; do + echo "$(tput setaf 4) $(date): testing: ${case} $(tput sgr 0)" + + CLIENT_LOG="$(mktemp)" + if ! GRPC_GO_LOG_SEVERITY_LEVEL=info withTimeout 20 go run ./interop/client \ + --use_tls \ + --server_host_override=foo.test.google.fr \ + --use_test_ca --test_case="${case}" \ + --service_config_json='{ "loadBalancingConfig": [{ "test_backend_metrics_load_balancer": {} }]}' \ + &> $CLIENT_LOG; then + fail "FAIL: test case ${case} + got server log: + $(cat $SERVER_LOG) + got client log: + $(cat $CLIENT_LOG) + " + else + pass "PASS: test case ${case}" + fi +done + +clean diff --git a/interop/observability/Dockerfile b/interop/observability/Dockerfile new file mode 100644 index 0000000..104a8e5 --- /dev/null +++ b/interop/observability/Dockerfile @@ -0,0 +1,53 @@ +# Copyright 2022 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# +# Stage 1: Build the interop test client and server +# + +FROM golang:1.25-bullseye as build + +WORKDIR /grpc-go +COPY . . + +WORKDIR /grpc-go/interop/observability +RUN go build -o server/ server/server.go && \ + go build -o client/ client/client.go + + + +# +# Stage 2: +# +# - Copy only the necessary files to reduce Docker image size. +# - Have an ENTRYPOINT script which will launch the interop test client or server +# with the given parameters. +# + +FROM golang:1.24-bullseye + +ENV GRPC_GO_LOG_SEVERITY_LEVEL info +ENV GRPC_GO_LOG_VERBOSITY_LEVEL 2 + +WORKDIR /grpc-go/interop/observability/server +COPY --from=build /grpc-go/interop/observability/server/server . + +WORKDIR /grpc-go/interop/observability/client +COPY --from=build /grpc-go/interop/observability/client/client . + +WORKDIR /grpc-go/interop/observability +COPY --from=build /grpc-go/interop/observability/run.sh . + +ENTRYPOINT ["/grpc-go/interop/observability/run.sh"] diff --git a/interop/observability/build_docker.sh b/interop/observability/build_docker.sh new file mode 100755 index 0000000..ed7a181 --- /dev/null +++ b/interop/observability/build_docker.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright 2022 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex +cd "$(dirname "$0")"/../.. + +# Environment Variables: +# +# TAG_NAME: the docker image tag name +# + +echo Building ${TAG_NAME} + +docker build --no-cache -t ${TAG_NAME} -f ./interop/observability/Dockerfile . diff --git a/interop/observability/client/client.go b/interop/observability/client/client.go new file mode 100644 index 0000000..0850e70 --- /dev/null +++ b/interop/observability/client/client.go @@ -0,0 +1,84 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an interop client for Observability. +// +// See interop test case descriptions [here]. +// +// [here]: https://github.com/grpc/grpc/blob/master/doc/interop-test-descriptions.md +package main + +import ( + "context" + "flag" + "log" + "net" + "strconv" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/gcp/observability" + "google.golang.org/grpc/interop" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + serverHost = flag.String("server_host", "localhost", "The server host name") + serverPort = flag.Int("server_port", 10000, "The server port number") + testCase = flag.String("test_case", "large_unary", "The action to perform") + numTimes = flag.Int("num_times", 1, "Number of times to run the test case") +) + +func main() { + err := observability.Start(context.Background()) + if err != nil { + log.Fatalf("observability start failed: %v", err) + } + defer observability.End() + flag.Parse() + serverAddr := *serverHost + if *serverPort != 0 { + serverAddr = net.JoinHostPort(*serverHost, strconv.Itoa(*serverPort)) + } + conn, err := grpc.NewClient(serverAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("grpc.NewClient(%q) = %v", serverAddr, err) + } + defer conn.Close() + tc := testgrpc.NewTestServiceClient(conn) + ctx := context.Background() + for i := 0; i < *numTimes; i++ { + if *testCase == "ping_pong" { + interop.DoPingPong(ctx, tc) + } else if *testCase == "large_unary" { + interop.DoLargeUnaryCall(ctx, tc) + } else if *testCase == "custom_metadata" { + interop.DoCustomMetadata(ctx, tc) + } else { + log.Fatalf("Invalid test case: %s", *testCase) + } + } + // TODO(stanleycheung): remove this once the observability exporter plugin is able to + // gracefully flush observability data to cloud at shutdown + // TODO(stanleycheung): see if we can reduce the number 65 + const exporterSleepDuration = 65 * time.Second + log.Printf("Sleeping %v before closing...", exporterSleepDuration) + time.Sleep(exporterSleepDuration) +} diff --git a/interop/observability/go.mod b/interop/observability/go.mod new file mode 100644 index 0000000..e50701b --- /dev/null +++ b/interop/observability/go.mod @@ -0,0 +1,73 @@ +module google.golang.org/grpc/interop/observability + +go 1.25.0 + +require ( + google.golang.org/grpc v1.83.0 + google.golang.org/grpc/gcp/observability v1.0.1 +) + +require ( + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.23.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/logging v1.19.1 // indirect + cloud.google.com/go/longrunning v1.2.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect + cloud.google.com/go/trace v1.16.0 // indirect + contrib.go.opencensus.io/exporter/stackdriver v0.13.15-0.20230702191903-2de6d2748484 // indirect + github.com/aws/aws-sdk-go-v2 v1.43.6 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.37 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.36 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.45.6 // indirect + github.com/aws/smithy-go v1.27.8 // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.21 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/api v0.293.0 // indirect + google.golang.org/genproto v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/grpc/stats/opencensus v1.0.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) + +replace google.golang.org/grpc => ../.. + +replace google.golang.org/grpc/gcp/observability => ../../gcp/observability + +replace google.golang.org/grpc/stats/opencensus => ../../stats/opencensus diff --git a/interop/observability/run.sh b/interop/observability/run.sh new file mode 100755 index 0000000..b049466 --- /dev/null +++ b/interop/observability/run.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright 2022 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex +cd "$(dirname "$0")"/../.. + +if [ "$1" = "server" ] ; then + /grpc-go/interop/observability/server/server "${@:2}" + +elif [ "$1" = "client" ] ; then + /grpc-go/interop/observability/client/client "${@:2}" + +else + echo "Invalid action: $1. Usage:" + echo " $ .../run.sh [server|client] --server_host= --server_port= ..." + exit 1 +fi diff --git a/interop/observability/server/server.go b/interop/observability/server/server.go new file mode 100644 index 0000000..0620994 --- /dev/null +++ b/interop/observability/server/server.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is an interop server for Observability. +// +// See interop test case descriptions [here]. +// +// [here]: https://github.com/grpc/grpc/blob/master/doc/interop-test-descriptions.md +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/gcp/observability" + "google.golang.org/grpc/interop" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + port = flag.Int("port", 10000, "The server port") +) + +func main() { + err := observability.Start(context.Background()) + if err != nil { + log.Fatalf("observability start failed: %v", err) + } + defer observability.End() + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + server := grpc.NewServer() + defer server.Stop() + testgrpc.RegisterTestServiceServer(server, interop.NewTestServer()) + log.Printf("Observability interop server listening on %v", lis.Addr()) + server.Serve(lis) +} diff --git a/interop/orcalb.go b/interop/orcalb.go new file mode 100644 index 0000000..abf0a30 --- /dev/null +++ b/interop/orcalb.go @@ -0,0 +1,256 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package interop + +import ( + "context" + "fmt" + "sync" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/endpointsharding" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/resolver" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" +) + +var orcaLogger = grpclog.Component("orca") + +func init() { + balancer.Register(orcabb{}) +} + +type orcabb struct{} + +// Build creates a new orcab balancer. +func (orcabb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &orcab{ + ClientConn: cc, + stopOOBListeners: make(map[balancer.SubConn]func()), + oobState: &oobState{ + reports: make(map[balancer.SubConn]*v3orcapb.OrcaLoadReport), + }, + } + b.logger = internalgrpclog.NewPrefixLogger(orcaLogger, fmt.Sprintf("[%p] ", b)) + b.child = endpointsharding.NewBalancer(b, bOpts, balancer.Get(pickfirst.Name).Build, endpointsharding.Options{}) + b.logger.Infof("Created") + return b +} + +func (orcabb) Name() string { + return "test_backend_metrics_load_balancer" +} + +// orcab is the balancer for the test_backend_metrics_load_balancer policy. +// It delegates SubConn management to endpointsharding + pick_first and +// intercepts NewSubConn calls to register OOB listeners on READY SubConns. +type orcab struct { + // The following fields are initialized at build time and read-only after + // that and therefore do not need to be guarded by a mutex. + balancer.ClientConn // Embeds to intercept NewSubConn and UpdateState calls + child balancer.Balancer + oobState *oobState + logger *internalgrpclog.PrefixLogger + + // mu guards the fields below. + mu sync.Mutex + stopOOBListeners map[balancer.SubConn]func() +} + +func (b *orcab) UpdateClientConnState(s balancer.ClientConnState) error { + // Delegate to the child endpoint sharding balancer, which distributes + // state updates to its pick_first children. + return b.child.UpdateClientConnState(s) +} + +func (b *orcab) ResolverError(err error) { + // Will cause an inline picker update from endpoint sharding. + b.child.ResolverError(err) +} + +func (b *orcab) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + orcaLogger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *orcab) ExitIdle() { + // Propagate to the child endpoint sharding balancer. + b.child.ExitIdle() +} + +func (b *orcab) Close() { + b.mu.Lock() + listeners := b.stopOOBListeners + b.stopOOBListeners = nil + b.mu.Unlock() + for _, stop := range listeners { + stop() + } + b.child.Close() +} + +// NewSubConn intercepts SubConn creation from pick_first children to wrap the +// StateListener for OOB listener management on READY transitions. +func (b *orcab) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + // The variable sc is declared before the closure so the closure captures + // the variable, not its (zero) value. After ClientConn.NewSubConn assigns + // to sc, the closure sees the real SubConn when invoked. + var sc balancer.SubConn + oldListener := opts.StateListener + opts.StateListener = func(state balancer.SubConnState) { + b.updateSubConnState(sc, state) + if oldListener != nil { + oldListener(state) + } + } + sc, err := b.ClientConn.NewSubConn(addrs, opts) + if err != nil { + return nil, err + } + return sc, nil +} + +func (b *orcab) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.mu.Lock() + if b.stopOOBListeners == nil { + b.mu.Unlock() + return + } + + if state.ConnectivityState == connectivity.Ready { + oldStop := b.stopOOBListeners[sc] + if oldStop != nil { + oldStop() + } + stop := orca.RegisterOOBListener(sc, &orcaOOBListener{subConn: sc, balancer: b}, orca.OOBListenerOptions{ReportInterval: time.Second}) + b.stopOOBListeners[sc] = stop + b.mu.Unlock() + return + } + + stop := b.stopOOBListeners[sc] + if stop != nil { + delete(b.stopOOBListeners, sc) + defer func() { stop() }() + } + b.mu.Unlock() + + if state.ConnectivityState == connectivity.Shutdown { + b.oobState.mu.Lock() + delete(b.oobState.reports, sc) + b.oobState.mu.Unlock() + } +} + +// UpdateState intercepts state updates from endpointsharding to wrap the +// picker with ORCA load report handling. +func (b *orcab) UpdateState(state balancer.State) { + // If READY, wrap the picker to inject the ORCA Done callback; otherwise, + // pass through the child picker as-is. + if state.ConnectivityState == connectivity.Ready { + state = balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &orcaPicker{childPicker: state.Picker, oobState: b.oobState}, + } + } + b.ClientConn.UpdateState(state) +} + +type orcaOOBListener struct { + subConn balancer.SubConn + balancer *orcab +} + +// OnLoadReport implements orca.OOBListener. +func (l *orcaOOBListener) OnLoadReport(r *v3orcapb.OrcaLoadReport) { + l.balancer.oobState.mu.Lock() + defer l.balancer.oobState.mu.Unlock() + l.balancer.oobState.reports[l.subConn] = r +} + +type oobState struct { + mu sync.Mutex + reports map[balancer.SubConn]*v3orcapb.OrcaLoadReport +} +type orcaPicker struct { + childPicker balancer.Picker + oobState *oobState +} + +func (p *orcaPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + res, err := p.childPicker.Pick(info) + if err != nil { + return res, err + } + + p.oobState.mu.Lock() + lr := p.oobState.reports[res.SubConn] + p.oobState.mu.Unlock() + + origDone := res.Done + res.Done = func(di balancer.DoneInfo) { + perRPCLR, _ := di.ServerLoad.(*v3orcapb.OrcaLoadReport) + if perRPCLR != nil && (perRPCLR.CpuUtilization != 0 || perRPCLR.MemUtilization != 0 || len(perRPCLR.Utilization) > 0 || len(perRPCLR.RequestCost) > 0) { + setContextCMR(info.Ctx, perRPCLR) + } else if lr != nil { + setContextCMR(info.Ctx, lr) + } + if origDone != nil { + origDone(di) + } + } + return res, nil +} + +func setContextCMR(ctx context.Context, lr *v3orcapb.OrcaLoadReport) { + if r := orcaResultFromContext(ctx); r != nil { + *r = lr + } +} + +type orcaKey string + +var orcaCtxKey = orcaKey("orcaResult") + +// contextWithORCAResult sets a key in ctx with a pointer to an ORCA load +// report that is to be filled in by the "test_backend_metrics_load_balancer" +// LB policy's Picker's Done callback. +// +// If a per-call load report is provided from the server for the call, result +// will be filled with that, otherwise the most recent OOB load report is used. +// If no OOB report has been received, result is not modified. +func contextWithORCAResult(ctx context.Context, result **v3orcapb.OrcaLoadReport) context.Context { + return context.WithValue(ctx, orcaCtxKey, result) +} + +// orcaResultFromContext returns the ORCA load report stored in the context. +// The LB policy uses this to communicate the load report back to the interop +// client application. +func orcaResultFromContext(ctx context.Context) **v3orcapb.OrcaLoadReport { + v := ctx.Value(orcaCtxKey) + if v == nil { + return nil + } + return v.(**v3orcapb.OrcaLoadReport) +} diff --git a/interop/orcalb_test.go b/interop/orcalb_test.go new file mode 100644 index 0000000..945e900 --- /dev/null +++ b/interop/orcalb_test.go @@ -0,0 +1,363 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package interop + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + "google.golang.org/protobuf/testing/protocmp" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const defaultTestTimeout = 30 * time.Second + +type orcaServer struct { + *stubserver.StubServer + metricsRecorder orca.ServerMetricsRecorder +} + +func startORCAServer(t *testing.T) *orcaServer { + t.Helper() + + smr := orca.NewServerMetricsRecorder() + ts := NewTestServer(NewTestServerOptions{MetricsRecorder: smr}) + + stub := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return ts.UnaryCall(ctx, in) + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + return ts.FullDuplexCall(stream) + }, + } + + sopts := []grpc.ServerOption{orca.CallMetricsServerOption(nil)} + + oso := orca.ServiceOptions{ + ServerMetricsProvider: smr, + MinReportingInterval: time.Second, + } + internal.ORCAAllowAnyMinReportingInterval.(func(so *orca.ServiceOptions))(&oso) + sopts = append(sopts, stubserver.RegisterServiceServerOption(func(s grpc.ServiceRegistrar) { + if err := orca.Register(s, oso); err != nil { + t.Fatalf("Failed to register ORCA service: %v", err) + } + })) + + if err := stub.StartServer(sopts...); err != nil { + t.Fatalf("Error starting server: %v", err) + } + t.Cleanup(stub.Stop) + + return &orcaServer{ + StubServer: stub, + metricsRecorder: smr, + } +} + +func orcaSvcConfig() string { + return `{"loadBalancingConfig": [{"test_backend_metrics_load_balancer": {}}]}` +} + +// TestORCAPerRPCReport verifies that per-call ORCA load reports flow from the +// server through the orcaPicker's Done callback into the context. +func (s) TestORCAPerRPCReport(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv := startORCAServer(t) + if err := srv.StartClient(grpc.WithDefaultServiceConfig(orcaSvcConfig())); err != nil { + t.Fatalf("Error starting client: %v", err) + } + + DoORCAPerRPCTest(ctx, srv.Client) +} + +// TestORCAOOBReport verifies that OOB ORCA load reports flow through +// OnLoadReport and are returned by the picker when no per-call report is +// present. +func (s) TestORCAOOBReport(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv := startORCAServer(t) + if err := srv.StartClient(grpc.WithDefaultServiceConfig(orcaSvcConfig())); err != nil { + t.Fatalf("Error starting client: %v", err) + } + + DoORCAOOBTest(ctx, srv.Client) +} + +// TestORCAOOBFallback verifies the fallback behavior: when a per-call report +// is present but has all zero fields, the picker uses the most recent OOB +// report instead. +func (s) TestORCAOOBFallback(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv := startORCAServer(t) + if err := srv.StartClient(grpc.WithDefaultServiceConfig(orcaSvcConfig())); err != nil { + t.Fatalf("Error starting client: %v", err) + } + + stream, err := srv.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall failed: %v", err) + } + req := &testpb.StreamingOutputCallRequest{ + OrcaOobReport: &testpb.TestOrcaReport{ + CpuUtilization: 0.73, + MemoryUtilization: 0.55, + Utilization: map[string]float64{"util": 0.42}, + }, + ResponseParameters: []*testpb.ResponseParameters{{Size: 1}}, + } + if err := stream.Send(req); err != nil { + t.Fatalf("stream.Send failed: %v", err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("stream.Recv failed: %v", err) + } + + oobWant := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 0.73, + MemUtilization: 0.55, + Utilization: map[string]float64{"util": 0.42}, + } + pollORCAResult(ctx, t, srv.Client, oobWant) + + // OrcaPerQueryReport is nil: per-call report has all-zero fields, + // triggering the OOB fallback path in orcaPicker. + orcaRes := &v3orcapb.OrcaLoadReport{} + if _, err := srv.Client.UnaryCall(contextWithORCAResult(ctx, &orcaRes), &testpb.SimpleRequest{}); err != nil { + t.Fatalf("UnaryCall failed: %v", err) + } + if diff := cmp.Diff(orcaRes, oobWant, protocmp.Transform()); diff != "" { + t.Fatalf("ORCA load report with zero per-call mismatch (-got +want):\n%s", diff) + } +} + +// TestEndpoints_MultipleAddresses verifies client behavior when an endpoint has +// multiple addresses and the first is unreachable. It validates that the +// pick_first falls through the address list and connects via the first +// reachable address, so per-call ORCA reports flow correctly. +func (s) TestEndpoints_MultipleAddresses(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv := startORCAServer(t) + if err := srv.StartClient(grpc.WithDefaultServiceConfig(orcaSvcConfig())); err != nil { + t.Fatalf("Error starting client: %v", err) + } + + // First address is unreachable; pick_first falls through to the good one. + state := resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{ + {Addr: "bad-address"}, + {Addr: srv.Address}, + }}, + }, + } + srv.R.UpdateState(state) + // Transient errors while pick_first probes the bad address are expected. + want := &v3orcapb.OrcaLoadReport{CpuUtilization: 0.42, MemUtilization: 0.21} + for ; ctx.Err() == nil; <-time.After(100 * time.Millisecond) { + orcaRes := &v3orcapb.OrcaLoadReport{} + req := &testpb.SimpleRequest{ + OrcaPerQueryReport: &testpb.TestOrcaReport{ + CpuUtilization: 0.42, + MemoryUtilization: 0.21, + }, + } + if _, err := srv.Client.UnaryCall(contextWithORCAResult(ctx, &orcaRes), req); err != nil { + continue + } + if diff := cmp.Diff(orcaRes, want, protocmp.Transform()); diff != "" { + t.Fatalf("Per-RPC ORCA mismatch (-got +want):\n%s", diff) + } + return + } + t.Fatalf("timed out waiting for connection through good address") +} + +// TestMultipleEndpoints_OOBListeners verifies that N endpoints produce N +// independent pick_first instances and N independent OOB listeners, each +// receiving reports from its respective server. +func (s) TestMultipleEndpoints_OOBListeners(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srvA := startORCAServer(t) + srvB := startORCAServer(t) + + srvA.metricsRecorder.SetCPUUtilization(0.1) + srvB.metricsRecorder.SetCPUUtilization(0.9) + + r := manual.NewBuilderWithScheme("test") + cc, err := grpc.Dial(r.Scheme()+":///test", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(orcaSvcConfig()), + ) + if err != nil { + t.Fatalf("Failed to dial: %v", err) + } + defer cc.Close() + tc := testgrpc.NewTestServiceClient(cc) + + // One endpoint per server: endpointsharding creates two independent + // pick_first children, each with its own OOB listener. + state := resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: srvA.Address}}}, + {Addresses: []resolver.Address{{Addr: srvB.Address}}}, + }, + } + r.UpdateState(state) + // No per-call report triggers OOB fallback. With two active listeners, + // b.report alternates between servers; both values must eventually appear. + wantA := &v3orcapb.OrcaLoadReport{CpuUtilization: 0.1} + wantB := &v3orcapb.OrcaLoadReport{CpuUtilization: 0.9} + seenA, seenB := false, false + for ; ctx.Err() == nil && (!seenA || !seenB); <-time.After(100 * time.Millisecond) { + orcaRes := &v3orcapb.OrcaLoadReport{} + if _, err := tc.UnaryCall(contextWithORCAResult(ctx, &orcaRes), &testpb.SimpleRequest{}); err != nil { + continue + } + if diff := cmp.Diff(orcaRes, wantA, protocmp.Transform()); diff == "" { + seenA = true + } else if diff := cmp.Diff(orcaRes, wantB, protocmp.Transform()); diff == "" { + seenB = true + } + } + if ctx.Err() != nil { + t.Fatalf("timed out waiting for OOB reports from both endpoints; seenA=%v seenB=%v", seenA, seenB) + } +} + +// TestEndpointUpdate verifies client behavior in response to a resolver update +// that changes an endpoint's address. It ensures that the client disconnects +// from the old address and connects to the new address via pick_first. It +// also validates that the OOB listener for the old connection is stopped, +// and a new OOB listener is registered for the new connection once it becomes +// READY. +func (s) TestEndpointUpdate(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srvA := startORCAServer(t) + srvB := startORCAServer(t) + + r := manual.NewBuilderWithScheme("test") + cc, err := grpc.Dial(r.Scheme()+":///test", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(orcaSvcConfig()), + ) + if err != nil { + t.Fatalf("Failed to dial: %v", err) + } + defer cc.Close() + tc := testgrpc.NewTestServiceClient(cc) + + state := resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: srvA.Address}}}, + }, + } + r.UpdateState(state) + + orcaRes := &v3orcapb.OrcaLoadReport{} + req := &testpb.SimpleRequest{ + OrcaPerQueryReport: &testpb.TestOrcaReport{ + CpuUtilization: 0.11, + MemoryUtilization: 0.22, + }, + } + if _, err := tc.UnaryCall(contextWithORCAResult(ctx, &orcaRes), req); err != nil { + t.Fatalf("UnaryCall to srvA failed: %v", err) + } + wantA := &v3orcapb.OrcaLoadReport{CpuUtilization: 0.11, MemUtilization: 0.22} + if diff := cmp.Diff(orcaRes, wantA, protocmp.Transform()); diff != "" { + t.Fatalf("ORCA from srvA mismatch (-got +want):\n%s", diff) + } + + // Distinctive OOB metric on srvB to confirm the switch via OOB fallback. + srvB.metricsRecorder.SetCPUUtilization(0.77) + + state = resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: srvB.Address}}}, + }, + } + r.UpdateState(state) + // No per-call report forces OOB fallback. Once srvB's metric appears, + // all three are confirmed: pick_first reconnected, new OOB listener + // registered for srvB, old one for srvA stopped. + wantB := &v3orcapb.OrcaLoadReport{CpuUtilization: 0.77} + for ; ctx.Err() == nil; <-time.After(100 * time.Millisecond) { + orcaRes := &v3orcapb.OrcaLoadReport{} + if _, err := tc.UnaryCall(contextWithORCAResult(ctx, &orcaRes), &testpb.SimpleRequest{}); err != nil { + continue + } + if diff := cmp.Diff(orcaRes, wantB, protocmp.Transform()); diff == "" { + return + } + t.Logf("ORCA after endpoint update = %v; want %v; retrying...", orcaRes, wantB) + } + t.Fatalf("timed out waiting for srvB OOB report after endpoint update") +} + +func pollORCAResult(ctx context.Context, t *testing.T, tc testgrpc.TestServiceClient, want *v3orcapb.OrcaLoadReport) { + t.Helper() + for ; ctx.Err() == nil; <-time.After(time.Second) { + orcaRes := &v3orcapb.OrcaLoadReport{} + if _, err := tc.UnaryCall(contextWithORCAResult(ctx, &orcaRes), &testpb.SimpleRequest{}); err != nil { + t.Fatalf("UnaryCall failed: %v", err) + } + if diff := cmp.Diff(orcaRes, want, protocmp.Transform()); diff == "" { + return + } + t.Logf("ORCA load report = %v; want %v; retrying...", orcaRes, want) + } + t.Fatalf("Timed out waiting for expected ORCA load report %v", want) +} diff --git a/interop/server/server.go b/interop/server/server.go new file mode 100644 index 0000000..67fbc31 --- /dev/null +++ b/interop/server/server.go @@ -0,0 +1,97 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is an interop server. +// +// See interop test case descriptions [here]. +// +// [here]: https://github.com/grpc/grpc/blob/master/doc/interop-test-descriptions.md +package main + +import ( + "flag" + "net" + "strconv" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/alts" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/interop" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + useTLS = flag.Bool("use_tls", false, "Connection uses TLS if true, else plain TCP") + useALTS = flag.Bool("use_alts", false, "Connection uses ALTS if true (this option can only be used on GCP)") + altsHSAddr = flag.String("alts_handshaker_service_address", "", "ALTS handshaker gRPC service address") + certFile = flag.String("tls_cert_file", "", "The TLS cert file") + keyFile = flag.String("tls_key_file", "", "The TLS key file") + port = flag.Int("port", 10000, "The server port") + + logger = grpclog.Component("interop") +) + +func main() { + flag.Parse() + if *useTLS && *useALTS { + logger.Fatal("-use_tls and -use_alts cannot be both set to true") + } + p := strconv.Itoa(*port) + lis, err := net.Listen("tcp", ":"+p) + if err != nil { + logger.Fatalf("failed to listen: %v", err) + } + logger.Infof("interop server listening on %v", lis.Addr()) + opts := []grpc.ServerOption{orca.CallMetricsServerOption(nil)} + if *useTLS { + if *certFile == "" { + *certFile = testdata.Path("server1.pem") + } + if *keyFile == "" { + *keyFile = testdata.Path("server1.key") + } + creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile) + if err != nil { + logger.Fatalf("Failed to generate credentials: %v", err) + } + opts = append(opts, grpc.Creds(creds)) + } else if *useALTS { + altsOpts := alts.DefaultServerOptions() + if *altsHSAddr != "" { + altsOpts.HandshakerServiceAddress = *altsHSAddr + } + altsTC := alts.NewServerCreds(altsOpts) + opts = append(opts, grpc.Creds(altsTC)) + } + server := grpc.NewServer(opts...) + metricsRecorder := orca.NewServerMetricsRecorder() + sopts := orca.ServiceOptions{ + MinReportingInterval: time.Second, + ServerMetricsProvider: metricsRecorder, + } + internal.ORCAAllowAnyMinReportingInterval.(func(*orca.ServiceOptions))(&sopts) + orca.Register(server, sopts) + testgrpc.RegisterTestServiceServer(server, interop.NewTestServer(interop.NewTestServerOptions{MetricsRecorder: metricsRecorder})) + server.Serve(lis) +} diff --git a/interop/soak_tests.go b/interop/soak_tests.go new file mode 100644 index 0000000..6d1863d --- /dev/null +++ b/interop/soak_tests.go @@ -0,0 +1,203 @@ +/* +* +* Copyright 2014 gRPC authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package interop + +import ( + "bytes" + "context" + "fmt" + "os" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/benchmark/stats" + "google.golang.org/grpc/peer" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// SoakWorkerResults stores the aggregated results for a specific worker during the soak test. +type SoakWorkerResults struct { + iterationsSucceeded int + Failures int + Latencies *stats.Histogram +} + +// SoakIterationConfig holds the parameters required for a single soak iteration. +type SoakIterationConfig struct { + RequestSize int // The size of the request payload in bytes. + ResponseSize int // The expected size of the response payload in bytes. + Client testgrpc.TestServiceClient // The gRPC client to make the call. + CallOptions []grpc.CallOption // Call options for the RPC. +} + +// SoakTestConfig holds the configuration for the entire soak test. +type SoakTestConfig struct { + RequestSize int + ResponseSize int + PerIterationMaxAcceptableLatency time.Duration + MinTimeBetweenRPCs time.Duration + OverallTimeout time.Duration + ServerAddr string + NumWorkers int + Iterations int + MaxFailures int + ChannelForTest func() (*grpc.ClientConn, func()) +} + +func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latency time.Duration, err error) { + start := time.Now() + // Do a large-unary RPC. + // Create the request payload. + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, config.RequestSize) + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(config.ResponseSize), + Payload: pl, + } + // Perform the GRPC call. + var reply *testpb.SimpleResponse + reply, err = config.Client.UnaryCall(ctx, req, config.CallOptions...) + if err != nil { + err = fmt.Errorf("/TestService/UnaryCall RPC failed: %s", err) + return 0, err + } + // Validate response. + t := reply.GetPayload().GetType() + s := len(reply.GetPayload().GetBody()) + if t != testpb.PayloadType_COMPRESSABLE || s != config.ResponseSize { + err = fmt.Errorf("got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, config.ResponseSize) + return 0, err + } + // Calculate latency and return result. + latency = time.Since(start) + return latency, nil +} + +func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTime time.Time, workerID int, soakWorkerResults *SoakWorkerResults) { + timeoutDuration := config.OverallTimeout + soakIterationsPerWorker := config.Iterations / config.NumWorkers + if soakWorkerResults.Latencies == nil { + soakWorkerResults.Latencies = stats.NewHistogram(stats.HistogramOptions{ + NumBuckets: 20, + GrowthFactor: 1, + BaseBucketSize: 1, + MinValue: 0, + }) + } + + for i := 0; i < soakIterationsPerWorker; i++ { + if ctx.Err() != nil { + return + } + if time.Since(startTime) >= timeoutDuration { + fmt.Printf("Test exceeded overall timeout of %v, stopping...\n", config.OverallTimeout) + return + } + earliestNextStart := time.After(config.MinTimeBetweenRPCs) + currentChannel, cleanup := config.ChannelForTest() + defer cleanup() + client := testgrpc.NewTestServiceClient(currentChannel) + var p peer.Peer + iterationConfig := SoakIterationConfig{ + RequestSize: config.RequestSize, + ResponseSize: config.ResponseSize, + Client: client, + CallOptions: []grpc.CallOption{grpc.Peer(&p)}, + } + latency, err := doOneSoakIteration(ctx, iterationConfig) + if err != nil { + fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s failed: %s\n", workerID, i, 0, p.Addr, config.ServerAddr, err) + soakWorkerResults.Failures++ + <-earliestNextStart + continue + } + if latency > config.PerIterationMaxAcceptableLatency { + fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", workerID, i, latency.Milliseconds(), p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) + soakWorkerResults.Failures++ + <-earliestNextStart + continue + } + // Success: log the details of the iteration. + soakWorkerResults.Latencies.Add(latency.Milliseconds()) + soakWorkerResults.iterationsSucceeded++ + fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", workerID, i, latency.Milliseconds(), p.Addr, config.ServerAddr) + <-earliestNextStart + } +} + +// DoSoakTest runs large unary RPCs in a loop for a configurable number of times, with configurable failure thresholds. +// If resetChannel is false, then each RPC will be performed on tc. Otherwise, each RPC will be performed on a new +// stub that is created with the provided server address and dial options. +// TODO(mohanli-ml): Create SoakTestOptions as a parameter for this method. +func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { + if soakConfig.Iterations%soakConfig.NumWorkers != 0 { + fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by soakNumWThreads\n") + } + startTime := time.Now() + var wg sync.WaitGroup + soakWorkerResults := make([]SoakWorkerResults, soakConfig.NumWorkers) + for i := 0; i < soakConfig.NumWorkers; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + executeSoakTestInWorker(ctx, soakConfig, startTime, workerID, &soakWorkerResults[workerID]) + }(i) + } + // Wait for all goroutines to complete. + wg.Wait() + + // Handle results. + totalSuccesses := 0 + totalFailures := 0 + latencies := stats.NewHistogram(stats.HistogramOptions{ + NumBuckets: 20, + GrowthFactor: 1, + BaseBucketSize: 1, + MinValue: 0, + }) + for _, worker := range soakWorkerResults { + totalSuccesses += worker.iterationsSucceeded + totalFailures += worker.Failures + if worker.Latencies != nil { + // Add latencies from the worker's Histogram to the main latencies. + latencies.Merge(worker.Latencies) + } + } + var b bytes.Buffer + totalIterations := totalSuccesses + totalFailures + latencies.Print(&b) + fmt.Fprintf(os.Stderr, + "(server_uri: %s) soak test successes: %d / %d iterations. Total failures: %d. Latencies in milliseconds: %s\n", + soakConfig.ServerAddr, totalSuccesses, soakConfig.Iterations, totalFailures, b.String()) + + if totalIterations != soakConfig.Iterations { + logger.Fatalf("Soak test consumed all %v of time and quit early, ran %d out of %d iterations.\n", soakConfig.OverallTimeout, totalIterations, soakConfig.Iterations) + } + + if totalFailures > soakConfig.MaxFailures { + logger.Fatalf("Soak test total failures: %d exceeded max failures threshold: %d\n", totalFailures, soakConfig.MaxFailures) + } + if soakConfig.ChannelForTest != nil { + _, cleanup := soakConfig.ChannelForTest() + defer cleanup() + } +} diff --git a/interop/stress/client/main.go b/interop/stress/client/main.go new file mode 100644 index 0000000..457ff54 --- /dev/null +++ b/interop/stress/client/main.go @@ -0,0 +1,364 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// client starts an interop client to do stress test and a metrics server to report qps. +package main + +import ( + "context" + "flag" + "fmt" + rand "math/rand/v2" + "net" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/interop" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" + + _ "google.golang.org/grpc/xds/googledirectpath" // Register xDS resolver required for c2p directpath. + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + metricspb "google.golang.org/grpc/interop/stress/grpc_testing" +) + +const ( + googleDefaultCredsName = "google_default_credentials" + computeEngineCredsName = "compute_engine_channel_creds" +) + +var ( + serverAddresses = flag.String("server_addresses", "localhost:8080", "a list of server addresses") + testCases = flag.String("test_cases", "", "a list of test cases along with the relative weights") + testDurationSecs = flag.Int("test_duration_secs", -1, "test duration in seconds") + numChannelsPerServer = flag.Int("num_channels_per_server", 1, "Number of channels (i.e connections) to each server") + numStubsPerChannel = flag.Int("num_stubs_per_channel", 1, "Number of client stubs per each connection to server") + metricsPort = flag.Int("metrics_port", 8081, "The port at which the stress client exposes QPS metrics") + useTLS = flag.Bool("use_tls", false, "Connection uses TLS if true, else plain TCP") + testCA = flag.Bool("use_test_ca", false, "Whether to replace platform root CAs with test CA as the CA root") + tlsServerName = flag.String("server_host_override", "foo.test.google.fr", "The server name use to verify the hostname returned by TLS handshake if it is not empty. Otherwise, --server_host is used.") + caFile = flag.String("ca_file", "", "The file containing the CA root cert file") + customCredentialsType = flag.String("custom_credentials_type", "", "Custom credentials type to use") + + totalNumCalls int64 + logger = grpclog.Component("stress") +) + +// testCaseWithWeight contains the test case type and its weight. +type testCaseWithWeight struct { + name string + weight int +} + +// parseTestCases converts test case string to a list of struct testCaseWithWeight. +func parseTestCases(testCaseString string) []testCaseWithWeight { + testCaseStrings := strings.Split(testCaseString, ",") + testCases := make([]testCaseWithWeight, len(testCaseStrings)) + for i, str := range testCaseStrings { + testCaseNameAndWeight := strings.Split(str, ":") + if len(testCaseNameAndWeight) != 2 { + panic(fmt.Sprintf("invalid test case with weight: %s", str)) + } + // Check if test case is supported. + testCaseName := strings.ToLower(testCaseNameAndWeight[0]) + switch testCaseName { + case + "empty_unary", + "large_unary", + "client_streaming", + "server_streaming", + "ping_pong", + "empty_stream", + "timeout_on_sleeping_server", + "cancel_after_begin", + "cancel_after_first_response", + "status_code_and_message", + "custom_metadata": + default: + panic(fmt.Sprintf("unknown test type: %s", testCaseNameAndWeight[0])) + } + testCases[i].name = testCaseName + w, err := strconv.Atoi(testCaseNameAndWeight[1]) + if err != nil { + panic(fmt.Sprintf("%v", err)) + } + testCases[i].weight = w + } + return testCases +} + +// weightedRandomTestSelector defines a weighted random selector for test case types. +type weightedRandomTestSelector struct { + tests []testCaseWithWeight + totalWeight int +} + +// newWeightedRandomTestSelector constructs a weightedRandomTestSelector with the given list of testCaseWithWeight. +func newWeightedRandomTestSelector(tests []testCaseWithWeight) *weightedRandomTestSelector { + var totalWeight int + for _, t := range tests { + totalWeight += t.weight + } + return &weightedRandomTestSelector{tests, totalWeight} +} + +func (selector weightedRandomTestSelector) getNextTest() string { + random := rand.IntN(selector.totalWeight) + var weightSofar int + for _, test := range selector.tests { + weightSofar += test.weight + if random < weightSofar { + return test.name + } + } + panic("no test case selected by weightedRandomTestSelector") +} + +// gauge stores the qps of one interop client (one stub). +type gauge struct { + mutex sync.RWMutex + val int64 +} + +func (g *gauge) set(v int64) { + g.mutex.Lock() + defer g.mutex.Unlock() + g.val = v +} + +func (g *gauge) get() int64 { + g.mutex.RLock() + defer g.mutex.RUnlock() + return g.val +} + +// server implements metrics server functions. +type server struct { + metricspb.UnimplementedMetricsServiceServer + mutex sync.RWMutex + // gauges is a map from /stress_test/server_/channel_/stub_/qps to its qps gauge. + gauges map[string]*gauge +} + +// newMetricsServer returns a new metrics server. +func newMetricsServer() *server { + return &server{gauges: make(map[string]*gauge)} +} + +// GetAllGauges returns all gauges. +func (s *server) GetAllGauges(_ *metricspb.EmptyMessage, stream metricspb.MetricsService_GetAllGaugesServer) error { + s.mutex.RLock() + defer s.mutex.RUnlock() + + for name, gauge := range s.gauges { + if err := stream.Send(&metricspb.GaugeResponse{Name: name, Value: &metricspb.GaugeResponse_LongValue{LongValue: gauge.get()}}); err != nil { + return err + } + } + return nil +} + +// GetGauge returns the gauge for the given name. +func (s *server) GetGauge(_ context.Context, in *metricspb.GaugeRequest) (*metricspb.GaugeResponse, error) { + s.mutex.RLock() + defer s.mutex.RUnlock() + + if g, ok := s.gauges[in.Name]; ok { + return &metricspb.GaugeResponse{Name: in.Name, Value: &metricspb.GaugeResponse_LongValue{LongValue: g.get()}}, nil + } + return nil, status.Errorf(codes.InvalidArgument, "gauge with name %s not found", in.Name) +} + +// createGauge creates a gauge using the given name in metrics server. +func (s *server) createGauge(name string) *gauge { + s.mutex.Lock() + defer s.mutex.Unlock() + + if _, ok := s.gauges[name]; ok { + // gauge already exists. + panic(fmt.Sprintf("gauge %s already exists", name)) + } + var g gauge + s.gauges[name] = &g + return &g +} + +func startServer(server *server, port int) { + lis, err := net.Listen("tcp", ":"+strconv.Itoa(port)) + if err != nil { + logger.Fatalf("failed to listen: %v", err) + } + + s := grpc.NewServer() + metricspb.RegisterMetricsServiceServer(s, server) + s.Serve(lis) +} + +// performRPCs uses weightedRandomTestSelector to select test case and runs the tests. +func performRPCs(gauge *gauge, conn *grpc.ClientConn, selector *weightedRandomTestSelector, stop <-chan bool) { + client := testgrpc.NewTestServiceClient(conn) + var numCalls int64 + ctx := context.Background() + startTime := time.Now() + for { + test := selector.getNextTest() + switch test { + case "empty_unary": + interop.DoEmptyUnaryCall(ctx, client) + case "large_unary": + interop.DoLargeUnaryCall(ctx, client) + case "client_streaming": + interop.DoClientStreaming(ctx, client) + case "server_streaming": + interop.DoServerStreaming(ctx, client) + case "ping_pong": + interop.DoPingPong(ctx, client) + case "empty_stream": + interop.DoEmptyStream(ctx, client) + case "timeout_on_sleeping_server": + interop.DoTimeoutOnSleepingServer(ctx, client) + case "cancel_after_begin": + interop.DoCancelAfterBegin(ctx, client) + case "cancel_after_first_response": + interop.DoCancelAfterFirstResponse(ctx, client) + case "status_code_and_message": + interop.DoStatusCodeAndMessage(ctx, client) + case "custom_metadata": + interop.DoCustomMetadata(ctx, client) + } + numCalls++ + defer func() { atomic.AddInt64(&totalNumCalls, numCalls) }() + gauge.set(int64(float64(numCalls) / time.Since(startTime).Seconds())) + + select { + case <-stop: + return + default: + } + } +} + +func logParameterInfo(addresses []string, tests []testCaseWithWeight) { + logger.Infof("server_addresses: %s", *serverAddresses) + logger.Infof("test_cases: %s", *testCases) + logger.Infof("test_duration_secs: %d", *testDurationSecs) + logger.Infof("num_channels_per_server: %d", *numChannelsPerServer) + logger.Infof("num_stubs_per_channel: %d", *numStubsPerChannel) + logger.Infof("metrics_port: %d", *metricsPort) + logger.Infof("use_tls: %t", *useTLS) + logger.Infof("use_test_ca: %t", *testCA) + logger.Infof("server_host_override: %s", *tlsServerName) + logger.Infof("custom_credentials_type: %s", *customCredentialsType) + + logger.Infoln("addresses:") + for i, addr := range addresses { + logger.Infof("%d. %s\n", i+1, addr) + } + logger.Infoln("tests:") + for i, test := range tests { + logger.Infof("%d. %v\n", i+1, test) + } +} + +func newConn(address string, useTLS, testCA bool, tlsServerName string) (*grpc.ClientConn, error) { + var opts []grpc.DialOption + if *customCredentialsType != "" { + if *customCredentialsType == googleDefaultCredsName { + opts = append(opts, grpc.WithCredentialsBundle(google.NewDefaultCredentials())) + } else if *customCredentialsType == computeEngineCredsName { + opts = append(opts, grpc.WithCredentialsBundle(google.NewComputeEngineCredentials())) + } else { + logger.Fatalf("Unknown custom credentials: %v", *customCredentialsType) + } + } else if useTLS { + var sn string + if tlsServerName != "" { + sn = tlsServerName + } + var creds credentials.TransportCredentials + if testCA { + var err error + if *caFile == "" { + *caFile = testdata.Path("x509/server_ca_cert.pem") + } + creds, err = credentials.NewClientTLSFromFile(*caFile, sn) + if err != nil { + logger.Fatalf("Failed to create TLS credentials: %v", err) + } + } else { + creds = credentials.NewClientTLSFromCert(nil, sn) + } + opts = append(opts, grpc.WithTransportCredentials(creds)) + } else { + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + return grpc.NewClient(address, opts...) +} + +func main() { + flag.Parse() + resolver.SetDefaultScheme("dns") + addresses := strings.Split(*serverAddresses, ",") + tests := parseTestCases(*testCases) + logParameterInfo(addresses, tests) + testSelector := newWeightedRandomTestSelector(tests) + metricsServer := newMetricsServer() + + var wg sync.WaitGroup + wg.Add(len(addresses) * *numChannelsPerServer * *numStubsPerChannel) + stop := make(chan bool) + + for serverIndex, address := range addresses { + for connIndex := 0; connIndex < *numChannelsPerServer; connIndex++ { + conn, err := newConn(address, *useTLS, *testCA, *tlsServerName) + if err != nil { + logger.Fatalf("Fail to dial: %v", err) + } + defer conn.Close() + for clientIndex := 0; clientIndex < *numStubsPerChannel; clientIndex++ { + name := fmt.Sprintf("/stress_test/server_%d/channel_%d/stub_%d/qps", serverIndex+1, connIndex+1, clientIndex+1) + go func() { + defer wg.Done() + g := metricsServer.createGauge(name) + performRPCs(g, conn, testSelector, stop) + }() + } + + } + } + go startServer(metricsServer, *metricsPort) + if *testDurationSecs > 0 { + time.Sleep(time.Duration(*testDurationSecs) * time.Second) + close(stop) + } + wg.Wait() + fmt.Fprintf(os.Stdout, "Total calls made: %v\n", totalNumCalls) + logger.Infof(" ===== ALL DONE ===== ") +} diff --git a/interop/stress/grpc_testing/metrics.pb.go b/interop/stress/grpc_testing/metrics.pb.go new file mode 100644 index 0000000..5d42916 --- /dev/null +++ b/interop/stress/grpc_testing/metrics.pb.go @@ -0,0 +1,309 @@ +// Copyright 2015-2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Contains the definitions for a metrics service and the type of metrics +// exposed by the service. +// +// Currently, 'Gauge' (i.e a metric that represents the measured value of +// something at an instant of time) is the only metric type supported by the +// service. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: interop/stress/grpc_testing/metrics.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Response message containing the gauge name and value +type GaugeResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Types that are valid to be assigned to Value: + // + // *GaugeResponse_LongValue + // *GaugeResponse_DoubleValue + // *GaugeResponse_StringValue + Value isGaugeResponse_Value `protobuf_oneof:"value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GaugeResponse) Reset() { + *x = GaugeResponse{} + mi := &file_interop_stress_grpc_testing_metrics_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GaugeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GaugeResponse) ProtoMessage() {} + +func (x *GaugeResponse) ProtoReflect() protoreflect.Message { + mi := &file_interop_stress_grpc_testing_metrics_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GaugeResponse.ProtoReflect.Descriptor instead. +func (*GaugeResponse) Descriptor() ([]byte, []int) { + return file_interop_stress_grpc_testing_metrics_proto_rawDescGZIP(), []int{0} +} + +func (x *GaugeResponse) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *GaugeResponse) GetValue() isGaugeResponse_Value { + if x != nil { + return x.Value + } + return nil +} + +func (x *GaugeResponse) GetLongValue() int64 { + if x != nil { + if x, ok := x.Value.(*GaugeResponse_LongValue); ok { + return x.LongValue + } + } + return 0 +} + +func (x *GaugeResponse) GetDoubleValue() float64 { + if x != nil { + if x, ok := x.Value.(*GaugeResponse_DoubleValue); ok { + return x.DoubleValue + } + } + return 0 +} + +func (x *GaugeResponse) GetStringValue() string { + if x != nil { + if x, ok := x.Value.(*GaugeResponse_StringValue); ok { + return x.StringValue + } + } + return "" +} + +type isGaugeResponse_Value interface { + isGaugeResponse_Value() +} + +type GaugeResponse_LongValue struct { + LongValue int64 `protobuf:"varint,2,opt,name=long_value,json=longValue,proto3,oneof"` +} + +type GaugeResponse_DoubleValue struct { + DoubleValue float64 `protobuf:"fixed64,3,opt,name=double_value,json=doubleValue,proto3,oneof"` +} + +type GaugeResponse_StringValue struct { + StringValue string `protobuf:"bytes,4,opt,name=string_value,json=stringValue,proto3,oneof"` +} + +func (*GaugeResponse_LongValue) isGaugeResponse_Value() {} + +func (*GaugeResponse_DoubleValue) isGaugeResponse_Value() {} + +func (*GaugeResponse_StringValue) isGaugeResponse_Value() {} + +// Request message containing the gauge name +type GaugeRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GaugeRequest) Reset() { + *x = GaugeRequest{} + mi := &file_interop_stress_grpc_testing_metrics_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GaugeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GaugeRequest) ProtoMessage() {} + +func (x *GaugeRequest) ProtoReflect() protoreflect.Message { + mi := &file_interop_stress_grpc_testing_metrics_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GaugeRequest.ProtoReflect.Descriptor instead. +func (*GaugeRequest) Descriptor() ([]byte, []int) { + return file_interop_stress_grpc_testing_metrics_proto_rawDescGZIP(), []int{1} +} + +func (x *GaugeRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +type EmptyMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EmptyMessage) Reset() { + *x = EmptyMessage{} + mi := &file_interop_stress_grpc_testing_metrics_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EmptyMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmptyMessage) ProtoMessage() {} + +func (x *EmptyMessage) ProtoReflect() protoreflect.Message { + mi := &file_interop_stress_grpc_testing_metrics_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmptyMessage.ProtoReflect.Descriptor instead. +func (*EmptyMessage) Descriptor() ([]byte, []int) { + return file_interop_stress_grpc_testing_metrics_proto_rawDescGZIP(), []int{2} +} + +var File_interop_stress_grpc_testing_metrics_proto protoreflect.FileDescriptor + +const file_interop_stress_grpc_testing_metrics_proto_rawDesc = "" + + "\n" + + ")interop/stress/grpc_testing/metrics.proto\x12\fgrpc.testing\"\x97\x01\n" + + "\rGaugeResponse\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n" + + "\n" + + "long_value\x18\x02 \x01(\x03H\x00R\tlongValue\x12#\n" + + "\fdouble_value\x18\x03 \x01(\x01H\x00R\vdoubleValue\x12#\n" + + "\fstring_value\x18\x04 \x01(\tH\x00R\vstringValueB\a\n" + + "\x05value\"\"\n" + + "\fGaugeRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\"\x0e\n" + + "\fEmptyMessage2\xa0\x01\n" + + "\x0eMetricsService\x12I\n" + + "\fGetAllGauges\x12\x1a.grpc.testing.EmptyMessage\x1a\x1b.grpc.testing.GaugeResponse0\x01\x12C\n" + + "\bGetGauge\x12\x1a.grpc.testing.GaugeRequest\x1a\x1b.grpc.testing.GaugeResponseB4Z2google.golang.org/grpc/interop/stress/grpc_testingb\x06proto3" + +var ( + file_interop_stress_grpc_testing_metrics_proto_rawDescOnce sync.Once + file_interop_stress_grpc_testing_metrics_proto_rawDescData []byte +) + +func file_interop_stress_grpc_testing_metrics_proto_rawDescGZIP() []byte { + file_interop_stress_grpc_testing_metrics_proto_rawDescOnce.Do(func() { + file_interop_stress_grpc_testing_metrics_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_interop_stress_grpc_testing_metrics_proto_rawDesc), len(file_interop_stress_grpc_testing_metrics_proto_rawDesc))) + }) + return file_interop_stress_grpc_testing_metrics_proto_rawDescData +} + +var file_interop_stress_grpc_testing_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_interop_stress_grpc_testing_metrics_proto_goTypes = []any{ + (*GaugeResponse)(nil), // 0: grpc.testing.GaugeResponse + (*GaugeRequest)(nil), // 1: grpc.testing.GaugeRequest + (*EmptyMessage)(nil), // 2: grpc.testing.EmptyMessage +} +var file_interop_stress_grpc_testing_metrics_proto_depIdxs = []int32{ + 2, // 0: grpc.testing.MetricsService.GetAllGauges:input_type -> grpc.testing.EmptyMessage + 1, // 1: grpc.testing.MetricsService.GetGauge:input_type -> grpc.testing.GaugeRequest + 0, // 2: grpc.testing.MetricsService.GetAllGauges:output_type -> grpc.testing.GaugeResponse + 0, // 3: grpc.testing.MetricsService.GetGauge:output_type -> grpc.testing.GaugeResponse + 2, // [2:4] is the sub-list for method output_type + 0, // [0:2] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_interop_stress_grpc_testing_metrics_proto_init() } +func file_interop_stress_grpc_testing_metrics_proto_init() { + if File_interop_stress_grpc_testing_metrics_proto != nil { + return + } + file_interop_stress_grpc_testing_metrics_proto_msgTypes[0].OneofWrappers = []any{ + (*GaugeResponse_LongValue)(nil), + (*GaugeResponse_DoubleValue)(nil), + (*GaugeResponse_StringValue)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_interop_stress_grpc_testing_metrics_proto_rawDesc), len(file_interop_stress_grpc_testing_metrics_proto_rawDesc)), + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_interop_stress_grpc_testing_metrics_proto_goTypes, + DependencyIndexes: file_interop_stress_grpc_testing_metrics_proto_depIdxs, + MessageInfos: file_interop_stress_grpc_testing_metrics_proto_msgTypes, + }.Build() + File_interop_stress_grpc_testing_metrics_proto = out.File + file_interop_stress_grpc_testing_metrics_proto_goTypes = nil + file_interop_stress_grpc_testing_metrics_proto_depIdxs = nil +} diff --git a/interop/stress/grpc_testing/metrics.proto b/interop/stress/grpc_testing/metrics.proto new file mode 100644 index 0000000..301527e --- /dev/null +++ b/interop/stress/grpc_testing/metrics.proto @@ -0,0 +1,51 @@ +// Copyright 2015-2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Contains the definitions for a metrics service and the type of metrics +// exposed by the service. +// +// Currently, 'Gauge' (i.e a metric that represents the measured value of +// something at an instant of time) is the only metric type supported by the +// service. +syntax = "proto3"; + +option go_package = "google.golang.org/grpc/interop/stress/grpc_testing"; + +package grpc.testing; + +// Response message containing the gauge name and value +message GaugeResponse { + string name = 1; + oneof value { + int64 long_value = 2; + double double_value = 3; + string string_value = 4; + } +} + +// Request message containing the gauge name +message GaugeRequest { + string name = 1; +} + +message EmptyMessage {} + +service MetricsService { + // Returns the values of all the gauges that are currently being maintained by + // the service + rpc GetAllGauges(EmptyMessage) returns (stream GaugeResponse); + + // Returns the value of one gauge + rpc GetGauge(GaugeRequest) returns (GaugeResponse); +} diff --git a/interop/stress/grpc_testing/metrics_grpc.pb.go b/interop/stress/grpc_testing/metrics_grpc.pb.go new file mode 100644 index 0000000..6d13101 --- /dev/null +++ b/interop/stress/grpc_testing/metrics_grpc.pb.go @@ -0,0 +1,190 @@ +// Copyright 2015-2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Contains the definitions for a metrics service and the type of metrics +// exposed by the service. +// +// Currently, 'Gauge' (i.e a metric that represents the measured value of +// something at an instant of time) is the only metric type supported by the +// service. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: interop/stress/grpc_testing/metrics.proto + +package grpc_testing + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + MetricsService_GetAllGauges_FullMethodName = "/grpc.testing.MetricsService/GetAllGauges" + MetricsService_GetGauge_FullMethodName = "/grpc.testing.MetricsService/GetGauge" +) + +// MetricsServiceClient is the client API for MetricsService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type MetricsServiceClient interface { + // Returns the values of all the gauges that are currently being maintained by + // the service + GetAllGauges(ctx context.Context, in *EmptyMessage, opts ...grpc.CallOption) (grpc.ServerStreamingClient[GaugeResponse], error) + // Returns the value of one gauge + GetGauge(ctx context.Context, in *GaugeRequest, opts ...grpc.CallOption) (*GaugeResponse, error) +} + +type metricsServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewMetricsServiceClient(cc grpc.ClientConnInterface) MetricsServiceClient { + return &metricsServiceClient{cc} +} + +func (c *metricsServiceClient) GetAllGauges(ctx context.Context, in *EmptyMessage, opts ...grpc.CallOption) (grpc.ServerStreamingClient[GaugeResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &MetricsService_ServiceDesc.Streams[0], MetricsService_GetAllGauges_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[EmptyMessage, GaugeResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type MetricsService_GetAllGaugesClient = grpc.ServerStreamingClient[GaugeResponse] + +func (c *metricsServiceClient) GetGauge(ctx context.Context, in *GaugeRequest, opts ...grpc.CallOption) (*GaugeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GaugeResponse) + err := c.cc.Invoke(ctx, MetricsService_GetGauge_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// MetricsServiceServer is the server API for MetricsService service. +// All implementations must embed UnimplementedMetricsServiceServer +// for forward compatibility. +type MetricsServiceServer interface { + // Returns the values of all the gauges that are currently being maintained by + // the service + GetAllGauges(*EmptyMessage, grpc.ServerStreamingServer[GaugeResponse]) error + // Returns the value of one gauge + GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error) + mustEmbedUnimplementedMetricsServiceServer() +} + +// UnimplementedMetricsServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedMetricsServiceServer struct{} + +func (UnimplementedMetricsServiceServer) GetAllGauges(*EmptyMessage, grpc.ServerStreamingServer[GaugeResponse]) error { + return status.Error(codes.Unimplemented, "method GetAllGauges not implemented") +} +func (UnimplementedMetricsServiceServer) GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetGauge not implemented") +} +func (UnimplementedMetricsServiceServer) mustEmbedUnimplementedMetricsServiceServer() {} +func (UnimplementedMetricsServiceServer) testEmbeddedByValue() {} + +// UnsafeMetricsServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to MetricsServiceServer will +// result in compilation errors. +type UnsafeMetricsServiceServer interface { + mustEmbedUnimplementedMetricsServiceServer() +} + +func RegisterMetricsServiceServer(s grpc.ServiceRegistrar, srv MetricsServiceServer) { + // If the following call panics, it indicates UnimplementedMetricsServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&MetricsService_ServiceDesc, srv) +} + +func _MetricsService_GetAllGauges_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(EmptyMessage) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(MetricsServiceServer).GetAllGauges(m, &grpc.GenericServerStream[EmptyMessage, GaugeResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type MetricsService_GetAllGaugesServer = grpc.ServerStreamingServer[GaugeResponse] + +func _MetricsService_GetGauge_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GaugeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetricsServiceServer).GetGauge(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: MetricsService_GetGauge_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetricsServiceServer).GetGauge(ctx, req.(*GaugeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// MetricsService_ServiceDesc is the grpc.ServiceDesc for MetricsService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var MetricsService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.MetricsService", + HandlerType: (*MetricsServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetGauge", + Handler: _MetricsService_GetGauge_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "GetAllGauges", + Handler: _MetricsService_GetAllGauges_Handler, + ServerStreams: true, + }, + }, + Metadata: "interop/stress/grpc_testing/metrics.proto", +} diff --git a/interop/stress/metrics_client/main.go b/interop/stress/metrics_client/main.go new file mode 100644 index 0000000..84b4e6b --- /dev/null +++ b/interop/stress/metrics_client/main.go @@ -0,0 +1,86 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary metrics_client is a client to retrieve metrics from the server. +package main + +import ( + "context" + "flag" + "fmt" + "io" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/grpclog" + metricspb "google.golang.org/grpc/interop/stress/grpc_testing" +) + +var ( + metricsServerAddress = flag.String("metrics_server_address", "", "The metrics server addresses in the format :") + totalOnly = flag.Bool("total_only", false, "If true, this prints only the total value of all gauges") + + logger = grpclog.Component("stress") +) + +func printMetrics(client metricspb.MetricsServiceClient, totalOnly bool) { + stream, err := client.GetAllGauges(context.Background(), &metricspb.EmptyMessage{}) + if err != nil { + logger.Fatalf("failed to call GetAllGauges: %v", err) + } + + var ( + overallQPS int64 + rpcStatus error + ) + for { + gaugeResponse, err := stream.Recv() + if err != nil { + rpcStatus = err + break + } + if _, ok := gaugeResponse.GetValue().(*metricspb.GaugeResponse_LongValue); !ok { + panic(fmt.Sprintf("gauge %s is not a long value", gaugeResponse.Name)) + } + v := gaugeResponse.GetLongValue() + if !totalOnly { + logger.Infof("%s: %d", gaugeResponse.Name, v) + } + overallQPS += v + } + if rpcStatus != io.EOF { + logger.Fatalf("failed to finish server streaming: %v", rpcStatus) + } + logger.Infof("overall qps: %d", overallQPS) +} + +func main() { + flag.Parse() + if *metricsServerAddress == "" { + logger.Fatal("-metrics_server_address is unset") + } + + conn, err := grpc.NewClient(*metricsServerAddress, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + logger.Fatalf("grpc.NewClient(%q) = %v", metricsServerAddress, err) + } + defer conn.Close() + + c := metricspb.NewMetricsServiceClient(conn) + printMetrics(c, *totalOnly) +} diff --git a/interop/test_utils.go b/interop/test_utils.go new file mode 100644 index 0000000..3aa7bd3 --- /dev/null +++ b/interop/test_utils.go @@ -0,0 +1,990 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package interop contains functions used by interop client/server. +// +// See interop test case descriptions [here]. +// +// [here]: https://github.com/grpc/grpc/blob/master/doc/interop-test-descriptions.md +package interop + +import ( + "context" + "fmt" + "io" + "os" + "strings" + "sync" + "time" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + reqSizes = []int{27182, 8, 1828, 45904} + respSizes = []int{31415, 9, 2653, 58979} + largeReqSize = 271828 + largeRespSize = 314159 + initialMetadataKey = "x-grpc-test-echo-initial" + trailingMetadataKey = "x-grpc-test-echo-trailing-bin" + + logger = grpclog.Component("interop") +) + +// ClientNewPayload returns a payload of the given type and size. +func ClientNewPayload(t testpb.PayloadType, size int) *testpb.Payload { + if size < 0 { + logger.Fatalf("Requested a response with invalid length %d", size) + } + body := make([]byte, size) + switch t { + case testpb.PayloadType_COMPRESSABLE: + default: + logger.Fatalf("Unsupported payload type: %d", t) + } + return &testpb.Payload{ + Type: t, + Body: body, + } +} + +// DoEmptyUnaryCall performs a unary RPC with empty request and response messages. +func DoEmptyUnaryCall(ctx context.Context, tc testgrpc.TestServiceClient, args ...grpc.CallOption) { + reply, err := tc.EmptyCall(ctx, &testpb.Empty{}, args...) + if err != nil { + logger.Fatal("/TestService/EmptyCall RPC failed: ", err) + } + if !proto.Equal(&testpb.Empty{}, reply) { + logger.Fatalf("/TestService/EmptyCall receives %v, want %v", reply, testpb.Empty{}) + } +} + +// DoLargeUnaryCall performs a unary RPC with large payload in the request and response. +func DoLargeUnaryCall(ctx context.Context, tc testgrpc.TestServiceClient, args ...grpc.CallOption) { + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(largeRespSize), + Payload: pl, + } + reply, err := tc.UnaryCall(ctx, req, args...) + if err != nil { + logger.Fatal("/TestService/UnaryCall RPC failed: ", err) + } + t := reply.GetPayload().GetType() + s := len(reply.GetPayload().GetBody()) + if t != testpb.PayloadType_COMPRESSABLE || s != largeRespSize { + logger.Fatalf("Got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, largeRespSize) + } +} + +// DoClientStreaming performs a client streaming RPC. +func DoClientStreaming(ctx context.Context, tc testgrpc.TestServiceClient, args ...grpc.CallOption) { + stream, err := tc.StreamingInputCall(ctx, args...) + if err != nil { + logger.Fatalf("%v.StreamingInputCall(_) = _, %v", tc, err) + } + var sum int + for _, s := range reqSizes { + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, s) + req := &testpb.StreamingInputCallRequest{ + Payload: pl, + } + if err := stream.Send(req); err != nil { + logger.Fatalf("%v has error %v while sending %v", stream, err, req) + } + sum += s + } + reply, err := stream.CloseAndRecv() + if err != nil { + logger.Fatalf("%v.CloseAndRecv() got error %v, want %v", stream, err, nil) + } + if reply.GetAggregatedPayloadSize() != int32(sum) { + logger.Fatalf("%v.CloseAndRecv().GetAggregatePayloadSize() = %v; want %v", stream, reply.GetAggregatedPayloadSize(), sum) + } +} + +// DoServerStreaming performs a server streaming RPC. +func DoServerStreaming(ctx context.Context, tc testgrpc.TestServiceClient, args ...grpc.CallOption) { + respParam := make([]*testpb.ResponseParameters, len(respSizes)) + for i, s := range respSizes { + respParam[i] = &testpb.ResponseParameters{ + Size: int32(s), + } + } + req := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + } + stream, err := tc.StreamingOutputCall(ctx, req, args...) + if err != nil { + logger.Fatalf("%v.StreamingOutputCall(_) = _, %v", tc, err) + } + var rpcStatus error + var respCnt int + var index int + for { + reply, err := stream.Recv() + if err != nil { + rpcStatus = err + break + } + t := reply.GetPayload().GetType() + if t != testpb.PayloadType_COMPRESSABLE { + logger.Fatalf("Got the reply of type %d, want %d", t, testpb.PayloadType_COMPRESSABLE) + } + size := len(reply.GetPayload().GetBody()) + if size != respSizes[index] { + logger.Fatalf("Got reply body of length %d, want %d", size, respSizes[index]) + } + index++ + respCnt++ + } + if rpcStatus != io.EOF { + logger.Fatalf("Failed to finish the server streaming rpc: %v", rpcStatus) + } + if respCnt != len(respSizes) { + logger.Fatalf("Got %d reply, want %d", len(respSizes), respCnt) + } +} + +// DoPingPong performs ping-pong style bi-directional streaming RPC. +func DoPingPong(ctx context.Context, tc testgrpc.TestServiceClient, args ...grpc.CallOption) { + stream, err := tc.FullDuplexCall(ctx, args...) + if err != nil { + logger.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err) + } + var index int + for index < len(reqSizes) { + respParam := []*testpb.ResponseParameters{ + { + Size: int32(respSizes[index]), + }, + } + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, reqSizes[index]) + req := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: pl, + } + if err := stream.Send(req); err != nil { + logger.Fatalf("%v has error %v while sending %v", stream, err, req) + } + reply, err := stream.Recv() + if err != nil { + logger.Fatalf("%v.Recv() = %v", stream, err) + } + t := reply.GetPayload().GetType() + if t != testpb.PayloadType_COMPRESSABLE { + logger.Fatalf("Got the reply of type %d, want %d", t, testpb.PayloadType_COMPRESSABLE) + } + size := len(reply.GetPayload().GetBody()) + if size != respSizes[index] { + logger.Fatalf("Got reply body of length %d, want %d", size, respSizes[index]) + } + index++ + } + if err := stream.CloseSend(); err != nil { + logger.Fatalf("%v.CloseSend() got %v, want %v", stream, err, nil) + } + if _, err := stream.Recv(); err != io.EOF { + logger.Fatalf("%v failed to complele the ping pong test: %v", stream, err) + } +} + +// DoEmptyStream sets up a bi-directional streaming with zero message. +func DoEmptyStream(ctx context.Context, tc testgrpc.TestServiceClient, args ...grpc.CallOption) { + stream, err := tc.FullDuplexCall(ctx, args...) + if err != nil { + logger.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err) + } + if err := stream.CloseSend(); err != nil { + logger.Fatalf("%v.CloseSend() got %v, want %v", stream, err, nil) + } + if _, err := stream.Recv(); err != io.EOF { + logger.Fatalf("%v failed to complete the empty stream test: %v", stream, err) + } +} + +// DoTimeoutOnSleepingServer performs an RPC on a sleep server which causes RPC timeout. +func DoTimeoutOnSleepingServer(ctx context.Context, tc testgrpc.TestServiceClient, args ...grpc.CallOption) { + ctx, cancel := context.WithTimeout(ctx, 1*time.Millisecond) + defer cancel() + stream, err := tc.FullDuplexCall(ctx, args...) + if err != nil { + if status.Code(err) == codes.DeadlineExceeded { + return + } + logger.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err) + } + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, 27182) + req := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + Payload: pl, + } + if err := stream.Send(req); err != nil && err != io.EOF { + logger.Fatalf("%v.Send(_) = %v", stream, err) + } + if _, err := stream.Recv(); status.Code(err) != codes.DeadlineExceeded { + logger.Fatalf("%v.Recv() = _, %v, want error code %d", stream, err, codes.DeadlineExceeded) + } +} + +// DoComputeEngineCreds performs a unary RPC with compute engine auth. +func DoComputeEngineCreds(ctx context.Context, tc testgrpc.TestServiceClient, serviceAccount, oauthScope string) { + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(largeRespSize), + Payload: pl, + FillUsername: true, + FillOauthScope: true, + } + reply, err := tc.UnaryCall(ctx, req) + if err != nil { + logger.Fatal("/TestService/UnaryCall RPC failed: ", err) + } + user := reply.GetUsername() + scope := reply.GetOauthScope() + if user != serviceAccount { + logger.Fatalf("Got user name %q, want %q.", user, serviceAccount) + } + if !strings.Contains(oauthScope, scope) { + logger.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope) + } +} + +func getServiceAccountJSONKey(keyFile string) []byte { + jsonKey, err := os.ReadFile(keyFile) + if err != nil { + logger.Fatalf("Failed to read the service account key file: %v", err) + } + return jsonKey +} + +// DoServiceAccountCreds performs a unary RPC with service account auth. +func DoServiceAccountCreds(ctx context.Context, tc testgrpc.TestServiceClient, serviceAccountKeyFile, oauthScope string) { + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(largeRespSize), + Payload: pl, + FillUsername: true, + FillOauthScope: true, + } + reply, err := tc.UnaryCall(ctx, req) + if err != nil { + logger.Fatal("/TestService/UnaryCall RPC failed: ", err) + } + jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile) + user := reply.GetUsername() + scope := reply.GetOauthScope() + if !strings.Contains(string(jsonKey), user) { + logger.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey) + } + if !strings.Contains(oauthScope, scope) { + logger.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope) + } +} + +// DoJWTTokenCreds performs a unary RPC with JWT token auth. +func DoJWTTokenCreds(ctx context.Context, tc testgrpc.TestServiceClient, serviceAccountKeyFile string) { + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(largeRespSize), + Payload: pl, + FillUsername: true, + } + reply, err := tc.UnaryCall(ctx, req) + if err != nil { + logger.Fatal("/TestService/UnaryCall RPC failed: ", err) + } + jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile) + user := reply.GetUsername() + if !strings.Contains(string(jsonKey), user) { + logger.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey) + } +} + +// GetToken obtains an OAUTH token from the input. +func GetToken(ctx context.Context, serviceAccountKeyFile string, oauthScope string) *oauth2.Token { + jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile) + config, err := google.JWTConfigFromJSON(jsonKey, oauthScope) + if err != nil { + logger.Fatalf("Failed to get the config: %v", err) + } + token, err := config.TokenSource(ctx).Token() + if err != nil { + logger.Fatalf("Failed to get the token: %v", err) + } + return token +} + +// DoOauth2TokenCreds performs a unary RPC with OAUTH2 token auth. +func DoOauth2TokenCreds(ctx context.Context, tc testgrpc.TestServiceClient, serviceAccountKeyFile, oauthScope string) { + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(largeRespSize), + Payload: pl, + FillUsername: true, + FillOauthScope: true, + } + reply, err := tc.UnaryCall(ctx, req) + if err != nil { + logger.Fatal("/TestService/UnaryCall RPC failed: ", err) + } + jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile) + user := reply.GetUsername() + scope := reply.GetOauthScope() + if !strings.Contains(string(jsonKey), user) { + logger.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey) + } + if !strings.Contains(oauthScope, scope) { + logger.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope) + } +} + +// DoPerRPCCreds performs a unary RPC with per RPC OAUTH2 token. +func DoPerRPCCreds(ctx context.Context, tc testgrpc.TestServiceClient, serviceAccountKeyFile, oauthScope string) { + jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile) + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(largeRespSize), + Payload: pl, + FillUsername: true, + FillOauthScope: true, + } + token := GetToken(ctx, serviceAccountKeyFile, oauthScope) + kv := map[string]string{"authorization": token.Type() + " " + token.AccessToken} + ctx = metadata.NewOutgoingContext(ctx, metadata.MD{"authorization": []string{kv["authorization"]}}) + reply, err := tc.UnaryCall(ctx, req) + if err != nil { + logger.Fatal("/TestService/UnaryCall RPC failed: ", err) + } + user := reply.GetUsername() + scope := reply.GetOauthScope() + if !strings.Contains(string(jsonKey), user) { + logger.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey) + } + if !strings.Contains(oauthScope, scope) { + logger.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope) + } +} + +// DoGoogleDefaultCredentials performs a unary RPC with google default credentials +func DoGoogleDefaultCredentials(ctx context.Context, tc testgrpc.TestServiceClient, defaultServiceAccount string) { + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(largeRespSize), + Payload: pl, + FillUsername: true, + FillOauthScope: true, + } + reply, err := tc.UnaryCall(ctx, req) + if err != nil { + logger.Fatal("/TestService/UnaryCall RPC failed: ", err) + } + if reply.GetUsername() != defaultServiceAccount { + logger.Fatalf("Got user name %q; wanted %q. ", reply.GetUsername(), defaultServiceAccount) + } +} + +// DoComputeEngineChannelCredentials performs a unary RPC with compute engine channel credentials +func DoComputeEngineChannelCredentials(ctx context.Context, tc testgrpc.TestServiceClient, defaultServiceAccount string) { + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(largeRespSize), + Payload: pl, + FillUsername: true, + FillOauthScope: true, + } + reply, err := tc.UnaryCall(ctx, req) + if err != nil { + logger.Fatal("/TestService/UnaryCall RPC failed: ", err) + } + if reply.GetUsername() != defaultServiceAccount { + logger.Fatalf("Got user name %q; wanted %q. ", reply.GetUsername(), defaultServiceAccount) + } +} + +var testMetadata = metadata.MD{ + "key1": []string{"value1"}, + "key2": []string{"value2"}, +} + +// DoCancelAfterBegin cancels the RPC after metadata has been sent but before payloads are sent. +func DoCancelAfterBegin(ctx context.Context, tc testgrpc.TestServiceClient, args ...grpc.CallOption) { + ctx, cancel := context.WithCancel(metadata.NewOutgoingContext(ctx, testMetadata)) + stream, err := tc.StreamingInputCall(ctx, args...) + if err != nil { + logger.Fatalf("%v.StreamingInputCall(_) = _, %v", tc, err) + } + cancel() + _, err = stream.CloseAndRecv() + if status.Code(err) != codes.Canceled { + logger.Fatalf("%v.CloseAndRecv() got error code %d, want %d", stream, status.Code(err), codes.Canceled) + } +} + +// DoCancelAfterFirstResponse cancels the RPC after receiving the first message from the server. +func DoCancelAfterFirstResponse(ctx context.Context, tc testgrpc.TestServiceClient, args ...grpc.CallOption) { + ctx, cancel := context.WithCancel(ctx) + stream, err := tc.FullDuplexCall(ctx, args...) + if err != nil { + logger.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err) + } + respParam := []*testpb.ResponseParameters{ + { + Size: 31415, + }, + } + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, 27182) + req := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: pl, + } + if err := stream.Send(req); err != nil { + logger.Fatalf("%v has error %v while sending %v", stream, err, req) + } + if _, err := stream.Recv(); err != nil { + logger.Fatalf("%v.Recv() = %v", stream, err) + } + cancel() + if _, err := stream.Recv(); status.Code(err) != codes.Canceled { + logger.Fatalf("%v compleled with error code %d, want %d", stream, status.Code(err), codes.Canceled) + } +} + +var ( + initialMetadataValue = "test_initial_metadata_value" + trailingMetadataValue = "\x0a\x0b\x0a\x0b\x0a\x0b" + customMetadata = metadata.Pairs( + initialMetadataKey, initialMetadataValue, + trailingMetadataKey, trailingMetadataValue, + ) +) + +func validateMetadata(header, trailer metadata.MD) { + if len(header[initialMetadataKey]) != 1 { + logger.Fatalf("Expected exactly one header from server. Received %d", len(header[initialMetadataKey])) + } + if header[initialMetadataKey][0] != initialMetadataValue { + logger.Fatalf("Got header %s; want %s", header[initialMetadataKey][0], initialMetadataValue) + } + if len(trailer[trailingMetadataKey]) != 1 { + logger.Fatalf("Expected exactly one trailer from server. Received %d", len(trailer[trailingMetadataKey])) + } + if trailer[trailingMetadataKey][0] != trailingMetadataValue { + logger.Fatalf("Got trailer %s; want %s", trailer[trailingMetadataKey][0], trailingMetadataValue) + } +} + +// DoCustomMetadata checks that metadata is echoed back to the client. +func DoCustomMetadata(ctx context.Context, tc testgrpc.TestServiceClient, args ...grpc.CallOption) { + // Testing with UnaryCall. + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, 1) + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(1), + Payload: pl, + } + ctx = metadata.NewOutgoingContext(ctx, customMetadata) + var header, trailer metadata.MD + args = append(args, grpc.Header(&header), grpc.Trailer(&trailer)) + reply, err := tc.UnaryCall( + ctx, + req, + args..., + ) + if err != nil { + logger.Fatal("/TestService/UnaryCall RPC failed: ", err) + } + t := reply.GetPayload().GetType() + s := len(reply.GetPayload().GetBody()) + if t != testpb.PayloadType_COMPRESSABLE || s != 1 { + logger.Fatalf("Got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, 1) + } + validateMetadata(header, trailer) + + // Testing with FullDuplex. + stream, err := tc.FullDuplexCall(ctx, args...) + if err != nil { + logger.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + respParam := []*testpb.ResponseParameters{ + { + Size: 1, + }, + } + streamReq := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: pl, + } + if err := stream.Send(streamReq); err != nil { + logger.Fatalf("%v has error %v while sending %v", stream, err, streamReq) + } + streamHeader, err := stream.Header() + if err != nil { + logger.Fatalf("%v.Header() = %v", stream, err) + } + if _, err := stream.Recv(); err != nil { + logger.Fatalf("%v.Recv() = %v", stream, err) + } + if err := stream.CloseSend(); err != nil { + logger.Fatalf("%v.CloseSend() = %v, want ", stream, err) + } + if _, err := stream.Recv(); err != io.EOF { + logger.Fatalf("%v failed to complete the custom metadata test: %v", stream, err) + } + streamTrailer := stream.Trailer() + validateMetadata(streamHeader, streamTrailer) +} + +// DoStatusCodeAndMessage checks that the status code is propagated back to the client. +func DoStatusCodeAndMessage(ctx context.Context, tc testgrpc.TestServiceClient, args ...grpc.CallOption) { + var code int32 = 2 + msg := "test status message" + expectedErr := status.Error(codes.Code(code), msg) + respStatus := &testpb.EchoStatus{ + Code: code, + Message: msg, + } + // Test UnaryCall. + req := &testpb.SimpleRequest{ + ResponseStatus: respStatus, + } + if _, err := tc.UnaryCall(ctx, req, args...); err == nil || err.Error() != expectedErr.Error() { + logger.Fatalf("%v.UnaryCall(_, %v) = _, %v, want _, %v", tc, req, err, expectedErr) + } + // Test FullDuplexCall. + stream, err := tc.FullDuplexCall(ctx, args...) + if err != nil { + logger.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + streamReq := &testpb.StreamingOutputCallRequest{ + ResponseStatus: respStatus, + } + if err := stream.Send(streamReq); err != nil { + logger.Fatalf("%v has error %v while sending %v, want ", stream, err, streamReq) + } + if err := stream.CloseSend(); err != nil { + logger.Fatalf("%v.CloseSend() = %v, want ", stream, err) + } + if _, err = stream.Recv(); err.Error() != expectedErr.Error() { + logger.Fatalf("%v.Recv() returned error %v, want %v", stream, err, expectedErr) + } +} + +// DoSpecialStatusMessage verifies Unicode and whitespace is correctly processed +// in status message. +func DoSpecialStatusMessage(ctx context.Context, tc testgrpc.TestServiceClient, args ...grpc.CallOption) { + const ( + code int32 = 2 + msg string = "\t\ntest with whitespace\r\nand Unicode BMP ☺ and non-BMP 😈\t\n" + ) + expectedErr := status.Error(codes.Code(code), msg) + req := &testpb.SimpleRequest{ + ResponseStatus: &testpb.EchoStatus{ + Code: code, + Message: msg, + }, + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if _, err := tc.UnaryCall(ctx, req, args...); err == nil || err.Error() != expectedErr.Error() { + logger.Fatalf("%v.UnaryCall(_, %v) = _, %v, want _, %v", tc, req, err, expectedErr) + } +} + +// DoUnimplementedService attempts to call a method from an unimplemented service. +func DoUnimplementedService(ctx context.Context, tc testgrpc.UnimplementedServiceClient) { + _, err := tc.UnimplementedCall(ctx, &testpb.Empty{}) + if status.Code(err) != codes.Unimplemented { + logger.Fatalf("%v.UnimplementedCall() = _, %v, want _, %v", tc, status.Code(err), codes.Unimplemented) + } +} + +// DoUnimplementedMethod attempts to call an unimplemented method. +func DoUnimplementedMethod(ctx context.Context, cc *grpc.ClientConn) { + var req, reply proto.Message + if err := cc.Invoke(ctx, "/grpc.testing.TestService/UnimplementedCall", req, reply); err == nil || status.Code(err) != codes.Unimplemented { + logger.Fatalf("ClientConn.Invoke(_, _, _, _, _) = %v, want error code %s", err, codes.Unimplemented) + } +} + +// DoPickFirstUnary runs multiple RPCs (rpcCount) and checks that all requests +// are sent to the same backend. +func DoPickFirstUnary(ctx context.Context, tc testgrpc.TestServiceClient) { + const rpcCount = 100 + + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, 1) + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(1), + Payload: pl, + FillServerId: true, + } + // TODO(mohanli): Revert the timeout back to 10s once TD migrates to xdstp. + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + var serverID string + for i := 0; i < rpcCount; i++ { + resp, err := tc.UnaryCall(ctx, req) + if err != nil { + logger.Fatalf("iteration %d, failed to do UnaryCall: %v", i, err) + } + id := resp.ServerId + if id == "" { + logger.Fatalf("iteration %d, got empty server ID", i) + } + if i == 0 { + serverID = id + continue + } + if serverID != id { + logger.Fatalf("iteration %d, got different server ids: %q vs %q", i, serverID, id) + } + } +} + +type testServer struct { + testgrpc.UnimplementedTestServiceServer + + orcaMu sync.Mutex + metricsRecorder orca.ServerMetricsRecorder +} + +// NewTestServerOptions contains options that control the behavior of the test +// server returned by NewTestServer. +type NewTestServerOptions struct { + MetricsRecorder orca.ServerMetricsRecorder +} + +// NewTestServer creates a test server for test service. opts carries optional +// settings and does not need to be provided. If multiple opts are provided, +// only the first one is used. +func NewTestServer(opts ...NewTestServerOptions) testgrpc.TestServiceServer { + if len(opts) > 0 { + return &testServer{metricsRecorder: opts[0].MetricsRecorder} + } + return &testServer{} +} + +func (s *testServer) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return new(testpb.Empty), nil +} + +func serverNewPayload(t testpb.PayloadType, size int32) (*testpb.Payload, error) { + if size < 0 { + return nil, fmt.Errorf("requested a response with invalid length %d", size) + } + body := make([]byte, size) + switch t { + case testpb.PayloadType_COMPRESSABLE: + default: + return nil, fmt.Errorf("unsupported payload type: %d", t) + } + return &testpb.Payload{ + Type: t, + Body: body, + }, nil +} + +func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + st := in.GetResponseStatus() + if md, ok := metadata.FromIncomingContext(ctx); ok { + if initialMetadata, ok := md[initialMetadataKey]; ok { + header := metadata.Pairs(initialMetadataKey, initialMetadata[0]) + grpc.SendHeader(ctx, header) + } + if trailingMetadata, ok := md[trailingMetadataKey]; ok { + trailer := metadata.Pairs(trailingMetadataKey, trailingMetadata[0]) + grpc.SetTrailer(ctx, trailer) + } + } + if st != nil && st.Code != 0 { + return nil, status.Error(codes.Code(st.Code), st.Message) + } + pl, err := serverNewPayload(in.GetResponseType(), in.GetResponseSize()) + if err != nil { + return nil, err + } + if r, orcaData := orca.CallMetricsRecorderFromContext(ctx), in.GetOrcaPerQueryReport(); r != nil && orcaData != nil { + // Transfer the request's per-Call ORCA data to the call metrics + // recorder in the context, if present. + setORCAMetrics(r, orcaData) + } + return &testpb.SimpleResponse{ + Payload: pl, + }, nil +} + +func setORCAMetrics(r orca.ServerMetricsRecorder, orcaData *testpb.TestOrcaReport) { + r.SetCPUUtilization(orcaData.CpuUtilization) + r.SetMemoryUtilization(orcaData.MemoryUtilization) + if rq, ok := r.(orca.CallMetricsRecorder); ok { + for k, v := range orcaData.RequestCost { + rq.SetRequestCost(k, v) + } + } + for k, v := range orcaData.Utilization { + r.SetNamedUtilization(k, v) + } +} + +func (s *testServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error { + cs := args.GetResponseParameters() + for _, c := range cs { + if us := c.GetIntervalUs(); us > 0 { + time.Sleep(time.Duration(us) * time.Microsecond) + } + pl, err := serverNewPayload(args.GetResponseType(), c.GetSize()) + if err != nil { + return err + } + if err := stream.Send(&testpb.StreamingOutputCallResponse{ + Payload: pl, + }); err != nil { + return err + } + } + return nil +} + +func (s *testServer) StreamingInputCall(stream testgrpc.TestService_StreamingInputCallServer) error { + var sum int + for { + in, err := stream.Recv() + if err == io.EOF { + return stream.SendAndClose(&testpb.StreamingInputCallResponse{ + AggregatedPayloadSize: int32(sum), + }) + } + if err != nil { + return err + } + p := in.GetPayload().GetBody() + sum += len(p) + } +} + +func (s *testServer) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error { + if md, ok := metadata.FromIncomingContext(stream.Context()); ok { + if initialMetadata, ok := md[initialMetadataKey]; ok { + header := metadata.Pairs(initialMetadataKey, initialMetadata[0]) + stream.SendHeader(header) + } + if trailingMetadata, ok := md[trailingMetadataKey]; ok { + trailer := metadata.Pairs(trailingMetadataKey, trailingMetadata[0]) + stream.SetTrailer(trailer) + } + } + hasORCALock := false + for { + in, err := stream.Recv() + if err == io.EOF { + // read done. + return nil + } + if err != nil { + return err + } + st := in.GetResponseStatus() + if st != nil && st.Code != 0 { + return status.Error(codes.Code(st.Code), st.Message) + } + + if r, orcaData := s.metricsRecorder, in.GetOrcaOobReport(); r != nil && orcaData != nil { + // Transfer the request's OOB ORCA data to the server metrics recorder + // in the server, if present. + if !hasORCALock { + s.orcaMu.Lock() + defer s.orcaMu.Unlock() + hasORCALock = true + } + setORCAMetrics(r, orcaData) + } + + cs := in.GetResponseParameters() + for _, c := range cs { + if us := c.GetIntervalUs(); us > 0 { + time.Sleep(time.Duration(us) * time.Microsecond) + } + pl, err := serverNewPayload(in.GetResponseType(), c.GetSize()) + if err != nil { + return err + } + if err := stream.Send(&testpb.StreamingOutputCallResponse{ + Payload: pl, + }); err != nil { + return err + } + } + } +} + +func (s *testServer) HalfDuplexCall(stream testgrpc.TestService_HalfDuplexCallServer) error { + var msgBuf []*testpb.StreamingOutputCallRequest + for { + in, err := stream.Recv() + if err == io.EOF { + // read done. + break + } + if err != nil { + return err + } + msgBuf = append(msgBuf, in) + } + for _, m := range msgBuf { + cs := m.GetResponseParameters() + for _, c := range cs { + if us := c.GetIntervalUs(); us > 0 { + time.Sleep(time.Duration(us) * time.Microsecond) + } + pl, err := serverNewPayload(m.GetResponseType(), c.GetSize()) + if err != nil { + return err + } + if err := stream.Send(&testpb.StreamingOutputCallResponse{ + Payload: pl, + }); err != nil { + return err + } + } + } + return nil +} + +// DoORCAPerRPCTest performs a unary RPC that enables ORCA per-call reporting +// and verifies the load report sent back to the LB policy's Done callback. +func DoORCAPerRPCTest(ctx context.Context, tc testgrpc.TestServiceClient) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + orcaRes := &v3orcapb.OrcaLoadReport{} + _, err := tc.UnaryCall(contextWithORCAResult(ctx, &orcaRes), &testpb.SimpleRequest{ + OrcaPerQueryReport: &testpb.TestOrcaReport{ + CpuUtilization: 0.8210, + MemoryUtilization: 0.5847, + RequestCost: map[string]float64{"cost": 3456.32}, + Utilization: map[string]float64{"util": 0.30499}, + }, + }) + if err != nil { + logger.Fatalf("/TestService/UnaryCall RPC failed: ", err) + } + want := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 0.8210, + MemUtilization: 0.5847, + RequestCost: map[string]float64{"cost": 3456.32}, + Utilization: map[string]float64{"util": 0.30499}, + } + if !proto.Equal(orcaRes, want) { + logger.Fatalf("/TestService/UnaryCall RPC received ORCA load report %+v; want %+v", orcaRes, want) + } +} + +// DoORCAOOBTest performs a streaming RPC that enables ORCA OOB reporting and +// verifies the load report sent to the LB policy's OOB listener. +func DoORCAOOBTest(ctx context.Context, tc testgrpc.TestServiceClient) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + logger.Fatalf("/TestService/FullDuplexCall received error starting stream: %v", err) + } + err = stream.Send(&testpb.StreamingOutputCallRequest{ + OrcaOobReport: &testpb.TestOrcaReport{ + CpuUtilization: 0.8210, + MemoryUtilization: 0.5847, + Utilization: map[string]float64{"util": 0.30499}, + }, + ResponseParameters: []*testpb.ResponseParameters{{Size: 1}}, + }) + if err != nil { + logger.Fatalf("/TestService/FullDuplexCall received error sending: %v", err) + } + _, err = stream.Recv() + if err != nil { + logger.Fatalf("/TestService/FullDuplexCall received error receiving: %v", err) + } + + want := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 0.8210, + MemUtilization: 0.5847, + Utilization: map[string]float64{"util": 0.30499}, + } + checkORCAMetrics(ctx, tc, want) + + err = stream.Send(&testpb.StreamingOutputCallRequest{ + OrcaOobReport: &testpb.TestOrcaReport{ + CpuUtilization: 0.29309, + MemoryUtilization: 0.2, + Utilization: map[string]float64{"util": 0.2039}, + }, + ResponseParameters: []*testpb.ResponseParameters{{Size: 1}}, + }) + if err != nil { + logger.Fatalf("/TestService/FullDuplexCall received error sending: %v", err) + } + _, err = stream.Recv() + if err != nil { + logger.Fatalf("/TestService/FullDuplexCall received error receiving: %v", err) + } + + want = &v3orcapb.OrcaLoadReport{ + CpuUtilization: 0.29309, + MemUtilization: 0.2, + Utilization: map[string]float64{"util": 0.2039}, + } + checkORCAMetrics(ctx, tc, want) +} + +func checkORCAMetrics(ctx context.Context, tc testgrpc.TestServiceClient, want *v3orcapb.OrcaLoadReport) { + for ctx.Err() == nil { + orcaRes := &v3orcapb.OrcaLoadReport{} + if _, err := tc.UnaryCall(contextWithORCAResult(ctx, &orcaRes), &testpb.SimpleRequest{}); err != nil { + logger.Fatalf("/TestService/UnaryCall RPC failed: ", err) + } + if proto.Equal(orcaRes, want) { + return + } + logger.Infof("/TestService/UnaryCall RPC received ORCA load report %+v; want %+v", orcaRes, want) + time.Sleep(time.Second) + } + logger.Fatalf("timed out waiting for expected ORCA load report") +} diff --git a/interop/xds/client/Dockerfile b/interop/xds/client/Dockerfile new file mode 100644 index 0000000..f170ab3 --- /dev/null +++ b/interop/xds/client/Dockerfile @@ -0,0 +1,37 @@ +# Copyright 2021 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for building the xDS interop client. To build the image, run the +# following command from grpc-go directory: +# docker build -t -f interop/xds/client/Dockerfile . + +FROM golang:1.25-alpine as build + +# Make a grpc-go directory and copy the repo into it. +WORKDIR /go/src/grpc-go +COPY . . + +# Build a static binary without cgo so that we can copy just the binary in the +# final image, and can get rid of Go compiler and gRPC-Go dependencies. +RUN cd interop/xds/client && go build -tags osusergo,netgo . + +# Second stage of the build which copies over only the client binary and skips +# the Go compiler and gRPC repo from the earlier stage. This significantly +# reduces the docker image size. +FROM alpine +COPY --from=build /go/src/grpc-go/interop/xds/client/client . +ENV GRPC_GO_LOG_VERBOSITY_LEVEL=99 +ENV GRPC_GO_LOG_SEVERITY_LEVEL="info" +ENV GRPC_GO_LOG_FORMATTER="json" +ENTRYPOINT ["./client"] diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go new file mode 100644 index 0000000..d3ab7eb --- /dev/null +++ b/interop/xds/client/client.go @@ -0,0 +1,554 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client for xDS interop tests. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "net/http" + "os" + "strings" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/admin" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/grpclog" + _ "google.golang.org/grpc/interop/xds" // to register Custom LB. + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/reflection" + "google.golang.org/grpc/stats/opentelemetry" + "google.golang.org/grpc/stats/opentelemetry/csm" + "google.golang.org/grpc/status" + _ "google.golang.org/grpc/xds" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/sdk/metric" +) + +func init() { + rpcCfgs.Store([]*rpcConfig{{typ: unaryCall}}) +} + +type statsWatcherKey struct { + startID int32 + endID int32 +} + +// rpcInfo contains the rpc type and the hostname where the response is received +// from. +type rpcInfo struct { + typ string + hostname string +} + +type statsWatcher struct { + rpcsByPeer map[string]int32 + rpcsByType map[string]map[string]int32 + numFailures int32 + remainingRPCs int32 + chanHosts chan *rpcInfo +} + +func (watcher *statsWatcher) buildResp() *testpb.LoadBalancerStatsResponse { + rpcsByType := make(map[string]*testpb.LoadBalancerStatsResponse_RpcsByPeer, len(watcher.rpcsByType)) + for t, rpcsByPeer := range watcher.rpcsByType { + rpcsByType[t] = &testpb.LoadBalancerStatsResponse_RpcsByPeer{ + RpcsByPeer: rpcsByPeer, + } + } + + return &testpb.LoadBalancerStatsResponse{ + NumFailures: watcher.numFailures + watcher.remainingRPCs, + RpcsByPeer: watcher.rpcsByPeer, + RpcsByMethod: rpcsByType, + } +} + +type accumulatedStats struct { + mu sync.Mutex + numRPCsStartedByMethod map[string]int32 + numRPCsSucceededByMethod map[string]int32 + numRPCsFailedByMethod map[string]int32 + rpcStatusByMethod map[string]map[int32]int32 +} + +func convertRPCName(in string) string { + switch in { + case unaryCall: + return testpb.ClientConfigureRequest_UNARY_CALL.String() + case emptyCall: + return testpb.ClientConfigureRequest_EMPTY_CALL.String() + } + logger.Warningf("unrecognized rpc type: %s", in) + return in +} + +// copyStatsMap makes a copy of the map. +func copyStatsMap(originalMap map[string]int32) map[string]int32 { + newMap := make(map[string]int32, len(originalMap)) + for k, v := range originalMap { + newMap[k] = v + } + return newMap +} + +// copyStatsIntMap makes a copy of the map. +func copyStatsIntMap(originalMap map[int32]int32) map[int32]int32 { + newMap := make(map[int32]int32, len(originalMap)) + for k, v := range originalMap { + newMap[k] = v + } + return newMap +} + +func (as *accumulatedStats) makeStatsMap() map[string]*testpb.LoadBalancerAccumulatedStatsResponse_MethodStats { + m := make(map[string]*testpb.LoadBalancerAccumulatedStatsResponse_MethodStats) + for k, v := range as.numRPCsStartedByMethod { + m[k] = &testpb.LoadBalancerAccumulatedStatsResponse_MethodStats{RpcsStarted: v} + } + for k, v := range as.rpcStatusByMethod { + if m[k] == nil { + m[k] = &testpb.LoadBalancerAccumulatedStatsResponse_MethodStats{} + } + m[k].Result = copyStatsIntMap(v) + } + return m +} + +func (as *accumulatedStats) buildResp() *testpb.LoadBalancerAccumulatedStatsResponse { + as.mu.Lock() + defer as.mu.Unlock() + return &testpb.LoadBalancerAccumulatedStatsResponse{ + NumRpcsStartedByMethod: copyStatsMap(as.numRPCsStartedByMethod), + NumRpcsSucceededByMethod: copyStatsMap(as.numRPCsSucceededByMethod), + NumRpcsFailedByMethod: copyStatsMap(as.numRPCsFailedByMethod), + StatsPerMethod: as.makeStatsMap(), + } +} + +func (as *accumulatedStats) startRPC(rpcType string) { + as.mu.Lock() + defer as.mu.Unlock() + as.numRPCsStartedByMethod[convertRPCName(rpcType)]++ +} + +func (as *accumulatedStats) finishRPC(rpcType string, err error) { + as.mu.Lock() + defer as.mu.Unlock() + name := convertRPCName(rpcType) + if as.rpcStatusByMethod[name] == nil { + as.rpcStatusByMethod[name] = make(map[int32]int32) + } + as.rpcStatusByMethod[name][int32(status.Convert(err).Code())]++ + if err != nil { + as.numRPCsFailedByMethod[name]++ + return + } + as.numRPCsSucceededByMethod[name]++ +} + +var ( + failOnFailedRPC = flag.Bool("fail_on_failed_rpc", false, "Fail client if any RPCs fail after first success") + numChannels = flag.Int("num_channels", 1, "Num of channels") + printResponse = flag.Bool("print_response", false, "Write RPC response to stdout") + qps = flag.Int("qps", 1, "QPS per channel, for each type of RPC") + rpc = flag.String("rpc", "UnaryCall", "Types of RPCs to make, ',' separated string. RPCs can be EmptyCall or UnaryCall. Deprecated: Use Configure RPC to XdsUpdateClientConfigureServiceServer instead.") + rpcMetadata = flag.String("metadata", "", "The metadata to send with RPC, in format EmptyCall:key1:value1,UnaryCall:key2:value2. Deprecated: Use Configure RPC to XdsUpdateClientConfigureServiceServer instead.") + rpcTimeout = flag.Duration("rpc_timeout", 20*time.Second, "Per RPC timeout") + server = flag.String("server", "localhost:8080", "Address of server to connect to") + statsPort = flag.Int("stats_port", 8081, "Port to expose peer distribution stats service") + secureMode = flag.Bool("secure_mode", false, "If true, retrieve security configuration from the management server. Else, use insecure credentials.") + enableCSMObservability = flag.Bool("enable_csm_observability", false, "Whether to enable CSM Observability") + requestPayloadSize = flag.Int("request_payload_size", 0, "Ask the server to respond with SimpleResponse.payload.body of the given length (may not be implemented on the server).") + responsePayloadSize = flag.Int("response_payload_size", 0, "Ask the server to respond with SimpleResponse.payload.body of the given length (may not be implemented on the server).") + + rpcCfgs atomic.Value + + mu sync.Mutex + currentRequestID int32 + watchers = make(map[statsWatcherKey]*statsWatcher) + + accStats = accumulatedStats{ + numRPCsStartedByMethod: make(map[string]int32), + numRPCsSucceededByMethod: make(map[string]int32), + numRPCsFailedByMethod: make(map[string]int32), + rpcStatusByMethod: make(map[string]map[int32]int32), + } + + // 0 or 1 representing an RPC has succeeded. Use hasRPCSucceeded and + // setRPCSucceeded to access in a safe manner. + rpcSucceeded uint32 + + logger = grpclog.Component("interop") +) + +type statsService struct { + testgrpc.UnimplementedLoadBalancerStatsServiceServer +} + +func hasRPCSucceeded() bool { + return atomic.LoadUint32(&rpcSucceeded) > 0 +} + +func setRPCSucceeded() { + atomic.StoreUint32(&rpcSucceeded, 1) +} + +// Wait for the next LoadBalancerStatsRequest.GetNumRpcs to start and complete, +// and return the distribution of remote peers. This is essentially a clientside +// LB reporting mechanism that is designed to be queried by an external test +// driver when verifying that the client is distributing RPCs as expected. +func (s *statsService) GetClientStats(ctx context.Context, in *testpb.LoadBalancerStatsRequest) (*testpb.LoadBalancerStatsResponse, error) { + mu.Lock() + watcherKey := statsWatcherKey{currentRequestID, currentRequestID + in.GetNumRpcs()} + watcher, ok := watchers[watcherKey] + if !ok { + watcher = &statsWatcher{ + rpcsByPeer: make(map[string]int32), + rpcsByType: make(map[string]map[string]int32), + numFailures: 0, + remainingRPCs: in.GetNumRpcs(), + chanHosts: make(chan *rpcInfo), + } + watchers[watcherKey] = watcher + } + mu.Unlock() + + ctx, cancel := context.WithTimeout(ctx, time.Duration(in.GetTimeoutSec())*time.Second) + defer cancel() + + defer func() { + mu.Lock() + delete(watchers, watcherKey) + mu.Unlock() + }() + + // Wait until the requested RPCs have all been recorded or timeout occurs. + for { + select { + case info := <-watcher.chanHosts: + if info != nil { + watcher.rpcsByPeer[info.hostname]++ + + rpcsByPeerForType := watcher.rpcsByType[info.typ] + if rpcsByPeerForType == nil { + rpcsByPeerForType = make(map[string]int32) + watcher.rpcsByType[info.typ] = rpcsByPeerForType + } + rpcsByPeerForType[info.hostname]++ + } else { + watcher.numFailures++ + } + watcher.remainingRPCs-- + if watcher.remainingRPCs == 0 { + return watcher.buildResp(), nil + } + case <-ctx.Done(): + logger.Info("Timed out, returning partial stats") + return watcher.buildResp(), nil + } + } +} + +func (s *statsService) GetClientAccumulatedStats(context.Context, *testpb.LoadBalancerAccumulatedStatsRequest) (*testpb.LoadBalancerAccumulatedStatsResponse, error) { + return accStats.buildResp(), nil +} + +type configureService struct { + testgrpc.UnimplementedXdsUpdateClientConfigureServiceServer +} + +func (s *configureService) Configure(_ context.Context, in *testpb.ClientConfigureRequest) (*testpb.ClientConfigureResponse, error) { + rpcsToMD := make(map[testpb.ClientConfigureRequest_RpcType][]string) + for _, typ := range in.GetTypes() { + rpcsToMD[typ] = nil + } + for _, md := range in.GetMetadata() { + typ := md.GetType() + strs, ok := rpcsToMD[typ] + if !ok { + continue + } + rpcsToMD[typ] = append(strs, md.GetKey(), md.GetValue()) + } + cfgs := make([]*rpcConfig, 0, len(rpcsToMD)) + for typ, md := range rpcsToMD { + var rpcType string + switch typ { + case testpb.ClientConfigureRequest_UNARY_CALL: + rpcType = unaryCall + case testpb.ClientConfigureRequest_EMPTY_CALL: + rpcType = emptyCall + default: + return nil, fmt.Errorf("unsupported RPC type: %v", typ) + } + cfgs = append(cfgs, &rpcConfig{ + typ: rpcType, + md: metadata.Pairs(md...), + timeout: in.GetTimeoutSec(), + }) + } + rpcCfgs.Store(cfgs) + return &testpb.ClientConfigureResponse{}, nil +} + +const ( + unaryCall string = "UnaryCall" + emptyCall string = "EmptyCall" +) + +func parseRPCTypes(rpcStr string) []string { + if len(rpcStr) == 0 { + return []string{unaryCall} + } + + rpcs := strings.Split(rpcStr, ",") + ret := make([]string, 0, len(rpcStr)) + for _, r := range rpcs { + switch r { + case unaryCall, emptyCall: + ret = append(ret, r) + default: + flag.PrintDefaults() + log.Fatalf("unsupported RPC type: %v", r) + } + } + return ret +} + +type rpcConfig struct { + typ string + md metadata.MD + timeout int32 +} + +// parseRPCMetadata turns EmptyCall:key1:value1 into +// +// {typ: emptyCall, md: {key1:value1}}. +func parseRPCMetadata(rpcMetadataStr string, rpcs []string) []*rpcConfig { + rpcMetadataSplit := strings.Split(rpcMetadataStr, ",") + rpcsToMD := make(map[string][]string) + for _, rm := range rpcMetadataSplit { + rmSplit := strings.Split(rm, ":") + if len(rmSplit)%2 != 1 { + log.Fatalf("invalid metadata config %v, want EmptyCall:key1:value1", rm) + } + rpcsToMD[rmSplit[0]] = append(rpcsToMD[rmSplit[0]], rmSplit[1:]...) + } + ret := make([]*rpcConfig, 0, len(rpcs)) + for _, rpcT := range rpcs { + rpcC := &rpcConfig{ + typ: rpcT, + } + if md := rpcsToMD[string(rpcT)]; len(md) > 0 { + rpcC.md = metadata.Pairs(md...) + } + ret = append(ret, rpcC) + } + return ret +} + +func main() { + flag.Parse() + if *enableCSMObservability { + exporter, err := prometheus.New() + if err != nil { + logger.Fatalf("Failed to start prometheus exporter: %v", err) + } + provider := metric.NewMeterProvider( + metric.WithReader(exporter), + ) + var addr string + var ok bool + if addr, ok = os.LookupEnv("OTEL_EXPORTER_PROMETHEUS_HOST"); !ok { + addr = "" + } + var port string + if port, ok = os.LookupEnv("OTEL_EXPORTER_PROMETHEUS_PORT"); !ok { + port = "9464" + } + go func() { + if err := http.ListenAndServe(addr+":"+port, promhttp.Handler()); err != nil { + logger.Fatalf("error listening: %v", err) + } + }() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + cleanup := csm.EnableObservability(ctx, opentelemetry.Options{MetricsOptions: opentelemetry.MetricsOptions{MeterProvider: provider}}) + defer cleanup() + } + + rpcCfgs.Store(parseRPCMetadata(*rpcMetadata, parseRPCTypes(*rpc))) + + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *statsPort)) + if err != nil { + logger.Fatalf("failed to listen: %v", err) + } + s := grpc.NewServer() + defer s.Stop() + testgrpc.RegisterLoadBalancerStatsServiceServer(s, &statsService{}) + testgrpc.RegisterXdsUpdateClientConfigureServiceServer(s, &configureService{}) + reflection.Register(s) + cleanup, err := admin.Register(s) + if err != nil { + logger.Fatalf("Failed to register admin: %v", err) + } + defer cleanup() + go s.Serve(lis) + + creds := insecure.NewCredentials() + if *secureMode { + var err error + creds, err = xds.NewClientCredentials(xds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + logger.Fatalf("Failed to create xDS credentials: %v", err) + } + } + + clients := make([]testgrpc.TestServiceClient, *numChannels) + for i := 0; i < *numChannels; i++ { + conn, err := grpc.NewClient(*server, grpc.WithTransportCredentials(creds)) + if err != nil { + logger.Fatalf("grpc.NewClient(%q) = %v", *server, err) + } + defer conn.Close() + clients[i] = testgrpc.NewTestServiceClient(conn) + } + ticker := time.NewTicker(time.Second / time.Duration(*qps**numChannels)) + defer ticker.Stop() + sendRPCs(clients, ticker) +} + +func makeOneRPC(c testgrpc.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInfo, error) { + timeout := *rpcTimeout + if cfg.timeout != 0 { + timeout = time.Duration(cfg.timeout) * time.Second + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + if len(cfg.md) != 0 { + ctx = metadata.NewOutgoingContext(ctx, cfg.md) + } + info := rpcInfo{typ: cfg.typ} + + var ( + p peer.Peer + header metadata.MD + err error + ) + accStats.startRPC(cfg.typ) + switch cfg.typ { + case unaryCall: + sr := &testpb.SimpleRequest{FillServerId: true} + if *requestPayloadSize > 0 { + sr.Payload = &testpb.Payload{Body: make([]byte, *requestPayloadSize)} + } + if *responsePayloadSize > 0 { + sr.ResponseSize = int32(*responsePayloadSize) + } + sr.ResponseSize = int32(*responsePayloadSize) + var resp *testpb.SimpleResponse + resp, err = c.UnaryCall(ctx, sr, grpc.Peer(&p), grpc.Header(&header)) + // For UnaryCall, also read hostname from response, in case the server + // isn't updated to send headers. + if resp != nil { + info.hostname = resp.Hostname + } + case emptyCall: + _, err = c.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&p), grpc.Header(&header)) + } + accStats.finishRPC(cfg.typ, err) + if err != nil { + return nil, nil, err + } + + hosts := header["hostname"] + if len(hosts) > 0 { + info.hostname = hosts[0] + } + return &p, &info, err +} + +func sendRPCs(clients []testgrpc.TestServiceClient, ticker *time.Ticker) { + var i int + for range ticker.C { + // Get and increment request ID, and save a list of watchers that are + // interested in this RPC. + mu.Lock() + savedRequestID := currentRequestID + currentRequestID++ + savedWatchers := []*statsWatcher{} + for key, value := range watchers { + if key.startID <= savedRequestID && savedRequestID < key.endID { + savedWatchers = append(savedWatchers, value) + } + } + mu.Unlock() + + // Get the RPC metadata configurations from the Configure RPC. + cfgs := rpcCfgs.Load().([]*rpcConfig) + + c := clients[i] + for _, cfg := range cfgs { + go func(cfg *rpcConfig) { + p, info, err := makeOneRPC(c, cfg) + + for _, watcher := range savedWatchers { + // This sends an empty string if the RPC failed. + watcher.chanHosts <- info + } + if err != nil && *failOnFailedRPC && hasRPCSucceeded() { + logger.Fatalf("RPC failed: %v", err) + } + if err == nil { + setRPCSucceeded() + } + if *printResponse { + if err == nil { + if cfg.typ == unaryCall { + // Need to keep this format, because some tests are + // relying on stdout. + fmt.Printf("Greeting: Hello world, this is %s, from %v\n", info.hostname, p.Addr) + } else { + fmt.Printf("RPC %q, from host %s, addr %v\n", cfg.typ, info.hostname, p.Addr) + } + } else { + fmt.Printf("RPC %q, failed with %v\n", cfg.typ, err) + } + } + }(cfg) + } + i = (i + 1) % len(clients) + } +} diff --git a/interop/xds/custom_lb.go b/interop/xds/custom_lb.go new file mode 100644 index 0000000..a08d825 --- /dev/null +++ b/interop/xds/custom_lb.go @@ -0,0 +1,140 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xds contains various xds interop helpers for usage in interop tests. +package xds + +import ( + "encoding/json" + "fmt" + "sync" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/serviceconfig" +) + +func init() { + balancer.Register(rpcBehaviorBB{}) +} + +const name = "test.RpcBehaviorLoadBalancer" + +type rpcBehaviorBB struct{} + +func (rpcBehaviorBB) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &rpcBehaviorLB{ + ClientConn: cc, + } + // round_robin child to complete balancer tree with a usable leaf policy and + // have RPCs actually work. + builder := balancer.Get(roundrobin.Name) + if builder == nil { + // Shouldn't happen, defensive programming. Registered from import of + // roundrobin package. + return nil + } + rr := builder.Build(b, bOpts) + if rr == nil { + // Shouldn't happen, defensive programming. + return nil + } + b.Balancer = rr + return b +} + +func (rpcBehaviorBB) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + lbCfg := &lbConfig{} + if err := json.Unmarshal(s, lbCfg); err != nil { + return nil, fmt.Errorf("rpc-behavior-lb: unable to marshal lbConfig: %s, error: %v", string(s), err) + } + return lbCfg, nil + +} + +func (rpcBehaviorBB) Name() string { + return name +} + +type lbConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + RPCBehavior string `json:"rpcBehavior,omitempty"` +} + +// rpcBehaviorLB is a load balancer that wraps a round robin balancer and +// appends the rpc-behavior metadata field to any metadata in pick results based +// on what is specified in configuration. +type rpcBehaviorLB struct { + // embed a ClientConn to wrap only UpdateState() operation + balancer.ClientConn + // embed a Balancer to wrap only UpdateClientConnState() operation + balancer.Balancer + + mu sync.Mutex + cfg *lbConfig +} + +func (b *rpcBehaviorLB) UpdateClientConnState(s balancer.ClientConnState) error { + lbCfg, ok := s.BalancerConfig.(*lbConfig) + if !ok { + return fmt.Errorf("test.RpcBehaviorLoadBalancer:received config with unexpected type %T: %s", s.BalancerConfig, pretty.ToJSON(s.BalancerConfig)) + } + b.mu.Lock() + b.cfg = lbCfg + b.mu.Unlock() + return b.Balancer.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: s.ResolverState, + }) +} + +func (b *rpcBehaviorLB) UpdateState(state balancer.State) { + b.mu.Lock() + rpcBehavior := b.cfg.RPCBehavior + b.mu.Unlock() + + b.ClientConn.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: newRPCBehaviorPicker(state.Picker, rpcBehavior), + }) +} + +// rpcBehaviorPicker wraps a picker and adds the rpc-behavior metadata field +// into the child pick result's metadata. +type rpcBehaviorPicker struct { + childPicker balancer.Picker + rpcBehavior string +} + +// Pick appends the rpc-behavior metadata entry to the pick result of the child. +func (p *rpcBehaviorPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + pr, err := p.childPicker.Pick(info) + if err != nil { + return balancer.PickResult{}, err + } + pr.Metadata = metadata.Join(pr.Metadata, metadata.Pairs("rpc-behavior", p.rpcBehavior)) + return pr, nil +} + +func newRPCBehaviorPicker(childPicker balancer.Picker, rpcBehavior string) *rpcBehaviorPicker { + return &rpcBehaviorPicker{ + childPicker: childPicker, + rpcBehavior: rpcBehavior, + } +} diff --git a/interop/xds/custom_lb_test.go b/interop/xds/custom_lb_test.go new file mode 100644 index 0000000..1658b91 --- /dev/null +++ b/interop/xds/custom_lb_test.go @@ -0,0 +1,135 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" +) + +var defaultTestTimeout = 5 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestCustomLB tests the Custom LB for the interop client. It configures the +// custom lb as the top level Load Balancing policy of the channel, then asserts +// it can successfully make an RPC and also that the rpc behavior the Custom LB +// is configured with makes it's way to the server in metadata. +func (s) TestCustomLB(t *testing.T) { + errCh := testutils.NewChannel() + // Setup a backend which verifies the expected rpc-behavior metadata is + // present in the request. + backend := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + errCh.Send(errors.New("failed to receive metadata")) + return &testpb.SimpleResponse{}, nil + } + rpcBMD := md.Get("rpc-behavior") + if len(rpcBMD) != 1 { + errCh.Send(fmt.Errorf("received %d values for metadata key \"rpc-behavior\", want 1", len(rpcBMD))) + return &testpb.SimpleResponse{}, nil + } + wantVal := "error-code-0" + if rpcBMD[0] != wantVal { + errCh.Send(fmt.Errorf("metadata val for key \"rpc-behavior\": got val %v, want val %v", rpcBMD[0], wantVal)) + return &testpb.SimpleResponse{}, nil + } + // Success. + errCh.Send(nil) + return &testpb.SimpleResponse{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started good TestService backend at: %q", backend.Address) + defer backend.Stop() + + lbCfgJSON := `{ + "loadBalancingConfig": [ + { + "test.RpcBehaviorLoadBalancer": { + "rpcBehavior": "error-code-0" + } + } + ] + }` + + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(lbCfgJSON) + mr := manual.NewBuilderWithScheme("customlb-e2e") + defer mr.Close() + mr.InitialState(resolver.State{ + Addresses: []resolver.Address{ + {Addr: backend.Address}, + }, + ServiceConfig: sc, + }) + + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + + // Make a Unary RPC. This RPC should be successful due to the round_robin + // leaf balancer. Also, the custom load balancer should inject the + // "rpc-behavior" string it is configured with into the metadata sent to + // server. + if _, err := testServiceClient.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + val, err := errCh.Receive(ctx) + if err != nil { + t.Fatalf("error receiving from errCh: %v", err) + } + + // Should receive nil on the error channel which implies backend verified it + // correctly received the correct "rpc-behavior" metadata. + if err, ok := val.(error); ok { + t.Fatalf("error in backend verifications on metadata received: %v", err) + } +} diff --git a/interop/xds/go.mod b/interop/xds/go.mod new file mode 100644 index 0000000..ab71227 --- /dev/null +++ b/interop/xds/go.mod @@ -0,0 +1,56 @@ +module google.golang.org/grpc/interop/xds + +go 1.25.0 + +replace google.golang.org/grpc => ../.. + +require ( + github.com/prometheus/client_golang v1.24.1 + go.opentelemetry.io/otel/exporters/prometheus v0.67.0 + go.opentelemetry.io/otel/sdk/metric v1.45.0 + google.golang.org/grpc v1.83.0 +) + +require ( + cel.dev/expr v0.25.3 // indirect + cloud.google.com/go/auth v0.23.1 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.21 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/otlptranslator v1.0.0 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/api v0.293.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/interop/xds/go.sum b/interop/xds/go.sum new file mode 100644 index 0000000..55a39d7 --- /dev/null +++ b/interop/xds/go.sum @@ -0,0 +1,119 @@ +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go/auth v0.23.1 h1:1tPpBPG02lQHmoiAvs9egyCASqXP0xgobptjZzov/Jg= +cloud.google.com/go/auth v0.23.1/go.mod h1:4DhBRcqvtljQN3dJ57qtqbib5ZGCYE5f2crfiiC2EM0= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 h1:bN1gA3of5bXtbnLsRPrwfmbbe7A5UWFlcTHseujLnpc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0/go.mod h1:Yj5vHEz/aAepZGliRJsA6uvHAVAQyEwajq9ORCHPxzM= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be h1:SWe0x6yfglnxuvOiYgTTnNq7QD/yvqthOh1RBI8Bj8w= +github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be/go.mod h1:PYEOlng9XcrulfyWpm49jECTPV0LT4q8cO7fLW/xwgk= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.21 h1:OFdQ3tnCX/zaQ0Cedur3D3z7kI6HiLX9g3TiAN4/DFU= +github.com/googleapis/enterprise-certificate-proxy v0.3.21/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos= +github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0 h1:9jR0ZPRok9ryaOQ2Wx8rg5F7Aon59mxrqbVI60/vlBk= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0/go.mod h1:VSme3o2fvSg5bVg0dRzyHaj4Z5EVhG+g2Fde6LKzmQA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/exporters/prometheus v0.67.0 h1:7IefDa35e6V3NoiqIeLDMDxMFyZDk5qcoC0Ax4cC16E= +go.opentelemetry.io/otel/exporters/prometheus v0.67.0/go.mod h1:nsPI1awTg5Vmg1YrommL2mVarVGlqc4yXOoKAkPRD0c= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= +go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.293.0 h1:p9XIWOf63U4OgYx120ZwVU8+vl4XTPmWfgVPnmOAS9w= +google.golang.org/api v0.293.0/go.mod h1:6n5tjEB1gzwniZTepZ0g5u+wM7Bof5GeULCx/zh8ZE0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1 h1:lrupDmKL3p5kEX1M92oan027eCKcouzjuPbH6YBK+Rs= +google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:q/3oV3jAi5vwelxsVAprMBC8BcM2zmNe+IjRGd+9/ks= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 h1:utmqiUzZAEINbLmRyAt1QjVkMyLQ+lvIwCpLaB1rZfw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/interop/xds/server/Dockerfile b/interop/xds/server/Dockerfile new file mode 100644 index 0000000..08fc497 --- /dev/null +++ b/interop/xds/server/Dockerfile @@ -0,0 +1,37 @@ +# Copyright 2021 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for building the xDS interop server. To build the image, run the +# following command from grpc-go directory: +# docker build -t -f interop/xds/server/Dockerfile . + +FROM golang:1.25-alpine as build + +# Make a grpc-go directory and copy the repo into it. +WORKDIR /go/src/grpc-go +COPY . . + +# Build a static binary without cgo so that we can copy just the binary in the +# final image, and can get rid of the Go compiler and gRPC-Go dependencies. +RUN cd interop/xds/server && go build -tags osusergo,netgo . + +# Second stage of the build which copies over only the server binary and skips +# the Go compiler and gRPC repo from the earlier stage. This significantly +# reduces the docker image size. +FROM alpine +COPY --from=build /go/src/grpc-go/interop/xds/server/server . +ENV GRPC_GO_LOG_VERBOSITY_LEVEL=99 +ENV GRPC_GO_LOG_SEVERITY_LEVEL="info" +ENV GRPC_GO_LOG_FORMATTER="json" +ENTRYPOINT ["./server"] diff --git a/interop/xds/server/server.go b/interop/xds/server/server.go new file mode 100644 index 0000000..4b8fa28 --- /dev/null +++ b/interop/xds/server/server.go @@ -0,0 +1,342 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is the server used for xDS interop tests. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "net/http" + "os" + "strconv" + "strings" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/admin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/health" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/reflection" + "google.golang.org/grpc/stats/opentelemetry" + "google.golang.org/grpc/stats/opentelemetry/csm" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds" + + xdscreds "google.golang.org/grpc/credentials/xds" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/sdk/metric" +) + +var ( + port = flag.Int("port", 8080, "Listening port for test service") + maintenancePort = flag.Int("maintenance_port", 8081, "Listening port for maintenance services like health, reflection, channelz etc when -secure_mode is true. When -secure_mode is false, all these services will be registered on -port") + serverID = flag.String("server_id", "go_server", "Server ID included in response") + secureMode = flag.Bool("secure_mode", false, "If true, retrieve security configuration from the management server. Else, use insecure credentials.") + hostNameOverride = flag.String("host_name_override", "", "If set, use this as the hostname instead of the real hostname") + enableCSMObservability = flag.Bool("enable_csm_observability", false, "Whether to enable CSM Observability") + + logger = grpclog.Component("interop") +) + +const ( + rpcBehaviorMDKey = "rpc-behavior" + grpcPreviousRPCAttemptsMDKey = "grpc-previous-rpc-attempts" + sleepPfx = "sleep-" + keepOpenVal = "keep-open" + errorCodePfx = "error-code-" + succeedOnRetryPfx = "succeed-on-retry-attempt-" + hostnamePfx = "hostname=" +) + +func getHostname() string { + if *hostNameOverride != "" { + return *hostNameOverride + } + hostname, err := os.Hostname() + if err != nil { + log.Fatalf("failed to get hostname: %v", err) + } + return hostname +} + +// testServiceImpl provides an implementation of the TestService defined in +// grpc.testing package. +type testServiceImpl struct { + testgrpc.UnimplementedTestServiceServer + hostname string + serverID string +} + +func (s *testServiceImpl) EmptyCall(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + grpc.SetHeader(ctx, metadata.Pairs("hostname", s.hostname)) + return &testpb.Empty{}, nil +} + +func (s *testServiceImpl) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + response := &testpb.SimpleResponse{ServerId: s.serverID, Hostname: s.hostname} + if in.ResponseSize > 0 { + response.Payload = &testpb.Payload{ + Body: make([]byte, in.ResponseSize), + } + } + +forLoop: + for _, headerVal := range getRPCBehaviorMetadata(ctx) { + // A value can have a prefix "hostname=" followed by a space. + // In that case, the rest of the value should only be applied + // if the specified hostname matches the server's hostname. + if strings.HasPrefix(headerVal, hostnamePfx) { + splitVal := strings.Split(headerVal, " ") + if len(splitVal) <= 1 { + return nil, status.Errorf(codes.InvalidArgument, "invalid format for rpc-behavior header %v, must be 'hostname=
=' instead", headerVal) + } + + if s.hostname != splitVal[0][len(hostnamePfx):] { + continue forLoop + } + headerVal = splitVal[1] + } + + switch { + // If the value matches "sleep-", the server should wait + // the specified number of seconds before resuming + // behavior matching and RPC processing. + case strings.HasPrefix(headerVal, sleepPfx): + sleep, err := strconv.Atoi(headerVal[len(sleepPfx):]) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid format for rpc-behavior header %v, must be 'sleep-' instead", headerVal) + } + time.Sleep(time.Duration(sleep) * time.Second) + + // If the value matches "keep-open", the server should + // never respond to the request and behavior matching ends. + case strings.HasPrefix(headerVal, keepOpenVal): + <-ctx.Done() + return nil, nil + + // If the value matches "error-code-", the server should + // respond with the specified status code and behavior matching ends. + case strings.HasPrefix(headerVal, errorCodePfx): + code, err := strconv.Atoi(headerVal[len(errorCodePfx):]) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid format for rpc-behavior header %v, must be 'error-code-' instead", headerVal) + } + return nil, status.Errorf(codes.Code(code), "rpc failed as per the rpc-behavior header value: %v", headerVal) + + // If the value matches "success-on-retry-attempt-", and the + // value of the "grpc-previous-rpc-attempts" metadata field is equal to + // the specified number, the normal RPC processing should resume + // and behavior matching ends. + case strings.HasPrefix(headerVal, succeedOnRetryPfx): + wantRetry, err := strconv.Atoi(headerVal[len(succeedOnRetryPfx):]) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid format for rpc-behavior header %v, must be 'success-on-retry-attempt-' instead", headerVal) + } + + mdRetry := getMetadataValues(ctx, grpcPreviousRPCAttemptsMDKey) + curRetry, err := strconv.Atoi(mdRetry[0]) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid format for grpc-previous-rpc-attempts header: %v", mdRetry[0]) + } + + if curRetry == wantRetry { + break forLoop + } + } + } + + grpc.SetHeader(ctx, metadata.Pairs("hostname", s.hostname)) + return response, status.Error(codes.OK, "") +} + +func getRPCBehaviorMetadata(ctx context.Context) []string { + mdRPCBehavior := getMetadataValues(ctx, rpcBehaviorMDKey) + var rpcBehaviorMetadata []string + for _, mdVal := range mdRPCBehavior { + splitVals := strings.Split(mdVal, ",") + + for _, val := range splitVals { + headerVal := strings.TrimSpace(val) + if headerVal == "" { + continue + } + rpcBehaviorMetadata = append(rpcBehaviorMetadata, headerVal) + } + } + return rpcBehaviorMetadata +} + +func getMetadataValues(ctx context.Context, metadataKey string) []string { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + logger.Error("Failed to retrieve metadata from incoming RPC context") + return nil + } + return md.Get(metadataKey) +} + +// xdsUpdateHealthServiceImpl provides an implementation of the +// XdsUpdateHealthService defined in grpc.testing package. +type xdsUpdateHealthServiceImpl struct { + testgrpc.UnimplementedXdsUpdateHealthServiceServer + healthServer *health.Server +} + +func (x *xdsUpdateHealthServiceImpl) SetServing(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + x.healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING) + return &testpb.Empty{}, nil + +} + +func (x *xdsUpdateHealthServiceImpl) SetNotServing(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + x.healthServer.SetServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING) + return &testpb.Empty{}, nil +} + +func xdsServingModeCallback(addr net.Addr, args xds.ServingModeChangeArgs) { + logger.Infof("Serving mode callback for xDS server at %q invoked with mode: %q, err: %v", addr.String(), args.Mode, args.Err) +} + +func main() { + flag.Parse() + if *enableCSMObservability { + exporter, err := prometheus.New() + if err != nil { + logger.Fatalf("Failed to start prometheus exporter: %v", err) + } + var addr string + var ok bool + if addr, ok = os.LookupEnv("OTEL_EXPORTER_PROMETHEUS_HOST"); !ok { + addr = "" + } + var port string + if port, ok = os.LookupEnv("OTEL_EXPORTER_PROMETHEUS_PORT"); !ok { + port = "9464" + } + go func() { + if err := http.ListenAndServe(addr+":"+port, promhttp.Handler()); err != nil { + logger.Fatalf("error listening: %v", err) + } + }() + + provider := metric.NewMeterProvider( + metric.WithReader(exporter), + ) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + cleanup := csm.EnableObservability(ctx, opentelemetry.Options{MetricsOptions: opentelemetry.MetricsOptions{MeterProvider: provider}}) + defer cleanup() + } + + if *secureMode && *port == *maintenancePort { + logger.Fatal("-port and -maintenance_port must be different when -secure_mode is set") + } + + testService := &testServiceImpl{hostname: getHostname(), serverID: *serverID} + healthServer := health.NewServer() + updateHealthService := &xdsUpdateHealthServiceImpl{healthServer: healthServer} + + // If -secure_mode is not set, expose all services on -port with a regular + // gRPC server. + if !*secureMode { + addr := fmt.Sprintf(":%d", *port) + lis, err := net.Listen("tcp4", addr) + if err != nil { + logger.Fatalf("net.Listen(%s) failed: %v", addr, err) + } + + server := grpc.NewServer() + testgrpc.RegisterTestServiceServer(server, testService) + healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING) + healthgrpc.RegisterHealthServer(server, healthServer) + testgrpc.RegisterXdsUpdateHealthServiceServer(server, updateHealthService) + reflection.Register(server) + cleanup, err := admin.Register(server) + if err != nil { + logger.Fatalf("Failed to register admin services: %v", err) + } + defer cleanup() + if err := server.Serve(lis); err != nil { + logger.Errorf("Serve() failed: %v", err) + } + return + } + + // Create a listener on -port to expose the test service. + addr := fmt.Sprintf(":%d", *port) + testLis, err := net.Listen("tcp4", addr) + if err != nil { + logger.Fatalf("net.Listen(%s) failed: %v", addr, err) + } + + // Create server-side xDS credentials with a plaintext fallback. + creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + logger.Fatalf("Failed to create xDS credentials: %v", err) + } + + // Create an xDS enabled gRPC server, register the test service + // implementation and start serving. + testServer, err := xds.NewGRPCServer(grpc.Creds(creds), xds.ServingModeCallback(xdsServingModeCallback)) + if err != nil { + logger.Fatal("Failed to create an xDS enabled gRPC server: %v", err) + } + testgrpc.RegisterTestServiceServer(testServer, testService) + go func() { + if err := testServer.Serve(testLis); err != nil { + logger.Errorf("test server Serve() failed: %v", err) + } + }() + defer testServer.Stop() + + // Create a listener on -maintenance_port to expose other services. + addr = fmt.Sprintf(":%d", *maintenancePort) + maintenanceLis, err := net.Listen("tcp4", addr) + if err != nil { + logger.Fatalf("net.Listen(%s) failed: %v", addr, err) + } + + // Create a regular gRPC server and register the maintenance services on + // it and start serving. + maintenanceServer := grpc.NewServer() + healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING) + healthgrpc.RegisterHealthServer(maintenanceServer, healthServer) + testgrpc.RegisterXdsUpdateHealthServiceServer(maintenanceServer, updateHealthService) + reflection.Register(maintenanceServer) + cleanup, err := admin.Register(maintenanceServer) + if err != nil { + logger.Fatalf("Failed to register admin services: %v", err) + } + defer cleanup() + if err := maintenanceServer.Serve(maintenanceLis); err != nil { + logger.Errorf("maintenance server Serve() failed: %v", err) + } +} diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go new file mode 100644 index 0000000..d7e9fd3 --- /dev/null +++ b/interop/xds_federation/client.go @@ -0,0 +1,152 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an interop client. +package main + +import ( + "context" + "flag" + "log" + "strings" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/interop" + + _ "google.golang.org/grpc/balancer/grpclb" // Register the grpclb load balancing policy. + _ "google.golang.org/grpc/balancer/rls" // Register the RLS load balancing policy. + _ "google.golang.org/grpc/xds/googledirectpath" // Register xDS resolver required for c2p directpath. + + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +const ( + computeEngineCredsName = "compute_engine_channel_creds" + insecureCredsName = "INSECURE_CREDENTIALS" +) + +var ( + serverURIs = flag.String("server_uris", "", "Comma-separated list of sever URIs to make RPCs to") + credentialsTypes = flag.String("credentials_types", "", "Comma-separated list of credentials, each entry is used for the server of the corresponding index in server_uris. Supported values: compute_engine_channel_creds, INSECURE_CREDENTIALS") + soakIterations = flag.Int("soak_iterations", 10, "The number of iterations to use for the two soak tests: rpc_soak and channel_soak") + soakMaxFailures = flag.Int("soak_max_failures", 0, "The number of iterations in soak tests that are allowed to fail (either due to non-OK status code or exceeding the per-iteration max acceptable latency).") + soakPerIterationMaxAcceptableLatencyMs = flag.Int("soak_per_iteration_max_acceptable_latency_ms", 1000, "The number of milliseconds a single iteration in the two soak tests (rpc_soak and channel_soak) should take.") + soakOverallTimeoutSeconds = flag.Int("soak_overall_timeout_seconds", 10, "The overall number of seconds after which a soak test should stop and fail, if the desired number of iterations have not yet completed.") + soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") + soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") + soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") + soakNumThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") + testCase = flag.String("test_case", "rpc_soak", + `Configure different test cases. Valid options are: + rpc_soak: sends --soak_iterations large_unary RPCs; + channel_soak: sends --soak_iterations RPCs, rebuilding the channel each time`) + + logger = grpclog.Component("interop") +) + +type clientConfig struct { + conn *grpc.ClientConn + tc testgrpc.TestServiceClient + opts []grpc.DialOption + uri string +} + +func main() { + flag.Parse() + // validate flags + uris := strings.Split(*serverURIs, ",") + creds := strings.Split(*credentialsTypes, ",") + if len(uris) != len(creds) { + logger.Fatalf("Number of entries in --server_uris (%d) != number of entries in --credentials_types (%d)", len(uris), len(creds)) + } + for _, c := range creds { + if c != computeEngineCredsName && c != insecureCredsName { + logger.Fatalf("Unsupported credentials type: %v", c) + } + } + var clients []clientConfig + for i := range uris { + var opts []grpc.DialOption + switch creds[i] { + case computeEngineCredsName: + opts = append(opts, grpc.WithCredentialsBundle(google.NewComputeEngineCredentials())) + case insecureCredsName: + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + cc, err := grpc.NewClient(uris[i], opts...) + if err != nil { + logger.Fatalf("grpc.NewClient(%q) = %v", uris[i], err) + } + defer cc.Close() + clients = append(clients, clientConfig{ + conn: cc, + tc: testgrpc.NewTestServiceClient(cc), + opts: opts, + uri: uris[i], + }) + } + + // run soak tests with the different clients + logger.Infof("Clients running with test case %q", *testCase) + var wg sync.WaitGroup + var channelForTest func() (*grpc.ClientConn, func()) + ctx := context.Background() + for i := range clients { + wg.Add(1) + go func(c clientConfig) { + ctxWithDeadline, cancel := context.WithTimeout(ctx, time.Duration(*soakOverallTimeoutSeconds)*time.Second) + defer cancel() + switch *testCase { + case "rpc_soak": + channelForTest = func() (*grpc.ClientConn, func()) { return c.conn, func() {} } + case "channel_soak": + channelForTest = func() (*grpc.ClientConn, func()) { + cc, err := grpc.NewClient(c.uri, c.opts...) + if err != nil { + log.Fatalf("Failed to create shared channel: %v", err) + } + return cc, func() { cc.Close() } + } + default: + logger.Fatal("Unsupported test case: ", *testCase) + } + soakConfig := interop.SoakTestConfig{ + RequestSize: *soakRequestSize, + ResponseSize: *soakResponseSize, + PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, + MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, + OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, + ServerAddr: c.uri, + NumWorkers: *soakNumThreads, + Iterations: *soakIterations, + MaxFailures: *soakMaxFailures, + ChannelForTest: channelForTest, + } + interop.DoSoakTest(ctxWithDeadline, soakConfig) + logger.Infof("%s test done for server: %s", *testCase, c.uri) + wg.Done() + }(clients[i]) + } + wg.Wait() + logger.Infoln("All clients done!") +} diff --git a/keepalive/keepalive.go b/keepalive/keepalive.go new file mode 100644 index 0000000..eb42b19 --- /dev/null +++ b/keepalive/keepalive.go @@ -0,0 +1,99 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package keepalive defines configurable parameters for point-to-point +// healthcheck. +package keepalive + +import ( + "time" +) + +// ClientParameters is used to set keepalive parameters on the client-side. +// These configure how the client will actively probe to notice when a +// connection is broken and send pings so intermediaries will be aware of the +// liveness of the connection. Make sure these parameters are set in +// coordination with the keepalive policy on the server, as incompatible +// settings can result in closing of connection. +type ClientParameters struct { + // After a duration of this time if the client doesn't see any activity it + // pings the server to see if the transport is still alive. + // If set below 10s, a minimum value of 10s will be used instead. + // + // Note that gRPC servers have a default EnforcementPolicy.MinTime of 5 + // minutes (which means the client shouldn't ping more frequently than every + // 5 minutes). + // + // Though not ideal, it's not a strong requirement for Time to be less than + // EnforcementPolicy.MinTime. Time will automatically double if the server + // disconnects due to its enforcement policy. + // + // For more details, see + // https://github.com/grpc/proposal/blob/master/A8-client-side-keepalive.md + Time time.Duration + // After having pinged for keepalive check, the client waits for a duration + // of Timeout and if no activity is seen even after that the connection is + // closed. + // + // If keepalive is enabled, and this value is not explicitly set, the default + // is 20 seconds. + Timeout time.Duration + // If true, client sends keepalive pings even with no active RPCs. If false, + // when there are no active RPCs, Time and Timeout will be ignored and no + // keepalive pings will be sent. + PermitWithoutStream bool +} + +// ServerParameters is used to set keepalive and max-age parameters on the +// server-side. +type ServerParameters struct { + // MaxConnectionIdle is a duration for the amount of time after which an + // idle connection would be closed by sending a GoAway. Idleness duration is + // defined since the most recent time the number of outstanding RPCs became + // zero or the connection establishment. + MaxConnectionIdle time.Duration // The current default value is infinity. + // MaxConnectionAge is a duration for the maximum amount of time a + // connection may exist before it will be closed by sending a GoAway. A + // random jitter of +/-10% will be added to MaxConnectionAge to spread out + // connection storms. + MaxConnectionAge time.Duration // The current default value is infinity. + // MaxConnectionAgeGrace is an additive period after MaxConnectionAge after + // which the connection will be forcibly closed. + MaxConnectionAgeGrace time.Duration // The current default value is infinity. + // After a duration of this time if the server doesn't see any activity it + // pings the client to see if the transport is still alive. + // If set below 1s, a minimum value of 1s will be used instead. + Time time.Duration // The current default value is 2 hours. + // After having pinged for keepalive check, the server waits for a duration + // of Timeout and if no activity is seen even after that the connection is + // closed. + Timeout time.Duration // The current default value is 20 seconds. +} + +// EnforcementPolicy is used to set keepalive enforcement policy on the +// server-side. Server will close connection with a client that violates this +// policy. +type EnforcementPolicy struct { + // MinTime is the minimum amount of time a client should wait before sending + // a keepalive ping. + MinTime time.Duration // The current default value is 5 minutes. + // If true, server allows keepalive pings even when there are no active + // streams(RPCs). If false, and client sends ping when there are no active + // streams, server will send GOAWAY and close the connection. + PermitWithoutStream bool // false by default. +} diff --git a/mem/buffer_pool.go b/mem/buffer_pool.go new file mode 100644 index 0000000..aa12137 --- /dev/null +++ b/mem/buffer_pool.go @@ -0,0 +1,92 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package mem + +import ( + "fmt" + + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/mem" +) + +// BufferPool is a pool of buffers that can be shared and reused, resulting in +// decreased memory allocation. +type BufferPool interface { + // Get returns a buffer with specified length from the pool. + Get(length int) *[]byte + + // Put returns a buffer to the pool. + // + // The provided pointer must hold a prefix of the buffer obtained via + // BufferPool.Get to ensure the buffer's entire capacity can be re-used. + Put(*[]byte) +} + +var ( + defaultBufferPoolSizeExponents = []uint8{ + 8, + 12, // Go page size, 4KB + 14, // 16KB (max HTTP/2 frame size used by gRPC) + 15, // 32KB (default buffer size for io.Copy) + 20, // 1MB + } + defaultBufferPool BufferPool +) + +func init() { + var err error + defaultBufferPool, err = NewBinaryTieredBufferPool(defaultBufferPoolSizeExponents...) + if err != nil { + panic(fmt.Sprintf("Failed to create default buffer pool: %v", err)) + } + + internal.SetDefaultBufferPool = func(pool BufferPool) { + defaultBufferPool = pool + } +} + +// DefaultBufferPool returns the current default buffer pool. It is a BufferPool +// created with NewBufferPool that uses a set of default sizes optimized for +// expected workflows. +func DefaultBufferPool() BufferPool { + return defaultBufferPool +} + +// NewTieredBufferPool returns a BufferPool implementation that uses multiple +// underlying pools of the given pool sizes. +func NewTieredBufferPool(poolSizes ...int) BufferPool { + return mem.NewTieredBufferPool(poolSizes...) +} + +// NewBinaryTieredBufferPool returns a BufferPool backed by multiple sub-pools. +// This structure enables O(1) lookup time for Get and Put operations. +// +// The arguments provided are the exponents for the buffer capacities (powers +// of 2), not the raw byte sizes. For example, to create a pool of 16KB buffers +// (2^14 bytes), pass 14 as the argument. +func NewBinaryTieredBufferPool(powerOfTwoExponents ...uint8) (BufferPool, error) { + return mem.NewBinaryTieredBufferPool(powerOfTwoExponents...) +} + +// NopBufferPool is a buffer pool that returns new buffers without pooling. +type NopBufferPool struct { + mem.NopBufferPool +} + +var _ BufferPool = NopBufferPool{} diff --git a/mem/buffer_pool_test.go b/mem/buffer_pool_test.go new file mode 100644 index 0000000..fbce09f --- /dev/null +++ b/mem/buffer_pool_test.go @@ -0,0 +1,148 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package mem_test + +import ( + "bytes" + "fmt" + "testing" + "unsafe" + + "google.golang.org/grpc/mem" +) + +func (s) TestBufferPool(t *testing.T) { + var poolSizes = []int{4, 8, 16, 32} + pools := []mem.BufferPool{ + mem.NopBufferPool{}, + mem.NewTieredBufferPool(poolSizes...), + } + + testSizes := append([]int{1}, poolSizes...) + testSizes = append(testSizes, 64) + + for _, p := range pools { + for _, l := range testSizes { + bs := p.Get(l) + if len(*bs) != l { + t.Fatalf("Get(%d) returned buffer of length %d, want %d", l, len(*bs), l) + } + + p.Put(bs) + } + } +} + +func (s) TestBufferPoolClears(t *testing.T) { + const poolSize = 4 + pool := mem.NewTieredBufferPool(poolSize) + tests := []struct { + name string + bufferSize int + }{ + { + name: "sized_buffer_pool", + bufferSize: poolSize, + }, + { + name: "simple_buffer_pool", + bufferSize: poolSize + 1, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + for { + buf1 := pool.Get(tc.bufferSize) + copy(*buf1, "1234") + pool.Put(buf1) + + buf2 := pool.Get(tc.bufferSize) + if unsafe.SliceData(*buf1) != unsafe.SliceData(*buf2) { + pool.Put(buf2) + // This test is only relevant if a buffer is reused, otherwise try again. This + // can happen if a GC pause happens between putting the buffer back in the pool + // and getting a new one. + continue + } + + if !bytes.Equal(*buf1, make([]byte, tc.bufferSize)) { + t.Fatalf("buffer not cleared") + } + break + } + }) + } +} + +func (s) TestBufferPoolIgnoresShortBuffers(t *testing.T) { + pool := mem.NewTieredBufferPool(10, 20) + buf := pool.Get(1) + if cap(*buf) != 10 { + t.Fatalf("Get(1) returned buffer with capacity: %d, want 10", cap(*buf)) + } + + // Insert a short buffer into the pool, which is currently empty. + short := make([]byte, 1) + pool.Put(&short) + // Then immediately request a buffer that would be pulled from the pool where the + // short buffer would have been returned. If the short buffer is pulled from the + // pool, it could cause a panic. + pool.Get(10) +} + +func TestBinaryBufferPool(t *testing.T) { + poolSizes := []uint8{0, 2, 3, 4, 2, 3, 4} // duplicates will be ignored. + + testCases := []struct { + requestSize int + wantCapacity int + }{ + {requestSize: 0, wantCapacity: 0}, + {requestSize: 1, wantCapacity: 1}, + {requestSize: 2, wantCapacity: 4}, + {requestSize: 3, wantCapacity: 4}, + {requestSize: 4, wantCapacity: 4}, + {requestSize: 5, wantCapacity: 8}, + {requestSize: 6, wantCapacity: 8}, + {requestSize: 7, wantCapacity: 8}, + {requestSize: 8, wantCapacity: 8}, + {requestSize: 9, wantCapacity: 16}, + {requestSize: 15, wantCapacity: 16}, + {requestSize: 16, wantCapacity: 16}, + {requestSize: 17, wantCapacity: 4096}, // fallback pool returns sizes in multiples of 4096. + } + + for _, tc := range testCases { + t.Run(fmt.Sprintf("requestSize=%d", tc.requestSize), func(t *testing.T) { + pool, err := mem.NewBinaryTieredBufferPool(poolSizes...) + if err != nil { + t.Fatalf("Failed to create buffer pool: %v", err) + } + buf := pool.Get(tc.requestSize) + if cap(*buf) != tc.wantCapacity { + t.Errorf("Get(%d) returned buffer with capacity: %d, want %d", tc.requestSize, cap(*buf), tc.wantCapacity) + } + if len(*buf) != tc.requestSize { + t.Errorf("Get(%d) returned buffer with length: %d, want %d", tc.requestSize, len(*buf), tc.requestSize) + } + pool.Put(buf) + }) + } +} diff --git a/mem/buffer_slice.go b/mem/buffer_slice.go new file mode 100644 index 0000000..086e9f9 --- /dev/null +++ b/mem/buffer_slice.go @@ -0,0 +1,345 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package mem + +import ( + "fmt" + "io" +) + +const ( + // 32 KiB is what io.Copy uses. + readAllBufSize = 32 * 1024 +) + +// BufferSlice offers a means to represent data that spans one or more Buffer +// instances. A BufferSlice is meant to be immutable after creation, and methods +// like Ref create and return copies of the slice. This is why all methods have +// value receivers rather than pointer receivers. +// +// Note that any of the methods that read the underlying buffers such as Ref, +// Len or CopyTo etc., will panic if any underlying buffers have already been +// freed. It is recommended to not directly interact with any of the underlying +// buffers directly, rather such interactions should be mediated through the +// various methods on this type. +// +// By convention, any APIs that return (mem.BufferSlice, error) should reduce +// the burden on the caller by never returning a mem.BufferSlice that needs to +// be freed if the error is non-nil, unless explicitly stated. +type BufferSlice []Buffer + +// Len returns the sum of the length of all the Buffers in this slice. +// +// # Warning +// +// Invoking the built-in len on a BufferSlice will return the number of buffers +// in the slice, and *not* the value returned by this function. +func (s BufferSlice) Len() int { + var length int + for _, b := range s { + length += b.Len() + } + return length +} + +// Ref invokes Ref on each buffer in the slice. +func (s BufferSlice) Ref() { + for _, b := range s { + b.Ref() + } +} + +// Free invokes Buffer.Free() on each Buffer in the slice. +func (s BufferSlice) Free() { + for _, b := range s { + b.Free() + } +} + +// CopyTo copies each of the underlying Buffer's data into the given buffer, +// returning the number of bytes copied. Has the same semantics as the copy +// builtin in that it will copy as many bytes as it can, stopping when either dst +// is full or s runs out of data, returning the minimum of s.Len() and len(dst). +func (s BufferSlice) CopyTo(dst []byte) int { + off := 0 + for _, b := range s { + off += copy(dst[off:], b.ReadOnlyData()) + } + return off +} + +// Materialize concatenates all the underlying Buffer's data into a single +// contiguous buffer using CopyTo. +func (s BufferSlice) Materialize() []byte { + l := s.Len() + if l == 0 { + return nil + } + out := make([]byte, l) + s.CopyTo(out) + return out +} + +// MaterializeToBuffer functions like Materialize except that it writes the data +// to a single Buffer pulled from the given BufferPool. +// +// As a special case, if the input BufferSlice only actually has one Buffer, this +// function simply increases the refcount before returning said Buffer. Freeing this +// buffer won't release it until the BufferSlice is itself released. +func (s BufferSlice) MaterializeToBuffer(pool BufferPool) Buffer { + if len(s) == 1 { + s[0].Ref() + return s[0] + } + sLen := s.Len() + if sLen == 0 { + return emptyBuffer{} + } + buf := pool.Get(sLen) + s.CopyTo(*buf) + return NewBuffer(buf, pool) +} + +// Reader returns a new Reader for the input slice after taking references to +// each underlying buffer. +func (s BufferSlice) Reader() *Reader { + s.Ref() + return &Reader{ + data: s, + len: s.Len(), + } +} + +// Reader exposes a BufferSlice's data as an io.Reader, allowing it to interface +// with other systems. +// +// Buffers will be freed as they are read. +// +// A Reader can be constructed from a BufferSlice; alternatively the zero value +// of a Reader may be used after calling Reset on it. +type Reader struct { + data BufferSlice + len int + // The index into data[0].ReadOnlyData(). + bufferIdx int +} + +// Remaining returns the number of unread bytes remaining in the slice. +func (r *Reader) Remaining() int { + return r.len +} + +// Reset frees the currently held buffer slice and starts reading from the +// provided slice. This allows reusing the reader object. +func (r *Reader) Reset(s BufferSlice) { + r.data.Free() + s.Ref() + r.data = s + r.len = s.Len() + r.bufferIdx = 0 +} + +// Close frees the underlying BufferSlice and never returns an error. Subsequent +// calls to Read will return (0, io.EOF). +func (r *Reader) Close() error { + r.data.Free() + r.data = nil + r.len = 0 + return nil +} + +func (r *Reader) freeFirstBufferIfEmpty() bool { + if len(r.data) == 0 || r.bufferIdx != r.data[0].Len() { + return false + } + + r.data[0].Free() + r.data = r.data[1:] + r.bufferIdx = 0 + return true +} + +func (r *Reader) Read(buf []byte) (n int, _ error) { + if r.len == 0 { + return 0, io.EOF + } + + for len(buf) != 0 && r.len != 0 { + // Copy as much as possible from the first Buffer in the slice into the + // given byte slice. + data := r.data[0].ReadOnlyData() + copied := copy(buf, data[r.bufferIdx:]) + r.len -= copied // Reduce len by the number of bytes copied. + r.bufferIdx += copied // Increment the buffer index. + n += copied // Increment the total number of bytes read. + buf = buf[copied:] // Shrink the given byte slice. + + // If we have copied all the data from the first Buffer, free it and advance to + // the next in the slice. + r.freeFirstBufferIfEmpty() + } + + return n, nil +} + +// ReadByte reads a single byte. +func (r *Reader) ReadByte() (byte, error) { + if r.len == 0 { + return 0, io.EOF + } + + // There may be any number of empty buffers in the slice, clear them all until a + // non-empty buffer is reached. This is guaranteed to exit since r.len is not 0. + for r.freeFirstBufferIfEmpty() { + } + + b := r.data[0].ReadOnlyData()[r.bufferIdx] + r.len-- + r.bufferIdx++ + // Free the first buffer in the slice if the last byte was read + r.freeFirstBufferIfEmpty() + return b, nil +} + +var _ io.Writer = (*writer)(nil) + +type writer struct { + buffers *BufferSlice + pool BufferPool +} + +func (w *writer) Write(p []byte) (n int, err error) { + b := Copy(p, w.pool) + *w.buffers = append(*w.buffers, b) + return b.Len(), nil +} + +// NewWriter wraps the given BufferSlice and BufferPool to implement the +// io.Writer interface. Every call to Write copies the contents of the given +// buffer into a new Buffer pulled from the given pool and the Buffer is +// added to the given BufferSlice. +func NewWriter(buffers *BufferSlice, pool BufferPool) io.Writer { + return &writer{buffers: buffers, pool: pool} +} + +// ReadAll reads from r until an error or EOF and returns the data it read. +// A successful call returns err == nil, not err == EOF. Because ReadAll is +// defined to read from src until EOF, it does not treat an EOF from Read +// as an error to be reported. +// +// Important: A failed call returns a non-nil error and may also return +// partially read buffers. It is the responsibility of the caller to free the +// BufferSlice returned, or its memory will not be reused. +func ReadAll(r io.Reader, pool BufferPool) (BufferSlice, error) { + var result BufferSlice + if wt, ok := r.(io.WriterTo); ok { + // This is more optimal since wt knows the size of chunks it wants to + // write and, hence, we can allocate buffers of an optimal size to fit + // them. E.g. might be a single big chunk, and we wouldn't chop it + // into pieces. + w := NewWriter(&result, pool) + _, err := wt.WriteTo(w) + return result, err + } +nextBuffer: + for { + buf := pool.Get(readAllBufSize) + // We asked for 32KiB but may have been given a bigger buffer. + // Use all of it if that's the case. + *buf = (*buf)[:cap(*buf)] + usedCap := 0 + for { + n, err := r.Read((*buf)[usedCap:]) + usedCap += n + if err != nil { + if usedCap == 0 { + // Nothing in this buf, put it back + pool.Put(buf) + } else { + *buf = (*buf)[:usedCap] + result = append(result, NewBuffer(buf, pool)) + } + if err == io.EOF { + err = nil + } + return result, err + } + if len(*buf) == usedCap { + result = append(result, NewBuffer(buf, pool)) + continue nextBuffer + } + } + } +} + +// Discard skips the next n bytes, returning the number of bytes discarded. +// +// It frees buffers as they are fully consumed. +// +// If Discard skips fewer than n bytes, it also returns an error. +func (r *Reader) Discard(n int) (discarded int, err error) { + total := n + for n > 0 && r.len > 0 { + curData := r.data[0].ReadOnlyData() + curSize := min(n, len(curData)-r.bufferIdx) + n -= curSize + r.len -= curSize + r.bufferIdx += curSize + if r.bufferIdx >= len(curData) { + r.data[0].Free() + r.data = r.data[1:] + r.bufferIdx = 0 + } + } + discarded = total - n + if n > 0 { + return discarded, fmt.Errorf("insufficient bytes in reader") + } + return discarded, nil +} + +// Peek returns the next n bytes without advancing the reader. +// +// Peek appends results to the provided res slice and returns the updated slice. +// This pattern allows re-using the storage of res if it has sufficient +// capacity. +// +// The returned subslices are views into the underlying buffers and are only +// valid until the reader is advanced past the corresponding buffer. +// +// If Peek returns fewer than n bytes, it also returns an error. +func (r *Reader) Peek(n int, res [][]byte) ([][]byte, error) { + for i := 0; n > 0 && i < len(r.data); i++ { + curData := r.data[i].ReadOnlyData() + start := 0 + if i == 0 { + start = r.bufferIdx + } + curSize := min(n, len(curData)-start) + if curSize == 0 { + continue + } + res = append(res, curData[start:start+curSize]) + n -= curSize + } + if n > 0 { + return nil, fmt.Errorf("insufficient bytes in reader") + } + return res, nil +} diff --git a/mem/buffer_slice_test.go b/mem/buffer_slice_test.go new file mode 100644 index 0000000..822acbd --- /dev/null +++ b/mem/buffer_slice_test.go @@ -0,0 +1,676 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package mem_test + +import ( + "bytes" + "crypto/rand" + "errors" + "fmt" + "io" + "testing" + + "google.golang.org/grpc/mem" +) + +const ( + minReadSize = 1 + // Should match the constant in buffer_slice.go (another package) + readAllBufSize = 32 * 1024 // 32 KiB +) + +func newBuffer(data []byte, pool mem.BufferPool) mem.Buffer { + return mem.NewBuffer(&data, pool) +} + +func (s) TestBufferSlice_Len(t *testing.T) { + tests := []struct { + name string + in mem.BufferSlice + want int + }{ + { + name: "empty", + in: nil, + want: 0, + }, + { + name: "single", + in: mem.BufferSlice{newBuffer([]byte("abcd"), nil)}, + want: 4, + }, + { + name: "multiple", + in: mem.BufferSlice{ + newBuffer([]byte("abcd"), nil), + newBuffer([]byte("abcd"), nil), + newBuffer([]byte("abcd"), nil), + }, + want: 12, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.in.Len(); got != tt.want { + t.Errorf("BufferSlice.Len() = %v, want %v", got, tt.want) + } + }) + } +} + +func (s) TestBufferSlice_Ref(t *testing.T) { + // Create a new buffer slice and a reference to it. + bs := mem.BufferSlice{ + newBuffer([]byte("abcd"), nil), + newBuffer([]byte("abcd"), nil), + } + bs.Ref() + + // Free the original buffer slice and verify that the reference can still + // read data from it. + bs.Free() + got := bs.Materialize() + want := []byte("abcdabcd") + if !bytes.Equal(got, want) { + t.Errorf("BufferSlice.Materialize() = %s, want %s", string(got), string(want)) + } +} + +func (s) TestBufferSlice_MaterializeToBuffer(t *testing.T) { + tests := []struct { + name string + in mem.BufferSlice + pool mem.BufferPool + wantData []byte + }{ + { + name: "single", + in: mem.BufferSlice{newBuffer([]byte("abcd"), nil)}, + pool: nil, // MaterializeToBuffer should not use the pool in this case. + wantData: []byte("abcd"), + }, + { + name: "multiple", + in: mem.BufferSlice{ + newBuffer([]byte("abcd"), nil), + newBuffer([]byte("abcd"), nil), + newBuffer([]byte("abcd"), nil), + }, + pool: mem.DefaultBufferPool(), + wantData: []byte("abcdabcdabcd"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer tt.in.Free() + got := tt.in.MaterializeToBuffer(tt.pool) + defer got.Free() + if !bytes.Equal(got.ReadOnlyData(), tt.wantData) { + t.Errorf("BufferSlice.MaterializeToBuffer() = %s, want %s", string(got.ReadOnlyData()), string(tt.wantData)) + } + }) + } +} + +func (s) TestBufferSlice_Reader(t *testing.T) { + bs := mem.BufferSlice{ + newBuffer([]byte("abcd"), nil), + newBuffer([]byte("abcd"), nil), + newBuffer([]byte("abcd"), nil), + } + wantData := []byte("abcdabcdabcd") + + reader := bs.Reader() + var gotData []byte + // Read into a buffer of size 1 until EOF, and verify that the data matches. + for { + buf := make([]byte, 1) + n, err := reader.Read(buf) + if n > 0 { + gotData = append(gotData, buf[:n]...) + } + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("BufferSlice.Reader() failed unexpectedly: %v", err) + } + } + if !bytes.Equal(gotData, wantData) { + t.Errorf("BufferSlice.Reader() returned data %v, want %v", string(gotData), string(wantData)) + } + + // Reader should have released its references to the underlying buffers, but + // bs still holds its reference and it should be able to read data from it. + gotData = bs.Materialize() + if !bytes.Equal(gotData, wantData) { + t.Errorf("BufferSlice.Materialize() = %s, want %s", string(gotData), string(wantData)) + } +} + +// TestBufferSlice_ReadAll_Reads exercises ReadAll by allowing it to read +// various combinations of data, empty data, EOF. +func (s) TestBufferSlice_ReadAll_Reads(t *testing.T) { + testcases := []struct { + name string + reads []readStep + wantErr string + wantBufs int + }{ + { + name: "EOF", + reads: []readStep{ + { + err: io.EOF, + }, + }, + }, + { + name: "data,EOF", + reads: []readStep{ + { + n: minReadSize, + }, + { + err: io.EOF, + }, + }, + wantBufs: 1, + }, + { + name: "data+EOF", + reads: []readStep{ + { + n: minReadSize, + err: io.EOF, + }, + }, + wantBufs: 1, + }, + { + name: "0,data+EOF", + reads: []readStep{ + {}, + { + n: minReadSize, + err: io.EOF, + }, + }, + wantBufs: 1, + }, + { + name: "0,data,EOF", + reads: []readStep{ + {}, + { + n: minReadSize, + }, + { + err: io.EOF, + }, + }, + wantBufs: 1, + }, + { + name: "data,data+EOF", + reads: []readStep{ + { + n: minReadSize, + }, + { + n: minReadSize, + err: io.EOF, + }, + }, + wantBufs: 1, + }, + { + name: "error", + reads: []readStep{ + { + err: errors.New("boom"), + }, + }, + wantErr: "boom", + }, + { + name: "data+error", + reads: []readStep{ + { + n: minReadSize, + err: errors.New("boom"), + }, + }, + wantErr: "boom", + wantBufs: 1, + }, + { + name: "data,data+error", + reads: []readStep{ + { + n: minReadSize, + }, + { + n: minReadSize, + err: errors.New("boom"), + }, + }, + wantErr: "boom", + wantBufs: 1, + }, + { + name: "data,data+EOF - whole buf", + reads: []readStep{ + { + n: minReadSize, + }, + { + n: readAllBufSize - minReadSize, + err: io.EOF, + }, + }, + wantBufs: 1, + }, + { + name: "data,data,EOF - whole buf", + reads: []readStep{ + { + n: minReadSize, + }, + { + n: readAllBufSize - minReadSize, + }, + { + err: io.EOF, + }, + }, + wantBufs: 1, + }, + { + name: "data,data,EOF - 2 bufs", + reads: []readStep{ + { + n: readAllBufSize, + }, + { + n: minReadSize, + }, + { + n: readAllBufSize - minReadSize, + }, + { + n: minReadSize, + }, + { + err: io.EOF, + }, + }, + wantBufs: 3, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + pool := &testPool{ + allocated: make(map[*[]byte]struct{}), + } + r := &stepReader{ + reads: tc.reads, + } + data, err := mem.ReadAll(r, pool) + if tc.wantErr != "" { + if err == nil || err.Error() != tc.wantErr { + t.Fatalf("ReadAll() returned err %v, wanted %q", err, tc.wantErr) + } + } else { + if err != nil { + t.Fatal(err) + } + } + gotData := data.Materialize() + if !bytes.Equal(r.read, gotData) { + t.Fatalf("ReadAll() returned data %q, wanted %q", gotData, r.read) + } + if len(data) != tc.wantBufs { + t.Fatalf("ReadAll() returned %d bufs, wanted %d bufs", len(data), tc.wantBufs) + } + // all but last should be full buffers + for i := 0; i < len(data)-1; i++ { + if data[i].Len() != readAllBufSize { + t.Fatalf("ReadAll() returned data length %d, wanted %d", data[i].Len(), readAllBufSize) + } + } + data.Free() + if len(pool.allocated) > 0 { + t.Fatalf("got %d allocated buffers, wanted none", len(pool.allocated)) + } + }) + } +} + +func (s) TestBufferSlice_ReadAll_WriteTo(t *testing.T) { + testcases := []struct { + name string + size int + }{ + { + name: "small", + size: minReadSize, + }, + { + name: "exact size", + size: readAllBufSize, + }, + { + name: "big", + size: readAllBufSize * 3, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + pool := &testPool{ + allocated: make(map[*[]byte]struct{}), + } + buf := make([]byte, tc.size) + _, err := rand.Read(buf) + if err != nil { + t.Fatal(err) + } + r := bytes.NewBuffer(buf) + data, err := mem.ReadAll(r, pool) + if err != nil { + t.Fatal(err) + } + + gotData := data.Materialize() + if !bytes.Equal(buf, gotData) { + t.Fatalf("ReadAll() = %q, wanted %q", gotData, buf) + } + data.Free() + if len(pool.allocated) > 0 { + t.Fatalf("wanted no allocated buffers, got %d", len(pool.allocated)) + } + }) + } +} + +func ExampleNewWriter() { + var bs mem.BufferSlice + pool := mem.DefaultBufferPool() + writer := mem.NewWriter(&bs, pool) + + for _, data := range [][]byte{ + []byte("abcd"), + []byte("abcd"), + []byte("abcd"), + } { + n, err := writer.Write(data) + fmt.Printf("Wrote %d bytes, err: %v\n", n, err) + } + fmt.Println(string(bs.Materialize())) + // Output: + // Wrote 4 bytes, err: + // Wrote 4 bytes, err: + // Wrote 4 bytes, err: + // abcdabcdabcd +} + +var ( + _ io.Reader = (*stepReader)(nil) + _ mem.BufferPool = (*testPool)(nil) +) + +// readStep describes what a single stepReader.Read should do - how much data +// to return and what error to return. +type readStep struct { + n int + err error +} + +// stepReader implements io.Reader that reads specified amount of data and/or +// returns the specified error in specified steps. +// The read data is accumulated in the read field. +type stepReader struct { + reads []readStep + read []byte +} + +func (s *stepReader) Read(buf []byte) (int, error) { + if len(s.reads) == 0 { + panic("unexpected Read() call") + } + read := s.reads[0] + s.reads = s.reads[1:] + _, err := rand.Read(buf[:read.n]) + if err != nil { + panic(err) + } + s.read = append(s.read, buf[:read.n]...) + return read.n, read.err +} + +// testPool is an implementation of BufferPool that allows to ensure that: +// - there are matching Put calls for all Get calls. +// - there are no unexpected Put calls. +type testPool struct { + allocated map[*[]byte]struct{} +} + +func (t *testPool) Get(length int) *[]byte { + buf := make([]byte, length) + t.allocated[&buf] = struct{}{} + return &buf +} + +func (t *testPool) Put(buf *[]byte) { + if _, ok := t.allocated[buf]; !ok { + panic("unexpected put") + } + delete(t.allocated, buf) +} + +func (s) TestBufferSlice_Iteration(t *testing.T) { + tests := []struct { + name string + buffers [][]byte + operations func(t *testing.T, c *mem.Reader) + }{ + { + name: "empty", + operations: func(t *testing.T, r *mem.Reader) { + if r.Remaining() != 0 { + t.Fatalf("Remaining() = %v, want 0", r.Remaining()) + } + _, err := r.Peek(1, nil) + if err == nil { + t.Fatalf("Peek(1) returned error , want non-nil") + } + discarded, err := r.Discard(1) + if got, want := discarded, 0; got != want { + t.Fatalf("Discard(1) = %d, want %d", got, want) + } + if err == nil { + t.Fatalf("Discard(1) returned error , want non-nil") + } + if r.Remaining() != 0 { + t.Fatalf("Remaining() after Discard = %v, want 0", r.Remaining()) + } + }, + }, + { + name: "single_buffer", + buffers: [][]byte{[]byte("0123456789")}, + operations: func(t *testing.T, r *mem.Reader) { + if r.Remaining() != 10 { + t.Fatalf("Remaining() = %v, want 10", r.Remaining()) + } + + res := make([][]byte, 0, 10) + res, err := r.Peek(5, res) + if err != nil { + t.Fatalf("Peek(5) return error %v, want ", err) + } + if len(res) != 1 || !bytes.Equal(res[0], []byte("01234")) { + t.Fatalf("Peek(5) = %v, want [[01234]]", res) + } + if cap(res) != 10 { + t.Fatalf("Peek(5) did not use the provided slice.") + } + + discarded, err := r.Discard(5) + if got, want := discarded, 5; got != want { + t.Fatalf("Discard(5) = %d, want %d", got, want) + } + if err != nil { + t.Fatalf("Discard(5) return error %v, want ", err) + } + if r.Remaining() != 5 { + t.Fatalf("Remaining() after Discard(5) = %v, want 5", r.Remaining()) + } + res, err = r.Peek(5, res[:0]) + if err != nil { + t.Fatalf("Peek(5) return error %v, want ", err) + } + if len(res) != 1 || !bytes.Equal(res[0], []byte("56789")) { + t.Fatalf("Peek(5) after Discard(5) = %v, want [[56789]]", res) + } + + discarded, err = r.Discard(100) + if got, want := discarded, 5; got != want { + t.Fatalf("Discard(100) = %d, want %d", got, want) + } + if err == nil { + t.Fatalf("Discard(100) returned error , want non-nil") + } + if r.Remaining() != 0 { + t.Fatalf("Remaining() after Discard(100) = %v, want 0", r.Remaining()) + } + }, + }, + { + name: "multiple_buffers", + buffers: [][]byte{[]byte("012"), []byte("345"), []byte("6789")}, + operations: func(t *testing.T, r *mem.Reader) { + if r.Remaining() != 10 { + t.Fatalf("Remaining() = %v, want 10", r.Remaining()) + } + + res, err := r.Peek(5, nil) + if err != nil { + t.Fatalf("Peek(5) return error %v, want ", err) + } + if len(res) != 2 || !bytes.Equal(res[0], []byte("012")) || !bytes.Equal(res[1], []byte("34")) { + t.Fatalf("Peek(5) = %v, want [[012] [34]]", res) + } + + discarded, err := r.Discard(5) + if got, want := discarded, 5; got != want { + t.Fatalf("Discard(5) = %d, want %d", got, want) + } + if err != nil { + t.Fatalf("Discard(5) return error %v, want ", err) + } + if r.Remaining() != 5 { + t.Fatalf("Remaining() after Discard(5) = %v, want 5", r.Remaining()) + } + + res, err = r.Peek(5, res[:0]) + if err != nil { + t.Fatalf("Peek(5) return error %v, want ", err) + } + if len(res) != 2 || !bytes.Equal(res[0], []byte("5")) || !bytes.Equal(res[1], []byte("6789")) { + t.Fatalf("Peek(5) after advance = %v, want [[5] [6789]]", res) + } + }, + }, + { + name: "close", + buffers: [][]byte{[]byte("0123456789")}, + operations: func(t *testing.T, r *mem.Reader) { + r.Close() + if r.Remaining() != 0 { + t.Fatalf("Remaining() after Close = %v, want 0", r.Remaining()) + } + }, + }, + { + name: "reset", + buffers: [][]byte{[]byte("0123")}, + operations: func(t *testing.T, r *mem.Reader) { + newSlice := mem.BufferSlice{mem.SliceBuffer([]byte("56789"))} + r.Reset(newSlice) + if r.Remaining() != 5 { + t.Fatalf("Remaining() after Reset = %v, want 5", r.Remaining()) + } + res, err := r.Peek(5, nil) + if err != nil { + t.Fatalf("Peek(5) return error %v, want ", err) + } + if len(res) != 1 || !bytes.Equal(res[0], []byte("56789")) { + t.Fatalf("Peek(5) after Reset = %v, want [[56789]]", res) + } + }, + }, + { + name: "zero_ops", + buffers: [][]byte{[]byte("01234")}, + operations: func(t *testing.T, c *mem.Reader) { + if c.Remaining() != 5 { + t.Fatalf("Remaining() = %v, want 5", c.Remaining()) + } + res, err := c.Peek(0, nil) + if err != nil { + t.Fatalf("Peek(0) return error %v, want ", err) + } + if len(res) != 0 { + t.Fatalf("Peek(0) got slices: %v, want empty", res) + } + discarded, err := c.Discard(0) + if err != nil { + t.Fatalf("Discard(0) return error %v, want ", err) + } + if got, want := discarded, 0; got != want { + t.Fatalf("Discard(0) = %d, want %d", got, want) + } + if c.Remaining() != 5 { + t.Fatalf("Remaining() after Discard(0) = %v, want 5", c.Remaining()) + } + res, err = c.Peek(2, res[:0]) + if err != nil { + t.Fatalf("Peek(2) return error %v, want ", err) + } + if len(res) != 1 || !bytes.Equal(res[0], []byte("01")) { + t.Fatalf("Peek(2) after zero ops = %v, want [[01]]", res) + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var slice mem.BufferSlice + for _, b := range tt.buffers { + slice = append(slice, mem.SliceBuffer(b)) + } + c := slice.Reader() + slice.Free() + defer c.Close() + tt.operations(t, c) + }) + } +} diff --git a/mem/buffers.go b/mem/buffers.go new file mode 100644 index 0000000..9b355d4 --- /dev/null +++ b/mem/buffers.go @@ -0,0 +1,317 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package mem provides utilities that facilitate memory reuse in byte slices +// that are used as buffers. +// +// # Experimental +// +// Notice: All APIs in this package are EXPERIMENTAL and may be changed or +// removed in a later release. +package mem + +import ( + "fmt" + "sync" + "sync/atomic" + + "google.golang.org/grpc/internal/mem" +) + +// A Buffer represents a reference counted piece of data (in bytes) that can be +// acquired by a call to NewBuffer() or Copy(). A reference to a Buffer may be +// released by calling Free(), which invokes the free function given at creation +// only after all references are released. +// +// Note that a Buffer is not safe for concurrent access and instead each +// goroutine should use its own reference to the data, which can be acquired via +// a call to Ref(). +// +// Attempts to access the underlying data after releasing the reference to the +// Buffer will panic. +type Buffer interface { + // ReadOnlyData returns the underlying byte slice. Note that it is undefined + // behavior to modify the contents of this slice in any way. + ReadOnlyData() []byte + // Ref increases the reference counter for this Buffer. + Ref() + // Free decrements this Buffer's reference counter and frees the underlying + // byte slice if the counter reaches 0 as a result of this call. + Free() + // Len returns the Buffer's size. + Len() int + // Slice returns a new Buffer that is a view into this buffer's data + // from [start:end). The buffer is not modified. Panics if the buffer + // has been freed or if start/end are out of bounds. + Slice(start, end int) Buffer + + split(n int) (left, right Buffer) + read(buf []byte) (int, Buffer) +} + +var ( + bufferObjectPool = sync.Pool{New: func() any { return new(buffer) }} +) + +// IsBelowBufferPoolingThreshold returns true if the given size is less than or +// equal to the threshold for buffer pooling. This is used to determine whether +// to pool buffers or allocate them directly. +func IsBelowBufferPoolingThreshold(size int) bool { + return size <= mem.BufferPoolingThreshold +} + +type buffer struct { + refs atomic.Int32 + data []byte + + // rootBuf is the buffer responsible for returning origData to the pool + // once the reference count drops to 0. + // + // When a buffer is split, the new buffer inherits the rootBuf of the + // original and increments the root's reference count. For the + // initial buffer (the root), this field points to itself. + rootBuf *buffer + + // The following fields are only set for root buffers. + origData *[]byte + pool BufferPool +} + +func newBuffer() *buffer { + return bufferObjectPool.Get().(*buffer) +} + +// NewBuffer creates a new Buffer from the given data, initializing the reference +// counter to 1. The data will then be returned to the given pool when all +// references to the returned Buffer are released. As a special case to avoid +// additional allocations, if the given buffer pool is nil, the returned buffer +// will be a "no-op" Buffer where invoking Buffer.Free() does nothing and the +// underlying data is never freed. +// +// Note that the backing array of the given data is not copied. +func NewBuffer(data *[]byte, pool BufferPool) Buffer { + // Use the buffer's capacity instead of the length, otherwise buffers may + // not be reused under certain conditions. For example, if a large buffer + // is acquired from the pool, but fewer bytes than the buffering threshold + // are written to it, the buffer will not be returned to the pool. + if pool == nil || IsBelowBufferPoolingThreshold(cap(*data)) { + return (SliceBuffer)(*data) + } + b := newBuffer() + b.origData = data + b.data = *data + b.pool = pool + b.rootBuf = b + b.refs.Store(1) + return b +} + +// Copy creates a new Buffer from the given data, initializing the reference +// counter to 1. +// +// It acquires a []byte from the given pool and copies over the backing array +// of the given data. The []byte acquired from the pool is returned to the +// pool when all references to the returned Buffer are released. +func Copy(data []byte, pool BufferPool) Buffer { + if IsBelowBufferPoolingThreshold(len(data)) { + buf := make(SliceBuffer, len(data)) + copy(buf, data) + return buf + } + + buf := pool.Get(len(data)) + copy(*buf, data) + return NewBuffer(buf, pool) +} + +func (b *buffer) ReadOnlyData() []byte { + if b.rootBuf == nil { + panic("Cannot read freed buffer") + } + return b.data +} + +func (b *buffer) Ref() { + if b.refs.Add(1) <= 1 { + panic("Cannot ref freed buffer") + } +} + +func (b *buffer) Free() { + refs := b.refs.Add(-1) + if refs < 0 { + panic("Cannot free freed buffer") + } + if refs > 0 { + return + } + + b.data = nil + if b.rootBuf == b { + // This buffer is the owner of the data slice and its ref count reached + // 0, free the slice. + if b.pool != nil { + b.pool.Put(b.origData) + b.pool = nil + } + b.origData = nil + } else { + // This buffer doesn't own the data slice, decrement a ref on the root + // buffer. + b.rootBuf.Free() + } + + b.rootBuf = nil + bufferObjectPool.Put(b) +} + +func (b *buffer) Len() int { + return len(b.ReadOnlyData()) +} + +func (b *buffer) Slice(start, end int) Buffer { + if b.rootBuf == nil { + panic("Cannot slice freed buffer") + } + + data := b.data[start:end] // access the data to check slice bounds + + if len(data) == 0 { + return emptyBuffer{} + } + if len(data) == len(b.data) { + b.Ref() + return b + } + // We are creating a new reference (view) to a portion of the root buffer's + // data. Therefore, we must increment the reference count of the root buffer + // to ensure the underlying data is not freed while this view is still in + // use. + b.rootBuf.Ref() + s := newBuffer() + s.data = data + s.rootBuf = b.rootBuf + s.refs.Store(1) + return s +} + +func (b *buffer) split(n int) (Buffer, Buffer) { + if b.rootBuf == nil || b.rootBuf.refs.Add(1) <= 1 { + panic("Cannot split freed buffer") + } + + split := newBuffer() + split.data = b.data[n:] + split.rootBuf = b.rootBuf + split.refs.Store(1) + + b.data = b.data[:n] + + return b, split +} + +func (b *buffer) read(buf []byte) (int, Buffer) { + if b.rootBuf == nil { + panic("Cannot read freed buffer") + } + + n := copy(buf, b.data) + if n == len(b.data) { + b.Free() + return n, nil + } + + b.data = b.data[n:] + return n, b +} + +func (b *buffer) String() string { + return fmt.Sprintf("mem.Buffer(%p, data: %p, length: %d)", b, b.ReadOnlyData(), len(b.ReadOnlyData())) +} + +// ReadUnsafe reads bytes from the given Buffer into the provided slice. +// It does not perform safety checks. +func ReadUnsafe(dst []byte, buf Buffer) (int, Buffer) { + return buf.read(dst) +} + +// SplitUnsafe modifies the receiver to point to the first n bytes while it +// returns a new reference to the remaining bytes. The returned Buffer +// functions just like a normal reference acquired using Ref(). +func SplitUnsafe(buf Buffer, n int) (left, right Buffer) { + return buf.split(n) +} + +type emptyBuffer struct{} + +func (e emptyBuffer) ReadOnlyData() []byte { + return nil +} + +func (e emptyBuffer) Ref() {} +func (e emptyBuffer) Free() {} + +func (e emptyBuffer) Len() int { + return 0 +} + +func (e emptyBuffer) Slice(start, end int) Buffer { + if start != 0 || end != 0 { + panic(fmt.Sprintf("slice bounds out of range [%d:%d] with length 0", start, end)) + } + return e +} + +func (e emptyBuffer) split(int) (left, right Buffer) { + return e, e +} + +func (e emptyBuffer) read([]byte) (int, Buffer) { + return 0, e +} + +// SliceBuffer is a Buffer implementation that wraps a byte slice. It provides +// methods for reading, splitting, and managing the byte slice. +type SliceBuffer []byte + +// ReadOnlyData returns the byte slice. +func (s SliceBuffer) ReadOnlyData() []byte { return s } + +// Ref is a noop implementation of Ref. +func (s SliceBuffer) Ref() {} + +// Free is a noop implementation of Free. +func (s SliceBuffer) Free() {} + +// Len is a noop implementation of Len. +func (s SliceBuffer) Len() int { return len(s) } + +// Slice returns a new SliceBuffer that is a view into the receiver from [start:end). +func (s SliceBuffer) Slice(start, end int) Buffer { return s[start:end] } + +func (s SliceBuffer) split(n int) (left, right Buffer) { + return s[:n], s[n:] +} + +func (s SliceBuffer) read(buf []byte) (int, Buffer) { + n := copy(buf, s) + if n == len(s) { + return n, nil + } + return n, s[n:] +} diff --git a/mem/buffers_test.go b/mem/buffers_test.go new file mode 100644 index 0000000..4f8eb31 --- /dev/null +++ b/mem/buffers_test.go @@ -0,0 +1,493 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package mem_test + +import ( + "bytes" + "fmt" + "slices" + "testing" + + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/mem" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + internal.SetBufferPoolingThresholdForTesting.(func(int))(0) + + grpctest.RunSubTests(t, s{}) +} + +// Tests that a buffer created with NewBuffer, which when later freed, invokes +// the free function with the correct data. +func (s) TestBuffer_NewBufferAndFree(t *testing.T) { + data := "abcd" + freed := false + freeF := poolFunc(func(got *[]byte) { + if !bytes.Equal(*got, []byte(data)) { + t.Fatalf("Free function called with bytes %s, want %s", string(*got), data) + } + freed = true + }) + + buf := newBuffer([]byte(data), freeF) + if got := buf.ReadOnlyData(); !bytes.Equal(got, []byte(data)) { + t.Fatalf("Buffer contains data %s, want %s", string(got), string(data)) + } + + // Verify that the free function is invoked when all references are freed. + buf.Free() + if !freed { + t.Fatalf("Buffer not freed") + } +} + +// Tests that a buffer created with NewBuffer, on which an additional reference +// is acquired, which when later freed, invokes the free function with the +// correct data, but only after all references are released. +func (s) TestBuffer_NewBufferRefAndFree(t *testing.T) { + data := "abcd" + freed := false + freeF := poolFunc(func(got *[]byte) { + if !bytes.Equal(*got, []byte(data)) { + t.Fatalf("Free function called with bytes %s, want %s", string(*got), string(data)) + } + freed = true + }) + + buf := newBuffer([]byte(data), freeF) + if got := buf.ReadOnlyData(); !bytes.Equal(got, []byte(data)) { + t.Fatalf("Buffer contains data %s, want %s", string(got), string(data)) + } + + buf.Ref() + if got := buf.ReadOnlyData(); !bytes.Equal(got, []byte(data)) { + t.Fatalf("New reference to the Buffer contains data %s, want %s", string(got), string(data)) + } + + // Verify that the free function is not invoked when all references are yet + // to be freed. + buf.Free() + if freed { + t.Fatalf("Free function called before all references freed") + } + + // Verify that the free function is invoked when all references are freed. + buf.Free() + if !freed { + t.Fatalf("Buffer not freed") + } +} + +func (s) TestBuffer_NewBufferHandlesShortBuffers(t *testing.T) { + const threshold = 100 + + // Update the pooling threshold, since that's what's being tested. + internal.SetBufferPoolingThresholdForTesting.(func(int))(threshold) + t.Cleanup(func() { + internal.SetBufferPoolingThresholdForTesting.(func(int))(0) + }) + + // Make a pool with a buffer whose capacity is larger than the pooling + // threshold, but whose length is less than the threshold. + b := make([]byte, threshold/2, threshold*2) + pool := &singleBufferPool{ + t: t, + data: &b, + } + + // Get a Buffer, then free it. If NewBuffer decided that the Buffer + // shouldn't get pooled, Free will be a noop and singleBufferPool will not + // have been updated. + mem.NewBuffer(&b, pool).Free() + + if pool.data != nil { + t.Fatalf("Buffer not returned to pool") + } +} + +func (s) TestBuffer_FreeAfterFree(t *testing.T) { + buf := newBuffer([]byte("abcd"), mem.NopBufferPool{}) + if buf.Len() != 4 { + t.Fatalf("Buffer length is %d, want 4", buf.Len()) + } + + // Ensure that a double free does panic. + buf.Free() + defer checkForPanic(t, "Cannot free freed buffer") + buf.Free() +} + +type singleBufferPool struct { + t *testing.T + data *[]byte +} + +func (s *singleBufferPool) Get(length int) *[]byte { + if len(*s.data) != length { + s.t.Fatalf("Invalid requested length, got %d want %d", length, len(*s.data)) + } + return s.data +} + +func (s *singleBufferPool) Put(b *[]byte) { + if s.data != b { + s.t.Fatalf("Wrong buffer returned to pool, got %p want %p", b, s.data) + } + s.data = nil +} + +// Tests that a buffer created with Copy, which when later freed, returns the underlying +// byte slice to the buffer pool. +func (s) TestBuffer_CopyAndFree(t *testing.T) { + data := []byte("abcd") + testPool := &singleBufferPool{ + t: t, + data: &data, + } + + buf := mem.Copy(data, testPool) + if got := buf.ReadOnlyData(); !bytes.Equal(got, data) { + t.Fatalf("Buffer contains data %s, want %s", string(got), string(data)) + } + + // Verify that the free function is invoked when all references are freed. + buf.Free() + if testPool.data != nil { + t.Fatalf("Buffer not freed") + } +} + +// Tests that a buffer created with Copy, on which an additional reference is +// acquired, which when later freed, returns the underlying byte slice to the +// buffer pool. +func (s) TestBuffer_CopyRefAndFree(t *testing.T) { + data := []byte("abcd") + testPool := &singleBufferPool{ + t: t, + data: &data, + } + + buf := mem.Copy(data, testPool) + if got := buf.ReadOnlyData(); !bytes.Equal(got, data) { + t.Fatalf("Buffer contains data %s, want %s", string(got), string(data)) + } + + buf.Ref() + if got := buf.ReadOnlyData(); !bytes.Equal(got, []byte(data)) { + t.Fatalf("New reference to the Buffer contains data %s, want %s", string(got), string(data)) + } + + // Verify that the free function is not invoked when all references are yet + // to be freed. + buf.Free() + if testPool.data == nil { + t.Fatalf("Free function called before all references freed") + } + + // Verify that the free function is invoked when all references are freed. + buf.Free() + if testPool.data != nil { + t.Fatalf("Free never called") + } +} + +func (s) TestBuffer_ReadOnlyDataAfterFree(t *testing.T) { + // Verify that reading before freeing does not panic. + buf := newBuffer([]byte("abcd"), mem.NopBufferPool{}) + buf.ReadOnlyData() + + buf.Free() + defer checkForPanic(t, "Cannot read freed buffer") + buf.ReadOnlyData() +} + +func (s) TestBuffer_RefAfterFree(t *testing.T) { + // Verify that acquiring a ref before freeing does not panic. + buf := newBuffer([]byte("abcd"), mem.NopBufferPool{}) + buf.Ref() + + // This first call should not panic and bring the ref counter down to 1 + buf.Free() + // This second call actually frees the buffer + buf.Free() + defer checkForPanic(t, "Cannot ref freed buffer") + buf.Ref() +} + +func (s) TestBuffer_SplitAfterFree(t *testing.T) { + // Verify that splitting before freeing does not panic. + buf := newBuffer([]byte("abcd"), mem.NopBufferPool{}) + buf, bufSplit := mem.SplitUnsafe(buf, 2) + + bufSplit.Free() + buf.Free() + defer checkForPanic(t, "Cannot split freed buffer") + mem.SplitUnsafe(buf, 2) +} + +type poolFunc func(*[]byte) + +func (p poolFunc) Get(int) *[]byte { + panic("Get should never be called") +} + +func (p poolFunc) Put(i *[]byte) { + p(i) +} + +func (s) TestBuffer_Split(t *testing.T) { + freed := make(chan struct{}) + data := []byte{1, 2, 3, 4} + buf := mem.NewBuffer(&data, poolFunc(func(*[]byte) { + close(freed) + })) + + buf, split1 := mem.SplitUnsafe(buf, 2) + if !bytes.Equal(buf.ReadOnlyData(), data[:2]) { + t.Fatalf("Buffer did not contain expected data. got %v, want %v", buf.ReadOnlyData(), data[:2]) + } + if !bytes.Equal(split1.ReadOnlyData(), data[2:]) { + t.Fatalf("Buffer did not contain expected data. got %v, want %v", split1.ReadOnlyData(), data[2:]) + } + + // Check that splitting the buffer more than once works as intended. + split1, split2 := mem.SplitUnsafe(split1, 1) + if !bytes.Equal(split1.ReadOnlyData(), data[2:3]) { + t.Fatalf("Buffer did not contain expected data. got %v, want %v", split1.ReadOnlyData(), data[2:3]) + } + if !bytes.Equal(split2.ReadOnlyData(), data[3:]) { + t.Fatalf("Buffer did not contain expected data. got %v, want %v", split2.ReadOnlyData(), data[3:]) + } + + // If any of the following frees actually free the buffer, the test will fail. + buf.Free() + split2.Free() + select { + case <-freed: + t.Fatalf("Freed too early") + default: + } + + split1.Free() + + select { + case <-freed: + default: + t.Fatalf("Still not freed") + } +} + +func (s) TestBuffer_Slice(t *testing.T) { + freed := make(chan struct{}) + data := []byte{1, 2, 3, 4} + buf := mem.NewBuffer(&data, poolFunc(func(*[]byte) { + close(freed) + })) + + // Slice the buffer and verify the data. + slice1 := buf.Slice(1, 3) + if !bytes.Equal(slice1.ReadOnlyData(), data[1:3]) { + t.Fatalf("Buffer did not contain expected data. got %v, want %v", slice1.ReadOnlyData(), data[1:3]) + } + + // Verify the original buffer is not modified. + if !bytes.Equal(buf.ReadOnlyData(), data) { + t.Fatalf("Buffer did not contain expected data. got %v, want %v", buf.ReadOnlyData(), data) + } + + // Slice the slice. + slice2 := slice1.Slice(0, 1) + if !bytes.Equal(slice2.ReadOnlyData(), data[1:2]) { + t.Fatalf("Buffer did not contain expected data. got %v, want %v", slice2.ReadOnlyData(), data[1:2]) + } + + // Free original and first slice — root should not be freed yet. + buf.Free() + slice1.Free() + select { + case <-freed: + t.Fatalf("Freed too early") + default: + } + + // The last slice keeps the root alive. + if !bytes.Equal(slice2.ReadOnlyData(), data[1:2]) { + t.Fatalf("Buffer did not contain expected data. got %v, want %v", slice2.ReadOnlyData(), data[1:2]) + } + + slice2.Free() + + select { + case <-freed: + default: + t.Fatalf("Still not freed") + } +} + +func (s) TestBuffer_SliceAfterFree(t *testing.T) { + buf := newBuffer([]byte("abcd"), mem.NopBufferPool{}) + buf.Free() + defer checkForPanic(t, "Cannot slice freed buffer") + buf.Slice(0, 2) +} + +type namedCtor struct { + name string + newBuf func([]byte) mem.Buffer +} + +var ctors = []namedCtor{ + {name: "pooled", newBuf: newPooledBuffer}, + {name: "slice", newBuf: newSliceBuf}, +} + +func (s) TestBuffer_SliceBasic(t *testing.T) { + type sliceCase struct { + start, end int + want []byte + } + cases := []sliceCase{ + {1, 3, []byte{2, 3}}, + {0, 4, []byte{1, 2, 3, 4}}, + {4, 4, []byte{}}, + } + for _, c := range ctors { + for _, tc := range cases { + t.Run(fmt.Sprintf("%s[%d:%d]", c.name, tc.start, tc.end), func(t *testing.T) { + buf := c.newBuf([]byte{1, 2, 3, 4}) + got := buf.Slice(tc.start, tc.end) + if !bytes.Equal(got.ReadOnlyData(), tc.want) { + t.Fatalf("Buffer did not contain expected data. got %v, want %v", got.ReadOnlyData(), tc.want) + } + }) + } + } +} + +func (s) TestBuffer_SliceSubslice(t *testing.T) { + for _, c := range ctors { + t.Run(c.name+" subslice", func(t *testing.T) { + buf := c.newBuf([]byte{1, 2, 3, 4}) + slice := buf.Slice(1, 3) + slice2 := slice.Slice(0, 1) + if !bytes.Equal(slice2.ReadOnlyData(), []byte{2}) { + t.Fatalf("Buffer did not contain expected data. got %v, want %v", buf.ReadOnlyData(), []byte{2}) + } + }) + } +} + +func (s) TestBuffer_SliceEmpty(t *testing.T) { + allCtors := append(slices.Clip(ctors), namedCtor{name: "empty", newBuf: newEmptyBuf}) + for _, c := range allCtors { + t.Run(c.name, func(t *testing.T) { + buf := c.newBuf([]byte{1, 2, 3, 4}) + got := buf.Slice(0, 0) + if !bytes.Equal(got.ReadOnlyData(), nil) { + t.Fatalf("Buffer did not contain expected data. got %v, want %v", got.ReadOnlyData(), nil) + } + }) + } +} + +func (s) TestBuffer_SliceBoundsCheck(t *testing.T) { + type panicCase struct { + name string + start, end int + } + nonEmptyCases := []panicCase{ + {"end_out_of_bounds", 0, 5}, + {"start_negative", -1, 3}, + {"start_greater_than_end", 3, 0}, + } + tests := []struct { + namedCtor + panicCases []panicCase + }{ + { + namedCtor: namedCtor{ + name: "buffer", + newBuf: newPooledBuffer, + }, + panicCases: nonEmptyCases, + }, + { + namedCtor: namedCtor{ + name: "SliceBuffer", + newBuf: newSliceBuf, + }, + panicCases: nonEmptyCases, + }, + { + namedCtor: namedCtor{ + name: "emptyBuffer", + newBuf: newEmptyBuf, + }, + panicCases: []panicCase{ + {"end_out_of_bounds", 0, 1}, + {"start_negative", -1, 0}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for _, pc := range tt.panicCases { + t.Run(pc.name, func(t *testing.T) { + buf := tt.newBuf([]byte{1, 2, 3, 4}) + defer func() { + if recover() == nil { + t.Fatalf("Slice(%d, %d) did not panic", pc.start, pc.end) + } + }() + buf.Slice(pc.start, pc.end) + }) + } + }) + } +} + +func newPooledBuffer(data []byte) mem.Buffer { + return newBuffer(data, mem.NopBufferPool{}) +} + +func newSliceBuf(data []byte) mem.Buffer { + return mem.SliceBuffer(data) +} + +func newEmptyBuf([]byte) mem.Buffer { + var bs mem.BufferSlice + return bs.MaterializeToBuffer(mem.NopBufferPool{}) +} + +func checkForPanic(t *testing.T, wantErr string) { + t.Helper() + r := recover() + if r == nil { + t.Fatalf("Use after free did not panic") + } + if msg, ok := r.(string); !ok || msg != wantErr { + t.Fatalf("panic called with %v, want %s", r, wantErr) + } +} diff --git a/metadata/metadata.go b/metadata/metadata.go new file mode 100644 index 0000000..a3090ff --- /dev/null +++ b/metadata/metadata.go @@ -0,0 +1,426 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package metadata define the structure of the metadata supported by gRPC library. +// Please refer to https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md +// for more information about custom-metadata. +package metadata // import "google.golang.org/grpc/metadata" + +import ( + "context" + "fmt" + "sort" + "strings" + + "google.golang.org/grpc/internal" +) + +func init() { + internal.FromOutgoingContextRaw = fromOutgoingContextRaw +} + +// DecodeKeyValue returns k, v, nil. +// +// Deprecated: use k and v directly instead. +func DecodeKeyValue(k, v string) (string, string, error) { + return k, v, nil +} + +// MD is a mapping from metadata keys to values. Users should use the following +// two convenience functions New and Pairs to generate MD. +type MD map[string][]string + +// New creates an MD from a given key-value map. +// +// Only the following ASCII characters are allowed in keys: +// - digits: 0-9 +// - uppercase letters: A-Z (normalized to lower) +// - lowercase letters: a-z +// - special characters: -_. +// +// Uppercase letters are automatically converted to lowercase. +// +// Keys beginning with "grpc-" are reserved for grpc-internal use only and may +// result in errors if set in metadata. +func New(m map[string]string) MD { + md := make(MD, len(m)) + for k, val := range m { + key := strings.ToLower(k) + md[key] = append(md[key], val) + } + return md +} + +// Pairs returns an MD formed by the mapping of key, value ... +// Pairs panics if len(kv) is odd. +// +// Only the following ASCII characters are allowed in keys: +// - digits: 0-9 +// - uppercase letters: A-Z (normalized to lower) +// - lowercase letters: a-z +// - special characters: -_. +// +// Uppercase letters are automatically converted to lowercase. +// +// Keys beginning with "grpc-" are reserved for grpc-internal use only and may +// result in errors if set in metadata. +func Pairs(kv ...string) MD { + if len(kv)%2 == 1 { + panic(fmt.Sprintf("metadata: Pairs got the odd number of input pairs for metadata: %d", len(kv))) + } + md := make(MD, len(kv)/2) + for i := 0; i < len(kv); i += 2 { + key := strings.ToLower(kv[i]) + md[key] = append(md[key], kv[i+1]) + } + return md +} + +// loggableMetadataKeys is the set of metadata keys whose values are known not +// to carry credentials or other sensitive data, and are therefore safe to print +// verbatim in String. Values for any key not in this set are redacted. Keys are +// in metadata's canonical lowercase form. +// +// The list is intentionally restricted to standardized gRPC and HTTP/2 protocol +// headers. Everything else, including all application-defined keys, is +// censored. +// Note that this list might change as we add/remove support for +// metadata types. +var loggableMetadataKeys = map[string]bool{ + "content-type": true, + "te": true, + "user-agent": true, + "grpc-encoding": true, + "grpc-accept-encoding": true, + "grpc-timeout": true, + "grpc-status": true, + "grpc-message-type": true, + "grpc-previous-rpc-attempts": true, + "grpc-retry-pushback-ms": true, +} + +// String implements fmt.Stringer to allow metadata to be printed when stored in +// a context. +// +// To avoid accidentally leaking credentials or other sensitive data (for +// example via log(md), or by logging a value that happens to contain metadata), +// String reports only keys on an allowlist of standardized, non-sensitive +// protocol headers (see loggableMetadataKeys). Every other key is omitted +// entirely, along with its values, because a key name may itself be sensitive; +// only the number of omitted keys is reported, as "". This is a +// best-effort guard against accidents, not a security boundary: a caller that +// genuinely wants the full contents can still print map[string][]string(md) +// directly. +// +// Note that this only affects verbs that use the Stringer, such as %v and %s. +// The %#v verb prints the underlying map with all values and is not redacted. +// Users should not rely on the output of this method to be stable +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a later +// release. +func (md MD) String() string { + keys := make([]string, 0, len(md)) + for k := range md { + if loggableMetadataKeys[strings.ToLower(k)] { + keys = append(keys, k) + } + } + sort.Strings(keys) + + var sb strings.Builder + sb.WriteString("map[") + for i, k := range keys { + if i > 0 { + sb.WriteByte(' ') + } + fmt.Fprintf(&sb, "%s:%v", k, md[k]) + } + if redacted := len(md) - len(keys); redacted > 0 { + if len(keys) > 0 { + sb.WriteByte(' ') + } + fmt.Fprintf(&sb, "<%d redacted>", redacted) + } + sb.WriteByte(']') + return sb.String() +} + +// Len returns the number of items in md. +func (md MD) Len() int { + return len(md) +} + +// Copy returns a copy of md. +func (md MD) Copy() MD { + out := make(MD, len(md)) + for k, v := range md { + out[k] = copyOf(v) + } + return out +} + +// Get obtains the values for a given key. +// +// k is converted to lowercase before searching in md. +func (md MD) Get(k string) []string { + k = strings.ToLower(k) + return md[k] +} + +// Set sets the value of a given key with a slice of values. +// +// k is converted to lowercase before storing in md. +func (md MD) Set(k string, vals ...string) { + if len(vals) == 0 { + return + } + k = strings.ToLower(k) + md[k] = vals +} + +// Append adds the values to key k, not overwriting what was already stored at +// that key. +// +// k is converted to lowercase before storing in md. +func (md MD) Append(k string, vals ...string) { + if len(vals) == 0 { + return + } + k = strings.ToLower(k) + md[k] = append(md[k], vals...) +} + +// Delete removes the values for a given key k which is converted to lowercase +// before removing it from md. +func (md MD) Delete(k string) { + k = strings.ToLower(k) + delete(md, k) +} + +// Join joins any number of mds into a single MD. +// +// The order of values for each key is determined by the order in which the mds +// containing those values are presented to Join. +func Join(mds ...MD) MD { + out := MD{} + for _, md := range mds { + for k, v := range md { + out[k] = append(out[k], v...) + } + } + return out +} + +type mdIncomingKey struct{} +type mdOutgoingKey struct{} + +// NewIncomingContext creates a new context with incoming md attached. md must +// not be modified after calling this function. +func NewIncomingContext(ctx context.Context, md MD) context.Context { + return context.WithValue(ctx, mdIncomingKey{}, md) +} + +// NewOutgoingContext creates a new context with outgoing md attached. If used +// in conjunction with AppendToOutgoingContext, NewOutgoingContext will +// overwrite any previously-appended metadata. md must not be modified after +// calling this function. +func NewOutgoingContext(ctx context.Context, md MD) context.Context { + return context.WithValue(ctx, mdOutgoingKey{}, rawMD{md: md}) +} + +// AppendToOutgoingContext returns a new context with the provided kv merged +// with any existing metadata in the context. Please refer to the documentation +// of Pairs for a description of kv. +func AppendToOutgoingContext(ctx context.Context, kv ...string) context.Context { + if len(kv)%2 == 1 { + panic(fmt.Sprintf("metadata: AppendToOutgoingContext got an odd number of input pairs for metadata: %d", len(kv))) + } + md, _ := ctx.Value(mdOutgoingKey{}).(rawMD) + added := make([][]string, len(md.added)+1) + copy(added, md.added) + kvCopy := make([]string, 0, len(kv)) + for i := 0; i < len(kv); i += 2 { + kvCopy = append(kvCopy, strings.ToLower(kv[i]), kv[i+1]) + } + added[len(added)-1] = kvCopy + return context.WithValue(ctx, mdOutgoingKey{}, rawMD{md: md.md, added: added}) +} + +// FromIncomingContext returns the incoming metadata in ctx if it exists. +// +// All keys in the returned MD are lowercase. +func FromIncomingContext(ctx context.Context) (MD, bool) { + md, ok := ctx.Value(mdIncomingKey{}).(MD) + if !ok { + return nil, false + } + out := make(MD, len(md)) + for k, v := range md { + // We need to manually convert all keys to lower case, because MD is a + // map, and there's no guarantee that the MD attached to the context is + // created using our helper functions. + key := strings.ToLower(k) + out[key] = copyOf(v) + } + return out, true +} + +// ValueFromIncomingContext returns the metadata value corresponding to the metadata +// key from the incoming metadata if it exists. Keys are matched in a case insensitive +// manner. +func ValueFromIncomingContext(ctx context.Context, key string) []string { + md, ok := ctx.Value(mdIncomingKey{}).(MD) + if !ok { + return nil + } + + if v, ok := md[key]; ok { + return copyOf(v) + } + for k, v := range md { + // Case insensitive comparison: MD is a map, and there's no guarantee + // that the MD attached to the context is created using our helper + // functions. + if strings.EqualFold(k, key) { + return copyOf(v) + } + } + return nil +} + +func copyOf(v []string) []string { + vals := make([]string, len(v)) + copy(vals, v) + return vals +} + +// fromOutgoingContextRaw returns the un-merged, intermediary contents of rawMD. +// +// Remember to perform strings.ToLower on the keys, for both the returned MD (MD +// is a map, there's no guarantee it's created using our helper functions) and +// the extra kv pairs (AppendToOutgoingContext doesn't turn them into +// lowercase). +func fromOutgoingContextRaw(ctx context.Context) (MD, [][]string, bool) { + raw, ok := ctx.Value(mdOutgoingKey{}).(rawMD) + if !ok { + return nil, nil, false + } + + return raw.md, raw.added, true +} + +// FromOutgoingContext returns the outgoing metadata in ctx if it exists. +// +// All keys in the returned MD are lowercase. +func FromOutgoingContext(ctx context.Context) (MD, bool) { + raw, ok := ctx.Value(mdOutgoingKey{}).(rawMD) + if !ok { + return nil, false + } + + mdSize := len(raw.md) + for i := range raw.added { + mdSize += len(raw.added[i]) / 2 + } + + out := make(MD, mdSize) + for k, v := range raw.md { + // We need to manually convert all keys to lower case, because MD is a + // map, and there's no guarantee that the MD attached to the context is + // created using our helper functions. + key := strings.ToLower(k) + out[key] = copyOf(v) + } + for _, added := range raw.added { + if len(added)%2 == 1 { + panic(fmt.Sprintf("metadata: FromOutgoingContext got an odd number of input pairs for metadata: %d", len(added))) + } + + for i := 0; i < len(added); i += 2 { + key := strings.ToLower(added[i]) + out[key] = append(out[key], added[i+1]) + } + } + return out, ok +} + +// ValueFromOutgoingContext returns the metadata value corresponding to the +// metadata key from the outgoing metadata if it exists. Keys are matched in a +// case-insensitive manner. +// +// Unlike FromOutgoingContext, this does not copy the entire outgoing metadata. +func ValueFromOutgoingContext(ctx context.Context, key string) []string { + raw, ok := ctx.Value(mdOutgoingKey{}).(rawMD) + if !ok { + return nil + } + + key = strings.ToLower(key) + // Unlike ValueFromIncomingContext, this can't return as soon as raw.md is + // checked: raw.added still needs to be walked and accumulated regardless. + // The else below prevents that from overwriting an exact match with an + // unrelated case-insensitive one. + var matchedMD []string + if v, ok := raw.md[key]; ok { + matchedMD = v + } else { + for k, v := range raw.md { + // Case insensitive comparison: MD is a map, and there's no + // guarantee that the MD attached to the context is created using + // our helper functions. + if strings.EqualFold(k, key) { + matchedMD = v + break + } + } + } + + // Defer allocating vals until raw.added actually has a match, so a match + // found only in raw.md doesn't pay for both a copyOf and a growing + // append. + var vals []string + for _, added := range raw.added { + if len(added)%2 == 1 { + panic(fmt.Sprintf("metadata: ValueFromOutgoingContext got an odd number of input pairs for metadata: %d", len(added))) + } + + // Case insensitive, like FromOutgoingContext: added isn't guaranteed + // lowercase, though AppendToOutgoingContext already lowercases it in + // practice, so try the cheap exact match first. + for i := 0; i < len(added); i += 2 { + if added[i] == key || strings.EqualFold(added[i], key) { + if vals == nil { + vals = make([]string, 0, len(matchedMD)+1) + vals = append(vals, matchedMD...) + } + vals = append(vals, added[i+1]) + } + } + } + + if vals == nil && matchedMD != nil { + return copyOf(matchedMD) + } + return vals +} + +type rawMD struct { + md MD + added [][]string +} diff --git a/metadata/metadata_test.go b/metadata/metadata_test.go new file mode 100644 index 0000000..5fc1f0d --- /dev/null +++ b/metadata/metadata_test.go @@ -0,0 +1,677 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package metadata + +import ( + "context" + "fmt" + "reflect" + "strconv" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestPairsMD(t *testing.T) { + for _, test := range []struct { + // input + kv []string + // output + md MD + }{ + {[]string{}, MD{}}, + {[]string{"k1", "v1", "k1", "v2"}, MD{"k1": []string{"v1", "v2"}}}, + } { + md := Pairs(test.kv...) + if !reflect.DeepEqual(md, test.md) { + t.Fatalf("Pairs(%v) = %v, want %v", test.kv, md, test.md) + } + } +} + +func (s) TestCopy(t *testing.T) { + const key, val = "key", "val" + orig := Pairs(key, val) + cpy := orig.Copy() + if !reflect.DeepEqual(orig, cpy) { + t.Errorf("copied value not equal to the original, got %v, want %v", cpy, orig) + } + orig[key][0] = "foo" + if v := cpy[key][0]; v != val { + t.Errorf("change in original should not affect copy, got %q, want %q", v, val) + } +} + +func (s) TestJoin(t *testing.T) { + for _, test := range []struct { + mds []MD + want MD + }{ + {[]MD{}, MD{}}, + {[]MD{Pairs("foo", "bar")}, Pairs("foo", "bar")}, + {[]MD{Pairs("foo", "bar"), Pairs("foo", "baz")}, Pairs("foo", "bar", "foo", "baz")}, + {[]MD{Pairs("foo", "bar"), Pairs("foo", "baz"), Pairs("zip", "zap")}, Pairs("foo", "bar", "foo", "baz", "zip", "zap")}, + } { + md := Join(test.mds...) + if !reflect.DeepEqual(md, test.want) { + t.Errorf("context's metadata is %v, want %v", md, test.want) + } + } +} + +func (s) TestGet(t *testing.T) { + for _, test := range []struct { + md MD + key string + wantVals []string + }{ + {md: Pairs("My-Optional-Header", "42"), key: "My-Optional-Header", wantVals: []string{"42"}}, + {md: Pairs("Header", "42", "Header", "43", "Header", "44", "other", "1"), key: "HEADER", wantVals: []string{"42", "43", "44"}}, + {md: Pairs("HEADER", "10"), key: "HEADER", wantVals: []string{"10"}}, + } { + vals := test.md.Get(test.key) + if !reflect.DeepEqual(vals, test.wantVals) { + t.Errorf("value of metadata %v is %v, want %v", test.key, vals, test.wantVals) + } + } +} + +func (s) TestSet(t *testing.T) { + for _, test := range []struct { + md MD + setKey string + setVals []string + want MD + }{ + { + md: Pairs("My-Optional-Header", "42", "other-key", "999"), + setKey: "Other-Key", + setVals: []string{"1"}, + want: Pairs("my-optional-header", "42", "other-key", "1"), + }, + { + md: Pairs("My-Optional-Header", "42"), + setKey: "Other-Key", + setVals: []string{"1", "2", "3"}, + want: Pairs("my-optional-header", "42", "other-key", "1", "other-key", "2", "other-key", "3"), + }, + { + md: Pairs("My-Optional-Header", "42"), + setKey: "Other-Key", + setVals: []string{}, + want: Pairs("my-optional-header", "42"), + }, + } { + test.md.Set(test.setKey, test.setVals...) + if !reflect.DeepEqual(test.md, test.want) { + t.Errorf("value of metadata is %v, want %v", test.md, test.want) + } + } +} + +func (s) TestAppend(t *testing.T) { + for _, test := range []struct { + md MD + appendKey string + appendVals []string + want MD + }{ + { + md: Pairs("My-Optional-Header", "42"), + appendKey: "Other-Key", + appendVals: []string{"1"}, + want: Pairs("my-optional-header", "42", "other-key", "1"), + }, + { + md: Pairs("My-Optional-Header", "42"), + appendKey: "my-OptIoNal-HeAder", + appendVals: []string{"1", "2", "3"}, + want: Pairs("my-optional-header", "42", "my-optional-header", "1", + "my-optional-header", "2", "my-optional-header", "3"), + }, + { + md: Pairs("My-Optional-Header", "42"), + appendKey: "my-OptIoNal-HeAder", + appendVals: []string{}, + want: Pairs("my-optional-header", "42"), + }, + } { + test.md.Append(test.appendKey, test.appendVals...) + if !reflect.DeepEqual(test.md, test.want) { + t.Errorf("value of metadata is %v, want %v", test.md, test.want) + } + } +} + +func (s) TestDelete(t *testing.T) { + for _, test := range []struct { + md MD + deleteKey string + want MD + }{ + { + md: Pairs("My-Optional-Header", "42"), + deleteKey: "My-Optional-Header", + want: Pairs(), + }, + { + md: Pairs("My-Optional-Header", "42"), + deleteKey: "Other-Key", + want: Pairs("my-optional-header", "42"), + }, + { + md: Pairs("My-Optional-Header", "42"), + deleteKey: "my-OptIoNal-HeAder", + want: Pairs(), + }, + } { + test.md.Delete(test.deleteKey) + if !reflect.DeepEqual(test.md, test.want) { + t.Errorf("value of metadata is %v, want %v", test.md, test.want) + } + } +} + +func (s) TestFromIncomingContext(t *testing.T) { + md := Pairs( + "X-My-Header-1", "42", + ) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Verify that we lowercase if callers directly modify md + md["X-INCORRECT-UPPERCASE"] = []string{"foo"} + ctx = NewIncomingContext(ctx, md) + + result, found := FromIncomingContext(ctx) + if !found { + t.Fatal("FromIncomingContext must return metadata") + } + expected := MD{ + "x-my-header-1": []string{"42"}, + "x-incorrect-uppercase": []string{"foo"}, + } + if !reflect.DeepEqual(result, expected) { + t.Errorf("FromIncomingContext returned %#v, expected %#v", result, expected) + } + + // ensure modifying result does not modify the value in the context + result["new_key"] = []string{"foo"} + result["x-my-header-1"][0] = "mutated" + + result2, found := FromIncomingContext(ctx) + if !found { + t.Fatal("FromIncomingContext must return metadata") + } + if !reflect.DeepEqual(result2, expected) { + t.Errorf("FromIncomingContext after modifications returned %#v, expected %#v", result2, expected) + } +} + +func (s) TestValueFromIncomingContext(t *testing.T) { + md := Pairs( + "X-My-Header-1", "42", + "X-My-Header-2", "43-1", + "X-My-Header-2", "43-2", + "x-my-header-3", "44", + ) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Verify that we lowercase if callers directly modify md + md["X-INCORRECT-UPPERCASE"] = []string{"foo"} + ctx = NewIncomingContext(ctx, md) + + for _, test := range []struct { + key string + want []string + }{ + { + key: "x-my-header-1", + want: []string{"42"}, + }, + { + key: "x-my-header-2", + want: []string{"43-1", "43-2"}, + }, + { + key: "x-my-header-3", + want: []string{"44"}, + }, + { + key: "x-unknown", + want: nil, + }, + { + key: "x-incorrect-uppercase", + want: []string{"foo"}, + }, + } { + v := ValueFromIncomingContext(ctx, test.key) + if !reflect.DeepEqual(v, test.want) { + t.Errorf("value of metadata is %v, want %v", v, test.want) + } + } +} + +func (s) TestAppendToOutgoingContext(t *testing.T) { + // Pre-existing metadata + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx := NewOutgoingContext(tCtx, Pairs("k1", "v1", "k2", "v2")) + ctx = AppendToOutgoingContext(ctx, "k1", "v3") + ctx = AppendToOutgoingContext(ctx, "k1", "v4") + md, ok := FromOutgoingContext(ctx) + if !ok { + t.Errorf("Expected MD to exist in ctx, but got none") + } + want := Pairs("k1", "v1", "k1", "v3", "k1", "v4", "k2", "v2") + if !reflect.DeepEqual(md, want) { + t.Errorf("context's metadata is %v, want %v", md, want) + } + + // No existing metadata + ctx = AppendToOutgoingContext(tCtx, "k1", "v1") + md, ok = FromOutgoingContext(ctx) + if !ok { + t.Errorf("Expected MD to exist in ctx, but got none") + } + want = Pairs("k1", "v1") + if !reflect.DeepEqual(md, want) { + t.Errorf("context's metadata is %v, want %v", md, want) + } +} + +func (s) TestAppendToOutgoingContext_Repeated(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + for i := 0; i < 100; i = i + 2 { + ctx1 := AppendToOutgoingContext(ctx, "k", strconv.Itoa(i)) + ctx2 := AppendToOutgoingContext(ctx, "k", strconv.Itoa(i+1)) + + md1, _ := FromOutgoingContext(ctx1) + md2, _ := FromOutgoingContext(ctx2) + + if reflect.DeepEqual(md1, md2) { + t.Fatalf("md1, md2 = %v, %v; should not be equal", md1, md2) + } + + ctx = ctx1 + } +} + +func (s) TestAppendToOutgoingContext_FromKVSlice(t *testing.T) { + const k, v = "a", "b" + kv := []string{k, v} + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx := AppendToOutgoingContext(tCtx, kv...) + md, _ := FromOutgoingContext(ctx) + if md[k][0] != v { + t.Fatalf("md[%q] = %q; want %q", k, md[k], v) + } + kv[1] = "xxx" + md, _ = FromOutgoingContext(ctx) + if md[k][0] != v { + t.Fatalf("md[%q] = %q; want %q", k, md[k], v) + } +} + +func (s) TestValueFromOutgoingContext(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + md := Pairs( + "X-My-Header-1", "42", + "x-my-header-3", "44", + "k1", "v1", + "k2", "v2", + ) + // Verify that we match case-insensitively even if callers directly + // modify md. + md["X-INCORRECT-UPPERCASE"] = []string{"foo"} + ctx = NewOutgoingContext(ctx, md) + ctx = AppendToOutgoingContext(ctx, "x-my-header-2", "43-1") + ctx = AppendToOutgoingContext(ctx, "X-My-Header-2", "43-2") + ctx = AppendToOutgoingContext(ctx, "k1", "v3") + ctx = AppendToOutgoingContext(ctx, "k1", "v4") + + for _, test := range []struct { + key string + want []string + }{ + { + key: "x-my-header-1", + want: []string{"42"}, + }, + { + // Split across two AppendToOutgoingContext calls (raw.added) — + // must accumulate across both, in call order. + key: "x-my-header-2", + want: []string{"43-1", "43-2"}, + }, + { + key: "x-my-header-3", + want: []string{"44"}, + }, + { + key: "x-unknown", + want: nil, + }, + { + key: "x-incorrect-uppercase", + want: []string{"foo"}, + }, + { + // Present in both raw.md and two later AppendToOutgoingContext + // calls — must accumulate across all three, in order. + key: "k1", + want: []string{"v1", "v3", "v4"}, + }, + } { + v := ValueFromOutgoingContext(ctx, test.key) + if diff := cmp.Diff(test.want, v); diff != "" { + t.Errorf("ValueFromOutgoingContext(ctx, %q) returned unexpected diff (-want +got):\n%s", test.key, diff) + } + } +} + +func (s) TestValueFromOutgoingContext_NoMetadata(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if v := ValueFromOutgoingContext(ctx, "x-my-header-1"); v != nil { + t.Errorf("ValueFromOutgoingContext on context with no outgoing metadata = %v, want nil", v) + } +} + +func (s) TestValueFromOutgoingContext_AddedCaseInsensitive(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = context.WithValue(ctx, mdOutgoingKey{}, rawMD{added: [][]string{{"X-My-Header", "42"}}}) + + v := ValueFromOutgoingContext(ctx, "x-my-header") + if diff := cmp.Diff([]string{"42"}, v); diff != "" { + t.Errorf("ValueFromOutgoingContext(ctx, \"x-my-header\") returned unexpected diff (-want +got):\n%s", diff) + } +} + +func (s) TestValueFromOutgoingContext_PanicsOnOddPairs(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Fatal("ValueFromOutgoingContext did not panic on an odd number of pairs in added") + } + }() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = context.WithValue(ctx, mdOutgoingKey{}, rawMD{added: [][]string{{"key-without-a-value"}}}) + + ValueFromOutgoingContext(ctx, "key-without-a-value") +} + +// Old/slow approach to adding metadata to context +func Benchmark_AddingMetadata_ContextManipulationApproach(b *testing.B) { + // TODO: Add in N=1-100 tests once Go1.6 support is removed. + const num = 10 + for n := 0; n < b.N; n++ { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for i := 0; i < num; i++ { + md, _ := FromOutgoingContext(ctx) + NewOutgoingContext(ctx, Join(Pairs("k1", "v1", "k2", "v2"), md)) + } + } +} + +// Newer/faster approach to adding metadata to context +func BenchmarkAppendToOutgoingContext(b *testing.B) { + const num = 10 + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for n := 0; n < b.N; n++ { + for i := 0; i < num; i++ { + ctx = AppendToOutgoingContext(ctx, "k1", "v1", "k2", "v2") + } + } +} + +func BenchmarkFromOutgoingContext(b *testing.B) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = NewOutgoingContext(ctx, MD{"k3": {"v3", "v4"}}) + ctx = AppendToOutgoingContext(ctx, "k1", "v1", "k2", "v2") + + for n := 0; n < b.N; n++ { + FromOutgoingContext(ctx) + } +} + +// Reading one key out of many staged headers +func BenchmarkValueFromOutgoingContextVsFromOutgoingContext(b *testing.B) { + for _, numHeaders := range []int{1, 10, 50} { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + md := MD{} + for i := 0; i < numHeaders; i++ { + md[strconv.Itoa(i)] = []string{strconv.Itoa(i)} + } + ctx = NewOutgoingContext(ctx, md) + ctx = AppendToOutgoingContext(ctx, "target-key", "target-value") + + b.Run(fmt.Sprintf("FromOutgoingContext/n=%d", numHeaders), func(b *testing.B) { + for b.Loop() { + FromOutgoingContext(ctx) + } + }) + + b.Run(fmt.Sprintf("ValueFromOutgoingContext/n=%d", numHeaders), func(b *testing.B) { + for b.Loop() { + ValueFromOutgoingContext(ctx, "target-key") + } + }) + } +} + +func BenchmarkFromIncomingContext(b *testing.B) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + md := Pairs("X-My-Header-1", "42") + ctx = NewIncomingContext(ctx, md) + + b.ResetTimer() + for n := 0; n < b.N; n++ { + FromIncomingContext(ctx) + } +} + +func BenchmarkValueFromIncomingContext(b *testing.B) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + md := Pairs("X-My-Header-1", "42") + ctx = NewIncomingContext(ctx, md) + + b.Run("key-found", func(b *testing.B) { + for n := 0; n < b.N; n++ { + result := ValueFromIncomingContext(ctx, "x-my-header-1") + if len(result) != 1 { + b.Fatal("ensures not optimized away") + } + } + }) + + b.Run("key-not-found", func(b *testing.B) { + for n := 0; n < b.N; n++ { + result := ValueFromIncomingContext(ctx, "key-not-found") + if len(result) != 0 { + b.Fatal("ensures not optimized away") + } + } + }) +} + +func BenchmarkValueFromOutgoingContext(b *testing.B) { + // Realistic-length gRPC metadata keys, not "k1"/"k2"/"k3": short keys + // understate the cost of strings.EqualFold relative to a cheap exact + // match, which matters for the key-found cases below. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = NewOutgoingContext(ctx, MD{"content-type": {"application/grpc"}}) + ctx = AppendToOutgoingContext(ctx, "grpc-timeout", "10S", "x-request-id", "abc-def-123") + + b.Run("key-found", func(b *testing.B) { + for b.Loop() { + ValueFromOutgoingContext(ctx, "grpc-timeout") + } + }) + + b.Run("key-not-found", func(b *testing.B) { + for b.Loop() { + ValueFromOutgoingContext(ctx, "x-does-not-exist") + } + }) + + // Key present in both raw.md ("content-type") and raw.added (appended + // below) — exercises the accumulate-across-both path, not just + // append-to-nil. + bothCtx := AppendToOutgoingContext(ctx, "content-type", "application/grpc+proto") + b.Run("key-found-in-md-and-added", func(b *testing.B) { + for b.Loop() { + ValueFromOutgoingContext(bothCtx, "content-type") + } + }) +} + +// TestString verifies that String shows keys and values only for keys known to +// be safe to log, omits every other key (including its name) and reports only +// the count of omitted keys, and never panics on nil/empty inputs. +func (s) TestString(t *testing.T) { + tests := []struct { + name string + md MD + want string + }{ + { + name: "empty", + md: MD{}, + want: "map[]", + }, + { + name: "nil", + md: nil, + want: "map[]", + }, + { + name: "safe-key-shown", + md: Pairs("content-type", "application/grpc"), + want: "map[content-type:[application/grpc]]", + }, + { + // A safe key with multiple values shows all of them. + name: "safe-key-multi-value", + md: Pairs("grpc-accept-encoding", "gzip", "grpc-accept-encoding", "snappy"), + want: "map[grpc-accept-encoding:[gzip snappy]]", + }, + { + // nil/empty value slices must not panic; a safe key shows []. + name: "safe-key-nil-value", + md: MD{"content-type": nil}, + want: "map[content-type:[]]", + }, + { + // A redacted key with a nil value slice still redacts, no panic. + name: "redacted-key-nil-value", + md: MD{"authorization": nil}, + want: "map[<1 redacted>]", + }, + { + // Bracket characters in a safe value are passed through verbatim. + name: "safe-value-with-brackets", + md: MD{"content-type": []string{"a]b ["}}, + want: "map[content-type:[a]b []]", + }, + { + // Keys differing only in case are sorted byte-wise (uppercase + // first); both still match the case-insensitive safelist. + name: "case-only-collision", + md: MD{"Content-Type": []string{"a"}, "content-type": []string{"b"}}, + want: "map[Content-Type:[a] content-type:[b]]", + }, + { + // A sensitive key is omitted entirely, including its name; only the + // count is reported. + name: "sensitive-key-redacted", + md: Pairs("authorization", "Bearer super-secret-token"), + want: "map[<1 redacted>]", + }, + { + // Application-defined keys are omitted; the key name (which may + // itself be sensitive) is never shown. + name: "custom-key-redacted", + md: Pairs("x-my-app-secret", "hunter2"), + want: "map[<1 redacted>]", + }, + { + // A single key with multiple values counts as one redacted key, and + // the value count is not leaked. + name: "multi-value-redacted", + md: Pairs("cookie", "a=1", "cookie", "b=2"), + want: "map[<1 redacted>]", + }, + { + // Multiple redacted keys are reported by count only. + name: "multiple-redacted-keys", + md: Pairs("authorization", "tok", "cookie", "x"), + want: "map[<2 redacted>]", + }, + { + // Safe keys are shown (sorted) and non-safe keys are summarized by a + // trailing count. + name: "mixed-safe-and-redacted", + md: Pairs("grpc-encoding", "gzip", "authorization", "tok", "user-agent", "grpc-go/x"), + want: "map[grpc-encoding:[gzip] user-agent:[grpc-go/x] <1 redacted>]", + }, + { + // Keys built via map literal are not canonicalized to lowercase; + // the safelist match is case-insensitive so a safe key is still + // shown. + name: "uppercase-safe-key-shown", + md: MD{"Content-Type": []string{"application/grpc"}}, + want: "map[Content-Type:[application/grpc]]", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := test.md.String(); got != test.want { + t.Errorf("MD.String() = %q, want %q", got, test.want) + } + }) + } +} diff --git a/orca/call_metrics.go b/orca/call_metrics.go new file mode 100644 index 0000000..9ae7721 --- /dev/null +++ b/orca/call_metrics.go @@ -0,0 +1,196 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca + +import ( + "context" + "sync" + + "google.golang.org/grpc" + grpcinternal "google.golang.org/grpc/internal" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/orca/internal" + "google.golang.org/protobuf/proto" +) + +// CallMetricsRecorder allows a service method handler to record per-RPC +// metrics. It contains all utilization-based metrics from +// ServerMetricsRecorder as well as additional request cost metrics. +type CallMetricsRecorder interface { + ServerMetricsRecorder + + // SetRequestCost sets the relevant server metric. + SetRequestCost(name string, val float64) + // DeleteRequestCost deletes the relevant server metric to prevent it + // from being sent. + DeleteRequestCost(name string) + + // SetNamedMetric sets the relevant server metric. + SetNamedMetric(name string, val float64) + // DeleteNamedMetric deletes the relevant server metric to prevent it + // from being sent. + DeleteNamedMetric(name string) +} + +type callMetricsRecorderCtxKey struct{} + +// CallMetricsRecorderFromContext returns the RPC-specific custom metrics +// recorder embedded in the provided RPC context. +// +// Returns nil if no custom metrics recorder is found in the provided context, +// which will be the case when custom metrics reporting is not enabled. +func CallMetricsRecorderFromContext(ctx context.Context) CallMetricsRecorder { + rw, ok := ctx.Value(callMetricsRecorderCtxKey{}).(*recorderWrapper) + if !ok { + return nil + } + return rw.recorder() +} + +// recorderWrapper is a wrapper around a CallMetricsRecorder to ensure that +// concurrent calls to CallMetricsRecorderFromContext() results in only one +// allocation of the underlying metrics recorder, while also allowing for lazy +// initialization of the recorder itself. +type recorderWrapper struct { + once sync.Once + r CallMetricsRecorder + smp ServerMetricsProvider +} + +func (rw *recorderWrapper) recorder() CallMetricsRecorder { + rw.once.Do(func() { + rw.r = newServerMetricsRecorder() + }) + return rw.r +} + +// setTrailerMetadata adds a trailer metadata entry with key being set to +// `internal.TrailerMetadataKey` and value being set to the binary-encoded +// orca.OrcaLoadReport protobuf message. +// +// This function is called from the unary and streaming interceptors defined +// above. Any errors encountered here are not propagated to the caller because +// they are ignored there. Hence we simply log any errors encountered here at +// warning level, and return nothing. +func (rw *recorderWrapper) setTrailerMetadata(ctx context.Context) { + var sm *ServerMetrics + if rw.smp != nil { + sm = rw.smp.ServerMetrics() + sm.merge(rw.r.ServerMetrics()) + } else { + sm = rw.r.ServerMetrics() + } + + b, err := proto.Marshal(sm.toLoadReportProto()) + if err != nil { + logger.Warningf("Failed to marshal load report: %v", err) + return + } + if err := grpc.SetTrailer(ctx, metadata.Pairs(internal.TrailerMetadataKey, string(b))); err != nil { + logger.Warningf("Failed to set trailer metadata: %v", err) + } +} + +var joinServerOptions = grpcinternal.JoinServerOptions.(func(...grpc.ServerOption) grpc.ServerOption) + +// CallMetricsServerOption returns a server option which enables the reporting +// of per-RPC custom backend metrics for unary and streaming RPCs. +// +// Server applications interested in injecting custom backend metrics should +// pass the server option returned from this function as the first argument to +// grpc.NewServer(). +// +// Subsequently, server RPC handlers can retrieve a reference to the RPC +// specific custom metrics recorder [CallMetricsRecorder] to be used, via a call +// to CallMetricsRecorderFromContext(), and inject custom metrics at any time +// during the RPC lifecycle. +// +// The injected custom metrics will be sent as part of trailer metadata, as a +// binary-encoded [ORCA LoadReport] protobuf message, with the metadata key +// being set be "endpoint-load-metrics-bin". +// +// If a non-nil ServerMetricsProvider is provided, the gRPC server will +// transmit the metrics it provides, overwritten by any per-RPC metrics given +// to the CallMetricsRecorder. A ServerMetricsProvider is typically obtained +// by calling NewServerMetricsRecorder. +// +// [ORCA LoadReport]: https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto#L15 +func CallMetricsServerOption(smp ServerMetricsProvider) grpc.ServerOption { + return joinServerOptions(grpc.ChainUnaryInterceptor(unaryInt(smp)), grpc.ChainStreamInterceptor(streamInt(smp))) +} + +func unaryInt(smp ServerMetricsProvider) func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + return func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + // We don't allocate the metric recorder here. It will be allocated the + // first time the user calls CallMetricsRecorderFromContext(). + rw := &recorderWrapper{smp: smp} + ctxWithRecorder := newContextWithRecorderWrapper(ctx, rw) + + resp, err := handler(ctxWithRecorder, req) + + // It is safe to access the underlying metric recorder inside the wrapper at + // this point, as the user's RPC handler is done executing, and therefore + // there will be no more calls to CallMetricsRecorderFromContext(), which is + // where the metric recorder is lazy allocated. + if rw.r != nil { + rw.setTrailerMetadata(ctx) + } + return resp, err + } +} + +func streamInt(smp ServerMetricsProvider) func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + return func(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + // We don't allocate the metric recorder here. It will be allocated the + // first time the user calls CallMetricsRecorderFromContext(). + rw := &recorderWrapper{smp: smp} + ws := &wrappedStream{ + ServerStream: ss, + ctx: newContextWithRecorderWrapper(ss.Context(), rw), + } + + err := handler(srv, ws) + + // It is safe to access the underlying metric recorder inside the wrapper at + // this point, as the user's RPC handler is done executing, and therefore + // there will be no more calls to CallMetricsRecorderFromContext(), which is + // where the metric recorder is lazy allocated. + if rw.r != nil { + rw.setTrailerMetadata(ss.Context()) + } + return err + } +} + +func newContextWithRecorderWrapper(ctx context.Context, r *recorderWrapper) context.Context { + return context.WithValue(ctx, callMetricsRecorderCtxKey{}, r) +} + +// wrappedStream wraps the grpc.ServerStream received by the streaming +// interceptor. Overrides only the Context() method to return a context which +// contains a reference to the CallMetricsRecorder corresponding to this +// stream. +type wrappedStream struct { + grpc.ServerStream + ctx context.Context +} + +func (w *wrappedStream) Context() context.Context { + return w.ctx +} diff --git a/orca/call_metrics_test.go b/orca/call_metrics_test.go new file mode 100644 index 0000000..1db9d87 --- /dev/null +++ b/orca/call_metrics_test.go @@ -0,0 +1,287 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca_test + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/orca/internal" + "google.golang.org/protobuf/proto" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestE2ECallMetricsUnary tests the injection of custom backend metrics from +// the server application for a unary RPC, and verifies that expected load +// reports are received at the client. +func (s) TestE2ECallMetricsUnary(t *testing.T) { + tests := []struct { + desc string + injectMetrics bool + wantProto *v3orcapb.OrcaLoadReport + }{ + { + desc: "with custom backend metrics", + injectMetrics: true, + wantProto: &v3orcapb.OrcaLoadReport{ + CpuUtilization: 1.0, + MemUtilization: 0.9, + RequestCost: map[string]float64{"queryCost": 25.0}, + Utilization: map[string]float64{"queueSize": 0.75}, + }, + }, + { + desc: "with no custom backend metrics", + injectMetrics: false, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // A server option to enable reporting of per-call backend metrics. + smr := orca.NewServerMetricsRecorder() + callMetricsServerOption := orca.CallMetricsServerOption(smr) + smr.SetCPUUtilization(1.0) + + // An interceptor to injects custom backend metrics, added only when + // the injectMetrics field in the test is set. + injectingInterceptor := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + recorder := orca.CallMetricsRecorderFromContext(ctx) + if recorder == nil { + err := errors.New("Failed to retrieve per-RPC custom metrics recorder from the RPC context") + t.Error(err) + return nil, err + } + recorder.SetMemoryUtilization(0.9) + // This value will be overwritten by a write to the same metric + // from the server handler. + recorder.SetNamedUtilization("queueSize", 1.0) + return handler(ctx, req) + } + + // A stub server whose unary handler injects custom metrics, if the + // injectMetrics field in the test is set. It overwrites one of the + // values injected above, by the interceptor. + srv := stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + if !test.injectMetrics { + return &testpb.Empty{}, nil + } + recorder := orca.CallMetricsRecorderFromContext(ctx) + if recorder == nil { + err := errors.New("Failed to retrieve per-RPC custom metrics recorder from the RPC context") + t.Error(err) + return nil, err + } + recorder.SetRequestCost("queryCost", 25.0) + recorder.SetNamedUtilization("queueSize", 0.75) + return &testpb.Empty{}, nil + }, + } + + // Start the stub server with the appropriate server options. + sopts := []grpc.ServerOption{callMetricsServerOption} + if test.injectMetrics { + sopts = append(sopts, grpc.ChainUnaryInterceptor(injectingInterceptor)) + } + if err := srv.StartServer(sopts...); err != nil { + t.Fatalf("Failed to start server: %v", err) + } + defer srv.Stop() + + // Dial the stub server. + cc, err := grpc.NewClient(srv.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v", srv.Address, err) + } + defer cc.Close() + + // Make a unary RPC and expect the trailer metadata to contain the custom + // backend metrics as an ORCA LoadReport protobuf message. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + trailer := metadata.MD{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Trailer(&trailer)); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } + + gotProto, err := internal.ToLoadReport(trailer) + if err != nil { + t.Fatalf("When retrieving load report, got error: %v, want: ", err) + } + if test.wantProto != nil && !cmp.Equal(gotProto, test.wantProto, cmp.Comparer(proto.Equal)) { + t.Fatalf("Received load report in trailer: %s, want: %s", pretty.ToJSON(gotProto), pretty.ToJSON(test.wantProto)) + } + }) + } +} + +// TestE2ECallMetricsStreaming tests the injection of custom backend metrics +// from the server application for a streaming RPC, and verifies that expected +// load reports are received at the client. +func (s) TestE2ECallMetricsStreaming(t *testing.T) { + tests := []struct { + desc string + injectMetrics bool + wantProto *v3orcapb.OrcaLoadReport + }{ + { + desc: "with custom backend metrics", + injectMetrics: true, + wantProto: &v3orcapb.OrcaLoadReport{ + CpuUtilization: 1.0, + MemUtilization: 0.5, + RequestCost: map[string]float64{"queryCost": 0.25}, + Utilization: map[string]float64{"queueSize": 0.75}, + }, + }, + { + desc: "with no custom backend metrics", + injectMetrics: false, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // A server option to enable reporting of per-call backend metrics. + smr := orca.NewServerMetricsRecorder() + callMetricsServerOption := orca.CallMetricsServerOption(smr) + smr.SetCPUUtilization(1.0) + + // An interceptor which injects custom backend metrics, added only + // when the injectMetrics field in the test is set. + injectingInterceptor := func(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + recorder := orca.CallMetricsRecorderFromContext(ss.Context()) + if recorder == nil { + err := errors.New("Failed to retrieve per-RPC custom metrics recorder from the RPC context") + t.Error(err) + return err + } + recorder.SetMemoryUtilization(0.5) + // This value will be overwritten by a write to the same metric + // from the server handler. + recorder.SetNamedUtilization("queueSize", 1.0) + return handler(srv, ss) + } + + // A stub server whose streaming handler injects custom metrics, if + // the injectMetrics field in the test is set. It overwrites one of + // the values injected above, by the interceptor. + srv := stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if test.injectMetrics { + recorder := orca.CallMetricsRecorderFromContext(stream.Context()) + if recorder == nil { + err := errors.New("Failed to retrieve per-RPC custom metrics recorder from the RPC context") + t.Error(err) + return err + } + recorder.SetRequestCost("queryCost", 0.25) + recorder.SetNamedUtilization("queueSize", 0.75) + } + + // Streaming implementation replies with a dummy response until the + // client closes the stream (in which case it will see an io.EOF), + // or an error occurs while reading/writing messages. + for { + _, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + payload := &testpb.Payload{Body: make([]byte, 32)} + if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: payload}); err != nil { + return err + } + } + }, + } + + // Start the stub server with the appropriate server options. + sopts := []grpc.ServerOption{callMetricsServerOption} + if test.injectMetrics { + sopts = append(sopts, grpc.ChainStreamInterceptor(injectingInterceptor)) + } + if err := srv.StartServer(sopts...); err != nil { + t.Fatalf("Failed to start server: %v", err) + } + defer srv.Stop() + + // Dial the stub server. + cc, err := grpc.NewClient(srv.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v", srv.Address, err) + } + defer cc.Close() + + // Start the full duplex streaming RPC. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + tc := testgrpc.NewTestServiceClient(cc) + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall failed: %v", err) + } + + // Send one request to the server. + payload := &testpb.Payload{Body: make([]byte, 32)} + req := &testpb.StreamingOutputCallRequest{Payload: payload} + if err := stream.Send(req); err != nil { + t.Fatalf("stream.Send() failed: %v", err) + } + // Read one reply from the server. + if _, err := stream.Recv(); err != nil { + t.Fatalf("stream.Recv() failed: %v", err) + } + // Close the sending side. + if err := stream.CloseSend(); err != nil { + t.Fatalf("stream.CloseSend() failed: %v", err) + } + // Make sure it is safe to read the trailer. + for { + if _, err := stream.Recv(); err != nil { + break + } + } + + gotProto, err := internal.ToLoadReport(stream.Trailer()) + if err != nil { + t.Fatalf("When retrieving load report, got error: %v, want: ", err) + } + if test.wantProto != nil && !cmp.Equal(gotProto, test.wantProto, cmp.Comparer(proto.Equal)) { + t.Fatalf("Received load report in trailer: %s, want: %s", pretty.ToJSON(gotProto), pretty.ToJSON(test.wantProto)) + } + }) + } +} diff --git a/orca/internal/internal.go b/orca/internal/internal.go new file mode 100644 index 0000000..d1425c3 --- /dev/null +++ b/orca/internal/internal.go @@ -0,0 +1,71 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains orca-internal code, for testing purposes and to +// avoid polluting the godoc of the top-level orca package. +package internal + +import ( + "errors" + "fmt" + + ibackoff "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/proto" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" +) + +// AllowAnyMinReportingInterval prevents clamping of the MinReportingInterval +// configured via ServiceOptions, to a minimum of 30s. +// +// For testing purposes only. +var AllowAnyMinReportingInterval any // func(*ServiceOptions) + +// DefaultBackoffFunc is used by the producer to control its backoff behavior. +// +// For testing purposes only. +var DefaultBackoffFunc = ibackoff.DefaultExponential.Backoff + +// TrailerMetadataKey is the key in which the per-call backend metrics are +// transmitted. +const TrailerMetadataKey = "endpoint-load-metrics-bin" + +// ToLoadReport unmarshals a binary encoded [ORCA LoadReport] protobuf message +// from md and returns the corresponding struct. The load report is expected to +// be stored as the value for key "endpoint-load-metrics-bin". +// +// If no load report was found in the provided metadata, if multiple load +// reports are found, or if the load report found cannot be parsed, an error is +// returned. +// +// [ORCA LoadReport]: (https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto#L15) +func ToLoadReport(md metadata.MD) (*v3orcapb.OrcaLoadReport, error) { + vs := md.Get(TrailerMetadataKey) + if len(vs) == 0 { + return nil, nil + } + if len(vs) != 1 { + return nil, errors.New("multiple orca load reports found in provided metadata") + } + ret := new(v3orcapb.OrcaLoadReport) + if err := proto.Unmarshal([]byte(vs[0]), ret); err != nil { + return nil, fmt.Errorf("failed to unmarshal load report found in metadata: %v", err) + } + return ret, nil +} diff --git a/orca/orca.go b/orca/orca.go new file mode 100644 index 0000000..d0cb372 --- /dev/null +++ b/orca/orca.go @@ -0,0 +1,57 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package orca implements Open Request Cost Aggregation, which is an open +// standard for request cost aggregation and reporting by backends and the +// corresponding aggregation of such reports by L7 load balancers (such as +// Envoy) on the data plane. In a proxyless world with gRPC enabled +// applications, aggregation of such reports will be done by the gRPC client. +// +// # Experimental +// +// Notice: All APIs is this package are EXPERIMENTAL and may be changed or +// removed in a later release. +package orca + +import ( + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/balancerload" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/orca/internal" +) + +var logger = grpclog.Component("orca-backend-metrics") + +// loadParser implements the Parser interface defined in `internal/balancerload` +// package. This interface is used by the client stream to parse load reports +// sent by the server in trailer metadata. The parsed loads are then sent to +// balancers via balancer.DoneInfo. +// +// The grpc package cannot directly call toLoadReport() as that would cause an +// import cycle. Hence this roundabout method is used. +type loadParser struct{} + +func (loadParser) Parse(md metadata.MD) any { + lr, err := internal.ToLoadReport(md) + if err != nil { + logger.Infof("Parse failed: %v", err) + } + return lr +} + +func init() { + balancerload.SetParser(loadParser{}) +} diff --git a/orca/orca_test.go b/orca/orca_test.go new file mode 100644 index 0000000..ce5b7bb --- /dev/null +++ b/orca/orca_test.go @@ -0,0 +1,102 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca_test + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/orca/internal" + "google.golang.org/protobuf/proto" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const defaultTestTimeout = 5 * time.Second + +func (s) TestToLoadReport(t *testing.T) { + goodReport := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 1.0, + MemUtilization: 50.0, + RequestCost: map[string]float64{"queryCost": 25.0}, + Utilization: map[string]float64{"queueSize": 75.0}, + } + tests := []struct { + name string + md metadata.MD + want *v3orcapb.OrcaLoadReport + wantErr bool + }{ + { + name: "no load report in metadata", + md: metadata.MD{}, + wantErr: false, + }, + { + name: "badly marshaled load report", + md: func() metadata.MD { + return metadata.Pairs("endpoint-load-metrics-bin", string("foo-bar")) + }(), + wantErr: true, + }, + { + name: "multiple load reports", + md: func() metadata.MD { + b, _ := proto.Marshal(goodReport) + return metadata.Pairs("endpoint-load-metrics-bin", string(b), "endpoint-load-metrics-bin", string(b)) + }(), + wantErr: true, + }, + { + name: "good load report", + md: func() metadata.MD { + b, _ := proto.Marshal(goodReport) + return metadata.Pairs("endpoint-load-metrics-bin", string(b)) + }(), + want: goodReport, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := internal.ToLoadReport(test.md) + if (err != nil) != test.wantErr { + t.Fatalf("orca.ToLoadReport(%v) = %v, wantErr: %v", test.md, err, test.wantErr) + } + if test.wantErr { + return + } + if !cmp.Equal(got, test.want, cmp.Comparer(proto.Equal)) { + t.Fatalf("Extracted load report from metadata: %s, want: %s", pretty.ToJSON(got), pretty.ToJSON(test.want)) + } + }) + } +} diff --git a/orca/producer.go b/orca/producer.go new file mode 100644 index 0000000..87fd374 --- /dev/null +++ b/orca/producer.go @@ -0,0 +1,225 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package orca + +import ( + "context" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/orca/internal" + "google.golang.org/grpc/status" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + v3orcaservicegrpc "github.com/cncf/xds/go/xds/service/orca/v3" + v3orcaservicepb "github.com/cncf/xds/go/xds/service/orca/v3" + "google.golang.org/protobuf/types/known/durationpb" +) + +type producerBuilder struct{} + +// Build constructs and returns a producer and its cleanup function +func (*producerBuilder) Build(cci any) (balancer.Producer, func()) { + p := &producer{ + client: v3orcaservicegrpc.NewOpenRcaServiceClient(cci.(grpc.ClientConnInterface)), + intervals: make(map[time.Duration]int), + listeners: make(map[OOBListener]struct{}), + backoff: internal.DefaultBackoffFunc, + } + return p, func() { + p.mu.Lock() + if p.stop != nil { + p.stop() + p.stop = nil + } + p.mu.Unlock() + <-p.stopped + } +} + +var producerBuilderSingleton = &producerBuilder{} + +// OOBListener is used to receive out-of-band load reports as they arrive. +type OOBListener interface { + // OnLoadReport is called when a load report is received. + OnLoadReport(*v3orcapb.OrcaLoadReport) +} + +// OOBListenerOptions contains options to control how an OOBListener is called. +type OOBListenerOptions struct { + // ReportInterval specifies how often to request the server to provide a + // load report. May be provided less frequently if the server requires a + // longer interval, or may be provided more frequently if another + // subscriber requests a shorter interval. + ReportInterval time.Duration +} + +// RegisterOOBListener registers an out-of-band load report listener on a Ready +// sc. Any OOBListener may only be registered once per subchannel at a time. +// The returned stop function must be called when no longer needed. Do not +// register a single OOBListener more than once per SubConn. +func RegisterOOBListener(sc balancer.SubConn, l OOBListener, opts OOBListenerOptions) (stop func()) { + pr, closeFn := sc.GetOrBuildProducer(producerBuilderSingleton) + p := pr.(*producer) + + p.registerListener(l, opts.ReportInterval) + + // If stop is called multiple times, prevent it from having any effect on + // subsequent calls. + return sync.OnceFunc(func() { + p.unregisterListener(l, opts.ReportInterval) + closeFn() + }) +} + +type producer struct { + client v3orcaservicegrpc.OpenRcaServiceClient + + // backoff is called between stream attempts to determine how long to delay + // to avoid overloading a server experiencing problems. The attempt count + // is incremented when stream errors occur and is reset when the stream + // reports a result. + backoff func(int) time.Duration + stopped chan struct{} // closed when the run goroutine exits + + mu sync.Mutex + intervals map[time.Duration]int // map from interval time to count of listeners requesting that time + listeners map[OOBListener]struct{} // set of registered listeners + minInterval time.Duration + stop func() // stops the current run goroutine +} + +// registerListener adds the listener and its requested report interval to the +// producer. +func (p *producer) registerListener(l OOBListener, interval time.Duration) { + p.mu.Lock() + defer p.mu.Unlock() + + p.listeners[l] = struct{}{} + p.intervals[interval]++ + if len(p.listeners) == 1 || interval < p.minInterval { + p.minInterval = interval + p.updateRunLocked() + } +} + +// registerListener removes the listener and its requested report interval to +// the producer. +func (p *producer) unregisterListener(l OOBListener, interval time.Duration) { + p.mu.Lock() + defer p.mu.Unlock() + + delete(p.listeners, l) + p.intervals[interval]-- + if p.intervals[interval] == 0 { + delete(p.intervals, interval) + + if p.minInterval == interval { + p.recomputeMinInterval() + p.updateRunLocked() + } + } +} + +// recomputeMinInterval sets p.minInterval to the minimum key's value in +// p.intervals. +func (p *producer) recomputeMinInterval() { + first := true + for interval := range p.intervals { + if first || interval < p.minInterval { + p.minInterval = interval + first = false + } + } +} + +// updateRunLocked is called whenever the run goroutine needs to be started / +// stopped / restarted due to: 1. the initial listener being registered, 2. the +// final listener being unregistered, or 3. the minimum registered interval +// changing. +func (p *producer) updateRunLocked() { + if p.stop != nil { + p.stop() + p.stop = nil + } + if len(p.listeners) > 0 { + var ctx context.Context + ctx, p.stop = context.WithCancel(context.Background()) + p.stopped = make(chan struct{}) + go p.run(ctx, p.stopped, p.minInterval) + } +} + +// run manages the ORCA OOB stream on the subchannel. +func (p *producer) run(ctx context.Context, done chan struct{}, interval time.Duration) { + defer close(done) + + runStream := func() error { + resetBackoff, err := p.runStream(ctx, interval) + if status.Code(err) == codes.Unimplemented { + // Unimplemented; do not retry. + logger.Error("Server doesn't support ORCA OOB load reporting protocol; not listening for load reports.") + return err + } + // Retry for all other errors. + if code := status.Code(err); code != codes.Unavailable && code != codes.Canceled { + // TODO: Unavailable and Canceled should also ideally log an error, + // but for now we receive them when shutting down the ClientConn + // (Unavailable if the stream hasn't started yet, and Canceled if it + // happens mid-stream). Once we can determine the state or ensure + // the producer is stopped before the stream ends, we can log an + // error when it's not a natural shutdown. + logger.Error("Received unexpected stream error:", err) + } + if resetBackoff { + return backoff.ErrResetBackoff + } + return nil + } + backoff.RunF(ctx, runStream, p.backoff) +} + +// runStream runs a single stream on the subchannel and returns the resulting +// error, if any, and whether or not the run loop should reset the backoff +// timer to zero or advance it. +func (p *producer) runStream(ctx context.Context, interval time.Duration) (resetBackoff bool, err error) { + streamCtx, cancel := context.WithCancel(ctx) + defer cancel() + stream, err := p.client.StreamCoreMetrics(streamCtx, &v3orcaservicepb.OrcaLoadReportRequest{ + ReportInterval: durationpb.New(interval), + }) + if err != nil { + return false, err + } + + for { + report, err := stream.Recv() + if err != nil { + return resetBackoff, err + } + resetBackoff = true + p.mu.Lock() + for l := range p.listeners { + l.OnLoadReport(report) + } + p.mu.Unlock() + } +} diff --git a/orca/producer_test.go b/orca/producer_test.go new file mode 100644 index 0000000..9a0a68d --- /dev/null +++ b/orca/producer_test.go @@ -0,0 +1,572 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package orca_test + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/orca/internal" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + v3orcaservicegrpc "github.com/cncf/xds/go/xds/service/orca/v3" + v3orcaservicepb "github.com/cncf/xds/go/xds/service/orca/v3" +) + +// customLBB wraps a round robin LB policy but provides a ClientConn wrapper to +// add an ORCA OOB report producer for all created SubConns. +type customLBB struct{} + +func (customLBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + return balancer.Get(roundrobin.Name).Build(&ccWrapper{ClientConn: cc}, opts) +} + +func (customLBB) Name() string { return "customLB" } + +func init() { + balancer.Register(customLBB{}) +} + +type ccWrapper struct { + balancer.ClientConn +} + +func (w *ccWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + if len(addrs) != 1 { + panic(fmt.Sprintf("got addrs=%v; want len(addrs) == 1", addrs)) + } + var sc balancer.SubConn + opts.StateListener = func(scs balancer.SubConnState) { + if scs.ConnectivityState != connectivity.Ready { + return + } + l := getListenerInfo(addrs[0]) + l.listener.cleanup = orca.RegisterOOBListener(sc, l.listener, l.opts) + l.scChan <- sc + } + sc, err := w.ClientConn.NewSubConn(addrs, opts) + if err != nil { + return sc, err + } + return sc, nil +} + +// listenerInfo is stored in an address's attributes to allow ORCA +// listeners to be registered on subconns created for that address. +type listenerInfo struct { + listener *testOOBListener + opts orca.OOBListenerOptions + scChan chan balancer.SubConn // Pushed on by the LB policy +} + +type listenerInfoKey struct{} + +func setListenerInfo(addr resolver.Address, l *listenerInfo) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(listenerInfoKey{}, l) + return addr +} + +func getListenerInfo(addr resolver.Address) *listenerInfo { + return addr.Attributes.Value(listenerInfoKey{}).(*listenerInfo) +} + +// testOOBListener is a simple listener that pushes load reports to a channel. +type testOOBListener struct { + cleanup func() + loadReportCh chan *v3orcapb.OrcaLoadReport +} + +func newTestOOBListener() *testOOBListener { + return &testOOBListener{cleanup: func() {}, loadReportCh: make(chan *v3orcapb.OrcaLoadReport)} +} + +func (t *testOOBListener) Stop() { t.cleanup() } + +func (t *testOOBListener) OnLoadReport(r *v3orcapb.OrcaLoadReport) { + t.loadReportCh <- r +} + +// TestProducer is a basic, end-to-end style test of an LB policy with an +// OOBListener communicating with a server with an ORCA service. +func (s) TestProducer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Use a fixed backoff for stream recreation. + oldBackoff := internal.DefaultBackoffFunc + internal.DefaultBackoffFunc = func(int) time.Duration { return 10 * time.Millisecond } + defer func() { internal.DefaultBackoffFunc = oldBackoff }() + + // Initialize listener for our ORCA server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatal(err) + } + + // Register the OpenRCAService with a very short metrics reporting interval. + const shortReportingInterval = 50 * time.Millisecond + smr := orca.NewServerMetricsRecorder() + opts := orca.ServiceOptions{MinReportingInterval: shortReportingInterval, ServerMetricsProvider: smr} + internal.AllowAnyMinReportingInterval.(func(*orca.ServiceOptions))(&opts) + s := grpc.NewServer() + if err := orca.Register(s, opts); err != nil { + t.Fatalf("orca.Register failed: %v", err) + } + go s.Serve(lis) + defer s.Stop() + + // Create our client with an OOB listener in the LB policy it selects. + r := manual.NewBuilderWithScheme("whatever") + oobLis := newTestOOBListener() + + lisOpts := orca.OOBListenerOptions{ReportInterval: 50 * time.Millisecond} + li := &listenerInfo{scChan: make(chan balancer.SubConn, 1), listener: oobLis, opts: lisOpts} + addr := setListenerInfo(resolver.Address{Addr: lis.Addr().String()}, li) + r.InitialState(resolver.State{Addresses: []resolver.Address{addr}}) + dopts := []grpc.DialOption{ + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"customLB":{}}]}`), + grpc.WithResolvers(r), + grpc.WithTransportCredentials(insecure.NewCredentials()), + } + cc, err := grpc.NewClient("whatever:///whatever", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + // Set a few metrics and wait for them on the client side. + smr.SetCPUUtilization(10) + smr.SetMemoryUtilization(0.1) + smr.SetNamedUtilization("bob", 0.555) + loadReportWant := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 10, + MemUtilization: 0.1, + Utilization: map[string]float64{"bob": 0.555}, + } + +testReport: + for { + select { + case r := <-oobLis.loadReportCh: + t.Log("Load report received: ", r) + if proto.Equal(r, loadReportWant) { + // Success! + break testReport + } + case <-ctx.Done(): + t.Fatalf("timed out waiting for load report: %v", loadReportWant) + } + } + + // Change and add metrics and wait for them on the client side. + smr.SetCPUUtilization(0.5) + smr.SetMemoryUtilization(0.2) + smr.SetNamedUtilization("mary", 0.321) + loadReportWant = &v3orcapb.OrcaLoadReport{ + CpuUtilization: 0.5, + MemUtilization: 0.2, + Utilization: map[string]float64{"bob": 0.555, "mary": 0.321}, + } + + for { + select { + case r := <-oobLis.loadReportCh: + t.Log("Load report received: ", r) + if proto.Equal(r, loadReportWant) { + // Success! + return + } + case <-ctx.Done(): + t.Fatalf("timed out waiting for load report: %v", loadReportWant) + } + } + +} + +// fakeORCAService is a simple implementation of an ORCA service that pushes +// requests it receives from clients to a channel and sends responses from a +// channel back. This allows tests to verify the client is sending requests +// and processing responses properly. +type fakeORCAService struct { + v3orcaservicegrpc.UnimplementedOpenRcaServiceServer + + reqCh chan *v3orcaservicepb.OrcaLoadReportRequest + respCh chan any // either *v3orcapb.OrcaLoadReport or error +} + +func newFakeORCAService() *fakeORCAService { + return &fakeORCAService{ + reqCh: make(chan *v3orcaservicepb.OrcaLoadReportRequest), + respCh: make(chan any), + } +} + +func (f *fakeORCAService) close() { + close(f.respCh) +} + +func (f *fakeORCAService) StreamCoreMetrics(req *v3orcaservicepb.OrcaLoadReportRequest, stream v3orcaservicegrpc.OpenRcaService_StreamCoreMetricsServer) error { + select { + case f.reqCh <- req: + case <-stream.Context().Done(): + return stream.Context().Err() + } + for { + var resp any + select { + case resp = <-f.respCh: + case <-stream.Context().Done(): + return stream.Context().Err() + } + + if err, ok := resp.(error); ok { + return err + } + if err := stream.Send(resp.(*v3orcapb.OrcaLoadReport)); err != nil { + // In the event that a stream error occurs, a new stream will have + // been created that was waiting for this response message. Push + // it back onto the channel and return. + // + // This happens because we range over respCh. If we changed to + // instead select on respCh + stream.Context(), the same situation + // could still occur due to a race between noticing the two events, + // so such a workaround would still be needed to prevent flakiness. + f.respCh <- resp + return err + } + } +} + +// TestProducerBackoff verifies that the ORCA producer applies the proper +// backoff after stream failures. +func (s) TestProducerBackoff(t *testing.T) { + grpctest.ExpectErrorN("injected error", 4) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Provide a convenient way to expect backoff calls and return a minimal + // value. + const backoffShouldNotBeCalled = 9999 // Use to assert backoff function is not called. + const backoffAllowAny = -1 // Use to ignore any backoff calls. + expectedBackoff := atomic.Int32{} + expectedBackoff.Store(backoffAllowAny) + oldBackoff := internal.DefaultBackoffFunc + internal.DefaultBackoffFunc = func(got int) time.Duration { + if expectedBackoff.Load() == backoffShouldNotBeCalled { + t.Errorf("Unexpected backoff call; parameter = %v", got) + } else if expectedBackoff.Load() != backoffAllowAny { + if got != int(expectedBackoff.Load()) { + t.Errorf("Unexpected backoff received; got %v want %v", got, expectedBackoff.Load()) + } + } + return time.Millisecond + } + defer func() { internal.DefaultBackoffFunc = oldBackoff }() + + // Initialize listener for our ORCA server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatal(err) + } + + // Register our fake ORCA service. + s := grpc.NewServer() + fake := newFakeORCAService() + defer fake.close() + v3orcaservicegrpc.RegisterOpenRcaServiceServer(s, fake) + go s.Serve(lis) + defer s.Stop() + + // Define the report interval and a function to wait for it to be sent to + // the server. + const reportInterval = 123 * time.Second + awaitRequest := func(interval time.Duration) { + select { + case req := <-fake.reqCh: + if got := req.GetReportInterval().AsDuration(); got != interval { + t.Errorf("Unexpected report interval; got %v want %v", got, interval) + } + case <-ctx.Done(): + t.Fatalf("Did not receive client request") + } + } + + // Create our client with an OOB listener in the LB policy it selects. + r := manual.NewBuilderWithScheme("whatever") + oobLis := newTestOOBListener() + + lisOpts := orca.OOBListenerOptions{ReportInterval: reportInterval} + li := &listenerInfo{scChan: make(chan balancer.SubConn, 1), listener: oobLis, opts: lisOpts} + r.InitialState(resolver.State{Addresses: []resolver.Address{setListenerInfo(resolver.Address{Addr: lis.Addr().String()}, li)}}) + dopts := []grpc.DialOption{ + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"customLB":{}}]}`), + grpc.WithResolvers(r), + grpc.WithTransportCredentials(insecure.NewCredentials()), + } + cc, err := grpc.NewClient("whatever:///whatever", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient failed: %v", err) + } + cc.Connect() + defer cc.Close() + + // Define a load report to send and expect the client to see. + loadReportWant := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 10, + MemUtilization: 0.1, + Utilization: map[string]float64{"bob": 0.555}, + } + + // Unblock the fake. + awaitRequest(reportInterval) + fake.respCh <- loadReportWant + select { + case r := <-oobLis.loadReportCh: + t.Log("Load report received: ", r) + if proto.Equal(r, loadReportWant) { + // Success! + break + } + case <-ctx.Done(): + t.Fatalf("timed out waiting for load report: %v", loadReportWant) + } + + // The next request should be immediate, since there was a message + // received. + expectedBackoff.Store(backoffShouldNotBeCalled) + fake.respCh <- status.Errorf(codes.Internal, "injected error") + awaitRequest(reportInterval) + + // The next requests will need to backoff. + expectedBackoff.Store(0) + fake.respCh <- status.Errorf(codes.Internal, "injected error") + awaitRequest(reportInterval) + expectedBackoff.Store(1) + fake.respCh <- status.Errorf(codes.Internal, "injected error") + awaitRequest(reportInterval) + expectedBackoff.Store(2) + fake.respCh <- status.Errorf(codes.Internal, "injected error") + awaitRequest(reportInterval) + // The next request should be immediate, since there was a message + // received. + expectedBackoff.Store(backoffShouldNotBeCalled) + + // Send another valid response and wait for it on the client. + fake.respCh <- loadReportWant + select { + case r := <-oobLis.loadReportCh: + t.Log("Load report received: ", r) + if proto.Equal(r, loadReportWant) { + // Success! + break + } + case <-ctx.Done(): + t.Fatalf("timed out waiting for load report: %v", loadReportWant) + } +} + +// TestProducerMultipleListeners tests that multiple listeners works as +// expected in a producer: requesting the proper interval and delivering the +// update to all listeners. +func (s) TestProducerMultipleListeners(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Provide a convenient way to expect backoff calls and return a minimal + // value. + oldBackoff := internal.DefaultBackoffFunc + internal.DefaultBackoffFunc = func(int) time.Duration { + return time.Millisecond + } + defer func() { internal.DefaultBackoffFunc = oldBackoff }() + + // Initialize listener for our ORCA server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatal(err) + } + + // Register our fake ORCA service. + s := grpc.NewServer() + fake := newFakeORCAService() + defer fake.close() + v3orcaservicegrpc.RegisterOpenRcaServiceServer(s, fake) + go s.Serve(lis) + defer s.Stop() + + // Define the report interval and a function to wait for it to be sent to + // the server. + const reportInterval1 = 123 * time.Second + const reportInterval2 = 234 * time.Second + const reportInterval3 = 56 * time.Second + awaitRequest := func(interval time.Duration) { + select { + case req := <-fake.reqCh: + if got := req.GetReportInterval().AsDuration(); got != interval { + t.Errorf("Unexpected report interval; got %v want %v", got, interval) + } + case <-ctx.Done(): + t.Fatalf("Did not receive client request") + } + } + + // Create our client with an OOB listener in the LB policy it selects. + r := manual.NewBuilderWithScheme("whatever") + oobLis1 := newTestOOBListener() + lisOpts1 := orca.OOBListenerOptions{ReportInterval: reportInterval1} + li := &listenerInfo{scChan: make(chan balancer.SubConn, 1), listener: oobLis1, opts: lisOpts1} + r.InitialState(resolver.State{Addresses: []resolver.Address{setListenerInfo(resolver.Address{Addr: lis.Addr().String()}, li)}}) + cc, err := grpc.NewClient("whatever:///whatever", grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"customLB":{}}]}`), grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + defer cc.Close() + + // Ensure the OOB listener is stopped before the client is closed to avoid + // a potential irrelevant error in the logs. + defer oobLis1.Stop() + + oobLis2 := newTestOOBListener() + lisOpts2 := orca.OOBListenerOptions{ReportInterval: reportInterval2} + + oobLis3 := newTestOOBListener() + lisOpts3 := orca.OOBListenerOptions{ReportInterval: reportInterval3} + + // Define a load report to send and expect the client to see. + loadReportWant := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 10, + MemUtilization: 0.1, + Utilization: map[string]float64{"bob": 0.555}, + } + + // Receive reports and update counts for the three listeners. + var reportsMu sync.Mutex + var reportsReceived1, reportsReceived2, reportsReceived3 int + go func() { + for { + select { + case r := <-oobLis1.loadReportCh: + t.Log("Load report 1 received: ", r) + if !proto.Equal(r, loadReportWant) { + t.Errorf("Unexpected report received: %+v", r) + } + reportsMu.Lock() + reportsReceived1++ + reportsMu.Unlock() + case r := <-oobLis2.loadReportCh: + t.Log("Load report 2 received: ", r) + if !proto.Equal(r, loadReportWant) { + t.Errorf("Unexpected report received: %+v", r) + } + reportsMu.Lock() + reportsReceived2++ + reportsMu.Unlock() + case r := <-oobLis3.loadReportCh: + t.Log("Load report 3 received: ", r) + if !proto.Equal(r, loadReportWant) { + t.Errorf("Unexpected report received: %+v", r) + } + reportsMu.Lock() + reportsReceived3++ + reportsMu.Unlock() + case <-ctx.Done(): + // Test has ended; exit + return + } + } + }() + + // checkReports is a helper function to check the report counts for the three listeners. + checkReports := func(r1, r2, r3 int) { + t.Helper() + for ctx.Err() == nil { + reportsMu.Lock() + if r1 == reportsReceived1 && r2 == reportsReceived2 && r3 == reportsReceived3 { + // Success! + reportsMu.Unlock() + return + } + if reportsReceived1 > r1 || reportsReceived2 > r2 || reportsReceived3 > r3 { + reportsMu.Unlock() + t.Fatalf("received excess reports. got %v %v %v; want %v %v %v", reportsReceived1, reportsReceived2, reportsReceived3, r1, r2, r3) + return + } + reportsMu.Unlock() + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for reports received. got %v %v %v; want %v %v %v", reportsReceived1, reportsReceived2, reportsReceived3, r1, r2, r3) + } + + // Only 1 listener; expect reportInterval1 to be used and expect the report + // to be sent to the listener. + awaitRequest(reportInterval1) + fake.respCh <- loadReportWant + checkReports(1, 0, 0) + + sc := <-li.scChan + // Register listener 2 with a less frequent interval; no need to recreate + // stream. Report should go to both listeners. + oobLis2.cleanup = orca.RegisterOOBListener(sc, oobLis2, lisOpts2) + fake.respCh <- loadReportWant + checkReports(2, 1, 0) + + // Register listener 3 with a more frequent interval; stream is recreated + // with this interval. The next report will go to all three listeners. + oobLis3.cleanup = orca.RegisterOOBListener(sc, oobLis3, lisOpts3) + awaitRequest(reportInterval3) + fake.respCh <- loadReportWant + checkReports(3, 2, 1) + + // Another report without a change in listeners should go to all three listeners. + fake.respCh <- loadReportWant + checkReports(4, 3, 2) + + // Stop listener 2. This does not affect the interval as listener 3 is + // still the shortest. The next update goes to listeners 1 and 3. + oobLis2.Stop() + fake.respCh <- loadReportWant + checkReports(5, 3, 3) + + // Stop listener 3. This makes the interval longer. Reports should only + // go to listener 1 now. + oobLis3.Stop() + awaitRequest(reportInterval1) + fake.respCh <- loadReportWant + checkReports(6, 3, 3) + // Another report without a change in listeners should go to the first listener. + fake.respCh <- loadReportWant + checkReports(7, 3, 3) +} diff --git a/orca/server_metrics.go b/orca/server_metrics.go new file mode 100644 index 0000000..bb664d6 --- /dev/null +++ b/orca/server_metrics.go @@ -0,0 +1,352 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca + +import ( + "sync/atomic" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" +) + +// ServerMetrics is the data returned from a server to a client to describe the +// current state of the server and/or the cost of a request when used per-call. +type ServerMetrics struct { + CPUUtilization float64 // CPU utilization: [0, inf); unset=-1 + MemUtilization float64 // Memory utilization: [0, 1.0]; unset=-1 + AppUtilization float64 // Application utilization: [0, inf); unset=-1 + QPS float64 // queries per second: [0, inf); unset=-1 + EPS float64 // errors per second: [0, inf); unset=-1 + + // The following maps must never be nil. + + Utilization map[string]float64 // Custom fields: [0, 1.0] + RequestCost map[string]float64 // Custom fields: [0, inf); not sent OOB + NamedMetrics map[string]float64 // Custom fields: [0, inf); not sent OOB +} + +// toLoadReportProto dumps sm as an OrcaLoadReport proto. +func (sm *ServerMetrics) toLoadReportProto() *v3orcapb.OrcaLoadReport { + ret := &v3orcapb.OrcaLoadReport{ + Utilization: sm.Utilization, + RequestCost: sm.RequestCost, + NamedMetrics: sm.NamedMetrics, + } + if sm.CPUUtilization != -1 { + ret.CpuUtilization = sm.CPUUtilization + } + if sm.MemUtilization != -1 { + ret.MemUtilization = sm.MemUtilization + } + if sm.AppUtilization != -1 { + ret.ApplicationUtilization = sm.AppUtilization + } + if sm.QPS != -1 { + ret.RpsFractional = sm.QPS + } + if sm.EPS != -1 { + ret.Eps = sm.EPS + } + return ret +} + +// merge merges o into sm, overwriting any values present in both. +func (sm *ServerMetrics) merge(o *ServerMetrics) { + mergeMap(sm.Utilization, o.Utilization) + mergeMap(sm.RequestCost, o.RequestCost) + mergeMap(sm.NamedMetrics, o.NamedMetrics) + if o.CPUUtilization != -1 { + sm.CPUUtilization = o.CPUUtilization + } + if o.MemUtilization != -1 { + sm.MemUtilization = o.MemUtilization + } + if o.AppUtilization != -1 { + sm.AppUtilization = o.AppUtilization + } + if o.QPS != -1 { + sm.QPS = o.QPS + } + if o.EPS != -1 { + sm.EPS = o.EPS + } +} + +func mergeMap(a, b map[string]float64) { + for k, v := range b { + a[k] = v + } +} + +// ServerMetricsRecorder allows for recording and providing out of band server +// metrics. +type ServerMetricsRecorder interface { + ServerMetricsProvider + + // SetCPUUtilization sets the CPU utilization server metric. Must be + // greater than zero. + SetCPUUtilization(float64) + // DeleteCPUUtilization deletes the CPU utilization server metric to + // prevent it from being sent. + DeleteCPUUtilization() + + // SetMemoryUtilization sets the memory utilization server metric. Must be + // in the range [0, 1]. + SetMemoryUtilization(float64) + // DeleteMemoryUtilization deletes the memory utilization server metric to + // prevent it from being sent. + DeleteMemoryUtilization() + + // SetApplicationUtilization sets the application utilization server + // metric. Must be greater than zero. + SetApplicationUtilization(float64) + // DeleteApplicationUtilization deletes the application utilization server + // metric to prevent it from being sent. + DeleteApplicationUtilization() + + // SetQPS sets the Queries Per Second server metric. Must be greater than + // zero. + SetQPS(float64) + // DeleteQPS deletes the Queries Per Second server metric to prevent it + // from being sent. + DeleteQPS() + + // SetEPS sets the Errors Per Second server metric. Must be greater than + // zero. + SetEPS(float64) + // DeleteEPS deletes the Errors Per Second server metric to prevent it from + // being sent. + DeleteEPS() + + // SetNamedUtilization sets the named utilization server metric for the + // name provided. val must be in the range [0, 1]. + SetNamedUtilization(name string, val float64) + // DeleteNamedUtilization deletes the named utilization server metric for + // the name provided to prevent it from being sent. + DeleteNamedUtilization(name string) +} + +type serverMetricsRecorder struct { + state atomic.Pointer[ServerMetrics] // the current metrics +} + +// NewServerMetricsRecorder returns an in-memory store for ServerMetrics and +// allows for safe setting and retrieving of ServerMetrics. Also implements +// ServerMetricsProvider for use with NewService. +func NewServerMetricsRecorder() ServerMetricsRecorder { + return newServerMetricsRecorder() +} + +func newServerMetricsRecorder() *serverMetricsRecorder { + s := new(serverMetricsRecorder) + s.state.Store(&ServerMetrics{ + CPUUtilization: -1, + MemUtilization: -1, + AppUtilization: -1, + QPS: -1, + EPS: -1, + Utilization: make(map[string]float64), + RequestCost: make(map[string]float64), + NamedMetrics: make(map[string]float64), + }) + return s +} + +// ServerMetrics returns a copy of the current ServerMetrics. +func (s *serverMetricsRecorder) ServerMetrics() *ServerMetrics { + return copyServerMetrics(s.state.Load()) +} + +func copyMap(m map[string]float64) map[string]float64 { + ret := make(map[string]float64, len(m)) + for k, v := range m { + ret[k] = v + } + return ret +} + +func copyServerMetrics(sm *ServerMetrics) *ServerMetrics { + return &ServerMetrics{ + CPUUtilization: sm.CPUUtilization, + MemUtilization: sm.MemUtilization, + AppUtilization: sm.AppUtilization, + QPS: sm.QPS, + EPS: sm.EPS, + Utilization: copyMap(sm.Utilization), + RequestCost: copyMap(sm.RequestCost), + NamedMetrics: copyMap(sm.NamedMetrics), + } +} + +// SetCPUUtilization records a measurement for the CPU utilization metric. +func (s *serverMetricsRecorder) SetCPUUtilization(val float64) { + if val < 0 { + if logger.V(2) { + logger.Infof("Ignoring CPU Utilization value out of range: %v", val) + } + return + } + smCopy := copyServerMetrics(s.state.Load()) + smCopy.CPUUtilization = val + s.state.Store(smCopy) +} + +// DeleteCPUUtilization deletes the relevant server metric to prevent it from +// being sent. +func (s *serverMetricsRecorder) DeleteCPUUtilization() { + smCopy := copyServerMetrics(s.state.Load()) + smCopy.CPUUtilization = -1 + s.state.Store(smCopy) +} + +// SetMemoryUtilization records a measurement for the memory utilization metric. +func (s *serverMetricsRecorder) SetMemoryUtilization(val float64) { + if val < 0 || val > 1 { + if logger.V(2) { + logger.Infof("Ignoring Memory Utilization value out of range: %v", val) + } + return + } + smCopy := copyServerMetrics(s.state.Load()) + smCopy.MemUtilization = val + s.state.Store(smCopy) +} + +// DeleteMemoryUtilization deletes the relevant server metric to prevent it +// from being sent. +func (s *serverMetricsRecorder) DeleteMemoryUtilization() { + smCopy := copyServerMetrics(s.state.Load()) + smCopy.MemUtilization = -1 + s.state.Store(smCopy) +} + +// SetApplicationUtilization records a measurement for a generic utilization +// metric. +func (s *serverMetricsRecorder) SetApplicationUtilization(val float64) { + if val < 0 { + if logger.V(2) { + logger.Infof("Ignoring Application Utilization value out of range: %v", val) + } + return + } + smCopy := copyServerMetrics(s.state.Load()) + smCopy.AppUtilization = val + s.state.Store(smCopy) +} + +// DeleteApplicationUtilization deletes the relevant server metric to prevent +// it from being sent. +func (s *serverMetricsRecorder) DeleteApplicationUtilization() { + smCopy := copyServerMetrics(s.state.Load()) + smCopy.AppUtilization = -1 + s.state.Store(smCopy) +} + +// SetQPS records a measurement for the QPS metric. +func (s *serverMetricsRecorder) SetQPS(val float64) { + if val < 0 { + if logger.V(2) { + logger.Infof("Ignoring QPS value out of range: %v", val) + } + return + } + smCopy := copyServerMetrics(s.state.Load()) + smCopy.QPS = val + s.state.Store(smCopy) +} + +// DeleteQPS deletes the relevant server metric to prevent it from being sent. +func (s *serverMetricsRecorder) DeleteQPS() { + smCopy := copyServerMetrics(s.state.Load()) + smCopy.QPS = -1 + s.state.Store(smCopy) +} + +// SetEPS records a measurement for the EPS metric. +func (s *serverMetricsRecorder) SetEPS(val float64) { + if val < 0 { + if logger.V(2) { + logger.Infof("Ignoring EPS value out of range: %v", val) + } + return + } + smCopy := copyServerMetrics(s.state.Load()) + smCopy.EPS = val + s.state.Store(smCopy) +} + +// DeleteEPS deletes the relevant server metric to prevent it from being sent. +func (s *serverMetricsRecorder) DeleteEPS() { + smCopy := copyServerMetrics(s.state.Load()) + smCopy.EPS = -1 + s.state.Store(smCopy) +} + +// SetNamedUtilization records a measurement for a utilization metric uniquely +// identifiable by name. +func (s *serverMetricsRecorder) SetNamedUtilization(name string, val float64) { + if val < 0 || val > 1 { + if logger.V(2) { + logger.Infof("Ignoring Named Utilization value out of range: %v", val) + } + return + } + smCopy := copyServerMetrics(s.state.Load()) + smCopy.Utilization[name] = val + s.state.Store(smCopy) +} + +// DeleteNamedUtilization deletes any previously recorded measurement for a +// utilization metric uniquely identifiable by name. +func (s *serverMetricsRecorder) DeleteNamedUtilization(name string) { + smCopy := copyServerMetrics(s.state.Load()) + delete(smCopy.Utilization, name) + s.state.Store(smCopy) +} + +// SetRequestCost records a measurement for a utilization metric uniquely +// identifiable by name. +func (s *serverMetricsRecorder) SetRequestCost(name string, val float64) { + smCopy := copyServerMetrics(s.state.Load()) + smCopy.RequestCost[name] = val + s.state.Store(smCopy) +} + +// DeleteRequestCost deletes any previously recorded measurement for a +// utilization metric uniquely identifiable by name. +func (s *serverMetricsRecorder) DeleteRequestCost(name string) { + smCopy := copyServerMetrics(s.state.Load()) + delete(smCopy.RequestCost, name) + s.state.Store(smCopy) +} + +// SetNamedMetric records a measurement for a utilization metric uniquely +// identifiable by name. +func (s *serverMetricsRecorder) SetNamedMetric(name string, val float64) { + smCopy := copyServerMetrics(s.state.Load()) + smCopy.NamedMetrics[name] = val + s.state.Store(smCopy) +} + +// DeleteNamedMetric deletes any previously recorded measurement for a +// utilization metric uniquely identifiable by name. +func (s *serverMetricsRecorder) DeleteNamedMetric(name string) { + smCopy := copyServerMetrics(s.state.Load()) + delete(smCopy.NamedMetrics, name) + s.state.Store(smCopy) +} diff --git a/orca/server_metrics_test.go b/orca/server_metrics_test.go new file mode 100644 index 0000000..ecc80d0 --- /dev/null +++ b/orca/server_metrics_test.go @@ -0,0 +1,175 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestServerMetrics_Setters(t *testing.T) { + smr := NewServerMetricsRecorder() + + smr.SetCPUUtilization(0.1) + smr.SetMemoryUtilization(0.2) + smr.SetApplicationUtilization(0.3) + smr.SetQPS(0.4) + smr.SetEPS(0.5) + smr.SetNamedUtilization("x", 0.6) + + want := &ServerMetrics{ + CPUUtilization: 0.1, + MemUtilization: 0.2, + AppUtilization: 0.3, + QPS: 0.4, + EPS: 0.5, + Utilization: map[string]float64{"x": 0.6}, + NamedMetrics: map[string]float64{}, + RequestCost: map[string]float64{}, + } + + got := smr.ServerMetrics() + if d := cmp.Diff(got, want); d != "" { + t.Fatalf("unexpected server metrics: -got +want: %v", d) + } +} + +func (s) TestServerMetrics_Deleters(t *testing.T) { + smr := NewServerMetricsRecorder() + + smr.SetCPUUtilization(0.1) + smr.SetMemoryUtilization(0.2) + smr.SetApplicationUtilization(0.3) + smr.SetQPS(0.4) + smr.SetEPS(0.5) + smr.SetNamedUtilization("x", 0.6) + smr.SetNamedUtilization("y", 0.7) + + // Now delete everything except named_utilization "y". + smr.DeleteCPUUtilization() + smr.DeleteMemoryUtilization() + smr.DeleteApplicationUtilization() + smr.DeleteQPS() + smr.DeleteEPS() + smr.DeleteNamedUtilization("x") + + want := &ServerMetrics{ + CPUUtilization: -1, + MemUtilization: -1, + AppUtilization: -1, + QPS: -1, + EPS: -1, + Utilization: map[string]float64{"y": 0.7}, + NamedMetrics: map[string]float64{}, + RequestCost: map[string]float64{}, + } + + got := smr.ServerMetrics() + if d := cmp.Diff(got, want); d != "" { + t.Fatalf("unexpected server metrics: -got +want: %v", d) + } +} + +func (s) TestServerMetrics_Setters_Range(t *testing.T) { + smr := NewServerMetricsRecorder() + + smr.SetCPUUtilization(0.1) + smr.SetMemoryUtilization(0.2) + smr.SetApplicationUtilization(0.3) + smr.SetQPS(0.4) + smr.SetEPS(0.5) + smr.SetNamedUtilization("x", 0.6) + + // Negatives for all these fields should be ignored. + smr.SetCPUUtilization(-2) + smr.SetMemoryUtilization(-3) + smr.SetApplicationUtilization(-4) + smr.SetQPS(-0.1) + smr.SetEPS(-0.6) + smr.SetNamedUtilization("x", -2) + + // Memory and named utilizations over 1 are ignored. + smr.SetMemoryUtilization(1.1) + smr.SetNamedUtilization("x", 1.1) + + want := &ServerMetrics{ + CPUUtilization: 0.1, + MemUtilization: 0.2, + AppUtilization: 0.3, + QPS: 0.4, + EPS: 0.5, + Utilization: map[string]float64{"x": 0.6}, + NamedMetrics: map[string]float64{}, + RequestCost: map[string]float64{}, + } + + got := smr.ServerMetrics() + if d := cmp.Diff(got, want); d != "" { + t.Fatalf("unexpected server metrics: -got +want: %v", d) + } +} + +func (s) TestServerMetrics_Merge(t *testing.T) { + sm1 := &ServerMetrics{ + CPUUtilization: 0.1, + MemUtilization: 0.2, + AppUtilization: 0.3, + QPS: -1, + EPS: 0, + Utilization: map[string]float64{"x": 0.6}, + NamedMetrics: map[string]float64{"y": 0.2}, + RequestCost: map[string]float64{"a": 0.1}, + } + + sm2 := &ServerMetrics{ + CPUUtilization: -1, + AppUtilization: 0, + QPS: 0.9, + EPS: 20, + Utilization: map[string]float64{"x": 0.5, "y": 0.4}, + NamedMetrics: map[string]float64{"x": 0.1}, + RequestCost: map[string]float64{"a": 0.2}, + } + + want := &ServerMetrics{ + CPUUtilization: 0.1, + MemUtilization: 0, + AppUtilization: 0, + QPS: 0.9, + EPS: 20, + Utilization: map[string]float64{"x": 0.5, "y": 0.4}, + NamedMetrics: map[string]float64{"x": 0.1, "y": 0.2}, + RequestCost: map[string]float64{"a": 0.2}, + } + + sm1.merge(sm2) + if d := cmp.Diff(sm1, want); d != "" { + t.Fatalf("unexpected server metrics: -got +want: %v", d) + } +} diff --git a/orca/service.go b/orca/service.go new file mode 100644 index 0000000..ffe4b5b --- /dev/null +++ b/orca/service.go @@ -0,0 +1,161 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca + +import ( + "fmt" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal" + ointernal "google.golang.org/grpc/orca/internal" + "google.golang.org/grpc/status" + + v3orcaservicegrpc "github.com/cncf/xds/go/xds/service/orca/v3" + v3orcaservicepb "github.com/cncf/xds/go/xds/service/orca/v3" +) + +func init() { + ointernal.AllowAnyMinReportingInterval = func(so *ServiceOptions) { + so.allowAnyMinReportingInterval = true + } + internal.ORCAAllowAnyMinReportingInterval = ointernal.AllowAnyMinReportingInterval +} + +// minReportingInterval is the absolute minimum value supported for +// out-of-band metrics reporting from the ORCA service implementation +// provided by the orca package. +const minReportingInterval = 30 * time.Second + +// Service provides an implementation of the OpenRcaService as defined in the +// [ORCA] service protos. Instances of this type must be created via calls to +// Register() or NewService(). +// +// Server applications can use the SetXxx() and DeleteXxx() methods to record +// measurements corresponding to backend metrics, which eventually get pushed to +// clients who have initiated the SteamCoreMetrics streaming RPC. +// +// [ORCA]: https://github.com/cncf/xds/blob/main/xds/service/orca/v3/orca.proto +type Service struct { + v3orcaservicegrpc.UnimplementedOpenRcaServiceServer + + // Minimum reporting interval, as configured by the user, or the default. + minReportingInterval time.Duration + + smProvider ServerMetricsProvider +} + +// ServiceOptions contains options to configure the ORCA service implementation. +type ServiceOptions struct { + // ServerMetricsProvider is the provider to be used by the service for + // reporting OOB server metrics to clients. Typically obtained via + // NewServerMetricsRecorder. This field is required. + ServerMetricsProvider ServerMetricsProvider + + // MinReportingInterval sets the lower bound for how often out-of-band + // metrics are reported on the streaming RPC initiated by the client. If + // unspecified, negative or less than the default value of 30s, the default + // is used. Clients may request a higher value as part of the + // StreamCoreMetrics streaming RPC. + MinReportingInterval time.Duration + + // Allow a minReportingInterval which is less than the default of 30s. + // Used for testing purposes only. + allowAnyMinReportingInterval bool +} + +// A ServerMetricsProvider provides ServerMetrics upon request. +type ServerMetricsProvider interface { + // ServerMetrics returns the current set of server metrics. It should + // return a read-only, immutable copy of the data that is active at the + // time of the call. + ServerMetrics() *ServerMetrics +} + +// NewService creates a new ORCA service implementation configured using the +// provided options. +func NewService(opts ServiceOptions) (*Service, error) { + // The default minimum supported reporting interval value can be overridden + // for testing purposes through the orca internal package. + if opts.ServerMetricsProvider == nil { + return nil, fmt.Errorf("ServerMetricsProvider not specified") + } + if !opts.allowAnyMinReportingInterval { + if opts.MinReportingInterval < 0 || opts.MinReportingInterval < minReportingInterval { + opts.MinReportingInterval = minReportingInterval + } + } + service := &Service{ + minReportingInterval: opts.MinReportingInterval, + smProvider: opts.ServerMetricsProvider, + } + return service, nil +} + +// Register creates a new ORCA service implementation configured using the +// provided options and registers the same on the provided grpc Server. +func Register(s grpc.ServiceRegistrar, opts ServiceOptions) error { + service, err := NewService(opts) + if err != nil { + return err + } + v3orcaservicegrpc.RegisterOpenRcaServiceServer(s, service) + return nil +} + +// determineReportingInterval determines the reporting interval for out-of-band +// metrics. If the reporting interval is not specified in the request, or is +// negative or is less than the configured minimum (via +// ServiceOptions.MinReportingInterval), the latter is used. Else the value from +// the incoming request is used. +func (s *Service) determineReportingInterval(req *v3orcaservicepb.OrcaLoadReportRequest) time.Duration { + if req.GetReportInterval() == nil { + return s.minReportingInterval + } + dur := req.GetReportInterval().AsDuration() + if dur < s.minReportingInterval { + logger.Warningf("Received reporting interval %q is less than configured minimum: %v. Using minimum", dur, s.minReportingInterval) + return s.minReportingInterval + } + return dur +} + +func (s *Service) sendMetricsResponse(stream v3orcaservicegrpc.OpenRcaService_StreamCoreMetricsServer) error { + return stream.Send(s.smProvider.ServerMetrics().toLoadReportProto()) +} + +// StreamCoreMetrics streams custom backend metrics injected by the server +// application. +func (s *Service) StreamCoreMetrics(req *v3orcaservicepb.OrcaLoadReportRequest, stream v3orcaservicegrpc.OpenRcaService_StreamCoreMetricsServer) error { + ticker := time.NewTicker(s.determineReportingInterval(req)) + defer ticker.Stop() + + for { + if err := s.sendMetricsResponse(stream); err != nil { + return err + } + // Send a response containing the currently recorded metrics + select { + case <-stream.Context().Done(): + return status.Error(codes.Canceled, "Stream has ended.") + case <-ticker.C: + } + } +} diff --git a/orca/service_test.go b/orca/service_test.go new file mode 100644 index 0000000..0d8e8b1 --- /dev/null +++ b/orca/service_test.go @@ -0,0 +1,187 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca_test + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/orca/internal" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + v3orcaservicegrpc "github.com/cncf/xds/go/xds/service/orca/v3" + v3orcaservicepb "github.com/cncf/xds/go/xds/service/orca/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const requestsMetricKey = "test-service-requests" + +// TestE2E_CustomBackendMetrics_OutOfBand tests the injection of out-of-band +// custom backend metrics from the server application, and verifies that +// expected load reports are received at the client. +// +// TODO: Change this test to use the client API, when ready, to read the +// out-of-band metrics pushed by the server. +func (s) TestE2E_CustomBackendMetrics_OutOfBand(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatal(err) + } + + // Override the min reporting interval in the internal package. + const shortReportingInterval = 10 * time.Millisecond + smr := orca.NewServerMetricsRecorder() + opts := orca.ServiceOptions{MinReportingInterval: shortReportingInterval, ServerMetricsProvider: smr} + internal.AllowAnyMinReportingInterval.(func(*orca.ServiceOptions))(&opts) + + var requests atomic.Int64 + + stub := &stubserver.StubServer{ + Listener: lis, + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + newRequests := requests.Add(1) + + smr.SetNamedUtilization(requestsMetricKey, float64(newRequests)*0.01) + smr.SetCPUUtilization(50.0) + smr.SetMemoryUtilization(0.9) + smr.SetApplicationUtilization(1.2) + return &testpb.SimpleResponse{}, nil + }, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + smr.DeleteNamedUtilization(requestsMetricKey) + smr.SetCPUUtilization(0) + smr.SetMemoryUtilization(0) + smr.DeleteApplicationUtilization() + return &testpb.Empty{}, nil + }, + } + + // Assign the gRPC server to the stub server and start serving. + stub.S = grpc.NewServer() + // Register the OpenRCAService with a very short metrics reporting interval. + if err := orca.Register(stub.S, opts); err != nil { + t.Fatalf("orca.EnableOutOfBandMetricsReportingForTesting() failed: %v", err) + } + stubserver.StartTestService(t, stub) + defer stub.S.Stop() + t.Logf("Started gRPC server at %s...", lis.Addr().String()) + + // Dial the test server. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%s) failed: %v", lis.Addr().String(), err) + } + defer cc.Close() + + // Spawn a goroutine which sends 20 unary RPCs to the stub server. This + // will trigger the injection of custom backend metrics from the + // stubServer. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testStub := testgrpc.NewTestServiceClient(cc) + const numRequests = 20 + errCh := make(chan error, 1) + go func() { + for i := 0; i < numRequests; i++ { + if _, err := testStub.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + errCh <- fmt.Errorf("UnaryCall failed: %v", err) + return + } + time.Sleep(time.Millisecond) + } + errCh <- nil + }() + + // Start the server streaming RPC to receive custom backend metrics. + oobStub := v3orcaservicegrpc.NewOpenRcaServiceClient(cc) + stream, err := oobStub.StreamCoreMetrics(ctx, &v3orcaservicepb.OrcaLoadReportRequest{ReportInterval: durationpb.New(shortReportingInterval)}) + if err != nil { + t.Fatalf("Failed to create a stream for out-of-band metrics") + } + + // Wait for the server to push metrics which indicate the completion of all + // the unary RPCs made from the above goroutine. + for { + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for out-of-band custom backend metrics to match expected values") + case err := <-errCh: + if err != nil { + t.Fatal(err) + } + default: + } + + wantProto := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 50.0, + MemUtilization: 0.9, + ApplicationUtilization: 1.2, + Utilization: map[string]float64{requestsMetricKey: numRequests * 0.01}, + } + gotProto, err := stream.Recv() + if err != nil { + t.Fatalf("Recv() failed: %v", err) + } + if !cmp.Equal(gotProto, wantProto, cmp.Comparer(proto.Equal)) { + t.Logf("Received load report from stream: %s, want: %s", pretty.ToJSON(gotProto), pretty.ToJSON(wantProto)) + continue + } + // This means that we received the metrics which we expected. + break + } + + // The EmptyCall RPC is expected to delete earlier injected metrics. + if _, err := testStub.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } + // Wait for the server to push empty metrics which indicate the processing + // of the above EmptyCall RPC. + for { + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for out-of-band custom backend metrics to match expected values") + default: + } + + wantProto := &v3orcapb.OrcaLoadReport{} + gotProto, err := stream.Recv() + if err != nil { + t.Fatalf("Recv() failed: %v", err) + } + if !cmp.Equal(gotProto, wantProto, cmp.Comparer(proto.Equal)) { + t.Logf("Received load report from stream: %s, want: %s", pretty.ToJSON(gotProto), pretty.ToJSON(wantProto)) + continue + } + // This means that we received the metrics which we expected. + break + } +} diff --git a/peer/peer.go b/peer/peer.go new file mode 100644 index 0000000..499a49c --- /dev/null +++ b/peer/peer.go @@ -0,0 +1,83 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package peer defines various peer information associated with RPCs and +// corresponding utils. +package peer + +import ( + "context" + "fmt" + "net" + "strings" + + "google.golang.org/grpc/credentials" +) + +// Peer contains the information of the peer for an RPC, such as the address +// and authentication information. +type Peer struct { + // Addr is the peer address. + Addr net.Addr + // LocalAddr is the local address. + LocalAddr net.Addr + // AuthInfo is the authentication information of the transport. + // It is nil if there is no transport security being used. + AuthInfo credentials.AuthInfo +} + +// String ensures the Peer types implements the Stringer interface in order to +// allow to print a context with a peerKey value effectively. +func (p *Peer) String() string { + if p == nil { + return "Peer" + } + sb := &strings.Builder{} + sb.WriteString("Peer{") + if p.Addr != nil { + fmt.Fprintf(sb, "Addr: '%s', ", p.Addr.String()) + } else { + fmt.Fprintf(sb, "Addr: , ") + } + if p.LocalAddr != nil { + fmt.Fprintf(sb, "LocalAddr: '%s', ", p.LocalAddr.String()) + } else { + fmt.Fprintf(sb, "LocalAddr: , ") + } + if p.AuthInfo != nil { + fmt.Fprintf(sb, "AuthInfo: '%s'", p.AuthInfo.AuthType()) + } else { + fmt.Fprintf(sb, "AuthInfo: ") + } + sb.WriteString("}") + + return sb.String() +} + +type peerKey struct{} + +// NewContext creates a new context with peer information attached. +func NewContext(ctx context.Context, p *Peer) context.Context { + return context.WithValue(ctx, peerKey{}, p) +} + +// FromContext returns the peer information in ctx if it exists. +func FromContext(ctx context.Context) (p *Peer, ok bool) { + p, ok = ctx.Value(peerKey{}).(*Peer) + return +} diff --git a/peer/peer_test.go b/peer/peer_test.go new file mode 100644 index 0000000..2ffff9b --- /dev/null +++ b/peer/peer_test.go @@ -0,0 +1,101 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package peer + +import ( + "context" + "fmt" + "testing" + "time" + + "google.golang.org/grpc/credentials" +) + +const defaultTestTimeout = 10 * time.Second + +// A struct that implements AuthInfo interface and implements CommonAuthInfo() method. +type testAuthInfo struct { + credentials.CommonAuthInfo +} + +func (ta testAuthInfo) AuthType() string { + return fmt.Sprintf("testAuthInfo-%d", ta.SecurityLevel) +} + +type addr struct { + ipAddress string +} + +func (addr) Network() string { return "" } + +func (a *addr) String() string { return a.ipAddress } + +func TestPeerStringer(t *testing.T) { + testCases := []struct { + name string + peer *Peer + want string + }{ + { + name: "+Addr-LocalAddr+ValidAuth", + peer: &Peer{Addr: &addr{"example.com:1234"}, AuthInfo: testAuthInfo{credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}}}, + want: "Peer{Addr: 'example.com:1234', LocalAddr: , AuthInfo: 'testAuthInfo-3'}", + }, + { + name: "+Addr+LocalAddr+ValidAuth", + peer: &Peer{Addr: &addr{"example.com:1234"}, LocalAddr: &addr{"example.com:1234"}, AuthInfo: testAuthInfo{credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}}}, + want: "Peer{Addr: 'example.com:1234', LocalAddr: 'example.com:1234', AuthInfo: 'testAuthInfo-3'}", + }, + { + name: "+Addr-LocalAddr+emptyAuth", + peer: &Peer{Addr: &addr{"1.2.3.4:1234"}, AuthInfo: testAuthInfo{credentials.CommonAuthInfo{}}}, + want: "Peer{Addr: '1.2.3.4:1234', LocalAddr: , AuthInfo: 'testAuthInfo-0'}", + }, + { + name: "-Addr-LocalAddr+emptyAuth", + peer: &Peer{AuthInfo: testAuthInfo{}}, + want: "Peer{Addr: , LocalAddr: , AuthInfo: 'testAuthInfo-0'}", + }, + { + name: "zeroedPeer", + peer: &Peer{}, + want: "Peer{Addr: , LocalAddr: , AuthInfo: }", + }, + { + name: "nilPeer", + peer: nil, + want: "Peer", + }, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx = NewContext(ctx, tc.peer) + + p, ok := FromContext(ctx) + if !ok { + t.Fatalf("Unable to get peer from context") + } + if p.String() != tc.want { + t.Fatalf("Error using peer String(): expected %q, got %q", tc.want, p.String()) + } + }) + } +} diff --git a/picker_wrapper.go b/picker_wrapper.go new file mode 100644 index 0000000..0183ab2 --- /dev/null +++ b/picker_wrapper.go @@ -0,0 +1,221 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "fmt" + "io" + "sync/atomic" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/channelz" + istatus "google.golang.org/grpc/internal/status" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/status" +) + +// pickerGeneration stores a picker and a channel used to signal that a picker +// newer than this one is available. +type pickerGeneration struct { + // picker is the picker produced by the LB policy. May be nil if a picker + // has never been produced. + picker balancer.Picker + // blockingCh is closed when the picker has been invalidated because there + // is a new one available. + blockingCh chan struct{} +} + +// pickerWrapper is a wrapper of balancer.Picker. It blocks on certain pick +// actions and unblock when there's a picker update. +type pickerWrapper struct { + // If pickerGen holds a nil pointer, the pickerWrapper is closed. + pickerGen atomic.Pointer[pickerGeneration] +} + +func newPickerWrapper() *pickerWrapper { + pw := &pickerWrapper{} + pw.pickerGen.Store(&pickerGeneration{ + blockingCh: make(chan struct{}), + }) + return pw +} + +// updatePicker is called by UpdateState calls from the LB policy. It +// unblocks all blocked pick. +func (pw *pickerWrapper) updatePicker(p balancer.Picker) { + old := pw.pickerGen.Swap(&pickerGeneration{ + picker: p, + blockingCh: make(chan struct{}), + }) + close(old.blockingCh) +} + +// doneChannelzWrapper performs the following: +// - increments the calls started channelz counter +// - wraps the done function in the passed in result to increment the calls +// failed or calls succeeded channelz counter before invoking the actual +// done function. +func doneChannelzWrapper(acbw *acBalancerWrapper, result *balancer.PickResult) { + ac := acbw.ac + ac.incrCallsStarted() + done := result.Done + result.Done = func(b balancer.DoneInfo) { + if b.Err != nil && b.Err != io.EOF { + ac.incrCallsFailed() + } else { + ac.incrCallsSucceeded() + } + if done != nil { + done(b) + } + } +} + +type pick struct { + transport transport.ClientTransport // the selected transport + result balancer.PickResult // the contents of the pick from the LB policy + blocked bool // set if a picker call queued for a new picker +} + +// pick returns the transport that will be used for the RPC. +// It may block in the following cases: +// - there's no picker +// - the current picker returns ErrNoSubConnAvailable +// - the current picker returns other errors and failfast is false. +// - the subConn returned by the current picker is not READY +// When one of these situations happens, pick blocks until the picker gets updated. +func (pw *pickerWrapper) pick(ctx context.Context, failfast bool, info balancer.PickInfo) (pick, error) { + var ch chan struct{} + + var lastPickErr error + pickBlocked := false + + for { + pg := pw.pickerGen.Load() + if pg == nil { + return pick{}, ErrClientConnClosing + } + if pg.picker == nil { + ch = pg.blockingCh + } + if ch == pg.blockingCh { + // This could happen when either: + // - pw.picker is nil (the previous if condition), or + // - we have already called pick on the current picker. + select { + case <-ctx.Done(): + var errStr string + if lastPickErr != nil { + errStr = "latest balancer error: " + lastPickErr.Error() + } else { + errStr = fmt.Sprintf("%v while waiting for connections to become ready", ctx.Err()) + } + switch ctx.Err() { + case context.DeadlineExceeded: + return pick{}, status.Error(codes.DeadlineExceeded, errStr) + case context.Canceled: + return pick{}, status.Error(codes.Canceled, errStr) + } + case <-ch: + } + continue + } + + // If the channel is set, it means that the pick call had to wait for a + // new picker at some point. Either it's the first iteration and this + // function received the first picker, or a picker errored with + // ErrNoSubConnAvailable or errored with failfast set to false, which + // will trigger a continue to the next iteration. In the first case this + // conditional will hit if this call had to block (the channel is set). + // In the second case, the only way it will get to this conditional is + // if there is a new picker. + if ch != nil { + pickBlocked = true + } + + ch = pg.blockingCh + p := pg.picker + + pickResult, err := p.Pick(info) + if err != nil { + if err == balancer.ErrNoSubConnAvailable { + continue + } + if st, ok := status.FromError(err); ok { + // Status error: end the RPC unconditionally with this status. + // First restrict the code to the list allowed by gRFC A54. + if istatus.IsRestrictedControlPlaneCode(st) { + err = status.Errorf(codes.Internal, "received picker error with illegal status: %v", err) + } + return pick{}, dropError{error: err} + } + // For all other errors, wait for ready RPCs should block and other + // RPCs should fail with unavailable. + if !failfast { + lastPickErr = err + continue + } + return pick{}, status.Error(codes.Unavailable, err.Error()) + } + + acbw, ok := pickResult.SubConn.(*acBalancerWrapper) + if !ok { + logger.Errorf("subconn returned from pick is type %T, not *acBalancerWrapper", pickResult.SubConn) + continue + } + if t := acbw.ac.getReadyTransport(); t != nil { + if channelz.IsOn() { + doneChannelzWrapper(acbw, &pickResult) + } + return pick{transport: t, result: pickResult, blocked: pickBlocked}, nil + } + if pickResult.Done != nil { + // Calling done with nil error, no bytes sent and no bytes received. + // DoneInfo with default value works. + pickResult.Done(balancer.DoneInfo{}) + } + if logger.V(2) { + logger.Infof("blockingPicker: the picked transport is not ready, loop back to repick") + } + // If ok == false, ac.state is not READY. + // A valid picker always returns READY subConn. This means the state of ac + // just changed, and picker will be updated shortly. + // continue back to the beginning of the for loop to repick. + } +} + +func (pw *pickerWrapper) close() { + old := pw.pickerGen.Swap(nil) + close(old.blockingCh) +} + +// reset clears the pickerWrapper and prepares it for being used again when idle +// mode is exited. +func (pw *pickerWrapper) reset() { + old := pw.pickerGen.Swap(&pickerGeneration{blockingCh: make(chan struct{})}) + close(old.blockingCh) +} + +// dropError is a wrapper error that indicates the LB policy wishes to drop the +// RPC and not retry it. +type dropError struct { + error +} diff --git a/picker_wrapper_test.go b/picker_wrapper_test.go new file mode 100644 index 0000000..f20231a --- /dev/null +++ b/picker_wrapper_test.go @@ -0,0 +1,184 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/status" +) + +const goroutineCount = 5 + +var ( + testT = &testTransport{} + testSC = &acBalancerWrapper{ac: &addrConn{ + state: connectivity.Ready, + transport: testT, + }} + testSCNotReady = &acBalancerWrapper{ac: &addrConn{ + state: connectivity.TransientFailure, + }} +) + +type testTransport struct { + transport.ClientTransport +} + +type testingPicker struct { + err error + sc balancer.SubConn + maxCalled int64 +} + +func (p *testingPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + if atomic.AddInt64(&p.maxCalled, -1) < 0 { + return balancer.PickResult{}, fmt.Errorf("pick called to many times (> goroutineCount)") + } + if p.err != nil { + return balancer.PickResult{}, p.err + } + return balancer.PickResult{SubConn: p.sc}, nil +} + +func (s) TestBlockingPickTimeout(t *testing.T) { + bp := newPickerWrapper() + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + if _, err := bp.pick(ctx, true, balancer.PickInfo{}); status.Code(err) != codes.DeadlineExceeded { + t.Errorf("bp.pick returned error %v, want DeadlineExceeded", err) + } +} + +func (s) TestBlockingPick(t *testing.T) { + bp := newPickerWrapper() + // All goroutines should block because picker is nil in bp. + var finishedCount uint64 + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wg := sync.WaitGroup{} + wg.Add(goroutineCount) + for i := goroutineCount; i > 0; i-- { + go func() { + if pick, err := bp.pick(ctx, true, balancer.PickInfo{}); err != nil || pick.transport != testT { + t.Errorf("bp.pick returned transport: %v, error: %v, want transport: %v, error: nil", pick.transport, err, testT) + } + atomic.AddUint64(&finishedCount, 1) + wg.Done() + }() + } + time.Sleep(50 * time.Millisecond) + if c := atomic.LoadUint64(&finishedCount); c != 0 { + t.Errorf("finished goroutines count: %v, want 0", c) + } + bp.updatePicker(&testingPicker{sc: testSC, maxCalled: goroutineCount}) + // Wait for all pickers to finish before the context is cancelled. + wg.Wait() +} + +func (s) TestBlockingPickNoSubAvailable(t *testing.T) { + bp := newPickerWrapper() + var finishedCount uint64 + bp.updatePicker(&testingPicker{err: balancer.ErrNoSubConnAvailable, maxCalled: goroutineCount}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // All goroutines should block because picker returns no subConn available. + wg := sync.WaitGroup{} + wg.Add(goroutineCount) + for i := goroutineCount; i > 0; i-- { + go func() { + if pick, err := bp.pick(ctx, true, balancer.PickInfo{}); err != nil || pick.transport != testT { + t.Errorf("bp.pick returned transport: %v, error: %v, want transport: %v, error: nil", pick.transport, err, testT) + } + atomic.AddUint64(&finishedCount, 1) + wg.Done() + }() + } + time.Sleep(50 * time.Millisecond) + if c := atomic.LoadUint64(&finishedCount); c != 0 { + t.Errorf("finished goroutines count: %v, want 0", c) + } + bp.updatePicker(&testingPicker{sc: testSC, maxCalled: goroutineCount}) + // Wait for all pickers to finish before the context is cancelled. + wg.Wait() +} + +func (s) TestBlockingPickTransientWaitforready(t *testing.T) { + bp := newPickerWrapper() + bp.updatePicker(&testingPicker{err: balancer.ErrTransientFailure, maxCalled: goroutineCount}) + var finishedCount uint64 + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // All goroutines should block because picker returns transientFailure and + // picks are not failfast. + wg := sync.WaitGroup{} + wg.Add(goroutineCount) + for i := goroutineCount; i > 0; i-- { + go func() { + if pick, err := bp.pick(ctx, false, balancer.PickInfo{}); err != nil || pick.transport != testT { + t.Errorf("bp.pick returned transport: %v, error: %v, want transport: %v, error: nil", pick.transport, err, testT) + } + atomic.AddUint64(&finishedCount, 1) + wg.Done() + }() + } + time.Sleep(time.Millisecond) + if c := atomic.LoadUint64(&finishedCount); c != 0 { + t.Errorf("finished goroutines count: %v, want 0", c) + } + bp.updatePicker(&testingPicker{sc: testSC, maxCalled: goroutineCount}) + // Wait for all pickers to finish before the context is cancelled. + wg.Wait() +} + +func (s) TestBlockingPickSCNotReady(t *testing.T) { + bp := newPickerWrapper() + bp.updatePicker(&testingPicker{sc: testSCNotReady, maxCalled: goroutineCount}) + var finishedCount uint64 + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // All goroutines should block because subConn is not ready. + wg := sync.WaitGroup{} + wg.Add(goroutineCount) + for i := goroutineCount; i > 0; i-- { + go func() { + if pick, err := bp.pick(ctx, true, balancer.PickInfo{}); err != nil || pick.transport != testT { + t.Errorf("bp.pick returned transport: %v, error: %v, want transport: %v, error: nil", pick.transport, err, testT) + } + atomic.AddUint64(&finishedCount, 1) + wg.Done() + }() + } + time.Sleep(time.Millisecond) + if c := atomic.LoadUint64(&finishedCount); c != 0 { + t.Errorf("finished goroutines count: %v, want 0", c) + } + bp.updatePicker(&testingPicker{sc: testSC, maxCalled: goroutineCount}) + // Wait for all pickers to finish before the context is cancelled. + wg.Wait() +} diff --git a/preloader.go b/preloader.go new file mode 100644 index 0000000..1e783fe --- /dev/null +++ b/preloader.go @@ -0,0 +1,82 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "google.golang.org/grpc/codes" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/status" +) + +// PreparedMsg is responsible for creating a Marshalled and Compressed object. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type PreparedMsg struct { + // Struct for preparing msg before sending them + encodedData mem.BufferSlice + hdr []byte + payload mem.BufferSlice + pf payloadFormat +} + +// Encode marshalls and compresses the message using the codec and compressor for the stream. +func (p *PreparedMsg) Encode(s Stream, msg any) error { + ctx := s.Context() + rpcInfo, ok := rpcInfoFromContext(ctx) + if !ok { + return status.Errorf(codes.Internal, "grpc: unable to get rpcInfo") + } + + // check if the context has the relevant information to prepareMsg + if rpcInfo.preloaderInfo.codec == nil { + return status.Errorf(codes.Internal, "grpc: rpcInfo.preloaderInfo.codec is nil") + } + + // prepare the msg + data, err := encode(rpcInfo.preloaderInfo.codec, msg) + if err != nil { + return err + } + + materializedData := data.Materialize() + data.Free() + p.encodedData = mem.BufferSlice{mem.SliceBuffer(materializedData)} + + // TODO: it should be possible to grab the bufferPool from the underlying + // stream implementation with a type cast to its actual type (such as + // addrConnStream) and accessing the buffer pool directly. + var compData mem.BufferSlice + compData, p.pf, err = compress(p.encodedData, rpcInfo.preloaderInfo.cp, rpcInfo.preloaderInfo.comp, mem.DefaultBufferPool()) + if err != nil { + return err + } + + if p.pf.isCompressed() { + materializedCompData := compData.Materialize() + compData.Free() + compData = mem.BufferSlice{mem.SliceBuffer(materializedCompData)} + } + + p.hdr, p.payload = msgHeader(p.encodedData, compData, p.pf) + + return nil +} diff --git a/producer_ext_test.go b/producer_ext_test.go new file mode 100644 index 0000000..5a4e528 --- /dev/null +++ b/producer_ext_test.go @@ -0,0 +1,116 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc_test + +import ( + "context" + "strings" + "sync/atomic" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +// TestProducerStopsBeforeStateChange confirms that producers are stopped before +// any state change notification is delivered to the LB policy. +func (s) TestProducerStopsBeforeStateChange(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + name := strings.ReplaceAll(strings.ToLower(t.Name()), "/", "") + var lastProducer *testProducer + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + var sc balancer.SubConn + sc, err := bd.ClientConn.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{ + StateListener: func(scs balancer.SubConnState) { + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: scs.ConnectivityState, + // We do not pass a picker, but since we don't perform + // RPCs, that's okay. + }) + if !lastProducer.stopped.Load() { + t.Errorf("lastProducer not stopped before state change notification") + } + t.Logf("State is now %v; recreating producer", scs.ConnectivityState) + p, _ := sc.GetOrBuildProducer(producerBuilderSingleton) + lastProducer = p.(*testProducer) + }, + }) + if err != nil { + return err + } + p, _ := sc.GetOrBuildProducer(producerBuilderSingleton) + lastProducer = p.(*testProducer) + sc.Connect() + return nil + }, + } + stub.Register(name, bf) + + ss := stubserver.StubServer{ + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + return nil + }, + } + if err := ss.StartServer(); err != nil { + t.Fatal("Error starting server:", err) + } + defer ss.Stop() + + cc, err := grpc.NewClient("dns:///"+ss.Address, + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"`+name+`":{}}]}`), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + t.Fatalf("Error creating client: %v", err) + } + defer cc.Close() + + go cc.Connect() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + cc.Close() + testutils.AwaitState(ctx, t, cc, connectivity.Shutdown) +} + +type producerBuilder struct{} + +type testProducer struct { + // There should be no race accessing this field, but use an atomic since + // the race checker probably can't detect that. + stopped atomic.Bool +} + +// Build constructs and returns a producer and its cleanup function +func (*producerBuilder) Build(any) (balancer.Producer, func()) { + p := &testProducer{} + return p, func() { + p.stopped.Store(true) + } +} + +var producerBuilderSingleton = &producerBuilder{} diff --git a/profiling/cmd/catapult.go b/profiling/cmd/catapult.go new file mode 100644 index 0000000..6669b27 --- /dev/null +++ b/profiling/cmd/catapult.go @@ -0,0 +1,394 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "os" + "sort" + "strings" + + ppb "google.golang.org/grpc/profiling/proto" +) + +type jsonNode struct { + Name string `json:"name"` + Cat string `json:"cat"` + ID string `json:"id"` + Cname string `json:"cname"` + Phase string `json:"ph"` + Timestamp float64 `json:"ts"` + PID string `json:"pid"` + TID string `json:"tid"` +} + +// Catapult does not allow specifying colours manually; a 20-odd predefined +// labels are used (that don't make much sense outside the context of +// Chromium). See this for more details: +// +// https://github.com/catapult-project/catapult/blob/bef344f7017fc9e04f7049d0f58af6d9ce9f4ab6/tracing/tracing/base/color_scheme.html#L29 +func hashCname(tag string) string { + if strings.Contains(tag, "encoding") { + return "rail_response" + } + + if strings.Contains(tag, "compression") { + return "cq_build_passed" + } + + if strings.Contains(tag, "transport") { + if strings.Contains(tag, "blocking") { + return "rail_animation" + } + return "good" + } + + if strings.Contains(tag, "header") { + return "cq_build_attempt_failed" + } + + if tag == "/" { + return "heap_dump_stack_frame" + } + + if strings.Contains(tag, "flow") || strings.Contains(tag, "tmp") { + return "heap_dump_stack_frame" + } + + return "" +} + +// filterCounter identifies the counter-th instance of a timer of the type +// `filter` within a Stat. This, in conjunction with the counter data structure +// defined below, is used to draw flows between linked loopy writer/reader +// events with application goroutine events in trace-viewer. This is possible +// because enqueues and dequeues are ordered -- that is, the first dequeue must +// be dequeueing the first enqueue operation. +func filterCounter(stat *ppb.Stat, filter string, counter int) int { + localCounter := 0 + for i := 0; i < len(stat.Timers); i++ { + if stat.Timers[i].Tags == filter { + if localCounter == counter { + return i + } + localCounter++ + } + } + + return -1 +} + +// counter is state object used to store and retrieve the number of timers of a +// particular type that have been seen. +type counter struct { + c map[string]int +} + +func newCounter() *counter { + return &counter{c: make(map[string]int)} +} + +func (c *counter) GetAndInc(s string) int { + ret := c.c[s] + c.c[s]++ + return ret +} + +func catapultNs(sec int64, nsec int32) float64 { + return float64((sec * 1000000000) + int64(nsec)) +} + +// streamStatsCatapultJSONSingle processes a single proto Stat object to return +// an array of jsonNodes in trace-viewer's format. +func streamStatsCatapultJSONSingle(stat *ppb.Stat, baseSec int64, baseNsec int32) []jsonNode { + if len(stat.Timers) == 0 { + return nil + } + + connectionCounter := binary.BigEndian.Uint64(stat.Metadata[0:8]) + streamID := binary.BigEndian.Uint32(stat.Metadata[8:12]) + opid := fmt.Sprintf("/%s/%d/%d", stat.Tags, connectionCounter, streamID) + + var loopyReaderGoID, loopyWriterGoID int64 + for i := 0; i < len(stat.Timers) && (loopyReaderGoID == 0 || loopyWriterGoID == 0); i++ { + if strings.Contains(stat.Timers[i].Tags, "/loopyReader") { + loopyReaderGoID = stat.Timers[i].GoId + } else if strings.Contains(stat.Timers[i].Tags, "/loopyWriter") { + loopyWriterGoID = stat.Timers[i].GoId + } + } + + lrc, lwc := newCounter(), newCounter() + + var result []jsonNode + result = append(result, + jsonNode{ + Name: "loopyReaderTmp", + ID: opid, + Cname: hashCname("tmp"), + Phase: "i", + Timestamp: 0, + PID: fmt.Sprintf("/%s/%d/loopyReader", stat.Tags, connectionCounter), + TID: fmt.Sprintf("%d", loopyReaderGoID), + }, + jsonNode{ + Name: "loopyWriterTmp", + ID: opid, + Cname: hashCname("tmp"), + Phase: "i", + Timestamp: 0, + PID: fmt.Sprintf("/%s/%d/loopyWriter", stat.Tags, connectionCounter), + TID: fmt.Sprintf("%d", loopyWriterGoID), + }, + ) + + for i := 0; i < len(stat.Timers); i++ { + categories := stat.Tags + pid, tid := opid, fmt.Sprintf("%d", stat.Timers[i].GoId) + + if stat.Timers[i].GoId == loopyReaderGoID { + pid, tid = fmt.Sprintf("/%s/%d/loopyReader", stat.Tags, connectionCounter), fmt.Sprintf("%d", stat.Timers[i].GoId) + + var flowEndID int + var flowEndPID, flowEndTID string + switch stat.Timers[i].Tags { + case "/http2/recv/header": + flowEndID = filterCounter(stat, "/grpc/stream/recv/header", lrc.GetAndInc("/http2/recv/header")) + if flowEndID != -1 { + flowEndPID = opid + flowEndTID = fmt.Sprintf("%d", stat.Timers[flowEndID].GoId) + } else { + logger.Infof("cannot find %s/grpc/stream/recv/header for %s/http2/recv/header", opid, opid) + } + case "/http2/recv/dataFrame/loopyReader": + flowEndID = filterCounter(stat, "/recvAndDecompress", lrc.GetAndInc("/http2/recv/dataFrame/loopyReader")) + if flowEndID != -1 { + flowEndPID = opid + flowEndTID = fmt.Sprintf("%d", stat.Timers[flowEndID].GoId) + } else { + logger.Infof("cannot find %s/recvAndDecompress for %s/http2/recv/dataFrame/loopyReader", opid, opid) + } + default: + flowEndID = -1 + } + + if flowEndID != -1 { + flowID := fmt.Sprintf("lrc begin:/%d%s end:/%d%s begin:(%d, %s, %s) end:(%d, %s, %s)", connectionCounter, stat.Timers[i].Tags, connectionCounter, stat.Timers[flowEndID].Tags, i, pid, tid, flowEndID, flowEndPID, flowEndTID) + result = append(result, + jsonNode{ + Name: fmt.Sprintf("%s/flow", opid), + Cat: categories + ",flow", + ID: flowID, + Cname: hashCname("flow"), + Phase: "s", + Timestamp: catapultNs(stat.Timers[i].EndSec-baseSec, stat.Timers[i].EndNsec-baseNsec), + PID: pid, + TID: tid, + }, + jsonNode{ + Name: fmt.Sprintf("%s/flow", opid), + Cat: categories + ",flow", + ID: flowID, + Cname: hashCname("flow"), + Phase: "f", + Timestamp: catapultNs(stat.Timers[flowEndID].BeginSec-baseSec, stat.Timers[flowEndID].BeginNsec-baseNsec), + PID: flowEndPID, + TID: flowEndTID, + }, + ) + } + } else if stat.Timers[i].GoId == loopyWriterGoID { + pid, tid = fmt.Sprintf("/%s/%d/loopyWriter", stat.Tags, connectionCounter), fmt.Sprintf("%d", stat.Timers[i].GoId) + + var flowBeginID int + var flowBeginPID, flowBeginTID string + switch stat.Timers[i].Tags { + case "/http2/recv/header/loopyWriter/registerOutStream": + flowBeginID = filterCounter(stat, "/http2/recv/header", lwc.GetAndInc("/http2/recv/header/loopyWriter/registerOutStream")) + flowBeginPID = fmt.Sprintf("/%s/%d/loopyReader", stat.Tags, connectionCounter) + flowBeginTID = fmt.Sprintf("%d", loopyReaderGoID) + case "/http2/send/dataFrame/loopyWriter/preprocess": + flowBeginID = filterCounter(stat, "/transport/enqueue", lwc.GetAndInc("/http2/send/dataFrame/loopyWriter/preprocess")) + if flowBeginID != -1 { + flowBeginPID = opid + flowBeginTID = fmt.Sprintf("%d", stat.Timers[flowBeginID].GoId) + } else { + logger.Infof("cannot find /%d/transport/enqueue for /%d/http2/send/dataFrame/loopyWriter/preprocess", connectionCounter, connectionCounter) + } + default: + flowBeginID = -1 + } + + if flowBeginID != -1 { + flowID := fmt.Sprintf("lwc begin:/%d%s end:/%d%s begin:(%d, %s, %s) end:(%d, %s, %s)", connectionCounter, stat.Timers[flowBeginID].Tags, connectionCounter, stat.Timers[i].Tags, flowBeginID, flowBeginPID, flowBeginTID, i, pid, tid) + result = append(result, + jsonNode{ + Name: fmt.Sprintf("/%s/%d/%d/flow", stat.Tags, connectionCounter, streamID), + Cat: categories + ",flow", + ID: flowID, + Cname: hashCname("flow"), + Phase: "s", + Timestamp: catapultNs(stat.Timers[flowBeginID].EndSec-baseSec, stat.Timers[flowBeginID].EndNsec-baseNsec), + PID: flowBeginPID, + TID: flowBeginTID, + }, + jsonNode{ + Name: fmt.Sprintf("/%s/%d/%d/flow", stat.Tags, connectionCounter, streamID), + Cat: categories + ",flow", + ID: flowID, + Cname: hashCname("flow"), + Phase: "f", + Timestamp: catapultNs(stat.Timers[i].BeginSec-baseSec, stat.Timers[i].BeginNsec-baseNsec), + PID: pid, + TID: tid, + }, + ) + } + } + + result = append(result, + jsonNode{ + Name: fmt.Sprintf("%s%s", opid, stat.Timers[i].Tags), + Cat: categories, + ID: opid, + Cname: hashCname(stat.Timers[i].Tags), + Phase: "B", + Timestamp: catapultNs(stat.Timers[i].BeginSec-baseSec, stat.Timers[i].BeginNsec-baseNsec), + PID: pid, + TID: tid, + }, + jsonNode{ + Name: fmt.Sprintf("%s%s", opid, stat.Timers[i].Tags), + Cat: categories, + ID: opid, + Cname: hashCname(stat.Timers[i].Tags), + Phase: "E", + Timestamp: catapultNs(stat.Timers[i].EndSec-baseSec, stat.Timers[i].EndNsec-baseNsec), + PID: pid, + TID: tid, + }, + ) + } + + return result +} + +// timerBeginIsBefore compares two proto Timer objects to determine if the +// first comes before the second chronologically. +func timerBeginIsBefore(ti *ppb.Timer, tj *ppb.Timer) bool { + if ti.BeginSec == tj.BeginSec { + return ti.BeginNsec < tj.BeginNsec + } + return ti.BeginSec < tj.BeginSec +} + +// streamStatsCatapultJSON receives a *snapshot and the name of a JSON file to +// write to. The grpc-go profiling snapshot is processed and converted to a +// JSON format that can be understood by trace-viewer. +func streamStatsCatapultJSON(s *snapshot, streamStatsCatapultJSONFileName string) (err error) { + logger.Infof("calculating stream stats filters") + filterArray := strings.Split(*flagStreamStatsFilter, ",") + filter := make(map[string]bool) + for _, f := range filterArray { + filter[f] = true + } + + logger.Infof("filter stream stats for %s", *flagStreamStatsFilter) + var streamStats []*ppb.Stat + for _, stat := range s.StreamStats { + if _, ok := filter[stat.Tags]; ok { + streamStats = append(streamStats, stat) + } + } + + logger.Infof("sorting timers within all stats") + for id := range streamStats { + sort.Slice(streamStats[id].Timers, func(i, j int) bool { + return timerBeginIsBefore(streamStats[id].Timers[i], streamStats[id].Timers[j]) + }) + } + + logger.Infof("sorting stream stats") + sort.Slice(streamStats, func(i, j int) bool { + if len(streamStats[j].Timers) == 0 { + return true + } else if len(streamStats[i].Timers) == 0 { + return false + } + pi := binary.BigEndian.Uint64(streamStats[i].Metadata[0:8]) + pj := binary.BigEndian.Uint64(streamStats[j].Metadata[0:8]) + if pi == pj { + return timerBeginIsBefore(streamStats[i].Timers[0], streamStats[j].Timers[0]) + } + + return pi < pj + }) + + // Clip the last stat as it's from the /Profiling/GetStreamStats call that we + // made to retrieve the stats themselves. This likely happened millions of + // nanoseconds after the last stream we want to profile, so it'd just make + // the catapult graph less readable. + if len(streamStats) > 0 { + streamStats = streamStats[:len(streamStats)-1] + } + + // All timestamps use the earliest timestamp available as the reference. + logger.Infof("calculating the earliest timestamp across all timers") + var base *ppb.Timer + for _, stat := range streamStats { + for _, timer := range stat.Timers { + if base == nil || timerBeginIsBefore(base, timer) { + base = timer + } + } + } + + logger.Infof("converting %d stats to catapult JSON format", len(streamStats)) + var jsonNodes []jsonNode + for _, stat := range streamStats { + jsonNodes = append(jsonNodes, streamStatsCatapultJSONSingle(stat, base.BeginSec, base.BeginNsec)...) + } + + logger.Infof("marshalling catapult JSON") + b, err := json.Marshal(jsonNodes) + if err != nil { + logger.Errorf("cannot marshal JSON: %v", err) + return err + } + + logger.Infof("creating catapult JSON file") + streamStatsCatapultJSONFile, err := os.Create(streamStatsCatapultJSONFileName) + if err != nil { + logger.Errorf("cannot create file %s: %v", streamStatsCatapultJSONFileName, err) + return err + } + defer streamStatsCatapultJSONFile.Close() + + logger.Infof("writing catapult JSON to disk") + _, err = streamStatsCatapultJSONFile.Write(b) + if err != nil { + logger.Errorf("cannot write marshalled JSON: %v", err) + return err + } + + logger.Infof("successfully wrote catapult JSON file %s", streamStatsCatapultJSONFileName) + return nil +} diff --git a/profiling/cmd/flags.go b/profiling/cmd/flags.go new file mode 100644 index 0000000..589bb15 --- /dev/null +++ b/profiling/cmd/flags.go @@ -0,0 +1,76 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "flag" + "fmt" +) + +var flagAddress = flag.String("address", "", "address of a remote gRPC server with profiling turned on to retrieve stats from") +var flagTimeout = flag.Int("timeout", 0, "network operations timeout in seconds to remote target (0 indicates unlimited)") + +var flagRetrieveSnapshot = flag.Bool("retrieve-snapshot", false, "connect to remote target and retrieve a profiling snapshot locally for processing") +var flagSnapshot = flag.String("snapshot", "", "snapshot file to write to when retrieving profiling data or snapshot file to read from when processing profiling data") + +var flagEnableProfiling = flag.Bool("enable-profiling", false, "enable profiling in remote target") +var flagDisableProfiling = flag.Bool("disable-profiling", false, "disable profiling in remote target") + +var flagStreamStatsCatapultJSON = flag.String("stream-stats-catapult-json", "", "path to a file to write to after transforming a snapshot into catapult's JSON format") +var flagStreamStatsFilter = flag.String("stream-stats-filter", "server,client", "comma-separated list of stat tags to filter for") + +func exactlyOneOf(opts ...bool) bool { + first := true + for _, o := range opts { + if !o { + continue + } + + if !first { + return false + } + first = false + } + + return !first +} + +func parseArgs() error { + flag.Parse() + + if *flagAddress != "" { + if !exactlyOneOf(*flagEnableProfiling, *flagDisableProfiling, *flagRetrieveSnapshot) { + return fmt.Errorf("when -address is specified, you must include exactly only one of -enable-profiling, -disable-profiling, and -retrieve-snapshot") + } + + if *flagStreamStatsCatapultJSON != "" { + return fmt.Errorf("when -address is specified, you must not include -stream-stats-catapult-json") + } + } else { + if *flagEnableProfiling || *flagDisableProfiling || *flagRetrieveSnapshot { + return fmt.Errorf("when -address isn't specified, you must not include any of -enable-profiling, -disable-profiling, and -retrieve-snapshot") + } + + if *flagStreamStatsCatapultJSON == "" { + return fmt.Errorf("when -address isn't specified, you must include -stream-stats-catapult-json") + } + } + + return nil +} diff --git a/profiling/cmd/local.go b/profiling/cmd/local.go new file mode 100644 index 0000000..7c312ac --- /dev/null +++ b/profiling/cmd/local.go @@ -0,0 +1,68 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "encoding/gob" + "fmt" + "os" +) + +func loadSnapshot(snapshotFileName string) (*snapshot, error) { + logger.Infof("opening snapshot file %s", snapshotFileName) + snapshotFile, err := os.Open(snapshotFileName) + if err != nil { + logger.Errorf("cannot open %s: %v", snapshotFileName, err) + return nil, err + } + defer snapshotFile.Close() + + logger.Infof("decoding snapshot file %s", snapshotFileName) + s := &snapshot{} + decoder := gob.NewDecoder(snapshotFile) + if err = decoder.Decode(s); err != nil { + logger.Errorf("cannot decode %s: %v", snapshotFileName, err) + return nil, err + } + + return s, nil +} + +func localCommand() error { + if *flagSnapshot == "" { + return fmt.Errorf("-snapshot flag missing") + } + + s, err := loadSnapshot(*flagSnapshot) + if err != nil { + return err + } + + if *flagStreamStatsCatapultJSON == "" { + return fmt.Errorf("snapshot file specified without an action to perform") + } + + if *flagStreamStatsCatapultJSON != "" { + if err = streamStatsCatapultJSON(s, *flagStreamStatsCatapultJSON); err != nil { + return err + } + } + + return nil +} diff --git a/profiling/cmd/main.go b/profiling/cmd/main.go new file mode 100644 index 0000000..2328fbd --- /dev/null +++ b/profiling/cmd/main.go @@ -0,0 +1,53 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary cmd is a command-line tool for profiling management. It retrieves and +// processes data from the profiling service. +package main + +import ( + "os" + + "google.golang.org/grpc/grpclog" + ppb "google.golang.org/grpc/profiling/proto" +) + +var logger = grpclog.Component("profiling") + +type snapshot struct { + StreamStats []*ppb.Stat +} + +func main() { + if err := parseArgs(); err != nil { + logger.Errorf("error parsing flags: %v", err) + os.Exit(1) + } + + if *flagAddress != "" { + if err := remoteCommand(); err != nil { + logger.Errorf("error: %v", err) + os.Exit(1) + } + } else { + if err := localCommand(); err != nil { + logger.Errorf("error: %v", err) + os.Exit(1) + } + } +} diff --git a/profiling/cmd/remote.go b/profiling/cmd/remote.go new file mode 100644 index 0000000..e21ef14 --- /dev/null +++ b/profiling/cmd/remote.go @@ -0,0 +1,98 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "context" + "encoding/gob" + "fmt" + "os" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + ppb "google.golang.org/grpc/profiling/proto" +) + +func setEnabled(ctx context.Context, c ppb.ProfilingClient, enabled bool) error { + _, err := c.Enable(ctx, &ppb.EnableRequest{Enabled: enabled}) + if err != nil { + logger.Infof("error calling Enable: %v\n", err) + return err + } + + logger.Infof("successfully set enabled = %v", enabled) + return nil +} + +func retrieveSnapshot(ctx context.Context, c ppb.ProfilingClient, f string) error { + logger.Infof("getting stream stats") + resp, err := c.GetStreamStats(ctx, &ppb.GetStreamStatsRequest{}) + if err != nil { + logger.Errorf("error calling GetStreamStats: %v\n", err) + return err + } + s := &snapshot{StreamStats: resp.StreamStats} + + logger.Infof("creating snapshot file %s", f) + file, err := os.Create(f) + if err != nil { + logger.Errorf("cannot create %s: %v", f, err) + return err + } + defer file.Close() + + logger.Infof("encoding data and writing to snapshot file %s", f) + encoder := gob.NewEncoder(file) + err = encoder.Encode(s) + if err != nil { + logger.Infof("error encoding: %v", err) + return err + } + + logger.Infof("successfully wrote profiling snapshot to %s", f) + return nil +} + +func remoteCommand() error { + ctx := context.Background() + if *flagTimeout > 0 { + var cancel func() + ctx, cancel = context.WithTimeout(context.Background(), time.Duration(*flagTimeout)*time.Second) + defer cancel() + } + + logger.Infof("dialing %s", *flagAddress) + cc, err := grpc.NewClient(*flagAddress, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + logger.Fatalf("grpc.NewClient(%q) = %v", *flagAddress, err) + return err + } + defer cc.Close() + + c := ppb.NewProfilingClient(cc) + + if *flagEnableProfiling || *flagDisableProfiling { + return setEnabled(ctx, c, *flagEnableProfiling) + } + if *flagRetrieveSnapshot { + return retrieveSnapshot(ctx, c, *flagSnapshot) + } + return fmt.Errorf("what should I do with the remote target?") +} diff --git a/profiling/profiling.go b/profiling/profiling.go new file mode 100644 index 0000000..869054e --- /dev/null +++ b/profiling/profiling.go @@ -0,0 +1,40 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package profiling exposes methods to manage profiling within gRPC. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +package profiling + +import ( + internal "google.golang.org/grpc/internal/profiling" +) + +// Enable turns profiling on and off. This operation is safe for concurrent +// access from different goroutines. +// +// Note that this is the only operation that's accessible through the publicly +// exposed profiling package. Everything else (such as retrieving stats) must +// be done through the profiling service. This is allowed so that users can use +// heuristics to turn profiling on and off automatically. +func Enable(enabled bool) { + internal.Enable(enabled) +} diff --git a/profiling/proto/service.pb.go b/profiling/proto/service.pb.go new file mode 100644 index 0000000..12a54e2 --- /dev/null +++ b/profiling/proto/service.pb.go @@ -0,0 +1,455 @@ +// Copyright 2019 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: profiling/proto/service.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// EnableRequest defines the fields in a /Profiling/Enable method request to +// toggle profiling on and off within a gRPC program. +type EnableRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Setting this to true will enable profiling. Setting this to false will + // disable profiling. + Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnableRequest) Reset() { + *x = EnableRequest{} + mi := &file_profiling_proto_service_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnableRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnableRequest) ProtoMessage() {} + +func (x *EnableRequest) ProtoReflect() protoreflect.Message { + mi := &file_profiling_proto_service_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnableRequest.ProtoReflect.Descriptor instead. +func (*EnableRequest) Descriptor() ([]byte, []int) { + return file_profiling_proto_service_proto_rawDescGZIP(), []int{0} +} + +func (x *EnableRequest) GetEnabled() bool { + if x != nil { + return x.Enabled + } + return false +} + +// EnableResponse defines the fields in a /Profiling/Enable method response. +type EnableResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnableResponse) Reset() { + *x = EnableResponse{} + mi := &file_profiling_proto_service_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnableResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnableResponse) ProtoMessage() {} + +func (x *EnableResponse) ProtoReflect() protoreflect.Message { + mi := &file_profiling_proto_service_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnableResponse.ProtoReflect.Descriptor instead. +func (*EnableResponse) Descriptor() ([]byte, []int) { + return file_profiling_proto_service_proto_rawDescGZIP(), []int{1} +} + +// GetStreamStatsRequest defines the fields in a /Profiling/GetStreamStats +// method request to retrieve stream-level stats in a gRPC client/server. +type GetStreamStatsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetStreamStatsRequest) Reset() { + *x = GetStreamStatsRequest{} + mi := &file_profiling_proto_service_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetStreamStatsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetStreamStatsRequest) ProtoMessage() {} + +func (x *GetStreamStatsRequest) ProtoReflect() protoreflect.Message { + mi := &file_profiling_proto_service_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetStreamStatsRequest.ProtoReflect.Descriptor instead. +func (*GetStreamStatsRequest) Descriptor() ([]byte, []int) { + return file_profiling_proto_service_proto_rawDescGZIP(), []int{2} +} + +// GetStreamStatsResponse defines the fields in a /Profiling/GetStreamStats +// method response. +type GetStreamStatsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + StreamStats []*Stat `protobuf:"bytes,1,rep,name=stream_stats,json=streamStats,proto3" json:"stream_stats,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetStreamStatsResponse) Reset() { + *x = GetStreamStatsResponse{} + mi := &file_profiling_proto_service_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetStreamStatsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetStreamStatsResponse) ProtoMessage() {} + +func (x *GetStreamStatsResponse) ProtoReflect() protoreflect.Message { + mi := &file_profiling_proto_service_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetStreamStatsResponse.ProtoReflect.Descriptor instead. +func (*GetStreamStatsResponse) Descriptor() ([]byte, []int) { + return file_profiling_proto_service_proto_rawDescGZIP(), []int{3} +} + +func (x *GetStreamStatsResponse) GetStreamStats() []*Stat { + if x != nil { + return x.StreamStats + } + return nil +} + +// A Timer measures the start and end of execution of a component within +// gRPC that's being profiled. It includes a tag and some additional metadata +// to identify itself. +type Timer struct { + state protoimpl.MessageState `protogen:"open.v1"` + // tags is a comma-separated list of strings used to tag a timer. + Tags string `protobuf:"bytes,1,opt,name=tags,proto3" json:"tags,omitempty"` + // begin_sec and begin_nsec are the start epoch second and nanosecond, + // respectively, of the component profiled by this timer in UTC. begin_nsec + // must be a non-negative integer. + BeginSec int64 `protobuf:"varint,2,opt,name=begin_sec,json=beginSec,proto3" json:"begin_sec,omitempty"` + BeginNsec int32 `protobuf:"varint,3,opt,name=begin_nsec,json=beginNsec,proto3" json:"begin_nsec,omitempty"` + // end_sec and end_nsec are the end epoch second and nanosecond, + // respectively, of the component profiled by this timer in UTC. end_nsec + // must be a non-negative integer. + EndSec int64 `protobuf:"varint,4,opt,name=end_sec,json=endSec,proto3" json:"end_sec,omitempty"` + EndNsec int32 `protobuf:"varint,5,opt,name=end_nsec,json=endNsec,proto3" json:"end_nsec,omitempty"` + // go_id is the goroutine ID of the component being profiled. + GoId int64 `protobuf:"varint,6,opt,name=go_id,json=goId,proto3" json:"go_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Timer) Reset() { + *x = Timer{} + mi := &file_profiling_proto_service_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Timer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Timer) ProtoMessage() {} + +func (x *Timer) ProtoReflect() protoreflect.Message { + mi := &file_profiling_proto_service_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Timer.ProtoReflect.Descriptor instead. +func (*Timer) Descriptor() ([]byte, []int) { + return file_profiling_proto_service_proto_rawDescGZIP(), []int{4} +} + +func (x *Timer) GetTags() string { + if x != nil { + return x.Tags + } + return "" +} + +func (x *Timer) GetBeginSec() int64 { + if x != nil { + return x.BeginSec + } + return 0 +} + +func (x *Timer) GetBeginNsec() int32 { + if x != nil { + return x.BeginNsec + } + return 0 +} + +func (x *Timer) GetEndSec() int64 { + if x != nil { + return x.EndSec + } + return 0 +} + +func (x *Timer) GetEndNsec() int32 { + if x != nil { + return x.EndNsec + } + return 0 +} + +func (x *Timer) GetGoId() int64 { + if x != nil { + return x.GoId + } + return 0 +} + +// A Stat is a collection of Timers along with some additional +// metadata to tag and identify itself. +type Stat struct { + state protoimpl.MessageState `protogen:"open.v1"` + // tags is a comma-separated list of strings used to categorize a stat. + Tags string `protobuf:"bytes,1,opt,name=tags,proto3" json:"tags,omitempty"` + // timers is an array of Timers, each representing a different + // (but possibly overlapping) component within this stat. + Timers []*Timer `protobuf:"bytes,2,rep,name=timers,proto3" json:"timers,omitempty"` + // metadata is an array of bytes used to uniquely identify a stat with an + // undefined encoding format. For example, the Stats returned by the + // /Profiling/GetStreamStats service use the metadata field to encode the + // connection ID and the stream ID of each query. + Metadata []byte `protobuf:"bytes,3,opt,name=metadata,proto3" json:"metadata,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Stat) Reset() { + *x = Stat{} + mi := &file_profiling_proto_service_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Stat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Stat) ProtoMessage() {} + +func (x *Stat) ProtoReflect() protoreflect.Message { + mi := &file_profiling_proto_service_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Stat.ProtoReflect.Descriptor instead. +func (*Stat) Descriptor() ([]byte, []int) { + return file_profiling_proto_service_proto_rawDescGZIP(), []int{5} +} + +func (x *Stat) GetTags() string { + if x != nil { + return x.Tags + } + return "" +} + +func (x *Stat) GetTimers() []*Timer { + if x != nil { + return x.Timers + } + return nil +} + +func (x *Stat) GetMetadata() []byte { + if x != nil { + return x.Metadata + } + return nil +} + +var File_profiling_proto_service_proto protoreflect.FileDescriptor + +const file_profiling_proto_service_proto_rawDesc = "" + + "\n" + + "\x1dprofiling/proto/service.proto\x12\x19grpc.go.profiling.v1alpha\")\n" + + "\rEnableRequest\x12\x18\n" + + "\aenabled\x18\x01 \x01(\bR\aenabled\"\x10\n" + + "\x0eEnableResponse\"\x17\n" + + "\x15GetStreamStatsRequest\"\\\n" + + "\x16GetStreamStatsResponse\x12B\n" + + "\fstream_stats\x18\x01 \x03(\v2\x1f.grpc.go.profiling.v1alpha.StatR\vstreamStats\"\xa0\x01\n" + + "\x05Timer\x12\x12\n" + + "\x04tags\x18\x01 \x01(\tR\x04tags\x12\x1b\n" + + "\tbegin_sec\x18\x02 \x01(\x03R\bbeginSec\x12\x1d\n" + + "\n" + + "begin_nsec\x18\x03 \x01(\x05R\tbeginNsec\x12\x17\n" + + "\aend_sec\x18\x04 \x01(\x03R\x06endSec\x12\x19\n" + + "\bend_nsec\x18\x05 \x01(\x05R\aendNsec\x12\x13\n" + + "\x05go_id\x18\x06 \x01(\x03R\x04goId\"p\n" + + "\x04Stat\x12\x12\n" + + "\x04tags\x18\x01 \x01(\tR\x04tags\x128\n" + + "\x06timers\x18\x02 \x03(\v2 .grpc.go.profiling.v1alpha.TimerR\x06timers\x12\x1a\n" + + "\bmetadata\x18\x03 \x01(\fR\bmetadata2\xe1\x01\n" + + "\tProfiling\x12]\n" + + "\x06Enable\x12(.grpc.go.profiling.v1alpha.EnableRequest\x1a).grpc.go.profiling.v1alpha.EnableResponse\x12u\n" + + "\x0eGetStreamStats\x120.grpc.go.profiling.v1alpha.GetStreamStatsRequest\x1a1.grpc.go.profiling.v1alpha.GetStreamStatsResponseB(Z&google.golang.org/grpc/profiling/protob\x06proto3" + +var ( + file_profiling_proto_service_proto_rawDescOnce sync.Once + file_profiling_proto_service_proto_rawDescData []byte +) + +func file_profiling_proto_service_proto_rawDescGZIP() []byte { + file_profiling_proto_service_proto_rawDescOnce.Do(func() { + file_profiling_proto_service_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_profiling_proto_service_proto_rawDesc), len(file_profiling_proto_service_proto_rawDesc))) + }) + return file_profiling_proto_service_proto_rawDescData +} + +var file_profiling_proto_service_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_profiling_proto_service_proto_goTypes = []any{ + (*EnableRequest)(nil), // 0: grpc.go.profiling.v1alpha.EnableRequest + (*EnableResponse)(nil), // 1: grpc.go.profiling.v1alpha.EnableResponse + (*GetStreamStatsRequest)(nil), // 2: grpc.go.profiling.v1alpha.GetStreamStatsRequest + (*GetStreamStatsResponse)(nil), // 3: grpc.go.profiling.v1alpha.GetStreamStatsResponse + (*Timer)(nil), // 4: grpc.go.profiling.v1alpha.Timer + (*Stat)(nil), // 5: grpc.go.profiling.v1alpha.Stat +} +var file_profiling_proto_service_proto_depIdxs = []int32{ + 5, // 0: grpc.go.profiling.v1alpha.GetStreamStatsResponse.stream_stats:type_name -> grpc.go.profiling.v1alpha.Stat + 4, // 1: grpc.go.profiling.v1alpha.Stat.timers:type_name -> grpc.go.profiling.v1alpha.Timer + 0, // 2: grpc.go.profiling.v1alpha.Profiling.Enable:input_type -> grpc.go.profiling.v1alpha.EnableRequest + 2, // 3: grpc.go.profiling.v1alpha.Profiling.GetStreamStats:input_type -> grpc.go.profiling.v1alpha.GetStreamStatsRequest + 1, // 4: grpc.go.profiling.v1alpha.Profiling.Enable:output_type -> grpc.go.profiling.v1alpha.EnableResponse + 3, // 5: grpc.go.profiling.v1alpha.Profiling.GetStreamStats:output_type -> grpc.go.profiling.v1alpha.GetStreamStatsResponse + 4, // [4:6] is the sub-list for method output_type + 2, // [2:4] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_profiling_proto_service_proto_init() } +func file_profiling_proto_service_proto_init() { + if File_profiling_proto_service_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_profiling_proto_service_proto_rawDesc), len(file_profiling_proto_service_proto_rawDesc)), + NumEnums: 0, + NumMessages: 6, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_profiling_proto_service_proto_goTypes, + DependencyIndexes: file_profiling_proto_service_proto_depIdxs, + MessageInfos: file_profiling_proto_service_proto_msgTypes, + }.Build() + File_profiling_proto_service_proto = out.File + file_profiling_proto_service_proto_goTypes = nil + file_profiling_proto_service_proto_depIdxs = nil +} diff --git a/profiling/proto/service.proto b/profiling/proto/service.proto new file mode 100644 index 0000000..3f5f807 --- /dev/null +++ b/profiling/proto/service.proto @@ -0,0 +1,98 @@ +// Copyright 2019 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package grpc.go.profiling.v1alpha; + +// This package defines the proto messages and RPC services exposed by gRPC for +// profiling management. A reference client implementation to interact with +// this service is provided as a command-line application. This service can be +// used to toggle profiling on and off and retrieve stats from a gRPC +// application. +option go_package = "google.golang.org/grpc/profiling/proto"; + +// EnableRequest defines the fields in a /Profiling/Enable method request to +// toggle profiling on and off within a gRPC program. +message EnableRequest { + // Setting this to true will enable profiling. Setting this to false will + // disable profiling. + bool enabled = 1; +} + +// EnableResponse defines the fields in a /Profiling/Enable method response. +message EnableResponse { +} + +// GetStreamStatsRequest defines the fields in a /Profiling/GetStreamStats +// method request to retrieve stream-level stats in a gRPC client/server. +message GetStreamStatsRequest { +} + +// GetStreamStatsResponse defines the fields in a /Profiling/GetStreamStats +// method response. +message GetStreamStatsResponse { + repeated Stat stream_stats = 1; +} + +// A Timer measures the start and end of execution of a component within +// gRPC that's being profiled. It includes a tag and some additional metadata +// to identify itself. +message Timer { + // tags is a comma-separated list of strings used to tag a timer. + string tags = 1; + + // begin_sec and begin_nsec are the start epoch second and nanosecond, + // respectively, of the component profiled by this timer in UTC. begin_nsec + // must be a non-negative integer. + int64 begin_sec = 2; + int32 begin_nsec = 3; + + // end_sec and end_nsec are the end epoch second and nanosecond, + // respectively, of the component profiled by this timer in UTC. end_nsec + // must be a non-negative integer. + int64 end_sec = 4; + int32 end_nsec = 5; + + // go_id is the goroutine ID of the component being profiled. + int64 go_id = 6; +} + +// A Stat is a collection of Timers along with some additional +// metadata to tag and identify itself. +message Stat { + // tags is a comma-separated list of strings used to categorize a stat. + string tags = 1; + + // timers is an array of Timers, each representing a different + // (but possibly overlapping) component within this stat. + repeated Timer timers = 2; + + // metadata is an array of bytes used to uniquely identify a stat with an + // undefined encoding format. For example, the Stats returned by the + // /Profiling/GetStreamStats service use the metadata field to encode the + // connection ID and the stream ID of each query. + bytes metadata = 3; +} + +// The Profiling service exposes functions to remotely manage the gRPC +// profiling behaviour in a program. +service Profiling { + // Enable allows users to toggle profiling on and off remotely. + rpc Enable (EnableRequest) returns (EnableResponse); + + // GetStreamStats is used to retrieve an array of stream-level stats from a + // gRPC client/server. + rpc GetStreamStats (GetStreamStatsRequest) returns (GetStreamStatsResponse); +} diff --git a/profiling/proto/service_grpc.pb.go b/profiling/proto/service_grpc.pb.go new file mode 100644 index 0000000..33fc2d4 --- /dev/null +++ b/profiling/proto/service_grpc.pb.go @@ -0,0 +1,183 @@ +// Copyright 2019 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: profiling/proto/service.proto + +package proto + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Profiling_Enable_FullMethodName = "/grpc.go.profiling.v1alpha.Profiling/Enable" + Profiling_GetStreamStats_FullMethodName = "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats" +) + +// ProfilingClient is the client API for Profiling service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// The Profiling service exposes functions to remotely manage the gRPC +// profiling behaviour in a program. +type ProfilingClient interface { + // Enable allows users to toggle profiling on and off remotely. + Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error) + // GetStreamStats is used to retrieve an array of stream-level stats from a + // gRPC client/server. + GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error) +} + +type profilingClient struct { + cc grpc.ClientConnInterface +} + +func NewProfilingClient(cc grpc.ClientConnInterface) ProfilingClient { + return &profilingClient{cc} +} + +func (c *profilingClient) Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(EnableResponse) + err := c.cc.Invoke(ctx, Profiling_Enable_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *profilingClient) GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetStreamStatsResponse) + err := c.cc.Invoke(ctx, Profiling_GetStreamStats_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ProfilingServer is the server API for Profiling service. +// All implementations should embed UnimplementedProfilingServer +// for forward compatibility. +// +// The Profiling service exposes functions to remotely manage the gRPC +// profiling behaviour in a program. +type ProfilingServer interface { + // Enable allows users to toggle profiling on and off remotely. + Enable(context.Context, *EnableRequest) (*EnableResponse, error) + // GetStreamStats is used to retrieve an array of stream-level stats from a + // gRPC client/server. + GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) +} + +// UnimplementedProfilingServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedProfilingServer struct{} + +func (UnimplementedProfilingServer) Enable(context.Context, *EnableRequest) (*EnableResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Enable not implemented") +} +func (UnimplementedProfilingServer) GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetStreamStats not implemented") +} +func (UnimplementedProfilingServer) testEmbeddedByValue() {} + +// UnsafeProfilingServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ProfilingServer will +// result in compilation errors. +type UnsafeProfilingServer interface { + mustEmbedUnimplementedProfilingServer() +} + +func RegisterProfilingServer(s grpc.ServiceRegistrar, srv ProfilingServer) { + // If the following call panics, it indicates UnimplementedProfilingServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Profiling_ServiceDesc, srv) +} + +func _Profiling_Enable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(EnableRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ProfilingServer).Enable(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Profiling_Enable_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ProfilingServer).Enable(ctx, req.(*EnableRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Profiling_GetStreamStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetStreamStatsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ProfilingServer).GetStreamStats(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Profiling_GetStreamStats_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ProfilingServer).GetStreamStats(ctx, req.(*GetStreamStatsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// Profiling_ServiceDesc is the grpc.ServiceDesc for Profiling service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Profiling_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.go.profiling.v1alpha.Profiling", + HandlerType: (*ProfilingServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Enable", + Handler: _Profiling_Enable_Handler, + }, + { + MethodName: "GetStreamStats", + Handler: _Profiling_GetStreamStats_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "profiling/proto/service.proto", +} diff --git a/profiling/service/service.go b/profiling/service/service.go new file mode 100644 index 0000000..458056f --- /dev/null +++ b/profiling/service/service.go @@ -0,0 +1,150 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package service defines methods to register a gRPC client/service for a +// profiling service that is exposed in the same server. This service can be +// queried by a client to remotely manage the gRPC profiling behaviour of an +// application. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +package service + +import ( + "context" + "errors" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/profiling" + ppb "google.golang.org/grpc/profiling/proto" +) + +var logger = grpclog.Component("profiling") + +// ProfilingConfig defines configuration options for the Init method. +type ProfilingConfig struct { + // Setting this to true will enable profiling. + Enabled bool + + // Profiling uses a circular buffer (ring buffer) to store statistics for + // only the last few RPCs so that profiling stats do not grow unbounded. This + // parameter defines the upper limit on the number of RPCs for which + // statistics should be stored at any given time. An average RPC requires + // approximately 2-3 KiB of memory for profiling-related statistics, so + // choose an appropriate number based on the amount of memory you can afford. + StreamStatsSize uint32 + + // To expose the profiling service and its methods, a *grpc.Server must be + // provided. + Server *grpc.Server +} + +var errorNilServer = errors.New("profiling: no grpc.Server provided") + +// Init takes a *ProfilingConfig to initialize profiling (turned on/off +// depending on the value set in pc.Enabled) and register the profiling service +// in the server provided in pc.Server. +func Init(pc *ProfilingConfig) error { + if pc.Server == nil { + return errorNilServer + } + + if err := profiling.InitStats(pc.StreamStatsSize); err != nil { + return err + } + + ppb.RegisterProfilingServer(pc.Server, getProfilingServerInstance()) + + // Do this last after everything has been initialized and allocated. + profiling.Enable(pc.Enabled) + + return nil +} + +type profilingServer struct { + ppb.UnimplementedProfilingServer + drainMutex sync.Mutex +} + +var profilingServerInstance *profilingServer +var profilingServerOnce sync.Once + +// getProfilingServerInstance creates and returns a singleton instance of +// profilingServer. Only one instance of profilingServer is created to use a +// shared mutex across all profilingServer instances. +func getProfilingServerInstance() *profilingServer { + profilingServerOnce.Do(func() { + profilingServerInstance = &profilingServer{} + }) + + return profilingServerInstance +} + +func (s *profilingServer) Enable(_ context.Context, req *ppb.EnableRequest) (*ppb.EnableResponse, error) { + if req.Enabled { + logger.Infof("profilingServer: Enable: enabling profiling") + } else { + logger.Infof("profilingServer: Enable: disabling profiling") + } + profiling.Enable(req.Enabled) + + return &ppb.EnableResponse{}, nil +} + +func timerToProtoTimer(timer *profiling.Timer) *ppb.Timer { + return &ppb.Timer{ + Tags: timer.Tags, + BeginSec: timer.Begin.Unix(), + BeginNsec: int32(timer.Begin.Nanosecond()), + EndSec: timer.End.Unix(), + EndNsec: int32(timer.End.Nanosecond()), + GoId: timer.GoID, + } +} + +func statToProtoStat(stat *profiling.Stat) *ppb.Stat { + protoStat := &ppb.Stat{ + Tags: stat.Tags, + Timers: make([]*ppb.Timer, 0, len(stat.Timers)), + Metadata: stat.Metadata, + } + for _, t := range stat.Timers { + protoStat.Timers = append(protoStat.Timers, timerToProtoTimer(t)) + } + return protoStat +} + +func (s *profilingServer) GetStreamStats(context.Context, *ppb.GetStreamStatsRequest) (*ppb.GetStreamStatsResponse, error) { + // Since the drain operation is destructive, only one client request should + // be served at a time. + logger.Infof("profilingServer: GetStreamStats: processing request") + s.drainMutex.Lock() + results := profiling.StreamStats.Drain() + s.drainMutex.Unlock() + + logger.Infof("profilingServer: GetStreamStats: returning %v records", len(results)) + streamStats := make([]*ppb.Stat, 0) + for _, stat := range results { + streamStats = append(streamStats, statToProtoStat(stat.(*profiling.Stat))) + } + return &ppb.GetStreamStatsResponse{StreamStats: streamStats}, nil +} diff --git a/reflection/README.md b/reflection/README.md new file mode 100644 index 0000000..9ace83c --- /dev/null +++ b/reflection/README.md @@ -0,0 +1,18 @@ +# Reflection + +Package reflection implements server reflection service. + +The service implemented is defined in: https://github.com/grpc/grpc/blob/master/src/proto/grpc/reflection/v1/reflection.proto. + +To register server reflection on a gRPC server: +```go +import "google.golang.org/grpc/reflection" + +s := grpc.NewServer() +pb.RegisterYourOwnServer(s, &server{}) + +// Register reflection service on gRPC server. +reflection.Register(s) + +s.Serve(lis) +``` diff --git a/reflection/adapt.go b/reflection/adapt.go new file mode 100644 index 0000000..6997e47 --- /dev/null +++ b/reflection/adapt.go @@ -0,0 +1,57 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package reflection + +import ( + "google.golang.org/grpc/reflection/internal" + + v1reflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1" + v1reflectionpb "google.golang.org/grpc/reflection/grpc_reflection_v1" + v1alphareflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" +) + +// asV1Alpha returns an implementation of the v1alpha version of the reflection +// interface that delegates all calls to the given v1 version. +func asV1Alpha(svr v1reflectiongrpc.ServerReflectionServer) v1alphareflectiongrpc.ServerReflectionServer { + return v1AlphaServerImpl{svr: svr} +} + +type v1AlphaServerImpl struct { + svr v1reflectiongrpc.ServerReflectionServer +} + +func (s v1AlphaServerImpl) ServerReflectionInfo(stream v1alphareflectiongrpc.ServerReflection_ServerReflectionInfoServer) error { + return s.svr.ServerReflectionInfo(v1AlphaServerStreamAdapter{stream}) +} + +type v1AlphaServerStreamAdapter struct { + v1alphareflectiongrpc.ServerReflection_ServerReflectionInfoServer +} + +func (s v1AlphaServerStreamAdapter) Send(response *v1reflectionpb.ServerReflectionResponse) error { + return s.ServerReflection_ServerReflectionInfoServer.Send(internal.V1ToV1AlphaResponse(response)) +} + +func (s v1AlphaServerStreamAdapter) Recv() (*v1reflectionpb.ServerReflectionRequest, error) { + resp, err := s.ServerReflection_ServerReflectionInfoServer.Recv() + if err != nil { + return nil, err + } + return internal.V1AlphaToV1Request(resp), nil +} diff --git a/reflection/grpc_reflection_v1/reflection.pb.go b/reflection/grpc_reflection_v1/reflection.pb.go new file mode 100644 index 0000000..c8772eb --- /dev/null +++ b/reflection/grpc_reflection_v1/reflection.pb.go @@ -0,0 +1,777 @@ +// Copyright 2016 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Service exported by server reflection. A more complete description of how +// server reflection works can be found at +// https://github.com/grpc/grpc/blob/master/doc/server-reflection.md +// +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/reflection/v1/reflection.proto + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: grpc/reflection/v1/reflection.proto + +package grpc_reflection_v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// The message sent by the client when calling ServerReflectionInfo method. +type ServerReflectionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` + // To use reflection service, the client should set one of the following + // fields in message_request. The server distinguishes requests by their + // defined field and then handles them using corresponding methods. + // + // Types that are valid to be assigned to MessageRequest: + // + // *ServerReflectionRequest_FileByFilename + // *ServerReflectionRequest_FileContainingSymbol + // *ServerReflectionRequest_FileContainingExtension + // *ServerReflectionRequest_AllExtensionNumbersOfType + // *ServerReflectionRequest_ListServices + MessageRequest isServerReflectionRequest_MessageRequest `protobuf_oneof:"message_request"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerReflectionRequest) Reset() { + *x = ServerReflectionRequest{} + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerReflectionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerReflectionRequest) ProtoMessage() {} + +func (x *ServerReflectionRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerReflectionRequest.ProtoReflect.Descriptor instead. +func (*ServerReflectionRequest) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{0} +} + +func (x *ServerReflectionRequest) GetHost() string { + if x != nil { + return x.Host + } + return "" +} + +func (x *ServerReflectionRequest) GetMessageRequest() isServerReflectionRequest_MessageRequest { + if x != nil { + return x.MessageRequest + } + return nil +} + +func (x *ServerReflectionRequest) GetFileByFilename() string { + if x != nil { + if x, ok := x.MessageRequest.(*ServerReflectionRequest_FileByFilename); ok { + return x.FileByFilename + } + } + return "" +} + +func (x *ServerReflectionRequest) GetFileContainingSymbol() string { + if x != nil { + if x, ok := x.MessageRequest.(*ServerReflectionRequest_FileContainingSymbol); ok { + return x.FileContainingSymbol + } + } + return "" +} + +func (x *ServerReflectionRequest) GetFileContainingExtension() *ExtensionRequest { + if x != nil { + if x, ok := x.MessageRequest.(*ServerReflectionRequest_FileContainingExtension); ok { + return x.FileContainingExtension + } + } + return nil +} + +func (x *ServerReflectionRequest) GetAllExtensionNumbersOfType() string { + if x != nil { + if x, ok := x.MessageRequest.(*ServerReflectionRequest_AllExtensionNumbersOfType); ok { + return x.AllExtensionNumbersOfType + } + } + return "" +} + +func (x *ServerReflectionRequest) GetListServices() string { + if x != nil { + if x, ok := x.MessageRequest.(*ServerReflectionRequest_ListServices); ok { + return x.ListServices + } + } + return "" +} + +type isServerReflectionRequest_MessageRequest interface { + isServerReflectionRequest_MessageRequest() +} + +type ServerReflectionRequest_FileByFilename struct { + // Find a proto file by the file name. + FileByFilename string `protobuf:"bytes,3,opt,name=file_by_filename,json=fileByFilename,proto3,oneof"` +} + +type ServerReflectionRequest_FileContainingSymbol struct { + // Find the proto file that declares the given fully-qualified symbol name. + // This field should be a fully-qualified symbol name + // (e.g. .[.] or .). + FileContainingSymbol string `protobuf:"bytes,4,opt,name=file_containing_symbol,json=fileContainingSymbol,proto3,oneof"` +} + +type ServerReflectionRequest_FileContainingExtension struct { + // Find the proto file which defines an extension extending the given + // message type with the given field number. + FileContainingExtension *ExtensionRequest `protobuf:"bytes,5,opt,name=file_containing_extension,json=fileContainingExtension,proto3,oneof"` +} + +type ServerReflectionRequest_AllExtensionNumbersOfType struct { + // Finds the tag numbers used by all known extensions of the given message + // type, and appends them to ExtensionNumberResponse in an undefined order. + // Its corresponding method is best-effort: it's not guaranteed that the + // reflection service will implement this method, and it's not guaranteed + // that this method will provide all extensions. Returns + // StatusCode::UNIMPLEMENTED if it's not implemented. + // This field should be a fully-qualified type name. The format is + // . + AllExtensionNumbersOfType string `protobuf:"bytes,6,opt,name=all_extension_numbers_of_type,json=allExtensionNumbersOfType,proto3,oneof"` +} + +type ServerReflectionRequest_ListServices struct { + // List the full names of registered services. The content will not be + // checked. + ListServices string `protobuf:"bytes,7,opt,name=list_services,json=listServices,proto3,oneof"` +} + +func (*ServerReflectionRequest_FileByFilename) isServerReflectionRequest_MessageRequest() {} + +func (*ServerReflectionRequest_FileContainingSymbol) isServerReflectionRequest_MessageRequest() {} + +func (*ServerReflectionRequest_FileContainingExtension) isServerReflectionRequest_MessageRequest() {} + +func (*ServerReflectionRequest_AllExtensionNumbersOfType) isServerReflectionRequest_MessageRequest() { +} + +func (*ServerReflectionRequest_ListServices) isServerReflectionRequest_MessageRequest() {} + +// The type name and extension number sent by the client when requesting +// file_containing_extension. +type ExtensionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Fully-qualified type name. The format should be . + ContainingType string `protobuf:"bytes,1,opt,name=containing_type,json=containingType,proto3" json:"containing_type,omitempty"` + ExtensionNumber int32 `protobuf:"varint,2,opt,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExtensionRequest) Reset() { + *x = ExtensionRequest{} + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExtensionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExtensionRequest) ProtoMessage() {} + +func (x *ExtensionRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExtensionRequest.ProtoReflect.Descriptor instead. +func (*ExtensionRequest) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{1} +} + +func (x *ExtensionRequest) GetContainingType() string { + if x != nil { + return x.ContainingType + } + return "" +} + +func (x *ExtensionRequest) GetExtensionNumber() int32 { + if x != nil { + return x.ExtensionNumber + } + return 0 +} + +// The message sent by the server to answer ServerReflectionInfo method. +type ServerReflectionResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + ValidHost string `protobuf:"bytes,1,opt,name=valid_host,json=validHost,proto3" json:"valid_host,omitempty"` + OriginalRequest *ServerReflectionRequest `protobuf:"bytes,2,opt,name=original_request,json=originalRequest,proto3" json:"original_request,omitempty"` + // The server sets one of the following fields according to the message_request + // in the request. + // + // Types that are valid to be assigned to MessageResponse: + // + // *ServerReflectionResponse_FileDescriptorResponse + // *ServerReflectionResponse_AllExtensionNumbersResponse + // *ServerReflectionResponse_ListServicesResponse + // *ServerReflectionResponse_ErrorResponse + MessageResponse isServerReflectionResponse_MessageResponse `protobuf_oneof:"message_response"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerReflectionResponse) Reset() { + *x = ServerReflectionResponse{} + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerReflectionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerReflectionResponse) ProtoMessage() {} + +func (x *ServerReflectionResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerReflectionResponse.ProtoReflect.Descriptor instead. +func (*ServerReflectionResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{2} +} + +func (x *ServerReflectionResponse) GetValidHost() string { + if x != nil { + return x.ValidHost + } + return "" +} + +func (x *ServerReflectionResponse) GetOriginalRequest() *ServerReflectionRequest { + if x != nil { + return x.OriginalRequest + } + return nil +} + +func (x *ServerReflectionResponse) GetMessageResponse() isServerReflectionResponse_MessageResponse { + if x != nil { + return x.MessageResponse + } + return nil +} + +func (x *ServerReflectionResponse) GetFileDescriptorResponse() *FileDescriptorResponse { + if x != nil { + if x, ok := x.MessageResponse.(*ServerReflectionResponse_FileDescriptorResponse); ok { + return x.FileDescriptorResponse + } + } + return nil +} + +func (x *ServerReflectionResponse) GetAllExtensionNumbersResponse() *ExtensionNumberResponse { + if x != nil { + if x, ok := x.MessageResponse.(*ServerReflectionResponse_AllExtensionNumbersResponse); ok { + return x.AllExtensionNumbersResponse + } + } + return nil +} + +func (x *ServerReflectionResponse) GetListServicesResponse() *ListServiceResponse { + if x != nil { + if x, ok := x.MessageResponse.(*ServerReflectionResponse_ListServicesResponse); ok { + return x.ListServicesResponse + } + } + return nil +} + +func (x *ServerReflectionResponse) GetErrorResponse() *ErrorResponse { + if x != nil { + if x, ok := x.MessageResponse.(*ServerReflectionResponse_ErrorResponse); ok { + return x.ErrorResponse + } + } + return nil +} + +type isServerReflectionResponse_MessageResponse interface { + isServerReflectionResponse_MessageResponse() +} + +type ServerReflectionResponse_FileDescriptorResponse struct { + // This message is used to answer file_by_filename, file_containing_symbol, + // file_containing_extension requests with transitive dependencies. + // As the repeated label is not allowed in oneof fields, we use a + // FileDescriptorResponse message to encapsulate the repeated fields. + // The reflection service is allowed to avoid sending FileDescriptorProtos + // that were previously sent in response to earlier requests in the stream. + FileDescriptorResponse *FileDescriptorResponse `protobuf:"bytes,4,opt,name=file_descriptor_response,json=fileDescriptorResponse,proto3,oneof"` +} + +type ServerReflectionResponse_AllExtensionNumbersResponse struct { + // This message is used to answer all_extension_numbers_of_type requests. + AllExtensionNumbersResponse *ExtensionNumberResponse `protobuf:"bytes,5,opt,name=all_extension_numbers_response,json=allExtensionNumbersResponse,proto3,oneof"` +} + +type ServerReflectionResponse_ListServicesResponse struct { + // This message is used to answer list_services requests. + ListServicesResponse *ListServiceResponse `protobuf:"bytes,6,opt,name=list_services_response,json=listServicesResponse,proto3,oneof"` +} + +type ServerReflectionResponse_ErrorResponse struct { + // This message is used when an error occurs. + ErrorResponse *ErrorResponse `protobuf:"bytes,7,opt,name=error_response,json=errorResponse,proto3,oneof"` +} + +func (*ServerReflectionResponse_FileDescriptorResponse) isServerReflectionResponse_MessageResponse() { +} + +func (*ServerReflectionResponse_AllExtensionNumbersResponse) isServerReflectionResponse_MessageResponse() { +} + +func (*ServerReflectionResponse_ListServicesResponse) isServerReflectionResponse_MessageResponse() {} + +func (*ServerReflectionResponse_ErrorResponse) isServerReflectionResponse_MessageResponse() {} + +// Serialized FileDescriptorProto messages sent by the server answering +// a file_by_filename, file_containing_symbol, or file_containing_extension +// request. +type FileDescriptorResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Serialized FileDescriptorProto messages. We avoid taking a dependency on + // descriptor.proto, which uses proto2 only features, by making them opaque + // bytes instead. + FileDescriptorProto [][]byte `protobuf:"bytes,1,rep,name=file_descriptor_proto,json=fileDescriptorProto,proto3" json:"file_descriptor_proto,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileDescriptorResponse) Reset() { + *x = FileDescriptorResponse{} + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileDescriptorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileDescriptorResponse) ProtoMessage() {} + +func (x *FileDescriptorResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileDescriptorResponse.ProtoReflect.Descriptor instead. +func (*FileDescriptorResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{3} +} + +func (x *FileDescriptorResponse) GetFileDescriptorProto() [][]byte { + if x != nil { + return x.FileDescriptorProto + } + return nil +} + +// A list of extension numbers sent by the server answering +// all_extension_numbers_of_type request. +type ExtensionNumberResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Full name of the base type, including the package name. The format + // is . + BaseTypeName string `protobuf:"bytes,1,opt,name=base_type_name,json=baseTypeName,proto3" json:"base_type_name,omitempty"` + ExtensionNumber []int32 `protobuf:"varint,2,rep,packed,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExtensionNumberResponse) Reset() { + *x = ExtensionNumberResponse{} + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExtensionNumberResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExtensionNumberResponse) ProtoMessage() {} + +func (x *ExtensionNumberResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExtensionNumberResponse.ProtoReflect.Descriptor instead. +func (*ExtensionNumberResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{4} +} + +func (x *ExtensionNumberResponse) GetBaseTypeName() string { + if x != nil { + return x.BaseTypeName + } + return "" +} + +func (x *ExtensionNumberResponse) GetExtensionNumber() []int32 { + if x != nil { + return x.ExtensionNumber + } + return nil +} + +// A list of ServiceResponse sent by the server answering list_services request. +type ListServiceResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The information of each service may be expanded in the future, so we use + // ServiceResponse message to encapsulate it. + Service []*ServiceResponse `protobuf:"bytes,1,rep,name=service,proto3" json:"service,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListServiceResponse) Reset() { + *x = ListServiceResponse{} + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListServiceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListServiceResponse) ProtoMessage() {} + +func (x *ListServiceResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListServiceResponse.ProtoReflect.Descriptor instead. +func (*ListServiceResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{5} +} + +func (x *ListServiceResponse) GetService() []*ServiceResponse { + if x != nil { + return x.Service + } + return nil +} + +// The information of a single service used by ListServiceResponse to answer +// list_services request. +type ServiceResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Full name of a registered service, including its package name. The format + // is . + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServiceResponse) Reset() { + *x = ServiceResponse{} + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServiceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServiceResponse) ProtoMessage() {} + +func (x *ServiceResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServiceResponse.ProtoReflect.Descriptor instead. +func (*ServiceResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{6} +} + +func (x *ServiceResponse) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +// The error code and error message sent by the server when an error occurs. +type ErrorResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // This field uses the error codes defined in grpc::StatusCode. + ErrorCode int32 `protobuf:"varint,1,opt,name=error_code,json=errorCode,proto3" json:"error_code,omitempty"` + ErrorMessage string `protobuf:"bytes,2,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ErrorResponse) Reset() { + *x = ErrorResponse{} + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ErrorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ErrorResponse) ProtoMessage() {} + +func (x *ErrorResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ErrorResponse.ProtoReflect.Descriptor instead. +func (*ErrorResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{7} +} + +func (x *ErrorResponse) GetErrorCode() int32 { + if x != nil { + return x.ErrorCode + } + return 0 +} + +func (x *ErrorResponse) GetErrorMessage() string { + if x != nil { + return x.ErrorMessage + } + return "" +} + +var File_grpc_reflection_v1_reflection_proto protoreflect.FileDescriptor + +const file_grpc_reflection_v1_reflection_proto_rawDesc = "" + + "\n" + + "#grpc/reflection/v1/reflection.proto\x12\x12grpc.reflection.v1\"\xf3\x02\n" + + "\x17ServerReflectionRequest\x12\x12\n" + + "\x04host\x18\x01 \x01(\tR\x04host\x12*\n" + + "\x10file_by_filename\x18\x03 \x01(\tH\x00R\x0efileByFilename\x126\n" + + "\x16file_containing_symbol\x18\x04 \x01(\tH\x00R\x14fileContainingSymbol\x12b\n" + + "\x19file_containing_extension\x18\x05 \x01(\v2$.grpc.reflection.v1.ExtensionRequestH\x00R\x17fileContainingExtension\x12B\n" + + "\x1dall_extension_numbers_of_type\x18\x06 \x01(\tH\x00R\x19allExtensionNumbersOfType\x12%\n" + + "\rlist_services\x18\a \x01(\tH\x00R\flistServicesB\x11\n" + + "\x0fmessage_request\"f\n" + + "\x10ExtensionRequest\x12'\n" + + "\x0fcontaining_type\x18\x01 \x01(\tR\x0econtainingType\x12)\n" + + "\x10extension_number\x18\x02 \x01(\x05R\x0fextensionNumber\"\xae\x04\n" + + "\x18ServerReflectionResponse\x12\x1d\n" + + "\n" + + "valid_host\x18\x01 \x01(\tR\tvalidHost\x12V\n" + + "\x10original_request\x18\x02 \x01(\v2+.grpc.reflection.v1.ServerReflectionRequestR\x0foriginalRequest\x12f\n" + + "\x18file_descriptor_response\x18\x04 \x01(\v2*.grpc.reflection.v1.FileDescriptorResponseH\x00R\x16fileDescriptorResponse\x12r\n" + + "\x1eall_extension_numbers_response\x18\x05 \x01(\v2+.grpc.reflection.v1.ExtensionNumberResponseH\x00R\x1ballExtensionNumbersResponse\x12_\n" + + "\x16list_services_response\x18\x06 \x01(\v2'.grpc.reflection.v1.ListServiceResponseH\x00R\x14listServicesResponse\x12J\n" + + "\x0eerror_response\x18\a \x01(\v2!.grpc.reflection.v1.ErrorResponseH\x00R\rerrorResponseB\x12\n" + + "\x10message_response\"L\n" + + "\x16FileDescriptorResponse\x122\n" + + "\x15file_descriptor_proto\x18\x01 \x03(\fR\x13fileDescriptorProto\"j\n" + + "\x17ExtensionNumberResponse\x12$\n" + + "\x0ebase_type_name\x18\x01 \x01(\tR\fbaseTypeName\x12)\n" + + "\x10extension_number\x18\x02 \x03(\x05R\x0fextensionNumber\"T\n" + + "\x13ListServiceResponse\x12=\n" + + "\aservice\x18\x01 \x03(\v2#.grpc.reflection.v1.ServiceResponseR\aservice\"%\n" + + "\x0fServiceResponse\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\"S\n" + + "\rErrorResponse\x12\x1d\n" + + "\n" + + "error_code\x18\x01 \x01(\x05R\terrorCode\x12#\n" + + "\rerror_message\x18\x02 \x01(\tR\ferrorMessage2\x89\x01\n" + + "\x10ServerReflection\x12u\n" + + "\x14ServerReflectionInfo\x12+.grpc.reflection.v1.ServerReflectionRequest\x1a,.grpc.reflection.v1.ServerReflectionResponse(\x010\x01Bf\n" + + "\x15io.grpc.reflection.v1B\x15ServerReflectionProtoP\x01Z4google.golang.org/grpc/reflection/grpc_reflection_v1b\x06proto3" + +var ( + file_grpc_reflection_v1_reflection_proto_rawDescOnce sync.Once + file_grpc_reflection_v1_reflection_proto_rawDescData []byte +) + +func file_grpc_reflection_v1_reflection_proto_rawDescGZIP() []byte { + file_grpc_reflection_v1_reflection_proto_rawDescOnce.Do(func() { + file_grpc_reflection_v1_reflection_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_reflection_v1_reflection_proto_rawDesc), len(file_grpc_reflection_v1_reflection_proto_rawDesc))) + }) + return file_grpc_reflection_v1_reflection_proto_rawDescData +} + +var file_grpc_reflection_v1_reflection_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_grpc_reflection_v1_reflection_proto_goTypes = []any{ + (*ServerReflectionRequest)(nil), // 0: grpc.reflection.v1.ServerReflectionRequest + (*ExtensionRequest)(nil), // 1: grpc.reflection.v1.ExtensionRequest + (*ServerReflectionResponse)(nil), // 2: grpc.reflection.v1.ServerReflectionResponse + (*FileDescriptorResponse)(nil), // 3: grpc.reflection.v1.FileDescriptorResponse + (*ExtensionNumberResponse)(nil), // 4: grpc.reflection.v1.ExtensionNumberResponse + (*ListServiceResponse)(nil), // 5: grpc.reflection.v1.ListServiceResponse + (*ServiceResponse)(nil), // 6: grpc.reflection.v1.ServiceResponse + (*ErrorResponse)(nil), // 7: grpc.reflection.v1.ErrorResponse +} +var file_grpc_reflection_v1_reflection_proto_depIdxs = []int32{ + 1, // 0: grpc.reflection.v1.ServerReflectionRequest.file_containing_extension:type_name -> grpc.reflection.v1.ExtensionRequest + 0, // 1: grpc.reflection.v1.ServerReflectionResponse.original_request:type_name -> grpc.reflection.v1.ServerReflectionRequest + 3, // 2: grpc.reflection.v1.ServerReflectionResponse.file_descriptor_response:type_name -> grpc.reflection.v1.FileDescriptorResponse + 4, // 3: grpc.reflection.v1.ServerReflectionResponse.all_extension_numbers_response:type_name -> grpc.reflection.v1.ExtensionNumberResponse + 5, // 4: grpc.reflection.v1.ServerReflectionResponse.list_services_response:type_name -> grpc.reflection.v1.ListServiceResponse + 7, // 5: grpc.reflection.v1.ServerReflectionResponse.error_response:type_name -> grpc.reflection.v1.ErrorResponse + 6, // 6: grpc.reflection.v1.ListServiceResponse.service:type_name -> grpc.reflection.v1.ServiceResponse + 0, // 7: grpc.reflection.v1.ServerReflection.ServerReflectionInfo:input_type -> grpc.reflection.v1.ServerReflectionRequest + 2, // 8: grpc.reflection.v1.ServerReflection.ServerReflectionInfo:output_type -> grpc.reflection.v1.ServerReflectionResponse + 8, // [8:9] is the sub-list for method output_type + 7, // [7:8] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name +} + +func init() { file_grpc_reflection_v1_reflection_proto_init() } +func file_grpc_reflection_v1_reflection_proto_init() { + if File_grpc_reflection_v1_reflection_proto != nil { + return + } + file_grpc_reflection_v1_reflection_proto_msgTypes[0].OneofWrappers = []any{ + (*ServerReflectionRequest_FileByFilename)(nil), + (*ServerReflectionRequest_FileContainingSymbol)(nil), + (*ServerReflectionRequest_FileContainingExtension)(nil), + (*ServerReflectionRequest_AllExtensionNumbersOfType)(nil), + (*ServerReflectionRequest_ListServices)(nil), + } + file_grpc_reflection_v1_reflection_proto_msgTypes[2].OneofWrappers = []any{ + (*ServerReflectionResponse_FileDescriptorResponse)(nil), + (*ServerReflectionResponse_AllExtensionNumbersResponse)(nil), + (*ServerReflectionResponse_ListServicesResponse)(nil), + (*ServerReflectionResponse_ErrorResponse)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_reflection_v1_reflection_proto_rawDesc), len(file_grpc_reflection_v1_reflection_proto_rawDesc)), + NumEnums: 0, + NumMessages: 8, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_reflection_v1_reflection_proto_goTypes, + DependencyIndexes: file_grpc_reflection_v1_reflection_proto_depIdxs, + MessageInfos: file_grpc_reflection_v1_reflection_proto_msgTypes, + }.Build() + File_grpc_reflection_v1_reflection_proto = out.File + file_grpc_reflection_v1_reflection_proto_goTypes = nil + file_grpc_reflection_v1_reflection_proto_depIdxs = nil +} diff --git a/reflection/grpc_reflection_v1/reflection_grpc.pb.go b/reflection/grpc_reflection_v1/reflection_grpc.pb.go new file mode 100644 index 0000000..21142c6 --- /dev/null +++ b/reflection/grpc_reflection_v1/reflection_grpc.pb.go @@ -0,0 +1,138 @@ +// Copyright 2016 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Service exported by server reflection. A more complete description of how +// server reflection works can be found at +// https://github.com/grpc/grpc/blob/master/doc/server-reflection.md +// +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/reflection/v1/reflection.proto + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: grpc/reflection/v1/reflection.proto + +package grpc_reflection_v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + ServerReflection_ServerReflectionInfo_FullMethodName = "/grpc.reflection.v1.ServerReflection/ServerReflectionInfo" +) + +// ServerReflectionClient is the client API for ServerReflection service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ServerReflectionClient interface { + // The reflection service is structured as a bidirectional stream, ensuring + // all related requests go to a single server. + ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ServerReflectionRequest, ServerReflectionResponse], error) +} + +type serverReflectionClient struct { + cc grpc.ClientConnInterface +} + +func NewServerReflectionClient(cc grpc.ClientConnInterface) ServerReflectionClient { + return &serverReflectionClient{cc} +} + +func (c *serverReflectionClient) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ServerReflectionRequest, ServerReflectionResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &ServerReflection_ServiceDesc.Streams[0], ServerReflection_ServerReflectionInfo_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[ServerReflectionRequest, ServerReflectionResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type ServerReflection_ServerReflectionInfoClient = grpc.BidiStreamingClient[ServerReflectionRequest, ServerReflectionResponse] + +// ServerReflectionServer is the server API for ServerReflection service. +// All implementations should embed UnimplementedServerReflectionServer +// for forward compatibility. +type ServerReflectionServer interface { + // The reflection service is structured as a bidirectional stream, ensuring + // all related requests go to a single server. + ServerReflectionInfo(grpc.BidiStreamingServer[ServerReflectionRequest, ServerReflectionResponse]) error +} + +// UnimplementedServerReflectionServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedServerReflectionServer struct{} + +func (UnimplementedServerReflectionServer) ServerReflectionInfo(grpc.BidiStreamingServer[ServerReflectionRequest, ServerReflectionResponse]) error { + return status.Error(codes.Unimplemented, "method ServerReflectionInfo not implemented") +} +func (UnimplementedServerReflectionServer) testEmbeddedByValue() {} + +// UnsafeServerReflectionServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ServerReflectionServer will +// result in compilation errors. +type UnsafeServerReflectionServer interface { + mustEmbedUnimplementedServerReflectionServer() +} + +func RegisterServerReflectionServer(s grpc.ServiceRegistrar, srv ServerReflectionServer) { + // If the following call panics, it indicates UnimplementedServerReflectionServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&ServerReflection_ServiceDesc, srv) +} + +func _ServerReflection_ServerReflectionInfo_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(ServerReflectionServer).ServerReflectionInfo(&grpc.GenericServerStream[ServerReflectionRequest, ServerReflectionResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type ServerReflection_ServerReflectionInfoServer = grpc.BidiStreamingServer[ServerReflectionRequest, ServerReflectionResponse] + +// ServerReflection_ServiceDesc is the grpc.ServiceDesc for ServerReflection service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ServerReflection_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.reflection.v1.ServerReflection", + HandlerType: (*ServerReflectionServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "ServerReflectionInfo", + Handler: _ServerReflection_ServerReflectionInfo_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/reflection/v1/reflection.proto", +} diff --git a/reflection/grpc_reflection_v1alpha/reflection.pb.go b/reflection/grpc_reflection_v1alpha/reflection.pb.go new file mode 100644 index 0000000..b927bd4 --- /dev/null +++ b/reflection/grpc_reflection_v1alpha/reflection.pb.go @@ -0,0 +1,847 @@ +// Copyright 2016 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// Service exported by server reflection + +// Warning: this entire file is deprecated. Use this instead: +// https://github.com/grpc/grpc-proto/blob/master/grpc/reflection/v1/reflection.proto + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// grpc/reflection/v1alpha/reflection.proto is a deprecated file. + +package grpc_reflection_v1alpha + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// The message sent by the client when calling ServerReflectionInfo method. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +type ServerReflectionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` + // To use reflection service, the client should set one of the following + // fields in message_request. The server distinguishes requests by their + // defined field and then handles them using corresponding methods. + // + // Types that are valid to be assigned to MessageRequest: + // + // *ServerReflectionRequest_FileByFilename + // *ServerReflectionRequest_FileContainingSymbol + // *ServerReflectionRequest_FileContainingExtension + // *ServerReflectionRequest_AllExtensionNumbersOfType + // *ServerReflectionRequest_ListServices + MessageRequest isServerReflectionRequest_MessageRequest `protobuf_oneof:"message_request"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerReflectionRequest) Reset() { + *x = ServerReflectionRequest{} + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerReflectionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerReflectionRequest) ProtoMessage() {} + +func (x *ServerReflectionRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerReflectionRequest.ProtoReflect.Descriptor instead. +func (*ServerReflectionRequest) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{0} +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionRequest) GetHost() string { + if x != nil { + return x.Host + } + return "" +} + +func (x *ServerReflectionRequest) GetMessageRequest() isServerReflectionRequest_MessageRequest { + if x != nil { + return x.MessageRequest + } + return nil +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionRequest) GetFileByFilename() string { + if x != nil { + if x, ok := x.MessageRequest.(*ServerReflectionRequest_FileByFilename); ok { + return x.FileByFilename + } + } + return "" +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionRequest) GetFileContainingSymbol() string { + if x != nil { + if x, ok := x.MessageRequest.(*ServerReflectionRequest_FileContainingSymbol); ok { + return x.FileContainingSymbol + } + } + return "" +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionRequest) GetFileContainingExtension() *ExtensionRequest { + if x != nil { + if x, ok := x.MessageRequest.(*ServerReflectionRequest_FileContainingExtension); ok { + return x.FileContainingExtension + } + } + return nil +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionRequest) GetAllExtensionNumbersOfType() string { + if x != nil { + if x, ok := x.MessageRequest.(*ServerReflectionRequest_AllExtensionNumbersOfType); ok { + return x.AllExtensionNumbersOfType + } + } + return "" +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionRequest) GetListServices() string { + if x != nil { + if x, ok := x.MessageRequest.(*ServerReflectionRequest_ListServices); ok { + return x.ListServices + } + } + return "" +} + +type isServerReflectionRequest_MessageRequest interface { + isServerReflectionRequest_MessageRequest() +} + +type ServerReflectionRequest_FileByFilename struct { + // Find a proto file by the file name. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + FileByFilename string `protobuf:"bytes,3,opt,name=file_by_filename,json=fileByFilename,proto3,oneof"` +} + +type ServerReflectionRequest_FileContainingSymbol struct { + // Find the proto file that declares the given fully-qualified symbol name. + // This field should be a fully-qualified symbol name + // (e.g. .[.] or .). + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + FileContainingSymbol string `protobuf:"bytes,4,opt,name=file_containing_symbol,json=fileContainingSymbol,proto3,oneof"` +} + +type ServerReflectionRequest_FileContainingExtension struct { + // Find the proto file which defines an extension extending the given + // message type with the given field number. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + FileContainingExtension *ExtensionRequest `protobuf:"bytes,5,opt,name=file_containing_extension,json=fileContainingExtension,proto3,oneof"` +} + +type ServerReflectionRequest_AllExtensionNumbersOfType struct { + // Finds the tag numbers used by all known extensions of extendee_type, and + // appends them to ExtensionNumberResponse in an undefined order. + // Its corresponding method is best-effort: it's not guaranteed that the + // reflection service will implement this method, and it's not guaranteed + // that this method will provide all extensions. Returns + // StatusCode::UNIMPLEMENTED if it's not implemented. + // This field should be a fully-qualified type name. The format is + // . + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + AllExtensionNumbersOfType string `protobuf:"bytes,6,opt,name=all_extension_numbers_of_type,json=allExtensionNumbersOfType,proto3,oneof"` +} + +type ServerReflectionRequest_ListServices struct { + // List the full names of registered services. The content will not be + // checked. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ListServices string `protobuf:"bytes,7,opt,name=list_services,json=listServices,proto3,oneof"` +} + +func (*ServerReflectionRequest_FileByFilename) isServerReflectionRequest_MessageRequest() {} + +func (*ServerReflectionRequest_FileContainingSymbol) isServerReflectionRequest_MessageRequest() {} + +func (*ServerReflectionRequest_FileContainingExtension) isServerReflectionRequest_MessageRequest() {} + +func (*ServerReflectionRequest_AllExtensionNumbersOfType) isServerReflectionRequest_MessageRequest() { +} + +func (*ServerReflectionRequest_ListServices) isServerReflectionRequest_MessageRequest() {} + +// The type name and extension number sent by the client when requesting +// file_containing_extension. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +type ExtensionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Fully-qualified type name. The format should be . + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ContainingType string `protobuf:"bytes,1,opt,name=containing_type,json=containingType,proto3" json:"containing_type,omitempty"` + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ExtensionNumber int32 `protobuf:"varint,2,opt,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExtensionRequest) Reset() { + *x = ExtensionRequest{} + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExtensionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExtensionRequest) ProtoMessage() {} + +func (x *ExtensionRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExtensionRequest.ProtoReflect.Descriptor instead. +func (*ExtensionRequest) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{1} +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ExtensionRequest) GetContainingType() string { + if x != nil { + return x.ContainingType + } + return "" +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ExtensionRequest) GetExtensionNumber() int32 { + if x != nil { + return x.ExtensionNumber + } + return 0 +} + +// The message sent by the server to answer ServerReflectionInfo method. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +type ServerReflectionResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ValidHost string `protobuf:"bytes,1,opt,name=valid_host,json=validHost,proto3" json:"valid_host,omitempty"` + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + OriginalRequest *ServerReflectionRequest `protobuf:"bytes,2,opt,name=original_request,json=originalRequest,proto3" json:"original_request,omitempty"` + // The server set one of the following fields according to the message_request + // in the request. + // + // Types that are valid to be assigned to MessageResponse: + // + // *ServerReflectionResponse_FileDescriptorResponse + // *ServerReflectionResponse_AllExtensionNumbersResponse + // *ServerReflectionResponse_ListServicesResponse + // *ServerReflectionResponse_ErrorResponse + MessageResponse isServerReflectionResponse_MessageResponse `protobuf_oneof:"message_response"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerReflectionResponse) Reset() { + *x = ServerReflectionResponse{} + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerReflectionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerReflectionResponse) ProtoMessage() {} + +func (x *ServerReflectionResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerReflectionResponse.ProtoReflect.Descriptor instead. +func (*ServerReflectionResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{2} +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionResponse) GetValidHost() string { + if x != nil { + return x.ValidHost + } + return "" +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionResponse) GetOriginalRequest() *ServerReflectionRequest { + if x != nil { + return x.OriginalRequest + } + return nil +} + +func (x *ServerReflectionResponse) GetMessageResponse() isServerReflectionResponse_MessageResponse { + if x != nil { + return x.MessageResponse + } + return nil +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionResponse) GetFileDescriptorResponse() *FileDescriptorResponse { + if x != nil { + if x, ok := x.MessageResponse.(*ServerReflectionResponse_FileDescriptorResponse); ok { + return x.FileDescriptorResponse + } + } + return nil +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionResponse) GetAllExtensionNumbersResponse() *ExtensionNumberResponse { + if x != nil { + if x, ok := x.MessageResponse.(*ServerReflectionResponse_AllExtensionNumbersResponse); ok { + return x.AllExtensionNumbersResponse + } + } + return nil +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionResponse) GetListServicesResponse() *ListServiceResponse { + if x != nil { + if x, ok := x.MessageResponse.(*ServerReflectionResponse_ListServicesResponse); ok { + return x.ListServicesResponse + } + } + return nil +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionResponse) GetErrorResponse() *ErrorResponse { + if x != nil { + if x, ok := x.MessageResponse.(*ServerReflectionResponse_ErrorResponse); ok { + return x.ErrorResponse + } + } + return nil +} + +type isServerReflectionResponse_MessageResponse interface { + isServerReflectionResponse_MessageResponse() +} + +type ServerReflectionResponse_FileDescriptorResponse struct { + // This message is used to answer file_by_filename, file_containing_symbol, + // file_containing_extension requests with transitive dependencies. As + // the repeated label is not allowed in oneof fields, we use a + // FileDescriptorResponse message to encapsulate the repeated fields. + // The reflection service is allowed to avoid sending FileDescriptorProtos + // that were previously sent in response to earlier requests in the stream. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + FileDescriptorResponse *FileDescriptorResponse `protobuf:"bytes,4,opt,name=file_descriptor_response,json=fileDescriptorResponse,proto3,oneof"` +} + +type ServerReflectionResponse_AllExtensionNumbersResponse struct { + // This message is used to answer all_extension_numbers_of_type request. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + AllExtensionNumbersResponse *ExtensionNumberResponse `protobuf:"bytes,5,opt,name=all_extension_numbers_response,json=allExtensionNumbersResponse,proto3,oneof"` +} + +type ServerReflectionResponse_ListServicesResponse struct { + // This message is used to answer list_services request. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ListServicesResponse *ListServiceResponse `protobuf:"bytes,6,opt,name=list_services_response,json=listServicesResponse,proto3,oneof"` +} + +type ServerReflectionResponse_ErrorResponse struct { + // This message is used when an error occurs. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ErrorResponse *ErrorResponse `protobuf:"bytes,7,opt,name=error_response,json=errorResponse,proto3,oneof"` +} + +func (*ServerReflectionResponse_FileDescriptorResponse) isServerReflectionResponse_MessageResponse() { +} + +func (*ServerReflectionResponse_AllExtensionNumbersResponse) isServerReflectionResponse_MessageResponse() { +} + +func (*ServerReflectionResponse_ListServicesResponse) isServerReflectionResponse_MessageResponse() {} + +func (*ServerReflectionResponse_ErrorResponse) isServerReflectionResponse_MessageResponse() {} + +// Serialized FileDescriptorProto messages sent by the server answering +// a file_by_filename, file_containing_symbol, or file_containing_extension +// request. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +type FileDescriptorResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Serialized FileDescriptorProto messages. We avoid taking a dependency on + // descriptor.proto, which uses proto2 only features, by making them opaque + // bytes instead. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + FileDescriptorProto [][]byte `protobuf:"bytes,1,rep,name=file_descriptor_proto,json=fileDescriptorProto,proto3" json:"file_descriptor_proto,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileDescriptorResponse) Reset() { + *x = FileDescriptorResponse{} + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileDescriptorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileDescriptorResponse) ProtoMessage() {} + +func (x *FileDescriptorResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileDescriptorResponse.ProtoReflect.Descriptor instead. +func (*FileDescriptorResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{3} +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *FileDescriptorResponse) GetFileDescriptorProto() [][]byte { + if x != nil { + return x.FileDescriptorProto + } + return nil +} + +// A list of extension numbers sent by the server answering +// all_extension_numbers_of_type request. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +type ExtensionNumberResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Full name of the base type, including the package name. The format + // is . + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + BaseTypeName string `protobuf:"bytes,1,opt,name=base_type_name,json=baseTypeName,proto3" json:"base_type_name,omitempty"` + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ExtensionNumber []int32 `protobuf:"varint,2,rep,packed,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExtensionNumberResponse) Reset() { + *x = ExtensionNumberResponse{} + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExtensionNumberResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExtensionNumberResponse) ProtoMessage() {} + +func (x *ExtensionNumberResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExtensionNumberResponse.ProtoReflect.Descriptor instead. +func (*ExtensionNumberResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{4} +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ExtensionNumberResponse) GetBaseTypeName() string { + if x != nil { + return x.BaseTypeName + } + return "" +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ExtensionNumberResponse) GetExtensionNumber() []int32 { + if x != nil { + return x.ExtensionNumber + } + return nil +} + +// A list of ServiceResponse sent by the server answering list_services request. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +type ListServiceResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The information of each service may be expanded in the future, so we use + // ServiceResponse message to encapsulate it. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + Service []*ServiceResponse `protobuf:"bytes,1,rep,name=service,proto3" json:"service,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListServiceResponse) Reset() { + *x = ListServiceResponse{} + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListServiceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListServiceResponse) ProtoMessage() {} + +func (x *ListServiceResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListServiceResponse.ProtoReflect.Descriptor instead. +func (*ListServiceResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{5} +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ListServiceResponse) GetService() []*ServiceResponse { + if x != nil { + return x.Service + } + return nil +} + +// The information of a single service used by ListServiceResponse to answer +// list_services request. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +type ServiceResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Full name of a registered service, including its package name. The format + // is . + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServiceResponse) Reset() { + *x = ServiceResponse{} + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServiceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServiceResponse) ProtoMessage() {} + +func (x *ServiceResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServiceResponse.ProtoReflect.Descriptor instead. +func (*ServiceResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{6} +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServiceResponse) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +// The error code and error message sent by the server when an error occurs. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +type ErrorResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // This field uses the error codes defined in grpc::StatusCode. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ErrorCode int32 `protobuf:"varint,1,opt,name=error_code,json=errorCode,proto3" json:"error_code,omitempty"` + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ErrorMessage string `protobuf:"bytes,2,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ErrorResponse) Reset() { + *x = ErrorResponse{} + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ErrorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ErrorResponse) ProtoMessage() {} + +func (x *ErrorResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ErrorResponse.ProtoReflect.Descriptor instead. +func (*ErrorResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{7} +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ErrorResponse) GetErrorCode() int32 { + if x != nil { + return x.ErrorCode + } + return 0 +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ErrorResponse) GetErrorMessage() string { + if x != nil { + return x.ErrorMessage + } + return "" +} + +var File_grpc_reflection_v1alpha_reflection_proto protoreflect.FileDescriptor + +const file_grpc_reflection_v1alpha_reflection_proto_rawDesc = "" + + "\n" + + "(grpc/reflection/v1alpha/reflection.proto\x12\x17grpc.reflection.v1alpha\"\xf8\x02\n" + + "\x17ServerReflectionRequest\x12\x12\n" + + "\x04host\x18\x01 \x01(\tR\x04host\x12*\n" + + "\x10file_by_filename\x18\x03 \x01(\tH\x00R\x0efileByFilename\x126\n" + + "\x16file_containing_symbol\x18\x04 \x01(\tH\x00R\x14fileContainingSymbol\x12g\n" + + "\x19file_containing_extension\x18\x05 \x01(\v2).grpc.reflection.v1alpha.ExtensionRequestH\x00R\x17fileContainingExtension\x12B\n" + + "\x1dall_extension_numbers_of_type\x18\x06 \x01(\tH\x00R\x19allExtensionNumbersOfType\x12%\n" + + "\rlist_services\x18\a \x01(\tH\x00R\flistServicesB\x11\n" + + "\x0fmessage_request\"f\n" + + "\x10ExtensionRequest\x12'\n" + + "\x0fcontaining_type\x18\x01 \x01(\tR\x0econtainingType\x12)\n" + + "\x10extension_number\x18\x02 \x01(\x05R\x0fextensionNumber\"\xc7\x04\n" + + "\x18ServerReflectionResponse\x12\x1d\n" + + "\n" + + "valid_host\x18\x01 \x01(\tR\tvalidHost\x12[\n" + + "\x10original_request\x18\x02 \x01(\v20.grpc.reflection.v1alpha.ServerReflectionRequestR\x0foriginalRequest\x12k\n" + + "\x18file_descriptor_response\x18\x04 \x01(\v2/.grpc.reflection.v1alpha.FileDescriptorResponseH\x00R\x16fileDescriptorResponse\x12w\n" + + "\x1eall_extension_numbers_response\x18\x05 \x01(\v20.grpc.reflection.v1alpha.ExtensionNumberResponseH\x00R\x1ballExtensionNumbersResponse\x12d\n" + + "\x16list_services_response\x18\x06 \x01(\v2,.grpc.reflection.v1alpha.ListServiceResponseH\x00R\x14listServicesResponse\x12O\n" + + "\x0eerror_response\x18\a \x01(\v2&.grpc.reflection.v1alpha.ErrorResponseH\x00R\rerrorResponseB\x12\n" + + "\x10message_response\"L\n" + + "\x16FileDescriptorResponse\x122\n" + + "\x15file_descriptor_proto\x18\x01 \x03(\fR\x13fileDescriptorProto\"j\n" + + "\x17ExtensionNumberResponse\x12$\n" + + "\x0ebase_type_name\x18\x01 \x01(\tR\fbaseTypeName\x12)\n" + + "\x10extension_number\x18\x02 \x03(\x05R\x0fextensionNumber\"Y\n" + + "\x13ListServiceResponse\x12B\n" + + "\aservice\x18\x01 \x03(\v2(.grpc.reflection.v1alpha.ServiceResponseR\aservice\"%\n" + + "\x0fServiceResponse\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\"S\n" + + "\rErrorResponse\x12\x1d\n" + + "\n" + + "error_code\x18\x01 \x01(\x05R\terrorCode\x12#\n" + + "\rerror_message\x18\x02 \x01(\tR\ferrorMessage2\x93\x01\n" + + "\x10ServerReflection\x12\x7f\n" + + "\x14ServerReflectionInfo\x120.grpc.reflection.v1alpha.ServerReflectionRequest\x1a1.grpc.reflection.v1alpha.ServerReflectionResponse(\x010\x01Bs\n" + + "\x1aio.grpc.reflection.v1alphaB\x15ServerReflectionProtoP\x01Z9google.golang.org/grpc/reflection/grpc_reflection_v1alpha\xb8\x01\x01b\x06proto3" + +var ( + file_grpc_reflection_v1alpha_reflection_proto_rawDescOnce sync.Once + file_grpc_reflection_v1alpha_reflection_proto_rawDescData []byte +) + +func file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP() []byte { + file_grpc_reflection_v1alpha_reflection_proto_rawDescOnce.Do(func() { + file_grpc_reflection_v1alpha_reflection_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_grpc_reflection_v1alpha_reflection_proto_rawDesc), len(file_grpc_reflection_v1alpha_reflection_proto_rawDesc))) + }) + return file_grpc_reflection_v1alpha_reflection_proto_rawDescData +} + +var file_grpc_reflection_v1alpha_reflection_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_grpc_reflection_v1alpha_reflection_proto_goTypes = []any{ + (*ServerReflectionRequest)(nil), // 0: grpc.reflection.v1alpha.ServerReflectionRequest + (*ExtensionRequest)(nil), // 1: grpc.reflection.v1alpha.ExtensionRequest + (*ServerReflectionResponse)(nil), // 2: grpc.reflection.v1alpha.ServerReflectionResponse + (*FileDescriptorResponse)(nil), // 3: grpc.reflection.v1alpha.FileDescriptorResponse + (*ExtensionNumberResponse)(nil), // 4: grpc.reflection.v1alpha.ExtensionNumberResponse + (*ListServiceResponse)(nil), // 5: grpc.reflection.v1alpha.ListServiceResponse + (*ServiceResponse)(nil), // 6: grpc.reflection.v1alpha.ServiceResponse + (*ErrorResponse)(nil), // 7: grpc.reflection.v1alpha.ErrorResponse +} +var file_grpc_reflection_v1alpha_reflection_proto_depIdxs = []int32{ + 1, // 0: grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_extension:type_name -> grpc.reflection.v1alpha.ExtensionRequest + 0, // 1: grpc.reflection.v1alpha.ServerReflectionResponse.original_request:type_name -> grpc.reflection.v1alpha.ServerReflectionRequest + 3, // 2: grpc.reflection.v1alpha.ServerReflectionResponse.file_descriptor_response:type_name -> grpc.reflection.v1alpha.FileDescriptorResponse + 4, // 3: grpc.reflection.v1alpha.ServerReflectionResponse.all_extension_numbers_response:type_name -> grpc.reflection.v1alpha.ExtensionNumberResponse + 5, // 4: grpc.reflection.v1alpha.ServerReflectionResponse.list_services_response:type_name -> grpc.reflection.v1alpha.ListServiceResponse + 7, // 5: grpc.reflection.v1alpha.ServerReflectionResponse.error_response:type_name -> grpc.reflection.v1alpha.ErrorResponse + 6, // 6: grpc.reflection.v1alpha.ListServiceResponse.service:type_name -> grpc.reflection.v1alpha.ServiceResponse + 0, // 7: grpc.reflection.v1alpha.ServerReflection.ServerReflectionInfo:input_type -> grpc.reflection.v1alpha.ServerReflectionRequest + 2, // 8: grpc.reflection.v1alpha.ServerReflection.ServerReflectionInfo:output_type -> grpc.reflection.v1alpha.ServerReflectionResponse + 8, // [8:9] is the sub-list for method output_type + 7, // [7:8] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name +} + +func init() { file_grpc_reflection_v1alpha_reflection_proto_init() } +func file_grpc_reflection_v1alpha_reflection_proto_init() { + if File_grpc_reflection_v1alpha_reflection_proto != nil { + return + } + file_grpc_reflection_v1alpha_reflection_proto_msgTypes[0].OneofWrappers = []any{ + (*ServerReflectionRequest_FileByFilename)(nil), + (*ServerReflectionRequest_FileContainingSymbol)(nil), + (*ServerReflectionRequest_FileContainingExtension)(nil), + (*ServerReflectionRequest_AllExtensionNumbersOfType)(nil), + (*ServerReflectionRequest_ListServices)(nil), + } + file_grpc_reflection_v1alpha_reflection_proto_msgTypes[2].OneofWrappers = []any{ + (*ServerReflectionResponse_FileDescriptorResponse)(nil), + (*ServerReflectionResponse_AllExtensionNumbersResponse)(nil), + (*ServerReflectionResponse_ListServicesResponse)(nil), + (*ServerReflectionResponse_ErrorResponse)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_reflection_v1alpha_reflection_proto_rawDesc), len(file_grpc_reflection_v1alpha_reflection_proto_rawDesc)), + NumEnums: 0, + NumMessages: 8, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_reflection_v1alpha_reflection_proto_goTypes, + DependencyIndexes: file_grpc_reflection_v1alpha_reflection_proto_depIdxs, + MessageInfos: file_grpc_reflection_v1alpha_reflection_proto_msgTypes, + }.Build() + File_grpc_reflection_v1alpha_reflection_proto = out.File + file_grpc_reflection_v1alpha_reflection_proto_goTypes = nil + file_grpc_reflection_v1alpha_reflection_proto_depIdxs = nil +} diff --git a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go new file mode 100644 index 0000000..2a3f45d --- /dev/null +++ b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go @@ -0,0 +1,135 @@ +// Copyright 2016 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// Service exported by server reflection + +// Warning: this entire file is deprecated. Use this instead: +// https://github.com/grpc/grpc-proto/blob/master/grpc/reflection/v1/reflection.proto + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// grpc/reflection/v1alpha/reflection.proto is a deprecated file. + +package grpc_reflection_v1alpha + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + ServerReflection_ServerReflectionInfo_FullMethodName = "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo" +) + +// ServerReflectionClient is the client API for ServerReflection service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ServerReflectionClient interface { + // The reflection service is structured as a bidirectional stream, ensuring + // all related requests go to a single server. + ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ServerReflectionRequest, ServerReflectionResponse], error) +} + +type serverReflectionClient struct { + cc grpc.ClientConnInterface +} + +func NewServerReflectionClient(cc grpc.ClientConnInterface) ServerReflectionClient { + return &serverReflectionClient{cc} +} + +func (c *serverReflectionClient) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ServerReflectionRequest, ServerReflectionResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &ServerReflection_ServiceDesc.Streams[0], ServerReflection_ServerReflectionInfo_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[ServerReflectionRequest, ServerReflectionResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type ServerReflection_ServerReflectionInfoClient = grpc.BidiStreamingClient[ServerReflectionRequest, ServerReflectionResponse] + +// ServerReflectionServer is the server API for ServerReflection service. +// All implementations should embed UnimplementedServerReflectionServer +// for forward compatibility. +type ServerReflectionServer interface { + // The reflection service is structured as a bidirectional stream, ensuring + // all related requests go to a single server. + ServerReflectionInfo(grpc.BidiStreamingServer[ServerReflectionRequest, ServerReflectionResponse]) error +} + +// UnimplementedServerReflectionServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedServerReflectionServer struct{} + +func (UnimplementedServerReflectionServer) ServerReflectionInfo(grpc.BidiStreamingServer[ServerReflectionRequest, ServerReflectionResponse]) error { + return status.Error(codes.Unimplemented, "method ServerReflectionInfo not implemented") +} +func (UnimplementedServerReflectionServer) testEmbeddedByValue() {} + +// UnsafeServerReflectionServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ServerReflectionServer will +// result in compilation errors. +type UnsafeServerReflectionServer interface { + mustEmbedUnimplementedServerReflectionServer() +} + +func RegisterServerReflectionServer(s grpc.ServiceRegistrar, srv ServerReflectionServer) { + // If the following call panics, it indicates UnimplementedServerReflectionServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&ServerReflection_ServiceDesc, srv) +} + +func _ServerReflection_ServerReflectionInfo_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(ServerReflectionServer).ServerReflectionInfo(&grpc.GenericServerStream[ServerReflectionRequest, ServerReflectionResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type ServerReflection_ServerReflectionInfoServer = grpc.BidiStreamingServer[ServerReflectionRequest, ServerReflectionResponse] + +// ServerReflection_ServiceDesc is the grpc.ServiceDesc for ServerReflection service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ServerReflection_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.reflection.v1alpha.ServerReflection", + HandlerType: (*ServerReflectionServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "ServerReflectionInfo", + Handler: _ServerReflection_ServerReflectionInfo_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/reflection/v1alpha/reflection.proto", +} diff --git a/reflection/grpc_testing/proto2.pb.go b/reflection/grpc_testing/proto2.pb.go new file mode 100644 index 0000000..69417e2 --- /dev/null +++ b/reflection/grpc_testing/proto2.pb.go @@ -0,0 +1,138 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: reflection/grpc_testing/proto2.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ToBeExtended struct { + state protoimpl.MessageState `protogen:"open.v1"` + Foo *int32 `protobuf:"varint,1,req,name=foo" json:"foo,omitempty"` + extensionFields protoimpl.ExtensionFields + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ToBeExtended) Reset() { + *x = ToBeExtended{} + mi := &file_reflection_grpc_testing_proto2_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ToBeExtended) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ToBeExtended) ProtoMessage() {} + +func (x *ToBeExtended) ProtoReflect() protoreflect.Message { + mi := &file_reflection_grpc_testing_proto2_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ToBeExtended.ProtoReflect.Descriptor instead. +func (*ToBeExtended) Descriptor() ([]byte, []int) { + return file_reflection_grpc_testing_proto2_proto_rawDescGZIP(), []int{0} +} + +func (x *ToBeExtended) GetFoo() int32 { + if x != nil && x.Foo != nil { + return *x.Foo + } + return 0 +} + +var File_reflection_grpc_testing_proto2_proto protoreflect.FileDescriptor + +const file_reflection_grpc_testing_proto2_proto_rawDesc = "" + + "\n" + + "$reflection/grpc_testing/proto2.proto\x12\fgrpc.testing\"&\n" + + "\fToBeExtended\x12\x10\n" + + "\x03foo\x18\x01 \x02(\x05R\x03foo*\x04\b\n" + + "\x10\x1fB0Z.google.golang.org/grpc/reflection/grpc_testing" + +var ( + file_reflection_grpc_testing_proto2_proto_rawDescOnce sync.Once + file_reflection_grpc_testing_proto2_proto_rawDescData []byte +) + +func file_reflection_grpc_testing_proto2_proto_rawDescGZIP() []byte { + file_reflection_grpc_testing_proto2_proto_rawDescOnce.Do(func() { + file_reflection_grpc_testing_proto2_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_reflection_grpc_testing_proto2_proto_rawDesc), len(file_reflection_grpc_testing_proto2_proto_rawDesc))) + }) + return file_reflection_grpc_testing_proto2_proto_rawDescData +} + +var file_reflection_grpc_testing_proto2_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_reflection_grpc_testing_proto2_proto_goTypes = []any{ + (*ToBeExtended)(nil), // 0: grpc.testing.ToBeExtended +} +var file_reflection_grpc_testing_proto2_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_reflection_grpc_testing_proto2_proto_init() } +func file_reflection_grpc_testing_proto2_proto_init() { + if File_reflection_grpc_testing_proto2_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_reflection_grpc_testing_proto2_proto_rawDesc), len(file_reflection_grpc_testing_proto2_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_reflection_grpc_testing_proto2_proto_goTypes, + DependencyIndexes: file_reflection_grpc_testing_proto2_proto_depIdxs, + MessageInfos: file_reflection_grpc_testing_proto2_proto_msgTypes, + }.Build() + File_reflection_grpc_testing_proto2_proto = out.File + file_reflection_grpc_testing_proto2_proto_goTypes = nil + file_reflection_grpc_testing_proto2_proto_depIdxs = nil +} diff --git a/reflection/grpc_testing/proto2.proto b/reflection/grpc_testing/proto2.proto new file mode 100644 index 0000000..c90d809 --- /dev/null +++ b/reflection/grpc_testing/proto2.proto @@ -0,0 +1,24 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +option go_package = "google.golang.org/grpc/reflection/grpc_testing"; + +package grpc.testing; + +message ToBeExtended { + required int32 foo = 1; + extensions 10 to 30; +} diff --git a/reflection/grpc_testing/proto2_ext.pb.go b/reflection/grpc_testing/proto2_ext.pb.go new file mode 100644 index 0000000..cd032f4 --- /dev/null +++ b/reflection/grpc_testing/proto2_ext.pb.go @@ -0,0 +1,186 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: reflection/grpc_testing/proto2_ext.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Extension struct { + state protoimpl.MessageState `protogen:"open.v1"` + Whatzit *int32 `protobuf:"varint,1,opt,name=whatzit" json:"whatzit,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Extension) Reset() { + *x = Extension{} + mi := &file_reflection_grpc_testing_proto2_ext_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Extension) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Extension) ProtoMessage() {} + +func (x *Extension) ProtoReflect() protoreflect.Message { + mi := &file_reflection_grpc_testing_proto2_ext_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Extension.ProtoReflect.Descriptor instead. +func (*Extension) Descriptor() ([]byte, []int) { + return file_reflection_grpc_testing_proto2_ext_proto_rawDescGZIP(), []int{0} +} + +func (x *Extension) GetWhatzit() int32 { + if x != nil && x.Whatzit != nil { + return *x.Whatzit + } + return 0 +} + +var file_reflection_grpc_testing_proto2_ext_proto_extTypes = []protoimpl.ExtensionInfo{ + { + ExtendedType: (*ToBeExtended)(nil), + ExtensionType: (*int32)(nil), + Field: 13, + Name: "grpc.testing.foo", + Tag: "varint,13,opt,name=foo", + Filename: "reflection/grpc_testing/proto2_ext.proto", + }, + { + ExtendedType: (*ToBeExtended)(nil), + ExtensionType: (*Extension)(nil), + Field: 17, + Name: "grpc.testing.bar", + Tag: "bytes,17,opt,name=bar", + Filename: "reflection/grpc_testing/proto2_ext.proto", + }, + { + ExtendedType: (*ToBeExtended)(nil), + ExtensionType: (*SearchRequest)(nil), + Field: 19, + Name: "grpc.testing.baz", + Tag: "bytes,19,opt,name=baz", + Filename: "reflection/grpc_testing/proto2_ext.proto", + }, +} + +// Extension fields to ToBeExtended. +var ( + // optional int32 foo = 13; + E_Foo = &file_reflection_grpc_testing_proto2_ext_proto_extTypes[0] + // optional grpc.testing.Extension bar = 17; + E_Bar = &file_reflection_grpc_testing_proto2_ext_proto_extTypes[1] + // optional grpc.testing.SearchRequest baz = 19; + E_Baz = &file_reflection_grpc_testing_proto2_ext_proto_extTypes[2] +) + +var File_reflection_grpc_testing_proto2_ext_proto protoreflect.FileDescriptor + +const file_reflection_grpc_testing_proto2_ext_proto_rawDesc = "" + + "\n" + + "(reflection/grpc_testing/proto2_ext.proto\x12\fgrpc.testing\x1a$reflection/grpc_testing/proto2.proto\x1a\"reflection/grpc_testing/test.proto\"%\n" + + "\tExtension\x12\x18\n" + + "\awhatzit\x18\x01 \x01(\x05R\awhatzit:,\n" + + "\x03foo\x12\x1a.grpc.testing.ToBeExtended\x18\r \x01(\x05R\x03foo:E\n" + + "\x03bar\x12\x1a.grpc.testing.ToBeExtended\x18\x11 \x01(\v2\x17.grpc.testing.ExtensionR\x03bar:I\n" + + "\x03baz\x12\x1a.grpc.testing.ToBeExtended\x18\x13 \x01(\v2\x1b.grpc.testing.SearchRequestR\x03bazB0Z.google.golang.org/grpc/reflection/grpc_testing" + +var ( + file_reflection_grpc_testing_proto2_ext_proto_rawDescOnce sync.Once + file_reflection_grpc_testing_proto2_ext_proto_rawDescData []byte +) + +func file_reflection_grpc_testing_proto2_ext_proto_rawDescGZIP() []byte { + file_reflection_grpc_testing_proto2_ext_proto_rawDescOnce.Do(func() { + file_reflection_grpc_testing_proto2_ext_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_reflection_grpc_testing_proto2_ext_proto_rawDesc), len(file_reflection_grpc_testing_proto2_ext_proto_rawDesc))) + }) + return file_reflection_grpc_testing_proto2_ext_proto_rawDescData +} + +var file_reflection_grpc_testing_proto2_ext_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_reflection_grpc_testing_proto2_ext_proto_goTypes = []any{ + (*Extension)(nil), // 0: grpc.testing.Extension + (*ToBeExtended)(nil), // 1: grpc.testing.ToBeExtended + (*SearchRequest)(nil), // 2: grpc.testing.SearchRequest +} +var file_reflection_grpc_testing_proto2_ext_proto_depIdxs = []int32{ + 1, // 0: grpc.testing.foo:extendee -> grpc.testing.ToBeExtended + 1, // 1: grpc.testing.bar:extendee -> grpc.testing.ToBeExtended + 1, // 2: grpc.testing.baz:extendee -> grpc.testing.ToBeExtended + 0, // 3: grpc.testing.bar:type_name -> grpc.testing.Extension + 2, // 4: grpc.testing.baz:type_name -> grpc.testing.SearchRequest + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 3, // [3:5] is the sub-list for extension type_name + 0, // [0:3] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_reflection_grpc_testing_proto2_ext_proto_init() } +func file_reflection_grpc_testing_proto2_ext_proto_init() { + if File_reflection_grpc_testing_proto2_ext_proto != nil { + return + } + file_reflection_grpc_testing_proto2_proto_init() + file_reflection_grpc_testing_test_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_reflection_grpc_testing_proto2_ext_proto_rawDesc), len(file_reflection_grpc_testing_proto2_ext_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 3, + NumServices: 0, + }, + GoTypes: file_reflection_grpc_testing_proto2_ext_proto_goTypes, + DependencyIndexes: file_reflection_grpc_testing_proto2_ext_proto_depIdxs, + MessageInfos: file_reflection_grpc_testing_proto2_ext_proto_msgTypes, + ExtensionInfos: file_reflection_grpc_testing_proto2_ext_proto_extTypes, + }.Build() + File_reflection_grpc_testing_proto2_ext_proto = out.File + file_reflection_grpc_testing_proto2_ext_proto_goTypes = nil + file_reflection_grpc_testing_proto2_ext_proto_depIdxs = nil +} diff --git a/reflection/grpc_testing/proto2_ext.proto b/reflection/grpc_testing/proto2_ext.proto new file mode 100644 index 0000000..4ba2dc6 --- /dev/null +++ b/reflection/grpc_testing/proto2_ext.proto @@ -0,0 +1,32 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +option go_package = "google.golang.org/grpc/reflection/grpc_testing"; + +package grpc.testing; + +import "reflection/grpc_testing/proto2.proto"; +import "reflection/grpc_testing/test.proto"; + +extend ToBeExtended { + optional int32 foo = 13; + optional Extension bar = 17; + optional SearchRequest baz = 19; +} + +message Extension { + optional int32 whatzit = 1; +} diff --git a/reflection/grpc_testing/proto2_ext2.pb.go b/reflection/grpc_testing/proto2_ext2.pb.go new file mode 100644 index 0000000..f3c51f5 --- /dev/null +++ b/reflection/grpc_testing/proto2_ext2.pb.go @@ -0,0 +1,171 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: reflection/grpc_testing/proto2_ext2.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type AnotherExtension struct { + state protoimpl.MessageState `protogen:"open.v1"` + Whatchamacallit *int32 `protobuf:"varint,1,opt,name=whatchamacallit" json:"whatchamacallit,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AnotherExtension) Reset() { + *x = AnotherExtension{} + mi := &file_reflection_grpc_testing_proto2_ext2_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AnotherExtension) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AnotherExtension) ProtoMessage() {} + +func (x *AnotherExtension) ProtoReflect() protoreflect.Message { + mi := &file_reflection_grpc_testing_proto2_ext2_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AnotherExtension.ProtoReflect.Descriptor instead. +func (*AnotherExtension) Descriptor() ([]byte, []int) { + return file_reflection_grpc_testing_proto2_ext2_proto_rawDescGZIP(), []int{0} +} + +func (x *AnotherExtension) GetWhatchamacallit() int32 { + if x != nil && x.Whatchamacallit != nil { + return *x.Whatchamacallit + } + return 0 +} + +var file_reflection_grpc_testing_proto2_ext2_proto_extTypes = []protoimpl.ExtensionInfo{ + { + ExtendedType: (*ToBeExtended)(nil), + ExtensionType: (*string)(nil), + Field: 23, + Name: "grpc.testing.frob", + Tag: "bytes,23,opt,name=frob", + Filename: "reflection/grpc_testing/proto2_ext2.proto", + }, + { + ExtendedType: (*ToBeExtended)(nil), + ExtensionType: (*AnotherExtension)(nil), + Field: 29, + Name: "grpc.testing.nitz", + Tag: "bytes,29,opt,name=nitz", + Filename: "reflection/grpc_testing/proto2_ext2.proto", + }, +} + +// Extension fields to ToBeExtended. +var ( + // optional string frob = 23; + E_Frob = &file_reflection_grpc_testing_proto2_ext2_proto_extTypes[0] + // optional grpc.testing.AnotherExtension nitz = 29; + E_Nitz = &file_reflection_grpc_testing_proto2_ext2_proto_extTypes[1] +) + +var File_reflection_grpc_testing_proto2_ext2_proto protoreflect.FileDescriptor + +const file_reflection_grpc_testing_proto2_ext2_proto_rawDesc = "" + + "\n" + + ")reflection/grpc_testing/proto2_ext2.proto\x12\fgrpc.testing\x1a$reflection/grpc_testing/proto2.proto\"<\n" + + "\x10AnotherExtension\x12(\n" + + "\x0fwhatchamacallit\x18\x01 \x01(\x05R\x0fwhatchamacallit:.\n" + + "\x04frob\x12\x1a.grpc.testing.ToBeExtended\x18\x17 \x01(\tR\x04frob:N\n" + + "\x04nitz\x12\x1a.grpc.testing.ToBeExtended\x18\x1d \x01(\v2\x1e.grpc.testing.AnotherExtensionR\x04nitzB0Z.google.golang.org/grpc/reflection/grpc_testing" + +var ( + file_reflection_grpc_testing_proto2_ext2_proto_rawDescOnce sync.Once + file_reflection_grpc_testing_proto2_ext2_proto_rawDescData []byte +) + +func file_reflection_grpc_testing_proto2_ext2_proto_rawDescGZIP() []byte { + file_reflection_grpc_testing_proto2_ext2_proto_rawDescOnce.Do(func() { + file_reflection_grpc_testing_proto2_ext2_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_reflection_grpc_testing_proto2_ext2_proto_rawDesc), len(file_reflection_grpc_testing_proto2_ext2_proto_rawDesc))) + }) + return file_reflection_grpc_testing_proto2_ext2_proto_rawDescData +} + +var file_reflection_grpc_testing_proto2_ext2_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_reflection_grpc_testing_proto2_ext2_proto_goTypes = []any{ + (*AnotherExtension)(nil), // 0: grpc.testing.AnotherExtension + (*ToBeExtended)(nil), // 1: grpc.testing.ToBeExtended +} +var file_reflection_grpc_testing_proto2_ext2_proto_depIdxs = []int32{ + 1, // 0: grpc.testing.frob:extendee -> grpc.testing.ToBeExtended + 1, // 1: grpc.testing.nitz:extendee -> grpc.testing.ToBeExtended + 0, // 2: grpc.testing.nitz:type_name -> grpc.testing.AnotherExtension + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 2, // [2:3] is the sub-list for extension type_name + 0, // [0:2] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_reflection_grpc_testing_proto2_ext2_proto_init() } +func file_reflection_grpc_testing_proto2_ext2_proto_init() { + if File_reflection_grpc_testing_proto2_ext2_proto != nil { + return + } + file_reflection_grpc_testing_proto2_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_reflection_grpc_testing_proto2_ext2_proto_rawDesc), len(file_reflection_grpc_testing_proto2_ext2_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 2, + NumServices: 0, + }, + GoTypes: file_reflection_grpc_testing_proto2_ext2_proto_goTypes, + DependencyIndexes: file_reflection_grpc_testing_proto2_ext2_proto_depIdxs, + MessageInfos: file_reflection_grpc_testing_proto2_ext2_proto_msgTypes, + ExtensionInfos: file_reflection_grpc_testing_proto2_ext2_proto_extTypes, + }.Build() + File_reflection_grpc_testing_proto2_ext2_proto = out.File + file_reflection_grpc_testing_proto2_ext2_proto_goTypes = nil + file_reflection_grpc_testing_proto2_ext2_proto_depIdxs = nil +} diff --git a/reflection/grpc_testing/proto2_ext2.proto b/reflection/grpc_testing/proto2_ext2.proto new file mode 100644 index 0000000..9311546 --- /dev/null +++ b/reflection/grpc_testing/proto2_ext2.proto @@ -0,0 +1,30 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +option go_package = "google.golang.org/grpc/reflection/grpc_testing"; + +package grpc.testing; + +import "reflection/grpc_testing/proto2.proto"; + +extend ToBeExtended { + optional string frob = 23; + optional AnotherExtension nitz = 29; +} + +message AnotherExtension { + optional int32 whatchamacallit = 1; +} diff --git a/reflection/grpc_testing/test.pb.go b/reflection/grpc_testing/test.pb.go new file mode 100644 index 0000000..bcdc167 --- /dev/null +++ b/reflection/grpc_testing/test.pb.go @@ -0,0 +1,256 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: reflection/grpc_testing/test.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SearchResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Results []*SearchResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SearchResponse) Reset() { + *x = SearchResponse{} + mi := &file_reflection_grpc_testing_test_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SearchResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SearchResponse) ProtoMessage() {} + +func (x *SearchResponse) ProtoReflect() protoreflect.Message { + mi := &file_reflection_grpc_testing_test_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchResponse.ProtoReflect.Descriptor instead. +func (*SearchResponse) Descriptor() ([]byte, []int) { + return file_reflection_grpc_testing_test_proto_rawDescGZIP(), []int{0} +} + +func (x *SearchResponse) GetResults() []*SearchResponse_Result { + if x != nil { + return x.Results + } + return nil +} + +type SearchRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SearchRequest) Reset() { + *x = SearchRequest{} + mi := &file_reflection_grpc_testing_test_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SearchRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SearchRequest) ProtoMessage() {} + +func (x *SearchRequest) ProtoReflect() protoreflect.Message { + mi := &file_reflection_grpc_testing_test_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchRequest.ProtoReflect.Descriptor instead. +func (*SearchRequest) Descriptor() ([]byte, []int) { + return file_reflection_grpc_testing_test_proto_rawDescGZIP(), []int{1} +} + +func (x *SearchRequest) GetQuery() string { + if x != nil { + return x.Query + } + return "" +} + +type SearchResponse_Result struct { + state protoimpl.MessageState `protogen:"open.v1"` + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` + Title string `protobuf:"bytes,2,opt,name=title,proto3" json:"title,omitempty"` + Snippets []string `protobuf:"bytes,3,rep,name=snippets,proto3" json:"snippets,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SearchResponse_Result) Reset() { + *x = SearchResponse_Result{} + mi := &file_reflection_grpc_testing_test_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SearchResponse_Result) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SearchResponse_Result) ProtoMessage() {} + +func (x *SearchResponse_Result) ProtoReflect() protoreflect.Message { + mi := &file_reflection_grpc_testing_test_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchResponse_Result.ProtoReflect.Descriptor instead. +func (*SearchResponse_Result) Descriptor() ([]byte, []int) { + return file_reflection_grpc_testing_test_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *SearchResponse_Result) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *SearchResponse_Result) GetTitle() string { + if x != nil { + return x.Title + } + return "" +} + +func (x *SearchResponse_Result) GetSnippets() []string { + if x != nil { + return x.Snippets + } + return nil +} + +var File_reflection_grpc_testing_test_proto protoreflect.FileDescriptor + +const file_reflection_grpc_testing_test_proto_rawDesc = "" + + "\n" + + "\"reflection/grpc_testing/test.proto\x12\fgrpc.testing\"\x9d\x01\n" + + "\x0eSearchResponse\x12=\n" + + "\aresults\x18\x01 \x03(\v2#.grpc.testing.SearchResponse.ResultR\aresults\x1aL\n" + + "\x06Result\x12\x10\n" + + "\x03url\x18\x01 \x01(\tR\x03url\x12\x14\n" + + "\x05title\x18\x02 \x01(\tR\x05title\x12\x1a\n" + + "\bsnippets\x18\x03 \x03(\tR\bsnippets\"%\n" + + "\rSearchRequest\x12\x14\n" + + "\x05query\x18\x01 \x01(\tR\x05query2\xa6\x01\n" + + "\rSearchService\x12C\n" + + "\x06Search\x12\x1b.grpc.testing.SearchRequest\x1a\x1c.grpc.testing.SearchResponse\x12P\n" + + "\x0fStreamingSearch\x12\x1b.grpc.testing.SearchRequest\x1a\x1c.grpc.testing.SearchResponse(\x010\x01B0Z.google.golang.org/grpc/reflection/grpc_testingb\x06proto3" + +var ( + file_reflection_grpc_testing_test_proto_rawDescOnce sync.Once + file_reflection_grpc_testing_test_proto_rawDescData []byte +) + +func file_reflection_grpc_testing_test_proto_rawDescGZIP() []byte { + file_reflection_grpc_testing_test_proto_rawDescOnce.Do(func() { + file_reflection_grpc_testing_test_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_reflection_grpc_testing_test_proto_rawDesc), len(file_reflection_grpc_testing_test_proto_rawDesc))) + }) + return file_reflection_grpc_testing_test_proto_rawDescData +} + +var file_reflection_grpc_testing_test_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_reflection_grpc_testing_test_proto_goTypes = []any{ + (*SearchResponse)(nil), // 0: grpc.testing.SearchResponse + (*SearchRequest)(nil), // 1: grpc.testing.SearchRequest + (*SearchResponse_Result)(nil), // 2: grpc.testing.SearchResponse.Result +} +var file_reflection_grpc_testing_test_proto_depIdxs = []int32{ + 2, // 0: grpc.testing.SearchResponse.results:type_name -> grpc.testing.SearchResponse.Result + 1, // 1: grpc.testing.SearchService.Search:input_type -> grpc.testing.SearchRequest + 1, // 2: grpc.testing.SearchService.StreamingSearch:input_type -> grpc.testing.SearchRequest + 0, // 3: grpc.testing.SearchService.Search:output_type -> grpc.testing.SearchResponse + 0, // 4: grpc.testing.SearchService.StreamingSearch:output_type -> grpc.testing.SearchResponse + 3, // [3:5] is the sub-list for method output_type + 1, // [1:3] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_reflection_grpc_testing_test_proto_init() } +func file_reflection_grpc_testing_test_proto_init() { + if File_reflection_grpc_testing_test_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_reflection_grpc_testing_test_proto_rawDesc), len(file_reflection_grpc_testing_test_proto_rawDesc)), + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_reflection_grpc_testing_test_proto_goTypes, + DependencyIndexes: file_reflection_grpc_testing_test_proto_depIdxs, + MessageInfos: file_reflection_grpc_testing_test_proto_msgTypes, + }.Build() + File_reflection_grpc_testing_test_proto = out.File + file_reflection_grpc_testing_test_proto_goTypes = nil + file_reflection_grpc_testing_test_proto_depIdxs = nil +} diff --git a/reflection/grpc_testing/test.proto b/reflection/grpc_testing/test.proto new file mode 100644 index 0000000..c4b18de --- /dev/null +++ b/reflection/grpc_testing/test.proto @@ -0,0 +1,37 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +option go_package = "google.golang.org/grpc/reflection/grpc_testing"; + +package grpc.testing; + +message SearchResponse { + message Result { + string url = 1; + string title = 2; + repeated string snippets = 3; + } + repeated Result results = 1; +} + +message SearchRequest { + string query = 1; +} + +service SearchService { + rpc Search(SearchRequest) returns (SearchResponse); + rpc StreamingSearch(stream SearchRequest) returns (stream SearchResponse); +} diff --git a/reflection/grpc_testing/test_grpc.pb.go b/reflection/grpc_testing/test_grpc.pb.go new file mode 100644 index 0000000..ff0b81b --- /dev/null +++ b/reflection/grpc_testing/test_grpc.pb.go @@ -0,0 +1,168 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.27.1 +// source: reflection/grpc_testing/test.proto + +package grpc_testing + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + SearchService_Search_FullMethodName = "/grpc.testing.SearchService/Search" + SearchService_StreamingSearch_FullMethodName = "/grpc.testing.SearchService/StreamingSearch" +) + +// SearchServiceClient is the client API for SearchService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type SearchServiceClient interface { + Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) + StreamingSearch(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SearchRequest, SearchResponse], error) +} + +type searchServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewSearchServiceClient(cc grpc.ClientConnInterface) SearchServiceClient { + return &searchServiceClient{cc} +} + +func (c *searchServiceClient) Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SearchResponse) + err := c.cc.Invoke(ctx, SearchService_Search_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *searchServiceClient) StreamingSearch(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SearchRequest, SearchResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &SearchService_ServiceDesc.Streams[0], SearchService_StreamingSearch_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[SearchRequest, SearchResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type SearchService_StreamingSearchClient = grpc.BidiStreamingClient[SearchRequest, SearchResponse] + +// SearchServiceServer is the server API for SearchService service. +// All implementations must embed UnimplementedSearchServiceServer +// for forward compatibility. +type SearchServiceServer interface { + Search(context.Context, *SearchRequest) (*SearchResponse, error) + StreamingSearch(grpc.BidiStreamingServer[SearchRequest, SearchResponse]) error + mustEmbedUnimplementedSearchServiceServer() +} + +// UnimplementedSearchServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedSearchServiceServer struct{} + +func (UnimplementedSearchServiceServer) Search(context.Context, *SearchRequest) (*SearchResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Search not implemented") +} +func (UnimplementedSearchServiceServer) StreamingSearch(grpc.BidiStreamingServer[SearchRequest, SearchResponse]) error { + return status.Error(codes.Unimplemented, "method StreamingSearch not implemented") +} +func (UnimplementedSearchServiceServer) mustEmbedUnimplementedSearchServiceServer() {} +func (UnimplementedSearchServiceServer) testEmbeddedByValue() {} + +// UnsafeSearchServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SearchServiceServer will +// result in compilation errors. +type UnsafeSearchServiceServer interface { + mustEmbedUnimplementedSearchServiceServer() +} + +func RegisterSearchServiceServer(s grpc.ServiceRegistrar, srv SearchServiceServer) { + // If the following call panics, it indicates UnimplementedSearchServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&SearchService_ServiceDesc, srv) +} + +func _SearchService_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SearchRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SearchServiceServer).Search(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: SearchService_Search_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SearchServiceServer).Search(ctx, req.(*SearchRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SearchService_StreamingSearch_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SearchServiceServer).StreamingSearch(&grpc.GenericServerStream[SearchRequest, SearchResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type SearchService_StreamingSearchServer = grpc.BidiStreamingServer[SearchRequest, SearchResponse] + +// SearchService_ServiceDesc is the grpc.ServiceDesc for SearchService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var SearchService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.SearchService", + HandlerType: (*SearchServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Search", + Handler: _SearchService_Search_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "StreamingSearch", + Handler: _SearchService_StreamingSearch_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "reflection/grpc_testing/test.proto", +} diff --git a/reflection/internal/internal.go b/reflection/internal/internal.go new file mode 100644 index 0000000..902fc6d --- /dev/null +++ b/reflection/internal/internal.go @@ -0,0 +1,436 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains code that is shared by both reflection package and +// the test package. The packages are split in this way inorder to avoid +// dependency to deprecated package github.com/golang/protobuf. +package internal + +import ( + "io" + "sort" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protodesc" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + + v1reflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1" + v1reflectionpb "google.golang.org/grpc/reflection/grpc_reflection_v1" + v1alphareflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" + v1alphareflectionpb "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" +) + +// ServiceInfoProvider is an interface used to retrieve metadata about the +// services to expose. +type ServiceInfoProvider interface { + GetServiceInfo() map[string]grpc.ServiceInfo +} + +// ExtensionResolver is the interface used to query details about extensions. +// This interface is satisfied by protoregistry.GlobalTypes. +type ExtensionResolver interface { + protoregistry.ExtensionTypeResolver + RangeExtensionsByMessage(message protoreflect.FullName, f func(protoreflect.ExtensionType) bool) +} + +// ServerReflectionServer is the server API for ServerReflection service. +type ServerReflectionServer struct { + v1alphareflectiongrpc.UnimplementedServerReflectionServer + S ServiceInfoProvider + DescResolver protodesc.Resolver + ExtResolver ExtensionResolver +} + +// FileDescWithDependencies returns a slice of serialized fileDescriptors in +// wire format ([]byte). The fileDescriptors will include fd and all the +// transitive dependencies of fd with names not in sentFileDescriptors. +func (s *ServerReflectionServer) FileDescWithDependencies(fd protoreflect.FileDescriptor, sentFileDescriptors map[string]bool) ([][]byte, error) { + if fd.IsPlaceholder() { + // If the given root file is a placeholder, treat it + // as missing instead of serializing it. + return nil, protoregistry.NotFound + } + var r [][]byte + queue := []protoreflect.FileDescriptor{fd} + for len(queue) > 0 { + currentfd := queue[0] + queue = queue[1:] + if currentfd.IsPlaceholder() { + // Skip any missing files in the dependency graph. + continue + } + if sent := sentFileDescriptors[currentfd.Path()]; len(r) == 0 || !sent { + sentFileDescriptors[currentfd.Path()] = true + fdProto := protodesc.ToFileDescriptorProto(currentfd) + currentfdEncoded, err := proto.Marshal(fdProto) + if err != nil { + return nil, err + } + r = append(r, currentfdEncoded) + } + for i := 0; i < currentfd.Imports().Len(); i++ { + queue = append(queue, currentfd.Imports().Get(i)) + } + } + return r, nil +} + +// FileDescEncodingContainingSymbol finds the file descriptor containing the +// given symbol, finds all of its previously unsent transitive dependencies, +// does marshalling on them, and returns the marshalled result. The given symbol +// can be a type, a service or a method. +func (s *ServerReflectionServer) FileDescEncodingContainingSymbol(name string, sentFileDescriptors map[string]bool) ([][]byte, error) { + d, err := s.DescResolver.FindDescriptorByName(protoreflect.FullName(name)) + if err != nil { + return nil, err + } + return s.FileDescWithDependencies(d.ParentFile(), sentFileDescriptors) +} + +// FileDescEncodingContainingExtension finds the file descriptor containing +// given extension, finds all of its previously unsent transitive dependencies, +// does marshalling on them, and returns the marshalled result. +func (s *ServerReflectionServer) FileDescEncodingContainingExtension(typeName string, extNum int32, sentFileDescriptors map[string]bool) ([][]byte, error) { + xt, err := s.ExtResolver.FindExtensionByNumber(protoreflect.FullName(typeName), protoreflect.FieldNumber(extNum)) + if err != nil { + return nil, err + } + return s.FileDescWithDependencies(xt.TypeDescriptor().ParentFile(), sentFileDescriptors) +} + +// AllExtensionNumbersForTypeName returns all extension numbers for the given type. +func (s *ServerReflectionServer) AllExtensionNumbersForTypeName(name string) ([]int32, error) { + var numbers []int32 + s.ExtResolver.RangeExtensionsByMessage(protoreflect.FullName(name), func(xt protoreflect.ExtensionType) bool { + numbers = append(numbers, int32(xt.TypeDescriptor().Number())) + return true + }) + sort.Slice(numbers, func(i, j int) bool { + return numbers[i] < numbers[j] + }) + if len(numbers) == 0 { + // maybe return an error if given type name is not known + if _, err := s.DescResolver.FindDescriptorByName(protoreflect.FullName(name)); err != nil { + return nil, err + } + } + return numbers, nil +} + +// ListServices returns the names of services this server exposes. +func (s *ServerReflectionServer) ListServices() []*v1reflectionpb.ServiceResponse { + serviceInfo := s.S.GetServiceInfo() + resp := make([]*v1reflectionpb.ServiceResponse, 0, len(serviceInfo)) + for svc := range serviceInfo { + resp = append(resp, &v1reflectionpb.ServiceResponse{Name: svc}) + } + sort.Slice(resp, func(i, j int) bool { + return resp[i].Name < resp[j].Name + }) + return resp +} + +// ServerReflectionInfo is the reflection service handler. +func (s *ServerReflectionServer) ServerReflectionInfo(stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoServer) error { + sentFileDescriptors := make(map[string]bool) + for { + in, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + out := &v1reflectionpb.ServerReflectionResponse{ + ValidHost: in.Host, + OriginalRequest: in, + } + switch req := in.MessageRequest.(type) { + case *v1reflectionpb.ServerReflectionRequest_FileByFilename: + var b [][]byte + fd, err := s.DescResolver.FindFileByPath(req.FileByFilename) + if err == nil { + b, err = s.FileDescWithDependencies(fd, sentFileDescriptors) + } + if err != nil { + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ErrorResponse{ + ErrorResponse: &v1reflectionpb.ErrorResponse{ + ErrorCode: int32(codes.NotFound), + ErrorMessage: err.Error(), + }, + } + } else { + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse{ + FileDescriptorResponse: &v1reflectionpb.FileDescriptorResponse{FileDescriptorProto: b}, + } + } + case *v1reflectionpb.ServerReflectionRequest_FileContainingSymbol: + b, err := s.FileDescEncodingContainingSymbol(req.FileContainingSymbol, sentFileDescriptors) + if err != nil { + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ErrorResponse{ + ErrorResponse: &v1reflectionpb.ErrorResponse{ + ErrorCode: int32(codes.NotFound), + ErrorMessage: err.Error(), + }, + } + } else { + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse{ + FileDescriptorResponse: &v1reflectionpb.FileDescriptorResponse{FileDescriptorProto: b}, + } + } + case *v1reflectionpb.ServerReflectionRequest_FileContainingExtension: + typeName := req.FileContainingExtension.ContainingType + extNum := req.FileContainingExtension.ExtensionNumber + b, err := s.FileDescEncodingContainingExtension(typeName, extNum, sentFileDescriptors) + if err != nil { + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ErrorResponse{ + ErrorResponse: &v1reflectionpb.ErrorResponse{ + ErrorCode: int32(codes.NotFound), + ErrorMessage: err.Error(), + }, + } + } else { + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse{ + FileDescriptorResponse: &v1reflectionpb.FileDescriptorResponse{FileDescriptorProto: b}, + } + } + case *v1reflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType: + extNums, err := s.AllExtensionNumbersForTypeName(req.AllExtensionNumbersOfType) + if err != nil { + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ErrorResponse{ + ErrorResponse: &v1reflectionpb.ErrorResponse{ + ErrorCode: int32(codes.NotFound), + ErrorMessage: err.Error(), + }, + } + } else { + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_AllExtensionNumbersResponse{ + AllExtensionNumbersResponse: &v1reflectionpb.ExtensionNumberResponse{ + BaseTypeName: req.AllExtensionNumbersOfType, + ExtensionNumber: extNums, + }, + } + } + case *v1reflectionpb.ServerReflectionRequest_ListServices: + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ListServicesResponse{ + ListServicesResponse: &v1reflectionpb.ListServiceResponse{ + Service: s.ListServices(), + }, + } + default: + return status.Errorf(codes.InvalidArgument, "invalid MessageRequest: %v", in.MessageRequest) + } + + if err := stream.Send(out); err != nil { + return err + } + } +} + +// V1ToV1AlphaResponse converts a v1 ServerReflectionResponse to a v1alpha. +func V1ToV1AlphaResponse(v1 *v1reflectionpb.ServerReflectionResponse) *v1alphareflectionpb.ServerReflectionResponse { + var v1alpha v1alphareflectionpb.ServerReflectionResponse + v1alpha.ValidHost = v1.ValidHost + if v1.OriginalRequest != nil { + v1alpha.OriginalRequest = V1ToV1AlphaRequest(v1.OriginalRequest) + } + switch mr := v1.MessageResponse.(type) { + case *v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse: + if mr != nil { + v1alpha.MessageResponse = &v1alphareflectionpb.ServerReflectionResponse_FileDescriptorResponse{ + FileDescriptorResponse: &v1alphareflectionpb.FileDescriptorResponse{ + FileDescriptorProto: mr.FileDescriptorResponse.GetFileDescriptorProto(), + }, + } + } + case *v1reflectionpb.ServerReflectionResponse_AllExtensionNumbersResponse: + if mr != nil { + v1alpha.MessageResponse = &v1alphareflectionpb.ServerReflectionResponse_AllExtensionNumbersResponse{ + AllExtensionNumbersResponse: &v1alphareflectionpb.ExtensionNumberResponse{ + BaseTypeName: mr.AllExtensionNumbersResponse.GetBaseTypeName(), + ExtensionNumber: mr.AllExtensionNumbersResponse.GetExtensionNumber(), + }, + } + } + case *v1reflectionpb.ServerReflectionResponse_ListServicesResponse: + if mr != nil { + svcs := make([]*v1alphareflectionpb.ServiceResponse, len(mr.ListServicesResponse.GetService())) + for i, svc := range mr.ListServicesResponse.GetService() { + svcs[i] = &v1alphareflectionpb.ServiceResponse{ + Name: svc.GetName(), + } + } + v1alpha.MessageResponse = &v1alphareflectionpb.ServerReflectionResponse_ListServicesResponse{ + ListServicesResponse: &v1alphareflectionpb.ListServiceResponse{ + Service: svcs, + }, + } + } + case *v1reflectionpb.ServerReflectionResponse_ErrorResponse: + if mr != nil { + v1alpha.MessageResponse = &v1alphareflectionpb.ServerReflectionResponse_ErrorResponse{ + ErrorResponse: &v1alphareflectionpb.ErrorResponse{ + ErrorCode: mr.ErrorResponse.GetErrorCode(), + ErrorMessage: mr.ErrorResponse.GetErrorMessage(), + }, + } + } + default: + // no value set + } + return &v1alpha +} + +// V1AlphaToV1Request converts a v1alpha ServerReflectionRequest to a v1. +func V1AlphaToV1Request(v1alpha *v1alphareflectionpb.ServerReflectionRequest) *v1reflectionpb.ServerReflectionRequest { + var v1 v1reflectionpb.ServerReflectionRequest + v1.Host = v1alpha.Host + switch mr := v1alpha.MessageRequest.(type) { + case *v1alphareflectionpb.ServerReflectionRequest_FileByFilename: + v1.MessageRequest = &v1reflectionpb.ServerReflectionRequest_FileByFilename{ + FileByFilename: mr.FileByFilename, + } + case *v1alphareflectionpb.ServerReflectionRequest_FileContainingSymbol: + v1.MessageRequest = &v1reflectionpb.ServerReflectionRequest_FileContainingSymbol{ + FileContainingSymbol: mr.FileContainingSymbol, + } + case *v1alphareflectionpb.ServerReflectionRequest_FileContainingExtension: + if mr.FileContainingExtension != nil { + v1.MessageRequest = &v1reflectionpb.ServerReflectionRequest_FileContainingExtension{ + FileContainingExtension: &v1reflectionpb.ExtensionRequest{ + ContainingType: mr.FileContainingExtension.GetContainingType(), + ExtensionNumber: mr.FileContainingExtension.GetExtensionNumber(), + }, + } + } + case *v1alphareflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType: + v1.MessageRequest = &v1reflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType{ + AllExtensionNumbersOfType: mr.AllExtensionNumbersOfType, + } + case *v1alphareflectionpb.ServerReflectionRequest_ListServices: + v1.MessageRequest = &v1reflectionpb.ServerReflectionRequest_ListServices{ + ListServices: mr.ListServices, + } + default: + // no value set + } + return &v1 +} + +// V1ToV1AlphaRequest converts a v1 ServerReflectionRequest to a v1alpha. +func V1ToV1AlphaRequest(v1 *v1reflectionpb.ServerReflectionRequest) *v1alphareflectionpb.ServerReflectionRequest { + var v1alpha v1alphareflectionpb.ServerReflectionRequest + v1alpha.Host = v1.Host + switch mr := v1.MessageRequest.(type) { + case *v1reflectionpb.ServerReflectionRequest_FileByFilename: + if mr != nil { + v1alpha.MessageRequest = &v1alphareflectionpb.ServerReflectionRequest_FileByFilename{ + FileByFilename: mr.FileByFilename, + } + } + case *v1reflectionpb.ServerReflectionRequest_FileContainingSymbol: + if mr != nil { + v1alpha.MessageRequest = &v1alphareflectionpb.ServerReflectionRequest_FileContainingSymbol{ + FileContainingSymbol: mr.FileContainingSymbol, + } + } + case *v1reflectionpb.ServerReflectionRequest_FileContainingExtension: + if mr != nil { + v1alpha.MessageRequest = &v1alphareflectionpb.ServerReflectionRequest_FileContainingExtension{ + FileContainingExtension: &v1alphareflectionpb.ExtensionRequest{ + ContainingType: mr.FileContainingExtension.GetContainingType(), + ExtensionNumber: mr.FileContainingExtension.GetExtensionNumber(), + }, + } + } + case *v1reflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType: + if mr != nil { + v1alpha.MessageRequest = &v1alphareflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType{ + AllExtensionNumbersOfType: mr.AllExtensionNumbersOfType, + } + } + case *v1reflectionpb.ServerReflectionRequest_ListServices: + if mr != nil { + v1alpha.MessageRequest = &v1alphareflectionpb.ServerReflectionRequest_ListServices{ + ListServices: mr.ListServices, + } + } + default: + // no value set + } + return &v1alpha +} + +// V1AlphaToV1Response converts a v1alpha ServerReflectionResponse to a v1. +func V1AlphaToV1Response(v1alpha *v1alphareflectionpb.ServerReflectionResponse) *v1reflectionpb.ServerReflectionResponse { + var v1 v1reflectionpb.ServerReflectionResponse + v1.ValidHost = v1alpha.ValidHost + if v1alpha.OriginalRequest != nil { + v1.OriginalRequest = V1AlphaToV1Request(v1alpha.OriginalRequest) + } + switch mr := v1alpha.MessageResponse.(type) { + case *v1alphareflectionpb.ServerReflectionResponse_FileDescriptorResponse: + if mr != nil { + v1.MessageResponse = &v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse{ + FileDescriptorResponse: &v1reflectionpb.FileDescriptorResponse{ + FileDescriptorProto: mr.FileDescriptorResponse.GetFileDescriptorProto(), + }, + } + } + case *v1alphareflectionpb.ServerReflectionResponse_AllExtensionNumbersResponse: + if mr != nil { + v1.MessageResponse = &v1reflectionpb.ServerReflectionResponse_AllExtensionNumbersResponse{ + AllExtensionNumbersResponse: &v1reflectionpb.ExtensionNumberResponse{ + BaseTypeName: mr.AllExtensionNumbersResponse.GetBaseTypeName(), + ExtensionNumber: mr.AllExtensionNumbersResponse.GetExtensionNumber(), + }, + } + } + case *v1alphareflectionpb.ServerReflectionResponse_ListServicesResponse: + if mr != nil { + svcs := make([]*v1reflectionpb.ServiceResponse, len(mr.ListServicesResponse.GetService())) + for i, svc := range mr.ListServicesResponse.GetService() { + svcs[i] = &v1reflectionpb.ServiceResponse{ + Name: svc.GetName(), + } + } + v1.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ListServicesResponse{ + ListServicesResponse: &v1reflectionpb.ListServiceResponse{ + Service: svcs, + }, + } + } + case *v1alphareflectionpb.ServerReflectionResponse_ErrorResponse: + if mr != nil { + v1.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ErrorResponse{ + ErrorResponse: &v1reflectionpb.ErrorResponse{ + ErrorCode: mr.ErrorResponse.GetErrorCode(), + ErrorMessage: mr.ErrorResponse.GetErrorMessage(), + }, + } + } + default: + // no value set + } + return &v1 +} diff --git a/reflection/serverreflection.go b/reflection/serverreflection.go new file mode 100644 index 0000000..13a94e2 --- /dev/null +++ b/reflection/serverreflection.go @@ -0,0 +1,160 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* +Package reflection implements server reflection service. + +The service implemented is defined in: +https://github.com/grpc/grpc/blob/master/src/proto/grpc/reflection/v1alpha/reflection.proto. + +To register server reflection on a gRPC server: + + import "google.golang.org/grpc/reflection" + + s := grpc.NewServer() + pb.RegisterYourOwnServer(s, &server{}) + + // Register reflection service on gRPC server. + reflection.Register(s) + + s.Serve(lis) +*/ +package reflection // import "google.golang.org/grpc/reflection" + +import ( + "google.golang.org/grpc" + "google.golang.org/grpc/reflection/internal" + "google.golang.org/protobuf/reflect/protodesc" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + + v1reflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1" + v1alphareflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" +) + +// GRPCServer is the interface provided by a gRPC server. It is implemented by +// *grpc.Server, but could also be implemented by other concrete types. It acts +// as a registry, for accumulating the services exposed by the server. +type GRPCServer interface { + grpc.ServiceRegistrar + ServiceInfoProvider +} + +var _ GRPCServer = (*grpc.Server)(nil) + +// Register registers the server reflection service on the given gRPC server. +// Both the v1 and v1alpha versions are registered. +func Register(s GRPCServer) { + svr := NewServerV1(ServerOptions{Services: s}) + v1alphareflectiongrpc.RegisterServerReflectionServer(s, asV1Alpha(svr)) + v1reflectiongrpc.RegisterServerReflectionServer(s, svr) +} + +// RegisterV1 registers only the v1 version of the server reflection service +// on the given gRPC server. Many clients may only support v1alpha so most +// users should use Register instead, at least until clients have upgraded. +func RegisterV1(s GRPCServer) { + svr := NewServerV1(ServerOptions{Services: s}) + v1reflectiongrpc.RegisterServerReflectionServer(s, svr) +} + +// ServiceInfoProvider is an interface used to retrieve metadata about the +// services to expose. +// +// The reflection service is only interested in the service names, but the +// signature is this way so that *grpc.Server implements it. So it is okay +// for a custom implementation to return zero values for the +// grpc.ServiceInfo values in the map. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type ServiceInfoProvider interface { + GetServiceInfo() map[string]grpc.ServiceInfo +} + +// ExtensionResolver is the interface used to query details about extensions. +// This interface is satisfied by protoregistry.GlobalTypes. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type ExtensionResolver interface { + protoregistry.ExtensionTypeResolver + RangeExtensionsByMessage(message protoreflect.FullName, f func(protoreflect.ExtensionType) bool) +} + +// ServerOptions represents the options used to construct a reflection server. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type ServerOptions struct { + // The source of advertised RPC services. If not specified, the reflection + // server will report an empty list when asked to list services. + // + // This value will typically be a *grpc.Server. But the set of advertised + // services can be customized by wrapping a *grpc.Server or using an + // alternate implementation that returns a custom set of service names. + Services ServiceInfoProvider + // Optional resolver used to load descriptors. If not specified, + // protoregistry.GlobalFiles will be used. + DescriptorResolver protodesc.Resolver + // Optional resolver used to query for known extensions. If not specified, + // protoregistry.GlobalTypes will be used. + ExtensionResolver ExtensionResolver +} + +// NewServer returns a reflection server implementation using the given options. +// This can be used to customize behavior of the reflection service. Most usages +// should prefer to use Register instead. For backwards compatibility reasons, +// this returns the v1alpha version of the reflection server. For a v1 version +// of the reflection server, see NewServerV1. +// +// # Experimental +// +// Notice: This function is EXPERIMENTAL and may be changed or removed in a +// later release. +func NewServer(opts ServerOptions) v1alphareflectiongrpc.ServerReflectionServer { + return asV1Alpha(NewServerV1(opts)) +} + +// NewServerV1 returns a reflection server implementation using the given options. +// This can be used to customize behavior of the reflection service. Most usages +// should prefer to use Register instead. +// +// # Experimental +// +// Notice: This function is EXPERIMENTAL and may be changed or removed in a +// later release. +func NewServerV1(opts ServerOptions) v1reflectiongrpc.ServerReflectionServer { + if opts.DescriptorResolver == nil { + opts.DescriptorResolver = protoregistry.GlobalFiles + } + if opts.ExtensionResolver == nil { + opts.ExtensionResolver = protoregistry.GlobalTypes + } + return &internal.ServerReflectionServer{ + S: opts.Services, + DescResolver: opts.DescriptorResolver, + ExtResolver: opts.ExtensionResolver, + } +} diff --git a/reflection/test/serverreflection_test.go b/reflection/test/serverreflection_test.go new file mode 100644 index 0000000..65030ba --- /dev/null +++ b/reflection/test/serverreflection_test.go @@ -0,0 +1,856 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test_test + +import ( + "context" + "fmt" + "net" + "reflect" + "sort" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/reflection" + "google.golang.org/grpc/reflection/internal" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protodesc" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + "google.golang.org/protobuf/types/descriptorpb" + "google.golang.org/protobuf/types/dynamicpb" + + v1reflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1" + v1reflectionpb "google.golang.org/grpc/reflection/grpc_reflection_v1" + v1alphareflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" + pb "google.golang.org/grpc/reflection/grpc_testing" + pbv3 "google.golang.org/grpc/testdata/grpc_testing_not_regenerated" +) + +var ( + s = reflection.NewServerV1(reflection.ServerOptions{}).(*internal.ServerReflectionServer) + // fileDescriptor of each test proto file. + fdProto2Ext *descriptorpb.FileDescriptorProto + fdProto2Ext2 *descriptorpb.FileDescriptorProto + fdDynamic *descriptorpb.FileDescriptorProto + // reflection descriptors. + fdDynamicFile protoreflect.FileDescriptor + // fileDescriptor marshalled. + fdTestByte []byte + fdTestv3Byte []byte + fdProto2Byte []byte + fdProto2ExtByte []byte + fdProto2Ext2Byte []byte + fdDynamicByte []byte +) + +const defaultTestTimeout = 10 * time.Second + +type x struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, x{}) +} + +func loadFileDesc(filename string) (*descriptorpb.FileDescriptorProto, []byte) { + fd, err := protoregistry.GlobalFiles.FindFileByPath(filename) + if err != nil { + panic(err) + } + fdProto := protodesc.ToFileDescriptorProto(fd) + b, err := proto.Marshal(fdProto) + if err != nil { + panic(fmt.Sprintf("failed to marshal fd: %v", err)) + } + return fdProto, b +} + +func loadFileDescDynamic(b []byte) (*descriptorpb.FileDescriptorProto, protoreflect.FileDescriptor, []byte) { + m := new(descriptorpb.FileDescriptorProto) + if err := proto.Unmarshal(b, m); err != nil { + panic("failed to unmarshal dynamic proto raw descriptor") + } + + fd, err := protodesc.NewFile(m, nil) + if err != nil { + panic(err) + } + + err = protoregistry.GlobalFiles.RegisterFile(fd) + if err != nil { + panic(err) + } + + for i := 0; i < fd.Messages().Len(); i++ { + m := fd.Messages().Get(i) + if err := protoregistry.GlobalTypes.RegisterMessage(dynamicpb.NewMessageType(m)); err != nil { + panic(err) + } + } + + return m, fd, b +} + +func init() { + _, fdTestByte = loadFileDesc("reflection/grpc_testing/test.proto") + _, fdTestv3Byte = loadFileDesc("testv3.proto") + _, fdProto2Byte = loadFileDesc("reflection/grpc_testing/proto2.proto") + fdProto2Ext, fdProto2ExtByte = loadFileDesc("reflection/grpc_testing/proto2_ext.proto") + fdProto2Ext2, fdProto2Ext2Byte = loadFileDesc("reflection/grpc_testing/proto2_ext2.proto") + fdDynamic, fdDynamicFile, fdDynamicByte = loadFileDescDynamic(pbv3.FileDynamicProtoRawDesc) +} + +func (x) TestFileDescContainingExtension(t *testing.T) { + for _, test := range []struct { + st string + extNum int32 + want *descriptorpb.FileDescriptorProto + }{ + {"grpc.testing.ToBeExtended", 13, fdProto2Ext}, + {"grpc.testing.ToBeExtended", 17, fdProto2Ext}, + {"grpc.testing.ToBeExtended", 19, fdProto2Ext}, + {"grpc.testing.ToBeExtended", 23, fdProto2Ext2}, + {"grpc.testing.ToBeExtended", 29, fdProto2Ext2}, + } { + fd, err := s.FileDescEncodingContainingExtension(test.st, test.extNum, map[string]bool{}) + if err != nil { + t.Errorf("fileDescContainingExtension(%q) return error: %v", test.st, err) + continue + } + var actualFd descriptorpb.FileDescriptorProto + if err := proto.Unmarshal(fd[0], &actualFd); err != nil { + t.Errorf("fileDescContainingExtension(%q) return invalid bytes: %v", test.st, err) + continue + } + if !proto.Equal(&actualFd, test.want) { + t.Errorf("fileDescContainingExtension(%q) returned %q, but wanted %q", test.st, &actualFd, test.want) + } + } +} + +// intArray is used to sort []int32 +type intArray []int32 + +func (s intArray) Len() int { return len(s) } +func (s intArray) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s intArray) Less(i, j int) bool { return s[i] < s[j] } + +func (x) TestAllExtensionNumbersForTypeName(t *testing.T) { + for _, test := range []struct { + st string + want []int32 + }{ + {"grpc.testing.ToBeExtended", []int32{13, 17, 19, 23, 29}}, + } { + r, err := s.AllExtensionNumbersForTypeName(test.st) + sort.Sort(intArray(r)) + if err != nil || !reflect.DeepEqual(r, test.want) { + t.Errorf("allExtensionNumbersForType(%q) = %v, %v, want %v, ", test.st, r, err, test.want) + } + } +} + +func (x) TestFileDescWithDependencies(t *testing.T) { + depFile, err := protodesc.NewFile( + &descriptorpb.FileDescriptorProto{ + Name: proto.String("dep.proto"), + }, nil, + ) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + deps := &protoregistry.Files{} + if err := deps.RegisterFile(depFile); err != nil { + t.Fatalf("unexpected error: %s", err) + } + + rootFileProto := &descriptorpb.FileDescriptorProto{ + Name: proto.String("root.proto"), + Dependency: []string{ + "google/protobuf/descriptor.proto", + "reflection/grpc_testing/proto2_ext2.proto", + "dep.proto", + }, + } + + // dep.proto is in deps; the other imports come from protoregistry.GlobalFiles + resolver := &combinedResolver{first: protoregistry.GlobalFiles, second: deps} + rootFile, err := protodesc.NewFile(rootFileProto, resolver) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + // Create a file hierarchy that contains a placeholder for dep.proto + placeholderDep := placeholderFile{depFile} + placeholderDeps := &protoregistry.Files{} + if err := placeholderDeps.RegisterFile(placeholderDep); err != nil { + t.Fatalf("unexpected error: %s", err) + } + resolver = &combinedResolver{first: protoregistry.GlobalFiles, second: placeholderDeps} + + rootFileHasPlaceholderDep, err := protodesc.NewFile(rootFileProto, resolver) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + rootFileIsPlaceholder := placeholderFile{rootFile} + + // Full transitive dependency graph of root.proto includes five files: + // - root.proto + // - google/protobuf/descriptor.proto + // - reflection/grpc_testing/proto2_ext2.proto + // - reflection/grpc_testing/proto2.proto + // - dep.proto + + for _, test := range []struct { + name string + sent []string + root protoreflect.FileDescriptor + expect []string + }{ + { + name: "send_all", + root: rootFile, + // expect full transitive closure + expect: []string{ + "root.proto", + "google/protobuf/descriptor.proto", + "reflection/grpc_testing/proto2_ext2.proto", + "reflection/grpc_testing/proto2.proto", + "dep.proto", + }, + }, + { + name: "already_sent", + sent: []string{ + "root.proto", + "google/protobuf/descriptor.proto", + "reflection/grpc_testing/proto2_ext2.proto", + "reflection/grpc_testing/proto2.proto", + "dep.proto", + }, + root: rootFile, + // expect only the root to be re-sent + expect: []string{"root.proto"}, + }, + { + name: "some_already_sent", + sent: []string{ + "reflection/grpc_testing/proto2_ext2.proto", + "reflection/grpc_testing/proto2.proto", + }, + root: rootFile, + expect: []string{ + "root.proto", + "google/protobuf/descriptor.proto", + "dep.proto", + }, + }, + { + name: "root_is_placeholder", + root: rootFileIsPlaceholder, + // expect error, no files + }, + { + name: "placeholder_skipped", + root: rootFileHasPlaceholderDep, + // dep.proto is a placeholder so is skipped + expect: []string{ + "root.proto", + "google/protobuf/descriptor.proto", + "reflection/grpc_testing/proto2_ext2.proto", + "reflection/grpc_testing/proto2.proto", + }, + }, + { + name: "placeholder_skipped_and_some_sent", + sent: []string{ + "reflection/grpc_testing/proto2_ext2.proto", + "reflection/grpc_testing/proto2.proto", + }, + root: rootFileHasPlaceholderDep, + expect: []string{ + "root.proto", + "google/protobuf/descriptor.proto", + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + s := reflection.NewServerV1(reflection.ServerOptions{}).(*internal.ServerReflectionServer) + + sent := map[string]bool{} + for _, path := range test.sent { + sent[path] = true + } + + descriptors, err := s.FileDescWithDependencies(test.root, sent) + if len(test.expect) == 0 { + // if we're not expecting any files then we're expecting an error + if err == nil { + t.Fatalf("expecting an error; instead got %d files", len(descriptors)) + } + return + } + + checkDescriptorResults(t, descriptors, test.expect) + }) + } +} + +func checkDescriptorResults(t *testing.T, descriptors [][]byte, expect []string) { + t.Helper() + if len(descriptors) != len(expect) { + t.Errorf("expected result to contain %d descriptor(s); instead got %d", len(expect), len(descriptors)) + } + names := map[string]struct{}{} + for i, desc := range descriptors { + var descProto descriptorpb.FileDescriptorProto + if err := proto.Unmarshal(desc, &descProto); err != nil { + t.Fatalf("could not unmarshal descriptor result #%d", i+1) + } + names[descProto.GetName()] = struct{}{} + } + actual := make([]string, 0, len(names)) + for name := range names { + actual = append(actual, name) + } + sort.Strings(actual) + sort.Strings(expect) + if !reflect.DeepEqual(actual, expect) { + t.Fatalf("expected file descriptors for %v; instead got %v", expect, actual) + } +} + +type placeholderFile struct { + protoreflect.FileDescriptor +} + +func (placeholderFile) IsPlaceholder() bool { + return true +} + +type combinedResolver struct { + first, second protodesc.Resolver +} + +func (r *combinedResolver) FindFileByPath(path string) (protoreflect.FileDescriptor, error) { + file, err := r.first.FindFileByPath(path) + if err == nil { + return file, nil + } + return r.second.FindFileByPath(path) +} + +func (r *combinedResolver) FindDescriptorByName(name protoreflect.FullName) (protoreflect.Descriptor, error) { + desc, err := r.first.FindDescriptorByName(name) + if err == nil { + return desc, nil + } + return r.second.FindDescriptorByName(name) +} + +// Do end2end tests. + +type server struct { + pb.UnimplementedSearchServiceServer +} + +func (s *server) Search(context.Context, *pb.SearchRequest) (*pb.SearchResponse, error) { + return &pb.SearchResponse{}, nil +} + +func (s *server) StreamingSearch(pb.SearchService_StreamingSearchServer) error { + return nil +} + +type serverV3 struct{} + +func (s *serverV3) Search(context.Context, *pbv3.SearchRequestV3) (*pbv3.SearchResponseV3, error) { + return &pbv3.SearchResponseV3{}, nil +} + +func (s *serverV3) StreamingSearch(pbv3.SearchServiceV3_StreamingSearchServer) error { + return nil +} + +func (x) TestReflectionEnd2end(t *testing.T) { + // Start server. + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + s := grpc.NewServer() + pb.RegisterSearchServiceServer(s, &server{}) + pbv3.RegisterSearchServiceV3Server(s, &serverV3{}) + + registerDynamicProto(s, fdDynamic, fdDynamicFile) + + // Register reflection service on s. + reflection.Register(s) + go s.Serve(lis) + t.Cleanup(s.Stop) + + // Create client. + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("cannot connect to server: %v", err) + } + defer conn.Close() + + clientV1 := v1reflectiongrpc.NewServerReflectionClient(conn) + clientV1Alpha := v1alphareflectiongrpc.NewServerReflectionClient(conn) + testCases := []struct { + name string + client v1reflectiongrpc.ServerReflectionClient + }{ + { + name: "v1", + client: clientV1, + }, + { + name: "v1alpha", + client: v1AlphaClientAdapter{stub: clientV1Alpha}, + }, + } + for _, testCase := range testCases { + c := testCase.client + t.Run(testCase.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := c.ServerReflectionInfo(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("cannot get ServerReflectionInfo: %v", err) + } + + testFileByFilenameTransitiveClosure(t, stream, true) + testFileByFilenameTransitiveClosure(t, stream, false) + testFileByFilename(t, stream) + testFileByFilenameError(t, stream) + testFileContainingSymbol(t, stream) + testFileContainingSymbolError(t, stream) + testFileContainingExtension(t, stream) + testFileContainingExtensionError(t, stream) + testAllExtensionNumbersOfType(t, stream) + testAllExtensionNumbersOfTypeError(t, stream) + testListServices(t, stream) + }) + } +} + +func testFileByFilenameTransitiveClosure(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient, expectClosure bool) { + filename := "reflection/grpc_testing/proto2_ext2.proto" + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileByFilename{ + FileByFilename: filename, + }, + }); err != nil { + t.Fatalf("failed to send request: %v", err) + } + r, err := stream.Recv() + if err != nil { + // io.EOF is not ok. + t.Fatalf("failed to recv response: %v", err) + } + switch r.MessageResponse.(type) { + case *v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse: + if !reflect.DeepEqual(r.GetFileDescriptorResponse().FileDescriptorProto[0], fdProto2Ext2Byte) { + t.Errorf("FileByFilename(%v)\nreceived: %q,\nwant: %q", filename, r.GetFileDescriptorResponse().FileDescriptorProto[0], fdProto2Ext2Byte) + } + if expectClosure { + if len(r.GetFileDescriptorResponse().FileDescriptorProto) != 2 { + t.Errorf("FileByFilename(%v) returned %v file descriptors, expected 2", filename, len(r.GetFileDescriptorResponse().FileDescriptorProto)) + } else if !reflect.DeepEqual(r.GetFileDescriptorResponse().FileDescriptorProto[1], fdProto2Byte) { + t.Errorf("FileByFilename(%v)\nreceived: %q,\nwant: %q", filename, r.GetFileDescriptorResponse().FileDescriptorProto[1], fdProto2Byte) + } + } else if len(r.GetFileDescriptorResponse().FileDescriptorProto) != 1 { + t.Errorf("FileByFilename(%v) returned %v file descriptors, expected 1", filename, len(r.GetFileDescriptorResponse().FileDescriptorProto)) + } + default: + t.Errorf("FileByFilename(%v) = %v, want type ", filename, r.MessageResponse) + } +} + +func testFileByFilename(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { + for _, test := range []struct { + filename string + want []byte + }{ + {"reflection/grpc_testing/test.proto", fdTestByte}, + {"reflection/grpc_testing/proto2.proto", fdProto2Byte}, + {"reflection/grpc_testing/proto2_ext.proto", fdProto2ExtByte}, + {"dynamic.proto", fdDynamicByte}, + } { + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileByFilename{ + FileByFilename: test.filename, + }, + }); err != nil { + t.Fatalf("failed to send request: %v", err) + } + r, err := stream.Recv() + if err != nil { + // io.EOF is not ok. + t.Fatalf("failed to recv response: %v", err) + } + + switch r.MessageResponse.(type) { + case *v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse: + if !reflect.DeepEqual(r.GetFileDescriptorResponse().FileDescriptorProto[0], test.want) { + t.Errorf("FileByFilename(%v)\nreceived: %q,\nwant: %q", test.filename, r.GetFileDescriptorResponse().FileDescriptorProto[0], test.want) + } + default: + t.Errorf("FileByFilename(%v) = %v, want type ", test.filename, r.MessageResponse) + } + } +} + +func testFileByFilenameError(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { + for _, test := range []string{ + "test.poto", + "proo2.proto", + "proto2_et.proto", + } { + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileByFilename{ + FileByFilename: test, + }, + }); err != nil { + t.Fatalf("failed to send request: %v", err) + } + r, err := stream.Recv() + if err != nil { + // io.EOF is not ok. + t.Fatalf("failed to recv response: %v", err) + } + + switch r.MessageResponse.(type) { + case *v1reflectionpb.ServerReflectionResponse_ErrorResponse: + default: + t.Errorf("FileByFilename(%v) = %v, want type ", test, r.MessageResponse) + } + } +} + +func testFileContainingSymbol(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { + for _, test := range []struct { + symbol string + want []byte + }{ + {"grpc.testing.SearchService", fdTestByte}, + {"grpc.testing.SearchService.Search", fdTestByte}, + {"grpc.testing.SearchService.StreamingSearch", fdTestByte}, + {"grpc.testing.SearchResponse", fdTestByte}, + {"grpc.testing.ToBeExtended", fdProto2Byte}, + // Test support package v3. + {"grpc.testingv3.SearchServiceV3", fdTestv3Byte}, + {"grpc.testingv3.SearchServiceV3.Search", fdTestv3Byte}, + {"grpc.testingv3.SearchServiceV3.StreamingSearch", fdTestv3Byte}, + {"grpc.testingv3.SearchResponseV3", fdTestv3Byte}, + // search for field, oneof, enum, and enum value symbols, too + {"grpc.testingv3.SearchResponseV3.Result.snippets", fdTestv3Byte}, + {"grpc.testingv3.SearchResponseV3.Result.Value.val", fdTestv3Byte}, + {"grpc.testingv3.SearchResponseV3.Result.Value.str", fdTestv3Byte}, + {"grpc.testingv3.SearchResponseV3.State", fdTestv3Byte}, + {"grpc.testingv3.SearchResponseV3.FRESH", fdTestv3Byte}, + // Test dynamic symbols + {"grpc.testing.DynamicService", fdDynamicByte}, + {"grpc.testing.DynamicReq", fdDynamicByte}, + {"grpc.testing.DynamicRes", fdDynamicByte}, + } { + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileContainingSymbol{ + FileContainingSymbol: test.symbol, + }, + }); err != nil { + t.Fatalf("failed to send request: %v", err) + } + r, err := stream.Recv() + if err != nil { + // io.EOF is not ok. + t.Fatalf("failed to recv response: %v", err) + } + + switch r.MessageResponse.(type) { + case *v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse: + if !reflect.DeepEqual(r.GetFileDescriptorResponse().FileDescriptorProto[0], test.want) { + t.Errorf("FileContainingSymbol(%v)\nreceived: %q,\nwant: %q", test.symbol, r.GetFileDescriptorResponse().FileDescriptorProto[0], test.want) + } + default: + t.Errorf("FileContainingSymbol(%v) = %v, want type ", test.symbol, r.MessageResponse) + } + } +} + +func testFileContainingSymbolError(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { + for _, test := range []string{ + "grpc.testing.SearchService_", + "grpc.testing.SearchService.SearchE", + "grpc.testing_.SearchResponse", + "gpc.testing.ToBeExtended", + } { + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileContainingSymbol{ + FileContainingSymbol: test, + }, + }); err != nil { + t.Fatalf("failed to send request: %v", err) + } + r, err := stream.Recv() + if err != nil { + // io.EOF is not ok. + t.Fatalf("failed to recv response: %v", err) + } + + switch r.MessageResponse.(type) { + case *v1reflectionpb.ServerReflectionResponse_ErrorResponse: + default: + t.Errorf("FileContainingSymbol(%v) = %v, want type ", test, r.MessageResponse) + } + } +} + +func testFileContainingExtension(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { + for _, test := range []struct { + typeName string + extNum int32 + want []byte + }{ + {"grpc.testing.ToBeExtended", 13, fdProto2ExtByte}, + {"grpc.testing.ToBeExtended", 17, fdProto2ExtByte}, + {"grpc.testing.ToBeExtended", 19, fdProto2ExtByte}, + {"grpc.testing.ToBeExtended", 23, fdProto2Ext2Byte}, + {"grpc.testing.ToBeExtended", 29, fdProto2Ext2Byte}, + } { + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileContainingExtension{ + FileContainingExtension: &v1reflectionpb.ExtensionRequest{ + ContainingType: test.typeName, + ExtensionNumber: test.extNum, + }, + }, + }); err != nil { + t.Fatalf("failed to send request: %v", err) + } + r, err := stream.Recv() + if err != nil { + // io.EOF is not ok. + t.Fatalf("failed to recv response: %v", err) + } + + switch r.MessageResponse.(type) { + case *v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse: + if !reflect.DeepEqual(r.GetFileDescriptorResponse().FileDescriptorProto[0], test.want) { + t.Errorf("FileContainingExtension(%v, %v)\nreceived: %q,\nwant: %q", test.typeName, test.extNum, r.GetFileDescriptorResponse().FileDescriptorProto[0], test.want) + } + default: + t.Errorf("FileContainingExtension(%v, %v) = %v, want type ", test.typeName, test.extNum, r.MessageResponse) + } + } +} + +func testFileContainingExtensionError(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { + for _, test := range []struct { + typeName string + extNum int32 + }{ + {"grpc.testing.ToBExtended", 17}, + {"grpc.testing.ToBeExtended", 15}, + } { + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileContainingExtension{ + FileContainingExtension: &v1reflectionpb.ExtensionRequest{ + ContainingType: test.typeName, + ExtensionNumber: test.extNum, + }, + }, + }); err != nil { + t.Fatalf("failed to send request: %v", err) + } + r, err := stream.Recv() + if err != nil { + // io.EOF is not ok. + t.Fatalf("failed to recv response: %v", err) + } + + switch r.MessageResponse.(type) { + case *v1reflectionpb.ServerReflectionResponse_ErrorResponse: + default: + t.Errorf("FileContainingExtension(%v, %v) = %v, want type ", test.typeName, test.extNum, r.MessageResponse) + } + } +} + +func testAllExtensionNumbersOfType(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { + for _, test := range []struct { + typeName string + want []int32 + }{ + {"grpc.testing.ToBeExtended", []int32{13, 17, 19, 23, 29}}, + {"grpc.testing.DynamicReq", nil}, + } { + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType{ + AllExtensionNumbersOfType: test.typeName, + }, + }); err != nil { + t.Fatalf("failed to send request: %v", err) + } + r, err := stream.Recv() + if err != nil { + // io.EOF is not ok. + t.Fatalf("failed to recv response: %v", err) + } + + switch r.MessageResponse.(type) { + case *v1reflectionpb.ServerReflectionResponse_AllExtensionNumbersResponse: + extNum := r.GetAllExtensionNumbersResponse().ExtensionNumber + sort.Sort(intArray(extNum)) + if r.GetAllExtensionNumbersResponse().BaseTypeName != test.typeName || + !reflect.DeepEqual(extNum, test.want) { + t.Errorf("AllExtensionNumbersOfType(%v)\nreceived: %v,\nwant: {%q %v}", r.GetAllExtensionNumbersResponse(), test.typeName, test.typeName, test.want) + } + default: + t.Errorf("AllExtensionNumbersOfType(%v) = %v, want type ", test.typeName, r.MessageResponse) + } + } +} + +func testAllExtensionNumbersOfTypeError(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { + for _, test := range []string{ + "grpc.testing.ToBeExtendedE", + } { + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType{ + AllExtensionNumbersOfType: test, + }, + }); err != nil { + t.Fatalf("failed to send request: %v", err) + } + r, err := stream.Recv() + if err != nil { + // io.EOF is not ok. + t.Fatalf("failed to recv response: %v", err) + } + + switch r.MessageResponse.(type) { + case *v1reflectionpb.ServerReflectionResponse_ErrorResponse: + default: + t.Errorf("AllExtensionNumbersOfType(%v) = %v, want type ", test, r.MessageResponse) + } + } +} + +func testListServices(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_ListServices{}, + }); err != nil { + t.Fatalf("failed to send request: %v", err) + } + r, err := stream.Recv() + if err != nil { + // io.EOF is not ok. + t.Fatalf("failed to recv response: %v", err) + } + + switch r.MessageResponse.(type) { + case *v1reflectionpb.ServerReflectionResponse_ListServicesResponse: + services := r.GetListServicesResponse().Service + want := []string{ + "grpc.testingv3.SearchServiceV3", + "grpc.testing.SearchService", + "grpc.reflection.v1.ServerReflection", + "grpc.reflection.v1alpha.ServerReflection", + "grpc.testing.DynamicService", + } + // Compare service names in response with want. + if len(services) != len(want) { + t.Errorf("= %v, want service names: %v", services, want) + } + m := make(map[string]int) + for _, e := range services { + m[e.Name]++ + } + for _, e := range want { + if m[e] > 0 { + m[e]-- + continue + } + t.Errorf("ListService\nreceived: %v,\nwant: %q", services, want) + } + default: + t.Errorf("ListServices = %v, want type ", r.MessageResponse) + } +} + +func registerDynamicProto(srv *grpc.Server, fdp *descriptorpb.FileDescriptorProto, fd protoreflect.FileDescriptor) { + type emptyInterface any + + for i := 0; i < fd.Services().Len(); i++ { + s := fd.Services().Get(i) + + sd := &grpc.ServiceDesc{ + ServiceName: string(s.FullName()), + HandlerType: (*emptyInterface)(nil), + Metadata: fdp.GetName(), + } + + for j := 0; j < s.Methods().Len(); j++ { + m := s.Methods().Get(j) + sd.Methods = append(sd.Methods, grpc.MethodDesc{ + MethodName: string(m.Name()), + }) + } + + srv.RegisterService(sd, struct{}{}) + } +} + +type v1AlphaClientAdapter struct { + stub v1alphareflectiongrpc.ServerReflectionClient +} + +func (v v1AlphaClientAdapter) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient, error) { + stream, err := v.stub.ServerReflectionInfo(ctx, opts...) + if err != nil { + return nil, err + } + return v1AlphaClientStreamAdapter{stream}, nil +} + +type v1AlphaClientStreamAdapter struct { + v1alphareflectiongrpc.ServerReflection_ServerReflectionInfoClient +} + +func (s v1AlphaClientStreamAdapter) Send(request *v1reflectionpb.ServerReflectionRequest) error { + return s.ServerReflection_ServerReflectionInfoClient.Send(internal.V1ToV1AlphaRequest(request)) +} + +func (s v1AlphaClientStreamAdapter) Recv() (*v1reflectionpb.ServerReflectionResponse, error) { + resp, err := s.ServerReflection_ServerReflectionInfoClient.Recv() + if err != nil { + return nil, err + } + return internal.V1AlphaToV1Response(resp), nil +} diff --git a/resolver/dns/dns_resolver.go b/resolver/dns/dns_resolver.go new file mode 100644 index 0000000..ef3d6ed --- /dev/null +++ b/resolver/dns/dns_resolver.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package dns implements a dns resolver to be installed as the default resolver +// in grpc. +package dns + +import ( + "time" + + "google.golang.org/grpc/internal/resolver/dns" + "google.golang.org/grpc/resolver" +) + +// SetResolvingTimeout sets the maximum duration for DNS resolution requests. +// +// This function affects the global timeout used by all channels using the DNS +// name resolver scheme. +// +// It must be called only at application startup, before any gRPC calls are +// made. Modifying this value after initialization is not thread-safe. +// +// The default value is 30 seconds. Setting the timeout too low may result in +// premature timeouts during resolution, while setting it too high may lead to +// unnecessary delays in service discovery. Choose a value appropriate for your +// specific needs and network environment. +func SetResolvingTimeout(timeout time.Duration) { + dns.ResolvingTimeout = timeout +} + +// NewBuilder creates a dnsBuilder which is used to factory DNS resolvers. +// +// Deprecated: import grpc and use resolver.Get("dns") instead. +func NewBuilder() resolver.Builder { + return dns.NewBuilder() +} + +// SetMinResolutionInterval sets the default minimum interval at which DNS +// re-resolutions are allowed. This helps to prevent excessive re-resolution. +// +// It must be called only at application startup, before any gRPC calls are +// made. Modifying this value after initialization is not thread-safe. +func SetMinResolutionInterval(d time.Duration) { + dns.MinResolutionInterval = d +} diff --git a/resolver/manual/manual.go b/resolver/manual/manual.go new file mode 100644 index 0000000..82fcc2e --- /dev/null +++ b/resolver/manual/manual.go @@ -0,0 +1,130 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package manual defines a resolver that can be used to manually send resolved +// addresses to ClientConn. +package manual + +import ( + "sync" + + "google.golang.org/grpc/resolver" +) + +// NewBuilderWithScheme creates a new manual resolver builder with the given +// scheme. Every instance of the manual resolver may only ever be used with a +// single grpc.ClientConn. Otherwise, bad things will happen. +func NewBuilderWithScheme(scheme string) *Resolver { + return &Resolver{ + BuildCallback: func(resolver.Target, resolver.ClientConn, resolver.BuildOptions) {}, + UpdateStateCallback: func(error) {}, + ResolveNowCallback: func(resolver.ResolveNowOptions) {}, + CloseCallback: func() {}, + scheme: scheme, + } +} + +// Resolver is also a resolver builder. +// It's build() function always returns itself. +type Resolver struct { + // BuildCallback is called when the Build method is called. Must not be + // nil. Must not be changed after the resolver may be built. + BuildCallback func(resolver.Target, resolver.ClientConn, resolver.BuildOptions) + // UpdateStateCallback is called when the UpdateState method is called on + // the resolver. The value passed as argument to this callback is the value + // returned by the resolver.ClientConn. Must not be nil. Must not be + // changed after the resolver may be built. + UpdateStateCallback func(err error) + // ResolveNowCallback is called when the ResolveNow method is called on the + // resolver. Must not be nil. Must not be changed after the resolver may + // be built. + ResolveNowCallback func(resolver.ResolveNowOptions) + // CloseCallback is called when the Close method is called. Must not be + // nil. Must not be changed after the resolver may be built. + CloseCallback func() + scheme string + + // Fields actually belong to the resolver. + // Guards access to below fields. + mu sync.Mutex + cc resolver.ClientConn + // Storing the most recent state update makes this resolver resilient to + // restarts, which is possible with channel idleness. + lastSeenState *resolver.State +} + +// InitialState adds initial state to the resolver so that UpdateState doesn't +// need to be explicitly called after Dial. +func (r *Resolver) InitialState(s resolver.State) { + r.lastSeenState = &s +} + +// Build returns itself for Resolver, because it's both a builder and a resolver. +func (r *Resolver) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { + r.mu.Lock() + defer r.mu.Unlock() + // Call BuildCallback after locking to avoid a race when UpdateState or CC + // is called before Build returns. + r.BuildCallback(target, cc, opts) + r.cc = cc + if r.lastSeenState != nil { + err := r.cc.UpdateState(*r.lastSeenState) + go r.UpdateStateCallback(err) + } + return r, nil +} + +// Scheme returns the manual resolver's scheme. +func (r *Resolver) Scheme() string { + return r.scheme +} + +// ResolveNow is a noop for Resolver. +func (r *Resolver) ResolveNow(o resolver.ResolveNowOptions) { + r.ResolveNowCallback(o) +} + +// Close is a noop for Resolver. +func (r *Resolver) Close() { + r.CloseCallback() +} + +// UpdateState calls UpdateState(s) on the channel. If the resolver has not +// been Built before, this instead sets the initial state of the resolver, like +// InitialState. +func (r *Resolver) UpdateState(s resolver.State) { + r.mu.Lock() + defer r.mu.Unlock() + r.lastSeenState = &s + if r.cc == nil { + return + } + err := r.cc.UpdateState(s) + r.UpdateStateCallback(err) +} + +// CC returns r's ClientConn when r was last Built. Panics if the resolver has +// not been Built before. +func (r *Resolver) CC() resolver.ClientConn { + r.mu.Lock() + defer r.mu.Unlock() + if r.cc == nil { + panic("Manual resolver instance has not yet been built.") + } + return r.cc +} diff --git a/resolver/manual/manual_test.go b/resolver/manual/manual_test.go new file mode 100644 index 0000000..5a27dba --- /dev/null +++ b/resolver/manual/manual_test.go @@ -0,0 +1,63 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package manual_test + +import ( + "errors" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" +) + +func TestResolver(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{ + Addresses: []resolver.Address{ + {Addr: "address"}, + }, + }) + + t.Run("cc_panics", func(t *testing.T) { + defer func() { + want := "Manual resolver instance has not yet been built." + if r := recover(); r != want { + t.Errorf("expected panic %q, got %q", want, r) + } + }() + r.CC() + }) + + t.Run("happy_path", func(t *testing.T) { + cc, err := grpc.NewClient("whatever://localhost", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r)) + if err != nil { + t.Errorf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + r.UpdateState(resolver.State{Addresses: []resolver.Address{ + {Addr: "ok"}, + }}) + r.CC().ReportError(errors.New("example")) + }) +} diff --git a/resolver/map.go b/resolver/map.go new file mode 100644 index 0000000..789a5ab --- /dev/null +++ b/resolver/map.go @@ -0,0 +1,281 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "encoding/base64" + "iter" + "sort" + "strings" +) + +type addressMapEntry[T any] struct { + addr Address + value T +} + +// AddressMap is an AddressMapV2[any]. It will be deleted in an upcoming +// release of grpc-go. +// +// Deprecated: use the generic AddressMapV2 type instead. +type AddressMap = AddressMapV2[any] + +// AddressMapV2 is a map of addresses to arbitrary values taking into account +// Attributes. BalancerAttributes are ignored, as are Metadata and Type. +// Multiple accesses may not be performed concurrently. Must be created via +// NewAddressMap; do not construct directly. +type AddressMapV2[T any] struct { + // The underlying map is keyed by an Address with fields that we don't care + // about being set to their zero values. The only fields that we care about + // are `Addr`, `ServerName` and `Attributes`. Since we need to be able to + // distinguish between addresses with same `Addr` and `ServerName`, but + // different `Attributes`, we cannot store the `Attributes` in the map key. + // + // The comparison operation for structs work as follows: + // Struct values are comparable if all their fields are comparable. Two + // struct values are equal if their corresponding non-blank fields are equal. + // + // The value type of the map contains a slice of addresses which match the key + // in their `Addr` and `ServerName` fields and contain the corresponding value + // associated with them. + m map[Address]addressMapEntryList[T] +} + +func toMapKey(addr *Address) Address { + return Address{Addr: addr.Addr, ServerName: addr.ServerName} +} + +type addressMapEntryList[T any] []*addressMapEntry[T] + +// NewAddressMap creates a new AddressMapV2[any]. +// +// Deprecated: use the generic NewAddressMapV2 constructor instead. +func NewAddressMap() *AddressMap { + return NewAddressMapV2[any]() +} + +// NewAddressMapV2 creates a new AddressMapV2. +func NewAddressMapV2[T any]() *AddressMapV2[T] { + return &AddressMapV2[T]{m: make(map[Address]addressMapEntryList[T])} +} + +// find returns the index of addr in the addressMapEntry slice, or -1 if not +// present. +func (l addressMapEntryList[T]) find(addr Address) int { + for i, entry := range l { + // Attributes are the only thing to match on here, since `Addr` and + // `ServerName` are already equal. + if entry.addr.Attributes.Equal(addr.Attributes) { + return i + } + } + return -1 +} + +// Get returns the value for the address in the map, if present. +func (a *AddressMapV2[T]) Get(addr Address) (value T, ok bool) { + addrKey := toMapKey(&addr) + entryList := a.m[addrKey] + if entry := entryList.find(addr); entry != -1 { + return entryList[entry].value, true + } + return value, false +} + +// Set updates or adds the value to the address in the map. +func (a *AddressMapV2[T]) Set(addr Address, value T) { + addrKey := toMapKey(&addr) + entryList := a.m[addrKey] + if entry := entryList.find(addr); entry != -1 { + entryList[entry].value = value + return + } + a.m[addrKey] = append(entryList, &addressMapEntry[T]{addr: addr, value: value}) +} + +// Delete removes addr from the map. +func (a *AddressMapV2[T]) Delete(addr Address) { + addrKey := toMapKey(&addr) + entryList := a.m[addrKey] + entry := entryList.find(addr) + if entry == -1 { + return + } + if len(entryList) == 1 { + entryList = nil + } else { + copy(entryList[entry:], entryList[entry+1:]) + entryList = entryList[:len(entryList)-1] + } + a.m[addrKey] = entryList +} + +// Len returns the number of entries in the map. +func (a *AddressMapV2[T]) Len() int { + ret := 0 + for _, entryList := range a.m { + ret += len(entryList) + } + return ret +} + +// Keys returns a slice of all current map keys. +// Deprecated: Use AddressMapV2.All() instead. +func (a *AddressMapV2[T]) Keys() []Address { + ret := make([]Address, 0, a.Len()) + for _, entryList := range a.m { + for _, entry := range entryList { + ret = append(ret, entry.addr) + } + } + return ret +} + +// Values returns a slice of all current map values. +// Deprecated: Use AddressMapV2.All() instead. +func (a *AddressMapV2[T]) Values() []T { + ret := make([]T, 0, a.Len()) + for _, entryList := range a.m { + for _, entry := range entryList { + ret = append(ret, entry.value) + } + } + return ret +} + +// All returns an iterator over all elements. +func (a *AddressMapV2[T]) All() iter.Seq2[Address, T] { + return func(yield func(Address, T) bool) { + for _, entryList := range a.m { + for _, entry := range entryList { + if !yield(entry.addr, entry.value) { + return + } + } + } + } +} + +type endpointMapKey string + +// EndpointMap is a map of endpoints to arbitrary values keyed on only the +// unordered set of address strings within an endpoint. This map is not thread +// safe, thus it is unsafe to access concurrently. Must be created via +// NewEndpointMap; do not construct directly. +type EndpointMap[T any] struct { + endpoints map[endpointMapKey]endpointData[T] +} + +type endpointData[T any] struct { + // decodedKey stores the original key to avoid decoding when iterating on + // EndpointMap keys. + decodedKey Endpoint + value T +} + +// NewEndpointMap creates a new EndpointMap. +func NewEndpointMap[T any]() *EndpointMap[T] { + return &EndpointMap[T]{ + endpoints: make(map[endpointMapKey]endpointData[T]), + } +} + +// encodeEndpoint returns a string that uniquely identifies the unordered set of +// addresses within an endpoint. +func encodeEndpoint(e Endpoint) endpointMapKey { + addrs := make([]string, 0, len(e.Addresses)) + // base64 encoding the address strings restricts the characters present + // within the strings. This allows us to use a delimiter without the need of + // escape characters. + for _, addr := range e.Addresses { + addrs = append(addrs, base64.StdEncoding.EncodeToString([]byte(addr.Addr))) + } + sort.Strings(addrs) + // " " should not appear in base64 encoded strings. + return endpointMapKey(strings.Join(addrs, " ")) +} + +// Get returns the value for the address in the map, if present. +func (em *EndpointMap[T]) Get(e Endpoint) (value T, ok bool) { + val, found := em.endpoints[encodeEndpoint(e)] + if found { + return val.value, true + } + return value, false +} + +// Set updates or adds the value to the address in the map. +func (em *EndpointMap[T]) Set(e Endpoint, value T) { + en := encodeEndpoint(e) + em.endpoints[en] = endpointData[T]{ + decodedKey: Endpoint{Addresses: e.Addresses}, + value: value, + } +} + +// Len returns the number of entries in the map. +func (em *EndpointMap[T]) Len() int { + return len(em.endpoints) +} + +// Keys returns a slice of all current map keys, as endpoints specifying the +// addresses present in the endpoint keys, in which uniqueness is determined by +// the unordered set of addresses. Thus, endpoint information returned is not +// the full endpoint data (drops duplicated addresses and attributes) but can be +// used for EndpointMap accesses. +// Deprecated: Use EndpointMap.All() instead. +func (em *EndpointMap[T]) Keys() []Endpoint { + ret := make([]Endpoint, 0, len(em.endpoints)) + for _, en := range em.endpoints { + ret = append(ret, en.decodedKey) + } + return ret +} + +// Values returns a slice of all current map values. +// Deprecated: Use EndpointMap.All() instead. +func (em *EndpointMap[T]) Values() []T { + ret := make([]T, 0, len(em.endpoints)) + for _, val := range em.endpoints { + ret = append(ret, val.value) + } + return ret +} + +// All returns an iterator over all elements. +// The map keys are endpoints specifying the addresses present in the endpoint +// map, in which uniqueness is determined by the unordered set of addresses. +// Thus, endpoint information returned is not the full endpoint data (drops +// duplicated addresses and attributes) but can be used for EndpointMap +// accesses. +func (em *EndpointMap[T]) All() iter.Seq2[Endpoint, T] { + return func(yield func(Endpoint, T) bool) { + for _, en := range em.endpoints { + if !yield(en.decodedKey, en.value) { + return + } + } + } +} + +// Delete removes the specified endpoint from the map. +func (em *EndpointMap[T]) Delete(e Endpoint) { + en := encodeEndpoint(e) + delete(em.endpoints, en) +} diff --git a/resolver/map_test.go b/resolver/map_test.go new file mode 100644 index 0000000..55b4fff --- /dev/null +++ b/resolver/map_test.go @@ -0,0 +1,365 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "fmt" + "sort" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" +) + +// Note: each address is different from addr1 by one value. addr7 matches +// addr1, since the only difference is BalancerAttributes, which are not +// compared. +var ( + addr1 = Address{Addr: "a1", Attributes: attributes.New("a1", 3), ServerName: "s1"} + addr2 = Address{Addr: "a2", Attributes: attributes.New("a1", 3), ServerName: "s1"} + addr3 = Address{Addr: "a1", Attributes: attributes.New("a2", 3), ServerName: "s1"} + addr4 = Address{Addr: "a1", Attributes: attributes.New("a1", 2), ServerName: "s1"} + addr5 = Address{Addr: "a1", Attributes: attributes.New("a1", "3"), ServerName: "s1"} + addr6 = Address{Addr: "a1", Attributes: attributes.New("a1", 3), ServerName: "s2"} + addr7 = Address{Addr: "a1", Attributes: attributes.New("a1", 3), ServerName: "s1", BalancerAttributes: attributes.New("xx", 3)} + + endpoint1 = Endpoint{Addresses: []Address{{Addr: "addr1"}}} + endpoint2 = Endpoint{Addresses: []Address{{Addr: "addr2"}}} + endpoint3 = Endpoint{Addresses: []Address{{Addr: "addr3"}}} + endpoint4 = Endpoint{Addresses: []Address{{Addr: "addr4"}}} + endpoint5 = Endpoint{Addresses: []Address{{Addr: "addr5"}}} + endpoint6 = Endpoint{Addresses: []Address{{Addr: "addr6"}}} + endpoint7 = Endpoint{Addresses: []Address{{Addr: "addr7"}}} + endpoint12 = Endpoint{Addresses: []Address{{Addr: "addr1"}, {Addr: "addr2"}}} + endpoint21 = Endpoint{Addresses: []Address{{Addr: "addr2"}, {Addr: "addr1"}}} + endpoint123 = Endpoint{Addresses: []Address{{Addr: "addr1"}, {Addr: "addr2"}, {Addr: "addr3"}}} +) + +func (s) TestAddressMap_Length(t *testing.T) { + addrMap := NewAddressMapV2[any]() + if got := addrMap.Len(); got != 0 { + t.Fatalf("addrMap.Len() = %v; want 0", got) + } + for i := 0; i < 10; i++ { + addrMap.Set(addr1, nil) + if got, want := addrMap.Len(), 1; got != want { + t.Fatalf("addrMap.Len() = %v; want %v", got, want) + } + addrMap.Set(addr7, nil) // aliases addr1 + } + for i := 0; i < 10; i++ { + addrMap.Set(addr2, nil) + if got, want := addrMap.Len(), 2; got != want { + t.Fatalf("addrMap.Len() = %v; want %v", got, want) + } + } +} + +func (s) TestAddressMap_Get(t *testing.T) { + addrMap := NewAddressMapV2[int]() + addrMap.Set(addr1, 1) + + if got, ok := addrMap.Get(addr2); ok || got != 0 { + t.Fatalf("addrMap.Get(addr1) = %v, %v; want 0, false", got, ok) + } + + addrMap.Set(addr2, 2) + addrMap.Set(addr3, 3) + addrMap.Set(addr4, 4) + addrMap.Set(addr5, 5) + addrMap.Set(addr6, 6) + addrMap.Set(addr7, 7) // aliases addr1 + if got, ok := addrMap.Get(addr1); !ok || got != 7 { + t.Fatalf("addrMap.Get(addr1) = %v, %v; want %v, true", got, ok, 7) + } + if got, ok := addrMap.Get(addr2); !ok || got != 2 { + t.Fatalf("addrMap.Get(addr2) = %v, %v; want %v, true", got, ok, 2) + } + if got, ok := addrMap.Get(addr3); !ok || got != 3 { + t.Fatalf("addrMap.Get(addr3) = %v, %v; want %v, true", got, ok, 3) + } + if got, ok := addrMap.Get(addr4); !ok || got != 4 { + t.Fatalf("addrMap.Get(addr4) = %v, %v; want %v, true", got, ok, 4) + } + if got, ok := addrMap.Get(addr5); !ok || got != 5 { + t.Fatalf("addrMap.Get(addr5) = %v, %v; want %v, true", got, ok, 5) + } + if got, ok := addrMap.Get(addr6); !ok || got != 6 { + t.Fatalf("addrMap.Get(addr6) = %v, %v; want %v, true", got, ok, 6) + } + if got, ok := addrMap.Get(addr7); !ok || got != 7 { + t.Fatalf("addrMap.Get(addr7) = %v, %v; want %v, true", got, ok, 7) + } +} + +func (s) TestAddressMap_Delete(t *testing.T) { + addrMap := NewAddressMapV2[any]() + addrMap.Set(addr1, 1) + addrMap.Set(addr2, 2) + if got, want := addrMap.Len(), 2; got != want { + t.Fatalf("addrMap.Len() = %v; want %v", got, want) + } + addrMap.Delete(addr3) + addrMap.Delete(addr4) + addrMap.Delete(addr5) + addrMap.Delete(addr6) + addrMap.Delete(addr7) // aliases addr1 + if got, ok := addrMap.Get(addr1); ok || got != nil { + t.Fatalf("addrMap.Get(addr1) = %v, %v; want nil, false", got, ok) + } + if got, ok := addrMap.Get(addr7); ok || got != nil { + t.Fatalf("addrMap.Get(addr7) = %v, %v; want nil, false", got, ok) + } + if got, ok := addrMap.Get(addr2); !ok || got.(int) != 2 { + t.Fatalf("addrMap.Get(addr2) = %v, %v; want %v, true", got, ok, 2) + } +} + +func (s) TestAddressMap_Keys(t *testing.T) { + addrMap := NewAddressMapV2[int]() + addrMap.Set(addr1, 1) + addrMap.Set(addr2, 2) + addrMap.Set(addr3, 3) + addrMap.Set(addr4, 4) + addrMap.Set(addr5, 5) + addrMap.Set(addr6, 6) + addrMap.Set(addr7, 7) // aliases addr1 + + want := []Address{addr1, addr2, addr3, addr4, addr5, addr6} + got := addrMap.Keys() + if d := cmp.Diff(want, got, cmp.Transformer("sort", func(in []Address) []Address { + out := append([]Address(nil), in...) + sort.Slice(out, func(i, j int) bool { return fmt.Sprint(out[i]) < fmt.Sprint(out[j]) }) + return out + })); d != "" { + t.Fatalf("addrMap.Keys returned unexpected elements (-want, +got):\n%v", d) + } +} + +func (s) TestAddressMap_Values(t *testing.T) { + addrMap := NewAddressMapV2[int]() + addrMap.Set(addr1, 1) + addrMap.Set(addr2, 2) + addrMap.Set(addr3, 3) + addrMap.Set(addr4, 4) + addrMap.Set(addr5, 5) + addrMap.Set(addr6, 6) + addrMap.Set(addr7, 7) // aliases addr1 + + want := []int{2, 3, 4, 5, 6, 7} + got := addrMap.Values() + sort.Ints(got) + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("addrMap.Values returned unexpected elements (-want, +got):\n%v", diff) + } +} + +func (s) TestAddressMap_All(t *testing.T) { + addrMap := NewAddressMapV2[int]() + addrMap.Set(addr1, 1) + addrMap.Set(addr2, 2) + addrMap.Set(addr3, 3) + addrMap.Set(addr4, 4) + addrMap.Set(addr5, 5) + addrMap.Set(addr6, 6) + addrMap.Set(addr7, 7) // aliases addr1 + + type pair struct { + K Address + V int + } + + want := []pair{{addr1, 7}, {addr2, 2}, {addr3, 3}, {addr4, 4}, {addr5, 5}, {addr6, 6}} + var got []pair + for k, v := range addrMap.All() { + got = append(got, pair{k, v}) + } + if d := cmp.Diff(want, got, cmp.Transformer("sort", func(in []pair) []pair { + out := append([]pair(nil), in...) + sort.Slice(out, func(i, j int) bool { return out[i].V < out[j].V }) + return out + })); d != "" { + t.Fatalf("addrMap.All returned unexpected elements (-want, +got):\n%v", d) + } +} + +func (s) TestEndpointMap_Length(t *testing.T) { + em := NewEndpointMap[struct{}]() + // Should be empty at creation time. + if got := em.Len(); got != 0 { + t.Fatalf("em.Len() = %v; want 0", got) + } + // Add two endpoints with the same unordered set of addresses. This should + // amount to one endpoint. It should also not take into account attributes. + em.Set(endpoint12, struct{}{}) + em.Set(endpoint21, struct{}{}) + + if got := em.Len(); got != 1 { + t.Fatalf("em.Len() = %v; want 1", got) + } + + // Add another unique endpoint. This should cause the length to be 2. + em.Set(endpoint123, struct{}{}) + if got := em.Len(); got != 2 { + t.Fatalf("em.Len() = %v; want 2", got) + } +} + +func (s) TestEndpointMap_Get(t *testing.T) { + em := NewEndpointMap[int]() + em.Set(endpoint1, 1) + // The second endpoint endpoint21 should override. + em.Set(endpoint12, 1) + em.Set(endpoint21, 2) + em.Set(endpoint3, 3) + em.Set(endpoint4, 4) + em.Set(endpoint5, 5) + em.Set(endpoint6, 6) + em.Set(endpoint7, 7) + + if got, ok := em.Get(endpoint1); !ok || got != 1 { + t.Fatalf("em.Get(endpoint1) = %v, %v; want %v, true", got, ok, 1) + } + if got, ok := em.Get(endpoint12); !ok || got != 2 { + t.Fatalf("em.Get(endpoint12) = %v, %v; want %v, true", got, ok, 2) + } + if got, ok := em.Get(endpoint21); !ok || got != 2 { + t.Fatalf("em.Get(endpoint21) = %v, %v; want %v, true", got, ok, 2) + } + if got, ok := em.Get(endpoint3); !ok || got != 3 { + t.Fatalf("em.Get(endpoint1) = %v, %v; want %v, true", got, ok, 3) + } + if got, ok := em.Get(endpoint4); !ok || got != 4 { + t.Fatalf("em.Get(endpoint1) = %v, %v; want %v, true", got, ok, 4) + } + if got, ok := em.Get(endpoint5); !ok || got != 5 { + t.Fatalf("em.Get(endpoint1) = %v, %v; want %v, true", got, ok, 5) + } + if got, ok := em.Get(endpoint6); !ok || got != 6 { + t.Fatalf("em.Get(endpoint1) = %v, %v; want %v, true", got, ok, 6) + } + if got, ok := em.Get(endpoint7); !ok || got != 7 { + t.Fatalf("em.Get(endpoint1) = %v, %v; want %v, true", got, ok, 7) + } + if _, ok := em.Get(endpoint123); ok { + t.Fatalf("em.Get(endpoint123) = _, %v; want _, false", ok) + } +} + +func (s) TestEndpointMap_Delete(t *testing.T) { + em := NewEndpointMap[struct{}]() + // Initial state of system: [1, 2, 3, 12] + em.Set(endpoint1, struct{}{}) + em.Set(endpoint2, struct{}{}) + em.Set(endpoint3, struct{}{}) + em.Set(endpoint12, struct{}{}) + // Delete: [2, 21] + em.Delete(endpoint2) + em.Delete(endpoint21) + + // [1, 3] should be present: + if _, ok := em.Get(endpoint1); !ok { + t.Fatalf("em.Get(endpoint1) = %v, want true", ok) + } + if _, ok := em.Get(endpoint3); !ok { + t.Fatalf("em.Get(endpoint3) = %v, want true", ok) + } + // [2, 12] should not be present: + if _, ok := em.Get(endpoint2); ok { + t.Fatalf("em.Get(endpoint2) = %v, want false", ok) + } + if _, ok := em.Get(endpoint12); ok { + t.Fatalf("em.Get(endpoint12) = %v, want false", ok) + } + if _, ok := em.Get(endpoint21); ok { + t.Fatalf("em.Get(endpoint21) = %v, want false", ok) + } +} + +func (s) TestEndpointMap_Values(t *testing.T) { + em := NewEndpointMap[int]() + em.Set(endpoint1, 1) + // The second endpoint endpoint21 should override. + em.Set(endpoint12, 1) + em.Set(endpoint21, 2) + em.Set(endpoint3, 3) + em.Set(endpoint4, 4) + em.Set(endpoint5, 5) + em.Set(endpoint6, 6) + em.Set(endpoint7, 7) + want := []int{1, 2, 3, 4, 5, 6, 7} + got := em.Values() + sort.Ints(got) + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("em.Values() returned unexpected elements (-want, +got):\n%v", diff) + } +} + +func (s) TestEndpointMap_All(t *testing.T) { + em := NewEndpointMap[int]() + em.Set(endpoint1, 1) + // The second endpoint endpoint21 should override. + em.Set(endpoint12, 1) + em.Set(endpoint21, 2) + em.Set(endpoint3, 3) + em.Set(endpoint4, 4) + em.Set(endpoint5, 5) + em.Set(endpoint6, 6) + em.Set(endpoint7, 7) + + type pair struct { + K Endpoint + V int + } + + want := []pair{{endpoint1, 1}, {endpoint21, 2}, {endpoint3, 3}, {endpoint4, 4}, {endpoint5, 5}, {endpoint6, 6}, {endpoint7, 7}} + var got []pair + for k, v := range em.All() { + got = append(got, pair{k, v}) + } + if d := cmp.Diff(want, got, cmp.Transformer("sort", func(in []pair) []pair { + out := append([]pair(nil), in...) + sort.Slice(out, func(i, j int) bool { return out[i].V < out[j].V }) + return out + })); d != "" { + t.Fatalf("em.All returned unexpected elements (-want, +got):\n%v", d) + } +} + +// BenchmarkEndpointMap benchmarks map operations that are expected to run +// faster than O(n). This test doesn't run O(n) operations including listing +// keys and values. +func BenchmarkEndpointMap(b *testing.B) { + em := NewEndpointMap[any]() + for i := range b.N { + em.Set(Endpoint{ + Addresses: []Address{{Addr: fmt.Sprintf("%d.%d.%d.%d", i, i, i, i)}}, + }, i) + } + for i := range b.N { + em.Get(Endpoint{ + Addresses: []Address{{Addr: fmt.Sprintf("%d.%d.%d.%d", i, i, i, i)}}, + }) + } + for i := range b.N { + em.Delete(Endpoint{ + Addresses: []Address{{Addr: fmt.Sprintf("%d.%d.%d.%d", i, i, i, i)}}, + }) + } +} diff --git a/resolver/passthrough/passthrough.go b/resolver/passthrough/passthrough.go new file mode 100644 index 0000000..c8a0c3d --- /dev/null +++ b/resolver/passthrough/passthrough.go @@ -0,0 +1,26 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package passthrough implements a pass-through resolver. It sends the target +// name without scheme back to gRPC as resolved address. +// +// Deprecated: this package is imported by grpc and should not need to be +// imported directly by users. +package passthrough + +import _ "google.golang.org/grpc/internal/resolver/passthrough" // import for side effects after package was moved diff --git a/resolver/resolver.go b/resolver/resolver.go new file mode 100644 index 0000000..598ed21 --- /dev/null +++ b/resolver/resolver.go @@ -0,0 +1,359 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package resolver defines APIs for name resolution in gRPC. +// All APIs in this package are experimental. +package resolver + +import ( + "context" + "errors" + "fmt" + "net" + "net/url" + "strings" + + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/serviceconfig" +) + +var ( + // m is a map from scheme to resolver builder. + m = make(map[string]Builder) + // defaultScheme is the default scheme to use. + defaultScheme = "passthrough" +) + +// TODO(bar) install dns resolver in init(){}. + +// Register registers the resolver builder to the resolver map. b.Scheme will +// be used as the scheme registered with this builder. The registry is case +// sensitive, and schemes should not contain any uppercase characters. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple Resolvers are +// registered with the same name, the one registered last will take effect. +func Register(b Builder) { + m[b.Scheme()] = b +} + +// Get returns the resolver builder registered with the given scheme. +// +// If no builder is register with the scheme, nil will be returned. +func Get(scheme string) Builder { + if b, ok := m[scheme]; ok { + return b + } + return nil +} + +// SetDefaultScheme sets the default scheme that will be used. The default +// scheme is initially set to "passthrough". +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. The scheme set last overrides +// previously set values. +func SetDefaultScheme(scheme string) { + defaultScheme = scheme + internal.UserSetDefaultScheme = true +} + +// GetDefaultScheme gets the default scheme that will be used by grpc.Dial. If +// SetDefaultScheme is never called, the default scheme used by grpc.NewClient is "dns" instead. +func GetDefaultScheme() string { + return defaultScheme +} + +// Address represents a server the client connects to. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type Address struct { + // Addr is the server address on which a connection will be established. + Addr string + + // ServerName is the name of this address. + // If non-empty, the ServerName is used as the transport certification authority for + // the address, instead of the hostname from the Dial target string. In most cases, + // this should not be set. + // + // WARNING: ServerName must only be populated with trusted values. It + // is insecure to populate it with data from untrusted inputs since untrusted + // values could be used to bypass the authority checks performed by TLS. + ServerName string + + // Attributes contains arbitrary data about this address intended for + // consumption by the SubConn. + Attributes *attributes.Attributes + + // BalancerAttributes contains arbitrary data about this address intended + // for consumption by the LB policy. These attributes do not affect SubConn + // creation, connection establishment, handshaking, etc. + // + // Deprecated: when an Address is inside an Endpoint, this field should not + // be used, and it will eventually be removed entirely. + BalancerAttributes *attributes.Attributes + + // Metadata is the information associated with Addr, which may be used + // to make load balancing decision. + // + // Deprecated: use Attributes instead. + Metadata any +} + +// Equal returns whether a and o are identical. Metadata is compared directly, +// not with any recursive introspection. +// +// This method compares all fields of the address. When used to tell apart +// addresses during subchannel creation or connection establishment, it might be +// more appropriate for the caller to implement custom equality logic. +func (a Address) Equal(o Address) bool { + return a.Addr == o.Addr && a.ServerName == o.ServerName && + a.Attributes.Equal(o.Attributes) && + a.BalancerAttributes.Equal(o.BalancerAttributes) && + a.Metadata == o.Metadata +} + +// String returns JSON formatted string representation of the address. +func (a Address) String() string { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("{Addr: %q, ", a.Addr)) + sb.WriteString(fmt.Sprintf("ServerName: %q, ", a.ServerName)) + if a.Attributes != nil { + sb.WriteString(fmt.Sprintf("Attributes: %v, ", a.Attributes.String())) + } + if a.BalancerAttributes != nil { + sb.WriteString(fmt.Sprintf("BalancerAttributes: %v", a.BalancerAttributes.String())) + } + sb.WriteString("}") + return sb.String() +} + +// BuildOptions includes additional information for the builder to create +// the resolver. +type BuildOptions struct { + // DisableServiceConfig indicates whether a resolver implementation should + // fetch service config data. + DisableServiceConfig bool + // DialCreds is the transport credentials used by the ClientConn for + // communicating with the target gRPC service (set via + // WithTransportCredentials). In cases where a name resolution service + // requires the same credentials, the resolver may use this field. In most + // cases though, it is not appropriate, and this field may be ignored. + DialCreds credentials.TransportCredentials + // CredsBundle is the credentials bundle used by the ClientConn for + // communicating with the target gRPC service (set via + // WithCredentialsBundle). In cases where a name resolution service + // requires the same credentials, the resolver may use this field. In most + // cases though, it is not appropriate, and this field may be ignored. + CredsBundle credentials.Bundle + // Dialer is the custom dialer used by the ClientConn for dialling the + // target gRPC service (set via WithDialer). In cases where a name + // resolution service requires the same dialer, the resolver may use this + // field. In most cases though, it is not appropriate, and this field may + // be ignored. + Dialer func(context.Context, string) (net.Conn, error) + // Authority is the effective authority of the clientconn for which the + // resolver is built. + Authority string + // MetricsRecorder is the metrics recorder to do recording. + MetricsRecorder stats.MetricsRecorder +} + +// An Endpoint is one network endpoint, or server, which may have multiple +// addresses with which it can be accessed. +// TODO(i/8773) : make resolver.Endpoint and resolver.Address immutable +type Endpoint struct { + // Addresses contains a list of addresses used to access this endpoint. + Addresses []Address + + // Attributes contains arbitrary data about this endpoint intended for + // consumption by the LB policy. + Attributes *attributes.Attributes +} + +// State contains the current Resolver state relevant to the ClientConn. +type State struct { + // Addresses is the latest set of resolved addresses for the target. + // + // If a resolver sets Addresses but does not set Endpoints, one Endpoint + // will be created for each Address before the State is passed to the LB + // policy. The BalancerAttributes of each entry in Addresses will be set + // in Endpoints.Attributes, and be cleared in the Endpoint's Address's + // BalancerAttributes. + // + // Soon, Addresses will be deprecated and replaced fully by Endpoints. + Addresses []Address + + // Endpoints is the latest set of resolved endpoints for the target. + // + // If a resolver produces a State containing Endpoints but not Addresses, + // it must take care to ensure the LB policies it selects will support + // Endpoints. + Endpoints []Endpoint + + // ServiceConfig contains the result from parsing the latest service + // config. If it is nil, it indicates no service config is present or the + // resolver does not provide service configs. + ServiceConfig *serviceconfig.ParseResult + + // Attributes contains arbitrary data about the resolver intended for + // consumption by the load balancing policy. + Attributes *attributes.Attributes +} + +// ClientConn contains the callbacks for resolver to notify any updates +// to the gRPC ClientConn. +// +// This interface is to be implemented by gRPC. Users should not need a +// brand new implementation of this interface. For the situations like +// testing, the new implementation should embed this interface. This allows +// gRPC to add new methods to this interface. +type ClientConn interface { + // UpdateState updates the state of the ClientConn appropriately. + // + // If an error is returned, the resolver should try to resolve the + // target again. The resolver should use a backoff timer to prevent + // overloading the server with requests. If a resolver is certain that + // reresolving will not change the result, e.g. because it is + // a watch-based resolver, returned errors can be ignored. + // + // If the resolved State is the same as the last reported one, calling + // UpdateState can be omitted. + UpdateState(State) error + // ReportError notifies the ClientConn that the Resolver encountered an + // error. The ClientConn then forwards this error to the load balancing + // policy. + ReportError(error) + // NewAddress is called by resolver to notify ClientConn a new list + // of resolved addresses. + // The address list should be the complete list of resolved addresses. + // + // Deprecated: Use UpdateState instead. + NewAddress(addresses []Address) + // ParseServiceConfig parses the provided service config and returns an + // object that provides the parsed config. + ParseServiceConfig(serviceConfigJSON string) *serviceconfig.ParseResult +} + +// Target represents a target for gRPC, as specified in: +// https://github.com/grpc/grpc/blob/master/doc/naming.md. +// It is parsed from the target string that gets passed into Dial or DialContext +// by the user. And gRPC passes it to the resolver and the balancer. +// +// If the target follows the naming spec, and the parsed scheme is registered +// with gRPC, we will parse the target string according to the spec. If the +// target does not contain a scheme or if the parsed scheme is not registered +// (i.e. no corresponding resolver available to resolve the endpoint), we will +// apply the default scheme, and will attempt to reparse it. +type Target struct { + // URL contains the parsed dial target with an optional default scheme added + // to it if the original dial target contained no scheme or contained an + // unregistered scheme. Any query params specified in the original dial + // target can be accessed from here. + URL url.URL +} + +// Endpoint retrieves endpoint without leading "/" from either `URL.Path` +// or `URL.Opaque`. The latter is used when the former is empty. +func (t Target) Endpoint() string { + endpoint := t.URL.Path + if endpoint == "" { + endpoint = t.URL.Opaque + } + // For targets of the form "[scheme]://[authority]/endpoint, the endpoint + // value returned from url.Parse() contains a leading "/". Although this is + // in accordance with RFC 3986, we do not want to break existing resolver + // implementations which expect the endpoint without the leading "/". So, we + // end up stripping the leading "/" here. But this will result in an + // incorrect parsing for something like "unix:///path/to/socket". Since we + // own the "unix" resolver, we can workaround in the unix resolver by using + // the `URL` field. + return strings.TrimPrefix(endpoint, "/") +} + +// String returns the canonical string representation of Target. +func (t Target) String() string { + return t.URL.Scheme + "://" + t.URL.Host + "/" + t.Endpoint() +} + +// Builder creates a resolver that will be used to watch name resolution updates. +type Builder interface { + // Build creates a new resolver for the given target. + // + // gRPC dial calls Build synchronously, and fails if the returned error is + // not nil. + Build(target Target, cc ClientConn, opts BuildOptions) (Resolver, error) + // Scheme returns the scheme supported by this resolver. Scheme is defined + // at https://github.com/grpc/grpc/blob/master/doc/naming.md. The returned + // string should not contain uppercase characters, as they will not match + // the parsed target's scheme as defined in RFC 3986. + Scheme() string +} + +// ResolveNowOptions includes additional information for ResolveNow. +type ResolveNowOptions struct{} + +// Resolver watches for the updates on the specified target. +// Updates include address updates and service config updates. +type Resolver interface { + // ResolveNow will be called by gRPC to try to resolve the target name + // again. It's just a hint, resolver can ignore this if it's not necessary. + // + // It could be called multiple times concurrently. + ResolveNow(ResolveNowOptions) + // Close closes the resolver. + Close() +} + +// AuthorityOverrider is implemented by Builders that wish to override the +// default authority for the ClientConn. +// By default, the authority used is target.Endpoint(). +type AuthorityOverrider interface { + // OverrideAuthority returns the authority to use for a ClientConn with the + // given target. The implementation must generate it without blocking, + // typically in line, and must keep it unchanged. + // + // The returned string must be a valid ":authority" header value, i.e. be + // encoded according to + // [RFC3986](https://datatracker.ietf.org/doc/html/rfc3986#section-3.2) as + // necessary. + OverrideAuthority(Target) string +} + +// ValidateEndpoints validates endpoints from a petiole policy's perspective. +// Petiole policies should call this before calling into their children. See +// [gRPC A61](https://github.com/grpc/proposal/blob/master/A61-IPv4-IPv6-dualstack-backends.md) +// for details. +func ValidateEndpoints(endpoints []Endpoint) error { + if len(endpoints) == 0 { + return errors.New("endpoints list is empty") + } + + for _, endpoint := range endpoints { + for range endpoint.Addresses { + return nil + } + } + return errors.New("endpoints list contains no addresses") +} diff --git a/resolver/resolver_test.go b/resolver/resolver_test.go new file mode 100644 index 0000000..33d2e27 --- /dev/null +++ b/resolver/resolver_test.go @@ -0,0 +1,105 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "testing" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestValidateEndpoints tests different scenarios of resolver addresses being +// validated by the ValidateEndpoint helper. +func (s) TestValidateEndpoints(t *testing.T) { + addr1 := Address{Addr: "addr1"} + addr2 := Address{Addr: "addr2"} + addr3 := Address{Addr: "addr3"} + addr4 := Address{Addr: "addr4"} + tests := []struct { + name string + endpoints []Endpoint + wantErr bool + }{ + { + name: "duplicate-address-across-endpoints", + endpoints: []Endpoint{ + {Addresses: []Address{addr1}}, + {Addresses: []Address{addr1}}, + }, + wantErr: false, + }, + { + name: "duplicate-address-same-endpoint", + endpoints: []Endpoint{ + {Addresses: []Address{addr1, addr1}}, + }, + wantErr: false, + }, + { + name: "duplicate-address-across-endpoints-plural-addresses", + endpoints: []Endpoint{ + {Addresses: []Address{addr1, addr2, addr3}}, + {Addresses: []Address{addr3, addr4}}, + }, + wantErr: false, + }, + { + name: "no-shared-addresses", + endpoints: []Endpoint{ + {Addresses: []Address{addr1, addr2}}, + {Addresses: []Address{addr3, addr4}}, + }, + wantErr: false, + }, + { + name: "endpoint-with-no-addresses", + endpoints: []Endpoint{ + {Addresses: []Address{addr1, addr2}}, + {Addresses: []Address{}}, + }, + wantErr: false, + }, + { + name: "empty-endpoints-list", + endpoints: []Endpoint{}, + wantErr: true, + }, + { + name: "endpoint-list-with-no-addresses", + endpoints: []Endpoint{{}, {}}, + wantErr: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := ValidateEndpoints(test.endpoints) + if (err != nil) != test.wantErr { + t.Fatalf("ValidateEndpoints() wantErr: %v, got: %v", test.wantErr, err) + } + }) + } +} diff --git a/resolver/ringhash/attr.go b/resolver/ringhash/attr.go new file mode 100644 index 0000000..154f023 --- /dev/null +++ b/resolver/ringhash/attr.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package ringhash implements resolver related functions for the ring_hash +// load balancing policy. +package ringhash + +import ( + "google.golang.org/grpc/resolver" +) + +type hashKeyType string + +// hashKeyKey is the key to store the ring hash key attribute in +// a resolver.Endpoint attribute. +const hashKeyKey = hashKeyType("grpc.resolver.ringhash.hash_key") + +// SetHashKey sets the hash key for this endpoint. Combined with the ring_hash +// load balancing policy, it allows placing the endpoint on the ring based on an +// arbitrary string instead of the IP address. If hashKey is empty, the endpoint +// is returned unmodified. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func SetHashKey(endpoint resolver.Endpoint, hashKey string) resolver.Endpoint { + if hashKey == "" { + return endpoint + } + endpoint.Attributes = endpoint.Attributes.WithValue(hashKeyKey, hashKey) + return endpoint +} + +// HashKey returns the hash key attribute of endpoint. If this attribute is +// not set, it returns the empty string. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func HashKey(endpoint resolver.Endpoint) string { + hashKey, _ := endpoint.Attributes.Value(hashKeyKey).(string) + return hashKey +} diff --git a/resolver_balancer_ext_test.go b/resolver_balancer_ext_test.go new file mode 100644 index 0000000..7b6cf0f --- /dev/null +++ b/resolver_balancer_ext_test.go @@ -0,0 +1,306 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc_test + +import ( + "context" + "errors" + "fmt" + "runtime" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestResolverBalancerInteraction tests: +// 1. resolver.Builder.Build() -> +// 2. resolver.ClientConn.UpdateState() -> +// 3. balancer.Balancer.UpdateClientConnState() -> +// 4. balancer.ClientConn.ResolveNow() -> +// 5. resolver.Resolver.ResolveNow() -> +func (s) TestResolverBalancerInteraction(t *testing.T) { + name := strings.ReplaceAll(strings.ToLower(t.Name()), "/", "") + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + bd.ClientConn.ResolveNow(resolver.ResolveNowOptions{}) + return nil + }, + } + stub.Register(name, bf) + + rb := manual.NewBuilderWithScheme(name) + rb.BuildCallback = func(_ resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) { + sc := cc.ParseServiceConfig(`{"loadBalancingConfig": [{"` + name + `":{}}]}`) + cc.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: "test"}}, + ServiceConfig: sc, + }) + } + rnCh := make(chan struct{}) + rb.ResolveNowCallback = func(resolver.ResolveNowOptions) { close(rnCh) } + resolver.Register(rb) + + cc, err := grpc.NewClient(name+":///", grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient error: %v", err) + } + defer cc.Close() + cc.Connect() + select { + case <-rnCh: + case <-time.After(defaultTestTimeout): + t.Fatalf("timed out waiting for resolver.ResolveNow") + } +} + +type resolverBuilderWithErr struct { + resolver.Resolver + errCh <-chan error + scheme string +} + +func (b *resolverBuilderWithErr) Build(_ resolver.Target, _ resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) { + if err := <-b.errCh; err != nil { + return nil, err + } + return b, nil +} + +func (b *resolverBuilderWithErr) Scheme() string { + return b.scheme +} + +func (b *resolverBuilderWithErr) Close() {} + +// TestResolverBuildFailure tests: +// 1. resolver.Builder.Build() passes. +// 2. Channel enters idle mode. +// 3. An RPC happens. +// 4. resolver.Builder.Build() fails. +func (s) TestResolverBuildFailure(t *testing.T) { + enterIdle := internal.EnterIdleModeForTesting.(func(*grpc.ClientConn)) + name := strings.ReplaceAll(strings.ToLower(t.Name()), "/", "") + resErrCh := make(chan error, 1) + resolver.Register(&resolverBuilderWithErr{errCh: resErrCh, scheme: name}) + + resErrCh <- nil + cc, err := grpc.NewClient(name+":///", grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient error: %v", err) + } + defer cc.Close() + cc.Connect() + enterIdle(cc) + const errStr = "test error from resolver builder" + t.Log("pushing res err") + resErrCh <- errors.New(errStr) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := cc.Invoke(ctx, "/a/b", nil, nil); err == nil || !strings.Contains(err.Error(), errStr) { + t.Fatalf("Invoke = %v; want %v", err, errStr) + } +} + +// Tests the case where the resolver reports an error to the channel before +// reporting an update. Verifies that the channel eventually moves to +// TransientFailure and subsequent RPCs returns the error reported by the +// resolver to the user. +func (s) TestResolverReportError(t *testing.T) { + const resolverErr = "test resolver error" + r := manual.NewBuilderWithScheme("whatever") + r.BuildCallback = func(_ resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) { + cc.ReportError(errors.New(resolverErr)) + } + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("Error creating client: %v", err) + } + defer cc.Close() + cc.Connect() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + client := testgrpc.NewTestServiceClient(cc) + for range 5 { + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if code := status.Code(err); code != codes.Unavailable { + t.Fatalf("EmptyCall() = %v, want %v", err, codes.Unavailable) + } + if err == nil || !strings.Contains(err.Error(), resolverErr) { + t.Fatalf("EmptyCall() = %q, want %q", err, resolverErr) + } + } +} + +// TestEnterIdleDuringResolverUpdateState tests a scenario that used to deadlock +// while calling UpdateState at the same time as the resolver being closed while +// the channel enters idle mode. +func (s) TestEnterIdleDuringResolverUpdateState(t *testing.T) { + enterIdle := internal.EnterIdleModeForTesting.(func(*grpc.ClientConn)) + name := strings.ReplaceAll(strings.ToLower(t.Name()), "/", "") + + // Create a manual resolver that spams UpdateState calls until it is closed. + rb := manual.NewBuilderWithScheme(name) + var cancel context.CancelFunc + rb.BuildCallback = func(_ resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) { + var ctx context.Context + ctx, cancel = context.WithCancel(context.Background()) + go func() { + for ctx.Err() == nil { + cc.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "test"}}}) + } + }() + } + rb.CloseCallback = func() { + cancel() + } + resolver.Register(rb) + + cc, err := grpc.NewClient(name+":///", grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient error: %v", err) + } + defer cc.Close() + + // Enter/exit idle mode repeatedly. + for i := 0; i < 2000; i++ { + // Start a timer so we panic out of the deadlock and can see all the + // stack traces to debug the problem. + p := time.AfterFunc(time.Second, func() { + buf := make([]byte, 8192) + buf = buf[0:runtime.Stack(buf, true)] + t.Error("Timed out waiting for enterIdle") + panic(fmt.Sprint("Stack trace:\n", string(buf))) + }) + enterIdle(cc) + p.Stop() + cc.Connect() + } +} + +// TestEnterIdleDuringBalancerUpdateState tests calling UpdateState at the same +// time as the balancer being closed while the channel enters idle mode. +func (s) TestEnterIdleDuringBalancerUpdateState(t *testing.T) { + enterIdle := internal.EnterIdleModeForTesting.(func(*grpc.ClientConn)) + name := strings.ReplaceAll(strings.ToLower(t.Name()), "/", "") + + // Create a balancer that calls UpdateState once asynchronously, attempting + // to make the channel appear ready even after entering idle. + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + go func() { + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Ready}) + }() + return nil + }, + } + stub.Register(name, bf) + + rb := manual.NewBuilderWithScheme(name) + rb.BuildCallback = func(_ resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) { + cc.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "test"}}}) + } + resolver.Register(rb) + + cc, err := grpc.NewClient( + name+":///", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"`+name+`":{}}]}`)) + if err != nil { + t.Fatalf("grpc.NewClient error: %v", err) + } + defer cc.Close() + + // Enter/exit idle mode repeatedly. + for i := 0; i < 2000; i++ { + enterIdle(cc) + if got, want := cc.GetState(), connectivity.Idle; got != want { + t.Fatalf("cc state = %v; want %v", got, want) + } + cc.Connect() + } +} + +// TestEnterIdleDuringBalancerNewSubConn tests calling NewSubConn at the same +// time as the balancer being closed while the channel enters idle mode. +func (s) TestEnterIdleDuringBalancerNewSubConn(t *testing.T) { + channelz.TurnOn() + defer internal.ChannelzTurnOffForTesting() + enterIdle := internal.EnterIdleModeForTesting.(func(*grpc.ClientConn)) + name := strings.ReplaceAll(strings.ToLower(t.Name()), "/", "") + + // Create a balancer that calls NewSubConn once asynchronously, attempting + // to create a subchannel after going idle. + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + go func() { + bd.ClientConn.NewSubConn([]resolver.Address{{Addr: "test"}}, balancer.NewSubConnOptions{}) + }() + return nil + }, + } + stub.Register(name, bf) + + rb := manual.NewBuilderWithScheme(name) + rb.BuildCallback = func(_ resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) { + cc.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "test"}}}) + } + resolver.Register(rb) + + cc, err := grpc.NewClient( + name+":///", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"`+name+`":{}}]}`)) + if err != nil { + t.Fatalf("grpc.NewClient error: %v", err) + } + defer cc.Close() + + // Enter/exit idle mode repeatedly. + for i := 0; i < 2000; i++ { + enterIdle(cc) + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + t.Fatalf("Found channels: %v; expected 1 entry", tcs) + } + if got := tcs[0].SubChans(); len(got) != 0 { + t.Fatalf("Found subchannels: %v; expected 0 entries", got) + } + cc.Connect() + } +} diff --git a/resolver_test.go b/resolver_test.go new file mode 100644 index 0000000..68ebfae --- /dev/null +++ b/resolver_test.go @@ -0,0 +1,197 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "fmt" + "net" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" +) + +type wrapResolverBuilder struct { + resolver.Builder + scheme string +} + +func (w *wrapResolverBuilder) Scheme() string { + return w.scheme +} + +func init() { + resolver.Register(&wrapResolverBuilder{Builder: resolver.Get("passthrough"), scheme: "casetest"}) + resolver.Register(&wrapResolverBuilder{Builder: resolver.Get("dns"), scheme: "caseTest"}) +} + +func (s) TestResolverCaseSensitivity(t *testing.T) { + // This should find the "casetest" resolver instead of the "caseTest" + // resolver, even though the latter was registered later. "casetest" is + // "passthrough" and "caseTest" is "dns". With "passthrough" the dialer + // should see the target's address directly, but "dns" would be converted + // into a loopback IP (v4 or v6) address. + target := "caseTest:///localhost:1234" + addrCh := make(chan string, 1) + customDialer := func(_ context.Context, addr string) (net.Conn, error) { + select { + case addrCh <- addr: + default: + } + return nil, fmt.Errorf("not dialing with custom dialer") + } + + cc, err := NewClient(target, WithContextDialer(customDialer), WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Unexpected NewClient(%q) error: %v", target, err) + } + cc.Connect() + if got, want := <-addrCh, "localhost:1234"; got != want { + cc.Close() + t.Fatalf("Dialer got address %q; wanted %q", got, want) + } + cc.Close() + + // Clear addrCh for future use. + select { + case <-addrCh: + default: + } + + res := &wrapResolverBuilder{Builder: resolver.Get("dns"), scheme: "caseTest2"} + // This should not find the injected resolver due to the case not matching. + // This results in "passthrough" being used with the address as the whole + // target. + target = "caseTest2:///localhost:1234" + cc, err = NewClient(target, WithContextDialer(customDialer), withDefaultScheme("passthrough"), WithResolvers(res), WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Unexpected NewClient(%q) error: %v", target, err) + } + cc.Connect() + if got, want := <-addrCh, target; got != want { + cc.Close() + t.Fatalf("Dialer got address %q; wanted %q", got, want) + } + cc.Close() +} + +// TestResolverAddressesToEndpoints ensures one Endpoint is created for each +// entry in resolver.State.Addresses automatically. +func (s) TestResolverAddressesToEndpoints(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const scheme = "testresolveraddressestoendpoints" + r := manual.NewBuilderWithScheme(scheme) + + stateCh := make(chan balancer.ClientConnState, 1) + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(_ *stub.BalancerData, ccs balancer.ClientConnState) error { + stateCh <- ccs + return nil + }, + } + balancerName := "stub-balancer-" + scheme + stub.Register(balancerName, bf) + + a1 := attributes.New("x", "y") + a2 := attributes.New("a", "b") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: "addr1", BalancerAttributes: a1}, {Addr: "addr2", BalancerAttributes: a2}}}) + + cc, err := NewClient(r.Scheme()+":///", + WithTransportCredentials(insecure.NewCredentials()), + WithResolvers(r), + WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, balancerName))) + if err != nil { + t.Fatalf("NewClient() failed: %v", err) + } + cc.Connect() + defer cc.Close() + + select { + case got := <-stateCh: + want := []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "addr1"}}, Attributes: a1}, + {Addresses: []resolver.Address{{Addr: "addr2"}}, Attributes: a2}, + } + if diff := cmp.Diff(got.ResolverState.Endpoints, want); diff != "" { + t.Errorf("Did not receive expected endpoints. Diff (-got +want):\n%v", diff) + } + case <-ctx.Done(): + t.Fatalf("timed out waiting for endpoints") + } +} + +// Test ensures one Endpoint is created for each entry in +// resolver.State.Addresses automatically. The test calls the deprecated +// NewAddresses API to send a list of addresses to the channel. +func (s) TestResolverAddressesToEndpointsUsingNewAddresses(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const scheme = "testresolveraddressestoendpoints" + r := manual.NewBuilderWithScheme(scheme) + + stateCh := make(chan balancer.ClientConnState, 1) + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(_ *stub.BalancerData, ccs balancer.ClientConnState) error { + stateCh <- ccs + return nil + }, + } + balancerName := "stub-balancer-" + scheme + stub.Register(balancerName, bf) + + a1 := attributes.New("x", "y") + a2 := attributes.New("a", "b") + addrs := []resolver.Address{ + {Addr: "addr1", BalancerAttributes: a1}, + {Addr: "addr2", BalancerAttributes: a2}, + } + + cc, err := NewClient(r.Scheme()+":///", + WithTransportCredentials(insecure.NewCredentials()), + WithResolvers(r), + WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, balancerName))) + if err != nil { + t.Fatalf("NewClient() failed: %v", err) + } + cc.Connect() + defer cc.Close() + r.CC().NewAddress(addrs) + + select { + case got := <-stateCh: + want := []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "addr1"}}, Attributes: a1}, + {Addresses: []resolver.Address{{Addr: "addr2"}}, Attributes: a2}, + } + if diff := cmp.Diff(got.ResolverState.Endpoints, want); diff != "" { + t.Errorf("Did not receive expected endpoints. Diff (-got +want):\n%v", diff) + } + case <-ctx.Done(): + t.Fatalf("timed out waiting for endpoints") + } +} diff --git a/resolver_wrapper.go b/resolver_wrapper.go new file mode 100644 index 0000000..f8b0fd3 --- /dev/null +++ b/resolver_wrapper.go @@ -0,0 +1,222 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "strings" + "sync" + + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/resolver/delegatingresolver" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// ccResolverWrapper is a wrapper on top of cc for resolvers. +// It implements resolver.ClientConn interface. +type ccResolverWrapper struct { + // The following fields are initialized when the wrapper is created and are + // read-only afterwards, and therefore can be accessed without a mutex. + cc *ClientConn + ignoreServiceConfig bool + serializer *grpcsync.CallbackSerializer + serializerCancel context.CancelFunc + + resolver resolver.Resolver // only accessed within the serializer + + // The following fields are protected by mu. Caller must take cc.mu before + // taking mu. + mu sync.Mutex + curState resolver.State + closed bool +} + +// newCCResolverWrapper initializes the ccResolverWrapper. It can only be used +// after calling start, which builds the resolver. +func newCCResolverWrapper(cc *ClientConn) *ccResolverWrapper { + ctx, cancel := context.WithCancel(cc.ctx) + return &ccResolverWrapper{ + cc: cc, + ignoreServiceConfig: cc.dopts.disableServiceConfig, + serializer: grpcsync.NewCallbackSerializer(ctx), + serializerCancel: cancel, + } +} + +// start builds the name resolver using the resolver.Builder in cc and returns +// any error encountered. It must always be the first operation performed on +// any newly created ccResolverWrapper, except that close may be called instead. +func (ccr *ccResolverWrapper) start() error { + errCh := make(chan error) + ccr.serializer.TrySchedule(func(ctx context.Context) { + if ctx.Err() != nil { + errCh <- ctx.Err() + return + } + opts := resolver.BuildOptions{ + DisableServiceConfig: ccr.cc.dopts.disableServiceConfig, + DialCreds: ccr.cc.dopts.copts.TransportCredentials, + CredsBundle: ccr.cc.dopts.copts.CredsBundle, + Dialer: ccr.cc.dopts.copts.Dialer, + Authority: ccr.cc.authority, + MetricsRecorder: ccr.cc.metricsRecorderList, + } + var err error + // The delegating resolver is used unless: + // - A custom dialer is provided via WithContextDialer dialoption or + // - Proxy usage is disabled through WithNoProxy dialoption. + // In these cases, the resolver is built based on the scheme of target, + // using the appropriate resolver builder. + if ccr.cc.dopts.copts.Dialer != nil || !ccr.cc.dopts.useProxy { + ccr.resolver, err = ccr.cc.resolverBuilder.Build(ccr.cc.parsedTarget, ccr, opts) + } else { + ccr.resolver, err = delegatingresolver.New(ccr.cc.parsedTarget, ccr, opts, ccr.cc.resolverBuilder, ccr.cc.dopts.enableLocalDNSResolution) + } + errCh <- err + }) + return <-errCh +} + +func (ccr *ccResolverWrapper) resolveNow(o resolver.ResolveNowOptions) { + ccr.serializer.TrySchedule(func(ctx context.Context) { + if ctx.Err() != nil || ccr.resolver == nil { + return + } + ccr.resolver.ResolveNow(o) + }) +} + +// close initiates async shutdown of the wrapper. To determine the wrapper has +// finished shutting down, the channel should block on ccr.serializer.Done() +// without cc.mu held. +func (ccr *ccResolverWrapper) close() { + channelz.Info(logger, ccr.cc.channelz, "Closing the name resolver") + ccr.mu.Lock() + ccr.closed = true + ccr.mu.Unlock() + + ccr.serializer.TrySchedule(func(context.Context) { + if ccr.resolver == nil { + return + } + ccr.resolver.Close() + ccr.resolver = nil + }) + ccr.serializerCancel() +} + +// UpdateState is called by resolver implementations to report new state to gRPC +// which includes addresses and service config. +func (ccr *ccResolverWrapper) UpdateState(s resolver.State) error { + ccr.cc.mu.Lock() + ccr.mu.Lock() + if ccr.closed { + ccr.mu.Unlock() + ccr.cc.mu.Unlock() + return nil + } + if s.Endpoints == nil { + s.Endpoints = addressesToEndpoints(s.Addresses) + } + ccr.addChannelzTraceEvent(s) + ccr.curState = s + ccr.mu.Unlock() + return ccr.cc.updateResolverStateAndUnlock(s, nil) +} + +// ReportError is called by resolver implementations to report errors +// encountered during name resolution to gRPC. +func (ccr *ccResolverWrapper) ReportError(err error) { + ccr.cc.mu.Lock() + ccr.mu.Lock() + if ccr.closed { + ccr.mu.Unlock() + ccr.cc.mu.Unlock() + return + } + ccr.mu.Unlock() + channelz.Warningf(logger, ccr.cc.channelz, "ccResolverWrapper: reporting error to cc: %v", err) + ccr.cc.updateResolverStateAndUnlock(resolver.State{}, err) +} + +// NewAddress is called by the resolver implementation to send addresses to +// gRPC. +func (ccr *ccResolverWrapper) NewAddress(addrs []resolver.Address) { + ccr.cc.mu.Lock() + ccr.mu.Lock() + if ccr.closed { + ccr.mu.Unlock() + ccr.cc.mu.Unlock() + return + } + s := resolver.State{ + Addresses: addrs, + ServiceConfig: ccr.curState.ServiceConfig, + Endpoints: addressesToEndpoints(addrs), + } + ccr.addChannelzTraceEvent(s) + ccr.curState = s + ccr.mu.Unlock() + ccr.cc.updateResolverStateAndUnlock(s, nil) +} + +// ParseServiceConfig is called by resolver implementations to parse a JSON +// representation of the service config. +func (ccr *ccResolverWrapper) ParseServiceConfig(scJSON string) *serviceconfig.ParseResult { + return parseServiceConfig(scJSON, ccr.cc.dopts.maxCallAttempts) +} + +// addChannelzTraceEvent adds a channelz trace event containing the new +// state received from resolver implementations. +func (ccr *ccResolverWrapper) addChannelzTraceEvent(s resolver.State) { + if !logger.V(2) && !channelz.IsOn() { + return + } + var updates []string + var oldSC, newSC *ServiceConfig + var oldOK, newOK bool + if ccr.curState.ServiceConfig != nil { + oldSC, oldOK = ccr.curState.ServiceConfig.Config.(*ServiceConfig) + } + if s.ServiceConfig != nil { + newSC, newOK = s.ServiceConfig.Config.(*ServiceConfig) + } + if oldOK != newOK || (oldOK && newOK && oldSC.rawJSONString != newSC.rawJSONString) { + updates = append(updates, "service config updated") + } + if len(ccr.curState.Addresses) > 0 && len(s.Addresses) == 0 { + updates = append(updates, "resolver returned an empty address list") + } else if len(ccr.curState.Addresses) == 0 && len(s.Addresses) > 0 { + updates = append(updates, "resolver returned new addresses") + } + channelz.Infof(logger, ccr.cc.channelz, "Resolver state updated: %s (%v)", pretty.ToJSON(s), strings.Join(updates, "; ")) +} + +func addressesToEndpoints(addrs []resolver.Address) []resolver.Endpoint { + endpoints := make([]resolver.Endpoint, 0, len(addrs)) + for _, a := range addrs { + ep := resolver.Endpoint{Addresses: []resolver.Address{a}, Attributes: a.BalancerAttributes} + ep.Addresses[0].BalancerAttributes = nil + endpoints = append(endpoints, ep) + } + return endpoints +} diff --git a/rpc_util.go b/rpc_util.go new file mode 100644 index 0000000..6f1bb2e --- /dev/null +++ b/rpc_util.go @@ -0,0 +1,1205 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "compress/gzip" + "context" + "encoding/binary" + "fmt" + "io" + "math" + "strings" + "sync" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/encoding/proto" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpcutil" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +func init() { + internal.AcceptCompressors = acceptCompressors +} + +// Compressor defines the interface gRPC uses to compress a message. +// +// Deprecated: use package encoding. +type Compressor interface { + // Do compresses p into w. + Do(w io.Writer, p []byte) error + // Type returns the compression algorithm the Compressor uses. + Type() string +} + +type gzipCompressor struct { + pool sync.Pool +} + +// NewGZIPCompressor creates a Compressor based on GZIP. +// +// Deprecated: use package encoding/gzip. +func NewGZIPCompressor() Compressor { + c, _ := NewGZIPCompressorWithLevel(gzip.DefaultCompression) + return c +} + +// NewGZIPCompressorWithLevel is like NewGZIPCompressor but specifies the gzip compression level instead +// of assuming DefaultCompression. +// +// The error returned will be nil if the level is valid. +// +// Deprecated: use package encoding/gzip. +func NewGZIPCompressorWithLevel(level int) (Compressor, error) { + if level < gzip.DefaultCompression || level > gzip.BestCompression { + return nil, fmt.Errorf("grpc: invalid compression level: %d", level) + } + return &gzipCompressor{ + pool: sync.Pool{ + New: func() any { + w, err := gzip.NewWriterLevel(io.Discard, level) + if err != nil { + panic(err) + } + return w + }, + }, + }, nil +} + +func (c *gzipCompressor) Do(w io.Writer, p []byte) error { + z := c.pool.Get().(*gzip.Writer) + defer c.pool.Put(z) + z.Reset(w) + if _, err := z.Write(p); err != nil { + return err + } + return z.Close() +} + +func (c *gzipCompressor) Type() string { + return "gzip" +} + +// Decompressor defines the interface gRPC uses to decompress a message. +// +// Deprecated: use package encoding. +type Decompressor interface { + // Do reads the data from r and uncompress them. + Do(r io.Reader) ([]byte, error) + // Type returns the compression algorithm the Decompressor uses. + Type() string +} + +type gzipDecompressor struct { + pool sync.Pool +} + +// NewGZIPDecompressor creates a Decompressor based on GZIP. +// +// Deprecated: use package encoding/gzip. +func NewGZIPDecompressor() Decompressor { + return &gzipDecompressor{} +} + +func (d *gzipDecompressor) Do(r io.Reader) ([]byte, error) { + return d.doWithMaxSize(r, math.MaxInt64) +} + +// doWithMaxSize behaves like Do but caps the size of the decompressed +// payload at maxMessageSize+1 bytes. The Decompressor interface does not +// allow extra parameters, so callers inside the package type-assert to +// *gzipDecompressor to invoke this method directly. The +1 byte makes it +// possible for the caller to detect that the limit was exceeded and +// return ResourceExhausted instead of materializing an unbounded payload. +func (d *gzipDecompressor) doWithMaxSize(r io.Reader, maxMessageSize int64) ([]byte, error) { + var z *gzip.Reader + switch maybeZ := d.pool.Get().(type) { + case nil: + newZ, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + z = newZ + case *gzip.Reader: + z = maybeZ + if err := z.Reset(r); err != nil { + d.pool.Put(z) + return nil, err + } + } + + defer func() { + z.Close() + d.pool.Put(z) + }() + var src io.Reader = z + if maxMessageSize < math.MaxInt64 { + src = io.LimitReader(z, maxMessageSize+1) + } + return io.ReadAll(src) +} + +func (d *gzipDecompressor) Type() string { + return "gzip" +} + +// callInfo contains all related configuration and information about an RPC. +type callInfo struct { + compressorName string + failFast bool + maxReceiveMessageSize *int + maxSendMessageSize *int + creds credentials.PerRPCCredentials + contentSubtype string + codec baseCodec + maxRetryRPCBufferSize int + authority string + acceptedResponseCompressors []string +} + +func acceptedCompressorAllows(allowed []string, name string) bool { + if allowed == nil { + return true + } + if name == "" || name == encoding.Identity { + return true + } + for _, a := range allowed { + if a == name { + return true + } + } + return false +} + +func defaultCallInfo() *callInfo { + return &callInfo{ + failFast: true, + maxRetryRPCBufferSize: 256 * 1024, // 256KB + } +} + +func newAcceptedCompressionConfig(names []string) ([]string, error) { + if len(names) == 0 { + return nil, nil + } + var allowed []string + seen := make(map[string]struct{}, len(names)) + for _, name := range names { + name = strings.TrimSpace(name) + if name == "" || name == encoding.Identity { + continue + } + if !grpcutil.IsCompressorNameRegistered(name) { + return nil, status.Errorf(codes.InvalidArgument, "grpc: compressor %q is not registered", name) + } + if _, dup := seen[name]; dup { + continue + } + seen[name] = struct{}{} + allowed = append(allowed, name) + } + return allowed, nil +} + +// CallOption configures a Call before it starts or extracts information from +// a Call after it completes. +type CallOption interface { + // before is called before the call is sent to any server. If before + // returns a non-nil error, the RPC fails with that error. + before(*callInfo) error + + // after is called after the call has completed. after cannot return an + // error, so any failures should be reported via output parameters. + after(*callInfo, *csAttempt) +} + +// EmptyCallOption does not alter the Call configuration. +// It can be embedded in another structure to carry satellite data for use +// by interceptors. +type EmptyCallOption struct{} + +func (EmptyCallOption) before(*callInfo) error { return nil } +func (EmptyCallOption) after(*callInfo, *csAttempt) {} + +// StaticMethod returns a CallOption which specifies that a call is being made +// to a method that is static, which means the method is known at compile time +// and doesn't change at runtime. This can be used as a signal to stats plugins +// that this method is safe to include as a key to a measurement. +func StaticMethod() CallOption { + return StaticMethodCallOption{} +} + +// StaticMethodCallOption is a CallOption that specifies that a call comes +// from a static method. +type StaticMethodCallOption struct { + EmptyCallOption +} + +// Header returns a CallOptions that retrieves the header metadata +// for a unary RPC. +func Header(md *metadata.MD) CallOption { + return HeaderCallOption{HeaderAddr: md} +} + +// HeaderCallOption is a CallOption for collecting response header metadata. +// The metadata field will be populated *after* the RPC completes. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type HeaderCallOption struct { + HeaderAddr *metadata.MD +} + +func (o HeaderCallOption) before(*callInfo) error { return nil } +func (o HeaderCallOption) after(_ *callInfo, attempt *csAttempt) { + *o.HeaderAddr, _ = attempt.transportStream.Header() +} + +// Trailer returns a CallOptions that retrieves the trailer metadata +// for a unary RPC. +func Trailer(md *metadata.MD) CallOption { + return TrailerCallOption{TrailerAddr: md} +} + +// TrailerCallOption is a CallOption for collecting response trailer metadata. +// The metadata field will be populated *after* the RPC completes. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type TrailerCallOption struct { + TrailerAddr *metadata.MD +} + +func (o TrailerCallOption) before(*callInfo) error { return nil } +func (o TrailerCallOption) after(_ *callInfo, attempt *csAttempt) { + *o.TrailerAddr = attempt.transportStream.Trailer() +} + +// Peer returns a CallOption that retrieves peer information for a unary RPC. +// The peer field will be populated *after* the RPC completes. +func Peer(p *peer.Peer) CallOption { + return PeerCallOption{PeerAddr: p} +} + +// PeerCallOption is a CallOption for collecting the identity of the remote +// peer. The peer field will be populated *after* the RPC completes. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type PeerCallOption struct { + PeerAddr *peer.Peer +} + +func (o PeerCallOption) before(*callInfo) error { return nil } +func (o PeerCallOption) after(_ *callInfo, attempt *csAttempt) { + if x, ok := peer.FromContext(attempt.transportStream.Context()); ok { + *o.PeerAddr = *x + } +} + +// WaitForReady configures the RPC's behavior when the client is in +// TRANSIENT_FAILURE, which occurs when all addresses fail to connect. If +// waitForReady is false, the RPC will fail immediately. Otherwise, the client +// will wait until a connection becomes available or the RPC's deadline is +// reached. +// +// By default, RPCs do not "wait for ready". +func WaitForReady(waitForReady bool) CallOption { + return FailFastCallOption{FailFast: !waitForReady} +} + +// FailFast is the opposite of WaitForReady. +// +// Deprecated: use WaitForReady. +func FailFast(failFast bool) CallOption { + return FailFastCallOption{FailFast: failFast} +} + +// FailFastCallOption is a CallOption for indicating whether an RPC should fail +// fast or not. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type FailFastCallOption struct { + FailFast bool +} + +func (o FailFastCallOption) before(c *callInfo) error { + c.failFast = o.FailFast + return nil +} +func (o FailFastCallOption) after(*callInfo, *csAttempt) {} + +// OnFinish returns a CallOption that configures a callback to be called when +// the call completes. The error passed to the callback is the status of the +// RPC, and may be nil. The onFinish callback provided will only be called once +// by gRPC. This is mainly used to be used by streaming interceptors, to be +// notified when the RPC completes along with information about the status of +// the RPC. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func OnFinish(onFinish func(err error)) CallOption { + return OnFinishCallOption{ + OnFinish: onFinish, + } +} + +// OnFinishCallOption is CallOption that indicates a callback to be called when +// the call completes. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type OnFinishCallOption struct { + OnFinish func(error) +} + +func (o OnFinishCallOption) before(*callInfo) error { + return nil +} + +func (o OnFinishCallOption) after(*callInfo, *csAttempt) {} + +// MaxCallRecvMsgSize returns a CallOption which sets the maximum message size +// in bytes the client can receive. If this is not set, gRPC uses the default +// 4MB. +func MaxCallRecvMsgSize(bytes int) CallOption { + return MaxRecvMsgSizeCallOption{MaxRecvMsgSize: bytes} +} + +// MaxRecvMsgSizeCallOption is a CallOption that indicates the maximum message +// size in bytes the client can receive. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type MaxRecvMsgSizeCallOption struct { + MaxRecvMsgSize int +} + +func (o MaxRecvMsgSizeCallOption) before(c *callInfo) error { + c.maxReceiveMessageSize = &o.MaxRecvMsgSize + return nil +} +func (o MaxRecvMsgSizeCallOption) after(*callInfo, *csAttempt) {} + +// CallAuthority returns a CallOption that sets the HTTP/2 :authority header of +// an RPC to the specified value. When using CallAuthority, the credentials in +// use must implement the AuthorityValidator interface. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a later +// release. +func CallAuthority(authority string) CallOption { + return AuthorityOverrideCallOption{Authority: authority} +} + +// AuthorityOverrideCallOption is a CallOption that indicates the HTTP/2 +// :authority header value to use for the call. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a later +// release. +type AuthorityOverrideCallOption struct { + Authority string +} + +func (o AuthorityOverrideCallOption) before(c *callInfo) error { + c.authority = o.Authority + return nil +} + +func (o AuthorityOverrideCallOption) after(*callInfo, *csAttempt) {} + +// MaxCallSendMsgSize returns a CallOption which sets the maximum message size +// in bytes the client can send. If this is not set, gRPC uses the default +// `math.MaxInt32`. +func MaxCallSendMsgSize(bytes int) CallOption { + return MaxSendMsgSizeCallOption{MaxSendMsgSize: bytes} +} + +// MaxSendMsgSizeCallOption is a CallOption that indicates the maximum message +// size in bytes the client can send. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type MaxSendMsgSizeCallOption struct { + MaxSendMsgSize int +} + +func (o MaxSendMsgSizeCallOption) before(c *callInfo) error { + c.maxSendMessageSize = &o.MaxSendMsgSize + return nil +} +func (o MaxSendMsgSizeCallOption) after(*callInfo, *csAttempt) {} + +// PerRPCCredentials returns a CallOption that sets credentials.PerRPCCredentials +// for a call. +func PerRPCCredentials(creds credentials.PerRPCCredentials) CallOption { + return PerRPCCredsCallOption{Creds: creds} +} + +// PerRPCCredsCallOption is a CallOption that indicates the per-RPC +// credentials to use for the call. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type PerRPCCredsCallOption struct { + Creds credentials.PerRPCCredentials +} + +func (o PerRPCCredsCallOption) before(c *callInfo) error { + c.creds = o.Creds + return nil +} +func (o PerRPCCredsCallOption) after(*callInfo, *csAttempt) {} + +// UseCompressor returns a CallOption which sets the compressor used when +// sending the request. If WithCompressor is also set, UseCompressor has +// higher priority. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func UseCompressor(name string) CallOption { + return CompressorCallOption{CompressorType: name} +} + +// CompressorCallOption is a CallOption that indicates the compressor to use. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type CompressorCallOption struct { + CompressorType string +} + +func (o CompressorCallOption) before(c *callInfo) error { + c.compressorName = o.CompressorType + return nil +} +func (o CompressorCallOption) after(*callInfo, *csAttempt) {} + +// acceptCompressors returns a CallOption that limits the compression algorithms +// advertised in the grpc-accept-encoding header for response messages. +// Compression algorithms not in the provided list will not be advertised, and +// responses compressed with non-listed algorithms will be rejected. +func acceptCompressors(names ...string) CallOption { + cp := append([]string(nil), names...) + return acceptCompressorsCallOption{names: cp} +} + +// acceptCompressorsCallOption is a CallOption that limits response compression. +type acceptCompressorsCallOption struct { + names []string +} + +func (o acceptCompressorsCallOption) before(c *callInfo) error { + allowed, err := newAcceptedCompressionConfig(o.names) + if err != nil { + return err + } + c.acceptedResponseCompressors = allowed + return nil +} + +func (acceptCompressorsCallOption) after(*callInfo, *csAttempt) {} + +// CallContentSubtype returns a CallOption that will set the content-subtype +// for a call. For example, if content-subtype is "json", the Content-Type over +// the wire will be "application/grpc+json". The content-subtype is converted +// to lowercase before being included in Content-Type. See Content-Type on +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +// +// If ForceCodec is not also used, the content-subtype will be used to look up +// the Codec to use in the registry controlled by RegisterCodec. See the +// documentation on RegisterCodec for details on registration. The lookup of +// content-subtype is case-insensitive. If no such Codec is found, the call +// will result in an error with code codes.Internal. +// +// If ForceCodec is also used, that Codec will be used for all request and +// response messages, with the content-subtype set to the given contentSubtype +// here for requests. +func CallContentSubtype(contentSubtype string) CallOption { + return ContentSubtypeCallOption{ContentSubtype: strings.ToLower(contentSubtype)} +} + +// ContentSubtypeCallOption is a CallOption that indicates the content-subtype +// used for marshaling messages. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type ContentSubtypeCallOption struct { + ContentSubtype string +} + +func (o ContentSubtypeCallOption) before(c *callInfo) error { + c.contentSubtype = o.ContentSubtype + return nil +} +func (o ContentSubtypeCallOption) after(*callInfo, *csAttempt) {} + +// ForceCodec returns a CallOption that will set codec to be used for all +// request and response messages for a call. The result of calling Name() will +// be used as the content-subtype after converting to lowercase, unless +// CallContentSubtype is also used. +// +// See Content-Type on +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. Also see the documentation on RegisterCodec and +// CallContentSubtype for more details on the interaction between Codec and +// content-subtype. +// +// This function is provided for advanced users; prefer to use only +// CallContentSubtype to select a registered codec instead. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func ForceCodec(codec encoding.Codec) CallOption { + return ForceCodecCallOption{Codec: codec} +} + +// ForceCodecCallOption is a CallOption that indicates the codec used for +// marshaling messages. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type ForceCodecCallOption struct { + Codec encoding.Codec +} + +func (o ForceCodecCallOption) before(c *callInfo) error { + c.codec = newCodecV1Bridge(o.Codec) + return nil +} +func (o ForceCodecCallOption) after(*callInfo, *csAttempt) {} + +// ForceCodecV2 returns a CallOption that will set codec to be used for all +// request and response messages for a call. The result of calling Name() will +// be used as the content-subtype after converting to lowercase, unless +// CallContentSubtype is also used. +// +// See Content-Type on +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. Also see the documentation on RegisterCodec and +// CallContentSubtype for more details on the interaction between Codec and +// content-subtype. +// +// This function is provided for advanced users; prefer to use only +// CallContentSubtype to select a registered codec instead. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func ForceCodecV2(codec encoding.CodecV2) CallOption { + return ForceCodecV2CallOption{CodecV2: codec} +} + +// ForceCodecV2CallOption is a CallOption that indicates the codec used for +// marshaling messages. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type ForceCodecV2CallOption struct { + CodecV2 encoding.CodecV2 +} + +func (o ForceCodecV2CallOption) before(c *callInfo) error { + c.codec = o.CodecV2 + return nil +} + +func (o ForceCodecV2CallOption) after(*callInfo, *csAttempt) {} + +// CallCustomCodec behaves like ForceCodec, but accepts a grpc.Codec instead of +// an encoding.Codec. +// +// Deprecated: use ForceCodec instead. +func CallCustomCodec(codec Codec) CallOption { + return CustomCodecCallOption{Codec: codec} +} + +// CustomCodecCallOption is a CallOption that indicates the codec used for +// marshaling messages. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type CustomCodecCallOption struct { + Codec Codec +} + +func (o CustomCodecCallOption) before(c *callInfo) error { + c.codec = newCodecV0Bridge(o.Codec) + return nil +} +func (o CustomCodecCallOption) after(*callInfo, *csAttempt) {} + +// MaxRetryRPCBufferSize returns a CallOption that limits the amount of memory +// used for buffering this RPC's requests for retry purposes. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func MaxRetryRPCBufferSize(bytes int) CallOption { + return MaxRetryRPCBufferSizeCallOption{bytes} +} + +// MaxRetryRPCBufferSizeCallOption is a CallOption indicating the amount of +// memory to be used for caching this RPC for retry purposes. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type MaxRetryRPCBufferSizeCallOption struct { + MaxRetryRPCBufferSize int +} + +func (o MaxRetryRPCBufferSizeCallOption) before(c *callInfo) error { + c.maxRetryRPCBufferSize = o.MaxRetryRPCBufferSize + return nil +} +func (o MaxRetryRPCBufferSizeCallOption) after(*callInfo, *csAttempt) {} + +// The format of the payload: compressed or not? +type payloadFormat uint8 + +const ( + compressionNone payloadFormat = 0 // no compression + compressionMade payloadFormat = 1 // compressed +) + +func (pf payloadFormat) isCompressed() bool { + return pf == compressionMade +} + +type streamReader interface { + ReadMessageHeader(header []byte) error + Read(n int) (mem.BufferSlice, error) +} + +// noCopy may be embedded into structs which must not be copied +// after the first use. +// +// See https://golang.org/issues/8005#issuecomment-190753527 +// for details. +type noCopy struct { +} + +func (*noCopy) Lock() {} +func (*noCopy) Unlock() {} + +// parser reads complete gRPC messages from the underlying reader. +type parser struct { + _ noCopy + // r is the underlying reader. + // See the comment on recvMsg for the permissible + // error types. + r streamReader + + // The header of a gRPC message. Find more detail at + // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md + header [5]byte + + // bufferPool is the pool of shared receive buffers. + bufferPool mem.BufferPool +} + +// recvMsg reads a complete gRPC message from the stream. +// +// It returns the message and its payload (compression/encoding) +// format. The caller owns the returned msg memory. +// +// If there is an error, possible values are: +// - io.EOF, when no messages remain +// - io.ErrUnexpectedEOF +// - of type transport.ConnectionError +// - an error from the status package +// +// No other error values or types must be returned, which also means +// that the underlying streamReader must not return an incompatible +// error. +func (p *parser) recvMsg(maxReceiveMessageSize int) (payloadFormat, mem.BufferSlice, error) { + err := p.r.ReadMessageHeader(p.header[:]) + if err != nil { + return 0, nil, err + } + + pf := payloadFormat(p.header[0]) + length := binary.BigEndian.Uint32(p.header[1:]) + + if int64(length) > int64(maxInt) { + return 0, nil, status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max length allowed on current machine (%d vs. %d)", length, maxInt) + } + if int(length) > maxReceiveMessageSize { + return 0, nil, status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", length, maxReceiveMessageSize) + } + + data, err := p.r.Read(int(length)) + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return 0, nil, err + } + return pf, data, nil +} + +// encode serializes msg and returns a buffer containing the message, or an +// error if it is too large to be transmitted by grpc. If msg is nil, it +// generates an empty message. +func encode(c baseCodec, msg any) (mem.BufferSlice, error) { + if msg == nil { // NOTE: typed nils will not be caught by this check + return nil, nil + } + b, err := c.Marshal(msg) + if err != nil { + return nil, status.Errorf(codes.Internal, "grpc: error while marshaling: %v", err.Error()) + } + if bufSize := uint(b.Len()); bufSize > math.MaxUint32 { + b.Free() + return nil, status.Errorf(codes.ResourceExhausted, "grpc: message too large (%d bytes)", bufSize) + } + return b, nil +} + +// compress returns the input bytes compressed by compressor or cp. +// If both compressors are nil, or if the message has zero length, returns nil, +// indicating no compression was done. +// +// TODO(dfawley): eliminate cp parameter by wrapping Compressor in an encoding.Compressor. +func compress(in mem.BufferSlice, cp Compressor, compressor encoding.Compressor, pool mem.BufferPool) (mem.BufferSlice, payloadFormat, error) { + if (compressor == nil && cp == nil) || in.Len() == 0 { + return nil, compressionNone, nil + } + var out mem.BufferSlice + w := mem.NewWriter(&out, pool) + wrapErr := func(err error) error { + out.Free() + return status.Errorf(codes.Internal, "grpc: error while compressing: %v", err.Error()) + } + if compressor != nil { + z, err := compressor.Compress(w) + if err != nil { + return nil, compressionNone, wrapErr(err) + } + for _, b := range in { + if _, err := z.Write(b.ReadOnlyData()); err != nil { + return nil, compressionNone, wrapErr(err) + } + } + if err := z.Close(); err != nil { + return nil, compressionNone, wrapErr(err) + } + } else { + // This is obviously really inefficient since it fully materializes the data, but + // there is no way around this with the old Compressor API. At least it attempts + // to return the buffer to the provider, in the hopes it can be reused (maybe + // even by a subsequent call to this very function). + buf := in.MaterializeToBuffer(pool) + defer buf.Free() + if err := cp.Do(w, buf.ReadOnlyData()); err != nil { + return nil, compressionNone, wrapErr(err) + } + } + return out, compressionMade, nil +} + +const ( + payloadLen = 1 + sizeLen = 4 + headerLen = payloadLen + sizeLen +) + +// msgHeader returns a 5-byte header for the message being transmitted and the +// payload, which is compData if non-nil or data otherwise. +func msgHeader(data, compData mem.BufferSlice, pf payloadFormat) (hdr []byte, payload mem.BufferSlice) { + hdr = make([]byte, headerLen) + hdr[0] = byte(pf) + + var length uint32 + if pf.isCompressed() { + length = uint32(compData.Len()) + payload = compData + } else { + length = uint32(data.Len()) + payload = data + } + + // Write length of payload into buf + binary.BigEndian.PutUint32(hdr[payloadLen:], length) + return hdr, payload +} + +func outPayload(client bool, msg any, dataLength, payloadLength int, t time.Time) *stats.OutPayload { + return &stats.OutPayload{ + Client: client, + Payload: msg, + Length: dataLength, + WireLength: payloadLength + headerLen, + CompressedLength: payloadLength, + SentTime: t, + } +} + +func checkRecvPayload(pf payloadFormat, recvCompress string, haveCompressor bool, isServer bool) *status.Status { + switch pf { + case compressionNone: + case compressionMade: + if recvCompress == "" || recvCompress == encoding.Identity { + return status.New(codes.Internal, "grpc: compressed flag set with identity or empty encoding") + } + if !haveCompressor { + if isServer { + return status.Newf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", recvCompress) + } + return status.Newf(codes.Internal, "grpc: Decompressor is not installed for grpc-encoding %q", recvCompress) + } + default: + return status.Newf(codes.Internal, "grpc: received unexpected payload format %d", pf) + } + return nil +} + +type payloadInfo struct { + compressedLength int // The compressed length got from wire. + uncompressedBytes mem.BufferSlice +} + +func (p *payloadInfo) free() { + if p != nil && p.uncompressedBytes != nil { + p.uncompressedBytes.Free() + } +} + +// recvAndDecompress reads a message from the stream, decompressing it if necessary. +// +// Cancelling the returned cancel function releases the buffer back to the pool. So the caller should cancel as soon as +// the buffer is no longer needed. +// TODO: Refactor this function to reduce the number of arguments. +// See: https://google.github.io/styleguide/go/best-practices.html#function-argument-lists +func recvAndDecompress(p *parser, s recvCompressor, dc Decompressor, maxReceiveMessageSize int, payInfo *payloadInfo, compressor encoding.Compressor, isServer bool) (out mem.BufferSlice, err error) { + pf, compressed, err := p.recvMsg(maxReceiveMessageSize) + if err != nil { + return nil, err + } + + compressedLength := compressed.Len() + + if st := checkRecvPayload(pf, s.RecvCompress(), compressor != nil || dc != nil, isServer); st != nil { + compressed.Free() + return nil, st.Err() + } + + if pf.isCompressed() { + defer compressed.Free() + // To match legacy behavior, if the decompressor is set by WithDecompressor or RPCDecompressor, + // use this decompressor as the default. + out, err = decompress(compressor, compressed, dc, maxReceiveMessageSize, p.bufferPool) + if err != nil { + return nil, err + } + } else { + out = compressed + } + + if payInfo != nil { + payInfo.compressedLength = compressedLength + out.Ref() + payInfo.uncompressedBytes = out + } + + return out, nil +} + +// decompress processes the given data by decompressing it using either +// a custom decompressor or a standard compressor. If a custom decompressor +// is provided, it takes precedence. The function validates that +// the decompressed data does not exceed the specified maximum size and returns +// an error if this limit is exceeded. On success, it returns the decompressed +// data. Otherwise, it returns an error if decompression fails or the data +// exceeds the size limit. +func decompress(compressor encoding.Compressor, d mem.BufferSlice, dc Decompressor, maxReceiveMessageSize int, pool mem.BufferPool) (mem.BufferSlice, error) { + if dc != nil { + r := d.Reader() + // For the built-in gzip decompressor, bound the decompressed output + // at maxReceiveMessageSize+1 so that a small but highly compressed + // payload (a "zip bomb") cannot expand to gigabytes in memory before + // the post-decompression size check below has a chance to fire. The + // Decompressor interface does not accept an extra size parameter, + // so we type-assert to invoke a size-aware helper. Third-party + // Decompressor implementations keep the original Do behavior. + var uncompressed []byte + var err error + if gd, ok := dc.(*gzipDecompressor); ok { + uncompressed, err = gd.doWithMaxSize(r, int64(maxReceiveMessageSize)) + } else { + uncompressed, err = dc.Do(r) + } + if err != nil { + r.Close() // ensure buffers are reused + return nil, status.Errorf(codes.Internal, "grpc: failed to decompress the received message: %v", err) + } + if len(uncompressed) > maxReceiveMessageSize { + r.Close() // ensure buffers are reused + return nil, status.Errorf(codes.ResourceExhausted, "grpc: message after decompression larger than max (%d vs. %d)", len(uncompressed), maxReceiveMessageSize) + } + return mem.BufferSlice{mem.SliceBuffer(uncompressed)}, nil + } + if compressor != nil { + r := d.Reader() + dcReader, err := compressor.Decompress(r) + if err != nil { + r.Close() // ensure buffers are reused + return nil, status.Errorf(codes.Internal, "grpc: failed to decompress the message: %v", err) + } + if closer, ok := dcReader.(io.Closer); ok { + defer closer.Close() + } + + // Read at most one byte more than the limit from the decompressor. + // Unless the limit is MaxInt64, in which case, that's impossible, so + // apply no limit. + if limit := int64(maxReceiveMessageSize); limit < math.MaxInt64 { + dcReader = io.LimitReader(dcReader, limit+1) + } + out, err := mem.ReadAll(dcReader, pool) + if err != nil { + r.Close() // ensure buffers are reused + out.Free() + return nil, status.Errorf(codes.Internal, "grpc: failed to read decompressed data: %v", err) + } + + if out.Len() > maxReceiveMessageSize { + r.Close() // ensure buffers are reused + out.Free() + return nil, status.Errorf(codes.ResourceExhausted, "grpc: received message after decompression larger than max %d", maxReceiveMessageSize) + } + return out, nil + } + return nil, status.Errorf(codes.Internal, "grpc: no decompressor available for compressed payload") +} + +type recvCompressor interface { + RecvCompress() string +} + +// For the two compressor parameters, both should not be set, but if they are, +// dc takes precedence over compressor. +// TODO(dfawley): wrap the old compressor/decompressor using the new API? +func recv(p *parser, c baseCodec, s recvCompressor, dc Decompressor, m any, maxReceiveMessageSize int, payInfo *payloadInfo, compressor encoding.Compressor, isServer bool) error { + data, err := recvAndDecompress(p, s, dc, maxReceiveMessageSize, payInfo, compressor, isServer) + if err != nil { + return err + } + + // If the codec wants its own reference to the data, it can get it. Otherwise, always + // free the buffers. + defer data.Free() + + if err := c.Unmarshal(data, m); err != nil { + return status.Errorf(codes.Internal, "grpc: failed to unmarshal the received message: %v", err) + } + + return nil +} + +// Information about RPC +type rpcInfo struct { + failfast bool + preloaderInfo compressorInfo +} + +// Information about Preloader +// Responsible for storing codec, and compressors +// If stream (s) has context s.Context which stores rpcInfo that has non nil +// pointers to codec, and compressors, then we can use preparedMsg for Async message prep +// and reuse marshalled bytes +type compressorInfo struct { + codec baseCodec + cp Compressor + comp encoding.Compressor +} + +type rpcInfoContextKey struct{} + +func newContextWithRPCInfo(ctx context.Context, failfast bool, codec baseCodec, cp Compressor, comp encoding.Compressor) context.Context { + return context.WithValue(ctx, rpcInfoContextKey{}, &rpcInfo{ + failfast: failfast, + preloaderInfo: compressorInfo{ + codec: codec, + cp: cp, + comp: comp, + }, + }) +} + +func rpcInfoFromContext(ctx context.Context) (s *rpcInfo, ok bool) { + s, ok = ctx.Value(rpcInfoContextKey{}).(*rpcInfo) + return +} + +// Code returns the error code for err if it was produced by the rpc system. +// Otherwise, it returns codes.Unknown. +// +// Deprecated: use status.Code instead. +func Code(err error) codes.Code { + return status.Code(err) +} + +// ErrorDesc returns the error description of err if it was produced by the rpc system. +// Otherwise, it returns err.Error() or empty string when err is nil. +// +// Deprecated: use status.Convert and Message method instead. +func ErrorDesc(err error) string { + return status.Convert(err).Message() +} + +// Errorf returns an error containing an error code and a description; +// Errorf returns nil if c is OK. +// +// Deprecated: use status.Errorf instead. +func Errorf(c codes.Code, format string, a ...any) error { + return status.Errorf(c, format, a...) +} + +var errContextCanceled = status.Error(codes.Canceled, context.Canceled.Error()) +var errContextDeadline = status.Error(codes.DeadlineExceeded, context.DeadlineExceeded.Error()) + +// toRPCErr converts an error into an error from the status package. +func toRPCErr(err error) error { + switch err { + case nil, io.EOF: + return err + case context.DeadlineExceeded: + return errContextDeadline + case context.Canceled: + return errContextCanceled + case io.ErrUnexpectedEOF: + return status.Error(codes.Internal, err.Error()) + } + + switch e := err.(type) { + case transport.ConnectionError: + return status.Error(codes.Unavailable, e.Desc) + case *transport.NewStreamError: + return toRPCErr(e.Err) + } + + if _, ok := status.FromError(err); ok { + return err + } + + return status.Error(codes.Unknown, err.Error()) +} + +// setCallInfoCodec should only be called after CallOptions have been applied. +func setCallInfoCodec(c *callInfo) error { + if c.codec != nil { + // codec was already set by a CallOption; use it, but set the content + // subtype if it is not set. + if c.contentSubtype == "" { + // c.codec is a baseCodec to hide the difference between grpc.Codec and + // encoding.Codec (Name vs. String method name). We only support + // setting content subtype from encoding.Codec to avoid a behavior + // change with the deprecated version. + if ec, ok := c.codec.(encoding.CodecV2); ok { + c.contentSubtype = strings.ToLower(ec.Name()) + } + } + return nil + } + + if c.contentSubtype == "" { + // No codec specified in CallOptions; use proto by default. + c.codec = getCodec(proto.Name) + return nil + } + + // c.contentSubtype is already lowercased in CallContentSubtype + c.codec = getCodec(c.contentSubtype) + if c.codec == nil { + return status.Errorf(codes.Internal, "no codec registered for content-subtype %s", c.contentSubtype) + } + return nil +} + +// The SupportPackageIsVersion variables are referenced from generated protocol +// buffer files to ensure compatibility with the gRPC version used. The latest +// support package version is 9. +// +// Older versions are kept for compatibility. +// +// These constants should not be referenced from any other code. +const ( + SupportPackageIsVersion3 = true + SupportPackageIsVersion4 = true + SupportPackageIsVersion5 = true + SupportPackageIsVersion6 = true + SupportPackageIsVersion7 = true + SupportPackageIsVersion8 = true + SupportPackageIsVersion9 = true +) + +const grpcUA = "grpc-go/" + Version diff --git a/rpc_util_test.go b/rpc_util_test.go new file mode 100644 index 0000000..08a4a33 --- /dev/null +++ b/rpc_util_test.go @@ -0,0 +1,676 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "math" + "reflect" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/encoding" + _ "google.golang.org/grpc/encoding/gzip" + protoenc "google.golang.org/grpc/encoding/proto" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/status" + perfpb "google.golang.org/grpc/test/codec_perf" + "google.golang.org/protobuf/proto" +) + +const ( + defaultDecompressedData = "default decompressed data" + decompressionErrorMsg = "invalid compression format" +) + +type testCompressorForRegistry struct { + name string +} + +func (c *testCompressorForRegistry) Compress(w io.Writer) (io.WriteCloser, error) { + return &testWriteCloser{w}, nil +} + +func (c *testCompressorForRegistry) Decompress(r io.Reader) (io.Reader, error) { + return r, nil +} + +func (c *testCompressorForRegistry) Name() string { + return c.name +} + +type testWriteCloser struct { + io.Writer +} + +func (w *testWriteCloser) Close() error { + return nil +} + +func (s) TestNewAcceptedCompressionConfig(t *testing.T) { + // Register a test compressor for multi-compressor tests + testCompressor := &testCompressorForRegistry{name: "test-compressor"} + encoding.RegisterCompressor(testCompressor) + defer func() { + // Unregister the test compressor + encoding.RegisterCompressor(&testCompressorForRegistry{name: "test-compressor"}) + }() + + tests := []struct { + name string + input []string + wantAllowed []string + wantErr bool + }{ + { + name: "identity-only", + input: nil, + wantAllowed: nil, + }, + { + name: "single valid", + input: []string{"gzip"}, + wantAllowed: []string{"gzip"}, + }, + { + name: "dedupe and trim", + input: []string{" gzip ", "gzip"}, + wantAllowed: []string{"gzip"}, + }, + { + name: "ignores identity", + input: []string{"identity", "gzip"}, + wantAllowed: []string{"gzip"}, + }, + { + name: "explicit identity only", + input: []string{"identity"}, + wantAllowed: nil, + }, + { + name: "invalid compressor", + input: []string{"does-not-exist"}, + wantErr: true, + }, + { + name: "only whitespace", + input: []string{" ", "\t"}, + wantAllowed: nil, + }, + { + name: "multiple valid compressors", + input: []string{"gzip", "test-compressor"}, + wantAllowed: []string{"gzip", "test-compressor"}, + }, + { + name: "multiple with identity and whitespace", + input: []string{"gzip", "identity", " test-compressor ", " "}, + wantAllowed: []string{"gzip", "test-compressor"}, + }, + { + name: "empty string in list", + input: []string{"gzip", "", "test-compressor"}, + wantAllowed: []string{"gzip", "test-compressor"}, + }, + { + name: "mixed valid and invalid", + input: []string{"gzip", "invalid-comp"}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + allowed, err := newAcceptedCompressionConfig(tt.input) + if (err != nil) != tt.wantErr { + t.Fatalf("newAcceptedCompressionConfig(%v) error = %v, wantErr %v", tt.input, err, tt.wantErr) + } + if tt.wantErr { + return + } + if diff := cmp.Diff(tt.wantAllowed, allowed); diff != "" { + t.Fatalf("allowed diff (-want +got): %v", diff) + } + }) + } +} + +type fullReader struct { + data []byte +} + +func (f *fullReader) ReadMessageHeader(header []byte) error { + buf, err := f.Read(len(header)) + defer buf.Free() + if err != nil { + return err + } + + buf.CopyTo(header) + return nil +} + +func (f *fullReader) Read(n int) (mem.BufferSlice, error) { + if n == 0 { + return nil, nil + } + + if len(f.data) == 0 { + return nil, io.EOF + } + + if len(f.data) < n { + data := f.data + f.data = nil + return mem.BufferSlice{mem.SliceBuffer(data)}, io.ErrUnexpectedEOF + } + + buf := f.data[:n] + f.data = f.data[n:] + + return mem.BufferSlice{mem.SliceBuffer(buf)}, nil +} + +var _ CallOption = EmptyCallOption{} // ensure EmptyCallOption implements the interface + +func (s) TestSimpleParsing(t *testing.T) { + bigMsg := bytes.Repeat([]byte{'x'}, 1<<24) + for _, test := range []struct { + // input + p []byte + // outputs + err error + b []byte + pt payloadFormat + }{ + {nil, io.EOF, nil, compressionNone}, + {[]byte{0, 0, 0, 0, 0}, nil, nil, compressionNone}, + {[]byte{0, 0, 0, 0, 1, 'a'}, nil, []byte{'a'}, compressionNone}, + {[]byte{1, 0}, io.ErrUnexpectedEOF, nil, compressionNone}, + {[]byte{0, 0, 0, 0, 10, 'a'}, io.ErrUnexpectedEOF, nil, compressionNone}, + // Check that messages with length >= 2^24 are parsed. + {append([]byte{0, 1, 0, 0, 0}, bigMsg...), nil, bigMsg, compressionNone}, + } { + buf := &fullReader{test.p} + parser := &parser{r: buf, bufferPool: mem.DefaultBufferPool()} + pt, b, err := parser.recvMsg(math.MaxInt32) + if err != test.err || !bytes.Equal(b.Materialize(), test.b) || pt != test.pt { + t.Fatalf("parser{%v}.recvMsg(_) = %v, %v, %v\nwant %v, %v, %v", test.p, pt, b, err, test.pt, test.b, test.err) + } + } +} + +func (s) TestMultipleParsing(t *testing.T) { + // Set a byte stream consists of 3 messages with their headers. + p := []byte{0, 0, 0, 0, 1, 'a', 0, 0, 0, 0, 2, 'b', 'c', 0, 0, 0, 0, 1, 'd'} + b := &fullReader{p} + parser := &parser{r: b, bufferPool: mem.DefaultBufferPool()} + + wantRecvs := []struct { + pt payloadFormat + data []byte + }{ + {compressionNone, []byte("a")}, + {compressionNone, []byte("bc")}, + {compressionNone, []byte("d")}, + } + for i, want := range wantRecvs { + pt, data, err := parser.recvMsg(math.MaxInt32) + if err != nil || pt != want.pt || !reflect.DeepEqual(data.Materialize(), want.data) { + t.Fatalf("after %d calls, parser{%v}.recvMsg(_) = %v, %v, %v\nwant %v, %v, ", + i, p, pt, data, err, want.pt, want.data) + } + } + + pt, data, err := parser.recvMsg(math.MaxInt32) + if err != io.EOF { + t.Fatalf("after %d recvMsgs calls, parser{%v}.recvMsg(_) = %v, %v, %v\nwant _, _, %v", + len(wantRecvs), p, pt, data, err, io.EOF) + } +} + +func (s) TestEncode(t *testing.T) { + for _, test := range []struct { + // input + msg proto.Message + // outputs + hdr []byte + data []byte + err error + }{ + {nil, []byte{0, 0, 0, 0, 0}, []byte{}, nil}, + } { + data, err := encode(getCodec(protoenc.Name), test.msg) + if err != test.err || !bytes.Equal(data.Materialize(), test.data) { + t.Errorf("encode(_, %v) = %v, %v; want %v, %v", test.msg, data, err, test.data, test.err) + continue + } + if hdr, _ := msgHeader(data, nil, compressionNone); !bytes.Equal(hdr, test.hdr) { + t.Errorf("msgHeader(%v, false) = %v; want %v", data, hdr, test.hdr) + } + } +} + +func (s) TestCompress(t *testing.T) { + bestCompressor, err := NewGZIPCompressorWithLevel(gzip.BestCompression) + if err != nil { + t.Fatalf("Could not initialize gzip compressor with best compression.") + } + bestSpeedCompressor, err := NewGZIPCompressorWithLevel(gzip.BestSpeed) + if err != nil { + t.Fatalf("Could not initialize gzip compressor with best speed compression.") + } + + defaultCompressor, err := NewGZIPCompressorWithLevel(gzip.BestSpeed) + if err != nil { + t.Fatalf("Could not initialize gzip compressor with default compression.") + } + + level5, err := NewGZIPCompressorWithLevel(5) + if err != nil { + t.Fatalf("Could not initialize gzip compressor with level 5 compression.") + } + + for _, test := range []struct { + // input + data []byte + cp Compressor + dc Decompressor + // outputs + err error + }{ + {make([]byte, 1024), NewGZIPCompressor(), NewGZIPDecompressor(), nil}, + {make([]byte, 1024), bestCompressor, NewGZIPDecompressor(), nil}, + {make([]byte, 1024), bestSpeedCompressor, NewGZIPDecompressor(), nil}, + {make([]byte, 1024), defaultCompressor, NewGZIPDecompressor(), nil}, + {make([]byte, 1024), level5, NewGZIPDecompressor(), nil}, + } { + b := new(bytes.Buffer) + if err := test.cp.Do(b, test.data); err != test.err { + t.Fatalf("Compressor.Do(_, %v) = %v, want %v", test.data, err, test.err) + } + if b.Len() >= len(test.data) { + t.Fatalf("The compressor fails to compress data.") + } + if p, err := test.dc.Do(b); err != nil || !bytes.Equal(test.data, p) { + t.Fatalf("Decompressor.Do(%v) = %v, %v, want %v, ", b, p, err, test.data) + } + } +} + +func (s) TestToRPCErr(t *testing.T) { + for _, test := range []struct { + // input + errIn error + // outputs + errOut error + }{ + {transport.ErrConnClosing, status.Error(codes.Unavailable, transport.ErrConnClosing.Desc)}, + {io.ErrUnexpectedEOF, status.Error(codes.Internal, io.ErrUnexpectedEOF.Error())}, + } { + err := toRPCErr(test.errIn) + if _, ok := status.FromError(err); !ok { + t.Errorf("toRPCErr{%v} returned type %T, want %T", test.errIn, err, status.Error) + } + if !testutils.StatusErrEqual(err, test.errOut) { + t.Errorf("toRPCErr{%v} = %v \nwant %v", test.errIn, err, test.errOut) + } + } +} + +// bmEncode benchmarks encoding a Protocol Buffer message containing mSize +// bytes. +func bmEncode(b *testing.B, mSize int) { + cdc := getCodec(protoenc.Name) + msg := &perfpb.Buffer{Body: make([]byte, mSize)} + encodeData, _ := encode(cdc, msg) + encodedSz := int64(len(encodeData)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + encode(cdc, msg) + } + b.SetBytes(encodedSz) +} + +func BenchmarkEncode1B(b *testing.B) { + bmEncode(b, 1) +} + +func BenchmarkEncode1KiB(b *testing.B) { + bmEncode(b, 1024) +} + +func BenchmarkEncode8KiB(b *testing.B) { + bmEncode(b, 8*1024) +} + +func BenchmarkEncode64KiB(b *testing.B) { + bmEncode(b, 64*1024) +} + +func BenchmarkEncode512KiB(b *testing.B) { + bmEncode(b, 512*1024) +} + +func BenchmarkEncode1MiB(b *testing.B) { + bmEncode(b, 1024*1024) +} + +// bmCompressor benchmarks a compressor of a Protocol Buffer message containing +// mSize bytes. +func bmCompressor(b *testing.B, mSize int, cp Compressor) { + payload := make([]byte, mSize) + cBuf := bytes.NewBuffer(make([]byte, mSize)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + cp.Do(cBuf, payload) + cBuf.Reset() + } +} + +func BenchmarkGZIPCompressor1B(b *testing.B) { + bmCompressor(b, 1, NewGZIPCompressor()) +} + +func BenchmarkGZIPCompressor1KiB(b *testing.B) { + bmCompressor(b, 1024, NewGZIPCompressor()) +} + +func BenchmarkGZIPCompressor8KiB(b *testing.B) { + bmCompressor(b, 8*1024, NewGZIPCompressor()) +} + +func BenchmarkGZIPCompressor64KiB(b *testing.B) { + bmCompressor(b, 64*1024, NewGZIPCompressor()) +} + +func BenchmarkGZIPCompressor512KiB(b *testing.B) { + bmCompressor(b, 512*1024, NewGZIPCompressor()) +} + +func BenchmarkGZIPCompressor1MiB(b *testing.B) { + bmCompressor(b, 1024*1024, NewGZIPCompressor()) +} + +// mustCompress gzip-compresses input and returns it as a BufferSlice, +// failing the test if compression fails. +func mustCompress(t *testing.T, input []byte) mem.BufferSlice { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + if _, err := gz.Write(input); err != nil { + t.Fatalf("mustCompress() failed to write data: %v", err) + } + if err := gz.Close(); err != nil { + t.Fatalf("mustCompress() failed to close gzip writer: %v", err) + } + compressedData := buf.Bytes() + return mem.BufferSlice{mem.NewBuffer(&compressedData, nil)} +} + +// MockDecompressor is a mock implementation of a decompressor used for testing purposes. +// It simulates decompression behavior, returning either decompressed data or an error based on the ShouldError flag. +type MockDecompressor struct { + ShouldError bool // Flag to control whether the decompression should simulate an error. +} + +// Do simulates decompression. It returns a predefined error if ShouldError is true, +// or a fixed set of decompressed data if ShouldError is false. +func (m *MockDecompressor) Do(_ io.Reader) ([]byte, error) { + if m.ShouldError { + return nil, errors.New(decompressionErrorMsg) + } + return []byte(defaultDecompressedData), nil +} + +// Type returns the string identifier for the MockDecompressor. +func (m *MockDecompressor) Type() string { + return "MockDecompressor" +} + +// TestDecompress tests the decompress function behaves correctly for following scenarios +// decompress successfully when message is <= maxReceiveMessageSize +// errors when message > maxReceiveMessageSize +// decompress successfully when maxReceiveMessageSize is MaxInt +// errors when the decompressed message has an invalid format +// errors when the decompressed message exceeds the maxReceiveMessageSize. +func (s) TestDecompress(t *testing.T) { + compressor := encoding.GetCompressor("gzip") + validDecompressor := &MockDecompressor{ShouldError: false} + invalidFormatDecompressor := &MockDecompressor{ShouldError: true} + + testCases := []struct { + name string + input mem.BufferSlice + dc Decompressor + maxReceiveMessageSize int + want []byte + wantErr error + }{ + { + name: "Decompresses successfully with sufficient buffer size", + input: mustCompress(t, []byte("decompressed data")), + dc: nil, + maxReceiveMessageSize: 50, + want: []byte("decompressed data"), + wantErr: nil, + }, + { + name: "Fails due to exceeding maxReceiveMessageSize", + input: mustCompress(t, []byte("message that is too large")), + dc: nil, + maxReceiveMessageSize: len("message that is too large") - 1, + want: nil, + wantErr: status.Errorf(codes.ResourceExhausted, "grpc: received message after decompression larger than max %d", len("message that is too large")-1), + }, + { + name: "Decompresses to exactly maxReceiveMessageSize", + input: mustCompress(t, []byte("exact size message")), + dc: nil, + maxReceiveMessageSize: len("exact size message"), + want: []byte("exact size message"), + wantErr: nil, + }, + { + name: "Decompresses successfully with maxReceiveMessageSize MaxInt", + input: mustCompress(t, []byte("large message")), + dc: nil, + maxReceiveMessageSize: math.MaxInt, + want: []byte("large message"), + wantErr: nil, + }, + { + name: "Fails with decompression error due to invalid format", + input: mustCompress(t, []byte("invalid compressed data")), + dc: invalidFormatDecompressor, + maxReceiveMessageSize: 50, + want: nil, + wantErr: status.Errorf(codes.Internal, "grpc: failed to decompress the received message: %v", errors.New(decompressionErrorMsg)), + }, + { + name: "Fails with resourceExhausted error when decompressed message exceeds maxReceiveMessageSize", + input: mustCompress(t, []byte("large compressed data")), + dc: validDecompressor, + maxReceiveMessageSize: 20, + want: nil, + wantErr: status.Errorf(codes.ResourceExhausted, "grpc: message after decompression larger than max (%d vs. %d)", 25, 20), + }, + { + // Bombs the legacy gzipDecompressor with 1 MiB of zeros (which + // gzips down to a few KiB). doWithMaxSize must cap the read at + // maxRecv+1 bytes; the error reports exactly that size so we + // know only maxRecv+1 bytes were materialised. + name: "Legacy gzipDecompressor bounds decompressed bomb to maxReceiveMessageSize+1", + input: mustCompress(t, make([]byte, 1<<20)), + dc: NewGZIPDecompressor(), + maxReceiveMessageSize: 1024, + want: nil, + wantErr: status.Errorf(codes.ResourceExhausted, "grpc: message after decompression larger than max (%d vs. %d)", 1025, 1024), + }, + { + name: "Legacy gzipDecompressor decompresses successfully when payload fits", + input: mustCompress(t, []byte("hello legacy decompressor")), + dc: NewGZIPDecompressor(), + maxReceiveMessageSize: len("hello legacy decompressor"), + want: []byte("hello legacy decompressor"), + wantErr: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + output, err := decompress(compressor, tc.input, tc.dc, tc.maxReceiveMessageSize, mem.DefaultBufferPool()) + if !cmp.Equal(err, tc.wantErr, cmpopts.EquateErrors()) { + t.Fatalf("decompress() err = %v, wantErr = %v", err, tc.wantErr) + } + if !cmp.Equal(tc.want, output.Materialize()) { + t.Fatalf("decompress() output mismatch: got = %v, want = %v", output.Materialize(), tc.want) + } + }) + } +} + +type mockCompressor struct { + // Written to by the io.Reader on every call to Read. + ch chan<- struct{} +} + +func (m *mockCompressor) Compress(io.Writer) (io.WriteCloser, error) { + panic("unimplemented") +} + +func (m *mockCompressor) Decompress(io.Reader) (io.Reader, error) { + return m, nil +} + +func (m *mockCompressor) Read([]byte) (int, error) { + m.ch <- struct{}{} + return 1, io.EOF +} + +func (m *mockCompressor) Name() string { return "" } + +// Tests that the decompressor's Read method is not called after it returns EOF. +func (s) TestDecompress_NoReadAfterEOF(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + ch := make(chan struct{}, 10) + mc := &mockCompressor{ch: ch} + in := mem.BufferSlice{mem.NewBuffer(&[]byte{1, 2, 3}, nil)} + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + out, err := decompress(mc, in, nil, 1, mem.DefaultBufferPool()) + if err != nil { + t.Errorf("Unexpected error from decompress: %v", err) + return + } + out.Free() + }() + select { + case <-ch: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for call to compressor") + } + ctx, cancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer cancel() + select { + case <-ch: + t.Fatalf("Unexpected second compressor.Read call detected") + case <-ctx.Done(): + } + wg.Wait() +} + +type fakeCloseCompressor struct { + encoding.Compressor + io.Reader + ch chan<- struct{} +} + +func (m *fakeCloseCompressor) Decompress(r io.Reader) (io.Reader, error) { + zr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + m.Reader = zr + return m, nil +} + +func (m *fakeCloseCompressor) Close() error { + close(m.ch) + return nil +} + +// TestDecompress_ClosesReader tests that the reader returned by Decompress is +// closed when decompress returns. +func (s) TestDecompress_ClosesReader(t *testing.T) { + testCases := []struct { + name string + maxReceiveMessageSize int + wantCode codes.Code + }{ + { + name: "close_on_failure", + maxReceiveMessageSize: 1, + wantCode: codes.ResourceExhausted, + }, + { + name: "close_on_success", + maxReceiveMessageSize: 100, + wantCode: codes.OK, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + ch := make(chan struct{}) + compressor := &fakeCloseCompressor{ch: ch} + in := mustCompress(t, []byte("some data")) + out, err := decompress(compressor, in, nil, tc.maxReceiveMessageSize, mem.DefaultBufferPool()) + if status.Code(err) != tc.wantCode { + t.Fatalf("decompress() failed with error code %v, want %v", status.Code(err), tc.wantCode) + } + + out.Free() + + select { + case <-ch: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for Close to be called") + } + }) + } +} diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100755 index 0000000..3e641ba --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +fail_on_output() { + tee /dev/stderr | not read +} + +# not makes sure the command passed to it does not exit with a return code of 0. +not() { + # This is required instead of the earlier (! $COMMAND) because subshells and + # pipefail don't work the same on Darwin as in Linux. + ! "$@" +} + +# noret_grep will return 0 if zero or more lines were selected, and >1 if an +# error occurred. Suppresses grep's return code of 1 when there are no matches +# (for eg, empty file). +noret_grep() { + grep "$@" || [[ $? == 1 ]] +} + +die() { + echo "$@" >&2 + exit 1 +} diff --git a/scripts/gen-deps.sh b/scripts/gen-deps.sh new file mode 100755 index 0000000..f4d5ec6 --- /dev/null +++ b/scripts/gen-deps.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e # Exit on error +set -o pipefail # Fail a pipe if any sub-command fails. + +source "$(dirname $0)/common.sh" + +if [[ "$#" -ne 1 || ! -d "$1" ]]; then + echo "Specify a valid output directory as the first parameter." + exit 1 +fi + +SCRIPTS_DIR="$(dirname "$0")" +OUTPUT_DIR="$1" + +cd "${SCRIPTS_DIR}/.." + +git ls-files -- '*.go' | grep -v '\(^\|/\)\(internal\|examples\|benchmark\|interop\|test\|testdata\)\(/\|$\)' | xargs dirname | sort -u | while read d; do + pushd "$d" > /dev/null + pkg="$(echo "$d" | sed 's;\.;grpc;' | sed 's;/;_;g')" + go list -deps . | sort | noret_grep -v 'google.golang.org/grpc' >| "${OUTPUT_DIR}/$pkg" + popd > /dev/null +done diff --git a/scripts/install-protoc.sh b/scripts/install-protoc.sh new file mode 100755 index 0000000..7c450ee --- /dev/null +++ b/scripts/install-protoc.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# install-protoc.sh +# +# This script installs the Protocol Buffers compiler (protoc) to the specified +# directory. It is used to generate code from .proto files for gRPC +# communication. The script downloads the protoc binary from the official GitHub +# repository and installs it in the system. +# +# Usage: ./install-protoc.sh INSTALL_PATH +# +# Arguments: +# INSTALL_PATH: The path where the protoc binary will be installed. +# +# Note: This script requires internet connectivity to download the protoc binary. + +set -eu -o pipefail + +source "$(dirname $0)/common.sh" + +# The version of protoc that will be installed. +PROTOC_VERSION="27.1" + +main() { + if [[ "$#" -ne 1 ]]; then + die "Usage: $0 INSTALL_PATH" + fi + + INSTALL_PATH="${1}" + + if [[ ! -d "${INSTALL_PATH}" ]]; then + die "INSTALL_PATH (${INSTALL_PATH}) does not exist." + fi + + echo "Installing protoc version $PROTOC_VERSION to ${INSTALL_PATH}..." + + # Detect the hardware platform. + case "$(uname -m)" in + "x86_64") ARCH="x86_64";; + "aarch64") ARCH="aarch_64";; + "arm64") ARCH="aarch_64";; + *) die "Install unsuccessful. Hardware platform not supported by installer: $1";; + esac + + # Detect the Operating System. + case "$(uname -s)" in + "Darwin") OS="osx";; + "Linux") OS="linux";; + *) die "Install unsuccessful. OS not supported by installer: $2";; + esac + + # Check if the protoc binary with the right version is already installed. + if [[ -f "${INSTALL_PATH}/bin/protoc" ]]; then + if [[ "$("${INSTALL_PATH}/bin/protoc" --version)" == "libprotoc ${PROTOC_VERSION}" ]]; then + echo "protoc version ${PROTOC_VERSION} is already installed in ${INSTALL_PATH}" + return + fi + fi + + DOWNLOAD_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${OS}-${ARCH}.zip" + + # -L follows redirects. + # -O writes output to a file. + curl -LO "${DOWNLOAD_URL}" + + # Unzip the downloaded file and except readme.txt. + # The file structure should look like: + # INSTALL_PATH + # ├── bin + # │ └── protoc + # └── include + # └── other files... + unzip "protoc-${PROTOC_VERSION}-${OS}-${ARCH}.zip" -d "${INSTALL_PATH}" -x "readme.txt" + rm "protoc-${PROTOC_VERSION}-${OS}-${ARCH}.zip" + + # Make the protoc binary executable. ¯\_(ツ)_/¯ crazy, right? + chmod +x "${INSTALL_PATH}/bin/protoc" +} + +main "$@" diff --git a/scripts/regenerate.sh b/scripts/regenerate.sh new file mode 100755 index 0000000..55fa279 --- /dev/null +++ b/scripts/regenerate.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# +# Copyright 2020 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu -o pipefail + +WORKDIR="/tmp/grpc-go-tools" +mkdir -p "${WORKDIR}" + +"$(dirname "${0}")"/install-protoc.sh ${WORKDIR} + +export GOBIN="${WORKDIR}"/bin +export PATH="${GOBIN}:${PATH}" +mkdir -p "${GOBIN}" + +echo "removing existing generated files..." +# grpc_testing_not_regenerated/*.pb.go is not re-generated, +# see grpc_testing_not_regenerated/README.md for details. +find . -name '*.pb.go' | grep -v 'grpc_testing_not_regenerated' | xargs rm -f || true + +echo "Executing: go install google.golang.org/protobuf/cmd/protoc-gen-go..." +(cd test/tools && go install google.golang.org/protobuf/cmd/protoc-gen-go) + +echo "Executing: go install cmd/protoc-gen-go-grpc..." +(cd cmd/protoc-gen-go-grpc && go install .) + +echo "Pulling protos from https://github.com/grpc/grpc-proto..." +if [ -d "${WORKDIR}/grpc-proto" ]; then + (cd "${WORKDIR}/grpc-proto" && git pull) +else + git clone --quiet https://github.com/grpc/grpc-proto "${WORKDIR}/grpc-proto" +fi + +echo "Pulling protos from https://github.com/protocolbuffers/protobuf..." +if [ -d "${WORKDIR}/protobuf" ]; then + (cd "${WORKDIR}/protobuf" && git pull) +else + git clone --quiet https://github.com/protocolbuffers/protobuf "${WORKDIR}/protobuf" +fi + +# Pull in code.proto as a proto dependency +mkdir -p "${WORKDIR}/googleapis/google/rpc" +echo "Pulling code.proto from https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto..." +curl --silent https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto > "${WORKDIR}/googleapis/google/rpc/code.proto" + + +mkdir -p "${WORKDIR}/out" + +# Generates sources without the embed requirement +LEGACY_SOURCES=( + "${WORKDIR}/grpc-proto/grpc/binlog/v1/binarylog.proto" + "${WORKDIR}/grpc-proto/grpc/channelz/v1/channelz.proto" + "${WORKDIR}/grpc-proto/grpc/health/v1/health.proto" + "${WORKDIR}/grpc-proto/grpc/lb/v1/load_balancer.proto" + profiling/proto/service.proto + "${WORKDIR}/grpc-proto/grpc/reflection/v1alpha/reflection.proto" + "${WORKDIR}/grpc-proto/grpc/reflection/v1/reflection.proto" +) + +# Generates only the new gRPC Service symbols +SOURCES=( + $(git ls-files --exclude-standard --cached --others "*.proto" | grep -v '^profiling/proto/service.proto$') + "${WORKDIR}/grpc-proto/grpc/gcp/altscontext.proto" + "${WORKDIR}/grpc-proto/grpc/gcp/handshaker.proto" + "${WORKDIR}/grpc-proto/grpc/gcp/transport_security_common.proto" + "${WORKDIR}/grpc-proto/grpc/lookup/v1/rls.proto" + "${WORKDIR}/grpc-proto/grpc/lookup/v1/rls_config.proto" + "${WORKDIR}"/grpc-proto/grpc/testing/*.proto + "${WORKDIR}"/grpc-proto/grpc/core/*.proto +) + +# These options of the form 'Mfoo.proto=bar' instruct the codegen to use an +# import path of 'bar' in the generated code when 'foo.proto' is imported in +# one of the sources. +# +# Note that the protos listed here are all for testing purposes. All protos to +# be used externally should have a go_package option (and they don't need to be +# listed here). +OPTS=Mgrpc/core/stats.proto=google.golang.org/grpc/interop/grpc_testing/core,\ +Mgrpc/testing/benchmark_service.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/stats.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/report_qps_scenario_service.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/messages.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/worker_service.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/control.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/test.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/payloads.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/empty.proto=google.golang.org/grpc/interop/grpc_testing + +for src in "${SOURCES[@]}"; do + echo "protoc ${src}" + protoc --go_out="${OPTS}:${WORKDIR}/out" --go-grpc_out="${OPTS}:${WORKDIR}/out" \ + -I"." \ + -I"${WORKDIR}/grpc-proto" \ + -I"${WORKDIR}/googleapis" \ + -I"${WORKDIR}/protobuf/src" \ + "${src}" +done + +for src in "${LEGACY_SOURCES[@]}"; do + echo "protoc ${src}" + protoc --go_out="${OPTS}:${WORKDIR}/out" --go-grpc_out="${OPTS},require_unimplemented_servers=false:${WORKDIR}/out" \ + -I"." \ + -I"${WORKDIR}/grpc-proto" \ + -I"${WORKDIR}/googleapis" \ + -I"${WORKDIR}/protobuf/src" \ + "${src}" +done + +# The go_package option in grpc/lookup/v1/rls.proto doesn't match the +# current location. Move it into the right place. +mkdir -p "${WORKDIR}/out/google.golang.org/grpc/internal/proto/grpc_lookup_v1" +mv "${WORKDIR}"/out/google.golang.org/grpc/lookup/grpc_lookup_v1/* "${WORKDIR}/out/google.golang.org/grpc/internal/proto/grpc_lookup_v1" + +# grpc_testing_not_regenerated/*.pb.go are not re-generated, +# see grpc_testing_not_regenerated/README.md for details. +rm "${WORKDIR}"/out/google.golang.org/grpc/testdata/grpc_testing_not_regenerated/*.pb.go + +cp -R "${WORKDIR}"/out/google.golang.org/grpc/* . diff --git a/scripts/revive.toml b/scripts/revive.toml new file mode 100644 index 0000000..1e80fe1 --- /dev/null +++ b/scripts/revive.toml @@ -0,0 +1,60 @@ +# Enabled rules +[rule.blank-imports] + +[rule.context-as-argument] + +[rule.context-keys-type] + +[rule.dot-imports] + +[rule.errorf] + +[rule.error-return] + +[rule.error-strings] + +[rule.error-naming] + +[rule.exported] + +[rule.increment-decrement] + +[rule.indent-error-flow] + arguments = ["preserveScope"] + +[rule.package-comments] + +[rule.range] + +[rule.receiver-naming] + +[rule.superfluous-else] + arguments = ["preserveScope"] + +[rule.time-naming] + +[rule.unexported-return] + +[rule.unnecessary-stmt] + +[rule.unreachable-code] + +[rule.unused-parameter] + +[rule.use-any] + +[rule.useless-break] + +[rule.var-declaration] + +[rule.var-naming] + +# Disabled rules +[rule.empty-block] # Disabled to allow intentional no-op blocks (e.g., channel draining). + Disabled = true + +[rule.import-shadowing] # Disabled to allow intentional reuse of variable names that are the same as package imports. + Disabled = true + +[rule.redefines-builtin-id] # Disabled to allow intentional reuse of variable names that are the same as built-in functions. + Disabled = true diff --git a/scripts/vet-proto.sh b/scripts/vet-proto.sh new file mode 100755 index 0000000..db2d98c --- /dev/null +++ b/scripts/vet-proto.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -ex # Exit on error; debugging enabled. +set -o pipefail # Fail a pipe if any sub-command fails. + +# - Source them sweet sweet helpers. +source "$(dirname $0)/common.sh" + +# - Check to make sure it's safe to modify the user's git repo. +git status --porcelain | fail_on_output + +# - Undo any edits made by this script. +cleanup() { + git reset --hard HEAD +} +trap cleanup EXIT + +# - Installs protoc into your ${GOBIN} directory, if requested. +# ($GOBIN might not be the best place for the protoc binary, but is at least +# consistent with the place where all binaries installed by scripts in this repo +# go.) +if [[ "$1" = "-install" ]]; then + if [[ "${GITHUB_ACTIONS}" = "true" ]]; then + source ./scripts/install-protoc.sh "/home/runner/go" + else + die "run protoc installer https://github.com/grpc/grpc-go/blob/master/scripts/install-protoc.sh" + fi + echo SUCCESS + exit 0 +elif [[ "$#" -ne 0 ]]; then + die "Unknown argument(s): $*" +fi + +for MOD_FILE in $(find . -name 'go.mod'); do + MOD_DIR=$(dirname ${MOD_FILE}) + pushd ${MOD_DIR} + go generate ./... + popd +done + +# - Check that generated proto files are up to date. +git status --porcelain 2>&1 | fail_on_output || \ + (git status; git --no-pager diff; exit 1) + +echo SUCCESS +exit 0 diff --git a/scripts/vet.sh b/scripts/vet.sh new file mode 100755 index 0000000..4d58927 --- /dev/null +++ b/scripts/vet.sh @@ -0,0 +1,217 @@ +#!/bin/bash + +set -ex # Exit on error; debugging enabled. +set -o pipefail # Fail a pipe if any sub-command fails. + +source "$(dirname $0)/common.sh" + +# Check to make sure it's safe to modify the user's git repo. +git status --porcelain | fail_on_output + +# Undo any edits made by this script. +cleanup() { + git reset --hard HEAD +} +trap cleanup EXIT + +if [ -n "${GOROOT}" ]; then + PATH="${GOROOT}/bin:${PATH}" +fi +PATH="${HOME}/go/bin:${PATH}" +go version + +if [[ "$1" = "-install" ]]; then + # Install the pinned versions as defined in module tools. + pushd ./test/tools + go install \ + golang.org/x/tools/cmd/goimports \ + honnef.co/go/tools/cmd/staticcheck \ + github.com/client9/misspell/cmd/misspell \ + github.com/mgechev/revive + popd + exit 0 +elif [[ "$#" -ne 0 ]]; then + die "Unknown argument(s): $*" +fi + +# - Ensure all source files contain a copyright message. +# (Done in two parts because Darwin "git grep" has broken support for compound +# exclusion matches.) +(grep -L "DO NOT EDIT" $(git grep -L "\(Copyright [0-9]\{4,\} gRPC authors\)" -- '*.go') || true) | fail_on_output + +# - Make sure all tests in grpc and grpc/test use leakcheck via Teardown. +not grep 'func Test[^(]' -- *_test.go +not grep 'func Test[^(]' -- test/*.go + +# - Check for typos in test function names +git grep 'func (s) ' -- "*_test.go" | not grep -v 'func (s) Test' +git grep 'func [A-Z]' -- "*_test.go" | not grep -v 'func Test\|Benchmark\|Example' + +# - Do not use time.After except in tests. It has the potential to leak the +# timer since there is no way to stop it early. +git grep -l 'time.After(' -- "*.go" | not grep -v '_test.go\|soak_tests\|testutils' + +# - Do not use "interface{}"; use "any" instead. +git grep -l 'interface{}' -- "*.go" 2>&1 | not grep -v '\.pb\.go\|protoc-gen-go-grpc\|grpc_testing_not_regenerated' + +# - Do not call grpclog directly. Use grpclog.Component instead. +git grep -l -e 'grpclog.I' --or -e 'grpclog.W' --or -e 'grpclog.E' --or -e 'grpclog.F' --or -e 'grpclog.V' -- "*.go" | not grep -v '^grpclog/component.go\|^internal/grpctest/tlogger_test.go\|^internal/grpclog/prefix_logger.go' + +# - Ensure that the deprecated protobuf dependency is not used. +not git grep "\"github.com/golang/protobuf/*" -- "*.go" ':(exclude)testdata/grpc_testing_not_regenerated/*' + +# - Ensure all usages of grpc_testing package are renamed when importing. +not git grep "\(import \|^\s*\)\"google.golang.org/grpc/interop/grpc_testing" -- "*.go" + +# - Ensure that no trailing spaces are found. +not git grep -n '[[:blank:]]$' + +# - Ensure that all files have a terminating newline. +git ls-files | not xargs -I {} sh -c '[ -n "$(tail -c 1 "{}" 2>/dev/null)" ] && echo "{}: No terminating new line found"' | fail_on_output + +# - Ensure that no tabs are found in markdown files. +not git grep -n $'\t' -- '*.md' + +# - Ensure all xds proto imports are renamed to *pb or *grpc. +git grep '"github.com/envoyproxy/go-control-plane/envoy' -- '*.go' ':(exclude)*.pb.go' | not grep -v 'pb "\|grpc "' + +# - Ensure all context usages are done with timeout. +# Context tests under benchmark are excluded as they are testing the performance of context.Background() and context.TODO(). +git grep -e 'context.Background()' --or -e 'context.TODO()' -- "*_test.go" | grep -v "benchmark/primitives/context_test.go" | grep -v 'context.WithTimeout(' | not grep -v 'context.WithCancel(' + +# Disallow usage of net.ParseIP in favour of netip.ParseAddr as the former +# can't parse link local IPv6 addresses. +not git grep 'net.ParseIP' -- '*.go' + +# - Ensure that direct calls to regexp.Compile are not allowed in xDS production code. +# Use CompileSafeRegex instead to ensure full-string matching. +(git grep -n 'regexp.Compile(' -- '*xds*' ':(exclude)*_test.go' ':(exclude)internal/xds/matcher/string_matcher.go' ':(exclude)internal/xds/xdsclient/xdsresource/unmarshal_rds.go' || true) | fail_on_output || { + echo "Error: direct calls to regexp.Compile are not allowed in xDS production code." + echo "Please use CompileSafeRegex instead to ensure full-string matching." + exit 1 +} + +misspell -error . + +# Get the absolute path to revive.toml relative to the script location +REVIVE_CONFIG_PATH="$(dirname "$(realpath "$0")")/revive.toml" + +# - gofmt, goimports, go vet, go mod tidy. +# Perform these checks on each module inside gRPC. +for MOD_FILE in $(find . -name 'go.mod'); do + MOD_DIR=$(dirname ${MOD_FILE}) + pushd ${MOD_DIR} + go vet -all ./... | fail_on_output + gofmt -s -d -l . 2>&1 | fail_on_output + goimports -l . 2>&1 | not grep -vE "\.pb\.go" + + go mod tidy -compat=1.25 + git status --porcelain 2>&1 | fail_on_output || \ + (git status; git --no-pager diff; exit 1) + + # Error for violation of enabled lint rules in config excluding generated code. + revive \ + -set_exit_status=1 \ + -exclude "testdata/grpc_testing_not_regenerated/" \ + -exclude "**/*.pb.go" \ + -formatter plain \ + -config "${REVIVE_CONFIG_PATH}" \ + ./... + + # - Collection of static analysis checks + SC_OUT="$(mktemp)" + # By default, Staticcheck targets the Go version declared in go.mod via the go + # directive. For Go 1.21 and newer, that directive specifies the minimum + # required version of Go. + # If a version is provided to Staticcheck using the -go flag, and the go + # toolchain version is higher than the one in go.mod, Staticcheck will report + # errors for usages of new language features in the std lib code. + staticcheck -checks 'all' ./... >"${SC_OUT}" || true + + # Error for anything other than checks that need exclusions. + noret_grep -v "(ST1000)" "${SC_OUT}" | noret_grep -v "(SA1019)" | noret_grep -v "(ST1003)" | noret_grep -v "(ST1019)\|\(other import of\)" | not grep -v "(SA4000)" + + # Exclude underscore checks for generated code. + noret_grep "(ST1003)" "${SC_OUT}" | not grep -v '\(.pb.go:\)\|\(code_string_test.go:\)\|\(grpc_testing_not_regenerated\)' + + # Error for duplicate imports not including grpc protos. + noret_grep "(ST1019)\|\(other import of\)" "${SC_OUT}" | not grep -Fv 'XXXXX PleaseIgnoreUnused +channelz/grpc_channelz_v1" +go-control-plane/envoy +grpclb/grpc_lb_v1" +health/grpc_health_v1" +interop/grpc_testing" +orca/v3" +proto/grpc_gcp" +proto/grpc_lookup_v1" +examples/features/proto/echo" +reflection/grpc_reflection_v1" +reflection/grpc_reflection_v1alpha" +XXXXX PleaseIgnoreUnused' + + # Error for any package comments not in generated code. + noret_grep "(ST1000)" "${SC_OUT}" | not grep -v "\.pb\.go:" + + # Ignore a false positive when operands have side affects. + # TODO(https://github.com/dominikh/go-tools/issues/54): Remove this once the issue is fixed in staticcheck. + noret_grep "(SA4000)" "${SC_OUT}" | not grep -v -e "crl.go:[0-9]\+:[0-9]\+: identical expressions on the left and right side of the '||' operator (SA4000)" + + # Usage of the deprecated Logger interface from prefix_logger.go is the only + # allowed one. If any other files use the deprecated interface, this check + # will fails. Also, note that this same deprecation notice is also added to + # the list of ignored notices down below to allow for the usage in + # prefix_logger.go to not case vet failure. + noret_grep "(SA1019)" "${SC_OUT}" | noret_grep "internal.Logger is deprecated:" | not grep -v -e "grpclog/logger.go" + + # Only ignore the following deprecated types/fields/functions and exclude + # generated code. + noret_grep "(SA1019)" "${SC_OUT}" | not grep -Fv 'XXXXX PleaseIgnoreUnused +XXXXX Protobuf related deprecation errors: +"github.com/golang/protobuf +.pb.go: +grpc_testing_not_regenerated +: ptypes. +proto.RegisterType +XXXXX gRPC internal usage deprecation errors: +"google.golang.org/grpc +: grpc. +: v1alpha. +: v1alphareflectionpb. +BalancerAttributes is deprecated: +CredsBundle is deprecated: +GetMetadata is deprecated: +internal.Logger is deprecated: +Metadata is deprecated: use Attributes instead. +NewAddress is deprecated: +NewSubConn is deprecated: +OverrideServerName is deprecated: +RemoveSubConn is deprecated: +SecurityVersion is deprecated: +.ServerName is deprecated: +stats.PickerUpdated is deprecated: +Target is deprecated: Use the Target field in the BuildOptions instead. +UpdateAddresses is deprecated: +UpdateSubConnState is deprecated: +balancer.ErrTransientFailure is deprecated: +grpc/reflection/v1alpha/reflection.proto +SwitchTo is deprecated: +XXXXX xDS deprecated fields we support +.ContainsMatch +.ExactMatch +.PrefixMatch +.SafeRegexMatch +.SuffixMatch +GetContainsMatch +GetExactMatch +GetMatchSubjectAltNames +GetPrefixMatch +GetSafeRegexMatch +GetSuffixMatch +GetSourceIp +GetTlsCertificateCertificateProviderInstance +GetValidationContextCertificateProviderInstance +XXXXX PleaseIgnoreUnused' + popd +done + +echo SUCCESS diff --git a/security/advancedtls/advancedtls.go b/security/advancedtls/advancedtls.go new file mode 100644 index 0000000..cfb2237 --- /dev/null +++ b/security/advancedtls/advancedtls.go @@ -0,0 +1,690 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package advancedtls provides gRPC transport credentials that allow easy +// configuration of advanced TLS features. The APIs here give the user more +// customizable control to fit their security landscape, thus the "advanced" +// moniker. This package provides both interfaces and generally useful +// implementations of those interfaces, for example periodic credential +// reloading, support for certificate revocation lists, and customizable +// certificate verification behaviors. If the provided implementations do not +// fit a given use case, a custom implementation of the interface can be +// injected. +package advancedtls + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "reflect" + "time" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + credinternal "google.golang.org/grpc/internal/credentials" +) + +// CertificateChains represents a slice of certificate chains, each consisting +// of a sequence of certificates. Each chain represents a path from a leaf +// certificate up to a root certificate in the certificate hierarchy. +type CertificateChains [][]*x509.Certificate + +// HandshakeVerificationInfo contains information about a handshake needed for +// verification for use when implementing the `PostHandshakeVerificationFunc` +// The fields in this struct are read-only. +type HandshakeVerificationInfo struct { + // The target server name that the client connects to when establishing the + // connection. This field is only meaningful for client side. On server side, + // this field would be an empty string. + ServerName string + // The raw certificates sent from peer. + RawCerts [][]byte + // The verification chain obtained by checking peer RawCerts against the + // trust certificate bundle(s), if applicable. + VerifiedChains CertificateChains + // The leaf certificate sent from peer, if choosing to verify the peer + // certificate(s) and that verification passed. This field would be nil if + // either user chose not to verify or the verification failed. + Leaf *x509.Certificate + // ConnectionState is the TLS connection state for the connection. + ConnectionState tls.ConnectionState +} + +// PostHandshakeVerificationResults contains the information about results of +// PostHandshakeVerificationFunc. +// PostHandshakeVerificationResults is an empty struct for now. It may be extended in the +// future to include more information. +type PostHandshakeVerificationResults struct{} + +// PostHandshakeVerificationFunc is the function defined by users to perform +// custom verification checks after chain building and regular handshake +// verification has been completed. +// PostHandshakeVerificationFunc should return (nil, error) if the authorization +// should fail, with the error containing information on why it failed. +type PostHandshakeVerificationFunc func(params *HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) + +// ConnectionInfo contains the parameters available to users when +// implementing GetRootCertificates. +type ConnectionInfo struct { + // RawConn is the raw net.Conn representing a connection. + RawConn net.Conn + // RawCerts is the byte representation of the presented peer cert chain. + RawCerts [][]byte +} + +// RootCertificates is the result of GetRootCertificates. +// If users want to reload the root trust certificate, it is required to return +// the proper TrustCerts in GetRootCAs. +type RootCertificates struct { + // TrustCerts is the pool of trusted certificates. + TrustCerts *x509.CertPool +} + +// RootCertificateOptions contains options to obtain root trust certificates +// for both the client and the server. +// At most one field should be set. If none of them are set, we use the system +// default trust certificates. Setting more than one field will result in +// undefined behavior. +type RootCertificateOptions struct { + // If RootCertificates is set, it will be used every time when verifying + // the peer certificates, without performing root certificate reloading. + RootCertificates *x509.CertPool + // If GetRootCertificates is set, it will be invoked to obtain root certs for + // every new connection. + GetRootCertificates func(params *ConnectionInfo) (*RootCertificates, error) + // If RootProvider is set, we will use the root certs from the Provider's + // KeyMaterial() call in the new connections. The Provider must have initial + // credentials if specified. Otherwise, KeyMaterial() will block forever. + RootProvider certprovider.Provider +} + +// nonNilFieldCount returns the number of set fields in RootCertificateOptions. +func (o RootCertificateOptions) nonNilFieldCount() int { + cnt := 0 + rv := reflect.ValueOf(o) + for i := 0; i < rv.NumField(); i++ { + if !rv.Field(i).IsNil() { + cnt++ + } + } + return cnt +} + +// IdentityCertificateOptions contains options to obtain identity certificates +// for both the client and the server. +// At most one field should be set. Setting more than one field will result in undefined behavior. +type IdentityCertificateOptions struct { + // If Certificates is set, it will be used every time when needed to present + // identity certificates, without performing identity certificate reloading. + Certificates []tls.Certificate + // If GetIdentityCertificatesForClient is set, it will be invoked to obtain + // identity certs for every new connection. + // This field is only relevant when set on the client side. + GetIdentityCertificatesForClient func(*tls.CertificateRequestInfo) (*tls.Certificate, error) + // If GetIdentityCertificatesForServer is set, it will be invoked to obtain + // identity certs for every new connection. + // This field is only relevant when set on the server side. + GetIdentityCertificatesForServer func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) + // If IdentityProvider is set, we will use the identity certs from the + // Provider's KeyMaterial() call in the new connections. The Provider must + // have initial credentials if specified. Otherwise, KeyMaterial() will block + // forever. + IdentityProvider certprovider.Provider +} + +// nonNilFieldCount returns the number of set fields in IdentityCertificateOptions. +func (o IdentityCertificateOptions) nonNilFieldCount() int { + cnt := 0 + rv := reflect.ValueOf(o) + for i := 0; i < rv.NumField(); i++ { + if !rv.Field(i).IsNil() { + cnt++ + } + } + return cnt +} + +// VerificationType is the enum type that represents different levels of +// verification users could set, both on client side and on server side. +type VerificationType int + +const ( + // CertAndHostVerification indicates doing both certificate signature check + // and hostname check. + CertAndHostVerification VerificationType = iota + // CertVerification indicates doing certificate signature check only. Setting + // this field without proper custom verification check would leave the + // application susceptible to the MITM attack. + CertVerification + // SkipVerification indicates skipping both certificate signature check and + // hostname check. If setting this field, proper custom verification needs to + // be implemented in order to complete the authentication. Setting this field + // with a nil custom verification would raise an error. + SkipVerification +) + +// Options contains the fields a user can configure when setting up TLS clients +// and servers +type Options struct { + // IdentityOptions is OPTIONAL on client side. This field only needs to be + // set if mutual authentication is required on server side. + // IdentityOptions is REQUIRED on server side. + IdentityOptions IdentityCertificateOptions + // AdditionalPeerVerification is a custom verification check after certificate signature + // check. + // If this is set, we will perform this customized check after doing the + // normal check(s) indicated by setting VerificationType. + AdditionalPeerVerification PostHandshakeVerificationFunc + // RootOptions is OPTIONAL on server side. This field only needs to be set if + // mutual authentication is required(RequireClientCert is true). + RootOptions RootCertificateOptions + // If the server requires the client to send certificates. This value is only + // relevant when configuring options for the server. Is not used for + // client-side configuration. + RequireClientCert bool + // VerificationType defines what type of peer verification is done. See + // the `VerificationType` enum for the different options. + // Default: CertAndHostVerification + VerificationType VerificationType + // RevocationOptions is the configurations for certificate revocation checks. + // It could be nil if such checks are not needed. + RevocationOptions *RevocationOptions + // MinTLSVersion contains the minimum TLS version that is acceptable. + // The value should be set using tls.VersionTLSxx from https://pkg.go.dev/crypto/tls + // By default, TLS 1.2 is currently used as the minimum when acting as a + // client, and TLS 1.0 when acting as a server. TLS 1.0 is the minimum + // supported by this package, both as a client and as a server. This + // default may be changed over time affecting backwards compatibility. + MinTLSVersion uint16 + // MaxTLSVersion contains the maximum TLS version that is acceptable. + // The value should be set using tls.VersionTLSxx from https://pkg.go.dev/crypto/tls + // By default, the maximum version supported by this package is used, + // which is currently TLS 1.3. This default may be changed over time + // affecting backwards compatibility. + MaxTLSVersion uint16 + // CipherSuites is an unordered list of supported TLS 1.0–1.2 + // ciphersuites. TLS 1.3 ciphersuites are not configurable. If nil, a + // safe default list is used. + CipherSuites []uint16 + // CurvePreferences contains the elliptic curves that will be used during the + // key exchange, in preference order. If empty, the default will be used. + CurvePreferences []tls.CurveID + // serverNameOverride is for testing only and only relevant on the client + // side. If set to a non-empty string, it will override the virtual host + // name of authority (e.g. :authority header field) in requests and the + // target hostname used during server cert verification. + serverNameOverride string + // SkipServerAuthEKU is an UNSAFE option that, if set to true, skips checking + // for the Server Auth Extended Key Usage (EKU) extension in peer + // certificates during server authentication. + // + // Doing so goes against the RFC 5280 specification (Section 4.2.1.12), + // which mandates that the certificate key usage should be compatible with + // the purpose of the key. + // See: https://datatracker.ietf.org/doc/html/rfc5280#section-4.2.1.12 + // + // Behavior: + // - Client-side: If set to true, the client will verify the server's + // certificate using x509.ExtKeyUsageAny instead of requiring + // x509.ExtKeyUsageServerAuth. This is useful when the server's certificate + // does not possess the Server Auth EKU. + // - Server-side: No-op. + SkipServerAuthEKU bool +} + +func (o *Options) clientConfig() (*tls.Config, error) { + if o.VerificationType == SkipVerification && o.AdditionalPeerVerification == nil { + return nil, fmt.Errorf("client needs to provide custom verification mechanism if choose to skip default verification") + } + // If the MinTLSVersion isn't set, default to 1.2 + if o.MinTLSVersion == 0 { + o.MinTLSVersion = tls.VersionTLS12 + } + // If the MaxTLSVersion isn't set, default to 1.3 + if o.MaxTLSVersion == 0 { + o.MaxTLSVersion = tls.VersionTLS13 + } + // Make sure users didn't specify more than one fields in + // RootCertificateOptions and IdentityCertificateOptions. + if num := o.RootOptions.nonNilFieldCount(); num > 1 { + return nil, fmt.Errorf("at most one field in RootCertificateOptions could be specified") + } + if num := o.IdentityOptions.nonNilFieldCount(); num > 1 { + return nil, fmt.Errorf("at most one field in IdentityCertificateOptions could be specified") + } + if o.IdentityOptions.GetIdentityCertificatesForServer != nil { + return nil, fmt.Errorf("GetIdentityCertificatesForServer cannot be specified on the client side") + } + if o.MinTLSVersion > o.MaxTLSVersion { + return nil, fmt.Errorf("the minimum TLS version is larger than the maximum TLS version") + } + config := &tls.Config{ + ServerName: o.serverNameOverride, + // We have to set InsecureSkipVerify to true to skip the default checks and + // use the verification function we built from buildVerifyFunc. + InsecureSkipVerify: true, + MinVersion: o.MinTLSVersion, + MaxVersion: o.MaxTLSVersion, + CipherSuites: o.CipherSuites, + CurvePreferences: o.CurvePreferences, + } + // Propagate root-certificate-related fields in tls.Config. + switch { + case o.RootOptions.RootCertificates != nil: + config.RootCAs = o.RootOptions.RootCertificates + case o.RootOptions.GetRootCertificates != nil: + // In cases when users provide GetRootCertificates callback, since this + // callback is not contained in tls.Config, we have nothing to set here. + // We will invoke the callback in ClientHandshake. + case o.RootOptions.RootProvider != nil: + o.RootOptions.GetRootCertificates = func(*ConnectionInfo) (*RootCertificates, error) { + km, err := o.RootOptions.RootProvider.KeyMaterial(context.Background()) + if err != nil { + return nil, err + } + return &RootCertificates{TrustCerts: km.Roots}, nil + } + default: + // No root certificate options specified by user. Use the certificates + // stored in system default path as the last resort. + if o.VerificationType != SkipVerification { + systemRootCAs, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + config.RootCAs = systemRootCAs + } + } + // Propagate identity-certificate-related fields in tls.Config. + switch { + case o.IdentityOptions.Certificates != nil: + config.Certificates = o.IdentityOptions.Certificates + case o.IdentityOptions.GetIdentityCertificatesForClient != nil: + config.GetClientCertificate = o.IdentityOptions.GetIdentityCertificatesForClient + case o.IdentityOptions.IdentityProvider != nil: + config.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + km, err := o.IdentityOptions.IdentityProvider.KeyMaterial(context.Background()) + if err != nil { + return nil, err + } + if len(km.Certs) != 1 { + return nil, fmt.Errorf("there should always be only one identity cert chain on the client side in IdentityProvider") + } + return &km.Certs[0], nil + } + default: + // It's fine for users to not specify identity certificate options here. + } + return config, nil +} + +func (o *Options) serverConfig() (*tls.Config, error) { + if o.RequireClientCert && o.VerificationType == SkipVerification && o.AdditionalPeerVerification == nil { + return nil, fmt.Errorf("server needs to provide custom verification mechanism if choose to skip default verification, but require client certificate(s)") + } + // If the MinTLSVersion isn't set, default to 1.2 + if o.MinTLSVersion == 0 { + o.MinTLSVersion = tls.VersionTLS12 + } + // If the MaxTLSVersion isn't set, default to 1.3 + if o.MaxTLSVersion == 0 { + o.MaxTLSVersion = tls.VersionTLS13 + } + // Make sure users didn't specify more than one fields in + // RootCertificateOptions and IdentityCertificateOptions. + if num := o.RootOptions.nonNilFieldCount(); num > 1 { + return nil, fmt.Errorf("at most one field in RootCertificateOptions could be specified") + } + if num := o.IdentityOptions.nonNilFieldCount(); num > 1 { + return nil, fmt.Errorf("at most one field in IdentityCertificateOptions could be specified") + } + if o.IdentityOptions.GetIdentityCertificatesForClient != nil { + return nil, fmt.Errorf("GetIdentityCertificatesForClient cannot be specified on the server side") + } + if o.MinTLSVersion > o.MaxTLSVersion { + return nil, fmt.Errorf("the minimum TLS version is larger than the maximum TLS version") + } + clientAuth := tls.NoClientCert + if o.RequireClientCert { + // We have to set clientAuth to RequireAnyClientCert to force underlying + // TLS package to use the verification function we built from + // buildVerifyFunc. + clientAuth = tls.RequireAnyClientCert + } + config := &tls.Config{ + ClientAuth: clientAuth, + MinVersion: o.MinTLSVersion, + MaxVersion: o.MaxTLSVersion, + CipherSuites: o.CipherSuites, + CurvePreferences: o.CurvePreferences, + } + // Propagate root-certificate-related fields in tls.Config. + switch { + case o.RootOptions.RootCertificates != nil: + config.ClientCAs = o.RootOptions.RootCertificates + case o.RootOptions.GetRootCertificates != nil: + // In cases when users provide GetRootCertificates callback, since this + // callback is not contained in tls.Config, we have nothing to set here. + // We will invoke the callback in ServerHandshake. + case o.RootOptions.RootProvider != nil: + o.RootOptions.GetRootCertificates = func(*ConnectionInfo) (*RootCertificates, error) { + km, err := o.RootOptions.RootProvider.KeyMaterial(context.Background()) + if err != nil { + return nil, err + } + return &RootCertificates{TrustCerts: km.Roots}, nil + } + default: + // No root certificate options specified by user. Use the certificates + // stored in system default path as the last resort. + if o.VerificationType != SkipVerification && o.RequireClientCert { + systemRootCAs, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + config.ClientCAs = systemRootCAs + } + } + // Propagate identity-certificate-related fields in tls.Config. + switch { + case o.IdentityOptions.Certificates != nil: + config.Certificates = o.IdentityOptions.Certificates + case o.IdentityOptions.GetIdentityCertificatesForServer != nil: + config.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { + return buildGetCertificates(clientHello, o) + } + case o.IdentityOptions.IdentityProvider != nil: + o.IdentityOptions.GetIdentityCertificatesForServer = func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + km, err := o.IdentityOptions.IdentityProvider.KeyMaterial(context.Background()) + if err != nil { + return nil, err + } + var certChains []*tls.Certificate + for i := 0; i < len(km.Certs); i++ { + certChains = append(certChains, &km.Certs[i]) + } + return certChains, nil + } + config.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { + return buildGetCertificates(clientHello, o) + } + default: + return nil, fmt.Errorf("needs to specify at least one field in IdentityCertificateOptions") + } + return config, nil +} + +// advancedTLSCreds is the credentials required for authenticating a connection +// using TLS. +type advancedTLSCreds struct { + config *tls.Config + verifyFunc PostHandshakeVerificationFunc + getRootCertificates func(params *ConnectionInfo) (*RootCertificates, error) + isClient bool + revocationOptions *RevocationOptions + verificationType VerificationType + skipServerAuthEKU bool +} + +func (c advancedTLSCreds) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{ + SecurityProtocol: "tls", + SecurityVersion: "1.2", + ServerName: c.config.ServerName, + } +} + +func (c *advancedTLSCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + // Use local cfg to avoid clobbering ServerName if using multiple endpoints. + cfg := credinternal.CloneTLSConfig(c.config) + // We return the full authority name to users without stripping the trailing + // port. + cfg.ServerName = authority + + peerVerifiedChains := CertificateChains{} + cfg.VerifyPeerCertificate, cfg.VerifyConnection = buildVerifyFunc(c, cfg.ServerName, rawConn, &peerVerifiedChains) + conn := tls.Client(rawConn, cfg) + errChannel := make(chan error, 1) + go func() { + errChannel <- conn.Handshake() + close(errChannel) + }() + select { + case err := <-errChannel: + if err != nil { + conn.Close() + return nil, nil, err + } + case <-ctx.Done(): + conn.Close() + return nil, nil, ctx.Err() + } + info := credentials.TLSInfo{ + State: conn.ConnectionState(), + CommonAuthInfo: credentials.CommonAuthInfo{ + SecurityLevel: credentials.PrivacyAndIntegrity, + }, + } + info.SPIFFEID = credinternal.SPIFFEIDFromState(conn.ConnectionState()) + info.State.VerifiedChains = peerVerifiedChains + return credinternal.WrapSyscallConn(rawConn, conn), info, nil +} + +func (c *advancedTLSCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + cfg := credinternal.CloneTLSConfig(c.config) + peerVerifiedChains := CertificateChains{} + cfg.VerifyPeerCertificate, cfg.VerifyConnection = buildVerifyFunc(c, "", rawConn, &peerVerifiedChains) + conn := tls.Server(rawConn, cfg) + if err := conn.Handshake(); err != nil { + conn.Close() + return nil, nil, err + } + info := credentials.TLSInfo{ + State: conn.ConnectionState(), + CommonAuthInfo: credentials.CommonAuthInfo{ + SecurityLevel: credentials.PrivacyAndIntegrity, + }, + } + info.SPIFFEID = credinternal.SPIFFEIDFromState(conn.ConnectionState()) + info.State.VerifiedChains = peerVerifiedChains + return credinternal.WrapSyscallConn(rawConn, conn), info, nil +} + +func (c *advancedTLSCreds) Clone() credentials.TransportCredentials { + return &advancedTLSCreds{ + config: credinternal.CloneTLSConfig(c.config), + verifyFunc: c.verifyFunc, + getRootCertificates: c.getRootCertificates, + isClient: c.isClient, + revocationOptions: c.revocationOptions, + verificationType: c.verificationType, + skipServerAuthEKU: c.skipServerAuthEKU, + } +} + +func (c *advancedTLSCreds) OverrideServerName(serverNameOverride string) error { + c.config.ServerName = serverNameOverride + return nil +} + +type verifyPeerCertificateFunc func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error +type verifyConnectionFunc func(cs tls.ConnectionState) error + +// The function buildVerifyFunc constructs the verification callbacks used when +// users want root cert reloading, certificate revocation checking, and custom +// verification checks. +// We build custom verification callbacks because the standard library tls module: +// 1. does not have native support for dynamic root cert reloading. +// 2. ignores basic certificate verification when InsecureSkipVerify is true. +// +// It returns two callbacks that work together during the TLS handshake: +// 1. VerifyPeerCertificate: performs root CA reloading, peer certificate chain +// verification, and revocation checking, storing the verified chains and +// leaf cert in shared closure variables. +// 2. VerifyConnection: invoked after certificate verification and key exchange +// complete, providing access to tls.ConnectionState and invoking the user's +// custom verification function (verifyFunc). +// +// peerVerifiedChains(output param): verified chain of certs from leaf to the +// trust cert that the peer trusts. +// 1. For server it is, client certs + Root ca that the server trusts +// 2. For client it is, server certs + Root ca that the client trusts +func buildVerifyFunc(c *advancedTLSCreds, + serverName string, + rawConn net.Conn, + peerVerifiedChains *CertificateChains) (verifyPeerCertificateFunc, verifyConnectionFunc) { + var ( + chains CertificateChains + leafCert *x509.Certificate + rawCerts [][]byte + ) + verifyPeerCertFunc := func(rc [][]byte, verifiedChains [][]*x509.Certificate) error { + if len(rc) == 0 { + return fmt.Errorf("no peer certificates presented") + } + rawCerts = rc + chains = verifiedChains + rawCertList := make([]*x509.Certificate, len(rawCerts)) + for i, asn1Data := range rawCerts { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + return err + } + rawCertList[i] = cert + } + if c.verificationType == CertAndHostVerification || c.verificationType == CertVerification { + // perform possible trust credential reloading and certificate check + rootCAs := c.config.RootCAs + if !c.isClient { + rootCAs = c.config.ClientCAs + } + // Reload root CA certs. + if rootCAs == nil && c.getRootCertificates != nil { + results, err := c.getRootCertificates(&ConnectionInfo{ + RawConn: rawConn, + RawCerts: rawCerts, + }) + if err != nil { + return err + } + rootCAs = results.TrustCerts + } + // Verify peers' certificates against RootCAs and get verifiedChains. + keyUsages := []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + if c.isClient { + if c.skipServerAuthEKU { + keyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageAny} + } else { + keyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} + } + } + opts := x509.VerifyOptions{ + Roots: rootCAs, + CurrentTime: time.Now(), + Intermediates: x509.NewCertPool(), + KeyUsages: keyUsages, + } + for _, cert := range rawCertList[1:] { + opts.Intermediates.AddCert(cert) + } + // Perform default hostname check if specified. + if c.isClient && c.verificationType == CertAndHostVerification && serverName != "" { + parsedName, _, err := net.SplitHostPort(serverName) + if err != nil { + // If the serverName had no host port or if the serverName cannot be + // parsed, use it as-is. + parsedName = serverName + } + opts.DNSName = parsedName + } + var err error + chains, err = rawCertList[0].Verify(opts) + if err != nil { + return err + } + leafCert = rawCertList[0] + } + // Perform certificate revocation check if specified. + if c.revocationOptions != nil { + verifiedChains := chains + if verifiedChains == nil { + verifiedChains = CertificateChains{rawCertList} + } + if err := checkChainRevocation(verifiedChains, *c.revocationOptions); err != nil { + return err + } + } + *peerVerifiedChains = chains + return nil + } + if c.verifyFunc == nil { + return verifyPeerCertFunc, nil + } + return verifyPeerCertFunc, func(cs tls.ConnectionState) error { + _, err := c.verifyFunc(&HandshakeVerificationInfo{ + ServerName: serverName, + RawCerts: rawCerts, + VerifiedChains: chains, + Leaf: leafCert, + ConnectionState: cs, + }) + return err + } +} + +// NewClientCreds uses ClientOptions to construct a TransportCredentials based +// on TLS. +func NewClientCreds(o *Options) (credentials.TransportCredentials, error) { + conf, err := o.clientConfig() + if err != nil { + return nil, err + } + tc := &advancedTLSCreds{ + config: conf, + isClient: true, + getRootCertificates: o.RootOptions.GetRootCertificates, + verifyFunc: o.AdditionalPeerVerification, + revocationOptions: o.RevocationOptions, + verificationType: o.VerificationType, + skipServerAuthEKU: o.SkipServerAuthEKU, + } + tc.config.NextProtos = credinternal.AppendH2ToNextProtos(tc.config.NextProtos) + return tc, nil +} + +// NewServerCreds uses ServerOptions to construct a TransportCredentials based +// on TLS. +func NewServerCreds(o *Options) (credentials.TransportCredentials, error) { + conf, err := o.serverConfig() + if err != nil { + return nil, err + } + tc := &advancedTLSCreds{ + config: conf, + isClient: false, + getRootCertificates: o.RootOptions.GetRootCertificates, + verifyFunc: o.AdditionalPeerVerification, + revocationOptions: o.RevocationOptions, + verificationType: o.VerificationType, + skipServerAuthEKU: o.SkipServerAuthEKU, + } + tc.config.NextProtos = credinternal.AppendH2ToNextProtos(tc.config.NextProtos) + return tc, nil +} diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go new file mode 100644 index 0000000..7742953 --- /dev/null +++ b/security/advancedtls/advancedtls_integration_test.go @@ -0,0 +1,942 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package advancedtls + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "os" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/credentials/tls/certprovider/pemfile" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/security/advancedtls/internal/testutils" + "google.golang.org/grpc/security/advancedtls/testdata" +) + +const ( + // Default timeout for normal connections. + defaultTestTimeout = 5 * time.Second + // Intervals that set to monitor the credential updates. + credRefreshingInterval = 200 * time.Millisecond + // Time we wait for the credential updates to be picked up. + sleepInterval = 400 * time.Millisecond +) + +// stageInfo contains a stage number indicating the current phase of each +// integration test, and a mutex. +// Based on the stage number of current test, we will use different +// certificates and custom verification functions to check if our tests behave +// as expected. +type stageInfo struct { + mutex sync.Mutex + stage int +} + +func (s *stageInfo) increase() { + s.mutex.Lock() + defer s.mutex.Unlock() + s.stage = s.stage + 1 +} + +func (s *stageInfo) read() int { + s.mutex.Lock() + defer s.mutex.Unlock() + return s.stage +} + +func (s *stageInfo) reset() { + s.mutex.Lock() + defer s.mutex.Unlock() + s.stage = 0 +} + +type greeterServer struct { + pb.UnimplementedGreeterServer +} + +// sayHello is a simple implementation of the pb.GreeterServer SayHello method. +func (greeterServer) SayHello(_ context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + return &pb.HelloReply{Message: "Hello " + in.Name}, nil +} + +// TODO(ZhenLian): remove shouldFail to the function signature to provider +// tests. +func callAndVerify(ctx context.Context, msg string, client pb.GreeterClient, shouldFail bool) error { + _, err := client.SayHello(ctx, &pb.HelloRequest{Name: msg}) + if want, got := shouldFail == true, err != nil; got != want { + return fmt.Errorf("want and got mismatch, want shouldFail=%v, got fail=%v, rpc error: %v", want, got, err) + } + return nil +} + +// TODO(ZhenLian): remove shouldFail and add ...DialOption to the function +// signature to provider cleaner tests. +func callAndVerifyWithClientConn(ctx context.Context, address string, msg string, creds credentials.TransportCredentials, shouldFail bool) (*grpc.ClientConn, pb.GreeterClient, error) { + // Disable service config lookups as it results in a DNS lookup which can + // flake in CI. + conn, err := grpc.NewClient(address, grpc.WithTransportCredentials(creds), grpc.WithDisableServiceConfig()) + if err != nil { + return nil, nil, fmt.Errorf("client failed to connect to %s. Error: %v", address, err) + } + greetClient := pb.NewGreeterClient(conn) + err = callAndVerify(ctx, msg, greetClient, shouldFail) + if err != nil { + return nil, nil, err + } + return conn, greetClient, nil +} + +// The advanced TLS features are tested in different stages. +// At stage 0, we establish a good connection between client and server. +// At stage 1, we change one factor(it could be we change the server's +// certificate, or custom verification function, etc), and test if the +// following connections would be dropped. +// At stage 2, we re-establish the connection by changing the counterpart of +// the factor we modified in stage 1. +// (could be change the client's trust certificate, or change custom +// verification function, etc) +func (s) TestEnd2End(t *testing.T) { + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + t.Fatalf("cs.LoadCerts() failed, err: %v", err) + } + stage := &stageInfo{} + for _, test := range []struct { + desc string + clientCert []tls.Certificate + clientGetCert func(*tls.CertificateRequestInfo) (*tls.Certificate, error) + clientRoot *x509.CertPool + clientGetRoot func(params *ConnectionInfo) (*RootCertificates, error) + clientVerifyFunc PostHandshakeVerificationFunc + clientVerificationType VerificationType + serverCert []tls.Certificate + serverGetCert func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) + serverRoot *x509.CertPool + serverGetRoot func(params *ConnectionInfo) (*RootCertificates, error) + serverVerifyFunc PostHandshakeVerificationFunc + serverVerificationType VerificationType + }{ + // Test Scenarios: + // At initialization(stage = 0), client will be initialized with cert + // ClientCert1 and ClientTrust1, server with ServerCert1 and ServerTrust1. + // The mutual authentication works at the beginning, since ClientCert1 is + // trusted by ServerTrust1, and ServerCert1 by ClientTrust1. + // At stage 1, client changes ClientCert1 to ClientCert2. Since ClientCert2 + // is not trusted by ServerTrust1, following rpc calls are expected to + // fail, while the previous rpc calls are still good because those are + // already authenticated. + // At stage 2, the server changes ServerTrust1 to ServerTrust2, and we + // should see it again accepts the connection, since ClientCert2 is trusted + // by ServerTrust2. + { + desc: "test the reloading feature for client identity callback and server trust callback", + clientGetCert: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + switch stage.read() { + case 0: + return &cs.ClientCert1, nil + default: + return &cs.ClientCert2, nil + } + }, + clientRoot: cs.ClientTrust1, + clientVerifyFunc: func(*HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + return &PostHandshakeVerificationResults{}, nil + }, + clientVerificationType: CertVerification, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverGetRoot: func(*ConnectionInfo) (*RootCertificates, error) { + switch stage.read() { + case 0, 1: + return &RootCertificates{TrustCerts: cs.ServerTrust1}, nil + default: + return &RootCertificates{TrustCerts: cs.ServerTrust2}, nil + } + }, + serverVerifyFunc: func(*HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + return &PostHandshakeVerificationResults{}, nil + }, + serverVerificationType: CertVerification, + }, + // Test Scenarios: + // At initialization(stage = 0), client will be initialized with cert + // ClientCert1 and ClientTrust1, server with ServerCert1 and ServerTrust1. + // The mutual authentication works at the beginning, since ClientCert1 is + // trusted by ServerTrust1, and ServerCert1 by ClientTrust1. + // At stage 1, server changes ServerCert1 to ServerCert2. Since ServerCert2 + // is not trusted by ClientTrust1, following rpc calls are expected to + // fail, while the previous rpc calls are still good because those are + // already authenticated. + // At stage 2, the client changes ClientTrust1 to ClientTrust2, and we + // should see it again accepts the connection, since ServerCert2 is trusted + // by ClientTrust2. + { + desc: "test the reloading feature for server identity callback and client trust callback", + clientCert: []tls.Certificate{cs.ClientCert1}, + clientGetRoot: func(*ConnectionInfo) (*RootCertificates, error) { + switch stage.read() { + case 0, 1: + return &RootCertificates{TrustCerts: cs.ClientTrust1}, nil + default: + return &RootCertificates{TrustCerts: cs.ClientTrust2}, nil + } + }, + clientVerifyFunc: func(*HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + return &PostHandshakeVerificationResults{}, nil + }, + clientVerificationType: CertVerification, + serverGetCert: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + switch stage.read() { + case 0: + return []*tls.Certificate{&cs.ServerCert1}, nil + default: + return []*tls.Certificate{&cs.ServerCert2}, nil + } + }, + serverRoot: cs.ServerTrust1, + serverVerifyFunc: func(*HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + return &PostHandshakeVerificationResults{}, nil + }, + serverVerificationType: CertVerification, + }, + // Test Scenarios: + // At initialization(stage = 0), client will be initialized with cert + // ClientCert1 and ClientTrust1, server with ServerCert1 and ServerTrust1. + // The mutual authentication works at the beginning, since ClientCert1 + // trusted by ServerTrust1, ServerCert1 by ClientTrust1, and also the + // custom verification check allows the CommonName on ServerCert1. + // At stage 1, server changes ServerCert1 to ServerCert2, and client + // changes ClientTrust1 to ClientTrust2. Although ServerCert2 is trusted by + // ClientTrust2, our authorization check only accepts ServerCert1, and + // hence the following calls should fail. Previous connections should + // not be affected. + // At stage 2, the client changes authorization check to only accept + // ServerCert2. Now we should see the connection becomes normal again. + { + desc: "test client custom verification", + clientCert: []tls.Certificate{cs.ClientCert1}, + clientGetRoot: func(*ConnectionInfo) (*RootCertificates, error) { + switch stage.read() { + case 0: + return &RootCertificates{TrustCerts: cs.ClientTrust1}, nil + default: + return &RootCertificates{TrustCerts: cs.ClientTrust2}, nil + } + }, + clientVerifyFunc: func(params *HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + if len(params.RawCerts) == 0 { + return nil, fmt.Errorf("no peer certs") + } + cert, err := x509.ParseCertificate(params.RawCerts[0]) + if err != nil || cert == nil { + return nil, fmt.Errorf("failed to parse certificate: %v", err) + } + authzCheck := false + switch stage.read() { + case 0, 1: + // foo.bar.com is the common name on ServerCert1 + if cert.Subject.CommonName == "foo.bar.com" { + authzCheck = true + } + default: + // foo.bar.server2.com is the common name on ServerCert2 + if cert.Subject.CommonName == "foo.bar.server2.com" { + authzCheck = true + } + } + if authzCheck { + return &PostHandshakeVerificationResults{}, nil + } + return nil, fmt.Errorf("custom authz check fails") + }, + clientVerificationType: CertVerification, + serverGetCert: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + switch stage.read() { + case 0: + return []*tls.Certificate{&cs.ServerCert1}, nil + default: + return []*tls.Certificate{&cs.ServerCert2}, nil + } + }, + serverRoot: cs.ServerTrust1, + serverVerifyFunc: func(*HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + return &PostHandshakeVerificationResults{}, nil + }, + serverVerificationType: CertVerification, + }, + // Test Scenarios: + // At initialization(stage = 0), client will be initialized with cert + // ClientCert1 and ClientTrust1, server with ServerCert1 and ServerTrust1. + // The mutual authentication works at the beginning, since ClientCert1 + // trusted by ServerTrust1, ServerCert1 by ClientTrust1, and also the + // custom verification check on server side allows all connections. + // At stage 1, server disallows the connections by setting custom + // verification check. The following calls should fail. Previous + // connections should not be affected. + // At stage 2, server allows all the connections again and the + // authentications should go back to normal. + { + desc: "TestServerCustomVerification", + clientCert: []tls.Certificate{cs.ClientCert1}, + clientRoot: cs.ClientTrust1, + clientVerifyFunc: func(*HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + return &PostHandshakeVerificationResults{}, nil + }, + clientVerificationType: CertVerification, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverRoot: cs.ServerTrust1, + serverVerifyFunc: func(*HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + switch stage.read() { + case 0, 2: + return &PostHandshakeVerificationResults{}, nil + case 1: + return nil, fmt.Errorf("custom authz check fails") + default: + return nil, fmt.Errorf("custom authz check fails") + } + }, + serverVerificationType: CertVerification, + }, + } { + test := test + t.Run(test.desc, func(t *testing.T) { + // Start a server using ServerOptions in another goroutine. + serverOptions := &Options{ + IdentityOptions: IdentityCertificateOptions{ + Certificates: test.serverCert, + GetIdentityCertificatesForServer: test.serverGetCert, + }, + RootOptions: RootCertificateOptions{ + RootCertificates: test.serverRoot, + GetRootCertificates: test.serverGetRoot, + }, + RequireClientCert: true, + AdditionalPeerVerification: test.serverVerifyFunc, + VerificationType: test.serverVerificationType, + } + serverTLSCreds, err := NewServerCreds(serverOptions) + if err != nil { + t.Fatalf("failed to create server creds: %v", err) + } + s := grpc.NewServer(grpc.Creds(serverTLSCreds)) + defer s.Stop() + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer lis.Close() + addr := fmt.Sprintf("localhost:%v", lis.Addr().(*net.TCPAddr).Port) + pb.RegisterGreeterServer(s, greeterServer{}) + go s.Serve(lis) + clientOptions := &Options{ + IdentityOptions: IdentityCertificateOptions{ + Certificates: test.clientCert, + GetIdentityCertificatesForClient: test.clientGetCert, + }, + AdditionalPeerVerification: test.clientVerifyFunc, + RootOptions: RootCertificateOptions{ + RootCertificates: test.clientRoot, + GetRootCertificates: test.clientGetRoot, + }, + VerificationType: test.clientVerificationType, + } + clientTLSCreds, err := NewClientCreds(clientOptions) + if err != nil { + t.Fatalf("clientTLSCreds failed to create: %v", err) + } + // ------------------------Scenario 1------------------------------------ + // stage = 0, initial connection should succeed + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, greetClient, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 1", clientTLSCreds, false) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + // ---------------------------------------------------------------------- + stage.increase() + // ------------------------Scenario 2------------------------------------ + // stage = 1, previous connection should still succeed + err = callAndVerify(ctx, "rpc call 2", greetClient, false) + if err != nil { + t.Fatal(err) + } + // ------------------------Scenario 3------------------------------------ + // stage = 1, new connection should fail + ctx2, cancel2 := context.WithTimeout(context.Background(), defaultTestTimeout) + conn2, _, err := callAndVerifyWithClientConn(ctx2, addr, "rpc call 3", clientTLSCreds, true) + if err != nil { + t.Fatal(err) + } + defer conn2.Close() + // Immediately cancel the context so the dialing won't drag the entire timeout still it stops. + cancel2() + // ---------------------------------------------------------------------- + stage.increase() + // ------------------------Scenario 4------------------------------------ + // stage = 2, new connection should succeed + conn3, _, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 4", clientTLSCreds, false) + if err != nil { + t.Fatal(err) + } + defer conn3.Close() + // ---------------------------------------------------------------------- + stage.reset() + }) + } +} + +type tmpCredsFiles struct { + clientCertTmp *os.File + clientKeyTmp *os.File + clientTrustTmp *os.File + serverCertTmp *os.File + serverKeyTmp *os.File + serverTrustTmp *os.File +} + +// Create temp files that are used to hold credentials. +func createTmpFiles() (*tmpCredsFiles, error) { + tmpFiles := &tmpCredsFiles{} + var err error + tmpFiles.clientCertTmp, err = os.CreateTemp(os.TempDir(), "pre-") + if err != nil { + return nil, err + } + tmpFiles.clientKeyTmp, err = os.CreateTemp(os.TempDir(), "pre-") + if err != nil { + return nil, err + } + tmpFiles.clientTrustTmp, err = os.CreateTemp(os.TempDir(), "pre-") + if err != nil { + return nil, err + } + tmpFiles.serverCertTmp, err = os.CreateTemp(os.TempDir(), "pre-") + if err != nil { + return nil, err + } + tmpFiles.serverKeyTmp, err = os.CreateTemp(os.TempDir(), "pre-") + if err != nil { + return nil, err + } + tmpFiles.serverTrustTmp, err = os.CreateTemp(os.TempDir(), "pre-") + if err != nil { + return nil, err + } + return tmpFiles, nil +} + +// Copy the credential contents to the temporary files. +func (tmpFiles *tmpCredsFiles) copyCredsToTmpFiles() error { + if err := copyFileContents(testdata.Path("client_cert_1.pem"), tmpFiles.clientCertTmp.Name()); err != nil { + return err + } + if err := copyFileContents(testdata.Path("client_key_1.pem"), tmpFiles.clientKeyTmp.Name()); err != nil { + return err + } + if err := copyFileContents(testdata.Path("client_trust_cert_1.pem"), tmpFiles.clientTrustTmp.Name()); err != nil { + return err + } + if err := copyFileContents(testdata.Path("server_cert_1.pem"), tmpFiles.serverCertTmp.Name()); err != nil { + return err + } + if err := copyFileContents(testdata.Path("server_key_1.pem"), tmpFiles.serverKeyTmp.Name()); err != nil { + return err + } + if err := copyFileContents(testdata.Path("server_trust_cert_1.pem"), tmpFiles.serverTrustTmp.Name()); err != nil { + return err + } + return nil +} + +func (tmpFiles *tmpCredsFiles) removeFiles() { + os.Remove(tmpFiles.clientCertTmp.Name()) + os.Remove(tmpFiles.clientKeyTmp.Name()) + os.Remove(tmpFiles.clientTrustTmp.Name()) + os.Remove(tmpFiles.serverCertTmp.Name()) + os.Remove(tmpFiles.serverKeyTmp.Name()) + os.Remove(tmpFiles.serverTrustTmp.Name()) +} + +func copyFileContents(sourceFile, destinationFile string) error { + input, err := os.ReadFile(sourceFile) + if err != nil { + return err + } + err = os.WriteFile(destinationFile, input, 0644) + if err != nil { + return err + } + return nil +} + +// Create PEMFileProvider(s) watching the content changes of temporary +// files. +func createProviders(tmpFiles *tmpCredsFiles) (certprovider.Provider, certprovider.Provider, certprovider.Provider, certprovider.Provider, error) { + clientIdentityOptions := pemfile.Options{ + CertFile: tmpFiles.clientCertTmp.Name(), + KeyFile: tmpFiles.clientKeyTmp.Name(), + RefreshDuration: credRefreshingInterval, + } + clientIdentityProvider, err := pemfile.NewProvider(clientIdentityOptions) + if err != nil { + return nil, nil, nil, nil, err + } + clientRootOptions := pemfile.Options{ + RootFile: tmpFiles.clientTrustTmp.Name(), + RefreshDuration: credRefreshingInterval, + } + clientRootProvider, err := pemfile.NewProvider(clientRootOptions) + if err != nil { + return nil, nil, nil, nil, err + } + serverIdentityOptions := pemfile.Options{ + CertFile: tmpFiles.serverCertTmp.Name(), + KeyFile: tmpFiles.serverKeyTmp.Name(), + RefreshDuration: credRefreshingInterval, + } + serverIdentityProvider, err := pemfile.NewProvider(serverIdentityOptions) + if err != nil { + return nil, nil, nil, nil, err + } + serverRootOptions := pemfile.Options{ + RootFile: tmpFiles.serverTrustTmp.Name(), + RefreshDuration: credRefreshingInterval, + } + serverRootProvider, err := pemfile.NewProvider(serverRootOptions) + if err != nil { + return nil, nil, nil, nil, err + } + return clientIdentityProvider, clientRootProvider, serverIdentityProvider, serverRootProvider, nil +} + +// In order to test advanced TLS provider features, we used temporary files to +// hold credential data, and copy the contents under testdata/ to these tmp +// files. +// Initially, we establish a good connection with providers watching contents +// from tmp files. +// Next, we change the identity certs that IdentityProvider is watching. Since +// the identity key is not changed, the IdentityProvider should ignore the +// update, and the connection should still be good. +// Then the identity key is changed. This time IdentityProvider should pick +// up the update, and the connection should fail, due to the trust certs on the +// other side is not changed. +// Finally, the trust certs that other-side's RootProvider is watching get +// changed. The connection should go back to normal again. +func (s) TestPEMFileProviderEnd2End(t *testing.T) { + tmpFiles, err := createTmpFiles() + if err != nil { + t.Fatalf("createTmpFiles() failed, error: %v", err) + } + defer tmpFiles.removeFiles() + for _, test := range []struct { + desc string + certUpdateFunc func() + keyUpdateFunc func() + trustCertUpdateFunc func() + }{ + { + desc: "test the reloading feature for clientIdentityProvider and serverTrustProvider", + certUpdateFunc: func() { + err = copyFileContents(testdata.Path("client_cert_2.pem"), tmpFiles.clientCertTmp.Name()) + if err != nil { + t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("client_cert_2.pem"), tmpFiles.clientCertTmp.Name(), err) + } + }, + keyUpdateFunc: func() { + err = copyFileContents(testdata.Path("client_key_2.pem"), tmpFiles.clientKeyTmp.Name()) + if err != nil { + t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("client_key_2.pem"), tmpFiles.clientKeyTmp.Name(), err) + } + }, + trustCertUpdateFunc: func() { + err = copyFileContents(testdata.Path("server_trust_cert_2.pem"), tmpFiles.serverTrustTmp.Name()) + if err != nil { + t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("server_trust_cert_2.pem"), tmpFiles.serverTrustTmp.Name(), err) + } + }, + }, + { + desc: "test the reloading feature for serverIdentityProvider and clientTrustProvider", + certUpdateFunc: func() { + err = copyFileContents(testdata.Path("server_cert_2.pem"), tmpFiles.serverCertTmp.Name()) + if err != nil { + t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("server_cert_2.pem"), tmpFiles.serverCertTmp.Name(), err) + } + }, + keyUpdateFunc: func() { + err = copyFileContents(testdata.Path("server_key_2.pem"), tmpFiles.serverKeyTmp.Name()) + if err != nil { + t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("server_key_2.pem"), tmpFiles.serverKeyTmp.Name(), err) + } + }, + trustCertUpdateFunc: func() { + err = copyFileContents(testdata.Path("client_trust_cert_2.pem"), tmpFiles.clientTrustTmp.Name()) + if err != nil { + t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("client_trust_cert_2.pem"), tmpFiles.clientTrustTmp.Name(), err) + } + }, + }, + } { + test := test + t.Run(test.desc, func(t *testing.T) { + if err := tmpFiles.copyCredsToTmpFiles(); err != nil { + t.Fatalf("tmpFiles.copyCredsToTmpFiles() failed, error: %v", err) + } + clientIdentityProvider, clientRootProvider, serverIdentityProvider, serverRootProvider, err := createProviders(tmpFiles) + if err != nil { + t.Fatalf("createProviders(%v) failed, error: %v", tmpFiles, err) + } + defer clientIdentityProvider.Close() + defer clientRootProvider.Close() + defer serverIdentityProvider.Close() + defer serverRootProvider.Close() + // Start a server and create a client using advancedtls API with Provider. + serverOptions := &Options{ + IdentityOptions: IdentityCertificateOptions{ + IdentityProvider: serverIdentityProvider, + }, + RootOptions: RootCertificateOptions{ + RootProvider: serverRootProvider, + }, + RequireClientCert: true, + AdditionalPeerVerification: func(*HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + return &PostHandshakeVerificationResults{}, nil + }, + VerificationType: CertVerification, + } + serverTLSCreds, err := NewServerCreds(serverOptions) + if err != nil { + t.Fatalf("failed to create server creds: %v", err) + } + s := grpc.NewServer(grpc.Creds(serverTLSCreds)) + defer s.Stop() + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer lis.Close() + addr := fmt.Sprintf("localhost:%v", lis.Addr().(*net.TCPAddr).Port) + pb.RegisterGreeterServer(s, greeterServer{}) + go s.Serve(lis) + clientOptions := &Options{ + IdentityOptions: IdentityCertificateOptions{ + IdentityProvider: clientIdentityProvider, + }, + AdditionalPeerVerification: func(*HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + return &PostHandshakeVerificationResults{}, nil + }, + RootOptions: RootCertificateOptions{ + RootProvider: clientRootProvider, + }, + VerificationType: CertVerification, + } + clientTLSCreds, err := NewClientCreds(clientOptions) + if err != nil { + t.Fatalf("clientTLSCreds failed to create, error: %v", err) + } + + // At initialization, the connection should be good. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, greetClient, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 1", clientTLSCreds, false) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + // Make the identity cert change, and wait 1 second for the provider to + // pick up the change. + test.certUpdateFunc() + time.Sleep(sleepInterval) + // The already-established connection should not be affected. + err = callAndVerify(ctx, "rpc call 2", greetClient, false) + if err != nil { + t.Fatal(err) + } + // New connections should still be good, because the Provider didn't pick + // up the changes due to key-cert mismatch. + conn2, _, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 3", clientTLSCreds, false) + if err != nil { + t.Fatal(err) + } + defer conn2.Close() + // Make the identity key change, and wait 1 second for the provider to + // pick up the change. + test.keyUpdateFunc() + time.Sleep(sleepInterval) + // New connections should fail now, because the Provider picked the + // change, and *_cert_2.pem is not trusted by *_trust_cert_1.pem on the + // other side. + ctx2, cancel2 := context.WithTimeout(context.Background(), defaultTestTimeout) + conn3, _, err := callAndVerifyWithClientConn(ctx2, addr, "rpc call 4", clientTLSCreds, true) + if err != nil { + t.Fatal(err) + } + defer conn3.Close() + // Immediately cancel the context so the dialing won't drag the entire timeout still it stops. + cancel2() + // Make the trust cert change on the other side, and wait 1 second for + // the provider to pick up the change. + test.trustCertUpdateFunc() + time.Sleep(sleepInterval) + // New connections should be good, because the other side is using + // *_trust_cert_2.pem now. + conn4, _, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 5", clientTLSCreds, false) + if err != nil { + t.Fatal(err) + } + defer conn4.Close() + }) + } +} + +func (s) TestDefaultHostNameCheck(t *testing.T) { + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + t.Fatalf("cs.LoadCerts() failed, err: %v", err) + } + for _, test := range []struct { + desc string + clientRoot *x509.CertPool + clientVerificationType VerificationType + serverCert []tls.Certificate + serverVerificationType VerificationType + expectError bool + }{ + // Client side sets vType to CertAndHostVerification, and will do + // default hostname check. Server uses a cert without "localhost" or + // "127.0.0.1" as common name or SAN names, and will hence fail. + { + desc: "Bad default hostname check", + clientRoot: cs.ClientTrust1, + clientVerificationType: CertAndHostVerification, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverVerificationType: CertAndHostVerification, + expectError: true, + }, + // Client side sets vType to CertAndHostVerification, and will do + // default hostname check. Server uses a certificate with "localhost" as + // common name, and will hence pass the default hostname check. + { + desc: "Good default hostname check", + clientRoot: cs.ClientTrust1, + clientVerificationType: CertAndHostVerification, + serverCert: []tls.Certificate{cs.ServerPeerLocalhost1}, + serverVerificationType: CertAndHostVerification, + expectError: false, + }, + } { + test := test + t.Run(test.desc, func(t *testing.T) { + // Start a server using ServerOptions in another goroutine. + serverOptions := &Options{ + IdentityOptions: IdentityCertificateOptions{ + Certificates: test.serverCert, + }, + RequireClientCert: false, + VerificationType: test.serverVerificationType, + } + serverTLSCreds, err := NewServerCreds(serverOptions) + if err != nil { + t.Fatalf("failed to create server creds: %v", err) + } + s := grpc.NewServer(grpc.Creds(serverTLSCreds)) + defer s.Stop() + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer lis.Close() + addr := fmt.Sprintf("localhost:%v", lis.Addr().(*net.TCPAddr).Port) + pb.RegisterGreeterServer(s, greeterServer{}) + go s.Serve(lis) + clientOptions := &Options{ + RootOptions: RootCertificateOptions{ + RootCertificates: test.clientRoot, + }, + VerificationType: test.clientVerificationType, + } + clientTLSCreds, err := NewClientCreds(clientOptions) + if err != nil { + t.Fatalf("clientTLSCreds failed to create: %v", err) + } + shouldFail := false + if test.expectError { + shouldFail = true + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, _, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 1", clientTLSCreds, shouldFail) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + }) + } +} + +func (s) TestTLSVersions(t *testing.T) { + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + t.Fatalf("cs.LoadCerts() failed, err: %v", err) + } + for _, test := range []struct { + desc string + expectError bool + clientMinVersion uint16 + clientMaxVersion uint16 + serverMinVersion uint16 + serverMaxVersion uint16 + }{ + // Client side sets TLS version that is higher than required from the server side. + { + desc: "Client TLS version higher than server", + clientMinVersion: tls.VersionTLS13, + clientMaxVersion: tls.VersionTLS13, + serverMinVersion: tls.VersionTLS12, + serverMaxVersion: tls.VersionTLS12, + expectError: true, + }, + // Server side sets TLS version that is higher than required from the client side. + { + desc: "Server TLS version higher than client", + clientMinVersion: tls.VersionTLS12, + clientMaxVersion: tls.VersionTLS12, + serverMinVersion: tls.VersionTLS13, + serverMaxVersion: tls.VersionTLS13, + expectError: true, + }, + // Client and server set proper TLS versions. + { + desc: "Good TLS version settings", + clientMinVersion: tls.VersionTLS12, + clientMaxVersion: tls.VersionTLS13, + serverMinVersion: tls.VersionTLS12, + serverMaxVersion: tls.VersionTLS13, + expectError: false, + }, + { + desc: "Client 1.2 - 1.3 and server 1.2", + clientMinVersion: tls.VersionTLS12, + clientMaxVersion: tls.VersionTLS13, + serverMinVersion: tls.VersionTLS12, + serverMaxVersion: tls.VersionTLS12, + expectError: false, + }, + { + desc: "Client 1.2 - 1.3 and server 1.1 - 1.2", + clientMinVersion: tls.VersionTLS12, + clientMaxVersion: tls.VersionTLS13, + serverMinVersion: tls.VersionTLS11, + serverMaxVersion: tls.VersionTLS12, + expectError: false, + }, + { + desc: "Client 1.2 - 1.3 and server 1.3", + clientMinVersion: tls.VersionTLS12, + clientMaxVersion: tls.VersionTLS13, + serverMinVersion: tls.VersionTLS13, + serverMaxVersion: tls.VersionTLS13, + expectError: false, + }, + { + desc: "Client 1.2 - 1.2 and server 1.2 - 1.3", + clientMinVersion: tls.VersionTLS12, + clientMaxVersion: tls.VersionTLS12, + serverMinVersion: tls.VersionTLS12, + serverMaxVersion: tls.VersionTLS13, + expectError: false, + }, + { + desc: "Client 1.1 - 1.2 and server 1.2 - 1.3", + clientMinVersion: tls.VersionTLS11, + clientMaxVersion: tls.VersionTLS12, + serverMinVersion: tls.VersionTLS12, + serverMaxVersion: tls.VersionTLS13, + expectError: false, + }, + { + desc: "Client 1.3 and server 1.2 - 1.3", + clientMinVersion: tls.VersionTLS13, + clientMaxVersion: tls.VersionTLS13, + serverMinVersion: tls.VersionTLS12, + serverMaxVersion: tls.VersionTLS13, + expectError: false, + }, + } { + test := test + t.Run(test.desc, func(t *testing.T) { + // Start a server using ServerOptions in another goroutine. + serverOptions := &Options{ + IdentityOptions: IdentityCertificateOptions{ + Certificates: []tls.Certificate{cs.ServerPeerLocalhost1}, + }, + RequireClientCert: false, + VerificationType: CertAndHostVerification, + MinTLSVersion: test.serverMinVersion, + MaxTLSVersion: test.serverMaxVersion, + } + serverTLSCreds, err := NewServerCreds(serverOptions) + if err != nil { + t.Fatalf("failed to create server creds: %v", err) + } + s := grpc.NewServer(grpc.Creds(serverTLSCreds)) + defer s.Stop() + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer lis.Close() + addr := fmt.Sprintf("localhost:%v", lis.Addr().(*net.TCPAddr).Port) + pb.RegisterGreeterServer(s, greeterServer{}) + go s.Serve(lis) + clientOptions := &Options{ + RootOptions: RootCertificateOptions{ + RootCertificates: cs.ClientTrust1, + }, + VerificationType: CertAndHostVerification, + MinTLSVersion: test.clientMinVersion, + MaxTLSVersion: test.clientMaxVersion, + } + clientTLSCreds, err := NewClientCreds(clientOptions) + if err != nil { + t.Fatalf("clientTLSCreds failed to create: %v", err) + } + shouldFail := false + if test.expectError { + shouldFail = true + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, _, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 1", clientTLSCreds, shouldFail) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + }) + } +} diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go new file mode 100644 index 0000000..0bb6375 --- /dev/null +++ b/security/advancedtls/advancedtls_test.go @@ -0,0 +1,1324 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package advancedtls + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net" + "os" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/security/advancedtls/internal/testutils" + "google.golang.org/grpc/security/advancedtls/testdata" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type provType int + +const ( + provTypeRoot provType = iota + provTypeIdentity +) + +type fakeProvider struct { + pt provType + isClient bool + wantMultiCert bool + wantError bool +} + +func (f fakeProvider) KeyMaterial(context.Context) (*certprovider.KeyMaterial, error) { + if f.wantError { + return nil, fmt.Errorf("bad fakeProvider") + } + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + return nil, fmt.Errorf("cs.LoadCerts() failed, err: %v", err) + } + if f.pt == provTypeRoot && f.isClient { + return &certprovider.KeyMaterial{Roots: cs.ClientTrust1}, nil + } + if f.pt == provTypeRoot && !f.isClient { + return &certprovider.KeyMaterial{Roots: cs.ServerTrust1}, nil + } + if f.pt == provTypeIdentity && f.isClient { + if f.wantMultiCert { + return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1, cs.ClientCert2}}, nil + } + return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1}}, nil + } + if f.wantMultiCert { + return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ServerCert1, cs.ServerCert2}}, nil + } + return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ServerCert1}}, nil +} + +func (f fakeProvider) Close() {} + +func (s) TestClientOptionsConfigErrorCases(t *testing.T) { + tests := []struct { + desc string + clientVerificationType VerificationType + IdentityOptions IdentityCertificateOptions + RootOptions RootCertificateOptions + MinVersion uint16 + MaxVersion uint16 + }{ + { + desc: "Skip default verification and provide no root credentials", + clientVerificationType: SkipVerification, + }, + { + desc: "More than one fields in RootCertificateOptions is specified", + clientVerificationType: CertVerification, + RootOptions: RootCertificateOptions{ + RootCertificates: x509.NewCertPool(), + RootProvider: fakeProvider{}, + }, + }, + { + desc: "More than one fields in IdentityCertificateOptions is specified", + clientVerificationType: CertVerification, + IdentityOptions: IdentityCertificateOptions{ + GetIdentityCertificatesForClient: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return nil, nil + }, + IdentityProvider: fakeProvider{pt: provTypeIdentity}, + }, + }, + { + desc: "Specify GetIdentityCertificatesForServer", + IdentityOptions: IdentityCertificateOptions{ + GetIdentityCertificatesForServer: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + return nil, nil + }, + }, + }, + { + desc: "Invalid min/max TLS versions", + MinVersion: tls.VersionTLS13, + MaxVersion: tls.VersionTLS12, + }, + } + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + clientOptions := &Options{ + VerificationType: test.clientVerificationType, + IdentityOptions: test.IdentityOptions, + RootOptions: test.RootOptions, + MinTLSVersion: test.MinVersion, + MaxTLSVersion: test.MaxVersion, + } + _, err := clientOptions.clientConfig() + if err == nil { + t.Fatalf("ClientOptions{%v}.config() returns no err, wantErr != nil", clientOptions) + } + }) + } +} + +func (s) TestClientOptionsConfigSuccessCases(t *testing.T) { + tests := []struct { + desc string + clientVerificationType VerificationType + IdentityOptions IdentityCertificateOptions + RootOptions RootCertificateOptions + MinVersion uint16 + MaxVersion uint16 + cipherSuites []uint16 + curvePreferences []tls.CurveID + }{ + { + desc: "Use system default if no fields in RootCertificateOptions is specified", + clientVerificationType: CertVerification, + }, + { + desc: "Good case with mutual TLS", + clientVerificationType: CertVerification, + RootOptions: RootCertificateOptions{ + RootProvider: fakeProvider{}, + }, + IdentityOptions: IdentityCertificateOptions{ + IdentityProvider: fakeProvider{pt: provTypeIdentity}, + }, + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS13, + }, + { + desc: "Ciphersuite plumbing through client options", + cipherSuites: []uint16{ + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + }, + }, + { + desc: "CurvePreferences plumbing through client options", + curvePreferences: []tls.CurveID{ + tls.X25519, + tls.CurveP256, + }, + }, + { + desc: "MaxVersion default is applied when only MinVersion is set", + clientVerificationType: CertVerification, + IdentityOptions: IdentityCertificateOptions{ + Certificates: []tls.Certificate{}, + }, + RootOptions: RootCertificateOptions{ + RootProvider: fakeProvider{}, + }, + MinVersion: tls.VersionTLS12, + }, + } + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + clientOptions := &Options{ + VerificationType: test.clientVerificationType, + IdentityOptions: test.IdentityOptions, + RootOptions: test.RootOptions, + MinTLSVersion: test.MinVersion, + MaxTLSVersion: test.MaxVersion, + CipherSuites: test.cipherSuites, + CurvePreferences: test.curvePreferences, + } + clientConfig, err := clientOptions.clientConfig() + if err != nil { + t.Fatalf("ClientOptions{%v}.config() = %v, wantErr == nil", clientOptions, err) + } + // Verify that the system-provided certificates would be used + // when no verification method was set in clientOptions. + if clientOptions.RootOptions.RootCertificates == nil && + clientOptions.RootOptions.GetRootCertificates == nil && clientOptions.RootOptions.RootProvider == nil { + if clientConfig.RootCAs == nil { + t.Fatalf("Failed to assign system-provided certificates on the client side.") + } + } + if test.MinVersion != 0 { + if clientConfig.MinVersion != test.MinVersion { + t.Fatalf("Failed to assign min tls version.") + } + } else { + if clientConfig.MinVersion != tls.VersionTLS12 { + t.Fatalf("Default min tls version not set correctly") + } + } + if test.MaxVersion != 0 { + if clientConfig.MaxVersion != test.MaxVersion { + t.Fatalf("Failed to assign max tls version.") + } + } else { + if clientConfig.MaxVersion != tls.VersionTLS13 { + t.Fatalf("Default max tls version not set correctly") + } + } + if diff := cmp.Diff(clientConfig.CipherSuites, test.cipherSuites); diff != "" { + t.Errorf("cipherSuites diff (-want +got):\n%s", diff) + } + if diff := cmp.Diff(clientConfig.CurvePreferences, test.curvePreferences); diff != "" { + t.Errorf("curvePreferences diff (-want +got):\n%s", diff) + } + }) + } +} + +func (s) TestServerOptionsConfigErrorCases(t *testing.T) { + tests := []struct { + desc string + requireClientCert bool + serverVerificationType VerificationType + IdentityOptions IdentityCertificateOptions + RootOptions RootCertificateOptions + MinVersion uint16 + MaxVersion uint16 + }{ + { + desc: "Skip default verification and provide no root credentials", + requireClientCert: true, + serverVerificationType: SkipVerification, + }, + { + desc: "More than one fields in RootCertificateOptions is specified", + requireClientCert: true, + serverVerificationType: CertVerification, + RootOptions: RootCertificateOptions{ + RootCertificates: x509.NewCertPool(), + GetRootCertificates: func(*ConnectionInfo) (*RootCertificates, error) { + return nil, nil + }, + }, + }, + { + desc: "More than one fields in IdentityCertificateOptions is specified", + serverVerificationType: CertVerification, + IdentityOptions: IdentityCertificateOptions{ + Certificates: []tls.Certificate{}, + IdentityProvider: fakeProvider{pt: provTypeIdentity}, + }, + }, + { + desc: "no field in IdentityCertificateOptions is specified", + serverVerificationType: CertVerification, + }, + { + desc: "Specify GetIdentityCertificatesForClient", + IdentityOptions: IdentityCertificateOptions{ + GetIdentityCertificatesForClient: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return nil, nil + }, + }, + }, + { + desc: "Invalid min/max TLS versions", + MinVersion: tls.VersionTLS13, + MaxVersion: tls.VersionTLS12, + }, + } + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + serverOptions := &Options{ + VerificationType: test.serverVerificationType, + RequireClientCert: test.requireClientCert, + IdentityOptions: test.IdentityOptions, + RootOptions: test.RootOptions, + MinTLSVersion: test.MinVersion, + MaxTLSVersion: test.MaxVersion, + } + _, err := serverOptions.serverConfig() + if err == nil { + t.Fatalf("ServerOptions{%v}.serverConfig() returns no err, wantErr != nil", serverOptions) + } + }) + } +} + +func (s) TestServerOptionsConfigSuccessCases(t *testing.T) { + tests := []struct { + desc string + requireClientCert bool + serverVerificationType VerificationType + IdentityOptions IdentityCertificateOptions + RootOptions RootCertificateOptions + MinVersion uint16 + MaxVersion uint16 + cipherSuites []uint16 + curvePreferences []tls.CurveID + }{ + { + desc: "Use system default if no fields in RootCertificateOptions is specified", + requireClientCert: true, + serverVerificationType: CertVerification, + IdentityOptions: IdentityCertificateOptions{ + Certificates: []tls.Certificate{}, + }, + }, + { + desc: "Good case with mutual TLS", + requireClientCert: true, + serverVerificationType: CertVerification, + RootOptions: RootCertificateOptions{ + RootProvider: fakeProvider{}, + }, + IdentityOptions: IdentityCertificateOptions{ + GetIdentityCertificatesForServer: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + return nil, nil + }, + }, + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS13, + }, + { + desc: "Ciphersuite plumbing through server options", + IdentityOptions: IdentityCertificateOptions{ + Certificates: []tls.Certificate{}, + }, + RootOptions: RootCertificateOptions{ + RootCertificates: x509.NewCertPool(), + }, + cipherSuites: []uint16{ + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + }, + }, + { + desc: "MaxVersion default is applied when only MinVersion is set", + serverVerificationType: CertVerification, + IdentityOptions: IdentityCertificateOptions{ + Certificates: []tls.Certificate{}, + }, + RootOptions: RootCertificateOptions{ + RootProvider: fakeProvider{}, + }, + MinVersion: tls.VersionTLS12, + }, + { + desc: "CurvePreferences plumbing through server options", + IdentityOptions: IdentityCertificateOptions{ + Certificates: []tls.Certificate{}, + }, + RootOptions: RootCertificateOptions{ + RootCertificates: x509.NewCertPool(), + }, + curvePreferences: []tls.CurveID{ + tls.X25519, + tls.CurveP256, + }, + }, + } + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + serverOptions := &Options{ + VerificationType: test.serverVerificationType, + RequireClientCert: test.requireClientCert, + IdentityOptions: test.IdentityOptions, + RootOptions: test.RootOptions, + MinTLSVersion: test.MinVersion, + MaxTLSVersion: test.MaxVersion, + CipherSuites: test.cipherSuites, + CurvePreferences: test.curvePreferences, + } + serverConfig, err := serverOptions.serverConfig() + if err != nil { + t.Fatalf("ServerOptions{%v}.config() = %v, wantErr == nil", serverOptions, err) + } + // Verify that the system-provided certificates would be used + // when no verification method was set in serverOptions. + if serverOptions.RootOptions.RootCertificates == nil && + serverOptions.RootOptions.GetRootCertificates == nil && serverOptions.RootOptions.RootProvider == nil { + if serverConfig.ClientCAs == nil { + t.Fatalf("Failed to assign system-provided certificates on the server side.") + } + } + if diff := cmp.Diff(serverConfig.CipherSuites, test.cipherSuites); diff != "" { + t.Errorf("cipherSuites diff (-want +got):\n%s", diff) + } + if diff := cmp.Diff(serverConfig.CurvePreferences, test.curvePreferences); diff != "" { + t.Errorf("curvePreferences diff (-want +got):\n%s", diff) + } + }) + } +} + +func (s) TestClientServerHandshake(t *testing.T) { + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + t.Fatalf("cs.LoadCerts() failed, err: %v", err) + } + getRootCertificatesForClient := func(*ConnectionInfo) (*RootCertificates, error) { + return &RootCertificates{TrustCerts: cs.ClientTrust1}, nil + } + + clientVerifyFuncGood := func(params *HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + if params.ServerName == "" { + return nil, errors.New("client side server name should have a value") + } + if params.ConnectionState.CurveID == 0 { + return nil, errors.New("client side key exchange algorithm should not be zero") + } + // "foo.bar.com" is the common name on server certificate server_cert_1.pem. + if len(params.VerifiedChains) > 0 && (params.Leaf == nil || params.Leaf.Subject.CommonName != "foo.bar.com") { + return nil, errors.New("client side params parsing error") + } + + return &PostHandshakeVerificationResults{}, nil + } + verifyFuncBad := func(*HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + return nil, fmt.Errorf("custom verification function failed") + } + getRootCertificatesForServer := func(*ConnectionInfo) (*RootCertificates, error) { + return &RootCertificates{TrustCerts: cs.ServerTrust1}, nil + } + serverVerifyFunc := func(params *HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + if params.ServerName != "" { + return nil, errors.New("server side server name should not have a value") + } + if params.ConnectionState.CurveID == 0 { + return nil, errors.New("server side key exchange algorithm should not be zero") + } + // "foo.bar.hoo.com" is the common name on client certificate client_cert_1.pem. + if len(params.VerifiedChains) > 0 && (params.Leaf == nil || params.Leaf.Subject.CommonName != "foo.bar.hoo.com") { + return nil, errors.New("server side params parsing error") + } + + return &PostHandshakeVerificationResults{}, nil + } + getRootCertificatesForServerBad := func(*ConnectionInfo) (*RootCertificates, error) { + return nil, fmt.Errorf("bad root certificate reloading") + } + + getRootCertificatesForClientCRL := func(*ConnectionInfo) (*RootCertificates, error) { + return &RootCertificates{TrustCerts: cs.ClientTrust3}, nil + } + + getRootCertificatesForServerCRL := func(*ConnectionInfo) (*RootCertificates, error) { + return &RootCertificates{TrustCerts: cs.ServerTrust3}, nil + } + + makeStaticCRLRevocationOptions := func(crlPath string, denyUndetermined bool) *RevocationOptions { + rawCRL, err := os.ReadFile(crlPath) + if err != nil { + t.Fatalf("readFile(%v) failed err = %v", crlPath, err) + } + cRLProvider := NewStaticCRLProvider([][]byte{rawCRL}) + return &RevocationOptions{ + DenyUndetermined: denyUndetermined, + CRLProvider: cRLProvider, + } + } + + for _, test := range []struct { + desc string + clientCert []tls.Certificate + clientGetCert func(*tls.CertificateRequestInfo) (*tls.Certificate, error) + clientRoot *x509.CertPool + clientGetRoot func(params *ConnectionInfo) (*RootCertificates, error) + clientVerifyFunc PostHandshakeVerificationFunc + clientVerificationType VerificationType + clientRootProvider certprovider.Provider + clientIdentityProvider certprovider.Provider + clientRevocationOptions *RevocationOptions + clientCurvePreferences []tls.CurveID + clientExpectHandshakeError bool + serverMutualTLS bool + serverCert []tls.Certificate + serverGetCert func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) + serverRoot *x509.CertPool + serverGetRoot func(params *ConnectionInfo) (*RootCertificates, error) + serverVerifyFunc PostHandshakeVerificationFunc + serverVerificationType VerificationType + serverRootProvider certprovider.Provider + serverIdentityProvider certprovider.Provider + serverRevocationOptions *RevocationOptions + serverCurvePreferences []tls.CurveID + serverExpectError bool + }{ + // Client: nil setting except verifyFuncGood + // Server: only set serverCert with mutual TLS off + // Expected Behavior: success + // Reason: we will use verifyFuncGood to verify the server, + // if either clientCert or clientGetCert is not set + { + desc: "Client has no trust cert with verifyFuncGood; server sends peer cert", + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: SkipVerification, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverVerificationType: CertAndHostVerification, + }, + // Client: set clientGetRoot and clientVerifyFunc + // Server: only set serverCert with mutual TLS off + // Expected Behavior: success + { + desc: "Client sets reload root function with verifyFuncGood; server sends peer cert", + clientGetRoot: getRootCertificatesForClient, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverVerificationType: CertAndHostVerification, + }, + // Client: set clientGetRoot and bad clientVerifyFunc function + // Server: only set serverCert with mutual TLS off + // Expected Behavior: server side failure and client handshake failure + // Reason: custom verification function is bad + { + desc: "Client sets reload root function with verifyFuncBad; server sends peer cert", + clientGetRoot: getRootCertificatesForClient, + clientVerifyFunc: verifyFuncBad, + clientVerificationType: CertVerification, + clientExpectHandshakeError: true, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverVerificationType: CertVerification, + serverExpectError: true, + }, + // Client: set clientGetRoot, clientVerifyFunc and clientCert + // Server: set serverRoot and serverCert with mutual TLS on + // Expected Behavior: success + { + desc: "Client sets peer cert, reload root function with verifyFuncGood; server sets peer cert and root cert; mutualTLS", + clientCert: []tls.Certificate{cs.ClientCert1}, + clientGetRoot: getRootCertificatesForClient, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + serverMutualTLS: true, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverRoot: cs.ServerTrust1, + serverVerificationType: CertVerification, + }, + // Client: set clientGetRoot, clientVerifyFunc and clientCert + // Server: set serverGetRoot and serverCert with mutual TLS on + // Expected Behavior: success + { + desc: "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, reload root function; mutualTLS", + clientCert: []tls.Certificate{cs.ClientCert1}, + clientGetRoot: getRootCertificatesForClient, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + serverMutualTLS: true, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverGetRoot: getRootCertificatesForServer, + serverVerificationType: CertVerification, + }, + // Client: set clientGetRoot, clientVerifyFunc and clientCert + // Server: set serverGetRoot returning error and serverCert with mutual + // TLS on + // Expected Behavior: server side failure + // Reason: server side reloading returns failure + { + desc: "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, bad reload root function; mutualTLS", + clientCert: []tls.Certificate{cs.ClientCert1}, + clientGetRoot: getRootCertificatesForClient, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + serverMutualTLS: true, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverGetRoot: getRootCertificatesForServerBad, + serverVerificationType: CertVerification, + serverExpectError: true, + }, + // Client: set clientGetRoot, clientVerifyFunc and clientGetCert + // Server: set serverGetRoot and serverGetCert with mutual TLS on + // Expected Behavior: success + { + desc: "Client sets reload peer/root function with verifyFuncGood; Server sets reload peer/root function with verifyFuncGood; mutualTLS", + clientGetCert: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return &cs.ClientCert1, nil + }, + clientGetRoot: getRootCertificatesForClient, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + serverMutualTLS: true, + serverGetCert: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + return []*tls.Certificate{&cs.ServerCert1}, nil + }, + serverGetRoot: getRootCertificatesForServer, + serverVerifyFunc: serverVerifyFunc, + serverVerificationType: CertVerification, + }, + // Client: set everything but with the wrong peer cert not trusted by + // server + // Server: set serverGetRoot and serverGetCert with mutual TLS on + // Expected Behavior: server side returns failure because of + // certificate mismatch + { + desc: "Client sends wrong peer cert; Server sets reload peer/root function with verifyFuncGood; mutualTLS", + clientGetCert: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return &cs.ServerCert1, nil + }, + clientGetRoot: getRootCertificatesForClient, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + serverMutualTLS: true, + serverGetCert: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + return []*tls.Certificate{&cs.ServerCert1}, nil + }, + serverGetRoot: getRootCertificatesForServer, + serverVerifyFunc: serverVerifyFunc, + serverVerificationType: CertVerification, + serverExpectError: true, + }, + // Client: set everything but with the wrong trust cert not trusting server + // Server: set serverGetRoot and serverGetCert with mutual TLS on + // Expected Behavior: server side and client side return failure due to + // certificate mismatch and handshake failure + { + desc: "Client has wrong trust cert; Server sets reload peer/root function with verifyFuncGood; mutualTLS", + clientGetCert: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return &cs.ClientCert1, nil + }, + clientGetRoot: getRootCertificatesForServer, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + clientExpectHandshakeError: true, + serverMutualTLS: true, + serverGetCert: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + return []*tls.Certificate{&cs.ServerCert1}, nil + }, + serverGetRoot: getRootCertificatesForServer, + serverVerifyFunc: serverVerifyFunc, + serverVerificationType: CertVerification, + serverExpectError: true, + }, + // Client: set clientGetRoot, clientVerifyFunc and clientCert + // Server: set everything but with the wrong peer cert not trusted by + // client + // Expected Behavior: server side and client side return failure due to + // certificate mismatch and handshake failure + { + desc: "Client sets reload peer/root function with verifyFuncGood; Server sends wrong peer cert; mutualTLS", + clientGetCert: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return &cs.ClientCert1, nil + }, + clientGetRoot: getRootCertificatesForClient, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + serverMutualTLS: true, + serverGetCert: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + return []*tls.Certificate{&cs.ClientCert1}, nil + }, + serverGetRoot: getRootCertificatesForServer, + serverVerifyFunc: serverVerifyFunc, + serverVerificationType: CertVerification, + serverExpectError: true, + }, + // Client: set clientGetRoot, clientVerifyFunc and clientCert + // Server: set everything but with the wrong trust cert not trusting client + // Expected Behavior: server side and client side return failure due to + // certificate mismatch and handshake failure + { + desc: "Client sets reload peer/root function with verifyFuncGood; Server has wrong trust cert; mutualTLS", + clientGetCert: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return &cs.ClientCert1, nil + }, + clientGetRoot: getRootCertificatesForClient, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + clientExpectHandshakeError: true, + serverMutualTLS: true, + serverGetCert: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + return []*tls.Certificate{&cs.ServerCert1}, nil + }, + serverGetRoot: getRootCertificatesForClient, + serverVerifyFunc: serverVerifyFunc, + serverVerificationType: CertVerification, + serverExpectError: true, + }, + // Client: set clientGetRoot, clientVerifyFunc and clientCert + // Server: set serverGetRoot and serverCert, but with bad verifyFunc + // Expected Behavior: server side and client side return failure due to + // server custom check fails + { + desc: "Client sets peer cert, reload root function with verifyFuncGood; Server sets bad custom check; mutualTLS", + clientCert: []tls.Certificate{cs.ClientCert1}, + clientGetRoot: getRootCertificatesForClient, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + clientExpectHandshakeError: true, + serverMutualTLS: true, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverGetRoot: getRootCertificatesForServer, + serverVerifyFunc: verifyFuncBad, + serverVerificationType: CertVerification, + serverExpectError: true, + }, + // Client: set a clientIdentityProvider which will get multiple cert chains + // Server: set serverIdentityProvider and serverRootProvider with mutual TLS on + // Expected Behavior: server side failure due to multiple cert chains in + // clientIdentityProvider + { + desc: "Client sets multiple certs in clientIdentityProvider; Server sets root and identity provider; mutualTLS", + clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true, wantMultiCert: true}, + clientRootProvider: fakeProvider{isClient: true}, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + serverMutualTLS: true, + serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false}, + serverRootProvider: fakeProvider{isClient: false}, + serverVerificationType: CertVerification, + serverExpectError: true, + }, + // Client: set a bad clientIdentityProvider + // Server: set serverIdentityProvider and serverRootProvider with mutual TLS on + // Expected Behavior: server side failure due to bad clientIdentityProvider + { + desc: "Client sets bad clientIdentityProvider; Server sets root and identity provider; mutualTLS", + clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true, wantError: true}, + clientRootProvider: fakeProvider{isClient: true}, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + serverMutualTLS: true, + serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false}, + serverRootProvider: fakeProvider{isClient: false}, + serverVerificationType: CertVerification, + serverExpectError: true, + }, + // Client: set clientIdentityProvider and clientRootProvider + // Server: set bad serverRootProvider with mutual TLS on + // Expected Behavior: server side failure due to bad serverRootProvider + { + desc: "Client sets root and identity provider; Server sets bad root provider; mutualTLS", + clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true}, + clientRootProvider: fakeProvider{isClient: true}, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + serverMutualTLS: true, + serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false}, + serverRootProvider: fakeProvider{isClient: false, wantError: true}, + serverVerificationType: CertVerification, + serverExpectError: true, + }, + // Client: set clientIdentityProvider and clientRootProvider + // Server: set serverIdentityProvider and serverRootProvider with mutual TLS on + // Expected Behavior: success + { + desc: "Client sets root and identity provider; Server sets root and identity provider; mutualTLS", + clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true}, + clientRootProvider: fakeProvider{isClient: true}, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + serverMutualTLS: true, + serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false}, + serverRootProvider: fakeProvider{isClient: false}, + serverVerificationType: CertVerification, + }, + // Client: set clientIdentityProvider and clientRootProvider + // Server: set serverIdentityProvider getting multiple cert chains and serverRootProvider with mutual TLS on + // Expected Behavior: success, because server side has SNI + { + desc: "Client sets root and identity provider; Server sets multiple certs in serverIdentityProvider; mutualTLS", + clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true}, + clientRootProvider: fakeProvider{isClient: true}, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + serverMutualTLS: true, + serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false, wantMultiCert: true}, + serverRootProvider: fakeProvider{isClient: false}, + serverVerificationType: CertVerification, + }, + // Client: set valid credentials with the revocation config + // Server: set valid credentials with the revocation config + // Expected Behavior: success, because none of the certificate chains sent in the connection are revoked + { + desc: "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, reload root function; Client uses CRL; mutualTLS", + clientCert: []tls.Certificate{cs.ClientCertForCRL}, + clientGetRoot: getRootCertificatesForClientCRL, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + clientRevocationOptions: makeStaticCRLRevocationOptions(testdata.Path("crl/provider_crl_empty.pem"), false), + serverMutualTLS: true, + serverCert: []tls.Certificate{cs.ServerCertForCRL}, + serverGetRoot: getRootCertificatesForServerCRL, + serverVerificationType: CertVerification, + }, + // Client: set valid credentials with the revocation config + // Server: set revoked credentials with the revocation config + // Expected Behavior: fail, server creds are revoked + { + desc: "Client sets peer cert, reload root function with verifyFuncGood; Server sets revoked cert; Client uses CRL; mutualTLS", + clientCert: []tls.Certificate{cs.ClientCertForCRL}, + clientGetRoot: getRootCertificatesForClientCRL, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + clientRevocationOptions: makeStaticCRLRevocationOptions(testdata.Path("crl/provider_crl_server_revoked.pem"), true), + serverMutualTLS: true, + serverCert: []tls.Certificate{cs.ServerCertForCRL}, + serverGetRoot: getRootCertificatesForServerCRL, + serverVerificationType: CertVerification, + serverExpectError: true, + }, + // Client: set valid credentials with the revocation config + // Server: set valid credentials with the revocation config + // Expected Behavior: fail, because CRL is issued by the malicious CA. It + // can't be properly processed, and we don't allow RevocationUndetermined. + { + desc: "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, reload root function; Client uses CRL; mutualTLS", + clientCert: []tls.Certificate{cs.ClientCertForCRL}, + clientGetRoot: getRootCertificatesForClientCRL, + clientVerifyFunc: clientVerifyFuncGood, + clientVerificationType: CertVerification, + clientRevocationOptions: makeStaticCRLRevocationOptions(testdata.Path("crl/provider_malicious_crl_empty.pem"), true), + serverMutualTLS: true, + serverCert: []tls.Certificate{cs.ServerCertForCRL}, + serverGetRoot: getRootCertificatesForServerCRL, + serverVerificationType: CertVerification, + serverExpectError: true, + }, + { + desc: "Client and server set CurvePreferences to CurveP256", + clientCert: []tls.Certificate{cs.ClientCert1}, + clientGetRoot: getRootCertificatesForClient, + clientVerifyFunc: func(params *HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + if params.ConnectionState.CurveID != tls.CurveP256 { + return nil, fmt.Errorf("client got CurveID %v, want %v", params.ConnectionState.CurveID, tls.CurveP256) + } + return &PostHandshakeVerificationResults{}, nil + }, + clientVerificationType: CertVerification, + clientCurvePreferences: []tls.CurveID{tls.CurveP256}, + serverMutualTLS: true, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverRoot: cs.ServerTrust1, + serverVerifyFunc: func(params *HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + if params.ConnectionState.CurveID != tls.CurveP256 { + return nil, fmt.Errorf("server got CurveID %v, want %v", params.ConnectionState.CurveID, tls.CurveP256) + } + return &PostHandshakeVerificationResults{}, nil + }, + serverVerificationType: CertVerification, + serverCurvePreferences: []tls.CurveID{tls.CurveP256}, + }, + { + desc: "Client and server set CurvePreferences to X25519", + clientCert: []tls.Certificate{cs.ClientCert1}, + clientGetRoot: getRootCertificatesForClient, + clientVerifyFunc: func(params *HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + if params.ConnectionState.CurveID != tls.X25519 { + return nil, fmt.Errorf("client got CurveID %v, want %v", params.ConnectionState.CurveID, tls.X25519) + } + return &PostHandshakeVerificationResults{}, nil + }, + clientVerificationType: CertVerification, + clientCurvePreferences: []tls.CurveID{tls.X25519}, + serverMutualTLS: true, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverRoot: cs.ServerTrust1, + serverVerifyFunc: func(params *HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + if params.ConnectionState.CurveID != tls.X25519 { + return nil, fmt.Errorf("server got CurveID %v, want %v", params.ConnectionState.CurveID, tls.X25519) + } + return &PostHandshakeVerificationResults{}, nil + }, + serverVerificationType: CertVerification, + serverCurvePreferences: []tls.CurveID{tls.X25519}, + }, + { + desc: "Client and server set CurvePreferences to X25519MLKEM768", + clientCert: []tls.Certificate{cs.ClientCert1}, + clientGetRoot: getRootCertificatesForClient, + clientVerifyFunc: func(params *HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + if params.ConnectionState.CurveID != tls.X25519MLKEM768 { + return nil, fmt.Errorf("client got CurveID %v, want %v", params.ConnectionState.CurveID, tls.X25519MLKEM768) + } + return &PostHandshakeVerificationResults{}, nil + }, + clientVerificationType: CertVerification, + clientCurvePreferences: []tls.CurveID{tls.X25519MLKEM768}, + serverMutualTLS: true, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverRoot: cs.ServerTrust1, + serverVerifyFunc: func(params *HandshakeVerificationInfo) (*PostHandshakeVerificationResults, error) { + if params.ConnectionState.CurveID != tls.X25519MLKEM768 { + return nil, fmt.Errorf("server got CurveID %v, want %v", params.ConnectionState.CurveID, tls.X25519MLKEM768) + } + return &PostHandshakeVerificationResults{}, nil + }, + serverVerificationType: CertVerification, + serverCurvePreferences: []tls.CurveID{tls.X25519MLKEM768}, + }, + } { + test := test + t.Run(test.desc, func(t *testing.T) { + done := make(chan credentials.AuthInfo, 1) + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen: %v", err) + } + // Start a server using ServerOptions in another goroutine. + serverOptions := &Options{ + IdentityOptions: IdentityCertificateOptions{ + Certificates: test.serverCert, + GetIdentityCertificatesForServer: test.serverGetCert, + IdentityProvider: test.serverIdentityProvider, + }, + RootOptions: RootCertificateOptions{ + RootCertificates: test.serverRoot, + GetRootCertificates: test.serverGetRoot, + RootProvider: test.serverRootProvider, + }, + RequireClientCert: test.serverMutualTLS, + AdditionalPeerVerification: test.serverVerifyFunc, + VerificationType: test.serverVerificationType, + RevocationOptions: test.serverRevocationOptions, + CurvePreferences: test.serverCurvePreferences, + } + go func(done chan credentials.AuthInfo, lis net.Listener, serverOptions *Options) { + serverRawConn, err := lis.Accept() + if err != nil { + close(done) + return + } + serverTLS, err := NewServerCreds(serverOptions) + if err != nil { + serverRawConn.Close() + close(done) + return + } + _, serverAuthInfo, err := serverTLS.ServerHandshake(serverRawConn) + if err != nil { + serverRawConn.Close() + close(done) + return + } + done <- serverAuthInfo + }(done, lis, serverOptions) + defer lis.Close() + // Start a client using ClientOptions and connects to the server. + lisAddr := lis.Addr().String() + conn, err := net.Dial("tcp", lisAddr) + if err != nil { + t.Fatalf("Client failed to connect to %s. Error: %v", lisAddr, err) + } + defer conn.Close() + clientOptions := &Options{ + IdentityOptions: IdentityCertificateOptions{ + Certificates: test.clientCert, + GetIdentityCertificatesForClient: test.clientGetCert, + IdentityProvider: test.clientIdentityProvider, + }, + AdditionalPeerVerification: test.clientVerifyFunc, + RootOptions: RootCertificateOptions{ + RootCertificates: test.clientRoot, + GetRootCertificates: test.clientGetRoot, + RootProvider: test.clientRootProvider, + }, + VerificationType: test.clientVerificationType, + RevocationOptions: test.clientRevocationOptions, + CurvePreferences: test.clientCurvePreferences, + } + clientTLS, err := NewClientCreds(clientOptions) + if err != nil { + t.Fatalf("NewClientCreds failed: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, clientAuthInfo, handshakeErr := clientTLS.ClientHandshake(ctx, lisAddr, conn) + // wait until server sends serverAuthInfo or fails. + serverAuthInfo, ok := <-done + if !ok && test.serverExpectError { + return + } + if ok && test.serverExpectError || !ok && !test.serverExpectError { + t.Fatalf("Server side error mismatch, got %v, want %v", !ok, test.serverExpectError) + } + if handshakeErr != nil && test.clientExpectHandshakeError { + return + } + if handshakeErr != nil && !test.clientExpectHandshakeError || + handshakeErr == nil && test.clientExpectHandshakeError { + t.Fatalf("Expect error: %v, but err is %v", + test.clientExpectHandshakeError, handshakeErr) + } + if !compare(clientAuthInfo, serverAuthInfo) { + t.Fatalf("c.ClientHandshake(_, %v, _) = %v, want %v.", lisAddr, + clientAuthInfo, serverAuthInfo) + } + serverVerifiedChains := serverAuthInfo.(credentials.TLSInfo).State.VerifiedChains + if test.serverMutualTLS && !test.serverExpectError { + if len(serverVerifiedChains) == 0 { + t.Fatalf("server verified chains is empty") + } + var clientCert *tls.Certificate + if len(test.clientCert) > 0 { + clientCert = &test.clientCert[0] + } else if test.clientGetCert != nil { + cert, _ := test.clientGetCert(&tls.CertificateRequestInfo{}) + clientCert = cert + } else if test.clientIdentityProvider != nil { + km, _ := test.clientIdentityProvider.KeyMaterial(ctx) + clientCert = &km.Certs[0] + } + if !bytes.Equal((*serverVerifiedChains[0][0]).Raw, clientCert.Certificate[0]) { + t.Fatal("server verifiedChains leaf cert doesn't match client cert") + } + + var serverRoot *x509.CertPool + if test.serverRoot != nil { + serverRoot = test.serverRoot + } else if test.serverGetRoot != nil { + result, _ := test.serverGetRoot(&ConnectionInfo{}) + serverRoot = result.TrustCerts + } else if test.serverRootProvider != nil { + km, _ := test.serverRootProvider.KeyMaterial(ctx) + serverRoot = km.Roots + } + serverVerifiedChainsCp := x509.NewCertPool() + serverVerifiedChainsCp.AddCert(serverVerifiedChains[0][len(serverVerifiedChains[0])-1]) + if !serverVerifiedChainsCp.Equal(serverRoot) { + t.Fatalf("server verified chain hierarchy doesn't match") + } + } + clientVerifiedChains := clientAuthInfo.(credentials.TLSInfo).State.VerifiedChains + if test.serverMutualTLS && !test.clientExpectHandshakeError { + if len(clientVerifiedChains) == 0 { + t.Fatalf("client verified chains is empty") + } + var serverCert *tls.Certificate + if len(test.serverCert) > 0 { + serverCert = &test.serverCert[0] + } else if test.serverGetCert != nil { + cert, _ := test.serverGetCert(&tls.ClientHelloInfo{}) + serverCert = cert[0] + } else if test.serverIdentityProvider != nil { + km, _ := test.serverIdentityProvider.KeyMaterial(ctx) + serverCert = &km.Certs[0] + } + if !bytes.Equal((*clientVerifiedChains[0][0]).Raw, serverCert.Certificate[0]) { + t.Fatal("client verifiedChains leaf cert doesn't match server cert") + } + + var clientRoot *x509.CertPool + if test.clientRoot != nil { + clientRoot = test.clientRoot + } else if test.clientGetRoot != nil { + result, _ := test.clientGetRoot(&ConnectionInfo{}) + clientRoot = result.TrustCerts + } else if test.clientRootProvider != nil { + km, _ := test.clientRootProvider.KeyMaterial(ctx) + clientRoot = km.Roots + } + clientVerifiedChainsCp := x509.NewCertPool() + clientVerifiedChainsCp.AddCert(clientVerifiedChains[0][len(clientVerifiedChains[0])-1]) + if !clientVerifiedChainsCp.Equal(clientRoot) { + t.Fatalf("client verified chain hierarchy doesn't match") + } + } + }) + } +} + +func compare(a1, a2 credentials.AuthInfo) bool { + if a1.AuthType() != a2.AuthType() { + return false + } + switch a1.AuthType() { + case "tls": + state1 := a1.(credentials.TLSInfo).State + state2 := a2.(credentials.TLSInfo).State + if state1.Version == state2.Version && + state1.HandshakeComplete == state2.HandshakeComplete && + state1.CipherSuite == state2.CipherSuite && + state1.NegotiatedProtocol == state2.NegotiatedProtocol { + return true + } + return false + default: + return false + } +} + +func (s) TestAdvancedTLSOverrideServerName(t *testing.T) { + expectedServerName := "server.name" + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + t.Fatalf("cs.LoadCerts() failed, err: %v", err) + } + clientOptions := &Options{ + RootOptions: RootCertificateOptions{ + RootCertificates: cs.ClientTrust1, + }, + serverNameOverride: expectedServerName, + } + c, err := NewClientCreds(clientOptions) + if err != nil { + t.Fatalf("Client is unable to create credentials. Error: %v", err) + } + c.OverrideServerName(expectedServerName) + if c.Info().ServerName != expectedServerName { + t.Fatalf("c.Info().ServerName = %v, want %v", c.Info().ServerName, expectedServerName) + } +} + +func (s) TestGetCertificatesSNI(t *testing.T) { + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + t.Fatalf("cs.LoadCerts() failed, err: %v", err) + } + tests := []struct { + desc string + serverName string + // Use Common Name on the certificate to differentiate if we choose the right cert. The common name on all of the three certs are different. + wantCommonName string + }{ + { + desc: "Select ServerCert1", + // "foo.bar.com" is the common name on server certificate server_cert_1.pem. + serverName: "foo.bar.com", + wantCommonName: "foo.bar.com", + }, + { + desc: "Select serverCert3", + // "foo.bar.server3.com" is the common name on server certificate server_cert_3.pem. + // "google.com" is one of the DNS names on server certificate server_cert_3.pem. + serverName: "google.com", + wantCommonName: "foo.bar.server3.com", + }, + } + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + serverOptions := &Options{ + IdentityOptions: IdentityCertificateOptions{ + GetIdentityCertificatesForServer: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + return []*tls.Certificate{&cs.ServerCert1, &cs.ServerCert2, &cs.ServerPeer3}, nil + }, + }, + } + serverConfig, err := serverOptions.serverConfig() + if err != nil { + t.Fatalf("serverOptions.serverConfig() failed: %v", err) + } + pointFormatUncompressed := uint8(0) + clientHello := &tls.ClientHelloInfo{ + CipherSuites: []uint16{tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA}, + ServerName: test.serverName, + SupportedCurves: []tls.CurveID{tls.CurveP256}, + SupportedPoints: []uint8{pointFormatUncompressed}, + SupportedVersions: []uint16{tls.VersionTLS12}, + } + gotCertificate, err := serverConfig.GetCertificate(clientHello) + if err != nil { + t.Fatalf("serverConfig.GetCertificate(clientHello) failed: %v", err) + } + if gotCertificate == nil || len(gotCertificate.Certificate) == 0 { + t.Fatalf("Got nil or empty Certificate after calling serverConfig.GetCertificate.") + } + parsedCert, err := x509.ParseCertificate(gotCertificate.Certificate[0]) + if err != nil { + t.Fatalf("x509.ParseCertificate(%v) failed: %v", gotCertificate.Certificate[0], err) + } + if parsedCert == nil { + t.Fatalf("Got nil Certificate after calling x509.ParseCertificate.") + } + if parsedCert.Subject.CommonName != test.wantCommonName { + t.Errorf("Common name mismatch, got %v, want %v", parsedCert.Subject.CommonName, test.wantCommonName) + } + }) + } +} + +func (s) TestVerifyPeerCertificateZeroCerts(t *testing.T) { + creds := &advancedTLSCreds{ + verificationType: CertVerification, + isClient: true, + config: &tls.Config{}, + } + verifyPeerCertificate, _ := buildVerifyFunc(creds, "", nil, &CertificateChains{}) + // Calling verifyPeerCertificate with zero certificates. + const wantErr = "no peer certificates presented" + if err := verifyPeerCertificate(nil, nil); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Errorf("verifyPeerCertificate(nil, nil) = %v, want error containing %q", err, wantErr) + } +} + +func (s) TestServerAuthEKUValidation(t *testing.T) { + // Generate server cert with ClientAuth EKU (no ServerAuth EKU) + rootPool, serverCert := generateCertificates(t, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}) + + for _, test := range []struct { + desc string + skipServerAuthEKU bool + wantErr bool + }{ + { + desc: "SkipServerAuthEKU_False_HandshakeFails", + skipServerAuthEKU: false, + wantErr: true, + }, + { + desc: "SkipServerAuthEKU_True_HandshakeSucceeds", + skipServerAuthEKU: true, + wantErr: false, + }, + } { + t.Run(test.desc, func(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen: %v", err) + } + defer lis.Close() + + serverOptions := &Options{ + IdentityOptions: IdentityCertificateOptions{ + Certificates: []tls.Certificate{serverCert}, + }, + VerificationType: CertVerification, + } + done := make(chan credentials.AuthInfo, 1) + go func() { + serverRawConn, err := lis.Accept() + if err != nil { + close(done) + return + } + serverTLS, err := NewServerCreds(serverOptions) + if err != nil { + serverRawConn.Close() + close(done) + return + } + _, serverAuthInfo, err := serverTLS.ServerHandshake(serverRawConn) + if err != nil { + serverRawConn.Close() + close(done) + return + } + done <- serverAuthInfo + }() + + lisAddr := lis.Addr().String() + conn, err := net.Dial("tcp", lisAddr) + if err != nil { + t.Fatalf("Client failed to connect to %s. Error: %v", lisAddr, err) + } + defer conn.Close() + + clientOptions := &Options{ + RootOptions: RootCertificateOptions{ + RootCertificates: rootPool, + }, + VerificationType: CertVerification, + SkipServerAuthEKU: test.skipServerAuthEKU, + } + clientTLS, err := NewClientCreds(clientOptions) + if err != nil { + t.Fatalf("NewClientCreds failed: %v", err) + } + // Clone the credentials to ensure the cloned instance also respects SkipServerAuthEKU + clientTLS = clientTLS.Clone() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, _, handshakeErr := clientTLS.ClientHandshake(ctx, lisAddr, conn) + + // wait for server side to finish + select { + case <-done: + case <-time.After(defaultTestTimeout): + t.Fatalf("timed out waiting for server handshake to finish") + } + + gotErr := handshakeErr != nil + if gotErr != test.wantErr { + t.Fatalf("ClientHandshake() got error: %v, want error: %v", handshakeErr, test.wantErr) + } + }) + } +} diff --git a/security/advancedtls/certificate_utils_test.go b/security/advancedtls/certificate_utils_test.go new file mode 100644 index 0000000..08aad42 --- /dev/null +++ b/security/advancedtls/certificate_utils_test.go @@ -0,0 +1,81 @@ +/* + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package advancedtls + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "testing" + "time" +) + +func generateCertificates(t *testing.T, extKeyUsages []x509.ExtKeyUsage) (*x509.CertPool, tls.Certificate) { + t.Helper() + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("Failed to generate CA key: %v", err) + } + caTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "Test CA"}, + NotBefore: time.Now().Add(-1 * time.Hour), + NotAfter: time.Now().Add(1 * time.Hour), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + BasicConstraintsValid: true, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + if err != nil { + t.Fatalf("Failed to create CA cert: %v", err) + } + caCert, err := x509.ParseCertificate(caDER) + if err != nil { + t.Fatalf("Failed to parse CA cert: %v", err) + } + + serverKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("Failed to generate server key: %v", err) + } + serverTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "localhost"}, + NotBefore: time.Now().Add(-1 * time.Hour), + NotAfter: time.Now().Add(1 * time.Hour), + ExtKeyUsage: extKeyUsages, + DNSNames: []string{"localhost"}, + } + serverDER, err := x509.CreateCertificate(rand.Reader, serverTemplate, caCert, &serverKey.PublicKey, caKey) + if err != nil { + t.Fatalf("Failed to create server cert: %v", err) + } + + rootPool := x509.NewCertPool() + rootPool.AddCert(caCert) + + serverCert := tls.Certificate{ + Certificate: [][]byte{serverDER}, + PrivateKey: serverKey, + } + + return rootPool, serverCert +} diff --git a/security/advancedtls/crl.go b/security/advancedtls/crl.go new file mode 100644 index 0000000..b6d49bc --- /dev/null +++ b/security/advancedtls/crl.go @@ -0,0 +1,436 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package advancedtls + +import ( + "bytes" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/pem" + "errors" + "fmt" + "os" + + "golang.org/x/crypto/cryptobyte" + cbasn1 "golang.org/x/crypto/cryptobyte/asn1" + "google.golang.org/grpc/grpclog" +) + +var grpclogLogger = grpclog.Component("advancedtls") + +// RevocationOptions allows a user to configure certificate revocation behavior. +type RevocationOptions struct { + // DenyUndetermined controls if certificate chains with RevocationUndetermined + // revocation status are allowed to complete. + DenyUndetermined bool + // CRLProvider is an alternative to using RootDir directly for the + // X509_LOOKUP_hash_dir approach to CRL files. If set, the CRLProvider's CRL + // function will be called when looking up and fetching CRLs during the + // handshake. + CRLProvider CRLProvider +} + +// revocationStatus is the revocation status for a certificate or chain. +type revocationStatus int + +const ( + // RevocationUndetermined means we couldn't find or verify a CRL for the cert. + RevocationUndetermined revocationStatus = iota + // RevocationUnrevoked means we found the CRL for the cert and the cert is not revoked. + RevocationUnrevoked + // RevocationRevoked means we found the CRL and the cert is revoked. + RevocationRevoked +) + +// CRL contains a pkix.CertificateList and parsed extensions that aren't +// provided by the golang CRL parser. +// All CRLs should be loaded using NewCRL() for bytes directly or ReadCRLFile() +// to read directly from a filepath +type CRL struct { + certList *x509.RevocationList + // RFC5280, 5.2.1, all conforming CRLs must have a AKID with the ID method. + authorityKeyID []byte + rawIssuer []byte +} + +// NewCRL constructs new CRL from the provided byte array. +func NewCRL(b []byte) (*CRL, error) { + crl, err := parseRevocationList(b) + if err != nil { + return nil, fmt.Errorf("fail to parse CRL: %v", err) + } + crlExt, err := parseCRLExtensions(crl) + if err != nil { + return nil, fmt.Errorf("fail to parse CRL extensions: %v", err) + } + crlExt.rawIssuer, err = extractCRLIssuer(b) + if err != nil { + return nil, fmt.Errorf("fail to extract CRL issuer failed err= %v", err) + } + return crlExt, nil +} + +// ReadCRLFile reads a file from the provided path, and returns constructed CRL +// struct from it. +func ReadCRLFile(path string) (*CRL, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("cannot read file from provided path %q: %v", path, err) + } + crl, err := NewCRL(b) + if err != nil { + return nil, fmt.Errorf("cannot construct CRL from file %q: %v", path, err) + } + return crl, nil +} + +const tagDirectoryName = 4 + +var ( + // RFC5280, 5.2.4 id-ce-deltaCRLIndicator OBJECT IDENTIFIER ::= { id-ce 27 } + oidDeltaCRLIndicator = asn1.ObjectIdentifier{2, 5, 29, 27} + // RFC5280, 5.2.5 id-ce-issuingDistributionPoint OBJECT IDENTIFIER ::= { id-ce 28 } + oidIssuingDistributionPoint = asn1.ObjectIdentifier{2, 5, 29, 28} + // RFC5280, 5.3.3 id-ce-certificateIssuer OBJECT IDENTIFIER ::= { id-ce 29 } + oidCertificateIssuer = asn1.ObjectIdentifier{2, 5, 29, 29} + // RFC5290, 4.2.1.1 id-ce-authorityKeyIdentifier OBJECT IDENTIFIER ::= { id-ce 35 } + oidAuthorityKeyIdentifier = asn1.ObjectIdentifier{2, 5, 29, 35} +) + +// checkChainRevocation checks the verified certificate chain +// for revoked certificates based on RFC5280. +func checkChainRevocation(verifiedChains [][]*x509.Certificate, cfg RevocationOptions) error { + // Iterate the verified chains looking for one that is RevocationUnrevoked. + // A single RevocationUnrevoked chain is enough to allow the connection, and a single RevocationRevoked + // chain does not mean the connection should fail. + count := make(map[revocationStatus]int) + for _, chain := range verifiedChains { + switch checkChain(chain, cfg) { + case RevocationUnrevoked: + // If any chain is RevocationUnrevoked then return no error. + return nil + case RevocationRevoked: + // If this chain is revoked, keep looking for another chain. + count[RevocationRevoked]++ + continue + case RevocationUndetermined: + count[RevocationUndetermined]++ + if cfg.DenyUndetermined { + continue + } + return nil + } + } + return fmt.Errorf("no unrevoked chains found: %v", count) +} + +// checkChain will determine and check all certificates in chain against the CRL +// defined in the certificate with the following rules: +// 1. If any certificate is RevocationRevoked, return RevocationRevoked. +// 2. If any certificate is RevocationUndetermined, return RevocationUndetermined. +// 3. If all certificates are RevocationUnrevoked, return RevocationUnrevoked. +func checkChain(chain []*x509.Certificate, cfg RevocationOptions) revocationStatus { + chainStatus := RevocationUnrevoked + for _, c := range chain { + switch checkCert(c, chain, cfg) { + case RevocationRevoked: + // Easy case, if a cert in the chain is revoked, the chain is revoked. + return RevocationRevoked + case RevocationUndetermined: + // If we couldn't find the revocation status for a cert, the chain is at best RevocationUndetermined + // keep looking to see if we find a cert in the chain that's RevocationRevoked, + // but return RevocationUndetermined at a minimum. + chainStatus = RevocationUndetermined + case RevocationUnrevoked: + // Continue iterating up the cert chain. + continue + } + } + return chainStatus +} + +func fetchCRL(c *x509.Certificate, crlVerifyCrt []*x509.Certificate, cfg RevocationOptions) (*CRL, error) { + if cfg.CRLProvider == nil { + return nil, fmt.Errorf("trying to fetch CRL but CRLProvider is nil") + } + crl, err := cfg.CRLProvider.CRL(c) + if err != nil { + return nil, fmt.Errorf("CrlProvider failed err = %v", err) + } + if crl == nil { + return nil, fmt.Errorf("no CRL found for certificate's issuer") + } + if err := verifyCRL(crl, crlVerifyCrt); err != nil { + return nil, fmt.Errorf("verifyCRL() failed: %v", err) + } + return crl, nil +} + +// checkCert checks a single certificate against the CRL defined in the +// certificate. It will fetch and verify the CRL(s) defined in the root +// directory (or a CRLProvider) specified by cfg. If we can't load (and verify - +// see verifyCRL) any valid authoritative CRL files, the status is +// RevocationUndetermined. +// c is the certificate to check. +// crlVerifyCrt is the group of possible certificates to verify the crl. +func checkCert(c *x509.Certificate, crlVerifyCrt []*x509.Certificate, cfg RevocationOptions) revocationStatus { + crl, err := fetchCRL(c, crlVerifyCrt, cfg) + if err != nil { + // We couldn't load any valid CRL files for the certificate, so we don't + // know if it's RevocationUnrevoked or not. This is not necessarily a + // problem - it's not invalid to have no CRLs if you don't have any + // revocations for an issuer. It also might be an indication that the CRL + // file is invalid. + // We just return RevocationUndetermined and there is a setting for the user + // to control the handling of that. + grpclogLogger.Warningf("fetchCRL() err = %v", err) + return RevocationUndetermined + } + revocation, err := checkCertRevocation(c, crl) + if err != nil { + grpclogLogger.Warningf("checkCertRevocation(CRL %v) failed: %v", crl.certList.Issuer, err) + // We couldn't check the CRL file for some reason, so we don't know if it's RevocationUnrevoked or not. + return RevocationUndetermined + } + // Here we've gotten a CRL that loads and verifies. + // We only handle all-reasons CRL files, so this file + // is authoritative for the certificate. + return revocation +} + +func checkCertRevocation(c *x509.Certificate, crl *CRL) (revocationStatus, error) { + // Per section 5.3.3 we prime the certificate issuer with the CRL issuer. + // Subsequent entries use the previous entry's issuer. + rawEntryIssuer := crl.rawIssuer + + // Loop through all the revoked certificates. + for _, revCert := range crl.certList.RevokedCertificateEntries { + // 5.3 Loop through CRL entry extensions for needed information. + for _, ext := range revCert.Extensions { + if oidCertificateIssuer.Equal(ext.Id) { + extIssuer, err := parseCertIssuerExt(ext) + if err != nil { + grpclogLogger.Info(err) + if ext.Critical { + return RevocationUndetermined, err + } + // Since this is a non-critical extension, we can skip it even though + // there was a parsing failure. + continue + } + rawEntryIssuer = extIssuer + } else if ext.Critical { + return RevocationUndetermined, fmt.Errorf("checkCertRevocation: Unhandled critical extension: %v", ext.Id) + } + } + + // If the issuer and serial number appear in the CRL, the certificate is revoked. + if bytes.Equal(c.RawIssuer, rawEntryIssuer) && c.SerialNumber.Cmp(revCert.SerialNumber) == 0 { + // CRL contains the serial, so return revoked. + return RevocationRevoked, nil + } + } + // We did not find the serial in the CRL file that was valid for the cert + // so the certificate is not revoked. + return RevocationUnrevoked, nil +} + +func parseCertIssuerExt(ext pkix.Extension) ([]byte, error) { + // 5.3.3 Certificate Issuer + // CertificateIssuer ::= GeneralNames + // GeneralNames ::= SEQUENCE SIZE (1..MAX) OF GeneralName + var generalNames []asn1.RawValue + if rest, err := asn1.Unmarshal(ext.Value, &generalNames); err != nil || len(rest) != 0 { + return nil, fmt.Errorf("asn1.Unmarshal failed: %v", err) + } + + for _, generalName := range generalNames { + // GeneralName ::= CHOICE { + // otherName [0] OtherName, + // rfc822Name [1] IA5String, + // dNSName [2] IA5String, + // x400Address [3] ORAddress, + // directoryName [4] Name, + // ediPartyName [5] EDIPartyName, + // uniformResourceIdentifier [6] IA5String, + // iPAddress [7] OCTET STRING, + // registeredID [8] OBJECT IDENTIFIER } + if generalName.Tag == tagDirectoryName { + return generalName.Bytes, nil + } + } + // Conforming CRL issuers MUST include in this extension the + // distinguished name (DN) from the issuer field of the certificate that + // corresponds to this CRL entry. + // If we couldn't get a directoryName, we can't reason about this file so cert status is + // RevocationUndetermined. + return nil, errors.New("no DN found in certificate issuer") +} + +// RFC 5280, 4.2.1.1 +type authKeyID struct { + ID []byte `asn1:"optional,tag:0"` +} + +// RFC5280, 5.2.5 +// id-ce-issuingDistributionPoint OBJECT IDENTIFIER ::= { id-ce 28 } + +// IssuingDistributionPoint ::= SEQUENCE { +// distributionPoint [0] DistributionPointName OPTIONAL, +// onlyContainsUserCerts [1] BOOLEAN DEFAULT FALSE, +// onlyContainsCACerts [2] BOOLEAN DEFAULT FALSE, +// onlySomeReasons [3] ReasonFlags OPTIONAL, +// indirectCRL [4] BOOLEAN DEFAULT FALSE, +// onlyContainsAttributeCerts [5] BOOLEAN DEFAULT FALSE } + +// -- at most one of onlyContainsUserCerts, onlyContainsCACerts, +// -- and onlyContainsAttributeCerts may be set to TRUE. +type issuingDistributionPoint struct { + DistributionPoint asn1.RawValue `asn1:"optional,tag:0"` + OnlyContainsUserCerts bool `asn1:"optional,tag:1"` + OnlyContainsCACerts bool `asn1:"optional,tag:2"` + OnlySomeReasons asn1.BitString `asn1:"optional,tag:3"` + IndirectCRL bool `asn1:"optional,tag:4"` + OnlyContainsAttributeCerts bool `asn1:"optional,tag:5"` +} + +// parseCRLExtensions parses the extensions for a CRL +// and checks that they're supported by the parser. +func parseCRLExtensions(c *x509.RevocationList) (*CRL, error) { + if c == nil { + return nil, errors.New("c is nil, expected any value") + } + certList := &CRL{certList: c} + + for _, ext := range c.Extensions { + switch { + case oidDeltaCRLIndicator.Equal(ext.Id): + return nil, fmt.Errorf("delta CRLs unsupported") + + case oidAuthorityKeyIdentifier.Equal(ext.Id): + var a authKeyID + if rest, err := asn1.Unmarshal(ext.Value, &a); err != nil { + return nil, fmt.Errorf("asn1.Unmarshal failed: %v", err) + } else if len(rest) != 0 { + return nil, errors.New("trailing data after AKID extension") + } + certList.authorityKeyID = a.ID + + case oidIssuingDistributionPoint.Equal(ext.Id): + var dp issuingDistributionPoint + if rest, err := asn1.Unmarshal(ext.Value, &dp); err != nil { + return nil, fmt.Errorf("asn1.Unmarshal failed: %v", err) + } else if len(rest) != 0 { + return nil, errors.New("trailing data after IssuingDistributionPoint extension") + } + + if dp.OnlyContainsUserCerts || dp.OnlyContainsCACerts || dp.OnlyContainsAttributeCerts { + return nil, errors.New("CRL only contains some certificate types") + } + if dp.IndirectCRL { + return nil, errors.New("indirect CRLs unsupported") + } + if dp.OnlySomeReasons.BitLength != 0 { + return nil, errors.New("onlySomeReasons unsupported") + } + + case ext.Critical: + return nil, fmt.Errorf("unsupported critical extension: %v", ext.Id) + } + } + + if len(certList.authorityKeyID) == 0 { + return nil, errors.New("authority key identifier extension missing") + } + return certList, nil +} + +func verifyCRL(crl *CRL, chain []*x509.Certificate) error { + // RFC5280, 6.3.3 (f) Obtain and validate the certification path for the issuer of the complete CRL + // We intentionally limit our CRLs to be signed with the same certificate path as the certificate + // so we can use the chain from the connection. + + for _, c := range chain { + // Use the key where the subject and KIDs match. + // This departs from RFC4158, 3.5.12 which states that KIDs + // cannot eliminate certificates, but RFC5280, 5.2.1 states that + // "Conforming CRL issuers MUST use the key identifier method, and MUST + // include this extension in all CRLs issued." + // So, this is much simpler than RFC4158 and should be compatible. + if bytes.Equal(c.SubjectKeyId, crl.authorityKeyID) && bytes.Equal(c.RawSubject, crl.rawIssuer) { + // RFC5280, 6.3.3 (f) Key usage and cRLSign bit. + if c.KeyUsage != 0 && c.KeyUsage&x509.KeyUsageCRLSign == 0 { + return fmt.Errorf("verifyCRL: The certificate can't be used for issuing CRLs") + } + // RFC5280, 6.3.3 (g) Validate signature. + return crl.certList.CheckSignatureFrom(c) + } + } + return fmt.Errorf("verifyCRL: No certificates matched CRL issuer (%v)", crl.certList.Issuer) +} + +// pemType is the type of a PEM encoded CRL. +const pemType string = "X509 CRL" + +var crlPemPrefix = []byte("-----BEGIN X509 CRL") + +func crlPemToDer(crlBytes []byte) []byte { + block, _ := pem.Decode(crlBytes) + if block != nil && block.Type == pemType { + crlBytes = block.Bytes + } + return crlBytes +} + +// extractCRLIssuer extracts the raw ASN.1 encoding of the CRL issuer. Due to the design of +// pkix.CertificateList and pkix.RDNSequence, it is not possible to reliably marshal the +// parsed Issuer to its original raw encoding. +func extractCRLIssuer(crlBytes []byte) ([]byte, error) { + if bytes.HasPrefix(crlBytes, crlPemPrefix) { + crlBytes = crlPemToDer(crlBytes) + } + der := cryptobyte.String(crlBytes) + var issuer cryptobyte.String + // This doubled der.ReadASN1 is intentional, it modifies the input buffer + if !der.ReadASN1(&der, cbasn1.SEQUENCE) || + !der.ReadASN1(&der, cbasn1.SEQUENCE) || + !der.SkipOptionalASN1(cbasn1.INTEGER) || + !der.SkipASN1(cbasn1.SEQUENCE) || + !der.ReadASN1Element(&issuer, cbasn1.SEQUENCE) { + return nil, errors.New("extractCRLIssuer: invalid ASN.1 encoding") + } + return issuer, nil +} + +// parseRevocationList comes largely from here +// x509.go: +// https://github.com/golang/go/blob/e2f413402527505144beea443078649380e0c545/src/crypto/x509/x509.go#L1669-L1690 +// We must first convert PEM to DER to be able to use the new +// x509.ParseRevocationList instead of the deprecated x509.ParseCRL +func parseRevocationList(crlBytes []byte) (*x509.RevocationList, error) { + if bytes.HasPrefix(crlBytes, crlPemPrefix) { + crlBytes = crlPemToDer(crlBytes) + } + crl, err := x509.ParseRevocationList(crlBytes) + if err != nil { + return nil, err + } + return crl, nil +} diff --git a/security/advancedtls/crl_provider.go b/security/advancedtls/crl_provider.go new file mode 100644 index 0000000..4a2bf23 --- /dev/null +++ b/security/advancedtls/crl_provider.go @@ -0,0 +1,239 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package advancedtls + +import ( + "crypto/x509" + "fmt" + "os" + "sync" + "time" +) + +const defaultCRLRefreshDuration = 1 * time.Hour +const minCRLRefreshDuration = 1 * time.Minute + +// CRLProvider is the interface to be implemented to enable custom CRL provider +// behavior, as defined in [gRFC A69]. +// +// The interface defines how gRPC gets CRLs from the provider during handshakes, +// but doesn't prescribe a specific way to load and store CRLs. Such +// implementations can be used in RevocationOptions of advancedtls.ClientOptions +// and/or advancedtls.ServerOptions. +// Please note that checking CRLs is directly on the path of connection +// establishment, so implementations of the CRL function need to be fast, and +// slow things such as file IO should be done asynchronously. +// +// [gRFC A69]: https://github.com/grpc/proposal/pull/382 +type CRLProvider interface { + // CRL accepts x509 Cert and returns a related CRL struct, which can contain + // either an empty or non-empty list of revoked certificates. If an error is + // thrown or (nil, nil) is returned, it indicates that we can't load any + // authoritative CRL files (which may not necessarily be a problem). It's not + // considered invalid to have no CRLs if there are no revocations for an + // issuer. In such cases, the status of the check CRL operation is marked as + // RevocationUndetermined, as defined in [RFC5280 - Undetermined]. + // + // [RFC5280 - Undetermined]: https://datatracker.ietf.org/doc/html/rfc5280#section-6.3.3 + CRL(cert *x509.Certificate) (*CRL, error) +} + +// StaticCRLProvider implements CRLProvider interface by accepting raw content +// of CRL files at creation time and storing parsed CRL structs in-memory. +type StaticCRLProvider struct { + crls map[string]*CRL +} + +// NewStaticCRLProvider processes raw content of CRL files, adds parsed CRL +// structs into in-memory, and returns a new instance of the StaticCRLProvider. +func NewStaticCRLProvider(rawCRLs [][]byte) *StaticCRLProvider { + p := StaticCRLProvider{} + p.crls = make(map[string]*CRL) + for idx, rawCRL := range rawCRLs { + cRL, err := NewCRL(rawCRL) + if err != nil { + grpclogLogger.Warningf("Can't parse raw CRL number %v from the slice: %v", idx, err) + continue + } + p.addCRL(cRL) + } + return &p +} + +// AddCRL adds/updates provided CRL to in-memory storage. +func (p *StaticCRLProvider) addCRL(crl *CRL) { + key := crl.certList.Issuer.ToRDNSequence().String() + p.crls[key] = crl +} + +// CRL returns CRL struct if it was passed to NewStaticCRLProvider. +func (p *StaticCRLProvider) CRL(cert *x509.Certificate) (*CRL, error) { + return p.crls[cert.Issuer.ToRDNSequence().String()], nil +} + +// FileWatcherOptions represents a data structure holding a configuration for +// FileWatcherCRLProvider. +type FileWatcherOptions struct { + CRLDirectory string // Required: Path of the directory containing CRL files + RefreshDuration time.Duration // Optional: Time interval (default 1 hour) between CRLDirectory scans, can't be smaller than 1 minute + CRLReloadingFailedCallback func(err error) // Optional: Custom callback executed when a CRL file can’t be processed +} + +// FileWatcherCRLProvider implements the CRLProvider interface by periodically +// scanning CRLDirectory (see FileWatcherOptions) and storing CRL structs +// in-memory. Users should call Close to stop the background refresh of +// CRLDirectory. +type FileWatcherCRLProvider struct { + crls map[string]*CRL + opts FileWatcherOptions + mu sync.Mutex + stop chan struct{} + done chan struct{} +} + +// NewFileWatcherCRLProvider returns a new instance of the +// FileWatcherCRLProvider. It uses FileWatcherOptions to validate and apply +// configuration required for creating a new instance. The initial scan of +// CRLDirectory is performed inside this function. Users should call Close to +// stop the background refresh of CRLDirectory. +func NewFileWatcherCRLProvider(o FileWatcherOptions) (*FileWatcherCRLProvider, error) { + if err := o.validate(); err != nil { + return nil, err + } + provider := &FileWatcherCRLProvider{ + crls: make(map[string]*CRL), + opts: o, + stop: make(chan struct{}), + done: make(chan struct{}), + } + provider.scanCRLDirectory() + go provider.run() + return provider, nil +} + +func (o *FileWatcherOptions) validate() error { + // Checks relates to CRLDirectory. + if o.CRLDirectory == "" { + return fmt.Errorf("advancedtls: CRLDirectory needs to be specified") + } + if _, err := os.ReadDir(o.CRLDirectory); err != nil { + return fmt.Errorf("advancedtls: CRLDirectory %v is not readable: %v", o.CRLDirectory, err) + } + // Checks related to RefreshDuration. + if o.RefreshDuration == 0 { + o.RefreshDuration = defaultCRLRefreshDuration + } + if o.RefreshDuration < minCRLRefreshDuration { + grpclogLogger.Warningf("RefreshDuration must be at least 1 minute: provided value %v, minimum value %v will be used.", o.RefreshDuration, minCRLRefreshDuration) + o.RefreshDuration = minCRLRefreshDuration + } + return nil +} + +// Start starts watching the directory for CRL files and updates the provider accordingly. +func (p *FileWatcherCRLProvider) run() { + defer close(p.done) + ticker := time.NewTicker(p.opts.RefreshDuration) + defer ticker.Stop() + + for { + select { + case <-p.stop: + grpclogLogger.Infof("Scanning of CRLDirectory %v stopped", p.opts.CRLDirectory) + return + case <-ticker.C: + p.scanCRLDirectory() + } + } +} + +// Close waits till the background refresh of CRLDirectory of +// FileWatcherCRLProvider is done and then stops it. +func (p *FileWatcherCRLProvider) Close() { + close(p.stop) + <-p.done +} + +// scanCRLDirectory starts the process of scanning +// FileWatcherOptions.CRLDirectory and updating in-memory storage of CRL +// structs, as defined in [gRFC A69]. It's called periodically +// (see FileWatcherOptions.RefreshDuration) by run goroutine. +// +// [gRFC A69]: https://github.com/grpc/proposal/pull/382 +func (p *FileWatcherCRLProvider) scanCRLDirectory() { + dir, err := os.Open(p.opts.CRLDirectory) + if err != nil { + grpclogLogger.Errorf("Can't open CRLDirectory %v", p.opts.CRLDirectory, err) + if p.opts.CRLReloadingFailedCallback != nil { + p.opts.CRLReloadingFailedCallback(err) + } + } + defer dir.Close() + + files, err := dir.ReadDir(0) + if err != nil { + grpclogLogger.Errorf("Can't access files under CRLDirectory %v", p.opts.CRLDirectory, err) + if p.opts.CRLReloadingFailedCallback != nil { + p.opts.CRLReloadingFailedCallback(err) + } + } + + tempCRLs := make(map[string]*CRL) + successCounter := 0 + failCounter := 0 + for _, file := range files { + filePath := fmt.Sprintf("%s/%s", p.opts.CRLDirectory, file.Name()) + crl, err := ReadCRLFile(filePath) + if err != nil { + failCounter++ + grpclogLogger.Warningf("Can't add CRL from file %v under CRLDirectory %v", filePath, p.opts.CRLDirectory, err) + if p.opts.CRLReloadingFailedCallback != nil { + p.opts.CRLReloadingFailedCallback(err) + } + continue + } + tempCRLs[crl.certList.Issuer.ToRDNSequence().String()] = crl + successCounter++ + } + // Only if all the files are processed successfully we can swap maps (there + // might be deletions of entries in this case). + if len(files) == successCounter { + p.mu.Lock() + defer p.mu.Unlock() + p.crls = tempCRLs + grpclogLogger.Infof("Scan of CRLDirectory %v completed, %v files found and processed successfully, in-memory CRL storage flushed and repopulated", p.opts.CRLDirectory, len(files)) + } else { + // Since some of the files failed we can only add/update entries in the map. + p.mu.Lock() + defer p.mu.Unlock() + for key, value := range tempCRLs { + p.crls[key] = value + } + grpclogLogger.Infof("Scan of CRLDirectory %v completed, %v files found, %v files processing failed, %v entries of in-memory CRL storage added/updated", p.opts.CRLDirectory, len(files), failCounter, successCounter) + } +} + +// CRL retrieves the CRL associated with the given certificate's issuer DN from +// in-memory if it was loaded during FileWatcherOptions.CRLDirectory scan before +// the execution of this function. +func (p *FileWatcherCRLProvider) CRL(cert *x509.Certificate) (*CRL, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.crls[cert.Issuer.ToRDNSequence().String()], nil +} diff --git a/security/advancedtls/crl_provider_test.go b/security/advancedtls/crl_provider_test.go new file mode 100644 index 0000000..04fec3d --- /dev/null +++ b/security/advancedtls/crl_provider_test.go @@ -0,0 +1,356 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package advancedtls + +import ( + "crypto/x509" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/security/advancedtls/testdata" +) + +// TestStaticCRLProvider tests how StaticCRLProvider handles the major four +// cases for CRL checks. It loads the CRLs under crl directory, constructs +// unrevoked, revoked leaf, and revoked intermediate chains, as well as a chain +// without CRL for issuer, and checks that it’s correctly processed. +func (s) TestStaticCRLProvider(t *testing.T) { + rawCRLs := make([][]byte, 6) + for i := 1; i <= 6; i++ { + rawCRL, err := os.ReadFile(testdata.Path(fmt.Sprintf("crl/%d.crl", i))) + if err != nil { + t.Fatalf("readFile(%v) failed err = %v", fmt.Sprintf("crl/%d.crl", i), err) + } + rawCRLs = append(rawCRLs, rawCRL) + } + p := NewStaticCRLProvider(rawCRLs) + + // Each test data entry contains a description of a certificate chain, + // certificate chain itself, and if CRL is not expected to be found. + tests := []struct { + desc string + certs []*x509.Certificate + expectNoCRL bool + }{ + { + desc: "Unrevoked chain", + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + }, + { + desc: "Revoked Intermediate chain", + certs: makeChain(t, testdata.Path("crl/revokedInt.pem")), + }, + { + desc: "Revoked leaf chain", + certs: makeChain(t, testdata.Path("crl/revokedLeaf.pem")), + }, + { + desc: "Chain with no CRL for issuer", + certs: makeChain(t, testdata.Path("client_cert_1.pem")), + expectNoCRL: true, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + for _, c := range tt.certs { + crl, err := p.CRL(c) + if err != nil { + t.Fatalf("Expected error fetch from provider: %v", err) + } + if crl == nil && !tt.expectNoCRL { + t.Fatalf("CRL is unexpectedly nil") + } + } + }) + } +} + +// TestFileWatcherCRLProviderConfig checks creation of FileWatcherCRLProvider, +// and the validation of FileWatcherOptions configuration. The configurations include empty +// one, non existing CRLDirectory, invalid RefreshDuration, and the correct one. +func (s) TestFileWatcherCRLProviderConfig(t *testing.T) { + if _, err := NewFileWatcherCRLProvider(FileWatcherOptions{}); err == nil { + t.Fatalf("Empty FileWatcherOptions should not be allowed") + } + if _, err := NewFileWatcherCRLProvider(FileWatcherOptions{CRLDirectory: "I_do_not_exist"}); err == nil { + t.Fatalf("CRLDirectory must exist") + } + defaultProvider, err := NewFileWatcherCRLProvider(FileWatcherOptions{CRLDirectory: testdata.Path("crl")}) + if err != nil { + t.Fatal("Unexpected error:", err) + } + if defaultProvider.opts.RefreshDuration != defaultCRLRefreshDuration { + t.Fatalf("RefreshDuration for defaultCRLRefreshDuration case is not properly updated by validate() func") + } + defaultProvider.Close() + tooFastRefreshProvider, err := NewFileWatcherCRLProvider(FileWatcherOptions{ + CRLDirectory: testdata.Path("crl"), + RefreshDuration: 5 * time.Second, + }) + if err != nil { + t.Fatal("Unexpected error:", err) + } + if tooFastRefreshProvider.opts.RefreshDuration != minCRLRefreshDuration { + t.Fatalf("RefreshDuration for minCRLRefreshDuration case is not properly updated by validate() func") + } + tooFastRefreshProvider.Close() + + customCallback := func(err error) { + t.Logf("Custom error message: %v", err) + } + regularProvider, err := NewFileWatcherCRLProvider(FileWatcherOptions{ + CRLDirectory: testdata.Path("crl"), + RefreshDuration: 2 * time.Hour, + CRLReloadingFailedCallback: customCallback, + }) + if err != nil { + t.Fatal("Unexpected error while creating regular FileWatcherCRLProvider:", err) + } + if regularProvider.opts.RefreshDuration != 2*time.Hour { + t.Fatalf("Valid refreshDuration was incorrectly updated by validate() func") + } + regularProvider.Close() +} + +// TestFileWatcherCRLProvider tests how FileWatcherCRLProvider handles the major +// four cases for CRL checks. It scans the CRLs under crl directory to populate +// the in-memory storage. Then we construct unrevoked, revoked leaf, and revoked +// intermediate chains, as well as a chain without CRL for issuer, and check +// that it’s correctly processed. Additionally, we also check if number of +// invocations of custom callback is correct. +func (s) TestFileWatcherCRLProvider(t *testing.T) { + const nonCRLFilesUnderCRLDirectory = 18 + nonCRLFilesSet := make(map[string]struct{}) + customCallback := func(err error) { + if strings.Contains(err.Error(), "BUILD") { + return + } + nonCRLFilesSet[err.Error()] = struct{}{} + } + p, err := NewFileWatcherCRLProvider(FileWatcherOptions{ + CRLDirectory: testdata.Path("crl"), + RefreshDuration: 1 * time.Hour, + CRLReloadingFailedCallback: customCallback, + }) + if err != nil { + t.Fatal("Unexpected error while creating FileWatcherCRLProvider:", err) + } + + // Each test data entry contains a description of a certificate chain, + // certificate chain itself, and if CRL is not expected to be found. + tests := []struct { + desc string + certs []*x509.Certificate + expectNoCRL bool + }{ + { + desc: "Unrevoked chain", + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + }, + { + desc: "Revoked Intermediate chain", + certs: makeChain(t, testdata.Path("crl/revokedInt.pem")), + }, + { + desc: "Revoked leaf chain", + certs: makeChain(t, testdata.Path("crl/revokedLeaf.pem")), + }, + { + desc: "Chain with no CRL for issuer", + certs: makeChain(t, testdata.Path("client_cert_1.pem")), + expectNoCRL: true, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + for _, c := range tt.certs { + crl, err := p.CRL(c) + if err != nil { + t.Fatalf("Expected error fetch from provider: %v", err) + } + if crl == nil && !tt.expectNoCRL { + t.Fatalf("CRL is unexpectedly nil") + } + } + }) + } + p.Close() + if diff := cmp.Diff(len(nonCRLFilesSet), nonCRLFilesUnderCRLDirectory); diff != "" { + t.Errorf("Unexpected number Number of callback executions\ndiff (-got +want):\n%s", diff) + } +} + +// TestFileWatcherCRLProviderDirectoryScan tests how FileWatcherCRLProvider +// handles different contents of FileWatcherOptions.CRLDirectory. +// We update the content with various (correct and incorrect) CRL files and +// check if in-memory storage was properly updated. Please note that the same +// instance of FileWatcherCRLProvider is used for the whole test so test cases +// are not independent from each other. +func (s) TestFileWatcherCRLProviderDirectoryScan(t *testing.T) { + sourcePath := testdata.Path("crl") + targetPath := createTmpDir(t) + defer os.RemoveAll(targetPath) + p, err := NewFileWatcherCRLProvider(FileWatcherOptions{ + CRLDirectory: targetPath, + RefreshDuration: 1 * time.Hour, + }) + if err != nil { + t.Fatal("Unexpected error while creating FileWatcherCRLProvider:", err) + } + + // Each test data entry contains a description of CRL directory content + // (including the expected number of entries in the FileWatcherCRLProvider + // map), the name of the files to be copied there before executing the test + // case, and information regarding whether a specific certificate is expected + // to be found in the map. + tests := []struct { + desc string + crlFileNames []string + certFileNames []struct { + fileName string + expected bool + } + }{ + { + desc: "Simple addition (1 map entry)", + crlFileNames: []string{"1.crl"}, + certFileNames: []struct { + fileName string + expected bool + }{ + {"crl/unrevoked.pem", true}, + }, + }, + { + desc: "Addition and deletion (2 map entries)", + crlFileNames: []string{"3.crl", "5.crl"}, + certFileNames: []struct { + fileName string + expected bool + }{ + {"crl/revokedInt.pem", true}, + {"crl/revokedLeaf.pem", true}, + {"crl/unrevoked.pem", false}, + }, + }, + { + desc: "Addition and a corrupt file (3 map entries)", + crlFileNames: []string{"1.crl", "README.md"}, + certFileNames: []struct { + fileName string + expected bool + }{ + {"crl/revokedInt.pem", true}, + {"crl/revokedLeaf.pem", true}, + {"crl/unrevoked.pem", true}, + }}, + { + desc: "Full deletion (0 map entries)", + crlFileNames: []string{}, + certFileNames: []struct { + fileName string + expected bool + }{ + {"crl/revokedInt.pem", false}, + {"crl/revokedLeaf.pem", false}, + {"crl/unrevoked.pem", false}, + }}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + copyFiles(sourcePath, targetPath, tt.crlFileNames, t) + p.scanCRLDirectory() + for _, certFileName := range tt.certFileNames { + c := makeChain(t, testdata.Path(certFileName.fileName))[0] + crl, err := p.CRL(c) + if err != nil { + t.Errorf("Cannot fetch CRL from provider: %v", err) + } + if crl == nil && certFileName.expected { + t.Errorf("CRL is unexpectedly nil") + } + if crl != nil && !certFileName.expected { + t.Errorf("CRL is unexpectedly not nil") + } + } + }) + } + p.Close() +} + +func copyFiles(sourcePath string, targetPath string, fileNames []string, t *testing.T) { + t.Helper() + targetDir, err := os.Open(targetPath) + if err != nil { + t.Fatalf("Can't open dir %v: %v", targetPath, err) + } + defer targetDir.Close() + names, err := targetDir.Readdirnames(-1) + if err != nil { + t.Fatalf("Can't read dir %v: %v", targetPath, err) + } + for _, name := range names { + err = os.RemoveAll(filepath.Join(targetPath, name)) + if err != nil { + t.Fatalf("Can't remove file %v: %v", name, err) + } + } + for _, fileName := range fileNames { + destinationPath := filepath.Join(targetPath, fileName) + + sourceFile, err := os.Open(filepath.Join(sourcePath, fileName)) + if err != nil { + t.Fatalf("Can't open file %v: %v", fileName, err) + } + defer sourceFile.Close() + + destinationFile, err := os.Create(destinationPath) + if err != nil { + t.Fatalf("Can't create file %v: %v", destinationFile, err) + } + defer destinationFile.Close() + + _, err = io.Copy(destinationFile, sourceFile) + if err != nil { + t.Fatalf("Can't copy file %v to %v: %v", sourceFile, destinationFile, err) + } + } +} + +func createTmpDir(t *testing.T) string { + t.Helper() + + // Create a temp directory. Passing an empty string for the first argument + // uses the system temp directory. + dir, err := os.MkdirTemp("", "filewatcher*") + if err != nil { + t.Fatalf("os.MkdirTemp() failed: %v", err) + } + t.Logf("Using tmpdir: %s", dir) + return dir +} diff --git a/security/advancedtls/crl_test.go b/security/advancedtls/crl_test.go new file mode 100644 index 0000000..450ab56 --- /dev/null +++ b/security/advancedtls/crl_test.go @@ -0,0 +1,588 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package advancedtls + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/pem" + "fmt" + "math/big" + "net" + "net/netip" + "os" + "path" + "strings" + "testing" + "time" + + "google.golang.org/grpc/security/advancedtls/testdata" +) + +func TestUnsupportedCRLs(t *testing.T) { + crlBytesSomeReasons := []byte(`-----BEGIN X509 CRL----- +MIIEeDCCA2ACAQEwDQYJKoZIhvcNAQELBQAwQjELMAkGA1UEBhMCVVMxHjAcBgNV +BAoTFUdvb2dsZSBUcnVzdCBTZXJ2aWNlczETMBEGA1UEAxMKR1RTIENBIDFPMRcN +MjEwNDI2MTI1OTQxWhcNMjEwNTA2MTE1OTQwWjCCAn0wIgIRAPOOG3L4VLC7CAAA +AABxQgEXDTIxMDQxOTEyMTgxOFowIQIQUK0UwBZkVdQIAAAAAHFCBRcNMjEwNDE5 +MTIxODE4WjAhAhBRIXBJaKoQkQgAAAAAcULHFw0yMTA0MjAxMjE4MTdaMCICEQCv +qQWUq5UxmQgAAAAAcULMFw0yMTA0MjAxMjE4MTdaMCICEQDdv5k1kKwKTQgAAAAA +cUOQFw0yMTA0MjExMjE4MTZaMCICEQDGIEfR8N9sEAgAAAAAcUOWFw0yMTA0MjEx +MjE4MThaMCECEBHgbLXlj5yUCAAAAABxQ/IXDTIxMDQyMTIzMDAyNlowIQIQE1wT +2GGYqKwIAAAAAHFD7xcNMjEwNDIxMjMwMDI5WjAiAhEAo/bSyDjpVtsIAAAAAHFE +txcNMjEwNDIyMjMwMDI3WjAhAhARdCrSrHE0dAgAAAAAcUS/Fw0yMTA0MjIyMzAw +MjhaMCECEHONohfWn3wwCAAAAABxRX8XDTIxMDQyMzIzMDAyOVowIgIRAOYkiUPA +os4vCAAAAABxRYgXDTIxMDQyMzIzMDAyOFowIQIQRNTow5Eg2gEIAAAAAHFGShcN +MjEwNDI0MjMwMDI2WjAhAhBX32dH4/WQ6AgAAAAAcUZNFw0yMTA0MjQyMzAwMjZa +MCICEQDHnUM1vsaP/wgAAAAAcUcQFw0yMTA0MjUyMzAwMjZaMCECEEm5rvmL8sj6 +CAAAAABxRxQXDTIxMDQyNTIzMDAyN1owIQIQW16OQs4YQYkIAAAAAHFIABcNMjEw +NDI2MTI1NDA4WjAhAhAhSohpYsJtDQgAAAAAcUgEFw0yMTA0MjYxMjU0MDlaoGkw +ZzAfBgNVHSMEGDAWgBSY0fhuEOvPm+xgnxiQG6DrfQn9KzALBgNVHRQEBAICBngw +NwYDVR0cAQH/BC0wK6AmoCSGImh0dHA6Ly9jcmwucGtpLmdvb2cvR1RTMU8xY29y +ZS5jcmyBAf8wDQYJKoZIhvcNAQELBQADggEBADPBXbxVxMJ1HC7btXExRUpJHUlU +YbeCZGx6zj5F8pkopbmpV7cpewwhm848Fx4VaFFppZQZd92O08daEC6aEqoug4qF +z6ZrOLzhuKfpW8E93JjgL91v0FYN7iOcT7+ERKCwVEwEkuxszxs7ggW6OJYJNvHh +priIdmcPoiQ3ZrIRH0vE3BfUcNXnKFGATWuDkiRI0I4A5P7NiOf+lAuGZet3/eom +0chgts6sdau10GfeUpHUd4f8e93cS/QeLeG16z7LC8vRLstU3m3vrknpZbdGqSia +97w66mqcnQh9V0swZiEnVLmLufaiuDZJ+6nUzSvLqBlb/ei3T/tKV0BoKJA= +-----END X509 CRL-----`) + + crlBytesIndirect := []byte(`-----BEGIN X509 CRL----- +MIIDGjCCAgICAQEwDQYJKoZIhvcNAQELBQAwdjELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCkNhbGlmb3JuaWExFDASBgNVBAoTC1Rlc3RpbmcgTHRkMSowKAYDVQQLEyFU +ZXN0aW5nIEx0ZCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxEDAOBgNVBAMTB1Rlc3Qg +Q0EXDTIxMDExNjAyMjAxNloXDTIxMDEyMDA2MjAxNlowgfIwbAIBAhcNMjEwMTE2 +MDIyMDE2WjBYMAoGA1UdFQQDCgEEMEoGA1UdHQEB/wRAMD6kPDA6MQwwCgYDVQQG +EwNVU0ExDTALBgNVBAcTBGhlcmUxCzAJBgNVBAoTAnVzMQ4wDAYDVQQDEwVUZXN0 +MTAgAgEDFw0yMTAxMTYwMjIwMTZaMAwwCgYDVR0VBAMKAQEwYAIBBBcNMjEwMTE2 +MDIyMDE2WjBMMEoGA1UdHQEB/wRAMD6kPDA6MQwwCgYDVQQGEwNVU0ExDTALBgNV +BAcTBGhlcmUxCzAJBgNVBAoTAnVzMQ4wDAYDVQQDEwVUZXN0MqBjMGEwHwYDVR0j +BBgwFoAURJSDWAOfhGCryBjl8dsQjBitl3swCgYDVR0UBAMCAQEwMgYDVR0cAQH/ +BCgwJqAhoB+GHWh0dHA6Ly9jcmxzLnBraS5nb29nL3Rlc3QuY3JshAH/MA0GCSqG +SIb3DQEBCwUAA4IBAQBVXX67mr2wFPmEWCe6mf/wFnPl3xL6zNOl96YJtsd7ulcS +TEbdJpaUnWFQ23+Tpzdj/lI2aQhTg5Lvii3o+D8C5r/Jc5NhSOtVJJDI/IQLh4pG +NgGdljdbJQIT5D2Z71dgbq1ocxn8DefZIJjO3jp8VnAm7AIMX2tLTySzD2MpMeMq +XmcN4lG1e4nx+xjzp7MySYO42NRY3LkphVzJhu3dRBYhBKViRJxw9hLttChitJpF +6Kh6a0QzrEY/QDJGhE1VrAD2c5g/SKnHPDVoCWo4ACIICi76KQQSIWfIdp4W/SY3 +qsSIp8gfxSyzkJP+Ngkm2DdLjlJQCZ9R0MZP9Xj4 +-----END X509 CRL-----`) + + var tests = []struct { + desc string + in []byte + }{ + { + desc: "some reasons", + in: crlBytesSomeReasons, + }, + { + desc: "indirect", + in: crlBytesIndirect, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + crl, err := parseRevocationList(tt.in) + if err != nil { + t.Fatal(err) + } + if _, err := parseCRLExtensions(crl); err == nil { + t.Error("expected error got ok") + } + }) + } +} + +func TestCheckCertRevocation(t *testing.T) { + dummyCrlFile := []byte(`-----BEGIN X509 CRL----- +MIIDGjCCAgICAQEwDQYJKoZIhvcNAQELBQAwdjELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCkNhbGlmb3JuaWExFDASBgNVBAoTC1Rlc3RpbmcgTHRkMSowKAYDVQQLEyFU +ZXN0aW5nIEx0ZCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxEDAOBgNVBAMTB1Rlc3Qg +Q0EXDTIxMDExNjAyMjAxNloXDTIxMDEyMDA2MjAxNlowgfIwbAIBAhcNMjEwMTE2 +MDIyMDE2WjBYMAoGA1UdFQQDCgEEMEoGA1UdHQEB/wRAMD6kPDA6MQwwCgYDVQQG +EwNVU0ExDTALBgNVBAcTBGhlcmUxCzAJBgNVBAoTAnVzMQ4wDAYDVQQDEwVUZXN0 +MTAgAgEDFw0yMTAxMTYwMjIwMTZaMAwwCgYDVR0VBAMKAQEwYAIBBBcNMjEwMTE2 +MDIyMDE2WjBMMEoGA1UdHQEB/wRAMD6kPDA6MQwwCgYDVQQGEwNVU0ExDTALBgNV +BAcTBGhlcmUxCzAJBgNVBAoTAnVzMQ4wDAYDVQQDEwVUZXN0MqBjMGEwHwYDVR0j +BBgwFoAURJSDWAOfhGCryBjl8dsQjBitl3swCgYDVR0UBAMCAQEwMgYDVR0cAQH/ +BCgwJqAhoB+GHWh0dHA6Ly9jcmxzLnBraS5nb29nL3Rlc3QuY3JshAH/MA0GCSqG +SIb3DQEBCwUAA4IBAQBVXX67mr2wFPmEWCe6mf/wFnPl3xL6zNOl96YJtsd7ulcS +TEbdJpaUnWFQ23+Tpzdj/lI2aQhTg5Lvii3o+D8C5r/Jc5NhSOtVJJDI/IQLh4pG +NgGdljdbJQIT5D2Z71dgbq1ocxn8DefZIJjO3jp8VnAm7AIMX2tLTySzD2MpMeMq +XmcN4lG1e4nx+xjzp7MySYO42NRY3LkphVzJhu3dRBYhBKViRJxw9hLttChitJpF +6Kh6a0QzrEY/QDJGhE1VrAD2c5g/SKnHPDVoCWo4ACIICi76KQQSIWfIdp4W/SY3 +qsSIp8gfxSyzkJP+Ngkm2DdLjlJQCZ9R0MZP9Xj4 +-----END X509 CRL-----`) + crl, err := parseRevocationList(dummyCrlFile) + if err != nil { + t.Fatalf("parseRevocationList(dummyCrlFile) failed: %v", err) + } + crlExt := &CRL{certList: crl} + + var revocationTests = []struct { + desc string + in x509.Certificate + revoked revocationStatus + }{ + { + desc: "Single revoked", + in: x509.Certificate{ + Issuer: pkix.Name{ + Country: []string{"USA"}, + Locality: []string{"here"}, + Organization: []string{"us"}, + CommonName: "Test1", + }, + SerialNumber: big.NewInt(2), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationRevoked, + }, + { + desc: "Revoked no entry issuer", + in: x509.Certificate{ + Issuer: pkix.Name{ + Country: []string{"USA"}, + Locality: []string{"here"}, + Organization: []string{"us"}, + CommonName: "Test1", + }, + SerialNumber: big.NewInt(3), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationRevoked, + }, + { + desc: "Revoked new entry issuer", + in: x509.Certificate{ + Issuer: pkix.Name{ + Country: []string{"USA"}, + Locality: []string{"here"}, + Organization: []string{"us"}, + CommonName: "Test2", + }, + SerialNumber: big.NewInt(4), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationRevoked, + }, + { + desc: "Single unrevoked", + in: x509.Certificate{ + Issuer: pkix.Name{ + Country: []string{"USA"}, + Locality: []string{"here"}, + Organization: []string{"us"}, + CommonName: "Test2", + }, + SerialNumber: big.NewInt(1), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationUnrevoked, + }, + { + desc: "Single unrevoked Issuer", + in: x509.Certificate{ + Issuer: crl.Issuer, + SerialNumber: big.NewInt(2), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationUnrevoked, + }, + } + + for _, tt := range revocationTests { + rawIssuer, err := asn1.Marshal(tt.in.Issuer.ToRDNSequence()) + if err != nil { + t.Fatalf("asn1.Marshal(%v) failed: %v", tt.in.Issuer.ToRDNSequence(), err) + } + tt.in.RawIssuer = rawIssuer + t.Run(tt.desc, func(t *testing.T) { + rev, err := checkCertRevocation(&tt.in, crlExt) + if err != nil { + t.Errorf("checkCertRevocation(%v) err = %v", tt.in.Issuer, err) + } else if rev != tt.revoked { + t.Errorf("checkCertRevocation(%v(%v)) returned %v wanted %v", + tt.in.Issuer, tt.in.SerialNumber, rev, tt.revoked) + } + }) + } +} + +func makeChain(t *testing.T, name string) []*x509.Certificate { + t.Helper() + + certChain := make([]*x509.Certificate, 0) + + rest, err := os.ReadFile(name) + if err != nil { + t.Fatalf("os.ReadFile(%v) failed %v", name, err) + } + for len(rest) > 0 { + var block *pem.Block + block, rest = pem.Decode(rest) + c, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("ParseCertificate error %v", err) + } + t.Logf("Parsed Cert sub = %v iss = %v", c.Subject, c.Issuer) + certChain = append(certChain, c) + } + return certChain +} + +func loadCRL(t *testing.T, path string) *CRL { + crl, err := ReadCRLFile(path) + if err != nil { + t.Fatalf("ReadCRLFile(%v) failed err = %v", path, err) + } + return crl +} + +func checkRevocation(conn tls.ConnectionState, cfg RevocationOptions) error { + return checkChainRevocation(conn.VerifiedChains, cfg) +} + +func TestVerifyCrl(t *testing.T) { + tamperedSignature := loadCRL(t, testdata.Path("crl/1.crl")) + // Change the signature so it won't verify + tamperedSignature.certList.Signature[0]++ + tamperedContent := loadCRL(t, testdata.Path("crl/provider_crl_empty.pem")) + // Change the content so it won't find a match + tamperedContent.rawIssuer[0]++ + + verifyTests := []struct { + desc string + crl *CRL + certs []*x509.Certificate + cert *x509.Certificate + errWant string + }{ + { + desc: "Pass intermediate", + crl: loadCRL(t, testdata.Path("crl/1.crl")), + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + cert: makeChain(t, testdata.Path("crl/unrevoked.pem"))[1], + errWant: "", + }, + { + desc: "Pass leaf", + crl: loadCRL(t, testdata.Path("crl/2.crl")), + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + cert: makeChain(t, testdata.Path("crl/unrevoked.pem"))[2], + errWant: "", + }, + { + desc: "Fail wrong cert chain", + crl: loadCRL(t, testdata.Path("crl/3.crl")), + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + cert: makeChain(t, testdata.Path("crl/revokedInt.pem"))[1], + errWant: "No certificates matched", + }, + { + desc: "Fail no certs", + crl: loadCRL(t, testdata.Path("crl/1.crl")), + certs: []*x509.Certificate{}, + cert: makeChain(t, testdata.Path("crl/unrevoked.pem"))[1], + errWant: "No certificates matched", + }, + { + desc: "Fail Tampered signature", + crl: tamperedSignature, + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + cert: makeChain(t, testdata.Path("crl/unrevoked.pem"))[1], + errWant: "verification failure", + }, + { + desc: "Fail Tampered content", + crl: tamperedContent, + certs: makeChain(t, testdata.Path("crl/provider_client_trust_cert.pem")), + cert: makeChain(t, testdata.Path("crl/provider_client_trust_cert.pem"))[0], + errWant: "No certificates", + }, + { + desc: "Fail CRL by malicious CA", + crl: loadCRL(t, testdata.Path("crl/provider_malicious_crl_empty.pem")), + certs: makeChain(t, testdata.Path("crl/provider_client_trust_cert.pem")), + cert: makeChain(t, testdata.Path("crl/provider_client_trust_cert.pem"))[0], + errWant: "verification error", + }, + { + desc: "Fail KeyUsage without cRLSign bit", + crl: loadCRL(t, testdata.Path("crl/provider_malicious_crl_empty.pem")), + certs: makeChain(t, testdata.Path("crl/provider_malicious_client_trust_cert.pem")), + cert: makeChain(t, testdata.Path("crl/provider_malicious_client_trust_cert.pem"))[0], + errWant: "certificate can't be used", + }, + } + + for _, tt := range verifyTests { + t.Run(tt.desc, func(t *testing.T) { + err := verifyCRL(tt.crl, tt.certs) + switch { + case tt.errWant == "" && err != nil: + t.Errorf("Valid CRL did not verify err = %v", err) + case tt.errWant != "" && err == nil: + t.Error("Invalid CRL verified") + case tt.errWant != "" && !strings.Contains(err.Error(), tt.errWant): + t.Errorf("fetchIssuerCRL(_, %v, %v, _) = %v; want Contains(%v)", tt.cert.RawIssuer, tt.certs, err, tt.errWant) + } + }) + } +} + +func TestRevokedCert(t *testing.T) { + revokedIntChain := makeChain(t, testdata.Path("crl/revokedInt.pem")) + revokedLeafChain := makeChain(t, testdata.Path("crl/revokedLeaf.pem")) + validChain := makeChain(t, testdata.Path("crl/unrevoked.pem")) + rawCRLs := make([][]byte, 6) + for i := 1; i <= 6; i++ { + rawCRL, err := os.ReadFile(testdata.Path(fmt.Sprintf("crl/%d.crl", i))) + if err != nil { + t.Fatalf("readFile(%v) failed err = %v", fmt.Sprintf("crl/%d.crl", i), err) + } + rawCRLs = append(rawCRLs, rawCRL) + } + staticCRLProvider := NewStaticCRLProvider(rawCRLs) + directoryCRLProvider, err := NewFileWatcherCRLProvider(FileWatcherOptions{CRLDirectory: testdata.Path("crl")}) + if err != nil { + t.Fatalf("NewFileWatcherCRLProvider: err = %v", err) + } + defer directoryCRLProvider.Close() + + var revocationTests = []struct { + desc string + in tls.ConnectionState + revoked bool + denyUndetermined bool + }{ + { + desc: "Single unrevoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{validChain}}, + revoked: false, + }, + { + desc: "Single revoked intermediate", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{revokedIntChain}}, + revoked: true, + }, + { + desc: "Single revoked leaf", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{revokedLeafChain}}, + revoked: true, + }, + { + desc: "Multi one revoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{validChain, revokedLeafChain}}, + revoked: false, + }, + { + desc: "Multi revoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{revokedLeafChain, revokedIntChain}}, + revoked: true, + }, + { + desc: "Multi unrevoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{validChain, validChain}}, + revoked: false, + }, + { + desc: "Undetermined revoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{ + {&x509.Certificate{CRLDistributionPoints: []string{"test"}}}, + }}, + revoked: true, + denyUndetermined: true, + }, + { + desc: "Undetermined allowed", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{ + {&x509.Certificate{CRLDistributionPoints: []string{"test"}}}, + }}, + revoked: false, + }, + } + + for _, tt := range revocationTests { + t.Run(fmt.Sprintf("%v with x509 crl dir", tt.desc), func(t *testing.T) { + err := checkRevocation(tt.in, RevocationOptions{ + CRLProvider: directoryCRLProvider, + DenyUndetermined: tt.denyUndetermined, + }) + t.Logf("checkRevocation err = %v", err) + if tt.revoked && err == nil { + t.Error("Revoked certificate chain was allowed") + } else if !tt.revoked && err != nil { + t.Error("Unrevoked certificate not allowed") + } + }) + t.Run(fmt.Sprintf("%v with static provider", tt.desc), func(t *testing.T) { + err := checkRevocation(tt.in, RevocationOptions{ + DenyUndetermined: tt.denyUndetermined, + CRLProvider: staticCRLProvider, + }) + t.Logf("checkRevocation err = %v", err) + if tt.revoked && err == nil { + t.Error("Revoked certificate chain was allowed") + } else if !tt.revoked && err != nil { + t.Error("Unrevoked certificate not allowed") + } + }) + } +} + +func setupTLSConn(t *testing.T) (net.Listener, *x509.Certificate, *ecdsa.PrivateKey) { + t.Helper() + templ := x509.Certificate{ + SerialNumber: big.NewInt(5), + BasicConstraintsValid: true, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + Subject: pkix.Name{CommonName: "test-cert"}, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + IPAddresses: []net.IP{netip.MustParseAddr("::1").AsSlice()}, + CRLDistributionPoints: []string{"http://static.corp.google.com/crl/campus-sln/borg"}, + } + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("ecdsa.GenerateKey failed err = %v", err) + } + rawCert, err := x509.CreateCertificate(rand.Reader, &templ, &templ, key.Public(), key) + if err != nil { + t.Fatalf("x509.CreateCertificate failed err = %v", err) + } + cert, err := x509.ParseCertificate(rawCert) + if err != nil { + t.Fatalf("x509.ParseCertificate failed err = %v", err) + } + + srvCfg := tls.Config{ + Certificates: []tls.Certificate{ + { + Certificate: [][]byte{cert.Raw}, + PrivateKey: key, + }, + }, + } + l, err := tls.Listen("tcp6", "[::1]:0", &srvCfg) + if err != nil { + t.Fatalf("tls.Listen failed err = %v", err) + } + return l, cert, key +} + +// TestVerifyConnection will setup a client/server connection and check revocation in the real TLS dialer +func TestVerifyConnection(t *testing.T) { + lis, cert, key := setupTLSConn(t) + defer func() { + lis.Close() + }() + + var handshakeTests = []struct { + desc string + revoked []pkix.RevokedCertificate + success bool + }{ + { + desc: "Empty CRL", + revoked: []pkix.RevokedCertificate{}, + success: true, + }, + { + desc: "Revoked Cert", + revoked: []pkix.RevokedCertificate{ + { + SerialNumber: cert.SerialNumber, + RevocationTime: time.Now(), + }, + }, + success: false, + }, + } + for _, tt := range handshakeTests { + t.Run(tt.desc, func(t *testing.T) { + // Accept one connection. + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("tls.Accept failed err = %v", err) + } else { + conn.Write([]byte("Hello, World!")) + conn.Close() + } + }() + + dir, err := os.MkdirTemp("", "crl_dir") + if err != nil { + t.Fatalf("os.MkdirTemp failed err = %v", err) + } + defer os.RemoveAll(dir) + + template := &x509.RevocationList{ + RevokedCertificates: tt.revoked, + ThisUpdate: time.Now(), + NextUpdate: time.Now().Add(time.Hour), + Number: big.NewInt(1), + } + crl, err := x509.CreateRevocationList(rand.Reader, template, cert, key) + if err != nil { + t.Fatalf("templ.CreateRevocationList failed err = %v", err) + } + + err = os.WriteFile(path.Join(dir, fmt.Sprintf("%s.r0", cert.Subject.ToRDNSequence())), crl, 0777) + if err != nil { + t.Fatalf("os.WriteFile failed err = %v", err) + } + + cp := x509.NewCertPool() + cp.AddCert(cert) + provider, err := NewFileWatcherCRLProvider(FileWatcherOptions{CRLDirectory: dir}) + if err != nil { + t.Errorf("NewFileWatcherCRLProvider: err = %v", err) + } + defer provider.Close() + cliCfg := tls.Config{ + RootCAs: cp, + VerifyConnection: func(cs tls.ConnectionState) error { + return checkRevocation(cs, RevocationOptions{CRLProvider: provider}) + }, + } + conn, err := tls.Dial(lis.Addr().Network(), lis.Addr().String(), &cliCfg) + t.Logf("tls.Dial err = %v", err) + if tt.success && err != nil { + t.Errorf("Expected success got err = %v", err) + } + if !tt.success && err == nil { + t.Error("Expected error, but got success") + } + if err == nil { + conn.Close() + } + }) + } +} diff --git a/security/advancedtls/examples/credential_reloading_from_files/README.md b/security/advancedtls/examples/credential_reloading_from_files/README.md new file mode 100644 index 0000000..8e39d37 --- /dev/null +++ b/security/advancedtls/examples/credential_reloading_from_files/README.md @@ -0,0 +1,25 @@ +# Credential Reloading From Files + +Credential reloading is a feature supported in the advancedtls library. +A very common way to achieve this is to reload from files. + +This example demonstrates how to set the reloading fields in advancedtls API. +Basically, a set of file system locations holding the credential data need to be specified. +Once the credential data needs to be updated, users just change the credential data in the file system, and gRPC will pick up the changes automatically. + +A couple of things to note: + 1. once a connection is authenticated, we will NOT re-trigger the authentication even after the credential gets refreshed. + 2. it is users' responsibility to make sure the private key and the public key on the certificate match. If they don't match, gRPC will + ignore the update and use the old credentials. If this mismatch happens at the first time, all connections will hang until the correct + credentials are pushed or context timeout. + +## Try it +In directory `security/advancedtls/examples`: + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` diff --git a/security/advancedtls/examples/credential_reloading_from_files/client/main.go b/security/advancedtls/examples/credential_reloading_from_files/client/main.go new file mode 100644 index 0000000..abdd7b6 --- /dev/null +++ b/security/advancedtls/examples/credential_reloading_from_files/client/main.go @@ -0,0 +1,111 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// The client demonstrates how to use the credential reloading feature in +// advancedtls to make a mTLS connection to the server. +package main + +import ( + "context" + "flag" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/tls/certprovider/pemfile" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/security/advancedtls" + "google.golang.org/grpc/security/advancedtls/testdata" +) + +var address = "localhost:50051" + +const ( + // Default timeout for normal connections. + defaultTimeout = 2 * time.Second + // Intervals that set to monitor the credential updates. + credRefreshingInterval = 500 * time.Millisecond +) + +func main() { + tmpKeyFile := flag.String("key", "", "temporary key file path") + tmpCertFile := flag.String("cert", "", "temporary cert file path") + flag.Parse() + + if tmpKeyFile == nil || *tmpKeyFile == "" { + log.Fatalf("tmpKeyFile is nil or empty.") + } + if tmpCertFile == nil || *tmpCertFile == "" { + log.Fatalf("tmpCertFile is nil or empty.") + } + + // Initialize credential struct using reloading API. + identityOptions := pemfile.Options{ + CertFile: *tmpCertFile, + KeyFile: *tmpKeyFile, + RefreshDuration: credRefreshingInterval, + } + identityProvider, err := pemfile.NewProvider(identityOptions) + if err != nil { + log.Fatalf("pemfile.NewProvider(%v) failed: %v", identityOptions, err) + } + rootOptions := pemfile.Options{ + RootFile: testdata.Path("client_trust_cert_1.pem"), + RefreshDuration: credRefreshingInterval, + } + rootProvider, err := pemfile.NewProvider(rootOptions) + if err != nil { + log.Fatalf("pemfile.NewProvider(%v) failed: %v", rootOptions, err) + } + options := &advancedtls.Options{ + IdentityOptions: advancedtls.IdentityCertificateOptions{ + IdentityProvider: identityProvider, + }, + AdditionalPeerVerification: func(*advancedtls.HandshakeVerificationInfo) (*advancedtls.PostHandshakeVerificationResults, error) { + return &advancedtls.PostHandshakeVerificationResults{}, nil + }, + RootOptions: advancedtls.RootCertificateOptions{ + RootProvider: rootProvider, + }, + VerificationType: advancedtls.CertVerification, + } + clientTLSCreds, err := advancedtls.NewClientCreds(options) + if err != nil { + log.Fatalf("advancedtls.NewClientCreds(%v) failed: %v", options, err) + } + + // Make a connection using the credentials. + conn, err := grpc.NewClient(address, grpc.WithTransportCredentials(clientTLSCreds)) + if err != nil { + log.Fatalf("grpc.NewClient to %s failed: %v", address, err) + } + client := pb.NewGreeterClient(conn) + + // Send the requests every 0.5s. The credential is expected to be changed in + // the bash script. We don't cancel the context nor call conn.Close() here, + // since the bash script is expected to close the client goroutine. + for { + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + _, err = client.SayHello(ctx, &pb.HelloRequest{Name: "gRPC"}, grpc.WaitForReady(true)) + if err != nil { + log.Fatalf("client.SayHello failed: %v", err) + } + cancel() + time.Sleep(500 * time.Millisecond) + } +} diff --git a/security/advancedtls/examples/credential_reloading_from_files/server/main.go b/security/advancedtls/examples/credential_reloading_from_files/server/main.go new file mode 100644 index 0000000..bc1a323 --- /dev/null +++ b/security/advancedtls/examples/credential_reloading_from_files/server/main.go @@ -0,0 +1,112 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// The server demonstrates how to use the credential reloading feature in +// advancedtls to serve mTLS connections from the client. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/tls/certprovider/pemfile" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/security/advancedtls" + "google.golang.org/grpc/security/advancedtls/testdata" + + pb "google.golang.org/grpc/examples/helloworld/helloworld" +) + +var port = ":50051" + +// Intervals that set to monitor the credential updates. +const credRefreshingInterval = 1 * time.Minute + +type greeterServer struct { + pb.UnimplementedGreeterServer +} + +// sayHello is a simple implementation of the pb.GreeterServer SayHello method. +func (greeterServer) SayHello(_ context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + return &pb.HelloReply{Message: "Hello " + in.Name}, nil +} + +func main() { + flag.Parse() + fmt.Printf("server starting on port %s...\n", port) + + identityOptions := pemfile.Options{ + CertFile: testdata.Path("server_cert_1.pem"), + KeyFile: testdata.Path("server_key_1.pem"), + RefreshDuration: credRefreshingInterval, + } + identityProvider, err := pemfile.NewProvider(identityOptions) + if err != nil { + log.Fatalf("pemfile.NewProvider(%v) failed: %v", identityOptions, err) + } + defer identityProvider.Close() + rootOptions := pemfile.Options{ + RootFile: testdata.Path("server_trust_cert_1.pem"), + RefreshDuration: credRefreshingInterval, + } + rootProvider, err := pemfile.NewProvider(rootOptions) + if err != nil { + log.Fatalf("pemfile.NewProvider(%v) failed: %v", rootOptions, err) + } + defer rootProvider.Close() + + // Start a server and create a client using advancedtls API with Provider. + options := &advancedtls.Options{ + IdentityOptions: advancedtls.IdentityCertificateOptions{ + IdentityProvider: identityProvider, + }, + RootOptions: advancedtls.RootCertificateOptions{ + RootProvider: rootProvider, + }, + RequireClientCert: true, + AdditionalPeerVerification: func(params *advancedtls.HandshakeVerificationInfo) (*advancedtls.PostHandshakeVerificationResults, error) { + // This message is to show the certificate under the hood is actually reloaded. + fmt.Printf("Client common name: %s.\n", params.Leaf.Subject.CommonName) + return &advancedtls.PostHandshakeVerificationResults{}, nil + }, + VerificationType: advancedtls.CertVerification, + } + serverTLSCreds, err := advancedtls.NewServerCreds(options) + if err != nil { + log.Fatalf("advancedtls.NewServerCreds(%v) failed: %v", options, err) + } + s := grpc.NewServer(grpc.Creds(serverTLSCreds), grpc.KeepaliveParams(keepalive.ServerParameters{ + // Set the max connection time to be 0.5 s to force the client to + // re-establish the connection, and hence re-invoke the verification + // callback. + MaxConnectionAge: 500 * time.Millisecond, + })) + lis, err := net.Listen("tcp", port) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + pb.RegisterGreeterServer(s, greeterServer{}) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/security/advancedtls/examples/examples_test.sh b/security/advancedtls/examples/examples_test.sh new file mode 100755 index 0000000..2b91152 --- /dev/null +++ b/security/advancedtls/examples/examples_test.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# +# Copyright 2020 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set +e + +export TMPDIR=$(mktemp -d) +trap "rm -rf ${TMPDIR}" EXIT + +clean () { + for i in {1..10}; do + jobs -p | xargs -n1 pkill -P + # A simple "wait" just hangs sometimes. Running `jobs` seems to help. + sleep 1 + if jobs | read; then + return + fi + done + echo "$(tput setaf 1) clean failed to kill tests $(tput sgr 0)" + jobs + pstree + rm ${CLIENT_LOG} + rm ${SERVER_LOG} + rm ${KEY_FILE_PATH} + rm ${CERT_FILE_PATH} + exit 1 +} + +fail () { + echo "$(tput setaf 1) $1 $(tput sgr 0)" + clean + exit 1 +} + +pass () { + echo "$(tput setaf 2) $1 $(tput sgr 0)" +} + +EXAMPLES=( + "credential_reloading_from_files" +) + +declare -a EXPECTED_SERVER_OUTPUT=("Client common name: foo.bar.hoo.com" "Client common name: foo.bar.another.client.com") + +cd ./security/advancedtls/examples + +for example in ${EXAMPLES[@]}; do + echo "$(tput setaf 4) testing: ${example} $(tput sgr 0)" + + KEY_FILE_PATH=$(mktemp) + cat ../testdata/client_key_1.pem > ${KEY_FILE_PATH} + + CERT_FILE_PATH=$(mktemp) + cat ../testdata/client_cert_1.pem > ${CERT_FILE_PATH} + + # Build server. + if ! go build -o /dev/null ./${example}/*server/*.go; then + fail "failed to build server" + else + pass "successfully built server" + fi + + # Build client. + if ! go build -o /dev/null ./${example}/*client/*.go; then + fail "failed to build client" + else + pass "successfully built client" + fi + + # Start server. + SERVER_LOG="$(mktemp)" + go run ./$example/*server/*.go &> $SERVER_LOG & + + # Run client binary. + CLIENT_LOG="$(mktemp)" + go run ${example}/*client/*.go -key=${KEY_FILE_PATH} -cert=${CERT_FILE_PATH} &> $CLIENT_LOG & + + # Wait for the client to send some requests using old credentials. + sleep 4s + + # Switch to the new credentials. + cat ../testdata/another_client_key_1.pem > ${KEY_FILE_PATH} + cat ../testdata/another_client_cert_1.pem > ${CERT_FILE_PATH} + + # Wait for the client to send some requests using new credentials. + sleep 4s + + # Check server log for expected output. + for output in "${EXPECTED_SERVER_OUTPUT[@]}"; do + if ! grep -q "$output" $SERVER_LOG; then + fail "server log missing output: $output + got server log: + $(cat $SERVER_LOG) + " + else + pass "server log contains expected output: $output" + fi + done + + clean +done diff --git a/security/advancedtls/examples/go.mod b/security/advancedtls/examples/go.mod new file mode 100644 index 0000000..1012924 --- /dev/null +++ b/security/advancedtls/examples/go.mod @@ -0,0 +1,26 @@ +module google.golang.org/grpc/security/advancedtls/examples + +go 1.25.0 + +require ( + google.golang.org/grpc v1.83.0 + google.golang.org/grpc/examples v0.0.0-20260818050025-b6685c50ff98 + google.golang.org/grpc/security/advancedtls v1.0.0 +) + +require ( + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) + +replace google.golang.org/grpc => ../../.. + +replace google.golang.org/grpc/examples => ../../../examples + +replace google.golang.org/grpc/security/advancedtls => ../ diff --git a/security/advancedtls/examples/go.sum b/security/advancedtls/examples/go.sum new file mode 100644 index 0000000..e194afe --- /dev/null +++ b/security/advancedtls/examples/go.sum @@ -0,0 +1,30 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 h1:utmqiUzZAEINbLmRyAt1QjVkMyLQ+lvIwCpLaB1rZfw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/security/advancedtls/go.mod b/security/advancedtls/go.mod new file mode 100644 index 0000000..2ed534e --- /dev/null +++ b/security/advancedtls/go.mod @@ -0,0 +1,24 @@ +module google.golang.org/grpc/security/advancedtls + +go 1.25.0 + +require ( + github.com/google/go-cmp v0.7.0 + golang.org/x/crypto v0.55.0 + google.golang.org/grpc v1.83.0 + google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 +) + +require ( + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) + +replace google.golang.org/grpc => ../../ + +replace google.golang.org/grpc/examples => ../../examples diff --git a/security/advancedtls/go.sum b/security/advancedtls/go.sum new file mode 100644 index 0000000..e194afe --- /dev/null +++ b/security/advancedtls/go.sum @@ -0,0 +1,30 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 h1:utmqiUzZAEINbLmRyAt1QjVkMyLQ+lvIwCpLaB1rZfw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/security/advancedtls/internal/testutils/testutils.go b/security/advancedtls/internal/testutils/testutils.go new file mode 100644 index 0000000..dd26366 --- /dev/null +++ b/security/advancedtls/internal/testutils/testutils.go @@ -0,0 +1,129 @@ +/* + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package testutils contains helper functions for advancedtls. +package testutils + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" + + "google.golang.org/grpc/security/advancedtls/testdata" +) + +// CertStore contains all the certificates used in the integration tests. +type CertStore struct { + // ClientCert1 is the certificate sent by client to prove its identity. + // It is trusted by ServerTrust1. + ClientCert1 tls.Certificate + // ClientCert2 is the certificate sent by client to prove its identity. + // It is trusted by ServerTrust2. + ClientCert2 tls.Certificate + // ClientCertForCRL is the certificate sent by client to prove its identity. + // It is trusted by ServerTrust3. Used in CRL tests + ClientCertForCRL tls.Certificate + // ServerCert1 is the certificate sent by server to prove its identity. + // It is trusted by ClientTrust1. + ServerCert1 tls.Certificate + // ServerCert2 is the certificate sent by server to prove its identity. + // It is trusted by ClientTrust2. + ServerCert2 tls.Certificate + // ServerCertForCRL is a revoked certificate + // (this info is stored in provider_crl_server_revoked.pem). + ServerCertForCRL tls.Certificate + // ServerPeer3 is the certificate sent by server to prove its identity. + ServerPeer3 tls.Certificate + // ServerPeerLocalhost1 is the certificate sent by server to prove its + // identity. It has "localhost" as its common name, and is trusted by + // ClientTrust1. + ServerPeerLocalhost1 tls.Certificate + // ClientTrust1 is the root certificate used on the client side. + ClientTrust1 *x509.CertPool + // ClientTrust2 is the root certificate used on the client side. + ClientTrust2 *x509.CertPool + // ClientTrust3 is the root certificate used on the client side. + ClientTrust3 *x509.CertPool + // ServerTrust1 is the root certificate used on the server side. + ServerTrust1 *x509.CertPool + // ServerTrust2 is the root certificate used on the server side. + ServerTrust2 *x509.CertPool + // ServerTrust2 is the root certificate used on the server side. + ServerTrust3 *x509.CertPool +} + +func readTrustCert(fileName string) (*x509.CertPool, error) { + trustData, err := os.ReadFile(fileName) + if err != nil { + return nil, err + } + trustPool := x509.NewCertPool() + if !trustPool.AppendCertsFromPEM(trustData) { + return nil, fmt.Errorf("error loading trust certificates") + } + return trustPool, nil +} + +// LoadCerts function is used to load test certificates at the beginning of +// each integration test. +func (cs *CertStore) LoadCerts() error { + var err error + if cs.ClientCert1, err = tls.LoadX509KeyPair(testdata.Path("client_cert_1.pem"), testdata.Path("client_key_1.pem")); err != nil { + return err + } + if cs.ClientCert2, err = tls.LoadX509KeyPair(testdata.Path("client_cert_2.pem"), testdata.Path("client_key_2.pem")); err != nil { + return err + } + if cs.ClientCertForCRL, err = tls.LoadX509KeyPair(testdata.Path("crl/provider_client_cert.pem"), testdata.Path("crl/provider_client_cert.key")); err != nil { + return err + } + if cs.ServerCert1, err = tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), testdata.Path("server_key_1.pem")); err != nil { + return err + } + if cs.ServerCert2, err = tls.LoadX509KeyPair(testdata.Path("server_cert_2.pem"), testdata.Path("server_key_2.pem")); err != nil { + return err + } + if cs.ServerCertForCRL, err = tls.LoadX509KeyPair(testdata.Path("crl/provider_server_cert.pem"), testdata.Path("crl/provider_server_cert.key")); err != nil { + return err + } + if cs.ServerPeer3, err = tls.LoadX509KeyPair(testdata.Path("server_cert_3.pem"), testdata.Path("server_key_3.pem")); err != nil { + return err + } + if cs.ServerPeerLocalhost1, err = tls.LoadX509KeyPair(testdata.Path("server_cert_localhost_1.pem"), testdata.Path("server_key_localhost_1.pem")); err != nil { + return err + } + if cs.ClientTrust1, err = readTrustCert(testdata.Path("client_trust_cert_1.pem")); err != nil { + return err + } + if cs.ClientTrust2, err = readTrustCert(testdata.Path("client_trust_cert_2.pem")); err != nil { + return err + } + if cs.ClientTrust3, err = readTrustCert(testdata.Path("crl/provider_client_trust_cert.pem")); err != nil { + return err + } + if cs.ServerTrust1, err = readTrustCert(testdata.Path("server_trust_cert_1.pem")); err != nil { + return err + } + if cs.ServerTrust2, err = readTrustCert(testdata.Path("server_trust_cert_2.pem")); err != nil { + return err + } + if cs.ServerTrust3, err = readTrustCert(testdata.Path("crl/provider_server_trust_cert.pem")); err != nil { + return err + } + return nil +} diff --git a/security/advancedtls/sni.go b/security/advancedtls/sni.go new file mode 100644 index 0000000..916e272 --- /dev/null +++ b/security/advancedtls/sni.go @@ -0,0 +1,51 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package advancedtls + +import ( + "crypto/tls" + "fmt" +) + +// buildGetCertificates returns the certificate that matches the SNI field +// for the given ClientHelloInfo, defaulting to the first element of o.GetCertificates. +func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *Options) (*tls.Certificate, error) { + if o.IdentityOptions.GetIdentityCertificatesForServer == nil { + return nil, fmt.Errorf("function GetCertificates must be specified") + } + certificates, err := o.IdentityOptions.GetIdentityCertificatesForServer(clientHello) + if err != nil { + return nil, err + } + if len(certificates) == 0 { + return nil, fmt.Errorf("no certificates configured") + } + // If users pass in only one certificate, return that certificate. + if len(certificates) == 1 { + return certificates[0], nil + } + // Choose the SNI certificate using SupportsCertificate. + for _, cert := range certificates { + if err := clientHello.SupportsCertificate(cert); err == nil { + return cert, nil + } + } + // If nothing matches, return the first certificate. + return certificates[0], nil +} diff --git a/security/advancedtls/testdata/README.md b/security/advancedtls/testdata/README.md new file mode 100644 index 0000000..1001459 --- /dev/null +++ b/security/advancedtls/testdata/README.md @@ -0,0 +1,48 @@ +About This Directory +------------- +This testdata directory contains the certificates used in the tests of package advancedtls. + +How to Generate Test Certificates Using OpenSSL +------------- + +Supposing we are going to create a `subject_cert.pem` that is trusted by `ca_cert.pem`, here are the +commands we run: + +1. Generate the private key, `ca_key.pem`, and the cert `ca_cert.pem`, for the CA: + + ``` + $ openssl req -x509 -newkey rsa:4096 -keyout ca_key.pem -out ca_cert.pem -nodes -days $DURATION_DAYS + ``` + +2. Generate a private key `subject_key.pem` for the subject: + + ``` + $ openssl genrsa -out subject_key.pem 4096 + ``` + +3. Generate a CSR `csr.pem` using `subject_key.pem`: + + ``` + $ openssl req -new -key subject_key.pem -out csr.pem + ``` + For some cases, we might want to add some extra SAN fields in `subject_cert.pem`. + In those cases, we can create a configuration file(for example, localhost-openssl.cnf), and do the following: + ``` + $ openssl req -new -key subject_key.pem -out csr.pem -config $CONFIG_FILE_NAME + ``` + +4. Use `ca_key.pem` and `ca_cert.pem` to sign `csr.pem`, and get a certificate, `subject_cert.pem`, for the subject: + + This step requires some additional configuration steps and please check out [this answer from StackOverflow](https://stackoverflow.com/a/21340898) for more. + + ``` + $ openssl ca -config openssl-ca.cnf -policy signing_policy -extensions signing_req -out subject_cert.pem -in csr.pem -keyfile ca_key.pem -cert ca_cert.pem + ``` + Please see an example configuration template at `openssl-ca.cnf`. +5. Verify the `subject_cert.pem` is trusted by `ca_cert.pem`: + + + ``` + $ openssl verify -verbose -CAfile ca_cert.pem subject_cert.pem + + ``` diff --git a/security/advancedtls/testdata/another_client_cert_1.pem b/security/advancedtls/testdata/another_client_cert_1.pem new file mode 100644 index 0000000..32f5a3c --- /dev/null +++ b/security/advancedtls/testdata/another_client_cert_1.pem @@ -0,0 +1,122 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 1 (0x1) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=VA, O=Internet Widgits Pty Ltd, CN=foo.bar.hoo.ca.com + Validity + Not Before: Nov 7 17:11:57 2020 GMT + Not After : Mar 25 17:11:57 2048 GMT + Subject: C=US, ST=CA, O=Internet Widgits Pty Ltd, CN=foo.bar.another.client.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (4096 bit) + Modulus: + 00:ec:00:a0:69:4b:11:ae:30:21:15:64:4d:c3:cb: + 94:67:58:44:9a:5e:ca:c3:82:75:eb:6f:8d:b7:33: + ca:e0:69:5f:98:1f:17:e5:7d:04:dd:f9:af:63:3b: + 84:0e:48:2e:23:fc:b4:de:02:64:ad:0c:8f:6b:70: + 15:fe:00:98:47:cb:c3:ff:5c:fb:b4:22:cd:36:18: + 6e:4a:6d:de:0e:7a:90:d8:3d:af:76:1d:17:07:67: + 79:7f:ab:59:af:1f:37:99:b9:46:ea:db:b3:a4:a2: + 8a:e9:b3:41:83:04:71:82:2f:88:51:5d:b3:ba:7d: + 72:d3:ef:97:47:89:b4:d0:78:f1:d6:ef:8a:09:94: + 19:54:ae:08:8c:77:49:2f:6b:c6:c9:56:0c:fa:ec: + 77:76:f0:83:d8:83:d0:4d:b5:f5:d9:e3:12:85:5c: + 64:c5:82:60:73:20:fa:8a:36:00:f4:f2:bc:cf:48: + 08:94:5f:2b:39:88:4c:56:f5:65:30:67:41:78:99: + 7e:26:f5:ab:e7:3d:b0:a3:8c:55:c5:e1:12:39:23: + 00:68:88:c2:b1:43:2f:61:7c:d4:08:35:52:28:5d: + 93:3e:84:c9:8d:0a:37:df:75:06:f4:ae:1e:2a:1d: + e3:f9:0f:26:80:ad:4e:6a:c3:6a:0c:2e:0f:31:0f: + b2:39:b0:98:aa:1e:96:c7:0b:a3:70:6c:35:52:82: + 56:0f:27:e1:07:d5:89:a6:97:58:97:6f:9f:4f:db: + 0e:e6:ef:fd:62:f4:c3:d5:bb:af:ee:f7:5b:26:6d: + 79:c0:46:71:94:d1:6f:ea:2a:61:78:1c:c5:09:4d: + 93:35:bf:6d:1d:26:49:2e:84:9e:8f:48:9c:93:e0: + ef:bb:c7:85:f1:20:2b:8e:1e:da:18:76:db:c6:42: + 4f:87:ed:e9:44:90:ab:99:ea:d7:3a:89:f9:be:b5: + 3a:b8:5b:50:ec:7c:54:ce:d9:ff:9e:94:30:97:25: + e5:ce:13:b4:ee:06:56:39:0b:d4:51:77:a2:e8:3f: + b4:e7:60:2a:03:44:3a:93:ed:8a:f0:d7:90:f7:92: + fe:e9:19:57:37:24:20:f5:9c:64:d0:fc:13:d1:3e: + a5:1d:ea:14:23:8d:f6:ee:c0:5c:f4:27:82:87:90: + 6c:80:6d:ff:6f:1d:ab:2e:83:0f:73:4a:78:3d:40: + 7f:e6:f9:55:b4:ea:e8:83:f7:86:e0:c8:c1:d9:96: + a0:7d:38:08:72:88:d2:c5:90:e3:3e:2a:f9:64:26: + 52:6e:20:79:20:ea:99:1b:75:65:4b:12:93:22:09: + f9:ec:1b:af:74:a4:3c:a3:df:07:d0:50:95:ad:7e: + 55:60:25 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + D5:43:51:8B:A8:4C:84:D0:C8:DE:29:14:1B:15:7A:62:01:ED:FF:EC + X509v3 Authority Key Identifier: + keyid:B4:19:08:1C:FC:10:23:C5:30:86:22:BC:CB:B1:5F:AD:EA:7A:5D:F1 + + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + Signature Algorithm: sha256WithRSAEncryption + 61:a0:89:19:3e:e8:3d:35:bf:6e:5d:0c:d0:ec:36:85:d4:27: + 41:6b:21:0e:5e:ed:e9:e2:17:ef:2b:19:5d:1b:4a:52:7f:e5: + 66:db:24:40:4d:cb:f8:9d:92:5e:10:67:68:d0:f8:56:ab:da: + 3c:8c:4f:02:07:9e:22:49:45:de:bd:d2:92:e8:ef:9d:f2:68: + 97:0d:10:43:c7:b7:73:9e:76:14:7b:1d:b5:8e:df:df:7b:0d: + b3:48:36:21:98:67:3f:41:55:c7:26:09:99:fc:bc:92:8c:9a: + ac:64:76:9d:b2:b1:ca:41:e5:59:79:8d:76:f1:b4:bb:2e:28: + 8b:bb:73:c4:83:0e:f9:6e:62:2c:49:d8:00:e5:87:c7:53:1b: + 45:ec:29:a4:1c:20:82:bf:d1:f4:a4:23:98:b2:1e:41:a0:ee: + 1a:0c:7e:bd:00:84:f5:17:05:1e:a6:7b:fd:75:ee:b9:6d:6b: + 31:2a:0d:97:fa:57:86:57:25:44:8c:5e:e9:bc:78:30:13:77: + 70:87:78:e1:7a:fb:26:db:b9:95:d0:04:c9:93:26:a2:96:b5: + 2b:d5:d3:61:22:b3:ca:31:51:15:6d:51:0a:fb:22:6e:ca:16: + 61:a5:91:9c:e0:39:de:cb:ad:fe:23:51:15:34:42:ae:ac:82: + 80:9a:12:f6:f4:a6:8b:65:b5:f3:21:7b:78:ab:e2:53:83:1c: + 08:83:a5:1a:26:f4:a8:f2:cd:19:d1:85:99:99:45:c8:46:35: + 48:c4:fa:44:80:b0:04:67:48:36:c9:5e:44:fa:41:6e:a7:f2: + eb:22:9e:f8:d3:f2:0d:79:1b:33:78:c4:d0:60:e0:93:5f:69: + 6f:cf:f0:df:04:3d:5b:b3:ac:09:30:ae:32:ea:0f:9f:7b:2c: + 69:bf:f3:fd:6d:7c:a2:25:dc:15:82:df:e0:a5:84:64:39:26: + 43:28:0f:cb:2e:7f:fe:9e:c2:7e:20:d3:ca:6d:96:1c:0e:e1: + b1:ac:0d:2e:4d:97:6f:14:39:65:1a:65:9f:13:2d:0a:28:f3: + 2e:6d:30:af:c6:82:cb:33:20:86:17:de:52:0c:9b:a1:fb:2a: + 59:ca:b1:18:b7:56:06:52:46:9c:8d:36:c6:2e:61:f2:02:2b: + ec:72:15:40:3a:1f:e9:ad:ca:5e:19:aa:1a:73:b5:d4:4d:b9: + d5:b2:24:8c:b6:eb:e6:cc:b2:0e:12:c6:26:52:f7:7a:e7:2c: + 0b:1f:bc:24:1c:76:66:bb:2f:1d:ec:26:9e:53:5b:12:0a:e4: + ae:e3:bc:dd:21:a0:aa:cc:e4:9a:1d:8e:f7:16:36:b2:ed:97: + 81:5a:48:22:a8:f6:6a:2d +-----BEGIN CERTIFICATE----- +MIIFkTCCA3mgAwIBAgIBATANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCVkExITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEb +MBkGA1UEAwwSZm9vLmJhci5ob28uY2EuY29tMB4XDTIwMTEwNzE3MTE1N1oXDTQ4 +MDMyNTE3MTE1N1owYjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMSEwHwYDVQQK +DBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxIzAhBgNVBAMMGmZvby5iYXIuYW5v +dGhlci5jbGllbnQuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA +7ACgaUsRrjAhFWRNw8uUZ1hEml7Kw4J162+NtzPK4GlfmB8X5X0E3fmvYzuEDkgu +I/y03gJkrQyPa3AV/gCYR8vD/1z7tCLNNhhuSm3eDnqQ2D2vdh0XB2d5f6tZrx83 +mblG6tuzpKKK6bNBgwRxgi+IUV2zun1y0++XR4m00Hjx1u+KCZQZVK4IjHdJL2vG +yVYM+ux3dvCD2IPQTbX12eMShVxkxYJgcyD6ijYA9PK8z0gIlF8rOYhMVvVlMGdB +eJl+JvWr5z2wo4xVxeESOSMAaIjCsUMvYXzUCDVSKF2TPoTJjQo333UG9K4eKh3j ++Q8mgK1OasNqDC4PMQ+yObCYqh6WxwujcGw1UoJWDyfhB9WJppdYl2+fT9sO5u/9 +YvTD1buv7vdbJm15wEZxlNFv6ipheBzFCU2TNb9tHSZJLoSej0ick+Dvu8eF8SAr +jh7aGHbbxkJPh+3pRJCrmerXOon5vrU6uFtQ7HxUztn/npQwlyXlzhO07gZWOQvU +UXei6D+052AqA0Q6k+2K8NeQ95L+6RlXNyQg9Zxk0PwT0T6lHeoUI4327sBc9CeC +h5BsgG3/bx2rLoMPc0p4PUB/5vlVtOrog/eG4MjB2ZagfTgIcojSxZDjPir5ZCZS +biB5IOqZG3VlSxKTIgn57BuvdKQ8o98H0FCVrX5VYCUCAwEAAaNaMFgwHQYDVR0O +BBYEFNVDUYuoTITQyN4pFBsVemIB7f/sMB8GA1UdIwQYMBaAFLQZCBz8ECPFMIYi +vMuxX63qel3xMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMA0GCSqGSIb3DQEBCwUA +A4ICAQBhoIkZPug9Nb9uXQzQ7DaF1CdBayEOXu3p4hfvKxldG0pSf+Vm2yRATcv4 +nZJeEGdo0PhWq9o8jE8CB54iSUXevdKS6O+d8miXDRBDx7dznnYUex21jt/few2z +SDYhmGc/QVXHJgmZ/LySjJqsZHadsrHKQeVZeY128bS7LiiLu3PEgw75bmIsSdgA +5YfHUxtF7CmkHCCCv9H0pCOYsh5BoO4aDH69AIT1FwUepnv9de65bWsxKg2X+leG +VyVEjF7pvHgwE3dwh3jhevsm27mV0ATJkyailrUr1dNhIrPKMVEVbVEK+yJuyhZh +pZGc4Dney63+I1EVNEKurIKAmhL29KaLZbXzIXt4q+JTgxwIg6UaJvSo8s0Z0YWZ +mUXIRjVIxPpEgLAEZ0g2yV5E+kFup/LrIp740/INeRszeMTQYOCTX2lvz/DfBD1b +s6wJMK4y6g+feyxpv/P9bXyiJdwVgt/gpYRkOSZDKA/LLn/+nsJ+INPKbZYcDuGx +rA0uTZdvFDllGmWfEy0KKPMubTCvxoLLMyCGF95SDJuh+ypZyrEYt1YGUkacjTbG +LmHyAivschVAOh/prcpeGaoac7XUTbnVsiSMtuvmzLIOEsYmUvd65ywLH7wkHHZm +uy8d7CaeU1sSCuSu47zdIaCqzOSaHY73Fjay7ZeBWkgiqPZqLQ== +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/another_client_key_1.pem b/security/advancedtls/testdata/another_client_key_1.pem new file mode 100644 index 0000000..ad61ccf --- /dev/null +++ b/security/advancedtls/testdata/another_client_key_1.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEA7ACgaUsRrjAhFWRNw8uUZ1hEml7Kw4J162+NtzPK4GlfmB8X +5X0E3fmvYzuEDkguI/y03gJkrQyPa3AV/gCYR8vD/1z7tCLNNhhuSm3eDnqQ2D2v +dh0XB2d5f6tZrx83mblG6tuzpKKK6bNBgwRxgi+IUV2zun1y0++XR4m00Hjx1u+K +CZQZVK4IjHdJL2vGyVYM+ux3dvCD2IPQTbX12eMShVxkxYJgcyD6ijYA9PK8z0gI +lF8rOYhMVvVlMGdBeJl+JvWr5z2wo4xVxeESOSMAaIjCsUMvYXzUCDVSKF2TPoTJ +jQo333UG9K4eKh3j+Q8mgK1OasNqDC4PMQ+yObCYqh6WxwujcGw1UoJWDyfhB9WJ +ppdYl2+fT9sO5u/9YvTD1buv7vdbJm15wEZxlNFv6ipheBzFCU2TNb9tHSZJLoSe +j0ick+Dvu8eF8SArjh7aGHbbxkJPh+3pRJCrmerXOon5vrU6uFtQ7HxUztn/npQw +lyXlzhO07gZWOQvUUXei6D+052AqA0Q6k+2K8NeQ95L+6RlXNyQg9Zxk0PwT0T6l +HeoUI4327sBc9CeCh5BsgG3/bx2rLoMPc0p4PUB/5vlVtOrog/eG4MjB2ZagfTgI +cojSxZDjPir5ZCZSbiB5IOqZG3VlSxKTIgn57BuvdKQ8o98H0FCVrX5VYCUCAwEA +AQKCAgAzIaOfjHMlMSpJzzSGAjqB9X7Pj1AQ8dgIjV+/3InM+yeJ9tqfjumaCjm0 +nzVqPrs4cszg+NXFJF6CYYNyR8C2dXBeiE/EZHHfkYV7vLgKnQV6xEqapYzSvtl1 +DrPcnD/Yn2q9AaK3PbwpC/xanYDWOuQm9M02z20se9Fj33L8Y+fJsJZQovSmAxq5 +DDMgAhLMlkczqj3r2ApIw65C1/SPI4JkwHLY0/l/mBqQDUlByMGdizbIpqHf0ibw +BDTLOuPVdDP/zuRSsmvt0z7WI4BmPq4c99xuuWavkXMC4EKPmk6Hkg907kzSrjE2 +m+7PIzC8SksGQAYoXXRBdU03TPZI3PaHb/Lyb6TCVK4gmhAF+QsSobOkFsM72QpF +JBdcRgNsy/fZLFsPFH+g4zCRLrRcPrP1ifiqXOrDJ128JD5PqrEBMyXKQMkUB6qq ++0Hdm2kPWhs5tM4XY7X3xIQJ8AmZ/VPSHwecIUIu4SwdBlaDfQAZgTFER0vzweS5 +PPuJXp/bLW/gOa7hG7bKwmMTHR9ge+i/aMN1JEY6t2pY8AGH9RiH0WRcgiIC6eev +YTmpb8dd4GhatqueKN48UpzCK6CRmvLeBINAyOLybV4RM9cQb6g2sWTc0sgQR4tL +QMB0XAug6QOD7pXFa36JCCrkDDDojjzPaOLCCVtY03si6Yps1QKCAQEA9sKM7nr1 +QoZjo1C7KAu35l1vHVZoXPXm4z8UVNzHqRs+Bd8cut1nNm/kDE5ur5ceGkbSqGs2 +ktqGF/nZYVpUJBKyIXVO4wamUeSFdojllUkXJKOyT+qav1J9/FtOUaONRuc8u452 +HkBclijh4fRkkuM2SROvM7+5YGhQFR/CZN/RXenOD+7wPWa3/3cmV7fZff3dQjDb +7K7vzq/BQVsbfjVSzd3Zwuo0teFz0AEWWt2LE455v/gYP9laIRWxGbIxpRgIyu2e +M/CwRRQT1SOJ9dwGGYls2yTHTfD2hYJnw0fXXo7YyMntMhRazKvQllZF0CdpHZO2 +OBk7UxwNwnMLnwKCAQEA9Nb0RUAR2gDoDQSWoMrRZ3cXCbA1efzP82A/TcppB1En +qqPwuoDd+nTH6KuXaZJYzbGSADyHWzMUaokWz28wC/2ffOhIxSdbU5su/20QfuIN +6rb9BZv6NGjh35nH8ftfyx5tW26iV7mcveig2ixQLg62xTHB8p0FJM3QRIflZ3OK +FWCC/+vZVyiAtGL6xS3XKPHukSeD5zOFKcF3GB9KdSbwED7bdxQ06HGfoWOgCmYb +FnlUWaUZl/wIePYxUSjAmGLkctwH2x83MiRcvPLGI/lW3xzZFHxNj+K9/aJPAbyY +JvZmBeezXphzrU3GjStzrwwd9SQpM4TDkDMIGBk9uwKCAQEAg/KcMZmGNEBwXw/4 +Q/2gJIqps+JUhADpqXI9iPNVwFNU4wbe8f0aB73lD7+Q6EvCSQK9+lj6IaTAN2ne +l3QZsgBdSA7WVAdmQDwWMcAaI62ltm3iF2G3xb5yp9KbGoR+Mv/LNe+DscFwwMqz +noN0lCbzDDh+YwmOMsMUr3cAF7im17UB/vshc3PNx8kKs7UXk4uAGLjPoMwaZ0cL +68qv9NjGolaS7usVrHwV1Y//SC9XAuoYqFIdhWbQDwjuXnMuoL0tVnWhNtzpJMcL +o9kRGGrCyDz3/Ga6PC8xY0rL+VwdCe8QdK2lLDY+J1toejs/sYKhbrNhqLW1R0el +A+lIuQKCAQAWkoKup7t9l7vNB3FDna80lLwg/ofPmUkqrOLpLxIDxK2dg8O7zgmo +/382qispZn6daBOHxgzMkab+M2lQ8nVBhb5ga6HZ20kGKjZpAgsVR430563oCHtG +vaylSq4uVvh753A5j7eT0t7qeznpI1C5Dk43W+D/lw5UWE0tJEI4CWTfl6g8I+hD +qs5C0yU/bHx7n+JYq4XzmMJcGSP7q1bX+iEDvmfJUKmYDHGlFWQ50TQKHGF0ak4z +vt6hGEFvtAwdgHCDTlnDD9us2cFbAh7WTjR+GVDCHLuh2kudyIr0JAj6/phlTvkw +bWmsvpDhjvH5X2qboRvTThghgTLr1dflAoIBAQDn4g3+8Aqf7LTR5EbU64BU7j5R +1CddpJDBafV+lmhkO5oFPFXh72tJu+P1zIRBhCwWqBGCZZQK0LUzezrenyGJH+dk +ZlEr3axKfiv9WKRe1AB6lJSxtOgPy9Kzb5pMFEXipqFQ70i49MjRjJ6WuMRMznug +q/dkVq7q7K+4Dv+/PDHyXols5LsMTNs8bypS0wCubklAk12NxRdeHu4JXJPjtdyS +dQvQj3oheh4km5Q7EqxPnMRvxg2FK7RNwvZ/Q70dCXb8jh7u4Ty4sq5tK4RGyAkL +Zfbng79N0J/+kvRFczaVIvCwSbfexPx/BtGsA1HnfAb2ADug8RlhEwHM9E8G +-----END RSA PRIVATE KEY----- diff --git a/security/advancedtls/testdata/client_cert_1.pem b/security/advancedtls/testdata/client_cert_1.pem new file mode 100644 index 0000000..8a3534d --- /dev/null +++ b/security/advancedtls/testdata/client_cert_1.pem @@ -0,0 +1,122 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 5 (0x5) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=VA, O=Internet Widgits Pty Ltd, CN=foo.bar.hoo.ca.com + Validity + Not Before: Nov 15 19:15:24 2019 GMT + Not After : Aug 29 19:15:24 2293 GMT + Subject: C=US, ST=CA, O=Internet Widgits Pty Ltd, CN=foo.bar.hoo.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (4096 bit) + Modulus: + 00:c3:3e:b5:d8:bc:73:5f:b4:e5:60:a8:73:0e:8c: + 9c:ff:28:2c:a9:bc:68:12:8b:ae:3e:aa:7b:f5:b9: + 2d:b3:73:f8:9e:64:e9:8e:26:ca:36:11:5f:f9:73: + 39:f0:19:55:d1:ba:f4:4e:2b:ab:ee:01:3b:eb:f0: + 5f:6e:b3:24:39:c1:f5:f0:bd:6a:d0:6c:56:cf:96: + 33:5d:05:48:c4:b5:b3:3e:55:8e:89:8e:6b:79:c5: + 3b:99:1c:e3:03:69:0f:74:a2:97:7b:bf:c4:11:1d: + da:d7:cb:87:0d:90:25:64:29:3e:f6:62:bc:f9:a5: + 56:de:56:e1:27:77:51:1a:30:f1:88:89:01:c2:c8: + 35:40:d3:2e:2d:30:ef:d7:de:3b:28:15:4a:a4:a9: + ba:f0:40:f0:79:3a:16:f9:ae:52:32:c3:52:ad:53: + 9c:94:07:d5:9b:63:50:90:ff:f1:8c:fd:4e:59:b8: + 5e:0a:73:9b:b4:b7:60:e1:7c:07:02:50:74:f3:48: + 69:6a:74:7c:b2:96:70:86:19:2f:82:4c:95:57:aa: + 4c:2f:38:75:8b:9b:a1:3e:7d:dd:da:bf:d2:a4:a3: + 3a:02:17:43:35:0a:52:03:f5:fb:1a:a1:60:28:c3: + e7:41:eb:4a:0c:f4:43:6e:81:64:ba:41:8d:61:40: + 97:9f:e2:67:51:7c:2d:2f:17:72:b9:a0:27:5c:fc: + e3:b6:a6:de:f4:1e:34:95:2c:c5:7f:13:c4:bb:25: + 76:3e:3b:39:b6:36:d0:60:17:1e:c7:01:9c:3d:65: + 9a:96:4c:d8:4c:10:85:32:76:c7:6e:53:64:80:c9: + 33:1a:44:39:a7:c7:69:d3:64:c3:4c:06:20:56:d2: + eb:d9:65:56:02:65:c4:ba:72:db:89:c4:00:3f:89: + f4:75:d5:6d:83:ce:ad:66:fb:73:f8:8e:bb:dc:01: + c0:4f:86:c1:57:45:68:34:3f:55:1f:0e:ef:82:3f: + 9a:26:1c:9c:8d:88:5e:27:ab:b6:b9:58:a7:c5:b0: + 36:0f:99:ba:d8:cc:89:41:ed:ab:26:b8:8a:16:17: + 21:67:b6:4d:83:d1:dd:53:de:67:ab:76:a3:af:f8: + 60:99:29:6a:0a:4f:f2:ad:32:54:69:33:8c:f2:ca: + 9b:d6:59:cd:8c:69:cd:3f:d3:8f:05:28:d1:29:04: + bf:b2:de:98:0f:9d:62:13:6d:fe:de:be:2d:c6:be: + d6:f8:10:cb:b5:b3:4f:ad:a4:60:36:b3:19:29:29: + b9:b4:37:5d:13:e7:36:cb:f9:fa:7f:9e:63:7e:f3: + 05:ee:9e:e6:4d:ff:e3:46:a4:7b:1f:12:72:89:b6: + 10:5f:bd + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 7F:9D:9C:C6:86:DF:9E:07:93:94:EF:18:2D:0A:0A:50:AA:1F:A2:B7 + X509v3 Authority Key Identifier: + keyid:B4:19:08:1C:FC:10:23:C5:30:86:22:BC:CB:B1:5F:AD:EA:7A:5D:F1 + + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + Signature Algorithm: sha256WithRSAEncryption + 31:b0:6d:25:5e:8e:9b:73:01:ac:08:b9:a6:70:8e:de:18:fd: + b8:2b:bb:2d:7c:c0:84:20:c8:d2:32:8a:d9:ca:24:b9:75:e2: + c8:91:40:db:0a:4e:e5:05:bd:a6:bb:22:85:3c:8e:be:d3:65: + 0a:7f:cd:c0:fc:eb:94:61:91:30:53:1d:4d:9f:4e:d7:38:0f: + ab:d9:3d:a1:1c:48:c1:e6:3c:35:cc:db:47:31:a6:b2:44:b8: + db:34:6c:28:20:49:ff:e1:2b:cd:48:e1:7e:78:7a:05:0a:31: + 3d:dd:45:51:95:06:ad:5c:8c:0e:ff:0c:98:77:4f:5c:42:dc: + da:d8:d3:30:58:e4:3c:ef:b3:64:3f:f2:e2:19:d9:36:04:1a: + b4:87:c2:1b:89:5d:52:17:fb:27:a2:83:2d:55:6d:1f:80:d5: + a7:ea:20:b0:0a:23:4d:0f:48:36:ae:42:f9:fc:c8:86:f4:69: + 30:e8:cd:52:34:62:ee:b9:fd:12:4b:ba:4d:a2:75:47:d4:b6: + b2:dd:ea:6f:6b:a2:86:f5:c0:3b:06:09:c1:5f:30:96:b6:79: + 32:45:b3:d1:8c:0a:d2:58:d3:39:2f:21:ba:7a:3e:a7:38:cc: + 88:16:1e:75:62:30:fd:79:a3:1d:a9:bd:df:66:dc:b9:f5:79: + bc:fb:bd:bd:e5:f0:46:60:d1:03:7b:58:06:00:f5:d8:36:a0: + a9:b0:2d:4f:4e:1b:6f:17:f0:d9:51:0c:25:a2:48:ac:e3:f4: + a6:52:59:84:83:e3:79:df:ca:9e:5c:24:d3:f9:55:39:8c:3e: + 2a:91:3f:53:0b:d4:22:55:c7:a3:80:41:05:e3:41:7d:16:d1: + af:a2:1e:f7:fa:ee:f3:a7:6e:19:66:af:dd:23:39:5a:33:f9: + 61:3d:e7:90:e2:9a:0e:8e:8b:a0:3b:27:55:e2:ed:09:c5:ca: + 71:14:95:10:be:03:8e:2a:6d:48:c5:85:a5:f4:39:0e:2d:f5: + 64:50:f4:b6:35:f9:63:58:d0:5d:09:01:f9:bc:99:60:dc:25: + 94:36:3b:ee:b9:9d:23:2f:52:80:9c:f1:e4:9b:5f:a4:37:c9: + 63:32:cf:ca:d6:2a:b7:3b:c8:10:54:21:ca:03:d3:ae:0e:da: + cd:08:fe:71:10:f8:db:d4:e6:cf:d2:59:9b:3d:96:4a:a8:80: + 42:69:ff:7f:4b:4b:52:42:aa:e7:e9:6e:7f:84:98:f5:13:16: + 14:b0:4e:22:a6:80:03:29:6b:2e:33:ac:05:b5:75:25:58:72: + 34:ff:ad:95:f0:52:9e:46:81:91:7b:6c:12:b1:43:af:70:06: + 03:d8:c8:cb:4a:85:f2:37 +-----BEGIN CERTIFICATE----- +MIIFiDCCA3CgAwIBAgIBBTANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCVkExITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEb +MBkGA1UEAwwSZm9vLmJhci5ob28uY2EuY29tMCAXDTE5MTExNTE5MTUyNFoYDzIy +OTMwODI5MTkxNTI0WjBXMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExITAfBgNV +BAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEYMBYGA1UEAwwPZm9vLmJhci5o +b28uY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAwz612LxzX7Tl +YKhzDoyc/ygsqbxoEouuPqp79bkts3P4nmTpjibKNhFf+XM58BlV0br0Tiur7gE7 +6/BfbrMkOcH18L1q0GxWz5YzXQVIxLWzPlWOiY5recU7mRzjA2kPdKKXe7/EER3a +18uHDZAlZCk+9mK8+aVW3lbhJ3dRGjDxiIkBwsg1QNMuLTDv1947KBVKpKm68EDw +eToW+a5SMsNSrVOclAfVm2NQkP/xjP1OWbheCnObtLdg4XwHAlB080hpanR8spZw +hhkvgkyVV6pMLzh1i5uhPn3d2r/SpKM6AhdDNQpSA/X7GqFgKMPnQetKDPRDboFk +ukGNYUCXn+JnUXwtLxdyuaAnXPzjtqbe9B40lSzFfxPEuyV2Pjs5tjbQYBcexwGc +PWWalkzYTBCFMnbHblNkgMkzGkQ5p8dp02TDTAYgVtLr2WVWAmXEunLbicQAP4n0 +ddVtg86tZvtz+I673AHAT4bBV0VoND9VHw7vgj+aJhycjYheJ6u2uVinxbA2D5m6 +2MyJQe2rJriKFhchZ7ZNg9HdU95nq3ajr/hgmSlqCk/yrTJUaTOM8sqb1lnNjGnN +P9OPBSjRKQS/st6YD51iE23+3r4txr7W+BDLtbNPraRgNrMZKSm5tDddE+c2y/n6 +f55jfvMF7p7mTf/jRqR7HxJyibYQX70CAwEAAaNaMFgwHQYDVR0OBBYEFH+dnMaG +354Hk5TvGC0KClCqH6K3MB8GA1UdIwQYMBaAFLQZCBz8ECPFMIYivMuxX63qel3x +MAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMA0GCSqGSIb3DQEBCwUAA4ICAQAxsG0l +Xo6bcwGsCLmmcI7eGP24K7stfMCEIMjSMorZyiS5deLIkUDbCk7lBb2muyKFPI6+ +02UKf83A/OuUYZEwUx1Nn07XOA+r2T2hHEjB5jw1zNtHMaayRLjbNGwoIEn/4SvN +SOF+eHoFCjE93UVRlQatXIwO/wyYd09cQtza2NMwWOQ877NkP/LiGdk2BBq0h8Ib +iV1SF/snooMtVW0fgNWn6iCwCiNND0g2rkL5/MiG9Gkw6M1SNGLuuf0SS7pNonVH +1Lay3epva6KG9cA7BgnBXzCWtnkyRbPRjArSWNM5LyG6ej6nOMyIFh51YjD9eaMd +qb3fZty59Xm8+7295fBGYNEDe1gGAPXYNqCpsC1PThtvF/DZUQwlokis4/SmUlmE +g+N538qeXCTT+VU5jD4qkT9TC9QiVcejgEEF40F9FtGvoh73+u7zp24ZZq/dIzla +M/lhPeeQ4poOjougOydV4u0JxcpxFJUQvgOOKm1IxYWl9DkOLfVkUPS2NfljWNBd +CQH5vJlg3CWUNjvuuZ0jL1KAnPHkm1+kN8ljMs/K1iq3O8gQVCHKA9OuDtrNCP5x +EPjb1ObP0lmbPZZKqIBCaf9/S0tSQqrn6W5/hJj1ExYUsE4ipoADKWsuM6wFtXUl +WHI0/62V8FKeRoGRe2wSsUOvcAYD2MjLSoXyNw== +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/client_cert_2.pem b/security/advancedtls/testdata/client_cert_2.pem new file mode 100644 index 0000000..f39cff4 --- /dev/null +++ b/security/advancedtls/testdata/client_cert_2.pem @@ -0,0 +1,122 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 6 (0x6) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=CA, O=Internet Widgits Pty Ltd, CN=foo.bar.server2.trust.com + Validity + Not Before: Jan 9 22:47:15 2020 GMT + Not After : Oct 23 22:47:15 2293 GMT + Subject: C=US, ST=CA, O=Internet Widgits Pty Ltd, CN=foo.bar.client2.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (4096 bit) + Modulus: + 00:b9:3e:c6:3b:cb:d6:77:4b:17:d4:8b:91:27:f4: + 62:01:60:8d:01:2f:0a:a8:b1:d6:e3:59:d6:25:3a: + a1:7f:2f:5d:ef:02:f9:6f:4f:72:db:75:ce:0b:22: + a2:05:7c:e0:7c:a3:d3:c8:fa:87:c0:6c:a9:47:00: + ed:52:2b:ba:95:36:36:1a:d3:59:1e:a7:30:a7:48: + 38:7f:1a:7a:3f:84:cf:83:f0:fe:60:61:9e:c0:46: + ce:44:b5:37:83:ef:14:6c:9a:ea:3b:fe:37:8a:ab: + ea:28:59:43:f0:d7:1a:a0:57:a6:5e:a7:3f:46:95: + 92:fb:44:77:68:ee:41:ca:57:1b:de:4c:80:ea:16: + b7:25:c5:b2:e5:d4:47:a7:bb:8d:f5:53:9d:a3:0e: + d0:eb:59:5e:7a:6d:8e:a1:8e:f3:b7:b1:4a:8b:f1: + 8a:01:f1:e1:14:85:dc:91:ce:25:7a:fd:db:17:b8: + 15:60:34:4b:f5:35:df:bd:22:65:b9:85:4a:7a:39: + 74:c0:88:c9:15:61:62:a8:4b:b6:ae:87:0b:2d:5f: + 2b:c6:13:c5:9c:1b:63:c0:23:73:6f:24:5e:e1:f9: + f5:ed:82:81:51:90:4a:08:7f:6e:4f:bd:27:00:b2: + b4:be:a8:0b:65:95:22:a4:c7:24:5b:07:5f:3c:66: + 55:2d:af:ec:d3:f7:ca:e6:07:44:09:6f:da:a2:f3: + c9:4b:1f:9b:d7:e0:0c:6c:a0:be:4d:4c:6c:c5:3a: + bb:0d:a1:c4:82:75:42:ba:c0:10:d2:93:a4:0e:4e: + 41:9a:c2:3c:68:ae:17:92:ec:4b:4f:ca:ef:09:7c: + b2:6d:16:31:15:31:67:78:02:0a:57:6b:60:4e:7f: + cb:0a:27:a5:cd:dd:d9:29:a5:a2:e8:d8:f5:e9:8c: + a3:16:72:9d:b9:94:3e:ef:b1:70:27:2e:16:0f:06: + f9:50:81:99:a2:aa:b2:74:d8:b9:24:0d:08:f4:ff: + 16:c1:2b:32:ad:d1:7d:c2:db:ed:e5:8c:52:26:ed: + 8c:04:af:86:9e:a1:5f:48:81:20:79:bc:57:58:25: + 89:85:02:ba:e1:5f:66:e4:4a:30:2e:6d:3b:89:2c: + 4f:e9:02:6a:e9:9e:b3:6c:7e:9d:1b:a9:37:3e:bf: + 06:ec:ce:d6:d7:6e:e3:e2:5c:2a:fd:98:dd:4d:59: + e8:43:be:44:fe:ee:0a:64:fe:fc:e3:4d:88:23:27: + 46:a7:f0:b5:80:c4:d8:2c:ad:02:a9:68:a7:d5:64: + 74:b9:14:21:68:c9:f5:3c:62:73:ed:b2:be:10:89: + 1f:d0:1d:1b:8a:ef:5e:6b:4b:08:15:25:4d:9c:b6: + f4:2a:0b + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + E0:27:7D:90:FC:81:7F:F3:EE:97:CE:65:A2:AD:D2:1E:CC:D5:2B:0F + X509v3 Authority Key Identifier: + keyid:63:88:EA:4D:D0:3E:EF:5E:F8:43:91:75:40:E4:16:AB:15:B3:32:B9 + + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + Signature Algorithm: sha256WithRSAEncryption + 8c:81:8f:65:38:2c:db:69:34:26:47:62:b7:5d:4e:67:41:c2: + 67:b2:97:72:51:84:f5:73:8e:cf:9d:0f:a2:91:1e:ec:e4:72: + 6f:08:da:26:06:c0:f0:11:fd:b8:ac:23:c7:cf:35:ac:d0:90: + e3:da:f0:8b:7b:55:16:00:5f:82:92:40:07:12:d1:ae:06:13: + c0:5d:7c:9b:64:d7:35:86:59:c3:8d:cd:b9:a8:17:03:2e:b5: + d4:8b:18:11:cf:8d:90:74:8f:12:f6:53:99:66:d8:50:b6:c6: + ef:c8:e3:bc:26:74:67:cb:6d:34:bd:c6:58:38:ef:4b:5e:56: + 80:37:2d:25:64:31:96:6e:8d:13:ff:21:63:c9:ec:8f:b6:05: + 5a:8b:b5:ae:88:50:af:00:c4:c7:9d:9b:88:a3:05:6c:63:85: + 46:1a:b1:6b:32:11:cc:0c:a6:75:44:a2:39:c6:58:c8:2a:f8: + 08:8c:9a:12:c2:49:e0:03:da:fa:f7:67:a3:7b:91:71:46:24: + 71:83:3f:a9:a0:a9:4f:e5:77:9d:a4:49:2b:0e:69:dd:47:93: + b9:4d:82:3d:f7:12:b1:02:0e:ec:4c:98:76:c2:48:81:30:68: + 7c:04:90:e7:a7:e5:0f:44:cf:48:e3:04:1b:9c:4a:0f:20:25: + ce:74:13:83:96:d8:78:69:a0:1c:e4:9e:8d:1b:0c:9f:e8:43: + 29:72:82:96:98:6e:8e:8b:0c:0e:18:4e:dd:62:e8:e9:5c:77: + 64:40:5b:c3:44:3d:21:0f:3f:ef:04:c8:83:f0:af:cc:be:9c: + b5:6b:32:c3:26:66:a0:06:bc:7b:b0:c8:54:8f:0a:d7:57:bb: + c7:d9:7a:7f:3e:61:ab:64:03:cc:32:44:a1:71:6f:9a:cc:80: + a6:e6:de:2d:8e:8a:2f:ca:bf:63:42:24:de:3f:c2:47:a4:e2: + fb:3d:6f:70:3f:6f:cb:bd:61:40:af:c9:59:75:99:39:9d:65: + e4:89:48:fc:14:1c:ad:03:fc:5f:a2:69:be:4d:a1:a3:ad:6b: + e7:f8:8d:13:64:f8:76:7d:04:af:61:f9:9c:39:68:68:99:bc: + ec:53:b9:d1:e7:f3:c2:c9:87:42:f0:26:8f:47:c3:6d:de:2a: + f5:df:b4:58:f2:1e:f5:6c:29:0b:dd:de:ea:1a:88:21:a4:d1: + bb:7f:54:c5:cd:75:71:4e:ef:d0:50:f8:ff:a2:0f:d5:02:fd: + 51:52:86:b8:30:db:4f:e0:3b:f1:91:45:72:49:df:a4:17:97: + 25:ca:12:9d:61:9d:29:2c:e4:5f:da:c7:3c:ee:4c:65:5d:2f: + 38:a6:7d:8b:52:af:af:18 +-----BEGIN CERTIFICATE----- +MIIFkzCCA3ugAwIBAgIBBjANBgkqhkiG9w0BAQsFADBhMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEi +MCAGA1UEAwwZZm9vLmJhci5zZXJ2ZXIyLnRydXN0LmNvbTAgFw0yMDAxMDkyMjQ3 +MTVaGA8yMjkzMTAyMzIyNDcxNVowWzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNB +MSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxHDAaBgNVBAMME2Zv +by5iYXIuY2xpZW50Mi5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC +AQC5PsY7y9Z3SxfUi5En9GIBYI0BLwqosdbjWdYlOqF/L13vAvlvT3Lbdc4LIqIF +fOB8o9PI+ofAbKlHAO1SK7qVNjYa01kepzCnSDh/Gno/hM+D8P5gYZ7ARs5EtTeD +7xRsmuo7/jeKq+ooWUPw1xqgV6Zepz9GlZL7RHdo7kHKVxveTIDqFrclxbLl1Een +u431U52jDtDrWV56bY6hjvO3sUqL8YoB8eEUhdyRziV6/dsXuBVgNEv1Nd+9ImW5 +hUp6OXTAiMkVYWKoS7auhwstXyvGE8WcG2PAI3NvJF7h+fXtgoFRkEoIf25PvScA +srS+qAtllSKkxyRbB188ZlUtr+zT98rmB0QJb9qi88lLH5vX4AxsoL5NTGzFOrsN +ocSCdUK6wBDSk6QOTkGawjxorheS7EtPyu8JfLJtFjEVMWd4AgpXa2BOf8sKJ6XN +3dkppaLo2PXpjKMWcp25lD7vsXAnLhYPBvlQgZmiqrJ02LkkDQj0/xbBKzKt0X3C +2+3ljFIm7YwEr4aeoV9IgSB5vFdYJYmFArrhX2bkSjAubTuJLE/pAmrpnrNsfp0b +qTc+vwbsztbXbuPiXCr9mN1NWehDvkT+7gpk/vzjTYgjJ0an8LWAxNgsrQKpaKfV +ZHS5FCFoyfU8YnPtsr4QiR/QHRuK715rSwgVJU2ctvQqCwIDAQABo1owWDAdBgNV +HQ4EFgQU4Cd9kPyBf/Pul85loq3SHszVKw8wHwYDVR0jBBgwFoAUY4jqTdA+7174 +Q5F1QOQWqxWzMrkwCQYDVR0TBAIwADALBgNVHQ8EBAMCBaAwDQYJKoZIhvcNAQEL +BQADggIBAIyBj2U4LNtpNCZHYrddTmdBwmeyl3JRhPVzjs+dD6KRHuzkcm8I2iYG +wPAR/bisI8fPNazQkOPa8It7VRYAX4KSQAcS0a4GE8BdfJtk1zWGWcONzbmoFwMu +tdSLGBHPjZB0jxL2U5lm2FC2xu/I47wmdGfLbTS9xlg470teVoA3LSVkMZZujRP/ +IWPJ7I+2BVqLta6IUK8AxMedm4ijBWxjhUYasWsyEcwMpnVEojnGWMgq+AiMmhLC +SeAD2vr3Z6N7kXFGJHGDP6mgqU/ld52kSSsOad1Hk7lNgj33ErECDuxMmHbCSIEw +aHwEkOen5Q9Ez0jjBBucSg8gJc50E4OW2HhpoBzkno0bDJ/oQylygpaYbo6LDA4Y +Tt1i6Olcd2RAW8NEPSEPP+8EyIPwr8y+nLVrMsMmZqAGvHuwyFSPCtdXu8fZen8+ +YatkA8wyRKFxb5rMgKbm3i2Oii/Kv2NCJN4/wkek4vs9b3A/b8u9YUCvyVl1mTmd +ZeSJSPwUHK0D/F+iab5NoaOta+f4jRNk+HZ9BK9h+Zw5aGiZvOxTudHn88LJh0Lw +Jo9Hw23eKvXftFjyHvVsKQvd3uoaiCGk0bt/VMXNdXFO79BQ+P+iD9UC/VFShrgw +20/gO/GRRXJJ36QXlyXKEp1hnSks5F/axzzuTGVdLzimfYtSr68Y +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/client_key_1.pem b/security/advancedtls/testdata/client_key_1.pem new file mode 100644 index 0000000..3be62fa --- /dev/null +++ b/security/advancedtls/testdata/client_key_1.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAwz612LxzX7TlYKhzDoyc/ygsqbxoEouuPqp79bkts3P4nmTp +jibKNhFf+XM58BlV0br0Tiur7gE76/BfbrMkOcH18L1q0GxWz5YzXQVIxLWzPlWO +iY5recU7mRzjA2kPdKKXe7/EER3a18uHDZAlZCk+9mK8+aVW3lbhJ3dRGjDxiIkB +wsg1QNMuLTDv1947KBVKpKm68EDweToW+a5SMsNSrVOclAfVm2NQkP/xjP1OWbhe +CnObtLdg4XwHAlB080hpanR8spZwhhkvgkyVV6pMLzh1i5uhPn3d2r/SpKM6AhdD +NQpSA/X7GqFgKMPnQetKDPRDboFkukGNYUCXn+JnUXwtLxdyuaAnXPzjtqbe9B40 +lSzFfxPEuyV2Pjs5tjbQYBcexwGcPWWalkzYTBCFMnbHblNkgMkzGkQ5p8dp02TD +TAYgVtLr2WVWAmXEunLbicQAP4n0ddVtg86tZvtz+I673AHAT4bBV0VoND9VHw7v +gj+aJhycjYheJ6u2uVinxbA2D5m62MyJQe2rJriKFhchZ7ZNg9HdU95nq3ajr/hg +mSlqCk/yrTJUaTOM8sqb1lnNjGnNP9OPBSjRKQS/st6YD51iE23+3r4txr7W+BDL +tbNPraRgNrMZKSm5tDddE+c2y/n6f55jfvMF7p7mTf/jRqR7HxJyibYQX70CAwEA +AQKCAgEAjT9s5yNOhEqmNssmkbwASEeUKCd5UxFiOUu06gvRmCWqE00F+iTt3Tes +qxZFMAHkKBqMa5EEjOavpvz6zWckKfS8LDGceLQoCX2sIvuTrVuWFN5og/NYpXue +piJTyT/UQpjt5kTRX2Ct1bgUOCe0JUYBmtXLyP9oXOmVcavMLJqD4jbb40Jb5E3i +9iaVHSJUwabFnWJ9LxqL3ee8f10xcjAEPAhlGmKgkg3DV2MSKOGIMThEMGN6nb6c +hAPqPi5erTIRsUYcgEZ9mUXXLPiigg1dmDvMLfelK0R7n6luhlTfvmt9331b4Cmw +Q4/DtTokr3e81qpPrj5F1Mlfsp+8EFd8Ucwtu37DFiwpqYpDeMYxbbVC1toa5QQy +6VRa7NQLTHXfRp6mmaf37KnganLYOqX0vF8LMIn2O11jEnfdAfqSkHY8JzvNG9OJ +71LO5FXa7VEfOGfu1lNXScFN3yqukjr2aPo8bd9hIIw4ZEvJtto+0hBBTF4ttJ+r +R5j+h764A6vqxBo8Oh60sahY7sYBD0BIZT/hmxqaEC1PUpPfveGzWcr6r/xb30ak +DhrbWsH2/St8NjCL/9u86K/KyQB8nDwOQlTC/gB+SxLCp9KcEG3HuNVMFtV/pic/ +lzqChT9p+2/F+iv/aIb69FcBuGMfljdrsnnrc9954nco8sXgmDECggEBAOPbmWLb +vnCzZ8VdbqsxlXeF9Ype9tyINE8az9rG0A15tTXtHCwYQRpTi8PWevaTYtJvzj5Z +7DnMH0B56q8p+oMyEP8YkfOQK6OavW9ehSui1y7KSgjFsFXuXPUR6BnLPUZW/BuD +UHrbjspFREWZBrm2y0tOk2sYZirqg9r+Hl1yAZXeXXkAK+UdNygRuoG3by8Nemql +wA22pLu6J0dZ04AQX4ERdxJcTxLx3wf3tpFltSWdsJr6kuevNBcdvpq/Xc9M91bW +n8POxMWIBTZTC5nDTJd9nCip1J8jACFII5evr/L+O3Bwda4k/B277D/DVUtKhhcD +UBucDcLXQro6eHMCggEBANtb9ZBYw3R/JbZrNwShB379I5p2SN30mSbkNB+0PYYx +WNX5YQADFlulG5/spPD15dyHdYWDWI+c40ZXAKfgN12it6id+eUC5hbx4+N9E6yP +4+9mkvPiV2HpIOLSb/fDReJsE/d6l0Fwqh2xGCN6adLSa3DCy4q9IP9pIJHOAkeY +kdBQwtXH0xo9hM25/ZFnWGmhugRvllB2rPCEPFxhsuS1ExgEPLm1T8DnueQTGuEf +lAXUj/s+RVcGgHgQ/ONv9O6uEhmZYST+ZFu3sb2Rq6YwNUjbIiaMuzMgEHAdQ4C4 +xYQDC0Bnf3Lt1iszvypwAxjPBcHhVeqzTL4l1sn4Kw8CggEAEK59Fk28LYgU6tAi +UAo7RRrblRvKuu6F1dzCpuOzS6lDaQVI8Ll92q2PJ/FF41N7AqkI0mvG7ZxSFWhX +lCdgncZGlEZ6OPivGTU09ThYS4+KbXSF4wqGFGR1DcQX1/uXKtUnc+QzOitk0s4r +Z2UCpwoI7CR+inKo2C9/I8NC+dhk4VH8SeWHUSjIZviVTPXe//Tep3wnCVn7yXqh +cYnUACYyt8JNk1yKtXpbt7uc9BwcHPrkeRQrOScMizy0PaQQ/CJIYWUpIS68HTIO +H6II0WMI8nZRvnBgjp4DXmxnnq1QFlwigeLZ2rv+cTbW3vwv/GkiVAD8FmlgYIld +60BonQKCAQBAP2fmFklxBoiKLE7Z+TwT0pqp8/kVoT12KaKmoojek/d7/GWPtlfH +Ec3Mgmgw9ySS+c3PBBBdR8s9X+AeS0qMD0uRhGubysSPdduUVp77jM1q4fUqn2GO +mNR7+ry2qaf/UD5s3qgMj64TsjnqskDqcZzsUvGAujI+/JCAhAEg7SvQAsd+C9/l +sJ0EEHSXMNixX5/3CqPQ/2FZtLFlMWxPFkX4Y81RayxnyLcmeP4Hb9NP/dkJ8kwm +2A2qnPckujbX7X35p3XPev7z6hKR/mdy7m284AnZlqCBseN+ouORgQzAxI94Fpg6 +ljSDRM255ULS8leyWIhsjIVur/CACUK7AoIBAHRqbtOLnfrDS8VlI+V0GtNJXLVS +XDlgTtPNMaDWMKxVLFNwF8MeY5pf1QHNa399bOumZUlmRfZ+AcJYDTyfF366Eoh6 +yatmoQKMJotsQWln9iGWv7wqTP7omrL+Y053R1ypdY4k/4Yf9ptykiCBIUwYqjxk ++NvIcf8r0cZZjsx7SlkjhGGhFHkeFewhbPm7o8bolZ26Nf/luNGuJSOzGac88Sq5 +9jSKbkWTI4Rukw3n73AAKkdbLmGkIw81BnMbXH3bBoB+fdmILgIFx61D1QeCipOQ +WJIht2SLm8UXfYAQLGL2kQ2+C531uFvV+hzNA1H5KHj1Lo4BD2ogjjePdFY= +-----END RSA PRIVATE KEY----- diff --git a/security/advancedtls/testdata/client_key_2.pem b/security/advancedtls/testdata/client_key_2.pem new file mode 100644 index 0000000..c56b1c1 --- /dev/null +++ b/security/advancedtls/testdata/client_key_2.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAuT7GO8vWd0sX1IuRJ/RiAWCNAS8KqLHW41nWJTqhfy9d7wL5 +b09y23XOCyKiBXzgfKPTyPqHwGypRwDtUiu6lTY2GtNZHqcwp0g4fxp6P4TPg/D+ +YGGewEbORLU3g+8UbJrqO/43iqvqKFlD8NcaoFemXqc/RpWS+0R3aO5Bylcb3kyA +6ha3JcWy5dRHp7uN9VOdow7Q61leem2OoY7zt7FKi/GKAfHhFIXckc4lev3bF7gV +YDRL9TXfvSJluYVKejl0wIjJFWFiqEu2rocLLV8rxhPFnBtjwCNzbyRe4fn17YKB +UZBKCH9uT70nALK0vqgLZZUipMckWwdfPGZVLa/s0/fK5gdECW/aovPJSx+b1+AM +bKC+TUxsxTq7DaHEgnVCusAQ0pOkDk5BmsI8aK4XkuxLT8rvCXyybRYxFTFneAIK +V2tgTn/LCielzd3ZKaWi6Nj16YyjFnKduZQ+77FwJy4WDwb5UIGZoqqydNi5JA0I +9P8WwSsyrdF9wtvt5YxSJu2MBK+GnqFfSIEgebxXWCWJhQK64V9m5EowLm07iSxP +6QJq6Z6zbH6dG6k3Pr8G7M7W127j4lwq/ZjdTVnoQ75E/u4KZP78402IIydGp/C1 +gMTYLK0CqWin1WR0uRQhaMn1PGJz7bK+EIkf0B0biu9ea0sIFSVNnLb0KgsCAwEA +AQKCAgBtWJWxJFBzWFs3ti630/Sp9XEmOrti+p7q0tOqZCKCLdaXyDyurMoSq0Y1 +onrbHGxyhk30O5Y4SqvdYrmzoGZhv39OdGUNyAjbJbFbrahtqBrKOk4dXGJWAzWs +rv+XHGAE/6i2QwhMDdCJgq+tEXwBG9vz0WtzYcVCFpcZ1FH3e1XS8XvDMidn33wL +WDP32akhH/tUDeHamoU/ZT4lNXm9e6SSWMBrB3kiISYi1vme0QwrwxizEguoMeXh +AdXkHb7pyNKW9+cifLq8tvydps89OAlhwbgKvswx1XtFJsXvRBob2cY1/CMHQxk9 +bl0Ad3xjclRP4Sly9K4MIZzgzVMHRCstG1K60cDVK5GeiBkXHKfihgXIIk5iILjH +jplpTx54KEtC+NTd0/i9DsK6/DKcATt+AAPgjoEy2giSgfTpZqyMgLgIAvYKgrYF +SME7jm4rFe950VpR7vBVtBtXKnea39/75uwbTAjL6kpqvDARM7MWb4R25voOmlo+ +6Jzw4VyktVb/p7HLq0ayONGGBIF3H3P+wnvhulHR1I/OHhNwnYsH5mFju7t5qO3H +ot/DxLOTmV8PkrHgfGwvbmwF5E66dpv4m5oCYHn8SiCEsXF1PkVrnSE1yeuzq681 +tAnaLPRO2UXlpe4I1CY45a/WTPoXCfxJdtjjLchY10bZXV+dEQKCAQEA6eSN8as7 +aJa7ljqh4Qf9LkD2lDzvYlyxzuwIh5d4+4YoctaiZdHWxXMeI6xPTrS2gTQVyCW/ +9edq9822Xo6ti4RK2yab1ewAcBDEBpDdTrcQZ9k9f5HVrCXyEuajKBCN0j5uGsPQ +cwv415xyfj/fudH/xj+FwstBnc6YDxHGC6SdhXghhLCfAJJROneu7W8eQuqt8tKo +eOGheiTo/WPGkNOPu6BXW5/lxMXXCPsqPJS6MBAphFDkCp+deXw4xjL4sKyqRWFY +HFH17tzPiyCPdOEnuytFJcrK7+0svACdwYbypbJpHSvjWmPwoB9+58mFODF4Lvub +ZD5VviRyDerf6QKCAQEAysEbRyEFquN+6PPhiS9wYdjHHiJXBtfmXf2fKBCRrLJ3 +y+/qPaViyEBgb7mKblaFBluitKevg7Oge6VY22moMRTR8L9zU6mKPjt2OiHmwsB+ +L0+8Z1wTO7knBJq8dwCc8Y1gpU+fWGoz5vYAWDX03yJeLsW9OG/pKm0tAFEY4GxJ +qVIz2NRjBc6ojisWN+QTonxQXkevaXw0sIL7Ol1pW0zQIXVkrzjvxV1KfdXwhXLI +jdxs5NrVOGNLCtrW8+vLBTbCuOWSJIzJOEMUH6UYhQCXLM5T+snEL3S0U46yqHOG +FcepRU2ncsHEz5eMN+JA8N6/ZVv2eIXfub/59dOV0wKCAQEAk3nsUmRgijr4vunr +ZkOuTTri/2dInaHK76j+W9iTjSzzVi2lqkPcgxVp/J5KR1tE9ETOMywyVK/9T5Cj +HA4kuSLKPFKk0gcD46V+pJE1KcveCUz+LPDcZLZsY6SPXdTKR7XboP608cWruu/H +dXl67OTPvMYS5ldY4VMBqAbR9Edwl1a+87aWGzsnApGyd72nvBPTaJeRaN8D/UtG +qXb/HhR3vZuFWZ2BuEfypZQQ9q/kkieuteJ3V4d7OL2t4rMDAgttNWACuaCoTFto +ddYq/kx1y9ultwWeXhgTK9vLnNolJ3tOMfmZWkZH0/7n+uijGmJ+4Ej/mv5+++xp +CgN9+QKCAQAstfDB+rI5QPmXfVBa5C8wJJGkP4ZZZ/rQ90DFoQG+x4xLWJibB4GF +D0001gGE22dyQ3rZw7CcplvZaFjz6ZTBXgn9wPo5lMV7e7lSkG9GuxQYcsjlMhS7 +stS72zN8OpJhYf/R9ID7ClBvugfRa/SX0Ahc4BYd/++2/2RREZEezEJiKFJumkdL +3Iqm7zFzGcSKrEc8wyoXZOBpnDiyYi79hy7OcgjF6xRUvYHTxf3IL8uyHM2Wmfsy ++BJwTlngaDrY536BL37OuI0W7xPc9pc1nS+5Hba/MwckP+QUGP+kzfTfkKvvMHSg +hcJU1OKC4E3Z0AT84Q60/TCc0YzZfNMpAoIBAA3Bb4lau9KWjVMza2fLdLmPqMM0 +MSCU7jo+xGH49YgET+lGFy00lbdIENMP0nv8pr7IKFy3pbMsZRHG53VPylUXSvdE +UJdW+7X/d5G8VVDypypgtSptD96kAU/ctq/Ty7uZw621vvTMuwokRTsL5ipE24ys +aA7M5GrMer9wrp3q7RNz64MVrnqJEFc4waFn9W7ZWG2i/upTj1oFcFF57QJz793m +KnFy7cOApEBahRFIkW3AuVdg0pJuYTRsrvfjYvFD5eKEON4qSXPxAgRl2zLR8i0x +jbKCySBaSFSYrnWs9Tt4QEiEYLGNe1WoCfxaUHCvM+d50GiZeJQkXCT3m80= +-----END RSA PRIVATE KEY----- diff --git a/security/advancedtls/testdata/client_trust_cert_1.pem b/security/advancedtls/testdata/client_trust_cert_1.pem new file mode 100644 index 0000000..26b6112 --- /dev/null +++ b/security/advancedtls/testdata/client_trust_cert_1.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFeTCCA2GgAwIBAgIURfVPAG6lOcTq29Ht/6KNbaKhm7IwDQYJKoZIhvcNAQEL +BQAwSzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxITAf +BgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAgFw0xOTExMDQyMTQyNTFa +GA8yMjI1MDMwOTIxNDI1MVowSzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQww +CgYDVQQHDANTVkwxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDCC +AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAL95INSZmsn3AgGX2Z9bu+6T +7n9VevXuD5bv/oc/Wpfb3d2rueBizpo1xRWMiNFq59wf+6o80LlRsJwqRCnMo7AU +FrR9zaryGzKa7mqkGnrX0oPIe/KP6sw+nUNtpr7m3PMpi3CkqTtZ1Oo1PhphyEu/ +pTF2mmS/Blaez3BKFuX/zMLbnxedUgKD2ok8VqOPUGr8uQ4IPp3eZL9FptQ8Tt7N +7H4U/Wng4YNvCBDSlLP60NGycJ8yrdL1aJSWAHJ1vDbYlSo3JeO4cjHdGUq6HTx0 +7USgg6ZX2OSEiE/DXRQbu7QkGjetssaURmUjUB3vbFCqmWZ9HDtkE3YYhk2vks6H +PQXlunNHUS7Ain+IgYsqK9cNRLWqcBbdo2IEKYYwAaK0xsQox/m4TuWacHMZw4Tg +Zh2Y984n6Hyq1H5FgWMYpng45VihT/iKZpD0r0vUBsDJQSQzsQkHjIJe6333gtey +8nWXm/dcRUZotcL+eJ6essniJ0ZBFz2m2DB/BKJ/5rA3hf1uQCPAdaLCho0QVUeE +gQShwTiP0og/0V6dHhqoDjnEnII9ZItGVn0NTl688a9VpzPGyCDcNtTuB089KtLs +UcE0vLtEmhlM0NI3CpP+ahQfxmF6i37VEzBoEzZfyzeaO2MrvmH7djfP3HYmx85M +yutAMo9NSpGWOiPYi/lFAgMBAAGjUzBRMB0GA1UdDgQWBBRapdqxmdTlDuYelOr/ +/GLi7QnxBjAfBgNVHSMEGDAWgBRapdqxmdTlDuYelOr//GLi7QnxBjAPBgNVHRMB +Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBpeytWYsMSa3sTZD1pA39IvQ4v +xAbzmJB6P86sjmt2hNkCZTJYnOdSnwr8mrsKnLgjf/6kpg2OO3RqN+aXUnDeCpjl +9mzDJ4kcoVxbOXYAhTkhgugT/b8HOlMEvC+3yULrKN9deUmQPCdyVOtMBnTHE3++ +VyAlP7T7R2xbgeVpOurO8SM7NLt04xxbtOS1cPVkhU2AE0+KlQBQYK9bProHTynR +zhVixFWtyS9asrMUwGh/xye85xTpy+unUxkzZoPqYvK6YiKv/WB3U3SURuLcOqAD +T/BMpOUwbYAP0KVEL5K82uo7csADLMwBkPvFhDsMaFGTkUdb014NQqRjTXOOK8ad +Xwnm4ur4GgT0Rr/iJ990JTOQkWYWlW3ZO6DSiUZCqeRWWhju290+aviOcrJeXS0V +XKkeJiYjbdnvFp/LIcg+V+n/HCDwwQgC3vQqlwd8PNvl0gKRX4EGjV+1lobZoKvD +WdIuSIIUkIDbv547n2ldp3GhJHIft6jlTOLAd3jonURO2/lZVyxj8yGFPbTWRUa0 +aK7IWkcOzof0+v2BrEhQQoL+lwJahqYEPSKw6WNehQxYWaxr3TL/xeawFzEVW1ve +v8Vh1LvZ/qyucpP3dgDuj7gpVg0xshKpKEbGwzPKMz9PGcHJvgF1GOXVRIdoB9nU +IdXOcawI6rpqTXrTgA== +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/client_trust_cert_2.pem b/security/advancedtls/testdata/client_trust_cert_2.pem new file mode 100644 index 0000000..01bb113 --- /dev/null +++ b/security/advancedtls/testdata/client_trust_cert_2.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFpTCCA42gAwIBAgIURc12C7/2O090oCXCOxpatu7h4m8wDQYJKoZIhvcNAQEL +BQAwYTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMSEwHwYDVQQKDBhJbnRlcm5l +dCBXaWRnaXRzIFB0eSBMdGQxIjAgBgNVBAMMGWZvby5iYXIuY2xpZW50Mi50cnVz +dC5jb20wIBcNMjAwMTA5MjI0OTU1WhgPMjIyNTA1MTQyMjQ5NTVaMGExCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJDQTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ +dHkgTHRkMSIwIAYDVQQDDBlmb28uYmFyLmNsaWVudDIudHJ1c3QuY29tMIICIjAN +BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxWqK9pdKrJUupCKUCcaVGWLq1wh6 +9I8YhLuWv73R15FbImFeh3085o009a4symIMSgfn8iPrN97WAMn4OSxAmJfSs0FA +DBmfpf0JOloFM5GHYVgGpdkFCiEjnJ+eTvIxRwsvbeT9EsLkeOVb8syr9bp/w5ZW +oh5Su9b7pwpAanQx67dxq2lCndVjxZLKgAXO23m70xoFOKwVaynxcUdnYVskFy30 +SRhB9h0w7I0L1pb5F1BTrsMgBLtrg81JCQzdmgoTKnn8AHDnA+rwe7ushXE3eCrm +Uxj4n2OYc2siSXBQFyUa/x18kgubS/FPJNHYFPqnyw+g+yk9hraq3OQ8XwHA03eg +1TZkttQwfUV3g2gywDaC6e2PGl2q8+h1g/7kaSu9yiihlMfgQoa7cmC+j1MKAgki +FEkyuQtYGx08rAKL/Gllmgm0VxT9jO33YnuZBqDbfnF3PYGBo4ZW9ERyJDguPTI6 +6Ms68uO/B10mNePwOunlKwJxnYZkDnGcqVZpm0RCt5IFWIk+b0ek1OhpzEeGmQp+ +xLWzC+O62WVmW5B2aKmJ/jV4MUOA9HFELrbh0kS+Odp1ANgFr0UKQK1O04Hex+7O +3rnHHzeAjHk8SzZRENKFp0Srf5L9GpDb4/FDmNM1XWw2g12R7nD69dNvC6OCiRvi +8TQxRAMYqSU8XKcCAwEAAaNTMFEwHQYDVR0OBBYEFAF0qURhPXq7wjLN0O0g2jrE +xgLoMB8GA1UdIwQYMBaAFAF0qURhPXq7wjLN0O0g2jrExgLoMA8GA1UdEwEB/wQF +MAMBAf8wDQYJKoZIhvcNAQELBQADggIBABHATLUgMaHJwT5Rc5P/vPeIu09zZyK5 +avol+tSGbMmcWAUK9gYlivyqcPzeJ6m5+GJ2WkfumdhkUY7XclddxEGyw6q/eRE6 +nirt84TFlc2QleSFFg84lwTLT6wE6Ym9+qC3C2b0nOgUeGl5J9itoYqDTOp5gF7Q +Ileh2+9aZSnbaR9W3QgRteTIq+9cVnBZExwgrLa6/Iam0x1ERtd/U94prO57D6mE +Wspvj3wfn7oUfTsTGuBjq20xjmQEGxMF+zgMTJGgkOUxwIGrhXWlK80GX6ff9tJJ +3WQ1lBG2BE1eB3NWLuyQjtO0Jl9bfrpz5sUyXMWyGD9bOz/qFLLdi1AxPAu4qIWt +j8avS4DavUtU3LJarW2IVIrVVSs+hg+mrzMpjso0/8QI7kG5hV4vvD6bOxMZzoBW +g6M9+eXYsp03HjNI34Je/w5tcUY90Jfk3mVxz1hTRh1Hj5EhtSlmwxLdBgRe1fdM +Y3gsHP/OFk7MpMFWZQmxZhsfrV1Nfh1XeznKuUCx0EaGPuZcjKeqUroYvlSWKLl9 +F2VfCIo0hKE1VZ9G1QxVuB65N+sdgotyj45LCn51HV1unYqY7Lsnmvbyxgevz1Sv +X9kF21BV+lBLQq8aQGyGwk2RfUVlVp2cKvWHqVT+qF9QgW66Dt1gU7+m9qC4jCTO +2OGZ/CvtfsXA +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/client_trust_key_1.pem b/security/advancedtls/testdata/client_trust_key_1.pem new file mode 100644 index 0000000..237f075 --- /dev/null +++ b/security/advancedtls/testdata/client_trust_key_1.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC/eSDUmZrJ9wIB +l9mfW7vuk+5/VXr17g+W7/6HP1qX293dq7ngYs6aNcUVjIjRaufcH/uqPNC5UbCc +KkQpzKOwFBa0fc2q8hsymu5qpBp619KDyHvyj+rMPp1Dbaa+5tzzKYtwpKk7WdTq +NT4aYchLv6UxdppkvwZWns9wShbl/8zC258XnVICg9qJPFajj1Bq/LkOCD6d3mS/ +RabUPE7ezex+FP1p4OGDbwgQ0pSz+tDRsnCfMq3S9WiUlgBydbw22JUqNyXjuHIx +3RlKuh08dO1EoIOmV9jkhIhPw10UG7u0JBo3rbLGlEZlI1Ad72xQqplmfRw7ZBN2 +GIZNr5LOhz0F5bpzR1EuwIp/iIGLKivXDUS1qnAW3aNiBCmGMAGitMbEKMf5uE7l +mnBzGcOE4GYdmPfOJ+h8qtR+RYFjGKZ4OOVYoU/4imaQ9K9L1AbAyUEkM7EJB4yC +Xut994LXsvJ1l5v3XEVGaLXC/nienrLJ4idGQRc9ptgwfwSif+awN4X9bkAjwHWi +woaNEFVHhIEEocE4j9KIP9FenR4aqA45xJyCPWSLRlZ9DU5evPGvVaczxsgg3DbU +7gdPPSrS7FHBNLy7RJoZTNDSNwqT/moUH8Zheot+1RMwaBM2X8s3mjtjK75h+3Y3 +z9x2JsfOTMrrQDKPTUqRljoj2Iv5RQIDAQABAoICAC+ehV66cPenudUBmfr7Cos0 +OU1rye/d6/yi5U9nnzVDVjNqIQlAKZfKpaBNWj2S8+UYAzP8egCM43qDPH6UyWTi +Kh9rZjoMil0UkRTuiTNh95YUx1a1GjT/oYcCf0TdD7hd7bLvELOVDNHOugo/pVvJ +ZuEdWRqTM5VZW8fWdUlwS9FuY2uxEZNUjYYx/m4hF2P0RGXMAR6sD6xOO0ZvVUIu +PpHA0KGDbzKL65qbdKYqS8LLOR0usnJT3FWP1L6ir1OIm9hq7L5sweHK1h5ymRDP +F69IqFU3Zda3a1tDACQfHZiYnfiY92xRtgwzMxquz+Zj91C47suKgRiO0uABOWY7 +rRCE4aVihSH8hjW2U9tRJBZdpyTlk5wyfoBlVGHOHXhHR0LIBJHcffB3Zwm8BkGd +OR/+4b8yqBBDGC/Bt9dIxM0QdLgmdWO0oywXircEzv6O+l3LGeQg2d7dkWKmMGxi +chnVJVq/txuZVw2+nifI2NueOlc28dIy+GkQqXFVVFgqLCNd7K7wfZ6OVHpb0qx1 +fXYtk1Vsx/3YgVKcbHpOKxiJK2xFVtIepooTSHuohZEX+kVtSvh664bmUJ0eZpdN +lkKUhgRfFLtXS6eBPlocZFzWJKUJQ+0b1l4W9G73m68XbByH9dUEe0K1i1ERXcp3 +RsSKmouSK04RKbEmyKXhAoIBAQDjGdJvFYRIpHFgsX3cZXC6t6OwSybAM+9g7LOR +jfDZasYs9Tonh2y2MhGKqjKdGQ513Ni1WRzuGrItuyrru+PCLCbattw/GNAlkIZ4 +Kqfuex8Ys9TqeW3qfBnIbpc4Sgcrjke5nqTdksoYVMM94alZVLOpsaOvdUwa2keS +MgSwHh6qNVw2/Vz20i8RTzJg3fkhdEQG+atLMl2+J+LhFRlMEACnEJM29UNcuQ3N +TinMRwiSSSCwQpTGMXi61tuJ2m+6cdmHUChX8QoYh9jZLBtjxwEAkGAvzZQeGgxc +bafofHey/ZoZg+DSXgwlKJffmc0huswELB5CiJx43Q5JBj65AoIBAQDX1q0MVxaK +iGsb9g9QE5iO+HD31TBp15rZgyJtUXJWHSFIIs2yCfD2nlxDTKp1KIdLCHj0vnca +/9VGy2h2MNnoU665JCg9wJI9Tgbs7raIM5tQMaJxysGhP/jkY1v0l2fc58+TxgFy +xzqbUeti2t3aQUISWzGukDlwTrQWW7/2DK4JE8pBhDI3n0n6A+eZaeNwWr2ChUlB +1syO5mQpvltM3IWJ/B5CHi5dzRupslnFkIGTzXFhf4kCXzxJb/JLY4XLHSxhiWWg +GvjObbb2FTPgYc+HanpDxM5eRW2oH0hyJUoKR4IrxvvSwGJQD2FejZzRhEI0/L94 +D59Ri1nALyjtAoIBAQC8878ircRirG+pBAS0W7JvqFuJUv3q7Us+WbMOaAr82toI +jgDU4tiQvxfZR8LU8wQVDKtCN+LaOVwGsLQFb08RP6sUTxDxbrPAjX9UfCk9QzOc +WgPNEztg3eCV423uZ6mPk9IZnuWNdZSwqdXIpvlAWjkh96s5UV8A+JyUBwnffzAE +bmFLX4L52edPf5VrA0VFkHcJVrIu3rkgfg9HN0bVAnuIhUH3eBmUDGRvbZlZXcDD +9hQ8kyk1vfO1gQ8oo5ZSimdzLj5i7Sp5Po4uI4Smf+1Visp8+49BfGrMfHA3/1eY +lWih0hg88AMq55t1b4I9ji4xSoPi18dYyJQaLhgBAoIBAQDMWsNpJaN/8n2G8ce5 +x3PwGaXL4Jt/+tTwEEquOikJA3eZdupOIT92IKW2SoYxevftwM3U2+ilNYhXCQuU +q9gFMgYB4QwAu606QgAooDNObZ4lpXjqSFBgPdOHWdOclyWNcCWHAjgo1hzVJhC5 +fgQDOzo1awZ1ArR/cuTrLl9ntMWqboRW17U8GKLQBpZnGGxw2lkHlO6xWZA/1D8N +jt+evEPrSzvS2gSIZ0RDvUtl1NX6fM9WwouUJVtNJKLBYi8xCiQVDSOdHSxpNlO+ +VoDRd4on6lZsh4/kjdOvFD9hY5Dgfqfuju2qst/icU19WpMZhCGzTYJzSEdNy6Rk +Y8JZAoIBAGFKZq43NwicIrBTIUKgvLntuNtvkCgBp9awGMptRPqVOkOSkFJLxL6v +pvSjQLLsvoHmgw9DHFYi9D5bWmdNIV/8rPch8XiNyBmjitAMq5siL6cZmswpjwIN +V81q7zt5bRvVJWGXL4JrfUL79bWlzPRBB+jYn2ktsdoz+vQR9tj5ohrOkjwnLSwj +bqhTawwMey4q5LeZPyegkEojx5U/pp/spisT16v9dkGbxgLc7wcmT/7vU2IWY+Es +7WX5FhV0jmj4zESGD5CNtBxkTyBmKJYSXxLZ4ZjS8v3Ua8DkQUdlD73STVK9Lxdp ++xZ1BJ0Xfq/t2SnXDABwi9hvqTNOGqY= +-----END PRIVATE KEY----- diff --git a/security/advancedtls/testdata/client_trust_key_2.pem b/security/advancedtls/testdata/client_trust_key_2.pem new file mode 100644 index 0000000..2a8af36 --- /dev/null +++ b/security/advancedtls/testdata/client_trust_key_2.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDFaor2l0qslS6k +IpQJxpUZYurXCHr0jxiEu5a/vdHXkVsiYV6HfTzmjTT1rizKYgxKB+fyI+s33tYA +yfg5LECYl9KzQUAMGZ+l/Qk6WgUzkYdhWAal2QUKISOcn55O8jFHCy9t5P0SwuR4 +5VvyzKv1un/DllaiHlK71vunCkBqdDHrt3GraUKd1WPFksqABc7bebvTGgU4rBVr +KfFxR2dhWyQXLfRJGEH2HTDsjQvWlvkXUFOuwyAEu2uDzUkJDN2aChMqefwAcOcD +6vB7u6yFcTd4KuZTGPifY5hzayJJcFAXJRr/HXySC5tL8U8k0dgU+qfLD6D7KT2G +tqrc5DxfAcDTd6DVNmS21DB9RXeDaDLANoLp7Y8aXarz6HWD/uRpK73KKKGUx+BC +hrtyYL6PUwoCCSIUSTK5C1gbHTysAov8aWWaCbRXFP2M7fdie5kGoNt+cXc9gYGj +hlb0RHIkOC49Mjroyzry478HXSY14/A66eUrAnGdhmQOcZypVmmbREK3kgVYiT5v +R6TU6GnMR4aZCn7EtbML47rZZWZbkHZoqYn+NXgxQ4D0cUQutuHSRL452nUA2AWv +RQpArU7Tgd7H7s7euccfN4CMeTxLNlEQ0oWnRKt/kv0akNvj8UOY0zVdbDaDXZHu +cPr1028Lo4KJG+LxNDFEAxipJTxcpwIDAQABAoICAG9UwV+FPKCNVQtNUM0eh3EU +nrl719NZa4tXOxGQ2+lE2O9Pl/6yuwiN86Llge70Ulfhk4WzifAtI+S4AdtEQH2N +iU576sGoJad3Rp/4qlxFouJbwQwAkl3/CFVIkv+UiAO3pBzGeY3+CNjBCBSqJgPj +FDBZ9StiDGhQOgUeu+sM8iYrgtgW+XGHKMgAG2ENZXXSdgD7+JvYOBACTF4E1aFK +w9Sqnswl+PTxy2hrtpRi+cCTFU5GTiU9CMoAmEKZVdOMAPkAaARbp3xHHy24TffH +PG/xSYjtWTCR+ySD84cU5qXW0B21JE48a2ztfiOWj9Rs8vmKK8/YlxEErOD7eatg +v1e47Ygv8JBLhEvk38HQYv3EdsV/2jAXg7K3d1s4znyiJdHg2ujCuTiR98auDh5S +Er3yFG38KKagw9I7yli/S5B7RKbhjHIunBfCA2W6cJVyA7smBllQ3YraVZWWWKIX +Z9UeZrA+KoBssg16c2Pwyg0X8HuDN39n9YwTFqj2VCrap71NYNn9G0q40aI7duaA +Ehl/NOBPyBMnXbnocj+0QkuKwW/i4wMRKREkzTGRHI1fXy0/LRO4Adc3ZUXaOxZx +aIM/BnNhuifk7rBk8VHAngWxRj3vfVP4lgqmizczHQ5hHO15Tb6Rhng3LfqeDJjZ +NOgdYMNm7epr5OsMjgApAoIBAQDrVAqnLm8jkBJHyrM577RVqCrPOUsndVZ86lg+ +cN4oyg6CWyNWJyKBYHpEyAx7d6qSOyRwMZfXlupXJga/sUQzvRdS96jbcBRMLXfN +ObHFRbgFF4xIuvqhUagzrMhtRPchh4dOQND8mpzRQoAvKryJrlm1o62AK1v/94a8 +K4Tbtpogfc/si2RimHeNc5dilBiNRhrewA4xXYvZ2xhNBfHD1AP8O3wSsmd3aI9J +PAqLaDCFuA+h8qa0qQmQC1Rehf031PEHGWmluEGuxfA6eeCQha5fzMPj7qWIN5RL +X7oGji+dj+pyKfKGbOnNTNJzHi7ppnh2R2saf19+j7joadGtAoIBAQDWwfNZPQkS +5tEHzDeEyG+oWBn9OxMaBoJ1VEuZNrcjSbqgDxwcyczUTO6dpINz2ve7Dv3s0V8T +75YI16jorpT6iJr3oD+6F28PD3jghgCTtEJoFbojdffBXXTvGU1UwtJ5eTTBpKRe +mOuxNL8dhMqCnmDVZ+4DQSWQ8h29xshVuymnlADfSqZC/zYLjLZrPFj2Lv/QVsvt +7V+D4UFlNI9aEYgnlsMa5A7MfTr7M1cEDhfUz7QpUufZRzGvVx4gk98lF6AzuvRI +tdcpOJUAowU8XchtI8x5NubtF04e0lpmlQMKhq8eZ7+URmYwZIROim4KV2eYL0M/ +PB4Jl6otwbojAoIBAQDDABb7xaxuiZm8R6kQHyMNv5YJtO4jukV6qS2KQDi3EAfJ +2P+FClS7ZFis2iANx3FeTwe4uD+cc/+nS2lYOunK/atwIqyXeV44aYzWUDKQx17f +SU4DjnzUZDe+6jQC55zo+ccS/v6t8uhzNmnFq+IjLIhFzWWdyVAo4NGS53TmI3+/ +4MEEv9TlJnYajmgpVZKqribh4b9hBKU4Vybh3EUkAnFy90+upoq6Fbh19Py/3Awp +IgZCKjIdjdzQsbKtyNW1CAzZ1yMGIZK74mVX71o4J64A0Eqae0xLfdKySpZ5jCTE +qVaaV0wSO/nZFwlkPuSc1EcJq9CCWn2lAC8210jZAoIBAEI/uIsp2fe7vnXyWJoc +nt1GuFW2+JCJu4roQx3zlBFNuEWSA7EZy5ceWGnHC0odHVjWKhz5BaSHvzfhF1kY +KhsTMwL6q04D1p3Fvxs8G0d1Txr+wNoZlSFQbDcqDgH8y6Lvcgfee1o3QFX9GIvJ +oBMlOmf61KCqYyVQmz4k6T4RK6tna9F2HM4EHq73bHquNh9TplSlwekW1eVAAsVu +rl4xlFfqGSvdeHc6loxRbSFyG4XpwQESczVC0h/t9vxDwY2WuTPcE2mutr4fl0+H ++qCBqceJSJWICzrOeqnlaD/G7hY8MB9oD+B0yydYirwT1hhYmDuJMOx75iQ9ZiER +ZxMCggEAUenerHVg6/+T0IwSeWPjR3GtJ+SWij44n99ojhq67rXaJ2jHuMaC0t4N ++VsspSISO71PuOgjQNjdN8xn8QaYBcLt8HMAcZFLJnDnbhfJ4iNbToIWhKqwjtKW +8eMeNziz9kE9jazOt8l9ErRiXmxZ7P7P4fnARtX0+X2TU2r1pYFt21Mj6yrqVkj5 +d4EMIl8NrHxoHhdGXN78yI6eoAxBwanLdILVw51PHShXODiCnA22lzzialsVxp09 +wV0LJJv2AsnVHxFdYCVZjDKG6WDL/U8PgDgsznhkCOuvzLPUtM2rAxgq//QtJygY +QBqTUW3bGnskKC0gOUqWO3Kd9zCnbA== +-----END PRIVATE KEY----- diff --git a/security/advancedtls/testdata/crl/0b35a562.r0 b/security/advancedtls/testdata/crl/0b35a562.r0 new file mode 120000 index 0000000..1a84eab --- /dev/null +++ b/security/advancedtls/testdata/crl/0b35a562.r0 @@ -0,0 +1 @@ +5.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/0b35a562.r1 b/security/advancedtls/testdata/crl/0b35a562.r1 new file mode 120000 index 0000000..6e6f109 --- /dev/null +++ b/security/advancedtls/testdata/crl/0b35a562.r1 @@ -0,0 +1 @@ +1.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/1.crl b/security/advancedtls/testdata/crl/1.crl new file mode 100644 index 0000000..5b12ded --- /dev/null +++ b/security/advancedtls/testdata/crl/1.crl @@ -0,0 +1,10 @@ +-----BEGIN X509 CRL----- +MIIBYDCCAQYCAQEwCgYIKoZIzj0EAwIwgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpH +b29nbGUgTExDMSYwEQYDVQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNs +bjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAyMS0wMi0wMlQwNzozMDozNi0wODowMCkX +DTIxMDIwMjE1MzAzNloXDTIxMDIwOTE1MzAzNlqgLzAtMB8GA1UdIwQYMBaAFPQN +tnCIBcG4ReQgoVi0kPgTROseMAoGA1UdFAQDAgEAMAoGCCqGSM49BAMCA0gAMEUC +IQDB9WEPBPHEo5xjCv8CT9okockJJnkLDOus6FypVLqj5QIgYw9/PYLwb41/Uc+4 +LLTAsfdDWh7xBJmqvVQglMoJOEc= +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/1ab871c8.r0 b/security/advancedtls/testdata/crl/1ab871c8.r0 new file mode 120000 index 0000000..f2cd877 --- /dev/null +++ b/security/advancedtls/testdata/crl/1ab871c8.r0 @@ -0,0 +1 @@ +2.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/2.crl b/security/advancedtls/testdata/crl/2.crl new file mode 100644 index 0000000..5ca9afd --- /dev/null +++ b/security/advancedtls/testdata/crl/2.crl @@ -0,0 +1,10 @@ +-----BEGIN X509 CRL----- +MIIBYDCCAQYCAQEwCgYIKoZIzj0EAwIwgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpH +b29nbGUgTExDMSYwEQYDVQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNs +bjEsMCoGA1UEAxMjbm9kZSBDQSAoMjAyMS0wMi0wMlQwNzozMDozNi0wODowMCkX +DTIxMDIwMjE1MzAzNloXDTIxMDIwOTE1MzAzNlqgLzAtMB8GA1UdIwQYMBaAFBjo +V5Jnk/gp1k7fmWwkvTk/cF/IMAoGA1UdFAQDAgEAMAoGCCqGSM49BAMCA0gAMEUC +IQDgjA1Vj/pNFtNRL0vFEdapmFoArHM2+rn4IiP8jYLsCAIgAj2KEHbbtJ3zl5XP +WVW6ZyW7r3wIX+Bt3vLJWPrQtf8= +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/2f11f022.r0 b/security/advancedtls/testdata/crl/2f11f022.r0 new file mode 100644 index 0000000..e570f17 --- /dev/null +++ b/security/advancedtls/testdata/crl/2f11f022.r0 @@ -0,0 +1,7 @@ +-----BEGIN X509 CRL----- +MIHnMFICAQEwDQYJKoZIhvcNAQEMBQAwDDEKMAgGAioDDAI6KRcNMDkxMTEwMjMw +MDAwWhcNMDkxMTExMDAwMDAwWqASMBAwDgYDVR0jBAcwBYADAQIDMA0GCSqGSIb3 +DQEBDAUAA4GBAMl2sjOjtOQ+OCsRyjM0IvqTn7lmdGJMvpYAym367JBamJPCbYrL +MifCjCA1ra7gG0MweZbpm4SG2YLakwi1/B+XhApQ5VVv5SwDn6Yy5zr9ePLEF7Iy +sP86e9s5XfOusLTW+Spre8q1vi7pJrRvUxhJGuUuLoM6Uhvh65ViilDJ +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/3.crl b/security/advancedtls/testdata/crl/3.crl new file mode 100644 index 0000000..d37ad22 --- /dev/null +++ b/security/advancedtls/testdata/crl/3.crl @@ -0,0 +1,11 @@ +-----BEGIN X509 CRL----- +MIIBiDCCAS8CAQEwCgYIKoZIzj0EAwIwgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpH +b29nbGUgTExDMSYwEQYDVQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNs +bjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAyMS0wMi0wMlQwNzozMTo1NC0wODowMCkX +DTIxMDIwMjE1MzE1NFoXDTIxMDIwOTE1MzE1NFowJzAlAhQAroEYW855BRqTrlov +5cBCGvkutxcNMjEwMjAyMTUzMTU0WqAvMC0wHwYDVR0jBBgwFoAUeq/TQ959KbWk +/um08jSTXogXpWUwCgYDVR0UBAMCAQEwCgYIKoZIzj0EAwIDRwAwRAIgaSOIhJDg +wOLYlbXkmxW0cqy/AfOUNYbz5D/8/FfvhosCICftg7Vzlu0Nh83jikyjy+wtkiJt +ZYNvGFQ3Sp2L3A9e +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/4.crl b/security/advancedtls/testdata/crl/4.crl new file mode 100644 index 0000000..d4ee6f7 --- /dev/null +++ b/security/advancedtls/testdata/crl/4.crl @@ -0,0 +1,10 @@ +-----BEGIN X509 CRL----- +MIIBYDCCAQYCAQEwCgYIKoZIzj0EAwIwgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpH +b29nbGUgTExDMSYwEQYDVQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNs +bjEsMCoGA1UEAxMjbm9kZSBDQSAoMjAyMS0wMi0wMlQwNzozMTo1NC0wODowMCkX +DTIxMDIwMjE1MzE1NFoXDTIxMDIwOTE1MzE1NFqgLzAtMB8GA1UdIwQYMBaAFIVn +8tIFgZpIdhomgYJ2c5ULLzpSMAoGA1UdFAQDAgEAMAoGCCqGSM49BAMCA0gAMEUC +ICupTvOqgAyRa1nn7+Pe/1vvlJPAQ8gUfTQsQ6XX3v6oAiEA08B2PsK6aTEwzjry +pXqhlUNZFzgaXrVVQuEJbyJ1qoU= +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/5.crl b/security/advancedtls/testdata/crl/5.crl new file mode 100644 index 0000000..d1c24f0 --- /dev/null +++ b/security/advancedtls/testdata/crl/5.crl @@ -0,0 +1,10 @@ +-----BEGIN X509 CRL----- +MIIBXzCCAQYCAQEwCgYIKoZIzj0EAwIwgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpH +b29nbGUgTExDMSYwEQYDVQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNs +bjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAyMS0wMi0wMlQwNzozMjo1Ny0wODowMCkX +DTIxMDIwMjE1MzI1N1oXDTIxMDIwOTE1MzI1N1qgLzAtMB8GA1UdIwQYMBaAFN+g +xTAtSTlb5Qqvrbp4rZtsaNzqMAoGA1UdFAQDAgEAMAoGCCqGSM49BAMCA0cAMEQC +IHrRKjieY7w7gxvpkJAdszPZBlaSSp/c9wILutBTy7SyAiAwhaHfgas89iRfaBs2 +EhGIeK39A+kSzqu6qEQBHpK36g== +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/6.crl b/security/advancedtls/testdata/crl/6.crl new file mode 100644 index 0000000..87ef378 --- /dev/null +++ b/security/advancedtls/testdata/crl/6.crl @@ -0,0 +1,11 @@ +-----BEGIN X509 CRL----- +MIIBiDCCAS8CAQEwCgYIKoZIzj0EAwIwgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpH +b29nbGUgTExDMSYwEQYDVQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNs +bjEsMCoGA1UEAxMjbm9kZSBDQSAoMjAyMS0wMi0wMlQwNzozMjo1Ny0wODowMCkX +DTIxMDIwMjE1MzI1N1oXDTIxMDIwOTE1MzI1N1owJzAlAhQAxSe/pGmyvzN7mxm5 +6ZJTYUXYuhcNMjEwMjAyMTUzMjU3WqAvMC0wHwYDVR0jBBgwFoAUpZ30UJXB4lI9 +j2SzodCtRFckrRcwCgYDVR0UBAMCAQEwCgYIKoZIzj0EAwIDRwAwRAIgRg3u7t3b +oyV5FhMuGGzWnfIwnKclpT8imnp8tEN253sCIFUY7DjiDohwu4Zup3bWs1OaZ3q3 +cm+j0H/oe8zzCAgp +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/71eac5a2.r0 b/security/advancedtls/testdata/crl/71eac5a2.r0 new file mode 120000 index 0000000..9f37924 --- /dev/null +++ b/security/advancedtls/testdata/crl/71eac5a2.r0 @@ -0,0 +1 @@ +4.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/7a1799af.r0 b/security/advancedtls/testdata/crl/7a1799af.r0 new file mode 120000 index 0000000..f34df5b --- /dev/null +++ b/security/advancedtls/testdata/crl/7a1799af.r0 @@ -0,0 +1 @@ +3.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/8828a7e6.r0 b/security/advancedtls/testdata/crl/8828a7e6.r0 new file mode 120000 index 0000000..70bead2 --- /dev/null +++ b/security/advancedtls/testdata/crl/8828a7e6.r0 @@ -0,0 +1 @@ +6.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/README.md b/security/advancedtls/testdata/crl/README.md new file mode 100644 index 0000000..2e76c8a --- /dev/null +++ b/security/advancedtls/testdata/crl/README.md @@ -0,0 +1,92 @@ +# CRL Test Data + +This directory contains cert chains and CRL files for revocation testing. + +To print the chain, use a command like, + +```shell +openssl crl2pkcs7 -nocrl -certfile security/crl/x509/client/testdata/revokedLeaf.pem | openssl pkcs7 -print_certs -text -noout +``` + +The crl file symlinks are generated with `openssl rehash` + +## unrevoked.pem + +A certificate chain with CRL files and unrevoked certs + +* Subject: C=US, ST=California, L=Mountain View, O=Google LLC, OU=Production, + OU=campus-sln, CN=Root CA (2021-02-02T07:30:36-08:00) + * 1.crl + +NOTE: 1.crl file is symlinked with 5.crl to simulate two issuers that hash to +the same value to test that loading multiple files works. + +* Subject: C=US, ST=California, L=Mountain View, O=Google LLC, OU=Production, + OU=campus-sln, CN=node CA (2021-02-02T07:30:36-08:00) + * 2.crl + +## revokedInt.pem + +Certificate chain where the intermediate is revoked + +* Subject: C=US, ST=California, L=Mountain View, O=Google LLC, OU=Production, + OU=campus-sln, CN=Root CA (2021-02-02T07:31:54-08:00) + * 3.crl +* Subject: C=US, ST=California, L=Mountain View, O=Google LLC, OU=Production, + OU=campus-sln, CN=node CA (2021-02-02T07:31:54-08:00) + * 4.crl + +## revokedLeaf.pem + +Certificate chain where the leaf is revoked + +* Subject: C=US, ST=California, L=Mountain View, O=Google LLC, OU=Production, + OU=campus-sln, CN=Root CA (2021-02-02T07:32:57-08:00) + * 5.crl +* Subject: C=US, ST=California, L=Mountain View, O=Google LLC, OU=Production, + OU=campus-sln, CN=node CA (2021-02-02T07:32:57-08:00) + * 6.crl + +## Test Data for testing CRL providers functionality + +To generate test data please run provider_create.sh script. All the files have +`provider_` prefix. + +We need to generate the following artifacts for testing CRL provider: +* server self signed CA cert +* client self signed CA cert +* server cert signed by client CA +* client cert signed by server CA +* empty crl file +* crl file containing information about revoked server cert +* crl file by 'malicious' CA which contains the same issuer with original CA + + +All the commands are provided in provider_create.sh script. Please find the +description below. + +1. The first two commands generate self signed CAs for client and server: + - provider_server_trust_key.pem + - provider_server_trust_cert.pem + - provider_client_trust_key.pem + - provider_client_trust_cert.pem + +2. Generate client and server certs signed by the CAs above: + - provider_server_cert.pem + - provider_client_cert.pem + +3. The next 2 commands create 2 files needed for CRL issuing: + - provider_crlnumber.txt + - provider_index.txt + +4. The next 3 commands generate an empty CRL file and a CRL file containing +revoked server cert: + - provider_crl_empty.pem + - provider_crl_server_revoked.pem + +5. The final section contains commands to generate CRL file by 'malicious' CA. +Note that we use Subject Key Identifier from previously created +provider_client_trust_cert.pem to generate malicious certs / CRL. + - provider_malicious_client_trust_key.pem + - provider_malicious_client_trust_cert.pem + - provider_malicious_crl_empty.pem diff --git a/security/advancedtls/testdata/crl/deee447d.r0 b/security/advancedtls/testdata/crl/deee447d.r0 new file mode 120000 index 0000000..1a84eab --- /dev/null +++ b/security/advancedtls/testdata/crl/deee447d.r0 @@ -0,0 +1 @@ +5.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/provider_ca.cnf b/security/advancedtls/testdata/crl/provider_ca.cnf new file mode 100644 index 0000000..4807e24 --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_ca.cnf @@ -0,0 +1,14 @@ +[req] +distinguished_name = req_distinguished_name +prompt = no +x509_extensions = extensions + +[req_distinguished_name] +C = US +ST = CA +L = SVL +O = Internet Widgits Pty Ltd + +[extensions] +basicConstraints = CA:true +subjectKeyIdentifier = hash diff --git a/security/advancedtls/testdata/crl/provider_client_cert.key b/security/advancedtls/testdata/crl/provider_client_cert.key new file mode 100644 index 0000000..5d1d4ee --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_client_cert.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQCZnZTnMzrKXLTu +xhCDb3zufXGknUBTq2LJS6C4dxq02XDt+pgwGaglO3CQHTNq2F1s3EEmD2eMMIjh +Dk1D6DCLcyvMrbNzNFcwaounr8wYdhyzW9DhWBTBVFU7RqL0jnjLaQGbnGSVGqy3 +O8cpX+LqPrROrTIyDt63zBwk4LhcIxAACaUpDzmZzqk6IcUKv27E3vDS2dUmPUgQ +tpHiBgZ1VIJkm6GTGrTWNvInbAUbyMP/1sF7Uo59zPLaPIbtPD6Kau2kvAgpLz70 +Zx5XTQGvSSyXu/P3C7AtX/sq7f/wyR/1QHX1FZQN02nCCnBeBBF62sqSmS/pnIHr +/ZSW4d3AoaLUl6raKsSSYlXXGEPCK0rWAMh2HsT3qbU4ya93PclSMhx5BFPVw1kA +132ifP4RD29+MjcqiNC7S4bn2aNNWeWTD5o/y2Sd7ep4eZUpQ0upj/w4g0R/arNg +NYDzfq0th8EERn1y9fku2RONkkEnb49xUijwWIKJgEVrz4XVTOpkRxo/V9nzwmdh +50esv7L3OiE8ogAE2wcKKSWkIwVTOINbdc5ZdXyAVzy/J51zZnPztrnmNOyiIlUn +6Dcvl1gK5LJaBMzWODCMf0tkeQtSOSwb8Yar+vguonRj+mByZ7cBgvosDMe6ST6t +ff5+wMBBPpUBmeMbshkxeXo5MVN7UwIDAQABAoICADYQykmdNEHo4x1uxH0eDiB6 +Mjc3yV4pYflrDsQd115zcVWw70NupEmGZBW00VX3lNotoxhcL5udsW9Uc0lFPWC3 +RmEheZlAgLdfqIt6SiEJ4QwXcqr9L0DkB6N3Nv9P7Z/Z82DraFM6MjPDbFNZlinP +q/JM7u/DYAvcYMNx/DEhfg4lVuXen+1MWS9Gl20y9y9/L89mL6jnxKdtOmcMq9U7 +yzDRvcpiiecjK5NIZ3vh62nuEebvpYsNSSQaTXrNy004WS+zpkOoh0XJzVn4lnE1 +FebJlhILIRS3RVVUstHyV7cf2uRJlfRPxlcvCWFtmSFeRBHYrI5SMT/33gZEtKD+ +JmwNkAzV8xGoyYPHbfMT1IsVo6oOQHgix/D31j6N1Q+EtmZ+z7rMUFQLLG514x0m +kg0g/rKCOOH+tbSXITn16loPaW/ydQgJ5CjEn8q6aXNbjrOoQDh2FZICRQF2orGl +EH1mQ1l709t0/hLOIn1WfGLY5hP+zBf+Wdi0tUrLf1Bngr0YuDpv3equJst2F5EH +mdLnXCNvvigsWGwXTGFt0RO+sr7+/NLgjw3joE1yD5lkZwcOZVDp1Z4FUa67NOiH +vwVZwLyWREFs7uZ/Y7cTGFvHynkDnn3WFDb+c0h34A5X/oqKaJhKOXnidT7oY8wq +plNqsCgztu2h12p7LsixAoIBAQDJkHyoa2NOW/0p7J0krKuSx5I70kYIKapqcf5+ +5REYi/8T/qv4KsSTYStZQ7DWLFdfF99Mi4nuCB+39WwBsVZYg2TRRRILILkO4HBK +GeimsAuRXj7vX0axSLM/aFpUHmppOB6SrybAlSJoGpBVIvk6+DaqOfwPBHEVljNT +1FCl8AB8mfA9EGNUiJwlHj7jqrSD6iPRQH8ULf6nPq6z6uZnFOXFPLOgen+pBFZy +68MWu22TY3f9jSFpqFFp5peielZG3Q3Mf6p1CPf69s5bzN2AJhw5bFQWkx20QsL5 +s8GbSDcjIgI+s4EZIw4D/o6+J6PzgRoJQcdq946XUzXZxN/nAoIBAQDDGhDcAHbc +aNqFgtA89HGmqBRmCcqU4WsWpBOmOsG661uTyNbHUxGKw6T5v5My+hcqJaLQfwnr +hYDVV4M0J34KU1S8axJ+e++3Jg9HpeDOdfN8pugqIJGkjQ7aRZKfPxfqR/ckF9FK +LLgPdzRfzDZRmwKSWPVBCNGlr7J+sKhykcDgdraMqpYWoaKp+QV6gji7d2syvLHW +5qFr652Dm7F+c+zTDtIvWjGLGu0au3zrwV1RQkgQ1aqBgmBT1S0+rPalV0AAva9E ++f2Vy8Tycgl6rYiQJSqnPz+JxfXxny8+Nay9mG3lN9HlN5zE7MQnDutPtiSYjy8q +ZUztarupI8u1AoIBADpPWTCjuFu/0tIhCCjG5u+UWmKB5w6PdyRKC/SLsdFnFoij +QP6O6MU19ANjyLF8rF3vGwMazvEUWpCuJ+upcLA0eqLrl1euxLpgBTv6mMo33XDV +UeGPr3Sz8l7iglcZYXFE8ds/XjeSLRzuqlhmwLDlg3LlSVzSzSAQjpKuthH7BzkE +k3Im3oVi68D1Kf5UsNoEjw4G2Xxt/eBGCuYziynA6uOPNuuy5GFxxsyCFbLqz702 +pkysWkEllz/KnI09VN41Lru9JwOqb4qjgXkfH+jlnX6jLwRE1PAD7EGuVdDlKEY6 +nWmkJjGuaWyQZJzv/McBzxVkeRshuJdgVBDGmnsCggEADFMdNYih+ZJ2G3EEDpWy +iECd6UQ9E+KZjTiYNSwJCPHNOyy4xKauuQFa7pv3hITf6b2u51TfH42zccaxdx33 +jFdvRufMp0jU/9Dbrj8AUIqK8xjoGaEtEiQHCCrU9FJcBGS/a/xFMFZa2j9Bg7u2 +wrj0FKKh+5W/CKRstiwauAIVGRjmt0QfbxaO3AXrHq4TP2Rv1SiuY1D2aYbc0G+J +at+P2lVZWbxs3Mi6qbGmVo5EgtmZC9czijLeOu2AijEK867rUCCrbcQNDOVub5Jc +nu9PbSur4hzQurdSrgzMQzXIz8FNT+mSzNQShy4dxgnfO43aCfkhlaAImAbiC/FC +8QKCAQAJ2EJDwTYdEhH4D+klAS852dCFb4/wV8cj2t4eKAF3hY6VVBvkQNaYzxyw +aWecUqIAuTORU7RQkYjz4eVu1ghVOlXKqtThpkwSpE234AZLU4AsXTG0ivefVwgG +NQlHwv/qejiNChgVlnc9Xgn70CmNmB6I12VTXxd8hy6Nd1AmwGCW2k2y1WgLPSZH +pHfyAkbAczK5GBZTvZ85QICW0VuAXSPaIdlkiQZyjArbw7uo9T3URZgkp7T/jq1C +/6bGOoB/mbB77SnCJDm51CWBkAfvSvItr2eisNp0KgO8/o5JAZL66SUOWPtbaj+3 +SbKH08tME1PaqI0WEXB49lAKzB1G +-----END PRIVATE KEY----- diff --git a/security/advancedtls/testdata/crl/provider_client_cert.pem b/security/advancedtls/testdata/crl/provider_client_cert.pem new file mode 100644 index 0000000..077e350 --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_client_cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFgTCCA2mgAwIBAgIUBpIR/E6Ufmm2cndWnoFZ0Ntsp1wwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMSEwHwYDVQQKDBhJbnRlcm5l +dCBXaWRnaXRzIFB0eSBMdGQxGzAZBgNVBAMMEmZvby5iYXIuaG9vLmNhLmNvbTAe +Fw0yNjAyMTIwNzU0MTlaFw0zNjAyMTAwNzU0MTlaMFcxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJDQTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRgw +FgYDVQQDDA9mb28uYmFyLmhvby5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw +ggIKAoICAQCZnZTnMzrKXLTuxhCDb3zufXGknUBTq2LJS6C4dxq02XDt+pgwGagl +O3CQHTNq2F1s3EEmD2eMMIjhDk1D6DCLcyvMrbNzNFcwaounr8wYdhyzW9DhWBTB +VFU7RqL0jnjLaQGbnGSVGqy3O8cpX+LqPrROrTIyDt63zBwk4LhcIxAACaUpDzmZ +zqk6IcUKv27E3vDS2dUmPUgQtpHiBgZ1VIJkm6GTGrTWNvInbAUbyMP/1sF7Uo59 +zPLaPIbtPD6Kau2kvAgpLz70Zx5XTQGvSSyXu/P3C7AtX/sq7f/wyR/1QHX1FZQN +02nCCnBeBBF62sqSmS/pnIHr/ZSW4d3AoaLUl6raKsSSYlXXGEPCK0rWAMh2HsT3 +qbU4ya93PclSMhx5BFPVw1kA132ifP4RD29+MjcqiNC7S4bn2aNNWeWTD5o/y2Sd +7ep4eZUpQ0upj/w4g0R/arNgNYDzfq0th8EERn1y9fku2RONkkEnb49xUijwWIKJ +gEVrz4XVTOpkRxo/V9nzwmdh50esv7L3OiE8ogAE2wcKKSWkIwVTOINbdc5ZdXyA +Vzy/J51zZnPztrnmNOyiIlUn6Dcvl1gK5LJaBMzWODCMf0tkeQtSOSwb8Yar+vgu +onRj+mByZ7cBgvosDMe6ST6tff5+wMBBPpUBmeMbshkxeXo5MVN7UwIDAQABo0Iw +QDAdBgNVHQ4EFgQUJWYH52GSJRDUMz8SwIg8BBDEJ5YwHwYDVR0jBBgwFoAUXWT8 +wjMCB+STUF9WMHSZ2nmm6lMwDQYJKoZIhvcNAQELBQADggIBABOAJk56dKeYBUW7 +81idI/guOqLleBQm1m/qjciQ8tl6eMCfXj1866ud6nzThcRXOcIHnON3R4DPfxt5 +AHSWBLPZDkGXvwEY4OAKL2g9Vwg/YAe561AuAKtfQUGyuSGo+S91WgTnQPpq5ZBE +pQwyMa4/QOj740kfNBlrIIhAoekCrS3sO1GXu52SYlKWDk/RP4fZnfzkbjTrkUIP +0wmNbg6mFGgNvgXewbPWWk/oyT22Z8Fi1pRZnF+D1n3MlhBaZcfe9bIWPbSiiqYc +4Hhx4gSyEWSwI8nSGCjfeRQ0/zsoKRyrfBMaJPUGd28blJLazEbvfE66t1BSB7bC +LnEXTWcztuvv4eeQ4zofcDoOQ9RPAdpmomPMsvzpqS6dXZ1KeUEsvrEqxzA15Xy0 +XrP0rJlMUKkVNMh+1epAfL/rJWOARpZiFbR+Gt81DSw5f99rXlDuQa/IEvts2Rf7 +Si7RKEHDWUYa+Uwd+EVAqhT+6Gc7lG6DPDgrjC9Nbnjris4joUNgkzFBkmkVcnUN +YCz7vBxQY+8WAKJFEeKI7GDdbbdL8O5XTMztG97Mn9psX1ygd+FRr9gkaa6chmYo +CFmaYwUqAXZc5sccNopCrT8xRXvPsfqcObJC+ZTTcy2iG9AHPRONuoMX74euuHv7 +fdOHdNs2zxer/DvOI2S0ldauBQcR +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/crl/provider_client_trust_cert.pem b/security/advancedtls/testdata/crl/provider_client_trust_cert.pem new file mode 100644 index 0000000..5fbe2fa --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_client_trust_cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFdzCCA1+gAwIBAgIUBsTfmfdYI6inXskJAgjSkYhI1P8wDQYJKoZIhvcNAQEL +BQAwSzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxITAf +BgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNjAyMTIwNzU0MTha +Fw0zNjAyMTAwNzU0MThaMEsxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoG +A1UEBwwDU1ZMMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggIi +MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCgWUiV3/qMIn3pCX3/C8JOsWqk +ZoatJMqSUXY50cX4KfRwfB9RxyAc/48pjVUouaHCLPS+/nfycPgWtzU4c4U4QMH8 +TKdmi/9g6HeAH/jzWuASqWV8irs+jMA2F3G18n7GgZwRdzIg0X2Bvi/T/mnhhlAK +Or3CwV8XLWBu65z09XA7YICCYZSG/7g6R+jpOb2eVybYtYey+PRd+OEmdlX9r90X +/aF+Xy2gMpqGeSN/OX4AAWNnss0Zl6OFjZYeyrwdGEJh1p++quVNQsiQU+onP5oC +H9m4XL5d9MiGiB36IOAITQd3mbeHm/I0nqCMq5fxqWF4CZna6+E4/kNWC4rKm9TT +DRKqmyJy0VR8lCBdh1lIvGlhJXEVoFvgqpmwwmNMjYW/1OG5CTYMpoSNlIIo+XSn +xWTfjgF6Ex3T4A2igg+fwcG7ApNRUIkdHmMVdYJMlWD11sZrlFCgy0Ia9luFFbZ+ +0+DQd0kc2xYN7kFnefrL/McTM4AMnx1jpMky1B45GsyS1Fc5MEQTXxYkMoBP5FVP +6ccsvuvBizAVQWTGNYzL+oDOkg1J1+J2q4aqbiod7vfZdfwSmZrzlLJGIhbxNh29 +6e3YdHmoDYbrZZPjPVzRCWS9FlBT3QvTU+uLC6wL/ije6ex++7KvzgIrkIAuEHfe +qEXg09dz9wuXHO/IswIDAQABo1MwUTAdBgNVHQ4EFgQUbcyHqiFVin7tmZVf2ucb +A1bt7UMwHwYDVR0jBBgwFoAUbcyHqiFVin7tmZVf2ucbA1bt7UMwDwYDVR0TAQH/ +BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAQNrBr1axMAl39YsP/CHbcrTCRjTk +zP2X4Cm8ZbQcJC1Y2hWLD3YUmj5DldG+9D08ub6VTzeQ2GVQmT8ESxEPWXk+5sdN +4MDhoQPLsicvQ6wPsqJ6tIQrJY21W5e5kQY5SBc/yDxyDbTaHJTnU6NhEtuuIXIc +ove4rq0hLs3FnAYcU1wT2/8m2Kohvfmk/T1b7LocBFIgerAE42qVS3ginFDMZOGS +Ejf3JchTfwNYWGIFRokpvBPX3Z34n0yx/Da+zRNLZ0QVrk1MXgUwWc6mRlQ2eLXV +N0hBvi6mVLArxxitmUx3CynAh99iPXyd+wso/vU8mvB0AIIsEXSKqx0j3d/r3Xkz +EhOmnKt84y3YaouUhejigiWqvTmi+MWc2KDFdJSkZ0bRuHsuPORKlUMAdLQl7ca5 +ksQ+iGlhX3g8F8A2DkBEQTLIg4dSVq1x1UDIHO5c02BKsftuyv5TP/wdrnE3E2qm +hMrnTl3Q5mZAUHkMOJtVxqSS/+xd4ar5JADk73OOkqKr3JpSdW+Uy2f0tix/I8aV +76lElQvewBsx7dIpL5dKloh8Ev71s9/Z9fnHefacSeRusXofuV3c1yeKELxGKpUh +eVMAx9SkJbTCNc5MnU632JhAKdClRXe+tQw8lThfpmIWsinT1jTc59T5eG3U/Fu5 +jGq8soZUfvgGyds= +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/crl/provider_client_trust_key.pem b/security/advancedtls/testdata/crl/provider_client_trust_key.pem new file mode 100644 index 0000000..c76a891 --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_client_trust_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCgWUiV3/qMIn3p +CX3/C8JOsWqkZoatJMqSUXY50cX4KfRwfB9RxyAc/48pjVUouaHCLPS+/nfycPgW +tzU4c4U4QMH8TKdmi/9g6HeAH/jzWuASqWV8irs+jMA2F3G18n7GgZwRdzIg0X2B +vi/T/mnhhlAKOr3CwV8XLWBu65z09XA7YICCYZSG/7g6R+jpOb2eVybYtYey+PRd ++OEmdlX9r90X/aF+Xy2gMpqGeSN/OX4AAWNnss0Zl6OFjZYeyrwdGEJh1p++quVN +QsiQU+onP5oCH9m4XL5d9MiGiB36IOAITQd3mbeHm/I0nqCMq5fxqWF4CZna6+E4 +/kNWC4rKm9TTDRKqmyJy0VR8lCBdh1lIvGlhJXEVoFvgqpmwwmNMjYW/1OG5CTYM +poSNlIIo+XSnxWTfjgF6Ex3T4A2igg+fwcG7ApNRUIkdHmMVdYJMlWD11sZrlFCg +y0Ia9luFFbZ+0+DQd0kc2xYN7kFnefrL/McTM4AMnx1jpMky1B45GsyS1Fc5MEQT +XxYkMoBP5FVP6ccsvuvBizAVQWTGNYzL+oDOkg1J1+J2q4aqbiod7vfZdfwSmZrz +lLJGIhbxNh296e3YdHmoDYbrZZPjPVzRCWS9FlBT3QvTU+uLC6wL/ije6ex++7Kv +zgIrkIAuEHfeqEXg09dz9wuXHO/IswIDAQABAoICABy5se+rhspo+W5wdWJl3GLV +lLmr5k+JSkLpFgloo5MADRrDmabASef3/lEe0RUxICHVhOjcVGeZxk/ndUDOLRz1 +OOs2XjzYMqFNM+8/iw0ph/+/2f70KXQcqehmzcefEAgGuwtD87Z+YQIHrLDJIHW2 +5orWmB0WIC9aQS7Nxbn6aCcy9AKncYC3ueyy6i4x2l7N1Rc4ef1dbQWSqt8FjwUZ +5r1AAhjN+zH6LsWNWQcXKRPeK80tcmG10SUBRtXwUr/Rkz7MwKRbICX1o1F7vvRW +CAR+aTYGY1IYon27T8d58Th7eC18W60PClZT3oYkzV0ND3l+GtJltFzN7AkzmyLF +Gha8k6yYi+ZolQl2USn+16ydmpfgrjf4Zg3PAnKLOZrn2D4/CdrxXBWex7m+cos6 +zyJGA4HdTCB3a+dhuPbpxIuIdcb0XX4h3Ud6Shi0521/N3phQJ+7sT5vrULY7usA +I9BGUZE28uNt/Hclj8zB0FUK4IAm7FURRUvhVZD3GY8lwmdwcyfjmBDSAhBIYRXS +zpoKHG9Q5DPnOik3veS/56xyEnpnRZ1x9jszwiYw/gprAXymPS3hyJJpYSBxMAjB +yXnpwBrAQF5hCFF6xbdXYBZH+Xgl2PM8tIneViJMBdF1drQq/rmj5DcjZG2iUy22 +w1wEGQatQoqAQ+4dM1clAoIBAQDUBqBIBoKm0Kya7eJZmYx04vGR3F965Ec0U39h +5g9kOb2C2rkzglyBj18RWxq7H3dJJ91ZkMthLhbzJtKmPBxOJNqqxfxE5VBntf4a +bEORDCNMKy7RP9pPwV6DrZ1zCev45rcfWrlSHBPfoTk3ThPjHW3LJheQcKguLGfL +EE3V2hqg8Ok1cV/y3V4CvVpwQrtgATCW19boOyC/jhfqQ6b1Ko42MgzLDm6MAbMO +yqm1yOT6D+bBPn9Cw9OweJDc++XkqnOS+Y08m1fyVULiltb217eWk8JTVrHoe0aO +YEylJS/1OQNUozjuSShSt5C8I/FbC4D3dS36Dc0OeqR5oDaPAoIBAQDBmuUeddRf +G9zbdDJi2hq9XXZJXg+gUkope9fYt5vK/LbWLjZ/LiX1YirouLTAJqlUaAXvMlU+ +vRYCyvPNXUpasquZi0ycdg1lV+ItrNueDPzYMi1x5U9Vew3MCk/1V2vjU8GCfC3r +iFLkf4X3PkjhckEzIedRBQkFpH/8L52Ojyp+k+M15PrbhSkqLSZ2n8jqlYpQ15ow +lIEahXhVMn1l/OgL6p5fJkOhiNfkGBMvlPema+hgYZmZGG0G+0YUgMw9l4lfYaf1 +HDbNyEwAqBJpWtD9VlBIx04fhm5Kiunszz28geHN2bH5F96OllfqcRqCT6SUSFuU +2Mz5G1Atjf2dAoIBAQCkb5shIQN08oPbCEEy8jYPdO6bDAl02tQqTdOODonDVlCW +yqE1xJWP2ayGXlzF4Sp3LxapXvWMod0kqOhYCmh9ZnG8Xh4/JIWOWYP/5BUmyf4a +FaeSm23pyvNNNnnU/U3oVK6S/56YgrQbDQO906zyyCEdm6ZM3EJixQeeYj+raiKa +zRxg2VPrnClMAKTCSc7eLy4K3syKgUjtpvr/MYarv1xZxclMNh4gMTU4dI7YMDz9 +fxWcq6axFgT4aRkYebga9uL5itcxuNylUeC0sP14pWZ5vpDIZ4VE406eHytyLPwb +uCLQImKF03EVbc4vS8TksnBL+rI0qz1sTEuBFHMbAoIBAAfQTZD2JnUUNcyxmtr8 +fHnKDN0XK8BHsfCMrAB2IJaHroVkZhSp8yQ9KwgrdDgRF2JttFecC261yO6h7EcM +jdStQ1m2Eoh6Bz5g1qMLR+3QDmBXXhgrrhEAH2VtwR5gwdzx42x1wJCx9s58CxcY +b26R4unCY4iUkHGm2vd9oHlBc/CZ3WCudiVn8WpfWh8Ngdld4bAzk2iEhdVhL6MY +n5D/LQpWFMA4ViBt2nC9PAD+nSQdVMqXBdD0+GmAuKpZLGUL+aJc2Z993QRfIhog +rmWMIcUnt6PIT3HcRhVCGADTuGUkRM2/DHzGJthQXwn6OJyrxDOr2+5c9aDUJl9A +TM0CggEAbTBirDqvfnDnWm1YNCSlo7YYtKzc8FnO53wRqxwsV7FNdzQgie6pruqV +HoN6/kKb5XSu5onOdHvpK4WGsLSyNPPj2qKgD4FwDn047PCdrlTGgbTjsYrp3KTW +B9qebbpv5gGGHgzs0szhzaRYjKUeBR0yH47r6uek2umlSU5pT5+N+FjxLMREyWao +kqdYy+IA6CeMioiumqoGHGM0bFWOmys/2MBwCESTtVok1D/nsdx1TtPnlaqv7lyK +aTGFHJGaZ2lrhYXiMA2pI//mXzFFll3fkReL2d9dWPATWjqDLYfYpG7ocHNf/ACy +CfazzeNWNCkJ7unLnt7usIkhposepw== +-----END PRIVATE KEY----- diff --git a/security/advancedtls/testdata/crl/provider_create.sh b/security/advancedtls/testdata/crl/provider_create.sh new file mode 100755 index 0000000..a4de2a1 --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_create.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +# The script contains a sequence of commands described in README.md +# Generate client/server self signed CAs and certs. +openssl req -x509 -newkey rsa:4096 \ + -keyout provider_server_trust_key.pem \ + -out provider_server_trust_cert.pem \ + -days 3650 \ + -config provider_ca.cnf \ + -nodes + +openssl req -x509 -newkey rsa:4096 \ + -keyout provider_client_trust_key.pem \ + -out provider_client_trust_cert.pem \ + -days 3650 \ + -config provider_ca.cnf \ + -nodes + +openssl req -newkey rsa:4096 \ + -keyout provider_server_cert.key \ + -out provider_new_cert.csr \ + -nodes \ + -subj "/C=US/ST=CA/L=DUMMYCITY/O=Internet Widgits Pty Ltd/CN=foo.bar.com" \ + -sha256 + +openssl x509 -req \ + -in provider_new_cert.csr \ + -out provider_server_cert.pem \ + -CA provider_client_trust_cert.pem \ + -CAkey provider_client_trust_key.pem \ + -CAcreateserial \ + -days 3650 \ + -sha256 \ + -extfile provider_extensions.conf + +openssl req -newkey rsa:4096 \ + -keyout provider_client_cert.key \ + -out provider_new_cert.csr \ + -nodes \ + -subj "/C=US/ST=CA/O=Internet Widgits Pty Ltd/CN=foo.bar.hoo.com" \ + -sha256 + +openssl x509 -req \ + -in provider_new_cert.csr \ + -out provider_client_cert.pem \ + -CA provider_server_trust_cert.pem \ + -CAkey provider_server_trust_key.pem \ + -CAcreateserial \ + -days 3650 \ + -sha256 \ + -extfile provider_extensions.conf + +# Generate files need for CRL issuing. + +echo "1000" > provider_crlnumber.txt + +touch provider_index.txt + +# Generate two CRLs. + +openssl ca -gencrl \ + -keyfile provider_client_trust_key.pem \ + -cert provider_client_trust_cert.pem \ + -out provider_crl_empty.pem \ + -config provider_crl.cnf + +openssl ca -revoke provider_server_cert.pem \ + -keyfile provider_client_trust_key.pem \ + -cert provider_client_trust_cert.pem \ + -config provider_crl.cnf + +openssl ca -gencrl \ + -keyfile provider_client_trust_key.pem \ + -cert provider_client_trust_cert.pem \ + -out provider_crl_server_revoked.pem \ + -config provider_crl.cnf + +# Generate malicious CRLs. + +openssl genrsa \ + -out provider_malicious_client_trust_key.pem 4096 + +SubjectKeyIdentifier=$(openssl x509 -in provider_client_trust_cert.pem \ + -noout \ + -text \ + | awk '/Subject Key Identifier/ {getline; print $1;}') + + +sed "s/subjectKeyIdentifier = hash/subjectKeyIdentifier = $SubjectKeyIdentifier/g" \ + provider_extensions.conf > provider_extensions.conf.tmp && + mv provider_extensions.conf.tmp provider_extensions.conf + +openssl req -new \ + -key provider_malicious_client_trust_key.pem \ + -out cert_malicious_request.csr \ + -subj "/C=US/ST=CA/L=SVL/O=Internet Widgits Pty Ltd" + +openssl x509 -req \ + -in cert_malicious_request.csr \ + -signkey provider_malicious_client_trust_key.pem \ + -out provider_malicious_client_trust_cert.pem \ + -days 3650 \ + -extfile provider_extensions.conf \ + -extensions extensions + +openssl ca -gencrl \ + -keyfile provider_malicious_client_trust_key.pem \ + -cert provider_malicious_client_trust_cert.pem \ + -out provider_malicious_crl_empty.pem \ + -config provider_crl.cnf + +sed "s/subjectKeyIdentifier = .*/subjectKeyIdentifier = hash/g" \ + provider_extensions.conf > provider_extensions.conf.tmp && + mv provider_extensions.conf.tmp provider_extensions.conf + +rm -f *.csr provider_{index.txt*,crlnumber.txt*,ca_client.cnf,ca_server.cnf} *.srl diff --git a/security/advancedtls/testdata/crl/provider_crl.cnf b/security/advancedtls/testdata/crl/provider_crl.cnf new file mode 100644 index 0000000..3f12ddf --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_crl.cnf @@ -0,0 +1,14 @@ +[ ca ] +default_ca = my_ca + +[ my_ca ] +default_md = sha256 +database = provider_index.txt +crlnumber = provider_crlnumber.txt +default_crl_days = 30 +default_crl_hours = 1 +crl_extensions = crl_ext + +[crl_ext] +# Authority Key Identifier extension +authorityKeyIdentifier=keyid:always,issuer:always diff --git a/security/advancedtls/testdata/crl/provider_crl_empty.pem b/security/advancedtls/testdata/crl/provider_crl_empty.pem new file mode 100644 index 0000000..f198b9d --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_crl_empty.pem @@ -0,0 +1,20 @@ +-----BEGIN X509 CRL----- +MIIDMTCCARkCAQEwDQYJKoZIhvcNAQELBQAwSzELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMQwwCgYDVQQHDANTVkwxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMg +UHR5IEx0ZBcNMjYwMjEyMDc1NDE5WhcNMjYwMzE0MDg1NDE5WqCBmTCBljCBhgYD +VR0jBH8wfYAUbcyHqiFVin7tmZVf2ucbA1bt7UOhT6RNMEsxCzAJBgNVBAYTAlVT +MQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGSCFAbE35n3WCOop17JCQII0pGISNT/MAsGA1UdFAQEAgIQ +ADANBgkqhkiG9w0BAQsFAAOCAgEADP80NJCab+u4X35GltIMNcedwPFUObfvu06r +DVTm75x6OB4mZCFmNIu1AXWecAPqU7tOnBkfQTDbTNNSHsdVnqXcLO8jiQJ5vePt +OoXqPdFO35OCBWm4+ZnRBn3x1WiRyig4PTgalhPklTKLqY5bIF9tyKH3EdY29CP4 +x9TvVoAlAaOM3D8mf2ms9g8I1UPfWjtltuLmL2HJHCiTTFsNm1gnR9UEa8bkeQeP +hxpekE/btZ5XWYKauH03UtVHkUMQuH/pUFhCCPAYwPCDYyE8e4xtjHpak79HvtSd +P/mSlgn9Cce4TOJxryD5KNmsRjcJiUVS3eirr1hhbDD4rDZzFmEWyjYMBQiEcBAl +FMtkomhT1i5+rpVjV2oS/DDwyJe9WhUzc+JhhmyfpJR5IlYtVK6rXgHtfTypv9Pn +bS1SD1KBcOyibixTGoiJWrPjaWbcEM3rQE6cJpkK/8xFvOhmmvpYrxVtyivnLPxO +0pYja34OF1o0AllVHADUjww98tvXjSRNowqBEkEGCtQfIYnbh6WG6m1c6OIMmCPO +dT0Vyk0EOtAIBDNnlQaUOQt3Gmlxi/T3OwAyM7nyAga+qBZNbfJr2zg6GkbIRkbB +8crvyouCUiudUFtxFFL9O1/alDbcIAYCzRk1NL+fPbBHRr5LzSKmyq/DcPKreo19 +XaplQm8= +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/provider_crl_server_revoked.pem b/security/advancedtls/testdata/crl/provider_crl_server_revoked.pem new file mode 100644 index 0000000..b724223 --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_crl_server_revoked.pem @@ -0,0 +1,20 @@ +-----BEGIN X509 CRL----- +MIIDWjCCAUICAQEwDQYJKoZIhvcNAQELBQAwSzELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMQwwCgYDVQQHDANTVkwxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMg +UHR5IEx0ZBcNMjYwMjEyMDc1NDE5WhcNMjYwMzE0MDg1NDE5WjAnMCUCFBvcyvmo +ckWEbX+0FnMz8yaiiqgFFw0yNjAyMTIwNzU0MTlaoIGZMIGWMIGGBgNVHSMEfzB9 +gBRtzIeqIVWKfu2ZlV/a5xsDVu3tQ6FPpE0wSzELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMQwwCgYDVQQHDANTVkwxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMg +UHR5IEx0ZIIUBsTfmfdYI6inXskJAgjSkYhI1P8wCwYDVR0UBAQCAhABMA0GCSqG +SIb3DQEBCwUAA4ICAQAH7QS5Gbd30cyWj/ATmYuDG2pWOLkTh/kTVHG9aAAv0UsQ +s5M4tTxNX50FcPE3aobVOL0pEAdjq3Cnss1BWQkJqDtewHrV99XmcTBMAvxo0SFi ++fRSgwDR9NaXsDPFJZIn4uHAKKxvA2qBCqKCPJySfIUdzVZ4nsPHAh0LPBGNuf7+ +UVakIhdp8jUQavWqlO1Y3VF5IFpiIQEWFQR9KAbZsuH6FsAeC2S+HUzGaVgmALCw +t6OldN4bMSOJakGDwY+3Ih3oNqcYxqAlq+N72X30eonPDloPfvImfqRgJVhD1y5X +vxnln0e47d6I6QUMdNmgbAP1Gs49ntJFk+t99E1XTDsxc7PHNeQ+lHJEh2WsdXrB +m0Ds8YbfN9dsWFiqhILyydm3ei1+JvNGMaQnaVAdUw9wsJDwbVdEDMq13VPDY3H9 +jyL7UNOjOSlNJQYpATORtzznzEdq8iZgCyo3cbyBFgXiQ9qdck0FDRXZYp0SPQB7 +X+1EjYzFgrPmRMZdItMw147Fe87uj1XGvjo5XID6EOCw/xxVIZCLXijCqDtW44X8 +U1W305YxjugQw/wgCUdGFvtThEPUkTKC4O5olBSP6msd10eBY224Lgw9Cn2x0Z5/ +QrgyPv7ef9tglfT+mP6BIqkfUWLiGiaxGQvSgY+vH7CoZEqZ5uZ9m7NArbHMag== +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/provider_extensions.conf b/security/advancedtls/testdata/crl/provider_extensions.conf new file mode 100644 index 0000000..52cd09a --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_extensions.conf @@ -0,0 +1,5 @@ +[extensions] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment diff --git a/security/advancedtls/testdata/crl/provider_malicious_client_trust_cert.pem b/security/advancedtls/testdata/crl/provider_malicious_client_trust_cert.pem new file mode 100644 index 0000000..d2a15eb --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_malicious_client_trust_cert.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFXTCCA0WgAwIBAgIUaExYOx/op7TEm3y6Ykmhy8CWn8owDQYJKoZIhvcNAQEL +BQAwSzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxITAf +BgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNjAyMTIwNzU0MTla +Fw0zNjAyMTAwNzU0MTlaMEsxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoG +A1UEBwwDU1ZMMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggIi +MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCwqLQQN8XrAdohlLIdP5LyeYtL +1S9ZtXFBMdmxtHpLZCuY2orlOyQeaFZrut3DRXmedPFH2n0lk5GbRQAytyo0bSvx +aWJSI+gG6rZsxebFqn9nF7eTFUxpkna9hGCpvhmaMplSjNrKCoVHfkkkBVoHHpW3 +vnrcf71mFZAQ+8SbfT7i3UVv2/4Mvhe22gYDa5U0sE9eqd1T4NnNBajTZzX/v1tv +qZMfzJiKmhyr/uI0pxUrefZ6uPwUwPh3/vMTN6ZkeUTQZQdNlYOj3sqtU4hCKKeE +XJ0VlyMfyPltDuentnzawiARvcj5MXKF6XuOKqQ8BDYQqVtWM+05+K7UU4bcBZOw +aXRAgMpbP8lwCTq9QnNGsjDk2w0tCHE9OBcbbEXWn+eRbIkxAd6oniwI9mTWxA95 +srGBudB5eXAEREGJXoCpeq6R9vIKGWrKXUjAecEyjdaa4gG68HGv1jhYSBeXuIHg +boOAZlEYPs+tT3QmHiv9lA3uAPa+o1Kt90RBoVZlrYNNt2oWg2Q/9ZKw/biIpAxK +UlaMW1yoR+TBZgeH3mcHGTPM2LlCt7NaTTUMeADb5K3GgpO2MIX4xF7bVhGxZeuJ +tUzp8hZLJkJ9QV2iyEfVLp0v9YgnIy3RqdNNJCIszqRjnsMPISapge8phRfAG2t3 +68E3aMv2j46QZ6liNQIDAQABozkwNzAdBgNVHQ4EFgQUbcyHqiFVin7tmZVf2ucb +A1bt7UMwCQYDVR0TBAIwADALBgNVHQ8EBAMCBaAwDQYJKoZIhvcNAQELBQADggIB +AD8jDS+k1jB6ax0vYU27cmximElKbBfZn0Rhrd5JlUuockKaMmSY26s7BSY1bkOA +W5wFpwwmADGm1OOK6cyEJADzA51ZupnD7M0nHSlhIr23Z+8bsUmO8JsniWFBNisd +h9SxAZt3DdLt5RGIrObdSnCE+C7JrzLFELC1W9B7H7anxNFDi+Sj9YR1jV1yLe96 +GoIGwXFtUlPm4cEJTsttl/3hDNEJrL+psBx65w7CEg1bsuV7AYTmevrNTjyqOIvo +Z5UBobfhQGErx1hcEbsnXnW0RQEkemclWz7xNOE3/GSOC2smAVeLcHKhJneLsCo7 +fDnGD1xuFNCC02EGem2CT2tyBBMk8NJSJsCtcvIm3FYJkkMfJf2pDVDjMl1JgRiO +ixJZ2sdFff/I74VUtCjWD1iKzXitFtTaShNTJUZxsDEb3EMv6NAZFGHoUWjgq/xQ +gFc6vUnzFxG++cjah9EUjDKU3WqnfMlCpcQTh1C3mw6a0pCUHPFandpbfE10RWyN +d9CaFbIhMaHtjRtdmlDe7IGkjv8roaN5IscCKediYHkgkDF74Q92NTzQZQUmU2Wk +xyMRhT3pMDKHu3WL2JPg6UrJK2qXF69KyEiYAPXJXlfBQBq5XZLINeiQsCDhCYas +aOzjFYS00hGVKrYBdNc15PlZhR93d/nRPNcdxAAfRrXy +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/crl/provider_malicious_client_trust_key.pem b/security/advancedtls/testdata/crl/provider_malicious_client_trust_key.pem new file mode 100644 index 0000000..83fb7a7 --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_malicious_client_trust_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCwqLQQN8XrAdoh +lLIdP5LyeYtL1S9ZtXFBMdmxtHpLZCuY2orlOyQeaFZrut3DRXmedPFH2n0lk5Gb +RQAytyo0bSvxaWJSI+gG6rZsxebFqn9nF7eTFUxpkna9hGCpvhmaMplSjNrKCoVH +fkkkBVoHHpW3vnrcf71mFZAQ+8SbfT7i3UVv2/4Mvhe22gYDa5U0sE9eqd1T4NnN +BajTZzX/v1tvqZMfzJiKmhyr/uI0pxUrefZ6uPwUwPh3/vMTN6ZkeUTQZQdNlYOj +3sqtU4hCKKeEXJ0VlyMfyPltDuentnzawiARvcj5MXKF6XuOKqQ8BDYQqVtWM+05 ++K7UU4bcBZOwaXRAgMpbP8lwCTq9QnNGsjDk2w0tCHE9OBcbbEXWn+eRbIkxAd6o +niwI9mTWxA95srGBudB5eXAEREGJXoCpeq6R9vIKGWrKXUjAecEyjdaa4gG68HGv +1jhYSBeXuIHgboOAZlEYPs+tT3QmHiv9lA3uAPa+o1Kt90RBoVZlrYNNt2oWg2Q/ +9ZKw/biIpAxKUlaMW1yoR+TBZgeH3mcHGTPM2LlCt7NaTTUMeADb5K3GgpO2MIX4 +xF7bVhGxZeuJtUzp8hZLJkJ9QV2iyEfVLp0v9YgnIy3RqdNNJCIszqRjnsMPISap +ge8phRfAG2t368E3aMv2j46QZ6liNQIDAQABAoICABz4SXEIaGc5ykbr/sp/mK41 +QuPYbbepIs21alTzOwPehSy/l/vv9yQaaaOohQHnBIL4+/FT0eaF3OCoz9fBREVN +KuHfrY05UK1Ds89CI+5B37ss/0B6Q2njqB+7k+t6HnI0PoL0UNPFpmYbPkzPKNyW +hihX7wd2Vj2lpxa15t+1ygiDj0XA/pgh2zsz5T0N2S3HBPkJ458D9kuiFDW4zPjn +7UtyNEuIbnJ6Q93rwjYuuTDEQyiMaeBQZd6tuzhpTZrX7TI8gxZUwL3whO2oD2YE +CzaQRn1aJ9soRsj84v3UlgR8xawLIqhE1NpDd+zQLtcdvKz+TNll0g8J6kes++YW +m4hBgSMeOYRPtqCV/CalTi289Ndm9pYfrlefF1WVVlXFKfHCXM2qSRSh5/bdPPFC +joBm+WJuuPsk9JpSekDMl8VCvinxjWB/8LXvY61mqx9dng+5YtfLPKYHxvfehR0e +R2XAY4zSB2ocB+ZqX1ZZuLxf6TQXM908JIJWXbcLBu9JggUeWG8JGtKC5sv5OmoQ +Tizi7QIZoZU16saTJQcE8Sd3Ek0pNzJ3yzj4CwfpX8EQb+FmLW+f9H2D+j3qBdt0 +WJw97hUrCkSOodi+XpLV40cL/X1EH/uT8jN6paUNJR95SwwBX54kRK2fhmbRJ8tN +RDPE1VxvmqFCWHuXwjFxAoIBAQDe7xjuA/QkL0kM5mTChd/FKtCKHY5s+EVxU/O8 +3B9Isazo372lEH8+jOuYUn6fz3m8AIbpwtkl728FnU1icRQUIaovD3HBh9GxVp+X +TqEl8o8MPzUno+DCTYrBFuEpobbjsmSoAoCM+L1xzsR/NQ6Yqh03ufaj4Jme+BGf +4Fsr9cmi/60smP8szo91sxMb0tb6fOYCIUkE58GiANLgeGxe5V9LdgmorsPBmjVJ +bWmULmGSyambEhHt+fJ5N7TBUL6tS4gUYMLimDev+8H7pMMEBAYqyvJA1yJa6hG3 +l6OXHBYLArqtGXXC1OljNMyQ7PxyyMsaeFYOe8MLnGVtWYIxAoIBAQDK3IZCaPoG +a+4r4qBDD/Y7xrHh0QCNLdMFcZdEbvZJfYT5vU1X67Jo8jXEWSlo3S5O+Gvqls/5 +NKV7E41Iz3UDqNxZ36x3xRq/vlKWD3JBLGM76Fd3jL5i5F3DnNNqfpTISuyZIyxX +zNfkoHmpolOr82mRC7zmSPvozmN2Sh+Gi9RY2dpJBki+/ESJJ+5w84I1I9Dh9ybS +Yc0SPHWJH6ZGPrGO/i8m0GmPKUtVu5KQRPhoIN6t1IEfPKmwHTDIGm9D8mppesLH ++W0CiXdi0pzbaMjtsQqssytoftZSpEmB62PISCerJXHy79jBzKeavOzUiqnjBYXO +phl+GrL8CDtFAoIBAQDXtzZBQbQgu1yNTfabv5zalWY67wSc7tOLKHgF/F2NIte0 +cqN4MHFf3k4uE00RaFpcD4p8cZ1bNscQALkbk12haT3a0a/6W2kNl0tPxrbqGD5Q +1Gyj0dAvU4b69h+kACYPR9RcOieXwSzXDgNXL8jS8nDZNmmxAyjDCTlMoXS3Idsd +jRdNuzHXcnygoEnSN37r3KVX1EtqorLcBr4GlKAQZxSB2VLZVVp4YLZFOcjaqNKj +jd0+/wo4Yw48Oyn9kRsgZqDjTwnk5vOjpxF6ZWCK5zXsfHpTQZitria8ps+V7Yhq +2RY3XxZzE0BOTY3QgnB4xVC6aUykMR93gbsnR2BRAoIBAGH7HYXZ+lk7rC/aKBn8 +DaxVjflJ34BRD8ljUfKlvaNFUwLk4gSDPQrfYgTSI+QYYJOX/VezPARb30mQ6f6u +5Q+9caCfHkhDFWZjYLRGBaNg8xUuZYDCo/pT3s3qY1rehLIxLhHRIUvPDr/ImrrK +Qqucx/JcvwJjYfjSJswthQiRZSD9KCd5N423fsrYVJyNoOhVwkBCBag7wLb8KLDw +bnkjMtOkBdYzd6jEAzUHggTYqzpwFd3T9AHIZRVBJByiV/dzkN4dgxLfcD13dAhx +PX3kIJhdmJBNgbvY91+3JiHwNaO45iAN1/nEyubgGFvuwFzwFJooQrbLFykHcEe3 +GhkCggEBAJuQ3yGOrKsj/COSA5HVifXcgC9EyLJtFBP7KlPzYLi1FXjr4yx4//7M +ZYbtQk6gEvNYumvsl7bhqGX2LuZXBmjix7WMGjChGC35Z68cp8CRYVtldkUGhWmm +fF1ha1aSaOUPIa4a7it6mAuWiccPNWG17blgID0hphC8FL5pPSmCpz1fGvcONCV/ +uyfn5NbSG6bjKNKAADnKVT+XmwSglajljOKVbM3YJqmfS6Rwb86zWwsrS1waOmYB +NooJFHnxMy2oXjnGJHNt/MZD4Sb495HiLWyIS4Tskyu2DypJFdHdK7Atk/u7ZNas +s8ldeq6PBcV8lVJoy/kET6KgDhugg+U= +-----END PRIVATE KEY----- diff --git a/security/advancedtls/testdata/crl/provider_malicious_crl_empty.pem b/security/advancedtls/testdata/crl/provider_malicious_crl_empty.pem new file mode 100644 index 0000000..6fdb432 --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_malicious_crl_empty.pem @@ -0,0 +1,20 @@ +-----BEGIN X509 CRL----- +MIIDWjCCAUICAQEwDQYJKoZIhvcNAQELBQAwSzELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMQwwCgYDVQQHDANTVkwxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMg +UHR5IEx0ZBcNMjYwMjEyMDc1NDE5WhcNMjYwMzE0MDg1NDE5WjAnMCUCFBvcyvmo +ckWEbX+0FnMz8yaiiqgFFw0yNjAyMTIwNzU0MTlaoIGZMIGWMIGGBgNVHSMEfzB9 +gBRtzIeqIVWKfu2ZlV/a5xsDVu3tQ6FPpE0wSzELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMQwwCgYDVQQHDANTVkwxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMg +UHR5IEx0ZIIUaExYOx/op7TEm3y6Ykmhy8CWn8owCwYDVR0UBAQCAhACMA0GCSqG +SIb3DQEBCwUAA4ICAQAheuHgVWPt4TNAtR6nhdmEBvI+g5VtrJeGAskKKPRjeGVs +eQdEc5w7bJ5D0oRftU/G1k/P7U4H9Moo7jZ7K3XyRgaMAw0RjeTFIs45tJLFuyqq +5MIygJBnfhAazFWZ49vorBnei2A+9Pwlvn2JowVjQ9Eo3VEfo4fIgWwrZISdKADr +I4cVyeEjNDOznJzwPen4QiOVbZu3pMSG10UfCI0OhuSlR86a5nR0Hi8yBCdWyDTl +mrj2o54ywIVYl96j8oC50slm0VGfde5Xy89gypb9/aXQv+bKk3zicw+N3CiXeVGi +Y9i2BWprSMl4qDI6M2gT63z1zDvPwCQHhjRpc2o1Wri6k5jF+zXezF1kwNa/Hru6 +slMR/4RhDsRBsfA+pg7Axg6DGD+yan5NTkvShWTYQW2KXt364pH95i34uOT+0L50 +RF9lE8rtO6gxFGWSkIOvwqiWsNxXqE4ovhwiI/ewDt37s3UFES0AainBNTcmhxXG +2uF8BQdGGatF7pQ1Qx7Nxq5Xr3WZinxcTu55P4ZETBJaGqT/nPHGh1oc+mRQtZ2p +TjReOOo4BDAaGjni0GNIO89XIgyzpI9NaOAo61gxfjVRI5eBl39vzg3QRgi92TQp +GSPZ+CVZKejvCLtqZrCrBBVmjoZTIkD4XJ3xlaRTAMYpS7CNh0MNcGobprissA== +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/provider_server_cert.key b/security/advancedtls/testdata/crl/provider_server_cert.key new file mode 100644 index 0000000..1ebcbb4 --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_server_cert.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDgJa+GWpCRg6uA +/NXFs8tLBO4qAZP0k1Wpr2ku3glFs7utO6ZwXG92oVT0qF2UqXdRPVsVjFkHpfta +ZZOsTAwghAdF+fGdb4x9g6g63lKCzbSUUxh/WSJFCXCS0cgrQ/bayFoWn0M54j3L +4+yEe6Ub08/vhGyO7lkt5SquqfwChjPapHGzlwZLZ6clt06jrm6D/m6GZypjmjRv +peDYj6dkIHSt4ZjYQP7R7Gw1YsntyGjTwwPIulTB65YZghzFZsw6N2iMwW6iVKrT +gW+LNNh4lqUTzDs/m6Lhks45unA27zB+RsQ8QRLXSboyAR27SO43PokejArCBTva +nZuTwYrx9aBfvxssdojGGU1j47jbAOCewi5OEp8/m41Wa4JUO+DMQzcdBQH9TxfW +WafowUSMoBCrSNrkpyhcbLe4A7hcYl3jTppr4dGDiDQDP40T/HM5U1AjdiqqRjct +pu8KfsgNxDOfinzOJvRGhPBO8yzKYj9y9P35OGsX6TDT36YAgO7wInP+JxVs+Bd2 +Ds2gbbv2T4mljnGvLV8sNXjgE47SCgQ011odtSrXe9maopbr7vU/WvVDT5kp8/W5 +JpjrfNuW1lVYlBcmM7tRlOW0mm0eczNASKU26ZO99HKKmTNRPM8av2unN4EVLxSK +8kGkQgxRMGt8vz8Xf9H5ZDTIq3tfIwIDAQABAoICADBdUVNfJUSQsafiZkoDcoEc +bjtYHdGvHNPBSqPXOw70KjHF5jLmbxKc2xTzY0XZjypTX9z1bJxu3x5xPnz158P4 +WQ1rUgwTbrACgYE6SXl541YB3A5WcEOPNuAnLdbQEmrAwleRQ9Mwkv70jitD9qtb +d8mJvFbW7R0vDpejjAILbRLnWrVSiAQrOHC17dz5nVUTyEtt5UKammfg1fREguzi +5+pA/FblJ1aqeerHByUskhnnQWDFe9Zf/AJDBew+MyD4pbGBZ1rVhAqhKi1SWMck +UjhEYCBod6vOnnrsVJw13jPRFzSdIfCcMvXpMb4gjW4UK0gRFZ0pvfF9An3OTClj +z8sUVl9w3f8viw3NOxVEBCcJOFtKHE6F17Ww8BcoeWSo1hDvNneHdN7znTXy5/jq +hZ+ja8Lg+dDmNwc8z3ScgRSYYSVxlUuSQF2TRg69akXLwQQLaPeJ7I5/DEq49OO7 +xyjHlPZAj3CBABfeoycUjZS1x2s2GOFN+ogTyGhYqcNjNJKR1xRqeWINqBHfzHDU +nbly8R9Ond5mu9/KGfea3oho9aBcbfHUatGKcoMnxh3MKQTX2bytYD6o5+EANyyb +E3A9QBKCDhgALtGwB/jH2DnSEVuL4t7hA0v56j8hq0pEkhAC81bAzitFBlS6XS5k +eARwQgaIov5YePY1sAdJAoIBAQD4+PoOHlo870jcFNNXOFWeg0DqdlBgEGzGaLJ8 +kYm27JLl12SxpdTZRqw1YFkEw1gSAZ2qWu8deK/eVHd8tQVe8ZVNye9Ewu1aadYy +6VKvHd0RkmXJVJ/+n9fTVNtkeOuhPRPCFBhE8YpiuZOpyvWmZgtbm4PQfRf4rhcw +lrS+QyHSVQi4+blJaZD9VBK9q2y23e6+ThDBpyEi+DdNdWfMW843NKn2b+DXsZBX +7f/qP7wiF2YrzJa0rz/QXlY5UDNLwm4mgkOXEpxoE3mxvWA0yzHLcaJv9HrdJzFm +k79Z1rw5IXzai+mphR5iqeoG05Yw+3JX6NH+pYET1/py/GNtAoIBAQDmeVN6G6fA +BwOFMhhyIKLx/Up7vxQL9bGS71D0LSoFySv7cIYsl+dl5Z0ytHq3WL9YOkf70MkL +Ymk6qS1gfp3HveUV8hujeGXsZsX+SlL4sysWZ7L8hulC1Ac/NnBAmtRHqRuVLl3D +o4csW+ZzM1i7HPY2DIpFFDoScHzhvY6K5IE6tJG1jO8WbvU3PHsrnVDuhmuCTWWd ++IJTlEq+yjfOykkNx53Q+RbuXlnjLg+7L66XxJdob8koMePIMrERpfxm2ug+QPG1 +NuQFikWAiztlVCF+B/CQBsyya0hywnRKAc1fxx9kaPJP+cMS0DJHyXixAaE3zqUK +0oO6jSpa7KLPAoIBAHti8JHUqwXt1F71lzNvgMcY5zALSJQDL9U4h9RCUEyue7Ka +h5WeyJiRdMDTKeq5YMkzsc5+WGhzqjz2AW11TN8bvNGbVQ/vxq97KiN7DHYqx6dw +tS8M2GnZD531PPFTF/uFiGgziz+HsPxyWeLY6dr4UYKp1K6bgCjHJkj3N0XfgUB6 +0eLSJ+hg//D8HHmRHkSWQj5f89/1EvAAsW+a0sEtckpbuCugkH50ykM5eQ2/Xl5K +2GC2eK28+FQsnHC09WcDSZGeFx8kowzVMgdLAgXH+bqIa2cuc0FsrgfXCwrb59Ys +aXLpyfgwN7nP2WJes908kBNPF9sqbIjIDZ+0wxUCggEBAJEQWIyJD6L8Ryj1CRH8 +nNM3nEQbVuDYOnbDju7B5qtRng9bGfjDe/BVAAbENmFkyLsdo+VJ2uEBhj5X8anE +yEbKrYCMrPzNcUnEvmZ6HZNQIpStnKj5uaRIlG8jMrBXQ7n/JM9XKclUCmbPSVPF +Q9oyNn56kiU8v/iPOOtVRn0Bqp1qvjPJi0tRd6LWvKgNEr2vecfAM2+k1VMKCang +2hOcmzLDLAA2aEqMtIMboOcu8fOw16PyiGh2TmraDT2Qfje3HWrhscFf1VHvYCOy +fyYeOB59nNrqjLjYcfdZkZxrWfso+AdZTvsrt68FwEAS/ZZb8j+QH62aJzOqWrh0 +LV0CggEBAJXTvwMd3mGBAg45NEmYrpSZDETyDUiPW2S5aUbHFxa7qhcq+b36264y +4kwICT40NwzRW75Xc/zkqdc7tazzZPLI0Rr2kLLPzun8ykl+QA4FX5y/5k1Q5+3R +IGeshrDLUgysVZzBuQpREWp3xfwFXzvkuRA6n8D3l9pwMiYgZrhQGw6pNDrSSooh +995gITDkMYpEQK1wyBkcC9AbUbNcu9fYxhN5YARyl+ppGHX3ogak0JhldlDU6Jyk +KjYO1eKxyc9FjUiodrkMG8po9iWPfwkErDlK+eHTxUwlsWpVS1QzGDHBPGHRvV4U +x3/2R1yeRE6qET/XFsGYa/nxAIHY8Ak= +-----END PRIVATE KEY----- diff --git a/security/advancedtls/testdata/crl/provider_server_cert.pem b/security/advancedtls/testdata/crl/provider_server_cert.pem new file mode 100644 index 0000000..02b92ab --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_server_cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFgjCCA2qgAwIBAgIUG9zK+ahyRYRtf7QWczPzJqKKqAUwDQYJKoZIhvcNAQEL +BQAwSzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxITAf +BgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNjAyMTIwNzU0MTha +Fw0zNjAyMTAwNzU0MThaMGcxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTESMBAG +A1UEBwwJRFVNTVlDSVRZMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBM +dGQxFDASBgNVBAMMC2Zvby5iYXIuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8A +MIICCgKCAgEA4CWvhlqQkYOrgPzVxbPLSwTuKgGT9JNVqa9pLt4JRbO7rTumcFxv +dqFU9KhdlKl3UT1bFYxZB6X7WmWTrEwMIIQHRfnxnW+MfYOoOt5Sgs20lFMYf1ki +RQlwktHIK0P22shaFp9DOeI9y+PshHulG9PP74Rsju5ZLeUqrqn8AoYz2qRxs5cG +S2enJbdOo65ug/5uhmcqY5o0b6Xg2I+nZCB0reGY2ED+0exsNWLJ7cho08MDyLpU +weuWGYIcxWbMOjdojMFuolSq04FvizTYeJalE8w7P5ui4ZLOObpwNu8wfkbEPEES +10m6MgEdu0juNz6JHowKwgU72p2bk8GK8fWgX78bLHaIxhlNY+O42wDgnsIuThKf +P5uNVmuCVDvgzEM3HQUB/U8X1lmn6MFEjKAQq0ja5KcoXGy3uAO4XGJd406aa+HR +g4g0Az+NE/xzOVNQI3YqqkY3LabvCn7IDcQzn4p8zib0RoTwTvMsymI/cvT9+Thr +F+kw09+mAIDu8CJz/icVbPgXdg7NoG279k+JpY5xry1fLDV44BOO0goENNdaHbUq +13vZmqKW6+71P1r1Q0+ZKfP1uSaY63zbltZVWJQXJjO7UZTltJptHnMzQEilNumT +vfRyipkzUTzPGr9rpzeBFS8UivJBpEIMUTBrfL8/F3/R+WQ0yKt7XyMCAwEAAaNC +MEAwHQYDVR0OBBYEFBkCLxjvqldzbym9yZG2rexNVOyhMB8GA1UdIwQYMBaAFG3M +h6ohVYp+7ZmVX9rnGwNW7e1DMA0GCSqGSIb3DQEBCwUAA4ICAQCXpe2iNV0LXSxX +F82i5WJ4Ihbb9raOciij1f06Dd6mFFsfTDYExQ2VxsKwW7LhUwSW7SaO599BhNLl +IsKDtIaz+BPfGjZw2dPUJfZXyCYuvDDg9msPDG4JKc6KqpOjmRO3FhUSQGOm5FZH +79OQc+TaGQkEBSEdC6MsCsQi/Ixuz2hyUWtcnIA5X1W5ZJfFH0G78CXHc4H4eRm+ +0E3GbpKTVRFPUJdhALPZ39obO5IpRg4uNcWQvwku0thoAaeJcic1Z0ZMybKVeNTE +BvMzot/2D6/hDEqln9o8RoQmdcBCM67eGZHB/LQ1YOf+Y/kMmoGq89RMtrx9VO4l +TpBnXxwkGCJ9Ex9vfkAliPpmowT+iH1OwYJrJkCfW24SWnwBgAZtb01akHoHRUTZ +FYdihaYt6ejzjVwlUJSWpqS9CnS8e5Nn83HJrKfkihhiEJJv0YZn30ohrWVWe47Y +2rU7onzFNddUpsLKAuwjawQirF7Ooez5YJ+mYaZ57QLOL9QdKjA+diUVP09TJu+x +pTvGpEdWF3f6YlAMreepm3honuugHm8/5+5P55zfCa9LRQP0IneQR2L9Y5Hbm0LB +z/1NpQ6YgvnQccQwq2zxfenntwuG/nE4DFgl3fFh7SaXbJ5VK0hP3a6qfAJO1DR5 +oI53aw/lvbD0GineMx5byejYrj7Zvg== +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/crl/provider_server_trust_cert.pem b/security/advancedtls/testdata/crl/provider_server_trust_cert.pem new file mode 100644 index 0000000..de79295 --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_server_trust_cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFlTCCA32gAwIBAgIUJJfx0gNOeQy4ksfwESsekJHnYFUwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMSEwHwYDVQQKDBhJbnRlcm5l +dCBXaWRnaXRzIFB0eSBMdGQxGzAZBgNVBAMMEmZvby5iYXIuaG9vLmNhLmNvbTAe +Fw0yNjAyMTIwNzU0MThaFw0zNjAyMTAwNzU0MThaMFoxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJWQTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRsw +GQYDVQQDDBJmb28uYmFyLmhvby5jYS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4IC +DwAwggIKAoICAQDzJH0OV7h7bTnKOJdrUzV+zFefVobL6JL/rOp8ND4MmVyoh+Cl +kg2JyZUUEVrFPFgXzzIqBB2YWZd8KjsSufir02LIsCmc5j/h9QykCJYCWhgu9xbM +zpRP6Wo1aZOg3WbXylzYEj7vFAylzcoCIocHXC9019hQmSnRKs5L+jHPvi16RSq8 +SY/we3Z91yB+39EtSY6zkPgtwHnowbeDex0fjNXH3GfzSFoejXwkMIv/yg595AHn +sx35qjKGfe4zaP3ddndnoC9mKIn0Z5b8q4fTzP0p8m0RseqJOl641IeTK15+gEMt +/aqtZBsi9ya2sCIEB8sF6+xLHcSsr7D9TBGSS80jz9i5kCZcsq9aIsJlzTBVRL+R +0gVPnGA35XM8i9f+2OKxrR3rF5yXX2DA7o0tHYWnDlB+c/YedAAJOZ1nn/nTVZV+ +oaC69DOknwfMnbBXxNivduCP9MTygWimrwAxXr/58Lk3xApmzc/ZYO2EbhNOx2sb +0iJ73/Q4zJBx4qn3e397LoYU4WKhaUV4lQqUKdbuIpaAb8p/9RJz4IRy9UzAdqyc +oGr1WJkSyt6xcAJ0kCHeZszdbSSFkOOYYwW65x/7vAsvgBsgY+nmiHH+LtKqIK87 +Ub8LJiMZii6aeMGwXD7sV3x2PzqwsoxtftaIOVIz8K459p+dY8qpSAFD4wIDAQAB +o1MwUTAdBgNVHQ4EFgQUXWT8wjMCB+STUF9WMHSZ2nmm6lMwHwYDVR0jBBgwFoAU +XWT8wjMCB+STUF9WMHSZ2nmm6lMwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B +AQsFAAOCAgEAt6pZJZVvBPJjK48BVWJur7PKSdaTK06carAGvuzWY2jn++bNqOxr +BYM9KaPemouw8TxFDp6MuXjHkLSeggYXqR48AzCwg78Kegdfox9Ki29Lg3PbW9II +rqZm3Uz7pDhR+1ydQ6S6Jn3Vddf+l/V1REF0hY8Y5XkJex+Li6Xpjk81TTS0LYd1 +76vZo10YNRxCrByEvl9/UqaTFhSYn7ZDFULx0wGySMRr3SVApbqA0fSWg/d2ksY3 +Ec82F597Tegu3jUyjw8pAHA+gPDQuALDsoThbRX9qL+6shyLsSYO4G++XreQkVTj +dUcyVQUHxrjYZ6OHD7BXBd2C/tUcCBoJOh1dbA5nQa+xa8NURb/U6/zRvNon4MdH +RMiLHFHLHxTZYZA4yR8yAnwNbtFfYq1r50xhAAdOT7siat4gVvtgCVWJaG/V6iPG +vXXI7eHFwFrNPpACDsROTIqlHjiLQ2FOcHuWqSR7OtBXqvnKMUQ5bEkxIAN0Qdgu +DNfr2NkfZK1CLsJGz1+DA1jcNiGcN5AEwaZSeP8/UUzmUfMkeDm9teHTxgE7GUim +33NNbIYxG2Z1ay55G2Il9JB5rIcefpLcmtiRSumvHAD3zTMT9eNSU0nszeTkwsNI +EYmJVgNDZSkjM2+5Bin11LszKVmZsCwwtWKj+5i12FOhfFLugGkSFmw= +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/crl/provider_server_trust_key.pem b/security/advancedtls/testdata/crl/provider_server_trust_key.pem new file mode 100644 index 0000000..5dfe9a9 --- /dev/null +++ b/security/advancedtls/testdata/crl/provider_server_trust_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDzJH0OV7h7bTnK +OJdrUzV+zFefVobL6JL/rOp8ND4MmVyoh+Clkg2JyZUUEVrFPFgXzzIqBB2YWZd8 +KjsSufir02LIsCmc5j/h9QykCJYCWhgu9xbMzpRP6Wo1aZOg3WbXylzYEj7vFAyl +zcoCIocHXC9019hQmSnRKs5L+jHPvi16RSq8SY/we3Z91yB+39EtSY6zkPgtwHno +wbeDex0fjNXH3GfzSFoejXwkMIv/yg595AHnsx35qjKGfe4zaP3ddndnoC9mKIn0 +Z5b8q4fTzP0p8m0RseqJOl641IeTK15+gEMt/aqtZBsi9ya2sCIEB8sF6+xLHcSs +r7D9TBGSS80jz9i5kCZcsq9aIsJlzTBVRL+R0gVPnGA35XM8i9f+2OKxrR3rF5yX +X2DA7o0tHYWnDlB+c/YedAAJOZ1nn/nTVZV+oaC69DOknwfMnbBXxNivduCP9MTy +gWimrwAxXr/58Lk3xApmzc/ZYO2EbhNOx2sb0iJ73/Q4zJBx4qn3e397LoYU4WKh +aUV4lQqUKdbuIpaAb8p/9RJz4IRy9UzAdqycoGr1WJkSyt6xcAJ0kCHeZszdbSSF +kOOYYwW65x/7vAsvgBsgY+nmiHH+LtKqIK87Ub8LJiMZii6aeMGwXD7sV3x2Pzqw +soxtftaIOVIz8K459p+dY8qpSAFD4wIDAQABAoIB/2GzlcSGsujPrUfBok5zCvW8 +Mf8Mr6NWmdFbDpuMTFeMERPUIajVz+jF4EMk+OU9htqmOvyQ7ZZKln5ZbseSCXwN +SJlyXNcIYLl2hwH+4ItJDN3+8YOHaPABdxt51rc8r5EoGNjQkmMmc6vTdOm/vmxY +GSkG9wajvNwNTDyJbkMBBxi2nDLKDp2e0m2oc6kjqwCHr66OfCBr6Zv3A/N4Y8wz +fYeEKvM+YlxT4lbU/qRP2xOZ2Valri41bZKHSqrCC0Dyhin8YLZ/wD2AsKW2BMg4 +dYJljtKCUproPcOYVVGABZnw5TpeLqafZrdugqWCN3bhlIbMn4rzz3gwhUdZjJPY +eSxwY6dtE4lISaxdXKUmcCir23EirUgfUD8UHj3LZQsv8lHZP21SZ6CR+puZEzT+ +KIf15aYZ6vI5jqfzcpSemif2ar8VHJmpEewaJUTxDivpcj8joh/gvKpzItuWSV+z +6nX4f/0sYDasT0hZcYA7c+0LK5seiGxpZKyMdvYHGARpTf69f8rzIlN8VNaWzGzP +y1MDWUYX8Zj8cOpH6y4L2ySOE78UtxGSZtYmaFOONcgf8lukN4dt/55e5MPoWNSg +pj1ugkYuEo1jPdfD4R8TDN1p1ekfsJPA6hqSqArDByHRaLmYHj4HUvLdkiqEkHAA +VaZRXax5zsmPChVtw+ECggEBAP4Q57g3i+OuVTUBVzT9adtEE/hsVffvn6+v8iUb +vmYxehjXxn9+k5vnGqjgxVF9DuHoueEA3E4S6EdhCpevbNa8DqI9VK1oeEWq5fd5 +6K8NVI1e4GSXsIb3rzycJ3/CvJRQUlfS2HMFJ+FLi7Tcze0U7FfIgoR8JDaqKY+C +3PiAGuhAYhfT22EoB9IG3ajjt6q/xZJ8oyVtyGXw4/PEKiRCrIs4968gNaQkOR9Y +36hURE66M2rro/51s8M726gQ1u/yNZF8evnqExwPuHzZMyEdFqCPQlqjdsCw1Mck +dfgrfH0z1Gz3aaJ3QNdTJ8HgMFWiXt7zhGpwOv4B5kUWgJkCggEBAPT+S//U4fX1 +T07xUEpoqmMAJvMg9rsPdJcHNn5MauN477e8S1YVWb6P2ia1iAcFrOMrot9shV8D +cfp2SJECS/wua962ew0GoQ7JZ47uq49DMNmk6Rjb6Kbzgr/9ZSXmc/OWodIExMFK +ElUIoiiiuNNy2KceRwAMwUst0pMoxQc9qJKNuK3rU279pXcYlBZjT0rTTbjXJhcT +QsUidQaXbbEH+/VmV9X1jFp0REeyS0WWhOZT2y8IucVWFXup4zP4B9fL94UYol0U +LgCFUaBOF+HJM/8jmiznTl60aKlYB97e9dLgaPMGbg88Wi115MltXZ+MISIHVdw+ +ooNt5fwm6dsCggEBAOBxvsMXDxXMZKm2zXuag3GY6quDyU6G+eMS5C+0Grfc7agU +tt7ayzvnJb4bEzWx7PvVxJ/pSrYOLfUg3WKzLstkxui8lZm7uMSS/SVrJQvAEvnw +3vr+powVM1GwAZT0S/QaISREt8Kkw15chsb4aVMQMNo74FF5+ePw31ZQnTVKtnqG +piG2nw9tdbstJZSV1yOY1slaIiZmnaqw7C/lE/WEkTlM0kJ7iee/uFbhBHSIPO/v +voyuLuoUwQGwV5RZjAfdSUWFWn59MvSPTsO8fVa7g8nDxTKdRcNkdBSZOT5L7GYO +65J333IPN9EBPRYhH5IHJxh+uHPvQa4zr2FJR8ECggEAGlIr1huH/0XQtHmGl8fw +7lv400wqVeSOMR+sQhtxrGi00HehXGlE0A1icaA1MhPxBCMPB5QALDoRh8gnebTW +dwyBuFbAl6Crswv+XUPVGLouSGPIS0xtDWrCFGBj+pNsx7HaRtcOUHnlyE76JQV+ +d/Exx5qgJCg6qBoPMpJBwMpmDwC846qRty33FfmFB3a58R8MjZxH0ljWZpmSGxgv +x8JE1pOdQjZ0Us+E+nL0VVCbjKdbuAzm8IWUH0ocR9wQ86VNPXZtEzT00EeCgoeB +V1enh3TKvqJHMDOPvwnfJpmrrXFAtUNulaJ1SccNwnpGgbqrYJ2LIvNzwE0RVWrU +2wKCAQEAtZI3+rQiMrT5AumTSV28jTEv7yrLhqhOxbrFOKd+N/39Lzjk2RQzzIkB +huxn90fqkc9NlD0wC4fy5QPnlg/6zt45jbkti6amdlmRXyTaippAmGSQwZH12b16 +2VaUqU1LC4KlBunHq3hW0zD2xjfpMuPeocCXB0hq9fjHA/jVsJ3XVgb5w44+htmE +PPhqNt0i/5U6b28kKzOZpEdIWDYP8PN6KM2JHn55t92nIIML4MUZL6w2F8dRyYzy +mAD/MQU4axn8fUh9PGgQQQgavtPXKhVkRe9/fkrYwtJNM9EznVhy8brfORVNzIlA +2sRyY6DlN7u92FW/rqGXURvtONgnHA== +-----END PRIVATE KEY----- diff --git a/security/advancedtls/testdata/crl/revokedInt.pem b/security/advancedtls/testdata/crl/revokedInt.pem new file mode 100644 index 0000000..8b7282f --- /dev/null +++ b/security/advancedtls/testdata/crl/revokedInt.pem @@ -0,0 +1,58 @@ +-----BEGIN CERTIFICATE----- +MIIDAzCCAqmgAwIBAgITAWjKwm2dNQvkO62Jgyr5rAvVQzAKBggqhkjOPQQDAjCB +pTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDU1v +dW50YWluIFZpZXcxEzARBgNVBAoTCkdvb2dsZSBMTEMxJjARBgNVBAsTClByb2R1 +Y3Rpb24wEQYDVQQLEwpjYW1wdXMtc2xuMSwwKgYDVQQDEyNSb290IENBICgyMDIx +LTAyLTAyVDA3OjMxOjU0LTA4OjAwKTAgFw0yMTAyMDIxNTMxNTRaGA85OTk5MTIz +MTIzNTk1OVowgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYw +FAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYD +VQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjUm9v +dCBDQSAoMjAyMS0wMi0wMlQwNzozMTo1NC0wODowMCkwWTATBgcqhkjOPQIBBggq +hkjOPQMBBwNCAAQhA0/puhTtSxbVVHseVhL2z7QhpPyJs5Q4beKi7tpaYRDmVn6p +Phh+jbRzg8Qj4gKI/Q1rrdm4rKer63LHpdWdo4GzMIGwMA4GA1UdDwEB/wQEAwIB +BjAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDwYDVR0TAQH/BAUwAwEB +/zAdBgNVHQ4EFgQUeq/TQ959KbWk/um08jSTXogXpWUwHwYDVR0jBBgwFoAUeq/T +Q959KbWk/um08jSTXogXpWUwLgYDVR0RBCcwJYYjc3BpZmZlOi8vY2FtcHVzLXNs +bi5wcm9kLmdvb2dsZS5jb20wCgYIKoZIzj0EAwIDSAAwRQIgOSQZvyDPQwVOWnpF +zWvI+DS2yXIj/2T2EOvJz2XgcK4CIQCL0mh/+DxLiO4zzbInKr0mxpGSxSeZCUk7 +1ZF7AeLlbw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDizCCAzKgAwIBAgIUAK6BGFvOeQUak65aL+XAQhr5LrcwCgYIKoZIzj0EAwIw +gaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1N +b3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYDVQQLEwpQcm9k +dWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAy +MS0wMi0wMlQwNzozMTo1NC0wODowMCkwIBcNMjEwMjAyMTUzMTU0WhgPOTk5OTEy +MzEyMzU5NTlaMIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEW +MBQGA1UEBxMNTW91bnRhaW4gVmlldzETMBEGA1UEChMKR29vZ2xlIExMQzEmMBEG +A1UECxMKUHJvZHVjdGlvbjARBgNVBAsTCmNhbXB1cy1zbG4xLDAqBgNVBAMTI25v +ZGUgQ0EgKDIwMjEtMDItMDJUMDc6MzE6NTQtMDg6MDApMFkwEwYHKoZIzj0CAQYI +KoZIzj0DAQcDQgAEye6UOlBos8Q3FFBiLahD9BaLTA18bO4MTPyv35T3lppvxD5X +U/AnEllOnx5OMtMjMBbIQjSkMbiQ9xNXoSqB6aOCATowggE2MA4GA1UdDwEB/wQE +AwIBBjAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDwYDVR0TAQH/BAUw +AwEB/zAdBgNVHQ4EFgQUhWfy0gWBmkh2GiaBgnZzlQsvOlIwHwYDVR0jBBgwFoAU +eq/TQ959KbWk/um08jSTXogXpWUwMwYDVR0RBCwwKoYoc3BpZmZlOi8vbm9kZS5j +YW1wdXMtc2xuLnByb2QuZ29vZ2xlLmNvbTA7BgNVHR4BAf8EMTAvoC0wK4YpY3Nj +cy10ZWFtLm5vZGUuY2FtcHVzLXNsbi5wcm9kLmdvb2dsZS5jb20wQgYDVR0fBDsw +OTA3oDWgM4YxaHR0cDovL3N0YXRpYy5jb3JwLmdvb2dsZS5jb20vY3JsL2NhbXB1 +cy1zbG4vbm9kZTAKBggqhkjOPQQDAgNHADBEAiA79rPu6ZO1/0qB6RxL7jVz1200 +UTo8ioB4itbTzMnJqAIgJqp/Rc8OhpsfzQX8XnIIkl+SewT+tOxJT1MHVNMlVhc= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIC0DCCAnWgAwIBAgITXQ2c/C27OGqk4Pbu+MNJlOtpYTAKBggqhkjOPQQDAjCB +pTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDU1v +dW50YWluIFZpZXcxEzARBgNVBAoTCkdvb2dsZSBMTEMxJjARBgNVBAsTClByb2R1 +Y3Rpb24wEQYDVQQLEwpjYW1wdXMtc2xuMSwwKgYDVQQDEyNub2RlIENBICgyMDIx +LTAyLTAyVDA3OjMxOjU0LTA4OjAwKTAgFw0yMTAyMDIxNTMxNTRaGA85OTk5MTIz +MTIzNTk1OVowADBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABN2/1le5d3hS/piw +hrNMHjd7gPEjzXwtuXQTzdV+aaeOf3ldnC6OnEF/bggym9MldQSJZLXPYSaoj430 +Vu5PRNejggEkMIIBIDAOBgNVHQ8BAf8EBAMCB4AwHQYDVR0lBBYwFAYIKwYBBQUH +AwIGCCsGAQUFBwMBMB0GA1UdDgQWBBTEewP3JgrJPekWWGGjChVqaMhaqTAfBgNV +HSMEGDAWgBSFZ/LSBYGaSHYaJoGCdnOVCy86UjBrBgNVHREBAf8EYTBfghZqemFi +MTIucHJvZC5nb29nbGUuY29thkVzcGlmZmU6Ly9jc2NzLXRlYW0ubm9kZS5jYW1w +dXMtc2xuLnByb2QuZ29vZ2xlLmNvbS9yb2xlL2JvcmctYWRtaW4tY28wQgYDVR0f +BDswOTA3oDWgM4YxaHR0cDovL3N0YXRpYy5jb3JwLmdvb2dsZS5jb20vY3JsL2Nh +bXB1cy1zbG4vbm9kZTAKBggqhkjOPQQDAgNJADBGAiEA9w4qp3nHpXo+6d7mZc69 +QoALfP5ynfBCArt8bAlToo8CIQCgc/lTfl2BtBko+7h/w6pKxLeuoQkvCL5gHFyK +LXE6vA== +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/crl/revokedLeaf.pem b/security/advancedtls/testdata/crl/revokedLeaf.pem new file mode 100644 index 0000000..b7541ab --- /dev/null +++ b/security/advancedtls/testdata/crl/revokedLeaf.pem @@ -0,0 +1,59 @@ +-----BEGIN CERTIFICATE----- +MIIDAzCCAqmgAwIBAgITTwodm6C4ZabFVUVa5yBw0TbzJTAKBggqhkjOPQQDAjCB +pTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDU1v +dW50YWluIFZpZXcxEzARBgNVBAoTCkdvb2dsZSBMTEMxJjARBgNVBAsTClByb2R1 +Y3Rpb24wEQYDVQQLEwpjYW1wdXMtc2xuMSwwKgYDVQQDEyNSb290IENBICgyMDIx +LTAyLTAyVDA3OjMyOjU3LTA4OjAwKTAgFw0yMTAyMDIxNTMyNTdaGA85OTk5MTIz +MTIzNTk1OVowgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYw +FAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYD +VQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjUm9v +dCBDQSAoMjAyMS0wMi0wMlQwNzozMjo1Ny0wODowMCkwWTATBgcqhkjOPQIBBggq +hkjOPQMBBwNCAARoZnzQWvAoyhvCLA2cFIK17khSaA9aA+flS5X9fLRt4RsfPCx3 +kim7wYKQSmBhQdc1UM4h3969r1c1Fvsh2H9qo4GzMIGwMA4GA1UdDwEB/wQEAwIB +BjAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDwYDVR0TAQH/BAUwAwEB +/zAdBgNVHQ4EFgQU36DFMC1JOVvlCq+tunitm2xo3OowHwYDVR0jBBgwFoAU36DF +MC1JOVvlCq+tunitm2xo3OowLgYDVR0RBCcwJYYjc3BpZmZlOi8vY2FtcHVzLXNs +bi5wcm9kLmdvb2dsZS5jb20wCgYIKoZIzj0EAwIDSAAwRQIgN7S9dQOQzNih92ag +7c5uQxuz+M6wnxWj/uwGQIIghRUCIQD2UDH6kkRSYQuyP0oN7XYO3XFjmZ2Yer6m +1ZS8fyWYYA== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDjTCCAzKgAwIBAgIUAOmArBu9gihLTlqP3W7Et0UoocEwCgYIKoZIzj0EAwIw +gaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1N +b3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYDVQQLEwpQcm9k +dWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAy +MS0wMi0wMlQwNzozMjo1Ny0wODowMCkwIBcNMjEwMjAyMTUzMjU3WhgPOTk5OTEy +MzEyMzU5NTlaMIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEW +MBQGA1UEBxMNTW91bnRhaW4gVmlldzETMBEGA1UEChMKR29vZ2xlIExMQzEmMBEG +A1UECxMKUHJvZHVjdGlvbjARBgNVBAsTCmNhbXB1cy1zbG4xLDAqBgNVBAMTI25v +ZGUgQ0EgKDIwMjEtMDItMDJUMDc6MzI6NTctMDg6MDApMFkwEwYHKoZIzj0CAQYI +KoZIzj0DAQcDQgAEfrgVEVQfSEFeCF1/FGeW7oq0yxecenT1BESfj4Z0zJ8p7P9W +bj1o6Rn6dUNlEhGrx7E3/4NFJ0cL1BSNGHkjiqOCATowggE2MA4GA1UdDwEB/wQE +AwIBBjAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDwYDVR0TAQH/BAUw +AwEB/zAdBgNVHQ4EFgQUpZ30UJXB4lI9j2SzodCtRFckrRcwHwYDVR0jBBgwFoAU +36DFMC1JOVvlCq+tunitm2xo3OowMwYDVR0RBCwwKoYoc3BpZmZlOi8vbm9kZS5j +YW1wdXMtc2xuLnByb2QuZ29vZ2xlLmNvbTA7BgNVHR4BAf8EMTAvoC0wK4YpY3Nj +cy10ZWFtLm5vZGUuY2FtcHVzLXNsbi5wcm9kLmdvb2dsZS5jb20wQgYDVR0fBDsw +OTA3oDWgM4YxaHR0cDovL3N0YXRpYy5jb3JwLmdvb2dsZS5jb20vY3JsL2NhbXB1 +cy1zbG4vbm9kZTAKBggqhkjOPQQDAgNJADBGAiEAnuONgMqmbBlj4ibw5BgDtZUM +pboACSFJtEOJu4Yqjt0CIQDI5193J4wUcAY0BK0vO9rRfbNOIc+4ke9ieBDPSuhm +mA== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIICzzCCAnagAwIBAgIUAMUnv6Rpsr8ze5sZuemSU2FF2LowCgYIKoZIzj0EAwIw +gaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1N +b3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYDVQQLEwpQcm9k +dWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjbm9kZSBDQSAoMjAy +MS0wMi0wMlQwNzozMjo1Ny0wODowMCkwIBcNMjEwMjAyMTUzMjU3WhgPOTk5OTEy +MzEyMzU5NTlaMAAwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASCmYiIHUux5WFz +S0ksJzAPL7YTEh5o5MdXgLPB/WM6x9sVsQDSYU0PF5qc9vPNhkQzGBW79dkBnxhW +AGJkFr1Po4IBJDCCASAwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUF +BwMCBggrBgEFBQcDATAdBgNVHQ4EFgQUCR1CGEdlks0qcxCExO0rP1B/Z7UwHwYD +VR0jBBgwFoAUpZ30UJXB4lI9j2SzodCtRFckrRcwawYDVR0RAQH/BGEwX4IWanph +YjEyLnByb2QuZ29vZ2xlLmNvbYZFc3BpZmZlOi8vY3Njcy10ZWFtLm5vZGUuY2Ft +cHVzLXNsbi5wcm9kLmdvb2dsZS5jb20vcm9sZS9ib3JnLWFkbWluLWNvMEIGA1Ud +HwQ7MDkwN6A1oDOGMWh0dHA6Ly9zdGF0aWMuY29ycC5nb29nbGUuY29tL2NybC9j +YW1wdXMtc2xuL25vZGUwCgYIKoZIzj0EAwIDRwAwRAIgK9vQYNoL8HlEwWv89ioG +aQ1+8swq6Bo/5mJBrdVLvY8CIGxo6M9vJkPdObmetWNC+lmKuZDoqJWI0AAmBT2J +mR2r +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/crl/unrevoked.pem b/security/advancedtls/testdata/crl/unrevoked.pem new file mode 100644 index 0000000..5c5fc58 --- /dev/null +++ b/security/advancedtls/testdata/crl/unrevoked.pem @@ -0,0 +1,58 @@ +-----BEGIN CERTIFICATE----- +MIIDBDCCAqqgAwIBAgIUALy864QhnkTdceLH52k2XVOe8IQwCgYIKoZIzj0EAwIw +gaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1N +b3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYDVQQLEwpQcm9k +dWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAy +MS0wMi0wMlQwNzozMDozNi0wODowMCkwIBcNMjEwMjAyMTUzMDM2WhgPOTk5OTEy +MzEyMzU5NTlaMIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEW +MBQGA1UEBxMNTW91bnRhaW4gVmlldzETMBEGA1UEChMKR29vZ2xlIExMQzEmMBEG +A1UECxMKUHJvZHVjdGlvbjARBgNVBAsTCmNhbXB1cy1zbG4xLDAqBgNVBAMTI1Jv +b3QgQ0EgKDIwMjEtMDItMDJUMDc6MzA6MzYtMDg6MDApMFkwEwYHKoZIzj0CAQYI +KoZIzj0DAQcDQgAEYv/JS5hQ5kIgdKqYZWTKCO/6gloHAmIb1G8lmY0oXLXYNHQ4 +qHN7/pPtlcHQp0WK/hM8IGvgOUDoynA8mj0H9KOBszCBsDAOBgNVHQ8BAf8EBAMC +AQYwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMB +Af8wHQYDVR0OBBYEFPQNtnCIBcG4ReQgoVi0kPgTROseMB8GA1UdIwQYMBaAFPQN +tnCIBcG4ReQgoVi0kPgTROseMC4GA1UdEQQnMCWGI3NwaWZmZTovL2NhbXB1cy1z +bG4ucHJvZC5nb29nbGUuY29tMAoGCCqGSM49BAMCA0gAMEUCIQDwBn20DB4X/7Uk +Q5BR8JxQYUPxOfvuedjfeA8bPvQ2FwIgOEWa0cXJs1JxarILJeCXtdXvBgu6LEGQ +3Pk/bgz8Gek= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDizCCAzKgAwIBAgIUAM/6RKQ7Vke0i4xp5LaAqV73cmIwCgYIKoZIzj0EAwIw +gaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1N +b3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYDVQQLEwpQcm9k +dWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAy +MS0wMi0wMlQwNzozMDozNi0wODowMCkwIBcNMjEwMjAyMTUzMDM2WhgPOTk5OTEy +MzEyMzU5NTlaMIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEW +MBQGA1UEBxMNTW91bnRhaW4gVmlldzETMBEGA1UEChMKR29vZ2xlIExMQzEmMBEG +A1UECxMKUHJvZHVjdGlvbjARBgNVBAsTCmNhbXB1cy1zbG4xLDAqBgNVBAMTI25v +ZGUgQ0EgKDIwMjEtMDItMDJUMDc6MzA6MzYtMDg6MDApMFkwEwYHKoZIzj0CAQYI +KoZIzj0DAQcDQgAEllnhxmMYiUPUgRGmenbnm10gXpM94zHx3D1/HumPs6arjYuT +Zlhx81XL+g4bu4HII2qcGdP+Hqj/MMFNDI9z4aOCATowggE2MA4GA1UdDwEB/wQE +AwIBBjAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDwYDVR0TAQH/BAUw +AwEB/zAdBgNVHQ4EFgQUGOhXkmeT+CnWTt+ZbCS9OT9wX8gwHwYDVR0jBBgwFoAU +9A22cIgFwbhF5CChWLSQ+BNE6x4wMwYDVR0RBCwwKoYoc3BpZmZlOi8vbm9kZS5j +YW1wdXMtc2xuLnByb2QuZ29vZ2xlLmNvbTA7BgNVHR4BAf8EMTAvoC0wK4YpY3Nj +cy10ZWFtLm5vZGUuY2FtcHVzLXNsbi5wcm9kLmdvb2dsZS5jb20wQgYDVR0fBDsw +OTA3oDWgM4YxaHR0cDovL3N0YXRpYy5jb3JwLmdvb2dsZS5jb20vY3JsL2NhbXB1 +cy1zbG4vbm9kZTAKBggqhkjOPQQDAgNHADBEAiA86egqPw0qyapAeMGbHxrmYZYa +i5ARQsSKRmQixgYizQIgW+2iRWN6Kbqt4WcwpmGv/xDckdRXakF5Ign/WUDO5u4= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIICzzCCAnWgAwIBAgITYjjKfYZUKQNUjNyF+hLDGpHJKTAKBggqhkjOPQQDAjCB +pTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDU1v +dW50YWluIFZpZXcxEzARBgNVBAoTCkdvb2dsZSBMTEMxJjARBgNVBAsTClByb2R1 +Y3Rpb24wEQYDVQQLEwpjYW1wdXMtc2xuMSwwKgYDVQQDEyNub2RlIENBICgyMDIx +LTAyLTAyVDA3OjMwOjM2LTA4OjAwKTAgFw0yMTAyMDIxNTMwMzZaGA85OTk5MTIz +MTIzNTk1OVowADBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABD4r4+nCgZExYF8v +CLvGn0lY/cmam8mAkJDXRN2Ja2t+JwaTOptPmbbXft+1NTk5gCg5wB+FJCnaV3I/ +HaxEhBWjggEkMIIBIDAOBgNVHQ8BAf8EBAMCB4AwHQYDVR0lBBYwFAYIKwYBBQUH +AwIGCCsGAQUFBwMBMB0GA1UdDgQWBBTTCjXX1Txjc00tBg/5cFzpeCSKuDAfBgNV +HSMEGDAWgBQY6FeSZ5P4KdZO35lsJL05P3BfyDBrBgNVHREBAf8EYTBfghZqemFi +MTIucHJvZC5nb29nbGUuY29thkVzcGlmZmU6Ly9jc2NzLXRlYW0ubm9kZS5jYW1w +dXMtc2xuLnByb2QuZ29vZ2xlLmNvbS9yb2xlL2JvcmctYWRtaW4tY28wQgYDVR0f +BDswOTA3oDWgM4YxaHR0cDovL3N0YXRpYy5jb3JwLmdvb2dsZS5jb20vY3JsL2Nh +bXB1cy1zbG4vbm9kZTAKBggqhkjOPQQDAgNIADBFAiBq3URViNyMLpvzZHC1Y+4L ++35guyIJfjHu08P3S8/xswIhAJtWSQ1ZtozdOzGxg7GfUo4hR+5SP6rBTgIqXEfq +48fW +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/localhost-openssl.cnf b/security/advancedtls/testdata/localhost-openssl.cnf new file mode 100644 index 0000000..5f46206 --- /dev/null +++ b/security/advancedtls/testdata/localhost-openssl.cnf @@ -0,0 +1,23 @@ +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req + +[req_distinguished_name] +countryName = Country Name (2 letter code) +countryName_default = US +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = Illinois +localityName = Locality Name (eg, city) +localityName_default = Chicago +organizationName = Organization Name (eg, company) +organizationName_default = Example, Co. +commonName = Common Name (eg, YOUR name) +commonName_max = 64 + +[v3_req] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names + +[alt_names] +DNS.1 = localhost diff --git a/security/advancedtls/testdata/openssl-ca.cnf b/security/advancedtls/testdata/openssl-ca.cnf new file mode 100644 index 0000000..0dc31e4 --- /dev/null +++ b/security/advancedtls/testdata/openssl-ca.cnf @@ -0,0 +1,87 @@ +base_dir = . +certificate = $base_dir/cacert.pem # The CA certificate +private_key = $base_dir/cakey.pem # The CA private key +new_certs_dir = $base_dir # Location for new certs after signing +database = $base_dir/index.txt # Database index file +serial = $base_dir/serial.txt # The current serial number + +unique_subject = no # Set to 'no' to allow creation of + # several certificates with same subject. + +HOME = . +RANDFILE = $ENV::HOME/.rnd + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +[ CA_default ] + +default_days = 10000 # How long to certify for +default_crl_days = 30 # How long before next CRL +default_md = sha256 # Use public key default MD +preserve = no # Keep passed DN ordering + +x509_extensions = ca_extensions # The extensions to add to the cert + +email_in_dn = no # Don't concat the email in the DN +copy_extensions = copy # Required to copy SANs from CSR to cert + +#################################################################### +[ req ] +default_bits = 4096 +default_keyfile = cakey.pem +distinguished_name = ca_distinguished_name +x509_extensions = ca_extensions +string_mask = utf8only + +#################################################################### +[ ca_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = Maryland + +localityName = Locality Name (eg, city) +localityName_default = Baltimore + +organizationName = Organization Name (eg, company) +organizationName_default = Test CA, Limited + +organizationalUnitName = Organizational Unit (eg, division) +organizationalUnitName_default = Server Research Department + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_default = Test CA + +emailAddress = Email Address +emailAddress_default = test@example.com + +#################################################################### +[ ca_extensions ] + +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always, issuer +basicConstraints = critical, CA:true +keyUsage = keyCertSign, cRLSign + + + + +#################################################################### +[ signing_policy ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ signing_req ] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment diff --git a/security/advancedtls/testdata/server_cert_1.pem b/security/advancedtls/testdata/server_cert_1.pem new file mode 100644 index 0000000..e3902a9 --- /dev/null +++ b/security/advancedtls/testdata/server_cert_1.pem @@ -0,0 +1,122 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 3 (0x3) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=CA, L=SVL, O=Internet Widgits Pty Ltd + Validity + Not Before: Nov 4 21:43:00 2019 GMT + Not After : Aug 18 21:43:00 2293 GMT + Subject: C=US, ST=CA, L=DUMMYCITY, O=Internet Widgits Pty Ltd, CN=foo.bar.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (4096 bit) + Modulus: + 00:ec:3f:24:2d:91:3a:bd:c3:fc:15:72:42:b3:fb: + 28:e6:04:a3:be:26:20:e6:ea:30:a8:aa:48:78:36: + 0e:0b:99:29:3b:4b:f9:f1:d5:bf:bd:0c:13:7c:ea: + 52:06:f4:bc:34:9e:2b:c0:b4:82:2c:87:fa:2f:e2: + cd:7c:d7:b9:e1:8f:04:71:6d:85:77:ae:18:40:e4: + b1:3a:4a:6b:e5:33:bf:3e:65:db:cf:94:64:87:1a: + 20:46:c0:37:3a:9f:93:3f:d4:4f:ac:c4:e4:e0:28: + b6:0f:28:53:2a:cf:b9:fe:50:f2:ef:47:dc:7e:b6: + 60:c2:47:85:b8:cb:ca:48:5b:fa:9f:8a:97:30:01: + f4:b3:51:0f:68:e1:60:ab:2f:a0:ad:fc:f0:10:4f: + 60:e1:92:db:be:83:04:5c:40:87:ce:51:3e:9a:9e: + d6:1c:1b:19:cb:8c:c2:6c:57:74:6f:7b:af:94:3d: + 53:ad:17:a5:99:69:7c:41:f5:3e:7a:5b:48:c7:78: + ff:d7:3b:a8:1f:f7:30:e7:83:26:78:e2:cb:a2:8f: + 58:92:61:cd:ca:e9:b8:d1:80:c0:40:58:e9:d8:d3: + 42:64:82:8f:e4:0c:b9:b1:36:db:9f:65:3f:3f:5b: + 24:59:31:b3:60:0c:fa:41:5a:1b:b8:9d:ec:99:37: + 90:fa:b5:e7:3f:cb:7c:e0:f9:ed:ea:27:ce:15:24: + c7:77:3b:45:45:2d:19:8e:2e:7f:65:0e:85:df:66: + 50:69:24:2c:a4:6a:07:e5:3f:eb:28:84:53:94:4d: + 5f:9c:a8:65:a6:50:4c:c0:35:06:40:6a:a5:62:b1: + 93:60:e5:1c:85:28:34:9b:29:81:6f:e2:4f:cd:15: + 30:b9:19:d7:4b:bb:30:0c:4b:2d:64:fe:3b:dd:0e: + a4:25:2c:4a:5c:de:d7:74:1f:5e:93:7b:1c:e8:c8: + fa:72:1f:4a:eb:8d:3f:98:e4:55:98:b8:e0:8a:29: + 92:33:af:75:6b:05:84:05:d3:0c:2c:07:78:bc:0e: + b2:6d:a7:00:35:c4:53:1f:7b:e6:ba:07:72:a8:24: + c1:0a:a7:c4:46:e6:f2:6f:3a:79:23:00:0b:b8:e5: + 1f:e0:e2:ee:c6:13:a3:57:d9:86:1a:95:f7:a3:04: + f1:46:d5:5f:21:d2:aa:d2:30:fb:f6:cb:e0:da:24: + c6:c3:30:2f:d2:1f:21:fe:bc:0f:99:ac:ac:9b:65: + 9b:e4:83:9a:00:b8:2f:40:fc:3b:42:d3:7a:e8:b7: + 52:d7:f4:67:2a:a5:f7:eb:78:f1:0a:56:8b:56:12: + d5:48:d8:48:70:ab:b8:69:5a:21:d3:71:b0:59:9d: + 17:b4:4b + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + C0:82:DA:FA:69:46:30:AE:FF:6F:CD:BB:93:49:94:A6:D0:E2:17:EB + X509v3 Authority Key Identifier: + keyid:5A:A5:DA:B1:99:D4:E5:0E:E6:1E:94:EA:FF:FC:62:E2:ED:09:F1:06 + + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + Signature Algorithm: sha256WithRSAEncryption + 36:fd:cf:ec:f5:20:4b:52:dc:2e:38:f3:92:b1:e4:b6:a1:06: + 86:aa:2d:c0:e6:f5:0a:58:97:a9:e3:be:13:09:61:79:ed:d4: + 41:83:26:ad:ee:0b:43:83:d1:dd:19:1a:e8:7b:b2:1f:fe:d4: + c1:57:7d:6d:6b:d4:42:ea:7d:cd:34:8c:a4:1f:5b:3b:fa:de: + bb:2f:ae:56:b6:18:e5:53:a9:a3:99:58:ad:36:be:19:54:61: + 0d:52:b6:a7:53:fc:60:e5:ff:f5:7f:82:3f:c1:49:06:cd:b2: + af:25:ee:de:bd:e0:e5:5e:ad:0b:dc:2e:b1:ec:7a:52:6f:9d: + e0:b9:84:18:db:49:53:ee:df:93:ee:8b:9d:9b:8e:3b:2a:82: + 86:7f:45:c8:dd:d1:b0:40:17:ed:63:52:a1:5b:6e:d3:5c:a2: + 72:05:fb:3a:39:71:0d:b4:2c:9d:15:23:1b:1f:8d:ac:89:dc: + c9:56:f2:19:c7:f3:2f:bb:d5:de:40:17:f1:52:ea:e8:93:ff: + 56:43:f5:1d:cb:c0:51:52:25:d7:b0:81:a9:0e:4d:92:24:e7: + 10:81:c7:31:26:ac:cb:66:c1:3f:f6:5f:69:7b:74:87:0d:b0: + 8c:27:d4:24:29:59:e9:5b:a2:cb:0c:c0:f5:9b:1d:42:38:6b: + e3:c3:43:1e:ba:df:b1:51:0a:b7:33:55:26:39:01:2f:9f:c7: + 88:ac:2f:4a:89:f3:69:de:72:43:48:49:08:59:36:86:84:09: + db:6a:82:84:3e:71:6a:9d:f9:bd:d8:b5:1e:7c:2c:29:e1:27: + 45:4c:47:5b:88:b8:e6:fa:9d:9b:ff:d4:e9:8d:2d:5e:64:7f: + 27:87:b2:8c:d8:7e:f5:52:3c:c4:d8:30:03:24:d7:ac:f8:53: + 91:80:98:42:24:5a:6b:cb:34:48:57:e0:82:ac:96:d9:55:6c: + c2:c3:8c:19:7c:56:39:0a:a8:f1:b8:77:64:70:83:a8:04:c8: + 3a:5d:0b:00:4c:e5:ba:f1:40:e5:57:cd:d9:67:48:21:e9:9c: + d3:f2:b8:01:b8:d1:c0:d1:3a:44:c0:97:db:e6:bc:8f:2e:33: + d5:e2:38:3d:d7:7b:50:13:01:36:28:61:cc:28:98:3c:f8:21: + 5d:8c:fe:f5:d0:ab:e0:60:ec:36:22:8d:0b:71:30:1b:3d:56: + ae:96:e9:d2:89:c2:43:8b:ef:25:b7:d6:0d:82:e6:5a:c6:91: + 8a:ad:8c:28:2a:2b:5c:4e:a1:de:cb:7d:cb:29:11:a2:66:c8: + a1:33:35:75:16:fe:28:0b:78:31:0a:1f:fa:d0:a8:f4:f1:69: + c7:97:1e:5d:fb:53:08:b5 +-----BEGIN CERTIFICATE----- +MIIFiTCCA3GgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBLMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExDDAKBgNVBAcMA1NWTDEhMB8GA1UECgwYSW50ZXJuZXQgV2lk +Z2l0cyBQdHkgTHRkMCAXDTE5MTEwNDIxNDMwMFoYDzIyOTMwODE4MjE0MzAwWjBn +MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExEjAQBgNVBAcMCURVTU1ZQ0lUWTEh +MB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRQwEgYDVQQDDAtmb28u +YmFyLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOw/JC2ROr3D +/BVyQrP7KOYEo74mIObqMKiqSHg2DguZKTtL+fHVv70ME3zqUgb0vDSeK8C0giyH ++i/izXzXueGPBHFthXeuGEDksTpKa+Uzvz5l28+UZIcaIEbANzqfkz/UT6zE5OAo +tg8oUyrPuf5Q8u9H3H62YMJHhbjLykhb+p+KlzAB9LNRD2jhYKsvoK388BBPYOGS +276DBFxAh85RPpqe1hwbGcuMwmxXdG97r5Q9U60XpZlpfEH1PnpbSMd4/9c7qB/3 +MOeDJnjiy6KPWJJhzcrpuNGAwEBY6djTQmSCj+QMubE2259lPz9bJFkxs2AM+kFa +G7id7Jk3kPq15z/LfOD57eonzhUkx3c7RUUtGY4uf2UOhd9mUGkkLKRqB+U/6yiE +U5RNX5yoZaZQTMA1BkBqpWKxk2DlHIUoNJspgW/iT80VMLkZ10u7MAxLLWT+O90O +pCUsSlze13QfXpN7HOjI+nIfSuuNP5jkVZi44IopkjOvdWsFhAXTDCwHeLwOsm2n +ADXEUx975roHcqgkwQqnxEbm8m86eSMAC7jlH+Di7sYTo1fZhhqV96ME8UbVXyHS +qtIw+/bL4NokxsMwL9IfIf68D5msrJtlm+SDmgC4L0D8O0LTeui3Utf0Zyql9+t4 +8QpWi1YS1UjYSHCruGlaIdNxsFmdF7RLAgMBAAGjWjBYMB0GA1UdDgQWBBTAgtr6 +aUYwrv9vzbuTSZSm0OIX6zAfBgNVHSMEGDAWgBRapdqxmdTlDuYelOr//GLi7Qnx +BjAJBgNVHRMEAjAAMAsGA1UdDwQEAwIFoDANBgkqhkiG9w0BAQsFAAOCAgEANv3P +7PUgS1LcLjjzkrHktqEGhqotwOb1CliXqeO+Ewlhee3UQYMmre4LQ4PR3Rka6Huy +H/7UwVd9bWvUQup9zTSMpB9bO/reuy+uVrYY5VOpo5lYrTa+GVRhDVK2p1P8YOX/ +9X+CP8FJBs2yryXu3r3g5V6tC9wusex6Um+d4LmEGNtJU+7fk+6LnZuOOyqChn9F +yN3RsEAX7WNSoVtu01yicgX7OjlxDbQsnRUjGx+NrIncyVbyGcfzL7vV3kAX8VLq +6JP/VkP1HcvAUVIl17CBqQ5NkiTnEIHHMSasy2bBP/ZfaXt0hw2wjCfUJClZ6Vui +ywzA9ZsdQjhr48NDHrrfsVEKtzNVJjkBL5/HiKwvSonzad5yQ0hJCFk2hoQJ22qC +hD5xap35vdi1HnwsKeEnRUxHW4i45vqdm//U6Y0tXmR/J4eyjNh+9VI8xNgwAyTX +rPhTkYCYQiRaa8s0SFfggqyW2VVswsOMGXxWOQqo8bh3ZHCDqATIOl0LAEzluvFA +5VfN2WdIIemc0/K4AbjRwNE6RMCX2+a8jy4z1eI4Pdd7UBMBNihhzCiYPPghXYz+ +9dCr4GDsNiKNC3EwGz1Wrpbp0onCQ4vvJbfWDYLmWsaRiq2MKCorXE6h3st9yykR +ombIoTM1dRb+KAt4MQof+tCo9PFpx5ceXftTCLU= +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/server_cert_1.txt b/security/advancedtls/testdata/server_cert_1.txt new file mode 100644 index 0000000..f6c1ec9 --- /dev/null +++ b/security/advancedtls/testdata/server_cert_1.txt @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 3 (0x3) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C = US, ST = CA, L = SVL, O = Internet Widgits Pty Ltd + Validity + Not Before: Nov 4 21:43:00 2019 GMT + Not After : Aug 18 21:43:00 2293 GMT + Subject: C = US, ST = CA, L = DUMMYCITY, O = Internet Widgits Pty Ltd, CN = foo.bar.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (4096 bit) + Modulus: + 00:ec:3f:24:2d:91:3a:bd:c3:fc:15:72:42:b3:fb: + 28:e6:04:a3:be:26:20:e6:ea:30:a8:aa:48:78:36: + 0e:0b:99:29:3b:4b:f9:f1:d5:bf:bd:0c:13:7c:ea: + 52:06:f4:bc:34:9e:2b:c0:b4:82:2c:87:fa:2f:e2: + cd:7c:d7:b9:e1:8f:04:71:6d:85:77:ae:18:40:e4: + b1:3a:4a:6b:e5:33:bf:3e:65:db:cf:94:64:87:1a: + 20:46:c0:37:3a:9f:93:3f:d4:4f:ac:c4:e4:e0:28: + b6:0f:28:53:2a:cf:b9:fe:50:f2:ef:47:dc:7e:b6: + 60:c2:47:85:b8:cb:ca:48:5b:fa:9f:8a:97:30:01: + f4:b3:51:0f:68:e1:60:ab:2f:a0:ad:fc:f0:10:4f: + 60:e1:92:db:be:83:04:5c:40:87:ce:51:3e:9a:9e: + d6:1c:1b:19:cb:8c:c2:6c:57:74:6f:7b:af:94:3d: + 53:ad:17:a5:99:69:7c:41:f5:3e:7a:5b:48:c7:78: + ff:d7:3b:a8:1f:f7:30:e7:83:26:78:e2:cb:a2:8f: + 58:92:61:cd:ca:e9:b8:d1:80:c0:40:58:e9:d8:d3: + 42:64:82:8f:e4:0c:b9:b1:36:db:9f:65:3f:3f:5b: + 24:59:31:b3:60:0c:fa:41:5a:1b:b8:9d:ec:99:37: + 90:fa:b5:e7:3f:cb:7c:e0:f9:ed:ea:27:ce:15:24: + c7:77:3b:45:45:2d:19:8e:2e:7f:65:0e:85:df:66: + 50:69:24:2c:a4:6a:07:e5:3f:eb:28:84:53:94:4d: + 5f:9c:a8:65:a6:50:4c:c0:35:06:40:6a:a5:62:b1: + 93:60:e5:1c:85:28:34:9b:29:81:6f:e2:4f:cd:15: + 30:b9:19:d7:4b:bb:30:0c:4b:2d:64:fe:3b:dd:0e: + a4:25:2c:4a:5c:de:d7:74:1f:5e:93:7b:1c:e8:c8: + fa:72:1f:4a:eb:8d:3f:98:e4:55:98:b8:e0:8a:29: + 92:33:af:75:6b:05:84:05:d3:0c:2c:07:78:bc:0e: + b2:6d:a7:00:35:c4:53:1f:7b:e6:ba:07:72:a8:24: + c1:0a:a7:c4:46:e6:f2:6f:3a:79:23:00:0b:b8:e5: + 1f:e0:e2:ee:c6:13:a3:57:d9:86:1a:95:f7:a3:04: + f1:46:d5:5f:21:d2:aa:d2:30:fb:f6:cb:e0:da:24: + c6:c3:30:2f:d2:1f:21:fe:bc:0f:99:ac:ac:9b:65: + 9b:e4:83:9a:00:b8:2f:40:fc:3b:42:d3:7a:e8:b7: + 52:d7:f4:67:2a:a5:f7:eb:78:f1:0a:56:8b:56:12: + d5:48:d8:48:70:ab:b8:69:5a:21:d3:71:b0:59:9d: + 17:b4:4b + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + C0:82:DA:FA:69:46:30:AE:FF:6F:CD:BB:93:49:94:A6:D0:E2:17:EB + X509v3 Authority Key Identifier: + keyid:5A:A5:DA:B1:99:D4:E5:0E:E6:1E:94:EA:FF:FC:62:E2:ED:09:F1:06 + + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + Signature Algorithm: sha256WithRSAEncryption + 36:fd:cf:ec:f5:20:4b:52:dc:2e:38:f3:92:b1:e4:b6:a1:06: + 86:aa:2d:c0:e6:f5:0a:58:97:a9:e3:be:13:09:61:79:ed:d4: + 41:83:26:ad:ee:0b:43:83:d1:dd:19:1a:e8:7b:b2:1f:fe:d4: + c1:57:7d:6d:6b:d4:42:ea:7d:cd:34:8c:a4:1f:5b:3b:fa:de: + bb:2f:ae:56:b6:18:e5:53:a9:a3:99:58:ad:36:be:19:54:61: + 0d:52:b6:a7:53:fc:60:e5:ff:f5:7f:82:3f:c1:49:06:cd:b2: + af:25:ee:de:bd:e0:e5:5e:ad:0b:dc:2e:b1:ec:7a:52:6f:9d: + e0:b9:84:18:db:49:53:ee:df:93:ee:8b:9d:9b:8e:3b:2a:82: + 86:7f:45:c8:dd:d1:b0:40:17:ed:63:52:a1:5b:6e:d3:5c:a2: + 72:05:fb:3a:39:71:0d:b4:2c:9d:15:23:1b:1f:8d:ac:89:dc: + c9:56:f2:19:c7:f3:2f:bb:d5:de:40:17:f1:52:ea:e8:93:ff: + 56:43:f5:1d:cb:c0:51:52:25:d7:b0:81:a9:0e:4d:92:24:e7: + 10:81:c7:31:26:ac:cb:66:c1:3f:f6:5f:69:7b:74:87:0d:b0: + 8c:27:d4:24:29:59:e9:5b:a2:cb:0c:c0:f5:9b:1d:42:38:6b: + e3:c3:43:1e:ba:df:b1:51:0a:b7:33:55:26:39:01:2f:9f:c7: + 88:ac:2f:4a:89:f3:69:de:72:43:48:49:08:59:36:86:84:09: + db:6a:82:84:3e:71:6a:9d:f9:bd:d8:b5:1e:7c:2c:29:e1:27: + 45:4c:47:5b:88:b8:e6:fa:9d:9b:ff:d4:e9:8d:2d:5e:64:7f: + 27:87:b2:8c:d8:7e:f5:52:3c:c4:d8:30:03:24:d7:ac:f8:53: + 91:80:98:42:24:5a:6b:cb:34:48:57:e0:82:ac:96:d9:55:6c: + c2:c3:8c:19:7c:56:39:0a:a8:f1:b8:77:64:70:83:a8:04:c8: + 3a:5d:0b:00:4c:e5:ba:f1:40:e5:57:cd:d9:67:48:21:e9:9c: + d3:f2:b8:01:b8:d1:c0:d1:3a:44:c0:97:db:e6:bc:8f:2e:33: + d5:e2:38:3d:d7:7b:50:13:01:36:28:61:cc:28:98:3c:f8:21: + 5d:8c:fe:f5:d0:ab:e0:60:ec:36:22:8d:0b:71:30:1b:3d:56: + ae:96:e9:d2:89:c2:43:8b:ef:25:b7:d6:0d:82:e6:5a:c6:91: + 8a:ad:8c:28:2a:2b:5c:4e:a1:de:cb:7d:cb:29:11:a2:66:c8: + a1:33:35:75:16:fe:28:0b:78:31:0a:1f:fa:d0:a8:f4:f1:69: + c7:97:1e:5d:fb:53:08:b5 diff --git a/security/advancedtls/testdata/server_cert_2.pem b/security/advancedtls/testdata/server_cert_2.pem new file mode 100644 index 0000000..3312b03 --- /dev/null +++ b/security/advancedtls/testdata/server_cert_2.pem @@ -0,0 +1,122 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 7 (0x7) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=CA, O=Internet Widgits Pty Ltd, CN=foo.bar.client2.trust.com + Validity + Not Before: Jan 9 22:51:54 2020 GMT + Not After : Oct 23 22:51:54 2293 GMT + Subject: C=US, ST=CA, O=Internet Widgits Pty Ltd, CN=foo.bar.server2.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (4096 bit) + Modulus: + 00:b1:0b:d3:7e:5b:61:30:db:b0:5f:3f:6d:d2:e0: + 3b:c6:4c:88:95:f5:7e:fd:cd:aa:20:5d:08:b9:6e: + 41:db:c4:ed:0d:f8:bc:cb:b4:ee:c5:87:11:05:a0: + ac:12:3b:4e:0b:4c:e4:43:e4:17:89:c1:ae:b4:13: + 58:1c:31:58:6a:f2:01:ed:df:66:e9:f9:2e:9c:c5: + 85:e6:02:db:36:f4:f3:07:39:75:30:f1:b5:55:5b: + 46:2f:87:b0:d4:a0:ab:57:df:30:45:ae:bd:b0:49: + 9a:fc:ba:5e:bc:d0:5d:86:f4:24:45:4a:d5:4d:5b: + b6:ba:e8:b7:a1:3b:c3:2f:46:2e:b3:ad:2c:63:03: + df:cb:f4:56:62:91:bd:bc:23:00:af:a2:7a:3d:6f: + f1:33:81:60:0e:bc:20:f5:8a:49:5f:ec:58:bc:64: + d5:47:36:a0:2b:b8:1f:76:25:01:89:3e:ff:52:69: + 95:03:8f:bb:14:2f:1a:38:a3:9f:c1:45:20:22:77: + 70:97:5e:25:51:b8:3d:5d:89:7a:bb:15:12:cd:1d: + 96:d2:9c:72:67:12:85:72:6e:27:7a:ef:25:da:af: + 49:26:8d:eb:a0:34:a4:4d:64:c3:63:33:77:5d:ad: + 53:c7:ee:51:32:7b:cc:43:bb:86:8d:f9:52:ba:35: + 23:0e:30:5d:dc:3b:25:63:c1:e3:5f:4b:b2:02:fc: + fe:5b:18:7f:84:aa:f3:71:e4:16:b5:98:bc:73:c5: + 58:13:41:38:eb:f3:a2:fa:8c:98:bd:f1:10:ee:b6: + fe:7e:a5:81:c7:5e:f2:72:54:8e:db:09:f0:35:42: + ca:b7:86:c2:48:b2:c6:18:08:ac:d1:f0:5d:de:b0: + b8:25:8b:3b:bd:61:48:0f:71:3f:ed:97:72:02:c9: + 44:5d:0c:00:fc:30:ca:5d:1c:e5:13:1b:3a:d0:ce: + d9:36:a0:db:f5:c2:ad:a6:95:26:4e:7b:29:2d:fc: + c4:04:1d:47:6e:03:59:68:1e:7a:20:6d:e8:a8:e1: + 3c:57:59:f8:3d:2f:16:61:7e:24:e5:13:ca:48:0a: + e6:f0:60:a3:2d:93:0b:8f:93:eb:b5:d1:06:26:52: + c0:63:1f:fc:9b:73:fe:91:c3:04:40:32:8d:09:d5: + 9e:c4:f6:0b:61:3d:9f:a1:d7:94:a2:e1:3d:b6:bb: + 60:26:74:89:33:25:18:0f:c3:88:db:10:5e:a0:5b: + f4:ee:d0:18:ab:36:50:c5:44:9b:6d:ba:ea:e2:6e: + 52:3a:55:49:a3:72:ae:04:af:1d:f6:f2:83:27:17: + 8b:9a:98:0a:f5:44:b1:c8:f2:a9:c8:ed:b0:75:ca: + 52:25:f3 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 74:BD:18:0B:32:AF:D0:51:8E:4C:4C:8D:B2:F6:4E:B8:6D:AB:BD:BA + X509v3 Authority Key Identifier: + keyid:01:74:A9:44:61:3D:7A:BB:C2:32:CD:D0:ED:20:DA:3A:C4:C6:02:E8 + + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + Signature Algorithm: sha256WithRSAEncryption + b5:63:0c:d8:ed:af:74:2d:4c:94:36:41:05:2a:f2:ef:45:e5: + 6a:0c:76:0c:f3:90:25:e0:54:56:f3:26:23:95:7e:24:74:6b: + fd:02:0a:bc:33:ba:e8:e8:8f:a3:b3:85:2e:59:4c:cf:e3:85: + 1a:d6:70:5c:7c:86:e2:7a:11:99:a8:fa:43:9a:bf:50:54:00: + 9e:6a:7b:72:7f:c5:20:89:6e:18:6c:46:64:ce:44:44:47:4d: + 87:b5:fc:cf:f3:b9:9f:45:a3:cb:b0:91:00:96:2d:29:68:8b: + ff:c7:e0:f1:b7:8d:31:c2:01:be:5b:51:1d:af:42:b1:17:22: + bc:91:e4:d9:b9:96:6d:64:40:79:6c:71:ed:f6:e5:49:16:0a: + e3:bc:18:95:2e:89:ba:c4:a5:ce:ba:ab:3a:32:eb:bc:d8:91: + cd:f2:ee:d1:fc:67:3a:51:00:92:bd:b8:68:0b:54:04:d5:07: + 0b:97:11:2c:42:64:7c:47:c1:68:b4:eb:21:c4:e4:ad:17:a7: + 16:b9:e0:e6:cd:04:c6:89:36:40:d4:4b:c3:f7:7e:26:6b:3a: + d7:68:b3:b2:da:00:65:13:c8:fa:d0:1c:2e:10:ba:71:3e:0f: + aa:8b:d0:ff:b7:3e:83:9c:bc:b3:d1:52:0c:9f:3f:21:4a:10: + dc:8f:ab:38:45:d4:2c:2a:15:2d:71:45:fe:91:a2:d8:d9:dd: + 0c:dc:a7:d9:cd:1b:f5:35:fe:14:ba:c5:1f:ed:ee:fb:87:cc: + 87:a1:08:c2:2e:ff:5d:af:b3:3d:6e:11:94:79:0b:28:e6:83: + 4e:fc:28:8f:7f:00:85:79:7f:3a:d1:07:ee:6e:fa:94:c4:0b: + 4b:2c:05:b1:68:00:e8:37:bc:b8:b2:03:5c:5a:ca:13:f2:68: + 57:df:ac:fc:da:be:27:24:7e:6d:c4:a9:53:2d:f2:43:0e:30: + 9c:82:d5:fb:f1:a2:0a:83:e0:a5:d8:9f:09:3e:99:c8:39:d6: + 69:6d:d6:c2:27:70:59:05:3c:3c:7d:d6:41:6a:b4:9c:1f:70: + 7e:3e:ee:6f:67:de:95:1d:eb:31:8b:11:c8:0d:a1:25:4e:08: + ef:3a:11:2d:a7:98:0d:a1:d9:30:2d:da:d2:a0:05:6b:34:38: + a6:87:b2:bd:0f:9c:51:cc:e0:2e:a2:1b:a3:a0:a6:eb:1f:0a: + 22:70:59:f0:0b:c9:bd:94:4e:1d:65:3b:99:5d:8e:6c:18:82: + 1d:b5:cc:6f:14:21:c4:89:07:9b:81:1d:9a:79:ff:bf:fd:ce: + e4:77:11:0f:47:21:dc:d9:79:f3:40:26:56:5c:b4:86:32:8e: + 28:b9:14:e7:b3:fe:86:47 +-----BEGIN CERTIFICATE----- +MIIFkzCCA3ugAwIBAgIBBzANBgkqhkiG9w0BAQsFADBhMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEi +MCAGA1UEAwwZZm9vLmJhci5jbGllbnQyLnRydXN0LmNvbTAgFw0yMDAxMDkyMjUx +NTRaGA8yMjkzMTAyMzIyNTE1NFowWzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNB +MSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxHDAaBgNVBAMME2Zv +by5iYXIuc2VydmVyMi5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC +AQCxC9N+W2Ew27BfP23S4DvGTIiV9X79zaogXQi5bkHbxO0N+LzLtO7FhxEFoKwS +O04LTORD5BeJwa60E1gcMVhq8gHt32bp+S6cxYXmAts29PMHOXUw8bVVW0Yvh7DU +oKtX3zBFrr2wSZr8ul680F2G9CRFStVNW7a66LehO8MvRi6zrSxjA9/L9FZikb28 +IwCvono9b/EzgWAOvCD1iklf7Fi8ZNVHNqAruB92JQGJPv9SaZUDj7sULxo4o5/B +RSAid3CXXiVRuD1diXq7FRLNHZbSnHJnEoVybid67yXar0kmjeugNKRNZMNjM3dd +rVPH7lEye8xDu4aN+VK6NSMOMF3cOyVjweNfS7IC/P5bGH+EqvNx5Ba1mLxzxVgT +QTjr86L6jJi98RDutv5+pYHHXvJyVI7bCfA1Qsq3hsJIssYYCKzR8F3esLglizu9 +YUgPcT/tl3ICyURdDAD8MMpdHOUTGzrQztk2oNv1wq2mlSZOeykt/MQEHUduA1lo +Hnogbeio4TxXWfg9LxZhfiTlE8pICubwYKMtkwuPk+u10QYmUsBjH/ybc/6RwwRA +Mo0J1Z7E9gthPZ+h15Si4T22u2AmdIkzJRgPw4jbEF6gW/Tu0BirNlDFRJttuuri +blI6VUmjcq4Erx328oMnF4uamAr1RLHI8qnI7bB1ylIl8wIDAQABo1owWDAdBgNV +HQ4EFgQUdL0YCzKv0FGOTEyNsvZOuG2rvbowHwYDVR0jBBgwFoAUAXSpRGE9ervC +Ms3Q7SDaOsTGAugwCQYDVR0TBAIwADALBgNVHQ8EBAMCBaAwDQYJKoZIhvcNAQEL +BQADggIBALVjDNjtr3QtTJQ2QQUq8u9F5WoMdgzzkCXgVFbzJiOVfiR0a/0CCrwz +uujoj6OzhS5ZTM/jhRrWcFx8huJ6EZmo+kOav1BUAJ5qe3J/xSCJbhhsRmTORERH +TYe1/M/zuZ9Fo8uwkQCWLSloi//H4PG3jTHCAb5bUR2vQrEXIryR5Nm5lm1kQHls +ce325UkWCuO8GJUuibrEpc66qzoy67zYkc3y7tH8ZzpRAJK9uGgLVATVBwuXESxC +ZHxHwWi06yHE5K0Xpxa54ObNBMaJNkDUS8P3fiZrOtdos7LaAGUTyPrQHC4QunE+ +D6qL0P+3PoOcvLPRUgyfPyFKENyPqzhF1CwqFS1xRf6RotjZ3Qzcp9nNG/U1/hS6 +xR/t7vuHzIehCMIu/12vsz1uEZR5Cyjmg078KI9/AIV5fzrRB+5u+pTEC0ssBbFo +AOg3vLiyA1xayhPyaFffrPzavickfm3EqVMt8kMOMJyC1fvxogqD4KXYnwk+mcg5 +1mlt1sIncFkFPDx91kFqtJwfcH4+7m9n3pUd6zGLEcgNoSVOCO86ES2nmA2h2TAt +2tKgBWs0OKaHsr0PnFHM4C6iG6OgpusfCiJwWfALyb2UTh1lO5ldjmwYgh21zG8U +IcSJB5uBHZp5/7/9zuR3EQ9HIdzZefNAJlZctIYyjii5FOez/oZH +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/server_cert_2.txt b/security/advancedtls/testdata/server_cert_2.txt new file mode 100644 index 0000000..dc69bed --- /dev/null +++ b/security/advancedtls/testdata/server_cert_2.txt @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 7 (0x7) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C = US, ST = CA, O = Internet Widgits Pty Ltd, CN = foo.bar.client2.trust.com + Validity + Not Before: Jan 9 22:51:54 2020 GMT + Not After : Oct 23 22:51:54 2293 GMT + Subject: C = US, ST = CA, O = Internet Widgits Pty Ltd, CN = foo.bar.server2.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (4096 bit) + Modulus: + 00:b1:0b:d3:7e:5b:61:30:db:b0:5f:3f:6d:d2:e0: + 3b:c6:4c:88:95:f5:7e:fd:cd:aa:20:5d:08:b9:6e: + 41:db:c4:ed:0d:f8:bc:cb:b4:ee:c5:87:11:05:a0: + ac:12:3b:4e:0b:4c:e4:43:e4:17:89:c1:ae:b4:13: + 58:1c:31:58:6a:f2:01:ed:df:66:e9:f9:2e:9c:c5: + 85:e6:02:db:36:f4:f3:07:39:75:30:f1:b5:55:5b: + 46:2f:87:b0:d4:a0:ab:57:df:30:45:ae:bd:b0:49: + 9a:fc:ba:5e:bc:d0:5d:86:f4:24:45:4a:d5:4d:5b: + b6:ba:e8:b7:a1:3b:c3:2f:46:2e:b3:ad:2c:63:03: + df:cb:f4:56:62:91:bd:bc:23:00:af:a2:7a:3d:6f: + f1:33:81:60:0e:bc:20:f5:8a:49:5f:ec:58:bc:64: + d5:47:36:a0:2b:b8:1f:76:25:01:89:3e:ff:52:69: + 95:03:8f:bb:14:2f:1a:38:a3:9f:c1:45:20:22:77: + 70:97:5e:25:51:b8:3d:5d:89:7a:bb:15:12:cd:1d: + 96:d2:9c:72:67:12:85:72:6e:27:7a:ef:25:da:af: + 49:26:8d:eb:a0:34:a4:4d:64:c3:63:33:77:5d:ad: + 53:c7:ee:51:32:7b:cc:43:bb:86:8d:f9:52:ba:35: + 23:0e:30:5d:dc:3b:25:63:c1:e3:5f:4b:b2:02:fc: + fe:5b:18:7f:84:aa:f3:71:e4:16:b5:98:bc:73:c5: + 58:13:41:38:eb:f3:a2:fa:8c:98:bd:f1:10:ee:b6: + fe:7e:a5:81:c7:5e:f2:72:54:8e:db:09:f0:35:42: + ca:b7:86:c2:48:b2:c6:18:08:ac:d1:f0:5d:de:b0: + b8:25:8b:3b:bd:61:48:0f:71:3f:ed:97:72:02:c9: + 44:5d:0c:00:fc:30:ca:5d:1c:e5:13:1b:3a:d0:ce: + d9:36:a0:db:f5:c2:ad:a6:95:26:4e:7b:29:2d:fc: + c4:04:1d:47:6e:03:59:68:1e:7a:20:6d:e8:a8:e1: + 3c:57:59:f8:3d:2f:16:61:7e:24:e5:13:ca:48:0a: + e6:f0:60:a3:2d:93:0b:8f:93:eb:b5:d1:06:26:52: + c0:63:1f:fc:9b:73:fe:91:c3:04:40:32:8d:09:d5: + 9e:c4:f6:0b:61:3d:9f:a1:d7:94:a2:e1:3d:b6:bb: + 60:26:74:89:33:25:18:0f:c3:88:db:10:5e:a0:5b: + f4:ee:d0:18:ab:36:50:c5:44:9b:6d:ba:ea:e2:6e: + 52:3a:55:49:a3:72:ae:04:af:1d:f6:f2:83:27:17: + 8b:9a:98:0a:f5:44:b1:c8:f2:a9:c8:ed:b0:75:ca: + 52:25:f3 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 74:BD:18:0B:32:AF:D0:51:8E:4C:4C:8D:B2:F6:4E:B8:6D:AB:BD:BA + X509v3 Authority Key Identifier: + keyid:01:74:A9:44:61:3D:7A:BB:C2:32:CD:D0:ED:20:DA:3A:C4:C6:02:E8 + + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + Signature Algorithm: sha256WithRSAEncryption + b5:63:0c:d8:ed:af:74:2d:4c:94:36:41:05:2a:f2:ef:45:e5: + 6a:0c:76:0c:f3:90:25:e0:54:56:f3:26:23:95:7e:24:74:6b: + fd:02:0a:bc:33:ba:e8:e8:8f:a3:b3:85:2e:59:4c:cf:e3:85: + 1a:d6:70:5c:7c:86:e2:7a:11:99:a8:fa:43:9a:bf:50:54:00: + 9e:6a:7b:72:7f:c5:20:89:6e:18:6c:46:64:ce:44:44:47:4d: + 87:b5:fc:cf:f3:b9:9f:45:a3:cb:b0:91:00:96:2d:29:68:8b: + ff:c7:e0:f1:b7:8d:31:c2:01:be:5b:51:1d:af:42:b1:17:22: + bc:91:e4:d9:b9:96:6d:64:40:79:6c:71:ed:f6:e5:49:16:0a: + e3:bc:18:95:2e:89:ba:c4:a5:ce:ba:ab:3a:32:eb:bc:d8:91: + cd:f2:ee:d1:fc:67:3a:51:00:92:bd:b8:68:0b:54:04:d5:07: + 0b:97:11:2c:42:64:7c:47:c1:68:b4:eb:21:c4:e4:ad:17:a7: + 16:b9:e0:e6:cd:04:c6:89:36:40:d4:4b:c3:f7:7e:26:6b:3a: + d7:68:b3:b2:da:00:65:13:c8:fa:d0:1c:2e:10:ba:71:3e:0f: + aa:8b:d0:ff:b7:3e:83:9c:bc:b3:d1:52:0c:9f:3f:21:4a:10: + dc:8f:ab:38:45:d4:2c:2a:15:2d:71:45:fe:91:a2:d8:d9:dd: + 0c:dc:a7:d9:cd:1b:f5:35:fe:14:ba:c5:1f:ed:ee:fb:87:cc: + 87:a1:08:c2:2e:ff:5d:af:b3:3d:6e:11:94:79:0b:28:e6:83: + 4e:fc:28:8f:7f:00:85:79:7f:3a:d1:07:ee:6e:fa:94:c4:0b: + 4b:2c:05:b1:68:00:e8:37:bc:b8:b2:03:5c:5a:ca:13:f2:68: + 57:df:ac:fc:da:be:27:24:7e:6d:c4:a9:53:2d:f2:43:0e:30: + 9c:82:d5:fb:f1:a2:0a:83:e0:a5:d8:9f:09:3e:99:c8:39:d6: + 69:6d:d6:c2:27:70:59:05:3c:3c:7d:d6:41:6a:b4:9c:1f:70: + 7e:3e:ee:6f:67:de:95:1d:eb:31:8b:11:c8:0d:a1:25:4e:08: + ef:3a:11:2d:a7:98:0d:a1:d9:30:2d:da:d2:a0:05:6b:34:38: + a6:87:b2:bd:0f:9c:51:cc:e0:2e:a2:1b:a3:a0:a6:eb:1f:0a: + 22:70:59:f0:0b:c9:bd:94:4e:1d:65:3b:99:5d:8e:6c:18:82: + 1d:b5:cc:6f:14:21:c4:89:07:9b:81:1d:9a:79:ff:bf:fd:ce: + e4:77:11:0f:47:21:dc:d9:79:f3:40:26:56:5c:b4:86:32:8e: + 28:b9:14:e7:b3:fe:86:47 diff --git a/security/advancedtls/testdata/server_cert_3.pem b/security/advancedtls/testdata/server_cert_3.pem new file mode 100644 index 0000000..bfd2d09 --- /dev/null +++ b/security/advancedtls/testdata/server_cert_3.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDyTCCArGgAwIBAgIUeoNdEiqhXVkpcYsmHaKiVS5W/tQwDQYJKoZIhvcNAQEL +BQAwPjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMREwDwYDVQQHDAhTYW4gSm9z +ZTEPMA0GA1UECgwGR29vZ2xlMB4XDTIwMDcxNjE2NTMxOVoXDTQwMDcxMTE2NTMx +OVowgZMxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTERMA8GA1UEBwwIU2FuIEpv +c2UxEjAQBgNVBAoMCUVuZCBQb2ludDEOMAwGA1UECwwFSW5mcmExIjAgBgkqhkiG +9w0BCQEWE2NpbmR5eHVlQGdvb2dsZS5jb20xHDAaBgNVBAMME2Zvby5iYXIuc2Vy +dmVyMy5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDcIyJEt3ZA +xPn7H5f/IwXS8NcoAXWP8L6rWndcg+EWayx7W+wmUsFKGSFGzrFPPCFmKO8MrQqp +8LSAxHAVtOC6Uw+INWJJw9BRlx2nvV7hfbqu3OnPkPVkN/siUQCqnEKJQHliNT9X +Dl4/Mav75uQSWb3Vfi3KtG7mzPFNNbbe4yfHyGbC4e9RtKkGimDSJ413s3m4+scD +vtpCcCXj9XXZNdCwD1CL3kNdmOdhgfkDBP+AMLBFKZKqpCo6m0s4JJTiej13dc27 +wTrnkFm1CP77SV+kQlWg5DAcVXYJkN9FqNqExqIPS/SxMk7H+3qSQACbttQK9UmC +n3qR3pGbwqzNAgMBAAGjaTBnMB8GA1UdIwQYMBaAFG4bi8k0dOd7jSpPQQ6YUDAU +ARaxMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgTwMCwGA1UdEQQlMCOCCmdvb2dsZS5j +b22CCWFwcGxlLmNvbYIKYW1hem9uLmNvbTANBgkqhkiG9w0BAQsFAAOCAQEAn5aW +HEHNTDmcgC25oEtCj+IkoAslgFze4ZqkSz0HCzx76vj3AfMmIEvqB0W74wKqeZgm +V0D7I0xHkM3ILH4RjoCotpol3nLooIPFflA6Z1ILTRZl8mE5kfBSHzKdPS0egOf6 +kgrNYgJjBEtGNsmq8RKxAHVVAPgH88di0JnQDN5LcV9ZBKTQM2R7EM6a8eWD/Jsi +uujbNtdNERssSBV+Oil93MbsEcOT1RSKKxAiVvkHkR+45GRB889xBnqelcDVqcMK +Vcdp6X7aD5/Bu/4fq9AZlcHSEQDixNtjp/pQR0B5FsCGrb5OAz0B2t9jykDiIyj4 +4lxhQz8ykXf7ue0/ag== +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/server_cert_3.txt b/security/advancedtls/testdata/server_cert_3.txt new file mode 100644 index 0000000..2b24d79 --- /dev/null +++ b/security/advancedtls/testdata/server_cert_3.txt @@ -0,0 +1,60 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 7a:83:5d:12:2a:a1:5d:59:29:71:8b:26:1d:a2:a2:55:2e:56:fe:d4 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C = US, ST = CA, L = San Jose, O = Google + Validity + Not Before: Jul 16 16:53:19 2020 GMT + Not After : Jul 11 16:53:19 2040 GMT + Subject: C = US, ST = CA, L = San Jose, O = End Point, OU = Infra, emailAddress = cindyxue@google.com, CN = foo.bar.server3.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (2048 bit) + Modulus: + 00:dc:23:22:44:b7:76:40:c4:f9:fb:1f:97:ff:23: + 05:d2:f0:d7:28:01:75:8f:f0:be:ab:5a:77:5c:83: + e1:16:6b:2c:7b:5b:ec:26:52:c1:4a:19:21:46:ce: + b1:4f:3c:21:66:28:ef:0c:ad:0a:a9:f0:b4:80:c4: + 70:15:b4:e0:ba:53:0f:88:35:62:49:c3:d0:51:97: + 1d:a7:bd:5e:e1:7d:ba:ae:dc:e9:cf:90:f5:64:37: + fb:22:51:00:aa:9c:42:89:40:79:62:35:3f:57:0e: + 5e:3f:31:ab:fb:e6:e4:12:59:bd:d5:7e:2d:ca:b4: + 6e:e6:cc:f1:4d:35:b6:de:e3:27:c7:c8:66:c2:e1: + ef:51:b4:a9:06:8a:60:d2:27:8d:77:b3:79:b8:fa: + c7:03:be:da:42:70:25:e3:f5:75:d9:35:d0:b0:0f: + 50:8b:de:43:5d:98:e7:61:81:f9:03:04:ff:80:30: + b0:45:29:92:aa:a4:2a:3a:9b:4b:38:24:94:e2:7a: + 3d:77:75:cd:bb:c1:3a:e7:90:59:b5:08:fe:fb:49: + 5f:a4:42:55:a0:e4:30:1c:55:76:09:90:df:45:a8: + da:84:c6:a2:0f:4b:f4:b1:32:4e:c7:fb:7a:92:40: + 00:9b:b6:d4:0a:f5:49:82:9f:7a:91:de:91:9b:c2: + ac:cd + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Authority Key Identifier: + keyid:6E:1B:8B:C9:34:74:E7:7B:8D:2A:4F:41:0E:98:50:30:14:01:16:B1 + + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment, Data Encipherment + X509v3 Subject Alternative Name: + DNS:google.com, DNS:apple.com, DNS:amazon.com + Signature Algorithm: sha256WithRSAEncryption + 9f:96:96:1c:41:cd:4c:39:9c:80:2d:b9:a0:4b:42:8f:e2:24: + a0:0b:25:80:5c:de:e1:9a:a4:4b:3d:07:0b:3c:7b:ea:f8:f7: + 01:f3:26:20:4b:ea:07:45:bb:e3:02:aa:79:98:26:57:40:fb: + 23:4c:47:90:cd:c8:2c:7e:11:8e:80:a8:b6:9a:25:de:72:e8: + a0:83:c5:7e:50:3a:67:52:0b:4d:16:65:f2:61:39:91:f0:52: + 1f:32:9d:3d:2d:1e:80:e7:fa:92:0a:cd:62:02:63:04:4b:46: + 36:c9:aa:f1:12:b1:00:75:55:00:f8:07:f3:c7:62:d0:99:d0: + 0c:de:4b:71:5f:59:04:a4:d0:33:64:7b:10:ce:9a:f1:e5:83: + fc:9b:22:ba:e8:db:36:d7:4d:11:1b:2c:48:15:7e:3a:29:7d: + dc:c6:ec:11:c3:93:d5:14:8a:2b:10:22:56:f9:07:91:1f:b8: + e4:64:41:f3:cf:71:06:7a:9e:95:c0:d5:a9:c3:0a:55:c7:69: + e9:7e:da:0f:9f:c1:bb:fe:1f:ab:d0:19:95:c1:d2:11:00:e2: + c4:db:63:a7:fa:50:47:40:79:16:c0:86:ad:be:4e:03:3d:01: + da:df:63:ca:40:e2:23:28:f8:e2:5c:61:43:3f:32:91:77:fb: + b9:ed:3f:6a diff --git a/security/advancedtls/testdata/server_cert_localhost_1.pem b/security/advancedtls/testdata/server_cert_localhost_1.pem new file mode 100644 index 0000000..5ec2125 --- /dev/null +++ b/security/advancedtls/testdata/server_cert_localhost_1.pem @@ -0,0 +1,124 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 5 (0x5) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=CA, L=SVL, O=Internet Widgits Pty Ltd + Validity + Not Before: Dec 15 18:05:59 2020 GMT + Not After : May 2 18:05:59 2048 GMT + Subject: C=US, ST=Illinois, L=Chicago, O=Example, Co., CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (4096 bit) + Modulus: + 00:d3:4d:01:08:9d:80:de:15:fa:0a:52:ed:2c:f6: + 4b:9d:c5:67:2d:b6:41:33:d7:5b:b4:fd:f0:a8:5c: + dd:8a:8d:1f:2d:12:5d:47:88:09:d4:96:ee:63:10: + f9:9d:9a:6c:34:c4:ec:a9:0d:95:6c:48:bc:17:d0: + 77:53:93:f8:44:8a:0b:0b:a2:4d:1d:53:f7:55:a7: + ed:6a:35:ad:1d:af:79:bd:d5:c6:5b:96:24:9e:4d: + d8:e9:21:93:e1:93:3b:5d:c4:e3:90:1a:36:d0:f7: + bb:8c:22:e9:d2:f8:29:19:bf:24:c6:52:21:0e:d6: + 3a:73:48:ba:e7:81:34:ba:23:21:57:c3:51:0e:59: + 51:9f:a6:07:56:17:c4:aa:1b:7b:6c:36:6b:ab:32: + 5e:2e:f7:70:dc:eb:f3:da:3b:39:4e:a4:8c:bf:40: + 72:ef:00:1e:46:c9:07:6a:93:4c:36:b7:c3:2e:7c: + c5:c1:85:9f:6b:4d:2d:fd:3c:9b:9a:cf:52:b2:fa: + ba:f8:ce:cc:8b:dc:57:7b:ed:37:69:c9:80:dc:a6: + 2c:a7:e4:4b:8c:77:cb:2e:28:65:64:61:a4:c8:33: + d9:f8:7d:b7:7c:4f:b7:f4:07:5a:89:ae:2a:59:8c: + ac:00:c7:ce:b0:d0:9b:cd:4d:83:39:55:bb:72:0f: + 62:d4:5f:8b:c3:c7:e2:79:95:53:eb:a0:26:0d:52: + 7b:4d:40:56:66:2b:55:67:f5:1a:c9:e8:a8:49:bd: + e7:e4:31:9a:e1:8d:80:f2:cc:ab:3d:70:f6:fe:75: + cb:aa:1b:12:d0:6f:d3:c8:df:f1:bd:ce:2b:21:42: + e5:2c:bd:c6:c8:c4:bb:4a:3d:92:48:0d:49:a8:96: + c0:51:ed:30:64:81:dd:ce:05:d4:ff:07:87:59:0b: + 13:41:8e:1d:58:e4:47:0c:00:97:12:e4:67:94:24: + 67:ef:ed:1e:85:89:df:85:78:10:1f:7f:b2:e8:af: + e7:0f:c7:ec:aa:01:67:b6:0a:9a:23:83:90:1d:0a: + 2a:37:80:c3:26:d7:f1:24:29:b3:d8:37:7c:5c:f8: + e3:08:96:1a:34:2d:fa:ff:75:e8:70:25:e2:ab:51: + 9d:f7:8a:23:52:89:02:c8:71:ea:cd:a2:89:b6:eb: + 6e:c9:fd:fb:dc:63:e8:f9:db:d7:57:d2:0c:fc:56: + 9c:16:a8:67:fe:17:88:07:e8:f8:c5:7a:33:72:ad: + b4:5a:43:5b:49:be:79:8b:00:18:83:f7:19:00:07: + 94:2c:38:96:61:a7:55:a8:30:db:7e:07:f6:c5:a3: + 65:1e:93:2d:6b:5e:d9:a6:0f:fa:c2:19:9b:59:4c: + 41:ba:07 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + F3:DC:6A:5B:B7:CE:E9:E1:4D:3E:C4:AE:B7:8E:39:E3:6D:CA:AF:C7 + X509v3 Authority Key Identifier: + keyid:5A:A5:DA:B1:99:D4:E5:0E:E6:1E:94:EA:FF:FC:62:E2:ED:09:F1:06 + + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Subject Alternative Name: + DNS:localhost + Signature Algorithm: sha256WithRSAEncryption + 54:13:3d:55:d3:4b:d8:85:f0:54:a8:33:5c:a1:9f:87:79:31: + 34:7e:52:b9:46:ea:97:43:fd:0a:9b:ae:2c:8b:74:cd:51:d5: + 63:b3:b0:b4:19:a1:da:73:b4:e2:47:0c:d9:33:4a:c6:aa:27: + 41:bc:4d:15:74:42:59:eb:41:8e:28:49:f0:55:89:13:f8:f0: + 35:1f:e9:b2:ae:48:79:e7:15:a2:aa:59:e0:fd:91:c8:7f:ba: + aa:a4:2e:77:2a:e4:62:fe:c2:83:51:dc:58:83:f8:7f:b2:47: + 68:8b:1f:9d:9b:12:25:f2:0f:7d:06:a4:9a:be:a3:af:2d:27: + 32:4a:20:fe:5f:98:d0:5d:6a:10:c9:04:49:54:26:60:20:6e: + 38:f2:91:a4:24:e7:ed:d2:e4:aa:88:7e:9c:6d:0a:1e:30:97: + a9:9c:45:35:09:fc:45:6c:32:d0:db:79:04:fa:03:7c:36:e5: + 27:72:1d:77:a1:d4:12:86:53:bf:93:4e:f7:da:7a:a1:4b:5b: + 42:e4:6b:9b:a5:58:f2:ef:30:2e:a6:6f:f7:63:fd:16:b0:35: + 81:99:3d:41:dc:da:34:46:ea:1f:02:cc:5e:dc:67:1c:62:68: + 76:45:5b:ea:f3:2f:a5:0f:2e:b8:d0:df:ad:8f:3b:03:b4:36: + 21:d4:4d:99:09:76:22:a4:61:b4:59:ca:8b:42:5d:64:2a:9a: + 4f:7f:67:65:38:4a:c6:e7:8b:ff:36:0d:42:4b:60:03:77:99: + 71:1f:ac:f4:83:bb:06:5c:22:fa:ed:4c:c5:37:22:22:11:85: + 1a:ab:06:e3:14:c8:71:bf:79:1f:b4:22:64:ea:65:5e:99:c5: + ca:3a:1e:d2:22:96:24:df:65:b1:42:7e:72:21:6b:de:49:cd: + 09:45:4d:3e:ff:45:22:f4:01:d5:09:5b:dd:22:cc:25:8f:b2: + 5e:b3:35:f8:b3:6d:6e:2b:bd:98:f2:eb:5b:0e:58:31:f3:90: + ec:d9:41:9d:e4:4d:e7:6a:ff:32:63:d1:45:f0:fd:a6:7d:34: + 16:e4:c8:0c:79:39:5d:46:e1:28:f1:13:49:a6:02:07:a2:ca: + de:ae:4a:3a:f7:0b:e4:27:49:33:c8:29:2f:cd:6f:0e:5a:be: + e0:5d:c0:98:7e:15:97:ea:a5:ba:84:3b:ab:8d:aa:81:d7:7f: + df:7c:b9:e7:11:6b:7f:82:6e:62:57:d9:ba:25:60:1d:93:49: + eb:48:91:88:2c:22:74:dc:50:cf:6c:44:bd:04:84:fb:97:53: + 57:62:56:8b:b1:5c:a2:06:ef:c4:01:21:d6:73:4f:58:79:60: + 75:09:53:7d:cb:8c:a2:86 +-----BEGIN CERTIFICATE----- +MIIFkzCCA3ugAwIBAgIBBTANBgkqhkiG9w0BAQsFADBLMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExDDAKBgNVBAcMA1NWTDEhMB8GA1UECgwYSW50ZXJuZXQgV2lk +Z2l0cyBQdHkgTHRkMB4XDTIwMTIxNTE4MDU1OVoXDTQ4MDUwMjE4MDU1OVowXTEL +MAkGA1UEBhMCVVMxETAPBgNVBAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdv +MRUwEwYDVQQKDAxFeGFtcGxlLCBDby4xEjAQBgNVBAMMCWxvY2FsaG9zdDCCAiIw +DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANNNAQidgN4V+gpS7Sz2S53FZy22 +QTPXW7T98Khc3YqNHy0SXUeICdSW7mMQ+Z2abDTE7KkNlWxIvBfQd1OT+ESKCwui +TR1T91Wn7Wo1rR2veb3VxluWJJ5N2Okhk+GTO13E45AaNtD3u4wi6dL4KRm/JMZS +IQ7WOnNIuueBNLojIVfDUQ5ZUZ+mB1YXxKobe2w2a6syXi73cNzr89o7OU6kjL9A +cu8AHkbJB2qTTDa3wy58xcGFn2tNLf08m5rPUrL6uvjOzIvcV3vtN2nJgNymLKfk +S4x3yy4oZWRhpMgz2fh9t3xPt/QHWomuKlmMrADHzrDQm81NgzlVu3IPYtRfi8PH +4nmVU+ugJg1Se01AVmYrVWf1GsnoqEm95+QxmuGNgPLMqz1w9v51y6obEtBv08jf +8b3OKyFC5Sy9xsjEu0o9kkgNSaiWwFHtMGSB3c4F1P8Hh1kLE0GOHVjkRwwAlxLk +Z5QkZ+/tHoWJ34V4EB9/suiv5w/H7KoBZ7YKmiODkB0KKjeAwybX8SQps9g3fFz4 +4wiWGjQt+v916HAl4qtRnfeKI1KJAshx6s2iibbrbsn9+9xj6Pnb11fSDPxWnBao +Z/4XiAfo+MV6M3KttFpDW0m+eYsAGIP3GQAHlCw4lmGnVagw234H9sWjZR6TLWte +2aYP+sIZm1lMQboHAgMBAAGjcDBuMB0GA1UdDgQWBBTz3Gpbt87p4U0+xK63jjnj +bcqvxzAfBgNVHSMEGDAWgBRapdqxmdTlDuYelOr//GLi7QnxBjAJBgNVHRMEAjAA +MAsGA1UdDwQEAwIFoDAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZIhvcNAQEL +BQADggIBAFQTPVXTS9iF8FSoM1yhn4d5MTR+UrlG6pdD/QqbriyLdM1R1WOzsLQZ +odpztOJHDNkzSsaqJ0G8TRV0QlnrQY4oSfBViRP48DUf6bKuSHnnFaKqWeD9kch/ +uqqkLncq5GL+woNR3FiD+H+yR2iLH52bEiXyD30GpJq+o68tJzJKIP5fmNBdahDJ +BElUJmAgbjjykaQk5+3S5KqIfpxtCh4wl6mcRTUJ/EVsMtDbeQT6A3w25SdyHXeh +1BKGU7+TTvfaeqFLW0Lka5ulWPLvMC6mb/dj/RawNYGZPUHc2jRG6h8CzF7cZxxi +aHZFW+rzL6UPLrjQ362POwO0NiHUTZkJdiKkYbRZyotCXWQqmk9/Z2U4Ssbni/82 +DUJLYAN3mXEfrPSDuwZcIvrtTMU3IiIRhRqrBuMUyHG/eR+0ImTqZV6Zxco6HtIi +liTfZbFCfnIha95JzQlFTT7/RSL0AdUJW90izCWPsl6zNfizbW4rvZjy61sOWDHz +kOzZQZ3kTedq/zJj0UXw/aZ9NBbkyAx5OV1G4SjxE0mmAgeiyt6uSjr3C+QnSTPI +KS/Nbw5avuBdwJh+FZfqpbqEO6uNqoHXf998uecRa3+CbmJX2bolYB2TSetIkYgs +InTcUM9sRL0EhPuXU1diVouxXKIG78QBIdZzT1h5YHUJU33LjKKG +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/server_key_1.pem b/security/advancedtls/testdata/server_key_1.pem new file mode 100644 index 0000000..927a03d --- /dev/null +++ b/security/advancedtls/testdata/server_key_1.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEA7D8kLZE6vcP8FXJCs/so5gSjviYg5uowqKpIeDYOC5kpO0v5 +8dW/vQwTfOpSBvS8NJ4rwLSCLIf6L+LNfNe54Y8EcW2Fd64YQOSxOkpr5TO/PmXb +z5RkhxogRsA3Op+TP9RPrMTk4Ci2DyhTKs+5/lDy70fcfrZgwkeFuMvKSFv6n4qX +MAH0s1EPaOFgqy+grfzwEE9g4ZLbvoMEXECHzlE+mp7WHBsZy4zCbFd0b3uvlD1T +rRelmWl8QfU+eltIx3j/1zuoH/cw54MmeOLLoo9YkmHNyum40YDAQFjp2NNCZIKP +5Ay5sTbbn2U/P1skWTGzYAz6QVobuJ3smTeQ+rXnP8t84Pnt6ifOFSTHdztFRS0Z +ji5/ZQ6F32ZQaSQspGoH5T/rKIRTlE1fnKhlplBMwDUGQGqlYrGTYOUchSg0mymB +b+JPzRUwuRnXS7swDEstZP473Q6kJSxKXN7XdB9ek3sc6Mj6ch9K640/mORVmLjg +iimSM691awWEBdMMLAd4vA6ybacANcRTH3vmugdyqCTBCqfERubybzp5IwALuOUf +4OLuxhOjV9mGGpX3owTxRtVfIdKq0jD79svg2iTGwzAv0h8h/rwPmaysm2Wb5IOa +ALgvQPw7QtN66LdS1/RnKqX363jxClaLVhLVSNhIcKu4aVoh03GwWZ0XtEsCAwEA +AQKCAgAiGesq+K+1/LhCkD+4oySAL2NDa1WMf3mOnyXe1E6qte0RtiHaGrSWoUue +2GQGxQT1w28lXej8bJRcnSx0PN+EA5TsmpaNc//kPh6m/18bsqCEbUeRayYnqknG +bLCMMcSbjhYCJlmzUa0V+wgmQd3jK+QlTgYx9Dl7Ub+nsSL91ukSZnr0XxPnXmgP +B5lgnHthIgW1FQAzD3PQyDC08EuqKGgVAaB+ZhsPGr5lzSntfbkWeNO/RI6O2n8p +NjFSkCKtSHYFp4LZOmFAydmf0Xz7dh2e46dFBv+6ng8iOrNmrPgEciQ7Eusq/XQu +SfsbNhjFFzuBPd5R2KPvvjwM0cyHXP8H84zOb/2LZ49J/RADG7CGpxCGF+27R8ex +JpQJysA0T7A4JhzvwS3t5BFP5DHr+1gJW5z6RAr5kMqW65EOOIokzRcejnu6gNee ++cYAGrUjxRoi/+ba23SaUwmYUfvxWeWtwG7ybIUGtMdHiAt+KegO8nNFvMUE0/un +TIGyrrvhmq/L0Y4EoKOTZTJ1Qf6FSdCfimtMhoaMEehZ+squSA+lWjJQ2uLe3qC9 +24n4rFyHl3rvSW12uHiYWWkGbnLtzlqL+uL3Yi7yb49PSDSHawNUKctS9tySRh0v +I7H7GSFKWi+P85vdzWc4F2bWxA4bWZQ+LtfVa7sdEZggoO8OgQKCAQEA+zShQBmD +Ao2sTW+rWpl0KdpbdO/+5eXu59yCYxxuwaWdzE+Hqw5Zvz0oL7/KWckTQV9Cg9vx +pt8FYPOuuJfXCD5kUXnZsdXS2qUGIuLMxX3aUrxuw0NeckF42iFom0+nO8NWtWzh +xnO59OQOLj/VER2QhI2fVAMIw62wZIR6pSDYCM2Rn/J4X00r8vLIUQ7ifGqw3uV7 +cezyenfpb5Gli+OmQCtcI4wvZUKdxDA5WjcqEqpTcxfb8emiENfP3J0FKfumYw6Y +rTM2SI2cpDzC05TF8PaucO6A48f39920d9AP+5WdExJ5XFsFFaPX7WM6w71iixr6 +Ntp1DO2VHnVcUQKCAQEA8MFsW2o9sAr18sJj06azaFk0otw9wz79aqrag/uSgJDL +FYiGixdRXVfT3m4/DYHVRSh3NPHcbh6KdaO1eJ2HOL11GwkclipIQhC8xvnbfYKb +xg49StUhyD1HxVXI+iIRW8jtJ6Fx+HltGlPp1muEdrehTbOTQz7Oc9TYm8aFNtWP +yPDqiAeOsy5v30oxKTm3D3i4hD0COcNXqKbMSI8iBULhIF0b7wU70qaILLiX/xoZ +zG5ipnPdsZQHC9y26j+2NAur+JCMHQFiapWctTOQRmX27LY+aQm7JtyUw/x+GGx/ +Ixc0gqoW05ngfMr3McMJ3f+kSc1FeaTe+ERG5sZL2wKCAQBsO7/iS1usJPiBIMUW +sxle0wsmtiUATvKBifvP0jdSThZQKlAM/pDimeoPsLXxu3YFa5LQF1rmCB9cJ4I3 +XIy0q5UzmamXOsavl/yt2URbLx97GF8s2ID//3+flFdq24X1dPOOFcytYb1Ua1JE +0RHvXuqeghqM6wXCsbpXhNEHBsCuAkxlOuZsQWbXNY3jhuNEsf9k+kEW0/2hkLrO +bFWEkWBXM5duZX8iRPKOzixX137ULfjolPYaJAzE7wdLSYgpD5kgAvD7Zx5TYliE +Vv2mhepHKTH9zHVSLx2C+U5BdS79uffEeOg7R6hIK6DkUiXGonmr78KxEazvFgpy +5iQRAoIBAQDXtna/8ZEUCr4TpNiM6vAUrtjakztDlUy6Jhtj5iR9zT4pLQpf1aSx +XeAXi/AyygGs1XT5mztF71df0C7owzxFOnuSnbdfVMMpbpW2MmjXLA8mhdulERIT +t9R2m0ZX1+51rrHOsHjNiP6YeFcsJ2modR+x3xQzTDLu1ea+rEDvwKn0AOgiuaLC +KPlTt8YUigHbeu7YjVFRMBV6pviiipyQ2jucI9DDeI0BUPTyHPMTPu+em8kIGwin +81nc5wV9HVjDiTGspNblpjfoB+VA9dJvQSzdKu0AcBef2kPw1mqkt5GyfzgtWvjY +3yakqbaSf453unYZKjL1qyOcjpB4dXPBAoIBAGzkqzHE0Izjd0jH59gxIBZvz864 +A6lrC+ltMtCYcVdDfvShgnTINnf7RYQ4U2HPB+IJ3IMRysT0LYdGDhp5Y8Zi73YO +KLGSl+P4jzs2z+MsavXk/wPi2xwc3htKHu8P6EFm50lR4jxNEXveGscwIm+wgr0F +W7gJsJSVeB3aK10dn1hfBo1J/8mimz3mZxpYIb/v+x5DYvwik657C+6p7RmylrZx +20jwy6L6d+qWL5V8H+KZoyRMb3xfsvHiOAUgFaNa+XivzRFeVqHYl9Cr1hpL0I8j +21Nm0f7u3QAGTrgjmPPNBI2lRoDbrOOO49R5rQne41iw9ahqSYfmOEYDTs8= +-----END RSA PRIVATE KEY----- diff --git a/security/advancedtls/testdata/server_key_2.pem b/security/advancedtls/testdata/server_key_2.pem new file mode 100644 index 0000000..74e572d --- /dev/null +++ b/security/advancedtls/testdata/server_key_2.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEAsQvTflthMNuwXz9t0uA7xkyIlfV+/c2qIF0IuW5B28TtDfi8 +y7TuxYcRBaCsEjtOC0zkQ+QXicGutBNYHDFYavIB7d9m6fkunMWF5gLbNvTzBzl1 +MPG1VVtGL4ew1KCrV98wRa69sEma/LpevNBdhvQkRUrVTVu2uui3oTvDL0Yus60s +YwPfy/RWYpG9vCMAr6J6PW/xM4FgDrwg9YpJX+xYvGTVRzagK7gfdiUBiT7/UmmV +A4+7FC8aOKOfwUUgIndwl14lUbg9XYl6uxUSzR2W0pxyZxKFcm4neu8l2q9JJo3r +oDSkTWTDYzN3Xa1Tx+5RMnvMQ7uGjflSujUjDjBd3DslY8HjX0uyAvz+Wxh/hKrz +ceQWtZi8c8VYE0E46/Oi+oyYvfEQ7rb+fqWBx17yclSO2wnwNULKt4bCSLLGGAis +0fBd3rC4JYs7vWFID3E/7ZdyAslEXQwA/DDKXRzlExs60M7ZNqDb9cKtppUmTnsp +LfzEBB1HbgNZaB56IG3oqOE8V1n4PS8WYX4k5RPKSArm8GCjLZMLj5PrtdEGJlLA +Yx/8m3P+kcMEQDKNCdWexPYLYT2fodeUouE9trtgJnSJMyUYD8OI2xBeoFv07tAY +qzZQxUSbbbrq4m5SOlVJo3KuBK8d9vKDJxeLmpgK9USxyPKpyO2wdcpSJfMCAwEA +AQKCAgEAmB9YNs7fgLKTJhQDElk3Ixipl2gcGIm5bxthHqsdDW90XDfoSIQLUU/P +kW1PzE6GrXEBBVCb5PK1YObqIzdHCIUuoSv+anV/1pZliY/UubDYjNGS314f99p4 +QOivSNNQxizwdj9Bn5JvCE4+jq/eXNGzxJIbGt/97zV8ap5GBH2iLSJT7DPs/HrS +KtmdFGVi9oZ90AI6Vo4IckC1dSTADRqv2BgvpYPLNiV7avE7E6k8ipxLvIaoMRyT +xCzbXJ4/kT3dUUJEgKX0nEU/XjYqNHIDIK3qIqQoY31AkQGhHfjUurrgxYPV1OYK +eFdFbgk63qPnwp/akCw13hFnQrXbiqt+ecpH82aGA5XW7wdngo59Ehpy7XWwG4Zn +MuyNVusSRUcclWD8PydLaweAKizjRzfVW/6nVtKiYTfkscArQTMZwEdkFkT/ZwcG +OSPTyf3hSUSmd17HPCvHm66jX2EVfB+MQfQhDulcbPyzvNDNHg83miMv0nrnWiHe +viOxT7M6pdJwMdHH7KfkkJmx+HJYDa8GwdGyCh2+dTfq42hRFvHNUMTCNrupwTO7 +yrxFnKMo/c5z6m6OvYyCh5k/wAkpbgZi5/k1EQG9uo7E7crO9AdMuzAgR1bvcU48 +MjJvxxh51J5A/VqV3RZR9CNomfLQ3WD6xVZUuvAyspRf3meO2akCggEBAOasi0oL +eEXNSLRlW+OxBenEL2Ke/GuAVy1+TkUAgNtHawUNK81FWSDIjv/+gB7WDZ2CaLUw +5UY6QigQ5Qjme0cE8QPnAdbCev0LSrXXbZ1aCF546szZu3VYVdU4s4PHcOojAzKk +pHYIYfbD11VHK5f5Ve8qt/I+DDGGALldfzgdSwx8K7n01Uu6zmeOvpXXirfR10AS +BOU9m/O2K5qk8g1MD33xqQjEk5BKdpgz9zfyWYlPj4rdo4IFK0em9bnwPJLPDu58 +F7DbKoAH5a5GY3bsODzWMWMhThpNTTvmqgZ1bLPBepnREslQ5Mf8MJYG0WU6QPNx +7tErFtpgY9PDEzcCggEBAMR7/PmV6fIpkEeAo490csFl1uoeiFEUF62SosJD2lpx ++iUirGAqs0c59NtzS8PzheDuU6S1EAvMcd5uJetST4NH/Yw4T1xjKFqkRC6Wlq/x +iokaH8SDizFx20dRCsJiNaxqqyr/RrVVYv27R2ihtW2482NNIl/bG/GgESZKN0hb +yHplWH0UoAwwSsJDRASi4CcrS30khjr/W3LKIo2iXVEd00P+Wbin9Vo7SgrpVujS +P2jrd0pp33yxZetur8XESnAjOiyStZ2tcapp1rFvj8i2YS9Zxd9bRXoaHd2XPvb2 +hm2l6VtqLZVpJyUlTNvWqWmM84EZAPSfB3BSMI/AGSUCggEAHioOBN6/GZGgokZm +3710Yn9PGvxjUcN0ovRTU96e+w25xu1T/wHEh+7yFDO5mU6wdRpqitccBDT2Fbsv +2BwbnsvcoIAC04yW/KQPXvwOz3bIhWIWgjcutkeY4csKXn8kGtn9PxAcmXq7JMOz +Uul9n9/xBtd1Om42tfsp+RNq4XGjMLzEEwsbIU4KU6xs67dF4ofEOBKjJT8LN7Fo +vk43gNmjZPrG+eiKy2GRZJHXEC/W2YfX43bcPNJkOHhyxZ/Oq/v7neAIUQ433oop +1MJLm2+EYyA3URk312SoZt7g+Ps9/budRqP6auzzHduylsvJcg1OFQefDScvU9sq +8rQdvQKCAQALgzhPZ3lNtyG9DsyGm0weCNmO3jsehQ7eHLlsqI0iv4rooh93gwj+ +I2c1dIv770jo5Q4BmJpYFqKVZd7S6v+9sXopvSLpRuYWaYmVMT2jEYQMhHtYCF0f +iIxQoW7/9MEwWQ+udUavWVFzjIWim9cFltCsANkCxNPeVIKsu6yBkN8uTMHiklLO +ZAX9W/OgUerQYLkLnBhBXLT/BNkBc4IEPrsiQMUBDNZTcyXjfciZ27fbbfCPa6Ss +qbhPEy05aUbzSx0df3skwgTm90ydGOxT1lvbamctryti/CTD1xjZX5iA1DfYI2CI +YKDqjET0nJ9Qj/G0nsJvkuHcsvQleBwBAoIBAQCvilpLyh8XzVY3TAsvVEaHpoNp +y2sIwDiI2elZOBcQkeUbsD4bhA1iF/4cpI9tgl7ApK8ZmmcRiKh1/PIHI4Ru4nB4 +bNqn7FP32vKyJ5O0o7bQBGGJIpLe0rVUmJ+ROB5PLw5aG/oo1bVKoMuw/u4o1z9S +90qI3LW6jNs+7UOELH6Gex6rfA+9xi//7NDUlJhQST++mS2pTm1/cq4RIaWx6EyF +N1hqjcWESyS1EtZYKp+/Mx4PDQKDAm2f9mjuTViW15EdcOBlMm40ZKRbxIJImlEe +fjZBgqsDoQKK0yYcQiVimMNc5vtNaT38lVu1NxvKJg1OeTboMBISLUOzZqQj +-----END RSA PRIVATE KEY----- diff --git a/security/advancedtls/testdata/server_key_3.pem b/security/advancedtls/testdata/server_key_3.pem new file mode 100644 index 0000000..ae8ff1a --- /dev/null +++ b/security/advancedtls/testdata/server_key_3.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDcIyJEt3ZAxPn7 +H5f/IwXS8NcoAXWP8L6rWndcg+EWayx7W+wmUsFKGSFGzrFPPCFmKO8MrQqp8LSA +xHAVtOC6Uw+INWJJw9BRlx2nvV7hfbqu3OnPkPVkN/siUQCqnEKJQHliNT9XDl4/ +Mav75uQSWb3Vfi3KtG7mzPFNNbbe4yfHyGbC4e9RtKkGimDSJ413s3m4+scDvtpC +cCXj9XXZNdCwD1CL3kNdmOdhgfkDBP+AMLBFKZKqpCo6m0s4JJTiej13dc27wTrn +kFm1CP77SV+kQlWg5DAcVXYJkN9FqNqExqIPS/SxMk7H+3qSQACbttQK9UmCn3qR +3pGbwqzNAgMBAAECggEAU0Te84tKKdnYjUs4HYRL8ay0VienJpl0JjEEMXSZMfe8 +TbVJsH1hK/wxgC0zGLuwDoqxUeQqwnmQbZzgoPVYhGJi360BztFI/XPh/c8+EqGS +eg6KSr+UcyJR1ns5e0+8Q1qmD6YAnZeLwu+xFIoT/3T+v8EI5UI3KQqgxAnrcIdc +RWqIwWLkkPm1QVRYsvRaTvmgFd2LcIT9AdIruP/VsqF3GEzvEQSr0lgmwOe0izLb +HKfZr+2JwOppwTLGhKo1wUyUUsglXCBOcFYAA03xdviQWBuCeKXamNrdB7M0T4zZ +THheNRQq2g7bTYnncCYKcrNa5VY5wmHY767mf6ahAQKBgQD+AGk21CsRgiKCGzUI +wTRynutAMkX55U1bud2+OMpzH9omvnccX5ezq5WTw/jZfz+jrUBF8YKSTtb3InUg +yXcpJ/XjRDFMZp1Avy/44rOlYg1QYMD4JK96f8bbd1yej5NVw45V9synaJHuvoDV +bkbZu00S0X8Pgvlh354MUH3CLQKBgQDd3oQdKMZgsX8gtmcQfQoEZ0VvlOYmTM5W +Kw+24iGrBkfBt+NuKx8qm1CpoFGx4G2+TgttMywjF9RG3R2uGqbJZbCOAXzjRMQJ +L9PuTiGAdYD3fA5cTmnrrNEhPydtRhF2M3p8FFeQtwsEBYreXux25rbmVOYTFMgJ +hVUW9RdZIQKBgQDwEYdgMQw70hm3iuuHSMS/iQCkfl+xH08MYRH6FkcSpIpVkDOX +96m0QXpwXQs41pJZqwhSkz9r9WQr1L+Lq58aoRBAK1XE9j+u0IUQ4YQVziTzUV9R +qarJRze2eoxpuR3yM5C2IzuvBqDXW+r8zuvcIrFoFeXXzVzTar1AulsCSQKBgQCa +UD+3QDrp2co/6F26vB0RfvpuZzPEA7undv/RBWrBVvblp46Je3iL28a4lAb+Hsh1 +ijasVuEl71b3iqcwBt1mSlIIEsTYFWX7tcZDgxgODqwKdcBPN0K4ZlR2OUSk3g0b +Fybj0gotXwJMY8Z4b7Er6b/gZ8A2GUggRxotg34fwQKBgEh6a88SgTyu1KcR2yzN +Zbs8MEfZ8hqfUj+GL0+6y9KQd4uSIngyHWGNETE9dCObNz5HEdJIpUNvB3vfN0Vk +f1EEPDQLQKJjii7jZ9U9XfPVaUhqIVH3Aupmb30H3AQw7XOF23g63k9Yg1FWtDT6 +4CuBsUvjXywL4yHrWK+BuSXF +-----END PRIVATE KEY----- diff --git a/security/advancedtls/testdata/server_key_localhost_1.pem b/security/advancedtls/testdata/server_key_localhost_1.pem new file mode 100644 index 0000000..1185eff --- /dev/null +++ b/security/advancedtls/testdata/server_key_localhost_1.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEA000BCJ2A3hX6ClLtLPZLncVnLbZBM9dbtP3wqFzdio0fLRJd +R4gJ1JbuYxD5nZpsNMTsqQ2VbEi8F9B3U5P4RIoLC6JNHVP3VaftajWtHa95vdXG +W5Yknk3Y6SGT4ZM7XcTjkBo20Pe7jCLp0vgpGb8kxlIhDtY6c0i654E0uiMhV8NR +DllRn6YHVhfEqht7bDZrqzJeLvdw3Ovz2js5TqSMv0By7wAeRskHapNMNrfDLnzF +wYWfa00t/Tybms9Ssvq6+M7Mi9xXe+03acmA3KYsp+RLjHfLLihlZGGkyDPZ+H23 +fE+39Adaia4qWYysAMfOsNCbzU2DOVW7cg9i1F+Lw8fieZVT66AmDVJ7TUBWZitV +Z/UayeioSb3n5DGa4Y2A8syrPXD2/nXLqhsS0G/TyN/xvc4rIULlLL3GyMS7Sj2S +SA1JqJbAUe0wZIHdzgXU/weHWQsTQY4dWORHDACXEuRnlCRn7+0ehYnfhXgQH3+y +6K/nD8fsqgFntgqaI4OQHQoqN4DDJtfxJCmz2Dd8XPjjCJYaNC36/3XocCXiq1Gd +94ojUokCyHHqzaKJtutuyf373GPo+dvXV9IM/FacFqhn/heIB+j4xXozcq20WkNb +Sb55iwAYg/cZAAeULDiWYadVqDDbfgf2xaNlHpMta17Zpg/6whmbWUxBugcCAwEA +AQKCAgBswO5uQ7qnE8Kc+6+M+7tRmd+QHIUUrJxL3IO39AwmmpnYNeKCxZbhr0lE +/eCr6GYXBuAT5qTolcsRqr8v6jHW/QHQXBm6pZPgp0y/5J6Ub9OGDHhKfU2dmM2y +uBCIAqKEkajaa1OZXFhQOUwFxKpK0SGZXX4cR9DPszhXnR3JS/mGVUXrz7b+J5MR +EaysLPbqbFwgQg1NuReC7YKV6POG8ZRrfz1om7P5lNBXXzbT1uMDkz6pax/xN0kb +VM118Y1MB1aiZrXKqn7wjth9fzPu3SyQwSTNSH7v4+TDtKn+TQm8JuCAf/tbA0nr +IRQ1AP0qbayJPuVh1qpaoTCX9SlU29QwahuXmjqZzbtXlre64Psfmx6fGpTrFVY4 +Irhd4If/VwtbHCK6hU4c8GsIori+5rgquKBQgawQgDXCcF4671RJlwQnYm9YT5cb +q84wgBfGbXODHV1A+qJL3J+K/YzwxZKi7w9hXE+MsBoDiiUPekTBZA3GkFRCXwpN +nnb/BpnHnf3Fvy+BcmjzlzaPJZ1mwLJJEH0/R820U59S5m2fh7Gwy9+ZZsSNsJVW +fGaMZBGhOgAiBLd7zl7If4Eza14U499P8Rl8+4RiaMjnQC8xyI5eb49DtKcPdDp9 +d+ZQiXTQeFWriCeXTQlEV8uUCpRosfUtJC7xjljQWHeNeMpnAQKCAQEA7u13pfYd +XH4BpO+wlRwjb4ADc6wLWFOm0nySNB3fzDrOix0vAGvm7sjtIGVYjcHAbjG2oflH +siqgFaT5iQUdvo5AGZU9rp6f4qctJQuSuti4eXE1P+YSGyNALISE49lpi/z3lwJ7 +K3zxT0FywS2pvHEMc3+s+T0otik285/TJ7yg9d5UTS3TAgLzkan6wzGCAos935Ok +0/pJXWk3jjmc7q9cMGPDbX2kb67j4BuqbDbiCNTdfL2BOErwZw6+lissp8TPVNf2 +03Hj24GQGO/A9H8U47M2bcY63UgcflmBwKvXVSnCVpYl9UjY/A4j45w2zjAX/lm1 +3mjTBiwQGVR1hwKCAQEA4mYt2SuyS6AkdpANUrzISS0FvLTy/Qb3iU7iBkSBqShe +XV/a520uTJ+5v0m6ifGsafrjKJ0Rb3GYAug/mzWItTuRArIH5hTNd+FaAeC9XBEv +dvUnAnffvocvAnmpQdO/kJLY54rA1KnuIJ30Xy5653LgQLur/b2EG7Fljc9vHqI8 +Emp7sLwPPFzelR+ryTak8GimUHm7psPJryvAeB8ak68TAmlBkLKrdO6oE3O1fJoO +cFbZG3r2KdRPZgBpegskOOgk0GQIG7z3G0auZpiwViRJLTBda1lLLYnVBcT2Qq10 +AQebj1pDC7Op33kZMNtQtrb6bvEdPhEsaWqcfZm3gQKCAQAvgcojlq8539gl2n7q +9yBYoESPcGsFEgT+n0RW1oXUTvEYmiHpXIsbeZokseIMtbS0dHAS/sTxuSYBh78S +LpE+fXxjWdhc6y9xWrpQPl/bhRIRG6Bx5yY8fSLadzMRNv6UliUIwraI7BvzHVla +7eBtFrFaGc3j9PQuXD2P7XyHzyrWGHH8sprdMIcLtJemziZCqTsRRIMmnwKNb0lb +nzsD/pw/Bucp0yyqBEVNH1Mglz0Ucnbjwa566fOpGjZtF4KWjTyIazSp0GB1Gerz ++mAMfWRC7jRpWVwE+byoptV04PY8+cOpgctkXSq/23PpYvtGvitXKLFP2tnyxToi +PzfrAoIBAGk9+HgosOQo2GppAliAu1YQ4MbdEst+bplcmwMw21lIE72yLm9AOLKT +2WPLoTQ4rN5DK0+Y3B8DHhfT4KWE2DzvKLSpD7Tr3KuqjQ2sbDodHwRcZ7rlAJRw +APFUntKj3TwWl0/jF0qEh9aPtqZ8U9O9efN9ijEU5RF+gGfQkqYZ4nTpHQCGG0sD +HNETfOa3SSscapukSw/1mY6ddwYf51nZm6uWRE1AUSW1P1pzgl0evDGKnbgBi+bb +8+DFtkJuZXMyrtJUfdRvHiuGytGUjvwsN/wSrIqXYrQTi3v4GEXcnb1QzQZxfhM1 +fHUOtSAaA0Y8fuQNn3tXvl5umbplN4ECggEBAIPDR2/Igmze9sCqZF06fexBZUcB +j6gsjY7zJYppAO6tniyLkQaOHoCZGvMA2xxcR1+4F+QQvLOkRTPxXHqZgNwze9oK +Y9QLkhsu0Q3A0+C/YC3HGL90HKndRx6pLHVhrBg3vxoTjQpPkpVT6UYnMGnRr5D5 +qluURPPYR56WX76rMud0yDTB/MeNsNjdBn8ukr1LwR40ircFgmy/O+FdR9mpPOVY +3DIDysyzug3IeIoa0QSy7aKHHGd9pTHJ3EA/tfoTCmr9iywKxsgPp1fcSjt8Nfg7 +mVo2qbVoqchrp0tFG2/MoxqiOlRMtWX6N37btlMxifj28BDE63hG53W4bGA= +-----END RSA PRIVATE KEY----- diff --git a/security/advancedtls/testdata/server_trust_cert_1.pem b/security/advancedtls/testdata/server_trust_cert_1.pem new file mode 100644 index 0000000..9612046 --- /dev/null +++ b/security/advancedtls/testdata/server_trust_cert_1.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFlzCCA3+gAwIBAgIUdkt73feqv3fH1K1fBBp2ryU4TUMwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMSEwHwYDVQQKDBhJbnRlcm5l +dCBXaWRnaXRzIFB0eSBMdGQxGzAZBgNVBAMMEmZvby5iYXIuaG9vLmNhLmNvbTAg +Fw0xOTExMTUxOTE1MTFaGA8yMjI1MDMyMDE5MTUxMVowWjELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAlZBMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQx +GzAZBgNVBAMMEmZvby5iYXIuaG9vLmNhLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAMLHM55ChPP7dk+1uWz2DSkLE/xkoBaagJkjXuBgpgtdcxfp +hG9qTQ/vPAnqRFNSqQPU/A0dbKlnlK2ibrSb1LD4CPiXMqMAVojbEBRiNZK2+E1p +6FDaUO/CiurX8QPsVHUp2Zol38FoGdHL3tHSEf2xfJzs1Ka4g54FASOn+wJSAdAG +Ai+TUT137NqmeIVMIhg8x2vtKJpIH016mBqPENpccb3wsk9kNLj9TonKP16Nkngm +YKdLBnhB5Coz9gFqnTFEXp54ESOKttNtAAdFfhBqJhYMAdoFxSsuDdpr23Nyfuzf +uT5QnIffD0JCxH6bGYpMgJMVLiWJSuZ6wohFl04lwQTj3UXC8GU9o8YGC1UnvJoZ +rTgC8bM+yNJnEsrU90dPMLAi6qN5pl0y18/jtyaP5YXjv2TCGAjmB3dUyFa4nCg+ +7w9tAi4pC3cBusN7e4cOseOM/23qKbcudHWAQ46VkTMs36DQyzxZutgZUI9lesol +o3eCR00v4N3Uf0yXff866EaDg3NmcZzhn1stJMHJMkhPOQZZmD8dd3Pi4DuQZMa/ +74vMcjLxXo2xKTQklBUDCAFVEIR0y0oHwYUCk+AuS0PAXbGred0KOs6Ey8c68JYZ +OfgD/jjY/emYzyNeGGKUkMtNA9xUqWNEnqmIQgpMndzy1c5UlnGpoOt/cfztAgMB +AAGjUzBRMB0GA1UdDgQWBBS0GQgc/BAjxTCGIrzLsV+t6npd8TAfBgNVHSMEGDAW +gBS0GQgc/BAjxTCGIrzLsV+t6npd8TAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4ICAQARMgDs180PFUSjXrRQ6hfpmYaauoq2atSYfzECfo92iemE1KiK +qUAVX/fjRy69/r6BKAo/j8F7BJVRqKhfiZm+EUIGWCtNRpVCx6WCtfJ1G8rEEH+U +E4kNpPC1OyVAhFMYKFJVXkyzpxjggLeY0bGs7BrX4wSid7vj6HM/pzfOShvB6qv0 +VfpAGwTKnqw64kpy+9QPwS0sDH17oJAteJ3WeRopsqCjK9eXljmGBKVZjv2m9/TT +7Jd6VCBm/x1yxPeuJfPTxkfGR3UEcKPgXG84N0nfbTLspQcBf2QqQtW4yL/PyRC/ +8sFAPanSkNc3u1ERQub0oUtd+jQalvxXqW1N0GAJHLvtXa5Etrz3WMfOVdqthEKK +CjGXdt4JoO+gvCGZH9jKa6HTgy+0QZrbOxBsJpbxSjXrJOeeJ2OgGZg8qBe5LqUD +Z3o45x6j3RiQrK24luZE/6A25VUvUke4Hr9oTBQFgMlIPuTeRw6XGkNzScaPrXEU +MnijDX8n7OcME+lCVCpgSd1SZzkTn4JYqlx8U33j1hRD1m5quO9+GOLQpWvZC5A5 +FsikGXULKuIxVCJMuCXeWdY1aDAJ/6cwz77eDzNkySUfDEhxjQGhCmNlNDHN3dCM +NtSqXJSDIwqikj7izot3evkoYa9j6w3qkNyg9fyGbdNHXp135RP5HIhqjA== +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/server_trust_cert_2.pem b/security/advancedtls/testdata/server_trust_cert_2.pem new file mode 100644 index 0000000..4a3c973 --- /dev/null +++ b/security/advancedtls/testdata/server_trust_cert_2.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFpTCCA42gAwIBAgIUTdt7HKlUedh94k4eA+nlamVgGSkwDQYJKoZIhvcNAQEL +BQAwYTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMSEwHwYDVQQKDBhJbnRlcm5l +dCBXaWRnaXRzIFB0eSBMdGQxIjAgBgNVBAMMGWZvby5iYXIuc2VydmVyMi50cnVz +dC5jb20wIBcNMjAwMTA5MjI0NDA5WhgPMjIyNTA1MTQyMjQ0MDlaMGExCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJDQTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ +dHkgTHRkMSIwIAYDVQQDDBlmb28uYmFyLnNlcnZlcjIudHJ1c3QuY29tMIICIjAN +BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA2KadB/xdPMRDW/LhFGJcbzVU+yoS +iRudc8w0Wq/0XpQwcDjxrq6v5XuzFIZU46Wb2g+eALNMjW7zv4BLFwEU0+CvMYWt +vgbTA2A07sU7P3WA8uZwjB25nkk0iMVBclL+g1XABnfXNobbKB/dyKArlyzFBV+w +rpV17RdkfXfGjeFWpfxF7KF4Wzh86XKSDYSQQE4kcQqSxDeZfRwm02jaXuPDmvUw +KFIxcfEW/3SadulFvOKgHWjUEirTGsT+8B8fWsfeJjGRmFcc1+utpOoOaC1+sRe6 +xTe7JJB9F13mZxEPJuFxuBvjmGiSXkyLWhVWeqzhTipojZ69mYzAxMs8AVWrYeru +EKuf3MlABub8dgDLocvOYD3A0IDm5173pU5RPW9tA2jBNLnyEF+wYFLjtFfYQesl +UlldccG+nZowaeUsiUPhTBzwAYSCdB+imtJxIT0xdOQCo+h9ASvnPpgk6AYaU/2d +gsFY39CvKmTFYlH2EGIJK3MWm6YT3T1fTTUgs/s++CkLzwAXpna4w8SLDl3IdeLX +lMiXhnoNr3uYeusxkJp5rtUHBsYPbH4Ec4erNRgbUuBnHJe4nlC6LCCycLHywhBr +niPPxyNBZzvrmRrVwx5xNEQn8r4ffftpASY/uePJK2wtrZop7mWFo/OnfMO6y/3C +22FK5wIbVLLsDlECAwEAAaNTMFEwHQYDVR0OBBYEFGOI6k3QPu9e+EORdUDkFqsV +szK5MB8GA1UdIwQYMBaAFGOI6k3QPu9e+EORdUDkFqsVszK5MA8GA1UdEwEB/wQF +MAMBAf8wDQYJKoZIhvcNAQELBQADggIBAMJtk0AbpT3pu+2G+NK3D4T2brrP66An +lRlQxDXQ0uKunGYMgam+sJWMz3agviekRVQk9Vog9FwiGoYsS3X6ojLrA1FXp/8h +oVXNmW8R87IS2KyPbzTmO+0OvO/KhYmA0USIhAmj645fyy8dGCQQOZCSfXE5/zCM +ODnrgeai3qw+KB4aGJ6fgDKMdPbyl7fyvu5EWDIycuij9S8FQJ7m2gWolxFAN4/c +nnWr/s6n8AQrb+k4Dp50nOrDA7JUEnFfQcBuJpDN2v5MD1/x83R1ZVuqNa+fOgrW +DdSm/XbaPpzZa/R6iJQxG8mNpNEjMnBq7WCa1tLLd7MrdxzrwaFdfRiMj91b/A4W +GZbX7SMrByI/6M01YoTdsPW2i/EDxJjghSGkvwuA2MPe8UqXELn5wpTXTDgCsj8V +j25GUupDB8Dm5aocLEFHiUwzAGcy19zVqepTaM4w//iA1qUuaG7DE8pVzL9XFxm+ +L1CGfxSTqdbqWa9PcLUoTI/8n6KQdK+vczgY4y+aUOZdGgLcVoO1BF6McnNPiihk +d+HdWb0xGjw63XsV5kC41y6mHBQJdJTm0CE+yZ1e6gt+YEZCELxpxg530J0CngHs +tCftzNI8o2pQhDhhKzxxGiA1cuzrrLDpdqNZo6VNm5tyYPicVbicZoJSbNDxohEJ +rzhu9hQ7iDV5 +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/server_trust_key_1.pem b/security/advancedtls/testdata/server_trust_key_1.pem new file mode 100644 index 0000000..b9c072a --- /dev/null +++ b/security/advancedtls/testdata/server_trust_key_1.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDCxzOeQoTz+3ZP +tbls9g0pCxP8ZKAWmoCZI17gYKYLXXMX6YRvak0P7zwJ6kRTUqkD1PwNHWypZ5St +om60m9Sw+Aj4lzKjAFaI2xAUYjWStvhNaehQ2lDvworq1/ED7FR1KdmaJd/BaBnR +y97R0hH9sXyc7NSmuIOeBQEjp/sCUgHQBgIvk1E9d+zapniFTCIYPMdr7SiaSB9N +epgajxDaXHG98LJPZDS4/U6Jyj9ejZJ4JmCnSwZ4QeQqM/YBap0xRF6eeBEjirbT +bQAHRX4QaiYWDAHaBcUrLg3aa9tzcn7s37k+UJyH3w9CQsR+mxmKTICTFS4liUrm +esKIRZdOJcEE491FwvBlPaPGBgtVJ7yaGa04AvGzPsjSZxLK1PdHTzCwIuqjeaZd +MtfP47cmj+WF479kwhgI5gd3VMhWuJwoPu8PbQIuKQt3AbrDe3uHDrHjjP9t6im3 +LnR1gEOOlZEzLN+g0Ms8WbrYGVCPZXrKJaN3gkdNL+Dd1H9Ml33/OuhGg4NzZnGc +4Z9bLSTByTJITzkGWZg/HXdz4uA7kGTGv++LzHIy8V6NsSk0JJQVAwgBVRCEdMtK +B8GFApPgLktDwF2xq3ndCjrOhMvHOvCWGTn4A/442P3pmM8jXhhilJDLTQPcVKlj +RJ6piEIKTJ3c8tXOVJZxqaDrf3H87QIDAQABAoICAGXTJ7wDgGfgPNCc6uv4kZa0 +UOVwYXSPnszv/ciFHijw2JtWm8J3KwQ6iAOS8dcxbmQvcvkUOdsx6DsBoKhQktdV +Q7NZr8IhChwPkY9mbCVf+9zUkfu6tfcxl9f/veLUKK77iuOYCyqb1mukDb9Y98jN +gZyz/tONwFjauua+CW4EGyh6C6h9dkoRKMSBpJ3i2Cwdkg9s8v382Ehz35J62k+d +ZmTqsPzqINnYqrdEAO7YSgr/3SV4BlDV+YbKlT/WUYkQ+foUQLl46e0LnakvfiDs +rS53Znxo6dOSBvH50sa+w3Xn23qlP7+UL/Du4LRjNu3i4pCB0RcUeBCXep0s7FSm +ZjhxZvFpFBin5NjoCrtwCwl+ijJfprKnNBPD0X+cpYKNuw7QBPufPUvLmje2m9mi +R9GTqMF9Ur2ZqERU9NQ7hPPYYBJ6Fu6xWi8tsu8919FOn0sxTaWcAMmN9cp4sQ9M +fLnMNQdsySp7YtEQ2cXMQv0SyId3q+rfM5wSNH0YO548X0pWApjHFUSj8qZDgXIH +4TJzPfpGcvCVXPBujcKSKocme3PcDRXjXwBV39fuZ0A/1DUusJKU7gYZN1ZR4jrI +TGEmf8AvFZUxeJ7w2QnlRYBhMWUnItGAA39YCIsBin7GRD5IgpINCM9ccxrykbuH +2RDahIVs7uXfBdTu3h2NAoIBAQD5bVlrJkQFbtEM7blCQe4ffEEzxsx15U9ZmIzu +YtfvXGevs5mAzP5NcQOWcmD+wvd69alN08E0sWeje9fEXUyILD0ZIhRMNn6DQVI0 +DtKLfOCtTtDabh7PBl3v7W54pzH217KUzE3Ob29rd7ALJebSgyVMtLGQ6gLe0HWy +immFpnOm8qbCSabhfR6ZdiIok+ST9lFpAmTnkz+dgQLWJQ4MpM0AqUD5OiYahyFj +7LggVXWSDAQCqfZbVr1KLOfjoVtsUGChSwzpFxlEmB1wXM64Q4t5gYTsT2gew8pz +bNSE7OqKTHv7foqS8tfISSi7JlJ3LL9VR9Dld1nBAN5+qXyfAoIBAQDH6S/P1/W/ +WRUTZuhJfKNz8zG9fG4AwKqJazf4YA/XsOwMFyDRLSMv7DDfwbXutUhItwPuDQPy +3qG0jhL/ipIJNwney148eBU8SunEgKnZm6bNk+08VL+o/9/mZkRKG6uzPDdeUVwo +CSZvLtJWo6f0IrIFtzd07fANqZ6CFnyQDA4o3bc1Eq8t0rbWE1fNMVKKBn/b1N6y +tgDVyGKpj4ZuwLDGZ+gQLSYdH9v8xp8pzDxallE2HP2dtqt01FEEMXsIW6tZ04l4 +/VRdAXi8ro1nWus0yiX2RonAbcnJ7zVM/YdDMFU7DawzjQMiO4UT4OPPVxe+2tNV +R9ra6owoQA7zAoIBAQCzLYlpvqBoorXMKs3FuiT8Oz9/mVTxcFwzSbIb4aerTF8z +yboA28HnEcN5BQuGl7o+e1E3FmIZn0OLHoDekANVYyo07tVT9mWllnwd53P6PigM +d6zy7N526+T5YT/Vro3m/AZOfAF8xXJt6hntuDl7ijh2ROu15VVQiMG0E1hAaVV1 +XaTLtysJmt8rcMCTE8LFQ9IxtEWWUaIGXFIUUaQpEw4tZmjFYK9UqTQkWz3eBGYk +FzueSkguTz5FlcKzNAu/4HG6DHbmzvAY5YloWVMq7WK5U4CQXW63gwDhMBHut165 +IL6D6OBVNdwrBdsbrijZcay075Ux8i3oxt4OcWSTAoIBAEYiOPPiAAUxa4NzButB +Htb+6uRfUvhQn4O2adxpVyWEnEthkdHQ1Bdr9XmKrBki4Ekia+6IAmqiUHjXnzKn +mrRA6uWO03DDcC/G2FxoBy6gvNRCoWgZE2Rm4FYkarDVJFetOH+Oa5ZgH2vCMWjT +4Yh045+9t2b+Usl4SHO7D9g5Yn5TyoKEG5En650PDC6gryRdQ14MQFTSJVjbBEIY +aEFSuLHiojeKn2R4WOVFiXFQhZwCQFuLsC40d9J06jdeZJt6DZNl80TPG1nFumX3 +lwQ7kWjjwo20EX/BBJojob9w8pNP0Zb2JQOw5PiNiRKAQ2vqUhpTCvFQVCeZQbKd +RqECggEBAIZh7qdFBFcCGzoRYnn+eNaJTxGDIRCZIn5Ur8SBUYEIE6+aB5ecTaLK +eBfSCl9lmVaol6P3T/fXVyUwCscPU6FaeWGe9v89+Y/JqM1zGWtXqWI9Lcvowmb0 +f5AenJXAjtcFUakB3xYyOakBzAHLEnacwaTPGR8s186hNXl9PV5sTFDN89IGhh9G +hCQyNtiyNbckQOYzO4yoDQiYfcsTZ57DWtfFvRP3T4A08fgmUzkr0jYoy1dPP1g/ +GBsgOVNr+LLgj353GqwrsHnG0Y+JarOfb31HcgR9fi4w7PruQ3ioQQaKINJBpfzH +HASpvDH+panUrtqSvjDZMuDvkA6qft8= +-----END PRIVATE KEY----- diff --git a/security/advancedtls/testdata/server_trust_key_2.pem b/security/advancedtls/testdata/server_trust_key_2.pem new file mode 100644 index 0000000..d04d176 --- /dev/null +++ b/security/advancedtls/testdata/server_trust_key_2.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQDYpp0H/F08xENb +8uEUYlxvNVT7KhKJG51zzDRar/RelDBwOPGurq/le7MUhlTjpZvaD54As0yNbvO/ +gEsXARTT4K8xha2+BtMDYDTuxTs/dYDy5nCMHbmeSTSIxUFyUv6DVcAGd9c2htso +H93IoCuXLMUFX7CulXXtF2R9d8aN4Val/EXsoXhbOHzpcpINhJBATiRxCpLEN5l9 +HCbTaNpe48Oa9TAoUjFx8Rb/dJp26UW84qAdaNQSKtMaxP7wHx9ax94mMZGYVxzX +662k6g5oLX6xF7rFN7skkH0XXeZnEQ8m4XG4G+OYaJJeTItaFVZ6rOFOKmiNnr2Z +jMDEyzwBVath6u4Qq5/cyUAG5vx2AMuhy85gPcDQgObnXvelTlE9b20DaME0ufIQ +X7BgUuO0V9hB6yVSWV1xwb6dmjBp5SyJQ+FMHPABhIJ0H6Ka0nEhPTF05AKj6H0B +K+c+mCToBhpT/Z2CwVjf0K8qZMViUfYQYgkrcxabphPdPV9NNSCz+z74KQvPABem +drjDxIsOXch14teUyJeGeg2ve5h66zGQmnmu1QcGxg9sfgRzh6s1GBtS4Gccl7ie +ULosILJwsfLCEGueI8/HI0FnO+uZGtXDHnE0RCfyvh99+2kBJj+548krbC2tminu +ZYWj86d8w7rL/cLbYUrnAhtUsuwOUQIDAQABAoICAQCioY/Hat3iu8GEyHHFh4Cz +ymkckZyQZ7ZuMqAqY2MhjERAOb7SzjckIRNxGNWofazcqFSHWhDhKqS24Gt9vUYR +NtzMY/jkaOMF6bZSdqPfIynFLM7Xn4izFWjmMozKcRq1JC2drWBUgi8Jk8I81F9k +gCr1ubs7kt6PN7wrozndT4Zn21PyKdPbRjAeXe7dTuGqI/6fDLzXppUFoZhToqYq +DPfM3rljyy9qxPvqj3FUShAbllNzQDnR2WvW8IIfZn12/An6ycLthJcWTshuv3RJ +J72u2o1NdmR5Mi102PwX6mphWWKwPd8/jWAygWsqGFJujFAlCRirFrplBY+/KoDD +bcJz7jek7elO09SGA20W2G9DHRvUr4fknUsXCUj5PCGehDQrfYFeKFt3t383i765 +WIXZmak1owxPtSuOmVbXqEVvwBkQ990E0+qxKeo1Tn1aANBZZVVb1LgJ75Zkmqrp +ARRb0h75G9cKZYex+3mgjECsBWurk2eriHS2D3RfJzlDpoZWqiMhMjC32kJQonws +0X7fgGs0vl2gPxq1xAs0QLjV6BgcYwJF7QdhEXiJUUKaB7aDBpVr+7jbVl8eIoql +zPE9owqQHhN5POSEnu76RPByYHt2twHXBpF0SFWKx7Nu0DpNqNqexVpqMNlz2Ehk +tjY6xm/hdWRLw0cNUI/4AQKCAQEA9sBDIEmXbnF0RtuKXMkkDOaeS3QM6vDQbRPf +itfuC9+B+qeLUkA4yyMLYml5mNuHawx34NoClmruESw/ASqgcqCxX/R9qGBJzkXn +saN6uF1ZQKKngzZ3UdrbVf7R1RBikHZHN0/Sn7mdhwM/CyQnD6/H0EOHLTk90v1d +Ctz8zOn6yqCqpyLedQteZavO3WKLzzothzS9WmblgALuYfGG/bRhNIkoM4JtkLsB +4hdp0n/tbIEIbMNAtvemXDO4N0VvMOa4m5if26tYI0vGIvkqHc+Oe5Jx1w3u9G6J +n+gF4hvdgpa3hpmIyP6o97hmyviP4I1KaonT5lHk9UvbKVEz8QKCAQEA4MWFJ5+7 +dpjvHLH9p1iBEbtdpd3Nd6wdcNjGErFdMEdsSdEQgVdbQVSbnCPqr9d79lqrzIsM +wfV6AND15SfAVdD4BS7DQI5RxwQCnMU+Z4knGTUZp72TtuWYLDQ1zkKbVfl9U97a +jtCz+YYp/GHJxHF+TVW8ltvPmNja+Cccf1DfXXwJG539Rl7NGULaBurRn4rNKNA2 +JmNB5DEnI+34ly+DBt5KKzbUc1nL6dO2ddnl7uokgDW3B6xDSZ+tdLFhYPSrX1em +VhxxvteLTqv9hyLu9u5f6wxphyo6GSMXTA8Yc+ID0GNLLb8kJmi97jFYVxRbWxev +QtOJGRjn631gYQKCAQEAoBFk+kMDG0A6H+U3Qq2w1zWbpnLoFliVvMzRjO46nDUn +yoR5mqfSr+RR9Etb+E8g786szY5fc1h2i2lajdUrNHEN36NpCJs+BbPPc6sLZyIX +Thi19iaVDOKeupCNalwwtGomFLmRdtAgYn82nHGdbU2on2/O9wVVF9QIUY296Og4 +Ks5DJh02llMDr4zeqzrMW2fwNO9/jm+FnZ9JKPxXh6lGDaCUFaYckXDe7d4mZclb +KbIi1vtqtca9gr6CWEiQsvZY94bw3L2wdWUoaXOdYK1OTtdXRhzh0GsMmFEZz+4n +qhk/gO+Ejm61Cc3z0OOh4heGGMrETXr+vimxSIJG4QKCAQBTYfLbmC36+RD7HCx1 +ACghY9iBx56JXpgtXL1eAd4IIvbRC3WMBdQckD6J1ekiAlZCNbC12H+LFH2F//64 +W97F9xeLFKXqNOGxapNthN55mi+e8kvqJjG+D74758JuGdd2NW+AxZNel52sW1EI +B17KOTAZkEy9yh1hHlFc7WVs9ZtnGrRmQl3K1TBQxrQLDOFmxh8FnPf5lajD9lgG +xCkMLNv2mE/7aAO4Jv+2ZouxfHwH/WQ9C7AycH0lus6mE4eEaD+KxwE1wKeRnHRZ +YwRSNWtgv11l3Nzo/4k9+f6SgKcZlibED5G8DsRiW0jaLAQRicO6LzcdG0wou0yN +150BAoIBAQD0dDgOjnlXzvw8OXFcNn41K9U/oXzO/cNyxbRZP4wY0f7PEUIoF2gJ +OZ4bTAXA5PQxs3fwKfC1UKN129mTcJy9HnJGJQKBRwN+W/SRbnw7yR93idwe1kGy +iGGBO1bORbgj9y40QamZgnGqDRxsYmwCVss6mamtyNJtwobkWK4Wb33Uex6ZXyFK +wJ5htqviYe5oYo2Yor9ok5Xf66npmYTtv5STAhKjk+PTvlTGckwr4zEWvkgnXHJd +XDNx0r6O6FhkxPMIlLfX5fsaCL0jBxX+tkh/vYuF70JnZAQmEphRsLljVCr2jIQs +m4DEMelbu4jDoUwmms+yra/9chKHzaRB +-----END PRIVATE KEY----- diff --git a/security/advancedtls/testdata/testdata.go b/security/advancedtls/testdata/testdata.go new file mode 100644 index 0000000..25dfccb --- /dev/null +++ b/security/advancedtls/testdata/testdata.go @@ -0,0 +1,43 @@ +/* + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package testdata contains functionality to find data files in tests. +package testdata + +import ( + "path/filepath" + "runtime" +) + +// basepath is the root directory of this package. +var basepath string + +func init() { + _, currentFile, _, _ := runtime.Caller(0) + basepath = filepath.Dir(currentFile) +} + +// Path returns the absolute path the given relative file or directory path, +// relative to the google.golang.org/grpc/testdata directory in the user's GOPATH. +// If rel is already absolute, it is returned unmodified. +func Path(rel string) string { + if filepath.IsAbs(rel) { + return rel + } + + return filepath.Join(basepath, rel) +} diff --git a/server.go b/server.go new file mode 100644 index 0000000..4e9acef --- /dev/null +++ b/server.go @@ -0,0 +1,1991 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "errors" + "fmt" + "io" + "math" + "net" + "net/http" + "reflect" + "runtime" + "runtime/pprof" + "strings" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/encoding/proto" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/binarylog" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpcutil" + istats "google.golang.org/grpc/internal/stats" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" + "google.golang.org/grpc/tap" +) + +const ( + defaultServerMaxReceiveMessageSize = 1024 * 1024 * 4 + defaultServerMaxSendMessageSize = math.MaxInt32 + + // Server transports are tracked in a map which is keyed on listener + // address. For regular gRPC traffic, connections are accepted in Serve() + // through a call to Accept(), and we use the actual listener address as key + // when we add it to the map. But for connections received through + // ServeHTTP(), we do not have a listener and hence use this dummy value. + listenerAddressForServeHTTP = "listenerAddressForServeHTTP" +) + +func init() { + internal.GetServerCredentials = func(srv *Server) credentials.TransportCredentials { + return srv.opts.creds + } + internal.IsRegisteredMethod = func(srv *Server, method string) bool { + return srv.isRegisteredMethod(method) + } + internal.ServerFromContext = serverFromContext + internal.AddGlobalServerOptions = func(opt ...ServerOption) { + globalServerOptions = append(globalServerOptions, opt...) + } + internal.ClearGlobalServerOptions = func() { + globalServerOptions = nil + } + internal.BinaryLogger = binaryLogger + internal.JoinServerOptions = newJoinServerOption + internal.BufferPool = bufferPool + internal.MetricsRecorderForServer = func(srv *Server) estats.MetricsRecorder { + return istats.NewMetricsRecorderList(srv.opts.statsHandlers) + } + internal.XDSFilterWrapperOption = xdsFilterWrapperOption +} + +var statusOK = status.New(codes.OK, "") +var logger = grpclog.Component("core") + +// MethodHandler is a function type that processes a unary RPC method call. +type MethodHandler func(srv any, ctx context.Context, dec func(any) error, interceptor UnaryServerInterceptor) (any, error) + +// MethodDesc represents an RPC service's method specification. +type MethodDesc struct { + MethodName string + Handler MethodHandler +} + +// ServiceDesc represents an RPC service's specification. +type ServiceDesc struct { + ServiceName string + // The pointer to the service interface. Used to check whether the user + // provided implementation satisfies the interface requirements. + HandlerType any + Methods []MethodDesc + Streams []StreamDesc + Metadata any +} + +// serviceInfo wraps information about a service. It is very similar to +// ServiceDesc and is constructed from it for internal purposes. +type serviceInfo struct { + serviceImpl any // Implementation for the methods in this service. + streams map[string]*StreamDesc // Streaming descriptors and wrapped unary descriptors for this service. + mdata any +} + +// Server is a gRPC server to serve RPC requests. +type Server struct { + opts serverOptions + statsHandler stats.Handler + + mu sync.Mutex // guards following + lis map[net.Listener]bool + // conns contains all active server transports. It is a map keyed on a + // listener address with the value being the set of active transports + // belonging to that listener. + conns map[string]map[transport.ServerTransport]bool + serve bool + drain bool + cv *sync.Cond // signaled when connections close for GracefulStop + services map[string]*serviceInfo // service name -> service info + events traceEventLog + + quit *grpcsync.Event + done *grpcsync.Event + channelzRemoveOnce sync.Once + serveWG sync.WaitGroup // counts active Serve goroutines for Stop/GracefulStop + handlersWG sync.WaitGroup // counts active method handler goroutines + + channelz *channelz.Server + + serverWorkerChannel chan func() + serverWorkerChannelClose func() +} + +type serverOptions struct { + creds credentials.TransportCredentials + codec baseCodec + cp Compressor + dc Decompressor + unaryInt UnaryServerInterceptor + streamInt StreamServerInterceptor + chainUnaryInts []UnaryServerInterceptor + chainStreamInts []StreamServerInterceptor + binaryLogger binarylog.Logger + inTapHandle tap.ServerInHandle + statsHandlers []stats.Handler + maxConcurrentStreams uint32 + maxReceiveMessageSize int + maxSendMessageSize int + unknownStreamDesc *StreamDesc + keepaliveParams keepalive.ServerParameters + keepalivePolicy keepalive.EnforcementPolicy + initialWindowSize int32 + initialConnWindowSize int32 + writeBufferSize int + readBufferSize int + sharedWriteBuffer bool + connectionTimeout time.Duration + maxHeaderListSize *uint32 + headerTableSize *uint32 + numServerWorkers uint32 + bufferPool mem.BufferPool + waitForHandlers bool + staticWindowSize bool + streamWrapper func(ServerStream) (ServerStream, error) +} + +var defaultServerOptions = serverOptions{ + maxConcurrentStreams: math.MaxUint32, + maxReceiveMessageSize: defaultServerMaxReceiveMessageSize, + maxSendMessageSize: defaultServerMaxSendMessageSize, + connectionTimeout: 120 * time.Second, + writeBufferSize: defaultWriteBufSize, + sharedWriteBuffer: true, + readBufferSize: defaultReadBufSize, + bufferPool: mem.DefaultBufferPool(), +} +var globalServerOptions []ServerOption + +// A ServerOption sets options such as credentials, codec and keepalive parameters, etc. +type ServerOption interface { + apply(*serverOptions) +} + +// EmptyServerOption does not alter the server configuration. It can be embedded +// in another structure to build custom server options. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type EmptyServerOption struct{} + +func (EmptyServerOption) apply(*serverOptions) {} + +// funcServerOption wraps a function that modifies serverOptions into an +// implementation of the ServerOption interface. +type funcServerOption struct { + f func(*serverOptions) +} + +func (fdo *funcServerOption) apply(do *serverOptions) { + fdo.f(do) +} + +func newFuncServerOption(f func(*serverOptions)) *funcServerOption { + return &funcServerOption{ + f: f, + } +} + +// joinServerOption provides a way to combine arbitrary number of server +// options into one. +type joinServerOption struct { + opts []ServerOption +} + +func (mdo *joinServerOption) apply(do *serverOptions) { + for _, opt := range mdo.opts { + opt.apply(do) + } +} + +func newJoinServerOption(opts ...ServerOption) ServerOption { + return &joinServerOption{opts: opts} +} + +// SharedWriteBuffer allows reusing per-connection transport write buffer. +// If this option is set to true every connection will release the buffer after +// flushing the data on the wire. +// +// Deprecated: shared write buffer is enabled by default. SharedWriteBuffer +// will be removed in a future release. +func SharedWriteBuffer(val bool) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.sharedWriteBuffer = val + }) +} + +// WriteBufferSize determines how much data can be batched before doing a write +// on the wire. The default value for this buffer is 32KB. Zero or negative +// values will disable the write buffer such that each write will be on underlying +// connection. Note: A Send call may not directly translate to a write. +func WriteBufferSize(s int) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.writeBufferSize = s + }) +} + +// ReadBufferSize lets you set the size of read buffer, this determines how much +// data can be read at most for one read syscall. The default value for this +// buffer is 32KB. Zero or negative values will disable read buffer for a +// connection so data framer can access the underlying conn directly. +func ReadBufferSize(s int) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.readBufferSize = s + }) +} + +// InitialWindowSize returns a ServerOption that sets window size for stream. +// The lower bound for window size is 64K and any value smaller than that will be ignored. +func InitialWindowSize(s int32) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.initialWindowSize = s + o.staticWindowSize = true + }) +} + +// InitialConnWindowSize returns a ServerOption that sets window size for a connection. +// The lower bound for window size is 64K and any value smaller than that will be ignored. +func InitialConnWindowSize(s int32) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.initialConnWindowSize = s + o.staticWindowSize = true + }) +} + +// StaticStreamWindowSize returns a ServerOption to set the initial stream +// window size to the value provided and disables dynamic flow control. +// The lower bound for window size is 64K and any value smaller than that +// will be ignored. +// +// Note that this also disables dynamic flow control for the connection, +// falling back to a default static connection-level window of 64KB. To +// use a larger connection-level window, you must also use the +// [StaticConnWindowSize] ServerOption. +// +// Most users should not configure static flow control windows unless +// operating in a memory-constrained environment. +func StaticStreamWindowSize(s int32) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.initialWindowSize = s + o.staticWindowSize = true + }) +} + +// StaticConnWindowSize returns a ServerOption to set the initial connection +// window size to the value provided and disables dynamic flow control. +// The lower bound for window size is 64K and any value smaller than that +// will be ignored. +// +// Note that this also disables dynamic flow control for individual streams, +// falling back to a default static connection-level window of 64KB. To +// explicitly configure the stream-level window size, you must also use the +// [StaticStreamWindowSize] ServerOption. +// +// Most users should not configure static flow control windows unless +// operating in a memory-constrained environment. +func StaticConnWindowSize(s int32) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.initialConnWindowSize = s + o.staticWindowSize = true + }) +} + +// KeepaliveParams returns a ServerOption that sets keepalive and max-age parameters for the server. +func KeepaliveParams(kp keepalive.ServerParameters) ServerOption { + if kp.Time > 0 && kp.Time < internal.KeepaliveMinServerPingTime { + logger.Warning("Adjusting keepalive ping interval to minimum period of 1s") + kp.Time = internal.KeepaliveMinServerPingTime + } + + return newFuncServerOption(func(o *serverOptions) { + o.keepaliveParams = kp + }) +} + +// KeepaliveEnforcementPolicy returns a ServerOption that sets keepalive enforcement policy for the server. +func KeepaliveEnforcementPolicy(kep keepalive.EnforcementPolicy) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.keepalivePolicy = kep + }) +} + +// CustomCodec returns a ServerOption that sets a codec for message marshaling and unmarshaling. +// +// This will override any lookups by content-subtype for Codecs registered with RegisterCodec. +// +// Deprecated: register codecs using encoding.RegisterCodec. The server will +// automatically use registered codecs based on the incoming requests' headers. +// See also +// https://github.com/grpc/grpc-go/blob/master/Documentation/encoding.md#using-a-codec. +// Will be supported throughout 1.x. +func CustomCodec(codec Codec) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.codec = newCodecV0Bridge(codec) + }) +} + +// ForceServerCodec returns a ServerOption that sets a codec for message +// marshaling and unmarshaling. +// +// This will override any lookups by content-subtype for Codecs registered +// with RegisterCodec. +// +// See Content-Type on +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. Also see the documentation on RegisterCodec and +// CallContentSubtype for more details on the interaction between encoding.Codec +// and content-subtype. +// +// This function is provided for advanced users; prefer to register codecs +// using encoding.RegisterCodec. +// The server will automatically use registered codecs based on the incoming +// requests' headers. See also +// https://github.com/grpc/grpc-go/blob/master/Documentation/encoding.md#using-a-codec. +// Will be supported throughout 1.x. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func ForceServerCodec(codec encoding.Codec) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.codec = newCodecV1Bridge(codec) + }) +} + +// ForceServerCodecV2 is the equivalent of ForceServerCodec, but for the new +// CodecV2 interface. +// +// Will be supported throughout 1.x. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func ForceServerCodecV2(codecV2 encoding.CodecV2) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.codec = codecV2 + }) +} + +// RPCCompressor returns a ServerOption that sets a compressor for outbound +// messages. For backward compatibility, all outbound messages will be sent +// using this compressor, regardless of incoming message compression. By +// default, server messages will be sent using the same compressor with which +// request messages were sent. +// +// Deprecated: use encoding.RegisterCompressor instead. Will be supported +// throughout 1.x. +func RPCCompressor(cp Compressor) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.cp = cp + }) +} + +// RPCDecompressor returns a ServerOption that sets a decompressor for inbound +// messages. It has higher priority than decompressors registered via +// encoding.RegisterCompressor. +// +// Deprecated: use encoding.RegisterCompressor instead. Will be supported +// throughout 1.x. +func RPCDecompressor(dc Decompressor) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.dc = dc + }) +} + +// MaxMsgSize returns a ServerOption to set the max message size in bytes the server can receive. +// If this is not set, gRPC uses the default limit. +// +// Deprecated: use MaxRecvMsgSize instead. Will be supported throughout 1.x. +func MaxMsgSize(m int) ServerOption { + return MaxRecvMsgSize(m) +} + +// MaxRecvMsgSize returns a ServerOption to set the max message size in bytes the server can receive. +// If this is not set, gRPC uses the default 4MB. +func MaxRecvMsgSize(m int) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.maxReceiveMessageSize = m + }) +} + +// MaxSendMsgSize returns a ServerOption to set the max message size in bytes the server can send. +// If this is not set, gRPC uses the default `math.MaxInt32`. +func MaxSendMsgSize(m int) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.maxSendMessageSize = m + }) +} + +// MaxConcurrentStreams returns a ServerOption that will apply a limit on the number +// of concurrent streams to each ServerTransport. +func MaxConcurrentStreams(n uint32) ServerOption { + if n == 0 { + n = math.MaxUint32 + } + return newFuncServerOption(func(o *serverOptions) { + o.maxConcurrentStreams = n + }) +} + +// Creds returns a ServerOption that sets credentials for server connections. +func Creds(c credentials.TransportCredentials) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.creds = c + }) +} + +// UnaryInterceptor returns a ServerOption that sets the UnaryServerInterceptor for the +// server. Only one unary interceptor can be installed. The construction of multiple +// interceptors (e.g., chaining) can be implemented at the caller. +func UnaryInterceptor(i UnaryServerInterceptor) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + if o.unaryInt != nil { + panic("The unary server interceptor was already set and may not be reset.") + } + o.unaryInt = i + }) +} + +// ChainUnaryInterceptor returns a ServerOption that specifies the chained interceptor +// for unary RPCs. The first interceptor will be the outer most, +// while the last interceptor will be the inner most wrapper around the real call. +// All unary interceptors added by this method will be chained. +func ChainUnaryInterceptor(interceptors ...UnaryServerInterceptor) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.chainUnaryInts = append(o.chainUnaryInts, interceptors...) + }) +} + +// StreamInterceptor returns a ServerOption that sets the StreamServerInterceptor for the +// server. Only one stream interceptor can be installed. +func StreamInterceptor(i StreamServerInterceptor) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + if o.streamInt != nil { + panic("The stream server interceptor was already set and may not be reset.") + } + o.streamInt = i + }) +} + +// ChainStreamInterceptor returns a ServerOption that specifies the chained interceptor +// for streaming RPCs. The first interceptor will be the outer most, +// while the last interceptor will be the inner most wrapper around the real call. +// All stream interceptors added by this method will be chained. +func ChainStreamInterceptor(interceptors ...StreamServerInterceptor) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.chainStreamInts = append(o.chainStreamInts, interceptors...) + }) +} + +// InTapHandle returns a ServerOption that sets the tap handle for all the server +// transport to be created. Only one can be installed. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func InTapHandle(h tap.ServerInHandle) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + if o.inTapHandle != nil { + panic("The tap handle was already set and may not be reset.") + } + o.inTapHandle = h + }) +} + +// StatsHandler returns a ServerOption that sets the stats handler for the server. +func StatsHandler(h stats.Handler) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + if h == nil { + logger.Error("ignoring nil parameter in grpc.StatsHandler ServerOption") + // Do not allow a nil stats handler, which would otherwise cause + // panics. + return + } + o.statsHandlers = append(o.statsHandlers, h) + }) +} + +// binaryLogger returns a ServerOption that can set the binary logger for the +// server. +func binaryLogger(bl binarylog.Logger) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.binaryLogger = bl + }) +} + +// UnknownServiceHandler returns a ServerOption that allows for adding a custom +// unknown service handler. The provided method is a bidi-streaming RPC service +// handler that will be invoked instead of returning the "unimplemented" gRPC +// error whenever a request is received for an unregistered service or method. +// The handling function and stream interceptor (if set) have full access to +// the ServerStream, including its Context. +func UnknownServiceHandler(streamHandler StreamHandler) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.unknownStreamDesc = &StreamDesc{ + StreamName: "unknown_service_handler", + Handler: streamHandler, + // We need to assume that the users of the streamHandler will want to use both. + ClientStreams: true, + ServerStreams: true, + } + }) +} + +// ConnectionTimeout returns a ServerOption that sets the timeout for +// connection establishment (up to and including HTTP/2 handshaking) for all +// new connections. If this is not set, the default is 120 seconds. A zero or +// negative value will result in an immediate timeout. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func ConnectionTimeout(d time.Duration) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.connectionTimeout = d + }) +} + +// MaxHeaderListSizeServerOption is a ServerOption that sets the max +// (uncompressed) size of header list that the server is prepared to accept. +type MaxHeaderListSizeServerOption struct { + MaxHeaderListSize uint32 +} + +func (o MaxHeaderListSizeServerOption) apply(so *serverOptions) { + so.maxHeaderListSize = &o.MaxHeaderListSize +} + +// MaxHeaderListSize returns a ServerOption that sets the max (uncompressed) size +// of header list that the server is prepared to accept. +func MaxHeaderListSize(s uint32) ServerOption { + return MaxHeaderListSizeServerOption{ + MaxHeaderListSize: s, + } +} + +// HeaderTableSize returns a ServerOption that sets the size of dynamic +// header table for stream. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func HeaderTableSize(s uint32) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.headerTableSize = &s + }) +} + +// NumStreamWorkers returns a ServerOption that sets the number of worker +// goroutines that should be used to process incoming streams. Setting this to +// zero (default) will disable workers and spawn a new goroutine for each +// stream. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func NumStreamWorkers(numServerWorkers uint32) ServerOption { + // TODO: If/when this API gets stabilized (i.e. stream workers become the + // only way streams are processed), change the behavior of the zero value to + // a sane default. Preliminary experiments suggest that a value equal to the + // number of CPUs available is most performant; requires thorough testing. + return newFuncServerOption(func(o *serverOptions) { + o.numServerWorkers = numServerWorkers + }) +} + +// WaitForHandlers cause Stop to wait until all outstanding method handlers have +// exited before returning. If false, Stop will return as soon as all +// connections have closed, but method handlers may still be running. By +// default, Stop does not wait for method handlers to return. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func WaitForHandlers(w bool) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.waitForHandlers = w + }) +} + +func bufferPool(bufferPool mem.BufferPool) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.bufferPool = bufferPool + }) +} + +// xdsFilterWrapperOption returns a ServerOption that sets the server-level +// stream wrapper (used internally by xDS server filters). +func xdsFilterWrapperOption(w func(ServerStream) (ServerStream, error)) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.streamWrapper = w + }) +} + +// serverWorkerResetThreshold defines how often the stack must be reset. Every +// N requests, by spawning a new goroutine in its place, a worker can reset its +// stack so that large stacks don't live in memory forever. 2^16 should allow +// each goroutine stack to live for at least a few seconds in a typical +// workload (assuming a QPS of a few thousand requests/sec). +const serverWorkerResetThreshold = 1 << 16 + +// serverWorker blocks on a *transport.ServerStream channel forever and waits +// for data to be fed by serveStreams. This allows multiple requests to be +// processed by the same goroutine, removing the need for expensive stack +// re-allocations (see the runtime.morestack problem [1]). +// +// [1] https://github.com/golang/go/issues/18138 +func (s *Server) serverWorker() { + for completed := 0; completed < serverWorkerResetThreshold; completed++ { + f, ok := <-s.serverWorkerChannel + if !ok { + return + } + f() + } + go s.serverWorker() +} + +// initServerWorkers creates worker goroutines and a channel to process incoming +// connections to reduce the time spent overall on runtime.morestack. +func (s *Server) initServerWorkers() { + s.serverWorkerChannel = make(chan func()) + s.serverWorkerChannelClose = sync.OnceFunc(func() { + close(s.serverWorkerChannel) + }) + for i := uint32(0); i < s.opts.numServerWorkers; i++ { + go s.serverWorker() + } +} + +// NewServer creates a gRPC server which has no service registered and has not +// started to accept requests yet. +func NewServer(opt ...ServerOption) *Server { + opts := defaultServerOptions + for _, o := range globalServerOptions { + o.apply(&opts) + } + for _, o := range opt { + o.apply(&opts) + } + s := &Server{ + lis: make(map[net.Listener]bool), + opts: opts, + statsHandler: istats.NewCombinedHandler(opts.statsHandlers...), + conns: make(map[string]map[transport.ServerTransport]bool), + services: make(map[string]*serviceInfo), + quit: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + channelz: channelz.RegisterServer(""), + } + chainUnaryServerInterceptors(s) + chainStreamServerInterceptors(s) + s.cv = sync.NewCond(&s.mu) + if EnableTracing { + _, file, line, _ := runtime.Caller(1) + s.events = newTraceEventLog("grpc.Server", fmt.Sprintf("%s:%d", file, line)) + } + + if s.opts.numServerWorkers > 0 { + s.initServerWorkers() + } + + channelz.Info(logger, s.channelz, "Server created") + return s +} + +// printf records an event in s's event log, unless s has been stopped. +// REQUIRES s.mu is held. +func (s *Server) printf(format string, a ...any) { + if s.events != nil { + s.events.Printf(format, a...) + } +} + +// errorf records an error in s's event log, unless s has been stopped. +// REQUIRES s.mu is held. +func (s *Server) errorf(format string, a ...any) { + if s.events != nil { + s.events.Errorf(format, a...) + } +} + +// ServiceRegistrar wraps a single method that supports service registration. It +// enables users to pass concrete types other than grpc.Server to the service +// registration methods exported by the IDL generated code. +type ServiceRegistrar interface { + // RegisterService registers a service and its implementation to the + // concrete type implementing this interface. It may not be called + // once the server has started serving. + // desc describes the service and its methods and handlers. impl is the + // service implementation which is passed to the method handlers. + RegisterService(desc *ServiceDesc, impl any) +} + +// RegisterService registers a service and its implementation to the gRPC +// server. It is called from the IDL generated code. This must be called before +// invoking Serve. If ss is non-nil (for legacy code), its type is checked to +// ensure it implements sd.HandlerType. +func (s *Server) RegisterService(sd *ServiceDesc, ss any) { + if ss != nil { + ht := reflect.TypeOf(sd.HandlerType).Elem() + st := reflect.TypeOf(ss) + if !st.Implements(ht) { + logger.Fatalf("grpc: Server.RegisterService found the handler of type %v that does not satisfy %v", st, ht) + } + } + s.register(sd, ss) +} + +func (s *Server) register(sd *ServiceDesc, ss any) { + s.mu.Lock() + defer s.mu.Unlock() + s.printf("RegisterService(%q)", sd.ServiceName) + if s.serve { + logger.Fatalf("grpc: Server.RegisterService after Server.Serve for %q", sd.ServiceName) + } + if _, ok := s.services[sd.ServiceName]; ok { + logger.Fatalf("grpc: Server.RegisterService found duplicate service registration for %q", sd.ServiceName) + } + info := &serviceInfo{ + serviceImpl: ss, + streams: make(map[string]*StreamDesc), + mdata: sd.Metadata, + } + for i := range sd.Streams { + d := &sd.Streams[i] + info.streams[d.StreamName] = d + } + for i := range sd.Methods { + d := &sd.Methods[i] + info.streams[d.MethodName] = &StreamDesc{ + StreamName: d.MethodName, + Handler: s.wrapUnaryHandler(d), + ServerStreams: false, + ClientStreams: false, + } + } + s.services[sd.ServiceName] = info +} + +// MethodInfo contains the information of an RPC including its method name and type. +type MethodInfo struct { + // Name is the method name only, without the service name or package name. + Name string + // IsClientStream indicates whether the RPC is a client streaming RPC. + IsClientStream bool + // IsServerStream indicates whether the RPC is a server streaming RPC. + IsServerStream bool +} + +// ServiceInfo contains unary RPC method info, streaming RPC method info and metadata for a service. +type ServiceInfo struct { + Methods []MethodInfo + // Metadata is the metadata specified in ServiceDesc when registering service. + Metadata any +} + +// GetServiceInfo returns a map from service names to ServiceInfo. +// Service names include the package names, in the form of .. +func (s *Server) GetServiceInfo() map[string]ServiceInfo { + ret := make(map[string]ServiceInfo) + for n, srv := range s.services { + methods := make([]MethodInfo, 0, len(srv.streams)) + // Iterate over unary methods first to maintain backward compatibility of order. + for m, d := range srv.streams { + if !d.ClientStreams && !d.ServerStreams { + methods = append(methods, MethodInfo{ + Name: m, + IsClientStream: false, + IsServerStream: false, + }) + } + } + // Iterate over streaming methods. + for m, d := range srv.streams { + if d.ClientStreams || d.ServerStreams { + methods = append(methods, MethodInfo{ + Name: m, + IsClientStream: d.ClientStreams, + IsServerStream: d.ServerStreams, + }) + } + } + + ret[n] = ServiceInfo{ + Methods: methods, + Metadata: srv.mdata, + } + } + return ret +} + +// ErrServerStopped indicates that the operation is now illegal because of +// the server being stopped. +var ErrServerStopped = errors.New("grpc: the server has been stopped") + +type listenSocket struct { + net.Listener + channelz *channelz.Socket +} + +func (l *listenSocket) Close() error { + err := l.Listener.Close() + channelz.RemoveEntry(l.channelz.ID) + channelz.Info(logger, l.channelz, "ListenSocket deleted") + return err +} + +// Serve accepts incoming connections on the listener lis, creating a new +// ServerTransport and service goroutine for each. The service goroutines +// read gRPC requests and then call the registered handlers to reply to them. +// Serve returns when lis.Accept fails with fatal errors. lis will be closed when +// this method returns. +// Serve will return a non-nil error unless Stop or GracefulStop is called. +// +// Note: All supported releases of Go (as of December 2023) override the OS +// defaults for TCP keepalive time and interval to 15s. To enable TCP keepalive +// with OS defaults for keepalive time and interval, callers need to do the +// following two things: +// - pass a net.Listener created by calling the Listen method on a +// net.ListenConfig with the `KeepAlive` field set to a negative value. This +// will result in the Go standard library not overriding OS defaults for TCP +// keepalive interval and time. But this will also result in the Go standard +// library not enabling TCP keepalives by default. +// - override the Accept method on the passed in net.Listener and set the +// SO_KEEPALIVE socket option to enable TCP keepalives, with OS defaults. +func (s *Server) Serve(lis net.Listener) error { + s.mu.Lock() + s.printf("serving") + s.serve = true + if s.lis == nil { + // Serve called after Stop or GracefulStop. + s.mu.Unlock() + lis.Close() + return ErrServerStopped + } + + s.serveWG.Add(1) + defer func() { + s.serveWG.Done() + if s.quit.HasFired() { + // Stop or GracefulStop called; block until done and return nil. + <-s.done.Done() + } + }() + + ls := &listenSocket{ + Listener: lis, + channelz: channelz.RegisterSocket(&channelz.Socket{ + SocketType: channelz.SocketTypeListen, + Parent: s.channelz, + RefName: lis.Addr().String(), + LocalAddr: lis.Addr(), + SocketOptions: channelz.GetSocketOption(lis)}, + ), + } + s.lis[ls] = true + + defer func() { + s.mu.Lock() + if s.lis != nil && s.lis[ls] { + ls.Close() + delete(s.lis, ls) + } + s.mu.Unlock() + }() + + s.mu.Unlock() + channelz.Info(logger, ls.channelz, "ListenSocket created") + + var tempDelay time.Duration // how long to sleep on accept failure + for { + rawConn, err := lis.Accept() + if err != nil { + if ne, ok := err.(interface { + Temporary() bool + }); ok && ne.Temporary() { + if tempDelay == 0 { + tempDelay = 5 * time.Millisecond + } else { + tempDelay *= 2 + tempDelay = min(tempDelay, 1*time.Second) + } + s.mu.Lock() + s.printf("Accept error: %v; retrying in %v", err, tempDelay) + s.mu.Unlock() + timer := time.NewTimer(tempDelay) + select { + case <-timer.C: + case <-s.quit.Done(): + timer.Stop() + return nil + } + continue + } + s.mu.Lock() + s.printf("done serving; Accept = %v", err) + s.mu.Unlock() + + if s.quit.HasFired() { + return nil + } + return err + } + tempDelay = 0 + // Start a new goroutine to deal with rawConn so we don't stall this Accept + // loop goroutine. + // + // Make sure we account for the goroutine so GracefulStop doesn't nil out + // s.conns before this conn can be added. + s.serveWG.Add(1) + go func() { + s.handleRawConn(lis.Addr().String(), rawConn) + s.serveWG.Done() + }() + } +} + +// handleRawConn forks a goroutine to handle a just-accepted connection that +// has not had any I/O performed on it yet. +func (s *Server) handleRawConn(lisAddr string, rawConn net.Conn) { + if s.quit.HasFired() { + rawConn.Close() + return + } + rawConn.SetDeadline(time.Now().Add(s.opts.connectionTimeout)) + + // Finish handshaking (HTTP2) + st := s.newHTTP2Transport(rawConn) + rawConn.SetDeadline(time.Time{}) + if st == nil { + return + } + + if cc, ok := rawConn.(interface { + PassServerTransport(transport.ServerTransport) + }); ok { + cc.PassServerTransport(st) + } + + if !s.addConn(lisAddr, st) { + return + } + go func() { + s.serveStreams(context.Background(), st, rawConn) + s.removeConn(lisAddr, st) + }() +} + +// newHTTP2Transport sets up a http/2 transport (using the +// gRPC http2 server transport in transport/http2_server.go). +func (s *Server) newHTTP2Transport(c net.Conn) transport.ServerTransport { + config := &transport.ServerConfig{ + MaxStreams: s.opts.maxConcurrentStreams, + ConnectionTimeout: s.opts.connectionTimeout, + Credentials: s.opts.creds, + InTapHandle: s.opts.inTapHandle, + StatsHandler: s.statsHandler, + KeepaliveParams: s.opts.keepaliveParams, + KeepalivePolicy: s.opts.keepalivePolicy, + InitialWindowSize: s.opts.initialWindowSize, + InitialConnWindowSize: s.opts.initialConnWindowSize, + WriteBufferSize: s.opts.writeBufferSize, + ReadBufferSize: s.opts.readBufferSize, + SharedWriteBuffer: s.opts.sharedWriteBuffer, + ChannelzParent: s.channelz, + MaxHeaderListSize: s.opts.maxHeaderListSize, + HeaderTableSize: s.opts.headerTableSize, + BufferPool: s.opts.bufferPool, + StaticWindowSize: s.opts.staticWindowSize, + } + st, err := transport.NewServerTransport(c, config) + if err != nil { + s.mu.Lock() + s.errorf("NewServerTransport(%q) failed: %v", c.RemoteAddr(), err) + s.mu.Unlock() + // ErrConnDispatched means that the connection was dispatched away from + // gRPC; those connections should be left open. + if err != credentials.ErrConnDispatched { + // Don't log on ErrConnDispatched and io.EOF to prevent log spam. + if err != io.EOF { + channelz.Info(logger, s.channelz, "grpc: Server.Serve failed to create ServerTransport: ", err) + } + c.Close() + } + return nil + } + + return st +} + +func (s *Server) serveStreams(ctx context.Context, st transport.ServerTransport, rawConn net.Conn) { + ctx = transport.SetConnection(ctx, rawConn) + ctx = peer.NewContext(ctx, st.Peer()) + if s.statsHandler != nil { + ctx = s.statsHandler.TagConn(ctx, &stats.ConnTagInfo{ + RemoteAddr: st.Peer().Addr, + LocalAddr: st.Peer().LocalAddr, + }) + s.statsHandler.HandleConn(ctx, &stats.ConnBegin{}) + } + + defer func() { + st.Close(errors.New("finished serving streams for the server transport")) + if s.statsHandler != nil { + s.statsHandler.HandleConn(ctx, &stats.ConnEnd{}) + } + }() + + streamQuota := newHandlerQuota(s.opts.maxConcurrentStreams) + st.HandleStreams(ctx, func(stream *transport.ServerStream) { + s.handlersWG.Add(1) + streamQuota.acquire() + f := func() { + defer streamQuota.release() + defer s.handlersWG.Done() + s.handleStream(st, stream) + } + + if s.opts.numServerWorkers > 0 { + select { + case s.serverWorkerChannel <- f: + return + default: + // If all stream workers are busy, fallback to the default code path. + } + } + go f() + }) +} + +var _ http.Handler = (*Server)(nil) + +// ServeHTTP implements the Go standard library's http.Handler +// interface by responding to the gRPC request r, by looking up +// the requested gRPC method in the gRPC server s. +// +// The provided HTTP request must have arrived on an HTTP/2 +// connection. When using the Go standard library's server, +// practically this means that the Request must also have arrived +// over TLS. +// +// To share one port (such as 443 for https) between gRPC and an +// existing http.Handler, use a root http.Handler such as: +// +// if r.ProtoMajor == 2 && strings.HasPrefix( +// r.Header.Get("Content-Type"), "application/grpc") { +// grpcServer.ServeHTTP(w, r) +// } else { +// yourMux.ServeHTTP(w, r) +// } +// +// Note that ServeHTTP uses Go's HTTP/2 server implementation which is totally +// separate from grpc-go's HTTP/2 server. Performance and features may vary +// between the two paths. ServeHTTP does not support some gRPC features +// available through grpc-go's HTTP/2 server. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + st, err := transport.NewServerHandlerTransport(w, r, s.statsHandler, s.opts.bufferPool) + if err != nil { + // Errors returned from transport.NewServerHandlerTransport have + // already been written to w. + return + } + if !s.addConn(listenerAddressForServeHTTP, st) { + return + } + defer s.removeConn(listenerAddressForServeHTTP, st) + s.serveStreams(r.Context(), st, nil) +} + +func (s *Server) addConn(addr string, st transport.ServerTransport) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.conns == nil { + st.Close(errors.New("Server.addConn called when server has already been stopped")) + return false + } + if s.drain { + // Transport added after we drained our existing conns: drain it + // immediately. + st.Drain("") + } + + if s.conns[addr] == nil { + // Create a map entry if this is the first connection on this listener. + s.conns[addr] = make(map[transport.ServerTransport]bool) + } + s.conns[addr][st] = true + return true +} + +func (s *Server) removeConn(addr string, st transport.ServerTransport) { + s.mu.Lock() + defer s.mu.Unlock() + + conns := s.conns[addr] + if conns != nil { + delete(conns, st) + if len(conns) == 0 { + // If the last connection for this address is being removed, also + // remove the map entry corresponding to the address. This is used + // in GracefulStop() when waiting for all connections to be closed. + delete(s.conns, addr) + } + s.cv.Broadcast() + } +} + +func (s *Server) incrCallsStarted() { + s.channelz.ServerMetrics.CallsStarted.Add(1) + s.channelz.ServerMetrics.LastCallStartedTimestamp.Store(time.Now().UnixNano()) +} + +func (s *Server) incrCallsSucceeded() { + s.channelz.ServerMetrics.CallsSucceeded.Add(1) +} + +func (s *Server) incrCallsFailed() { + s.channelz.ServerMetrics.CallsFailed.Add(1) +} + +// chainUnaryServerInterceptors chains all unary server interceptors into one. +func chainUnaryServerInterceptors(s *Server) { + // Prepend opts.unaryInt to the chaining interceptors if it exists, since unaryInt will + // be executed before any other chained interceptors. + interceptors := s.opts.chainUnaryInts + if s.opts.unaryInt != nil { + interceptors = append([]UnaryServerInterceptor{s.opts.unaryInt}, s.opts.chainUnaryInts...) + } + + var chainedInt UnaryServerInterceptor + if len(interceptors) == 0 { + chainedInt = nil + } else if len(interceptors) == 1 { + chainedInt = interceptors[0] + } else { + chainedInt = chainUnaryInterceptors(interceptors) + } + + s.opts.unaryInt = chainedInt +} + +func chainUnaryInterceptors(interceptors []UnaryServerInterceptor) UnaryServerInterceptor { + return func(ctx context.Context, req any, info *UnaryServerInfo, handler UnaryHandler) (any, error) { + return interceptors[0](ctx, req, info, getChainUnaryHandler(interceptors, 0, info, handler)) + } +} + +func getChainUnaryHandler(interceptors []UnaryServerInterceptor, curr int, info *UnaryServerInfo, finalHandler UnaryHandler) UnaryHandler { + if curr == len(interceptors)-1 { + return finalHandler + } + return func(ctx context.Context, req any) (any, error) { + return interceptors[curr+1](ctx, req, info, getChainUnaryHandler(interceptors, curr+1, info, finalHandler)) + } +} + +// chainStreamServerInterceptors chains all stream server interceptors into one. +func chainStreamServerInterceptors(s *Server) { + // Prepend opts.streamInt to the chaining interceptors if it exists, since streamInt will + // be executed before any other chained interceptors. + interceptors := s.opts.chainStreamInts + if s.opts.streamInt != nil { + interceptors = append([]StreamServerInterceptor{s.opts.streamInt}, s.opts.chainStreamInts...) + } + + var chainedInt StreamServerInterceptor + if len(interceptors) == 0 { + chainedInt = nil + } else if len(interceptors) == 1 { + chainedInt = interceptors[0] + } else { + chainedInt = chainStreamInterceptors(interceptors) + } + + s.opts.streamInt = chainedInt +} + +func chainStreamInterceptors(interceptors []StreamServerInterceptor) StreamServerInterceptor { + return func(srv any, ss ServerStream, info *StreamServerInfo, handler StreamHandler) error { + return interceptors[0](srv, ss, info, getChainStreamHandler(interceptors, 0, info, handler)) + } +} + +func getChainStreamHandler(interceptors []StreamServerInterceptor, curr int, info *StreamServerInfo, finalHandler StreamHandler) StreamHandler { + if curr == len(interceptors)-1 { + return finalHandler + } + return func(srv any, stream ServerStream) error { + return interceptors[curr+1](srv, stream, info, getChainStreamHandler(interceptors, curr+1, info, finalHandler)) + } +} + +// wrapUnaryHandler converts a standard Unary Method Descriptor into a +// StreamHandler, allowing Unary RPCs to be processed via the same unified +// pipeline as Streaming RPCs. +func (s *Server) wrapUnaryHandler(md *MethodDesc) StreamHandler { + return func(srv any, stream ServerStream) error { + df := func(v any) error { return stream.RecvMsg(v) } + reply, err := md.Handler(srv, stream.Context(), df, s.opts.unaryInt) + if err != nil { + return err + } + return stream.SendMsg(reply) + } +} + +func (s *Server) processRPC(ctx context.Context, stream *transport.ServerStream, info *serviceInfo, sd *StreamDesc, trInfo *traceInfo) (err error) { + if channelz.IsOn() { + s.incrCallsStarted() + } + sh := s.statsHandler + var statsBegin *stats.Begin + if sh != nil { + statsBegin = &stats.Begin{ + BeginTime: time.Now(), + IsClientStream: sd.ClientStreams, + IsServerStream: sd.ServerStreams, + } + sh.HandleRPC(ctx, statsBegin) + } + ctx = NewContextWithServerTransportStream(ctx, stream) + ss := &serverStream{ + ctx: ctx, + s: stream, + p: parser{r: stream, bufferPool: s.opts.bufferPool}, + codec: s.getCodec(stream.ContentSubtype()), + desc: sd, + maxReceiveMessageSize: s.opts.maxReceiveMessageSize, + maxSendMessageSize: s.opts.maxSendMessageSize, + trInfo: trInfo, + statsHandler: sh, + } + + if sh != nil || trInfo != nil || channelz.IsOn() { + // The deferred error handling for tracing, stats handler and channelz are + // combined into one function to reduce stack usage -- a defer takes ~56-64 + // bytes on the stack, so overflowing the stack will require a stack + // re-allocation, which is expensive. + // + // To maintain behavior similar to separate deferred statements, statements + // should be executed in the reverse order. That is, tracing first, stats + // handler second, and channelz last. Note that panics *within* defers will + // lead to different behavior, but that's an acceptable compromise; that + // would be undefined behavior territory anyway. + defer func() { + if trInfo != nil { + ss.mu.Lock() + if err != nil && err != io.EOF { + ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []any{err}}, true) + ss.trInfo.tr.SetError() + } + ss.trInfo.tr.Finish() + ss.trInfo.tr = nil + ss.mu.Unlock() + } + + if sh != nil { + end := &stats.End{ + BeginTime: statsBegin.BeginTime, + EndTime: time.Now(), + } + if err != nil && err != io.EOF { + end.Error = toRPCErr(err) + } + sh.HandleRPC(ctx, end) + } + + if channelz.IsOn() { + if err != nil && err != io.EOF { + s.incrCallsFailed() + } else { + s.incrCallsSucceeded() + } + } + }() + } + + if ml := binarylog.GetMethodLogger(stream.Method()); ml != nil { + ss.binlogs = append(ss.binlogs, ml) + } + if s.opts.binaryLogger != nil { + if ml := s.opts.binaryLogger.GetMethodLogger(stream.Method()); ml != nil { + ss.binlogs = append(ss.binlogs, ml) + } + } + if len(ss.binlogs) != 0 { + md, _ := metadata.FromIncomingContext(ctx) + logEntry := &binarylog.ClientHeader{ + Header: md, + MethodName: stream.Method(), + PeerAddr: nil, + } + if deadline, ok := ctx.Deadline(); ok { + logEntry.Timeout = time.Until(deadline) + if logEntry.Timeout < 0 { + logEntry.Timeout = 0 + } + } + if a := md[":authority"]; len(a) > 0 { + logEntry.Authority = a[0] + } + if peer, ok := peer.FromContext(ss.Context()); ok { + logEntry.PeerAddr = peer.Addr + } + for _, binlog := range ss.binlogs { + binlog.Log(ctx, logEntry) + } + } + + // If dc is set and matches the stream's compression, use it. Otherwise, try + // to find a matching registered compressor for decomp. + if rc := stream.RecvCompress(); s.opts.dc != nil && s.opts.dc.Type() == rc { + ss.decompressorV0 = s.opts.dc + } else if rc != "" && rc != encoding.Identity { + ss.decompressorV1 = encoding.GetCompressor(rc) + if ss.decompressorV1 == nil { + st := status.Newf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", rc) + ss.s.WriteStatus(st) + return st.Err() + } + } + + // If cp is set, use it. Otherwise, attempt to compress the response using + // the incoming message compression method. + // + // NOTE: this needs to be ahead of all handling, https://github.com/grpc/grpc-go/issues/686. + if s.opts.cp != nil { + ss.compressorV0 = s.opts.cp + ss.sendCompressorName = s.opts.cp.Type() + } else if rc := stream.RecvCompress(); rc != "" && rc != encoding.Identity { + // Legacy compressor not specified; attempt to respond with same encoding. + ss.compressorV1 = encoding.GetCompressor(rc) + if ss.compressorV1 != nil { + ss.sendCompressorName = rc + } + } + + if ss.sendCompressorName != "" { + if err := stream.SetSendCompress(ss.sendCompressorName); err != nil { + return status.Errorf(codes.Internal, "grpc: failed to set send compressor: %v", err) + } + } + + ss.ctx = newContextWithRPCInfo(ss.ctx, false, ss.codec, ss.compressorV0, ss.compressorV1) + + // Execute the xDS HTTP filter interceptors. + var appErr error + var wrappedStream ServerStream = ss + if s.opts.streamWrapper != nil { + wrappedStream, appErr = s.opts.streamWrapper(ss) + } + + if appErr == nil { + if trInfo != nil { + trInfo.tr.LazyLog(&trInfo.firstLine, false) + } + var server any + if info != nil { + server = info.serviceImpl + } + if s.opts.streamInt == nil || (!sd.ClientStreams && !sd.ServerStreams) { + // If there is no stream interceptor, or if this is a unary RPC, call + // the handler directly. The wrapped unary handler will call the unary + // interceptor if it exists. + appErr = sd.Handler(server, wrappedStream) + } else { + info := &StreamServerInfo{ + FullMethod: stream.Method(), + IsClientStream: sd.ClientStreams, + IsServerStream: sd.ServerStreams, + } + appErr = s.opts.streamInt(server, wrappedStream, info, sd.Handler) + } + } + if appErr != nil { + appStatus, ok := status.FromError(appErr) + if !ok { + // Convert non-status application error to a status error with code + // Unknown, but handle context errors specifically. + appStatus = status.FromContextError(appErr) + appErr = appStatus.Err() + } + if trInfo != nil { + ss.mu.Lock() + ss.trInfo.tr.LazyLog(stringer(appStatus.Message()), true) + ss.trInfo.tr.SetError() + ss.mu.Unlock() + } + if len(ss.binlogs) != 0 { + if !ss.serverHeaderBinlogged { + if h, _ := ss.s.Header(); h.Len() > 0 { + sh := &binarylog.ServerHeader{ + Header: h, + } + ss.serverHeaderBinlogged = true + for _, binlog := range ss.binlogs { + binlog.Log(ctx, sh) + } + } + } + st := &binarylog.ServerTrailer{ + Trailer: ss.s.Trailer(), + Err: appErr, + } + for _, binlog := range ss.binlogs { + binlog.Log(ctx, st) + } + } + if err := ss.s.WriteStatus(appStatus); err != nil { + channelz.Warningf(logger, s.channelz, "grpc: Server.processRPC failed to write status: %v", err) + } + return appErr + } + if trInfo != nil { + ss.mu.Lock() + ss.trInfo.tr.LazyLog(stringer("OK"), false) + ss.mu.Unlock() + } + if len(ss.binlogs) != 0 { + st := &binarylog.ServerTrailer{ + Trailer: ss.s.Trailer(), + Err: appErr, + } + for _, binlog := range ss.binlogs { + binlog.Log(ctx, st) + } + } + if err := ss.s.WriteStatus(statusOK); err != nil { + channelz.Warningf(logger, s.channelz, "grpc: Server.processRPC failed to write status: %v", err) + } + return err +} + +func (s *Server) handleMalformedMethodName(stream *transport.ServerStream, ti *traceInfo) { + if ti != nil { + ti.tr.LazyLog(&fmtStringer{"Malformed method name %q", []any{stream.Method()}}, true) + ti.tr.SetError() + } + errDesc := fmt.Sprintf("malformed method name: %q", stream.Method()) + if err := stream.WriteStatus(status.New(codes.Unimplemented, errDesc)); err != nil { + if ti != nil { + ti.tr.LazyLog(&fmtStringer{"%v", []any{err}}, true) + ti.tr.SetError() + } + channelz.Warningf(logger, s.channelz, "grpc: Server.handleStream failed to write status: %v", err) + } + if ti != nil { + ti.tr.Finish() + } +} + +func (s *Server) handleStream(t transport.ServerTransport, stream *transport.ServerStream) { + ctx := stream.Context() + ctx = contextWithServer(ctx, s) + if envconfig.LabelServerGoroutines&envconfig.GoroutineLabelServerMethod != 0 { + // This method always runs in its own goroutine, so we can set a + // goroutine label without needing to restore a previous context. + ctx = pprof.WithLabels(ctx, pprof.Labels("grpc.method", stream.Method())) + pprof.SetGoroutineLabels(ctx) + } + var ti *traceInfo + if EnableTracing { + tr := newTrace("grpc.Recv."+methodFamily(stream.Method()), stream.Method()) + ctx = newTraceContext(ctx, tr) + ti = &traceInfo{ + tr: tr, + firstLine: firstLine{ + client: false, + remoteAddr: t.Peer().Addr, + }, + } + if dl, ok := ctx.Deadline(); ok { + ti.firstLine.deadline = time.Until(dl) + } + } + + sm, found := strings.CutPrefix(stream.Method(), "/") + if !found { + s.handleMalformedMethodName(stream, ti) + return + } + pos := strings.LastIndex(sm, "/") + if pos == -1 { + s.handleMalformedMethodName(stream, ti) + return + } + service := sm[:pos] + method := sm[pos+1:] + + // FromIncomingContext is expensive: skip if there are no statsHandlers + if s.statsHandler != nil { + md, _ := metadata.FromIncomingContext(ctx) + ctx = s.statsHandler.TagRPC(ctx, &stats.RPCTagInfo{FullMethodName: stream.Method()}) + s.statsHandler.HandleRPC(ctx, &stats.InHeader{ + FullMethod: stream.Method(), + RemoteAddr: t.Peer().Addr, + LocalAddr: t.Peer().LocalAddr, + Compression: stream.RecvCompress(), + WireLength: stream.HeaderWireLength(), + Header: md, + }) + } + // To have calls in stream callouts work. Will delete once all stats handler + // calls come from the gRPC layer. + stream.SetContext(ctx) + + srv, knownService := s.services[service] + if knownService { + if sd, ok := srv.streams[method]; ok { + s.processRPC(ctx, stream, srv, sd, ti) + return + } + } + // Unknown service, or known server unknown method. + if unknownDesc := s.opts.unknownStreamDesc; unknownDesc != nil { + s.processRPC(ctx, stream, nil, unknownDesc, ti) + return + } + var errDesc string + if !knownService { + errDesc = fmt.Sprintf("unknown service %v", service) + } else { + errDesc = fmt.Sprintf("unknown method %v for service %v", method, service) + } + if ti != nil { + ti.tr.LazyPrintf("%s", errDesc) + ti.tr.SetError() + } + if err := stream.WriteStatus(status.New(codes.Unimplemented, errDesc)); err != nil { + if ti != nil { + ti.tr.LazyLog(&fmtStringer{"%v", []any{err}}, true) + ti.tr.SetError() + } + channelz.Warningf(logger, s.channelz, "grpc: Server.handleStream failed to write status: %v", err) + } + if ti != nil { + ti.tr.Finish() + } +} + +// The key to save ServerTransportStream in the context. +type streamKey struct{} + +// NewContextWithServerTransportStream creates a new context from ctx and +// attaches stream to it. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func NewContextWithServerTransportStream(ctx context.Context, stream ServerTransportStream) context.Context { + return context.WithValue(ctx, streamKey{}, stream) +} + +// ServerTransportStream is a minimal interface that a transport stream must +// implement. This can be used to mock an actual transport stream for tests of +// handler code that use, for example, grpc.SetHeader (which requires some +// stream to be in context). +// +// See also NewContextWithServerTransportStream. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type ServerTransportStream interface { + Method() string + SetHeader(md metadata.MD) error + SendHeader(md metadata.MD) error + SetTrailer(md metadata.MD) error +} + +// ServerTransportStreamFromContext returns the ServerTransportStream saved in +// ctx. Returns nil if the given context has no stream associated with it +// (which implies it is not an RPC invocation context). +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func ServerTransportStreamFromContext(ctx context.Context) ServerTransportStream { + s, _ := ctx.Value(streamKey{}).(ServerTransportStream) + return s +} + +// Stop stops the gRPC server. It immediately closes all open +// connections and listeners. +// It cancels all active RPCs on the server side and the corresponding +// pending RPCs on the client side will get notified by connection +// errors. +func (s *Server) Stop() { + s.stop(false) +} + +// GracefulStop stops the gRPC server gracefully. It stops the server from +// accepting new connections and RPCs and blocks until all the pending RPCs are +// finished. +func (s *Server) GracefulStop() { + s.stop(true) +} + +func (s *Server) stop(graceful bool) { + s.quit.Fire() + defer s.done.Fire() + + s.channelzRemoveOnce.Do(func() { channelz.RemoveEntry(s.channelz.ID) }) + s.mu.Lock() + s.closeListenersLocked() + // Wait for serving threads to be ready to exit. Only then can we be sure no + // new conns will be created. + s.mu.Unlock() + s.serveWG.Wait() + + s.mu.Lock() + defer s.mu.Unlock() + + if graceful { + s.drainAllServerTransportsLocked() + } else { + s.closeServerTransportsLocked() + } + + for len(s.conns) != 0 { + s.cv.Wait() + } + s.conns = nil + + if s.opts.numServerWorkers > 0 { + // Closing the channel (only once, via sync.OnceFunc) after all the + // connections have been closed above ensures that there are no + // goroutines executing the callback passed to st.HandleStreams (where + // the channel is written to). + s.serverWorkerChannelClose() + } + + if graceful || s.opts.waitForHandlers { + s.handlersWG.Wait() + } + + if s.events != nil { + s.events.Finish() + s.events = nil + } +} + +// s.mu must be held by the caller. +func (s *Server) closeServerTransportsLocked() { + for _, conns := range s.conns { + for st := range conns { + st.Close(errors.New("Server.Stop called")) + } + } +} + +// s.mu must be held by the caller. +func (s *Server) drainAllServerTransportsLocked() { + if !s.drain { + for _, conns := range s.conns { + for st := range conns { + st.Drain("graceful_stop") + } + } + s.drain = true + } +} + +// s.mu must be held by the caller. +func (s *Server) closeListenersLocked() { + for lis := range s.lis { + lis.Close() + } + s.lis = nil +} + +// contentSubtype must be lowercase +// cannot return nil +func (s *Server) getCodec(contentSubtype string) baseCodec { + if s.opts.codec != nil { + return s.opts.codec + } + if contentSubtype == "" { + return getCodec(proto.Name) + } + codec := getCodec(contentSubtype) + if codec == nil { + logger.Warningf("Unsupported codec %q. Defaulting to %q for now. This will start to fail in future releases.", contentSubtype, proto.Name) + return getCodec(proto.Name) + } + return codec +} + +type serverKey struct{} + +// serverFromContext gets the Server from the context. +func serverFromContext(ctx context.Context) *Server { + s, _ := ctx.Value(serverKey{}).(*Server) + return s +} + +// contextWithServer sets the Server in the context. +func contextWithServer(ctx context.Context, server *Server) context.Context { + return context.WithValue(ctx, serverKey{}, server) +} + +// isRegisteredMethod returns whether the passed in method is registered as a +// method on the server. /service/method and service/method will match if the +// service and method are registered on the server. +func (s *Server) isRegisteredMethod(serviceMethod string) bool { + if serviceMethod != "" && serviceMethod[0] == '/' { + serviceMethod = serviceMethod[1:] + } + pos := strings.LastIndex(serviceMethod, "/") + if pos == -1 { // Invalid method name syntax. + return false + } + service := serviceMethod[:pos] + method := serviceMethod[pos+1:] + srv, knownService := s.services[service] + if knownService { + if _, ok := srv.streams[method]; ok { + return true + } + } + return false +} + +// SetHeader sets the header metadata to be sent from the server to the client. +// The context provided must be the context passed to the server's handler. +// +// Streaming RPCs should prefer the SetHeader method of the ServerStream. +// +// When called multiple times, all the provided metadata will be merged. All +// the metadata will be sent out when one of the following happens: +// +// - grpc.SendHeader is called, or for streaming handlers, stream.SendHeader. +// - The first response message is sent. For unary handlers, this occurs when +// the handler returns; for streaming handlers, this can happen when stream's +// SendMsg method is called. +// - An RPC status is sent out (error or success). This occurs when the handler +// returns. +// +// SetHeader will fail if called after any of the events above. +// +// The error returned is compatible with the status package. However, the +// status code will often not match the RPC status as seen by the client +// application, and therefore, should not be relied upon for this purpose. +func SetHeader(ctx context.Context, md metadata.MD) error { + if md.Len() == 0 { + return nil + } + stream := ServerTransportStreamFromContext(ctx) + if stream == nil { + return status.Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) + } + return stream.SetHeader(md) +} + +// SendHeader sends header metadata. It may be called at most once, and may not +// be called after any event that causes headers to be sent (see SetHeader for +// a complete list). The provided md and headers set by SetHeader() will be +// sent. +// +// The error returned is compatible with the status package. However, the +// status code will often not match the RPC status as seen by the client +// application, and therefore, should not be relied upon for this purpose. +func SendHeader(ctx context.Context, md metadata.MD) error { + stream := ServerTransportStreamFromContext(ctx) + if stream == nil { + return status.Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) + } + if err := stream.SendHeader(md); err != nil { + return toRPCErr(err) + } + return nil +} + +// SetSendCompressor sets a compressor for outbound messages from the server. +// It must not be called after any event that causes headers to be sent +// (see ServerStream.SetHeader for the complete list). Provided compressor is +// used when below conditions are met: +// +// - compressor is registered via encoding.RegisterCompressor +// - compressor name must exist in the client advertised compressor names +// sent in grpc-accept-encoding header. Use ClientSupportedCompressors to +// get client supported compressor names. +// +// The context provided must be the context passed to the server's handler. +// It must be noted that compressor name encoding.Identity disables the +// outbound compression. +// By default, server messages will be sent using the same compressor with +// which request messages were sent. +// +// It is not safe to call SetSendCompressor concurrently with SendHeader and +// SendMsg. +// +// # Experimental +// +// Notice: This function is EXPERIMENTAL and may be changed or removed in a +// later release. +func SetSendCompressor(ctx context.Context, name string) error { + stream, ok := ServerTransportStreamFromContext(ctx).(*transport.ServerStream) + if !ok || stream == nil { + return fmt.Errorf("failed to fetch the stream from the given context") + } + + if err := validateSendCompressor(name, stream.ClientAdvertisedCompressors()); err != nil { + return fmt.Errorf("unable to set send compressor: %w", err) + } + + return stream.SetSendCompress(name) +} + +// ClientSupportedCompressors returns compressor names advertised by the client +// via grpc-accept-encoding header. +// +// The context provided must be the context passed to the server's handler. +// +// # Experimental +// +// Notice: This function is EXPERIMENTAL and may be changed or removed in a +// later release. +func ClientSupportedCompressors(ctx context.Context) ([]string, error) { + stream, ok := ServerTransportStreamFromContext(ctx).(*transport.ServerStream) + if !ok || stream == nil { + return nil, fmt.Errorf("failed to fetch the stream from the given context %v", ctx) + } + + return stream.ClientAdvertisedCompressors(), nil +} + +// SetTrailer sets the trailer metadata that will be sent when an RPC returns. +// When called more than once, all the provided metadata will be merged. +// +// The error returned is compatible with the status package. However, the +// status code will often not match the RPC status as seen by the client +// application, and therefore, should not be relied upon for this purpose. +func SetTrailer(ctx context.Context, md metadata.MD) error { + if md.Len() == 0 { + return nil + } + stream := ServerTransportStreamFromContext(ctx) + if stream == nil { + return status.Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) + } + return stream.SetTrailer(md) +} + +// Method returns the method string for the server context. The returned +// string is in the format of "/service/method". +func Method(ctx context.Context) (string, bool) { + s := ServerTransportStreamFromContext(ctx) + if s == nil { + return "", false + } + return s.Method(), true +} + +// validateSendCompressor returns an error when given compressor name cannot be +// handled by the server or the client based on the advertised compressors. +func validateSendCompressor(name string, clientCompressors []string) error { + if name == encoding.Identity { + return nil + } + + if !grpcutil.IsCompressorNameRegistered(name) { + return fmt.Errorf("compressor not registered %q", name) + } + + for _, c := range clientCompressors { + if c == name { + return nil // found match + } + } + return fmt.Errorf("client does not support compressor %q", name) +} + +// atomicSemaphore implements a blocking, counting semaphore. acquire should be +// called synchronously; release may be called asynchronously. +type atomicSemaphore struct { + n atomic.Int64 + wait chan struct{} +} + +func (q *atomicSemaphore) acquire() { + if q.n.Add(-1) < 0 { + // We ran out of quota. Block until a release happens. + <-q.wait + } +} + +func (q *atomicSemaphore) release() { + // N.B. the "<= 0" check below should allow for this to work with multiple + // concurrent calls to acquire, but also note that with synchronous calls to + // acquire, as our system does, n will never be less than -1. There are + // fairness issues (queuing) to consider if this was to be generalized. + if q.n.Add(1) <= 0 { + // An acquire was waiting on us. Unblock it. + q.wait <- struct{}{} + } +} + +func newHandlerQuota(n uint32) *atomicSemaphore { + a := &atomicSemaphore{wait: make(chan struct{}, 1)} + a.n.Store(int64(n)) + return a +} diff --git a/server_ext_test.go b/server_ext_test.go new file mode 100644 index 0000000..579b9d7 --- /dev/null +++ b/server_ext_test.go @@ -0,0 +1,332 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc_test + +import ( + "context" + "io" + "runtime" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestServer_MaxHandlers ensures that no more than MaxConcurrentStreams server +// handlers are active at one time. +func (s) TestServer_MaxHandlers(t *testing.T) { + started := make(chan struct{}) + blockCalls := grpcsync.NewEvent() + + // This stub server does not properly respect the stream context, so it will + // not exit when the context is canceled. + ss := stubserver.StubServer{ + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + started <- struct{}{} + <-blockCalls.Done() + return nil + }, + } + if err := ss.Start([]grpc.ServerOption{grpc.MaxConcurrentStreams(1)}); err != nil { + t.Fatal("Error starting server:", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start one RPC to the server. + ctx1, cancel1 := context.WithCancel(ctx) + _, err := ss.Client.FullDuplexCall(ctx1) + if err != nil { + t.Fatal("Error staring call:", err) + } + + // Wait for the handler to be invoked. + select { + case <-started: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for RPC to start on server.") + } + + // Cancel it on the client. The server handler will still be running. + cancel1() + + ctx2, cancel2 := context.WithCancel(ctx) + defer cancel2() + s, err := ss.Client.FullDuplexCall(ctx2) + if err != nil { + t.Fatal("Error staring call:", err) + } + + // After 100ms, allow the first call to unblock. That should allow the + // second RPC to run and finish. + select { + case <-started: + blockCalls.Fire() + t.Fatalf("RPC started unexpectedly.") + case <-time.After(100 * time.Millisecond): + blockCalls.Fire() + } + + select { + case <-started: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for second RPC to start on server.") + } + if _, err := s.Recv(); err != io.EOF { + t.Fatal("Received unexpected RPC error:", err) + } +} + +// Tests the case where the stream worker goroutine option is enabled, and a +// number of RPCs are initiated around the same time that Stop() is called. This +// used to result in a write to a closed channel. This test verifies that there +// is no panic. +func (s) TestStreamWorkers_RPCsAndStop(t *testing.T) { + ss := stubserver.StartTestService(t, nil, grpc.NumStreamWorkers(uint32(runtime.NumCPU()))) + // This deferred stop takes care of stopping the server when one of the + // below grpc.NewClient fail, and the test exits early. + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + const numChannels = 20 + const numRPCLoops = 20 + + // Create a bunch of clientconns and ensure that they are READY by making an + // RPC on them. + ccs := make([]*grpc.ClientConn, numChannels) + for i := 0; i < numChannels; i++ { + var err error + ccs[i], err = grpc.NewClient(ss.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("[iteration: %d] grpc.NewClient(%s) failed: %v", i, ss.Address, err) + } + defer ccs[i].Close() + client := testgrpc.NewTestServiceClient(ccs[i]) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + } + + // Make a bunch of concurrent RPCs on the above clientconns. These will + // eventually race with Stop(), and will start to fail. + var wg sync.WaitGroup + for i := 0; i < numChannels; i++ { + client := testgrpc.NewTestServiceClient(ccs[i]) + for j := 0; j < numRPCLoops; j++ { + wg.Add(1) + go func(client testgrpc.TestServiceClient) { + defer wg.Done() + for { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + continue + } + if code := status.Code(err); code == codes.Unavailable { + // Once Stop() has been called on the server, we expect + // subsequent calls to fail with Unavailable. + return + } + t.Errorf("EmptyCall() failed: %v", err) + return + } + }(client) + } + } + + // Call Stop() concurrently with the above RPC attempts. + ss.Stop() + wg.Wait() +} + +// Tests the case where the stream worker goroutine option is enabled, and both +// Stop() and GracefulStop() care called. This used to result in a close of a +// closed channel. This test verifies that there is no panic. +func (s) TestStreamWorkers_GracefulStopAndStop(t *testing.T) { + ss := stubserver.StartTestService(t, nil, grpc.NumStreamWorkers(uint32(runtime.NumCPU()))) + defer ss.Stop() + + if err := ss.StartClient(grpc.WithTransportCredentials(insecure.NewCredentials())); err != nil { + t.Fatalf("Failed to create client to stub server: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(ss.CC) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + ss.S.GracefulStop() +} + +// Tests the WaitForHandlers ServerOption by leaving an RPC running while Stop +// is called, and ensures Stop doesn't return until the handler returns. +func (s) TestServer_WaitForHandlers(t *testing.T) { + started := grpcsync.NewEvent() + blockCalls := grpcsync.NewEvent() + + // This stub server does not properly respect the stream context, so it will + // not exit when the context is canceled. + ss := stubserver.StubServer{ + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + started.Fire() + <-blockCalls.Done() + return nil + }, + } + if err := ss.Start([]grpc.ServerOption{grpc.WaitForHandlers(true)}); err != nil { + t.Fatal("Error starting server:", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start one RPC to the server. + ctx1, cancel1 := context.WithCancel(ctx) + _, err := ss.Client.FullDuplexCall(ctx1) + if err != nil { + t.Fatal("Error staring call:", err) + } + + // Wait for the handler to be invoked. + select { + case <-started.Done(): + case <-ctx.Done(): + t.Fatalf("Timed out waiting for RPC to start on server.") + } + + // Cancel it on the client. The server handler will still be running. + cancel1() + + // Close the connection. This might be sufficient to allow the server to + // return if it doesn't properly wait for outstanding method handlers to + // return. + ss.CC.Close() + + // Try to Stop() the server, which should block indefinitely (until + // blockCalls is fired). + stopped := grpcsync.NewEvent() + go func() { + ss.S.Stop() + stopped.Fire() + }() + + // Wait 100ms and ensure stopped does not fire. + select { + case <-stopped.Done(): + trace := make([]byte, 4096) + trace = trace[0:runtime.Stack(trace, true)] + blockCalls.Fire() + t.Fatalf("Server returned from Stop() illegally. Stack trace:\n%v", string(trace)) + case <-time.After(100 * time.Millisecond): + // Success; unblock the call and wait for stopped. + blockCalls.Fire() + } + + select { + case <-stopped.Done(): + case <-ctx.Done(): + t.Fatalf("Timed out waiting for second RPC to start on server.") + } +} + +// Tests that GracefulStop will wait for all method handlers to return by +// blocking a handler and ensuring GracefulStop doesn't return until after it is +// unblocked. +func (s) TestServer_GracefulStopWaits(t *testing.T) { + started := grpcsync.NewEvent() + blockCalls := grpcsync.NewEvent() + + // This stub server does not properly respect the stream context, so it will + // not exit when the context is canceled. + ss := stubserver.StubServer{ + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + started.Fire() + <-blockCalls.Done() + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatal("Error starting server:", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start one RPC to the server. + ctx1, cancel1 := context.WithCancel(ctx) + _, err := ss.Client.FullDuplexCall(ctx1) + if err != nil { + t.Fatal("Error staring call:", err) + } + + // Wait for the handler to be invoked. + select { + case <-started.Done(): + case <-ctx.Done(): + t.Fatalf("Timed out waiting for RPC to start on server.") + } + + // Cancel it on the client. The server handler will still be running. + cancel1() + + // Close the connection. This might be sufficient to allow the server to + // return if it doesn't properly wait for outstanding method handlers to + // return. + ss.CC.Close() + + // Try to Stop() the server, which should block indefinitely (until + // blockCalls is fired). + stopped := grpcsync.NewEvent() + go func() { + ss.S.GracefulStop() + stopped.Fire() + }() + + // Wait 100ms and ensure stopped does not fire. + select { + case <-stopped.Done(): + trace := make([]byte, 4096) + trace = trace[0:runtime.Stack(trace, true)] + blockCalls.Fire() + t.Fatalf("Server returned from Stop() illegally. Stack trace:\n%v", string(trace)) + case <-time.After(100 * time.Millisecond): + // Success; unblock the call and wait for stopped. + blockCalls.Fire() + } + + select { + case <-stopped.Done(): + case <-ctx.Done(): + t.Fatalf("Timed out waiting for second RPC to start on server.") + } +} diff --git a/server_test.go b/server_test.go new file mode 100644 index 0000000..07200fb --- /dev/null +++ b/server_test.go @@ -0,0 +1,235 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "net" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/status" +) + +type emptyServiceServer any + +type testServer struct{} + +func errorDesc(err error) string { + if s, ok := status.FromError(err); ok { + return s.Message() + } + return err.Error() +} + +func (s) TestStopBeforeServe(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to create listener: %v", err) + } + + server := NewServer() + server.Stop() + err = server.Serve(lis) + if err != ErrServerStopped { + t.Fatalf("server.Serve() error = %v, want %v", err, ErrServerStopped) + } + + // server.Serve is responsible for closing the listener, even if the + // server was already stopped. + err = lis.Close() + if got, want := errorDesc(err), "use of closed"; !strings.Contains(got, want) { + t.Errorf("Close() error = %q, want %q", got, want) + } +} + +func (s) TestGracefulStop(t *testing.T) { + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to create listener: %v", err) + } + + server := NewServer() + go func() { + // make sure Serve() is called + time.Sleep(time.Millisecond * 500) + server.GracefulStop() + }() + + err = server.Serve(lis) + if err != nil { + t.Fatalf("Serve() returned non-nil error on GracefulStop: %v", err) + } +} + +func (s) TestGetServiceInfo(t *testing.T) { + testSd := ServiceDesc{ + ServiceName: "grpc.testing.EmptyService", + HandlerType: (*emptyServiceServer)(nil), + Methods: []MethodDesc{ + { + MethodName: "EmptyCall", + Handler: nil, + }, + }, + Streams: []StreamDesc{ + { + StreamName: "EmptyStream", + Handler: nil, + ServerStreams: false, + ClientStreams: true, + }, + }, + Metadata: []int{0, 2, 1, 3}, + } + + server := NewServer() + server.RegisterService(&testSd, &testServer{}) + + info := server.GetServiceInfo() + want := map[string]ServiceInfo{ + "grpc.testing.EmptyService": { + Methods: []MethodInfo{ + { + Name: "EmptyCall", + IsClientStream: false, + IsServerStream: false, + }, + { + Name: "EmptyStream", + IsClientStream: true, + IsServerStream: false, + }}, + Metadata: []int{0, 2, 1, 3}, + }, + } + + if !reflect.DeepEqual(info, want) { + t.Errorf("GetServiceInfo() = %+v, want %+v", info, want) + } +} + +func (s) TestRetryChainedInterceptor(t *testing.T) { + var records []int + i1 := func(ctx context.Context, req any, _ *UnaryServerInfo, handler UnaryHandler) (resp any, err error) { + records = append(records, 1) + // call handler twice to simulate a retry here. + handler(ctx, req) + return handler(ctx, req) + } + i2 := func(ctx context.Context, req any, _ *UnaryServerInfo, handler UnaryHandler) (resp any, err error) { + records = append(records, 2) + return handler(ctx, req) + } + i3 := func(ctx context.Context, req any, _ *UnaryServerInfo, handler UnaryHandler) (resp any, err error) { + records = append(records, 3) + return handler(ctx, req) + } + + ii := chainUnaryInterceptors([]UnaryServerInterceptor{i1, i2, i3}) + + handler := func(context.Context, any) (any, error) { + return nil, nil + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + ii(ctx, nil, nil, handler) + if !cmp.Equal(records, []int{1, 2, 3, 2, 3}) { + t.Fatalf("retry failed on chained interceptors: %v", records) + } +} + +func (s) TestStreamContext(t *testing.T) { + expectedStream := &transport.ServerStream{} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = NewContextWithServerTransportStream(ctx, expectedStream) + + s := ServerTransportStreamFromContext(ctx) + stream, ok := s.(*transport.ServerStream) + if !ok || expectedStream != stream { + t.Fatalf("GetStreamFromContext(%v) = %v, %t, want: %v, true", ctx, stream, ok, expectedStream) + } +} + +func BenchmarkChainUnaryInterceptor(b *testing.B) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for _, n := range []int{1, 3, 5, 10} { + n := n + b.Run(strconv.Itoa(n), func(b *testing.B) { + interceptors := make([]UnaryServerInterceptor, 0, n) + for i := 0; i < n; i++ { + interceptors = append(interceptors, func( + ctx context.Context, req any, _ *UnaryServerInfo, handler UnaryHandler, + ) (any, error) { + return handler(ctx, req) + }) + } + + s := NewServer(ChainUnaryInterceptor(interceptors...)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := s.opts.unaryInt(ctx, nil, nil, + func(context.Context, any) (any, error) { + return nil, nil + }, + ); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func BenchmarkChainStreamInterceptor(b *testing.B) { + for _, n := range []int{1, 3, 5, 10} { + n := n + b.Run(strconv.Itoa(n), func(b *testing.B) { + interceptors := make([]StreamServerInterceptor, 0, n) + for i := 0; i < n; i++ { + interceptors = append(interceptors, func( + srv any, ss ServerStream, _ *StreamServerInfo, handler StreamHandler, + ) error { + return handler(srv, ss) + }) + } + + s := NewServer(ChainStreamInterceptor(interceptors...)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := s.opts.streamInt(nil, nil, nil, func(any, ServerStream) error { + return nil + }); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/service_config.go b/service_config.go new file mode 100644 index 0000000..8d451e0 --- /dev/null +++ b/service_config.go @@ -0,0 +1,360 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/gracefulswitch" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +const maxInt = int(^uint(0) >> 1) + +// MethodConfig defines the configuration recommended by the service providers for a +// particular method. +// +// Deprecated: Users should not use this struct. Service config should be received +// through name resolver, as specified here +// https://github.com/grpc/grpc/blob/master/doc/service_config.md +type MethodConfig = internalserviceconfig.MethodConfig + +// ServiceConfig is provided by the service provider and contains parameters for how +// clients that connect to the service should behave. +// +// Deprecated: Users should not use this struct. Service config should be received +// through name resolver, as specified here +// https://github.com/grpc/grpc/blob/master/doc/service_config.md +type ServiceConfig struct { + serviceconfig.Config + + // lbConfig is the service config's load balancing configuration. If + // lbConfig and LB are both present, lbConfig will be used. + lbConfig serviceconfig.LoadBalancingConfig + + // Methods contains a map for the methods in this service. If there is an + // exact match for a method (i.e. /service/method) in the map, use the + // corresponding MethodConfig. If there's no exact match, look for the + // default config for the service (/service/) and use the corresponding + // MethodConfig if it exists. Otherwise, the method has no MethodConfig to + // use. + Methods map[string]MethodConfig + + // If a retryThrottlingPolicy is provided, gRPC will automatically throttle + // retry attempts and hedged RPCs when the client’s ratio of failures to + // successes exceeds a threshold. + // + // For each server name, the gRPC client will maintain a token_count which is + // initially set to maxTokens, and can take values between 0 and maxTokens. + // + // Every outgoing RPC (regardless of service or method invoked) will change + // token_count as follows: + // + // - Every failed RPC will decrement the token_count by 1. + // - Every successful RPC will increment the token_count by tokenRatio. + // + // If token_count is less than or equal to maxTokens / 2, then RPCs will not + // be retried and hedged RPCs will not be sent. + retryThrottling *retryThrottlingPolicy + // healthCheckConfig must be set as one of the requirement to enable LB channel + // health check. + healthCheckConfig *healthCheckConfig + // rawJSONString stores service config json string that get parsed into + // this service config struct. + rawJSONString string +} + +// healthCheckConfig defines the go-native version of the LB channel health check config. +type healthCheckConfig struct { + // serviceName is the service name to use in the health-checking request. + ServiceName string +} + +type jsonRetryPolicy struct { + MaxAttempts int + InitialBackoff internalserviceconfig.Duration + MaxBackoff internalserviceconfig.Duration + BackoffMultiplier float64 + RetryableStatusCodes []codes.Code +} + +// retryThrottlingPolicy defines the go-native version of the retry throttling +// policy defined by the service config here: +// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#integration-with-service-config +type retryThrottlingPolicy struct { + // The number of tokens starts at maxTokens. The token_count will always be + // between 0 and maxTokens. + // + // This field is required and must be greater than zero. + MaxTokens float64 + // The amount of tokens to add on each successful RPC. Typically this will + // be some number between 0 and 1, e.g., 0.1. + // + // This field is required and must be greater than zero. Up to 3 decimal + // places are supported. + TokenRatio float64 +} + +type jsonName struct { + Service string + Method string +} + +var ( + errDuplicatedName = errors.New("duplicated name") + errEmptyServiceNonEmptyMethod = errors.New("cannot combine empty 'service' and non-empty 'method'") +) + +func (j jsonName) generatePath() (string, error) { + if j.Service == "" { + if j.Method != "" { + return "", errEmptyServiceNonEmptyMethod + } + return "", nil + } + res := "/" + j.Service + "/" + if j.Method != "" { + res += j.Method + } + return res, nil +} + +// TODO(lyuxuan): delete this struct after cleaning up old service config implementation. +type jsonMC struct { + Name *[]jsonName + WaitForReady *bool + Timeout *internalserviceconfig.Duration + MaxRequestMessageBytes *int64 + MaxResponseMessageBytes *int64 + RetryPolicy *jsonRetryPolicy +} + +// TODO(lyuxuan): delete this struct after cleaning up old service config implementation. +type jsonSC struct { + LoadBalancingPolicy *string + LoadBalancingConfig *json.RawMessage + MethodConfig *[]jsonMC + RetryThrottling *retryThrottlingPolicy + HealthCheckConfig *healthCheckConfig +} + +func init() { + internal.ParseServiceConfig = func(js string) *serviceconfig.ParseResult { + return parseServiceConfig(js, defaultMaxCallAttempts) + } +} + +func parseServiceConfig(js string, maxAttempts int) *serviceconfig.ParseResult { + if len(js) == 0 { + return &serviceconfig.ParseResult{Err: fmt.Errorf("no JSON service config provided")} + } + var rsc jsonSC + err := json.Unmarshal([]byte(js), &rsc) + if err != nil { + logger.Warningf("grpc: unmarshalling service config %s: %v", js, err) + return &serviceconfig.ParseResult{Err: err} + } + sc := ServiceConfig{ + Methods: make(map[string]MethodConfig), + retryThrottling: rsc.RetryThrottling, + healthCheckConfig: rsc.HealthCheckConfig, + rawJSONString: js, + } + c := rsc.LoadBalancingConfig + if c == nil { + name := pickfirst.Name + if rsc.LoadBalancingPolicy != nil { + name = *rsc.LoadBalancingPolicy + } + if balancer.Get(name) == nil { + name = pickfirst.Name + } + cfg := []map[string]any{{name: struct{}{}}} + strCfg, err := json.Marshal(cfg) + if err != nil { + return &serviceconfig.ParseResult{Err: fmt.Errorf("unexpected error marshaling simple LB config: %w", err)} + } + r := json.RawMessage(strCfg) + c = &r + } + cfg, err := gracefulswitch.ParseConfig(*c) + if err != nil { + return &serviceconfig.ParseResult{Err: err} + } + sc.lbConfig = cfg + + if rsc.MethodConfig == nil { + return &serviceconfig.ParseResult{Config: &sc} + } + + paths := map[string]struct{}{} + for _, m := range *rsc.MethodConfig { + if m.Name == nil { + continue + } + + mc := MethodConfig{ + WaitForReady: m.WaitForReady, + Timeout: (*time.Duration)(m.Timeout), + } + if mc.RetryPolicy, err = convertRetryPolicy(m.RetryPolicy, maxAttempts); err != nil { + logger.Warningf("grpc: unmarshalling service config %s: %v", js, err) + return &serviceconfig.ParseResult{Err: err} + } + if m.MaxRequestMessageBytes != nil { + if *m.MaxRequestMessageBytes > int64(maxInt) { + mc.MaxReqSize = newInt(maxInt) + } else { + mc.MaxReqSize = newInt(int(*m.MaxRequestMessageBytes)) + } + } + if m.MaxResponseMessageBytes != nil { + if *m.MaxResponseMessageBytes > int64(maxInt) { + mc.MaxRespSize = newInt(maxInt) + } else { + mc.MaxRespSize = newInt(int(*m.MaxResponseMessageBytes)) + } + } + for i, n := range *m.Name { + path, err := n.generatePath() + if err != nil { + logger.Warningf("grpc: error unmarshalling service config %s due to methodConfig[%d]: %v", js, i, err) + return &serviceconfig.ParseResult{Err: err} + } + + if _, ok := paths[path]; ok { + err = errDuplicatedName + logger.Warningf("grpc: error unmarshalling service config %s due to methodConfig[%d]: %v", js, i, err) + return &serviceconfig.ParseResult{Err: err} + } + paths[path] = struct{}{} + sc.Methods[path] = mc + } + } + + if sc.retryThrottling != nil { + if mt := sc.retryThrottling.MaxTokens; mt <= 0 || mt > 1000 { + return &serviceconfig.ParseResult{Err: fmt.Errorf("invalid retry throttling config: maxTokens (%v) out of range (0, 1000]", mt)} + } + if tr := sc.retryThrottling.TokenRatio; tr <= 0 { + return &serviceconfig.ParseResult{Err: fmt.Errorf("invalid retry throttling config: tokenRatio (%v) may not be negative", tr)} + } + } + return &serviceconfig.ParseResult{Config: &sc} +} + +func isValidRetryPolicy(jrp *jsonRetryPolicy) bool { + return jrp.MaxAttempts > 1 && + jrp.InitialBackoff > 0 && + jrp.MaxBackoff > 0 && + jrp.BackoffMultiplier > 0 && + len(jrp.RetryableStatusCodes) > 0 +} + +func convertRetryPolicy(jrp *jsonRetryPolicy, maxAttempts int) (p *internalserviceconfig.RetryPolicy, err error) { + if jrp == nil { + return nil, nil + } + + if !isValidRetryPolicy(jrp) { + return nil, fmt.Errorf("invalid retry policy (%+v): ", jrp) + } + + if jrp.MaxAttempts < maxAttempts { + maxAttempts = jrp.MaxAttempts + } + rp := &internalserviceconfig.RetryPolicy{ + MaxAttempts: maxAttempts, + InitialBackoff: time.Duration(jrp.InitialBackoff), + MaxBackoff: time.Duration(jrp.MaxBackoff), + BackoffMultiplier: jrp.BackoffMultiplier, + RetryableStatusCodes: make(map[codes.Code]bool), + } + for _, code := range jrp.RetryableStatusCodes { + rp.RetryableStatusCodes[code] = true + } + return rp, nil +} + +func minPointers(a, b *int) *int { + if *a < *b { + return a + } + return b +} + +func getMaxSize(mcMax, doptMax *int, defaultVal int) *int { + if mcMax == nil && doptMax == nil { + return &defaultVal + } + if mcMax != nil && doptMax != nil { + return minPointers(mcMax, doptMax) + } + if mcMax != nil { + return mcMax + } + return doptMax +} + +func newInt(b int) *int { + return &b +} + +func init() { + internal.EqualServiceConfigForTesting = equalServiceConfig +} + +// equalServiceConfig compares two configs. The rawJSONString field is ignored, +// because they may diff in white spaces. +// +// If any of them is NOT *ServiceConfig, return false. +func equalServiceConfig(a, b serviceconfig.Config) bool { + if a == nil && b == nil { + return true + } + aa, ok := a.(*ServiceConfig) + if !ok { + return false + } + bb, ok := b.(*ServiceConfig) + if !ok { + return false + } + aaRaw := aa.rawJSONString + aa.rawJSONString = "" + bbRaw := bb.rawJSONString + bb.rawJSONString = "" + defer func() { + aa.rawJSONString = aaRaw + bb.rawJSONString = bbRaw + }() + // Using reflect.DeepEqual instead of cmp.Equal because many balancer + // configs are unexported, and cmp.Equal cannot compare unexported fields + // from unexported structs. + return reflect.DeepEqual(aa, bb) +} diff --git a/service_config_test.go b/service_config_test.go new file mode 100644 index 0000000..5dc4e6d --- /dev/null +++ b/service_config_test.go @@ -0,0 +1,609 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "encoding/json" + "fmt" + "reflect" + "testing" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/balancer/gracefulswitch" + "google.golang.org/grpc/serviceconfig" + + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" +) + +type parseTestCase struct { + name string + scjs string + wantSC *ServiceConfig + wantErr bool +} + +func lbConfigFor(t *testing.T, name string, cfg serviceconfig.LoadBalancingConfig) serviceconfig.LoadBalancingConfig { + if name == "" { + name = "pick_first" + cfg = struct { + serviceconfig.LoadBalancingConfig + }{} + } + d := []map[string]any{{name: cfg}} + strCfg, err := json.Marshal(d) + t.Logf("strCfg = %v", string(strCfg)) + if err != nil { + t.Fatalf("Error parsing config: %v", err) + } + parsedCfg, err := gracefulswitch.ParseConfig(strCfg) + if err != nil { + t.Fatalf("Error parsing config: %v", err) + } + return parsedCfg +} + +func runParseTests(t *testing.T, testCases []parseTestCase) { + t.Helper() + for i, c := range testCases { + name := c.name + if name == "" { + name = fmt.Sprint(i) + } + t.Run(name, func(t *testing.T) { + scpr := parseServiceConfig(c.scjs, defaultMaxCallAttempts) + var sc *ServiceConfig + sc, _ = scpr.Config.(*ServiceConfig) + if !c.wantErr { + c.wantSC.rawJSONString = c.scjs + } + if c.wantErr != (scpr.Err != nil) || !reflect.DeepEqual(sc, c.wantSC) { + t.Fatalf("parseServiceConfig(%s) = %+v, %v, want %+v, %v", c.scjs, sc, scpr.Err, c.wantSC, c.wantErr) + } + }) + } +} + +type pbbData struct { + serviceconfig.LoadBalancingConfig + Foo string + Bar int +} + +type parseBalancerBuilder struct{} + +func (parseBalancerBuilder) Name() string { + return "pbb" +} + +func (parseBalancerBuilder) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + d := pbbData{} + if err := json.Unmarshal(c, &d); err != nil { + return nil, err + } + return d, nil +} + +func (parseBalancerBuilder) Build(balancer.ClientConn, balancer.BuildOptions) balancer.Balancer { + panic("unimplemented") +} + +func init() { + balancer.Register(parseBalancerBuilder{}) +} + +func (s) TestParseLBConfig(t *testing.T) { + testcases := []parseTestCase{ + { + scjs: `{ + "loadBalancingConfig": [{"pbb": { "foo": "hi" } }] +}`, + wantSC: &ServiceConfig{ + Methods: make(map[string]MethodConfig), + lbConfig: lbConfigFor(t, "pbb", pbbData{Foo: "hi"}), + }, + wantErr: false, + }, + } + runParseTests(t, testcases) +} + +func (s) TestParseNoLBConfigSupported(t *testing.T) { + // We have a loadBalancingConfig field but will not encounter a supported + // policy. The config will be considered invalid in this case. + testcases := []parseTestCase{ + { + scjs: `{ + "loadBalancingConfig": [{"not_a_balancer1": {} }, {"not_a_balancer2": {}}] +}`, + wantErr: true, + }, { + scjs: `{"loadBalancingConfig": []}`, + wantErr: true, + }, + } + runParseTests(t, testcases) +} + +func (s) TestParseLoadBalancer(t *testing.T) { + testcases := []parseTestCase{ + { + scjs: `{ + "loadBalancingPolicy": "round_robin", + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "waitForReady": true + } + ] +}`, + wantSC: &ServiceConfig{ + Methods: map[string]MethodConfig{ + "/foo/Bar": { + WaitForReady: newBool(true), + }, + }, + lbConfig: lbConfigFor(t, "round_robin", nil), + }, + wantErr: false, + }, + { + scjs: `{ + "loadBalancingPolicy": 1, + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "waitForReady": false + } + ] +}`, + wantErr: true, + }, + } + runParseTests(t, testcases) +} + +func (s) TestParseWaitForReady(t *testing.T) { + testcases := []parseTestCase{ + { + scjs: `{ + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "waitForReady": true + } + ] +}`, + wantSC: &ServiceConfig{ + Methods: map[string]MethodConfig{ + "/foo/Bar": { + WaitForReady: newBool(true), + }, + }, + lbConfig: lbConfigFor(t, "", nil), + }, + }, + { + scjs: `{ + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "waitForReady": false + } + ] +}`, + wantSC: &ServiceConfig{ + Methods: map[string]MethodConfig{ + "/foo/Bar": { + WaitForReady: newBool(false), + }, + }, + lbConfig: lbConfigFor(t, "", nil), + }, + }, + { + scjs: `{ + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "waitForReady": fall + }, + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "waitForReady": true + } + ] +}`, + wantErr: true, + }, + } + + runParseTests(t, testcases) +} + +func (s) TestParseTimeOut(t *testing.T) { + testcases := []parseTestCase{ + { + scjs: `{ + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "timeout": "1s" + } + ] +}`, + wantSC: &ServiceConfig{ + Methods: map[string]MethodConfig{ + "/foo/Bar": { + Timeout: newDuration(time.Second), + }, + }, + lbConfig: lbConfigFor(t, "", nil), + }, + }, + { + scjs: `{ + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "timeout": "3c" + } + ] +}`, + wantErr: true, + }, + { + scjs: `{ + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "timeout": "3c" + }, + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "timeout": "1s" + } + ] +}`, + wantErr: true, + }, + } + + runParseTests(t, testcases) +} + +func (s) TestParseMsgSize(t *testing.T) { + testcases := []parseTestCase{ + { + scjs: `{ + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "maxRequestMessageBytes": 1024, + "maxResponseMessageBytes": 2048 + } + ] +}`, + wantSC: &ServiceConfig{ + Methods: map[string]MethodConfig{ + "/foo/Bar": { + MaxReqSize: newInt(1024), + MaxRespSize: newInt(2048), + }, + }, + lbConfig: lbConfigFor(t, "", nil), + }, + }, + { + scjs: `{ + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "maxRequestMessageBytes": "1024", + "maxResponseMessageBytes": "2048" + }, + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "maxRequestMessageBytes": 1024, + "maxResponseMessageBytes": 2048 + } + ] +}`, + wantErr: true, + }, + } + + runParseTests(t, testcases) +} +func (s) TestParseDefaultMethodConfig(t *testing.T) { + dc := &ServiceConfig{ + Methods: map[string]MethodConfig{ + "": {WaitForReady: newBool(true)}, + }, + lbConfig: lbConfigFor(t, "", nil), + } + + runParseTests(t, []parseTestCase{ + { + scjs: `{ + "methodConfig": [{ + "name": [{}], + "waitForReady": true + }] +}`, + wantSC: dc, + }, + { + scjs: `{ + "methodConfig": [{ + "name": [{"service": null}], + "waitForReady": true + }] +}`, + wantSC: dc, + }, + { + scjs: `{ + "methodConfig": [{ + "name": [{"service": ""}], + "waitForReady": true + }] +}`, + wantSC: dc, + }, + { + scjs: `{ + "methodConfig": [{ + "name": [{"method": "Bar"}], + "waitForReady": true + }] +}`, + wantErr: true, + }, + { + scjs: `{ + "methodConfig": [{ + "name": [{"service": "", "method": "Bar"}], + "waitForReady": true + }] +}`, + wantErr: true, + }, + }) +} + +func (s) TestParseMethodConfigDuplicatedName(t *testing.T) { + runParseTests(t, []parseTestCase{ + { + scjs: `{ + "methodConfig": [{ + "name": [ + {"service": "foo"}, + {"service": "foo"} + ], + "waitForReady": true + }] +}`, + wantErr: true, + }, + }) +} + +func (s) TestParseRetryPolicy(t *testing.T) { + runParseTests(t, []parseTestCase{ + { + name: "valid", + scjs: `{ + "methodConfig": [{ + "name": [{"service": "foo"}], + "retryPolicy": { + "maxAttempts": 2, + "initialBackoff": "2s", + "maxBackoff": "10s", + "backoffMultiplier": 2, + "retryableStatusCodes": ["UNAVAILABLE"] + } + }] + }`, + wantSC: &ServiceConfig{ + Methods: map[string]MethodConfig{ + "/foo/": { + RetryPolicy: &internalserviceconfig.RetryPolicy{ + MaxAttempts: 2, + InitialBackoff: 2 * time.Second, + MaxBackoff: 10 * time.Second, + BackoffMultiplier: 2, + RetryableStatusCodes: map[codes.Code]bool{codes.Unavailable: true}, + }, + }, + }, + lbConfig: lbConfigFor(t, "", nil), + }, + }, + { + name: "negative maxAttempts", + scjs: `{ + "methodConfig": [{ + "name": [{"service": "foo"}], + "retryPolicy": { + "maxAttempts": -1, + "initialBackoff": "2s", + "maxBackoff": "10s", + "backoffMultiplier": 2, + "retryableStatusCodes": ["UNAVAILABLE"] + } + }] + }`, + wantErr: true, + }, + { + name: "missing maxAttempts", + scjs: `{ + "methodConfig": [{ + "name": [{"service": "foo"}], + "retryPolicy": { + "initialBackoff": "2s", + "maxBackoff": "10s", + "backoffMultiplier": 2, + "retryableStatusCodes": ["UNAVAILABLE"] + } + }] + }`, + wantErr: true, + }, + { + name: "zero initialBackoff", + scjs: `{ + "methodConfig": [{ + "name": [{"service": "foo"}], + "retryPolicy": { + "maxAttempts": 2, + "initialBackoff": "0s", + "maxBackoff": "10s", + "backoffMultiplier": 2, + "retryableStatusCodes": ["UNAVAILABLE"] + } + }] + }`, + wantErr: true, + }, + { + name: "zero maxBackoff", + scjs: `{ + "methodConfig": [{ + "name": [{"service": "foo"}], + "retryPolicy": { + "maxAttempts": 2, + "initialBackoff": "2s", + "maxBackoff": "0s", + "backoffMultiplier": 2, + "retryableStatusCodes": ["UNAVAILABLE"] + } + }] + }`, + wantErr: true, + }, + { + name: "zero backoffMultiplier", + scjs: `{ + "methodConfig": [{ + "name": [{"service": "foo"}], + "retryPolicy": { + "maxAttempts": 2, + "initialBackoff": "2s", + "maxBackoff": "10s", + "backoffMultiplier": 0, + "retryableStatusCodes": ["UNAVAILABLE"] + } + }] + }`, + wantErr: true, + }, + { + name: "no retryable codes", + scjs: `{ + "methodConfig": [{ + "name": [{"service": "foo"}], + "retryPolicy": { + "maxAttempts": 2, + "initialBackoff": "2s", + "maxBackoff": "10s", + "backoffMultiplier": 2, + "retryableStatusCodes": [] + } + }] + }`, + wantErr: true, + }, + }) +} + +func newBool(b bool) *bool { + return &b +} + +func newDuration(b time.Duration) *time.Duration { + return &b +} diff --git a/serviceconfig/serviceconfig.go b/serviceconfig/serviceconfig.go new file mode 100644 index 0000000..35e7a20 --- /dev/null +++ b/serviceconfig/serviceconfig.go @@ -0,0 +1,44 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package serviceconfig defines types and methods for operating on gRPC +// service configs. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +package serviceconfig + +// Config represents an opaque data structure holding a service config. +type Config interface { + isServiceConfig() +} + +// LoadBalancingConfig represents an opaque data structure holding a load +// balancing config. +type LoadBalancingConfig interface { + isLoadBalancingConfig() +} + +// ParseResult contains a service config or an error. Exactly one must be +// non-nil. +type ParseResult struct { + Config Config + Err error +} diff --git a/stats/handlers.go b/stats/handlers.go new file mode 100644 index 0000000..67194a5 --- /dev/null +++ b/stats/handlers.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package stats + +import ( + "context" + "net" +) + +// ConnTagInfo defines the relevant information needed by connection context tagger. +type ConnTagInfo struct { + // RemoteAddr is the remote address of the corresponding connection. + RemoteAddr net.Addr + // LocalAddr is the local address of the corresponding connection. + LocalAddr net.Addr +} + +// RPCTagInfo defines the relevant information needed by RPC context tagger. +type RPCTagInfo struct { + // FullMethodName is the RPC method in the format of /package.service/method. + FullMethodName string + // FailFast indicates if this RPC is failfast. + // This field is only valid on client side, it's always false on server side. + FailFast bool + // NameResolutionDelay indicates if the RPC needed to wait for the + // initial name resolver update before it could begin. This should only + // happen if the channel is IDLE when the RPC is started. Note that + // all retry or hedging attempts for an RPC that experienced a delay + // will have it set. + // + // This field is only valid on the client side; it is always false on + // the server side. + NameResolutionDelay bool +} + +// Handler defines the interface for the related stats handling (e.g., RPCs, connections). +type Handler interface { + // TagRPC can attach some information to the given context. + // The context used for the rest lifetime of the RPC will be derived from + // the returned context. + TagRPC(context.Context, *RPCTagInfo) context.Context + // HandleRPC processes the RPC stats. + HandleRPC(context.Context, RPCStats) + + // TagConn can attach some information to the given context. + // The returned context will be used for stats handling. + // For conn stats handling, the context used in HandleConn for this + // connection will be derived from the context returned. + // For RPC stats handling, + // - On server side, the context used in HandleRPC for all RPCs on this + // connection will be derived from the context returned. + // - On client side, the context is not derived from the context returned. + TagConn(context.Context, *ConnTagInfo) context.Context + // HandleConn processes the Conn stats. + HandleConn(context.Context, ConnStats) +} diff --git a/stats/metrics.go b/stats/metrics.go new file mode 100644 index 0000000..641c8e9 --- /dev/null +++ b/stats/metrics.go @@ -0,0 +1,81 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stats + +import "maps" + +// MetricSet is a set of metrics to record. Once created, MetricSet is immutable, +// however Add and Remove can make copies with specific metrics added or +// removed, respectively. +// +// Do not construct directly; use NewMetricSet instead. +type MetricSet struct { + // metrics are the set of metrics to initialize. + metrics map[string]bool +} + +// NewMetricSet returns a MetricSet containing metricNames. +func NewMetricSet(metricNames ...string) *MetricSet { + newMetrics := make(map[string]bool) + for _, metric := range metricNames { + newMetrics[metric] = true + } + return &MetricSet{metrics: newMetrics} +} + +// Metrics returns the metrics set. The returned map is read-only and must not +// be modified. +func (m *MetricSet) Metrics() map[string]bool { + return m.metrics +} + +// Add adds the metricNames to the metrics set and returns a new copy with the +// additional metrics. +func (m *MetricSet) Add(metricNames ...string) *MetricSet { + newMetrics := make(map[string]bool) + for metric := range m.metrics { + newMetrics[metric] = true + } + + for _, metric := range metricNames { + newMetrics[metric] = true + } + return &MetricSet{metrics: newMetrics} +} + +// Join joins the metrics passed in with the metrics set, and returns a new copy +// with the merged metrics. +func (m *MetricSet) Join(metrics *MetricSet) *MetricSet { + newMetrics := make(map[string]bool) + maps.Copy(newMetrics, m.metrics) + maps.Copy(newMetrics, metrics.metrics) + return &MetricSet{metrics: newMetrics} +} + +// Remove removes the metricNames from the metrics set and returns a new copy +// with the metrics removed. +func (m *MetricSet) Remove(metricNames ...string) *MetricSet { + newMetrics := make(map[string]bool) + for metric := range m.metrics { + newMetrics[metric] = true + } + + for _, metric := range metricNames { + delete(newMetrics, metric) + } + return &MetricSet{metrics: newMetrics} +} diff --git a/stats/opencensus/client_metrics.go b/stats/opencensus/client_metrics.go new file mode 100644 index 0000000..4d45845 --- /dev/null +++ b/stats/opencensus/client_metrics.go @@ -0,0 +1,150 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opencensus + +import ( + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" +) + +var ( + keyClientMethod = tag.MustNewKey("grpc_client_method") + keyClientStatus = tag.MustNewKey("grpc_client_status") +) + +// Measures, which are recorded by client stats handler: Note that due to the +// nature of how stats handlers are called on gRPC's client side, the per rpc +// unit is actually per attempt throughout this definition file. +var ( + clientSentMessagesPerRPC = stats.Int64("grpc.io/client/sent_messages_per_rpc", "Number of messages sent in the RPC (always 1 for non-streaming RPCs).", stats.UnitDimensionless) + clientSentBytesPerRPC = stats.Int64("grpc.io/client/sent_bytes_per_rpc", "Total bytes sent across all request messages per RPC.", stats.UnitBytes) + clientSentCompressedBytesPerRPC = stats.Int64("grpc.io/client/sent_compressed_message_bytes_per_rpc", "Total compressed bytes sent across all request messages per RPC.", stats.UnitBytes) + clientReceivedMessagesPerRPC = stats.Int64("grpc.io/client/received_messages_per_rpc", "Number of response messages received per RPC (always 1 for non-streaming RPCs).", stats.UnitDimensionless) + clientReceivedBytesPerRPC = stats.Int64("grpc.io/client/received_bytes_per_rpc", "Total bytes received across all response messages per RPC.", stats.UnitBytes) + clientReceivedCompressedBytesPerRPC = stats.Int64("grpc.io/client/received_compressed_message_bytes_per_rpc", "Total compressed bytes received across all response messages per RPC.", stats.UnitBytes) + clientRoundtripLatency = stats.Float64("grpc.io/client/roundtrip_latency", "Time between first byte of request sent to last byte of response received, or terminal error.", stats.UnitMilliseconds) + clientStartedRPCs = stats.Int64("grpc.io/client/started_rpcs", "The total number of client RPCs ever opened, including those that have not completed.", stats.UnitDimensionless) + clientServerLatency = stats.Float64("grpc.io/client/server_latency", `Propagated from the server and should have the same value as "grpc.io/server/latency".`, stats.UnitMilliseconds) + // Per call measure: + clientAPILatency = stats.Float64("grpc.io/client/api_latency", "The end-to-end time the gRPC library takes to complete an RPC from the application’s perspective", stats.UnitMilliseconds) +) + +var ( + // ClientSentMessagesPerRPCView is the distribution of sent messages per + // RPC, keyed on method. + ClientSentMessagesPerRPCView = &view.View{ + Measure: clientSentMessagesPerRPC, + Name: "grpc.io/client/sent_messages_per_rpc", + Description: "Distribution of sent messages per RPC, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: countDistribution, + } + // ClientReceivedMessagesPerRPCView is the distribution of received messages + // per RPC, keyed on method. + ClientReceivedMessagesPerRPCView = &view.View{ + Measure: clientReceivedMessagesPerRPC, + Name: "grpc.io/client/received_messages_per_rpc", + Description: "Distribution of received messages per RPC, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: countDistribution, + } + // ClientSentBytesPerRPCView is the distribution of sent bytes per RPC, + // keyed on method. + ClientSentBytesPerRPCView = &view.View{ + Measure: clientSentBytesPerRPC, + Name: "grpc.io/client/sent_bytes_per_rpc", + Description: "Distribution of sent bytes per RPC, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: bytesDistribution, + } + // ClientSentCompressedMessageBytesPerRPCView is the distribution of + // compressed sent message bytes per RPC, keyed on method. + ClientSentCompressedMessageBytesPerRPCView = &view.View{ + Measure: clientSentCompressedBytesPerRPC, + Name: "grpc.io/client/sent_compressed_message_bytes_per_rpc", + Description: "Distribution of sent compressed message bytes per RPC, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: bytesDistribution, + } + // ClientReceivedBytesPerRPCView is the distribution of received bytes per + // RPC, keyed on method. + ClientReceivedBytesPerRPCView = &view.View{ + Measure: clientReceivedBytesPerRPC, + Name: "grpc.io/client/received_bytes_per_rpc", + Description: "Distribution of received bytes per RPC, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: bytesDistribution, + } + // ClientReceivedCompressedMessageBytesPerRPCView is the distribution of + // compressed received message bytes per RPC, keyed on method. + ClientReceivedCompressedMessageBytesPerRPCView = &view.View{ + Measure: clientReceivedCompressedBytesPerRPC, + Name: "grpc.io/client/received_compressed_message_bytes_per_rpc", + Description: "Distribution of received compressed message bytes per RPC, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: bytesDistribution, + } + // ClientStartedRPCsView is the count of opened RPCs, keyed on method. + ClientStartedRPCsView = &view.View{ + Measure: clientStartedRPCs, + Name: "grpc.io/client/started_rpcs", + Description: "Number of opened client RPCs, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: view.Count(), + } + // ClientCompletedRPCsView is the count of completed RPCs, keyed on method + // and status. + ClientCompletedRPCsView = &view.View{ + Measure: clientRoundtripLatency, + Name: "grpc.io/client/completed_rpcs", + Description: "Number of completed RPCs by method and status.", + TagKeys: []tag.Key{keyClientMethod, keyClientStatus}, + Aggregation: view.Count(), + } + // ClientRoundtripLatencyView is the distribution of round-trip latency in + // milliseconds per RPC, keyed on method. + ClientRoundtripLatencyView = &view.View{ + Measure: clientRoundtripLatency, + Name: "grpc.io/client/roundtrip_latency", + Description: "Distribution of round-trip latency, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: millisecondsDistribution, + } + + // The following metric is per call: + + // ClientAPILatencyView is the distribution of client api latency for the + // full RPC call, keyed on method and status. + ClientAPILatencyView = &view.View{ + Measure: clientAPILatency, + Name: "grpc.io/client/api_latency", + Description: "Distribution of client api latency, by method and status", + TagKeys: []tag.Key{keyClientMethod, keyClientStatus}, + Aggregation: millisecondsDistribution, + } +) + +// DefaultClientViews is the set of client views which are considered the +// minimum required to monitor client side performance. +var DefaultClientViews = []*view.View{ + ClientSentBytesPerRPCView, + ClientReceivedBytesPerRPCView, + ClientRoundtripLatencyView, + ClientCompletedRPCsView, + ClientStartedRPCsView, +} diff --git a/stats/opencensus/e2e_test.go b/stats/opencensus/e2e_test.go new file mode 100644 index 0000000..d4cd441 --- /dev/null +++ b/stats/opencensus/e2e_test.go @@ -0,0 +1,1619 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opencensus + +import ( + "context" + "errors" + "fmt" + "io" + "reflect" + "sort" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + "go.opencensus.io/trace" + + "google.golang.org/grpc" + "google.golang.org/grpc/encoding/gzip" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/leakcheck" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func init() { + // OpenCensus, once included in binary, will spawn a global goroutine + // recorder that is not controllable by application. + // https://github.com/census-instrumentation/opencensus-go/issues/1191 + leakcheck.RegisterIgnoreGoroutine("go.opencensus.io/stats/view.(*worker).start") +} + +var defaultTestTimeout = 5 * time.Second + +type fakeExporter struct { + t *testing.T + + mu sync.RWMutex + seenViews map[string]*viewInformation + seenSpans []spanInformation +} + +// viewInformation is information Exported from the view package through +// ExportView relevant to testing, i.e. a reasonably non flaky expectation of +// desired emissions to Exporter. +type viewInformation struct { + aggType view.AggType + aggBuckets []float64 + desc string + tagKeys []tag.Key + rows []*view.Row +} + +func (fe *fakeExporter) ExportView(vd *view.Data) { + fe.mu.Lock() + defer fe.mu.Unlock() + fe.seenViews[vd.View.Name] = &viewInformation{ + aggType: vd.View.Aggregation.Type, + aggBuckets: vd.View.Aggregation.Buckets, + desc: vd.View.Description, + tagKeys: vd.View.TagKeys, + rows: vd.Rows, + } +} + +// compareRows compares rows with respect to the information desired to test. +// Both the tags representing the rows and also the data of the row are tested +// for equality. Rows are in nondeterministic order when ExportView is called, +// but handled inside this function by sorting. +func compareRows(rows []*view.Row, rows2 []*view.Row) bool { + if len(rows) != len(rows2) { + return false + } + // Sort both rows according to the same rule. This is to take away non + // determinism in the row ordering passed to the Exporter, while keeping the + // row data. + sort.Slice(rows, func(i, j int) bool { + return rows[i].String() > rows[j].String() + }) + + sort.Slice(rows2, func(i, j int) bool { + return rows2[i].String() > rows2[j].String() + }) + + for i, row := range rows { + if !cmp.Equal(row.Tags, rows2[i].Tags, cmp.Comparer(func(a tag.Key, b tag.Key) bool { + return a.Name() == b.Name() + })) { + return false + } + if !compareData(row.Data, rows2[i].Data) { + return false + } + } + return true +} + +// compareData returns whether the two aggregation data's are equal to each +// other with respect to parts of the data desired for correct emission. The +// function first makes sure the two types of aggregation data are the same, and +// then checks the equality for the respective aggregation data type. +func compareData(ad view.AggregationData, ad2 view.AggregationData) bool { + if ad == nil && ad2 == nil { + return true + } + if ad == nil || ad2 == nil { + return false + } + if reflect.TypeOf(ad) != reflect.TypeOf(ad2) { + return false + } + switch ad1 := ad.(type) { + case *view.DistributionData: + dd2 := ad2.(*view.DistributionData) + // Count and Count Per Buckets are reasonable for correctness, + // especially since we verify equality of bucket endpoints elsewhere. + if ad1.Count != dd2.Count { + return false + } + for i, count := range ad1.CountPerBucket { + if count != dd2.CountPerBucket[i] { + return false + } + } + case *view.CountData: + cd2 := ad2.(*view.CountData) + return ad1.Value == cd2.Value + + // gRPC open census plugin does not have these next two types of aggregation + // data types present, for now just check for type equality between the two + // aggregation data points (done above). + // case *view.SumData + // case *view.LastValueData: + } + return true +} + +func (vi *viewInformation) Equal(vi2 *viewInformation) bool { + if vi == nil && vi2 == nil { + return true + } + if vi == nil || vi2 == nil { + return false + } + if vi.aggType != vi2.aggType { + return false + } + if !cmp.Equal(vi.aggBuckets, vi2.aggBuckets) { + return false + } + if vi.desc != vi2.desc { + return false + } + if !cmp.Equal(vi.tagKeys, vi2.tagKeys, cmp.Comparer(func(a tag.Key, b tag.Key) bool { + return a.Name() == b.Name() + })) { + return false + } + if !compareRows(vi.rows, vi2.rows) { + return false + } + return true +} + +// distributionDataLatencyCount checks if the view information contains the +// desired distribution latency total count that falls in buckets of 5 seconds or +// less. This must be called with non nil view information that is aggregated +// with distribution data. Returns a nil error if correct count information +// found, non nil error if correct information not found. +func distributionDataLatencyCount(vi *viewInformation, countWant int64, wantTags [][]tag.Tag) error { + var totalCount int64 + var largestIndexWithFive int + for i, bucket := range vi.aggBuckets { + // Distribution for latency is measured in milliseconds, so 5 * 1000 = + // 5000. + if bucket > 5000 { + largestIndexWithFive = i + break + } + } + // Sort rows by string name. This is to take away non determinism in the row + // ordering passed to the Exporter, while keeping the row data. + sort.Slice(vi.rows, func(i, j int) bool { + return vi.rows[i].String() > vi.rows[j].String() + }) + // Iterating through rows sums up data points for all methods. In this case, + // a data point for the unary and for the streaming RPC. + for i, row := range vi.rows { + // The method names corresponding to unary and streaming call should + // have the leading slash removed. + if diff := cmp.Diff(row.Tags, wantTags[i], cmp.Comparer(func(a tag.Key, b tag.Key) bool { + return a.Name() == b.Name() + })); diff != "" { + return fmt.Errorf("wrong tag keys for unary method -got, +want: %v", diff) + } + // This could potentially have an extra measurement in buckets above 5s, + // but that's fine. Count of buckets that could contain up to 5s is a + // good enough assertion. + for i, count := range row.Data.(*view.DistributionData).CountPerBucket { + if i >= largestIndexWithFive { + break + } + totalCount = totalCount + count + } + } + if totalCount != countWant { + return fmt.Errorf("wrong total count for counts under 5: %v, wantCount: %v", totalCount, countWant) + } + return nil +} + +// waitForServerCompletedRPCs waits until both Unary and Streaming metric rows +// appear, in two separate rows, for server completed RPC's view. Returns an +// error if the Unary and Streaming metric are not found within the passed +// context's timeout. +func waitForServerCompletedRPCs(ctx context.Context) error { + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + rows, err := view.RetrieveData("grpc.io/server/completed_rpcs") + if err != nil { + continue + } + unaryFound := false + streamingFound := false + for _, row := range rows { + for _, tag := range row.Tags { + if tag.Value == "grpc.testing.TestService/UnaryCall" { + unaryFound = true + break + } else if tag.Value == "grpc.testing.TestService/FullDuplexCall" { + streamingFound = true + break + } + } + if unaryFound && streamingFound { + return nil + } + } + } + return fmt.Errorf("timeout when waiting for Unary and Streaming rows to be present for \"grpc.io/server/completed_rpcs\"") +} + +// TestAllMetricsOneFunction tests emitted metrics from gRPC. It registers all +// the metrics provided by this package. It then configures a system with a gRPC +// Client and gRPC server with the OpenCensus Dial and Server Option configured, +// and makes a Unary RPC and a Streaming RPC. These two RPCs should cause +// certain emissions for each registered metric through the OpenCensus View +// package. +func (s) TestAllMetricsOneFunction(t *testing.T) { + allViews := []*view.View{ + ClientStartedRPCsView, + ServerStartedRPCsView, + ClientCompletedRPCsView, + ServerCompletedRPCsView, + ClientSentBytesPerRPCView, + ClientSentCompressedMessageBytesPerRPCView, + ServerSentBytesPerRPCView, + ServerSentCompressedMessageBytesPerRPCView, + ClientReceivedBytesPerRPCView, + ClientReceivedCompressedMessageBytesPerRPCView, + ServerReceivedBytesPerRPCView, + ServerReceivedCompressedMessageBytesPerRPCView, + ClientSentMessagesPerRPCView, + ServerSentMessagesPerRPCView, + ClientReceivedMessagesPerRPCView, + ServerReceivedMessagesPerRPCView, + ClientRoundtripLatencyView, + ServerLatencyView, + ClientAPILatencyView, + } + view.Register(allViews...) + // Unregister unconditionally in this defer to correctly cleanup globals in + // error conditions. + defer view.Unregister(allViews...) + fe := &fakeExporter{ + t: t, + seenViews: make(map[string]*viewInformation), + } + view.RegisterExporter(fe) + defer view.UnregisterExporter(fe) + + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + _, err := stream.Recv() + if err == io.EOF { + return nil + } + } + }, + } + if err := ss.Start([]grpc.ServerOption{ServerOption(TraceOptions{})}, DialOption(TraceOptions{})); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Make two RPC's, a unary RPC and a streaming RPC. These should cause + // certain metrics to be emitted. + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}, grpc.UseCompressor(gzip.Name)); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + + cmtk := tag.MustNewKey("grpc_client_method") + smtk := tag.MustNewKey("grpc_server_method") + cstk := tag.MustNewKey("grpc_client_status") + sstk := tag.MustNewKey("grpc_server_status") + wantMetrics := []struct { + metric *view.View + wantVI *viewInformation + wantTags [][]tag.Tag // for non deterministic (i.e. latency) metrics. First dimension represents rows. + }{ + { + metric: ClientStartedRPCsView, + wantVI: &viewInformation{ + aggType: view.AggTypeCount, + aggBuckets: []float64{}, + desc: "Number of opened client RPCs, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + }, + }, + }, + { + metric: ServerStartedRPCsView, + wantVI: &viewInformation{ + aggType: view.AggTypeCount, + aggBuckets: []float64{}, + desc: "Number of opened server RPCs, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + }, + }, + }, + { + metric: ClientCompletedRPCsView, + wantVI: &viewInformation{ + aggType: view.AggTypeCount, + aggBuckets: []float64{}, + desc: "Number of completed RPCs by method and status.", + tagKeys: []tag.Key{ + cmtk, + cstk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + { + Key: cstk, + Value: "OK", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + { + Key: cstk, + Value: "OK", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + }, + }, + }, + { + metric: ServerCompletedRPCsView, + wantVI: &viewInformation{ + aggType: view.AggTypeCount, + aggBuckets: []float64{}, + desc: "Number of completed RPCs by method and status.", + tagKeys: []tag.Key{ + smtk, + sstk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + { + Key: sstk, + Value: "OK", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + { + Key: sstk, + Value: "OK", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + }, + }, + }, + { + metric: ClientSentBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of sent bytes per RPC, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ClientSentCompressedMessageBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of sent compressed message bytes per RPC, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ServerSentBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of sent bytes per RPC, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ServerSentCompressedMessageBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of sent compressed message bytes per RPC, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ClientReceivedBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of received bytes per RPC, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ClientReceivedCompressedMessageBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of received compressed message bytes per RPC, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ServerReceivedBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of received bytes per RPC, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ServerReceivedCompressedMessageBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of received compressed message bytes per RPC, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ClientSentMessagesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: countDistributionBounds, + desc: "Distribution of sent messages per RPC, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ServerSentMessagesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: countDistributionBounds, + desc: "Distribution of sent messages per RPC, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ClientReceivedMessagesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: countDistributionBounds, + desc: "Distribution of received messages per RPC, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ServerReceivedMessagesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: countDistributionBounds, + desc: "Distribution of received messages per RPC, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ClientRoundtripLatencyView, + wantTags: [][]tag.Tag{ + { + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + { + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + }, + }, + { + metric: ServerLatencyView, + wantTags: [][]tag.Tag{ + { + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + { + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + }, + }, + // Per call metrics: + { + metric: ClientAPILatencyView, + wantTags: [][]tag.Tag{ + { + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + { + Key: cstk, + Value: "OK", + }, + }, + { + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + { + Key: cstk, + Value: "OK", + }, + }, + }, + }, + } + // Server Side stats.End call happens asynchronously for both Unary and + // Streaming calls with respect to the RPC returning client side. Thus, add + // a sync point at the global view package level for these two rows to be + // recorded, which will be synchronously uploaded to exporters right after. + if err := waitForServerCompletedRPCs(ctx); err != nil { + t.Fatal(err) + } + view.Unregister(allViews...) + // Assert the expected emissions for each metric match the expected + // emissions. + for _, wantMetric := range wantMetrics { + metricName := wantMetric.metric.Name + var vi *viewInformation + if vi = fe.seenViews[metricName]; vi == nil { + t.Fatalf("couldn't find %v in the views exported, never collected", metricName) + } + + // For latency metrics, there is a lot of non determinism about + // the exact milliseconds of RPCs that finish. Thus, rather than + // declare the exact data you want, make sure the latency + // measurement points for the two RPCs above fall within buckets + // that fall into less than 5 seconds, which is the rpc timeout. + if metricName == "grpc.io/client/roundtrip_latency" || metricName == "grpc.io/server/server_latency" || metricName == "grpc.io/client/api_latency" { + // RPCs have a context timeout of 5s, so all the recorded + // measurements (one per RPC - two total) should fall within 5 + // second buckets. + if err := distributionDataLatencyCount(vi, 2, wantMetric.wantTags); err != nil { + t.Fatalf("Invalid OpenCensus export view data for metric %v: %v", metricName, err) + } + continue + } + if diff := cmp.Diff(vi, wantMetric.wantVI); diff != "" { + t.Fatalf("got unexpected viewInformation for metric %v, diff (-got, +want): %v", metricName, diff) + } + // Note that this test only fatals with one error if a metric fails. + // This is fine, as all are expected to pass so if a single one fails + // you can figure it out and iterate as needed. + } +} + +// TestOpenCensusTags tests this instrumentation code's ability to propagate +// OpenCensus tags across the wire. It also tests the server stats handler's +// functionality of adding the server method tag for the application to see. The +// test makes a Unary RPC without a tag map and with a tag map, and expects to +// see a tag map at the application layer with server method tag in the first +// case, and a tag map at the application layer with the populated tag map plus +// server method tag in second case. +func (s) TestOpenCensusTags(t *testing.T) { + // This stub servers functions represent the application layer server side. + // This is the intended feature being tested: that open census tags + // populated at the client side application layer end up at the server side + // application layer with the server method tag key in addition to the map + // populated at the client side application layer if populated. + tmCh := testutils.NewChannel() + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + // Do the sends of the tag maps for assertions in this main testing + // goroutine. Do the receives and assertions in a forked goroutine. + if tm := tag.FromContext(ctx); tm != nil { + tmCh.Send(tm) + } else { + tmCh.Send(errors.New("no tag map received server side")) + } + return &testpb.SimpleResponse{}, nil + }, + } + if err := ss.Start([]grpc.ServerOption{ServerOption(TraceOptions{})}, DialOption(TraceOptions{})); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + key1 := tag.MustNewKey("key 1") + wg := sync.WaitGroup{} + wg.Add(1) + readerErrCh := testutils.NewChannel() + // Spawn a goroutine to receive and validation two tag maps received by the + // server application code. + go func() { + defer wg.Done() + unaryCallMethodName := "grpc.testing.TestService/UnaryCall" + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Attempt to receive the tag map from the first RPC. + if tm, err := tmCh.Receive(ctx); err == nil { + tagMap, ok := tm.(*tag.Map) + // Shouldn't happen, this test sends only *tag.Map type on channel. + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", tm)) + } + // keyServerMethod should be present in this tag map received server + // side. + val, ok := tagMap.Value(keyServerMethod) + if !ok { + readerErrCh.Send(fmt.Errorf("no key: %v present in OpenCensus tag map", keyServerMethod.Name())) + } + if val != unaryCallMethodName { + readerErrCh.Send(fmt.Errorf("serverMethod received: %v, want server method: %v", val, unaryCallMethodName)) + } + } else { + readerErrCh.Send(fmt.Errorf("error while waiting for a tag map: %v", err)) + } + readerErrCh.Send(nil) + + // Attempt to receive the tag map from the second RPC. + if tm, err := tmCh.Receive(ctx); err == nil { + tagMap, ok := tm.(*tag.Map) + // Shouldn't happen, this test sends only *tag.Map type on channel. + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", tm)) + } + // key1: "value1" populated in the tag map client side should make + // its way to server. + val, ok := tagMap.Value(key1) + if !ok { + readerErrCh.Send(fmt.Errorf("no key: %v present in OpenCensus tag map", key1.Name())) + } + if val != "value1" { + readerErrCh.Send(fmt.Errorf("key %v received: %v, want server method: %v", key1.Name(), val, unaryCallMethodName)) + } + // keyServerMethod should be appended to tag map as well. + val, ok = tagMap.Value(keyServerMethod) + if !ok { + readerErrCh.Send(fmt.Errorf("no key: %v present in OpenCensus tag map", keyServerMethod.Name())) + } + if val != unaryCallMethodName { + readerErrCh.Send(fmt.Errorf("key: %v received: %v, want server method: %v", keyServerMethod.Name(), val, unaryCallMethodName)) + } + } else { + readerErrCh.Send(fmt.Errorf("error while waiting for second tag map: %v", err)) + } + readerErrCh.Send(nil) + }() + + // Make a unary RPC without populating an OpenCensus tag map. The server + // side should receive an OpenCensus tag map containing only the + // keyServerMethod. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Should receive a nil error from the readerErrCh, meaning the reader + // goroutine successfully received a tag map with the keyServerMethod + // populated. + if chErr, err := readerErrCh.Receive(ctx); chErr != nil || err != nil { + if err != nil { + t.Fatalf("Should have received something from error channel: %v", err) + } + if chErr != nil { + t.Fatalf("Should have received a nil error from channel, instead received: %v", chErr) + } + } + + tm := &tag.Map{} + ctx = tag.NewContext(ctx, tm) + ctx, err := tag.New(ctx, tag.Upsert(key1, "value1")) + // Setup steps like this can fatal, so easier to do the RPC's and subsequent + // sends of the tag maps of the RPC's in main goroutine and have the + // corresponding receives and assertions in a forked goroutine. + if err != nil { + t.Fatalf("Error creating tag map: %v", err) + } + // Make a unary RPC with a populated OpenCensus tag map. The server side + // should receive an OpenCensus tag map containing this populated tag map + // with the keyServerMethod tag appended to it. + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + if chErr, err := readerErrCh.Receive(ctx); chErr != nil || err != nil { + if err != nil { + t.Fatalf("Should have received something from error channel: %v", err) + } + if chErr != nil { + t.Fatalf("Should have received a nil error from channel, instead received: %v", chErr) + } + } + wg.Wait() +} + +// compareSpanContext only checks the equality of the trace options, which +// represent whether the span should be sampled. The other fields are checked +// for presence in later assertions. +func compareSpanContext(sc trace.SpanContext, sc2 trace.SpanContext) bool { + return sc.TraceOptions.IsSampled() == sc2.TraceOptions.IsSampled() +} + +func compareMessageEvents(me []trace.MessageEvent, me2 []trace.MessageEvent) bool { + if len(me) != len(me2) { + return false + } + // Order matters here, message events are deterministic so no flakiness to + // test. + for i, e := range me { + e2 := me2[i] + if e.EventType != e2.EventType { + return false + } + if e.MessageID != e2.MessageID { + return false + } + if e.UncompressedByteSize != e2.UncompressedByteSize { + return false + } + if e.CompressedByteSize != e2.CompressedByteSize { + return false + } + } + return true +} + +// compareLinks compares the type of link received compared to the wanted link. +func compareLinks(ls []trace.Link, ls2 []trace.Link) bool { + if len(ls) != len(ls2) { + return false + } + for i, l := range ls { + l2 := ls2[i] + if l.Type != l2.Type { + return false + } + } + return true +} + +// spanInformation is the information received about the span. This is a subset +// of information that is important to verify that gRPC has knobs over, which +// goes through a stable OpenCensus API with well defined behavior. This keeps +// the robustness of assertions over time. +type spanInformation struct { + // SpanContext either gets pulled off the wire in certain cases server side + // or created. + sc trace.SpanContext + parentSpanID trace.SpanID + spanKind int + name string + message string + messageEvents []trace.MessageEvent + status trace.Status + links []trace.Link + hasRemoteParent bool + childSpanCount int +} + +// validateTraceAndSpanIDs checks for consistent trace ID across the full trace. +// It also asserts each span has a corresponding generated SpanID, and makes +// sure in the case of a server span and a client span, the server span points +// to the client span as its parent. This is assumed to be called with spans +// from the same RPC (thus the same trace). If called with spanInformation slice +// of length 2, it assumes first span is a server span which points to second +// span as parent and second span is a client span. These assertions are +// orthogonal to pure equality assertions, as this data is generated at runtime, +// so can only test relations between IDs (i.e. this part of the data has the +// same ID as this part of the data). +// +// Returns an error in the case of a failing assertion, non nil error otherwise. +func validateTraceAndSpanIDs(sis []spanInformation) error { + var traceID trace.TraceID + for i, si := range sis { + // Trace IDs should all be consistent across every span, since this + // function assumes called with Span from one RPC, which all fall under + // one trace. + if i == 0 { + traceID = si.sc.TraceID + } else { + if !cmp.Equal(si.sc.TraceID, traceID) { + return fmt.Errorf("TraceIDs should all be consistent: %v, %v", si.sc.TraceID, traceID) + } + } + // Due to the span IDs being 8 bytes, the documentation states that it + // is practically a mathematical uncertainty in practice to create two + // colliding IDs. Thus, for a presence check (the ID was actually + // generated, I will simply compare to the zero value, even though a + // zero value is a theoretical possibility of generation). This is + // because in practice, this zero value defined by this test will never + // collide with the generated ID. + if cmp.Equal(si.sc.SpanID, trace.SpanID{}) { + return errors.New("span IDs should be populated from the creation of the span") + } + } + // If the length of spans of an RPC is 2, it means there is a server span + // which exports first and a client span which exports second. Thus, the + // server span should point to the client span as its parent, represented + // by its ID. + if len(sis) == 2 { + if !cmp.Equal(sis[0].parentSpanID, sis[1].sc.SpanID) { + return fmt.Errorf("server span should point to the client span as its parent. parentSpanID: %v, clientSpanID: %v", sis[0].parentSpanID, sis[1].sc.SpanID) + } + } + return nil +} + +// Equal compares the constant data of the exported span information that is +// important for correctness known before runtime. +func (si spanInformation) Equal(si2 spanInformation) bool { + if !compareSpanContext(si.sc, si2.sc) { + return false + } + + if si.spanKind != si2.spanKind { + return false + } + if si.name != si2.name { + return false + } + if si.message != si2.message { + return false + } + // Ignore attribute comparison because Java doesn't even populate any so not + // important for correctness. + if !compareMessageEvents(si.messageEvents, si2.messageEvents) { + return false + } + if !cmp.Equal(si.status, si2.status) { + return false + } + // compare link type as link type child is important. + if !compareLinks(si.links, si2.links) { + return false + } + if si.hasRemoteParent != si2.hasRemoteParent { + return false + } + return si.childSpanCount == si2.childSpanCount +} + +func (fe *fakeExporter) ExportSpan(sd *trace.SpanData) { + fe.mu.Lock() + defer fe.mu.Unlock() + + // Persist the subset of data received that is important for correctness and + // to make various assertions on later. Keep the ordering as ordering of + // spans is deterministic in the context of one RPC. + gotSI := spanInformation{ + sc: sd.SpanContext, + parentSpanID: sd.ParentSpanID, + spanKind: sd.SpanKind, + name: sd.Name, + message: sd.Message, + // annotations - ignore + // attributes - ignore, I just left them in from previous but no spec + // for correctness so no need to test. Java doesn't even have any + // attributes. + messageEvents: sd.MessageEvents, + status: sd.Status, + links: sd.Links, + hasRemoteParent: sd.HasRemoteParent, + childSpanCount: sd.ChildSpanCount, + } + fe.seenSpans = append(fe.seenSpans, gotSI) +} + +// waitForServerSpan waits until a server span appears somewhere in the span +// list in an exporter. Returns an error if no server span found within the +// passed context's timeout. +func waitForServerSpan(ctx context.Context, fe *fakeExporter) error { + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + fe.mu.Lock() + for _, seenSpan := range fe.seenSpans { + if seenSpan.spanKind == trace.SpanKindServer { + fe.mu.Unlock() + return nil + } + } + fe.mu.Unlock() + } + return fmt.Errorf("timeout when waiting for server span to be present in exporter") +} + +// TestSpan tests emitted spans from gRPC. It configures a system with a gRPC +// Client and gRPC server with the OpenCensus Dial and Server Option configured, +// and makes a Unary RPC and a Streaming RPC. This should cause spans with +// certain information to be emitted from client and server side for each RPC. +func (s) TestSpan(t *testing.T) { + fe := &fakeExporter{ + t: t, + } + trace.RegisterExporter(fe) + defer trace.UnregisterExporter(fe) + + so := TraceOptions{ + TS: trace.ProbabilitySampler(1), + DisableTrace: false, + } + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + _, err := stream.Recv() + if err == io.EOF { + return nil + } + } + }, + } + if err := ss.Start([]grpc.ServerOption{ServerOption(so)}, DialOption(so)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Make a Unary RPC. This should cause a span with message events + // corresponding to the request message and response message to be emitted + // both from the client and the server. + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + wantSI := []spanInformation{ + { + sc: trace.SpanContext{ + TraceOptions: 1, + }, + name: "Attempt.grpc.testing.TestService.UnaryCall", + messageEvents: []trace.MessageEvent{ + { + EventType: trace.MessageEventTypeSent, + MessageID: 1, // First msg send so 1 (see comment above) + UncompressedByteSize: 2, + CompressedByteSize: 2, + }, + { + EventType: trace.MessageEventTypeRecv, + MessageID: 1, // First msg recv so 1 (see comment above) + }, + }, + hasRemoteParent: false, + }, + { + // Sampling rate of 100 percent, so this should populate every span + // with the information that this span is being sampled. Here and + // every other span emitted in this test. + sc: trace.SpanContext{ + TraceOptions: 1, + }, + spanKind: trace.SpanKindServer, + name: "grpc.testing.TestService.UnaryCall", + // message id - "must be calculated as two different counters + // starting from 1 one for sent messages and one for received + // message. This way we guarantee that the values will be consistent + // between different implementations. In case of unary calls only + // one sent and one received message will be recorded for both + // client and server spans." + messageEvents: []trace.MessageEvent{ + { + EventType: trace.MessageEventTypeRecv, + MessageID: 1, // First msg recv so 1 (see comment above) + UncompressedByteSize: 2, + CompressedByteSize: 2, + }, + { + EventType: trace.MessageEventTypeSent, + MessageID: 1, // First msg send so 1 (see comment above) + }, + }, + links: []trace.Link{ + { + Type: trace.LinkTypeChild, + }, + }, + // For some reason, status isn't populated in the data sent to the + // exporter. This seems wrong, but it didn't send status in old + // instrumentation code, so I'm iffy on it but fine. + hasRemoteParent: true, + }, + { + sc: trace.SpanContext{ + TraceOptions: 1, + }, + spanKind: trace.SpanKindClient, + name: "grpc.testing.TestService.UnaryCall", + hasRemoteParent: false, + childSpanCount: 1, + }, + } + if err := waitForServerSpan(ctx, fe); err != nil { + t.Fatal(err) + } + var spanInfoSort = func(i, j int) bool { + // This will order into attempt span (which has an unset span kind to + // not prepend Sent. to span names in backends), then call span, then + // server span. + return fe.seenSpans[i].spanKind < fe.seenSpans[j].spanKind + } + fe.mu.Lock() + // Sort the underlying seen Spans for cmp.Diff assertions and ID + // relationship assertions. + sort.Slice(fe.seenSpans, spanInfoSort) + if diff := cmp.Diff(fe.seenSpans, wantSI); diff != "" { + fe.mu.Unlock() + t.Fatalf("got unexpected spans, diff (-got, +want): %v", diff) + } + if err := validateTraceAndSpanIDs(fe.seenSpans); err != nil { + fe.mu.Unlock() + t.Fatalf("Error in runtime data assertions: %v", err) + } + if !cmp.Equal(fe.seenSpans[1].parentSpanID, fe.seenSpans[0].sc.SpanID) { + t.Fatalf("server span should point to the client attempt span as its parent. parentSpanID: %v, clientAttemptSpanID: %v", fe.seenSpans[1].parentSpanID, fe.seenSpans[0].sc.SpanID) + } + if !cmp.Equal(fe.seenSpans[0].parentSpanID, fe.seenSpans[2].sc.SpanID) { + t.Fatalf("client attempt span should point to the client call span as its parent. parentSpanID: %v, clientCallSpanID: %v", fe.seenSpans[0].parentSpanID, fe.seenSpans[2].sc.SpanID) + } + + fe.seenSpans = nil + fe.mu.Unlock() + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %v", err) + } + // Send two messages. This should be recorded in the emitted spans message + // events, with message IDs which increase for each message. + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("stream.Send failed: %v", err) + } + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("stream.Send failed: %v", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + + wantSI = []spanInformation{ + { + sc: trace.SpanContext{ + TraceOptions: 1, + }, + name: "Attempt.grpc.testing.TestService.FullDuplexCall", + messageEvents: []trace.MessageEvent{ + { + EventType: trace.MessageEventTypeSent, + MessageID: 1, // First msg send so 1 + }, + { + EventType: trace.MessageEventTypeSent, + MessageID: 2, // Second msg send so 2 + }, + }, + hasRemoteParent: false, + }, + { + sc: trace.SpanContext{ + TraceOptions: 1, + }, + spanKind: trace.SpanKindServer, + name: "grpc.testing.TestService.FullDuplexCall", + links: []trace.Link{ + { + Type: trace.LinkTypeChild, + }, + }, + messageEvents: []trace.MessageEvent{ + { + EventType: trace.MessageEventTypeRecv, + MessageID: 1, // First msg recv so 1 + }, + { + EventType: trace.MessageEventTypeRecv, + MessageID: 2, // Second msg recv so 2 + }, + }, + hasRemoteParent: true, + }, + { + sc: trace.SpanContext{ + TraceOptions: 1, + }, + spanKind: trace.SpanKindClient, + name: "grpc.testing.TestService.FullDuplexCall", + hasRemoteParent: false, + childSpanCount: 1, + }, + } + if err := waitForServerSpan(ctx, fe); err != nil { + t.Fatal(err) + } + fe.mu.Lock() + defer fe.mu.Unlock() + // Sort the underlying seen Spans for cmp.Diff assertions and ID + // relationship assertions. + sort.Slice(fe.seenSpans, spanInfoSort) + if diff := cmp.Diff(fe.seenSpans, wantSI); diff != "" { + t.Fatalf("got unexpected spans, diff (-got, +want): %v", diff) + } + if err := validateTraceAndSpanIDs(fe.seenSpans); err != nil { + t.Fatalf("Error in runtime data assertions: %v", err) + } + if !cmp.Equal(fe.seenSpans[1].parentSpanID, fe.seenSpans[0].sc.SpanID) { + t.Fatalf("server span should point to the client attempt span as its parent. parentSpanID: %v, clientAttemptSpanID: %v", fe.seenSpans[1].parentSpanID, fe.seenSpans[0].sc.SpanID) + } + if !cmp.Equal(fe.seenSpans[0].parentSpanID, fe.seenSpans[2].sc.SpanID) { + t.Fatalf("client attempt span should point to the client call span as its parent. parentSpanID: %v, clientCallSpanID: %v", fe.seenSpans[0].parentSpanID, fe.seenSpans[2].sc.SpanID) + } +} diff --git a/stats/opencensus/go.mod b/stats/opencensus/go.mod new file mode 100644 index 0000000..dd4977c --- /dev/null +++ b/stats/opencensus/go.mod @@ -0,0 +1,20 @@ +module google.golang.org/grpc/stats/opencensus + +go 1.25.0 + +require ( + github.com/google/go-cmp v0.7.0 + go.opencensus.io v0.24.0 + google.golang.org/grpc v1.83.0 +) + +require ( + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) + +replace google.golang.org/grpc => ../.. diff --git a/stats/opencensus/opencensus.go b/stats/opencensus/opencensus.go new file mode 100644 index 0000000..ff13537 --- /dev/null +++ b/stats/opencensus/opencensus.go @@ -0,0 +1,251 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package opencensus implements opencensus instrumentation code for gRPC-Go +// clients and servers. +package opencensus + +import ( + "context" + "strings" + "time" + + ocstats "go.opencensus.io/stats" + "go.opencensus.io/tag" + "go.opencensus.io/trace" + + "google.golang.org/grpc" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +var ( + joinDialOptions = internal.JoinDialOptions.(func(...grpc.DialOption) grpc.DialOption) +) + +// TraceOptions are the tracing options for opencensus instrumentation. +type TraceOptions struct { + // TS is the Sampler used for tracing. + TS trace.Sampler + // DisableTrace determines whether traces are disabled for an OpenCensus + // Dial or Server option. will overwrite any global option setting. + DisableTrace bool +} + +// DialOption returns a dial option which enables OpenCensus instrumentation +// code for a grpc.ClientConn. +// +// Client applications interested in instrumenting their grpc.ClientConn should +// pass the dial option returned from this function as the first dial option to +// grpc.Dial(). +// +// Using this option will always lead to instrumentation, however in order to +// use the data an exporter must be registered with the OpenCensus trace package +// for traces and the OpenCensus view package for metrics. Client side has +// retries, so a Unary and Streaming Interceptor are registered to handle per +// RPC traces/metrics, and a Stats Handler is registered to handle per RPC +// attempt trace/metrics. These three components registered work together in +// conjunction, and do not work standalone. It is not supported to use this +// alongside another stats handler dial option. +func DialOption(to TraceOptions) grpc.DialOption { + csh := &clientStatsHandler{to: to} + return joinDialOptions(grpc.WithChainUnaryInterceptor(csh.unaryInterceptor), grpc.WithChainStreamInterceptor(csh.streamInterceptor), grpc.WithStatsHandler(csh)) +} + +// ServerOption returns a server option which enables OpenCensus instrumentation +// code for a grpc.Server. +// +// Server applications interested in instrumenting their grpc.Server should +// pass the server option returned from this function as the first argument to +// grpc.NewServer(). +// +// Using this option will always lead to instrumentation, however in order to +// use the data an exporter must be registered with the OpenCensus trace package +// for traces and the OpenCensus view package for metrics. Server side does not +// have retries, so a registered Stats Handler is the only option that is +// returned. It is not supported to use this alongside another stats handler +// server option. +func ServerOption(to TraceOptions) grpc.ServerOption { + return grpc.StatsHandler(&serverStatsHandler{to: to}) +} + +// createCallSpan creates a call span if tracing is enabled, which will be put +// in the context provided if created. +func (csh *clientStatsHandler) createCallSpan(ctx context.Context, method string) (context.Context, *trace.Span) { + var span *trace.Span + if !csh.to.DisableTrace { + mn := strings.ReplaceAll(removeLeadingSlash(method), "/", ".") + ctx, span = trace.StartSpan(ctx, mn, trace.WithSampler(csh.to.TS), trace.WithSpanKind(trace.SpanKindClient)) + } + return ctx, span +} + +// perCallTracesAndMetrics records per call spans and metrics. +func perCallTracesAndMetrics(err error, span *trace.Span, startTime time.Time, method string) { + s := status.Convert(err) + if span != nil { + span.SetStatus(trace.Status{Code: int32(s.Code()), Message: s.Message()}) + span.End() + } + callLatency := float64(time.Since(startTime)) / float64(time.Millisecond) + ocstats.RecordWithOptions(context.Background(), + ocstats.WithTags( + tag.Upsert(keyClientMethod, removeLeadingSlash(method)), + tag.Upsert(keyClientStatus, canonicalString(s.Code())), + ), + ocstats.WithMeasurements( + clientAPILatency.M(callLatency), + ), + ) +} + +// unaryInterceptor handles per RPC context management. It also handles per RPC +// tracing and stats by creating a top level call span and recording the latency +// for the full RPC call. +func (csh *clientStatsHandler) unaryInterceptor(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + startTime := time.Now() + ctx, span := csh.createCallSpan(ctx, method) + err := invoker(ctx, method, req, reply, cc, opts...) + perCallTracesAndMetrics(err, span, startTime, method) + return err +} + +// streamInterceptor handles per RPC context management. It also handles per RPC +// tracing and stats by creating a top level call span and recording the latency +// for the full RPC call. +func (csh *clientStatsHandler) streamInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + startTime := time.Now() + ctx, span := csh.createCallSpan(ctx, method) + callback := func(err error) { + perCallTracesAndMetrics(err, span, startTime, method) + } + opts = append([]grpc.CallOption{grpc.OnFinish(callback)}, opts...) + s, err := streamer(ctx, desc, cc, method, opts...) + if err != nil { + return nil, err + } + return s, nil +} + +type rpcInfo struct { + mi *metricsInfo + ti *traceInfo +} + +type rpcInfoKey struct{} + +func setRPCInfo(ctx context.Context, ri *rpcInfo) context.Context { + return context.WithValue(ctx, rpcInfoKey{}, ri) +} + +// getRPCInfo returns the rpcInfo stored in the context, or nil +// if there isn't one. +func getRPCInfo(ctx context.Context) *rpcInfo { + ri, _ := ctx.Value(rpcInfoKey{}).(*rpcInfo) + return ri +} + +// SpanContextFromContext returns the Span Context about the Span in the +// context. Returns false if no Span in the context. +func SpanContextFromContext(ctx context.Context) (trace.SpanContext, bool) { + ri, ok := ctx.Value(rpcInfoKey{}).(*rpcInfo) + if !ok { + return trace.SpanContext{}, false + } + if ri.ti == nil || ri.ti.span == nil { + return trace.SpanContext{}, false + } + sc := ri.ti.span.SpanContext() + return sc, true +} + +type clientStatsHandler struct { + to TraceOptions +} + +// TagConn exists to satisfy stats.Handler. +func (csh *clientStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +// HandleConn exists to satisfy stats.Handler. +func (csh *clientStatsHandler) HandleConn(context.Context, stats.ConnStats) {} + +// TagRPC implements per RPC attempt context management. +func (csh *clientStatsHandler) TagRPC(ctx context.Context, rti *stats.RPCTagInfo) context.Context { + ctx, mi := csh.statsTagRPC(ctx, rti) + var ti *traceInfo + if !csh.to.DisableTrace { + ctx, ti = csh.traceTagRPC(ctx, rti) + } + ri := &rpcInfo{ + mi: mi, + ti: ti, + } + return setRPCInfo(ctx, ri) +} + +func (csh *clientStatsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { + ri := getRPCInfo(ctx) + if ri == nil { + // Shouldn't happen because TagRPC populates this information. + return + } + recordRPCData(ctx, rs, ri.mi) + if !csh.to.DisableTrace { + populateSpan(ctx, rs, ri.ti) + } +} + +type serverStatsHandler struct { + to TraceOptions +} + +// TagConn exists to satisfy stats.Handler. +func (ssh *serverStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +// HandleConn exists to satisfy stats.Handler. +func (ssh *serverStatsHandler) HandleConn(context.Context, stats.ConnStats) {} + +// TagRPC implements per RPC context management. +func (ssh *serverStatsHandler) TagRPC(ctx context.Context, rti *stats.RPCTagInfo) context.Context { + ctx, mi := ssh.statsTagRPC(ctx, rti) + var ti *traceInfo + if !ssh.to.DisableTrace { + ctx, ti = ssh.traceTagRPC(ctx, rti) + } + ri := &rpcInfo{ + mi: mi, + ti: ti, + } + return setRPCInfo(ctx, ri) +} + +// HandleRPC implements per RPC tracing and stats implementation. +func (ssh *serverStatsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { + ri := getRPCInfo(ctx) + if ri == nil { + // Shouldn't happen because TagRPC populates this information. + return + } + recordRPCData(ctx, rs, ri.mi) + if !ssh.to.DisableTrace { + populateSpan(ctx, rs, ri.ti) + } +} diff --git a/stats/opencensus/server_metrics.go b/stats/opencensus/server_metrics.go new file mode 100644 index 0000000..4f087c2 --- /dev/null +++ b/stats/opencensus/server_metrics.go @@ -0,0 +1,135 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opencensus + +import ( + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" +) + +var ( + keyServerMethod = tag.MustNewKey("grpc_server_method") + keyServerStatus = tag.MustNewKey("grpc_server_status") +) + +// Measures, which are recorded by server stats handler: Note that on gRPC's +// server side, the per rpc unit is truly per rpc, as there is no concept of a +// rpc attempt server side. +var ( + serverReceivedMessagesPerRPC = stats.Int64("grpc.io/server/received_messages_per_rpc", "Number of messages received in each RPC. Has value 1 for non-streaming RPCs.", stats.UnitDimensionless) // the collection/measurement point of this measure handles the /rpc aspect of it + serverReceivedBytesPerRPC = stats.Int64("grpc.io/server/received_bytes_per_rpc", "Total bytes received across all messages per RPC.", stats.UnitBytes) + serverReceivedCompressedBytesPerRPC = stats.Int64("grpc.io/server/received_compressed_bytes_per_rpc", "Total compressed bytes received across all messages per RPC.", stats.UnitBytes) + serverSentMessagesPerRPC = stats.Int64("grpc.io/server/sent_messages_per_rpc", "Number of messages sent in each RPC. Has value 1 for non-streaming RPCs.", stats.UnitDimensionless) + serverSentBytesPerRPC = stats.Int64("grpc.io/server/sent_bytes_per_rpc", "Total bytes sent in across all response messages per RPC.", stats.UnitBytes) + serverSentCompressedBytesPerRPC = stats.Int64("grpc.io/server/sent_compressed_bytes_per_rpc", "Total compressed bytes sent in across all response messages per RPC.", stats.UnitBytes) + serverStartedRPCs = stats.Int64("grpc.io/server/started_rpcs", "The total number of server RPCs ever opened, including those that have not completed.", stats.UnitDimensionless) + serverLatency = stats.Float64("grpc.io/server/server_latency", "Time between first byte of request received to last byte of response sent, or terminal error.", stats.UnitMilliseconds) +) + +var ( + // ServerSentMessagesPerRPCView is the distribution of sent messages per + // RPC, keyed on method. + ServerSentMessagesPerRPCView = &view.View{ + Name: "grpc.io/server/sent_messages_per_rpc", + Description: "Distribution of sent messages per RPC, by method.", + TagKeys: []tag.Key{keyServerMethod}, + Measure: serverSentMessagesPerRPC, + Aggregation: countDistribution, + } + // ServerReceivedMessagesPerRPCView is the distribution of received messages + // per RPC, keyed on method. + ServerReceivedMessagesPerRPCView = &view.View{ + Name: "grpc.io/server/received_messages_per_rpc", + Description: "Distribution of received messages per RPC, by method.", + TagKeys: []tag.Key{keyServerMethod}, + Measure: serverReceivedMessagesPerRPC, + Aggregation: countDistribution, + } + // ServerSentBytesPerRPCView is the distribution of received bytes per RPC, + // keyed on method. + ServerSentBytesPerRPCView = &view.View{ + Name: "grpc.io/server/sent_bytes_per_rpc", + Description: "Distribution of sent bytes per RPC, by method.", + Measure: serverSentBytesPerRPC, + TagKeys: []tag.Key{keyServerMethod}, + Aggregation: bytesDistribution, + } + // ServerSentCompressedMessageBytesPerRPCView is the distribution of + // received compressed message bytes per RPC, keyed on method. + ServerSentCompressedMessageBytesPerRPCView = &view.View{ + Name: "grpc.io/server/sent_compressed_message_bytes_per_rpc", + Description: "Distribution of sent compressed message bytes per RPC, by method.", + Measure: serverSentCompressedBytesPerRPC, + TagKeys: []tag.Key{keyServerMethod}, + Aggregation: bytesDistribution, + } + // ServerReceivedBytesPerRPCView is the distribution of sent bytes per RPC, + // keyed on method. + ServerReceivedBytesPerRPCView = &view.View{ + Name: "grpc.io/server/received_bytes_per_rpc", + Description: "Distribution of received bytes per RPC, by method.", + Measure: serverReceivedBytesPerRPC, + TagKeys: []tag.Key{keyServerMethod}, + Aggregation: bytesDistribution, + } + // ServerReceivedCompressedMessageBytesPerRPCView is the distribution of + // sent compressed message bytes per RPC, keyed on method. + ServerReceivedCompressedMessageBytesPerRPCView = &view.View{ + Name: "grpc.io/server/received_compressed_message_bytes_per_rpc", + Description: "Distribution of received compressed message bytes per RPC, by method.", + Measure: serverReceivedCompressedBytesPerRPC, + TagKeys: []tag.Key{keyServerMethod}, + Aggregation: bytesDistribution, + } + // ServerStartedRPCsView is the count of opened RPCs, keyed on method. + ServerStartedRPCsView = &view.View{ + Measure: serverStartedRPCs, + Name: "grpc.io/server/started_rpcs", + Description: "Number of opened server RPCs, by method.", + TagKeys: []tag.Key{keyServerMethod}, + Aggregation: view.Count(), + } + // ServerCompletedRPCsView is the count of completed RPCs, keyed on + // method and status. + ServerCompletedRPCsView = &view.View{ + Name: "grpc.io/server/completed_rpcs", + Description: "Number of completed RPCs by method and status.", + TagKeys: []tag.Key{keyServerMethod, keyServerStatus}, + Measure: serverLatency, + Aggregation: view.Count(), + } + // ServerLatencyView is the distribution of server latency in milliseconds + // per RPC, keyed on method. + ServerLatencyView = &view.View{ + Name: "grpc.io/server/server_latency", + Description: "Distribution of server latency in milliseconds, by method.", + TagKeys: []tag.Key{keyServerMethod}, + Measure: serverLatency, + Aggregation: millisecondsDistribution, + } +) + +// DefaultServerViews is the set of server views which are considered the +// minimum required to monitor server side performance. +var DefaultServerViews = []*view.View{ + ServerReceivedBytesPerRPCView, + ServerSentBytesPerRPCView, + ServerLatencyView, + ServerCompletedRPCsView, + ServerStartedRPCsView, +} diff --git a/stats/opencensus/stats.go b/stats/opencensus/stats.go new file mode 100644 index 0000000..fecf55a --- /dev/null +++ b/stats/opencensus/stats.go @@ -0,0 +1,226 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opencensus + +import ( + "context" + "strings" + "sync/atomic" + "time" + + ocstats "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +var logger = grpclog.Component("opencensus-instrumentation") + +var canonicalString = internal.CanonicalString.(func(codes.Code) string) + +var ( + // bounds separate variable for testing purposes. + bytesDistributionBounds = []float64{1024, 2048, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864, 268435456, 1073741824, 4294967296} + bytesDistribution = view.Distribution(bytesDistributionBounds...) + millisecondsDistribution = view.Distribution(0.01, 0.05, 0.1, 0.3, 0.6, 0.8, 1, 2, 3, 4, 5, 6, 8, 10, 13, 16, 20, 25, 30, 40, 50, 65, 80, 100, 130, 160, 200, 250, 300, 400, 500, 650, 800, 1000, 2000, 5000, 10000, 20000, 50000, 100000) + countDistributionBounds = []float64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536} + countDistribution = view.Distribution(countDistributionBounds...) +) + +func removeLeadingSlash(mn string) string { + return strings.TrimLeft(mn, "/") +} + +// metricsInfo is data used for recording metrics about the rpc attempt client +// side, and the overall rpc server side. +type metricsInfo struct { + // access these counts atomically for hedging in the future + // number of messages sent from side (client || server) + sentMsgs int64 + // number of bytes sent (within each message) from side (client || server) + sentBytes int64 + // number of bytes after compression (within each message) from side (client || server) + sentCompressedBytes int64 + // number of messages received on side (client || server) + recvMsgs int64 + // number of bytes received (within each message) received on side (client + // || server) + recvBytes int64 + // number of compressed bytes received (within each message) received on + // side (client || server) + recvCompressedBytes int64 + + startTime time.Time + method string +} + +// statsTagRPC creates a recording object to derive measurements from in the +// context, scoping the recordings to per RPC Attempt client side (scope of the +// context). It also populates the gRPC Metadata within the context with any +// opencensus specific tags set by the application in the context, binary +// encoded to send across the wire. +func (csh *clientStatsHandler) statsTagRPC(ctx context.Context, info *stats.RPCTagInfo) (context.Context, *metricsInfo) { + mi := &metricsInfo{ + startTime: time.Now(), + method: info.FullMethodName, + } + + // Populate gRPC Metadata with OpenCensus tag map if set by application. + if tm := tag.FromContext(ctx); tm != nil { + ctx = metadata.AppendToOutgoingContext(ctx, "grpc-tags-bin", string(tag.Encode(tm))) + } + return ctx, mi +} + +// statsTagRPC creates a recording object to derive measurements from in the +// context, scoping the recordings to per RPC server side (scope of the +// context). It also deserializes the opencensus tags set in the context's gRPC +// Metadata, and adds a server method tag to the opencensus tags. If multiple +// tags exist, it adds the last one. +func (ssh *serverStatsHandler) statsTagRPC(ctx context.Context, info *stats.RPCTagInfo) (context.Context, *metricsInfo) { + mi := &metricsInfo{ + startTime: time.Now(), + method: info.FullMethodName, + } + + if tgValues := metadata.ValueFromIncomingContext(ctx, "grpc-tags-bin"); len(tgValues) > 0 { + tagsBin := []byte(tgValues[len(tgValues)-1]) + if tags, err := tag.Decode(tagsBin); err == nil { + ctx = tag.NewContext(ctx, tags) + } + } + + // We can ignore the error here because in the error case, the context + // passed in is returned. If the call errors, the server side application + // layer won't get this key server method information in the tag map, but + // this instrumentation code will function as normal. + ctx, _ = tag.New(ctx, tag.Upsert(keyServerMethod, removeLeadingSlash(info.FullMethodName))) + return ctx, mi +} + +func recordRPCData(ctx context.Context, s stats.RPCStats, mi *metricsInfo) { + if mi == nil { + // Shouldn't happen, as gRPC calls TagRPC which populates the metricsInfo in + // context. + logger.Error("ctx passed into stats handler metrics event handling has no metrics data present") + return + } + switch st := s.(type) { + case *stats.InHeader, *stats.OutHeader, *stats.InTrailer, *stats.OutTrailer, *stats.PickerUpdated: + // Headers, Trailers, and picker updates are not relevant to the measures, + // as the measures concern number of messages and bytes for messages. This + // aligns with flow control. + case *stats.Begin: + recordDataBegin(ctx, mi, st) + case *stats.OutPayload: + recordDataOutPayload(mi, st) + case *stats.InPayload: + recordDataInPayload(mi, st) + case *stats.End: + recordDataEnd(ctx, mi, st) + default: + // Shouldn't happen. gRPC calls into stats handler, and will never not + // be one of the types above. + logger.Errorf("Received unexpected stats type (%T) with data: %v", s, s) + } +} + +// recordDataBegin takes a measurement related to the RPC beginning, +// client/server started RPCs dependent on the caller. +func recordDataBegin(ctx context.Context, mi *metricsInfo, b *stats.Begin) { + if b.Client { + ocstats.RecordWithOptions(ctx, + ocstats.WithTags(tag.Upsert(keyClientMethod, removeLeadingSlash(mi.method))), + ocstats.WithMeasurements(clientStartedRPCs.M(1))) + return + } + ocstats.RecordWithOptions(ctx, + ocstats.WithTags(tag.Upsert(keyServerMethod, removeLeadingSlash(mi.method))), + ocstats.WithMeasurements(serverStartedRPCs.M(1))) +} + +// recordDataOutPayload records the length in bytes of outgoing messages and +// increases total count of sent messages both stored in the RPCs (attempt on +// client side) context for use in taking measurements at RPC end. +func recordDataOutPayload(mi *metricsInfo, op *stats.OutPayload) { + atomic.AddInt64(&mi.sentMsgs, 1) + atomic.AddInt64(&mi.sentBytes, int64(op.Length)) + atomic.AddInt64(&mi.sentCompressedBytes, int64(op.CompressedLength)) +} + +// recordDataInPayload records the length in bytes of incoming messages and +// increases total count of sent messages both stored in the RPCs (attempt on +// client side) context for use in taking measurements at RPC end. +func recordDataInPayload(mi *metricsInfo, ip *stats.InPayload) { + atomic.AddInt64(&mi.recvMsgs, 1) + atomic.AddInt64(&mi.recvBytes, int64(ip.Length)) + atomic.AddInt64(&mi.recvCompressedBytes, int64(ip.CompressedLength)) +} + +// recordDataEnd takes per RPC measurements derived from information derived +// from the lifetime of the RPC (RPC attempt client side). +func recordDataEnd(ctx context.Context, mi *metricsInfo, e *stats.End) { + // latency bounds for distribution data (speced millisecond bounds) have + // fractions, thus need a float. + latency := float64(time.Since(mi.startTime)) / float64(time.Millisecond) + var st string + if e.Error != nil { + s, _ := status.FromError(e.Error) + st = canonicalString(s.Code()) + } else { + st = "OK" + } + + // TODO: Attach trace data through attachments?!?! + + if e.Client { + ocstats.RecordWithOptions(ctx, + ocstats.WithTags( + tag.Upsert(keyClientMethod, removeLeadingSlash(mi.method)), + tag.Upsert(keyClientStatus, st)), + ocstats.WithMeasurements( + clientSentBytesPerRPC.M(atomic.LoadInt64(&mi.sentBytes)), + clientSentCompressedBytesPerRPC.M(atomic.LoadInt64(&mi.sentCompressedBytes)), + clientSentMessagesPerRPC.M(atomic.LoadInt64(&mi.sentMsgs)), + clientReceivedMessagesPerRPC.M(atomic.LoadInt64(&mi.recvMsgs)), + clientReceivedBytesPerRPC.M(atomic.LoadInt64(&mi.recvBytes)), + clientReceivedCompressedBytesPerRPC.M(atomic.LoadInt64(&mi.recvCompressedBytes)), + clientRoundtripLatency.M(latency), + clientServerLatency.M(latency), + )) + return + } + ocstats.RecordWithOptions(ctx, + ocstats.WithTags( + tag.Upsert(keyServerMethod, removeLeadingSlash(mi.method)), + tag.Upsert(keyServerStatus, st), + ), + ocstats.WithMeasurements( + serverSentBytesPerRPC.M(atomic.LoadInt64(&mi.sentBytes)), + serverSentCompressedBytesPerRPC.M(atomic.LoadInt64(&mi.sentCompressedBytes)), + serverSentMessagesPerRPC.M(atomic.LoadInt64(&mi.sentMsgs)), + serverReceivedMessagesPerRPC.M(atomic.LoadInt64(&mi.recvMsgs)), + serverReceivedBytesPerRPC.M(atomic.LoadInt64(&mi.recvBytes)), + serverReceivedCompressedBytesPerRPC.M(atomic.LoadInt64(&mi.recvCompressedBytes)), + serverLatency.M(latency))) +} diff --git a/stats/opencensus/trace.go b/stats/opencensus/trace.go new file mode 100644 index 0000000..55a058e --- /dev/null +++ b/stats/opencensus/trace.go @@ -0,0 +1,130 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opencensus + +import ( + "context" + "strings" + "sync/atomic" + + "go.opencensus.io/trace" + "go.opencensus.io/trace/propagation" + + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +// traceInfo is data used for recording traces. +type traceInfo struct { + span *trace.Span + countSentMsg uint32 + countRecvMsg uint32 +} + +// traceTagRPC populates context with a new span, and serializes information +// about this span into gRPC Metadata. +func (csh *clientStatsHandler) traceTagRPC(ctx context.Context, rti *stats.RPCTagInfo) (context.Context, *traceInfo) { + // TODO: get consensus on whether this method name of "s.m" is correct. + mn := "Attempt." + strings.ReplaceAll(removeLeadingSlash(rti.FullMethodName), "/", ".") + // Returned context is ignored because will populate context with data that + // wraps the span instead. Don't set span kind client on this attempt span + // to prevent backend from prepending span name with "Sent.". + _, span := trace.StartSpan(ctx, mn, trace.WithSampler(csh.to.TS)) + + tcBin := propagation.Binary(span.SpanContext()) + return metadata.AppendToOutgoingContext(ctx, "grpc-trace-bin", string(tcBin)), &traceInfo{ + span: span, + countSentMsg: 0, // msg events scoped to scope of context, per attempt client side + countRecvMsg: 0, + } +} + +// traceTagRPC populates context with new span data, with a parent based on the +// spanContext deserialized from context passed in (wire data in gRPC metadata) +// if present. If multiple spanContexts exist, it takes the last one. +func (ssh *serverStatsHandler) traceTagRPC(ctx context.Context, rti *stats.RPCTagInfo) (context.Context, *traceInfo) { + mn := strings.ReplaceAll(removeLeadingSlash(rti.FullMethodName), "/", ".") + + var tcBin []byte + if tcValues := metadata.ValueFromIncomingContext(ctx, "grpc-trace-bin"); len(tcValues) > 0 { + tcBin = []byte(tcValues[len(tcValues)-1]) + } + + var span *trace.Span + if sc, ok := propagation.FromBinary(tcBin); ok { + // Returned context is ignored because will populate context with data + // that wraps the span instead. + _, span = trace.StartSpanWithRemoteParent(ctx, mn, sc, trace.WithSpanKind(trace.SpanKindServer), trace.WithSampler(ssh.to.TS)) + span.AddLink(trace.Link{TraceID: sc.TraceID, SpanID: sc.SpanID, Type: trace.LinkTypeChild}) + } else { + // Returned context is ignored because will populate context with data + // that wraps the span instead. + _, span = trace.StartSpan(ctx, mn, trace.WithSpanKind(trace.SpanKindServer), trace.WithSampler(ssh.to.TS)) + } + + return ctx, &traceInfo{ + span: span, + countSentMsg: 0, + countRecvMsg: 0, + } +} + +// populateSpan populates span information based on stats passed in (invariants +// of the RPC lifecycle), and also ends span which triggers the span to be +// exported. +func populateSpan(_ context.Context, rs stats.RPCStats, ti *traceInfo) { + if ti == nil || ti.span == nil { + // Shouldn't happen, tagRPC call comes before this function gets called + // which populates this information. + logger.Error("ctx passed into stats handler tracing event handling has no span present") + return + } + span := ti.span + + switch rs := rs.(type) { + case *stats.Begin: + // Note: Go always added these attributes even though they are not + // defined by the OpenCensus gRPC spec. Thus, they are unimportant for + // correctness. + span.AddAttributes( + trace.BoolAttribute("Client", rs.Client), + trace.BoolAttribute("FailFast", rs.FailFast), + ) + case *stats.PickerUpdated: + span.Annotate(nil, "Delayed LB pick complete") + case *stats.InPayload: + // message id - "must be calculated as two different counters starting + // from one for sent messages and one for received messages." + mi := atomic.AddUint32(&ti.countRecvMsg, 1) + span.AddMessageReceiveEvent(int64(mi), int64(rs.Length), int64(rs.CompressedLength)) + case *stats.OutPayload: + mi := atomic.AddUint32(&ti.countSentMsg, 1) + span.AddMessageSendEvent(int64(mi), int64(rs.Length), int64(rs.CompressedLength)) + case *stats.End: + if rs.Error != nil { + // "The mapping between gRPC canonical codes and OpenCensus codes + // can be found here", which implies 1:1 mapping to gRPC statuses + // (OpenCensus statuses are based off gRPC statuses and a subset). + s := status.Convert(rs.Error) + span.SetStatus(trace.Status{Code: int32(s.Code()), Message: s.Message()}) + } else { + span.SetStatus(trace.Status{Code: trace.StatusCodeOK}) // could get rid of this else conditional and just leave as 0 value, but this makes it explicit + } + span.End() + } +} diff --git a/stats/opentelemetry/client_metrics.go b/stats/opentelemetry/client_metrics.go new file mode 100644 index 0000000..3ec48e0 --- /dev/null +++ b/stats/opentelemetry/client_metrics.go @@ -0,0 +1,287 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opentelemetry + +import ( + "context" + "maps" + "sync/atomic" + "time" + + otelattribute "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/metric" + "google.golang.org/grpc" + estats "google.golang.org/grpc/experimental/stats" + istats "google.golang.org/grpc/internal/stats" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +type clientMetricsHandler struct { + estats.MetricsRecorder + options Options + clientMetrics clientMetrics +} + +func (h *clientMetricsHandler) initializeMetrics() { + // Will set no metrics to record, logically making this stats handler a + // no-op. + if h.options.MetricsOptions.MeterProvider == nil { + return + } + + meter := h.options.MetricsOptions.MeterProvider.Meter("grpc-go", otelmetric.WithInstrumentationVersion(grpc.Version)) + if meter == nil { + return + } + + metrics := h.options.MetricsOptions.Metrics + if metrics == nil { + metrics = DefaultMetrics() + } + + h.clientMetrics.attemptStarted = createInt64Counter(metrics.Metrics(), "grpc.client.attempt.started", meter, otelmetric.WithUnit("{attempt}"), otelmetric.WithDescription("Number of client call attempts started.")) + h.clientMetrics.attemptDuration = createFloat64Histogram(metrics.Metrics(), "grpc.client.attempt.duration", meter, otelmetric.WithUnit("s"), otelmetric.WithDescription("End-to-end time taken to complete a client call attempt."), otelmetric.WithExplicitBucketBoundaries(DefaultLatencyBounds...)) + h.clientMetrics.attemptSentTotalCompressedMessageSize = createInt64Histogram(metrics.Metrics(), "grpc.client.attempt.sent_total_compressed_message_size", meter, otelmetric.WithUnit("By"), otelmetric.WithDescription("Compressed message bytes sent per client call attempt."), otelmetric.WithExplicitBucketBoundaries(DefaultSizeBounds...)) + h.clientMetrics.attemptRcvdTotalCompressedMessageSize = createInt64Histogram(metrics.Metrics(), "grpc.client.attempt.rcvd_total_compressed_message_size", meter, otelmetric.WithUnit("By"), otelmetric.WithDescription("Compressed message bytes received per call attempt."), otelmetric.WithExplicitBucketBoundaries(DefaultSizeBounds...)) + h.clientMetrics.callDuration = createFloat64Histogram(metrics.Metrics(), "grpc.client.call.duration", meter, otelmetric.WithUnit("s"), otelmetric.WithDescription("Time taken by gRPC to complete an RPC from application's perspective."), otelmetric.WithExplicitBucketBoundaries(DefaultLatencyBounds...)) + + rm := ®istryMetrics{ + optionalLabels: h.options.MetricsOptions.OptionalLabels, + } + h.MetricsRecorder = rm + rm.registerMetrics(metrics, meter) +} + +// getOrCreateCallInfo returns the existing callInfo from context if present, +// or creates and attaches a new one. +func getOrCreateCallInfo(ctx context.Context, cc *grpc.ClientConn, method string, opts ...grpc.CallOption) (context.Context, *callInfo) { + ci := getCallInfo(ctx) + if ci == nil { + ci = &callInfo{ + target: cc.CanonicalTarget(), + method: determineMethod(method, opts...), + } + ctx = context.WithValue(ctx, callInfoKey{}, ci) + } + return ctx, ci +} + +func (h *clientMetricsHandler) unaryInterceptor(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + ctx, ci := getOrCreateCallInfo(ctx, cc, method, opts...) + + if h.options.MetricsOptions.pluginOption != nil { + md := h.options.MetricsOptions.pluginOption.GetMetadata() + for k, vs := range md { + for _, v := range vs { + ctx = metadata.AppendToOutgoingContext(ctx, k, v) + } + } + } + + startTime := time.Now() + err := invoker(ctx, method, req, reply, cc, opts...) + h.perCallMetrics(ctx, err, startTime, ci) + return err +} + +// determineMethod determines the method to record attributes with. This will be +// "other" if StaticMethod isn't specified or if method filter is set and +// specifies, the method name as is otherwise. +func determineMethod(method string, opts ...grpc.CallOption) string { + for _, opt := range opts { + if _, ok := opt.(grpc.StaticMethodCallOption); ok { + return removeLeadingSlash(method) + } + } + return "other" +} + +func (h *clientMetricsHandler) streamInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + ctx, ci := getOrCreateCallInfo(ctx, cc, method, opts...) + + if h.options.MetricsOptions.pluginOption != nil { + md := h.options.MetricsOptions.pluginOption.GetMetadata() + for k, vs := range md { + for _, v := range vs { + ctx = metadata.AppendToOutgoingContext(ctx, k, v) + } + } + } + + startTime := time.Now() + callback := func(err error) { + h.perCallMetrics(ctx, err, startTime, ci) + } + opts = append([]grpc.CallOption{grpc.OnFinish(callback)}, opts...) + return streamer(ctx, desc, cc, method, opts...) +} + +// perCallMetrics records per call metrics for both unary and stream calls. +func (h *clientMetricsHandler) perCallMetrics(ctx context.Context, err error, startTime time.Time, ci *callInfo) { + callLatency := float64(time.Since(startTime)) / float64(time.Second) + attributes := []otelattribute.KeyValue{ + otelattribute.String("grpc.method", ci.method), + otelattribute.String("grpc.target", ci.target), + otelattribute.String("grpc.status", canonicalString(status.Code(err))), + } + for _, o := range h.options.MetricsOptions.OptionalLabels { + if o == "grpc.client.call.custom" { + label := estats.CustomLabelFromContext(ctx) + attributes = append(attributes, otelattribute.String(o, label)) + } + } + attrs := otelmetric.WithAttributeSet(otelattribute.NewSet(attributes...)) + h.clientMetrics.callDuration.Record(ctx, callLatency, attrs) +} + +// TagConn exists to satisfy stats.Handler. +func (h *clientMetricsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +// HandleConn exists to satisfy stats.Handler. +func (h *clientMetricsHandler) HandleConn(context.Context, stats.ConnStats) {} + +// TagRPC implements per RPC attempt context management for metrics. +func (h *clientMetricsHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + ctx, ri := getOrCreateClientRPCInfo(ctx, info) + // Numerous stats handlers can be used for the same channel. This callback + // ensures that all label updates are propagated to the rpc attempt info across + // derived contexts. + ctx = istats.RegisterTelemetryLabelCallback(ctx, func(labels map[string]string) { + maps.Copy(ri.ai.xdsLabels, labels) + }) + + return ctx +} + +// HandleRPC handles per RPC stats implementation. +func (h *clientMetricsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { + ri := clientRPCInfo(ctx) + if ri == nil { + logger.Error("ctx passed into client side stats handler metrics event handling has no client attempt data present") + return + } + h.processRPCEvent(ctx, rs, ri.ai) +} + +func (h *clientMetricsHandler) processRPCEvent(ctx context.Context, s stats.RPCStats, ai *attemptInfo) { + switch st := s.(type) { + case *stats.Begin: + ci := getCallInfo(ctx) + if ci == nil { + logger.Error("ctx passed into client side stats handler metrics event handling has no metrics data present") + return + } + + attributes := []otelattribute.KeyValue{ + otelattribute.String("grpc.method", ci.method), + otelattribute.String("grpc.target", ci.target), + } + for _, o := range h.options.MetricsOptions.OptionalLabels { + if o == "grpc.client.call.custom" { + label := estats.CustomLabelFromContext(ctx) + attributes = append(attributes, otelattribute.String(o, label)) + } + } + + attrs := otelmetric.WithAttributeSet(otelattribute.NewSet(attributes...)) + h.clientMetrics.attemptStarted.Add(ctx, 1, attrs) + case *stats.OutPayload: + atomic.AddInt64(&ai.sentCompressedBytes, int64(st.CompressedLength)) + case *stats.InPayload: + atomic.AddInt64(&ai.recvCompressedBytes, int64(st.CompressedLength)) + case *stats.InHeader: + h.setLabelsFromPluginOption(ai, st.Header) + case *stats.InTrailer: + h.setLabelsFromPluginOption(ai, st.Trailer) + case *stats.End: + h.processRPCEnd(ctx, ai, st) + default: + } +} + +func (h *clientMetricsHandler) setLabelsFromPluginOption(ai *attemptInfo, incomingMetadata metadata.MD) { + if ai.pluginOptionLabels == nil && h.options.MetricsOptions.pluginOption != nil { + labels := h.options.MetricsOptions.pluginOption.GetLabels(incomingMetadata) + if labels == nil { + labels = map[string]string{} // Shouldn't return a nil map. Make it empty if so to ignore future Get Calls for this Attempt. + } + ai.pluginOptionLabels = labels + } +} + +func (h *clientMetricsHandler) processRPCEnd(ctx context.Context, ai *attemptInfo, e *stats.End) { + ci := getCallInfo(ctx) + if ci == nil { + logger.Error("ctx passed into client side stats handler metrics event handling has no metrics data present") + return + } + latency := float64(time.Since(ai.startTime)) / float64(time.Second) + st := "OK" + if e.Error != nil { + s, _ := status.FromError(e.Error) + st = canonicalString(s.Code()) + } + + attributes := []otelattribute.KeyValue{ + otelattribute.String("grpc.method", ci.method), + otelattribute.String("grpc.target", ci.target), + otelattribute.String("grpc.status", st), + } + + for k, v := range ai.pluginOptionLabels { + attributes = append(attributes, otelattribute.String(k, v)) + } + + for _, o := range h.options.MetricsOptions.OptionalLabels { + // TODO: Add a filter for converting to unknown if not present in the + // CSM Plugin Option layer by adding an optional labels API. + if val, ok := ai.xdsLabels[o]; ok { + attributes = append(attributes, otelattribute.String(o, val)) + } else if o == "grpc.client.call.custom" { + label := estats.CustomLabelFromContext(ctx) + attributes = append(attributes, otelattribute.String(o, label)) + } + } + + // Allocate vararg slice once. + opts := []otelmetric.RecordOption{otelmetric.WithAttributeSet(otelattribute.NewSet(attributes...))} + h.clientMetrics.attemptDuration.Record(ctx, latency, opts...) + h.clientMetrics.attemptSentTotalCompressedMessageSize.Record(ctx, atomic.LoadInt64(&ai.sentCompressedBytes), opts...) + h.clientMetrics.attemptRcvdTotalCompressedMessageSize.Record(ctx, atomic.LoadInt64(&ai.recvCompressedBytes), opts...) +} + +const ( + // ClientAttemptStartedMetricName is the number of client call attempts + // started. + ClientAttemptStartedMetricName string = "grpc.client.attempt.started" + // ClientAttemptDurationMetricName is the end-to-end time taken to complete + // a client call attempt. + ClientAttemptDurationMetricName string = "grpc.client.attempt.duration" + // ClientAttemptSentCompressedTotalMessageSizeMetricName is the compressed + // message bytes sent per client call attempt. + ClientAttemptSentCompressedTotalMessageSizeMetricName string = "grpc.client.attempt.sent_total_compressed_message_size" + // ClientAttemptRcvdCompressedTotalMessageSizeMetricName is the compressed + // message bytes received per call attempt. + ClientAttemptRcvdCompressedTotalMessageSizeMetricName string = "grpc.client.attempt.rcvd_total_compressed_message_size" + // ClientCallDurationMetricName is the time taken by gRPC to complete an RPC + // from application's perspective. + ClientCallDurationMetricName string = "grpc.client.call.duration" +) diff --git a/stats/opentelemetry/client_tracing.go b/stats/opentelemetry/client_tracing.go new file mode 100644 index 0000000..64cda94 --- /dev/null +++ b/stats/opentelemetry/client_tracing.go @@ -0,0 +1,155 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opentelemetry + +import ( + "context" + "log" + "strings" + + "go.opentelemetry.io/otel/attribute" + otelcodes "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc" + grpccodes "google.golang.org/grpc/codes" + "google.golang.org/grpc/stats" + otelinternaltracing "google.golang.org/grpc/stats/opentelemetry/internal/tracing" + "google.golang.org/grpc/status" +) + +const ( + delayedResolutionEventName = "Delayed name resolution complete" + tracerName = "grpc-go" +) + +type clientTracingHandler struct { + options Options +} + +func (h *clientTracingHandler) initializeTraces() { + if h.options.TraceOptions.TracerProvider == nil { + log.Printf("TracerProvider is not provided in client TraceOptions") + return + } +} + +func (h *clientTracingHandler) unaryInterceptor(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + ctx, _ = getOrCreateCallInfo(ctx, cc, method, opts...) + + var span trace.Span + ctx, span = h.createCallTraceSpan(ctx, method) + err := invoker(ctx, method, req, reply, cc, opts...) + h.finishTrace(err, span) + return err +} + +func (h *clientTracingHandler) streamInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + ctx, _ = getOrCreateCallInfo(ctx, cc, method, opts...) + + var span trace.Span + ctx, span = h.createCallTraceSpan(ctx, method) + callback := func(err error) { h.finishTrace(err, span) } + opts = append([]grpc.CallOption{grpc.OnFinish(callback)}, opts...) + return streamer(ctx, desc, cc, method, opts...) +} + +// finishTrace sets the span status based on the RPC result and ends the span. +// It is used to finalize tracing for both unary and streaming calls. +func (h *clientTracingHandler) finishTrace(err error, ts trace.Span) { + s := status.Convert(err) + if s.Code() == grpccodes.OK { + ts.SetStatus(otelcodes.Ok, s.Message()) + } else { + ts.SetStatus(otelcodes.Error, s.Message()) + } + ts.End() +} + +// traceTagRPC populates provided context with a new span using the +// TextMapPropagator supplied in trace options and internal itracing.carrier. +// It creates a new outgoing carrier which serializes information about this +// span into gRPC Metadata, if TextMapPropagator is provided in the trace +// options. if TextMapPropagator is not provided, it returns the context as is. +// +// Note: The passed attemptInfo pointer (ai) is mutated in-place. Fields such as +// ai.traceSpan are updated directly. No new attemptInfo is returned. +func (h *clientTracingHandler) traceTagRPC(ctx context.Context, ai *attemptInfo, nameResolutionDelayed bool) context.Context { + // Add a "Delayed name resolution complete" event to the call span + // if there was name resolution delay. In case of multiple retry attempts, + // ensure that event is added only once. + callSpan := trace.SpanFromContext(ctx) + ci := getCallInfo(ctx) + if nameResolutionDelayed && !ci.nameResolutionEventAdded.Swap(true) && callSpan.SpanContext().IsValid() { + callSpan.AddEvent(delayedResolutionEventName) + } + mn := "Attempt." + strings.Replace(ai.method, "/", ".", -1) + tracer := h.options.TraceOptions.TracerProvider.Tracer(tracerName, trace.WithInstrumentationVersion(grpc.Version)) + ctx, span := tracer.Start(ctx, mn) + carrier := otelinternaltracing.NewOutgoingCarrier(ctx) + h.options.TraceOptions.TextMapPropagator.Inject(ctx, carrier) + ai.traceSpan = span + return carrier.Context() +} + +// createCallTraceSpan creates a call span to put in the provided context using +// provided TraceProvider. If TraceProvider is nil, it returns context as is. +func (h *clientTracingHandler) createCallTraceSpan(ctx context.Context, method string) (context.Context, trace.Span) { + mn := "Sent." + strings.Replace(removeLeadingSlash(method), "/", ".", -1) + tracer := h.options.TraceOptions.TracerProvider.Tracer(tracerName, trace.WithInstrumentationVersion(grpc.Version)) + ctx, span := tracer.Start(ctx, mn, trace.WithSpanKind(trace.SpanKindClient)) + return ctx, span +} + +// TagConn exists to satisfy stats.Handler for tracing. +func (h *clientTracingHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +// HandleConn exists to satisfy stats.Handler for tracing. +func (h *clientTracingHandler) HandleConn(context.Context, stats.ConnStats) {} + +// TagRPC implements per RPC attempt context management for traces. +func (h *clientTracingHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + ctx, ri := getOrCreateClientRPCInfo(ctx, info) + ci := getCallInfo(ctx) + if ci == nil { + logger.Error("context passed into client side stats handler (TagRPC) has no call info") + return ctx + } + ctx = h.traceTagRPC(ctx, ri.ai, info.NameResolutionDelay) + return ctx +} + +// HandleRPC handles per RPC tracing implementation. +func (h *clientTracingHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { + ri := clientRPCInfo(ctx) + if ri == nil { + logger.Error("ctx passed into client side tracing handler trace event handling has no client attempt data present") + return + } + + // Client-specific Begin attributes. + if begin, ok := rs.(*stats.Begin); ok { + ci := getCallInfo(ctx) + previousRPCAttempts := ci.previousRPCAttempts.Add(1) - 1 + ri.ai.traceSpan.SetAttributes( + attribute.Int64("previous-rpc-attempts", int64(previousRPCAttempts)), + attribute.Bool("transparent-retry", begin.IsTransparentRetryAttempt), + ) + } + populateSpan(rs, ri.ai) +} diff --git a/stats/opentelemetry/csm/observability.go b/stats/opentelemetry/csm/observability.go new file mode 100644 index 0000000..4db5d21 --- /dev/null +++ b/stats/opentelemetry/csm/observability.go @@ -0,0 +1,84 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package csm + +import ( + "context" + "net/url" + + "google.golang.org/grpc" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/stats/opentelemetry" + otelinternal "google.golang.org/grpc/stats/opentelemetry/internal" +) + +// EnableObservability sets up CSM Observability for the binary globally. +// +// The CSM Stats Plugin is instantiated with local labels and metadata exchange +// labels pulled from the environment, and emits metadata exchange labels from +// the peer and local labels. Context timeouts do not trigger an error, but set +// certain labels to "unknown". +// +// This function is not thread safe, and should only be invoked once in main +// before any channels or servers are created. Returns a cleanup function to be +// deferred in main. +func EnableObservability(ctx context.Context, options opentelemetry.Options) func() { + csmPluginOption := newPluginOption(ctx) + clientSideOTelWithCSM := dialOptionWithCSMPluginOption(options, csmPluginOption) + clientSideOTel := opentelemetry.DialOption(options) + internal.AddGlobalPerTargetDialOptions.(func(opt any))(&perTargetDialOption{ + clientSideOTelWithCSM: clientSideOTelWithCSM, + clientSideOTel: clientSideOTel, + }) + + serverSideOTelWithCSM := serverOptionWithCSMPluginOption(options, csmPluginOption) + internal.AddGlobalServerOptions.(func(opt ...grpc.ServerOption))(serverSideOTelWithCSM) + + return func() { + internal.ClearGlobalServerOptions() + internal.ClearGlobalPerTargetDialOptions() + } +} + +type perTargetDialOption struct { + clientSideOTelWithCSM grpc.DialOption + clientSideOTel grpc.DialOption +} + +func (o *perTargetDialOption) DialOptionForTarget(parsedTarget url.URL) grpc.DialOption { + if determineTargetCSM(&parsedTarget) { + return o.clientSideOTelWithCSM + } + return o.clientSideOTel +} + +func dialOptionWithCSMPluginOption(options opentelemetry.Options, po otelinternal.PluginOption) grpc.DialOption { + options.MetricsOptions.OptionalLabels = []string{"csm.service_name", "csm.service_namespace_name"} // Attach the two xDS Optional Labels for this component to not filter out. + return dialOptionSetCSM(options, po) +} + +func dialOptionSetCSM(options opentelemetry.Options, po otelinternal.PluginOption) grpc.DialOption { + otelinternal.SetPluginOption.(func(options *opentelemetry.Options, po otelinternal.PluginOption))(&options, po) + return opentelemetry.DialOption(options) +} + +func serverOptionWithCSMPluginOption(options opentelemetry.Options, po otelinternal.PluginOption) grpc.ServerOption { + otelinternal.SetPluginOption.(func(options *opentelemetry.Options, po otelinternal.PluginOption))(&options, po) + return opentelemetry.ServerOption(options) +} diff --git a/stats/opentelemetry/csm/observability_test.go b/stats/opentelemetry/csm/observability_test.go new file mode 100644 index 0000000..f68896c --- /dev/null +++ b/stats/opentelemetry/csm/observability_test.go @@ -0,0 +1,649 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package csm + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "testing" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/encoding/gzip" + estats "google.golang.org/grpc/experimental/stats" + istats "google.golang.org/grpc/internal/stats" + "google.golang.org/grpc/internal/stubserver" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats/opentelemetry" + itestutils "google.golang.org/grpc/stats/opentelemetry/internal/testutils" +) + +// setupEnv configures the environment for CSM Observability Testing. It sets +// the bootstrap env var to a bootstrap file with a nodeID provided. It sets CSM +// Env Vars as well, and mocks the resource detector's returned attribute set to +// simulate the environment. It registers a cleanup function on the provided t +// to restore the environment to its original state. +func setupEnv(t *testing.T, resourceDetectorEmissions map[string]string, meshID, csmCanonicalServiceName, csmWorkloadName string) { + oldCSMMeshID, csmMeshIDPresent := os.LookupEnv("CSM_MESH_ID") + oldCSMCanonicalServiceName, csmCanonicalServiceNamePresent := os.LookupEnv("CSM_CANONICAL_SERVICE_NAME") + oldCSMWorkloadName, csmWorkloadNamePresent := os.LookupEnv("CSM_WORKLOAD_NAME") + os.Setenv("CSM_MESH_ID", meshID) + os.Setenv("CSM_CANONICAL_SERVICE_NAME", csmCanonicalServiceName) + os.Setenv("CSM_WORKLOAD_NAME", csmWorkloadName) + + var attributes []attribute.KeyValue + for k, v := range resourceDetectorEmissions { + attributes = append(attributes, attribute.String(k, v)) + } + // Return the attributes configured as part of the test in place + // of reading from resource. + attrSet := attribute.NewSet(attributes...) + origGetAttrSet := getAttrSetFromResourceDetector + getAttrSetFromResourceDetector = func(context.Context) *attribute.Set { + return &attrSet + } + t.Cleanup(func() { + if csmMeshIDPresent { + os.Setenv("CSM_MESH_ID", oldCSMMeshID) + } else { + os.Unsetenv("CSM_MESH_ID") + } + if csmCanonicalServiceNamePresent { + os.Setenv("CSM_CANONICAL_SERVICE_NAME", oldCSMCanonicalServiceName) + } else { + os.Unsetenv("CSM_CANONICAL_SERVICE_NAME") + } + if csmWorkloadNamePresent { + os.Setenv("CSM_WORKLOAD_NAME", oldCSMWorkloadName) + } else { + os.Unsetenv("CSM_WORKLOAD_NAME") + } + + getAttrSetFromResourceDetector = origGetAttrSet + }) +} + +// TestCSMPluginOptionUnary tests the CSM Plugin Option and labels. It +// configures the environment for the CSM Plugin Option to read from. It then +// configures a system with a gRPC Client and gRPC server with the OpenTelemetry +// Dial and Server Option configured with a CSM Plugin Option with a certain +// unary handler set to induce different ways of setting metadata exchange +// labels, and makes a Unary RPC. This RPC should cause certain recording for +// each registered metric observed through a Manual Metrics Reader on the +// provided OpenTelemetry SDK's Meter Provider. The CSM Labels emitted from the +// plugin option should be attached to the relevant metrics. +func (s) TestCSMPluginOptionUnary(t *testing.T) { + resourceDetectorEmissions := map[string]string{ + "cloud.platform": "gcp_kubernetes_engine", + "cloud.region": "cloud_region_val", // availability_zone isn't present, so this should become location + "cloud.account.id": "cloud_account_id_val", + "k8s.namespace.name": "k8s_namespace_name_val", + "k8s.cluster.name": "k8s_cluster_name_val", + } + const meshID = "mesh_id" + const csmCanonicalServiceName = "csm_canonical_service_name" + const csmWorkloadName = "csm_workload_name" + setupEnv(t, resourceDetectorEmissions, meshID, csmCanonicalServiceName, csmWorkloadName) + + attributesWant := map[string]string{ + "csm.workload_canonical_service": csmCanonicalServiceName, // from env + "csm.mesh_id": "mesh_id", // from bootstrap env var + + // No xDS Labels - this happens in a test below. + + "csm.remote_workload_type": "gcp_kubernetes_engine", + "csm.remote_workload_canonical_service": csmCanonicalServiceName, + "csm.remote_workload_project_id": "cloud_account_id_val", + "csm.remote_workload_cluster_name": "k8s_cluster_name_val", + "csm.remote_workload_namespace_name": "k8s_namespace_name_val", + "csm.remote_workload_location": "cloud_region_val", + "csm.remote_workload_name": csmWorkloadName, + } + + var csmLabels []attribute.KeyValue + for k, v := range attributesWant { + csmLabels = append(csmLabels, attribute.String(k, v)) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + unaryCompressedSize := float64(itestutils.GzipCompressedMessageSize(t, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: make([]byte, 10000)}})) + tests := []struct { + name string + // To test the different operations for Unary RPC's from the interceptor + // level that can plumb metadata exchange header in. + unaryCallFunc func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) + opts itestutils.MetricDataOptions + }{ + { + name: "normal-flow", + unaryCallFunc: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{Payload: &testpb.Payload{ + Body: make([]byte, len(in.GetPayload().GetBody())), + }}, nil + }, + opts: itestutils.MetricDataOptions{ + CSMLabels: csmLabels, + UnaryCompressedMessageSize: unaryCompressedSize, + }, + }, + { + name: "trailers-only", + unaryCallFunc: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return nil, errors.New("some error") // return an error and no message - this triggers trailers only - no messages or headers sent + }, + opts: itestutils.MetricDataOptions{ + CSMLabels: csmLabels, + UnaryCallFailed: true, + }, + }, + { + name: "set-header", + unaryCallFunc: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + grpc.SetHeader(ctx, metadata.New(map[string]string{"some-metadata": "some-metadata-val"})) + + return &testpb.SimpleResponse{Payload: &testpb.Payload{ + Body: make([]byte, len(in.GetPayload().GetBody())), + }}, nil + }, + opts: itestutils.MetricDataOptions{ + CSMLabels: csmLabels, + UnaryCompressedMessageSize: unaryCompressedSize, + }, + }, + { + name: "send-header", + unaryCallFunc: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + grpc.SendHeader(ctx, metadata.New(map[string]string{"some-metadata": "some-metadata-val"})) + + return &testpb.SimpleResponse{Payload: &testpb.Payload{ + Body: make([]byte, len(in.GetPayload().GetBody())), + }}, nil + }, + opts: itestutils.MetricDataOptions{ + CSMLabels: csmLabels, + UnaryCompressedMessageSize: unaryCompressedSize, + }, + }, + { + name: "send-msg", + unaryCallFunc: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{Payload: &testpb.Payload{ + Body: make([]byte, len(in.GetPayload().GetBody())), + }}, nil + }, + opts: itestutils.MetricDataOptions{ + CSMLabels: csmLabels, + UnaryCompressedMessageSize: unaryCompressedSize, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + ss := &stubserver.StubServer{UnaryCallF: test.unaryCallFunc} + po := newPluginOption(ctx) + sopts := []grpc.ServerOption{ + serverOptionWithCSMPluginOption(opentelemetry.Options{ + MetricsOptions: opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics(), + }}, po), + } + dopts := []grpc.DialOption{dialOptionWithCSMPluginOption(opentelemetry.Options{ + MetricsOptions: opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics(), + OptionalLabels: []string{"csm.service_name", "csm.service_namespace_name"}, + }, + }, po)} + if err := ss.Start(sopts, dopts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + var request *testpb.SimpleRequest + if test.opts.UnaryCompressedMessageSize != 0 { + request = &testpb.SimpleRequest{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }} + } + // Make a Unary RPC. These should cause certain metrics to be + // emitted, which should be able to be observed through the Metric + // Reader. + ss.Client.UnaryCall(ctx, request, grpc.UseCompressor(gzip.Name)) + rm := &metricdata.ResourceMetrics{} + reader.Collect(ctx, rm) + + gotMetrics := map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + gotMetrics[m.Name] = m + } + } + + opts := test.opts + opts.Target = ss.Target + wantMetrics := itestutils.MetricDataUnary(opts) + gotMetrics = itestutils.WaitForServerMetrics(ctx, t, reader, gotMetrics, wantMetrics) + itestutils.CompareMetrics(t, gotMetrics, wantMetrics) + }) + } +} + +// TestCSMPluginOptionStreaming tests the CSM Plugin Option and labels. It +// configures the environment for the CSM Plugin Option to read from. It then +// configures a system with a gRPC Client and gRPC server with the OpenTelemetry +// Dial and Server Option configured with a CSM Plugin Option with a certain +// streaming handler set to induce different ways of setting metadata exchange +// labels, and makes a Streaming RPC. This RPC should cause certain recording +// for each registered metric observed through a Manual Metrics Reader on the +// provided OpenTelemetry SDK's Meter Provider. The CSM Labels emitted from the +// plugin option should be attached to the relevant metrics. +func (s) TestCSMPluginOptionStreaming(t *testing.T) { + resourceDetectorEmissions := map[string]string{ + "cloud.platform": "gcp_kubernetes_engine", + "cloud.region": "cloud_region_val", // availability_zone isn't present, so this should become location + "cloud.account.id": "cloud_account_id_val", + "k8s.namespace.name": "k8s_namespace_name_val", + "k8s.cluster.name": "k8s_cluster_name_val", + } + const meshID = "mesh_id" + const csmCanonicalServiceName = "csm_canonical_service_name" + const csmWorkloadName = "csm_workload_name" + setupEnv(t, resourceDetectorEmissions, meshID, csmCanonicalServiceName, csmWorkloadName) + + attributesWant := map[string]string{ + "csm.workload_canonical_service": csmCanonicalServiceName, // from env + "csm.mesh_id": "mesh_id", // from bootstrap env var + + // No xDS Labels - this happens in a test below. + + "csm.remote_workload_type": "gcp_kubernetes_engine", + "csm.remote_workload_canonical_service": csmCanonicalServiceName, + "csm.remote_workload_project_id": "cloud_account_id_val", + "csm.remote_workload_cluster_name": "k8s_cluster_name_val", + "csm.remote_workload_namespace_name": "k8s_namespace_name_val", + "csm.remote_workload_location": "cloud_region_val", + "csm.remote_workload_name": csmWorkloadName, + } + + var csmLabels []attribute.KeyValue + for k, v := range attributesWant { + csmLabels = append(csmLabels, attribute.String(k, v)) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + streamingCompressedSize := float64(itestutils.GzipCompressedMessageSize(t, &testpb.StreamingOutputCallRequest{Payload: &testpb.Payload{Body: make([]byte, 10000)}})) + tests := []struct { + name string + // To test the different operations for Streaming RPC's from the + // interceptor level that can plumb metadata exchange header in. + streamingCallFunc func(stream testgrpc.TestService_FullDuplexCallServer) error + opts itestutils.MetricDataOptions + }{ + { + name: "trailers-only", + streamingCallFunc: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + if _, err := stream.Recv(); err == io.EOF { + return nil + } + } + }, + opts: itestutils.MetricDataOptions{ + CSMLabels: csmLabels, + }, + }, + { + name: "set-header", + streamingCallFunc: func(stream testgrpc.TestService_FullDuplexCallServer) error { + stream.SetHeader(metadata.New(map[string]string{"some-metadata": "some-metadata-val"})) + for { + if _, err := stream.Recv(); err == io.EOF { + return nil + } + } + }, + opts: itestutils.MetricDataOptions{ + CSMLabels: csmLabels, + }, + }, + { + name: "send-header", + streamingCallFunc: func(stream testgrpc.TestService_FullDuplexCallServer) error { + stream.SendHeader(metadata.New(map[string]string{"some-metadata": "some-metadata-val"})) + for { + if _, err := stream.Recv(); err == io.EOF { + return nil + } + } + }, + opts: itestutils.MetricDataOptions{ + CSMLabels: csmLabels, + }, + }, + { + name: "send-msg", + streamingCallFunc: func(stream testgrpc.TestService_FullDuplexCallServer) error { + stream.Send(&testpb.StreamingOutputCallResponse{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}) + for { + if _, err := stream.Recv(); err == io.EOF { + return nil + } + } + }, + opts: itestutils.MetricDataOptions{ + CSMLabels: csmLabels, + StreamingCompressedMessageSize: streamingCompressedSize, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + ss := &stubserver.StubServer{FullDuplexCallF: test.streamingCallFunc} + po := newPluginOption(ctx) + sopts := []grpc.ServerOption{ + serverOptionWithCSMPluginOption(opentelemetry.Options{ + MetricsOptions: opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics(), + }}, po), + } + dopts := []grpc.DialOption{dialOptionWithCSMPluginOption(opentelemetry.Options{ + MetricsOptions: opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics(), + OptionalLabels: []string{"csm.service_name", "csm.service_namespace_name"}, + }, + }, po)} + if err := ss.Start(sopts, dopts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + stream, err := ss.Client.FullDuplexCall(ctx, grpc.UseCompressor(gzip.Name)) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + if test.opts.StreamingCompressedMessageSize != 0 { + if err := stream.Send(&testpb.StreamingOutputCallRequest{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}); err != nil { + t.Fatalf("stream.Send failed") + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("stream.Recv failed with error: %v", err) + } + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err) + } + + rm := &metricdata.ResourceMetrics{} + reader.Collect(ctx, rm) + + gotMetrics := map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + gotMetrics[m.Name] = m + } + } + + opts := test.opts + opts.Target = ss.Target + wantMetrics := itestutils.MetricDataStreaming(opts) + gotMetrics = itestutils.WaitForServerMetrics(ctx, t, reader, gotMetrics, wantMetrics) + itestutils.CompareMetrics(t, gotMetrics, wantMetrics) + }) + } +} + +// TestXDSLabels tests that xDS Labels get emitted from OpenTelemetry metrics. +// This test configures OpenTelemetry with the CSM Plugin Option, and xDS +// Optional Labels turned on. It then configures a test balancer that updates +// labels, simulating the cluster_impl picker. It then makes a unary RPC, and +// expects xDS Labels labels to be attached to emitted relevant metrics. Full +// xDS System alongside OpenTelemetry will be tested with interop. (there is a +// test for xDS -> Stats handler and this tests -> OTel -> emission). It also +// tests the optional per call locality label in the same manner. +func (s) TestXDSLabels(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + balancerName := "o11y_balancer_xds_labels" + balancer.Register(base.NewBalancerBuilder(balancerName, &o11yPickerBuilder{}, base.Config{})) + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + ss := &stubserver.StubServer{ + UnaryCallF: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{Payload: &testpb.Payload{ + Body: make([]byte, len(in.GetPayload().GetBody())), + }}, nil + }, + } + + po := newPluginOption(ctx) + sc := fmt.Sprintf(`{"loadBalancingConfig": [{"%s": {}}]}`, balancerName) + dopts := []grpc.DialOption{dialOptionSetCSM(opentelemetry.Options{ + MetricsOptions: opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics(), + OptionalLabels: []string{"csm.service_name", "csm.service_namespace_name", "grpc.lb.locality", "grpc.lb.backend_service", "grpc.client.call.custom"}, + }, + }, po), grpc.WithDefaultServiceConfig(sc)} + if err := ss.Start(nil, dopts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + + defer ss.Stop() + ss.Client.UnaryCall(estats.NewContextWithCustomLabel(ctx, "my-custom-label"), &testpb.SimpleRequest{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}, grpc.UseCompressor(gzip.Name)) + + rm := &metricdata.ResourceMetrics{} + reader.Collect(ctx, rm) + + gotMetrics := map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + gotMetrics[m.Name] = m + } + } + + unaryMethodAttr := attribute.String("grpc.method", "grpc.testing.TestService/UnaryCall") + targetAttr := attribute.String("grpc.target", ss.Target) + unaryStatusAttr := attribute.String("grpc.status", "OK") + + serviceNameAttr := attribute.String("csm.service_name", "service_name_val") + serviceNamespaceAttr := attribute.String("csm.service_namespace_name", "service_namespace_val") + localityAttr := attribute.String("grpc.lb.locality", "grpc.lb.locality_val") + backendServiceAttr := attribute.String("grpc.lb.backend_service", "grpc.lb.backend_service_val") + meshIDAttr := attribute.String("csm.mesh_id", "unknown") + workloadCanonicalServiceAttr := attribute.String("csm.workload_canonical_service", "unknown") + remoteWorkloadTypeAttr := attribute.String("csm.remote_workload_type", "unknown") + remoteWorkloadCanonicalServiceAttr := attribute.String("csm.remote_workload_canonical_service", "unknown") + customLabelAttr := attribute.String("grpc.client.call.custom", "my-custom-label") + + unaryMethodClientSideEnd := []attribute.KeyValue{ + unaryMethodAttr, + targetAttr, + unaryStatusAttr, + serviceNameAttr, + serviceNamespaceAttr, + localityAttr, + backendServiceAttr, + meshIDAttr, + workloadCanonicalServiceAttr, + remoteWorkloadTypeAttr, + remoteWorkloadCanonicalServiceAttr, + customLabelAttr, + } + + unaryCompressedBytesSentRecv := int64(itestutils.GzipCompressedMessageSize(t, &testpb.SimpleRequest{ + Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }, + })) // Fixed 10000 bytes with gzip. + unaryBucketCounts := []uint64{0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0} + unaryExtrema := metricdata.NewExtrema(unaryCompressedBytesSentRecv) + wantMetrics := []metricdata.Metrics{ + { + Name: "grpc.client.attempt.started", + Description: "Number of client call attempts started.", + Unit: "{attempt}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(unaryMethodAttr, targetAttr, customLabelAttr), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, // Doesn't have xDS Labels, CSM Labels start from header or trailer from server, whichever comes first, so this doesn't need it + { + Name: "grpc.client.attempt.duration", + Description: "End-to-end time taken to complete a client call attempt.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attribute.NewSet(unaryMethodClientSideEnd...), + Count: 1, + Bounds: itestutils.DefaultLatencyBounds, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.client.attempt.sent_total_compressed_message_size", + Description: "Compressed message bytes sent per client call attempt.", + Unit: "By", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(unaryMethodClientSideEnd...), + Count: 1, + Bounds: itestutils.DefaultSizeBounds, + BucketCounts: unaryBucketCounts, + Min: unaryExtrema, + Max: unaryExtrema, + Sum: unaryCompressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.client.attempt.rcvd_total_compressed_message_size", + Description: "Compressed message bytes received per call attempt.", + Unit: "By", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(unaryMethodClientSideEnd...), + Count: 1, + Bounds: itestutils.DefaultSizeBounds, + BucketCounts: unaryBucketCounts, + Min: unaryExtrema, + Max: unaryExtrema, + Sum: unaryCompressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.client.call.duration", + Description: "Time taken by gRPC to complete an RPC from application's perspective.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attribute.NewSet(unaryMethodAttr, targetAttr, unaryStatusAttr, customLabelAttr), + Count: 1, + Bounds: itestutils.DefaultLatencyBounds, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + } + + gotMetrics = itestutils.WaitForServerMetrics(ctx, t, reader, gotMetrics, wantMetrics) + itestutils.CompareMetrics(t, gotMetrics, wantMetrics) +} + +// TestObservability tests that Observability global function compiles and runs +// without error. The actual functionality of this function will be verified in +// interop tests. +func (s) TestObservability(*testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cleanup := EnableObservability(ctx, opentelemetry.Options{}) + cleanup() +} + +type o11yPickerBuilder struct{} + +func (*o11yPickerBuilder) Build(info base.PickerBuildInfo) balancer.Picker { + return &o11yPicker{info: info} +} + +type o11yPicker struct { + info base.PickerBuildInfo +} + +func (p *o11yPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + // This simulates the behavior of the cluster_impl picker. + // It runs after TagRPC has already registered the default telemetry + // callback. + istats.UpdateLabels(info.Ctx, map[string]string{ + "csm.service_name": "service_name_val", + "csm.service_namespace_name": "service_namespace_val", + + "grpc.lb.locality": "grpc.lb.locality_val", + "grpc.lb.backend_service": "grpc.lb.backend_service_val", + }) + + for sc := range p.info.ReadySCs { + return balancer.PickResult{SubConn: sc}, nil + } + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable +} diff --git a/stats/opentelemetry/csm/pluginoption.go b/stats/opentelemetry/csm/pluginoption.go new file mode 100644 index 0000000..62a8d38 --- /dev/null +++ b/stats/opentelemetry/csm/pluginoption.go @@ -0,0 +1,279 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package csm contains utilities for Google Cloud Service Mesh observability. +package csm + +import ( + "context" + "encoding/base64" + "net/url" + "os" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats/opentelemetry/internal" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" + + "go.opentelemetry.io/contrib/detectors/gcp" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/resource" +) + +var logger = grpclog.Component("csm-observability-plugin") + +// pluginOption emits CSM Labels from the environment and metadata exchange +// for csm channels and all servers. +// +// Do not use this directly; use newPluginOption instead. +type pluginOption struct { + // localLabels are the labels that identify the local environment a binary + // is run in, and will be emitted from the CSM Plugin Option. + localLabels map[string]string + // metadataExchangeLabelsEncoded are the metadata exchange labels to be sent + // as the value of metadata key "x-envoy-peer-metadata" in proto wire format + // and base 64 encoded. This gets sent out from all the servers running in + // this process and for csm channels. + metadataExchangeLabelsEncoded string +} + +// newPluginOption returns a new pluginOption with local labels and metadata +// exchange labels derived from the environment. +func newPluginOption(ctx context.Context) internal.PluginOption { + localLabels, metadataExchangeLabelsEncoded := constructMetadataFromEnv(ctx) + + return &pluginOption{ + localLabels: localLabels, + metadataExchangeLabelsEncoded: metadataExchangeLabelsEncoded, + } +} + +// NewLabelsMD returns a metadata.MD with the CSM labels as an encoded protobuf +// Struct as the value of "x-envoy-peer-metadata". +func (cpo *pluginOption) GetMetadata() metadata.MD { + return metadata.New(map[string]string{ + metadataExchangeKey: cpo.metadataExchangeLabelsEncoded, + }) +} + +// GetLabels gets the CSM peer labels from the metadata provided. It returns +// "unknown" for labels not found. Labels returned depend on the remote type. +// Additionally, local labels determined at initialization time are appended to +// labels returned, in addition to the optionalLabels provided. +func (cpo *pluginOption) GetLabels(md metadata.MD) map[string]string { + labels := map[string]string{ // Remote labels if type is unknown (i.e. unset or error processing x-envoy-peer-metadata) + "csm.remote_workload_type": "unknown", + "csm.remote_workload_canonical_service": "unknown", + } + // Append the local labels. + for k, v := range cpo.localLabels { + labels[k] = v + } + + val := md.Get("x-envoy-peer-metadata") + // This can't happen if corresponding csm client because of proto wire + // format encoding, but since it is arbitrary off the wire be safe. + if len(val) != 1 { + logger.Warningf("length of md values of \"x-envoy-peer-metadata\" is not 1, is %v", len(val)) + return labels + } + + protoWireFormat, err := base64.RawStdEncoding.DecodeString(val[0]) + if err != nil { + logger.Warningf("error base 64 decoding value of \"x-envoy-peer-metadata\": %v", err) + return labels + } + + spb := &structpb.Struct{} + if err := proto.Unmarshal(protoWireFormat, spb); err != nil { + logger.Warningf("error unmarshalling value of \"x-envoy-peer-metadata\" into proto: %v", err) + return labels + } + + fields := spb.GetFields() + + labels["csm.remote_workload_type"] = getFromMetadata("type", fields) + // The value of “csm.remote_workload_canonical_service” comes from + // MetadataExchange with the key “canonical_service”. (Note that this should + // be read even if the remote type is unknown.) + labels["csm.remote_workload_canonical_service"] = getFromMetadata("canonical_service", fields) + + // Unset/unknown types, and types that aren't GKE or GCP return early with + // just local labels, remote_workload_type and + // remote_workload_canonical_service labels. + workloadType := labels["csm.remote_workload_type"] + if workloadType != "gcp_kubernetes_engine" && workloadType != "gcp_compute_engine" { + return labels + } + // GKE and GCE labels. + labels["csm.remote_workload_project_id"] = getFromMetadata("project_id", fields) + labels["csm.remote_workload_location"] = getFromMetadata("location", fields) + labels["csm.remote_workload_name"] = getFromMetadata("workload_name", fields) + if workloadType == "gcp_compute_engine" { + return labels + } + + // GKE only labels. + labels["csm.remote_workload_cluster_name"] = getFromMetadata("cluster_name", fields) + labels["csm.remote_workload_namespace_name"] = getFromMetadata("namespace_name", fields) + return labels +} + +// getFromMetadata gets the value for the metadata key from the protobuf +// metadata. Returns "unknown" if the metadata is not found in the protobuf +// metadata, or if the value is not a string value. Returns the string value +// otherwise. +func getFromMetadata(metadataKey string, metadata map[string]*structpb.Value) string { + if metadata != nil { + if metadataVal, ok := metadata[metadataKey]; ok { + if _, ok := metadataVal.GetKind().(*structpb.Value_StringValue); ok { + return metadataVal.GetStringValue() + } + } + } + return "unknown" +} + +// getFromResource gets the value for the resource key from the attribute set. +// Returns "unknown" if the resourceKey is not found in the attribute set or is +// not a string value, the string value otherwise. +func getFromResource(resourceKey attribute.Key, set *attribute.Set) string { + if set != nil { + if resourceVal, ok := set.Value(resourceKey); ok && resourceVal.Type() == attribute.STRING { + return resourceVal.AsString() + } + } + return "unknown" +} + +// getEnv returns "unknown" if environment variable is unset, the environment +// variable otherwise. +func getEnv(name string) string { + if val, ok := os.LookupEnv(name); ok { + return val + } + return "unknown" +} + +var ( + // This function will be overridden in unit tests. + getAttrSetFromResourceDetector = func(ctx context.Context) *attribute.Set { + r, err := resource.New(ctx, resource.WithFromEnv(), resource.WithDetectors(gcp.NewDetector())) + if err != nil { + logger.Warningf("error reading OpenTelemetry resource: %v", err) + } + if r != nil { + // It's possible for resource.New to return partial data alongside + // an error. In this case, use partial data and set "unknown" for + // labels missing. + return r.Set() + } + return nil + } +) + +// constructMetadataFromEnv creates local labels and labels to send to the peer +// using metadata exchange based off resource detection and environment +// variables. +// +// Returns local labels, and base 64 encoded protobuf.Struct containing metadata +// exchange labels. +func constructMetadataFromEnv(ctx context.Context) (map[string]string, string) { + set := getAttrSetFromResourceDetector(ctx) + + labels := make(map[string]string) + labels["type"] = getFromResource("cloud.platform", set) + labels["canonical_service"] = getEnv("CSM_CANONICAL_SERVICE_NAME") + + // If type is not GCE or GKE only metadata exchange labels are "type" and + // "canonical_service". + cloudPlatformVal := labels["type"] + if cloudPlatformVal != "gcp_kubernetes_engine" && cloudPlatformVal != "gcp_compute_engine" { + return initializeLocalAndMetadataLabels(labels) + } + + // GCE and GKE labels: + labels["workload_name"] = getEnv("CSM_WORKLOAD_NAME") + + locationVal := "unknown" + if resourceVal, ok := set.Value("cloud.availability_zone"); ok && resourceVal.Type() == attribute.STRING { + locationVal = resourceVal.AsString() + } else if resourceVal, ok = set.Value("cloud.region"); ok && resourceVal.Type() == attribute.STRING { + locationVal = resourceVal.AsString() + } + labels["location"] = locationVal + + labels["project_id"] = getFromResource("cloud.account.id", set) + if cloudPlatformVal == "gcp_compute_engine" { + return initializeLocalAndMetadataLabels(labels) + } + + // GKE specific labels: + labels["namespace_name"] = getFromResource("k8s.namespace.name", set) + labels["cluster_name"] = getFromResource("k8s.cluster.name", set) + return initializeLocalAndMetadataLabels(labels) +} + +// initializeLocalAndMetadataLabels csm local labels for a CSM Plugin Option to +// record. It also builds out a base 64 encoded protobuf.Struct containing the +// metadata exchange labels to be sent as part of metadata exchange from a CSM +// Plugin Option. +func initializeLocalAndMetadataLabels(labels map[string]string) (map[string]string, string) { + // The value of “csm.workload_canonical_service” comes from + // “CSM_CANONICAL_SERVICE_NAME” env var, “unknown” if unset. + val := labels["canonical_service"] + localLabels := make(map[string]string) + localLabels["csm.workload_canonical_service"] = val + localLabels["csm.mesh_id"] = getEnv("CSM_MESH_ID") + + // Metadata exchange labels - can go ahead and encode into proto, and then + // base64. + pbLabels := &structpb.Struct{ + Fields: map[string]*structpb.Value{}, + } + for k, v := range labels { + pbLabels.Fields[k] = structpb.NewStringValue(v) + } + protoWireFormat, err := proto.Marshal(pbLabels) + metadataExchangeLabelsEncoded := "" + if err == nil { + metadataExchangeLabelsEncoded = base64.RawStdEncoding.EncodeToString(protoWireFormat) + } + // else - This behavior triggers server side to reply (if sent from a gRPC + // Client within this binary) with the metadata exchange labels. Even if + // client side has a problem marshaling proto into wire format, it can + // still use server labels so send an empty string as the value of + // x-envoy-peer-metadata. The presence of this metadata exchange header + // will cause server side to respond with metadata exchange labels. + + return localLabels, metadataExchangeLabelsEncoded +} + +// metadataExchangeKey is the key for HTTP metadata exchange. +const metadataExchangeKey = "x-envoy-peer-metadata" + +func determineTargetCSM(parsedTarget *url.URL) bool { + // On the client-side, the channel target is used to determine if a channel is a + // CSM channel or not. CSM channels need to have an “xds” scheme and a + // "traffic-director-global.xds.googleapis.com" authority. In the cases where no + // authority is mentioned, the authority is assumed to be CSM. MetadataExchange + // is performed only for CSM channels. Non-metadata exchange labels are detected + // as described below. + return parsedTarget.Scheme == "xds" && (parsedTarget.Host == "" || parsedTarget.Host == "traffic-director-global.xds.googleapis.com") +} diff --git a/stats/opentelemetry/csm/pluginoption_test.go b/stats/opentelemetry/csm/pluginoption_test.go new file mode 100644 index 0000000..55e9597 --- /dev/null +++ b/stats/opentelemetry/csm/pluginoption_test.go @@ -0,0 +1,490 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package csm + +import ( + "context" + "encoding/base64" + "fmt" + "net/url" + "os" + "testing" + "time" + + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/metadata" + + "github.com/google/go-cmp/cmp" + "go.opentelemetry.io/otel/attribute" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +var defaultTestTimeout = 5 * time.Second + +// clearEnv unsets all the environment variables relevant to the csm +// pluginOption. +func clearEnv() { + os.Unsetenv(envconfig.XDSBootstrapFileContentEnv) + os.Unsetenv(envconfig.XDSBootstrapFileNameEnv) + + os.Unsetenv("CSM_CANONICAL_SERVICE_NAME") + os.Unsetenv("CSM_WORKLOAD_NAME") +} + +func (s) TestGetLabels(t *testing.T) { + clearEnv() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cpo := newPluginOption(ctx) + + tests := []struct { + name string + unsetHeader bool // Should trigger "unknown" labels + twoValues bool // Should trigger "unknown" labels + metadataExchangeLabels map[string]string + labelsWant map[string]string + }{ + { + name: "unset-labels", + metadataExchangeLabels: nil, + labelsWant: map[string]string{ + "csm.workload_canonical_service": "unknown", + "csm.mesh_id": "unknown", + + "csm.remote_workload_type": "unknown", + "csm.remote_workload_canonical_service": "unknown", + }, + }, + { + name: "metadata-partially-set", + metadataExchangeLabels: map[string]string{ + "type": "not-gce-or-gke", + "ignore-this": "ignore-this", + }, + labelsWant: map[string]string{ + "csm.workload_canonical_service": "unknown", + "csm.mesh_id": "unknown", + + "csm.remote_workload_type": "not-gce-or-gke", + "csm.remote_workload_canonical_service": "unknown", + }, + }, + { + name: "google-compute-engine", + metadataExchangeLabels: map[string]string{ // All of these labels get emitted when type is "gcp_compute_engine". + "type": "gcp_compute_engine", + "canonical_service": "canonical_service_val", + "project_id": "unique-id", + "location": "us-east", + "workload_name": "workload_name_val", + }, + labelsWant: map[string]string{ + "csm.workload_canonical_service": "unknown", + "csm.mesh_id": "unknown", + + "csm.remote_workload_type": "gcp_compute_engine", + "csm.remote_workload_canonical_service": "canonical_service_val", + "csm.remote_workload_project_id": "unique-id", + "csm.remote_workload_location": "us-east", + "csm.remote_workload_name": "workload_name_val", + }, + }, + // unset should go to unknown, ignore GKE labels that are not relevant + // to GCE. + { + name: "google-compute-engine-labels-partially-set-with-extra", + metadataExchangeLabels: map[string]string{ + "type": "gcp_compute_engine", + "canonical_service": "canonical_service_val", + "project_id": "unique-id", + "location": "us-east", + // "workload_name": "", unset workload name - should become "unknown" + "namespace_name": "should-be-ignored", + "cluster_name": "should-be-ignored", + }, + labelsWant: map[string]string{ + "csm.workload_canonical_service": "unknown", + "csm.mesh_id": "unknown", + + "csm.remote_workload_type": "gcp_compute_engine", + "csm.remote_workload_canonical_service": "canonical_service_val", + "csm.remote_workload_project_id": "unique-id", + "csm.remote_workload_location": "us-east", + "csm.remote_workload_name": "unknown", + }, + }, + { + name: "google-kubernetes-engine", + metadataExchangeLabels: map[string]string{ + "type": "gcp_kubernetes_engine", + "canonical_service": "canonical_service_val", + "project_id": "unique-id", + "namespace_name": "namespace_name_val", + "cluster_name": "cluster_name_val", + "location": "us-east", + "workload_name": "workload_name_val", + }, + labelsWant: map[string]string{ + "csm.workload_canonical_service": "unknown", + "csm.mesh_id": "unknown", + + "csm.remote_workload_type": "gcp_kubernetes_engine", + "csm.remote_workload_canonical_service": "canonical_service_val", + "csm.remote_workload_project_id": "unique-id", + "csm.remote_workload_cluster_name": "cluster_name_val", + "csm.remote_workload_namespace_name": "namespace_name_val", + "csm.remote_workload_location": "us-east", + "csm.remote_workload_name": "workload_name_val", + }, + }, + { + name: "google-kubernetes-engine-labels-partially-set", + metadataExchangeLabels: map[string]string{ + "type": "gcp_kubernetes_engine", + "canonical_service": "canonical_service_val", + "project_id": "unique-id", + "namespace_name": "namespace_name_val", + // "cluster_name": "", cluster_name unset, should become "unknown" + "location": "us-east", + // "workload_name": "", workload_name unset, should become "unknown" + }, + labelsWant: map[string]string{ + "csm.workload_canonical_service": "unknown", + "csm.mesh_id": "unknown", + + "csm.remote_workload_type": "gcp_kubernetes_engine", + "csm.remote_workload_canonical_service": "canonical_service_val", + "csm.remote_workload_project_id": "unique-id", + "csm.remote_workload_cluster_name": "unknown", + "csm.remote_workload_namespace_name": "namespace_name_val", + "csm.remote_workload_location": "us-east", + "csm.remote_workload_name": "unknown", + }, + }, + { + name: "unset-header", + metadataExchangeLabels: map[string]string{ + "type": "gcp_kubernetes_engine", + "canonical_service": "canonical_service_val", + "project_id": "unique-id", + "namespace_name": "namespace_name_val", + "cluster_name": "cluster_name_val", + "location": "us-east", + "workload_name": "workload_name_val", + }, + unsetHeader: true, + labelsWant: map[string]string{ + "csm.workload_canonical_service": "unknown", + "csm.mesh_id": "unknown", + + "csm.remote_workload_type": "unknown", + "csm.remote_workload_canonical_service": "unknown", + }, + }, + { + name: "two-header-values", + metadataExchangeLabels: map[string]string{ + "type": "gcp_kubernetes_engine", + "canonical_service": "canonical_service_val", + "project_id": "unique-id", + "namespace_name": "namespace_name_val", + "cluster_name": "cluster_name_val", + "location": "us-east", + "workload_name": "workload_name_val", + }, + twoValues: true, + labelsWant: map[string]string{ + "csm.workload_canonical_service": "unknown", + "csm.mesh_id": "unknown", + + "csm.remote_workload_type": "unknown", + "csm.remote_workload_canonical_service": "unknown", + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pbLabels := &structpb.Struct{ + Fields: map[string]*structpb.Value{}, + } + for k, v := range test.metadataExchangeLabels { + pbLabels.Fields[k] = structpb.NewStringValue(v) + } + protoWireFormat, err := proto.Marshal(pbLabels) + if err != nil { + t.Fatalf("Error marshaling proto: %v", err) + } + metadataExchangeLabelsEncoded := base64.RawStdEncoding.EncodeToString(protoWireFormat) + md := metadata.New(map[string]string{ + metadataExchangeKey: metadataExchangeLabelsEncoded, + }) + if test.unsetHeader { + md.Delete(metadataExchangeKey) + } + if test.twoValues { + md.Append(metadataExchangeKey, "extra-val") + } + + labelsGot := cpo.GetLabels(md) + if diff := cmp.Diff(labelsGot, test.labelsWant); diff != "" { + t.Fatalf("cpo.GetLabels returned unexpected value (-got, +want): %v", diff) + } + }) + } +} + +// TestDetermineTargetCSM tests the helper function that determines whether a +// target is relevant to CSM or not, based off the rules outlined in design. +func (s) TestDetermineTargetCSM(t *testing.T) { + tests := []struct { + name string + target string + targetCSM bool + }{ + { + name: "dns:///localhost", + target: "normal-target-here", + targetCSM: false, + }, + { + name: "xds-no-authority", + target: "xds:///localhost", + targetCSM: true, + }, + { + name: "xds-traffic-director-authority", + target: "xds://traffic-director-global.xds.googleapis.com/localhost", + targetCSM: true, + }, + { + name: "xds-not-traffic-director-authority", + target: "xds://not-traffic-director-authority/localhost", + targetCSM: false, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + parsedTarget, err := url.Parse(test.target) + if err != nil { + t.Fatalf("test target %v failed to parse: %v", test.target, err) + } + if got := determineTargetCSM(parsedTarget); got != test.targetCSM { + t.Fatalf("cpo.determineTargetCSM(%v): got %v, want %v", test.target, got, test.targetCSM) + } + }) + } +} + +// TestSetLabels tests the setting of labels, which snapshots the resource and +// environment. It mocks the resource and environment, and then calls into +// labels creation. It verifies to local labels created and metadata exchange +// labels emitted from the setLabels function. +func (s) TestSetLabels(t *testing.T) { + clearEnv() + tests := []struct { + name string + resourceKeyValues map[string]string + csmCanonicalServiceNamePopulated bool + csmWorkloadNamePopulated bool + meshIDPopulated bool + localLabelsWant map[string]string + metadataExchangeLabelsWant map[string]string + }{ + { + name: "no-type", + csmCanonicalServiceNamePopulated: true, + meshIDPopulated: true, + resourceKeyValues: map[string]string{}, + localLabelsWant: map[string]string{ + "csm.workload_canonical_service": "canonical_service_name_val", // env var populated so should be set. + "csm.mesh_id": "mesh_id", // env var populated so should be set. + }, + metadataExchangeLabelsWant: map[string]string{ + "type": "unknown", + "canonical_service": "canonical_service_name_val", // env var populated so should be set. + }, + }, + { + name: "gce", + csmWorkloadNamePopulated: true, + resourceKeyValues: map[string]string{ + "cloud.platform": "gcp_compute_engine", + // csm workload name is an env var + "cloud.availability_zone": "cloud_availability_zone_val", + "cloud.region": "should-be-ignored", // cloud.availability_zone takes precedence + "cloud.account.id": "cloud_account_id_val", + }, + localLabelsWant: map[string]string{ + "csm.workload_canonical_service": "unknown", + "csm.mesh_id": "unknown", + }, + metadataExchangeLabelsWant: map[string]string{ + "type": "gcp_compute_engine", + "canonical_service": "unknown", + "workload_name": "workload_name_val", + "location": "cloud_availability_zone_val", + "project_id": "cloud_account_id_val", + }, + }, + { + name: "gce-half-unset", + resourceKeyValues: map[string]string{ + "cloud.platform": "gcp_compute_engine", + // csm workload name is an env var + "cloud.availability_zone": "cloud_availability_zone_val", + "cloud.region": "should-be-ignored", // cloud.availability_zone takes precedence + }, + localLabelsWant: map[string]string{ + "csm.workload_canonical_service": "unknown", + "csm.mesh_id": "unknown", + }, + metadataExchangeLabelsWant: map[string]string{ + "type": "gcp_compute_engine", + "canonical_service": "unknown", + "workload_name": "unknown", + "location": "cloud_availability_zone_val", + "project_id": "unknown", + }, + }, + { + name: "gke", + resourceKeyValues: map[string]string{ + "cloud.platform": "gcp_kubernetes_engine", + // csm workload name is an env var + "cloud.region": "cloud_region_val", // availability_zone isn't present, so this should become location + "cloud.account.id": "cloud_account_id_val", + "k8s.namespace.name": "k8s_namespace_name_val", + "k8s.cluster.name": "k8s_cluster_name_val", + }, + localLabelsWant: map[string]string{ + "csm.workload_canonical_service": "unknown", + "csm.mesh_id": "unknown", + }, + metadataExchangeLabelsWant: map[string]string{ + "type": "gcp_kubernetes_engine", + "canonical_service": "unknown", + "workload_name": "unknown", + "location": "cloud_region_val", + "project_id": "cloud_account_id_val", + "namespace_name": "k8s_namespace_name_val", + "cluster_name": "k8s_cluster_name_val", + }, + }, + { + name: "gke-half-unset", + resourceKeyValues: map[string]string{ // unset should become unknown + "cloud.platform": "gcp_kubernetes_engine", + // csm workload name is an env var + "cloud.region": "cloud_region_val", // availability_zone isn't present, so this should become location + // "cloud.account.id": "", // unset - should become unknown + "k8s.namespace.name": "k8s_namespace_name_val", + // "k8s.cluster.name": "", // unset - should become unknown + }, + localLabelsWant: map[string]string{ + "csm.workload_canonical_service": "unknown", + "csm.mesh_id": "unknown", + }, + metadataExchangeLabelsWant: map[string]string{ + "type": "gcp_kubernetes_engine", + "canonical_service": "unknown", + "workload_name": "unknown", + "location": "cloud_region_val", + "project_id": "unknown", + "namespace_name": "k8s_namespace_name_val", + "cluster_name": "unknown", + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + func() { + if test.csmCanonicalServiceNamePopulated { + os.Setenv("CSM_CANONICAL_SERVICE_NAME", "canonical_service_name_val") + defer os.Unsetenv("CSM_CANONICAL_SERVICE_NAME") + } + if test.csmWorkloadNamePopulated { + os.Setenv("CSM_WORKLOAD_NAME", "workload_name_val") + defer os.Unsetenv("CSM_WORKLOAD_NAME") + } + if test.meshIDPopulated { + os.Setenv("CSM_MESH_ID", "mesh_id") + defer os.Unsetenv("CSM_MESH_ID") + } + var attributes []attribute.KeyValue + for k, v := range test.resourceKeyValues { + attributes = append(attributes, attribute.String(k, v)) + } + // Return the attributes configured as part of the test in place + // of reading from resource. + attrSet := attribute.NewSet(attributes...) + origGetAttrSet := getAttrSetFromResourceDetector + getAttrSetFromResourceDetector = func(context.Context) *attribute.Set { + return &attrSet + } + defer func() { getAttrSetFromResourceDetector = origGetAttrSet }() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + localLabelsGot, mdEncoded := constructMetadataFromEnv(ctx) + if diff := cmp.Diff(localLabelsGot, test.localLabelsWant); diff != "" { + t.Fatalf("constructMetadataFromEnv() want: %v, got %v", test.localLabelsWant, localLabelsGot) + } + + verifyMetadataExchangeLabels(mdEncoded, test.metadataExchangeLabelsWant) + }() + }) + } +} + +func verifyMetadataExchangeLabels(mdEncoded string, mdLabelsWant map[string]string) error { + protoWireFormat, err := base64.RawStdEncoding.DecodeString(mdEncoded) + if err != nil { + return fmt.Errorf("error base 64 decoding metadata val: %v", err) + } + spb := &structpb.Struct{} + if err := proto.Unmarshal(protoWireFormat, spb); err != nil { + return fmt.Errorf("error unmarshaling proto wire format: %v", err) + } + fields := spb.GetFields() + for k, v := range mdLabelsWant { + if val, ok := fields[k]; !ok { + if _, ok := val.GetKind().(*structpb.Value_StringValue); !ok { + return fmt.Errorf("struct value for key %v should be string type", k) + } + if val.GetStringValue() != v { + return fmt.Errorf("struct value for key %v got: %v, want %v", k, val.GetStringValue(), v) + } + } + } + if len(mdLabelsWant) != len(fields) { + return fmt.Errorf("len(mdLabelsWant) = %v, len(mdLabelsGot) = %v", len(mdLabelsWant), len(fields)) + } + return nil +} diff --git a/stats/opentelemetry/e2e_test.go b/stats/opentelemetry/e2e_test.go new file mode 100644 index 0000000..2bb89b9 --- /dev/null +++ b/stats/opentelemetry/e2e_test.go @@ -0,0 +1,2772 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opentelemetry_test + +import ( + "context" + "fmt" + "io" + "net" + "slices" + "strconv" + "sync" + "syscall" + "testing" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/health" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + + otelcodes "go.opentelemetry.io/otel/codes" + oteltrace "go.opentelemetry.io/otel/trace" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3clientsideweightedroundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3" + v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/baggage" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" + "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/encoding/gzip" + experimental "google.golang.org/grpc/experimental/opentelemetry" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + itestutils "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + setup "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/stats/opentelemetry" + "google.golang.org/grpc/stats/opentelemetry/internal/testutils" + "google.golang.org/grpc/status" +) + +var defaultTestTimeout = 5 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// traceSpanInfo is the information received about the trace span. It contains +// subset of information that is needed to verify if correct trace is being +// attributed to the rpc. +type traceSpanInfo struct { + spanKind string + name string + events []trace.Event + attributes []attribute.KeyValue + status otelcodes.Code +} + +// defaultMetricsOptions creates default metrics options +func defaultMetricsOptions(_ *testing.T, methodAttributeFilter func(string) bool) (*opentelemetry.MetricsOptions, *metric.ManualReader) { + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + metricsOptions := &opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics(), + MethodAttributeFilter: methodAttributeFilter, + } + return metricsOptions, reader +} + +// defaultTraceOptions function to create default trace options +func defaultTraceOptions(_ *testing.T) (*experimental.TraceOptions, *tracetest.InMemoryExporter) { + spanExporter := tracetest.NewInMemoryExporter() + spanProcessor := trace.NewSimpleSpanProcessor(spanExporter) + tracerProvider := trace.NewTracerProvider(trace.WithSpanProcessor(spanProcessor)) + textMapPropagator := propagation.NewCompositeTextMapPropagator(opentelemetry.GRPCTraceBinPropagator{}) + traceOptions := &experimental.TraceOptions{ + TracerProvider: tracerProvider, + TextMapPropagator: textMapPropagator, + } + return traceOptions, spanExporter +} + +// setupStubServer creates a stub server with OpenTelemetry component configured on client +// and server side and returns the server. +func setupStubServer(t *testing.T, metricsOptions *opentelemetry.MetricsOptions, traceOptions *experimental.TraceOptions) *stubserver.StubServer { + ss := &stubserver.StubServer{ + UnaryCallF: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{Payload: &testpb.Payload{ + Body: make([]byte, len(in.GetPayload().GetBody())), + }}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + _, err := stream.Recv() + if err == io.EOF { + return nil + } + } + }, + } + + otelOptions := opentelemetry.Options{} + if metricsOptions != nil { + otelOptions.MetricsOptions = *metricsOptions + } + if traceOptions != nil { + otelOptions.TraceOptions = *traceOptions + } + + if err := ss.Start([]grpc.ServerOption{opentelemetry.ServerOption(otelOptions)}, + opentelemetry.DialOption(otelOptions)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + return ss +} + +// waitForTraceSpans waits until the in-memory span exporter has received the +// expected trace spans based on span name and kind. It polls the exporter at a +// short interval until the desired spans are available or the context is +// cancelled. +// +// Returns the collected spans or an error if the context deadline is exceeded +// before the expected spans are exported. +func waitForTraceSpans(ctx context.Context, exporter *tracetest.InMemoryExporter, wantSpans []traceSpanInfo) (tracetest.SpanStubs, error) { + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + spans := exporter.GetSpans() + if len(spans) < len(wantSpans) { + continue + } + missingAnySpan := false + for _, wantSpan := range wantSpans { + if !slices.ContainsFunc(spans, func(span tracetest.SpanStub) bool { + return span.Name == wantSpan.name && span.SpanKind.String() == wantSpan.spanKind + }) { + missingAnySpan = true + } + } + if !missingAnySpan { + return spans, nil + } + } + return nil, fmt.Errorf("error waiting for complete trace spans %v: %v", wantSpans, ctx.Err()) +} + +// validateTraces first first groups the received spans by their TraceID. For +// each trace group, it identifies the client, server, and attempt spans for +// both unary and streaming RPCs. It checks that the expected spans are +// present and that the server spans have the correct parent (attempt span). +// Finally, it compares the content of each span (name, kind, attributes, +// events) against the provided expected spans information. +func validateTraces(t *testing.T, spans tracetest.SpanStubs, wantSpanInfos []traceSpanInfo) { + // Group spans by TraceID. + traceSpans := make(map[oteltrace.TraceID][]tracetest.SpanStub) + for _, span := range spans { + traceID := span.SpanContext.TraceID() + traceSpans[traceID] = append(traceSpans[traceID], span) + } + + // For each trace group, verify relationships and content. + for traceID, spans := range traceSpans { + var unaryClient, unaryServer, unaryAttempt *tracetest.SpanStub + var streamClient, streamServer, streamAttempt *tracetest.SpanStub + var isUnary, isStream bool + + for _, span := range spans { + switch { + case span.Name == "Sent.grpc.testing.TestService.UnaryCall": + isUnary = true + if span.SpanKind == oteltrace.SpanKindClient { + unaryClient = &span + } + case span.Name == "Recv.grpc.testing.TestService.UnaryCall": + isUnary = true + if span.SpanKind == oteltrace.SpanKindServer { + unaryServer = &span + } + case span.Name == "Attempt.grpc.testing.TestService.UnaryCall": + isUnary = true + unaryAttempt = &span + + case span.Name == "Sent.grpc.testing.TestService.FullDuplexCall": + isStream = true + if span.SpanKind == oteltrace.SpanKindClient { + streamClient = &span + } + case span.Name == "Recv.grpc.testing.TestService.FullDuplexCall": + isStream = true + if span.SpanKind == oteltrace.SpanKindServer { + streamServer = &span + } + case span.Name == "Attempt.grpc.testing.TestService.FullDuplexCall": + isStream = true + streamAttempt = &span + } + } + + if isUnary { + // Verify Unary Call Spans. + if unaryClient == nil { + t.Error("Unary call client span not found") + } + if unaryServer == nil { + t.Error("Unary call server span not found") + } + if unaryAttempt == nil { + t.Error("Unary call attempt span not found") + } + // Check TraceID consistency. + if unaryClient != nil && unaryClient.SpanContext.TraceID() != traceID || unaryServer.SpanContext.TraceID() != traceID { + t.Error("Unary call spans have inconsistent TraceIDs") + } + // Check parent-child relationship via SpanID. + if unaryServer != nil && unaryServer.Parent.SpanID() != unaryAttempt.SpanContext.SpanID() { + t.Error("Unary server span parent does not match attempt span ID") + } + } + + if isStream { + // Verify Streaming Call Spans. + if streamClient == nil { + t.Error("Streaming call client span not found") + } + if streamServer == nil { + t.Error("Streaming call server span not found") + } + if streamAttempt == nil { + t.Error("Streaming call attempt span not found") + } + // Check TraceID consistency. + if streamClient != nil && streamClient.SpanContext.TraceID() != traceID || streamServer.SpanContext.TraceID() != traceID { + t.Error("Streaming call spans have inconsistent TraceIDs") + } + if streamServer != nil && streamServer.Parent.SpanID() != streamAttempt.SpanContext.SpanID() { + t.Error("Streaming server span parent does not match attempt span ID") + } + } + } + + // Convert spans to traceSpanInfo for cmp.Diff comparison. + actualSpanInfos := make([]traceSpanInfo, len(spans)) + for i, s := range spans { + actualSpanInfos[i] = traceSpanInfo{ + name: s.Name, + spanKind: s.SpanKind.String(), + attributes: s.Attributes, + events: s.Events, + status: s.Status.Code, + } + } + opts := []cmp.Option{ + cmpopts.SortSlices(func(a, b traceSpanInfo) bool { + if a.name == b.name { + return a.spanKind < b.spanKind + } + return a.name < b.name + }), + cmpopts.SortSlices(func(a, b trace.Event) bool { return a.Name < b.Name }), + cmpopts.IgnoreFields(trace.Event{}, "Time"), + cmpopts.EquateComparable(attribute.KeyValue{}, attribute.Value{}, attribute.Set{}), + cmpopts.IgnoreFields(tracetest.SpanStub{}, "InstrumentationScope"), + cmp.AllowUnexported(traceSpanInfo{}), + } + if diff := cmp.Diff(wantSpanInfos, actualSpanInfos, opts...); diff != "" { + t.Errorf("Spans mismatch (-want +got):\n%s", diff) + } +} + +// TestMethodAttributeFilter tests the method attribute filter. The method +// filter set should bucket the grpc.method attribute into "other" if the method +// attribute filter specifies. +func (s) TestMethodAttributeFilter(t *testing.T) { + maf := func(str string) bool { + // Will allow duplex/any other type of RPC. + return str != testgrpc.TestService_UnaryCall_FullMethodName + } + mo, reader := defaultMetricsOptions(t, maf) + ss := setupStubServer(t, mo, nil) + defer ss.Stop() + + // Make a Unary and Streaming RPC. The Unary RPC should be filtered by the + // method attribute filter, and the Full Duplex (Streaming) RPC should not. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err) + } + rm := &metricdata.ResourceMetrics{} + reader.Collect(ctx, rm) + gotMetrics := map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + gotMetrics[m.Name] = m + } + } + + wantMetrics := []metricdata.Metrics{ + { + Name: "grpc.client.attempt.started", + Description: "Number of client call attempts started.", + Unit: "{attempt}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("grpc.method", "grpc.testing.TestService/UnaryCall"), attribute.String("grpc.target", ss.Target)), + Value: 1, + }, + { + Attributes: attribute.NewSet(attribute.String("grpc.method", "grpc.testing.TestService/FullDuplexCall"), attribute.String("grpc.target", ss.Target)), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "grpc.server.call.duration", + Description: "End-to-end time taken to complete a call from server transport's perspective.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { // Method should go to "other" due to the method attribute filter. + Attributes: attribute.NewSet(attribute.String("grpc.method", "other"), attribute.String("grpc.status", "OK")), + Count: 1, + Bounds: testutils.DefaultLatencyBounds, + }, + { + Attributes: attribute.NewSet(attribute.String("grpc.method", "grpc.testing.TestService/FullDuplexCall"), attribute.String("grpc.status", "OK")), + Count: 1, + Bounds: testutils.DefaultLatencyBounds, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + } + + gotMetrics = testutils.WaitForServerMetrics(ctx, t, reader, gotMetrics, wantMetrics) + testutils.CompareMetrics(t, gotMetrics, wantMetrics) +} + +// TestAllMetricsOneFunction tests emitted metrics from OpenTelemetry +// instrumentation component. It then configures a system with a gRPC Client and +// gRPC server with the OpenTelemetry Dial and Server Option configured +// specifying all the metrics provided by this package, and makes a Unary RPC +// and a Streaming RPC. These two RPCs should cause certain recording for each +// registered metric observed through a Manual Metrics Reader on the provided +// OpenTelemetry SDK's Meter Provider. It then makes an RPC that is unregistered +// on the Client (no StaticMethodCallOption set) and Server. The method +// attribute on subsequent metrics should be bucketed in "other". +func (s) TestAllMetricsOneFunction(t *testing.T) { + mo, reader := defaultMetricsOptions(t, nil) + ss := setupStubServer(t, mo, nil) + defer ss.Stop() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Make two RPC's, a unary RPC and a streaming RPC. These should cause + // certain metrics to be emitted, which should be observed through the + // Metric Reader. + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}, grpc.UseCompressor(gzip.Name)); err != nil { // Deterministic compression. + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err) + } + + rm := &metricdata.ResourceMetrics{} + reader.Collect(ctx, rm) + + gotMetrics := map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + gotMetrics[m.Name] = m + } + } + + compressedSize := testutils.GzipCompressedMessageSize(t, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: make([]byte, 10000)}}) + wantMetrics := testutils.MetricData(testutils.MetricDataOptions{ + Target: ss.Target, + UnaryCompressedMessageSize: float64(compressedSize), + }) + gotMetrics = testutils.WaitForServerMetrics(ctx, t, reader, gotMetrics, wantMetrics) + testutils.CompareMetrics(t, gotMetrics, wantMetrics) + + stream, err = ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err) + } + // This Invoke doesn't pass the StaticMethodCallOption. Thus, the method + // attribute should become "other" on client side metrics. Since it is also + // not registered on the server either, it should also become "other" on the + // server metrics method attribute. + ss.CC.Invoke(ctx, "/grpc.testing.TestService/UnregisteredCall", nil, nil, []grpc.CallOption{}...) + ss.CC.Invoke(ctx, "/grpc.testing.TestService/UnregisteredCall", nil, nil, []grpc.CallOption{}...) + ss.CC.Invoke(ctx, "/grpc.testing.TestService/UnregisteredCall", nil, nil, []grpc.CallOption{}...) + + rm = &metricdata.ResourceMetrics{} + reader.Collect(ctx, rm) + gotMetrics = map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + gotMetrics[m.Name] = m + } + } + unaryMethodAttr := attribute.String("grpc.method", "grpc.testing.TestService/UnaryCall") + duplexMethodAttr := attribute.String("grpc.method", "grpc.testing.TestService/FullDuplexCall") + + targetAttr := attribute.String("grpc.target", ss.Target) + otherMethodAttr := attribute.String("grpc.method", "other") + wantMetrics = []metricdata.Metrics{ + { + Name: "grpc.client.attempt.started", + Description: "Number of client call attempts started.", + Unit: "{attempt}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(unaryMethodAttr, targetAttr), + Value: 1, + }, + { + Attributes: attribute.NewSet(duplexMethodAttr, targetAttr), + Value: 2, + }, + { + Attributes: attribute.NewSet(otherMethodAttr, targetAttr), + Value: 3, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "grpc.server.call.started", + Description: "Number of server calls started.", + Unit: "{call}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(unaryMethodAttr), + Value: 1, + }, + { + Attributes: attribute.NewSet(duplexMethodAttr), + Value: 2, + }, + { + Attributes: attribute.NewSet(otherMethodAttr), + Value: 3, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + } + for _, metric := range wantMetrics { + val, ok := gotMetrics[metric.Name] + if !ok { + t.Fatalf("Metric %v not present in recorded metrics", metric.Name) + } + if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) { + t.Fatalf("Metrics data type not equal for metric: %v", metric.Name) + } + } +} + +// clusterWithLBConfiguration returns a cluster resource with the proto message +// passed Marshaled to an any and specified through the load_balancing_policy +// field. +func clusterWithLBConfiguration(t *testing.T, clusterName, edsServiceName string, secLevel e2e.SecurityLevel, m proto.Message) *v3clusterpb.Cluster { + cluster := e2e.DefaultCluster(clusterName, edsServiceName, secLevel) + cluster.LoadBalancingPolicy = &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: itestutils.MarshalAny(t, m), + }, + }, + }, + } + return cluster +} + +func metricsDataFromReader(ctx context.Context, reader *metric.ManualReader) map[string]metricdata.Metrics { + rm := &metricdata.ResourceMetrics{} + reader.Collect(ctx, rm) + gotMetrics := map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + gotMetrics[m.Name] = m + } + } + return gotMetrics +} + +// TestWRRMetrics tests the metrics emitted from the WRR LB Policy. It +// configures WRR as an endpoint picking policy through xDS on a ClientConn +// alongside an OpenTelemetry stats handler. It makes a few RPC's, and then +// sleeps for a bit to allow weight to expire. It then asserts OpenTelemetry +// metrics atoms are eventually present for all four WRR Metrics, alongside the +// correct target and locality label for each metric. +func (s) TestWRRMetrics(t *testing.T) { + cmr := orca.NewServerMetricsRecorder().(orca.CallMetricsRecorder) + backend1 := stubserver.StartTestService(t, &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + if r := orca.CallMetricsRecorderFromContext(ctx); r != nil { + // Copy metrics from what the test set in cmr into r. + sm := cmr.(orca.ServerMetricsProvider).ServerMetrics() + r.SetApplicationUtilization(sm.AppUtilization) + r.SetQPS(sm.QPS) + r.SetEPS(sm.EPS) + } + return &testpb.Empty{}, nil + }, + }, orca.CallMetricsServerOption(nil)) + port1 := itestutils.ParsePort(t, backend1.Address) + defer backend1.Stop() + + cmr.SetQPS(10.0) + cmr.SetApplicationUtilization(1.0) + + backend2 := stubserver.StartTestService(t, &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + if r := orca.CallMetricsRecorderFromContext(ctx); r != nil { + // Copy metrics from what the test set in cmr into r. + sm := cmr.(orca.ServerMetricsProvider).ServerMetrics() + r.SetApplicationUtilization(sm.AppUtilization) + r.SetQPS(sm.QPS) + r.SetEPS(sm.EPS) + } + return &testpb.Empty{}, nil + }, + }, orca.CallMetricsServerOption(nil)) + port2 := itestutils.ParsePort(t, backend2.Address) + defer backend2.Stop() + + const serviceName = "my-service-client-side-xds" + + // Start an xDS management server. + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + wrrConfig := &v3wrrlocalitypb.WrrLocality{ + EndpointPickingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: itestutils.MarshalAny(t, &v3clientsideweightedroundrobinpb.ClientSideWeightedRoundRobin{ + EnableOobLoadReport: &wrapperspb.BoolValue{ + Value: false, + }, + // BlackoutPeriod long enough to cause load report + // weight to trigger in the scope of test case. + // WeightExpirationPeriod will cause the load report + // weight for backend 1 to expire. + BlackoutPeriod: durationpb.New(5 * time.Millisecond), + WeightExpirationPeriod: durationpb.New(500 * time.Millisecond), + WeightUpdatePeriod: durationpb.New(time.Second), + ErrorUtilizationPenalty: &wrapperspb.FloatValue{Value: 1}, + }), + }, + }, + }, + }, + } + + routeConfigName := "route-" + serviceName + clusterName := "cluster-" + serviceName + endpointsName := "endpoints-" + serviceName + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{clusterWithLBConfiguration(t, clusterName, endpointsName, e2e.SecurityLevelNone, wrrConfig)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: endpointsName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Backends: []e2e.BackendOptions{{Ports: []uint32{port1}}, {Ports: []uint32{port2}}}, + Weight: 1, + }, + }, + })}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + + mo := opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics().Add("grpc.lb.wrr.rr_fallback", "grpc.lb.wrr.endpoint_weight_not_yet_usable", "grpc.lb.wrr.endpoint_weight_stale", "grpc.lb.wrr.endpoint_weights"), + OptionalLabels: []string{"grpc.lb.locality", "grpc.lb.backend_service"}, + } + + target := fmt.Sprintf("xds:///%s", serviceName) + cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver), opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo})) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + // Make 100 RPC's. The two backends will send back load reports per call + // giving the two SubChannels weights which will eventually expire. Two + // backends needed as for only one backend, WRR does not recompute the + // scheduler. + receivedExpectedMetrics := grpcsync.NewEvent() + go func() { + for !receivedExpectedMetrics.HasFired() && ctx.Err() == nil { + client.EmptyCall(ctx, &testpb.Empty{}) + time.Sleep(2 * time.Millisecond) + } + }() + + targetAttr := attribute.String("grpc.target", target) + localityAttr := attribute.String("grpc.lb.locality", `{region="region-1", zone="zone-1", sub_zone="subzone-1"}`) + backendServiceAttr := attribute.String("grpc.lb.backend_service", clusterName) + + wantMetrics := []metricdata.Metrics{ + { + Name: "grpc.lb.wrr.rr_fallback", + Description: "EXPERIMENTAL. Number of scheduler updates in which there were not enough endpoints with valid weight, which caused the WRR policy to fall back to RR behavior.", + Unit: "{update}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(targetAttr, localityAttr, backendServiceAttr), + Value: 1, // value ignored + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + + { + Name: "grpc.lb.wrr.endpoint_weight_not_yet_usable", + Description: "EXPERIMENTAL. Number of endpoints from each scheduler update that don't yet have usable weight information (i.e., either the load report has not yet been received, or it is within the blackout period).", + Unit: "{endpoint}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(targetAttr, localityAttr, backendServiceAttr), + Value: 1, // value ignored + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "grpc.lb.wrr.endpoint_weights", + Description: "EXPERIMENTAL. Weight of each endpoint, recorded on every scheduler update. Endpoints without usable weights will be recorded as weight 0.", + Unit: "{endpoint}", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attribute.NewSet(targetAttr, localityAttr, backendServiceAttr), + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + } + + if err := pollForWantMetrics(ctx, t, reader, wantMetrics); err != nil { + t.Fatal(err) + } + receivedExpectedMetrics.Fire() + + // Poll for 5 seconds for weight expiration metric. No more RPC's are being + // made, so weight should expire on a subsequent scheduler update. + eventuallyWantMetric := metricdata.Metrics{ + Name: "grpc.lb.wrr.endpoint_weight_stale", + Description: "EXPERIMENTAL. Number of endpoints from each scheduler update whose latest weight is older than the expiration period.", + Unit: "{endpoint}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(targetAttr, localityAttr, backendServiceAttr), + Value: 1, // value ignored + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + } + + if err := pollForWantMetrics(ctx, t, reader, []metricdata.Metrics{eventuallyWantMetric}); err != nil { + t.Fatal(err) + } +} + +// pollForWantMetrics polls for the wantMetrics to show up on reader. Returns an +// error if metric is present but not equal to expected, or if the wantMetrics +// do not show up during the context timeout. +func pollForWantMetrics(ctx context.Context, t *testing.T, reader *metric.ManualReader, wantMetrics []metricdata.Metrics) error { + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + gotMetrics := metricsDataFromReader(ctx, reader) + containsAllMetrics := true + for _, metric := range wantMetrics { + val, ok := gotMetrics[metric.Name] + if !ok { + containsAllMetrics = false + break + } + if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreValue(), metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) { + return fmt.Errorf("metrics data type not equal for metric: %v", metric.Name) + } + } + if containsAllMetrics { + return nil + } + time.Sleep(5 * time.Millisecond) + } + + return fmt.Errorf("error waiting for metrics %v: %v", wantMetrics, ctx.Err()) +} + +// TestMetricsAndTracesOptionEnabled verifies the integration of metrics and traces +// emitted by the OpenTelemetry instrumentation in a gRPC environment. It sets up a +// stub server with both metrics and traces enabled, and tests the correct emission +// of metrics and traces during a Unary RPC and a Streaming RPC. The test ensures +// that the emitted metrics reflect the operations performed, including the size of +// the compressed message, and verifies that tracing information is correctly recorded. +func (s) TestMetricsAndTracesOptionEnabled(t *testing.T) { + // Create default metrics options + mo, reader := defaultMetricsOptions(t, nil) + // Create default trace options + to, exporter := defaultTraceOptions(t) + + ss := setupStubServer(t, mo, to) + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout*2) + defer cancel() + + // Make two RPC's, a unary RPC and a streaming RPC. These should cause + // certain metrics and traces to be emitted which should be observed + // through metrics reader and span exporter respectively. + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}, grpc.UseCompressor(gzip.Name)); err != nil { // Deterministic compression. + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err) + } + + // Verify metrics + rm := &metricdata.ResourceMetrics{} + reader.Collect(ctx, rm) + + gotMetrics := map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + gotMetrics[m.Name] = m + } + } + + compressedSize := testutils.GzipCompressedMessageSize(t, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: make([]byte, 10000)}}) + wantMetrics := testutils.MetricData(testutils.MetricDataOptions{ + Target: ss.Target, + UnaryCompressedMessageSize: float64(compressedSize), + }) + gotMetrics = testutils.WaitForServerMetrics(ctx, t, reader, gotMetrics, wantMetrics) + testutils.CompareMetrics(t, gotMetrics, wantMetrics) + + wantSpanInfos := []traceSpanInfo{ + { + name: "Recv.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + { + Key: "sequence-number", + Value: attribute.IntValue(0), + }, + { + Key: "message-size", + Value: attribute.IntValue(10006), + }, + { + Key: "message-size-compressed", + Value: attribute.IntValue(compressedSize), + }, + }, + }, + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + { + Key: "sequence-number", + Value: attribute.IntValue(0), + }, + { + Key: "message-size", + Value: attribute.IntValue(10006), + }, + { + Key: "message-size-compressed", + Value: attribute.IntValue(compressedSize), + }, + }, + }, + }, + }, + { + name: "Attempt.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, + attributes: []attribute.KeyValue{ + { + Key: "previous-rpc-attempts", + Value: attribute.IntValue(0), + }, + { + Key: "transparent-retry", + Value: attribute.BoolValue(false), + }, + }, + events: []trace.Event{ + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + { + Key: "sequence-number", + Value: attribute.IntValue(0), + }, + { + Key: "message-size", + Value: attribute.IntValue(10006), + }, + { + Key: "message-size-compressed", + Value: attribute.IntValue(compressedSize), + }, + }, + }, + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + { + Key: "sequence-number", + Value: attribute.IntValue(0), + }, + { + Key: "message-size", + Value: attribute.IntValue(10006), + }, + { + Key: "message-size-compressed", + Value: attribute.IntValue(compressedSize), + }, + }, + }, + }, + }, + { + name: "Sent.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, + attributes: nil, + events: nil, + }, + { + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, + events: nil, + }, + { + name: "Sent.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, + attributes: nil, + events: nil, + }, + { + name: "Attempt.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, + attributes: []attribute.KeyValue{ + { + Key: "previous-rpc-attempts", + Value: attribute.IntValue(0), + }, + { + Key: "transparent-retry", + Value: attribute.BoolValue(false), + }, + }, + events: nil, + }, + } + + spans, err := waitForTraceSpans(ctx, exporter, wantSpanInfos) + if err != nil { + t.Fatal(err) + } + validateTraces(t, spans, wantSpanInfos) +} + +// TestTracingOnlyOptionEnabled verifies that tracing works correctly when only +// tracing is enabled (metrics are disabled). It ensures that the method name +// is correctly populated in both client and server spans. +func (s) TestTracingOnlyOptionEnabled(t *testing.T) { + opts, exporter := defaultTraceOptions(t) + + ss := setupStubServer(t, nil, opts) + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{ + Body: make([]byte, 100), + }}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + + wantSpanInfos := []traceSpanInfo{ + { + name: "Recv.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + {Key: "sequence-number", Value: attribute.IntValue(0)}, + {Key: "message-size", Value: attribute.IntValue(104)}, + }, + }, + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + {Key: "sequence-number", Value: attribute.IntValue(0)}, + {Key: "message-size", Value: attribute.IntValue(104)}, + }, + }, + }, + }, + { + name: "Attempt.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, + attributes: []attribute.KeyValue{ + {Key: "previous-rpc-attempts", Value: attribute.IntValue(0)}, + {Key: "transparent-retry", Value: attribute.BoolValue(false)}, + }, + events: []trace.Event{ + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + {Key: "sequence-number", Value: attribute.IntValue(0)}, + {Key: "message-size", Value: attribute.IntValue(104)}, + }, + }, + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + {Key: "sequence-number", Value: attribute.IntValue(0)}, + {Key: "message-size", Value: attribute.IntValue(104)}, + }, + }, + }, + }, + { + name: "Sent.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, + }, + } + + spans, err := waitForTraceSpans(ctx, exporter, wantSpanInfos) + if err != nil { + t.Fatal(err) + } + validateTraces(t, spans, wantSpanInfos) +} + +// TestSpan verifies that the gRPC Trace Binary propagator correctly +// propagates span context between a client and server using the grpc- +// trace-bin header. It sets up a stub server with OpenTelemetry tracing +// enabled, makes a unary RPC, and streaming RPC as well. +// +// Verification: +// - Verifies that the span context is correctly propagated from the client +// to the server, including the trace ID and span ID. +// - Verifies that the server can access the span context and create +// child spans as expected during the RPC calls. +// - Verifies that the tracing information is recorded accurately in +// the OpenTelemetry backend. +func (s) TestSpan(t *testing.T) { + mo, _ := defaultMetricsOptions(t, nil) + // Using defaultTraceOptions to set up OpenTelemetry with an in-memory exporter. + to, exporter := defaultTraceOptions(t) + // Start the server with trace options. + ss := setupStubServer(t, mo, to) + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Make two RPC's, a unary RPC and a streaming RPC. These should cause + // certain traces to be emitted, which should be observed through the + // span exporter. + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err) + } + + wantSpanInfos := []traceSpanInfo{ + { + name: "Recv.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + { + Key: "sequence-number", + Value: attribute.IntValue(0), + }, + { + Key: "message-size", + Value: attribute.IntValue(10006), + }, + }, + }, + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + { + Key: "sequence-number", + Value: attribute.IntValue(0), + }, + { + Key: "message-size", + Value: attribute.IntValue(10006), + }, + }, + }, + }, + }, + { + name: "Attempt.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, + attributes: []attribute.KeyValue{ + { + Key: "previous-rpc-attempts", + Value: attribute.IntValue(0), + }, + { + Key: "transparent-retry", + Value: attribute.BoolValue(false), + }, + }, + events: []trace.Event{ + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + { + Key: "sequence-number", + Value: attribute.IntValue(0), + }, + { + Key: "message-size", + Value: attribute.IntValue(10006), + }, + }, + }, + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + { + Key: "sequence-number", + Value: attribute.IntValue(0), + }, + { + Key: "message-size", + Value: attribute.IntValue(10006), + }, + }, + }, + }, + }, + { + name: "Sent.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, + attributes: nil, + events: nil, + }, + { + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, + events: nil, + }, + { + name: "Sent.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, + attributes: nil, + events: nil, + }, + { + name: "Attempt.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, + attributes: []attribute.KeyValue{ + { + Key: "previous-rpc-attempts", + Value: attribute.IntValue(0), + }, + { + Key: "transparent-retry", + Value: attribute.BoolValue(false), + }, + }, + events: nil, + }, + } + + spans, err := waitForTraceSpans(ctx, exporter, wantSpanInfos) + if err != nil { + t.Fatal(err) + } + validateTraces(t, spans, wantSpanInfos) +} + +// TestSpan_WithW3CContextPropagator sets up a stub server with OpenTelemetry tracing +// enabled, makes a unary and a streaming RPC, and then asserts that the correct +// number of spans are created with the expected spans. +// +// Verification: +// - Verifies that the correct number of spans are created for both unary and +// streaming RPCs. +// - Verifies that the spans have the expected names and attributes, ensuring +// they accurately reflect the operations performed. +// - Verifies that the trace ID and span ID are correctly assigned and accessible +// in the OpenTelemetry backend. +func (s) TestSpan_WithW3CContextPropagator(t *testing.T) { + mo, _ := defaultMetricsOptions(t, nil) + // Using defaultTraceOptions to set up OpenTelemetry with an in-memory exporter + to, exporter := defaultTraceOptions(t) + // Set the W3CContextPropagator as part of TracingOptions. + to.TextMapPropagator = propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}) + // Start the server with OpenTelemetry options + ss := setupStubServer(t, mo, to) + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Make two RPC's, a unary RPC and a streaming RPC. These should cause + // certain traces to be emitted, which should be observed through the + // span exporter. + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err) + } + + wantSpanInfos := []traceSpanInfo{ + { + name: "Recv.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + { + Key: "sequence-number", + Value: attribute.IntValue(0), + }, + { + Key: "message-size", + Value: attribute.IntValue(10006), + }, + }, + }, + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + { + Key: "sequence-number", + Value: attribute.IntValue(0), + }, + { + Key: "message-size", + Value: attribute.IntValue(10006), + }, + }, + }, + }, + }, + { + name: "Attempt.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, + attributes: []attribute.KeyValue{ + { + Key: "previous-rpc-attempts", + Value: attribute.IntValue(0), + }, + { + Key: "transparent-retry", + Value: attribute.BoolValue(false), + }, + }, + events: []trace.Event{ + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + { + Key: "sequence-number", + Value: attribute.IntValue(0), + }, + { + Key: "message-size", + Value: attribute.IntValue(10006), + }, + }, + }, + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + { + Key: "sequence-number", + Value: attribute.IntValue(0), + }, + { + Key: "message-size", + Value: attribute.IntValue(10006), + }, + }, + }, + }, + }, + { + name: "Sent.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, + attributes: nil, + events: nil, + }, + { + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, + events: nil, + }, + { + name: "Sent.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, + attributes: nil, + events: nil, + }, + { + name: "Attempt.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, + attributes: []attribute.KeyValue{ + { + Key: "previous-rpc-attempts", + Value: attribute.IntValue(0), + }, + { + Key: "transparent-retry", + Value: attribute.BoolValue(false), + }, + }, + events: nil, + }, + } + + spans, err := waitForTraceSpans(ctx, exporter, wantSpanInfos) + if err != nil { + t.Fatal(err) + } + validateTraces(t, spans, wantSpanInfos) +} + +// TestMetricsAndTracesDisabled verifies that RPCs call succeed as expected +// when metrics and traces are disabled in the OpenTelemetry instrumentation. +func (s) TestMetricsAndTracesDisabled(t *testing.T) { + ss := &stubserver.StubServer{ + UnaryCallF: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{Payload: &testpb.Payload{ + Body: make([]byte, len(in.GetPayload().GetBody())), + }}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + _, err := stream.Recv() + if err == io.EOF { + return nil + } + } + }, + } + + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Make two RPCs, a unary RPC and a streaming RPC. + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %v", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err) + } +} + +// TestRPCSpanErrorStatus verifies that errors during RPC calls are correctly +// reflected in the span status. It simulates a unary RPC that returns an error +// and checks that the span's status is set to error with the appropriate message. +func (s) TestRPCSpanErrorStatus(t *testing.T) { + mo, _ := defaultMetricsOptions(t, nil) + // Using defaultTraceOptions to set up OpenTelemetry with an in-memory exporter + to, exporter := defaultTraceOptions(t) + const rpcErrorMsg = "unary call: internal server error" + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return nil, fmt.Errorf("%v", rpcErrorMsg) + }, + } + + otelOptions := opentelemetry.Options{ + MetricsOptions: *mo, + TraceOptions: *to, + } + + if err := ss.Start([]grpc.ServerOption{opentelemetry.ServerOption(otelOptions)}, + opentelemetry.DialOption(otelOptions)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}) + + // Verify spans has error status with rpcErrorMsg as error message. + for ; len(exporter.GetSpans()) == 0 && ctx.Err() == nil; <-time.After(time.Millisecond) { + // wait until trace spans are collected + } + spans := exporter.GetSpans() + if got, want := spans[0].Status.Description, rpcErrorMsg; got != want { + t.Fatalf("got rpc error %s, want %s", spans[0].Status.Description, rpcErrorMsg) + } +} + +const delayedResolutionEventName = "Delayed name resolution complete" + +type testCCWrapper struct { + balancer.ClientConn + pickInvoked *grpcsync.Event +} + +func (t *testCCWrapper) UpdateState(state balancer.State) { + state.Picker = &blockingPicker{ + p: state.Picker, + pickInvoked: t.pickInvoked, + } + // Delay propagating the Ready connectivity state until Pick has been invoked + // at least once. This ensures the "Delayed LB pick complete" condition. + if state.ConnectivityState == connectivity.Ready { + go func() { + <-t.pickInvoked.Done() + t.ClientConn.UpdateState(state) + }() + return + } + t.ClientConn.UpdateState(state) +} + +type blockingPicker struct { + p balancer.Picker + pickInvoked *grpcsync.Event +} + +func (p *blockingPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + p.pickInvoked.Fire() + return p.p.Pick(info) +} + +// registerBlockingBalancer registers a stub balancer that delegates to +// pick_first. It wraps the ClientConn to intercept state updates and wraps +// the picker to detect calls to Pick. This guarantees the "Delayed LB pick +// complete" event is emitted reliably. +func registerBlockingBalancer(t *testing.T, balancerName string) { + t.Helper() + stub.Register(balancerName, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + cc := &testCCWrapper{ + ClientConn: bd.ClientConn, + pickInvoked: grpcsync.NewEvent(), + } + bd.ChildBalancer = balancer.Get(pickfirst.Name).Build(cc, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + ExitIdle: func(bd *stub.BalancerData) { + bd.ChildBalancer.ExitIdle() + }, + }) +} + +// TestTraceSpan_WithRetriesAndNameResolutionDelay verifies that +// "Delayed name resolution complete" event is recorded in the call trace span +// only once if any of the retry attempt encountered a delay in name resolution +func (s) TestTraceSpan_WithRetriesAndNameResolutionDelay(t *testing.T) { + tests := []struct { + name string + setupStub func() *stubserver.StubServer + doCall func(context.Context, testgrpc.TestServiceClient) error + wantSpanInfos []traceSpanInfo + }{ + { + name: "unary", + setupStub: func() *stubserver.StubServer { + return &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + md, _ := metadata.FromIncomingContext(ctx) + headerAttempts := 0 + if h := md["grpc-previous-rpc-attempts"]; len(h) > 0 { + headerAttempts, _ = strconv.Atoi(h[0]) + } + if headerAttempts < 2 { + return nil, status.Errorf(codes.Unavailable, "retry (%d)", headerAttempts) + } + return &testpb.SimpleResponse{}, nil + }, + } + }, + doCall: func(ctx context.Context, client testgrpc.TestServiceClient) error { + _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}) + return err + }, + wantSpanInfos: []traceSpanInfo{ + { + name: "Recv.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Error, + attributes: nil, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + // RPC attempt #1 + { + name: "Attempt.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Error, + attributes: []attribute.KeyValue{ + attribute.Int("previous-rpc-attempts", 0), + attribute.Bool("transparent-retry", false), + }, + events: []trace.Event{ + { + Name: "Delayed LB pick complete", + }, + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + { + name: "Recv.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Error, + attributes: nil, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + // RPC attempt #2 + { + name: "Attempt.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Error, + attributes: []attribute.KeyValue{ + attribute.Int("previous-rpc-attempts", 1), + attribute.Bool("transparent-retry", false), + }, + events: []trace.Event{ + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + { + name: "Recv.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + // RPC attempt #3 + { + name: "Attempt.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, + attributes: []attribute.KeyValue{ + attribute.Int("previous-rpc-attempts", 2), + attribute.Bool("transparent-retry", false), + }, + events: []trace.Event{ + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + { + name: "Sent.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, + attributes: nil, + events: []trace.Event{ + {Name: delayedResolutionEventName}, + }, + }, + }, + }, + { + name: "streaming", + setupStub: func() *stubserver.StubServer { + return &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil && err != io.EOF { + return err + } + md, _ := metadata.FromIncomingContext(stream.Context()) + headerAttempts := 0 + if h := md["grpc-previous-rpc-attempts"]; len(h) > 0 { + headerAttempts, _ = strconv.Atoi(h[0]) + } + if headerAttempts < 2 { + return status.Errorf(codes.Unavailable, "retry (%d)", headerAttempts) + } + for { + _, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + } + }, + } + }, + doCall: func(ctx context.Context, client testgrpc.TestServiceClient) error { + stream, err := client.FullDuplexCall(ctx) + if err != nil { + return err + } + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + return err + } + if err := stream.CloseSend(); err != nil { + return err + } + _, err = stream.Recv() + if err != nil && err != io.EOF { + return err + } + return nil + }, + wantSpanInfos: []traceSpanInfo{ + { + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Error, + attributes: nil, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + // RPC attempt #1 + { + name: "Attempt.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Error, + attributes: []attribute.KeyValue{ + attribute.Int("previous-rpc-attempts", 0), + attribute.Bool("transparent-retry", false), + }, + events: []trace.Event{ + { + Name: "Delayed LB pick complete", + }, + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + { + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Error, + attributes: nil, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + // RPC attempt #2 + { + name: "Attempt.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Error, + attributes: []attribute.KeyValue{ + attribute.Int("previous-rpc-attempts", 1), + attribute.Bool("transparent-retry", false), + }, + events: []trace.Event{ + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + { + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + // RPC attempt #3 + { + name: "Attempt.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, + attributes: []attribute.KeyValue{ + attribute.Int("previous-rpc-attempts", 2), + attribute.Bool("transparent-retry", false), + }, + events: []trace.Event{ + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + { + name: "Sent.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, + attributes: nil, + events: []trace.Event{ + {Name: delayedResolutionEventName}, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resolutionWait := grpcsync.NewEvent() + prevHook := internal.NewStreamWaitingForResolver + internal.NewStreamWaitingForResolver = func() { resolutionWait.Fire() } + defer func() { internal.NewStreamWaitingForResolver = prevHook }() + + blockingBalancerName := "test_blocking_balancer" + registerBlockingBalancer(t, blockingBalancerName) + + mo, _ := defaultMetricsOptions(t, nil) + to, exporter := defaultTraceOptions(t) + rb := manual.NewBuilderWithScheme("delayed") + ss := tt.setupStub() + opts := opentelemetry.Options{MetricsOptions: *mo, TraceOptions: *to} + if err := ss.Start([]grpc.ServerOption{opentelemetry.ServerOption(opts)}); err != nil { + t.Fatal(err) + } + defer ss.Stop() + + serviceConfig := fmt.Sprintf(`{ + "loadBalancingConfig": [{"%s":{}}], + "methodConfig": [{ + "name": [{"service": "grpc.testing.TestService"}], + "retryPolicy": { + "maxAttempts": 3, + "initialBackoff": "0.05s", + "maxBackoff": "0.2s", + "backoffMultiplier": 1.0, + "retryableStatusCodes": ["UNAVAILABLE"] + } + }] + }`, blockingBalancerName) + cc, err := grpc.NewClient( + rb.Scheme()+":///test.server", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(rb), + opentelemetry.DialOption(opts), + grpc.WithDefaultServiceConfig(serviceConfig), + ) + if err != nil { + t.Fatal(err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + go func() { + <-resolutionWait.Done() + rb.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}}) + }() + if err := tt.doCall(ctx, client); err != nil { + t.Fatalf("%s call failed: %v", tt.name, err) + } + spans, err := waitForTraceSpans(ctx, exporter, tt.wantSpanInfos) + if err != nil { + t.Fatal(err) + } + validateTraces(t, spans, tt.wantSpanInfos) + }) + } +} + +// TestStreamingRPC_TraceSequenceNumbers verifies that sequence numbers +// are incremented correctly for multiple messages sent and received +// during a streaming RPC. +func (s) TestStreamingRPC_TraceSequenceNumbers(t *testing.T) { + mo, _ := defaultMetricsOptions(t, nil) + to, exporter := defaultTraceOptions(t) + ss := setupStubServer(t, mo, to) + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + const numMessages = 3 + var wantOutboundEvents, wantInboundEvents []trace.Event + for i := range numMessages { + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("stream.Send() failed at message %d: %v", i, err) + } + wantOutboundEvents = append(wantOutboundEvents, trace.Event{ + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", i), + attribute.Int("message-size", 0), + }, + }) + wantInboundEvents = append(wantInboundEvents, trace.Event{ + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", i), + attribute.Int("message-size", 0), + }, + }) + } + stream.CloseSend() + _, err = stream.Recv() + if err != io.EOF { + t.Fatalf("stream.Recv() got unexpected err=%v; want io.EOF", err) + } + + wantSpanInfos := []traceSpanInfo{ + { + name: "Sent.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, + events: nil, + attributes: nil, + }, + { + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + events: wantInboundEvents, + attributes: nil, + }, + { + name: "Attempt.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, + events: wantOutboundEvents, + attributes: []attribute.KeyValue{ + attribute.Int("previous-rpc-attempts", 0), + attribute.Bool("transparent-retry", false), + }, + }, + } + + spans, err := waitForTraceSpans(ctx, exporter, wantSpanInfos) + if err != nil { + t.Fatal(err) + } + validateTraces(t, spans, wantSpanInfos) +} + +// TestSubChannelMetrics tests subchannel metrics emitted during connection +// lifecycle events (connect, disconnect, failure). +func (s) TestSubChannelMetrics(t *testing.T) { + // Start a single backend server. + backend := stubserver.StartTestService(t, nil) + port := itestutils.ParsePort(t, backend.Address) + defer backend.Stop() + + // Configure xDS for that single backend. + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + const serviceName = "my-service-client-side-xds" + clusterName := "cluster-" + serviceName + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Setup Telemetry. + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + mo := opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics().Add( + "grpc.subchannel.connection_attempts_succeeded", + "grpc.subchannel.open_connections", + "grpc.subchannel.disconnections", + "grpc.subchannel.connection_attempts_failed", + ), + OptionalLabels: []string{ + "grpc.lb.locality", + "grpc.lb.backend_service", + "grpc.security_level", + "grpc.disconnect_error", + }, + } + + target := fmt.Sprintf("xds:///%s", serviceName) + cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver), opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo})) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("rpc failed: %v", err) + } + + targetAttr := attribute.String("grpc.target", target) + localityAttr := attribute.String("grpc.lb.locality", `{region="region-1", zone="zone-1", sub_zone="subzone-1"}`) + backendServiceAttr := attribute.String("grpc.lb.backend_service", clusterName) + disconnectionReasonAttr := attribute.String("grpc.disconnect_error", "GOAWAY NO_ERROR") + securityLevelAttr := attribute.String("grpc.security_level", "NoSecurity") + + // Verify Connect Metrics. + wantMetrics := []metricdata.Metrics{ + { + Name: "grpc.subchannel.connection_attempts_succeeded", + Description: "EXPERIMENTAL. Number of successful connection attempts.", + Unit: "{attempt}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(targetAttr, backendServiceAttr, localityAttr), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "grpc.subchannel.open_connections", + Description: "EXPERIMENTAL. Number of open connections.", + Unit: "{attempt}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(targetAttr, backendServiceAttr, securityLevelAttr, localityAttr), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: false, + }, + }, + } + if err := pollForWantMetrics(ctx, t, reader, wantMetrics); err != nil { + t.Fatal(err) + } + + // Stop backend gracefully to trigger Disconnect Metrics with GOAWAY NO_ERROR. + backend.S.GracefulStop() + + disconnectionWantMetrics := []metricdata.Metrics{ + { + Name: "grpc.subchannel.disconnections", + Description: "EXPERIMENTAL. Number of times the selected subchannel becomes disconnected.", + Unit: "{disconnection}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(targetAttr, backendServiceAttr, localityAttr, disconnectionReasonAttr), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "grpc.subchannel.connection_attempts_failed", + Description: "EXPERIMENTAL. Number of failed connection attempts.", + Unit: "{attempt}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(targetAttr, backendServiceAttr, localityAttr), + Value: 1, // It will try to reconnect at least once + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + } + + if err := pollForWantMetrics(ctx, t, reader, disconnectionWantMetrics); err != nil { + t.Fatal(err) + } +} + +// TestSubChannelMetrics_ConnectionLifecycle verifies the subchannel metrics +// emitted during a connection lifecycle - a failed connection attempt, a +// successful connection attempt and a disconnection - on a gRPC channel +// configured with the pick_first LB policy. +func (s) TestSubChannelMetrics_ConnectionLifecycle(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + ss := stubserver.StartTestService(t, nil) + defer ss.Stop() + + pfConfig := fmt.Sprintf(`{"loadBalancingConfig": [{%q: {}}]}`, pickfirst.Name) + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(pfConfig) + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{ + ServiceConfig: sc, + Addresses: []resolver.Address{{Addr: "bad address"}}}, + ) // Will trigger connection failed. + + grpcTarget := r.Scheme() + ":///" + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + mo := opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics().Add("grpc.subchannel.disconnections", "grpc.subchannel.connection_attempts_succeeded", "grpc.subchannel.connection_attempts_failed"), + } + + cc, err := grpc.NewClient(grpcTarget, opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo}), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("NewClient() failed with error: %v", err) + } + defer cc.Close() + + tsc := testgrpc.NewTestServiceClient(cc) + if _, err := tsc.EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Fatalf("EmptyCall() passed when expected to fail") + } + + r.UpdateState(resolver.State{ + ServiceConfig: sc, + Addresses: []resolver.Address{{Addr: ss.Address}}, + }) // Will trigger successful connection metric. + if _, err := tsc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Stop the server, that should send signal to disconnect, which will + // eventually emit disconnection metric before ClientConn goes IDLE. + ss.Stop() + itestutils.AwaitState(ctx, t, cc, connectivity.Idle) + wantMetrics := []metricdata.Metrics{ + { + Name: "grpc.subchannel.connection_attempts_succeeded", + Description: "EXPERIMENTAL. Number of successful connection attempts.", + Unit: "{attempt}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget)), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "grpc.subchannel.connection_attempts_failed", + Description: "EXPERIMENTAL. Number of failed connection attempts.", + Unit: "{attempt}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget)), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "grpc.subchannel.disconnections", + Description: "EXPERIMENTAL. Number of times the selected subchannel becomes disconnected.", + Unit: "{disconnection}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("grpc.target", grpcTarget)), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + } + + gotMetrics := metricsDataFromReader(ctx, reader) + for _, metric := range wantMetrics { + val, ok := gotMetrics[metric.Name] + if !ok { + t.Fatalf("Metric %v not present in recorded metrics", metric.Name) + } + if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) { + t.Fatalf("Metrics data type not equal for metric: %v", metric.Name) + } + } +} + +func (s) TestHealthStreamNoOtelErrorLog(t *testing.T) { + mo, _ := defaultMetricsOptions(t, nil) + to, _ := defaultTraceOptions(t) + otelOptions := opentelemetry.Options{ + MetricsOptions: *mo, + TraceOptions: *to, + } + + healthcheck := health.NewServer() + + backend := stubserver.StartTestService(t, &stubserver.StubServer{ + EmptyCallF: func(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + }, stubserver.RegisterServiceServerOption(func(s grpc.ServiceRegistrar) { + healthgrpc.RegisterHealthServer(s, healthcheck) + })) + defer backend.Stop() + + healthcheck.SetServingStatus("", healthpb.HealthCheckResponse_SERVING) + + // Dial with healthCheckConfig to trigger the internal health stream. + sc := `{"loadBalancingConfig": [{"round_robin":{}}], "healthCheckConfig": {"serviceName": ""}}` + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + opentelemetry.DialOption(otelOptions), + grpc.WithDefaultServiceConfig(sc), + } + + cc, err := grpc.NewClient(backend.Address, dopts...) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + defer cc.Close() + + // Perform an RPC to ensure the subchannel connects and health stream initializes. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } +} + +// errorConn wraps a standard net.Conn to allow manual error injection. +// It is used to simulate transport-level failures (like ECONNRESET) that +// are otherwise difficult to trigger deterministically in a loopback environment. +type errorConn struct { + net.Conn + mu sync.Mutex + err error +} + +func (c *errorConn) setError(err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.err = err +} + +func (c *errorConn) getErr() error { + c.mu.Lock() + defer c.mu.Unlock() + return c.err +} + +// Read overrides the underlying net.Conn.Read to prioritize injected errors. +func (c *errorConn) Read(b []byte) (int, error) { + if err := c.getErr(); err != nil { + return 0, err + } + n, err := c.Conn.Read(b) + if injectErr := c.getErr(); injectErr != nil { + return 0, injectErr + } + return n, err +} + +// TestSubChannelDisconnectionLabels verifies that gRPC correctly labels +// subchannel disconnection metrics based on the root cause of the disconnect. +func (s) TestSubChannelDisconnectionLabels(t *testing.T) { + tests := []struct { + name string + wantLabel string + action func(backend *stubserver.StubServer, cc *grpc.ClientConn, injectErr func(error)) + }{ + { + name: "GoAway", + wantLabel: "GOAWAY NO_ERROR", + action: func(backend *stubserver.StubServer, _ *grpc.ClientConn, _ func(error)) { + backend.S.GracefulStop() + }, + }, + { + name: "IO_Error", + wantLabel: "unknown", + action: func(_ *stubserver.StubServer, _ *grpc.ClientConn, injectErr func(error)) { + injectErr(io.EOF) + }, + }, + { + name: "SubchannelShutdown", + wantLabel: "subchannel shutdown", + action: func(_ *stubserver.StubServer, cc *grpc.ClientConn, _ func(error)) { + cc.Close() + }, + }, + { + name: "ConnectionReset", + wantLabel: "connection reset", + action: func(_ *stubserver.StubServer, _ *grpc.ClientConn, injectErr func(error)) { + injectErr(syscall.ECONNRESET) + }, + }, + { + name: "ConnectionTimeout", + wantLabel: "connection timed out", + action: func(_ *stubserver.StubServer, _ *grpc.ClientConn, injectErr func(error)) { + injectErr(context.DeadlineExceeded) + }, + }, + { + name: "SocketError", + wantLabel: "socket error", + action: func(_ *stubserver.StubServer, _ *grpc.ClientConn, injectErr func(error)) { + injectErr(syscall.ECONNREFUSED) + }, + }, + { + name: "ConnectionAborted", + wantLabel: "connection aborted", + action: func(_ *stubserver.StubServer, _ *grpc.ClientConn, injectErr func(error)) { + injectErr(syscall.ECONNABORTED) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runDisconnectScenario(t, tt.name, tt.wantLabel, tt.action) + }) + } +} + +// runDisconnectScenario sets up a functional xDS environment with OpenTelemetry enabled. +// It performs the following steps: +// 1. Starts a backend and configures an xDS management server. +// 2. Establishes a gRPC connection using a custom dialer to capture the underlying transport. +// 3. Performs a warm-up RPC to ensure the subchannel is active. +// 4. Executes the provided 'action' callback to trigger a specific disconnection event. +// 5. Polls the OpenTelemetry metric reader to verify the 'grpc.disconnect_error' label matches wantLabel. +func runDisconnectScenario(t *testing.T, name, wantLabel string, action func(*stubserver.StubServer, *grpc.ClientConn, func(error))) { + backend := stubserver.StartTestService(t, nil) + port := itestutils.ParsePort(t, backend.Address) + defer backend.Stop() + + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + serviceName := "service-" + name + clusterName := "cluster-" + serviceName + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update xDS resources: %v", err) + } + + // Setup Telemetry + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + mo := opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics().Add("grpc.subchannel.disconnections"), + OptionalLabels: []string{"grpc.lb.locality", "grpc.lb.backend_service", "grpc.disconnect_error"}, + } + + connCh := make(chan *errorConn, 1) + dialer := func(ctx context.Context, addr string) (net.Conn, error) { + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", addr) + if err != nil { + return nil, err + } + activeConn := &errorConn{Conn: conn} + select { + case connCh <- activeConn: + case <-ctx.Done(): + activeConn.Conn.Close() + return nil, ctx.Err() + } + return activeConn, nil + } + + target := fmt.Sprintf("xds:///%s", serviceName) + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(xdsResolver), + grpc.WithContextDialer(dialer), + opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo}), + } + cc, err := grpc.NewClient(target, dopts...) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + defer cc.Close() + + // Perform warm-up call to ensure the subchannel and connection are ready. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("Warm-up EmptyCall failed: %v", err) + } + + var activeConn *errorConn + select { + case activeConn = <-connCh: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for connection from dialer: %v", ctx.Err()) + } + + // injectErr provides a way for test cases to force the transport to fail + // with a specific error and immediately close the physical socket. + injectErr := func(err error) { + activeConn.setError(err) + activeConn.Conn.Close() + } + + // Trigger the disconnection scenario + action(backend, cc, injectErr) + + // Define expected metric state + wantMetrics := []metricdata.Metrics{ + { + Name: "grpc.subchannel.disconnections", + Description: "EXPERIMENTAL. Number of times the selected subchannel becomes disconnected.", + Unit: "{disconnection}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.String("grpc.target", target), + attribute.String("grpc.lb.backend_service", clusterName), + attribute.String("grpc.lb.locality", `{region="region-1", zone="zone-1", sub_zone="subzone-1"}`), + attribute.String("grpc.disconnect_error", wantLabel), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + } + + // Poll for metrics to allow for OTel asynchronous processing + if err := pollForWantMetrics(ctx, t, reader, wantMetrics); err != nil { + t.Fatalf("Metric verification failed for case %s: %v", name, err) + } +} + +// TestRelayContextCollisionMetrics verifies that when an application acts as +// both a server and a client using the same context, the client metrics do not +// inherit or overwrite the server's telemetry metadata (e.g., grpc.method). +func (s) TestRelayContextCollisionMetrics(t *testing.T) { + backendMetricsOpts, _ := defaultMetricsOptions(t, nil) + backendServer := setupStubServer(t, backendMetricsOpts, nil) + backendServer.EmptyCallF = func(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + return nil, status.Error(codes.Unimplemented, "EmptyCall not implemented") + } + defer backendServer.Stop() + + relayMetricsOpts, relayMetricsReader := defaultMetricsOptions(t, nil) + otelOpts := opentelemetry.Options{MetricsOptions: *relayMetricsOpts} + + relayServer := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + relayCC, err := grpc.NewClient( + backendServer.Address, + grpc.WithTransportCredentials(insecure.NewCredentials()), + opentelemetry.DialOption(otelOpts), + ) + if err != nil { + return nil, fmt.Errorf("failed to create relay client: %v", err) + } + defer relayCC.Close() + client := testgrpc.NewTestServiceClient(relayCC) + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if status.Code(err) != codes.Unimplemented { + t.Errorf("Expected Unimplemented error, got: %v", err) + } + return &testpb.SimpleResponse{}, nil + }, + } + if err := relayServer.Start([]grpc.ServerOption{opentelemetry.ServerOption(otelOpts)}, opentelemetry.DialOption(otelOpts)); err != nil { + t.Fatalf("Failed to start relay server: %v", err) + } + defer relayServer.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if _, err := relayServer.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected UnaryCall error: %v", err) + } + + // Verify Server Metric Identity is retained. + if err := checkMetricWithMethod(ctx, relayMetricsReader, "grpc.server.call.started", "grpc.testing.TestService/UnaryCall"); err != nil { + t.Fatal(err) + } + + // Verify Client Metric Identity correctly resolved to "grpc.testing.TestService/EmptyCall". + if err := checkMetricWithMethod(ctx, relayMetricsReader, "grpc.client.attempt.started", "grpc.testing.TestService/EmptyCall"); err != nil { + t.Fatal(err) + } +} + +// TestRelayContextCollisionTracing verifies that span context is correctly +// propagated from incoming server requests to outgoing client requests without +// the client span accidentally adopting the server's identity or breaking the +// trace chain. +func (s) TestRelayContextCollisionTracing(t *testing.T) { + backendTraceOpts, _ := defaultTraceOptions(t) + backendServer := setupStubServer(t, nil, backendTraceOpts) + backendServer.EmptyCallF = func(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + return nil, status.Error(codes.Unimplemented, "EmptyCall not implemented") + } + defer backendServer.Stop() + + relayTraceOpts, relayTraceExporter := defaultTraceOptions(t) + otelOpts := opentelemetry.Options{TraceOptions: *relayTraceOpts} + + relayServer := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + relayCC, err := grpc.NewClient( + backendServer.Address, + grpc.WithTransportCredentials(insecure.NewCredentials()), + opentelemetry.DialOption(otelOpts), + ) + if err != nil { + return nil, fmt.Errorf("failed to create relay client: %v", err) + } + defer relayCC.Close() + client := testgrpc.NewTestServiceClient(relayCC) + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if status.Code(err) != codes.Unimplemented { + t.Errorf("Expected Unimplemented error, got: %v", err) + } + return &testpb.SimpleResponse{}, nil + }, + } + if err := relayServer.Start([]grpc.ServerOption{opentelemetry.ServerOption(otelOpts)}, opentelemetry.DialOption(otelOpts)); err != nil { + t.Fatalf("Failed to start relay server: %v", err) + } + defer relayServer.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + _, _ = relayServer.Client.UnaryCall(ctx, &testpb.SimpleRequest{}) + + wantSpans := []traceSpanInfo{ + {name: "Recv.grpc.testing.TestService.UnaryCall", spanKind: "server"}, + {name: "Sent.grpc.testing.TestService.EmptyCall", spanKind: "client"}, + } + spans, err := waitForTraceSpans(ctx, relayTraceExporter, wantSpans) + if err != nil { + t.Fatalf("Failed to wait for spans: %v", err) + } + + var srvTraceID, cliTraceID oteltrace.TraceID + for _, span := range spans { + if span.Name == "Recv.grpc.testing.TestService.UnaryCall" && span.SpanKind == oteltrace.SpanKindServer { + srvTraceID = span.SpanContext.TraceID() + } + if span.Name == "Sent.grpc.testing.TestService.EmptyCall" && span.SpanKind == oteltrace.SpanKindClient { + cliTraceID = span.SpanContext.TraceID() + } + } + if !srvTraceID.IsValid() || !cliTraceID.IsValid() { + t.Fatalf("Invalid trace IDs found. Server: %s, Client: %s", srvTraceID, cliTraceID) + } + + if srvTraceID != cliTraceID { + t.Errorf("Trace continuity broken: Server TraceID %s != Client TraceID %s", srvTraceID, cliTraceID) + } +} + +// baggageToMap converts the members of an OpenTelemetry baggage into a +// key/value map for easy comparison in tests. +func baggageToMap(b baggage.Baggage) map[string]string { + m := make(map[string]string, b.Len()) + for _, member := range b.Members() { + m[member.Key()] = member.Value() + } + return m +} + +// TestRPC_BaggagePropagation verifies that OpenTelemetry baggage set on the +// client-side context is propagated through the gRPC pipeline and is available +// in the server handler's context, when the W3C Baggage propagator is +// configured in the trace options. It exercises both a unary and a streaming +// RPC. +func (s) TestRPC_BaggagePropagation(t *testing.T) { + member1, err := baggage.NewMember("key1", "value1") + if err != nil { + t.Fatalf("baggage.NewMember(key1, value1) failed: %v", err) + } + member2, err := baggage.NewMember("key2", "value2") + if err != nil { + t.Fatalf("baggage.NewMember(key2, value2) failed: %v", err) + } + bag, err := baggage.New(member1, member2) + if err != nil { + t.Fatalf("baggage.New() failed: %v", err) + } + wantBaggage := map[string]string{"key1": "value1", "key2": "value2"} + + // serverBaggage receives the baggage observed by the server handler for + // each RPC. It is verified after each RPC, so a buffer of one suffices. + serverBaggage := make(chan map[string]string, 1) + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + serverBaggage <- baggageToMap(baggage.FromContext(ctx)) + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + serverBaggage <- baggageToMap(baggage.FromContext(stream.Context())) + for { + if _, err := stream.Recv(); err != nil { + if err == io.EOF { + return nil + } + return err + } + } + }, + } + + to, _ := defaultTraceOptions(t) + // Configure the W3C Baggage propagator so baggage is carried in metadata. + to.TextMapPropagator = propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}) + otelOptions := opentelemetry.Options{TraceOptions: *to} + if err := ss.Start([]grpc.ServerOption{opentelemetry.ServerOption(otelOptions)}, opentelemetry.DialOption(otelOptions)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = baggage.ContextWithBaggage(ctx, bag) + + verifyServerBaggage := func(rpc string) { + t.Helper() + select { + case got := <-serverBaggage: + if diff := cmp.Diff(wantBaggage, got); diff != "" { + t.Errorf("Baggage received by server for %s RPC mismatch (-want +got):\n%s", rpc, diff) + } + case <-ctx.Done(): + t.Fatalf("Timed out waiting for server to receive baggage for %s RPC", rpc) + } + } + + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + verifyServerBaggage("unary") + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %v", err) + } + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err) + } + verifyServerBaggage("streaming") +} + +// checkMetricWithMethod verifies that a metric with the specified name contains +// a data point matching the target grpc.method. It does not poll. +func checkMetricWithMethod(ctx context.Context, reader *metric.ManualReader, metricName, method string) error { + metrics := metricsDataFromReader(ctx, reader) + if m, ok := metrics[metricName]; ok { + if sum, ok := m.Data.(metricdata.Sum[int64]); ok { + for _, dp := range sum.DataPoints { + if val, ok := dp.Attributes.Value("grpc.method"); ok && val.AsString() == method { + return nil + } + } + } + } + return fmt.Errorf("metric %q with method %q not found", metricName, method) +} diff --git a/stats/opentelemetry/example_test.go b/stats/opentelemetry/example_test.go new file mode 100644 index 0000000..f232c16 --- /dev/null +++ b/stats/opentelemetry/example_test.go @@ -0,0 +1,151 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opentelemetry_test + +import ( + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/stats/opentelemetry" + + "go.opentelemetry.io/otel/sdk/metric" +) + +func ExampleDialOption_basic() { + // This is setting default bounds for a view. Setting these bounds through + // meter provider from SDK is recommended, as API calls in this module + // provide default bounds, but these calls are not guaranteed to be stable + // and API implementors are not required to implement bounds. Setting bounds + // through SDK ensures the bounds get picked up. The specific fields in + // Aggregation take precedence over defaults from API. For any fields unset + // in aggregation, defaults get picked up, so can have a mix of fields from + // SDK and fields created from API call. The overridden views themselves + // also follow same logic, only the specific views being created in the SDK + // use SDK information, the rest are created from API call. + reader := metric.NewManualReader() + provider := metric.NewMeterProvider( + metric.WithReader(reader), + metric.WithView(metric.NewView(metric.Instrument{ + Name: "grpc.client.call.duration", + }, + metric.Stream{ + Aggregation: metric.AggregationExplicitBucketHistogram{ + Boundaries: opentelemetry.DefaultSizeBounds, // The specific fields set in SDK take precedence over API. + }, + }, + )), + ) + + opts := opentelemetry.Options{ + MetricsOptions: opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics(), // equivalent to unset - distinct from empty + }, + } + do := opentelemetry.DialOption(opts) + cc, err := grpc.NewClient("", do, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + // Handle err. + } + defer cc.Close() +} + +func ExampleServerOption_methodFilter() { + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + opts := opentelemetry.Options{ + MetricsOptions: opentelemetry.MetricsOptions{ + MeterProvider: provider, + // Because Metrics is unset, the user will get default metrics. + MethodAttributeFilter: func(str string) bool { + // Will allow duplex/any other type of RPC. + return str != "/grpc.testing.TestService/UnaryCall" + }, + }, + } + cc, err := grpc.NewClient("some-target", opentelemetry.DialOption(opts), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + // Handle err. + } + defer cc.Close() +} + +func ExampleMetrics_excludeSome() { + // To exclude specific metrics, initialize Options as follows: + opts := opentelemetry.Options{ + MetricsOptions: opentelemetry.MetricsOptions{ + Metrics: opentelemetry.DefaultMetrics().Remove(opentelemetry.ClientAttemptDurationMetricName, opentelemetry.ClientAttemptRcvdCompressedTotalMessageSizeMetricName), + }, + } + do := opentelemetry.DialOption(opts) + cc, err := grpc.NewClient("", do, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + // Handle err. + } + defer cc.Close() +} + +func ExampleMetrics_disableAll() { + // To disable all metrics, initialize Options as follows: + opts := opentelemetry.Options{ + MetricsOptions: opentelemetry.MetricsOptions{ + Metrics: stats.NewMetricSet(), // Distinct to nil, which creates default metrics. This empty set creates no metrics. + }, + } + do := opentelemetry.DialOption(opts) + cc, err := grpc.NewClient("", do, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + // Handle err. + } + defer cc.Close() +} + +func ExampleMetrics_enableSome() { + // To only create specific metrics, initialize Options as follows: + opts := opentelemetry.Options{ + MetricsOptions: opentelemetry.MetricsOptions{ + Metrics: stats.NewMetricSet(opentelemetry.ClientAttemptDurationMetricName, opentelemetry.ClientAttemptRcvdCompressedTotalMessageSizeMetricName), // only create these metrics + }, + } + do := opentelemetry.DialOption(opts) + cc, err := grpc.NewClient("", do, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { // might fail vet + // Handle err. + } + defer cc.Close() +} + +func ExampleOptions_addExperimentalMetrics() { + opts := opentelemetry.Options{ + MetricsOptions: opentelemetry.MetricsOptions{ + // These are example experimental gRPC metrics, which are disabled + // by default and must be explicitly enabled. For the full, + // up-to-date list of metrics, see: + // https://grpc.io/docs/guides/opentelemetry-metrics/#instruments + Metrics: opentelemetry.DefaultMetrics().Add( + "grpc.subchannel.connection_attempts_succeeded", + "grpc.subchannel.connection_attempts_failed", + ), + }, + } + do := opentelemetry.DialOption(opts) + cc, err := grpc.NewClient("", do, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + // Handle error. + } + defer cc.Close() +} diff --git a/stats/opentelemetry/grpc_trace_bin_propagator.go b/stats/opentelemetry/grpc_trace_bin_propagator.go new file mode 100644 index 0000000..e8a3986 --- /dev/null +++ b/stats/opentelemetry/grpc_trace_bin_propagator.go @@ -0,0 +1,119 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package opentelemetry + +import ( + "context" + + otelpropagation "go.opentelemetry.io/otel/propagation" + oteltrace "go.opentelemetry.io/otel/trace" +) + +// gRPCTraceBinHeaderKey is the gRPC metadata header key `grpc-trace-bin` used +// to propagate trace context in binary format. +const grpcTraceBinHeaderKey = "grpc-trace-bin" + +// GRPCTraceBinPropagator is an OpenTelemetry TextMapPropagator which is used +// to extract and inject trace context data from and into headers exchanged by +// gRPC applications. It propagates trace data in binary format using the +// `grpc-trace-bin` header. +type GRPCTraceBinPropagator struct{} + +// Inject sets OpenTelemetry span context from the Context into the carrier as +// a `grpc-trace-bin` header if span context is valid. +// +// If span context is not valid, it returns without setting `grpc-trace-bin` +// header. +func (GRPCTraceBinPropagator) Inject(ctx context.Context, carrier otelpropagation.TextMapCarrier) { + sc := oteltrace.SpanFromContext(ctx) + if !sc.SpanContext().IsValid() { + return + } + + bd := toBinary(sc.SpanContext()) + carrier.Set(grpcTraceBinHeaderKey, string(bd)) +} + +// Extract reads OpenTelemetry span context from the `grpc-trace-bin` header of +// carrier into the provided context, if present. +// +// If a valid span context is retrieved from `grpc-trace-bin`, it returns a new +// context containing the extracted OpenTelemetry span context marked as +// remote. +// +// If `grpc-trace-bin` header is not present, it returns the context as is. +func (GRPCTraceBinPropagator) Extract(ctx context.Context, carrier otelpropagation.TextMapCarrier) context.Context { + h := carrier.Get(grpcTraceBinHeaderKey) + if h == "" { + return ctx + } + + sc, ok := fromBinary([]byte(h)) + if !ok { + return ctx + } + return oteltrace.ContextWithRemoteSpanContext(ctx, sc) +} + +// Fields returns the keys whose values are set with Inject. +// +// GRPCTraceBinPropagator always returns a slice containing only +// `grpc-trace-bin` key because it only sets the `grpc-trace-bin` header for +// propagating trace context. +func (GRPCTraceBinPropagator) Fields() []string { + return []string{grpcTraceBinHeaderKey} +} + +// toBinary returns the binary format representation of a SpanContext. +// +// If sc is the zero value, returns nil. +func toBinary(sc oteltrace.SpanContext) []byte { + if sc.Equal(oteltrace.SpanContext{}) { + return nil + } + var b [29]byte + traceID := oteltrace.TraceID(sc.TraceID()) + copy(b[2:18], traceID[:]) + b[18] = 1 + spanID := oteltrace.SpanID(sc.SpanID()) + copy(b[19:27], spanID[:]) + b[27] = 2 + b[28] = byte(oteltrace.TraceFlags(sc.TraceFlags())) + return b[:] +} + +// fromBinary returns the SpanContext represented by b with Remote set to true. +// +// It returns with zero value SpanContext and false, if any of the +// below condition is not satisfied: +// - Valid header: len(b) = 29 +// - Valid version: b[0] = 0 +// - Valid traceID prefixed with 0: b[1] = 0 +// - Valid spanID prefixed with 1: b[18] = 1 +// - Valid traceFlags prefixed with 2: b[27] = 2 +func fromBinary(b []byte) (oteltrace.SpanContext, bool) { + if len(b) != 29 || b[0] != 0 || b[1] != 0 || b[18] != 1 || b[27] != 2 { + return oteltrace.SpanContext{}, false + } + + return oteltrace.SpanContext{}.WithTraceID( + oteltrace.TraceID(b[2:18])).WithSpanID( + oteltrace.SpanID(b[19:27])).WithTraceFlags( + oteltrace.TraceFlags(b[28])).WithRemote(true), true +} diff --git a/stats/opentelemetry/grpc_trace_bin_propagator_test.go b/stats/opentelemetry/grpc_trace_bin_propagator_test.go new file mode 100644 index 0000000..2d575af --- /dev/null +++ b/stats/opentelemetry/grpc_trace_bin_propagator_test.go @@ -0,0 +1,219 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package opentelemetry + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + oteltrace "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc/metadata" + itracing "google.golang.org/grpc/stats/opentelemetry/internal/tracing" +) + +var validSpanContext = oteltrace.SpanContext{}.WithTraceID( + oteltrace.TraceID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}).WithSpanID( + oteltrace.SpanID{17, 18, 19, 20, 21, 22, 23, 24}).WithTraceFlags( + oteltrace.TraceFlags(1)) + +// TestInject_ValidSpanContext verifies that the GRPCTraceBinPropagator +// correctly injects a valid OpenTelemetry span context as `grpc-trace-bin` +// header in the provided carrier's context metadata. +// +// It verifies that if a valid span context is injected, same span context can +// can be retreived from the carrier's context metadata. +func (s) TestInject_ValidSpanContext(t *testing.T) { + p := GRPCTraceBinPropagator{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := itracing.NewOutgoingCarrier(ctx) + ctx = oteltrace.ContextWithSpanContext(ctx, validSpanContext) + + p.Inject(ctx, c) + + md, _ := metadata.FromOutgoingContext(c.Context()) + gotH := md.Get(grpcTraceBinHeaderKey) + if gotH[len(gotH)-1] == "" { + t.Fatalf("got empty value from Carrier's context metadata grpc-trace-bin header, want valid span context: %v", validSpanContext) + } + gotSC, ok := fromBinary([]byte(gotH[len(gotH)-1])) + if !ok { + t.Fatalf("got invalid span context %v from Carrier's context metadata grpc-trace-bin header, want valid span context: %v", gotSC, validSpanContext) + } + if cmp.Equal(validSpanContext, gotSC) { + t.Fatalf("got span context = %v, want span contexts %v", gotSC, validSpanContext) + } +} + +// TestInject_InvalidSpanContext verifies that the GRPCTraceBinPropagator does +// not inject an invalid OpenTelemetry span context as `grpc-trace-bin` header +// in the provided carrier's context metadata. +// +// If an invalid span context is injected, it verifies that `grpc-trace-bin` +// header is not set in the carrier's context metadata. +func (s) TestInject_InvalidSpanContext(t *testing.T) { + p := GRPCTraceBinPropagator{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := itracing.NewOutgoingCarrier(ctx) + ctx = oteltrace.ContextWithSpanContext(ctx, oteltrace.SpanContext{}) + + p.Inject(ctx, c) + + md, _ := metadata.FromOutgoingContext(c.Context()) + if gotH := md.Get(grpcTraceBinHeaderKey); len(gotH) > 0 { + t.Fatalf("got %v value from Carrier's context metadata grpc-trace-bin header, want empty", gotH) + } +} + +// TestExtract verifies that the GRPCTraceBinPropagator correctly extracts +// OpenTelemetry span context data from the provided context using carrier. +// +// If a valid span context was injected, it verifies same trace span context +// is extracted from carrier's metadata for `grpc-trace-bin` header key. +// +// If invalid span context was injected, it verifies that valid trace span +// context is not extracted. +func (s) TestExtract(t *testing.T) { + tests := []struct { + name string + wantSC oteltrace.SpanContext // expected span context from carrier + }{ + { + name: "valid OpenTelemetry span context", + wantSC: validSpanContext.WithRemote(true), + }, + { + name: "invalid OpenTelemetry span context", + wantSC: oteltrace.SpanContext{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + p := GRPCTraceBinPropagator{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ctx = metadata.NewIncomingContext(ctx, metadata.MD{grpcTraceBinHeaderKey: []string{string(toBinary(test.wantSC))}}) + + c := itracing.NewIncomingCarrier(ctx) + + tCtx := p.Extract(ctx, c) + got := oteltrace.SpanContextFromContext(tCtx) + if !got.Equal(test.wantSC) { + t.Fatalf("got span context: %v, want span context: %v", got, test.wantSC) + } + }) + } +} + +// TestBinary verifies that the toBinary() function correctly serializes a valid +// OpenTelemetry span context into its binary format representation. If span +// context is invalid, it verifies that serialization is nil. +func (s) TestToBinary(t *testing.T) { + tests := []struct { + name string + sc oteltrace.SpanContext + want []byte + }{ + { + name: "valid context", + sc: validSpanContext, + want: toBinary(validSpanContext), + }, + { + name: "zero value context", + sc: oteltrace.SpanContext{}, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := toBinary(test.sc); !cmp.Equal(got, test.want) { + t.Fatalf("binary() = %v, want %v", got, test.want) + } + }) + } +} + +// TestFromBinary verifies that the fromBinary() function correctly +// deserializes a binary format representation of a valid OpenTelemetry span +// context into its corresponding span context format. If span context's binary +// representation is invalid, it verifies that deserialization is zero value +// span context. +func (s) TestFromBinary(t *testing.T) { + tests := []struct { + name string + b []byte + want oteltrace.SpanContext + ok bool + }{ + { + name: "valid", + b: []byte{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 2, 1}, + want: validSpanContext.WithRemote(true), + ok: true, + }, + { + name: "invalid length", + b: []byte{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 2}, + want: oteltrace.SpanContext{}, + ok: false, + }, + { + name: "invalid version", + b: []byte{1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 2, 1}, + want: oteltrace.SpanContext{}, + ok: false, + }, + { + name: "invalid traceID field ID", + b: []byte{0, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 2, 1}, + want: oteltrace.SpanContext{}, + ok: false, + }, + { + name: "invalid spanID field ID", + b: []byte{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, 17, 18, 19, 20, 21, 22, 23, 24, 2, 1}, + want: oteltrace.SpanContext{}, + ok: false, + }, + { + name: "invalid traceFlags field ID", + b: []byte{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 1, 1}, + want: oteltrace.SpanContext{}, + ok: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, ok := fromBinary(test.b) + if ok != test.ok { + t.Fatalf("fromBinary() ok = %v, want %v", ok, test.ok) + return + } + if !got.Equal(test.want) { + t.Fatalf("fromBinary() got = %v, want %v", got, test.want) + } + }) + } +} diff --git a/stats/opentelemetry/internal/pluginoption.go b/stats/opentelemetry/internal/pluginoption.go new file mode 100644 index 0000000..b595aa8 --- /dev/null +++ b/stats/opentelemetry/internal/pluginoption.go @@ -0,0 +1,41 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package internal defines the PluginOption interface. +package internal + +import ( + "google.golang.org/grpc/metadata" +) + +// SetPluginOption sets the plugin option on Options. +var SetPluginOption any // func(*Options, PluginOption) + +// PluginOption is the interface which represents a plugin option for the +// OpenTelemetry instrumentation component. This plugin option emits labels from +// metadata and also creates metadata containing labels. These labels are +// intended to be added to applicable OpenTelemetry metrics recorded in the +// OpenTelemetry instrumentation component. +// +// In the future, we hope to stabilize and expose this API to allow plugins to +// inject labels of their choosing into metrics recorded. +type PluginOption interface { + // GetMetadata creates a MD with metadata exchange labels. + GetMetadata() metadata.MD + // GetLabels emits labels to be attached to metrics for the RPC that + // contains the provided incomingMetadata. + GetLabels(incomingMetadata metadata.MD) map[string]string +} diff --git a/stats/opentelemetry/internal/testutils/testutils.go b/stats/opentelemetry/internal/testutils/testutils.go new file mode 100644 index 0000000..28691fe --- /dev/null +++ b/stats/opentelemetry/internal/testutils/testutils.go @@ -0,0 +1,853 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package testutils contains helpers for OpenTelemetry tests. +package testutils + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "slices" + "testing" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" + "google.golang.org/protobuf/proto" +) + +// Redefine default bounds here to avoid a cyclic dependency with top level +// opentelemetry package. Could define once through internal, but would make +// external opentelemetry godoc less readable. +var ( + // DefaultLatencyBounds are the default bounds for latency metrics. + DefaultLatencyBounds = []float64{0, 0.00001, 0.00005, 0.0001, 0.0003, 0.0006, 0.0008, 0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.008, 0.01, 0.013, 0.016, 0.02, 0.025, 0.03, 0.04, 0.05, 0.065, 0.08, 0.1, 0.13, 0.16, 0.2, 0.25, 0.3, 0.4, 0.5, 0.65, 0.8, 1, 2, 5, 10, 20, 50, 100} + // DefaultSizeBounds are the default bounds for metrics which record size. + DefaultSizeBounds = []float64{0, 1024, 2048, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864, 268435456, 1073741824, 4294967296} +) + +// waitForServerCompletedRPCs waits until the unary and streaming stats.End +// calls are finished processing. It does this by waiting for the expected +// metric triggered by stats.End to appear through the passed in metrics reader. +// +// Returns a new gotMetrics map containing the metric data being polled for, or +// an error if failed to wait for metric. +func waitForServerCompletedRPCs(ctx context.Context, reader metric.Reader, wantMetric metricdata.Metrics) (map[string]metricdata.Metrics, error) { + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + rm := &metricdata.ResourceMetrics{} + reader.Collect(ctx, rm) + gotMetrics := map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + gotMetrics[m.Name] = m + } + } + val, ok := gotMetrics[wantMetric.Name] + if !ok { + continue + } + switch data := val.Data.(type) { + case metricdata.Histogram[int64]: + if len(wantMetric.Data.(metricdata.Histogram[int64]).DataPoints) > len(data.DataPoints) { + continue + } + case metricdata.Histogram[float64]: + if len(wantMetric.Data.(metricdata.Histogram[float64]).DataPoints) > len(data.DataPoints) { + continue + } + } + return gotMetrics, nil + } + return nil, fmt.Errorf("error waiting for metric %v: %v", wantMetric, ctx.Err()) +} + +// checkDataPointWithinFiveSeconds checks if the metric passed in contains a +// histogram with dataPoints that fall within buckets that are <=5. Returns an +// error if check fails. +func checkDataPointWithinFiveSeconds(metric metricdata.Metrics) error { + histo, ok := metric.Data.(metricdata.Histogram[float64]) + if !ok { + return fmt.Errorf("metric data is not histogram") + } + for _, dataPoint := range histo.DataPoints { + var boundWithFive int + for i, bound := range dataPoint.Bounds { + if bound >= 5 { + boundWithFive = i + } + } + foundPoint := false + for i, count := range dataPoint.BucketCounts { + if i >= boundWithFive { + return fmt.Errorf("data point not found in bucket <=5 seconds") + } + if count == 1 { + foundPoint = true + break + } + } + if !foundPoint { + return fmt.Errorf("no data point found for metric") + } + } + return nil +} + +// MetricDataOptions are the options used to configure the metricData emissions +// of expected metrics data from NewMetricData. +type MetricDataOptions struct { + // CSMLabels are the csm labels to attach to metrics which receive csm + // labels (all A66 expect client call and started RPC's client and server + // side). + CSMLabels []attribute.KeyValue + // Target is the target of the client and server. + Target string + // UnaryCallFailed is whether the Unary Call failed, which would trigger + // trailers only. + UnaryCallFailed bool + // UnaryCompressedMessageSize is the compressed message size of the Unary + // RPC. This assumes both client and server sent the same message size. + UnaryCompressedMessageSize float64 + // StreamingCompressedMessageSize is the compressed message size of the + // Streaming RPC. This assumes both client and server sent the same message + // size. + StreamingCompressedMessageSize float64 +} + +// createBucketCounts creates a list of bucket counts based off the +// recordingPoints and bounds. Both recordingPoints and bounds are assumed to be +// in order. +func createBucketCounts(recordingPoints []float64, bounds []float64) []uint64 { + var bucketCounts []uint64 + var recordingPointIndex int + for _, bound := range bounds { + var bucketCount uint64 + if recordingPointIndex >= len(recordingPoints) { + bucketCounts = append(bucketCounts, bucketCount) + continue + } + for recordingPoints[recordingPointIndex] <= bound { + bucketCount++ + recordingPointIndex++ + if recordingPointIndex >= len(recordingPoints) { + break + } + } + bucketCounts = append(bucketCounts, bucketCount) + } + // The rest of the recording points are last bound -> infinity. + bucketCounts = append(bucketCounts, uint64(len(recordingPoints)-recordingPointIndex)) + return bucketCounts +} + +// MetricDataUnary returns a list of expected metrics defined in A66 for a +// client and server for one unary RPC. +func MetricDataUnary(options MetricDataOptions) []metricdata.Metrics { + methodAttr := attribute.String("grpc.method", "grpc.testing.TestService/UnaryCall") + targetAttr := attribute.String("grpc.target", options.Target) + statusAttr := attribute.String("grpc.status", "OK") + if options.UnaryCallFailed { + statusAttr = attribute.String("grpc.status", "UNKNOWN") + } + clientSideEnd := []attribute.KeyValue{ + methodAttr, + targetAttr, + statusAttr, + } + serverSideEnd := []attribute.KeyValue{ + methodAttr, + statusAttr, + } + clientSideEnd = append(clientSideEnd, options.CSMLabels...) + serverSideEnd = append(serverSideEnd, options.CSMLabels...) + compressedBytesSentRecv := int64(options.UnaryCompressedMessageSize) + bucketCounts := createBucketCounts([]float64{options.UnaryCompressedMessageSize}, DefaultSizeBounds) + extrema := metricdata.NewExtrema(int64(options.UnaryCompressedMessageSize)) + return []metricdata.Metrics{ + { + Name: "grpc.client.attempt.started", + Description: "Number of client call attempts started.", + Unit: "{attempt}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(methodAttr, targetAttr), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "grpc.client.attempt.duration", + Description: "End-to-end time taken to complete a client call attempt.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attribute.NewSet(clientSideEnd...), + Count: 1, + Bounds: DefaultLatencyBounds, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.client.attempt.sent_total_compressed_message_size", + Description: "Compressed message bytes sent per client call attempt.", + Unit: "By", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(clientSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: bucketCounts, + Min: extrema, + Max: extrema, + Sum: compressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.client.attempt.rcvd_total_compressed_message_size", + Description: "Compressed message bytes received per call attempt.", + Unit: "By", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(clientSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: bucketCounts, + Min: extrema, + Max: extrema, + Sum: compressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.client.call.duration", + Description: "Time taken by gRPC to complete an RPC from application's perspective.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attribute.NewSet(methodAttr, targetAttr, statusAttr), + Count: 1, + Bounds: DefaultLatencyBounds, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.server.call.started", + Description: "Number of server calls started.", + Unit: "{call}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(methodAttr), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "grpc.server.call.sent_total_compressed_message_size", + Unit: "By", + Description: "Compressed message bytes sent per server call.", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(serverSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: bucketCounts, + Min: extrema, + Max: extrema, + Sum: compressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.server.call.rcvd_total_compressed_message_size", + Unit: "By", + Description: "Compressed message bytes received per server call.", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(serverSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: bucketCounts, + Min: extrema, + Max: extrema, + Sum: compressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.server.call.duration", + Description: "End-to-end time taken to complete a call from server transport's perspective.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attribute.NewSet(serverSideEnd...), + Count: 1, + Bounds: DefaultLatencyBounds, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + } +} + +// MetricDataStreaming returns a list of expected metrics defined in A66 for a +// client and server for one streaming RPC. +func MetricDataStreaming(options MetricDataOptions) []metricdata.Metrics { + methodAttr := attribute.String("grpc.method", "grpc.testing.TestService/FullDuplexCall") + targetAttr := attribute.String("grpc.target", options.Target) + statusAttr := attribute.String("grpc.status", "OK") + clientSideEnd := []attribute.KeyValue{ + methodAttr, + targetAttr, + statusAttr, + } + serverSideEnd := []attribute.KeyValue{ + methodAttr, + statusAttr, + } + clientSideEnd = append(clientSideEnd, options.CSMLabels...) + serverSideEnd = append(serverSideEnd, options.CSMLabels...) + compressedBytesSentRecv := int64(options.StreamingCompressedMessageSize) + bucketCounts := createBucketCounts([]float64{options.StreamingCompressedMessageSize}, DefaultSizeBounds) + extrema := metricdata.NewExtrema(int64(options.StreamingCompressedMessageSize)) + return []metricdata.Metrics{ + { + Name: "grpc.client.attempt.started", + Description: "Number of client call attempts started.", + Unit: "{attempt}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(methodAttr, targetAttr), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "grpc.client.attempt.duration", + Description: "End-to-end time taken to complete a client call attempt.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attribute.NewSet(clientSideEnd...), + Count: 1, + Bounds: DefaultLatencyBounds, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.client.attempt.sent_total_compressed_message_size", + Description: "Compressed message bytes sent per client call attempt.", + Unit: "By", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(clientSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: bucketCounts, + Min: extrema, + Max: extrema, + Sum: compressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.client.attempt.rcvd_total_compressed_message_size", + Description: "Compressed message bytes received per call attempt.", + Unit: "By", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(clientSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: bucketCounts, + Min: extrema, + Max: extrema, + Sum: compressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.client.call.duration", + Description: "Time taken by gRPC to complete an RPC from application's perspective.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attribute.NewSet(methodAttr, targetAttr, statusAttr), + Count: 1, + Bounds: DefaultLatencyBounds, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.server.call.started", + Description: "Number of server calls started.", + Unit: "{call}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(methodAttr), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "grpc.server.call.sent_total_compressed_message_size", + Unit: "By", + Description: "Compressed message bytes sent per server call.", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(serverSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: bucketCounts, + Min: extrema, + Max: extrema, + Sum: compressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.server.call.rcvd_total_compressed_message_size", + Unit: "By", + Description: "Compressed message bytes received per server call.", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(serverSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: bucketCounts, + Min: extrema, + Max: extrema, + Sum: compressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.server.call.duration", + Description: "End-to-end time taken to complete a call from server transport's perspective.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attribute.NewSet(serverSideEnd...), + Count: 1, + Bounds: DefaultLatencyBounds, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + } +} + +// MetricData returns a metricsDataSlice for A66 metrics for client and server +// with a unary RPC and streaming RPC with certain compression and message flow +// sent. If csmAttributes is set to true, the corresponding CSM Metrics (not +// client side call metrics, or started on client and server side). +func MetricData(options MetricDataOptions) []metricdata.Metrics { + unaryMethodAttr := attribute.String("grpc.method", "grpc.testing.TestService/UnaryCall") + duplexMethodAttr := attribute.String("grpc.method", "grpc.testing.TestService/FullDuplexCall") + targetAttr := attribute.String("grpc.target", options.Target) + unaryStatusAttr := attribute.String("grpc.status", "OK") + streamingStatusAttr := attribute.String("grpc.status", "OK") + if options.UnaryCallFailed { + unaryStatusAttr = attribute.String("grpc.status", "UNKNOWN") + } + unaryMethodClientSideEnd := []attribute.KeyValue{ + unaryMethodAttr, + targetAttr, + unaryStatusAttr, + } + streamingMethodClientSideEnd := []attribute.KeyValue{ + duplexMethodAttr, + targetAttr, + streamingStatusAttr, + } + unaryMethodServerSideEnd := []attribute.KeyValue{ + unaryMethodAttr, + unaryStatusAttr, + } + streamingMethodServerSideEnd := []attribute.KeyValue{ + duplexMethodAttr, + streamingStatusAttr, + } + + unaryMethodClientSideEnd = append(unaryMethodClientSideEnd, options.CSMLabels...) + streamingMethodClientSideEnd = append(streamingMethodClientSideEnd, options.CSMLabels...) + unaryMethodServerSideEnd = append(unaryMethodServerSideEnd, options.CSMLabels...) + streamingMethodServerSideEnd = append(streamingMethodServerSideEnd, options.CSMLabels...) + unaryCompressedBytesSentRecv := int64(options.UnaryCompressedMessageSize) + unaryBucketCounts := createBucketCounts([]float64{options.UnaryCompressedMessageSize}, DefaultSizeBounds) + unaryExtrema := metricdata.NewExtrema(int64(options.UnaryCompressedMessageSize)) + + streamingCompressedBytesSentRecv := int64(options.StreamingCompressedMessageSize) + streamingBucketCounts := createBucketCounts([]float64{options.StreamingCompressedMessageSize}, DefaultSizeBounds) + streamingExtrema := metricdata.NewExtrema(int64(options.StreamingCompressedMessageSize)) + + return []metricdata.Metrics{ + { + Name: "grpc.client.attempt.started", + Description: "Number of client call attempts started.", + Unit: "{attempt}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(unaryMethodAttr, targetAttr), + Value: 1, + }, + { + Attributes: attribute.NewSet(duplexMethodAttr, targetAttr), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "grpc.client.attempt.duration", + Description: "End-to-end time taken to complete a client call attempt.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attribute.NewSet(unaryMethodClientSideEnd...), + Count: 1, + Bounds: DefaultLatencyBounds, + }, + { + Attributes: attribute.NewSet(streamingMethodClientSideEnd...), + Count: 1, + Bounds: DefaultLatencyBounds, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.client.attempt.sent_total_compressed_message_size", + Description: "Compressed message bytes sent per client call attempt.", + Unit: "By", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(unaryMethodClientSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: unaryBucketCounts, + Min: unaryExtrema, + Max: unaryExtrema, + Sum: unaryCompressedBytesSentRecv, + }, + { + Attributes: attribute.NewSet(streamingMethodClientSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: streamingBucketCounts, + Min: streamingExtrema, + Max: streamingExtrema, + Sum: streamingCompressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.client.attempt.rcvd_total_compressed_message_size", + Description: "Compressed message bytes received per call attempt.", + Unit: "By", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(unaryMethodClientSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: unaryBucketCounts, + Min: unaryExtrema, + Max: unaryExtrema, + Sum: unaryCompressedBytesSentRecv, + }, + { + Attributes: attribute.NewSet(streamingMethodClientSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: streamingBucketCounts, + Min: streamingExtrema, + Max: streamingExtrema, + Sum: streamingCompressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.client.call.duration", + Description: "Time taken by gRPC to complete an RPC from application's perspective.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attribute.NewSet(unaryMethodAttr, targetAttr, unaryStatusAttr), + Count: 1, + Bounds: DefaultLatencyBounds, + }, + { + Attributes: attribute.NewSet(duplexMethodAttr, targetAttr, streamingStatusAttr), + Count: 1, + Bounds: DefaultLatencyBounds, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.server.call.started", + Description: "Number of server calls started.", + Unit: "{call}", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(unaryMethodAttr), + Value: 1, + }, + { + Attributes: attribute.NewSet(duplexMethodAttr), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "grpc.server.call.sent_total_compressed_message_size", + Unit: "By", + Description: "Compressed message bytes sent per server call.", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(unaryMethodServerSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: unaryBucketCounts, + Min: unaryExtrema, + Max: unaryExtrema, + Sum: unaryCompressedBytesSentRecv, + }, + { + Attributes: attribute.NewSet(streamingMethodServerSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: streamingBucketCounts, + Min: streamingExtrema, + Max: streamingExtrema, + Sum: streamingCompressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.server.call.rcvd_total_compressed_message_size", + Unit: "By", + Description: "Compressed message bytes received per server call.", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(unaryMethodServerSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: unaryBucketCounts, + Min: unaryExtrema, + Max: unaryExtrema, + Sum: unaryCompressedBytesSentRecv, + }, + { + Attributes: attribute.NewSet(streamingMethodServerSideEnd...), + Count: 1, + Bounds: DefaultSizeBounds, + BucketCounts: streamingBucketCounts, + Min: streamingExtrema, + Max: streamingExtrema, + Sum: streamingCompressedBytesSentRecv, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "grpc.server.call.duration", + Description: "End-to-end time taken to complete a call from server transport's perspective.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attribute.NewSet(unaryMethodServerSideEnd...), + Count: 1, + Bounds: DefaultLatencyBounds, + }, + { + Attributes: attribute.NewSet(streamingMethodServerSideEnd...), + Count: 1, + Bounds: DefaultLatencyBounds, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + } +} + +// GzipCompressedMessageSize returns the byte length of the proto message after +// gzip compression. +func GzipCompressedMessageSize(t *testing.T, m proto.Message) int { + t.Helper() + data, err := proto.Marshal(m) + if err != nil { + t.Fatalf("proto.Marshal failed: %v", err) + } + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := zw.Write(data); err != nil { + t.Fatalf("gzip.Write failed: %v", err) + } + if err := zw.Close(); err != nil { + t.Fatalf("gzip.Close failed: %v", err) + } + return buf.Len() +} + +// CompareMetrics asserts wantMetrics are what we expect. For duration metrics +// makes sure the data point is within possible testing time (five seconds from +// context timeout). +func CompareMetrics(t *testing.T, gotMetrics map[string]metricdata.Metrics, wantMetrics []metricdata.Metrics) { + for _, metric := range wantMetrics { + val, ok := gotMetrics[metric.Name] + if !ok { + t.Errorf("Metric %v not present in recorded metrics", metric.Name) + continue + } + + if metric.Name == "grpc.server.call.sent_total_compressed_message_size" || metric.Name == "grpc.server.call.rcvd_total_compressed_message_size" { + val := gotMetrics[metric.Name] + if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) { + t.Errorf("Metrics data type not equal for metric: %v", metric.Name) + } + continue + } + + // If one of the duration metrics, ignore the bucket counts, and make + // sure it count falls within a bucket <= 5 seconds (maximum duration of + // test due to context). + if metric.Name == "grpc.client.attempt.duration" || metric.Name == "grpc.client.call.duration" || metric.Name == "grpc.server.call.duration" { + val := gotMetrics[metric.Name] + if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars(), metricdatatest.IgnoreValue()) { + t.Errorf("Metrics data type not equal for metric: %v", metric.Name) + } + if err := checkDataPointWithinFiveSeconds(val); err != nil { + t.Errorf("Data point not within five seconds for metric %v: %v", metric.Name, err) + } + continue + } + + if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) { + t.Errorf("Metrics data type not equal for metric: %v", metric.Name) + } + } +} + +// WaitForServerMetrics waits for eventual server metrics (not emitted +// synchronously with client side rpc returning). +func WaitForServerMetrics(ctx context.Context, t *testing.T, mr *metric.ManualReader, gotMetrics map[string]metricdata.Metrics, wantMetrics []metricdata.Metrics) map[string]metricdata.Metrics { + terminalMetrics := []string{ + "grpc.server.call.sent_total_compressed_message_size", + "grpc.server.call.rcvd_total_compressed_message_size", + "grpc.client.attempt.duration", + "grpc.client.call.duration", + "grpc.server.call.duration", + } + for _, metric := range wantMetrics { + if !slices.Contains(terminalMetrics, metric.Name) { + continue + } + // Sync the metric reader to see the event because stats.End is + // handled async server side. Thus, poll until metrics created from + // stats.End show up. + var err error + if gotMetrics, err = waitForServerCompletedRPCs(ctx, mr, metric); err != nil { // move to shared helper + t.Fatal(err) + } + } + + return gotMetrics +} diff --git a/stats/opentelemetry/internal/tracing/carrier.go b/stats/opentelemetry/internal/tracing/carrier.go new file mode 100644 index 0000000..214102a --- /dev/null +++ b/stats/opentelemetry/internal/tracing/carrier.go @@ -0,0 +1,131 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package tracing implements the OpenTelemetry carrier for context propagation +// in gRPC tracing. +package tracing + +import ( + "context" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" +) + +var logger = grpclog.Component("otel-plugin") + +// IncomingCarrier is a TextMapCarrier that uses incoming `context.Context` to +// retrieve any propagated key-value pairs in text format. +type IncomingCarrier struct { + ctx context.Context +} + +// NewIncomingCarrier creates a new `IncomingCarrier` with the given context. +// The incoming carrier should be used with propagator's `Extract()` method in +// the incoming rpc path. +func NewIncomingCarrier(ctx context.Context) *IncomingCarrier { + return &IncomingCarrier{ctx: ctx} +} + +// Get returns the string value associated with the passed key from the +// carrier's incoming context metadata. +// +// It returns an empty string if the key is not present in the carrier's +// context or if the value associated with the key is empty. +// +// If multiple values are present for a key, it returns the last one. +func (c *IncomingCarrier) Get(key string) string { + values := metadata.ValueFromIncomingContext(c.ctx, key) + if len(values) == 0 { + return "" + } + return values[len(values)-1] +} + +// Set just logs an error. It implements the `TextMapCarrier` interface but +// should not be used with `IncomingCarrier`. +func (c *IncomingCarrier) Set(string, string) { + logger.Error("Set() should not be used with IncomingCarrier.") +} + +// Keys returns the keys stored in the carrier's context metadata. It returns +// keys from incoming context metadata. +func (c *IncomingCarrier) Keys() []string { + md, ok := metadata.FromIncomingContext(c.ctx) + if !ok { + return nil + } + keys := make([]string, 0, len(md)) + for key := range md { + keys = append(keys, key) + } + return keys +} + +// Context returns the underlying context associated with the +// `IncomingCarrier“. +func (c *IncomingCarrier) Context() context.Context { + return c.ctx +} + +// OutgoingCarrier is a TextMapCarrier that uses outgoing `context.Context` to +// store any propagated key-value pairs in text format. +type OutgoingCarrier struct { + ctx context.Context +} + +// NewOutgoingCarrier creates a new Carrier with the given context. The +// outgoing carrier should be used with propagator's `Inject()` method in the +// outgoing rpc path. +func NewOutgoingCarrier(ctx context.Context) *OutgoingCarrier { + return &OutgoingCarrier{ctx: ctx} +} + +// Get just logs an error and returns an empty string. It implements the +// `TextMapCarrier` interface but should not be used with `OutgoingCarrier`. +func (c *OutgoingCarrier) Get(string) string { + logger.Error("Get() should not be used with `OutgoingCarrier`") + return "" +} + +// Set stores the key-value pair in the carrier's outgoing context metadata. +// +// If the key already exists, given value is appended to the last. +func (c *OutgoingCarrier) Set(key, value string) { + c.ctx = metadata.AppendToOutgoingContext(c.ctx, key, value) +} + +// Keys returns the keys stored in the carrier's context metadata. It returns +// keys from outgoing context metadata. +func (c *OutgoingCarrier) Keys() []string { + md, ok := metadata.FromOutgoingContext(c.ctx) + if !ok { + return nil + } + keys := make([]string, 0, len(md)) + for key := range md { + keys = append(keys, key) + } + return keys +} + +// Context returns the underlying context associated with the +// `OutgoingCarrier“. +func (c *OutgoingCarrier) Context() context.Context { + return c.ctx +} diff --git a/stats/opentelemetry/internal/tracing/carrier_test.go b/stats/opentelemetry/internal/tracing/carrier_test.go new file mode 100644 index 0000000..a2e22be --- /dev/null +++ b/stats/opentelemetry/internal/tracing/carrier_test.go @@ -0,0 +1,190 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * htestp://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package tracing + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/metadata" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestIncomingCarrier verifies that `IncomingCarrier.Get()` returns correct +// value for the corresponding key in the carrier's context metadata, if key is +// present. If key is not present, it verifies that empty string is returned. +// +// If multiple values are present for a key, it verifies that last value is +// returned. +// +// If key ends with `-bin`, it verifies that a correct binary value is returned +// in the string format for the binary header. +func (s) TestIncomingCarrier(t *testing.T) { + tests := []struct { + name string + md metadata.MD + key string + want string + wantKeys []string + }{ + { + name: "existing key", + md: metadata.Pairs("key1", "value1"), + key: "key1", + want: "value1", + wantKeys: []string{"key1"}, + }, + { + name: "non-existing key", + md: metadata.Pairs("key1", "value1"), + key: "key2", + want: "", + wantKeys: []string{"key1"}, + }, + { + name: "empty key", + md: metadata.MD{}, + key: "key1", + want: "", + wantKeys: []string{}, + }, + { + name: "more than one key/value pair", + md: metadata.MD{"key1": []string{"value1"}, "key2": []string{"value2"}}, + key: "key2", + want: "value2", + wantKeys: []string{"key1", "key2"}, + }, + { + name: "more than one value for a key", + md: metadata.MD{"key1": []string{"value1", "value2"}}, + key: "key1", + want: "value2", + wantKeys: []string{"key1"}, + }, + { + name: "grpc-trace-bin key", + md: metadata.Pairs("grpc-trace-bin", string([]byte{0x01, 0x02, 0x03})), + key: "grpc-trace-bin", + want: string([]byte{0x01, 0x02, 0x03}), + wantKeys: []string{"grpc-trace-bin"}, + }, + { + name: "grpc-trace-bin key with another string key", + md: metadata.MD{"key1": []string{"value1"}, "grpc-trace-bin": []string{string([]byte{0x01, 0x02, 0x03})}}, + key: "grpc-trace-bin", + want: string([]byte{0x01, 0x02, 0x03}), + wantKeys: []string{"key1", "grpc-trace-bin"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := NewIncomingCarrier(metadata.NewIncomingContext(ctx, test.md)) + got := c.Get(test.key) + if got != test.want { + t.Fatalf("c.Get() = %s, want %s", got, test.want) + } + if gotKeys := c.Keys(); !cmp.Equal(test.wantKeys, gotKeys, cmpopts.SortSlices(func(a, b string) bool { return a < b })) { + t.Fatalf("c.Keys() = keys %v, want %v", gotKeys, test.wantKeys) + } + }) + } +} + +// TestOutgoingCarrier verifies that a key-value pair is set in carrier's +// context metadata using `OutgoingCarrier.Set()`. If key is not present, it +// verifies that key-value pair is insterted. If key is already present, it +// verifies that new value is appended at the end of list for the existing key. +// +// If key ends with `-bin`, it verifies that a binary value is set for +// `-bin` header in string format. +// +// It also verifies that both existing and newly inserted keys are present in +// the carrier's context using `Carrier.Keys()`. +func (s) TestOutgoingCarrier(t *testing.T) { + tests := []struct { + name string + initialMD metadata.MD + setKey string + setValue string + wantValue string // expected value of the set key + wantKeys []string + }{ + { + name: "new key", + initialMD: metadata.MD{}, + setKey: "key1", + setValue: "value1", + wantValue: "value1", + wantKeys: []string{"key1"}, + }, + { + name: "add to existing key", + initialMD: metadata.MD{"key1": []string{"oldvalue"}}, + setKey: "key1", + setValue: "newvalue", + wantValue: "newvalue", + wantKeys: []string{"key1"}, + }, + { + name: "new key with different existing key", + initialMD: metadata.MD{"key2": []string{"value2"}}, + setKey: "key1", + setValue: "value1", + wantValue: "value1", + wantKeys: []string{"key2", "key1"}, + }, + { + name: "grpc-trace-bin binary key", + initialMD: metadata.MD{"key1": []string{"value1"}}, + setKey: "grpc-trace-bin", + setValue: string([]byte{0x01, 0x02, 0x03}), + wantValue: string([]byte{0x01, 0x02, 0x03}), + wantKeys: []string{"key1", "grpc-trace-bin"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := NewOutgoingCarrier(metadata.NewOutgoingContext(ctx, test.initialMD)) + c.Set(test.setKey, test.setValue) + if gotKeys := c.Keys(); !cmp.Equal(test.wantKeys, gotKeys, cmpopts.SortSlices(func(a, b string) bool { return a < b })) { + t.Fatalf("c.Keys() = keys %v, want %v", gotKeys, test.wantKeys) + } + if md, ok := metadata.FromOutgoingContext(c.Context()); ok && md.Get(test.setKey)[len(md.Get(test.setKey))-1] != test.wantValue { + t.Fatalf("got value %s, want %s, for key %s", md.Get(test.setKey)[len(md.Get(test.setKey))-1], test.wantValue, test.setKey) + } + }) + } +} diff --git a/stats/opentelemetry/metricsregistry_test.go b/stats/opentelemetry/metricsregistry_test.go new file mode 100644 index 0000000..d23c748 --- /dev/null +++ b/stats/opentelemetry/metricsregistry_test.go @@ -0,0 +1,341 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opentelemetry + +import ( + "context" + "testing" + "time" + + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + + "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" +) + +var defaultTestTimeout = 5 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type metricsRecorderForTest interface { + estats.MetricsRecorder + initializeMetrics() +} + +func newClientStatsHandler(options MetricsOptions) metricsRecorderForTest { + return &clientMetricsHandler{options: Options{MetricsOptions: options}} +} + +func newServerStatsHandler(options MetricsOptions) metricsRecorderForTest { + return &serverMetricsHandler{options: Options{MetricsOptions: options}} +} + +// TestMetricsRegistryMetrics tests the OpenTelemetry behavior with respect to +// registered metrics. It registers metrics in the metrics registry. It then +// creates an OpenTelemetry client and server stats handler This test then makes +// measurements on those instruments using one of the stats handlers, then tests +// the expected metrics emissions, which includes default metrics and optional +// label assertions. +func (s) TestMetricsRegistryMetrics(t *testing.T) { + cleanup := internal.SnapshotMetricRegistryForTesting() + defer cleanup() + + intCountHandle1 := estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "int-counter-1", + Description: "Sum of calls from test", + Unit: "int", + Labels: []string{"int counter 1 label key"}, + OptionalLabels: []string{"int counter 1 optional label key"}, + Default: true, + }) + // A non default metric. If not specified in OpenTelemetry constructor, this + // will become a no-op, so measurements recorded on it won't show up in + // emitted metrics. + intCountHandle2 := estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "int-counter-2", + Description: "Sum of calls from test", + Unit: "int", + Labels: []string{"int counter 2 label key"}, + OptionalLabels: []string{"int counter 2 optional label key"}, + Default: false, + }) + // Register another non default metric. This will get added to the default + // metrics set in the OpenTelemetry constructor options, so metrics recorded + // on this should show up in metrics emissions. + intCountHandle3 := estats.RegisterInt64Count(estats.MetricDescriptor{ + Name: "int-counter-3", + Description: "sum of calls from test", + Unit: "int", + Labels: []string{"int counter 3 label key"}, + OptionalLabels: []string{"int counter 3 optional label key"}, + Default: false, + }) + floatCountHandle := estats.RegisterFloat64Count(estats.MetricDescriptor{ + Name: "float-counter", + Description: "sum of calls from test", + Unit: "float", + Labels: []string{"float counter label key"}, + OptionalLabels: []string{"float counter optional label key"}, + Default: true, + }) + bounds := []float64{0, 5, 10} + intHistoHandle := estats.RegisterInt64Histo(estats.MetricDescriptor{ + Name: "int-histo", + Description: "histogram of call values from tests", + Unit: "int", + Labels: []string{"int histo label key"}, + OptionalLabels: []string{"int histo optional label key"}, + Default: true, + Bounds: bounds, + }) + floatHistoHandle := estats.RegisterFloat64Histo(estats.MetricDescriptor{ + Name: "float-histo", + Description: "histogram of call values from tests", + Unit: "float", + Labels: []string{"float histo label key"}, + OptionalLabels: []string{"float histo optional label key"}, + Default: true, + Bounds: bounds, + }) + intGaugeHandle := estats.RegisterInt64Gauge(estats.MetricDescriptor{ + Name: "simple-gauge", + Description: "the most recent int emitted by test", + Unit: "int", + Labels: []string{"int gauge label key"}, + OptionalLabels: []string{"int gauge optional label key"}, + Default: true, + }) + intAsyncHandle := estats.RegisterInt64AsyncGauge(estats.MetricDescriptor{ + Name: "async-gauge", + Description: "async gauge value from test", + Unit: "int", + Labels: []string{"async label key"}, + OptionalLabels: []string{"async optional label key"}, + Default: true, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Only float optional labels are configured, so only float optional labels should show up. + // All required labels should show up. + wantMetrics := []metricdata.Metrics{ + { + Name: "int-counter-1", + Description: "Sum of calls from test", + Unit: "int", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("int counter 1 label key", "int counter 1 label value")), // No optional label, not float. + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "int-counter-3", + Description: "sum of calls from test", + Unit: "int", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("int counter 3 label key", "int counter 3 label value")), // No optional label, not float. + Value: 4, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "float-counter", + Description: "sum of calls from test", + Unit: "float", + Data: metricdata.Sum[float64]{ + DataPoints: []metricdata.DataPoint[float64]{ + { + Attributes: attribute.NewSet(attribute.String("float counter label key", "float counter label value"), attribute.String("float counter optional label key", "float counter optional label value")), + Value: 1.2, + }, + }, + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + }, + }, + { + Name: "int-histo", + Description: "histogram of call values from tests", + Unit: "int", + Data: metricdata.Histogram[int64]{ + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("int histo label key", "int histo label value")), // No optional label, not float. + Count: 1, + Bounds: bounds, + BucketCounts: []uint64{0, 1, 0, 0}, + Min: metricdata.NewExtrema(int64(3)), + Max: metricdata.NewExtrema(int64(3)), + Sum: 3, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "float-histo", + Description: "histogram of call values from tests", + Unit: "float", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: attribute.NewSet(attribute.String("float histo label key", "float histo label value"), attribute.String("float histo optional label key", "float histo optional label value")), + Count: 1, + Bounds: bounds, + BucketCounts: []uint64{0, 1, 0, 0}, + Min: metricdata.NewExtrema(float64(4.3)), + Max: metricdata.NewExtrema(float64(4.3)), + Sum: 4.3, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "simple-gauge", + Description: "the most recent int emitted by test", + Unit: "int", + Data: metricdata.Gauge[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet(attribute.String("int gauge label key", "int gauge label value")), // No optional label, not float. + Value: 8, + }, + }, + }, + }, + { + Name: "async-gauge", + Description: "async gauge value from test", + Unit: "int", + Data: metricdata.Gauge[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + // Note: Only the required label is expected because optional labels + // for "async optional label key" are not enabled in MetricsOptions below. + Attributes: attribute.NewSet(attribute.String("async label key", "async label value")), + Value: 999, + }, + }, + }, + }, + } + + for _, test := range []struct { + name string + constructor func(options MetricsOptions) metricsRecorderForTest + }{ + { + name: "client stats handler", + constructor: newClientStatsHandler, + }, + { + name: "server stats handler", + constructor: newServerStatsHandler, + }, + } { + t.Run(test.name, func(t *testing.T) { + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + // This configures the defaults alongside int counter 3. All the instruments + // registered except int counter 2 and 3 are default, so all measurements + // recorded should show up in reader collected metrics except those for int + // counter 2. + // This also only toggles the float count and float histo optional labels, + // so only those should show up in metrics emissions. All the required + // labels should show up in metrics emissions. + mo := MetricsOptions{ + Metrics: DefaultMetrics().Add("int-counter-3"), + OptionalLabels: []string{"float counter optional label key", "float histo optional label key"}, + MeterProvider: provider, + } + mr := test.constructor(mo) + mr.initializeMetrics() + // These Record calls are guaranteed at a layer underneath OpenTelemetry for + // labels emitted to match the length of labels + optional labels. + intCountHandle1.Record(mr, 1, []string{"int counter 1 label value", "int counter 1 optional label value"}...) + // int-counter-2 is not part of metrics specified (not default), so this + // record call shouldn't show up in the reader. + intCountHandle2.Record(mr, 2, []string{"int counter 2 label value", "int counter 2 optional label value"}...) + // int-counter-3 is part of metrics specified, so this call should show up + // in the reader. + intCountHandle3.Record(mr, 4, []string{"int counter 3 label value", "int counter 3 optional label value"}...) + + // All future recording points should show up in emissions as all of these are defaults. + floatCountHandle.Record(mr, 1.2, []string{"float counter label value", "float counter optional label value"}...) + intHistoHandle.Record(mr, 3, []string{"int histo label value", "int histo optional label value"}...) + floatHistoHandle.Record(mr, 4.3, []string{"float histo label value", "float histo optional label value"}...) + intGaugeHandle.Record(mr, 7, []string{"int gauge label value", "int gauge optional label value"}...) + // This second gauge call should take the place of the previous gauge call. + intGaugeHandle.Record(mr, 8, []string{"int gauge label value", "int gauge optional label value"}...) + reporter := estats.AsyncMetricReporterFunc(func(r estats.AsyncMetricsRecorder) error { + // We record value 999. + // Note: We pass both required and optional labels, but the expectation (Step 2) + // knows that the optional one will be dropped based on 'mo' config. + intAsyncHandle.Record(r, 999, "async label value", "async optional label value") + return nil + }) + mr.RegisterAsyncReporter(reporter, intAsyncHandle) + rm := &metricdata.ResourceMetrics{} + reader.Collect(ctx, rm) + gotMetrics := map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + gotMetrics[m.Name] = m + } + } + for _, metric := range wantMetrics { + val, ok := gotMetrics[metric.Name] + if !ok { + t.Fatalf("Metric %v not present in recorded metrics", metric.Name) + } + if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) { + t.Fatalf("Metrics data type not equal for metric: %v", metric.Name) + } + } + + // int-counter-2 is not a default metric and wasn't specified in + // constructor, so emissions should not show up. + if _, ok := gotMetrics["int-counter-2"]; ok { + t.Fatalf("Metric int-counter-2 present in recorded metrics, was not configured") + } + }) + } +} diff --git a/stats/opentelemetry/opentelemetry.go b/stats/opentelemetry/opentelemetry.go new file mode 100644 index 0000000..878e4eb --- /dev/null +++ b/stats/opentelemetry/opentelemetry.go @@ -0,0 +1,626 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package opentelemetry implements opentelemetry instrumentation code for +// gRPC-Go clients and servers. +// +// For details on configuring opentelemetry and various instruments that this +// package creates, see +// [gRPC OpenTelemetry Metrics](https://grpc.io/docs/guides/opentelemetry-metrics/). +package opentelemetry + +import ( + "context" + "strings" + "sync/atomic" + "time" + + otelattribute "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/metric/noop" + "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + experimental "google.golang.org/grpc/experimental/opentelemetry" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/stats" + otelinternal "google.golang.org/grpc/stats/opentelemetry/internal" +) + +func init() { + otelinternal.SetPluginOption = func(o *Options, po otelinternal.PluginOption) { + o.MetricsOptions.pluginOption = po + // Log an error if one of the options is missing. + if (o.TraceOptions.TextMapPropagator == nil) != (o.TraceOptions.TracerProvider == nil) { + logger.Warning("Tracing will not be recorded because traceOptions are not set properly: one of TextMapPropagator or TracerProvider is missing") + } + } +} + +var ( + logger = grpclog.Component("otel-plugin") + canonicalString = internal.CanonicalString.(func(codes.Code) string) + joinDialOptions = internal.JoinDialOptions.(func(...grpc.DialOption) grpc.DialOption) +) + +// Options are the options for OpenTelemetry instrumentation. +type Options struct { + // MetricsOptions are the metrics options for OpenTelemetry instrumentation. + MetricsOptions MetricsOptions + // TraceOptions are the tracing options for OpenTelemetry instrumentation. + TraceOptions experimental.TraceOptions +} + +func (o *Options) isMetricsEnabled() bool { + return o.MetricsOptions.MeterProvider != nil +} + +func (o *Options) isTracingEnabled() bool { + return o.TraceOptions.TracerProvider != nil +} + +// MetricsOptions are the metrics options for OpenTelemetry instrumentation. +type MetricsOptions struct { + // MeterProvider is the MeterProvider instance that will be used to create + // instruments. To enable metrics collection, set a meter provider. If + // unset, no metrics will be recorded. + MeterProvider otelmetric.MeterProvider + + // Metrics are the metrics to instrument. Will create instrument and record telemetry + // for corresponding metric supported by the client and server + // instrumentation components if applicable. If not set, the default metrics + // will be recorded. + Metrics *stats.MetricSet + + // MethodAttributeFilter is a function that determines whether to record the + // method name of RPCs as an attribute, or to bucket into "other". Take care + // to limit the values allowed, as allowing too many will increase + // cardinality and could cause severe memory or performance problems. + // + // This only applies for server-side metrics. For clients, to record the + // method name in the attributes, pass grpc.StaticMethodCallOption to Invoke + // or NewStream. Note that when using protobuf generated clients, this + // CallOption is included automatically. + MethodAttributeFilter func(string) bool + + // OptionalLabels specifies a list of optional labels to enable on any + // metrics that support them. + OptionalLabels []string + + // pluginOption is used to get labels to attach to certain metrics, if set. + pluginOption otelinternal.PluginOption +} + +// DialOption returns a dial option which enables OpenTelemetry instrumentation +// code for a grpc.ClientConn. +// +// Client applications interested in instrumenting their grpc.ClientConn should +// pass the dial option returned from this function as a dial option to +// grpc.NewClient(). +// +// For the metrics supported by this instrumentation code, specify the client +// metrics to record in metrics options. Also provide an implementation of a +// MeterProvider. If the passed in Meter Provider does not have the view +// configured for an individual metric turned on, the API call in this component +// will create a default view for that metric. +// +// For the traces supported by this instrumentation code, provide an +// implementation of a TextMapPropagator and OpenTelemetry TracerProvider. +func DialOption(o Options) grpc.DialOption { + var metricsOpts, tracingOpts []grpc.DialOption + + if o.isMetricsEnabled() { + metricsHandler := &clientMetricsHandler{options: o} + metricsHandler.initializeMetrics() + metricsOpts = append(metricsOpts, grpc.WithChainUnaryInterceptor(metricsHandler.unaryInterceptor), grpc.WithChainStreamInterceptor(metricsHandler.streamInterceptor), grpc.WithStatsHandler(metricsHandler)) + } + if o.isTracingEnabled() { + tracingHandler := &clientTracingHandler{options: o} + tracingHandler.initializeTraces() + tracingOpts = append(tracingOpts, grpc.WithChainUnaryInterceptor(tracingHandler.unaryInterceptor), grpc.WithChainStreamInterceptor(tracingHandler.streamInterceptor), grpc.WithStatsHandler(tracingHandler)) + } + return joinDialOptions(append(metricsOpts, tracingOpts...)...) +} + +var joinServerOptions = internal.JoinServerOptions.(func(...grpc.ServerOption) grpc.ServerOption) + +// ServerOption returns a server option which enables OpenTelemetry +// instrumentation code for a grpc.Server. +// +// Server applications interested in instrumenting their grpc.Server should pass +// the server option returned from this function as an argument to +// grpc.NewServer(). +// +// For the metrics supported by this instrumentation code, specify the server +// metrics to record in metrics options. Also provide an implementation of a +// MeterProvider. If the passed in Meter Provider does not have the view +// configured for an individual metric turned on, the API call in this component +// will create a default view for that metric. +// +// For the traces supported by this instrumentation code, provide an +// implementation of a TextMapPropagator and OpenTelemetry TracerProvider. +func ServerOption(o Options) grpc.ServerOption { + var metricsOpts, tracingOpts []grpc.ServerOption + + if o.isMetricsEnabled() { + metricsHandler := &serverMetricsHandler{options: o} + metricsHandler.initializeMetrics() + metricsOpts = append(metricsOpts, grpc.ChainUnaryInterceptor(metricsHandler.unaryInterceptor), grpc.ChainStreamInterceptor(metricsHandler.streamInterceptor), grpc.StatsHandler(metricsHandler)) + } + if o.isTracingEnabled() { + tracingHandler := &serverTracingHandler{options: o} + tracingHandler.initializeTraces() + tracingOpts = append(tracingOpts, grpc.StatsHandler(tracingHandler)) + } + return joinServerOptions(append(metricsOpts, tracingOpts...)...) +} + +// callInfo is information pertaining to the lifespan of the RPC client side. +type callInfo struct { + target string + + method string + + // nameResolutionEventAdded is set when the resolver delay trace event + // is added. Prevents duplicate events, since it is reported per-attempt. + nameResolutionEventAdded atomic.Bool + // previousRPCAttempts holds the count of RPC attempts that have happened + // before current attempt. Transparent retries are excluded. + previousRPCAttempts atomic.Uint32 +} + +type callInfoKey struct{} + +// getCallInfo returns the callInfo stored in the context, or nil +// if there isn't one. +func getCallInfo(ctx context.Context) *callInfo { + ci, _ := ctx.Value(callInfoKey{}).(*callInfo) + return ci +} + +// rpcInfo is RPC information scoped to the RPC attempt life span client side, +// and the RPC life span server side. +type rpcInfo struct { + ai *attemptInfo +} + +type clientRPCInfoKey struct{} +type serverRPCInfoKey struct{} + +// clientRPCInfo returns the rpcInfo stored in the context for client, or nil +// if there isn't one. +func clientRPCInfo(ctx context.Context) *rpcInfo { + ri, _ := ctx.Value(clientRPCInfoKey{}).(*rpcInfo) + return ri +} + +// serverRPCInfo returns the rpcInfo stored in the context for server, or nil +// if there isn't one. +func serverRPCInfo(ctx context.Context) *rpcInfo { + ri, _ := ctx.Value(serverRPCInfoKey{}).(*rpcInfo) + return ri +} + +func getOrCreateClientRPCInfo(ctx context.Context, info *stats.RPCTagInfo) (context.Context, *rpcInfo) { + ri := clientRPCInfo(ctx) + if ri != nil { + return ctx, ri + } + ri = &rpcInfo{ + ai: &attemptInfo{ + startTime: time.Now(), + method: removeLeadingSlash(info.FullMethodName), + xdsLabels: map[string]string{ + // The defaults for all the per call labels from a plugin that + // executes on the callpath that this OpenTelemetry component + // currently supports. + "grpc.lb.locality": "", + "grpc.lb.backend_service": "", + }, + }, + } + return context.WithValue(ctx, clientRPCInfoKey{}, ri), ri +} + +func getOrCreateServerRPCInfo(ctx context.Context, info *stats.RPCTagInfo, options MetricsOptions) (context.Context, *rpcInfo) { + ri := serverRPCInfo(ctx) + if ri != nil { + return ctx, ri + } + method := info.FullMethodName + if options.MethodAttributeFilter != nil { + if !options.MethodAttributeFilter(method) { + method = "other" + } + } + server := internal.ServerFromContext.(func(context.Context) *grpc.Server)(ctx) + if server == nil { // Shouldn't happen, defensive programming. + logger.Error("ctx passed into server side stats handler has no grpc server ref") + method = "other" + } else { + isRegisteredMethod := internal.IsRegisteredMethod.(func(*grpc.Server, string) bool) + if !isRegisteredMethod(server, method) { + method = "other" + } + } + + ri = &rpcInfo{ + ai: &attemptInfo{ + startTime: time.Now(), + method: removeLeadingSlash(method), + }, + } + return context.WithValue(ctx, serverRPCInfoKey{}, ri), ri +} + +func removeLeadingSlash(mn string) string { + return strings.TrimLeft(mn, "/") +} + +// attemptInfo is RPC information scoped to the RPC attempt life span client +// side, and the RPC life span server side. +type attemptInfo struct { + // access these counts atomically for hedging in the future: + // number of bytes after compression (within each message) from side (client + // || server). + sentCompressedBytes int64 + // number of compressed bytes received (within each message) received on + // side (client || server). + recvCompressedBytes int64 + + startTime time.Time + method string + + pluginOptionLabels map[string]string // pluginOptionLabels to attach to metrics emitted + xdsLabels map[string]string + + // traceSpan is data used for recording traces. + traceSpan trace.Span + // message counters for sent and received messages (used for + // generating message IDs), and the number of previous RPC attempts for the + // associated call. + countSentMsg uint32 + countRecvMsg uint32 +} + +type clientMetrics struct { + // "grpc.client.attempt.started" + attemptStarted otelmetric.Int64Counter + // "grpc.client.attempt.duration" + attemptDuration otelmetric.Float64Histogram + // "grpc.client.attempt.sent_total_compressed_message_size" + attemptSentTotalCompressedMessageSize otelmetric.Int64Histogram + // "grpc.client.attempt.rcvd_total_compressed_message_size" + attemptRcvdTotalCompressedMessageSize otelmetric.Int64Histogram + // "grpc.client.call.duration" + callDuration otelmetric.Float64Histogram +} + +type serverMetrics struct { + // "grpc.server.call.started" + callStarted otelmetric.Int64Counter + // "grpc.server.call.sent_total_compressed_message_size" + callSentTotalCompressedMessageSize otelmetric.Int64Histogram + // "grpc.server.call.rcvd_total_compressed_message_size" + callRcvdTotalCompressedMessageSize otelmetric.Int64Histogram + // "grpc.server.call.duration" + callDuration otelmetric.Float64Histogram +} + +func createInt64Counter(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Int64CounterOption) otelmetric.Int64Counter { + if _, ok := setOfMetrics[metricName]; !ok { + return noop.Int64Counter{} + } + ret, err := meter.Int64Counter(string(metricName), options...) + if err != nil { + logger.Errorf("failed to register metric \"%v\", will not record: %v", metricName, err) + return noop.Int64Counter{} + } + return ret +} + +func createInt64UpDownCounter(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Int64UpDownCounterOption) otelmetric.Int64UpDownCounter { + if _, ok := setOfMetrics[metricName]; !ok { + return noop.Int64UpDownCounter{} + } + ret, err := meter.Int64UpDownCounter(string(metricName), options...) + if err != nil { + logger.Errorf("Failed to register metric \"%v\", will not record: %v", metricName, err) + return noop.Int64UpDownCounter{} + } + return ret +} + +func createFloat64Counter(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Float64CounterOption) otelmetric.Float64Counter { + if _, ok := setOfMetrics[metricName]; !ok { + return noop.Float64Counter{} + } + ret, err := meter.Float64Counter(string(metricName), options...) + if err != nil { + logger.Errorf("failed to register metric \"%v\", will not record: %v", metricName, err) + return noop.Float64Counter{} + } + return ret +} + +func createInt64Histogram(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Int64HistogramOption) otelmetric.Int64Histogram { + if _, ok := setOfMetrics[metricName]; !ok { + return noop.Int64Histogram{} + } + ret, err := meter.Int64Histogram(string(metricName), options...) + if err != nil { + logger.Errorf("failed to register metric \"%v\", will not record: %v", metricName, err) + return noop.Int64Histogram{} + } + return ret +} + +func createFloat64Histogram(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Float64HistogramOption) otelmetric.Float64Histogram { + if _, ok := setOfMetrics[metricName]; !ok { + return noop.Float64Histogram{} + } + ret, err := meter.Float64Histogram(string(metricName), options...) + if err != nil { + logger.Errorf("failed to register metric \"%v\", will not record: %v", metricName, err) + return noop.Float64Histogram{} + } + return ret +} + +func createInt64Gauge(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Int64GaugeOption) otelmetric.Int64Gauge { + if _, ok := setOfMetrics[metricName]; !ok { + return noop.Int64Gauge{} + } + ret, err := meter.Int64Gauge(string(metricName), options...) + if err != nil { + logger.Errorf("failed to register metric \"%v\", will not record: %v", metricName, err) + return noop.Int64Gauge{} + } + return ret +} + +// createInt64ObservableGauge initializes an OTel Int64ObservableGauge if the metric is enabled. +func createInt64ObservableGauge(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Int64ObservableGaugeOption) otelmetric.Int64ObservableGauge { + if _, ok := setOfMetrics[metricName]; !ok { + n, _ := noop.NewMeterProvider().Meter("noop").Int64ObservableGauge("noop") + return n + } + ret, err := meter.Int64ObservableGauge(metricName, options...) + if err != nil { + logger.Errorf("failed to register metric %q, will not record: %v", metricName, err) + n, _ := noop.NewMeterProvider().Meter("noop").Int64ObservableGauge("noop") + return n + } + return ret +} + +func optionFromLabels(labelKeys []string, optionalLabelKeys []string, optionalLabels []string, labelVals ...string) otelmetric.MeasurementOption { + var attributes []otelattribute.KeyValue + + // Once it hits here lower level has guaranteed length of labelVals matches + // labelKeys + optionalLabelKeys. + for i, label := range labelKeys { + attributes = append(attributes, otelattribute.String(label, labelVals[i])) + } + + for i, label := range optionalLabelKeys { + for _, optLabel := range optionalLabels { // o(n) could build out a set but n is currently capped at < 5 + if label == optLabel { + attributes = append(attributes, otelattribute.String(label, labelVals[i+len(labelKeys)])) + } + } + } + return otelmetric.WithAttributeSet(otelattribute.NewSet(attributes...)) +} + +// registryMetrics implements MetricsRecorder for the client and server stats +// handlers. +type registryMetrics struct { + internal.EnforceMetricsRecorderEmbedding + intCounts map[*estats.MetricDescriptor]otelmetric.Int64Counter + floatCounts map[*estats.MetricDescriptor]otelmetric.Float64Counter + intHistos map[*estats.MetricDescriptor]otelmetric.Int64Histogram + floatHistos map[*estats.MetricDescriptor]otelmetric.Float64Histogram + intGauges map[*estats.MetricDescriptor]otelmetric.Int64Gauge + intUpDownCounts map[*estats.MetricDescriptor]otelmetric.Int64UpDownCounter + + // Asynchronous (Observable) Instruments + intObservableGauges map[*estats.MetricDescriptor]otelmetric.Int64ObservableGauge + + meter otelmetric.Meter + optionalLabels []string +} + +func (rm *registryMetrics) registerMetrics(metrics *stats.MetricSet, meter otelmetric.Meter) { + rm.meter = meter + rm.intCounts = make(map[*estats.MetricDescriptor]otelmetric.Int64Counter) + rm.floatCounts = make(map[*estats.MetricDescriptor]otelmetric.Float64Counter) + rm.intHistos = make(map[*estats.MetricDescriptor]otelmetric.Int64Histogram) + rm.floatHistos = make(map[*estats.MetricDescriptor]otelmetric.Float64Histogram) + rm.intGauges = make(map[*estats.MetricDescriptor]otelmetric.Int64Gauge) + rm.intUpDownCounts = make(map[*estats.MetricDescriptor]otelmetric.Int64UpDownCounter) + rm.intObservableGauges = make(map[*estats.MetricDescriptor]otelmetric.Int64ObservableGauge) + + for metric := range metrics.Metrics() { + desc := estats.DescriptorForMetric(metric) + if desc == nil { + // Either the metric was per call or the metric is not registered. + // Thus, if this component ever receives the desc as a handle in + // record it will be a no-op. + continue + } + switch desc.Type { + case estats.MetricTypeIntCount: + rm.intCounts[desc] = createInt64Counter(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description)) + case estats.MetricTypeFloatCount: + rm.floatCounts[desc] = createFloat64Counter(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description)) + case estats.MetricTypeIntHisto: + rm.intHistos[desc] = createInt64Histogram(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description), otelmetric.WithExplicitBucketBoundaries(desc.Bounds...)) + case estats.MetricTypeFloatHisto: + rm.floatHistos[desc] = createFloat64Histogram(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description), otelmetric.WithExplicitBucketBoundaries(desc.Bounds...)) + case estats.MetricTypeIntGauge: + rm.intGauges[desc] = createInt64Gauge(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description)) + case estats.MetricTypeIntUpDownCount: + rm.intUpDownCounts[desc] = createInt64UpDownCounter(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description)) + case estats.MetricTypeIntAsyncGauge: + rm.intObservableGauges[desc] = createInt64ObservableGauge(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description)) + } + } +} + +func (rm *registryMetrics) RecordInt64Count(handle *estats.Int64CountHandle, incr int64, labels ...string) { + desc := handle.Descriptor() + if ic, ok := rm.intCounts[desc]; ok { + ao := optionFromLabels(desc.Labels, desc.OptionalLabels, rm.optionalLabels, labels...) + ic.Add(context.TODO(), incr, ao) + } +} + +func (rm *registryMetrics) RecordInt64UpDownCount(handle *estats.Int64UpDownCountHandle, incr int64, labels ...string) { + desc := handle.Descriptor() + if ic, ok := rm.intUpDownCounts[desc]; ok { + ao := optionFromLabels(desc.Labels, desc.OptionalLabels, rm.optionalLabels, labels...) + ic.Add(context.TODO(), incr, ao) + } +} + +func (rm *registryMetrics) RecordFloat64Count(handle *estats.Float64CountHandle, incr float64, labels ...string) { + desc := handle.Descriptor() + if fc, ok := rm.floatCounts[desc]; ok { + ao := optionFromLabels(desc.Labels, desc.OptionalLabels, rm.optionalLabels, labels...) + fc.Add(context.TODO(), incr, ao) + } +} + +func (rm *registryMetrics) RecordInt64Histo(handle *estats.Int64HistoHandle, incr int64, labels ...string) { + desc := handle.Descriptor() + if ih, ok := rm.intHistos[desc]; ok { + ao := optionFromLabels(desc.Labels, desc.OptionalLabels, rm.optionalLabels, labels...) + ih.Record(context.TODO(), incr, ao) + } +} + +func (rm *registryMetrics) RecordFloat64Histo(handle *estats.Float64HistoHandle, incr float64, labels ...string) { + desc := handle.Descriptor() + if fh, ok := rm.floatHistos[desc]; ok { + ao := optionFromLabels(desc.Labels, desc.OptionalLabels, rm.optionalLabels, labels...) + fh.Record(context.TODO(), incr, ao) + } +} + +func (rm *registryMetrics) RecordInt64Gauge(handle *estats.Int64GaugeHandle, incr int64, labels ...string) { + desc := handle.Descriptor() + if ig, ok := rm.intGauges[desc]; ok { + ao := optionFromLabels(desc.Labels, desc.OptionalLabels, rm.optionalLabels, labels...) + ig.Record(context.TODO(), incr, ao) + } +} + +// RegisterAsyncReporter will register a callback with the underlying OpenTelemetry +// Meter for the provided descriptors. +// +// It will map the provided descriptors to their corresponding OTel Observable +// instruments. If no instruments match the descriptors, registration is +// skipped. +// +// The returned cleanup function unregisters the callback from the Meter. +// RegisterAsyncReporter registers a callback with the OpenTelemetry Meter. +func (rm *registryMetrics) RegisterAsyncReporter(reporter estats.AsyncMetricReporter, metrics ...estats.AsyncMetric) func() { + observables := make([]otelmetric.Observable, 0, len(metrics)) + observableMap := make(map[*estats.MetricDescriptor]otelmetric.Observable, len(metrics)) + + for _, m := range metrics { + d := m.Descriptor() + if inst, ok := rm.intObservableGauges[d]; ok { + observables = append(observables, inst) + observableMap[d] = inst + } + } + + if len(observables) == 0 { + return func() {} + } + + cbWrapper := func(_ context.Context, o otelmetric.Observer) error { + adapter := &observerAdapter{ + observableMap: observableMap, + optionalLabels: rm.optionalLabels, + delegate: o, + } + reporter.Report(adapter) + return nil + } + + reg, err := rm.meter.RegisterCallback(cbWrapper, observables...) + if err != nil { + logger.Warningf("grpc: failed to register callback for async metrics: %v", err) + return func() {} + } + + return func() { + err = reg.Unregister() + if err != nil { + logger.Errorf("grpc: failed to unregister callback for async metrics: %v", err) + } + } +} + +// Users of this component should use these bucket boundaries as part of their +// SDK MeterProvider passed in. This component sends this as "advice" to the +// API, which works, however this stability is not guaranteed, so for safety the +// SDK Meter Provider provided should set these bounds for corresponding +// metrics. +var ( + // DefaultLatencyBounds are the default bounds for latency metrics. + DefaultLatencyBounds = []float64{0, 0.00001, 0.00005, 0.0001, 0.0003, 0.0006, 0.0008, 0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.008, 0.01, 0.013, 0.016, 0.02, 0.025, 0.03, 0.04, 0.05, 0.065, 0.08, 0.1, 0.13, 0.16, 0.2, 0.25, 0.3, 0.4, 0.5, 0.65, 0.8, 1, 2, 5, 10, 20, 50, 100} // provide "advice" through API, SDK should set this too + // DefaultSizeBounds are the default bounds for metrics which record size. + DefaultSizeBounds = []float64{0, 1024, 2048, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864, 268435456, 1073741824, 4294967296} + // defaultPerCallMetrics are the default metrics provided by this module. + defaultPerCallMetrics = stats.NewMetricSet(ClientAttemptStartedMetricName, ClientAttemptDurationMetricName, ClientAttemptSentCompressedTotalMessageSizeMetricName, ClientAttemptRcvdCompressedTotalMessageSizeMetricName, ClientCallDurationMetricName, ServerCallStartedMetricName, ServerCallSentCompressedTotalMessageSizeMetricName, ServerCallRcvdCompressedTotalMessageSizeMetricName, ServerCallDurationMetricName) +) + +// DefaultMetrics returns a set of default OpenTelemetry metrics. +// +// This should only be invoked after init time. +func DefaultMetrics() *stats.MetricSet { + return defaultPerCallMetrics.Join(estats.DefaultMetrics) +} + +type observerAdapter struct { + observableMap map[*estats.MetricDescriptor]otelmetric.Observable + optionalLabels []string + delegate otelmetric.Observer +} + +// RecordInt64AsyncGauge records the measurement alongside labels on the int +// gauge associated with the provided handle. +func (a *observerAdapter) RecordInt64AsyncGauge(handle *estats.Int64AsyncGaugeHandle, val int64, labels ...string) { + desc := handle.Descriptor() + observable, ok := a.observableMap[desc] + if !ok { + return + } + + ao := optionFromLabels(desc.Labels, desc.OptionalLabels, a.optionalLabels, labels...) + + switch obs := observable.(type) { + case otelmetric.Int64ObservableGauge: + a.delegate.ObserveInt64(obs, val, ao) + default: + } +} diff --git a/stats/opentelemetry/server_metrics.go b/stats/opentelemetry/server_metrics.go new file mode 100644 index 0000000..7ec0e4b --- /dev/null +++ b/stats/opentelemetry/server_metrics.go @@ -0,0 +1,254 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opentelemetry + +import ( + "context" + "sync/atomic" + "time" + + otelattribute "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/metric" + + "google.golang.org/grpc" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +type serverMetricsHandler struct { + estats.MetricsRecorder + options Options + serverMetrics serverMetrics +} + +func (h *serverMetricsHandler) initializeMetrics() { + // Will set no metrics to record, logically making this stats handler a + // no-op. + if h.options.MetricsOptions.MeterProvider == nil { + return + } + + meter := h.options.MetricsOptions.MeterProvider.Meter("grpc-go", otelmetric.WithInstrumentationVersion(grpc.Version)) + if meter == nil { + return + } + metrics := h.options.MetricsOptions.Metrics + if metrics == nil { + metrics = DefaultMetrics() + } + + h.serverMetrics.callStarted = createInt64Counter(metrics.Metrics(), "grpc.server.call.started", meter, otelmetric.WithUnit("{call}"), otelmetric.WithDescription("Number of server calls started.")) + h.serverMetrics.callSentTotalCompressedMessageSize = createInt64Histogram(metrics.Metrics(), "grpc.server.call.sent_total_compressed_message_size", meter, otelmetric.WithUnit("By"), otelmetric.WithDescription("Compressed message bytes sent per server call."), otelmetric.WithExplicitBucketBoundaries(DefaultSizeBounds...)) + h.serverMetrics.callRcvdTotalCompressedMessageSize = createInt64Histogram(metrics.Metrics(), "grpc.server.call.rcvd_total_compressed_message_size", meter, otelmetric.WithUnit("By"), otelmetric.WithDescription("Compressed message bytes received per server call."), otelmetric.WithExplicitBucketBoundaries(DefaultSizeBounds...)) + h.serverMetrics.callDuration = createFloat64Histogram(metrics.Metrics(), "grpc.server.call.duration", meter, otelmetric.WithUnit("s"), otelmetric.WithDescription("End-to-end time taken to complete a call from server transport's perspective."), otelmetric.WithExplicitBucketBoundaries(DefaultLatencyBounds...)) + + rm := ®istryMetrics{ + optionalLabels: h.options.MetricsOptions.OptionalLabels, + } + h.MetricsRecorder = rm + rm.registerMetrics(metrics, meter) +} + +// attachLabelsTransportStream intercepts SetHeader and SendHeader calls of the +// underlying ServerTransportStream to attach metadataExchangeLabels. +type attachLabelsTransportStream struct { + grpc.ServerTransportStream + + attachedLabels atomic.Bool + metadataExchangeLabels metadata.MD +} + +func (s *attachLabelsTransportStream) SetHeader(md metadata.MD) error { + if !s.attachedLabels.Swap(true) { + s.ServerTransportStream.SetHeader(s.metadataExchangeLabels) + } + return s.ServerTransportStream.SetHeader(md) +} + +func (s *attachLabelsTransportStream) SendHeader(md metadata.MD) error { + if !s.attachedLabels.Swap(true) { + s.ServerTransportStream.SetHeader(s.metadataExchangeLabels) + } + + return s.ServerTransportStream.SendHeader(md) +} + +func (h *serverMetricsHandler) unaryInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + var metadataExchangeLabels metadata.MD + if h.options.MetricsOptions.pluginOption != nil { + metadataExchangeLabels = h.options.MetricsOptions.pluginOption.GetMetadata() + } + + sts := grpc.ServerTransportStreamFromContext(ctx) + + alts := &attachLabelsTransportStream{ + ServerTransportStream: sts, + metadataExchangeLabels: metadataExchangeLabels, + } + ctx = grpc.NewContextWithServerTransportStream(ctx, alts) + + res, err := handler(ctx, req) + if err != nil { // maybe trailers-only if headers haven't already been sent + if !alts.attachedLabels.Swap(true) { + alts.SetTrailer(alts.metadataExchangeLabels) + } + } else { // headers will be written; a message was sent + if !alts.attachedLabels.Swap(true) { + alts.SetHeader(alts.metadataExchangeLabels) + } + } + + return res, err +} + +// attachLabelsStream embeds a grpc.ServerStream, and intercepts the +// SetHeader/SendHeader/SendMsg/SendTrailer call to attach metadata exchange +// labels. +type attachLabelsStream struct { + grpc.ServerStream + + attachedLabels atomic.Bool + metadataExchangeLabels metadata.MD +} + +func (s *attachLabelsStream) SetHeader(md metadata.MD) error { + if !s.attachedLabels.Swap(true) { + s.ServerStream.SetHeader(s.metadataExchangeLabels) + } + + return s.ServerStream.SetHeader(md) +} + +func (s *attachLabelsStream) SendHeader(md metadata.MD) error { + if !s.attachedLabels.Swap(true) { + s.ServerStream.SetHeader(s.metadataExchangeLabels) + } + + return s.ServerStream.SendHeader(md) +} + +func (s *attachLabelsStream) SendMsg(m any) error { + if !s.attachedLabels.Swap(true) { + s.ServerStream.SetHeader(s.metadataExchangeLabels) + } + return s.ServerStream.SendMsg(m) +} + +func (h *serverMetricsHandler) streamInterceptor(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + var metadataExchangeLabels metadata.MD + if h.options.MetricsOptions.pluginOption != nil { + metadataExchangeLabels = h.options.MetricsOptions.pluginOption.GetMetadata() + } + als := &attachLabelsStream{ + ServerStream: ss, + metadataExchangeLabels: metadataExchangeLabels, + } + err := handler(srv, als) + + // Add metadata exchange labels to trailers if never sent in headers, + // irrespective of whether or not RPC failed. + if !als.attachedLabels.Load() { + als.SetTrailer(als.metadataExchangeLabels) + } + return err +} + +// TagConn exists to satisfy stats.Handler. +func (h *serverMetricsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +// HandleConn exists to satisfy stats.Handler. +func (h *serverMetricsHandler) HandleConn(context.Context, stats.ConnStats) {} + +// TagRPC implements per RPC context management for metrics. +func (h *serverMetricsHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + ctx, _ = getOrCreateServerRPCInfo(ctx, info, h.options.MetricsOptions) + return ctx +} + +// HandleRPC handles per RPC stats implementation. +func (h *serverMetricsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { + ri := serverRPCInfo(ctx) + if ri == nil { + logger.Error("ctx passed into server side stats handler metrics event handling has no server call data present") + return + } + h.processRPCData(ctx, rs, ri.ai) +} + +func (h *serverMetricsHandler) processRPCData(ctx context.Context, s stats.RPCStats, ai *attemptInfo) { + switch st := s.(type) { + case *stats.InHeader: + if ai.pluginOptionLabels == nil && h.options.MetricsOptions.pluginOption != nil { + labels := h.options.MetricsOptions.pluginOption.GetLabels(st.Header) + if labels == nil { + labels = map[string]string{} // Shouldn't return a nil map. Make it empty if so to ignore future Get Calls for this Attempt. + } + ai.pluginOptionLabels = labels + } + attrs := otelmetric.WithAttributeSet(otelattribute.NewSet( + otelattribute.String("grpc.method", ai.method), + )) + h.serverMetrics.callStarted.Add(ctx, 1, attrs) + case *stats.OutPayload: + atomic.AddInt64(&ai.sentCompressedBytes, int64(st.CompressedLength)) + case *stats.InPayload: + atomic.AddInt64(&ai.recvCompressedBytes, int64(st.CompressedLength)) + case *stats.End: + h.processRPCEnd(ctx, ai, st) + default: + } +} + +func (h *serverMetricsHandler) processRPCEnd(ctx context.Context, ai *attemptInfo, e *stats.End) { + latency := float64(time.Since(ai.startTime)) / float64(time.Second) + st := "OK" + if e.Error != nil { + s, _ := status.FromError(e.Error) + st = canonicalString(s.Code()) + } + attributes := []otelattribute.KeyValue{ + otelattribute.String("grpc.method", ai.method), + otelattribute.String("grpc.status", st), + } + for k, v := range ai.pluginOptionLabels { + attributes = append(attributes, otelattribute.String(k, v)) + } + + // Allocate vararg slice once. + opts := []otelmetric.RecordOption{otelmetric.WithAttributeSet(otelattribute.NewSet(attributes...))} + h.serverMetrics.callDuration.Record(ctx, latency, opts...) + h.serverMetrics.callSentTotalCompressedMessageSize.Record(ctx, atomic.LoadInt64(&ai.sentCompressedBytes), opts...) + h.serverMetrics.callRcvdTotalCompressedMessageSize.Record(ctx, atomic.LoadInt64(&ai.recvCompressedBytes), opts...) +} + +const ( + // ServerCallStartedMetricName is the number of server calls started. + ServerCallStartedMetricName string = "grpc.server.call.started" + // ServerCallSentCompressedTotalMessageSizeMetricName is the compressed + // message bytes sent per server call. + ServerCallSentCompressedTotalMessageSizeMetricName string = "grpc.server.call.sent_total_compressed_message_size" + // ServerCallRcvdCompressedTotalMessageSizeMetricName is the compressed + // message bytes received per server call. + ServerCallRcvdCompressedTotalMessageSizeMetricName string = "grpc.server.call.rcvd_total_compressed_message_size" + // ServerCallDurationMetricName is the end-to-end time taken to complete a + // call from server transport's perspective. + ServerCallDurationMetricName string = "grpc.server.call.duration" +) diff --git a/stats/opentelemetry/server_tracing.go b/stats/opentelemetry/server_tracing.go new file mode 100644 index 0000000..952e787 --- /dev/null +++ b/stats/opentelemetry/server_tracing.go @@ -0,0 +1,84 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opentelemetry + +import ( + "context" + "log" + "strings" + + "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc" + "google.golang.org/grpc/stats" + otelinternaltracing "google.golang.org/grpc/stats/opentelemetry/internal/tracing" +) + +type serverTracingHandler struct { + options Options +} + +func (h *serverTracingHandler) initializeTraces() { + if h.options.TraceOptions.TracerProvider == nil { + log.Printf("TracerProvider is not provided in server TraceOptions") + return + } +} + +// TagRPC implements per RPC attempt context management for traces. +func (h *serverTracingHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + ctx, ri := getOrCreateServerRPCInfo(ctx, info, h.options.MetricsOptions) + ctx, _ = h.traceTagRPC(ctx, ri.ai) + return ctx +} + +// traceTagRPC populates context with new span data using the TextMapPropagator +// supplied in trace options and internal itracing.Carrier. It creates a new +// incoming carrier which extracts an existing span context (if present) by +// deserializing from provided context. If valid span context is extracted, it +// is set as parent of the new span otherwise new span remains the root span. +// If TextMapPropagator is not provided in the trace options, it returns context +// as is. +func (h *serverTracingHandler) traceTagRPC(ctx context.Context, ai *attemptInfo) (context.Context, *attemptInfo) { + mn := "Recv." + strings.Replace(ai.method, "/", ".", -1) + var span trace.Span + tracer := h.options.TraceOptions.TracerProvider.Tracer(tracerName, trace.WithInstrumentationVersion(grpc.Version)) + ctx = h.options.TraceOptions.TextMapPropagator.Extract(ctx, otelinternaltracing.NewIncomingCarrier(ctx)) + // If the context.Context provided in `ctx` to tracer.Start(), contains a + // span then the newly-created Span will be a child of that span, + // otherwise it will be a root span. + ctx, span = tracer.Start(ctx, mn, trace.WithSpanKind(trace.SpanKindServer)) + ai.traceSpan = span + return ctx, ai +} + +// HandleRPC handles per RPC tracing implementation. +func (h *serverTracingHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { + ri := serverRPCInfo(ctx) + if ri == nil { + logger.Error("ctx passed into server side tracing handler trace event handling has no server call data present") + return + } + populateSpan(rs, ri.ai) +} + +// TagConn exists to satisfy stats.Handler for tracing. +func (h *serverTracingHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +// HandleConn exists to satisfy stats.Handler for tracing. +func (h *serverTracingHandler) HandleConn(context.Context, stats.ConnStats) {} diff --git a/stats/opentelemetry/trace.go b/stats/opentelemetry/trace.go new file mode 100644 index 0000000..3ee66d1 --- /dev/null +++ b/stats/opentelemetry/trace.go @@ -0,0 +1,74 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opentelemetry + +import ( + "go.opentelemetry.io/otel/attribute" + otelcodes "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +// populateSpan populates span information based on stats passed in, representing +// invariants of the RPC lifecycle. It ends the span, triggering its export. +// This function handles attempt spans on the client-side and call spans on the +// server-side. +func populateSpan(rs stats.RPCStats, ai *attemptInfo) { + if ai == nil || ai.traceSpan == nil { + // Shouldn't happen, tagRPC call comes before this function gets called + // which populates this information. + logger.Error("ctx passed into stats handler tracing event handling has no traceSpan present") + return + } + span := ai.traceSpan + + switch rs := rs.(type) { + case *stats.DelayedPickComplete: + span.AddEvent("Delayed LB pick complete") + case *stats.InPayload: + // message id - "must be calculated as two different counters starting + // from one for sent messages and one for received messages." + attrs := []attribute.KeyValue{ + attribute.Int64("sequence-number", int64(ai.countRecvMsg)), + attribute.Int64("message-size", int64(rs.Length)), + } + if rs.CompressedLength != rs.Length { + attrs = append(attrs, attribute.Int64("message-size-compressed", int64(rs.CompressedLength))) + } + span.AddEvent("Inbound message", trace.WithAttributes(attrs...)) + ai.countRecvMsg++ + case *stats.OutPayload: + attrs := []attribute.KeyValue{ + attribute.Int64("sequence-number", int64(ai.countSentMsg)), + attribute.Int64("message-size", int64(rs.Length)), + } + if rs.CompressedLength != rs.Length { + attrs = append(attrs, attribute.Int64("message-size-compressed", int64(rs.CompressedLength))) + } + span.AddEvent("Outbound message", trace.WithAttributes(attrs...)) + ai.countSentMsg++ + case *stats.End: + if rs.Error != nil { + s := status.Convert(rs.Error) + span.SetStatus(otelcodes.Error, s.Message()) + } else { + span.SetStatus(otelcodes.Ok, "Ok") + } + span.End() + } +} diff --git a/stats/stats.go b/stats/stats.go new file mode 100644 index 0000000..c699672 --- /dev/null +++ b/stats/stats.go @@ -0,0 +1,321 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package stats is for collecting and reporting various network and RPC stats. +// This package is for monitoring purpose only. All fields are read-only. +// All APIs are experimental. +package stats // import "google.golang.org/grpc/stats" + +import ( + "context" + "net" + "time" + + "google.golang.org/grpc/metadata" +) + +// RPCStats contains stats information about RPCs. +type RPCStats interface { + isRPCStats() + // IsClient returns true if this RPCStats is from client side. + IsClient() bool +} + +// Begin contains stats for the start of an RPC attempt. +// +// - Server-side: Triggered after `InHeader`, as headers are processed +// before the RPC lifecycle begins. +// - Client-side: The first stats event recorded. +// +// FailFast is only valid if this Begin is from client side. +type Begin struct { + // Client is true if this Begin is from client side. + Client bool + // BeginTime is the time when the RPC attempt begins. + BeginTime time.Time + // FailFast indicates if this RPC is failfast. + FailFast bool + // IsClientStream indicates whether the RPC is a client streaming RPC. + IsClientStream bool + // IsServerStream indicates whether the RPC is a server streaming RPC. + IsServerStream bool + // IsTransparentRetryAttempt indicates whether this attempt was initiated + // due to transparently retrying a previous attempt. + IsTransparentRetryAttempt bool +} + +// IsClient indicates if the stats information is from client side. +func (s *Begin) IsClient() bool { return s.Client } + +func (s *Begin) isRPCStats() {} + +// DelayedPickComplete indicates that the RPC is unblocked following a delay in +// selecting a connection for the call. +type DelayedPickComplete struct{} + +// IsClient indicates DelayedPickComplete is available on the client. +func (*DelayedPickComplete) IsClient() bool { return true } + +func (*DelayedPickComplete) isRPCStats() {} + +// PickerUpdated indicates that the RPC is unblocked following a delay in +// selecting a connection for the call. +// +// Deprecated: will be removed in a future release; use DelayedPickComplete +// instead. +type PickerUpdated = DelayedPickComplete + +// InPayload contains stats about an incoming payload. +type InPayload struct { + // Client is true if this InPayload is from client side. + Client bool + // Payload is the payload with original type. This may be modified after + // the call to HandleRPC which provides the InPayload returns and must be + // copied if needed later. + Payload any + + // Length is the size of the uncompressed payload data. Does not include any + // framing (gRPC or HTTP/2). + Length int + // CompressedLength is the size of the compressed payload data. Does not + // include any framing (gRPC or HTTP/2). Same as Length if compression not + // enabled. + CompressedLength int + // WireLength is the size of the compressed payload data plus gRPC framing. + // Does not include HTTP/2 framing. + WireLength int + + // RecvTime is the time when the payload is received. + RecvTime time.Time +} + +// IsClient indicates if the stats information is from client side. +func (s *InPayload) IsClient() bool { return s.Client } + +func (s *InPayload) isRPCStats() {} + +// InHeader contains stats about header reception. +// +// - Server-side: The first stats event after the RPC request is received. +type InHeader struct { + // Client is true if this InHeader is from client side. + Client bool + // WireLength is the wire length of header. + WireLength int + // Compression is the compression algorithm used for the RPC. + Compression string + // Header contains the header metadata received. + Header metadata.MD + + // The following fields are valid only if Client is false. + // FullMethod is the full RPC method string, i.e., /package.service/method. + FullMethod string + // RemoteAddr is the remote address of the corresponding connection. + RemoteAddr net.Addr + // LocalAddr is the local address of the corresponding connection. + LocalAddr net.Addr +} + +// IsClient indicates if the stats information is from client side. +func (s *InHeader) IsClient() bool { return s.Client } + +func (s *InHeader) isRPCStats() {} + +// InTrailer contains stats about trailer reception. +type InTrailer struct { + // Client is true if this InTrailer is from client side. + Client bool + // WireLength is the wire length of trailer. + WireLength int + // Trailer contains the trailer metadata received from the server. This + // field is only valid if this InTrailer is from the client side. + Trailer metadata.MD +} + +// IsClient indicates if the stats information is from client side. +func (s *InTrailer) IsClient() bool { return s.Client } + +func (s *InTrailer) isRPCStats() {} + +// OutPayload contains stats about an outgoing payload. +type OutPayload struct { + // Client is true if this OutPayload is from client side. + Client bool + // Payload is the payload with original type. This may be modified after + // the call to HandleRPC which provides the OutPayload returns and must be + // copied if needed later. + Payload any + // Length is the size of the uncompressed payload data. Does not include any + // framing (gRPC or HTTP/2). + Length int + // CompressedLength is the size of the compressed payload data. Does not + // include any framing (gRPC or HTTP/2). Same as Length if compression not + // enabled. + CompressedLength int + // WireLength is the size of the compressed payload data plus gRPC framing. + // Does not include HTTP/2 framing. + WireLength int + // SentTime is the time when the payload is sent. + SentTime time.Time +} + +// IsClient indicates if this stats information is from client side. +func (s *OutPayload) IsClient() bool { return s.Client } + +func (s *OutPayload) isRPCStats() {} + +// OutHeader contains stats about header transmission. +// +// - Client-side: Only occurs after 'Begin', as headers are always the first +// thing sent on a stream. +type OutHeader struct { + // Client is true if this OutHeader is from client side. + Client bool + // Compression is the compression algorithm used for the RPC. + Compression string + // Header contains the header metadata sent. + Header metadata.MD + // Authority is the :authority pseudo-header sent for the RPC. + // It is valid only if Client is true. + Authority string + + // The following fields are valid only if Client is true. + // FullMethod is the full RPC method string, i.e., /package.service/method. + FullMethod string + // RemoteAddr is the remote address of the corresponding connection. + RemoteAddr net.Addr + // LocalAddr is the local address of the corresponding connection. + LocalAddr net.Addr +} + +// IsClient indicates if this stats information is from client side. +func (s *OutHeader) IsClient() bool { return s.Client } + +func (s *OutHeader) isRPCStats() {} + +// OutTrailer contains stats about trailer transmission. +type OutTrailer struct { + // Client is true if this OutTrailer is from client side. + Client bool + // WireLength is the wire length of trailer. + // + // Deprecated: This field is never set. The length is not known when this + // message is emitted because the trailer fields are compressed with hpack + // after that. + WireLength int + // Trailer contains the trailer metadata sent to the client. This + // field is only valid if this OutTrailer is from the server side. + Trailer metadata.MD +} + +// IsClient indicates if this stats information is from client side. +func (s *OutTrailer) IsClient() bool { return s.Client } + +func (s *OutTrailer) isRPCStats() {} + +// End contains stats about RPC completion. +type End struct { + // Client is true if this End is from client side. + Client bool + // BeginTime is the time when the RPC began. + BeginTime time.Time + // EndTime is the time when the RPC ends. + EndTime time.Time + // Trailer contains the trailer metadata received from the server. This + // field is only valid if this End is from the client side. + // Deprecated: use Trailer in InTrailer instead. + Trailer metadata.MD + // Error is the error the RPC ended with. It is an error generated from + // status.Status and can be converted back to status.Status using + // status.FromError if non-nil. + Error error +} + +// IsClient indicates if this is from client side. +func (s *End) IsClient() bool { return s.Client } + +func (s *End) isRPCStats() {} + +// ConnStats contains stats information about connections. +type ConnStats interface { + isConnStats() + // IsClient returns true if this ConnStats is from client side. + IsClient() bool +} + +// ConnBegin contains stats about connection establishment. +type ConnBegin struct { + // Client is true if this ConnBegin is from client side. + Client bool +} + +// IsClient indicates if this is from client side. +func (s *ConnBegin) IsClient() bool { return s.Client } + +func (s *ConnBegin) isConnStats() {} + +// ConnEnd contains stats about connection termination. +type ConnEnd struct { + // Client is true if this ConnEnd is from client side. + Client bool +} + +// IsClient indicates if this is from client side. +func (s *ConnEnd) IsClient() bool { return s.Client } + +func (s *ConnEnd) isConnStats() {} + +// SetTags attaches stats tagging data to the context, which will be sent in +// the outgoing RPC with the header grpc-tags-bin. Subsequent calls to +// SetTags will overwrite the values from earlier calls. +// +// Deprecated: set the `grpc-tags-bin` header in the metadata instead. +func SetTags(ctx context.Context, b []byte) context.Context { + return metadata.AppendToOutgoingContext(ctx, "grpc-tags-bin", string(b)) +} + +// Tags returns the tags from the context for the inbound RPC. +// +// Deprecated: obtain the `grpc-tags-bin` header from metadata instead. +func Tags(ctx context.Context) []byte { + traceValues := metadata.ValueFromIncomingContext(ctx, "grpc-tags-bin") + if len(traceValues) == 0 { + return nil + } + return []byte(traceValues[len(traceValues)-1]) +} + +// SetTrace attaches stats tagging data to the context, which will be sent in +// the outgoing RPC with the header grpc-trace-bin. Subsequent calls to +// SetTrace will overwrite the values from earlier calls. +// +// Deprecated: set the `grpc-trace-bin` header in the metadata instead. +func SetTrace(ctx context.Context, b []byte) context.Context { + return metadata.AppendToOutgoingContext(ctx, "grpc-trace-bin", string(b)) +} + +// Trace returns the trace from the context for the inbound RPC. +// +// Deprecated: obtain the `grpc-trace-bin` header from metadata instead. +func Trace(ctx context.Context) []byte { + traceValues := metadata.ValueFromIncomingContext(ctx, "grpc-trace-bin") + if len(traceValues) == 0 { + return nil + } + return []byte(traceValues[len(traceValues)-1]) +} diff --git a/stats/stats_test.go b/stats/stats_test.go new file mode 100644 index 0000000..86faba1 --- /dev/null +++ b/stats/stats_test.go @@ -0,0 +1,1540 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package stats_test + +import ( + "context" + "fmt" + "io" + "reflect" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func init() { + grpc.EnableTracing = false +} + +type connCtxKey struct{} +type rpcCtxKey struct{} + +var ( + // For headers sent to server: + testMetadata = metadata.MD{ + "key1": []string{"value1"}, + "key2": []string{"value2"}, + "user-agent": []string{fmt.Sprintf("test/0.0.1 grpc-go/%s", grpc.Version)}, + } + // For headers sent from server: + testHeaderMetadata = metadata.MD{ + "hkey1": []string{"headerValue1"}, + "hkey2": []string{"headerValue2"}, + } + // For trailers sent from server: + testTrailerMetadata = metadata.MD{ + "tkey1": []string{"trailerValue1"}, + "tkey2": []string{"trailerValue2"}, + } + // The id for which the service handler should return error. + errorID int32 = 32202 +) + +func idToPayload(id int32) *testpb.Payload { + return &testpb.Payload{Body: []byte{byte(id), byte(id >> 8), byte(id >> 16), byte(id >> 24)}} +} + +func payloadToID(p *testpb.Payload) int32 { + if p == nil || len(p.Body) != 4 { + panic("invalid payload") + } + return int32(p.Body[0]) + int32(p.Body[1])<<8 + int32(p.Body[2])<<16 + int32(p.Body[3])<<24 +} + +func setIncomingStats(ctx context.Context, mdKey string, b []byte) context.Context { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + md = metadata.MD{} + } + md.Set(mdKey, string(b)) + return metadata.NewIncomingContext(ctx, md) +} + +func getOutgoingStats(ctx context.Context, mdKey string) []byte { + md, ok := metadata.FromOutgoingContext(ctx) + if !ok { + return nil + } + tagValues := md.Get(mdKey) + if len(tagValues) == 0 { + return nil + } + return []byte(tagValues[len(tagValues)-1]) +} + +func newStatsTestStubServer() *stubserver.StubServer { + return &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + if err := grpc.SendHeader(ctx, testHeaderMetadata); err != nil { + return nil, status.Errorf(status.Code(err), "grpc.SendHeader(_, %v) = %v, want ", testHeaderMetadata, err) + } + if err := grpc.SetTrailer(ctx, testTrailerMetadata); err != nil { + return nil, status.Errorf(status.Code(err), "grpc.SetTrailer(_, %v) = %v, want ", testTrailerMetadata, err) + } + + if id := payloadToID(in.Payload); id == errorID { + return nil, fmt.Errorf("got error id: %v", id) + } + + return &testpb.SimpleResponse{Payload: in.Payload}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if err := stream.SendHeader(testHeaderMetadata); err != nil { + return status.Errorf(status.Code(err), "%v.SendHeader(%v) = %v, want %v", stream, testHeaderMetadata, err, nil) + } + stream.SetTrailer(testTrailerMetadata) + for { + in, err := stream.Recv() + if err == io.EOF { + // read done. + return nil + } + if err != nil { + return err + } + + if id := payloadToID(in.Payload); id == errorID { + return fmt.Errorf("got error id: %v", id) + } + + if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: in.Payload}); err != nil { + return err + } + } + }, + StreamingInputCallF: func(stream testgrpc.TestService_StreamingInputCallServer) error { + if err := stream.SendHeader(testHeaderMetadata); err != nil { + return status.Errorf(status.Code(err), "%v.SendHeader(%v) = %v, want %v", stream, testHeaderMetadata, err, nil) + } + stream.SetTrailer(testTrailerMetadata) + for { + in, err := stream.Recv() + if err == io.EOF { + // read done. + return stream.SendAndClose(&testpb.StreamingInputCallResponse{AggregatedPayloadSize: 0}) + } + if err != nil { + return err + } + + if id := payloadToID(in.Payload); id == errorID { + return fmt.Errorf("got error id: %v", id) + } + } + }, + StreamingOutputCallF: func(req *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error { + if err := stream.SendHeader(testHeaderMetadata); err != nil { + return status.Errorf(status.Code(err), "%v.SendHeader(%v) = %v, want %v", stream, testHeaderMetadata, err, nil) + } + stream.SetTrailer(testTrailerMetadata) + + if id := payloadToID(req.Payload); id == errorID { + return fmt.Errorf("got error id: %v", id) + } + + for i := 0; i < 5; i++ { + if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: req.Payload}); err != nil { + return err + } + } + return nil + }, + } +} + +// test is an end-to-end test. It should be created with the newTest +// func, modified as needed, and then started with its startServer method. +// It should be cleaned up with the tearDown method. +type test struct { + t *testing.T + compress string + clientStatsHandlers []stats.Handler + serverStatsHandlers []stats.Handler + + // srv and srvAddr are set once startServer is called. + srv stubserver.GRPCServer + srvAddr string + + cc *grpc.ClientConn // nil until requested via clientConn +} + +func (te *test) tearDown() { + if te.cc != nil { + te.cc.Close() + te.cc = nil + } + te.srv.Stop() +} + +type testConfig struct { + compress string +} + +// newTest returns a new test using the provided testing.T and +// environment. It is returned with default values. Tests should +// modify it before calling its startServer and clientConn methods. +func newTest(t *testing.T, tc *testConfig, chs []stats.Handler, shs []stats.Handler) *test { + te := &test{ + t: t, + compress: tc.compress, + clientStatsHandlers: chs, + serverStatsHandlers: shs, + } + return te +} + +// startServer starts a gRPC server listening. Callers should defer a +// call to te.tearDown to clean up. +func (te *test) startServer(ss *stubserver.StubServer) { + var opts []grpc.ServerOption + if te.compress == "gzip" { + opts = append(opts, + grpc.RPCCompressor(grpc.NewGZIPCompressor()), + grpc.RPCDecompressor(grpc.NewGZIPDecompressor()), + ) + } + for _, sh := range te.serverStatsHandlers { + opts = append(opts, grpc.StatsHandler(sh)) + } + if err := ss.StartServer(opts...); err != nil { + te.t.Fatalf("StubServer.StartServer() failed: %v", err) + } + te.srv = ss.S + te.srvAddr = ss.Address +} + +func (te *test) clientConn(ctx context.Context) *grpc.ClientConn { + if te.cc != nil { + return te.cc + } + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithUserAgent("test/0.0.1"), + } + if te.compress == "gzip" { + opts = append(opts, + grpc.WithCompressor(grpc.NewGZIPCompressor()), + grpc.WithDecompressor(grpc.NewGZIPDecompressor()), + ) + } + for _, sh := range te.clientStatsHandlers { + opts = append(opts, grpc.WithStatsHandler(sh)) + } + + var err error + te.cc, err = grpc.NewClient(te.srvAddr, opts...) + if err != nil { + te.t.Fatalf("grpc.NewClient(%q) failed: %v", te.srvAddr, err) + } + te.cc.Connect() + testutils.AwaitState(ctx, te.t, te.cc, connectivity.Ready) + return te.cc +} + +type rpcType int + +const ( + unaryRPC rpcType = iota + clientStreamRPC + serverStreamRPC + fullDuplexStreamRPC +) + +type rpcConfig struct { + count int // Number of requests and responses for streaming RPCs. + success bool // Whether the RPC should succeed or return error. + failfast bool + authority string // Authority override for the RPC. + callType rpcType // Type of RPC. +} + +func (te *test) doUnaryCall(c *rpcConfig) (*testpb.SimpleRequest, *testpb.SimpleResponse, error) { + var ( + resp *testpb.SimpleResponse + req *testpb.SimpleRequest + err error + ) + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + tc := testgrpc.NewTestServiceClient(te.clientConn(tCtx)) + if c.success { + req = &testpb.SimpleRequest{Payload: idToPayload(errorID + 1)} + } else { + req = &testpb.SimpleRequest{Payload: idToPayload(errorID)} + } + + callOpts := []grpc.CallOption{grpc.WaitForReady(!c.failfast)} + if c.authority != "" { + callOpts = append(callOpts, grpc.CallAuthority(c.authority)) + } + resp, err = tc.UnaryCall(metadata.NewOutgoingContext(tCtx, testMetadata), req, callOpts...) + return req, resp, err +} + +func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]proto.Message, []proto.Message, error) { + var ( + reqs []proto.Message + resps []proto.Message + err error + ) + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + tc := testgrpc.NewTestServiceClient(te.clientConn(tCtx)) + stream, err := tc.FullDuplexCall(metadata.NewOutgoingContext(tCtx, testMetadata), grpc.WaitForReady(!c.failfast)) + if err != nil { + return reqs, resps, err + } + var startID int32 + if !c.success { + startID = errorID + } + for i := 0; i < c.count; i++ { + req := &testpb.StreamingOutputCallRequest{ + Payload: idToPayload(int32(i) + startID), + } + reqs = append(reqs, req) + if err = stream.Send(req); err != nil { + return reqs, resps, err + } + var resp *testpb.StreamingOutputCallResponse + if resp, err = stream.Recv(); err != nil { + return reqs, resps, err + } + resps = append(resps, resp) + } + if err = stream.CloseSend(); err != nil && err != io.EOF { + return reqs, resps, err + } + if _, err = stream.Recv(); err != io.EOF { + return reqs, resps, err + } + + return reqs, resps, nil +} + +func (te *test) doClientStreamCall(c *rpcConfig) ([]proto.Message, *testpb.StreamingInputCallResponse, error) { + var ( + reqs []proto.Message + resp *testpb.StreamingInputCallResponse + err error + ) + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + tc := testgrpc.NewTestServiceClient(te.clientConn(tCtx)) + stream, err := tc.StreamingInputCall(metadata.NewOutgoingContext(tCtx, testMetadata), grpc.WaitForReady(!c.failfast)) + if err != nil { + return reqs, resp, err + } + var startID int32 + if !c.success { + startID = errorID + } + for i := 0; i < c.count; i++ { + req := &testpb.StreamingInputCallRequest{ + Payload: idToPayload(int32(i) + startID), + } + reqs = append(reqs, req) + if err = stream.Send(req); err != nil { + return reqs, resp, err + } + } + resp, err = stream.CloseAndRecv() + return reqs, resp, err +} + +func (te *test) doServerStreamCall(c *rpcConfig) (*testpb.StreamingOutputCallRequest, []proto.Message, error) { + var ( + req *testpb.StreamingOutputCallRequest + resps []proto.Message + err error + ) + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + tc := testgrpc.NewTestServiceClient(te.clientConn(tCtx)) + + var startID int32 + if !c.success { + startID = errorID + } + req = &testpb.StreamingOutputCallRequest{Payload: idToPayload(startID)} + stream, err := tc.StreamingOutputCall(metadata.NewOutgoingContext(tCtx, testMetadata), req, grpc.WaitForReady(!c.failfast)) + if err != nil { + return req, resps, err + } + for { + var resp *testpb.StreamingOutputCallResponse + resp, err := stream.Recv() + if err == io.EOF { + return req, resps, nil + } else if err != nil { + return req, resps, err + } + resps = append(resps, resp) + } +} + +type expectedData struct { + method string + isClientStream bool + isServerStream bool + serverAddr string + compression string + authority string + reqIdx int + requests []proto.Message + respIdx int + responses []proto.Message + err error + failfast bool +} + +type gotData struct { + ctx context.Context + client bool + s any // This could be RPCStats or ConnStats. +} + +const ( + begin int = iota + end + inPayload + inHeader + inTrailer + outPayload + outHeader + // TODO: test outTrailer ? + connBegin + connEnd +) + +func checkBegin(t *testing.T, d *gotData, e *expectedData) { + var ( + ok bool + st *stats.Begin + ) + if st, ok = d.s.(*stats.Begin); !ok { + t.Fatalf("got %T, want Begin", d.s) + } + if d.ctx == nil { + t.Fatalf("d.ctx = nil, want ") + } + if st.BeginTime.IsZero() { + t.Fatalf("st.BeginTime = %v, want ", st.BeginTime) + } + if d.client { + if st.FailFast != e.failfast { + t.Fatalf("st.FailFast = %v, want %v", st.FailFast, e.failfast) + } + } + if st.IsClientStream != e.isClientStream { + t.Fatalf("st.IsClientStream = %v, want %v", st.IsClientStream, e.isClientStream) + } + if st.IsServerStream != e.isServerStream { + t.Fatalf("st.IsServerStream = %v, want %v", st.IsServerStream, e.isServerStream) + } +} + +func checkInHeader(t *testing.T, d *gotData, e *expectedData) { + var ( + ok bool + st *stats.InHeader + ) + if st, ok = d.s.(*stats.InHeader); !ok { + t.Fatalf("got %T, want InHeader", d.s) + } + if d.ctx == nil { + t.Fatalf("d.ctx = nil, want ") + } + if st.Compression != e.compression { + t.Fatalf("st.Compression = %v, want %v", st.Compression, e.compression) + } + if d.client { + // additional headers might be injected so instead of testing equality, test that all the + // expected headers keys have the expected header values. + for key := range testHeaderMetadata { + if !reflect.DeepEqual(st.Header.Get(key), testHeaderMetadata.Get(key)) { + t.Fatalf("st.Header[%s] = %v, want %v", key, st.Header.Get(key), testHeaderMetadata.Get(key)) + } + } + } else { + if st.FullMethod != e.method { + t.Fatalf("st.FullMethod = %s, want %v", st.FullMethod, e.method) + } + if st.LocalAddr.String() != e.serverAddr { + t.Fatalf("st.LocalAddr = %v, want %v", st.LocalAddr, e.serverAddr) + } + // additional headers might be injected so instead of testing equality, test that all the + // expected headers keys have the expected header values. + for key := range testMetadata { + if !reflect.DeepEqual(st.Header.Get(key), testMetadata.Get(key)) { + t.Fatalf("st.Header[%s] = %v, want %v", key, st.Header.Get(key), testMetadata.Get(key)) + } + } + + if connInfo, ok := d.ctx.Value(connCtxKey{}).(*stats.ConnTagInfo); ok { + if connInfo.RemoteAddr != st.RemoteAddr { + t.Fatalf("connInfo.RemoteAddr = %v, want %v", connInfo.RemoteAddr, st.RemoteAddr) + } + if connInfo.LocalAddr != st.LocalAddr { + t.Fatalf("connInfo.LocalAddr = %v, want %v", connInfo.LocalAddr, st.LocalAddr) + } + } else { + t.Fatalf("got context %v, want one with connCtxKey", d.ctx) + } + if rpcInfo, ok := d.ctx.Value(rpcCtxKey{}).(*stats.RPCTagInfo); ok { + if rpcInfo.FullMethodName != st.FullMethod { + t.Fatalf("rpcInfo.FullMethod = %s, want %v", rpcInfo.FullMethodName, st.FullMethod) + } + } else { + t.Fatalf("got context %v, want one with rpcCtxKey", d.ctx) + } + } +} + +func checkInPayload(t *testing.T, d *gotData, e *expectedData) { + var ( + ok bool + st *stats.InPayload + ) + if st, ok = d.s.(*stats.InPayload); !ok { + t.Fatalf("got %T, want InPayload", d.s) + } + if d.ctx == nil { + t.Fatalf("d.ctx = nil, want ") + } + + var idx *int + var payloads []proto.Message + if d.client { + idx = &e.respIdx + payloads = e.responses + } else { + idx = &e.reqIdx + payloads = e.requests + } + + wantPayload := payloads[*idx] + if diff := cmp.Diff(wantPayload, st.Payload.(proto.Message), protocmp.Transform()); diff != "" { + t.Fatalf("unexpected difference in st.Payload (-want +got):\n%s", diff) + } + *idx++ + if st.Length != proto.Size(wantPayload) { + t.Fatalf("st.Length = %v, want %v", st.Length, proto.Size(wantPayload)) + } + + // Below are sanity checks that WireLength and RecvTime are populated. + // TODO: check values of WireLength and RecvTime. + if st.Length > 0 && st.CompressedLength == 0 { + t.Fatalf("st.WireLength = %v with non-empty data, want ", + st.CompressedLength) + } + if st.RecvTime.IsZero() { + t.Fatalf("st.ReceivedTime = %v, want ", st.RecvTime) + } +} + +func checkInTrailer(t *testing.T, d *gotData, _ *expectedData) { + var ( + ok bool + st *stats.InTrailer + ) + if st, ok = d.s.(*stats.InTrailer); !ok { + t.Fatalf("got %T, want InTrailer", d.s) + } + if d.ctx == nil { + t.Fatalf("d.ctx = nil, want ") + } + if !st.Client { + t.Fatalf("st IsClient = false, want true") + } + if !reflect.DeepEqual(st.Trailer, testTrailerMetadata) { + t.Fatalf("st.Trailer = %v, want %v", st.Trailer, testTrailerMetadata) + } +} + +func checkOutHeader(t *testing.T, d *gotData, e *expectedData) { + var ( + ok bool + st *stats.OutHeader + ) + if st, ok = d.s.(*stats.OutHeader); !ok { + t.Fatalf("got %T, want OutHeader", d.s) + } + if d.ctx == nil { + t.Fatalf("d.ctx = nil, want ") + } + if st.Compression != e.compression { + t.Fatalf("st.Compression = %v, want %v", st.Compression, e.compression) + } + if d.client { + if st.FullMethod != e.method { + t.Fatalf("st.FullMethod = %s, want %v", st.FullMethod, e.method) + } + if st.RemoteAddr.String() != e.serverAddr { + t.Fatalf("st.RemoteAddr = %v, want %v", st.RemoteAddr, e.serverAddr) + } + wantAuthority := e.authority + if wantAuthority == "" { + wantAuthority = e.serverAddr + } + if st.Authority != wantAuthority { + t.Fatalf("st.Authority = %s, want %s", st.Authority, wantAuthority) + } + // additional headers might be injected so instead of testing equality, test that all the + // expected headers keys have the expected header values. + for key := range testMetadata { + if !reflect.DeepEqual(st.Header.Get(key), testMetadata.Get(key)) { + t.Fatalf("st.Header[%s] = %v, want %v", key, st.Header.Get(key), testMetadata.Get(key)) + } + } + + if rpcInfo, ok := d.ctx.Value(rpcCtxKey{}).(*stats.RPCTagInfo); ok { + if rpcInfo.FullMethodName != st.FullMethod { + t.Fatalf("rpcInfo.FullMethod = %s, want %v", rpcInfo.FullMethodName, st.FullMethod) + } + } else { + t.Fatalf("got context %v, want one with rpcCtxKey", d.ctx) + } + } else { + // additional headers might be injected so instead of testing equality, test that all the + // expected headers keys have the expected header values. + for key := range testHeaderMetadata { + if !reflect.DeepEqual(st.Header.Get(key), testHeaderMetadata.Get(key)) { + t.Fatalf("st.Header[%s] = %v, want %v", key, st.Header.Get(key), testHeaderMetadata.Get(key)) + } + } + } +} + +func checkOutPayload(t *testing.T, d *gotData, e *expectedData) { + var ( + ok bool + st *stats.OutPayload + ) + if st, ok = d.s.(*stats.OutPayload); !ok { + t.Fatalf("got %T, want OutPayload", d.s) + } + if d.ctx == nil { + t.Fatalf("d.ctx = nil, want ") + } + + var idx *int + var payloads []proto.Message + if d.client { + idx = &e.reqIdx + payloads = e.requests + } else { + idx = &e.respIdx + payloads = e.responses + } + + expectedPayload := payloads[*idx] + if !proto.Equal(st.Payload.(proto.Message), expectedPayload) { + t.Fatalf("st.Payload = %v, want %v", st.Payload, expectedPayload) + } + *idx++ + if st.Length != proto.Size(expectedPayload) { + t.Fatalf("st.Length = %v, want %v", st.Length, proto.Size(expectedPayload)) + } + + // Below are sanity checks that Length, CompressedLength and SentTime are populated. + // TODO: check values of WireLength and SentTime. + if st.Length > 0 && st.WireLength == 0 { + t.Fatalf("st.WireLength = %v with non-empty data, want ", + st.WireLength) + } + if st.SentTime.IsZero() { + t.Fatalf("st.SentTime = %v, want ", st.SentTime) + } +} + +func checkOutTrailer(t *testing.T, d *gotData, _ *expectedData) { + var ( + ok bool + st *stats.OutTrailer + ) + if st, ok = d.s.(*stats.OutTrailer); !ok { + t.Fatalf("got %T, want OutTrailer", d.s) + } + if d.ctx == nil { + t.Fatalf("d.ctx = nil, want ") + } + if st.Client { + t.Fatalf("st IsClient = true, want false") + } + if !reflect.DeepEqual(st.Trailer, testTrailerMetadata) { + t.Fatalf("st.Trailer = %v, want %v", st.Trailer, testTrailerMetadata) + } +} + +func checkEnd(t *testing.T, d *gotData, e *expectedData) { + var ( + ok bool + st *stats.End + ) + if st, ok = d.s.(*stats.End); !ok { + t.Fatalf("got %T, want End", d.s) + } + if d.ctx == nil { + t.Fatalf("d.ctx = nil, want ") + } + if st.BeginTime.IsZero() { + t.Fatalf("st.BeginTime = %v, want ", st.BeginTime) + } + if st.EndTime.IsZero() { + t.Fatalf("st.EndTime = %v, want ", st.EndTime) + } + + actual, ok := status.FromError(st.Error) + if !ok { + t.Fatalf("expected st.Error to be a statusError, got %v (type %T)", st.Error, st.Error) + } + + expectedStatus, _ := status.FromError(e.err) + if actual.Code() != expectedStatus.Code() || actual.Message() != expectedStatus.Message() { + t.Fatalf("st.Error = %v, want %v", st.Error, e.err) + } + + if st.Client { + if !reflect.DeepEqual(st.Trailer, testTrailerMetadata) { + t.Fatalf("st.Trailer = %v, want %v", st.Trailer, testTrailerMetadata) + } + } else { + if st.Trailer != nil { + t.Fatalf("st.Trailer = %v, want nil", st.Trailer) + } + } +} + +func checkConnBegin(t *testing.T, d *gotData) { + var ( + ok bool + st *stats.ConnBegin + ) + if st, ok = d.s.(*stats.ConnBegin); !ok { + t.Fatalf("got %T, want ConnBegin", d.s) + } + if d.ctx == nil { + t.Fatalf("d.ctx = nil, want ") + } + st.IsClient() // TODO remove this. +} + +func checkConnEnd(t *testing.T, d *gotData) { + var ( + ok bool + st *stats.ConnEnd + ) + if st, ok = d.s.(*stats.ConnEnd); !ok { + t.Fatalf("got %T, want ConnEnd", d.s) + } + if d.ctx == nil { + t.Fatalf("d.ctx = nil, want ") + } + st.IsClient() // TODO remove this. +} + +type statshandler struct { + mu sync.Mutex + gotRPC []*gotData + gotConn []*gotData +} + +func (h *statshandler) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context { + return context.WithValue(ctx, connCtxKey{}, info) +} + +func (h *statshandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + return context.WithValue(ctx, rpcCtxKey{}, info) +} + +func (h *statshandler) HandleConn(ctx context.Context, s stats.ConnStats) { + h.mu.Lock() + defer h.mu.Unlock() + h.gotConn = append(h.gotConn, &gotData{ctx, s.IsClient(), s}) +} + +func (h *statshandler) HandleRPC(ctx context.Context, s stats.RPCStats) { + h.mu.Lock() + defer h.mu.Unlock() + h.gotRPC = append(h.gotRPC, &gotData{ctx, s.IsClient(), s}) +} + +func checkConnStats(t *testing.T, got []*gotData) { + if len(got) <= 0 || len(got)%2 != 0 { + for i, g := range got { + t.Errorf(" - %v, %T = %+v, ctx: %v", i, g.s, g.s, g.ctx) + } + t.Fatalf("got %v stats, want even positive number", len(got)) + } + // The first conn stats must be a ConnBegin. + checkConnBegin(t, got[0]) + // The last conn stats must be a ConnEnd. + checkConnEnd(t, got[len(got)-1]) +} + +func checkServerStats(t *testing.T, got []*gotData, expect *expectedData, checkFuncs []func(t *testing.T, d *gotData, e *expectedData)) { + if len(got) != len(checkFuncs) { + for i, g := range got { + t.Errorf(" - %v, %T", i, g.s) + } + t.Fatalf("got %v stats, want %v stats", len(got), len(checkFuncs)) + } + + for i, f := range checkFuncs { + f(t, got[i], expect) + } +} + +func testServerStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs []func(t *testing.T, d *gotData, e *expectedData)) { + h := &statshandler{} + te := newTest(t, tc, nil, []stats.Handler{h}) + te.startServer(newStatsTestStubServer()) + defer te.tearDown() + + var ( + reqs []proto.Message + resps []proto.Message + err error + method string + + isClientStream bool + isServerStream bool + + req proto.Message + resp proto.Message + e error + ) + + switch cc.callType { + case unaryRPC: + method = "/grpc.testing.TestService/UnaryCall" + req, resp, e = te.doUnaryCall(cc) + reqs = []proto.Message{req} + resps = []proto.Message{resp} + err = e + case clientStreamRPC: + method = "/grpc.testing.TestService/StreamingInputCall" + reqs, resp, e = te.doClientStreamCall(cc) + resps = []proto.Message{resp} + err = e + isClientStream = true + case serverStreamRPC: + method = "/grpc.testing.TestService/StreamingOutputCall" + req, resps, e = te.doServerStreamCall(cc) + reqs = []proto.Message{req} + err = e + isServerStream = true + case fullDuplexStreamRPC: + method = "/grpc.testing.TestService/FullDuplexCall" + reqs, resps, err = te.doFullDuplexCallRoundtrip(cc) + isClientStream = true + isServerStream = true + } + if cc.success != (err == nil) { + t.Fatalf("cc.success: %v, got error: %v", cc.success, err) + } + te.cc.Close() + te.srv.GracefulStop() // Wait for the server to stop. + + for { + h.mu.Lock() + if len(h.gotRPC) >= len(checkFuncs) { + h.mu.Unlock() + break + } + h.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } + + for { + h.mu.Lock() + if _, ok := h.gotConn[len(h.gotConn)-1].s.(*stats.ConnEnd); ok { + h.mu.Unlock() + break + } + h.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } + + expect := &expectedData{ + serverAddr: te.srvAddr, + compression: tc.compress, + method: method, + requests: reqs, + responses: resps, + err: err, + isClientStream: isClientStream, + isServerStream: isServerStream, + } + + h.mu.Lock() + checkConnStats(t, h.gotConn) + h.mu.Unlock() + checkServerStats(t, h.gotRPC, expect, checkFuncs) +} + +func (s) TestServerStatsUnaryRPC(t *testing.T) { + testServerStats(t, &testConfig{compress: ""}, &rpcConfig{success: true, callType: unaryRPC}, []func(t *testing.T, d *gotData, e *expectedData){ + checkInHeader, + checkBegin, + checkInPayload, + checkOutHeader, + checkOutPayload, + checkOutTrailer, + checkEnd, + }) +} + +func (s) TestServerStatsUnaryRPCError(t *testing.T) { + testServerStats(t, &testConfig{compress: ""}, &rpcConfig{success: false, callType: unaryRPC}, []func(t *testing.T, d *gotData, e *expectedData){ + checkInHeader, + checkBegin, + checkInPayload, + checkOutHeader, + checkOutTrailer, + checkEnd, + }) +} + +func (s) TestServerStatsClientStreamRPC(t *testing.T) { + count := 5 + checkFuncs := []func(t *testing.T, d *gotData, e *expectedData){ + checkInHeader, + checkBegin, + checkOutHeader, + } + ioPayFuncs := []func(t *testing.T, d *gotData, e *expectedData){ + checkInPayload, + } + for i := 0; i < count; i++ { + checkFuncs = append(checkFuncs, ioPayFuncs...) + } + checkFuncs = append(checkFuncs, + checkOutPayload, + checkOutTrailer, + checkEnd, + ) + testServerStats(t, &testConfig{compress: "gzip"}, &rpcConfig{count: count, success: true, callType: clientStreamRPC}, checkFuncs) +} + +func (s) TestServerStatsClientStreamRPCError(t *testing.T) { + count := 1 + testServerStats(t, &testConfig{compress: "gzip"}, &rpcConfig{count: count, success: false, callType: clientStreamRPC}, []func(t *testing.T, d *gotData, e *expectedData){ + checkInHeader, + checkBegin, + checkOutHeader, + checkInPayload, + checkOutTrailer, + checkEnd, + }) +} + +func (s) TestServerStatsServerStreamRPC(t *testing.T) { + count := 5 + checkFuncs := []func(t *testing.T, d *gotData, e *expectedData){ + checkInHeader, + checkBegin, + checkInPayload, + checkOutHeader, + } + ioPayFuncs := []func(t *testing.T, d *gotData, e *expectedData){ + checkOutPayload, + } + for i := 0; i < count; i++ { + checkFuncs = append(checkFuncs, ioPayFuncs...) + } + checkFuncs = append(checkFuncs, + checkOutTrailer, + checkEnd, + ) + testServerStats(t, &testConfig{compress: "gzip"}, &rpcConfig{count: count, success: true, callType: serverStreamRPC}, checkFuncs) +} + +func (s) TestServerStatsServerStreamRPCError(t *testing.T) { + count := 5 + testServerStats(t, &testConfig{compress: "gzip"}, &rpcConfig{count: count, success: false, callType: serverStreamRPC}, []func(t *testing.T, d *gotData, e *expectedData){ + checkInHeader, + checkBegin, + checkInPayload, + checkOutHeader, + checkOutTrailer, + checkEnd, + }) +} + +func (s) TestServerStatsFullDuplexRPC(t *testing.T) { + count := 5 + checkFuncs := []func(t *testing.T, d *gotData, e *expectedData){ + checkInHeader, + checkBegin, + checkOutHeader, + } + ioPayFuncs := []func(t *testing.T, d *gotData, e *expectedData){ + checkInPayload, + checkOutPayload, + } + for i := 0; i < count; i++ { + checkFuncs = append(checkFuncs, ioPayFuncs...) + } + checkFuncs = append(checkFuncs, + checkOutTrailer, + checkEnd, + ) + testServerStats(t, &testConfig{compress: "gzip"}, &rpcConfig{count: count, success: true, callType: fullDuplexStreamRPC}, checkFuncs) +} + +func (s) TestServerStatsFullDuplexRPCError(t *testing.T) { + count := 5 + testServerStats(t, &testConfig{compress: "gzip"}, &rpcConfig{count: count, success: false, callType: fullDuplexStreamRPC}, []func(t *testing.T, d *gotData, e *expectedData){ + checkInHeader, + checkBegin, + checkOutHeader, + checkInPayload, + checkOutTrailer, + checkEnd, + }) +} + +type checkFuncWithCount struct { + f func(t *testing.T, d *gotData, e *expectedData) + c int // expected count +} + +func checkClientStats(t *testing.T, got []*gotData, expect *expectedData, checkFuncs map[int]*checkFuncWithCount) { + var expectLen int + for _, v := range checkFuncs { + expectLen += v.c + } + if len(got) != expectLen { + for i, g := range got { + t.Errorf(" - %v, %T", i, g.s) + } + t.Fatalf("got %v stats, want %v stats", len(got), expectLen) + } + + var tagInfoInCtx *stats.RPCTagInfo + for i := 0; i < len(got); i++ { + if _, ok := got[i].s.(stats.RPCStats); ok { + tagInfoInCtxNew, _ := got[i].ctx.Value(rpcCtxKey{}).(*stats.RPCTagInfo) + if tagInfoInCtx != nil && tagInfoInCtx != tagInfoInCtxNew { + t.Fatalf("got context containing different tagInfo with stats %T", got[i].s) + } + tagInfoInCtx = tagInfoInCtxNew + } + } + + for _, s := range got { + switch s.s.(type) { + case *stats.Begin: + if checkFuncs[begin].c <= 0 { + t.Fatalf("unexpected stats: %T", s.s) + } + checkFuncs[begin].f(t, s, expect) + checkFuncs[begin].c-- + case *stats.OutHeader: + if checkFuncs[outHeader].c <= 0 { + t.Fatalf("unexpected stats: %T", s.s) + } + checkFuncs[outHeader].f(t, s, expect) + checkFuncs[outHeader].c-- + case *stats.OutPayload: + if checkFuncs[outPayload].c <= 0 { + t.Fatalf("unexpected stats: %T", s.s) + } + checkFuncs[outPayload].f(t, s, expect) + checkFuncs[outPayload].c-- + case *stats.InHeader: + if checkFuncs[inHeader].c <= 0 { + t.Fatalf("unexpected stats: %T", s.s) + } + checkFuncs[inHeader].f(t, s, expect) + checkFuncs[inHeader].c-- + case *stats.InPayload: + if checkFuncs[inPayload].c <= 0 { + t.Fatalf("unexpected stats: %T", s.s) + } + checkFuncs[inPayload].f(t, s, expect) + checkFuncs[inPayload].c-- + case *stats.InTrailer: + if checkFuncs[inTrailer].c <= 0 { + t.Fatalf("unexpected stats: %T", s.s) + } + checkFuncs[inTrailer].f(t, s, expect) + checkFuncs[inTrailer].c-- + case *stats.End: + if checkFuncs[end].c <= 0 { + t.Fatalf("unexpected stats: %T", s.s) + } + checkFuncs[end].f(t, s, expect) + checkFuncs[end].c-- + case *stats.ConnBegin: + if checkFuncs[connBegin].c <= 0 { + t.Fatalf("unexpected stats: %T", s.s) + } + checkFuncs[connBegin].f(t, s, expect) + checkFuncs[connBegin].c-- + case *stats.ConnEnd: + if checkFuncs[connEnd].c <= 0 { + t.Fatalf("unexpected stats: %T", s.s) + } + checkFuncs[connEnd].f(t, s, expect) + checkFuncs[connEnd].c-- + default: + t.Fatalf("unexpected stats: %T", s.s) + } + } +} + +func testClientStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs map[int]*checkFuncWithCount) { + h := &statshandler{} + te := newTest(t, tc, []stats.Handler{h}, nil) + te.startServer(newStatsTestStubServer()) + defer te.tearDown() + + var ( + reqs []proto.Message + resps []proto.Message + method string + err error + + isClientStream bool + isServerStream bool + + req proto.Message + resp proto.Message + e error + ) + switch cc.callType { + case unaryRPC: + method = "/grpc.testing.TestService/UnaryCall" + req, resp, e = te.doUnaryCall(cc) + reqs = []proto.Message{req} + resps = []proto.Message{resp} + err = e + case clientStreamRPC: + method = "/grpc.testing.TestService/StreamingInputCall" + reqs, resp, e = te.doClientStreamCall(cc) + resps = []proto.Message{resp} + err = e + isClientStream = true + case serverStreamRPC: + method = "/grpc.testing.TestService/StreamingOutputCall" + req, resps, e = te.doServerStreamCall(cc) + reqs = []proto.Message{req} + err = e + isServerStream = true + case fullDuplexStreamRPC: + method = "/grpc.testing.TestService/FullDuplexCall" + reqs, resps, err = te.doFullDuplexCallRoundtrip(cc) + isClientStream = true + isServerStream = true + } + if cc.success != (err == nil) { + t.Fatalf("cc.success: %v, got error: %v", cc.success, err) + } + te.cc.Close() + te.srv.GracefulStop() // Wait for the server to stop. + + lenRPCStats := 0 + for _, v := range checkFuncs { + lenRPCStats += v.c + } + for { + h.mu.Lock() + if len(h.gotRPC) >= lenRPCStats { + h.mu.Unlock() + break + } + h.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } + + for { + h.mu.Lock() + if _, ok := h.gotConn[len(h.gotConn)-1].s.(*stats.ConnEnd); ok { + h.mu.Unlock() + break + } + h.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } + + expect := &expectedData{ + serverAddr: te.srvAddr, + compression: tc.compress, + method: method, + requests: reqs, + responses: resps, + failfast: cc.failfast, + err: err, + isClientStream: isClientStream, + isServerStream: isServerStream, + authority: cc.authority, + } + + h.mu.Lock() + checkConnStats(t, h.gotConn) + h.mu.Unlock() + checkClientStats(t, h.gotRPC, expect, checkFuncs) +} + +func (s) TestClientStatsUnaryRPC(t *testing.T) { + testClientStats(t, &testConfig{compress: ""}, &rpcConfig{success: true, failfast: false, callType: unaryRPC}, map[int]*checkFuncWithCount{ + begin: {checkBegin, 1}, + outHeader: {checkOutHeader, 1}, + outPayload: {checkOutPayload, 1}, + inHeader: {checkInHeader, 1}, + inPayload: {checkInPayload, 1}, + inTrailer: {checkInTrailer, 1}, + end: {checkEnd, 1}, + }) +} + +func (s) TestClientStatsUnaryRPCAuthority(t *testing.T) { + testClientStats(t, &testConfig{compress: ""}, &rpcConfig{ + success: true, + failfast: false, + authority: "authority-override.example.com", + callType: unaryRPC, + }, map[int]*checkFuncWithCount{ + begin: {checkBegin, 1}, + outHeader: {checkOutHeader, 1}, + outPayload: {checkOutPayload, 1}, + inHeader: {checkInHeader, 1}, + inPayload: {checkInPayload, 1}, + inTrailer: {checkInTrailer, 1}, + end: {checkEnd, 1}, + }) +} + +func (s) TestClientStatsUnaryRPCError(t *testing.T) { + testClientStats(t, &testConfig{compress: ""}, &rpcConfig{success: false, failfast: false, callType: unaryRPC}, map[int]*checkFuncWithCount{ + begin: {checkBegin, 1}, + outHeader: {checkOutHeader, 1}, + outPayload: {checkOutPayload, 1}, + inHeader: {checkInHeader, 1}, + inTrailer: {checkInTrailer, 1}, + end: {checkEnd, 1}, + }) +} + +func (s) TestClientStatsClientStreamRPC(t *testing.T) { + count := 5 + testClientStats(t, &testConfig{compress: "gzip"}, &rpcConfig{count: count, success: true, failfast: false, callType: clientStreamRPC}, map[int]*checkFuncWithCount{ + begin: {checkBegin, 1}, + outHeader: {checkOutHeader, 1}, + inHeader: {checkInHeader, 1}, + outPayload: {checkOutPayload, count}, + inTrailer: {checkInTrailer, 1}, + inPayload: {checkInPayload, 1}, + end: {checkEnd, 1}, + }) +} + +func (s) TestClientStatsClientStreamRPCError(t *testing.T) { + count := 1 + testClientStats(t, &testConfig{compress: "gzip"}, &rpcConfig{count: count, success: false, failfast: false, callType: clientStreamRPC}, map[int]*checkFuncWithCount{ + begin: {checkBegin, 1}, + outHeader: {checkOutHeader, 1}, + inHeader: {checkInHeader, 1}, + outPayload: {checkOutPayload, 1}, + inTrailer: {checkInTrailer, 1}, + end: {checkEnd, 1}, + }) +} + +func (s) TestClientStatsServerStreamRPC(t *testing.T) { + count := 5 + testClientStats(t, &testConfig{compress: "gzip"}, &rpcConfig{count: count, success: true, failfast: false, callType: serverStreamRPC}, map[int]*checkFuncWithCount{ + begin: {checkBegin, 1}, + outHeader: {checkOutHeader, 1}, + outPayload: {checkOutPayload, 1}, + inHeader: {checkInHeader, 1}, + inPayload: {checkInPayload, count}, + inTrailer: {checkInTrailer, 1}, + end: {checkEnd, 1}, + }) +} + +func (s) TestClientStatsServerStreamRPCError(t *testing.T) { + count := 5 + testClientStats(t, &testConfig{compress: "gzip"}, &rpcConfig{count: count, success: false, failfast: false, callType: serverStreamRPC}, map[int]*checkFuncWithCount{ + begin: {checkBegin, 1}, + outHeader: {checkOutHeader, 1}, + outPayload: {checkOutPayload, 1}, + inHeader: {checkInHeader, 1}, + inTrailer: {checkInTrailer, 1}, + end: {checkEnd, 1}, + }) +} + +func (s) TestClientStatsFullDuplexRPC(t *testing.T) { + count := 5 + testClientStats(t, &testConfig{compress: "gzip"}, &rpcConfig{count: count, success: true, failfast: false, callType: fullDuplexStreamRPC}, map[int]*checkFuncWithCount{ + begin: {checkBegin, 1}, + outHeader: {checkOutHeader, 1}, + outPayload: {checkOutPayload, count}, + inHeader: {checkInHeader, 1}, + inPayload: {checkInPayload, count}, + inTrailer: {checkInTrailer, 1}, + end: {checkEnd, 1}, + }) +} + +func (s) TestClientStatsFullDuplexRPCError(t *testing.T) { + count := 5 + testClientStats(t, &testConfig{compress: "gzip"}, &rpcConfig{count: count, success: false, failfast: false, callType: fullDuplexStreamRPC}, map[int]*checkFuncWithCount{ + begin: {checkBegin, 1}, + outHeader: {checkOutHeader, 1}, + outPayload: {checkOutPayload, 1}, + inHeader: {checkInHeader, 1}, + inTrailer: {checkInTrailer, 1}, + end: {checkEnd, 1}, + }) +} + +func (s) TestTags(t *testing.T) { + b := []byte{5, 2, 4, 3, 1} + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx := stats.SetTags(tCtx, b) + if tg := getOutgoingStats(ctx, "grpc-tags-bin"); !reflect.DeepEqual(tg, b) { + t.Errorf("getOutgoingStats(%v, grpc-tags-bin) = %v; want %v", ctx, tg, b) + } + if tg := stats.Tags(ctx); tg != nil { + t.Errorf("Tags(%v) = %v; want nil", ctx, tg) + } + + ctx = setIncomingStats(tCtx, "grpc-tags-bin", b) + if tg := stats.Tags(ctx); !reflect.DeepEqual(tg, b) { + t.Errorf("Tags(%v) = %v; want %v", ctx, tg, b) + } + if tg := getOutgoingStats(ctx, "grpc-tags-bin"); tg != nil { + t.Errorf("getOutgoingStats(%v, grpc-tags-bin) = %v; want nil", ctx, tg) + } +} + +func (s) TestTrace(t *testing.T) { + b := []byte{5, 2, 4, 3, 1} + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx := stats.SetTrace(tCtx, b) + if tr := getOutgoingStats(ctx, "grpc-trace-bin"); !reflect.DeepEqual(tr, b) { + t.Errorf("getOutgoingStats(%v, grpc-trace-bin) = %v; want %v", ctx, tr, b) + } + if tr := stats.Trace(ctx); tr != nil { + t.Errorf("Trace(%v) = %v; want nil", ctx, tr) + } + + ctx = setIncomingStats(tCtx, "grpc-trace-bin", b) + if tr := stats.Trace(ctx); !reflect.DeepEqual(tr, b) { + t.Errorf("Trace(%v) = %v; want %v", ctx, tr, b) + } + if tr := getOutgoingStats(ctx, "grpc-trace-bin"); tr != nil { + t.Errorf("getOutgoingStats(%v, grpc-trace-bin) = %v; want nil", ctx, tr) + } +} + +func (s) TestMultipleClientStatsHandler(t *testing.T) { + h := &statshandler{} + tc := &testConfig{compress: ""} + te := newTest(t, tc, []stats.Handler{h, h}, nil) + te.startServer(newStatsTestStubServer()) + defer te.tearDown() + + cc := &rpcConfig{success: false, failfast: false, callType: unaryRPC} + _, _, err := te.doUnaryCall(cc) + if cc.success != (err == nil) { + t.Fatalf("cc.success: %v, got error: %v", cc.success, err) + } + te.cc.Close() + te.srv.GracefulStop() // Wait for the server to stop. + + for start := time.Now(); time.Since(start) < defaultTestTimeout; { + h.mu.Lock() + if _, ok := h.gotRPC[len(h.gotRPC)-1].s.(*stats.End); ok && len(h.gotRPC) == 12 { + h.mu.Unlock() + break + } + h.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } + + for start := time.Now(); time.Since(start) < defaultTestTimeout; { + h.mu.Lock() + if _, ok := h.gotConn[len(h.gotConn)-1].s.(*stats.ConnEnd); ok && len(h.gotConn) == 4 { + h.mu.Unlock() + break + } + h.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } + + // Each RPC generates 6 stats events on the client-side, times 2 StatsHandler + if len(h.gotRPC) != 12 { + t.Fatalf("h.gotRPC: unexpected amount of RPCStats: %v != %v", len(h.gotRPC), 12) + } + + // Each connection generates 4 conn events on the client-side, times 2 StatsHandler + if len(h.gotConn) != 4 { + t.Fatalf("h.gotConn: unexpected amount of ConnStats: %v != %v", len(h.gotConn), 4) + } +} + +func (s) TestMultipleServerStatsHandler(t *testing.T) { + h := &statshandler{} + tc := &testConfig{compress: ""} + te := newTest(t, tc, nil, []stats.Handler{h, h}) + te.startServer(newStatsTestStubServer()) + defer te.tearDown() + + cc := &rpcConfig{success: false, failfast: false, callType: unaryRPC} + _, _, err := te.doUnaryCall(cc) + if cc.success != (err == nil) { + t.Fatalf("cc.success: %v, got error: %v", cc.success, err) + } + te.cc.Close() + te.srv.GracefulStop() // Wait for the server to stop. + + for start := time.Now(); time.Since(start) < defaultTestTimeout; { + h.mu.Lock() + if _, ok := h.gotRPC[len(h.gotRPC)-1].s.(*stats.End); ok { + h.mu.Unlock() + break + } + h.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } + + for start := time.Now(); time.Since(start) < defaultTestTimeout; { + h.mu.Lock() + if _, ok := h.gotConn[len(h.gotConn)-1].s.(*stats.ConnEnd); ok { + h.mu.Unlock() + break + } + h.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } + + // Each RPC generates 6 stats events on the server-side, times 2 StatsHandler + if len(h.gotRPC) != 12 { + t.Fatalf("h.gotRPC: unexpected amount of RPCStats: %v != %v", len(h.gotRPC), 12) + } + + // Each connection generates 4 conn events on the server-side, times 2 StatsHandler + if len(h.gotConn) != 4 { + t.Fatalf("h.gotConn: unexpected amount of ConnStats: %v != %v", len(h.gotConn), 4) + } +} + +// TestStatsHandlerCallsServerIsRegisteredMethod tests whether a stats handler +// gets access to a Server on the server side, and thus the method that the +// server owns which specifies whether a method is made or not. The test sets up +// a server with a unary call and full duplex call configured, and makes an RPC. +// Within the stats handler, asking the server whether unary or duplex method +// names are registered should return true, and any other query should return +// false. +func (s) TestStatsHandlerCallsServerIsRegisteredMethod(t *testing.T) { + wg := sync.WaitGroup{} + wg.Add(1) + stubStatsHandler := &testutils.StubStatsHandler{ + TagRPCF: func(ctx context.Context, _ *stats.RPCTagInfo) context.Context { + // OpenTelemetry instrumentation needs the passed in Server to determine if + // methods are registered in different handle calls in to record metrics. + // This tag RPC call context gets passed into every handle call, so can + // assert once here, since it maps to all the handle RPC calls that come + // after. These internal calls will be how the OpenTelemetry instrumentation + // component accesses this server and the subsequent helper on the server. + server := internal.ServerFromContext.(func(context.Context) *grpc.Server)(ctx) + if server == nil { + t.Errorf("stats handler received ctx has no server present") + } + isRegisteredMethod := internal.IsRegisteredMethod.(func(*grpc.Server, string) bool) + // /s/m and s/m are valid. + if !isRegisteredMethod(server, "/grpc.testing.TestService/UnaryCall") { + t.Errorf("UnaryCall should be a registered method according to server") + } + if !isRegisteredMethod(server, "grpc.testing.TestService/FullDuplexCall") { + t.Errorf("FullDuplexCall should be a registered method according to server") + } + if isRegisteredMethod(server, "/grpc.testing.TestService/DoesNotExistCall") { + t.Errorf("DoesNotExistCall should not be a registered method according to server") + } + if isRegisteredMethod(server, "/unknownService/UnaryCall") { + t.Errorf("/unknownService/UnaryCall should not be a registered method according to server") + } + wg.Done() + return ctx + }, + } + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + } + if err := ss.Start([]grpc.ServerOption{grpc.StatsHandler(stubStatsHandler)}); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + wg.Wait() +} diff --git a/status/status.go b/status/status.go new file mode 100644 index 0000000..a93360e --- /dev/null +++ b/status/status.go @@ -0,0 +1,162 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package status implements errors returned by gRPC. These errors are +// serialized and transmitted on the wire between server and client, and allow +// for additional data to be transmitted via the Details field in the status +// proto. gRPC service handlers should return an error created by this +// package, and gRPC clients should expect a corresponding error to be +// returned from the RPC call. +// +// This package upholds the invariants that a non-nil error may not +// contain an OK code, and an OK code must result in a nil error. +package status + +import ( + "context" + "errors" + "fmt" + + spb "google.golang.org/genproto/googleapis/rpc/status" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/status" +) + +// Status references google.golang.org/grpc/internal/status. It represents an +// RPC status code, message, and details. It is immutable and should be +// created with New, Newf, or FromProto. +// https://godoc.org/google.golang.org/grpc/internal/status +type Status = status.Status + +// New returns a Status representing c and msg. +func New(c codes.Code, msg string) *Status { + return status.New(c, msg) +} + +// Newf returns New(c, fmt.Sprintf(format, a...)). +func Newf(c codes.Code, format string, a ...any) *Status { + return New(c, fmt.Sprintf(format, a...)) +} + +// Error returns an error representing c and msg. If c is OK, returns nil. +func Error(c codes.Code, msg string) error { + return New(c, msg).Err() +} + +// Errorf returns Error(c, fmt.Sprintf(format, a...)). +func Errorf(c codes.Code, format string, a ...any) error { + return Error(c, fmt.Sprintf(format, a...)) +} + +// ErrorProto returns an error representing s. If s.Code is OK, returns nil. +func ErrorProto(s *spb.Status) error { + return FromProto(s).Err() +} + +// FromProto returns a Status representing s. +func FromProto(s *spb.Status) *Status { + return status.FromProto(s) +} + +// FromError returns a Status representation of err. +// +// - If err was produced by this package or implements the method `GRPCStatus() +// *Status` and `GRPCStatus()` does not return nil, or if err wraps a type +// satisfying this, the Status from `GRPCStatus()` is returned. For wrapped +// errors, the message returned contains the entire err.Error() text and not +// just the wrapped status. In that case, ok is true. +// +// - If err is nil, a Status is returned with codes.OK and no message, and ok +// is true. +// +// - If err implements the method `GRPCStatus() *Status` and `GRPCStatus()` +// returns nil (which maps to Codes.OK), or if err wraps a type +// satisfying this, a Status is returned with codes.Unknown and err's +// Error() message, and ok is false. +// +// - Otherwise, err is an error not compatible with this package. In this +// case, a Status is returned with codes.Unknown and err's Error() message, +// and ok is false. +func FromError(err error) (s *Status, ok bool) { + if err == nil { + return nil, true + } + type grpcstatus interface{ GRPCStatus() *Status } + if gs, ok := err.(grpcstatus); ok { + grpcStatus := gs.GRPCStatus() + if grpcStatus == nil { + // Error has status nil, which maps to codes.OK. There + // is no sensible behavior for this, so we turn it into + // an error with codes.Unknown and discard the existing + // status. + return New(codes.Unknown, err.Error()), false + } + return grpcStatus, true + } + var gs grpcstatus + if errors.As(err, &gs) { + grpcStatus := gs.GRPCStatus() + if grpcStatus == nil { + // Error wraps an error that has status nil, which maps + // to codes.OK. There is no sensible behavior for this, + // so we turn it into an error with codes.Unknown and + // discard the existing status. + return New(codes.Unknown, err.Error()), false + } + p := grpcStatus.Proto() + p.Message = err.Error() + return status.FromProto(p), true + } + return New(codes.Unknown, err.Error()), false +} + +// Convert is a convenience function which removes the need to handle the +// boolean return value from FromError. +func Convert(err error) *Status { + s, _ := FromError(err) + return s +} + +// Code returns the Code of the error if it is a Status error or if it wraps a +// Status error. If that is not the case, it returns codes.OK if err is nil, or +// codes.Unknown otherwise. +func Code(err error) codes.Code { + // Don't use FromError to avoid allocation of OK status. + if err == nil { + return codes.OK + } + + return Convert(err).Code() +} + +// FromContextError converts a context error or wrapped context error into a +// Status. It returns a Status with codes.OK if err is nil, or a Status with +// codes.Unknown if err is non-nil and not a context error. +func FromContextError(err error) *Status { + if err == nil { + return nil + } + if errors.Is(err, context.DeadlineExceeded) { + return New(codes.DeadlineExceeded, err.Error()) + } + if errors.Is(err, context.Canceled) { + return New(codes.Canceled, err.Error()) + } + return New(codes.Unknown, err.Error()) +} diff --git a/status/status_ext_test.go b/status/status_ext_test.go new file mode 100644 index 0000000..2598ca6 --- /dev/null +++ b/status/status_ext_test.go @@ -0,0 +1,317 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package status_test + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/protoadapt" + "google.golang.org/protobuf/testing/protocmp" + + testpb "google.golang.org/grpc/interop/grpc_testing" + tpb "google.golang.org/grpc/testdata/grpc_testing_not_regenerated" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func errWithDetails(t *testing.T, s *status.Status, details ...protoadapt.MessageV1) error { + t.Helper() + res, err := s.WithDetails(details...) + if err != nil { + t.Fatalf("(%v).WithDetails(%v) = %v, %v; want _, ", s, details, res, err) + } + return res.Err() +} + +func (s) TestErrorIs(t *testing.T) { + // Test errors. + testErr := status.Error(codes.Internal, "internal server error") + testErrWithDetails := errWithDetails(t, status.New(codes.Internal, "internal server error"), &testpb.Empty{}) + + // Test cases. + testCases := []struct { + err1, err2 error + want bool + }{ + {err1: testErr, err2: nil, want: false}, + {err1: testErr, err2: status.Error(codes.Internal, "internal server error"), want: true}, + {err1: testErr, err2: status.Error(codes.Internal, "internal error"), want: false}, + {err1: testErr, err2: status.Error(codes.Unknown, "internal server error"), want: false}, + {err1: testErr, err2: errors.New("non-grpc error"), want: false}, + {err1: testErrWithDetails, err2: status.Error(codes.Internal, "internal server error"), want: false}, + {err1: testErrWithDetails, err2: errWithDetails(t, status.New(codes.Internal, "internal server error"), &testpb.Empty{}), want: true}, + {err1: testErrWithDetails, err2: errWithDetails(t, status.New(codes.Internal, "internal server error"), &testpb.Empty{}, &testpb.Empty{}), want: false}, + } + + for _, tc := range testCases { + isError, ok := tc.err1.(interface{ Is(target error) bool }) + if !ok { + t.Errorf("(%v) does not implement is", tc.err1) + continue + } + + is := isError.Is(tc.err2) + if is != tc.want { + t.Errorf("(%v).Is(%v) = %t; want %t", tc.err1, tc.err2, is, tc.want) + } + } +} + +// TestStatusDetails tests how gRPC handles grpc-status-details-bin, especially +// in cases where it doesn't match the grpc-status trailer or contains arbitrary +// data. +func (s) TestStatusDetails(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + for _, serverType := range []struct { + name string + startServerFunc func(*stubserver.StubServer) error + }{{ + name: "normal server", + startServerFunc: func(ss *stubserver.StubServer) error { + return ss.StartServer() + }, + }, { + name: "handler server", + startServerFunc: func(ss *stubserver.StubServer) error { + return ss.StartHandlerServer() + }, + }} { + t.Run(serverType.name, func(t *testing.T) { + // Convenience function for making a status including details. + detailErr := func(c codes.Code, m string) error { + s, err := status.New(c, m).WithDetails(&testpb.SimpleRequest{ + Payload: &testpb.Payload{Body: []byte("detail msg")}, + }) + if err != nil { + t.Fatalf("Error adding details: %v", err) + } + return s.Err() + } + + serialize := func(err error) string { + buf, _ := proto.Marshal(status.Convert(err).Proto()) + return string(buf) + } + + testCases := []struct { + name string + trailerSent metadata.MD + errSent error + trailerWant []string + errWant error + errContains error + }{{ + name: "basic without details", + trailerSent: metadata.MD{}, + errSent: status.Error(codes.Aborted, "test msg"), + errWant: status.Error(codes.Aborted, "test msg"), + }, { + name: "basic without details passes through trailers", + trailerSent: metadata.MD{"grpc-status-details-bin": []string{"random text"}}, + errSent: status.Error(codes.Aborted, "test msg"), + trailerWant: []string{"random text"}, + errWant: status.Error(codes.Aborted, "test msg"), + }, { + name: "basic without details conflicts with manual details", + trailerSent: metadata.MD{"grpc-status-details-bin": []string{serialize(status.Error(codes.Canceled, "test msg"))}}, + errSent: status.Error(codes.Aborted, "test msg"), + trailerWant: []string{serialize(status.Error(codes.Canceled, "test msg"))}, + errContains: status.Error(codes.Internal, "mismatch"), + }, { + name: "basic with details", + trailerSent: metadata.MD{}, + errSent: detailErr(codes.Aborted, "test msg"), + trailerWant: []string{serialize(detailErr(codes.Aborted, "test msg"))}, + errWant: detailErr(codes.Aborted, "test msg"), + }, { + name: "basic with details discards user's trailers", + trailerSent: metadata.MD{"grpc-status-details-bin": []string{"will be ignored"}}, + errSent: detailErr(codes.Aborted, "test msg"), + trailerWant: []string{serialize(detailErr(codes.Aborted, "test msg"))}, + errWant: detailErr(codes.Aborted, "test msg"), + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Start a simple server that returns the trailer and error it receives from + // channels. + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + grpc.SetTrailer(ctx, tc.trailerSent) + return nil, tc.errSent + }, + } + if err := serverType.startServerFunc(ss); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + if err := ss.StartClient(); err != nil { + t.Fatalf("Error starting endpoint client: %v", err) + } + defer ss.Stop() + + trailerGot := metadata.MD{} + _, errGot := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Trailer(&trailerGot)) + gsdb := trailerGot["grpc-status-details-bin"] + if !cmp.Equal(gsdb, tc.trailerWant) { + t.Errorf("Trailer got: %v; want: %v", gsdb, tc.trailerWant) + } + if tc.errWant != nil && !testutils.StatusErrEqual(errGot, tc.errWant) { + t.Errorf("Err got: %v; want: %v", errGot, tc.errWant) + } + if tc.errContains != nil && (status.Code(errGot) != status.Code(tc.errContains) || !strings.Contains(status.Convert(errGot).Message(), status.Convert(tc.errContains).Message())) { + t.Errorf("Err got: %v; want: (Contains: %v)", errGot, tc.errWant) + } + }) + } + }) + } +} + +// TestStatus_ErrorDetailsMessageV1 verifies backward compatibility of the +// status.Details() method when using protobuf code generated with only the +// MessageV1 API implementation. +func (s) TestStatus_ErrorDetailsMessageV1(t *testing.T) { + details := []protoadapt.MessageV1{ + &tpb.SimpleMessage{Data: "abc"}, + } + s, err := status.New(codes.Aborted, "").WithDetails(details...) + if err != nil { + t.Fatalf("(%v).WithDetails(%+v) failed: %v", s, details, err) + } + gotDetails := s.Details() + for i, msg := range gotDetails { + if got, want := reflect.TypeOf(msg), reflect.TypeOf(details[i]); got != want { + t.Errorf("reflect.Typeof(%v) = %v, want = %v", msg, got, want) + } + if _, ok := msg.(protoadapt.MessageV1); !ok { + t.Errorf("(%v).Details() returned message that doesn't implement protoadapt.MessageV1: %v", s, msg) + } + if diff := cmp.Diff(msg, details[i], protocmp.Transform()); diff != "" { + t.Errorf("(%v).Details got unexpected output, diff (-got +want):\n%s", s, diff) + } + } +} + +// TestStatus_ErrorDetailsMessageV1AndV2 verifies that status.Details() method +// returns the same message types when using protobuf code generated with both the +// MessageV1 and MessageV2 API implementations. +func (s) TestStatus_ErrorDetailsMessageV1AndV2(t *testing.T) { + details := []protoadapt.MessageV1{ + &testpb.Empty{}, + } + s, err := status.New(codes.Aborted, "").WithDetails(details...) + if err != nil { + t.Fatalf("(%v).WithDetails(%+v) failed: %v", s, details, err) + } + gotDetails := s.Details() + for i, msg := range gotDetails { + if got, want := reflect.TypeOf(msg), reflect.TypeOf(details[i]); got != want { + t.Errorf("reflect.Typeof(%v) = %v, want = %v", msg, got, want) + } + if _, ok := msg.(protoadapt.MessageV1); !ok { + t.Errorf("(%v).Details() returned message that doesn't implement protoadapt.MessageV1: %v", s, msg) + } + if _, ok := msg.(protoadapt.MessageV2); !ok { + t.Errorf("(%v).Details() returned message that doesn't implement protoadapt.MessageV2: %v", s, msg) + } + if diff := cmp.Diff(msg, details[i], protocmp.Transform()); diff != "" { + t.Errorf("(%v).Details got unexpected output, diff (-got +want):\n%s", s, diff) + } + } +} + +func (s) TestFromError_Wrapped(t *testing.T) { + base := status.New(codes.Canceled, "inner canceled") + sWithDetails, err := base.WithDetails(&testpb.Empty{}) + if err != nil { + t.Fatalf("WithDetails failed: %v", err) + } + innerErr := sWithDetails.Err() + mustStatus := func(message string) *status.Status { + st, err := status.New(codes.Canceled, message).WithDetails(&testpb.Empty{}) + if err != nil { + t.Fatal(err) + } + return st + } + + testCases := []struct { + name string + err error + wantStatus *status.Status + }{ + { + name: "direct_error", + err: innerErr, + wantStatus: mustStatus("inner canceled"), + }, + { + name: "wrapped_error", + err: fmt.Errorf("wrapped: %w", innerErr), + wantStatus: mustStatus("wrapped: rpc error: code = Canceled desc = inner canceled"), + }, + { + name: "double_wrapped_error", + err: fmt.Errorf("outer: %w", fmt.Errorf("inner: %w", innerErr)), + wantStatus: mustStatus("outer: inner: rpc error: code = Canceled desc = inner canceled"), + }, + { + name: "double_wrapped_single_errorf", + err: fmt.Errorf("error: %w: %w", errors.New("test error"), innerErr), + wantStatus: mustStatus("error: test error: rpc error: code = Canceled desc = inner canceled"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, ok := status.FromError(tc.err) + if !ok { + t.Fatalf("status.FromError(%v) returned false; want true", tc.err) + } + if diff := cmp.Diff(tc.wantStatus, got, protocmp.Transform(), cmp.AllowUnexported(status.Status{})); diff != "" { + t.Fatalf("status.FromError(%v) got unexpected output, diff (-want +got):\n%s", tc.err, diff) + } + }) + } +} diff --git a/status/status_test.go b/status/status_test.go new file mode 100644 index 0000000..b0f1f4e --- /dev/null +++ b/status/status_test.go @@ -0,0 +1,482 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package status + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + cpb "google.golang.org/genproto/googleapis/rpc/code" + epb "google.golang.org/genproto/googleapis/rpc/errdetails" + spb "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/protoadapt" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/durationpb" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/status" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// errEqual is essentially a copy of testutils.StatusErrEqual(), to avoid a +// cyclic dependency. +func errEqual(err1, err2 error) bool { + status1, ok := FromError(err1) + if !ok { + return false + } + status2, ok := FromError(err2) + if !ok { + return false + } + return proto.Equal(status1.Proto(), status2.Proto()) +} + +func (s) TestErrorsWithSameParameters(t *testing.T) { + const description = "some description" + e1 := Errorf(codes.AlreadyExists, description) + e2 := Errorf(codes.AlreadyExists, description) + if e1 == e2 || !errEqual(e1, e2) { + t.Fatalf("Errors should be equivalent but unique - e1: %v, %v e2: %p, %v", e1.(*status.Error), e1, e2.(*status.Error), e2) + } +} + +func (s) TestFromToProto(t *testing.T) { + s := &spb.Status{ + Code: int32(codes.Internal), + Message: "test test test", + Details: []*anypb.Any{{TypeUrl: "foo", Value: []byte{3, 2, 1}}}, + } + + err := FromProto(s) + if got := err.Proto(); !proto.Equal(s, got) { + t.Fatalf("Expected errors to be identical - s: %v got: %v", s, got) + } +} + +func (s) TestFromNilProto(t *testing.T) { + tests := []*Status{nil, FromProto(nil)} + for _, s := range tests { + if c := s.Code(); c != codes.OK { + t.Errorf("s: %v - Expected s.Code() = OK; got %v", s, c) + } + if m := s.Message(); m != "" { + t.Errorf("s: %v - Expected s.Message() = \"\"; got %q", s, m) + } + if p := s.Proto(); p != nil { + t.Errorf("s: %v - Expected s.Proto() = nil; got %q", s, p) + } + if e := s.Err(); e != nil { + t.Errorf("s: %v - Expected s.Err() = nil; got %v", s, e) + } + } +} + +func (s) TestError(t *testing.T) { + err := Error(codes.Internal, "test description") + if got, want := err.Error(), "rpc error: code = Internal desc = test description"; got != want { + t.Fatalf("err.Error() = %q; want %q", got, want) + } + s, _ := FromError(err) + if got, want := s.Code(), codes.Internal; got != want { + t.Fatalf("err.Code() = %s; want %s", got, want) + } + if got, want := s.Message(), "test description"; got != want { + t.Fatalf("err.Message() = %s; want %s", got, want) + } +} + +func (s) TestErrorOK(t *testing.T) { + err := Error(codes.OK, "foo") + if err != nil { + t.Fatalf("Error(codes.OK, _) = %p; want nil", err.(*status.Error)) + } +} + +func (s) TestErrorProtoOK(t *testing.T) { + s := &spb.Status{Code: int32(codes.OK)} + if got := ErrorProto(s); got != nil { + t.Fatalf("ErrorProto(%v) = %v; want nil", s, got) + } +} + +func (s) TestFromError(t *testing.T) { + code, message := codes.Internal, "test description" + err := Error(code, message) + s, ok := FromError(err) + if !ok || s.Code() != code || s.Message() != message || s.Err() == nil { + t.Fatalf("FromError(%v) = %v, %v; want , true", err, s, ok, code, message) + } +} + +func (s) TestFromErrorOK(t *testing.T) { + code, message := codes.OK, "" + s, ok := FromError(nil) + if !ok || s.Code() != code || s.Message() != message || s.Err() != nil { + t.Fatalf("FromError(nil) = %v, %v; want , true", s, ok, code, message) + } +} + +type customError struct { + Code codes.Code + Message string + Details []*anypb.Any +} + +func (c customError) Error() string { + return fmt.Sprintf("rpc error: code = %s desc = %s", c.Code, c.Message) +} + +func (c customError) GRPCStatus() *Status { + return status.FromProto(&spb.Status{ + Code: int32(c.Code), + Message: c.Message, + Details: c.Details, + }) +} + +func (s) TestFromErrorImplementsInterface(t *testing.T) { + code, message := codes.Internal, "test description" + details := []*anypb.Any{{ + TypeUrl: "testUrl", + Value: []byte("testValue"), + }} + err := customError{ + Code: code, + Message: message, + Details: details, + } + s, ok := FromError(err) + if !ok || s.Code() != code || s.Message() != message || s.Err() == nil { + t.Fatalf("FromError(%v) = %v, %v; want , true", err, s, ok, code, message) + } + pd := s.Proto().GetDetails() + if len(pd) != 1 || !proto.Equal(pd[0], details[0]) { + t.Fatalf("s.Proto.GetDetails() = %v; want ", pd, details) + } +} + +func (s) TestFromErrorUnknownError(t *testing.T) { + code, message := codes.Unknown, "unknown error" + err := errors.New("unknown error") + s, ok := FromError(err) + if ok || s.Code() != code || s.Message() != message { + t.Fatalf("FromError(%v) = %v, %v; want , false", err, s, ok, code, message) + } +} + +func (s) TestFromErrorWrapped(t *testing.T) { + const code, message = codes.Internal, "test description" + err := fmt.Errorf("wrapped error: %w", Error(code, message)) + s, ok := FromError(err) + if !ok || s.Code() != code || s.Message() != err.Error() || s.Err() == nil { + t.Fatalf("FromError(%v) = %v, %v; want , true", err, s, ok, code, message) + } +} + +type customErrorNilStatus struct { +} + +func (c customErrorNilStatus) Error() string { + return "test" +} + +func (c customErrorNilStatus) GRPCStatus() *Status { + return nil +} + +func (s) TestFromErrorImplementsInterfaceReturnsOKStatus(t *testing.T) { + err := customErrorNilStatus{} + s, ok := FromError(err) + if ok || s.Code() != codes.Unknown || s.Message() != err.Error() { + t.Fatalf("FromError(%v) = %v, %v; want , true", err, s, ok, codes.Unknown, err.Error()) + } +} + +func (s) TestFromErrorImplementsInterfaceReturnsOKStatusWrapped(t *testing.T) { + err := fmt.Errorf("wrapping: %w", customErrorNilStatus{}) + s, ok := FromError(err) + if ok || s.Code() != codes.Unknown || s.Message() != err.Error() { + t.Fatalf("FromError(%v) = %v, %v; want , true", err, s, ok, codes.Unknown, err.Error()) + } +} + +func (s) TestFromErrorImplementsInterfaceWrapped(t *testing.T) { + const code, message = codes.Internal, "test description" + err := fmt.Errorf("wrapped error: %w", customError{Code: code, Message: message}) + s, ok := FromError(err) + if !ok || s.Code() != code || s.Message() != err.Error() || s.Err() == nil { + t.Fatalf("FromError(%v) = %v, %v; want , true", err, s, ok, code, message) + } +} + +func (s) TestCode(t *testing.T) { + const code = codes.Internal + err := Error(code, "test description") + if s := Code(err); s != code { + t.Fatalf("Code(%v) = %v; want ", err, s, code) + } +} + +func (s) TestCodeOK(t *testing.T) { + if s, code := Code(nil), codes.OK; s != code { + t.Fatalf("Code(%v) = %v; want ", nil, s, code) + } +} + +func (s) TestCodeImplementsInterface(t *testing.T) { + const code = codes.Internal + err := customError{Code: code, Message: "test description"} + if s := Code(err); s != code { + t.Fatalf("Code(%v) = %v; want ", err, s, code) + } +} + +func (s) TestCodeUnknownError(t *testing.T) { + const code = codes.Unknown + err := errors.New("unknown error") + if s := Code(err); s != code { + t.Fatalf("Code(%v) = %v; want ", err, s, code) + } +} + +func (s) TestCodeWrapped(t *testing.T) { + const code = codes.Internal + err := fmt.Errorf("wrapped: %w", Error(code, "test description")) + if s := Code(err); s != code { + t.Fatalf("Code(%v) = %v; want ", err, s, code) + } +} + +func (s) TestCodeImplementsInterfaceWrapped(t *testing.T) { + const code = codes.Internal + err := fmt.Errorf("wrapped: %w", customError{Code: code, Message: "test description"}) + if s := Code(err); s != code { + t.Fatalf("Code(%v) = %v; want ", err, s, code) + } +} + +func (s) TestConvertKnownError(t *testing.T) { + code, message := codes.Internal, "test description" + err := Error(code, message) + s := Convert(err) + if s.Code() != code || s.Message() != message { + t.Fatalf("Convert(%v) = %v; want ", err, s, code, message) + } +} + +func (s) TestConvertUnknownError(t *testing.T) { + code, message := codes.Unknown, "unknown error" + err := errors.New("unknown error") + s := Convert(err) + if s.Code() != code || s.Message() != message { + t.Fatalf("Convert(%v) = %v; want ", err, s, code, message) + } +} + +func (s) TestStatus_ErrorDetails(t *testing.T) { + tests := []struct { + code codes.Code + details []protoadapt.MessageV1 + }{ + { + code: codes.NotFound, + details: nil, + }, + { + code: codes.NotFound, + details: []protoadapt.MessageV1{ + &epb.ResourceInfo{ + ResourceType: "book", + ResourceName: "projects/1234/books/5678", + Owner: "User", + }, + }, + }, + { + code: codes.Internal, + details: []protoadapt.MessageV1{ + &epb.DebugInfo{ + StackEntries: []string{ + "first stack", + "second stack", + }, + }, + }, + }, + { + code: codes.Unavailable, + details: []protoadapt.MessageV1{ + &epb.RetryInfo{ + RetryDelay: &durationpb.Duration{Seconds: 60}, + }, + &epb.ResourceInfo{ + ResourceType: "book", + ResourceName: "projects/1234/books/5678", + Owner: "User", + }, + }, + }, + } + + for _, tc := range tests { + s, err := New(tc.code, "").WithDetails(tc.details...) + if err != nil { + t.Fatalf("(%v).WithDetails(%+v) failed: %v", str(s), tc.details, err) + } + details := s.Details() + if len(details) != len(tc.details) { + t.Fatalf("len(s.Details()) = %d, want = %d.", len(details), len(tc.details)) + } + for i := range details { + if !proto.Equal(details[i].(protoreflect.ProtoMessage), tc.details[i].(protoreflect.ProtoMessage)) { + t.Fatalf("(%v).Details()[%d] = %+v, want %+v", str(s), i, details[i], tc.details[i]) + } + } + } +} + +func (s) TestStatus_WithDetails_Fail(t *testing.T) { + tests := []*Status{ + nil, + FromProto(nil), + New(codes.OK, ""), + } + for _, s := range tests { + if s, err := s.WithDetails(); err == nil || s != nil { + t.Fatalf("(%v).WithDetails(%+v) = %v, %v; want nil, non-nil", str(s), []proto.Message{}, s, err) + } + } +} + +func (s) TestStatus_ErrorDetails_Fail(t *testing.T) { + tests := []struct { + s *Status + want []any + }{ + { + s: nil, + want: nil, + }, + { + s: FromProto(nil), + want: nil, + }, + { + s: New(codes.OK, ""), + want: []any{}, + }, + { + s: FromProto(&spb.Status{ + Code: int32(cpb.Code_CANCELLED), + Details: []*anypb.Any{ + { + TypeUrl: "", + Value: []byte{}, + }, + mustMarshalAny(&epb.ResourceInfo{ + ResourceType: "book", + ResourceName: "projects/1234/books/5678", + Owner: "User", + }), + }, + }), + want: []any{ + errors.New("invalid empty type URL"), + &epb.ResourceInfo{ + ResourceType: "book", + ResourceName: "projects/1234/books/5678", + Owner: "User", + }, + }, + }, + } + for _, tc := range tests { + details := tc.s.Details() + if len(details) != len(tc.want) { + t.Fatalf("len(s.Details()) = %d, want = %d.", len(details), len(tc.want)) + } + for i, d := range details { + // s.Details can either contain an error or a proto message. We + // want to do a compare the proto message for an Equal match, and + // for errors only check for presence. + if _, ok := d.(error); ok { + if (d != nil) != (tc.want[i] != nil) { + t.Fatalf("s.Details()[%v] was %v; want %v", i, d, tc.want[i]) + } + continue + } + if !cmp.Equal(d, tc.want[i], cmp.Comparer(proto.Equal)) { + t.Fatalf("s.Details()[%v] was %v; want %v", i, d, tc.want[i]) + } + } + } +} + +func str(s *Status) string { + if s == nil { + return "nil" + } + if s.Proto() == nil { + return "" + } + return fmt.Sprintf("", s.Code(), s.Message(), s.Details()) +} + +// mustMarshalAny converts a protobuf message to an any. +func mustMarshalAny(msg proto.Message) *anypb.Any { + m, err := anypb.New(msg) + if err != nil { + panic(fmt.Sprintf("anypb.New(%+v) failed: %v", msg, err)) + } + return m +} + +func (s) TestFromContextError(t *testing.T) { + testCases := []struct { + in error + want *Status + }{ + {in: nil, want: New(codes.OK, "")}, + {in: context.DeadlineExceeded, want: New(codes.DeadlineExceeded, context.DeadlineExceeded.Error())}, + {in: context.Canceled, want: New(codes.Canceled, context.Canceled.Error())}, + {in: errors.New("other"), want: New(codes.Unknown, "other")}, + {in: fmt.Errorf("wrapped: %w", context.DeadlineExceeded), want: New(codes.DeadlineExceeded, "wrapped: "+context.DeadlineExceeded.Error())}, + {in: fmt.Errorf("wrapped: %w", context.Canceled), want: New(codes.Canceled, "wrapped: "+context.Canceled.Error())}, + } + for _, tc := range testCases { + got := FromContextError(tc.in) + if got.Code() != tc.want.Code() || got.Message() != tc.want.Message() { + t.Errorf("FromContextError(%v) = %v; want %v", tc.in, got, tc.want) + } + } +} diff --git a/stream.go b/stream.go new file mode 100644 index 0000000..00b56e3 --- /dev/null +++ b/stream.go @@ -0,0 +1,2021 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "errors" + "fmt" + "io" + "math" + rand "math/rand/v2" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancerload" + "google.golang.org/grpc/internal/binarylog" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpcutil" + imetadata "google.golang.org/grpc/internal/metadata" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/serviceconfig" + istatus "google.golang.org/grpc/internal/status" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/mem" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +var metadataFromOutgoingContextRaw = internal.FromOutgoingContextRaw.(func(context.Context) (metadata.MD, [][]string, bool)) + +// StreamHandler defines the handler called by gRPC server to complete the +// execution of a streaming RPC. srv is the service implementation on which the +// RPC was invoked. +// +// If a StreamHandler returns an error, it should either be produced by the +// status package, or be one of the context errors. Otherwise, gRPC will use +// codes.Unknown as the status code and err.Error() as the status message of the +// RPC. +type StreamHandler func(srv any, stream ServerStream) error + +// StreamDesc represents a streaming RPC service's method specification. Used +// on the server when registering services and on the client when initiating +// new streams. +type StreamDesc struct { + // StreamName and Handler are only used when registering handlers on a + // server. + StreamName string // the name of the method excluding the service + Handler StreamHandler // the handler called for the method + + // ServerStreams and ClientStreams are used for registering handlers on a + // server as well as defining RPC behavior when passed to NewClientStream + // and ClientConn.NewStream. At least one must be true. + ServerStreams bool // indicates the server can perform streaming sends + ClientStreams bool // indicates the client can perform streaming sends +} + +// Stream defines the common interface a client or server stream has to satisfy. +// +// Deprecated: See ClientStream and ServerStream documentation instead. +type Stream interface { + // Deprecated: See ClientStream and ServerStream documentation instead. + Context() context.Context + // Deprecated: See ClientStream and ServerStream documentation instead. + SendMsg(m any) error + // Deprecated: See ClientStream and ServerStream documentation instead. + RecvMsg(m any) error +} + +// ClientStream defines the client-side behavior of a streaming RPC. +// +// All errors returned from ClientStream methods are compatible with the +// status package. +type ClientStream interface { + // Header returns the header metadata received from the server if there + // is any. It blocks if the metadata is not ready to read. If the metadata + // is nil and the error is also nil, then the stream was terminated without + // headers, and the status can be discovered by calling RecvMsg. + Header() (metadata.MD, error) + // Trailer returns the trailer metadata from the server, if there is any. + // It must only be called after stream.CloseAndRecv has returned, or + // stream.Recv has returned a non-nil error (including io.EOF). + Trailer() metadata.MD + // CloseSend closes the send direction of the stream. This method always + // returns a nil error. The status of the stream may be discovered using + // RecvMsg. It is also not safe to call CloseSend concurrently with SendMsg. + CloseSend() error + // Context returns the context for this stream. + // + // It should not be called until after Header or RecvMsg has returned. Once + // called, subsequent client-side retries are disabled. + Context() context.Context + // SendMsg is generally called by generated code. On error, SendMsg aborts + // the stream. If the error was generated by the client, the status is + // returned directly; otherwise, io.EOF is returned and the status of + // the stream may be discovered using RecvMsg. For unary or server-streaming + // RPCs (StreamDesc.ClientStreams is false), a nil error is returned + // unconditionally. + // + // SendMsg blocks until: + // - There is sufficient flow control to schedule m with the transport, or + // - The stream is done, or + // - The stream breaks. + // + // SendMsg does not wait until the message is received by the server. An + // untimely stream closure may result in lost messages. To ensure delivery, + // users should ensure the RPC completed successfully using RecvMsg. + // + // It is safe to have a goroutine calling SendMsg and another goroutine + // calling RecvMsg on the same stream at the same time, but it is not safe + // to call SendMsg on the same stream in different goroutines. It is also + // not safe to call CloseSend concurrently with SendMsg. + // + // It is not safe to modify the message after calling SendMsg. Tracing + // libraries and stats handlers may use the message lazily. + SendMsg(m any) error + // RecvMsg blocks until it receives a message into m or the stream is + // done. It returns io.EOF when the stream completes successfully. On + // any other error, the stream is aborted and the error contains the RPC + // status. + // + // It is safe to have a goroutine calling SendMsg and another goroutine + // calling RecvMsg on the same stream at the same time, but it is not + // safe to call RecvMsg on the same stream in different goroutines. + RecvMsg(m any) error +} + +// clientStreamWrapper wraps a ClientStream and handles SendMsg, CloseSend, and +// RecvMsg parities based on the nature of stream. +type clientStreamWrapper struct { + ClientStream + desc *StreamDesc + closeSendCalled atomic.Bool +} + +// CloseSend closes the send direction of the stream. The implementation ensures +// that CloseSend is only called once on the underlying ClientStream, even if +// CloseSend is called multiple times on the wrapper. +func (w *clientStreamWrapper) CloseSend() error { + if w.closeSendCalled.Swap(true) { + return nil + } + return w.ClientStream.CloseSend() +} + +// SendMsg sends message m across the stream. For RPCs where client can call +// SendMsg only once, i.e. only server-streaming RPCs, it converts io.EOF to nil +// and immediately calls CloseSend to trigger any interceptor hooks. +func (w *clientStreamWrapper) SendMsg(m any) error { + err := w.ClientStream.SendMsg(m) + // If the RPC is a client-streaming RPC, the client can send multiple + // messages. In this case, the client should handle any type of + // error,including io.EOF and call CloseSend once it is done sending messages. + if w.desc.ClientStreams { + return err + } + + if err == io.EOF { + // For non-client-streaming RPCs, we return nil instead of EOF on error + // because the generated code requires it. finish is not called; RecvMsg() + // will call it with the stream's status independently. + return nil + } + if err != nil { + return err + } + // In some scenarios (e.g., xDS), the same interceptors process both unary and + // streaming RPCs, relying on CloseSend to signal that no more messages are on + // the way. Although protobuf-generated stubs already invoke CloseSend for + // server-streaming RPCs, it is explicitly called here to ensure downstream + // interceptors are also notified when callers interact with the ClientStream + // API directly. + if err := w.CloseSend(); err != nil && err != io.EOF { + return err + } + return nil + +} + +// RecvMsg receives message m from the stream. For RPCs that call RecvMsg only +// once i.e. only client streaming RPCs, it calls the underlying RecvMsg a +// second time after receiving the first message to get the trailers. +func (w *clientStreamWrapper) RecvMsg(m any) error { + err := w.ClientStream.RecvMsg(m) + if err != nil { + return err + } + if w.desc.ServerStreams { + return nil + } + // Call RecvMsg again for non-server streaming RPCs to get the trailers and + // ensure RPC has completed successfully. + err = w.ClientStream.RecvMsg(m) + if err == io.EOF { + return nil + } + if err == nil { + return status.Error(codes.Internal, "cardinality violation: expected for non server-streaming RPCs, but received another message") + } + return err +} + +// defaultStreamInterceptor is a StreamClientInterceptor which wraps the +// ClientStream and is always invoked as the first interceptor. It consolidates +// behavior for different RPC types at the level closest to the application, +// which simplifies the underlying stream implementation and other interceptors +// by avoiding duplicate or scattered handling. +func defaultStreamInterceptor(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error) { + cs, err := streamer(ctx, desc, cc, method, opts...) + if err != nil { + return nil, err + } + return &clientStreamWrapper{ClientStream: cs, desc: desc}, nil +} + +// NewStream creates a new Stream for the client side. This is typically +// called by generated code. ctx is used for the lifetime of the stream. +// +// To ensure resources are not leaked due to the stream returned, one of the following +// actions must be performed: +// +// 1. Call Close on the ClientConn. +// 2. Cancel the context provided. +// 3. Call RecvMsg until a non-nil error is returned. A protobuf-generated +// client-streaming RPC, for instance, might use the helper function +// CloseAndRecv (note that CloseSend does not Recv, therefore is not +// guaranteed to release all resources). +// 4. Receive a non-nil, non-io.EOF error from Header or SendMsg. +// +// If none of the above happen, a goroutine and a context will be leaked, and grpc +// will not call the optionally-configured stats handler with a stats.End message. +func (cc *ClientConn) NewStream(ctx context.Context, desc *StreamDesc, method string, opts ...CallOption) (ClientStream, error) { + // allow interceptor to see all applicable call options, which means those + // configured as defaults from dial option as well as per-call options + opts = combine(cc.dopts.callOptions, opts) + + if cc.dopts.streamInt != nil { + return cc.dopts.streamInt(ctx, desc, cc, method, newClientStream, opts...) + } + return newClientStream(ctx, desc, cc, method, opts...) +} + +// NewClientStream is a wrapper for ClientConn.NewStream. +func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) { + return cc.NewStream(ctx, desc, method, opts...) +} + +var emptyMethodConfig = serviceconfig.MethodConfig{} + +// endOfClientStream performs cleanup actions required for both successful and +// failed streams. This includes incrementing channelz stats and invoking all +// registered OnFinish call options. +func endOfClientStream(cc *ClientConn, err error, opts ...CallOption) { + if channelz.IsOn() { + if err != nil { + cc.incrCallsFailed() + } else { + cc.incrCallsSucceeded() + } + } + + for _, o := range opts { + if o, ok := o.(OnFinishCallOption); ok { + o.OnFinish(err) + } + } +} + +// clientInterceptor is structurally identical to the ClientInterceptor defined +// in internal/xds/httpfilter/httpfilter.go. It is defined locally here so that +// we can type-assert the generic Interceptor field in iresolver.RPCConfig +// without introducing a dependency on xDS packages. +type clientInterceptor interface { + NewStream(ctx context.Context, ri iresolver.RPCInfo, newStream func(ctx context.Context, opts ...CallOption) (ClientStream, error), opts ...CallOption) (ClientStream, error) + Close() +} + +func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (_ ClientStream, err error) { + if channelz.IsOn() { + cc.incrCallsStarted() + } + defer func() { + if err != nil { + // Ensure cleanup when stream creation fails. + endOfClientStream(cc, err, opts...) + } + }() + + // Start tracking the RPC for idleness purposes. This is where a stream is + // created for both streaming and unary RPCs, and hence is a good place to + // track active RPC count. + cc.idlenessMgr.OnCallBegin() + + // Add a calloption, to decrement the active call count, that gets executed + // when the RPC completes. + opts = append([]CallOption{OnFinish(func(error) { cc.idlenessMgr.OnCallEnd() })}, opts...) + + if md, added, ok := metadataFromOutgoingContextRaw(ctx); ok { + // validate md + if err := imetadata.Validate(md); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + // validate added + for _, kvs := range added { + for i := 0; i < len(kvs); i += 2 { + if err := imetadata.ValidatePair(kvs[i], kvs[i+1]); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + } + } + } + // Provide an opportunity for the first RPC to see the first service config + // provided by the resolver. + nameResolutionDelayed, err := cc.waitForResolvedAddrs(ctx) + if err != nil { + return nil, err + } + + mc := &emptyMethodConfig + var onCommit func() + newStream := func(ctx context.Context, opts ...CallOption) (ClientStream, error) { + return newClientStreamWithParams(ctx, desc, cc, method, mc, onCommit, nameResolutionDelayed, opts...) + } + + rpcInfo := iresolver.RPCInfo{Context: ctx, Method: method, Authority: cc.authority} + rpcConfig, err := cc.safeConfigSelector.SelectConfig(rpcInfo) + if err != nil { + if st, ok := status.FromError(err); ok { + // Restrict the code to the list allowed by gRFC A54. + if istatus.IsRestrictedControlPlaneCode(st) { + err = status.Errorf(codes.Internal, "config selector returned illegal status: %v", err) + } + return nil, err + } + return nil, toRPCErr(err) + } + + if rpcConfig != nil { + if rpcConfig.Context != nil { + ctx = rpcConfig.Context + } + mc = &rpcConfig.MethodConfig + + if rpcConfig.OnCommitted != nil { + onCommit = rpcConfig.OnCommitted + // Register an OnFinish CallOption with the OnCommitted callback to + // ensure it is invoked on stream termination, even if the stream + // fails early before committing. Implementations of OnCommitted are + // expected to be idempotent (e.g., guarded by sync.Once), since both + // onCommit and OnFinish may run for a single RPC. + opts = append(opts, OnFinish(func(error) { rpcConfig.OnCommitted() })) + } + + if rpcConfig.Interceptor != nil { + rpcInfo.Context = nil + ns := newStream + if interceptor, ok := rpcConfig.Interceptor.(clientInterceptor); ok { + newStream = func(ctx context.Context, opts ...CallOption) (ClientStream, error) { + cs, err := interceptor.NewStream(ctx, rpcInfo, ns, opts...) + if err != nil { + return nil, toRPCErr(err) + } + return cs, nil + } + } else { + return nil, status.Errorf(codes.Internal, "invalid client interceptor type %T", rpcConfig.Interceptor) + } + } + } + + return newStream(ctx, opts...) +} + +func newClientStreamWithParams(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, mc *serviceconfig.MethodConfig, onCommit func(), nameResolutionDelayed bool, opts ...CallOption) (_ ClientStream, err error) { + callInfo := defaultCallInfo() + if mc.WaitForReady != nil { + callInfo.failFast = !*mc.WaitForReady + } + + // Possible context leak: + // The cancel function for the child context we create will only be called + // when RecvMsg returns a non-nil error, if the ClientConn is closed, or if + // an error is generated by SendMsg. + // https://github.com/grpc/grpc-go/issues/1818. + var cancel context.CancelFunc + if mc.Timeout != nil && *mc.Timeout >= 0 { + ctx, cancel = context.WithTimeout(ctx, *mc.Timeout) + } else { + ctx, cancel = context.WithCancel(ctx) + } + defer func() { + if err != nil { + cancel() + } + }() + + for _, o := range opts { + if err := o.before(callInfo); err != nil { + return nil, toRPCErr(err) + } + } + callInfo.maxSendMessageSize = getMaxSize(mc.MaxReqSize, callInfo.maxSendMessageSize, defaultClientMaxSendMessageSize) + callInfo.maxReceiveMessageSize = getMaxSize(mc.MaxRespSize, callInfo.maxReceiveMessageSize, defaultClientMaxReceiveMessageSize) + if err := setCallInfoCodec(callInfo); err != nil { + return nil, err + } + + callHdr := &transport.CallHdr{ + Host: cc.authority, + Method: method, + ContentSubtype: callInfo.contentSubtype, + Authority: callInfo.authority, + } + if allowed := callInfo.acceptedResponseCompressors; len(allowed) > 0 { + headerValue := strings.Join(allowed, ",") + callHdr.AcceptedCompressors = &headerValue + } + + // Set our outgoing compression according to the UseCompressor CallOption, if + // set. In that case, also find the compressor from the encoding package. + // Otherwise, use the compressor configured by the WithCompressor DialOption, + // if set. + var compressorV0 Compressor + var compressorV1 encoding.Compressor + if ct := callInfo.compressorName; ct != "" { + callHdr.SendCompress = ct + if ct != encoding.Identity { + compressorV1 = encoding.GetCompressor(ct) + if compressorV1 == nil { + return nil, status.Errorf(codes.Internal, "grpc: Compressor is not installed for requested grpc-encoding %q", ct) + } + } + } else if cc.dopts.compressorV0 != nil { + callHdr.SendCompress = cc.dopts.compressorV0.Type() + compressorV0 = cc.dopts.compressorV0 + } + if callInfo.creds != nil { + callHdr.Creds = callInfo.creds + } + + cs := &clientStream{ + callHdr: callHdr, + ctx: ctx, + methodConfig: mc, + opts: opts, + callInfo: callInfo, + cc: cc, + desc: desc, + codec: callInfo.codec, + compressorV0: compressorV0, + compressorV1: compressorV1, + cancel: cancel, + firstAttempt: true, + onCommit: onCommit, + nameResolutionDelay: nameResolutionDelayed, + } + if !cc.dopts.disableRetry { + cs.retryThrottler = cc.retryThrottler.Load().(*retryThrottler) + } + if ml := binarylog.GetMethodLogger(method); ml != nil { + cs.binlogs = append(cs.binlogs, ml) + } + if cc.dopts.binaryLogger != nil { + if ml := cc.dopts.binaryLogger.GetMethodLogger(method); ml != nil { + cs.binlogs = append(cs.binlogs, ml) + } + } + + // Pick the transport to use and create a new stream on the transport. + // Assign cs.attempt upon success. + op := func(a *csAttempt) error { + if err := a.getTransport(); err != nil { + return err + } + if err := a.newStream(); err != nil { + return err + } + // Because this operation is always called either here (while creating + // the clientStream) or by the retry code while locked when replaying + // the operation, it is safe to access cs.attempt directly. + cs.attempt = a + return nil + } + if err := cs.withRetry(op, func() { cs.bufferForRetryLocked(0, op, nil) }); err != nil { + return nil, err + } + + if len(cs.binlogs) != 0 { + md, _ := metadata.FromOutgoingContext(ctx) + logEntry := &binarylog.ClientHeader{ + OnClientSide: true, + Header: md, + MethodName: method, + Authority: cs.cc.authority, + } + if deadline, ok := ctx.Deadline(); ok { + logEntry.Timeout = time.Until(deadline) + if logEntry.Timeout < 0 { + logEntry.Timeout = 0 + } + } + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, logEntry) + } + } + + if desc != unaryStreamDesc { + // Listen on cc and stream contexts to cleanup when the user closes the + // ClientConn or cancels the stream context. In all other cases, an error + // should already be injected into the recv buffer by the transport, which + // the client will eventually receive, and then we will cancel the stream's + // context in clientStream.finish. + go func() { + select { + case <-cc.ctx.Done(): + cs.finish(ErrClientConnClosing) + case <-ctx.Done(): + cs.finish(toRPCErr(ctx.Err())) + } + }() + } + return cs, nil +} + +// newAttemptLocked creates a new csAttempt without a transport or stream. +func (cs *clientStream) newAttemptLocked(isTransparent bool) (*csAttempt, error) { + if err := cs.ctx.Err(); err != nil { + return nil, toRPCErr(err) + } + if err := cs.cc.ctx.Err(); err != nil { + return nil, ErrClientConnClosing + } + + ctx := newContextWithRPCInfo(cs.ctx, cs.callInfo.failFast, cs.callInfo.codec, cs.compressorV0, cs.compressorV1) + method := cs.callHdr.Method + var beginTime time.Time + sh := cs.cc.statsHandler + if sh != nil { + beginTime = time.Now() + ctx = sh.TagRPC(ctx, &stats.RPCTagInfo{ + FullMethodName: method, FailFast: cs.callInfo.failFast, + NameResolutionDelay: cs.nameResolutionDelay, + }) + sh.HandleRPC(ctx, &stats.Begin{ + Client: true, + BeginTime: beginTime, + FailFast: cs.callInfo.failFast, + IsClientStream: cs.desc.ClientStreams, + IsServerStream: cs.desc.ServerStreams, + IsTransparentRetryAttempt: isTransparent, + }) + } + + var trInfo *traceInfo + if EnableTracing { + trInfo = &traceInfo{ + tr: newTrace("grpc.Sent."+methodFamily(method), method), + firstLine: firstLine{ + client: true, + }, + } + if deadline, ok := ctx.Deadline(); ok { + trInfo.firstLine.deadline = time.Until(deadline) + } + trInfo.tr.LazyLog(&trInfo.firstLine, false) + ctx = newTraceContext(ctx, trInfo.tr) + } + + if cs.cc.parsedTarget.URL.Scheme == internal.GRPCResolverSchemeExtraMetadata { + // Add extra metadata (metadata that will be added by transport) to context + // so the balancer can see them. + ctx = grpcutil.WithExtraMetadata(ctx, metadata.Pairs( + "content-type", grpcutil.ContentType(cs.callHdr.ContentSubtype), + )) + } + + return &csAttempt{ + ctx: ctx, + beginTime: beginTime, + cs: cs, + decompressorV0: cs.cc.dopts.dc, + statsHandler: sh, + trInfo: trInfo, + }, nil +} + +func (a *csAttempt) getTransport() error { + cs := a.cs + + pickInfo := balancer.PickInfo{Ctx: a.ctx, FullMethodName: cs.callHdr.Method} + pick, err := cs.cc.pickerWrapper.pick(a.ctx, cs.callInfo.failFast, pickInfo) + a.transport, a.pickResult = pick.transport, pick.result + if err != nil { + if de, ok := err.(dropError); ok { + err = de.error + a.drop = true + } + return err + } + if a.trInfo != nil { + a.trInfo.firstLine.SetRemoteAddr(a.transport.Peer().Addr) + } + if pick.blocked && a.statsHandler != nil { + a.statsHandler.HandleRPC(a.ctx, &stats.DelayedPickComplete{}) + } + return nil +} + +func (a *csAttempt) newStream() error { + cs := a.cs + // The header is copied because the fields set below, notably the authority + // override taken from the pick result, describe the endpoint picked for + // this attempt only. Mutating the clientStream's header would carry them + // into a later attempt. + callHdr := *cs.callHdr + callHdr.PreviousAttempts = cs.numRetries + + // Merge metadata stored in PickResult, if any, with existing call metadata. + // It is safe to overwrite the csAttempt's context here, since all state + // maintained in it are local to the attempt. When the attempt has to be + // retried, a new instance of csAttempt will be created. + if a.pickResult.Metadata != nil { + if err := imetadata.Validate(a.pickResult.Metadata); err != nil { + return status.Error(codes.Internal, err.Error()) + } + // We currently do not have a function it the metadata package which + // merges given metadata with existing metadata in a context. Existing + // function `AppendToOutgoingContext()` takes a variadic argument of key + // value pairs. + // + // TODO: Make it possible to retrieve key value pairs from metadata.MD + // in a form passable to AppendToOutgoingContext(), or create a version + // of AppendToOutgoingContext() that accepts a metadata.MD. + md, _ := metadata.FromOutgoingContext(a.ctx) + md = metadata.Join(md, a.pickResult.Metadata) + a.ctx = metadata.NewOutgoingContext(a.ctx, md) + + // If the `CallAuthority` CallOption is not set, check if the LB picker + // has provided an authority override in the PickResult metadata and + // apply it, as specified in gRFC A81. + if cs.callInfo.authority == "" { + if authMD := a.pickResult.Metadata.Get(":authority"); len(authMD) > 0 { + callHdr.Authority = authMD[0] + } + } + } + s, err := a.transport.NewStream(a.ctx, &callHdr, a.statsHandler) + if err != nil { + nse, ok := err.(*transport.NewStreamError) + if !ok { + // Unexpected. + return err + } + + if nse.AllowTransparentRetry { + a.allowTransparentRetry = true + } + + // Unwrap and convert error. + return toRPCErr(nse.Err) + } + a.transportStream = s + a.ctx = s.Context() + a.parser = parser{r: s, bufferPool: a.cs.cc.dopts.copts.BufferPool} + return nil +} + +// clientStream implements a client side Stream. +type clientStream struct { + callHdr *transport.CallHdr + opts []CallOption + callInfo *callInfo + cc *ClientConn + desc *StreamDesc + + codec baseCodec + compressorV0 Compressor + compressorV1 encoding.Compressor + + cancel context.CancelFunc // cancels all attempts + + methodConfig *MethodConfig + + ctx context.Context // the application's context, wrapped by stats/tracing + + retryThrottler *retryThrottler // The throttler active when the RPC began. + + binlogs []binarylog.MethodLogger + + mu sync.Mutex + numRetries int // exclusive of transparent retry attempt(s) + numRetriesSincePushback int // retries since pushback; to reset backoff + // attempt is the active client stream attempt. + // The only place where it is written is the newAttemptLocked method and this method never writes nil. + // So, attempt can be nil only inside newClientStream function when clientStream is first created. + // One of the first things done after clientStream's creation, is to call newAttemptLocked which either + // assigns a non nil value to the attempt or returns an error. If an error is returned from newAttemptLocked, + // then newClientStream calls finish on the clientStream and returns. So, finish method is the only + // place where we need to check if the attempt is nil. + attempt *csAttempt + // TODO(hedging): hedging will have multiple attempts simultaneously. + onCommit func() + replayBuffer []replayOp // operations to replay on retry + replayBufferSize int // current size of replayBuffer + + // Bool fields are grouped at the tail to eliminate the alignment padding + // that would otherwise follow each bool when the next field is pointer- or + // int-sized. See https://github.com/grpc/grpc-go/issues/9280 for benchmarks. + // Add new bool fields here, not inline above. + + // Not guarded by mu. + sentLast bool // sent an end stream + receivedFirstMsg bool // set after the first message is received + // serverHeaderBinlogged is a boolean for whether server header has been + // logged. Server header will be logged when the first time one of those + // happens: stream.Header(), stream.Recv(). + // + // It's only read and used by Recv() and Header(), so it doesn't need to be + // synchronized. + serverHeaderBinlogged bool + // nameResolutionDelay indicates if there was a delay in the name resolution. + // This field is only valid on client side, it's always false on server side. + nameResolutionDelay bool + + // Guarded by mu. + firstAttempt bool // if true, transparent retry is valid + finished bool // TODO: replace with atomic cmpxchg or sync.Once? + committed bool // active attempt committed for retry? +} + +type replayOp struct { + op func(a *csAttempt) error + cleanup func() +} + +// csAttempt implements a single transport stream attempt within a +// clientStream. +type csAttempt struct { + ctx context.Context + cs *clientStream + transport transport.ClientTransport + transportStream *transport.ClientStream + parser parser + pickResult balancer.PickResult + + decompressorV0 Decompressor + decompressorV1 encoding.Compressor + + mu sync.Mutex // guards trInfo.tr + // trInfo may be nil (if EnableTracing is false). + // trInfo.tr is set when created (if EnableTracing is true), + // and cleared when the finish method is called. + trInfo *traceInfo + + statsHandler stats.Handler + beginTime time.Time + + // Bool fields are grouped at the tail to eliminate the alignment padding + // that would otherwise follow each bool when the next field is pointer- or + // int-sized. See https://github.com/grpc/grpc-go/issues/9347 for benchmarks. + // Add new bool fields here, not inline above. + + // Not guarded by mu. + decompressorSet bool + allowTransparentRetry bool // set for newStream errors that may be transparently retried + drop bool // set for pick errors that are returned as a status + + // Guarded by mu. + finished bool +} + +func (cs *clientStream) commitAttemptLocked() { + if !cs.committed && cs.onCommit != nil { + cs.onCommit() + } + cs.committed = true + for _, op := range cs.replayBuffer { + if op.cleanup != nil { + op.cleanup() + } + } + cs.replayBuffer = nil +} + +func (cs *clientStream) commitAttempt() { + cs.mu.Lock() + cs.commitAttemptLocked() + cs.mu.Unlock() +} + +// shouldRetry returns nil if the RPC should be retried; otherwise it returns +// the error that should be returned by the operation. If the RPC should be +// retried, the bool indicates whether it is being retried transparently. +func (a *csAttempt) shouldRetry(err error) (bool, error) { + cs := a.cs + + if cs.finished || cs.committed || a.drop { + // RPC is finished or committed or was dropped by the picker; cannot retry. + return false, err + } + if a.transportStream == nil && a.allowTransparentRetry { + return true, nil + } + // Wait for the trailers. + unprocessed := false + if a.transportStream != nil { + <-a.transportStream.Done() + unprocessed = a.transportStream.Unprocessed() + } + if cs.firstAttempt && unprocessed { + // First attempt, stream unprocessed: transparently retry. + return true, nil + } + if cs.cc.dopts.disableRetry { + return false, err + } + + pushback := 0 + hasPushback := false + if a.transportStream != nil { + if !a.transportStream.TrailersOnly() { + return false, err + } + + // TODO(retry): Move down if the spec changes to not check server pushback + // before considering this a failure for throttling. + sps := a.transportStream.Trailer()["grpc-retry-pushback-ms"] + if len(sps) == 1 { + var e error + if pushback, e = strconv.Atoi(sps[0]); e != nil || pushback < 0 { + channelz.Infof(logger, cs.cc.channelz, "Server retry pushback specified to abort (%q).", sps[0]) + cs.retryThrottler.throttle() // This counts as a failure for throttling. + return false, err + } + hasPushback = true + } else if len(sps) > 1 { + channelz.Warningf(logger, cs.cc.channelz, "Server retry pushback specified multiple values (%q); not retrying.", sps) + cs.retryThrottler.throttle() // This counts as a failure for throttling. + return false, err + } + } + + var code codes.Code + if a.transportStream != nil { + code = a.transportStream.Status().Code() + } else { + code = status.Code(err) + } + + rp := cs.methodConfig.RetryPolicy + if rp == nil || !rp.RetryableStatusCodes[code] { + return false, err + } + + // Note: the ordering here is important; we count this as a failure + // only if the code matched a retryable code. + if cs.retryThrottler.throttle() { + return false, err + } + if cs.numRetries+1 >= rp.MaxAttempts { + return false, fmt.Errorf("max retries exhausted: failed after %d attempts: %w", cs.numRetries+1, err) + } + + var dur time.Duration + if hasPushback { + dur = time.Millisecond * time.Duration(pushback) + cs.numRetriesSincePushback = 0 + } else { + fact := math.Pow(rp.BackoffMultiplier, float64(cs.numRetriesSincePushback)) + cur := min(float64(rp.InitialBackoff)*fact, float64(rp.MaxBackoff)) + // Apply jitter by multiplying with a random factor between 0.8 and 1.2 + cur *= 0.8 + 0.4*rand.Float64() + dur = time.Duration(int64(cur)) + cs.numRetriesSincePushback++ + } + + // TODO(dfawley): we could eagerly fail here if dur puts us past the + // deadline, but unsure if it is worth doing. + t := time.NewTimer(dur) + select { + case <-t.C: + cs.numRetries++ + return false, nil + case <-cs.ctx.Done(): + t.Stop() + return false, status.FromContextError(cs.ctx.Err()).Err() + } +} + +// Returns nil if a retry was performed and succeeded; error otherwise. +func (cs *clientStream) retryLocked(attempt *csAttempt, lastErr error) error { + for { + attempt.finish(toRPCErr(lastErr)) + isTransparent, err := attempt.shouldRetry(lastErr) + if err != nil { + cs.commitAttemptLocked() + return err + } + cs.firstAttempt = false + attempt, err = cs.newAttemptLocked(isTransparent) + if err != nil { + // Only returns error if the clientconn is closed or the context of + // the stream is canceled. + return err + } + // Note that the first op in replayBuffer always sets cs.attempt + // if it is able to pick a transport and create a stream. + if lastErr = cs.replayBufferLocked(attempt); lastErr == nil { + return nil + } + } +} + +func (cs *clientStream) Context() context.Context { + cs.commitAttempt() + // No need to lock before using attempt, since we know it is committed and + // cannot change. + if cs.attempt.transportStream != nil { + return cs.attempt.transportStream.Context() + } + return cs.ctx +} + +func (cs *clientStream) withRetry(op func(a *csAttempt) error, onSuccess func()) error { + cs.mu.Lock() + for { + if cs.committed { + cs.mu.Unlock() + // toRPCErr is used in case the error from the attempt comes from + // NewClientStream, which intentionally doesn't return a status + // error to allow for further inspection; all other errors should + // already be status errors. + return toRPCErr(op(cs.attempt)) + } + if len(cs.replayBuffer) == 0 { + // For the first op, which controls creation of the stream and + // assigns cs.attempt, we need to create a new attempt inline + // before executing the first op. On subsequent ops, the attempt + // is created immediately before replaying the ops. + var err error + if cs.attempt, err = cs.newAttemptLocked(false /* isTransparent */); err != nil { + cs.mu.Unlock() + cs.finish(err) + return err + } + } + a := cs.attempt + cs.mu.Unlock() + err := op(a) + cs.mu.Lock() + if a != cs.attempt { + // We started another attempt already. + continue + } + if err == io.EOF { + <-a.transportStream.Done() + } + if err == nil || (err == io.EOF && a.transportStream.Status().Code() == codes.OK) { + onSuccess() + cs.mu.Unlock() + return err + } + if err := cs.retryLocked(a, err); err != nil { + cs.mu.Unlock() + return err + } + } +} + +func (cs *clientStream) Header() (metadata.MD, error) { + var m metadata.MD + err := cs.withRetry(func(a *csAttempt) error { + var err error + m, err = a.transportStream.Header() + return toRPCErr(err) + }, cs.commitAttemptLocked) + + if m == nil && err == nil { + // The stream ended with success. Finish the clientStream. + err = io.EOF + } + + if err != nil { + cs.finish(err) + // Do not return the error. The user should get it by calling Recv(). + return nil, nil + } + + if len(cs.binlogs) != 0 && !cs.serverHeaderBinlogged && m != nil { + // Only log if binary log is on and header has not been logged, and + // there is actually headers to log. + logEntry := &binarylog.ServerHeader{ + OnClientSide: true, + Header: m, + PeerAddr: nil, + } + if peer, ok := peer.FromContext(cs.Context()); ok { + logEntry.PeerAddr = peer.Addr + } + cs.serverHeaderBinlogged = true + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, logEntry) + } + } + + return m, nil +} + +func (cs *clientStream) Trailer() metadata.MD { + // On RPC failure, we never need to retry, because usage requires that + // RecvMsg() returned a non-nil error before calling this function is valid. + // We would have retried earlier if necessary. + // + // Commit the attempt anyway, just in case users are not following those + // directions -- it will prevent races and should not meaningfully impact + // performance. + cs.commitAttempt() + if cs.attempt.transportStream == nil { + return nil + } + return cs.attempt.transportStream.Trailer() +} + +func (cs *clientStream) replayBufferLocked(attempt *csAttempt) error { + for _, f := range cs.replayBuffer { + if err := f.op(attempt); err != nil { + return err + } + } + return nil +} + +func (cs *clientStream) bufferForRetryLocked(sz int, op func(a *csAttempt) error, cleanup func()) { + // Note: we still will buffer if retry is disabled (for transparent retries). + if cs.committed { + return + } + cs.replayBufferSize += sz + if cs.replayBufferSize > cs.callInfo.maxRetryRPCBufferSize { + cs.commitAttemptLocked() + cleanup() + return + } + cs.replayBuffer = append(cs.replayBuffer, replayOp{op: op, cleanup: cleanup}) +} + +func (cs *clientStream) SendMsg(m any) (err error) { + defer func() { + if err != nil && err != io.EOF { + // Call finish on the client stream for errors generated by this SendMsg + // call, as these indicate problems created by this client. (Transport + // errors are converted to an io.EOF error in csAttempt.sendMsg; the real + // error will be returned from RecvMsg eventually in that case, or be + // retried.) + cs.finish(err) + } + }() + if cs.sentLast { + return status.Errorf(codes.Internal, "SendMsg called after CloseSend") + } + if !cs.desc.ClientStreams { + cs.sentLast = true + } + + // load hdr, payload, data + hdr, data, payload, pf, err := prepareMsg(m, cs.codec, cs.compressorV0, cs.compressorV1, cs.cc.dopts.copts.BufferPool) + if err != nil { + return err + } + + defer func() { + data.Free() + // only free payload if compression was made, and therefore it is a different set + // of buffers from data. + if pf.isCompressed() { + payload.Free() + } + }() + + dataLen := data.Len() + payloadLen := payload.Len() + // TODO(dfawley): should we be checking len(data) instead? + if payloadLen > *cs.callInfo.maxSendMessageSize { + return status.Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", payloadLen, *cs.callInfo.maxSendMessageSize) + } + + // always take an extra ref in case data == payload (i.e. when the data isn't + // compressed). The original ref will always be freed by the deferred free above. + payload.Ref() + op := func(a *csAttempt) error { + return a.sendMsg(m, hdr, payload, dataLen, payloadLen) + } + + // onSuccess is invoked when the op is captured for a subsequent retry. If the + // stream was established by a previous message and therefore retries are + // disabled, onSuccess will not be invoked, and payloadRef can be freed + // immediately. + onSuccessCalled := false + err = cs.withRetry(op, func() { + cs.bufferForRetryLocked(len(hdr)+payloadLen, op, payload.Free) + onSuccessCalled = true + }) + if !onSuccessCalled { + payload.Free() + } + if len(cs.binlogs) != 0 && err == nil { + cm := &binarylog.ClientMessage{ + OnClientSide: true, + Message: data.Materialize(), + } + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, cm) + } + } + return err +} + +func (cs *clientStream) RecvMsg(m any) error { + if len(cs.binlogs) != 0 && !cs.serverHeaderBinlogged { + // Call Header() to binary log header if it's not already logged. + cs.Header() + } + var recvInfo *payloadInfo + if len(cs.binlogs) != 0 { + recvInfo = &payloadInfo{} + defer recvInfo.free() + } + err := cs.withRetry(func(a *csAttempt) error { + return a.recvMsg(m, recvInfo) + }, cs.commitAttemptLocked) + if len(cs.binlogs) != 0 && err == nil { + sm := &binarylog.ServerMessage{ + OnClientSide: true, + Message: recvInfo.uncompressedBytes.Materialize(), + } + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, sm) + } + } + if err != nil { + // err != nil indicates end of stream. + cs.finish(err) + } + return err +} + +func (cs *clientStream) CloseSend() error { + if cs.sentLast { + // Return a nil error on repeated calls to this method. + return nil + } + cs.sentLast = true + op := func(a *csAttempt) error { + a.transportStream.Write(nil, nil, &transport.WriteOptions{Last: true}) + // Always return nil; io.EOF is the only error that might make sense + // instead, but there is no need to signal the client to call RecvMsg + // as the only use left for the stream after CloseSend is to call + // RecvMsg. This also matches historical behavior. + return nil + } + cs.withRetry(op, func() { cs.bufferForRetryLocked(0, op, nil) }) + if len(cs.binlogs) != 0 { + chc := &binarylog.ClientHalfClose{ + OnClientSide: true, + } + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, chc) + } + } + // We don't return an error here as we expect users to read all messages + // from the stream and get the RPC status from RecvMsg(). Note that + // SendMsg() must return an error when one occurs so the application + // knows to stop sending messages, but that does not apply here. + return nil +} + +func (cs *clientStream) finish(err error) { + if err == io.EOF { + // Ending a stream with EOF indicates a success. + err = nil + } + cs.mu.Lock() + if cs.finished { + cs.mu.Unlock() + return + } + cs.finished = true + cs.commitAttemptLocked() + attemptCreated := cs.attempt != nil + if attemptCreated { + cs.attempt.finish(err) + // after functions all rely upon having a stream. + if cs.attempt.transportStream != nil { + for _, o := range cs.opts { + o.after(cs.callInfo, cs.attempt) + } + } + } + + cs.mu.Unlock() + // Only one of cancel or trailer needs to be logged. + if len(cs.binlogs) != 0 { + switch err { + case errContextCanceled, errContextDeadline, ErrClientConnClosing: + c := &binarylog.Cancel{ + OnClientSide: true, + } + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, c) + } + default: + logEntry := &binarylog.ServerTrailer{ + OnClientSide: true, + Trailer: cs.Trailer(), + Err: err, + } + if peer, ok := peer.FromContext(cs.Context()); ok { + logEntry.PeerAddr = peer.Addr + } + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, logEntry) + } + } + } + if err == nil { + cs.retryThrottler.successfulRPC() + } + // If no attempt was ever created, stream creation has failed, and the + // cleanup is left to newClientStream, whose deferred cleanup invokes + // endOfClientStream if the call fails. Invoking it here as well would + // run the cleanup twice for the same call. + if attemptCreated { + endOfClientStream(cs.cc, err, cs.opts...) + } + cs.cancel() +} + +func (a *csAttempt) sendMsg(m any, hdr []byte, payld mem.BufferSlice, dataLength, payloadLength int) error { + cs := a.cs + if a.trInfo != nil { + a.mu.Lock() + if a.trInfo.tr != nil { + a.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true) + } + a.mu.Unlock() + } + if err := a.transportStream.Write(hdr, payld, &transport.WriteOptions{Last: !cs.desc.ClientStreams}); err != nil { + return io.EOF + } + if a.statsHandler != nil { + a.statsHandler.HandleRPC(a.ctx, outPayload(true, m, dataLength, payloadLength, time.Now())) + } + return nil +} + +func (a *csAttempt) recvMsg(m any, payInfo *payloadInfo) (err error) { + cs := a.cs + if a.statsHandler != nil && payInfo == nil { + payInfo = &payloadInfo{} + defer payInfo.free() + } + + if !a.decompressorSet { + // Block until we receive headers containing received message encoding. + if ct := a.transportStream.RecvCompress(); ct != "" && ct != encoding.Identity { + if a.decompressorV0 == nil || a.decompressorV0.Type() != ct { + // No configured decompressor, or it does not match the incoming + // message encoding; attempt to find a registered compressor that does. + a.decompressorV0 = nil + a.decompressorV1 = encoding.GetCompressor(ct) + } + // Validate that the compression method is acceptable for this call. + if !acceptedCompressorAllows(cs.callInfo.acceptedResponseCompressors, ct) { + return status.Errorf(codes.Internal, "grpc: peer compressed the response with %q which is not allowed by AcceptCompressors", ct) + } + } else { + // No compression is used; disable our decompressor. + a.decompressorV0 = nil + } + // Only initialize this state once per stream. + a.decompressorSet = true + } + if err := recv(&a.parser, cs.codec, a.transportStream, a.decompressorV0, m, *cs.callInfo.maxReceiveMessageSize, payInfo, a.decompressorV1, false); err != nil { + if err == io.EOF { + if statusErr := a.transportStream.Status().Err(); statusErr != nil { + return statusErr + } + // Received no msg and status OK for non-server streaming rpcs. + if !cs.desc.ServerStreams && !cs.receivedFirstMsg { + return status.Error(codes.Internal, "cardinality violation: received no response message from non-server-streaming RPC") + } + return io.EOF // indicates successful end of stream. + } + + return toRPCErr(err) + } + cs.receivedFirstMsg = true + if a.trInfo != nil { + a.mu.Lock() + if a.trInfo.tr != nil { + a.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true) + } + a.mu.Unlock() + } + if a.statsHandler != nil { + a.statsHandler.HandleRPC(a.ctx, &stats.InPayload{ + Client: true, + RecvTime: time.Now(), + Payload: m, + WireLength: payInfo.compressedLength + headerLen, + CompressedLength: payInfo.compressedLength, + Length: payInfo.uncompressedBytes.Len(), + }) + } + return nil +} + +func (a *csAttempt) finish(err error) { + a.mu.Lock() + if a.finished { + a.mu.Unlock() + return + } + a.finished = true + if err == io.EOF { + // Ending a stream with EOF indicates a success. + err = nil + } + var tr metadata.MD + if a.transportStream != nil { + a.transportStream.Close(err) + tr = a.transportStream.Trailer() + } + + if a.pickResult.Done != nil { + br := false + if a.transportStream != nil { + br = a.transportStream.BytesReceived() + } + a.pickResult.Done(balancer.DoneInfo{ + Err: err, + Trailer: tr, + BytesSent: a.transportStream != nil, + BytesReceived: br, + ServerLoad: balancerload.Parse(tr), + }) + } + if a.statsHandler != nil { + a.statsHandler.HandleRPC(a.ctx, &stats.End{ + Client: true, + BeginTime: a.beginTime, + EndTime: time.Now(), + Trailer: tr, + Error: err, + }) + } + if a.trInfo != nil && a.trInfo.tr != nil { + if err == nil { + a.trInfo.tr.LazyPrintf("RPC: [OK]") + } else { + a.trInfo.tr.LazyPrintf("RPC: [%v]", err) + a.trInfo.tr.SetError() + } + a.trInfo.tr.Finish() + a.trInfo.tr = nil + } + a.mu.Unlock() +} + +// newNonRetryClientStream creates a ClientStream with the specified transport, on the +// given addrConn. +// +// It's expected that the given transport is either the same one in addrConn, or +// is already closed. To avoid race, transport is specified separately, instead +// of using ac.transport. +// +// Main difference between this and ClientConn.NewStream: +// - no retry +// - no service config (or wait for service config) +// - no tracing or stats +func newNonRetryClientStream(ctx context.Context, desc *StreamDesc, method string, t transport.ClientTransport, ac *addrConn, opts ...CallOption) (_ ClientStream, err error) { + if t == nil { + // TODO: return RPC error here? + return nil, errors.New("transport provided is nil") + } + // defaultCallInfo contains unnecessary info(i.e. failfast, maxRetryRPCBufferSize), so we just initialize an empty struct. + c := &callInfo{} + + // Possible context leak: + // The cancel function for the child context we create will only be called + // when RecvMsg returns a non-nil error, if the ClientConn is closed, or if + // an error is generated by SendMsg. + // https://github.com/grpc/grpc-go/issues/1818. + ctx, cancel := context.WithCancel(ctx) + defer func() { + if err != nil { + cancel() + } + }() + + for _, o := range opts { + if err := o.before(c); err != nil { + return nil, toRPCErr(err) + } + } + c.maxReceiveMessageSize = getMaxSize(nil, c.maxReceiveMessageSize, defaultClientMaxReceiveMessageSize) + c.maxSendMessageSize = getMaxSize(nil, c.maxSendMessageSize, defaultServerMaxSendMessageSize) + if err := setCallInfoCodec(c); err != nil { + return nil, err + } + + callHdr := &transport.CallHdr{ + Host: ac.cc.authority, + Method: method, + ContentSubtype: c.contentSubtype, + } + + // Set our outgoing compression according to the UseCompressor CallOption, if + // set. In that case, also find the compressor from the encoding package. + // Otherwise, use the compressor configured by the WithCompressor DialOption, + // if set. + var cp Compressor + var comp encoding.Compressor + if ct := c.compressorName; ct != "" { + callHdr.SendCompress = ct + if ct != encoding.Identity { + comp = encoding.GetCompressor(ct) + if comp == nil { + return nil, status.Errorf(codes.Internal, "grpc: Compressor is not installed for requested grpc-encoding %q", ct) + } + } + } else if ac.cc.dopts.compressorV0 != nil { + callHdr.SendCompress = ac.cc.dopts.compressorV0.Type() + cp = ac.cc.dopts.compressorV0 + } + if c.creds != nil { + callHdr.Creds = c.creds + } + + // Use a special addrConnStream to avoid retry. + as := &addrConnStream{ + callHdr: callHdr, + ac: ac, + ctx: ctx, + cancel: cancel, + opts: opts, + callInfo: c, + desc: desc, + codec: c.codec, + sendCompressorV0: cp, + sendCompressorV1: comp, + decompressorV0: ac.cc.dopts.dc, + transport: t, + } + + // nil stats handler: internal streams like health and ORCA do not support telemetry. + s, err := as.transport.NewStream(as.ctx, as.callHdr, nil) + if err != nil { + err = toRPCErr(err) + return nil, err + } + as.transportStream = s + as.parser = parser{r: s, bufferPool: ac.dopts.copts.BufferPool} + ac.incrCallsStarted() + if desc != unaryStreamDesc { + // Listen on stream context to cleanup when the stream context is + // canceled. Also listen for the addrConn's context in case the + // addrConn is closed or reconnects to a different address. In all + // other cases, an error should already be injected into the recv + // buffer by the transport, which the client will eventually receive, + // and then we will cancel the stream's context in + // addrConnStream.finish. + go func() { + ac.mu.Lock() + acCtx := ac.ctx + ac.mu.Unlock() + select { + case <-acCtx.Done(): + as.finish(status.Error(codes.Canceled, "grpc: the SubConn is closing")) + case <-ctx.Done(): + as.finish(toRPCErr(ctx.Err())) + } + }() + } + return &clientStreamWrapper{ClientStream: as, desc: desc}, nil +} + +type addrConnStream struct { + transportStream *transport.ClientStream + ac *addrConn + callHdr *transport.CallHdr + cancel context.CancelFunc + opts []CallOption + callInfo *callInfo + transport transport.ClientTransport + ctx context.Context + desc *StreamDesc + codec baseCodec + sendCompressorV0 Compressor + sendCompressorV1 encoding.Compressor + decompressorV0 Decompressor + decompressorV1 encoding.Compressor + + // mu guards finished and is held for the entire finish method. + mu sync.Mutex + parser parser + + // Bool fields are grouped at the tail to eliminate the alignment padding + // that would otherwise follow each bool when the next field is pointer- or + // int-sized. See https://github.com/grpc/grpc-go/issues/9348 for benchmarks. + // Add new bool fields here, not inline above. + + // Not guarded by mu. + sentLast bool + receivedFirstMsg bool + decompressorSet bool + + finished bool // guarded by mu +} + +func (as *addrConnStream) Header() (metadata.MD, error) { + m, err := as.transportStream.Header() + if err != nil { + as.finish(toRPCErr(err)) + } + return m, err +} + +func (as *addrConnStream) Trailer() metadata.MD { + return as.transportStream.Trailer() +} + +func (as *addrConnStream) CloseSend() error { + if as.sentLast { + // Return a nil error on repeated calls to this method. + return nil + } + as.sentLast = true + + as.transportStream.Write(nil, nil, &transport.WriteOptions{Last: true}) + // Always return nil; io.EOF is the only error that might make sense + // instead, but there is no need to signal the client to call RecvMsg + // as the only use left for the stream after CloseSend is to call + // RecvMsg. This also matches historical behavior. + return nil +} + +func (as *addrConnStream) Context() context.Context { + return as.transportStream.Context() +} + +func (as *addrConnStream) SendMsg(m any) (err error) { + defer func() { + if err != nil && err != io.EOF { + // Call finish on the client stream for errors generated by this SendMsg + // call, as these indicate problems created by this client. (Transport + // errors are converted to an io.EOF error in csAttempt.sendMsg; the real + // error will be returned from RecvMsg eventually in that case, or be + // retried.) + as.finish(err) + } + }() + if as.sentLast { + return status.Errorf(codes.Internal, "SendMsg called after CloseSend") + } + if !as.desc.ClientStreams { + as.sentLast = true + } + + // load hdr, payload, data + hdr, data, payload, pf, err := prepareMsg(m, as.codec, as.sendCompressorV0, as.sendCompressorV1, as.ac.dopts.copts.BufferPool) + if err != nil { + return err + } + + defer func() { + data.Free() + // only free payload if compression was made, and therefore it is a different set + // of buffers from data. + if pf.isCompressed() { + payload.Free() + } + }() + + // TODO(dfawley): should we be checking len(data) instead? + if payload.Len() > *as.callInfo.maxSendMessageSize { + return status.Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", payload.Len(), *as.callInfo.maxSendMessageSize) + } + + if err := as.transportStream.Write(hdr, payload, &transport.WriteOptions{Last: !as.desc.ClientStreams}); err != nil { + return io.EOF + } + + return nil +} + +func (as *addrConnStream) RecvMsg(m any) (err error) { + defer func() { + if err != nil { + // err != nil indicates end of stream. + as.finish(err) + } + }() + + if !as.decompressorSet { + // Block until we receive headers containing received message encoding. + if ct := as.transportStream.RecvCompress(); ct != "" && ct != encoding.Identity { + if as.decompressorV0 == nil || as.decompressorV0.Type() != ct { + // No configured decompressor, or it does not match the incoming + // message encoding; attempt to find a registered compressor that does. + as.decompressorV0 = nil + as.decompressorV1 = encoding.GetCompressor(ct) + } + // Validate that the compression method is acceptable for this call. + if !acceptedCompressorAllows(as.callInfo.acceptedResponseCompressors, ct) { + return status.Errorf(codes.Internal, "grpc: peer compressed the response with %q which is not allowed by AcceptCompressors", ct) + } + } else { + // No compression is used; disable our decompressor. + as.decompressorV0 = nil + } + // Only initialize this state once per stream. + as.decompressorSet = true + } + if err := recv(&as.parser, as.codec, as.transportStream, as.decompressorV0, m, *as.callInfo.maxReceiveMessageSize, nil, as.decompressorV1, false); err != nil { + if err == io.EOF { + if statusErr := as.transportStream.Status().Err(); statusErr != nil { + return statusErr + } + // Received no msg and status OK for non-server streaming rpcs. + if !as.desc.ServerStreams && !as.receivedFirstMsg { + return status.Error(codes.Internal, "cardinality violation: received no response message from non-server-streaming RPC") + } + return io.EOF // indicates successful end of stream. + } + return toRPCErr(err) + } + as.receivedFirstMsg = true + return nil +} + +func (as *addrConnStream) finish(err error) { + as.mu.Lock() + if as.finished { + as.mu.Unlock() + return + } + as.finished = true + if err == io.EOF { + // Ending a stream with EOF indicates a success. + err = nil + } + if as.transportStream != nil { + as.transportStream.Close(err) + } + + if err != nil { + as.ac.incrCallsFailed() + } else { + as.ac.incrCallsSucceeded() + } + as.cancel() + as.mu.Unlock() +} + +// ServerStream defines the server-side behavior of a streaming RPC. +// +// Errors returned from ServerStream methods are compatible with the status +// package. However, the status code will often not match the RPC status as +// seen by the client application, and therefore, should not be relied upon for +// this purpose. +type ServerStream interface { + // SetHeader sets the header metadata. It may be called multiple times. + // When call multiple times, all the provided metadata will be merged. + // All the metadata will be sent out when one of the following happens: + // - ServerStream.SendHeader() is called; + // - The first response is sent out; + // - An RPC status is sent out (error or success). + SetHeader(metadata.MD) error + // SendHeader sends the header metadata. + // The provided md and headers set by SetHeader() will be sent. + // It fails if called multiple times. + SendHeader(metadata.MD) error + // SetTrailer sets the trailer metadata which will be sent with the RPC status. + // When called more than once, all the provided metadata will be merged. + SetTrailer(metadata.MD) + // Context returns the context for this stream. + Context() context.Context + // SendMsg sends a message. On error, SendMsg aborts the stream and the + // error is returned directly. + // + // SendMsg blocks until: + // - There is sufficient flow control to schedule m with the transport, or + // - The stream is done, or + // - The stream breaks. + // + // SendMsg does not wait until the message is received by the client. An + // untimely stream closure may result in lost messages. + // + // It is safe to have a goroutine calling SendMsg and another goroutine + // calling RecvMsg on the same stream at the same time, but it is not safe + // to call SendMsg on the same stream in different goroutines. + // + // It is not safe to modify the message after calling SendMsg. Tracing + // libraries and stats handlers may use the message lazily. + SendMsg(m any) error + // RecvMsg blocks until it receives a message into m or the stream is + // done. It returns io.EOF when the client has performed a CloseSend. On + // any non-EOF error, the stream is aborted and the error contains the + // RPC status. + // + // It is safe to have a goroutine calling SendMsg and another goroutine + // calling RecvMsg on the same stream at the same time, but it is not + // safe to call RecvMsg on the same stream in different goroutines. + RecvMsg(m any) error +} + +// serverStream implements a server side Stream. +type serverStream struct { + ctx context.Context + s *transport.ServerStream + p parser + codec baseCodec + desc *StreamDesc + + compressorV0 Compressor + compressorV1 encoding.Compressor + decompressorV0 Decompressor + decompressorV1 encoding.Compressor + + sendCompressorName string + + maxReceiveMessageSize int + maxSendMessageSize int + + // mu guards trInfo.tr after the service handler runs. + mu sync.Mutex + trInfo *traceInfo + + statsHandler stats.Handler + + binlogs []binarylog.MethodLogger + + // Bool fields are grouped at the tail to eliminate the alignment padding + // that would otherwise follow each bool when the next field is pointer- or + // int-sized. See https://github.com/grpc/grpc-go/issues/9349 for benchmarks. + // Add new bool fields here, not inline above. + + recvFirstMsg bool // set after the first message is received + // serverHeaderBinlogged indicates whether server header has been logged. It + // will happen when one of the following two happens: stream.SendHeader(), + // stream.Send(). + // + // It's only checked in send and sendHeader, doesn't need to be + // synchronized. + serverHeaderBinlogged bool +} + +func (ss *serverStream) Context() context.Context { + return ss.ctx +} + +func (ss *serverStream) SetHeader(md metadata.MD) error { + if md.Len() == 0 { + return nil + } + err := imetadata.Validate(md) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + return ss.s.SetHeader(md) +} + +func (ss *serverStream) SendHeader(md metadata.MD) error { + err := imetadata.Validate(md) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + + err = ss.s.SendHeader(md) + if len(ss.binlogs) != 0 && !ss.serverHeaderBinlogged { + h, _ := ss.s.Header() + sh := &binarylog.ServerHeader{ + Header: h, + } + ss.serverHeaderBinlogged = true + for _, binlog := range ss.binlogs { + binlog.Log(ss.ctx, sh) + } + } + return err +} + +func (ss *serverStream) SetTrailer(md metadata.MD) { + if md.Len() == 0 { + return + } + if err := imetadata.Validate(md); err != nil { + logger.Errorf("stream: failed to validate md when setting trailer, err: %v", err) + } + ss.s.SetTrailer(md) +} + +func (ss *serverStream) SendMsg(m any) (err error) { + defer func() { + if ss.trInfo != nil { + ss.mu.Lock() + if ss.trInfo.tr != nil { + if err == nil { + ss.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true) + } else { + ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []any{err}}, true) + ss.trInfo.tr.SetError() + } + } + ss.mu.Unlock() + } + if err != nil && err != io.EOF { + st, _ := status.FromError(toRPCErr(err)) + ss.s.WriteStatus(st) + // Non-user specified status was sent out. This should be an error + // case (as a server side Cancel maybe). + // + // This is not handled specifically now. User will return a final + // status from the service handler, we will log that error instead. + // This behavior is similar to an interceptor. + } + }() + + // Server handler could have set new compressor by calling SetSendCompressor. + // In case it is set, we need to use it for compressing outbound message. + if sendCompressorsName := ss.s.SendCompress(); sendCompressorsName != ss.sendCompressorName { + ss.compressorV1 = encoding.GetCompressor(sendCompressorsName) + ss.sendCompressorName = sendCompressorsName + } + + // load hdr, payload, data + hdr, data, payload, pf, err := prepareMsg(m, ss.codec, ss.compressorV0, ss.compressorV1, ss.p.bufferPool) + if err != nil { + return err + } + + defer func() { + data.Free() + // only free payload if compression was made, and therefore it is a different set + // of buffers from data. + if pf.isCompressed() { + payload.Free() + } + }() + + dataLen := data.Len() + payloadLen := payload.Len() + + // TODO(dfawley): should we be checking len(data) instead? + if payloadLen > ss.maxSendMessageSize { + return status.Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", payloadLen, ss.maxSendMessageSize) + } + if err := ss.s.Write(hdr, payload, &transport.WriteOptions{Last: false}); err != nil { + return toRPCErr(err) + } + + if len(ss.binlogs) != 0 { + if !ss.serverHeaderBinlogged { + h, _ := ss.s.Header() + sh := &binarylog.ServerHeader{ + Header: h, + } + ss.serverHeaderBinlogged = true + for _, binlog := range ss.binlogs { + binlog.Log(ss.ctx, sh) + } + } + sm := &binarylog.ServerMessage{ + Message: data.Materialize(), + } + for _, binlog := range ss.binlogs { + binlog.Log(ss.ctx, sm) + } + } + if ss.statsHandler != nil { + ss.statsHandler.HandleRPC(ss.s.Context(), outPayload(false, m, dataLen, payloadLen, time.Now())) + } + return nil +} + +func (ss *serverStream) RecvMsg(m any) (err error) { + defer func() { + if ss.trInfo != nil { + ss.mu.Lock() + if ss.trInfo.tr != nil { + if err == nil { + ss.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true) + } else if err != io.EOF { + ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []any{err}}, true) + ss.trInfo.tr.SetError() + } + } + ss.mu.Unlock() + } + if err != nil && err != io.EOF { + st, _ := status.FromError(toRPCErr(err)) + ss.s.WriteStatus(st) + // Non-user specified status was sent out. This should be an error + // case (as a server side Cancel maybe). + // + // This is not handled specifically now. User will return a final + // status from the service handler, we will log that error instead. + // This behavior is similar to an interceptor. + } + }() + var payInfo *payloadInfo + if ss.statsHandler != nil || len(ss.binlogs) != 0 { + payInfo = &payloadInfo{} + defer payInfo.free() + } + if err := recv(&ss.p, ss.codec, ss.s, ss.decompressorV0, m, ss.maxReceiveMessageSize, payInfo, ss.decompressorV1, true); err != nil { + if err == io.EOF { + if len(ss.binlogs) != 0 { + chc := &binarylog.ClientHalfClose{} + for _, binlog := range ss.binlogs { + binlog.Log(ss.ctx, chc) + } + } + // Received no request msg for non-client streaming rpcs. + if !ss.desc.ClientStreams && !ss.recvFirstMsg { + return status.Error(codes.Internal, "cardinality violation: received no request message from non-client-streaming RPC") + } + return err + } + if err == io.ErrUnexpectedEOF { + err = status.Error(codes.Internal, io.ErrUnexpectedEOF.Error()) + } + return toRPCErr(err) + } + ss.recvFirstMsg = true + if ss.statsHandler != nil { + ss.statsHandler.HandleRPC(ss.s.Context(), &stats.InPayload{ + RecvTime: time.Now(), + Payload: m, + Length: payInfo.uncompressedBytes.Len(), + WireLength: payInfo.compressedLength + headerLen, + CompressedLength: payInfo.compressedLength, + }) + } + if len(ss.binlogs) != 0 { + cm := &binarylog.ClientMessage{ + Message: payInfo.uncompressedBytes.Materialize(), + } + for _, binlog := range ss.binlogs { + binlog.Log(ss.ctx, cm) + } + } + + if ss.desc.ClientStreams { + // Subsequent messages should be received by subsequent RecvMsg calls. + return nil + } + // Special handling for non-client-stream rpcs. + // This recv expects EOF or errors, so we don't collect inPayload. + if err := recv(&ss.p, ss.codec, ss.s, ss.decompressorV0, m, ss.maxReceiveMessageSize, nil, ss.decompressorV1, true); err == io.EOF { + return nil + } else if err != nil { + return err + } + return status.Error(codes.Internal, "cardinality violation: received multiple request messages for non-client-streaming RPC") +} + +// MethodFromServerStream returns the method string for the input stream. +// The returned string is in the format of "/service/method". +func MethodFromServerStream(stream ServerStream) (string, bool) { + return Method(stream.Context()) +} + +// prepareMsg returns the hdr, payload and data using the compressors passed or +// using the passed preparedmsg. The returned boolean indicates whether +// compression was made and therefore whether the payload needs to be freed in +// addition to the returned data. Freeing the payload if the returned boolean is +// false can lead to undefined behavior. +func prepareMsg(m any, codec baseCodec, cp Compressor, comp encoding.Compressor, pool mem.BufferPool) (hdr []byte, data, payload mem.BufferSlice, pf payloadFormat, err error) { + if preparedMsg, ok := m.(*PreparedMsg); ok { + return preparedMsg.hdr, preparedMsg.encodedData, preparedMsg.payload, preparedMsg.pf, nil + } + // The input interface is not a prepared msg. + // Marshal and Compress the data at this point + data, err = encode(codec, m) + if err != nil { + return nil, nil, nil, 0, err + } + compData, pf, err := compress(data, cp, comp, pool) + if err != nil { + data.Free() + return nil, nil, nil, 0, err + } + hdr, payload = msgHeader(data, compData, pf) + return hdr, data, payload, pf, nil +} diff --git a/stream_interfaces.go b/stream_interfaces.go new file mode 100644 index 0000000..0037fee --- /dev/null +++ b/stream_interfaces.go @@ -0,0 +1,238 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +// ServerStreamingClient represents the client side of a server-streaming (one +// request, many responses) RPC. It is generic over the type of the response +// message. It is used in generated code. +type ServerStreamingClient[Res any] interface { + // Recv receives the next response message from the server. The client may + // repeatedly call Recv to read messages from the response stream. If + // io.EOF is returned, the stream has terminated with an OK status. Any + // other error is compatible with the status package and indicates the + // RPC's status code and message. + Recv() (*Res, error) + + // ClientStream is embedded to provide Context, Header, and Trailer + // functionality. No other methods in the ClientStream should be called + // directly. + ClientStream +} + +// ServerStreamingServer represents the server side of a server-streaming (one +// request, many responses) RPC. It is generic over the type of the response +// message. It is used in generated code. +// +// To terminate the response stream, return from the handler method and return +// an error from the status package, or use nil to indicate an OK status code. +type ServerStreamingServer[Res any] interface { + // Send sends a response message to the client. The server handler may + // call Send multiple times to send multiple messages to the client. An + // error is returned if the stream was terminated unexpectedly, and the + // handler method should return, as the stream is no longer usable. + Send(*Res) error + + // ServerStream is embedded to provide Context, SetHeader, SendHeader, and + // SetTrailer functionality. No other methods in the ServerStream should + // be called directly. + ServerStream +} + +// ClientStreamingClient represents the client side of a client-streaming (many +// requests, one response) RPC. It is generic over both the type of the request +// message stream and the type of the unary response message. It is used in +// generated code. +type ClientStreamingClient[Req any, Res any] interface { + // Send sends a request message to the server. The client may call Send + // multiple times to send multiple messages to the server. On error, Send + // aborts the stream. If the error was generated by the client, the status + // is returned directly. Otherwise, io.EOF is returned, and the status of + // the stream may be discovered using CloseAndRecv(). + Send(*Req) error + + // CloseAndRecv closes the request stream and waits for the server's + // response. This method must be called once and only once after sending + // all request messages. Any error returned is implemented by the status + // package. + CloseAndRecv() (*Res, error) + + // ClientStream is embedded to provide Context, Header, and Trailer + // functionality. No other methods in the ClientStream should be called + // directly. + ClientStream +} + +// ClientStreamingServer represents the server side of a client-streaming (many +// requests, one response) RPC. It is generic over both the type of the request +// message stream and the type of the unary response message. It is used in +// generated code. +// +// To terminate the RPC, call SendAndClose and return nil from the method +// handler or do not call SendAndClose and return an error from the status +// package. +type ClientStreamingServer[Req any, Res any] interface { + // Recv receives the next request message from the client. The server may + // repeatedly call Recv to read messages from the request stream. If + // io.EOF is returned, it indicates the client called CloseAndRecv on its + // ClientStreamingClient. Any other error indicates the stream was + // terminated unexpectedly, and the handler method should return, as the + // stream is no longer usable. + Recv() (*Req, error) + + // SendAndClose sends a single response message to the client and closes + // the stream. This method must be called once and only once after all + // request messages have been processed. Recv should not be called after + // calling SendAndClose. + SendAndClose(*Res) error + + // ServerStream is embedded to provide Context, SetHeader, SendHeader, and + // SetTrailer functionality. No other methods in the ServerStream should + // be called directly. + ServerStream +} + +// BidiStreamingClient represents the client side of a bidirectional-streaming +// (many requests, many responses) RPC. It is generic over both the type of the +// request message stream and the type of the response message stream. It is +// used in generated code. +type BidiStreamingClient[Req any, Res any] interface { + // Send sends a request message to the server. The client may call Send + // multiple times to send multiple messages to the server. On error, Send + // aborts the stream. If the error was generated by the client, the status + // is returned directly. Otherwise, io.EOF is returned, and the status of + // the stream may be discovered using Recv(). + Send(*Req) error + + // Recv receives the next response message from the server. The client may + // repeatedly call Recv to read messages from the response stream. If + // io.EOF is returned, the stream has terminated with an OK status. Any + // other error is compatible with the status package and indicates the + // RPC's status code and message. + Recv() (*Res, error) + + // ClientStream is embedded to provide Context, Header, Trailer, and + // CloseSend functionality. No other methods in the ClientStream should be + // called directly. + ClientStream +} + +// BidiStreamingServer represents the server side of a bidirectional-streaming +// (many requests, many responses) RPC. It is generic over both the type of the +// request message stream and the type of the response message stream. It is +// used in generated code. +// +// To terminate the stream, return from the handler method and return +// an error from the status package, or use nil to indicate an OK status code. +type BidiStreamingServer[Req any, Res any] interface { + // Recv receives the next request message from the client. The server may + // repeatedly call Recv to read messages from the request stream. If + // io.EOF is returned, it indicates the client called CloseSend on its + // BidiStreamingClient. Any other error indicates the stream was + // terminated unexpectedly, and the handler method should return, as the + // stream is no longer usable. + Recv() (*Req, error) + + // Send sends a response message to the client. The server handler may + // call Send multiple times to send multiple messages to the client. An + // error is returned if the stream was terminated unexpectedly, and the + // handler method should return, as the stream is no longer usable. + Send(*Res) error + + // ServerStream is embedded to provide Context, SetHeader, SendHeader, and + // SetTrailer functionality. No other methods in the ServerStream should + // be called directly. + ServerStream +} + +// GenericClientStream implements the ServerStreamingClient, ClientStreamingClient, +// and BidiStreamingClient interfaces. It is used in generated code. +type GenericClientStream[Req any, Res any] struct { + ClientStream +} + +var _ ServerStreamingClient[string] = (*GenericClientStream[int, string])(nil) +var _ ClientStreamingClient[int, string] = (*GenericClientStream[int, string])(nil) +var _ BidiStreamingClient[int, string] = (*GenericClientStream[int, string])(nil) + +// Send pushes one message into the stream of requests to be consumed by the +// server. The type of message which can be sent is determined by the Req type +// parameter of the GenericClientStream receiver. +func (x *GenericClientStream[Req, Res]) Send(m *Req) error { + return x.ClientStream.SendMsg(m) +} + +// Recv reads one message from the stream of responses generated by the server. +// The type of the message returned is determined by the Res type parameter +// of the GenericClientStream receiver. +func (x *GenericClientStream[Req, Res]) Recv() (*Res, error) { + m := new(Res) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// CloseAndRecv closes the sending side of the stream, then receives the unary +// response from the server. The type of message which it returns is determined +// by the Res type parameter of the GenericClientStream receiver. +func (x *GenericClientStream[Req, Res]) CloseAndRecv() (*Res, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(Res) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// GenericServerStream implements the ServerStreamingServer, ClientStreamingServer, +// and BidiStreamingServer interfaces. It is used in generated code. +type GenericServerStream[Req any, Res any] struct { + ServerStream +} + +var _ ServerStreamingServer[string] = (*GenericServerStream[int, string])(nil) +var _ ClientStreamingServer[int, string] = (*GenericServerStream[int, string])(nil) +var _ BidiStreamingServer[int, string] = (*GenericServerStream[int, string])(nil) + +// Send pushes one message into the stream of responses to be consumed by the +// client. The type of message which can be sent is determined by the Res +// type parameter of the serverStreamServer receiver. +func (x *GenericServerStream[Req, Res]) Send(m *Res) error { + return x.ServerStream.SendMsg(m) +} + +// SendAndClose pushes the unary response to the client. The type of message +// which can be sent is determined by the Res type parameter of the +// clientStreamServer receiver. +func (x *GenericServerStream[Req, Res]) SendAndClose(m *Res) error { + return x.ServerStream.SendMsg(m) +} + +// Recv reads one message from the stream of requests generated by the client. +// The type of the message returned is determined by the Req type parameter +// of the clientStreamServer receiver. +func (x *GenericServerStream[Req, Res]) Recv() (*Req, error) { + m := new(Req) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} diff --git a/stream_test.go b/stream_test.go new file mode 100644 index 0000000..4116fc0 --- /dev/null +++ b/stream_test.go @@ -0,0 +1,243 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc_test + +import ( + "context" + "io" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestStream_Header_TrailersOnly(t *testing.T) { + ss := stubserver.StubServer{ + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + return status.Errorf(codes.NotFound, "a test error") + }, + } + if err := ss.Start(nil); err != nil { + t.Fatal("Error starting server:", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + s, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatal("Error staring call", err) + } + if md, err := s.Header(); md != nil || err != nil { + t.Fatalf("s.Header() = %v, %v; want nil, nil", md, err) + } + if _, err := s.Recv(); status.Code(err) != codes.NotFound { + t.Fatalf("s.Recv() = _, %v; want _, err.Code()=codes.NotFound", err) + } +} + +// TestUnaryClient_ServerStreamingMismatch ensures that the client's +// non-streaming RecvMsg() logic correctly handles various error scenarios +// from the server. +// +// The Client initiates a Unary RPC (Invoke), forcing it to use the +// non-server-streaming `recvMsg` code path (where the bug was). +// The Server handles it as a Streaming RPC (FullDuplexCall), allowing us to +// send arbitrary sequences of messages and errors. +func (s) TestUnaryClient_ServerStreamingMismatch(t *testing.T) { + tests := []struct { + name string + fullDuplexCallF func(testgrpc.TestService_FullDuplexCallServer) error + wantErrorContains string + wantCode codes.Code + clientCallOptions []grpc.CallOption + }{ + { + name: "server_sends_error_after_message", + fullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil { + return err + } + return status.Error(codes.Internal, "server error after message") + }, + wantErrorContains: "server error after message", + wantCode: codes.Internal, + }, + { + name: "server_sends_second_message_exceeding_limit", + fullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if err := stream.Send(&testpb.StreamingOutputCallResponse{ + Payload: &testpb.Payload{Body: make([]byte, 1)}, + }); err != nil { + return err + } + return stream.Send(&testpb.StreamingOutputCallResponse{ + Payload: &testpb.Payload{Body: make([]byte, 10)}, + }) + }, + clientCallOptions: []grpc.CallOption{grpc.MaxCallRecvMsgSize(5)}, + wantErrorContains: "received message larger than max", + wantCode: codes.ResourceExhausted, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ss := &stubserver.StubServer{ + FullDuplexCallF: test.fullDuplexCallF, + } + if err := ss.Start(nil); err != nil { + t.Fatal("Error starting server:", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Invoke the streaming RPC method as a Unary RPC. This forces the client + // to use the non-streaming RecvMsg path, while the server handles it as + // a stream (allowing it to send messages and errors in ways a standard + // Unary server cannot). + err := ss.CC.Invoke(ctx, "/grpc.testing.TestService/FullDuplexCall", &testpb.StreamingOutputCallRequest{}, &testpb.StreamingOutputCallResponse{}, test.clientCallOptions...) + if err == nil { + t.Fatal("Client.Invoke returned nil, want error") + } + if status.Code(err) != test.wantCode { + t.Errorf("Unexpected error code: got %v, want %v", status.Code(err), test.wantCode) + } + if !strings.Contains(err.Error(), test.wantErrorContains) { + t.Errorf("Unexpected error message: got %v, want %v", err.Error(), test.wantErrorContains) + } + }) + } +} + +// interceptorStream wraps a ClientStream to record invocations of RecvMsg and +// CloseSend hooks across downstream interceptors. +type interceptorStream struct { + grpc.ClientStream + recvMsgCount int + closeSendCount int +} + +func (s *interceptorStream) RecvMsg(m any) error { + s.recvMsgCount++ + return s.ClientStream.RecvMsg(m) +} + +func (s *interceptorStream) CloseSend() error { + s.closeSendCount++ + return s.ClientStream.CloseSend() +} + +// TestDefaultStreamInterceptor verifies that defaultStreamInterceptor's +// behavior of automatically triggering CloseSend on non-client-streaming RPCs +// right after SendMsg, and calling RecvMsg a second time on +// non-server-streaming RPCs to consume trailers and io.EOF, are not visible to +// user-defined interceptors. +func (s) TestDefaultStreamInterceptor(t *testing.T) { + var iStream *interceptorStream + // Define a client-side stream interceptor that wraps the ClientStream to + // monitor hook invocations across RPC calls. + clientInt := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + cs, err := streamer(ctx, desc, cc, method, opts...) + if err != nil { + return nil, err + } + iStream = &interceptorStream{ClientStream: cs} + return iStream, nil + } + + // Setup a service implementing both a client-streaming RPC and a + // server-streaming RPC. + ss := &stubserver.StubServer{ + StreamingOutputCallF: func(_ *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error { + return stream.Send(&testpb.StreamingOutputCallResponse{}) + }, + StreamingInputCallF: func(stream testgrpc.TestService_StreamingInputCallServer) error { + for { + if _, err := stream.Recv(); err != nil { + if err == io.EOF { + return stream.SendAndClose(&testpb.StreamingInputCallResponse{}) + } + return err + } + } + }, + } + if err := ss.Start(nil, grpc.WithStreamInterceptor(clientInt)); err != nil { + t.Fatal("Error starting server:", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Make a client-streaming RPC. When CloseAndRecv invokes RecvMsg once to + // get the single reply message on a non-server-streaming RPC, + // defaultStreamInterceptor automatically calls a second RecvMsg on the + // underlying client stream to consume io.EOF and receive trailers. But this + // should not be visible to user interceptors. + stream, err := ss.Client.StreamingInputCall(ctx) + if err != nil { + t.Fatal("Error calling StreamingInputCall:", err) + } + if err := stream.Send(&testpb.StreamingInputCallRequest{}); err != nil { + t.Fatal("Error sending request:", err) + } + if _, err := stream.CloseAndRecv(); err != nil { + t.Fatal("Error running CloseAndRecv:", err) + } + if iStream.recvMsgCount != 1 { + t.Fatalf("RecvMsg was called %v times on user interceptor stream, want 1 time", iStream.recvMsgCount) + } + + // Make a server-streaming RPC. Since StreamingOutputCall is not + // client-streaming, the proto generated code invokes CloseSend after + // sending the single message. defaultStreamInterceptor also invokes + // CloseSend right after sending the request message (to handle cases where + // the user is using the ClientStream API instead of the proto generated + // code) to signal downstream interceptors, which in this case are xDS + // filters. The user interceptor should not see this CloseSend call. + if _, err := ss.Client.StreamingOutputCall(ctx, &testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatal("Error calling StreamingOutputCall:", err) + } + if iStream.closeSendCount != 1 { + t.Fatalf("CloseSend called %v times on user interceptor stream, want 1 times", iStream.closeSendCount) + } +} diff --git a/tap/tap.go b/tap/tap.go new file mode 100644 index 0000000..07f0125 --- /dev/null +++ b/tap/tap.go @@ -0,0 +1,62 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package tap defines the function handles which are executed on the transport +// layer of gRPC-Go and related information. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +package tap + +import ( + "context" + + "google.golang.org/grpc/metadata" +) + +// Info defines the relevant information needed by the handles. +type Info struct { + // FullMethodName is the string of grpc method (in the format of + // /package.service/method). + FullMethodName string + + // Header contains the header metadata received. + Header metadata.MD + + // TODO: More to be added. +} + +// ServerInHandle defines the function which runs before a new stream is +// created on the server side. If it returns a non-nil error, the stream will +// not be created and an error will be returned to the client. If the error +// returned is a status error, that status code and message will be used, +// otherwise PermissionDenied will be the code and err.Error() will be the +// message. +// +// It's intended to be used in situations where you don't want to waste the +// resources to accept the new stream (e.g. rate-limiting). For other general +// usages, please use interceptors. +// +// Note that it is executed in the per-connection I/O goroutine(s) instead of +// per-RPC goroutine. Therefore, users should NOT have any +// blocking/time-consuming work in this handle. Otherwise all the RPCs would +// slow down. Also, for the same reason, this handle won't be called +// concurrently by gRPC. +type ServerInHandle func(ctx context.Context, info *Info) (context.Context, error) diff --git a/test/authority_test.go b/test/authority_test.go new file mode 100644 index 0000000..3749b59 --- /dev/null +++ b/test/authority_test.go @@ -0,0 +1,237 @@ +//go:build linux +// +build linux + +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "net" + "os" + "strings" + "sync" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func authorityChecker(ctx context.Context, expectedAuthority string) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Error(codes.InvalidArgument, "failed to parse metadata") + } + auths, ok := md[":authority"] + if !ok { + return nil, status.Error(codes.InvalidArgument, "no authority header") + } + if len(auths) != 1 { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("no authority header, auths = %v", auths)) + } + if auths[0] != expectedAuthority { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority)) + } + return &testpb.Empty{}, nil +} + +func runUnixTest(t *testing.T, address, target, expectedAuthority string, dialer func(context.Context, string) (net.Conn, error)) { + if !strings.HasPrefix(target, "unix-abstract:") { + if err := os.RemoveAll(address); err != nil { + t.Fatalf("Error removing socket file %v: %v\n", address, err) + } + } + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + return authorityChecker(ctx, expectedAuthority) + }, + Network: "unix", + Address: address, + Target: target, + } + opts := []grpc.DialOption{} + if dialer != nil { + opts = append(opts, grpc.WithContextDialer(dialer)) + } + if err := ss.Start(nil, opts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}) + if err != nil { + t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil", err) + } +} + +type authorityTest struct { + name string + address string + target string + authority string + dialTargetWant string +} + +var authorityTests = []authorityTest{ + { + name: "UnixRelative", + address: "sock.sock", + target: "unix:sock.sock", + authority: "localhost", + dialTargetWant: "unix:sock.sock", + }, + { + name: "UnixAbsolute", + address: "/tmp/sock.sock", + target: "unix:/tmp/sock.sock", + authority: "localhost", + dialTargetWant: "unix:///tmp/sock.sock", + }, + { + name: "UnixAbsoluteAlternate", + address: "/tmp/sock.sock", + target: "unix:///tmp/sock.sock", + authority: "localhost", + dialTargetWant: "unix:///tmp/sock.sock", + }, + { + name: "UnixPassthrough", + address: "/tmp/sock.sock", + target: "passthrough:///unix:///tmp/sock.sock", + authority: "unix:%2F%2F%2Ftmp%2Fsock.sock", + dialTargetWant: "unix:///tmp/sock.sock", + }, + { + name: "UnixAbstract", + address: "@abc efg", + target: "unix-abstract:abc efg", + authority: "localhost", + dialTargetWant: "unix:@abc efg", + }, +} + +// TestUnix does end to end tests with the various supported unix target +// formats, ensuring that the authority is set as expected. +func (s) TestUnix(t *testing.T) { + for _, test := range authorityTests { + t.Run(test.name, func(t *testing.T) { + runUnixTest(t, test.address, test.target, test.authority, nil) + }) + } +} + +// TestUnixCustomDialer does end to end tests with various supported unix target +// formats, ensuring that the target sent to the dialer does NOT have the +// "unix:" prefix stripped. +func (s) TestUnixCustomDialer(t *testing.T) { + for _, test := range authorityTests { + t.Run(test.name+"WithDialer", func(t *testing.T) { + dialer := func(ctx context.Context, address string) (net.Conn, error) { + if address != test.dialTargetWant { + return nil, fmt.Errorf("expected target %v in custom dialer, instead got %v", test.dialTargetWant, address) + } + address = address[len("unix:"):] + return (&net.Dialer{}).DialContext(ctx, "unix", address) + } + runUnixTest(t, test.address, test.target, test.authority, dialer) + }) + } +} + +// TestColonPortAuthority does an end to end test with the target for grpc.NewClient +// being ":[port]". Ensures authority is "localhost:[port]". +func (s) TestColonPortAuthority(t *testing.T) { + expectedAuthority := "" + var authorityMu sync.Mutex + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + authorityMu.Lock() + defer authorityMu.Unlock() + return authorityChecker(ctx, expectedAuthority) + }, + Network: "tcp", + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + _, port, err := net.SplitHostPort(ss.Address) + if err != nil { + t.Fatalf("Failed splitting host from post: %v", err) + } + authorityMu.Lock() + expectedAuthority = "localhost:" + port + authorityMu.Unlock() + // ss.Start dials the server, but we explicitly test with ":[port]" + // as the target. + cc, err := grpc.NewClient(":"+port, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", ss.Target, err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err = testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}) + if err != nil { + t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil", err) + } +} + +// TestAuthorityReplacedWithResolverAddress tests the scenario where the resolver +// returned address contains a ServerName override. The test verifies that the +// :authority header value sent to the server as part of the http/2 HEADERS frame +// is set to the value specified in the resolver returned address. +func (s) TestAuthorityReplacedWithResolverAddress(t *testing.T) { + const expectedAuthority = "test.server.name" + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + return authorityChecker(ctx, expectedAuthority) + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address, ServerName: expectedAuthority}}}) + cc, err := grpc.NewClient(r.Scheme()+":///whatever", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", ss.Address, err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() rpc failed: %v", err) + } +} diff --git a/test/balancer_switching_test.go b/test/balancer_switching_test.go new file mode 100644 index 0000000..db543e6 --- /dev/null +++ b/test/balancer_switching_test.go @@ -0,0 +1,530 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + grpclbstate "google.golang.org/grpc/balancer/grpclb/state" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils/fakegrpclb" + pfutil "google.golang.org/grpc/internal/testutils/pickfirst" + rrutil "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const ( + loadBalancedServiceName = "foo.bar.service" + loadBalancedServicePort = 443 + wantGRPCLBTraceDesc = `Channel switches to new LB policy "grpclb"` + wantRoundRobinTraceDesc = `Channel switches to new LB policy "round_robin"` + pickFirstServiceConfig = `{"loadBalancingConfig": [{"pick_first":{}}]}` + grpclbServiceConfig = `{"loadBalancingConfig": [{"grpclb":{}}]}` + + // This is the number of stub backends set up at the start of each test. The + // first backend is used for the "grpclb" policy and the rest are used for + // other LB policies to test balancer switching. + backendCount = 3 +) + +// stubBackendsToResolverAddrs converts from a set of stub server backends to +// resolver addresses. Useful when pushing addresses to the manual resolver. +func stubBackendsToResolverAddrs(backends []*stubserver.StubServer) []resolver.Address { + addrs := make([]resolver.Address, len(backends)) + for i, backend := range backends { + addrs[i] = resolver.Address{Addr: backend.Address} + } + return addrs +} + +// setupBackendsAndFakeGRPCLB sets up backendCount number of stub server +// backends and a fake grpclb server for tests which exercise balancer switch +// scenarios involving grpclb. +// +// The fake grpclb server always returns the first of the configured stub +// backends as backend addresses. So, the tests are free to use the other +// backends with other LB policies to verify balancer switching scenarios. +// +// Returns a cleanup function to be invoked by the caller. +func setupBackendsAndFakeGRPCLB(t *testing.T) ([]*stubserver.StubServer, *fakegrpclb.Server, func()) { + backends, backendsCleanup := startBackendsForBalancerSwitch(t) + + lbServer, err := fakegrpclb.NewServer(fakegrpclb.ServerParams{ + LoadBalancedServiceName: loadBalancedServiceName, + LoadBalancedServicePort: loadBalancedServicePort, + BackendAddresses: []string{backends[0].Address}, + }) + if err != nil { + t.Fatalf("failed to create fake grpclb server: %v", err) + } + go func() { + if err := lbServer.Serve(); err != nil { + t.Errorf("fake grpclb Serve() failed: %v", err) + } + }() + + return backends, lbServer, func() { + backendsCleanup() + lbServer.Stop() + } +} + +// startBackendsForBalancerSwitch spins up a bunch of stub server backends +// exposing the TestService. Returns a cleanup function to be invoked by the +// caller. +func startBackendsForBalancerSwitch(t *testing.T) ([]*stubserver.StubServer, func()) { + t.Helper() + + backends := make([]*stubserver.StubServer, backendCount) + for i := 0; i < backendCount; i++ { + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + backends[i] = backend + } + return backends, func() { + for _, b := range backends { + b.Stop() + } + } +} + +// TestBalancerSwitch_Basic tests the basic scenario of switching from one LB +// policy to another, as specified in the service config. +func (s) TestBalancerSwitch_Basic(t *testing.T) { + backends, cleanup := startBackendsForBalancerSwitch(t) + defer cleanup() + addrs := stubBackendsToResolverAddrs(backends) + + r := manual.NewBuilderWithScheme("whatever") + + r.InitialState(resolver.State{Addresses: addrs}) + + cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + // Push a resolver update without an LB policy in the service config. The + // channel should pick the default LB policy, which is pick_first. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Push a resolver update with a service config specifying "round_robin". + r.UpdateState(resolver.State{ + Addresses: addrs, + ServiceConfig: parseServiceConfig(t, r, rrServiceConfig), + }) + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil { + t.Fatal(err) + } + + // Push another resolver update with a service config specifying "pick_first". + r.UpdateState(resolver.State{ + Addresses: addrs, + ServiceConfig: parseServiceConfig(t, r, pickFirstServiceConfig), + }) + if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } +} + +// TestBalancerSwitch_grpclbToPickFirst tests the scenario where the channel +// starts off "grpclb", switches to "pick_first" and back. +func (s) TestBalancerSwitch_grpclbToPickFirst(t *testing.T) { + backends, lbServer, cleanup := setupBackendsAndFakeGRPCLB(t) + defer cleanup() + + addrs := stubBackendsToResolverAddrs(backends) + r := manual.NewBuilderWithScheme("whatever") + target := fmt.Sprintf("%s:///%s", r.Scheme(), loadBalancedServiceName) + cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + + // Push a resolver update with a GRPCLB service config and a single address + // pointing to the grpclb server we created above. This will cause the + // channel to switch to the "grpclb" balancer, which returns a single + // backend address. + grpclbConfig := parseServiceConfig(t, r, grpclbServiceConfig) + state := resolver.State{ServiceConfig: grpclbConfig} + r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: lbServer.Address()}}})) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[0:1]); err != nil { + t.Fatal(err) + } + + // Push a resolver update containing a non-existent grpclb server address. + // This should not lead to a balancer switch. + const nonExistentServer = "non-existent-grpclb-server-address" + r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: nonExistentServer}}})) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil { + t.Fatal(err) + } + + // Push a resolver update containing no grpclb server address. This should + // lead to the channel using the default LB policy which is pick_first. The + // list of addresses pushed as part of this update is different from the one + // returned by the "grpclb" balancer. So, we should see RPCs going to the + // newly configured backends, as part of the balancer switch. + emptyConfig := parseServiceConfig(t, r, `{}`) + r.UpdateState(resolver.State{Addresses: addrs[1:], ServiceConfig: emptyConfig}) + if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// TestBalancerSwitch_pickFirstToGRPCLB tests the scenario where the channel +// starts off with "pick_first", switches to "grpclb" and back. +func (s) TestBalancerSwitch_pickFirstToGRPCLB(t *testing.T) { + backends, lbServer, cleanup := setupBackendsAndFakeGRPCLB(t) + defer cleanup() + + addrs := stubBackendsToResolverAddrs(backends) + r := manual.NewBuilderWithScheme("whatever") + target := fmt.Sprintf("%s:///%s", r.Scheme(), loadBalancedServiceName) + cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + // Set an empty initial resolver state. This should lead to the channel + // using the default LB policy which is pick_first. + r.InitialState(resolver.State{Addresses: addrs[1:]}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + // Push a resolver update with no service config and a single address pointing + // to the grpclb server we created above. This will cause the channel to + // switch to the "grpclb" balancer, which returns a single backend address. + grpclbConfig := parseServiceConfig(t, r, grpclbServiceConfig) + state := resolver.State{ServiceConfig: grpclbConfig} + r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: lbServer.Address()}}})) + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil { + t.Fatal(err) + } + + // Push a resolver update containing a non-existent grpclb server address. + // This should not lead to a balancer switch. + r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: "nonExistentServer"}}})) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil { + t.Fatal(err) + } + + // Switch to "pick_first" again by sending no grpclb server addresses. + emptyConfig := parseServiceConfig(t, r, `{}`) + r.UpdateState(resolver.State{Addresses: addrs[1:], ServiceConfig: emptyConfig}) + if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// TestBalancerSwitch_RoundRobinToGRPCLB tests the scenario where the channel +// starts off with "round_robin", switches to "grpclb" and back. +// +// Note that this test uses the deprecated `loadBalancingPolicy` field in the +// service config. +func (s) TestBalancerSwitch_RoundRobinToGRPCLB(t *testing.T) { + backends, lbServer, cleanup := setupBackendsAndFakeGRPCLB(t) + defer cleanup() + + addrs := stubBackendsToResolverAddrs(backends) + r := manual.NewBuilderWithScheme("whatever") + target := fmt.Sprintf("%s:///%s", r.Scheme(), loadBalancedServiceName) + cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + // Note the use of the deprecated `loadBalancingPolicy` field here instead + // of the now recommended `loadBalancingConfig` field. The logic in the + // ClientConn which decides which balancer to switch to looks at the + // following places in the given order of preference: + // - `loadBalancingConfig` field + // - addresses of type grpclb + // - `loadBalancingPolicy` field + // If we use the `loadBalancingPolicy` field, the switch to "grpclb" later on + // in the test will not happen as the ClientConn will continue to use the LB + // policy received in the first update. + scpr := parseServiceConfig(t, r, rrServiceConfig) + + // Push a resolver update with the service config specifying "round_robin". + r.UpdateState(resolver.State{Addresses: addrs[1:], ServiceConfig: scpr}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil { + t.Fatal(err) + } + + // Push a resolver update with grpclb and a single balancer address + // pointing to the grpclb server we created above. This will cause the + // channel to switch to the "grpclb" balancer, which returns a single + // backend address. + grpclbConfig := parseServiceConfig(t, r, grpclbServiceConfig) + state := resolver.State{ServiceConfig: grpclbConfig} + r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: lbServer.Address()}}})) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil { + t.Fatal(err) + } + + // Switch back to "round_robin". + r.UpdateState(resolver.State{Addresses: addrs[1:], ServiceConfig: scpr}) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil { + t.Fatal(err) + } +} + +// TestBalancerSwitch_grpclbNotRegistered tests the scenario where the grpclb +// balancer is not registered. Verifies that the ClientConn falls back to the +// default LB policy or the LB policy specified in the service config, and that +// addresses of type "grpclb" are filtered out. +func (s) TestBalancerSwitch_grpclbNotRegistered(t *testing.T) { + // Unregister the grpclb balancer builder for the duration of this test. + grpclbBuilder := balancer.Get("grpclb") + internal.BalancerUnregister(grpclbBuilder.Name()) + defer balancer.Register(grpclbBuilder) + + backends, cleanup := startBackendsForBalancerSwitch(t) + defer cleanup() + addrs := stubBackendsToResolverAddrs(backends) + + r := manual.NewBuilderWithScheme("whatever") + cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + + // Push a resolver update which contains a bunch of stub server backends and a + // grpclb server address. The latter should get the ClientConn to try and + // apply the grpclb policy. But since grpclb is not registered, it should + // fallback to the default LB policy which is pick_first. The ClientConn is + // also expected to filter out the grpclb address when sending the addresses + // list for pick_first. + grpclbAddr := []resolver.Address{{Addr: "non-existent-grpclb-server-address"}} + grpclbConfig := parseServiceConfig(t, r, `{"loadBalancingPolicy": "grpclb"}`) + state := resolver.State{ServiceConfig: grpclbConfig, Addresses: addrs} + r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: grpclbAddr})) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatalf("Pick_first backend readiness check failed: %v", err) + } + + // Push a resolver update with the same addresses, but with a service config + // specifying "round_robin". The ClientConn is expected to filter out the + // grpclb address when sending the addresses list to round_robin. + r.UpdateState(resolver.State{ + Addresses: addrs, + ServiceConfig: parseServiceConfig(t, r, rrServiceConfig), + }) + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil { + t.Fatalf("Round robin RPCs failed: %v", err) + } +} + +// TestBalancerSwitch_OldBalancerCallsShutdownInClose tests the scenario where +// the balancer being switched out calls Shutdown() in its Close() +// method. Verifies that this sequence of calls doesn't lead to a deadlock. +func (s) TestBalancerSwitch_OldBalancerCallsShutdownInClose(t *testing.T) { + // Register a stub balancer which calls Shutdown() from its Close(). + scChan := make(chan balancer.SubConn, 1) + uccsCalled := make(chan struct{}, 1) + stub.Register(t.Name(), stub.BalancerFuncs{ + UpdateClientConnState: func(data *stub.BalancerData, ccs balancer.ClientConnState) error { + sc, err := data.ClientConn.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{}) + if err != nil { + t.Errorf("failed to create subConn: %v", err) + } + scChan <- sc + close(uccsCalled) + return nil + }, + Close: func(*stub.BalancerData) { + (<-scChan).Shutdown() + }, + }) + + r := manual.NewBuilderWithScheme("whatever") + cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + defer cc.Close() + + // Push a resolver update specifying our stub balancer as the LB policy. + scpr := parseServiceConfig(t, r, fmt.Sprintf(`{"loadBalancingPolicy": "%v"}`, t.Name())) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: "dummy-address"}}, + ServiceConfig: scpr, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout waiting for UpdateClientConnState to be called: %v", ctx.Err()) + case <-uccsCalled: + } + + // The following service config update will switch balancer from our stub + // balancer to pick_first. The former will be closed, which will call + // sc.Shutdown() inline. + // + // This is to make sure the sc.Shutdown() from Close() doesn't cause a + // deadlock (e.g. trying to grab a mutex while it's already locked). + // + // Do it in a goroutine so this test will fail with a helpful message + // (though the goroutine will still leak). + done := make(chan struct{}) + go func() { + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: "dummy-address"}}, + ServiceConfig: parseServiceConfig(t, r, pickFirstServiceConfig), + }) + close(done) + }() + + select { + case <-ctx.Done(): + t.Fatalf("timeout waiting for resolver.UpdateState to finish: %v", ctx.Err()) + case <-done: + } +} + +// TestBalancerSwitch_Graceful tests the graceful switching of LB policies. It +// starts off by configuring "round_robin" on the channel and ensures that RPCs +// are successful. Then, it switches to a stub balancer which does not report a +// picker until instructed by the test do to so. At this point, the test +// verifies that RPCs are still successful using the old balancer. Then the test +// asks the new balancer to report a healthy picker and the test verifies that +// the RPCs get routed using the picker reported by the new balancer. +func (s) TestBalancerSwitch_Graceful(t *testing.T) { + backends, cleanup := startBackendsForBalancerSwitch(t) + defer cleanup() + addrs := stubBackendsToResolverAddrs(backends) + + r := manual.NewBuilderWithScheme("whatever") + cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + // Push a resolver update with the service config specifying "round_robin". + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + r.UpdateState(resolver.State{ + Addresses: addrs[1:], + ServiceConfig: parseServiceConfig(t, r, rrServiceConfig), + }) + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil { + t.Fatal(err) + } + + // Register a stub balancer which uses a "pick_first" balancer underneath and + // signals on a channel when it receives ClientConn updates. But it does not + // forward the ccUpdate to the underlying "pick_first" balancer until the test + // asks it to do so. This allows us to test the graceful switch functionality. + // Until the test asks the stub balancer to forward the ccUpdate, RPCs should + // get routed to the old balancer. And once the test gives the go ahead, RPCs + // should get routed to the new balancer. + ccUpdateCh := make(chan struct{}) + waitToProceed := make(chan struct{}) + stub.Register(t.Name(), stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + pf := balancer.Get(pickfirst.Name) + bd.ChildBalancer = pf.Build(bd.ClientConn, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + close(ccUpdateCh) + go func() { + <-waitToProceed + bd.ChildBalancer.UpdateClientConnState(ccs) + }() + return nil + }, + }) + + // Push a resolver update with the service config specifying our stub + // balancer. We should see a trace event for this balancer switch. But RPCs + // should still be routed to the old balancer since our stub balancer does not + // report a ready picker until we ask it to do so. + r.UpdateState(resolver.State{ + Addresses: addrs[:1], + ServiceConfig: r.CC().ParseServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%v": {}}]}`, t.Name())), + }) + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for a ClientConnState update on the new balancer") + case <-ccUpdateCh: + } + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil { + t.Fatalf("RPCs routed to old balancer failed: %v", err) + } + + // Ask our stub balancer to forward the earlier received ccUpdate to the + // underlying "pick_first" balancer which will result in a healthy picker + // being reported to the channel. RPCs should start using the new balancer. + close(waitToProceed) + if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatalf("RPCs routed to new balancer failed: %v", err) + } +} diff --git a/test/balancer_test.go b/test/balancer_test.go new file mode 100644 index 0000000..2131b0f --- /dev/null +++ b/test/balancer_test.go @@ -0,0 +1,1429 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "errors" + "fmt" + "net" + "reflect" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + + "google.golang.org/grpc" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/balancerload" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpcutil" + imetadata "google.golang.org/grpc/internal/metadata" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const testBalancerName = "testbalancer" + +// testBalancer creates one subconn with the first address from resolved +// addresses. +// +// It's used to test whether options for NewSubConn are applied correctly. +type testBalancer struct { + cc balancer.ClientConn + sc balancer.SubConn + + newSubConnOptions balancer.NewSubConnOptions + pickInfos []balancer.PickInfo + pickExtraMDs []metadata.MD + doneInfo []balancer.DoneInfo +} + +func (b *testBalancer) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { + b.cc = cc + return b +} + +func (*testBalancer) Name() string { + return testBalancerName +} + +func (*testBalancer) ResolverError(error) { + panic("not implemented") +} + +func (b *testBalancer) UpdateClientConnState(state balancer.ClientConnState) error { + // Only create a subconn at the first time. + if b.sc == nil { + var err error + b.newSubConnOptions.StateListener = b.updateSubConnState + b.sc, err = b.cc.NewSubConn(state.ResolverState.Addresses, b.newSubConnOptions) + if err != nil { + logger.Errorf("testBalancer: failed to NewSubConn: %v", err) + return nil + } + b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: &picker{err: balancer.ErrNoSubConnAvailable, bal: b}}) + b.sc.Connect() + } + return nil +} + +func (b *testBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) { + panic(fmt.Sprintf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, s)) +} + +func (b *testBalancer) updateSubConnState(s balancer.SubConnState) { + logger.Infof("testBalancer: updateSubConnState: %v", s) + + switch s.ConnectivityState { + case connectivity.Ready: + b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{bal: b}}) + case connectivity.Idle: + b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{bal: b, idle: true}}) + case connectivity.Connecting: + b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{err: balancer.ErrNoSubConnAvailable, bal: b}}) + case connectivity.TransientFailure: + b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{err: balancer.ErrTransientFailure, bal: b}}) + } +} + +func (b *testBalancer) Close() {} + +func (b *testBalancer) ExitIdle() {} + +type picker struct { + err error + bal *testBalancer + idle bool +} + +func (p *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + if p.err != nil { + return balancer.PickResult{}, p.err + } + if p.idle { + p.bal.sc.Connect() + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + } + extraMD, _ := grpcutil.ExtraMetadata(info.Ctx) + info.Ctx = nil // Do not validate context. + p.bal.pickInfos = append(p.bal.pickInfos, info) + p.bal.pickExtraMDs = append(p.bal.pickExtraMDs, extraMD) + return balancer.PickResult{SubConn: p.bal.sc, Done: func(d balancer.DoneInfo) { p.bal.doneInfo = append(p.bal.doneInfo, d) }}, nil +} + +func (s) TestCredsBundleFromBalancer(t *testing.T) { + balancer.Register(&testBalancer{ + newSubConnOptions: balancer.NewSubConnOptions{ + CredsBundle: &testCredsBundle{}, + }, + }) + te := newTest(t, env{name: "creds-bundle", network: "tcp", balancer: ""}) + te.tapHandle = authHandle + te.customDialOptions = []grpc.DialOption{ + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, testBalancerName)), + grpc.WithAuthority("x.test.example.com"), + } + creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("Failed to generate credentials %v", err) + } + te.customServerOptions = []grpc.ServerOption{ + grpc.Creds(creds), + } + te.startServer(&testServer{}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("Test failed. Reason: %v", err) + } +} + +func (s) TestPickExtraMetadata(t *testing.T) { + for _, e := range listTestEnv() { + testPickExtraMetadata(t, e) + } +} + +func testPickExtraMetadata(t *testing.T, e env) { + te := newTest(t, e) + b := &testBalancer{} + balancer.Register(b) + const ( + testUserAgent = "test-user-agent" + testSubContentType = "proto" + ) + + te.customDialOptions = []grpc.DialOption{ + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, testBalancerName)), + grpc.WithUserAgent(testUserAgent), + } + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + // Trigger the extra-metadata-adding code path. + defer func(old string) { internal.GRPCResolverSchemeExtraMetadata = old }(internal.GRPCResolverSchemeExtraMetadata) + internal.GRPCResolverSchemeExtraMetadata = "passthrough" + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %v", err, nil) + } + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.CallContentSubtype(testSubContentType)); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %v", err, nil) + } + + want := []metadata.MD{ + // First RPC doesn't have sub-content-type. + {"content-type": []string{"application/grpc"}}, + // Second RPC has sub-content-type "proto". + {"content-type": []string{"application/grpc+proto"}}, + } + if diff := cmp.Diff(want, b.pickExtraMDs); diff != "" { + t.Fatalf("unexpected diff in metadata (-want, +got): %s", diff) + } +} + +func (s) TestDoneInfo(t *testing.T) { + for _, e := range listTestEnv() { + testDoneInfo(t, e) + } +} + +func testDoneInfo(t *testing.T, e env) { + te := newTest(t, e) + b := &testBalancer{} + balancer.Register(b) + te.customDialOptions = []grpc.DialOption{ + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, testBalancerName)), + } + te.userAgent = failAppUA + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantErr := detailedError + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); !testutils.StatusErrEqual(err, wantErr) { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %v", status.Convert(err).Proto(), status.Convert(wantErr).Proto()) + } + if _, err := tc.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("TestService.UnaryCall(%v, _, _, _) = _, %v; want _, ", ctx, err) + } + + if len(b.doneInfo) < 1 || !testutils.StatusErrEqual(b.doneInfo[0].Err, wantErr) { + t.Fatalf("b.doneInfo = %v; want b.doneInfo[0].Err = %v", b.doneInfo, wantErr) + } + if len(b.doneInfo) < 2 || !reflect.DeepEqual(b.doneInfo[1].Trailer, testTrailerMetadata) { + t.Fatalf("b.doneInfo = %v; want b.doneInfo[1].Trailer = %v", b.doneInfo, testTrailerMetadata) + } + if len(b.pickInfos) != len(b.doneInfo) { + t.Fatalf("Got %d picks, but %d doneInfo, want equal amount", len(b.pickInfos), len(b.doneInfo)) + } + // To test done() is always called, even if it's returned with a non-Ready + // SubConn. + // + // Stop server and at the same time send RPCs. There are chances that picker + // is not updated in time, causing a non-Ready SubConn to be returned. + finished := make(chan struct{}) + go func() { + for i := 0; i < 20; i++ { + tc.UnaryCall(ctx, &testpb.SimpleRequest{}) + } + close(finished) + }() + te.srv.Stop() + <-finished + if len(b.pickInfos) != len(b.doneInfo) { + t.Fatalf("Got %d picks, %d doneInfo, want equal amount", len(b.pickInfos), len(b.doneInfo)) + } +} + +const loadMDKey = "X-Endpoint-Load-Metrics-Bin" + +type testLoadParser struct{} + +func (*testLoadParser) Parse(md metadata.MD) any { + vs := md.Get(loadMDKey) + if len(vs) == 0 { + return nil + } + return vs[0] +} + +func init() { + balancerload.SetParser(&testLoadParser{}) +} + +func (s) TestDoneLoads(t *testing.T) { + testDoneLoads(t) +} + +func testDoneLoads(t *testing.T) { + b := &testBalancer{} + balancer.Register(b) + + const testLoad = "test-load-,-should-be-orca" + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + grpc.SetTrailer(ctx, metadata.Pairs(loadMDKey, testLoad)) + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil, grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, testBalancerName))); err != nil { + t.Fatalf("error starting testing server: %v", err) + } + defer ss.Stop() + + tc := testgrpc.NewTestServiceClient(ss.CC) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %v", err, nil) + } + + piWant := []balancer.PickInfo{ + {FullMethodName: "/grpc.testing.TestService/EmptyCall"}, + } + if !reflect.DeepEqual(b.pickInfos, piWant) { + t.Fatalf("b.pickInfos = %v; want %v", b.pickInfos, piWant) + } + + if len(b.doneInfo) < 1 { + t.Fatalf("b.doneInfo = %v, want length 1", b.doneInfo) + } + gotLoad, _ := b.doneInfo[0].ServerLoad.(string) + if gotLoad != testLoad { + t.Fatalf("b.doneInfo[0].ServerLoad = %v; want = %v", b.doneInfo[0].ServerLoad, testLoad) + } +} + +type aiPicker struct { + result balancer.PickResult + err error +} + +func (aip *aiPicker) Pick(_ balancer.PickInfo) (balancer.PickResult, error) { + return aip.result, aip.err +} + +// attrTransportCreds is a transport credential implementation which stores +// Attributes from the ClientHandshakeInfo struct passed in the context locally +// for the test to inspect. +type attrTransportCreds struct { + credentials.TransportCredentials + attr *attributes.Attributes +} + +func (ac *attrTransportCreds) ClientHandshake(ctx context.Context, _ string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + ai := credentials.ClientHandshakeInfoFromContext(ctx) + ac.attr = ai.Attributes + return rawConn, nil, nil +} +func (ac *attrTransportCreds) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{} +} +func (ac *attrTransportCreds) Clone() credentials.TransportCredentials { + return nil +} + +// TestAddressAttributesInNewSubConn verifies that the Attributes passed from a +// balancer in the resolver.Address that is passes to NewSubConn reaches all the +// way to the ClientHandshake method of the credentials configured on the parent +// channel. +func (s) TestAddressAttributesInNewSubConn(t *testing.T) { + const ( + testAttrKey = "foo" + testAttrVal = "bar" + attrBalancerName = "attribute-balancer" + ) + + // Register a stub balancer which adds attributes to the first address that + // it receives and then calls NewSubConn on it. + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + addrs := ccs.ResolverState.Addresses + if len(addrs) == 0 { + return nil + } + + // Only use the first address. + attr := attributes.New(testAttrKey, testAttrVal) + addrs[0].Attributes = attr + var sc balancer.SubConn + sc, err := bd.ClientConn.NewSubConn([]resolver.Address{addrs[0]}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: state.ConnectivityState, Picker: &aiPicker{result: balancer.PickResult{SubConn: sc}, err: state.ConnectionError}}) + }, + }) + if err != nil { + return err + } + sc.Connect() + return nil + }, + } + stub.Register(attrBalancerName, bf) + t.Logf("Registered balancer %s...", attrBalancerName) + + r := manual.NewBuilderWithScheme("whatever") + t.Logf("Registered manual resolver with scheme %s...", r.Scheme()) + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + stub := &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + S: grpc.NewServer(), + } + stubserver.StartTestService(t, stub) + defer stub.S.Stop() + t.Logf("Started gRPC server at %s...", lis.Addr().String()) + + creds := &attrTransportCreds{} + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(creds), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, attrBalancerName)), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatal(err) + } + defer cc.Close() + tc := testgrpc.NewTestServiceClient(cc) + t.Log("Created a ClientConn...") + + // The first RPC should fail because there's no address. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err == nil || status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) + } + t.Log("Made an RPC which was expected to fail...") + + state := resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}} + r.UpdateState(state) + t.Logf("Pushing resolver state update: %v through the manual resolver", state) + + // The second RPC should succeed. + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() = _, %v, want _, ", err) + } + t.Log("Made an RPC which succeeded...") + + wantAttr := attributes.New(testAttrKey, testAttrVal) + if gotAttr := creds.attr; !cmp.Equal(gotAttr, wantAttr, cmp.AllowUnexported(attributes.Attributes{})) { + t.Fatalf("received attributes %v in creds, want %v", gotAttr, wantAttr) + } +} + +// TestMetadataInAddressAttributes verifies that the metadata added to +// address.Attributes will be sent with the RPCs. +func (s) TestMetadataInAddressAttributes(t *testing.T) { + const ( + testMDKey = "test-md" + testMDValue = "test-md-value" + mdBalancerName = "metadata-balancer" + ) + + // Register a stub balancer which adds metadata to the first address that it + // receives and then calls NewSubConn on it. + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + addrs := ccs.ResolverState.Addresses + if len(addrs) == 0 { + return nil + } + // Only use the first address. + var sc balancer.SubConn + sc, err := bd.ClientConn.NewSubConn([]resolver.Address{ + imetadata.Set(addrs[0], metadata.Pairs(testMDKey, testMDValue)), + }, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: state.ConnectivityState, Picker: &aiPicker{result: balancer.PickResult{SubConn: sc}, err: state.ConnectionError}}) + }, + }) + if err != nil { + return err + } + sc.Connect() + return nil + }, + } + stub.Register(mdBalancerName, bf) + t.Logf("Registered balancer %s...", mdBalancerName) + + testMDChan := make(chan []string, 1) + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if ok { + select { + case testMDChan <- md[testMDKey]: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil, grpc.WithDefaultServiceConfig( + fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, mdBalancerName), + )); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + // The RPC should succeed with the expected md. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() = _, %v, want _, ", err) + } + t.Log("Made an RPC which succeeded...") + + // The server should receive the test metadata. + md1 := <-testMDChan + if len(md1) == 0 || md1[0] != testMDValue { + t.Fatalf("got md: %v, want %v", md1, []string{testMDValue}) + } +} + +// TestServersSwap creates two servers and verifies the client switches between +// them when the name resolver reports the first and then the second. +func (s) TestServersSwap(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Initialize servers + reg := func(username string) (addr string, cleanup func()) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + + stub := &stubserver.StubServer{ + Listener: lis, + UnaryCallF: func(_ context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{Username: username}, nil + }, + S: grpc.NewServer(), + } + stubserver.StartTestService(t, stub) + return lis.Addr().String(), stub.S.Stop + } + const one = "1" + addr1, cleanup := reg(one) + defer cleanup() + const two = "2" + addr2, cleanup := reg(two) + defer cleanup() + + // Initialize client + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: addr1}}}) + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("Error creating client: %v", err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + + // Confirm we are connected to the first server + if res, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil || res.Username != one { + t.Fatalf("UnaryCall(_) = %v, %v; want {Username: %q}, nil", res, err, one) + } + + // Update resolver to report only the second server + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: addr2}}}) + + // Loop until new RPCs talk to server two. + for i := 0; i < 2000; i++ { + if res, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err) + } else if res.Username == two { + break // pass + } + time.Sleep(5 * time.Millisecond) + } +} + +func (s) TestWaitForReady(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Initialize server + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + const one = "1" + stub := &stubserver.StubServer{ + Listener: lis, + UnaryCallF: func(_ context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{Username: one}, nil + }, + S: grpc.NewServer(), + } + stubserver.StartTestService(t, stub) + defer stub.S.Stop() + + // Initialize client + r := manual.NewBuilderWithScheme("whatever") + + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("Error creating client: %v", err) + } + defer cc.Close() + cc.Connect() + client := testgrpc.NewTestServiceClient(cc) + + // Report an error so non-WFR RPCs will give up early. + r.CC().ReportError(errors.New("fake resolver error")) + + // Ensure the client is not connected to anything and fails non-WFR RPCs. + if res, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable { + t.Fatalf("UnaryCall(_) = %v, %v; want _, Code()=%v", res, err, codes.Unavailable) + } + + errChan := make(chan error, 1) + go func() { + if res, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.WaitForReady(true)); err != nil || res.Username != one { + errChan <- fmt.Errorf("UnaryCall(_) = %v, %v; want {Username: %q}, nil", res, err, one) + } + close(errChan) + }() + + select { + case err := <-errChan: + t.Errorf("unexpected receive from errChan before addresses provided") + t.Fatal(err.Error()) + case <-time.After(5 * time.Millisecond): + } + + // Resolve the server. The WFR RPC should unblock and use it. + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}) + + if err := <-errChan; err != nil { + t.Fatal(err.Error()) + } +} + +// authorityOverrideTransportCreds returns the configured authority value in its +// Info() method. +type authorityOverrideTransportCreds struct { + credentials.TransportCredentials + authorityOverride string +} + +func (ao *authorityOverrideTransportCreds) ClientHandshake(_ context.Context, _ string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + return rawConn, nil, nil +} +func (ao *authorityOverrideTransportCreds) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{ServerName: ao.authorityOverride} +} +func (ao *authorityOverrideTransportCreds) Clone() credentials.TransportCredentials { + return &authorityOverrideTransportCreds{authorityOverride: ao.authorityOverride} +} + +// TestAuthorityInBuildOptions tests that the Authority field in +// balancer.BuildOptions is setup correctly from gRPC. +func (s) TestAuthorityInBuildOptions(t *testing.T) { + const dialTarget = "test.server" + + tests := []struct { + name string + dopts []grpc.DialOption + wantAuthority string + }{ + { + name: "authority from dial target", + dopts: []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}, + wantAuthority: dialTarget, + }, + { + name: "authority from dial option", + dopts: []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithAuthority("authority-override"), + }, + wantAuthority: "authority-override", + }, + { + name: "authority from transport creds", + dopts: []grpc.DialOption{grpc.WithTransportCredentials(&authorityOverrideTransportCreds{authorityOverride: "authority-override-from-transport-creds"})}, + wantAuthority: "authority-override-from-transport-creds", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + authorityCh := make(chan string, 1) + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + select { + case authorityCh <- bd.BuildOptions.Authority: + default: + } + + addrs := ccs.ResolverState.Addresses + if len(addrs) == 0 { + return nil + } + + // Only use the first address. + var sc balancer.SubConn + sc, err := bd.ClientConn.NewSubConn([]resolver.Address{addrs[0]}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: state.ConnectivityState, Picker: &aiPicker{result: balancer.PickResult{SubConn: sc}, err: state.ConnectionError}}) + }, + }) + if err != nil { + return err + } + sc.Connect() + return nil + }, + } + balancerName := "stub-balancer-" + test.name + stub.Register(balancerName, bf) + t.Logf("Registered balancer %s...", balancerName) + + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatal(err) + } + + stub := &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + S: grpc.NewServer(), + } + stubserver.StartTestService(t, stub) + defer stub.S.Stop() + t.Logf("Started gRPC server at %s...", lis.Addr().String()) + + r := manual.NewBuilderWithScheme("whatever") + t.Logf("Registered manual resolver with scheme %s...", r.Scheme()) + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}) + + dopts := append([]grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, balancerName)), + }, test.dopts...) + cc, err := grpc.NewClient(r.Scheme()+":///"+dialTarget, dopts...) + if err != nil { + t.Fatal(err) + } + defer cc.Close() + tc := testgrpc.NewTestServiceClient(cc) + t.Log("Created a ClientConn...") + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() = _, %v, want _, ", err) + } + t.Log("Made an RPC which succeeded...") + + select { + case <-ctx.Done(): + t.Fatal("timeout when waiting for Authority in balancer.BuildOptions") + case gotAuthority := <-authorityCh: + if gotAuthority != test.wantAuthority { + t.Fatalf("Authority in balancer.BuildOptions is %s, want %s", gotAuthority, test.wantAuthority) + } + } + }) + } +} + +// testCCWrapper wraps a balancer.ClientConn and intercepts UpdateState and +// returns a custom picker which injects arbitrary metadata on a per-call basis. +type testCCWrapper struct { + balancer.ClientConn +} + +func (t *testCCWrapper) UpdateState(state balancer.State) { + state.Picker = &wrappedPicker{p: state.Picker} + t.ClientConn.UpdateState(state) +} + +const ( + metadataHeaderInjectedByBalancer = "metadata-header-injected-by-balancer" + metadataHeaderInjectedByApplication = "metadata-header-injected-by-application" + metadataValueInjectedByBalancer = "metadata-value-injected-by-balancer" + metadataValueInjectedByApplication = "metadata-value-injected-by-application" +) + +// wrappedPicker wraps the picker returned by the pick_first +type wrappedPicker struct { + p balancer.Picker +} + +func (wp *wrappedPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + res, err := wp.p.Pick(info) + if err != nil { + return balancer.PickResult{}, err + } + + if res.Metadata == nil { + res.Metadata = metadata.Pairs(metadataHeaderInjectedByBalancer, metadataValueInjectedByBalancer) + } else { + res.Metadata.Append(metadataHeaderInjectedByBalancer, metadataValueInjectedByBalancer) + } + return res, nil +} + +// TestMetadataInPickResult tests the scenario where an LB policy inject +// arbitrary metadata on a per-call basis and verifies that the injected +// metadata makes it all the way to the server RPC handler. +func (s) TestMetadataInPickResult(t *testing.T) { + t.Log("Starting test backend...") + mdChan := make(chan metadata.MD, 1) + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, _ := metadata.FromIncomingContext(ctx) + select { + case mdChan <- md: + case <-ctx.Done(): + return nil, ctx.Err() + } + return &testpb.Empty{}, nil + }, + } + if err := ss.StartServer(); err != nil { + t.Fatalf("Starting test backend: %v", err) + } + defer ss.Stop() + t.Logf("Started test backend at %q", ss.Address) + + // Register a test balancer that contains a pick_first balancer and forwards + // all calls from the ClientConn to it. For state updates from the + // pick_first balancer, it creates a custom picker which injects arbitrary + // metadata on a per-call basis. + stub.Register(t.Name(), stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + cc := &testCCWrapper{ClientConn: bd.ClientConn} + bd.ChildBalancer = balancer.Get(pickfirst.Name).Build(cc, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + }) + + t.Log("Creating ClientConn to test backend...") + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}}) + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, t.Name())), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient(): %v", err) + } + defer cc.Close() + tc := testgrpc.NewTestServiceClient(cc) + + t.Log("Making EmptyCall() RPC with custom metadata...") + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + md := metadata.Pairs(metadataHeaderInjectedByApplication, metadataValueInjectedByApplication) + ctx = metadata.NewOutgoingContext(ctx, md) + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() RPC: %v", err) + } + t.Log("EmptyCall() RPC succeeded") + + t.Log("Waiting for custom metadata to be received at the test backend...") + var gotMD metadata.MD + select { + case gotMD = <-mdChan: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for custom metadata to be received at the test backend") + } + + t.Log("Verifying custom metadata added by the client application is received at the test backend...") + wantMDVal := []string{metadataValueInjectedByApplication} + gotMDVal := gotMD.Get(metadataHeaderInjectedByApplication) + if !cmp.Equal(gotMDVal, wantMDVal) { + t.Fatalf("Mismatch in custom metadata received at test backend, got: %v, want %v", gotMDVal, wantMDVal) + } + + t.Log("Verifying custom metadata added by the LB policy is received at the test backend...") + wantMDVal = []string{metadataValueInjectedByBalancer} + gotMDVal = gotMD.Get(metadataHeaderInjectedByBalancer) + if !cmp.Equal(gotMDVal, wantMDVal) { + t.Fatalf("Mismatch in custom metadata received at test backend, got: %v, want %v", gotMDVal, wantMDVal) + } +} + +type invalidMetadataCCWrapper struct { + balancer.ClientConn +} + +func (t *invalidMetadataCCWrapper) UpdateState(state balancer.State) { + state.Picker = &invalidMetadataPicker{picker: state.Picker} + t.ClientConn.UpdateState(state) +} + +type invalidMetadataPicker struct { + picker balancer.Picker +} + +func (imp *invalidMetadataPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + res, err := imp.picker.Pick(info) + if err != nil { + return balancer.PickResult{}, err + } + res.Metadata = metadata.MD{"bad key": {"value"}} + return res, nil +} + +func (s) TestInvalidMetadataInPickResult(t *testing.T) { + ss := &stubserver.StubServer{EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }} + if err := ss.StartServer(); err != nil { + t.Fatalf("Starting test backend: %v", err) + } + defer ss.Stop() + + stub.Register(t.Name(), stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + cc := &invalidMetadataCCWrapper{ClientConn: bd.ClientConn} + bd.ChildBalancer = balancer.Get(pickfirst.Name).Build(cc, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + }) + + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}}) + cc, err := grpc.NewClient(r.Scheme()+":///test.server", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, t.Name())), + ) + if err != nil { + t.Fatalf("grpc.NewClient(): %v", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantErr := `header key "bad key" contains illegal characters not in [0-9a-z-_.]` + if _, err := testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Internal || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall() error = %v, want code %v and message containing %q", err, codes.Internal, wantErr) + } +} + +func (s) TestAddressMetadataValidation(t *testing.T) { + tests := []struct { + name string + md metadata.MD + wantErr string + }{ + { + name: "valid", + md: metadata.Pairs("valid-key", "value"), + }, + { + name: "invalid key", + md: metadata.Pairs("bad key", "value"), + wantErr: `header key "bad key" contains illegal characters not in [0-9a-z-_.]`, + }, + { + name: "invalid value", + md: metadata.Pairs("valid-key", "bad\x01value"), + wantErr: `header key "valid-key" contains value with non-printable ASCII characters`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testAddressMetadataValidation(t, test.md, test.wantErr) + }) + } +} + +func testAddressMetadataValidation(t *testing.T, addrMD metadata.MD, wantErr string) { + ss := &stubserver.StubServer{EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + if wantErr != "" { + return nil, status.Error(codes.Internal, "EmptyCall reached backend with invalid address metadata") + } + md, _ := metadata.FromIncomingContext(ctx) + if got := md.Get("valid-key"); !cmp.Equal(got, []string{"value"}) { + return nil, status.Errorf(codes.Internal, "metadata.Get(\"valid-key\") = %v, want [value]", got) + } + return &testpb.Empty{}, nil + }} + if err := ss.StartServer(); err != nil { + t.Fatalf("Starting test backend: %v", err) + } + defer ss.Stop() + + r := manual.NewBuilderWithScheme("whatever") + addr := imetadata.Set(resolver.Address{Addr: ss.Address}, addrMD) + r.InitialState(resolver.State{Addresses: []resolver.Address{addr}}) + cc, err := grpc.NewClient(r.Scheme()+":///test.server", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + ) + if err != nil { + t.Fatalf("grpc.NewClient(): %v", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err = testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}) + if wantErr == "" { + if err != nil { + t.Fatalf("EmptyCall() error = %v, want nil", err) + } + return + } + if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall() error = %v, want code %v and message containing %q", err, codes.Internal, wantErr) + } +} + +// TestSubConnShutdown confirms that the Shutdown method on subconns and +// RemoveSubConn method on ClientConn properly initiates subconn shutdown. +func (s) TestSubConnShutdown(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + testCases := []struct { + name string + shutdown func(cc balancer.ClientConn, sc balancer.SubConn) + }{{ + name: "ClientConn.RemoveSubConn", + shutdown: func(cc balancer.ClientConn, sc balancer.SubConn) { + cc.RemoveSubConn(sc) + }, + }, { + name: "SubConn.Shutdown", + shutdown: func(_ balancer.ClientConn, sc balancer.SubConn) { + sc.Shutdown() + }, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotShutdown := grpcsync.NewEvent() + + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + var sc balancer.SubConn + opts := balancer.NewSubConnOptions{ + StateListener: func(scs balancer.SubConnState) { + switch scs.ConnectivityState { + case connectivity.Connecting: + // Ignored. + case connectivity.Ready: + tc.shutdown(bd.ClientConn, sc) + case connectivity.Shutdown: + gotShutdown.Fire() + default: + t.Errorf("got unexpected state %q in listener", scs.ConnectivityState) + } + }, + } + sc, err := bd.ClientConn.NewSubConn(ccs.ResolverState.Addresses, opts) + if err != nil { + return err + } + sc.Connect() + // Report the state as READY to unblock ss.Start(), which waits for ready. + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Ready}) + return nil + }, + } + + testBalName := "shutdown-test-balancer-" + tc.name + stub.Register(testBalName, bf) + t.Logf("Registered balancer %s...", testBalName) + + ss := &stubserver.StubServer{} + if err := ss.Start(nil, grpc.WithDefaultServiceConfig( + fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, testBalName), + )); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + select { + case <-gotShutdown.Done(): + // Success + case <-ctx.Done(): + t.Fatalf("Timed out waiting for gotShutdown to be fired.") + } + }) + } +} + +type subConnStoringCCWrapper struct { + balancer.ClientConn + stateListener func(balancer.SubConnState) + scChan chan balancer.SubConn +} + +func (ccw *subConnStoringCCWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + if ccw.stateListener != nil { + origListener := opts.StateListener + opts.StateListener = func(scs balancer.SubConnState) { + ccw.stateListener(scs) + origListener(scs) + } + } + sc, err := ccw.ClientConn.NewSubConn(addrs, opts) + ccw.scChan <- sc + return sc, err +} + +// Test calls RegisterHealthListener on a SubConn to verify that expected health +// updates are sent only to the most recently registered listener. +func (s) TestSubConn_RegisterHealthListener(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + scChan := make(chan balancer.SubConn, 1) + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + cc := bd.ClientConn + ccw := &subConnStoringCCWrapper{ + ClientConn: cc, + scChan: scChan, + } + bd.ChildBalancer = balancer.Get(pickfirst.Name).Build(ccw, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + ExitIdle: func(bd *stub.BalancerData) { + bd.ChildBalancer.ExitIdle() + }, + } + + stub.Register(t.Name(), bf) + svcCfg := fmt.Sprintf(`{ "loadBalancingConfig": [{%q: {}}] }`, t.Name()) + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(svcCfg), + } + cc, err := grpc.NewClient(backend.Address, opts...) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", backend.Address, err) + + } + defer cc.Close() + + cc.Connect() + + var sc balancer.SubConn + select { + case sc = <-scChan: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for SubConn creation") + } + healthUpdateChan := make(chan balancer.SubConnState, 1) + + // Register listener while Ready and verify it gets a health update. + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + for i := 0; i < 2; i++ { + sc.RegisterHealthListener(func(scs balancer.SubConnState) { + healthUpdateChan <- scs + }) + select { + case scs := <-healthUpdateChan: + if scs.ConnectivityState != connectivity.Ready { + t.Fatalf("Received health update = %v, want = %v", scs.ConnectivityState, connectivity.Ready) + } + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for health update") + } + + // No further updates are expected. + select { + case scs := <-healthUpdateChan: + t.Fatalf("Received unexpected health update while channel is in state %v: %v", cc.GetState(), scs) + case <-time.After(defaultTestShortTimeout): + } + } + + // Make the SubConn enter IDLE and verify that health updates are recevied + // on registering a new listener. + backend.S.Stop() + backend.S = nil + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + if err := backend.StartServer(); err != nil { + t.Fatalf("Error while restarting the backend server: %v", err) + } + cc.Connect() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + sc.RegisterHealthListener(func(scs balancer.SubConnState) { + healthUpdateChan <- scs + }) + select { + case scs := <-healthUpdateChan: + if scs.ConnectivityState != connectivity.Ready { + t.Fatalf("Received health update = %v, want = %v", scs.ConnectivityState, connectivity.Ready) + } + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for health update") + } +} + +// Test calls RegisterHealthListener on a SubConn twice while handling the +// connectivity update. The test verifies that only the latest listener +// receives the health update. +func (s) TestSubConn_RegisterHealthListener_RegisterTwice(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + scChan := make(chan balancer.SubConn, 1) + readyUpdateResumeCh := make(chan struct{}) + readyUpdateReceivedCh := make(chan struct{}) + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + cc := bd.ClientConn + ccw := &subConnStoringCCWrapper{ + ClientConn: cc, + scChan: scChan, + stateListener: func(scs balancer.SubConnState) { + if scs.ConnectivityState != connectivity.Ready { + return + } + close(readyUpdateReceivedCh) + select { + case <-readyUpdateResumeCh: + case <-ctx.Done(): + t.Error("Context timed out waiting for update on ready channel") + } + }, + } + bd.ChildBalancer = balancer.Get(pickfirst.Name).Build(ccw, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + } + + stub.Register(t.Name(), bf) + svcCfg := fmt.Sprintf(`{ "loadBalancingConfig": [{%q: {}}] }`, t.Name()) + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(svcCfg), + } + cc, err := grpc.NewClient(backend.Address, opts...) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", backend.Address, err) + + } + defer cc.Close() + + cc.Connect() + + var sc balancer.SubConn + select { + case sc = <-scChan: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for SubConn creation") + } + + // Wait for the SubConn to enter READY. + select { + case <-readyUpdateReceivedCh: + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for SubConn to enter READY") + } + + healthChan1 := make(chan balancer.SubConnState, 1) + healthChan2 := make(chan balancer.SubConnState, 1) + + sc.RegisterHealthListener(func(scs balancer.SubConnState) { + healthChan1 <- scs + }) + sc.RegisterHealthListener(func(scs balancer.SubConnState) { + healthChan2 <- scs + }) + close(readyUpdateResumeCh) + + select { + case scs := <-healthChan2: + if scs.ConnectivityState != connectivity.Ready { + t.Fatalf("Received health update = %v, want = %v", scs.ConnectivityState, connectivity.Ready) + } + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for health update") + } + + // No updates should be received on the first listener. + select { + case scs := <-healthChan1: + t.Fatalf("Received unexpected health update on first listener: %v", scs) + case <-time.After(defaultTestShortTimeout): + } +} + +// Test calls RegisterHealthListener on a SubConn with a nil listener and +// verifies that the listener registered before the nil listener doesn't receive +// any further updates. +func (s) TestSubConn_RegisterHealthListener_NilListener(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + scChan := make(chan balancer.SubConn, 1) + readyUpdateResumeCh := make(chan struct{}) + readyUpdateReceivedCh := make(chan struct{}) + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + cc := bd.ClientConn + ccw := &subConnStoringCCWrapper{ + ClientConn: cc, + scChan: scChan, + stateListener: func(scs balancer.SubConnState) { + if scs.ConnectivityState != connectivity.Ready { + return + } + close(readyUpdateReceivedCh) + select { + case <-readyUpdateResumeCh: + case <-ctx.Done(): + t.Error("Context timed out waiting for update on ready channel") + } + }, + } + bd.ChildBalancer = balancer.Get(pickfirst.Name).Build(ccw, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + } + + stub.Register(t.Name(), bf) + svcCfg := fmt.Sprintf(`{ "loadBalancingConfig": [{%q: {}}] }`, t.Name()) + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(svcCfg), + } + cc, err := grpc.NewClient(backend.Address, opts...) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", backend.Address, err) + + } + defer cc.Close() + + cc.Connect() + + var sc balancer.SubConn + select { + case sc = <-scChan: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for SubConn creation") + } + + // Wait for the SubConn to enter READY. + select { + case <-readyUpdateReceivedCh: + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for SubConn to enter READY") + } + + healthChan := make(chan balancer.SubConnState, 1) + + sc.RegisterHealthListener(func(scs balancer.SubConnState) { + healthChan <- scs + }) + + // Registering a nil listener should invalidate the previously registered + // listener. + sc.RegisterHealthListener(nil) + close(readyUpdateResumeCh) + + // No updates should be received on the listener. + select { + case scs := <-healthChan: + t.Fatalf("Received unexpected health update on the listener: %v", scs) + case <-time.After(defaultTestShortTimeout): + } +} diff --git a/test/bufconn/bufconn.go b/test/bufconn/bufconn.go new file mode 100644 index 0000000..9f69a55 --- /dev/null +++ b/test/bufconn/bufconn.go @@ -0,0 +1,317 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package bufconn provides a net.Conn implemented by a buffer and related +// dialing and listening functionality. +package bufconn + +import ( + "context" + "fmt" + "io" + "net" + "sync" + "time" +) + +// Listener implements a net.Listener that creates local, buffered net.Conns +// via its Accept and Dial method. +type Listener struct { + mu sync.Mutex + sz int + ch chan net.Conn + done chan struct{} +} + +// Implementation of net.Error providing timeout +type netErrorTimeout struct { + error +} + +func (e netErrorTimeout) Timeout() bool { return true } +func (e netErrorTimeout) Temporary() bool { return false } + +var errClosed = fmt.Errorf("closed") +var errTimeout net.Error = netErrorTimeout{error: fmt.Errorf("i/o timeout")} + +// Listen returns a Listener that can only be contacted by its own Dialers and +// creates buffered connections between the two. +func Listen(sz int) *Listener { + return &Listener{sz: sz, ch: make(chan net.Conn), done: make(chan struct{})} +} + +// Accept blocks until Dial is called, then returns a net.Conn for the server +// half of the connection. +func (l *Listener) Accept() (net.Conn, error) { + select { + case <-l.done: + return nil, errClosed + case c := <-l.ch: + return c, nil + } +} + +// Close stops the listener. +func (l *Listener) Close() error { + l.mu.Lock() + defer l.mu.Unlock() + select { + case <-l.done: + // Already closed. + default: + close(l.done) + } + return nil +} + +// Addr reports the address of the listener. +func (l *Listener) Addr() net.Addr { return addr{} } + +// Dial creates an in-memory full-duplex network connection, unblocks Accept by +// providing it the server half of the connection, and returns the client half +// of the connection. +func (l *Listener) Dial() (net.Conn, error) { + return l.DialContext(context.Background()) +} + +// DialContext creates an in-memory full-duplex network connection, unblocks Accept by +// providing it the server half of the connection, and returns the client half +// of the connection. If ctx is Done, returns ctx.Err() +func (l *Listener) DialContext(ctx context.Context) (net.Conn, error) { + p1, p2 := newPipe(l.sz), newPipe(l.sz) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-l.done: + return nil, errClosed + case l.ch <- &conn{p1, p2}: + return &conn{p2, p1}, nil + } +} + +type pipe struct { + mu sync.Mutex + + // buf contains the data in the pipe. It is a ring buffer of fixed capacity, + // with r and w pointing to the offset to read and write, respectively. + // + // Data is read between [r, w) and written to [w, r), wrapping around the end + // of the slice if necessary. + // + // The buffer is empty if r == len(buf), otherwise if r == w, it is full. + // + // w and r are always in the range [0, cap(buf)) and [0, len(buf)]. + buf []byte + w, r int + + wwait sync.Cond + rwait sync.Cond + + // Indicate that a write/read timeout has occurred + wtimedout bool + rtimedout bool + + wtimer *time.Timer + rtimer *time.Timer + + closed bool + writeClosed bool +} + +func newPipe(sz int) *pipe { + p := &pipe{buf: make([]byte, 0, sz)} + p.wwait.L = &p.mu + p.rwait.L = &p.mu + + p.wtimer = time.AfterFunc(0, func() {}) + p.rtimer = time.AfterFunc(0, func() {}) + return p +} + +func (p *pipe) empty() bool { + return p.r == len(p.buf) +} + +func (p *pipe) full() bool { + return p.r < len(p.buf) && p.r == p.w +} + +func (p *pipe) Read(b []byte) (n int, err error) { + p.mu.Lock() + defer p.mu.Unlock() + // Block until p has data. + for { + if p.closed { + return 0, io.ErrClosedPipe + } + if !p.empty() { + break + } + if p.writeClosed { + return 0, io.EOF + } + if p.rtimedout { + return 0, errTimeout + } + + p.rwait.Wait() + } + wasFull := p.full() + + n = copy(b, p.buf[p.r:len(p.buf)]) + p.r += n + if p.r == cap(p.buf) { + p.r = 0 + p.buf = p.buf[:p.w] + } + + // Signal a blocked writer, if any + if wasFull { + p.wwait.Signal() + } + + return n, nil +} + +func (p *pipe) Write(b []byte) (n int, err error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { + return 0, io.ErrClosedPipe + } + for len(b) > 0 { + // Block until p is not full. + for { + if p.closed || p.writeClosed { + return 0, io.ErrClosedPipe + } + if !p.full() { + break + } + if p.wtimedout { + return 0, errTimeout + } + + p.wwait.Wait() + } + wasEmpty := p.empty() + + end := cap(p.buf) + if p.w < p.r { + end = p.r + } + x := copy(p.buf[p.w:end], b) + b = b[x:] + n += x + p.w += x + if p.w > len(p.buf) { + p.buf = p.buf[:p.w] + } + if p.w == cap(p.buf) { + p.w = 0 + } + + // Signal a blocked reader, if any. + if wasEmpty { + p.rwait.Signal() + } + } + return n, nil +} + +func (p *pipe) Close() error { + p.mu.Lock() + defer p.mu.Unlock() + p.closed = true + // Signal all blocked readers and writers to return an error. + p.rwait.Broadcast() + p.wwait.Broadcast() + return nil +} + +func (p *pipe) closeWrite() error { + p.mu.Lock() + defer p.mu.Unlock() + p.writeClosed = true + // Signal all blocked readers and writers to return an error. + p.rwait.Broadcast() + p.wwait.Broadcast() + return nil +} + +type conn struct { + io.Reader + io.Writer +} + +func (c *conn) Close() error { + err1 := c.Reader.(*pipe).Close() + err2 := c.Writer.(*pipe).closeWrite() + if err1 != nil { + return err1 + } + return err2 +} + +func (c *conn) SetDeadline(t time.Time) error { + c.SetReadDeadline(t) + c.SetWriteDeadline(t) + return nil +} + +func (c *conn) SetReadDeadline(t time.Time) error { + p := c.Reader.(*pipe) + p.mu.Lock() + defer p.mu.Unlock() + p.rtimer.Stop() + p.rtimedout = false + if !t.IsZero() { + p.rtimer = time.AfterFunc(time.Until(t), func() { + p.mu.Lock() + defer p.mu.Unlock() + p.rtimedout = true + p.rwait.Broadcast() + }) + } + return nil +} + +func (c *conn) SetWriteDeadline(t time.Time) error { + p := c.Writer.(*pipe) + p.mu.Lock() + defer p.mu.Unlock() + p.wtimer.Stop() + p.wtimedout = false + if !t.IsZero() { + p.wtimer = time.AfterFunc(time.Until(t), func() { + p.mu.Lock() + defer p.mu.Unlock() + p.wtimedout = true + p.wwait.Broadcast() + }) + } + return nil +} + +func (*conn) LocalAddr() net.Addr { return addr{} } +func (*conn) RemoteAddr() net.Addr { return addr{} } + +type addr struct{} + +func (addr) Network() string { return "bufconn" } +func (addr) String() string { return "bufconn" } diff --git a/test/bufconn/bufconn_test.go b/test/bufconn/bufconn_test.go new file mode 100644 index 0000000..7a98661 --- /dev/null +++ b/test/bufconn/bufconn_test.go @@ -0,0 +1,328 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package bufconn + +import ( + "fmt" + "io" + "net" + "reflect" + "testing" + "time" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func testRW(r io.Reader, w io.Writer) error { + for i := 0; i < 20; i++ { + d := make([]byte, i) + for j := 0; j < i; j++ { + d[j] = byte(i - j) + } + var rn int + var rerr error + b := make([]byte, i) + done := make(chan struct{}) + go func() { + for rn < len(b) && rerr == nil { + var x int + x, rerr = r.Read(b[rn:]) + rn += x + } + close(done) + }() + wn, werr := w.Write(d) + if wn != i || werr != nil { + return fmt.Errorf("%v: w.Write(%v) = %v, %v; want %v, nil", i, d, wn, werr, i) + } + select { + case <-done: + case <-time.After(500 * time.Millisecond): + return fmt.Errorf("%v: r.Read never returned", i) + } + if rn != i || rerr != nil { + return fmt.Errorf("%v: r.Read = %v, %v; want %v, nil", i, rn, rerr, i) + } + if !reflect.DeepEqual(b, d) { + return fmt.Errorf("%v: r.Read read %v; want %v", i, b, d) + } + } + return nil +} + +func (s) TestPipe(t *testing.T) { + p := newPipe(10) + if err := testRW(p, p); err != nil { + t.Fatal(err) + } +} + +func (s) TestPipeClose(t *testing.T) { + p := newPipe(10) + p.Close() + if _, err := p.Write(nil); err != io.ErrClosedPipe { + t.Fatalf("p.Write = _, %v; want _, %v", err, io.ErrClosedPipe) + } + if _, err := p.Read(nil); err != io.ErrClosedPipe { + t.Fatalf("p.Read = _, %v; want _, %v", err, io.ErrClosedPipe) + } +} + +func (s) TestConn(t *testing.T) { + p1, p2 := newPipe(10), newPipe(10) + c1, c2 := &conn{p1, p2}, &conn{p2, p1} + + if err := testRW(c1, c2); err != nil { + t.Fatal(err) + } + if err := testRW(c2, c1); err != nil { + t.Fatal(err) + } +} + +func (s) TestConnCloseWithData(t *testing.T) { + lis := Listen(7) + errChan := make(chan error, 1) + var lisConn net.Conn + go func() { + var err error + if lisConn, err = lis.Accept(); err != nil { + errChan <- err + } + close(errChan) + }() + dialConn, err := lis.Dial() + if err != nil { + t.Fatalf("Dial error: %v", err) + } + if err := <-errChan; err != nil { + t.Fatalf("Listen error: %v", err) + } + + // Write some data on both sides of the connection. + n, err := dialConn.Write([]byte("hello")) + if n != 5 || err != nil { + t.Fatalf("dialConn.Write([]byte{\"hello\"}) = %v, %v; want 5, ", n, err) + } + n, err = lisConn.Write([]byte("hello")) + if n != 5 || err != nil { + t.Fatalf("lisConn.Write([]byte{\"hello\"}) = %v, %v; want 5, ", n, err) + } + + // Close dial-side; writes from either side should fail. + dialConn.Close() + if _, err := lisConn.Write([]byte("hello")); err != io.ErrClosedPipe { + t.Fatalf("lisConn.Write() = _, ; want _, ") + } + if _, err := dialConn.Write([]byte("hello")); err != io.ErrClosedPipe { + t.Fatalf("dialConn.Write() = _, ; want _, ") + } + + // Read from both sides; reads on lisConn should work, but dialConn should + // fail. + buf := make([]byte, 6) + if _, err := dialConn.Read(buf); err != io.ErrClosedPipe { + t.Fatalf("dialConn.Read(buf) = %v, %v; want _, io.ErrClosedPipe", n, err) + } + n, err = lisConn.Read(buf) + if n != 5 || err != nil { + t.Fatalf("lisConn.Read(buf) = %v, %v; want 5, ", n, err) + } +} + +func (s) TestListener(t *testing.T) { + l := Listen(7) + var s net.Conn + var serr error + done := make(chan struct{}) + go func() { + s, serr = l.Accept() + close(done) + }() + c, cerr := l.Dial() + <-done + if cerr != nil || serr != nil { + t.Fatalf("cerr = %v, serr = %v; want nil, nil", cerr, serr) + } + if err := testRW(c, s); err != nil { + t.Fatal(err) + } + if err := testRW(s, c); err != nil { + t.Fatal(err) + } +} + +func (s) TestCloseWhileDialing(t *testing.T) { + l := Listen(7) + var c net.Conn + var err error + done := make(chan struct{}) + go func() { + c, err = l.Dial() + close(done) + }() + l.Close() + <-done + if c != nil || err != errClosed { + t.Fatalf("c, err = %v, %v; want nil, %v", c, err, errClosed) + } +} + +func (s) TestCloseWhileAccepting(t *testing.T) { + l := Listen(7) + var c net.Conn + var err error + done := make(chan struct{}) + go func() { + c, err = l.Accept() + close(done) + }() + l.Close() + <-done + if c != nil || err != errClosed { + t.Fatalf("c, err = %v, %v; want nil, %v", c, err, errClosed) + } +} + +func (s) TestDeadline(t *testing.T) { + sig := make(chan error, 2) + blockingWrite := func(conn net.Conn) { + _, err := conn.Write([]byte("0123456789")) + sig <- err + } + + blockingRead := func(conn net.Conn) { + _, err := conn.Read(make([]byte, 10)) + sig <- err + } + + p1, p2 := newPipe(5), newPipe(5) + c1, c2 := &conn{p1, p1}, &conn{p2, p2} + defer c1.Close() + defer c2.Close() + + // Test with deadline + c1.SetWriteDeadline(time.Now()) + + go blockingWrite(c1) + select { + case <-time.After(100 * time.Millisecond): + t.Fatalf("Write timeout timed out, c = %v", c1) + case err := <-sig: + if netErr, ok := err.(net.Error); ok { + if !netErr.Timeout() { + t.Fatalf("Write returned unexpected error, c = %v, err = %v", c1, netErr) + } + } else { + t.Fatalf("Write returned unexpected error, c = %v, err = %v", c1, err) + } + } + + c2.SetReadDeadline(time.Now()) + + go blockingRead(c2) + select { + case <-time.After(100 * time.Millisecond): + t.Fatalf("Read timeout timed out, c = %v", c2) + case err := <-sig: + if netErr, ok := err.(net.Error); ok { + if !netErr.Timeout() { + t.Fatalf("Read returned unexpected error, c = %v, err = %v", c2, netErr) + } + } else { + t.Fatalf("Read returned unexpected error, c = %v, err = %v", c2, err) + } + } + + // Test timing out pending reads/writes + c1.SetWriteDeadline(time.Time{}) + c2.SetReadDeadline(time.Time{}) + + go blockingWrite(c1) + select { + case <-time.After(100 * time.Millisecond): + case err := <-sig: + t.Fatalf("Write returned before timeout, err = %v", err) + } + + c1.SetWriteDeadline(time.Now()) + select { + case <-time.After(100 * time.Millisecond): + t.Fatalf("Write timeout timed out, c = %v", c1) + case err := <-sig: + if netErr, ok := err.(net.Error); ok { + if !netErr.Timeout() { + t.Fatalf("Write returned unexpected error, c = %v, err = %v", c1, netErr) + } + } else { + t.Fatalf("Write returned unexpected error, c = %v, err = %v", c1, err) + } + } + + go blockingRead(c2) + select { + case <-time.After(100 * time.Millisecond): + case err := <-sig: + t.Fatalf("Read returned before timeout, err = %v", err) + } + + c2.SetReadDeadline(time.Now()) + select { + case <-time.After(100 * time.Millisecond): + t.Fatalf("Read timeout timed out, c = %v", c2) + case err := <-sig: + if netErr, ok := err.(net.Error); ok { + if !netErr.Timeout() { + t.Fatalf("Read returned unexpected error, c = %v, err = %v", c2, netErr) + } + } else { + t.Fatalf("Read returned unexpected error, c = %v, err = %v", c2, err) + } + } + + // Test non-blocking read/write + c1, c2 = &conn{p1, p2}, &conn{p2, p1} + + c1.SetWriteDeadline(time.Now().Add(10 * time.Second)) + c2.SetReadDeadline(time.Now().Add(10 * time.Second)) + + // Not blocking here + go blockingWrite(c1) + go blockingRead(c2) + + // Read response from both routines + for i := 0; i < 2; i++ { + select { + case <-time.After(100 * time.Millisecond): + t.Fatalf("Read/Write timed out, c1 = %v, c2 = %v", c1, c2) + case err := <-sig: + if err != nil { + t.Fatalf("Read/Write failed to complete, c1 = %v, c2 = %v, err = %v", c1, c2, err) + } + } + } +} diff --git a/test/channelz_linux_test.go b/test/channelz_linux_test.go new file mode 100644 index 0000000..1be9e68 --- /dev/null +++ b/test/channelz_linux_test.go @@ -0,0 +1,95 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "testing" + "time" + + "google.golang.org/grpc/internal/channelz" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestCZSocketMetricsSocketOption(t *testing.T) { + envs := []env{tcpClearRREnv, tcpTLSRREnv} + for _, e := range envs { + testCZSocketMetricsSocketOption(t, e) + } +} + +func testCZSocketMetricsSocketOption(t *testing.T, e env) { + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + doSuccessfulUnaryCall(tc, t) + + time.Sleep(10 * time.Millisecond) + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + t.Fatalf("There should be one server, not %d", len(ss)) + } + skts := ss[0].ListenSockets() + if len(skts) != 1 { + t.Fatalf("There should be one listen socket, not %d", len(skts)) + } + for id := range skts { + sm := channelz.GetSocket(id) + if sm == nil || sm.SocketOptions == nil { + t.Fatalf("Unable to get server listen socket options") + } + } + ns, _ := channelz.GetServerSockets(ss[0].ID, 0, 0) + if len(ns) != 1 { + t.Fatalf("There should be one server normal socket, not %d", len(ns)) + } + if ns[0] == nil || ns[0].SocketOptions == nil { + t.Fatalf("Unable to get server normal socket options") + } + + tchan, _ := channelz.GetTopChannels(0, 0) + if len(tchan) != 1 { + t.Fatalf("There should only be one top channel, not %d", len(tchan)) + } + subChans := tchan[0].SubChans() + if len(subChans) != 1 { + t.Fatalf("There should only be one subchannel under top channel %d, not %d", tchan[0].ID, len(subChans)) + } + var id int64 + for id = range subChans { + break + } + sc := channelz.GetSubChannel(id) + if sc == nil { + t.Fatalf("There should only be one socket under subchannel %d, not 0", id) + } + skts = sc.Sockets() + if len(skts) != 1 { + t.Fatalf("There should only be one socket under subchannel %d, not %d", sc.ID, len(skts)) + } + for id = range skts { + break + } + skt := channelz.GetSocket(id) + if skt == nil || skt.SocketOptions == nil { + t.Fatalf("Unable to get client normal socket options") + } +} diff --git a/test/channelz_test.go b/test/channelz_test.go new file mode 100644 index 0000000..9384712 --- /dev/null +++ b/test/channelz_test.go @@ -0,0 +1,2165 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "golang.org/x/net/http2" + "google.golang.org/grpc" + _ "google.golang.org/grpc/balancer/grpclb" + grpclbstate "google.golang.org/grpc/balancer/grpclb/state" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func verifyResultWithDelay(f func() (bool, error)) error { + var ok bool + var err error + for i := 0; i < 1000; i++ { + if ok, err = f(); ok { + return nil + } + time.Sleep(10 * time.Millisecond) + } + return err +} + +func (s) TestCZServerRegistrationAndDeletion(t *testing.T) { + testcases := []struct { + total int + start int64 + max int + length int + end bool + }{ + {total: int(channelz.EntriesPerPage), start: 0, max: 0, length: channelz.EntriesPerPage, end: true}, + {total: int(channelz.EntriesPerPage) - 1, start: 0, max: 0, length: channelz.EntriesPerPage - 1, end: true}, + {total: int(channelz.EntriesPerPage) + 1, start: 0, max: 0, length: channelz.EntriesPerPage, end: false}, + {total: int(channelz.EntriesPerPage) + 1, start: int64(2*(channelz.EntriesPerPage+1) + 1), max: 0, length: 0, end: true}, + {total: int(channelz.EntriesPerPage), start: 0, max: 1, length: 1, end: false}, + {total: int(channelz.EntriesPerPage), start: 0, max: channelz.EntriesPerPage - 1, length: channelz.EntriesPerPage - 1, end: false}, + } + + for i, c := range testcases { + // Reset channelz IDs so `start` is valid. + channelz.IDGen.Reset() + + e := tcpClearRREnv + te := newTest(t, e) + te.startServers(&testServer{security: e.security}, c.total) + + ss, end := channelz.GetServers(c.start, c.max) + if len(ss) != c.length || end != c.end { + t.Fatalf("%d: GetServers(%d) = %+v (len of which: %d), end: %+v, want len(GetServers(%d)) = %d, end: %+v", i, c.start, ss, len(ss), end, c.start, c.length, c.end) + } + te.tearDown() + ss, end = channelz.GetServers(c.start, c.max) + if len(ss) != 0 || !end { + t.Fatalf("%d: GetServers(0) = %+v (len of which: %d), end: %+v, want len(GetServers(0)) = 0, end: true", i, ss, len(ss), end) + } + } +} + +func (s) TestCZGetChannel(t *testing.T) { + e := tcpClearRREnv + e.balancer = "" + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + r := manual.NewBuilderWithScheme("whatever") + addrs := []resolver.Address{{Addr: te.srvAddr}} + r.InitialState(resolver.State{Addresses: addrs}) + te.resolverScheme = r.Scheme() + te.clientConn(grpc.WithResolvers(r)) + defer te.tearDown() + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + target := tcs[0].ChannelMetrics.Target.Load() + wantTarget := "whatever:///" + te.srvAddr + if target == nil || *target != wantTarget { + return false, fmt.Errorf("Got channelz target=%v; want %q", target, wantTarget) + } + state := tcs[0].ChannelMetrics.State.Load() + if state == nil || *state != connectivity.Ready { + return false, fmt.Errorf("Got channelz state=%v; want %q", state, connectivity.Ready) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZGetSubChannel(t *testing.T) { + e := tcpClearRREnv + e.balancer = "" + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + r := manual.NewBuilderWithScheme("whatever") + addrs := []resolver.Address{{Addr: te.srvAddr}} + r.InitialState(resolver.State{Addresses: addrs}) + te.resolverScheme = r.Scheme() + te.clientConn(grpc.WithResolvers(r)) + defer te.tearDown() + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + scs := tcs[0].SubChans() + if len(scs) != 1 { + return false, fmt.Errorf("there should be one subchannel, not %d", len(scs)) + } + var scid int64 + for scid = range scs { + } + sc := channelz.GetSubChannel(scid) + if sc == nil { + return false, fmt.Errorf("subchannel with id %v is nil", scid) + } + target := sc.ChannelMetrics.Target.Load() + if target == nil || !strings.HasPrefix(*target, "localhost") { + t.Fatalf("subchannel target must never be set incorrectly; got: %v, want ", target) + } + state := sc.ChannelMetrics.State.Load() + if state == nil || *state != connectivity.Ready { + return false, fmt.Errorf("Got subchannel state=%v; want %q", state, connectivity.Ready) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZGetServer(t *testing.T) { + e := tcpClearRREnv + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + t.Fatalf("there should only be one server, not %d", len(ss)) + } + + serverID := ss[0].ID + srv := channelz.GetServer(serverID) + if srv == nil { + t.Fatalf("server %d does not exist", serverID) + } + if srv.ID != serverID { + t.Fatalf("server want id %d, but got %d", serverID, srv.ID) + } + + te.tearDown() + + if err := verifyResultWithDelay(func() (bool, error) { + srv := channelz.GetServer(serverID) + if srv != nil { + return false, fmt.Errorf("server %d should not exist", serverID) + } + + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZGetSocket(t *testing.T) { + e := tcpClearRREnv + te := newTest(t, e) + lis := te.listenAndServe(&testServer{security: e.security}, net.Listen) + defer te.tearDown() + + if err := verifyResultWithDelay(func() (bool, error) { + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + return false, fmt.Errorf("len(ss) = %v; want %v", len(ss), 1) + } + + serverID := ss[0].ID + srv := channelz.GetServer(serverID) + if srv == nil { + return false, fmt.Errorf("server %d does not exist", serverID) + } + if srv.ID != serverID { + return false, fmt.Errorf("srv.ID = %d; want %v", srv.ID, serverID) + } + + skts := srv.ListenSockets() + if got, want := len(skts), 1; got != want { + return false, fmt.Errorf("len(skts) = %v; want %v", got, want) + } + var sktID int64 + for sktID = range skts { + } + + skt := channelz.GetSocket(sktID) + if skt == nil { + return false, fmt.Errorf("socket %v does not exist", sktID) + } + + if got, want := skt.LocalAddr, lis.Addr(); got != want { + return false, fmt.Errorf("socket %v LocalAddr=%v; want %v", sktID, got, want) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZTopChannelRegistrationAndDeletion(t *testing.T) { + testcases := []struct { + total int + start int64 + max int + length int + end bool + }{ + {total: int(channelz.EntriesPerPage), start: 0, max: 0, length: channelz.EntriesPerPage, end: true}, + {total: int(channelz.EntriesPerPage) - 1, start: 0, max: 0, length: channelz.EntriesPerPage - 1, end: true}, + {total: int(channelz.EntriesPerPage) + 1, start: 0, max: 0, length: channelz.EntriesPerPage, end: false}, + {total: int(channelz.EntriesPerPage) + 1, start: int64(2*(channelz.EntriesPerPage+1) + 1), max: 0, length: 0, end: true}, + {total: int(channelz.EntriesPerPage), start: 0, max: 1, length: 1, end: false}, + {total: int(channelz.EntriesPerPage), start: 0, max: channelz.EntriesPerPage - 1, length: channelz.EntriesPerPage - 1, end: false}, + } + + for _, c := range testcases { + // Reset channelz IDs so `start` is valid. + channelz.IDGen.Reset() + + e := tcpClearRREnv + te := newTest(t, e) + var ccs []*grpc.ClientConn + for i := 0; i < c.total; i++ { + cc := te.clientConn() + te.cc = nil + // avoid making next dial blocking + te.srvAddr = "" + ccs = append(ccs, cc) + } + if err := verifyResultWithDelay(func() (bool, error) { + if tcs, end := channelz.GetTopChannels(c.start, c.max); len(tcs) != c.length || end != c.end { + return false, fmt.Errorf("getTopChannels(%d) = %+v (len of which: %d), end: %+v, want len(GetTopChannels(%d)) = %d, end: %+v", c.start, tcs, len(tcs), end, c.start, c.length, c.end) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + for _, cc := range ccs { + cc.Close() + } + + if err := verifyResultWithDelay(func() (bool, error) { + if tcs, end := channelz.GetTopChannels(c.start, c.max); len(tcs) != 0 || !end { + return false, fmt.Errorf("getTopChannels(0) = %+v (len of which: %d), end: %+v, want len(GetTopChannels(0)) = 0, end: true", tcs, len(tcs), end) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + te.tearDown() + } +} + +func (s) TestCZTopChannelRegistrationAndDeletionWhenNewClientFail(t *testing.T) { + // Make newclient fails (due to no transport security specified) + _, err := grpc.NewClient("fake.addr") + if err == nil { + t.Fatal("expecting newclient to fail") + } + if tcs, end := channelz.GetTopChannels(0, 0); tcs != nil || !end { + t.Fatalf("GetTopChannels(0, 0) = %v, %v, want , true", tcs, end) + } +} + +func (s) TestCZNestedChannelRegistrationAndDeletion(t *testing.T) { + e := tcpClearRREnv + // avoid calling API to set balancer type, which will void service config's change of balancer. + e.balancer = "" + te := newTest(t, e) + r := manual.NewBuilderWithScheme("whatever") + te.resolverScheme = r.Scheme() + te.clientConn(grpc.WithResolvers(r)) + resolvedAddrs := []resolver.Address{{Addr: "127.0.0.1:0", ServerName: "grpclb.server"}} + grpclbConfig := parseServiceConfig(t, r, `{"loadBalancingPolicy": "grpclb"}`) + r.UpdateState(grpclbstate.Set(resolver.State{ServiceConfig: grpclbConfig}, &grpclbstate.State{BalancerAddresses: resolvedAddrs})) + defer te.tearDown() + + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + if nestedChans := tcs[0].NestedChans(); len(nestedChans) != 1 { + return false, fmt.Errorf("there should be one nested channel from grpclb, not %d", len(nestedChans)) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: "127.0.0.1:0"}}, + ServiceConfig: parseServiceConfig(t, r, `{"loadBalancingPolicy": "round_robin"}`), + }) + + // wait for the shutdown of grpclb balancer + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + if nestedChans := tcs[0].NestedChans(); len(nestedChans) != 0 { + return false, fmt.Errorf("there should be 0 nested channel from grpclb, not %d", len(nestedChans)) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZClientSubChannelSocketRegistrationAndDeletion(t *testing.T) { + e := tcpClearRREnv + num := 3 // number of backends + te := newTest(t, e) + var svrAddrs []resolver.Address + te.startServers(&testServer{security: e.security}, num) + r := manual.NewBuilderWithScheme("whatever") + for _, a := range te.srvAddrs { + svrAddrs = append(svrAddrs, resolver.Address{Addr: a}) + } + r.InitialState(resolver.State{Addresses: svrAddrs}) + te.resolverScheme = r.Scheme() + te.clientConn(grpc.WithResolvers(r)) + defer te.tearDown() + // Here, we just wait for all sockets to be up. In the future, if we implement + // IDLE, we may need to make several rpc calls to create the sockets. + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + subChans := tcs[0].SubChans() + if len(subChans) != num { + return false, fmt.Errorf("there should be %d subchannel not %d", num, len(subChans)) + } + count := 0 + for k := range subChans { + sc := channelz.GetSubChannel(k) + if sc == nil { + return false, fmt.Errorf("got subchannel") + } + count += len(sc.Sockets()) + } + if count != num { + return false, fmt.Errorf("there should be %d sockets not %d", num, count) + } + + return true, nil + }); err != nil { + t.Fatal(err) + } + + r.UpdateState(resolver.State{Addresses: svrAddrs[:len(svrAddrs)-1]}) + + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + subChans := tcs[0].SubChans() + if len(subChans) != num-1 { + return false, fmt.Errorf("there should be %d subchannel not %d", num-1, len(subChans)) + } + count := 0 + for k := range subChans { + sc := channelz.GetSubChannel(k) + if sc == nil { + return false, fmt.Errorf("got subchannel") + } + count += len(sc.Sockets()) + } + if count != num-1 { + return false, fmt.Errorf("there should be %d sockets not %d", num-1, count) + } + + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZServerSocketRegistrationAndDeletion(t *testing.T) { + testcases := []struct { + total int + start int64 + max int + length int + end bool + }{ + {total: int(channelz.EntriesPerPage), start: 0, max: 0, length: channelz.EntriesPerPage, end: true}, + {total: int(channelz.EntriesPerPage) - 1, start: 0, max: 0, length: channelz.EntriesPerPage - 1, end: true}, + {total: int(channelz.EntriesPerPage) + 1, start: 0, max: 0, length: channelz.EntriesPerPage, end: false}, + {total: int(channelz.EntriesPerPage), start: 1, max: 0, length: channelz.EntriesPerPage - 1, end: true}, + {total: int(channelz.EntriesPerPage) + 1, start: int64(channelz.EntriesPerPage) + 1, max: 0, length: 0, end: true}, + {total: int(channelz.EntriesPerPage), start: 0, max: 1, length: 1, end: false}, + {total: int(channelz.EntriesPerPage), start: 0, max: channelz.EntriesPerPage - 1, length: channelz.EntriesPerPage - 1, end: false}, + } + + for _, c := range testcases { + // Reset channelz IDs so `start` is valid. + channelz.IDGen.Reset() + + e := tcpClearRREnv + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + var ccs []*grpc.ClientConn + for i := 0; i < c.total; i++ { + cc := te.clientConn() + te.cc = nil + ccs = append(ccs, cc) + } + + var svrID int64 + if err := verifyResultWithDelay(func() (bool, error) { + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + return false, fmt.Errorf("there should only be one server, not %d", len(ss)) + } + if got := len(ss[0].ListenSockets()); got != 1 { + return false, fmt.Errorf("there should only be one server listen socket, not %d", got) + } + + startID := c.start + if startID != 0 { + ns, _ := channelz.GetServerSockets(ss[0].ID, 0, c.total) + if int64(len(ns)) < c.start { + return false, fmt.Errorf("there should more than %d sockets, not %d", len(ns), c.start) + } + startID = ns[c.start-1].ID + 1 + } + + ns, end := channelz.GetServerSockets(ss[0].ID, startID, c.max) + if len(ns) != c.length || end != c.end { + return false, fmt.Errorf("GetServerSockets(%d) = %+v (len of which: %d), end: %+v, want len(GetServerSockets(%d)) = %d, end: %+v", c.start, ns, len(ns), end, c.start, c.length, c.end) + } + + svrID = ss[0].ID + return true, nil + }); err != nil { + t.Fatal(err) + } + + for _, cc := range ccs { + cc.Close() + } + + if err := verifyResultWithDelay(func() (bool, error) { + ns, _ := channelz.GetServerSockets(svrID, c.start, c.max) + if len(ns) != 0 { + return false, fmt.Errorf("there should be %d normal sockets not %d", 0, len(ns)) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + te.tearDown() + } +} + +func (s) TestCZServerListenSocketDeletion(t *testing.T) { + s := grpc.NewServer() + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + go s.Serve(lis) + if err := verifyResultWithDelay(func() (bool, error) { + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + return false, fmt.Errorf("there should only be one server, not %d", len(ss)) + } + skts := ss[0].ListenSockets() + if len(skts) != 1 { + return false, fmt.Errorf("there should only be one server listen socket, not %v", skts) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + lis.Close() + if err := verifyResultWithDelay(func() (bool, error) { + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + return false, fmt.Errorf("there should be 1 server, not %d", len(ss)) + } + skts := ss[0].ListenSockets() + if len(skts) != 0 { + return false, fmt.Errorf("there should only be %d server listen socket, not %v", 0, skts) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + s.Stop() +} + +func (s) TestCZRecursiveDeletionOfEntry(t *testing.T) { + // +--+TopChan+---+ + // | | + // v v + // +-+SubChan1+--+ SubChan2 + // | | + // v v + // Socket1 Socket2 + + topChan := channelz.RegisterChannel(nil, "") + subChan1 := channelz.RegisterSubChannel(topChan, "") + subChan2 := channelz.RegisterSubChannel(topChan, "") + skt1 := channelz.RegisterSocket(&channelz.Socket{SocketType: channelz.SocketTypeNormal, Parent: subChan1}) + skt2 := channelz.RegisterSocket(&channelz.Socket{SocketType: channelz.SocketTypeNormal, Parent: subChan1}) + + tcs, _ := channelz.GetTopChannels(0, 0) + if tcs == nil || len(tcs) != 1 { + t.Fatalf("There should be one TopChannel entry") + } + if len(tcs[0].SubChans()) != 2 { + t.Fatalf("There should be two SubChannel entries") + } + sc := channelz.GetSubChannel(subChan1.ID) + if sc == nil || len(sc.Sockets()) != 2 { + t.Fatalf("There should be two Socket entries") + } + + channelz.RemoveEntry(topChan.ID) + tcs, _ = channelz.GetTopChannels(0, 0) + if tcs == nil || len(tcs) != 1 { + t.Fatalf("There should be one TopChannel entry") + } + + channelz.RemoveEntry(subChan1.ID) + channelz.RemoveEntry(subChan2.ID) + tcs, _ = channelz.GetTopChannels(0, 0) + if tcs == nil || len(tcs) != 1 { + t.Fatalf("There should be one TopChannel entry") + } + if len(tcs[0].SubChans()) != 1 { + t.Fatalf("There should be one SubChannel entry") + } + + channelz.RemoveEntry(skt1.ID) + channelz.RemoveEntry(skt2.ID) + tcs, _ = channelz.GetTopChannels(0, 0) + if tcs != nil { + t.Fatalf("There should be no TopChannel entry") + } +} + +func (s) TestCZChannelMetrics(t *testing.T) { + e := tcpClearRREnv + num := 3 // number of backends + te := newTest(t, e) + te.maxClientSendMsgSize = newInt(8) + var svrAddrs []resolver.Address + te.startServers(&testServer{security: e.security}, num) + r := manual.NewBuilderWithScheme("whatever") + for _, a := range te.srvAddrs { + svrAddrs = append(svrAddrs, resolver.Address{Addr: a}) + } + r.InitialState(resolver.State{Addresses: svrAddrs}) + te.resolverScheme = r.Scheme() + cc := te.clientConn(grpc.WithResolvers(r)) + defer te.tearDown() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + + const smallSize = 1 + const largeSize = 8 + + largePayload, err := newPayload(testpb.PayloadType_COMPRESSABLE, largeSize) + if err != nil { + t.Fatal(err) + } + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(smallSize), + Payload: largePayload, + } + + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) + } + + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + defer stream.CloseSend() + // Here, we just wait for all sockets to be up. In the future, if we implement + // IDLE, we may need to make several rpc calls to create the sockets. + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + subChans := tcs[0].SubChans() + if len(subChans) != num { + return false, fmt.Errorf("there should be %d subchannel not %d", num, len(subChans)) + } + var cst, csu, cf int64 + for k := range subChans { + sc := channelz.GetSubChannel(k) + if sc == nil { + return false, fmt.Errorf("got subchannel") + } + cst += sc.ChannelMetrics.CallsStarted.Load() + csu += sc.ChannelMetrics.CallsSucceeded.Load() + cf += sc.ChannelMetrics.CallsFailed.Load() + } + if cst != 3 { + return false, fmt.Errorf("there should be 3 CallsStarted not %d", cst) + } + if csu != 1 { + return false, fmt.Errorf("there should be 1 CallsSucceeded not %d", csu) + } + if cf != 1 { + return false, fmt.Errorf("there should be 1 CallsFailed not %d", cf) + } + if got := tcs[0].ChannelMetrics.CallsStarted.Load(); got != 3 { + return false, fmt.Errorf("there should be 3 CallsStarted not %d", got) + } + if got := tcs[0].ChannelMetrics.CallsSucceeded.Load(); got != 1 { + return false, fmt.Errorf("there should be 1 CallsSucceeded not %d", got) + } + if got := tcs[0].ChannelMetrics.CallsFailed.Load(); got != 1 { + return false, fmt.Errorf("there should be 1 CallsFailed not %d", got) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZServerMetrics(t *testing.T) { + e := tcpClearRREnv + te := newTest(t, e) + te.maxServerReceiveMsgSize = newInt(8) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + + const smallSize = 1 + const largeSize = 8 + + largePayload, err := newPayload(testpb.PayloadType_COMPRESSABLE, largeSize) + if err != nil { + t.Fatal(err) + } + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(smallSize), + Payload: largePayload, + } + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) + } + + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + defer stream.CloseSend() + + if err := verifyResultWithDelay(func() (bool, error) { + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + return false, fmt.Errorf("there should only be one server, not %d", len(ss)) + } + if cs := ss[0].ServerMetrics.CallsStarted.Load(); cs != 3 { + return false, fmt.Errorf("there should be 3 CallsStarted not %d", cs) + } + if cs := ss[0].ServerMetrics.CallsSucceeded.Load(); cs != 1 { + return false, fmt.Errorf("there should be 1 CallsSucceeded not %d", cs) + } + if cf := ss[0].ServerMetrics.CallsFailed.Load(); cf != 1 { + return false, fmt.Errorf("there should be 1 CallsFailed not %d", cf) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +type testServiceClientWrapper struct { + testgrpc.TestServiceClient + mu sync.RWMutex + streamsCreated int +} + +func (t *testServiceClientWrapper) getCurrentStreamID() uint32 { + t.mu.RLock() + defer t.mu.RUnlock() + return uint32(2*t.streamsCreated - 1) +} + +func (t *testServiceClientWrapper) EmptyCall(ctx context.Context, in *testpb.Empty, opts ...grpc.CallOption) (*testpb.Empty, error) { + t.mu.Lock() + defer t.mu.Unlock() + t.streamsCreated++ + return t.TestServiceClient.EmptyCall(ctx, in, opts...) +} + +func (t *testServiceClientWrapper) UnaryCall(ctx context.Context, in *testpb.SimpleRequest, opts ...grpc.CallOption) (*testpb.SimpleResponse, error) { + t.mu.Lock() + defer t.mu.Unlock() + t.streamsCreated++ + return t.TestServiceClient.UnaryCall(ctx, in, opts...) +} + +func (t *testServiceClientWrapper) StreamingOutputCall(ctx context.Context, in *testpb.StreamingOutputCallRequest, opts ...grpc.CallOption) (testgrpc.TestService_StreamingOutputCallClient, error) { + t.mu.Lock() + defer t.mu.Unlock() + t.streamsCreated++ + return t.TestServiceClient.StreamingOutputCall(ctx, in, opts...) +} + +func (t *testServiceClientWrapper) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (testgrpc.TestService_StreamingInputCallClient, error) { + t.mu.Lock() + defer t.mu.Unlock() + t.streamsCreated++ + return t.TestServiceClient.StreamingInputCall(ctx, opts...) +} + +func (t *testServiceClientWrapper) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (testgrpc.TestService_FullDuplexCallClient, error) { + t.mu.Lock() + defer t.mu.Unlock() + t.streamsCreated++ + return t.TestServiceClient.FullDuplexCall(ctx, opts...) +} + +func (t *testServiceClientWrapper) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (testgrpc.TestService_HalfDuplexCallClient, error) { + t.mu.Lock() + defer t.mu.Unlock() + t.streamsCreated++ + return t.TestServiceClient.HalfDuplexCall(ctx, opts...) +} + +func doSuccessfulUnaryCall(tc testgrpc.TestServiceClient, t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } +} + +func doStreamingInputCallWithLargePayload(tc testgrpc.TestServiceClient, t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + s, err := tc.StreamingInputCall(ctx) + if err != nil { + t.Fatalf("TestService/StreamingInputCall(_) = _, %v, want ", err) + } + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 10000) + if err != nil { + t.Fatal(err) + } + s.Send(&testpb.StreamingInputCallRequest{Payload: payload}) +} + +func doServerSideFailedUnaryCall(tc testgrpc.TestServiceClient, t *testing.T) { + const smallSize = 1 + const largeSize = 2000 + + largePayload, err := newPayload(testpb.PayloadType_COMPRESSABLE, largeSize) + if err != nil { + t.Fatal(err) + } + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(smallSize), + Payload: largePayload, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) + } +} + +func doClientSideInitiatedFailedStream(tc testgrpc.TestServiceClient, t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want ", err) + } + + const smallSize = 1 + smallPayload, err := newPayload(testpb.PayloadType_COMPRESSABLE, smallSize) + if err != nil { + t.Fatal(err) + } + + sreq := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: []*testpb.ResponseParameters{ + {Size: smallSize}, + }, + Payload: smallPayload, + } + + if err := stream.Send(sreq); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = %v, want ", stream, err) + } + // By canceling the call, the client will send rst_stream to end the call, and + // the stream will failed as a result. + cancel() +} + +// This func is to be used to test client side counting of failed streams. +func doServerSideInitiatedFailedStreamWithRSTStream(tc testgrpc.TestServiceClient, t *testing.T, l *listenerWrapper) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want ", err) + } + + const smallSize = 1 + smallPayload, err := newPayload(testpb.PayloadType_COMPRESSABLE, smallSize) + if err != nil { + t.Fatal(err) + } + + sreq := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: []*testpb.ResponseParameters{ + {Size: smallSize}, + }, + Payload: smallPayload, + } + + if err := stream.Send(sreq); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = %v, want ", stream, err) + } + + rcw := l.getLastConn() + + if rcw != nil { + rcw.writeRSTStream(tc.(*testServiceClientWrapper).getCurrentStreamID(), http2.ErrCodeCancel) + } + if _, err := stream.Recv(); err == nil { + t.Fatalf("%v.Recv() = %v, want ", stream, err) + } +} + +// this func is to be used to test client side counting of failed streams. +func doServerSideInitiatedFailedStreamWithGoAway(ctx context.Context, tc testgrpc.TestServiceClient, t *testing.T, l *listenerWrapper) { + // This call is just to keep the transport from shutting down (socket will be deleted + // in this case, and we will not be able to get metrics). + s, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want ", err) + } + if err := s.Send(&testpb.StreamingOutputCallRequest{ResponseParameters: []*testpb.ResponseParameters{ + { + Size: 1, + }, + }}); err != nil { + t.Fatalf("s.Send() failed with error: %v", err) + } + if _, err := s.Recv(); err != nil { + t.Fatalf("s.Recv() failed with error: %v", err) + } + + s, err = tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want ", err) + } + if err := s.Send(&testpb.StreamingOutputCallRequest{ResponseParameters: []*testpb.ResponseParameters{ + { + Size: 1, + }, + }}); err != nil { + t.Fatalf("s.Send() failed with error: %v", err) + } + if _, err := s.Recv(); err != nil { + t.Fatalf("s.Recv() failed with error: %v", err) + } + + rcw := l.getLastConn() + if rcw != nil { + rcw.writeGoAway(tc.(*testServiceClientWrapper).getCurrentStreamID()-2, http2.ErrCodeCancel, []byte{}) + } + if _, err := s.Recv(); err == nil { + t.Fatalf("%v.Recv() = %v, want ", s, err) + } +} + +func (s) TestCZClientSocketMetricsStreamsAndMessagesCount(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + e := tcpClearRREnv + te := newTest(t, e) + te.maxServerReceiveMsgSize = newInt(20) + te.maxClientReceiveMsgSize = newInt(20) + rcw := te.startServerWithConnControl(&testServer{security: e.security}) + defer te.tearDown() + cc := te.clientConn() + tc := &testServiceClientWrapper{TestServiceClient: testgrpc.NewTestServiceClient(cc)} + + doSuccessfulUnaryCall(tc, t) + var scID, skID int64 + if err := verifyResultWithDelay(func() (bool, error) { + tchan, _ := channelz.GetTopChannels(0, 0) + if len(tchan) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tchan)) + } + subChans := tchan[0].SubChans() + if len(subChans) != 1 { + return false, fmt.Errorf("there should only be one subchannel under top channel %d, not %d", tchan[0].ID, len(subChans)) + } + + for scID = range subChans { + break + } + sc := channelz.GetSubChannel(scID) + if sc == nil { + return false, fmt.Errorf("there should only be one socket under subchannel %d, not 0", scID) + } + skts := sc.Sockets() + if len(skts) != 1 { + return false, fmt.Errorf("there should only be one socket under subchannel %d, not %d", sc.ID, len(skts)) + } + for skID = range skts { + break + } + skt := channelz.GetSocket(skID) + sktData := &skt.SocketMetrics + if sktData.StreamsStarted.Load() != 1 || sktData.StreamsSucceeded.Load() != 1 || sktData.MessagesSent.Load() != 1 || sktData.MessagesReceived.Load() != 1 { + return false, fmt.Errorf("channelz.GetSocket(%d), want (StreamsStarted.Load(), StreamsSucceeded.Load(), MessagesSent.Load(), MessagesReceived.Load()) = (1, 1, 1, 1), got (%d, %d, %d, %d)", skt.ID, sktData.StreamsStarted.Load(), sktData.StreamsSucceeded.Load(), sktData.MessagesSent.Load(), sktData.MessagesReceived.Load()) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + doServerSideFailedUnaryCall(tc, t) + if err := verifyResultWithDelay(func() (bool, error) { + skt := channelz.GetSocket(skID) + sktData := &skt.SocketMetrics + if sktData.StreamsStarted.Load() != 2 || sktData.StreamsSucceeded.Load() != 2 || sktData.MessagesSent.Load() != 2 || sktData.MessagesReceived.Load() != 1 { + return false, fmt.Errorf("channelz.GetSocket(%d), want (StreamsStarted.Load(), StreamsSucceeded.Load(), MessagesSent.Load(), MessagesReceived.Load()) = (2, 2, 2, 1), got (%d, %d, %d, %d)", skt.ID, sktData.StreamsStarted.Load(), sktData.StreamsSucceeded.Load(), sktData.MessagesSent.Load(), sktData.MessagesReceived.Load()) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + doClientSideInitiatedFailedStream(tc, t) + if err := verifyResultWithDelay(func() (bool, error) { + skt := channelz.GetSocket(skID) + sktData := &skt.SocketMetrics + if sktData.StreamsStarted.Load() != 3 || sktData.StreamsSucceeded.Load() != 2 || sktData.StreamsFailed.Load() != 1 || sktData.MessagesSent.Load() != 3 || sktData.MessagesReceived.Load() != 2 { + return false, fmt.Errorf("channelz.GetSocket(%d), want (StreamsStarted.Load(), StreamsSucceeded.Load(), StreamsFailed.Load(), MessagesSent.Load(), MessagesReceived.Load()) = (3, 2, 1, 3, 2), got (%d, %d, %d, %d, %d)", skt.ID, sktData.StreamsStarted.Load(), sktData.StreamsSucceeded.Load(), sktData.StreamsFailed.Load(), sktData.MessagesSent.Load(), sktData.MessagesReceived.Load()) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + doServerSideInitiatedFailedStreamWithRSTStream(tc, t, rcw) + if err := verifyResultWithDelay(func() (bool, error) { + skt := channelz.GetSocket(skID) + sktData := &skt.SocketMetrics + if sktData.StreamsStarted.Load() != 4 || sktData.StreamsSucceeded.Load() != 2 || sktData.StreamsFailed.Load() != 2 || sktData.MessagesSent.Load() != 4 || sktData.MessagesReceived.Load() != 3 { + return false, fmt.Errorf("channelz.GetSocket(%d), want (StreamsStarted.Load(), StreamsSucceeded.Load(), StreamsFailed.Load(), MessagesSent.Load(), MessagesReceived.Load()) = (4, 2, 2, 4, 3), got (%d, %d, %d, %d, %d)", skt.ID, sktData.StreamsStarted.Load(), sktData.StreamsSucceeded.Load(), sktData.StreamsFailed.Load(), sktData.MessagesSent.Load(), sktData.MessagesReceived.Load()) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + doServerSideInitiatedFailedStreamWithGoAway(ctx, tc, t, rcw) + if err := verifyResultWithDelay(func() (bool, error) { + skt := channelz.GetSocket(skID) + sktData := &skt.SocketMetrics + if sktData.StreamsStarted.Load() != 6 || sktData.StreamsSucceeded.Load() != 2 || sktData.StreamsFailed.Load() != 3 || sktData.MessagesSent.Load() != 6 || sktData.MessagesReceived.Load() != 5 { + return false, fmt.Errorf("channelz.GetSocket(%d), want (StreamsStarted.Load(), StreamsSucceeded.Load(), StreamsFailed.Load(), MessagesSent.Load(), MessagesReceived.Load()) = (6, 2, 3, 6, 5), got (%d, %d, %d, %d, %d)", skt.ID, sktData.StreamsStarted.Load(), sktData.StreamsSucceeded.Load(), sktData.StreamsFailed.Load(), sktData.MessagesSent.Load(), sktData.MessagesReceived.Load()) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +// This test is to complete TestCZClientSocketMetricsStreamsAndMessagesCount and +// TestCZServerSocketMetricsStreamsAndMessagesCount by adding the test case of +// server sending RST_STREAM to client due to client side flow control violation. +// It is separated from other cases due to setup incompatibly, i.e. max receive +// size violation will mask flow control violation. +func (s) TestCZClientAndServerSocketMetricsStreamsCountFlowControlRSTStream(t *testing.T) { + e := tcpClearRREnv + te := newTest(t, e) + te.serverInitialWindowSize = 65536 + // Avoid overflowing connection level flow control window, which will lead to + // transport being closed. + te.serverInitialConnWindowSize = 65536 * 2 + ts := &stubserver.StubServer{FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + stream.Send(&testpb.StreamingOutputCallResponse{}) + <-stream.Context().Done() + return status.Errorf(codes.DeadlineExceeded, "deadline exceeded or cancelled") + }} + te.startServer(ts) + defer te.tearDown() + cc, dw := te.clientConnWithConnControl() + tc := &testServiceClientWrapper{TestServiceClient: testgrpc.NewTestServiceClient(cc)} + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want ", err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("stream.Recv() = %v, want nil", err) + } + go func() { + payload := make([]byte, 16384) + for i := 0; i < 6; i++ { + dw.getRawConnWrapper().writeDataFrame(tc.getCurrentStreamID(), payload) + } + }() + if _, err := stream.Recv(); status.Code(err) != codes.ResourceExhausted { + t.Fatalf("stream.Recv() = %v, want error code: %v", err, codes.ResourceExhausted) + } + cancel() + + if err := verifyResultWithDelay(func() (bool, error) { + tchan, _ := channelz.GetTopChannels(0, 0) + if len(tchan) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tchan)) + } + subChans := tchan[0].SubChans() + if len(subChans) != 1 { + return false, fmt.Errorf("there should only be one subchannel under top channel %d, not %d", tchan[0].ID, len(subChans)) + } + var id int64 + for id = range subChans { + break + } + sc := channelz.GetSubChannel(id) + if sc == nil { + return false, fmt.Errorf("there should only be one socket under subchannel %d, not 0", id) + } + skts := sc.Sockets() + if len(skts) != 1 { + return false, fmt.Errorf("there should only be one socket under subchannel %d, not %d", sc.ID, len(skts)) + } + for id = range skts { + break + } + skt := channelz.GetSocket(id) + sktData := &skt.SocketMetrics + if sktData.StreamsStarted.Load() != 1 || sktData.StreamsSucceeded.Load() != 0 || sktData.StreamsFailed.Load() != 1 { + return false, fmt.Errorf("channelz.GetSocket(%d), want (StreamsStarted.Load(), StreamsSucceeded.Load(), StreamsFailed.Load()) = (1, 0, 1), got (%d, %d, %d)", skt.ID, sktData.StreamsStarted.Load(), sktData.StreamsSucceeded.Load(), sktData.StreamsFailed.Load()) + } + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + return false, fmt.Errorf("there should only be one server, not %d", len(ss)) + } + + ns, _ := channelz.GetServerSockets(ss[0].ID, 0, 0) + if len(ns) != 1 { + return false, fmt.Errorf("there should be one server normal socket, not %d", len(ns)) + } + sktData = &ns[0].SocketMetrics + if sktData.StreamsStarted.Load() != 1 || sktData.StreamsSucceeded.Load() != 0 || sktData.StreamsFailed.Load() != 1 { + return false, fmt.Errorf("server socket metric with ID %d, want (StreamsStarted.Load(), StreamsSucceeded.Load(), StreamsFailed.Load()) = (1, 0, 1), got (%d, %d, %d)", ns[0].ID, sktData.StreamsStarted.Load(), sktData.StreamsSucceeded.Load(), sktData.StreamsFailed.Load()) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZClientAndServerSocketMetricsFlowControl(t *testing.T) { + e := tcpClearRREnv + te := newTest(t, e) + // disable BDP + te.serverInitialWindowSize = 65536 + te.serverInitialConnWindowSize = 65536 + te.clientInitialWindowSize = 65536 + te.clientInitialConnWindowSize = 65536 + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + + for i := 0; i < 10; i++ { + doSuccessfulUnaryCall(tc, t) + } + + var cliSktID, svrSktID int64 + if err := verifyResultWithDelay(func() (bool, error) { + tchan, _ := channelz.GetTopChannels(0, 0) + if len(tchan) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tchan)) + } + subChans := tchan[0].SubChans() + if len(subChans) != 1 { + return false, fmt.Errorf("there should only be one subchannel under top channel %d, not %d", tchan[0].ID, len(subChans)) + } + var id int64 + for id = range subChans { + break + } + sc := channelz.GetSubChannel(id) + if sc == nil { + return false, fmt.Errorf("there should only be one socket under subchannel %d, not 0", id) + } + skts := sc.Sockets() + if len(skts) != 1 { + return false, fmt.Errorf("there should only be one socket under subchannel %d, not %d", sc.ID, len(skts)) + } + for id = range skts { + break + } + skt := channelz.GetSocket(id) + sktData := skt.EphemeralMetrics() + // 65536 - 5 (Length-Prefixed-Message size) * 10 = 65486 + if sktData.LocalFlowControlWindow != 65486 || sktData.RemoteFlowControlWindow != 65486 { + return false, fmt.Errorf("client: (LocalFlowControlWindow, RemoteFlowControlWindow) size should be (65536, 65486), not (%d, %d)", sktData.LocalFlowControlWindow, sktData.RemoteFlowControlWindow) + } + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + return false, fmt.Errorf("there should only be one server, not %d", len(ss)) + } + ns, _ := channelz.GetServerSockets(ss[0].ID, 0, 0) + sktData = ns[0].EphemeralMetrics() + if sktData.LocalFlowControlWindow != 65486 || sktData.RemoteFlowControlWindow != 65486 { + return false, fmt.Errorf("server: (LocalFlowControlWindow, RemoteFlowControlWindow) size should be (65536, 65486), not (%d, %d)", sktData.LocalFlowControlWindow, sktData.RemoteFlowControlWindow) + } + cliSktID, svrSktID = id, ss[0].ID + return true, nil + }); err != nil { + t.Fatal(err) + } + + doStreamingInputCallWithLargePayload(tc, t) + + if err := verifyResultWithDelay(func() (bool, error) { + skt := channelz.GetSocket(cliSktID) + sktData := skt.EphemeralMetrics() + // Local: 65536 - 5 (Length-Prefixed-Message size) * 10 = 65486 + // Remote: 65536 - 5 (Length-Prefixed-Message size) * 10 - 10011 = 55475 + if sktData.LocalFlowControlWindow != 65486 || sktData.RemoteFlowControlWindow != 55475 { + return false, fmt.Errorf("client: (LocalFlowControlWindow, RemoteFlowControlWindow) size should be (65486, 55475), not (%d, %d)", sktData.LocalFlowControlWindow, sktData.RemoteFlowControlWindow) + } + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + return false, fmt.Errorf("there should only be one server, not %d", len(ss)) + } + ns, _ := channelz.GetServerSockets(svrSktID, 0, 0) + sktData = ns[0].EphemeralMetrics() + if sktData.LocalFlowControlWindow != 55475 || sktData.RemoteFlowControlWindow != 65486 { + return false, fmt.Errorf("server: (LocalFlowControlWindow, RemoteFlowControlWindow) size should be (55475, 65486), not (%d, %d)", sktData.LocalFlowControlWindow, sktData.RemoteFlowControlWindow) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + // triggers transport flow control window update on server side, since unacked + // bytes should be larger than limit now. i.e. 50 + 20022 > 65536/4. + doStreamingInputCallWithLargePayload(tc, t) + if err := verifyResultWithDelay(func() (bool, error) { + skt := channelz.GetSocket(cliSktID) + sktData := skt.EphemeralMetrics() + // Local: 65536 - 5 (Length-Prefixed-Message size) * 10 = 65486 + // Remote: 65536 + if sktData.LocalFlowControlWindow != 65486 || sktData.RemoteFlowControlWindow != 65536 { + return false, fmt.Errorf("client: (LocalFlowControlWindow, RemoteFlowControlWindow) size should be (65486, 65536), not (%d, %d)", sktData.LocalFlowControlWindow, sktData.RemoteFlowControlWindow) + } + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + return false, fmt.Errorf("there should only be one server, not %d", len(ss)) + } + ns, _ := channelz.GetServerSockets(svrSktID, 0, 0) + sktData = ns[0].EphemeralMetrics() + if sktData.LocalFlowControlWindow != 65536 || sktData.RemoteFlowControlWindow != 65486 { + return false, fmt.Errorf("server: (LocalFlowControlWindow, RemoteFlowControlWindow) size should be (65536, 65486), not (%d, %d)", sktData.LocalFlowControlWindow, sktData.RemoteFlowControlWindow) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZClientSocketMetricsKeepAlive(t *testing.T) { + const keepaliveRate = 50 * time.Millisecond + defer func(t time.Duration) { internal.KeepaliveMinPingTime = t }(internal.KeepaliveMinPingTime) + internal.KeepaliveMinPingTime = keepaliveRate + e := tcpClearRREnv + te := newTest(t, e) + te.customDialOptions = append(te.customDialOptions, grpc.WithKeepaliveParams( + keepalive.ClientParameters{ + Time: keepaliveRate, + Timeout: 500 * time.Millisecond, + PermitWithoutStream: true, + })) + te.customServerOptions = append(te.customServerOptions, grpc.KeepaliveEnforcementPolicy( + keepalive.EnforcementPolicy{ + MinTime: keepaliveRate, + PermitWithoutStream: true, + })) + te.startServer(&testServer{security: e.security}) + cc := te.clientConn() // Dial the server + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + start := time.Now() + // Wait for at least two keepalives to be able to occur. + time.Sleep(2 * keepaliveRate) + defer te.tearDown() + if err := verifyResultWithDelay(func() (bool, error) { + tchan, _ := channelz.GetTopChannels(0, 0) + if len(tchan) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tchan)) + } + subChans := tchan[0].SubChans() + if len(subChans) != 1 { + return false, fmt.Errorf("there should only be one subchannel under top channel %d, not %d", tchan[0].ID, len(subChans)) + } + var id int64 + for id = range subChans { + break + } + sc := channelz.GetSubChannel(id) + if sc == nil { + return false, fmt.Errorf("there should only be one socket under subchannel %d, not 0", id) + } + skts := sc.Sockets() + if len(skts) != 1 { + return false, fmt.Errorf("there should only be one socket under subchannel %d, not %d", sc.ID, len(skts)) + } + for id = range skts { + break + } + skt := channelz.GetSocket(id) + want := int64(time.Since(start) / keepaliveRate) + if got := skt.SocketMetrics.KeepAlivesSent.Load(); got != want { + return false, fmt.Errorf("there should be %v KeepAlives sent, not %d", want, got) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZServerSocketMetricsStreamsAndMessagesCount(t *testing.T) { + e := tcpClearRREnv + te := newTest(t, e) + te.maxServerReceiveMsgSize = newInt(20) + te.maxClientReceiveMsgSize = newInt(20) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + cc, _ := te.clientConnWithConnControl() + tc := &testServiceClientWrapper{TestServiceClient: testgrpc.NewTestServiceClient(cc)} + + var svrID int64 + if err := verifyResultWithDelay(func() (bool, error) { + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + return false, fmt.Errorf("there should only be one server, not %d", len(ss)) + } + svrID = ss[0].ID + return true, nil + }); err != nil { + t.Fatal(err) + } + + doSuccessfulUnaryCall(tc, t) + if err := verifyResultWithDelay(func() (bool, error) { + ns, _ := channelz.GetServerSockets(svrID, 0, 0) + sktData := &ns[0].SocketMetrics + if sktData.StreamsStarted.Load() != 1 || sktData.StreamsSucceeded.Load() != 1 || sktData.StreamsFailed.Load() != 0 || sktData.MessagesSent.Load() != 1 || sktData.MessagesReceived.Load() != 1 { + return false, fmt.Errorf("server socket metric with ID %d, want (StreamsStarted.Load(), StreamsSucceeded.Load(), MessagesSent.Load(), MessagesReceived.Load()) = (1, 1, 1, 1), got (%d, %d, %d, %d, %d)", ns[0].ID, sktData.StreamsStarted.Load(), sktData.StreamsSucceeded.Load(), sktData.StreamsFailed.Load(), sktData.MessagesSent.Load(), sktData.MessagesReceived.Load()) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + doServerSideFailedUnaryCall(tc, t) + if err := verifyResultWithDelay(func() (bool, error) { + ns, _ := channelz.GetServerSockets(svrID, 0, 0) + sktData := &ns[0].SocketMetrics + if sktData.StreamsStarted.Load() != 2 || sktData.StreamsSucceeded.Load() != 2 || sktData.StreamsFailed.Load() != 0 || sktData.MessagesSent.Load() != 1 || sktData.MessagesReceived.Load() != 1 { + return false, fmt.Errorf("server socket metric with ID %d, want (StreamsStarted.Load(), StreamsSucceeded.Load(), StreamsFailed.Load(), MessagesSent.Load(), MessagesReceived.Load()) = (2, 2, 0, 1, 1), got (%d, %d, %d, %d, %d)", ns[0].ID, sktData.StreamsStarted.Load(), sktData.StreamsSucceeded.Load(), sktData.StreamsFailed.Load(), sktData.MessagesSent.Load(), sktData.MessagesReceived.Load()) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + doClientSideInitiatedFailedStream(tc, t) + if err := verifyResultWithDelay(func() (bool, error) { + ns, _ := channelz.GetServerSockets(svrID, 0, 0) + sktData := &ns[0].SocketMetrics + if sktData.StreamsStarted.Load() != 3 || sktData.StreamsSucceeded.Load() != 2 || sktData.StreamsFailed.Load() != 1 || sktData.MessagesSent.Load() != 2 || sktData.MessagesReceived.Load() != 2 { + return false, fmt.Errorf("server socket metric with ID %d, want (StreamsStarted.Load(), StreamsSucceeded.Load(), StreamsFailed.Load(), MessagesSent.Load(), MessagesReceived.Load()) = (3, 2, 1, 2, 2), got (%d, %d, %d, %d, %d)", ns[0].ID, sktData.StreamsStarted.Load(), sktData.StreamsSucceeded.Load(), sktData.StreamsFailed.Load(), sktData.MessagesSent.Load(), sktData.MessagesReceived.Load()) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZServerSocketMetricsKeepAlive(t *testing.T) { + defer func(t time.Duration) { internal.KeepaliveMinServerPingTime = t }(internal.KeepaliveMinServerPingTime) + internal.KeepaliveMinServerPingTime = 50 * time.Millisecond + + e := tcpClearRREnv + te := newTest(t, e) + // We setup the server keepalive parameters to send one keepalive every + // 50ms, and verify that the actual number of keepalives is very close to + // Time/50ms. We had a bug wherein the server was sending one keepalive + // every [Time+Timeout] instead of every [Time] period, and since Timeout + // is configured to a high value here, we should be able to verify that the + // fix works with the above mentioned logic. + kpOption := grpc.KeepaliveParams(keepalive.ServerParameters{ + Time: 50 * time.Millisecond, + Timeout: 5 * time.Second, + }) + te.customServerOptions = append(te.customServerOptions, kpOption) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + cc := te.clientConn() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Allow about 5 pings to happen (250ms/50ms). + time.Sleep(255 * time.Millisecond) + + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + t.Fatalf("there should be one server, not %d", len(ss)) + } + ns, _ := channelz.GetServerSockets(ss[0].ID, 0, 0) + if len(ns) != 1 { + t.Fatalf("there should be one server normal socket, not %d", len(ns)) + } + const wantMin, wantMax = 3, 7 + if got := ns[0].SocketMetrics.KeepAlivesSent.Load(); got < wantMin || got > wantMax { + t.Fatalf("got keepalivesCount: %v, want keepalivesCount: [%v,%v]", got, wantMin, wantMax) + } +} + +var cipherSuites = []string{ + "TLS_RSA_WITH_RC4_128_SHA", + "TLS_RSA_WITH_3DES_EDE_CBC_SHA", + "TLS_RSA_WITH_AES_128_CBC_SHA", + "TLS_RSA_WITH_AES_256_CBC_SHA", + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA", + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA", + "TLS_ECDHE_RSA_WITH_RC4_128_SHA", + "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_FALLBACK_SCSV", + "TLS_RSA_WITH_AES_128_CBC_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", + "TLS_AES_128_GCM_SHA256", + "TLS_AES_256_GCM_SHA384", + "TLS_CHACHA20_POLY1305_SHA256", +} + +func (s) TestCZSocketGetSecurityValueTLS(t *testing.T) { + e := tcpTLSRREnv + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + te.clientConn() + if err := verifyResultWithDelay(func() (bool, error) { + tchan, _ := channelz.GetTopChannels(0, 0) + if len(tchan) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tchan)) + } + subChans := tchan[0].SubChans() + if len(subChans) != 1 { + return false, fmt.Errorf("there should only be one subchannel under top channel %d, not %d", tchan[0].ID, len(subChans)) + } + var id int64 + for id = range subChans { + break + } + sc := channelz.GetSubChannel(id) + if sc == nil { + return false, fmt.Errorf("there should only be one socket under subchannel %d, not 0", id) + } + skts := sc.Sockets() + if len(skts) != 1 { + return false, fmt.Errorf("there should only be one socket under subchannel %d, not %d", sc.ID, len(skts)) + } + for id = range skts { + break + } + skt := channelz.GetSocket(id) + cert, _ := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + securityVal, ok := skt.Security.(*credentials.TLSChannelzSecurityValue) + if !ok { + return false, fmt.Errorf("the Security is of type: %T, want: *credentials.TLSChannelzSecurityValue", skt.Security) + } + if !cmp.Equal(securityVal.RemoteCertificate, cert.Certificate[0]) { + return false, fmt.Errorf("Security.RemoteCertificate got: %v, want: %v", securityVal.RemoteCertificate, cert.Certificate[0]) + } + for _, v := range cipherSuites { + if v == securityVal.StandardName { + return true, nil + } + } + return false, fmt.Errorf("Security.StandardName got: %v, want it to be one of %v", securityVal.StandardName, cipherSuites) + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZChannelTraceCreationDeletion(t *testing.T) { + e := tcpClearRREnv + // avoid calling API to set balancer type, which will void service config's change of balancer. + e.balancer = "" + te := newTest(t, e) + r := manual.NewBuilderWithScheme("whatever") + te.resolverScheme = r.Scheme() + te.clientConn(grpc.WithResolvers(r)) + resolvedAddrs := []resolver.Address{{Addr: "127.0.0.1:0", ServerName: "grpclb.server"}} + grpclbConfig := parseServiceConfig(t, r, `{"loadBalancingPolicy": "grpclb"}`) + r.UpdateState(grpclbstate.Set(resolver.State{ServiceConfig: grpclbConfig}, &grpclbstate.State{BalancerAddresses: resolvedAddrs})) + defer te.tearDown() + + var nestedConn int64 + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + nestedChans := tcs[0].NestedChans() + if len(nestedChans) != 1 { + return false, fmt.Errorf("there should be one nested channel from grpclb, not %d", len(nestedChans)) + } + for k := range nestedChans { + nestedConn = k + } + trace := tcs[0].Trace() + for _, e := range trace.Events { + if e.RefID == nestedConn && e.RefType != channelz.RefChannel { + return false, fmt.Errorf("nested channel trace event should have RefChannel as RefType") + } + } + ncm := channelz.GetChannel(nestedConn) + ncmTrace := ncm.Trace() + if ncmTrace == nil { + return false, fmt.Errorf("trace for nested channel should not be empty") + } + if len(ncmTrace.Events) == 0 { + return false, fmt.Errorf("there should be at least one trace event for nested channel not 0") + } + pattern := `Channel created` + if ok, _ := regexp.MatchString(pattern, ncmTrace.Events[0].Desc); !ok { + return false, fmt.Errorf("the first trace event should be %q, not %q", pattern, ncmTrace.Events[0].Desc) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: "127.0.0.1:0"}}, + ServiceConfig: parseServiceConfig(t, r, `{"loadBalancingPolicy": "round_robin"}`), + }) + + // wait for the shutdown of grpclb balancer + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + nestedChans := tcs[0].NestedChans() + if len(nestedChans) != 0 { + return false, fmt.Errorf("there should be 0 nested channel from grpclb, not %d", len(nestedChans)) + } + ncm := channelz.GetChannel(nestedConn) + if ncm == nil { + return false, fmt.Errorf("nested channel should still exist due to parent's trace reference") + } + trace := ncm.Trace() + if trace == nil { + return false, fmt.Errorf("trace for nested channel should not be empty") + } + if len(trace.Events) == 0 { + return false, fmt.Errorf("there should be at least one trace event for nested channel not 0") + } + pattern := `Channel created` + if ok, _ := regexp.MatchString(pattern, trace.Events[0].Desc); !ok { + return false, fmt.Errorf("the first trace event should be %q, not %q", pattern, trace.Events[0].Desc) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZSubChannelTraceCreationDeletion(t *testing.T) { + e := tcpClearRREnv + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}}) + te.resolverScheme = r.Scheme() + te.clientConn(grpc.WithResolvers(r)) + defer te.tearDown() + var subConn int64 + // Here, we just wait for all sockets to be up. In the future, if we implement + // IDLE, we may need to make several rpc calls to create the sockets. + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + subChans := tcs[0].SubChans() + if len(subChans) != 1 { + return false, fmt.Errorf("there should be 1 subchannel not %d", len(subChans)) + } + for k := range subChans { + subConn = k + } + trace := tcs[0].Trace() + for _, e := range trace.Events { + if e.RefID == subConn && e.RefType != channelz.RefSubChannel { + return false, fmt.Errorf("subchannel trace event should have RefType to be RefSubChannel") + } + } + scm := channelz.GetSubChannel(subConn) + if scm == nil { + return false, fmt.Errorf("subChannel does not exist") + } + scTrace := scm.Trace() + if scTrace == nil { + return false, fmt.Errorf("trace for subChannel should not be empty") + } + if len(scTrace.Events) == 0 { + return false, fmt.Errorf("there should be at least one trace event for subChannel not 0") + } + pattern := `Subchannel created` + if ok, _ := regexp.MatchString(pattern, scTrace.Events[0].Desc); !ok { + return false, fmt.Errorf("the first trace event should be %q, not %q", pattern, scTrace.Events[0].Desc) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, te.cc, connectivity.Ready) + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "fake address"}}}) + testutils.AwaitNotState(ctx, t, te.cc, connectivity.Ready) + + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + subChans := tcs[0].SubChans() + if len(subChans) != 1 { + return false, fmt.Errorf("there should be 1 subchannel not %d", len(subChans)) + } + scm := channelz.GetSubChannel(subConn) + if scm == nil { + return false, fmt.Errorf("subChannel should still exist due to parent's trace reference") + } + trace := scm.Trace() + if trace == nil { + return false, fmt.Errorf("trace for SubChannel should not be empty") + } + if len(trace.Events) == 0 { + return false, fmt.Errorf("there should be at least one trace event for subChannel not 0") + } + + pattern := `Subchannel deleted` + desc := trace.Events[len(trace.Events)-1].Desc + if ok, _ := regexp.MatchString(pattern, desc); !ok { + return false, fmt.Errorf("the last trace event should be %q, not %q", pattern, desc) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZChannelAddressResolutionChange(t *testing.T) { + e := tcpClearRREnv + e.balancer = "" + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + r := manual.NewBuilderWithScheme("whatever") + addrs := []resolver.Address{{Addr: te.srvAddr}} + r.InitialState(resolver.State{Addresses: addrs}) + te.resolverScheme = r.Scheme() + te.clientConn(grpc.WithResolvers(r)) + defer te.tearDown() + var cid int64 + // Here, we just wait for all sockets to be up. In the future, if we implement + // IDLE, we may need to make several rpc calls to create the sockets. + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + cid = tcs[0].ID + trace := tcs[0].Trace() + for i := len(trace.Events) - 1; i >= 0; i-- { + if strings.Contains(trace.Events[i].Desc, "resolver returned new addresses") { + break + } + if i == 0 { + return false, fmt.Errorf("events do not contain expected address resolution from empty address state. Got: %+v", trace.Events) + } + } + return true, nil + }); err != nil { + t.Fatal(err) + } + r.UpdateState(resolver.State{ + Addresses: addrs, + ServiceConfig: parseServiceConfig(t, r, `{"loadBalancingPolicy": "round_robin"}`), + }) + + if err := verifyResultWithDelay(func() (bool, error) { + cm := channelz.GetChannel(cid) + trace := cm.Trace() + for i := len(trace.Events) - 1; i >= 0; i-- { + if strings.Contains(trace.Events[i].Desc, fmt.Sprintf("Channel switches to new LB policy %q", roundrobin.Name)) { + break + } + if i == 0 { + return false, fmt.Errorf("events do not contain expected address resolution change of LB policy") + } + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + newSC := parseServiceConfig(t, r, `{ + "methodConfig": [ + { + "name": [ + { + "service": "grpc.testing.TestService", + "method": "EmptyCall" + } + ], + "waitForReady": false, + "timeout": ".001s" + } + ] +}`) + r.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: newSC}) + + if err := verifyResultWithDelay(func() (bool, error) { + cm := channelz.GetChannel(cid) + + var es []string + trace := cm.Trace() + for i := len(trace.Events) - 1; i >= 0; i-- { + if strings.Contains(trace.Events[i].Desc, "service config updated") { + break + } + es = append(es, trace.Events[i].Desc) + if i == 0 { + return false, fmt.Errorf("events do not contain expected address resolution of new service config\n Events:\n%v", strings.Join(es, "\n")) + } + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + r.UpdateState(resolver.State{Addresses: []resolver.Address{}, ServiceConfig: newSC}) + + if err := verifyResultWithDelay(func() (bool, error) { + cm := channelz.GetChannel(cid) + trace := cm.Trace() + for i := len(trace.Events) - 1; i >= 0; i-- { + if strings.Contains(trace.Events[i].Desc, "resolver returned an empty address list") { + break + } + if i == 0 { + return false, fmt.Errorf("events do not contain expected address resolution of empty address") + } + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZSubChannelPickedNewAddress(t *testing.T) { + e := tcpClearRREnv + e.balancer = "" + te := newTest(t, e) + te.startServers(&testServer{security: e.security}, 3) + r := manual.NewBuilderWithScheme("whatever") + var svrAddrs []resolver.Address + for _, a := range te.srvAddrs { + svrAddrs = append(svrAddrs, resolver.Address{Addr: a}) + } + r.InitialState(resolver.State{Addresses: svrAddrs}) + te.resolverScheme = r.Scheme() + cc := te.clientConn(grpc.WithResolvers(r)) + defer te.tearDown() + tc := testgrpc.NewTestServiceClient(cc) + // make sure the connection is up + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + te.srvs[0].Stop() + te.srvs[1].Stop() + // Here, we just wait for all sockets to be up. Make several rpc calls to + // create the sockets since we do not automatically reconnect. + done := make(chan struct{}) + defer close(done) + go func() { + for { + tc.EmptyCall(ctx, &testpb.Empty{}) + select { + case <-time.After(10 * time.Millisecond): + case <-done: + return + } + } + }() + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + subChans := tcs[0].SubChans() + if len(subChans) != 1 { + return false, fmt.Errorf("there should be 1 subchannel not %d", len(subChans)) + } + var subConn int64 + for k := range subChans { + subConn = k + } + scm := channelz.GetSubChannel(subConn) + trace := scm.Trace() + if trace == nil { + return false, fmt.Errorf("trace for SubChannel should not be empty") + } + if len(trace.Events) == 0 { + return false, fmt.Errorf("there should be at least one trace event for subChannel not 0") + } + for i := len(trace.Events) - 1; i >= 0; i-- { + if strings.Contains(trace.Events[i].Desc, fmt.Sprintf("Subchannel picks a new address %q to connect", te.srvAddrs[2])) { + break + } + if i == 0 { + return false, fmt.Errorf("events do not contain expected address resolution of subchannel picked new address") + } + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZSubChannelConnectivityState(t *testing.T) { + e := tcpClearRREnv + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}}) + te.resolverScheme = r.Scheme() + cc := te.clientConn(grpc.WithResolvers(r)) + defer te.tearDown() + tc := testgrpc.NewTestServiceClient(cc) + // make sure the connection is up + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + te.srv.Stop() + + var subConn int64 + if err := verifyResultWithDelay(func() (bool, error) { + // we need to obtain the SubChannel id before it gets deleted from Channel's children list (due + // to effect of r.UpdateState(resolver.State{Addresses:[]resolver.Address{}})) + if subConn == 0 { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + subChans := tcs[0].SubChans() + if len(subChans) != 1 { + return false, fmt.Errorf("there should be 1 subchannel not %d", len(subChans)) + } + for k := range subChans { + // get the SubChannel id for further trace inquiry. + subConn = k + t.Logf("SubChannel Id is %d", subConn) + } + } + scm := channelz.GetSubChannel(subConn) + if scm == nil { + return false, fmt.Errorf("subChannel should still exist due to parent's trace reference") + } + trace := scm.Trace() + if trace == nil { + return false, fmt.Errorf("trace for SubChannel should not be empty") + } + if len(trace.Events) == 0 { + return false, fmt.Errorf("there should be at least one trace event for subChannel not 0") + } + var ready, connecting, transient, shutdown int + t.Log("SubChannel trace events seen so far...") + for _, e := range trace.Events { + t.Log(e.Desc) + if strings.Contains(e.Desc, fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.TransientFailure)) { + transient++ + } + } + // Make sure the SubChannel has already seen transient failure before shutting it down through + // r.UpdateState(resolver.State{Addresses:[]resolver.Address{}}). + if transient == 0 { + return false, fmt.Errorf("transient failure has not happened on SubChannel yet") + } + transient = 0 + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "fake address"}}}) + t.Log("SubChannel trace events seen so far...") + for _, e := range trace.Events { + t.Log(e.Desc) + if strings.Contains(e.Desc, fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.Ready)) { + ready++ + } + if strings.Contains(e.Desc, fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.Connecting)) { + connecting++ + } + if strings.Contains(e.Desc, fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.TransientFailure)) { + transient++ + } + if strings.Contains(e.Desc, fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.Shutdown)) { + shutdown++ + } + } + // example: + // Subchannel Created + // Subchannel's connectivity state changed to CONNECTING + // Subchannel picked a new address: "localhost:36011" + // Subchannel's connectivity state changed to READY + // Subchannel's connectivity state changed to TRANSIENT_FAILURE + // Subchannel's connectivity state changed to CONNECTING + // Subchannel picked a new address: "localhost:36011" + // Subchannel's connectivity state changed to SHUTDOWN + // Subchannel Deleted + if ready != 1 || connecting < 1 || transient < 1 || shutdown != 1 { + return false, fmt.Errorf("got: ready = %d, connecting = %d, transient = %d, shutdown = %d, want: 1, >=1, >=1, 1", ready, connecting, transient, shutdown) + } + + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZChannelConnectivityState(t *testing.T) { + e := tcpClearRREnv + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}}) + te.resolverScheme = r.Scheme() + cc := te.clientConn(grpc.WithResolvers(r)) + defer te.tearDown() + tc := testgrpc.NewTestServiceClient(cc) + // make sure the connection is up + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + te.srv.Stop() + + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + + var ready, connecting, transient int + t.Log("Channel trace events seen so far...") + for _, e := range tcs[0].Trace().Events { + t.Log(e.Desc) + if strings.Contains(e.Desc, fmt.Sprintf("Channel Connectivity change to %v", connectivity.Ready)) { + ready++ + } + if strings.Contains(e.Desc, fmt.Sprintf("Channel Connectivity change to %v", connectivity.Connecting)) { + connecting++ + } + if strings.Contains(e.Desc, fmt.Sprintf("Channel Connectivity change to %v", connectivity.TransientFailure)) { + transient++ + } + } + + // example: + // Channel Created + // Addresses resolved (from empty address state): "localhost:40467" + // SubChannel (id: 4[]) Created + // Channel's connectivity state changed to CONNECTING + // Channel's connectivity state changed to READY + // Channel's connectivity state changed to TRANSIENT_FAILURE + // Channel's connectivity state changed to CONNECTING + // Channel's connectivity state changed to TRANSIENT_FAILURE + if ready != 1 || connecting < 1 || transient < 1 { + return false, fmt.Errorf("got: ready = %d, connecting = %d, transient = %d, want: 1, >=1, >=1", ready, connecting, transient) + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZTraceOverwriteChannelDeletion(t *testing.T) { + e := tcpClearRREnv + e.balancer = "" + te := newTest(t, e) + channelz.SetMaxTraceEntry(1) + defer channelz.ResetMaxTraceEntryToDefault() + r := manual.NewBuilderWithScheme("whatever") + te.resolverScheme = r.Scheme() + te.clientConn(grpc.WithResolvers(r)) + resolvedAddrs := []resolver.Address{{Addr: "127.0.0.1:0", ServerName: "grpclb.server"}} + grpclbConfig := parseServiceConfig(t, r, `{"loadBalancingPolicy": "grpclb"}`) + r.UpdateState(grpclbstate.Set(resolver.State{ServiceConfig: grpclbConfig}, &grpclbstate.State{BalancerAddresses: resolvedAddrs})) + defer te.tearDown() + var nestedConn int64 + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + nestedChans := tcs[0].NestedChans() + if len(nestedChans) != 1 { + return false, fmt.Errorf("there should be one nested channel from grpclb, not %d", len(nestedChans)) + } + for k := range nestedChans { + nestedConn = k + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: "127.0.0.1:0"}}, + ServiceConfig: parseServiceConfig(t, r, `{"loadBalancingPolicy": "round_robin"}`), + }) + + // wait for the shutdown of grpclb balancer + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + + if nestedChans := tcs[0].NestedChans(); len(nestedChans) != 0 { + return false, fmt.Errorf("there should be 0 nested channel from grpclb, not %d", len(nestedChans)) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + // If nested channel deletion is last trace event before the next validation, it will fail, as the top channel will hold a reference to it. + // This line forces a trace event on the top channel in that case. + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: "127.0.0.1:0"}}, + ServiceConfig: parseServiceConfig(t, r, `{"loadBalancingPolicy": "round_robin"}`), + }) + + // verify that the nested channel no longer exist due to trace referencing it got overwritten. + if err := verifyResultWithDelay(func() (bool, error) { + cm := channelz.GetChannel(nestedConn) + if cm != nil { + return false, fmt.Errorf("nested channel should have been deleted since its parent's trace should not contain any reference to it anymore") + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZTraceOverwriteSubChannelDeletion(t *testing.T) { + e := tcpClearRREnv + te := newTest(t, e) + channelz.SetMaxTraceEntry(1) + defer channelz.ResetMaxTraceEntryToDefault() + te.startServer(&testServer{security: e.security}) + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}}) + te.resolverScheme = r.Scheme() + te.clientConn(grpc.WithResolvers(r)) + defer te.tearDown() + var subConn int64 + // Here, we just wait for all sockets to be up. In the future, if we implement + // IDLE, we may need to make several rpc calls to create the sockets. + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + subChans := tcs[0].SubChans() + if len(subChans) != 1 { + return false, fmt.Errorf("there should be 1 subchannel not %d", len(subChans)) + } + for k := range subChans { + subConn = k + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, te.cc, connectivity.Ready) + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "fake address"}}}) + testutils.AwaitNotState(ctx, t, te.cc, connectivity.Ready) + + // verify that the subchannel no longer exist due to trace referencing it got overwritten. + if err := verifyResultWithDelay(func() (bool, error) { + cm := channelz.GetChannel(subConn) + if cm != nil { + return false, fmt.Errorf("subchannel should have been deleted since its parent's trace should not contain any reference to it anymore") + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestCZTraceTopChannelDeletionTraceClear(t *testing.T) { + e := tcpClearRREnv + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}}) + te.resolverScheme = r.Scheme() + te.clientConn(grpc.WithResolvers(r)) + var subConn int64 + // Here, we just wait for all sockets to be up. In the future, if we implement + // IDLE, we may need to make several rpc calls to create the sockets. + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + return false, fmt.Errorf("there should only be one top channel, not %d", len(tcs)) + } + subChans := tcs[0].SubChans() + if len(subChans) != 1 { + return false, fmt.Errorf("there should be 1 subchannel not %d", len(subChans)) + } + for k := range subChans { + subConn = k + } + return true, nil + }); err != nil { + t.Fatal(err) + } + te.tearDown() + // verify that the subchannel no longer exist due to parent channel got deleted and its trace cleared. + if err := verifyResultWithDelay(func() (bool, error) { + cm := channelz.GetChannel(subConn) + if cm != nil { + return false, fmt.Errorf("subchannel should have been deleted since its parent's trace should not contain any reference to it anymore") + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} diff --git a/test/clientconn_state_transition_test.go b/test/clientconn_state_transition_test.go new file mode 100644 index 0000000..3d365bd --- /dev/null +++ b/test/clientconn_state_transition_test.go @@ -0,0 +1,832 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "io" + "net" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/net/http2" + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// Keep reading until something causes the connection to die (EOF, server +// closed, etc). Useful as a tool for mindlessly keeping the connection +// healthy, since the client will error if things like client prefaces are not +// accepted in a timely fashion. +func keepReading(conn net.Conn) { + io.Copy(io.Discard, conn) +} + +type funcConnectivityStateSubscriber struct { + onMsg func(connectivity.State) +} + +func (f *funcConnectivityStateSubscriber) OnMessage(msg any) { + f.onMsg(msg.(connectivity.State)) +} + +func waitForState(ctx context.Context, t *testing.T, stateCh <-chan connectivity.State, want connectivity.State) { + t.Helper() + select { + case gotState := <-stateCh: + if gotState != want { + t.Fatalf("State is %s; want %s", gotState, want) + } + t.Logf("State is %s as expected", gotState) + case <-ctx.Done(): + t.Fatalf("Timed out waiting for state update: %s", want) + } +} + +// Tests for state transitions in various scenarios with a single address. +func (s) TestStateTransitions_SingleAddress(t *testing.T) { + for _, test := range []struct { + desc string + wantStates []connectivity.State + server func(net.Listener) net.Conn + }{ + { + desc: "ServerSendsPreface", + wantStates: []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + }, + server: func(lis net.Listener) net.Conn { + conn, err := lis.Accept() + if err != nil { + t.Error(err) + return nil + } + + go keepReading(conn) + + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings frame. %v", err) + return nil + } + + return conn + }, + }, + { + desc: "ConnectionClosesBeforeServerPreface", + wantStates: []connectivity.State{ + connectivity.Connecting, + connectivity.TransientFailure, + }, + server: func(lis net.Listener) net.Conn { + conn, err := lis.Accept() + if err != nil { + t.Error(err) + return nil + } + + conn.Close() + return nil + }, + }, + { + desc: "ConnectionClosesBeforeClientPreface", + wantStates: []connectivity.State{ + connectivity.Connecting, + connectivity.TransientFailure, + }, + server: func(lis net.Listener) net.Conn { + conn, err := lis.Accept() + if err != nil { + t.Error(err) + return nil + } + + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings frame. %v", err) + return nil + } + + conn.Close() + return nil + }, + }, + { + desc: "ServerNeverSendsPreface", + wantStates: []connectivity.State{ + connectivity.Connecting, + connectivity.TransientFailure, + }, + server: func(lis net.Listener) net.Conn { + conn, err := lis.Accept() + if err != nil { + t.Error(err) + return nil + } + + go keepReading(conn) + + return conn + }, + }, + } { + t.Run(test.desc, func(t *testing.T) { + testStateTransitionSingleAddress(t, test.wantStates, test.server) + }) + } +} + +func testStateTransitionSingleAddress(t *testing.T, wantStates []connectivity.State, server func(net.Listener) net.Conn) { + pl := testutils.NewPipeListener() + defer pl.Close() + + // Launch the server. + var conn net.Conn + var connMu sync.Mutex + go func() { + connMu.Lock() + conn = server(pl) + connMu.Unlock() + }() + + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDialer(pl.Dialer()), + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.Config{}, + MinConnectTimeout: 100 * time.Millisecond, + }), + } + cc, err := grpc.NewClient("passthrough:///", dopts...) + if err != nil { + t.Fatal(err) + } + defer cc.Close() + + // Ensure that the client is in IDLE before connecting. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Subscribe to state updates. + stateCh := make(chan connectivity.State, 1) + s := &funcConnectivityStateSubscriber{ + onMsg: func(s connectivity.State) { + select { + case stateCh <- s: + case <-ctx.Done(): + } + }, + } + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s) + + cc.Connect() + for _, wantState := range wantStates { + waitForState(ctx, t, stateCh, wantState) + } + + connMu.Lock() + defer connMu.Unlock() + if conn != nil { + err = conn.Close() + if err != nil { + t.Fatal(err) + } + } +} + +// Tests for state transitions when the READY connection is closed. +func (s) TestStateTransitions_ReadyToConnecting(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis.Close() + + sawReady := make(chan struct{}, 1) + defer close(sawReady) + + // Launch the server. + go func() { + conn, err := lis.Accept() + if err != nil { + t.Error(err) + return + } + + go keepReading(conn) + + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings frame. %v", err) + return + } + + // Prevents race between onPrefaceReceipt and onClose. + <-sawReady + + conn.Close() + }() + + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + defer cc.Close() + + // Ensure that the client is in IDLE before connecting. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Subscribe to state updates. + stateCh := make(chan connectivity.State, 1) + s := &funcConnectivityStateSubscriber{ + onMsg: func(s connectivity.State) { + select { + case stateCh <- s: + case <-ctx.Done(): + } + }, + } + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s) + + cc.Connect() + wantStates := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + } + for _, wantState := range wantStates { + waitForState(ctx, t, stateCh, wantState) + if wantState == connectivity.Ready { + sawReady <- struct{}{} + } + if wantState == connectivity.Idle { + cc.Connect() + } + } +} + +// Tests for state transitions when there are multiple addresses and all the +// addresses fail. +func (s) TestStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) { + lis1, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis1.Close() + + lis2, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis2.Close() + + server1Done := make(chan struct{}) + server2Done := make(chan struct{}) + + // Launch server 1. + go func() { + conn, err := lis1.Accept() + if err != nil { + t.Error(err) + return + } + + conn.Close() + close(server1Done) + }() + // Launch server 2. + go func() { + conn, err := lis2.Accept() + if err != nil { + t.Error(err) + return + } + + conn.Close() + close(server2Done) + }() + + rb := manual.NewBuilderWithScheme("whatever") + rb.InitialState(resolver.State{Addresses: []resolver.Address{ + {Addr: lis1.Addr().String()}, + {Addr: lis2.Addr().String()}, + }}) + + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(rb), + grpc.WithConnectParams(grpc.ConnectParams{ + // Set a really long back-off delay to ensure the subchannels stay + // in TRANSIENT_FAILURE and not enter IDLE. + Backoff: backoff.Config{BaseDelay: 1 * time.Hour}, + }), + } + cc, err := grpc.NewClient("whatever:///this-gets-overwritten", dopts...) + if err != nil { + t.Fatal(err) + } + defer cc.Close() + + // Ensure that the client is in IDLE before connecting. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Subscribe to state updates. + stateCh := make(chan connectivity.State, 1) + s := &funcConnectivityStateSubscriber{ + onMsg: func(s connectivity.State) { + select { + case stateCh <- s: + case <-ctx.Done(): + } + }, + } + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s) + + cc.Connect() + wantStates := []connectivity.State{ + connectivity.Connecting, + connectivity.TransientFailure, + } + for _, wantState := range wantStates { + waitForState(ctx, t, stateCh, wantState) + } + + select { + case <-ctx.Done(): + t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 1") + case <-server1Done: + } + select { + case <-ctx.Done(): + t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 2") + case <-server2Done: + } +} + +// Tests for state transitions with multiple addresses when the READY connection +// is closed. +func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) { + lis1, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis1.Close() + + // Never actually gets used; we just want it to be alive so that the + // resolver has two addresses to target. + lis2, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis2.Close() + + server1Done := make(chan struct{}) + sawReady := make(chan struct{}, 1) + defer close(sawReady) + + // Launch server 1. + go func() { + conn, err := lis1.Accept() + if err != nil { + t.Error(err) + return + } + + go keepReading(conn) + + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings frame. %v", err) + return + } + + <-sawReady + + conn.Close() + + close(server1Done) + }() + + rb := manual.NewBuilderWithScheme("whatever") + rb.InitialState(resolver.State{Addresses: []resolver.Address{ + {Addr: lis1.Addr().String()}, + {Addr: lis2.Addr().String()}, + }}) + cc, err := grpc.NewClient("whatever:///this-gets-overwritten", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(rb)) + if err != nil { + t.Fatal(err) + } + defer cc.Close() + + // Ensure that the client is in IDLE before connecting. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Subscribe to state updates. + stateCh := make(chan connectivity.State, 1) + s := &funcConnectivityStateSubscriber{ + onMsg: func(s connectivity.State) { + select { + case stateCh <- s: + case <-ctx.Done(): + } + }, + } + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s) + + cc.Connect() + wantStates := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + } + for _, wantState := range wantStates { + waitForState(ctx, t, stateCh, wantState) + if wantState == connectivity.Ready { + sawReady <- struct{}{} + } + if wantState == connectivity.Idle { + cc.Connect() + } + } + select { + case <-ctx.Done(): + t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 1") + case <-server1Done: + } +} + +// TestConnectivityStateSubscriber confirms updates sent by the balancer in +// rapid succession are not missed by the subscriber. +func (s) TestConnectivityStateSubscriber(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + sendStates := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + } + wantStates := append(sendStates, connectivity.Shutdown) + + const testBalName = "any" + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + // Send the expected states in rapid succession. + for _, s := range sendStates { + t.Logf("Sending state update %s", s) + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: s}) + } + return nil + }, + } + stub.Register(testBalName, bf) + + // Create the ClientConn. + const testResName = "any" + rb := manual.NewBuilderWithScheme(testResName) + cc, err := grpc.NewClient(testResName+":///", + grpc.WithResolvers(rb), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, testBalName)), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + // Subscribe to state updates. Use a buffer size of 1 to allow the + // Shutdown state to go into the channel when Close()ing. + connCh := make(chan connectivity.State, 1) + s := &funcConnectivityStateSubscriber{ + onMsg: func(s connectivity.State) { + select { + case connCh <- s: + case <-ctx.Done(): + } + if s == connectivity.Shutdown { + close(connCh) + } + }, + } + + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s) + + // Send an update from the resolver that will trigger the LB policy's UpdateClientConnState. + go rb.UpdateState(resolver.State{}) + + // Verify the resulting states. + for i, want := range wantStates { + if i == len(sendStates) { + // Trigger Shutdown to be sent by the channel. Use a goroutine to + // ensure the operation does not block. + cc.Close() + } + select { + case got := <-connCh: + if got != want { + t.Errorf("Update %v was %s; want %s", i, got, want) + } else { + t.Logf("Update %v was %s as expected", i, got) + } + case <-ctx.Done(): + t.Fatalf("Timed out waiting for state update %v: %s", i, want) + } + } +} + +// Test verifies that a channel starts off in IDLE and transitions to CONNECTING +// when Connect() is called, and stays there when there are no resolver updates. +func (s) TestStateTransitions_WithConnect_NoResolverUpdate(t *testing.T) { + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + + mr := manual.NewBuilderWithScheme("e2e-test") + defer mr.Close() + + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create new client: %v", err) + } + defer cc.Close() + + if state := cc.GetState(); state != connectivity.Idle { + t.Fatalf("Expected initial state to be IDLE, got %v", state) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // The channel should transition to CONNECTING automatically when Connect() + // is called. + cc.Connect() + testutils.AwaitState(ctx, t, cc, connectivity.Connecting) + + // Verify that the channel remains in CONNECTING state for a short time. + shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer shortCancel() + testutils.AwaitNoStateChange(shortCtx, t, cc, connectivity.Connecting) +} + +// Test verifies that a channel starts off in IDLE and transitions to CONNECTING +// when Connect() is called, and stays there when there are no resolver updates. +func (s) TestStateTransitions_WithRPC_NoResolverUpdate(t *testing.T) { + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + + mr := manual.NewBuilderWithScheme("e2e-test") + defer mr.Close() + + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create new client: %v", err) + } + defer cc.Close() + + if state := cc.GetState(); state != connectivity.Idle { + t.Fatalf("Expected initial state to be IDLE, got %v", state) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Make an RPC call to transition the channel to CONNECTING. + go func() { + if _, err := testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Errorf("Expected RPC to fail, but it succeeded") + } + }() + + // The channel should transition to CONNECTING automatically when an RPC + // is made. + testutils.AwaitState(ctx, t, cc, connectivity.Connecting) + + // The channel remains in CONNECTING state for a short time. + shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer shortCancel() + testutils.AwaitNoStateChange(shortCtx, t, cc, connectivity.Connecting) +} + +const testResolverBuildFailureScheme = "test-resolver-build-failure" + +// testResolverBuilder is a resolver builder that fails the first time its +// Build method is called, and succeeds thereafter. +type testResolverBuilder struct { + logger interface { + Logf(format string, args ...any) + } + buildCalled bool + manualR *manual.Resolver +} + +func (b *testResolverBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { + b.logger.Logf("testResolverBuilder: Build called with target: %v", target) + if !b.buildCalled { + b.buildCalled = true + b.logger.Logf("testResolverBuilder: returning build failure") + return nil, fmt.Errorf("simulated resolver build failure") + } + return b.manualR.Build(target, cc, opts) +} + +func (b *testResolverBuilder) Scheme() string { + return testResolverBuildFailureScheme +} + +// Tests for state transitions when the resolver initially fails to build. +func (s) TestStateTransitions_ResolverBuildFailure(t *testing.T) { + tests := []struct { + name string + exitIdleWithRPC bool + }{ + { + name: "exitIdleByConnecting", + exitIdleWithRPC: false, + }, + { + name: "exitIdleByRPC", + exitIdleWithRPC: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mr := manual.NewBuilderWithScheme("whatever" + tt.name) + backend := stubserver.StartTestService(t, nil) + defer backend.Stop() + mr.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(&testResolverBuilder{logger: t, manualR: mr}), + } + + cc, err := grpc.NewClient(testResolverBuildFailureScheme+":///", dopts...) + if err != nil { + t.Fatalf("Failed to create new client: %v", err) + } + defer cc.Close() + + // Ensure that the client is in IDLE before connecting. + if state := cc.GetState(); state != connectivity.Idle { + t.Fatalf("Expected initial state to be IDLE, got %v", state) + } + + // Subscribe to state updates. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stateCh := make(chan connectivity.State, 1) + s := &funcConnectivityStateSubscriber{ + onMsg: func(s connectivity.State) { + select { + case stateCh <- s: + case <-ctx.Done(): + } + }, + } + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s) + + if tt.exitIdleWithRPC { + // The first attempt to kick the channel is expected to return + // the resolver build error to the RPC. + const wantErr = "simulated resolver build failure" + for range 2 { + _, err := testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}) + if code := status.Code(err); code != codes.Unavailable { + t.Fatalf("EmptyCall RPC failed with code %v, want %v", err, codes.Unavailable) + } + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall RPC failed with error: %q, want %q", err, wantErr) + } + } + } else { + cc.Connect() + } + + wantStates := []connectivity.State{ + connectivity.Connecting, // When channel exits IDLE for the first time. + connectivity.TransientFailure, // Resolver build failure. + connectivity.Idle, // After idle timeout. + connectivity.Connecting, // When channel exits IDLE again. + connectivity.Ready, // Successful resolver build and connection to backend. + } + for _, wantState := range wantStates { + waitForState(ctx, t, stateCh, wantState) + switch wantState { + case connectivity.TransientFailure: + internal.EnterIdleModeForTesting.(func(*grpc.ClientConn))(cc) + case connectivity.Idle: + if tt.exitIdleWithRPC { + if _, err := testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall RPC failed: %v", err) + } + } else { + cc.Connect() + } + } + } + }) + } +} + +// Tests for state transitions when the resolver reports no addresses. +func (s) TestStateTransitions_WithRPC_ResolverUpdateContainsNoAddresses(t *testing.T) { + mr := manual.NewBuilderWithScheme("e2e-test") + mr.InitialState(resolver.State{}) + defer mr.Close() + + cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create new client: %v", err) + } + defer cc.Close() + + if state := cc.GetState(); state != connectivity.Idle { + t.Fatalf("Expected initial state to be IDLE, got %v", state) + } + + // Subscribe to state updates. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stateCh := make(chan connectivity.State, 1) + s := &funcConnectivityStateSubscriber{ + onMsg: func(s connectivity.State) { + select { + case stateCh <- s: + case <-ctx.Done(): + } + }, + } + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s) + + // Make an RPC call to transition the channel to CONNECTING. + const wantErr = "name resolver error: produced zero addresses" + for range 2 { + _, err := testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}) + if code := status.Code(err); code != codes.Unavailable { + t.Errorf("EmptyCall RPC failed with code %v, want %v", err, codes.Unavailable) + } + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Errorf("EmptyCall RPC failed with error: %q, want %q", err, wantErr) + } + } + + wantStates := []connectivity.State{ + connectivity.Connecting, // When channel exits IDLE for the first time. + connectivity.TransientFailure, // No endpoints from the resolver + connectivity.Idle, // After idle timeout. + } + for _, wantState := range wantStates { + waitForState(ctx, t, stateCh, wantState) + if wantState == connectivity.TransientFailure { + internal.EnterIdleModeForTesting.(func(*grpc.ClientConn))(cc) + } + } +} diff --git a/test/clientconn_test.go b/test/clientconn_test.go new file mode 100644 index 0000000..f56ab4d --- /dev/null +++ b/test/clientconn_test.go @@ -0,0 +1,197 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +// TestClientConnClose_WithPendingRPC tests the scenario where the channel has +// not yet received any update from the name resolver and hence RPCs are +// blocking. The test verifies that closing the ClientConn unblocks the RPC with +// the expected error code. +func (s) TestClientConnClose_WithPendingRPC(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + client := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + doneErrCh := make(chan error, 1) + go func() { + // This RPC would block until the ClientConn is closed, because the + // resolver has not provided its first update yet. + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if status.Code(err) != codes.Canceled || !strings.Contains(err.Error(), "client connection is closing") { + doneErrCh <- fmt.Errorf("EmptyCall() = %v, want %s", err, codes.Canceled) + } + doneErrCh <- nil + }() + + // Make sure that there is one pending RPC on the ClientConn before attempting + // to close it. If we don't do this, cc.Close() can happen before the above + // goroutine gets to make the RPC. + for { + if err := ctx.Err(); err != nil { + t.Fatal(err) + } + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + t.Fatalf("there should only be one top channel, not %d", len(tcs)) + } + started := tcs[0].ChannelMetrics.CallsStarted.Load() + completed := tcs[0].ChannelMetrics.CallsSucceeded.Load() + tcs[0].ChannelMetrics.CallsFailed.Load() + if (started - completed) == 1 { + break + } + time.Sleep(defaultTestShortTimeout) + } + cc.Close() + if err := <-doneErrCh; err != nil { + t.Fatal(err) + } +} + +type testStatsHandler struct { + nameResolutionDelayed bool +} + +// TagRPC is called when an RPC is initiated and allows adding metadata to the +// context. It checks if the RPC experienced a name resolution delay and +// updates the handler's state. +func (h *testStatsHandler) TagRPC(ctx context.Context, rpcInfo *stats.RPCTagInfo) context.Context { + h.nameResolutionDelayed = rpcInfo.NameResolutionDelay + return ctx +} + +// This method is required to satisfy the stats.Handler interface. +func (h *testStatsHandler) HandleRPC(_ context.Context, _ stats.RPCStats) {} + +// TagConn exists to satisfy stats.Handler. +func (h *testStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +// HandleConn exists to satisfy stats.Handler. +func (h *testStatsHandler) HandleConn(_ context.Context, _ stats.ConnStats) {} + +// TestClientConnRPC_WithoutNameResolutionDelay verify that if the resolution +// has already happened once before at the time of making RPC, the name +// resolution flag is not set indicating there was no delay in name resolution. +func (s) TestClientConnRPC_WithoutNameResolutionDelay(t *testing.T) { + statsHandler := &testStatsHandler{} + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil, grpc.WithStatsHandler(statsHandler)); err != nil { + t.Fatalf("Failed to start StubServer: %v", err) + } + defer ss.Stop() + + rb := manual.NewBuilderWithScheme("instant") + rb.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}}) + cc := ss.CC + defer cc.Close() + + cc.Connect() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + client := testgrpc.NewTestServiceClient(cc) + // Verify that the RPC succeeds. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("First RPC failed unexpectedly: %v", err) + } + // verifying that RPC was not blocked on resolver indicating there was no + // delay in name resolution. + if statsHandler.nameResolutionDelayed { + t.Fatalf("statsHandler.nameResolutionDelayed = %v; want false", statsHandler.nameResolutionDelayed) + } +} + +// TestStatsHandlerDetectsResolutionDelay verifies that if this is the +// first time resolution is happening at the time of making RPC, +// nameResolutionDelayed flag is set indicating there was a delay in name +// resolution waiting for resolver to return addresses. +func (s) TestClientConnRPC_WithNameResolutionDelay(t *testing.T) { + resolutionWait := grpcsync.NewEvent() + prevHook := internal.NewStreamWaitingForResolver + internal.NewStreamWaitingForResolver = func() { resolutionWait.Fire() } + defer func() { internal.NewStreamWaitingForResolver = prevHook }() + + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Failed to start StubServer: %v", err) + } + defer ss.Stop() + + statsHandler := &testStatsHandler{} + rb := manual.NewBuilderWithScheme("delayed") + cc, err := grpc.NewClient(rb.Scheme()+":///test.server", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(rb), + grpc.WithStatsHandler(statsHandler), + ) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + go func() { + <-resolutionWait.Done() + rb.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}}) + }() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall RPC failed: %v", err) + } + if !statsHandler.nameResolutionDelayed { + t.Fatalf("statsHandler.nameResolutionDelayed = %v; want true", statsHandler.nameResolutionDelayed) + } +} diff --git a/test/clienttester.go b/test/clienttester.go new file mode 100644 index 0000000..7e22309 --- /dev/null +++ b/test/clienttester.go @@ -0,0 +1,109 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package test + +import ( + "bytes" + "io" + "net" + "testing" + + "golang.org/x/net/http2" +) + +var ( + clientPreface = []byte(http2.ClientPreface) +) + +func newClientTester(t *testing.T, conn net.Conn) *clientTester { + ct := &clientTester{ + t: t, + conn: conn, + } + ct.fr = http2.NewFramer(conn, conn) + ct.greet() + return ct +} + +type clientTester struct { + t *testing.T + conn net.Conn + fr *http2.Framer +} + +// greet() performs the necessary steps for http2 connection establishment on +// the server side. +func (ct *clientTester) greet() { + ct.wantClientPreface() + ct.wantSettingsFrame() + ct.writeSettingsFrame() + ct.writeSettingsAck() + + for { + f, err := ct.fr.ReadFrame() + if err != nil { + ct.t.Errorf("error reading frame from client side: %v", err) + } + switch f := f.(type) { + case *http2.SettingsFrame: + if f.IsAck() { // HTTP/2 handshake completed. + return + } + default: + ct.t.Errorf("during greet, unexpected frame type %T", f) + } + } +} + +func (ct *clientTester) wantClientPreface() { + preface := make([]byte, len(clientPreface)) + if _, err := io.ReadFull(ct.conn, preface); err != nil { + ct.t.Errorf("Error at server-side while reading preface from client. Err: %v", err) + } + if !bytes.Equal(preface, clientPreface) { + ct.t.Errorf("received bogus greeting from client %q", preface) + } +} + +func (ct *clientTester) wantSettingsFrame() { + frame, err := ct.fr.ReadFrame() + if err != nil { + ct.t.Errorf("error reading initial settings frame from client: %v", err) + } + _, ok := frame.(*http2.SettingsFrame) + if !ok { + ct.t.Errorf("initial frame sent from client is not a settings frame, type %T", frame) + } +} + +func (ct *clientTester) writeSettingsFrame() { + if err := ct.fr.WriteSettings(); err != nil { + ct.t.Fatalf("Error writing initial SETTINGS frame from client to server: %v", err) + } +} + +func (ct *clientTester) writeSettingsAck() { + if err := ct.fr.WriteSettingsAck(); err != nil { + ct.t.Fatalf("Error writing ACK of client's SETTINGS: %v", err) + } +} + +func (ct *clientTester) writeGoAway(maxStreamID uint32, code http2.ErrCode, debugData []byte) { + if err := ct.fr.WriteGoAway(maxStreamID, code, debugData); err != nil { + ct.t.Fatalf("Error writing GOAWAY: %v", err) + } +} diff --git a/test/codec_perf/perf.pb.go b/test/codec_perf/perf.pb.go new file mode 100644 index 0000000..709b558 --- /dev/null +++ b/test/codec_perf/perf.pb.go @@ -0,0 +1,142 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Messages used for performance tests that may not reference grpc directly for +// reasons of import cycles. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.27.1 +// source: test/codec_perf/perf.proto + +package codec_perf + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Buffer is a message that contains a body of bytes that is used to exercise +// encoding and decoding overheads. +type Buffer struct { + state protoimpl.MessageState `protogen:"open.v1"` + Body []byte `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Buffer) Reset() { + *x = Buffer{} + mi := &file_test_codec_perf_perf_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Buffer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Buffer) ProtoMessage() {} + +func (x *Buffer) ProtoReflect() protoreflect.Message { + mi := &file_test_codec_perf_perf_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Buffer.ProtoReflect.Descriptor instead. +func (*Buffer) Descriptor() ([]byte, []int) { + return file_test_codec_perf_perf_proto_rawDescGZIP(), []int{0} +} + +func (x *Buffer) GetBody() []byte { + if x != nil { + return x.Body + } + return nil +} + +var File_test_codec_perf_perf_proto protoreflect.FileDescriptor + +const file_test_codec_perf_perf_proto_rawDesc = "" + + "\n" + + "\x1atest/codec_perf/perf.proto\x12\n" + + "codec.perf\"\x1c\n" + + "\x06Buffer\x12\x12\n" + + "\x04body\x18\x01 \x01(\fR\x04bodyB(Z&google.golang.org/grpc/test/codec_perfb\x06proto3" + +var ( + file_test_codec_perf_perf_proto_rawDescOnce sync.Once + file_test_codec_perf_perf_proto_rawDescData []byte +) + +func file_test_codec_perf_perf_proto_rawDescGZIP() []byte { + file_test_codec_perf_perf_proto_rawDescOnce.Do(func() { + file_test_codec_perf_perf_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_test_codec_perf_perf_proto_rawDesc), len(file_test_codec_perf_perf_proto_rawDesc))) + }) + return file_test_codec_perf_perf_proto_rawDescData +} + +var file_test_codec_perf_perf_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_test_codec_perf_perf_proto_goTypes = []any{ + (*Buffer)(nil), // 0: codec.perf.Buffer +} +var file_test_codec_perf_perf_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_test_codec_perf_perf_proto_init() } +func file_test_codec_perf_perf_proto_init() { + if File_test_codec_perf_perf_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_test_codec_perf_perf_proto_rawDesc), len(file_test_codec_perf_perf_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_test_codec_perf_perf_proto_goTypes, + DependencyIndexes: file_test_codec_perf_perf_proto_depIdxs, + MessageInfos: file_test_codec_perf_perf_proto_msgTypes, + }.Build() + File_test_codec_perf_perf_proto = out.File + file_test_codec_perf_perf_proto_goTypes = nil + file_test_codec_perf_perf_proto_depIdxs = nil +} diff --git a/test/codec_perf/perf.proto b/test/codec_perf/perf.proto new file mode 100644 index 0000000..7a678d1 --- /dev/null +++ b/test/codec_perf/perf.proto @@ -0,0 +1,27 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Messages used for performance tests that may not reference grpc directly for +// reasons of import cycles. +syntax = "proto3"; + +option go_package = "google.golang.org/grpc/test/codec_perf"; + +package codec.perf; + +// Buffer is a message that contains a body of bytes that is used to exercise +// encoding and decoding overheads. +message Buffer { + bytes body = 1; +} diff --git a/test/compressor_test.go b/test/compressor_test.go new file mode 100644 index 0000000..ebc42f2 --- /dev/null +++ b/test/compressor_test.go @@ -0,0 +1,694 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "bytes" + "compress/gzip" + "context" + "io" + "reflect" + "strings" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/experimental" + "google.golang.org/grpc/internal/grpcutil" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestUnsupportedEncodingResponse validates gRPC status codes +// for different client-server compression setups +// ensuring the correct behavior when compression is enabled or disabled on either side. +func (s) TestUnsupportedEncodingResponse(t *testing.T) { + tests := []struct { + name string + clientCompress bool + serverCompress bool + wantStatus codes.Code + }{ + { + name: "client_server_compression", + clientCompress: true, + serverCompress: true, + wantStatus: codes.OK, + }, + { + name: "client_compression", + clientCompress: true, + serverCompress: false, + wantStatus: codes.Unimplemented, + }, + { + name: "server_compression", + clientCompress: false, + serverCompress: true, + wantStatus: codes.Internal, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ss := &stubserver.StubServer{ + UnaryCallF: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{Payload: in.Payload}, nil + }, + } + sopts := []grpc.ServerOption{} + if test.serverCompress { + // Using deprecated methods to selectively apply compression + // only on the server side. With encoding.registerCompressor(), + // the compressor is applied globally, affecting client and server + sopts = append(sopts, grpc.RPCCompressor(newNopCompressor()), grpc.RPCDecompressor(newNopDecompressor())) + } + if err := ss.StartServer(sopts...); err != nil { + t.Fatalf("Error starting server: %v", err) + } + defer ss.Stop() + + dopts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} + if test.clientCompress { + // UseCompressor() requires the compressor to be registered + // using encoding.RegisterCompressor() which applies compressor globally, + // Hence, using deprecated WithCompressor() and WithDecompressor() + // to apply compression only on client. + dopts = append(dopts, grpc.WithCompressor(newNopCompressor()), grpc.WithDecompressor(newNopDecompressor())) + } + if err := ss.StartClient(dopts...); err != nil { + t.Fatalf("Error starting client: %v", err) + } + + payload := &testpb.SimpleRequest{ + Payload: &testpb.Payload{ + Body: []byte("test message"), + }, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := ss.Client.UnaryCall(ctx, payload) + if got, want := status.Code(err), test.wantStatus; got != want { + t.Errorf("Client.UnaryCall() = %v, want %v", got, want) + } + }) + } +} + +func (s) TestCompressServerHasNoSupport(t *testing.T) { + for _, e := range listTestEnv() { + testCompressServerHasNoSupport(t, e) + } +} + +func testCompressServerHasNoSupport(t *testing.T, e env) { + te := newTest(t, e) + te.serverCompression = false + te.clientCompression = false + te.clientNopCompression = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + tc := testgrpc.NewTestServiceClient(te.clientConn()) + + const argSize = 271828 + const respSize = 314159 + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, argSize) + if err != nil { + t.Fatal(err) + } + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: respSize, + Payload: payload, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.Unimplemented { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code %s", err, codes.Unimplemented) + } + // Streaming RPC + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + if _, err := stream.Recv(); err == nil || status.Code(err) != codes.Unimplemented { + t.Fatalf("%v.Recv() = %v, want error code %s", stream, err, codes.Unimplemented) + } +} + +func (s) TestCompressOK(t *testing.T) { + for _, e := range listTestEnv() { + testCompressOK(t, e) + } +} + +func testCompressOK(t *testing.T, e env) { + te := newTest(t, e) + te.serverCompression = true + te.clientCompression = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + tc := testgrpc.NewTestServiceClient(te.clientConn()) + + // Unary call + const argSize = 271828 + const respSize = 314159 + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, argSize) + if err != nil { + t.Fatal(err) + } + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: respSize, + Payload: payload, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("something", "something")) + if _, err := tc.UnaryCall(ctx, req); err != nil { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, ", err) + } + // Streaming RPC + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + respParam := []*testpb.ResponseParameters{ + { + Size: 31415, + }, + } + payload, err = newPayload(testpb.PayloadType_COMPRESSABLE, int32(31415)) + if err != nil { + t.Fatal(err) + } + sreq := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: payload, + } + if err := stream.Send(sreq); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) + } + stream.CloseSend() + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = %v, want ", stream, err) + } + if _, err := stream.Recv(); err != io.EOF { + t.Fatalf("%v.Recv() = %v, want io.EOF", stream, err) + } +} + +func (s) TestIdentityEncoding(t *testing.T) { + for _, e := range listTestEnv() { + testIdentityEncoding(t, e) + } +} + +func testIdentityEncoding(t *testing.T, e env) { + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + tc := testgrpc.NewTestServiceClient(te.clientConn()) + + // Unary call + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 5) + if err != nil { + t.Fatal(err) + } + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: 10, + Payload: payload, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("something", "something")) + if _, err := tc.UnaryCall(ctx, req); err != nil { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, ", err) + } + // Streaming RPC + stream, err := tc.FullDuplexCall(ctx, grpc.UseCompressor("identity")) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + payload, err = newPayload(testpb.PayloadType_COMPRESSABLE, int32(31415)) + if err != nil { + t.Fatal(err) + } + sreq := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: []*testpb.ResponseParameters{{Size: 10}}, + Payload: payload, + } + if err := stream.Send(sreq); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) + } + stream.CloseSend() + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = %v, want ", stream, err) + } + if _, err := stream.Recv(); err != io.EOF { + t.Fatalf("%v.Recv() = %v, want io.EOF", stream, err) + } +} + +// renameCompressor is a grpc.Compressor wrapper that allows customizing the +// Type() of another compressor. +type renameCompressor struct { + grpc.Compressor + name string +} + +func (r *renameCompressor) Type() string { return r.name } + +// renameDecompressor is a grpc.Decompressor wrapper that allows customizing the +// Type() of another Decompressor. +type renameDecompressor struct { + grpc.Decompressor + name string +} + +func (r *renameDecompressor) Type() string { return r.name } + +func (s) TestClientForwardsGrpcAcceptEncodingHeader(t *testing.T) { + wantGrpcAcceptEncodingCh := make(chan []string, 1) + defer close(wantGrpcAcceptEncodingCh) + + compressor := renameCompressor{Compressor: grpc.NewGZIPCompressor(), name: "testgzip"} + decompressor := renameDecompressor{Decompressor: grpc.NewGZIPDecompressor(), name: "testgzip"} + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Errorf(codes.Internal, "no metadata in context") + } + if got, want := md["grpc-accept-encoding"], <-wantGrpcAcceptEncodingCh; !reflect.DeepEqual(got, want) { + return nil, status.Errorf(codes.Internal, "got grpc-accept-encoding=%q; want [%q]", got, want) + } + return &testpb.Empty{}, nil + }, + } + if err := ss.Start([]grpc.ServerOption{grpc.RPCDecompressor(&decompressor)}); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + wantGrpcAcceptEncodingCh <- []string{"gzip"} + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) + } + + wantGrpcAcceptEncodingCh <- []string{"gzip"} + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, grpc.UseCompressor("gzip")); err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) + } + + // Use compressor directly which is not registered via + // encoding.RegisterCompressor. + if err := ss.StartClient(grpc.WithCompressor(&compressor)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + wantGrpcAcceptEncodingCh <- []string{"gzip,testgzip"} + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) + } +} + +func (s) TestUnregisteredSetSendCompressorFailure(t *testing.T) { + resCompressor := "snappy2" + wantErr := status.Error(codes.Unknown, "unable to set send compressor: compressor not registered \"snappy2\"") + + t.Run("unary", func(t *testing.T) { + testUnarySetSendCompressorFailure(t, resCompressor, wantErr) + }) + + t.Run("stream", func(t *testing.T) { + testStreamSetSendCompressorFailure(t, resCompressor, wantErr) + }) +} + +func testUnarySetSendCompressorFailure(t *testing.T, resCompressor string, wantErr error) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + if err := grpc.SetSendCompressor(ctx, resCompressor); err != nil { + return nil, err + } + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); !equalError(err, wantErr) { + t.Fatalf("Unexpected unary call error, got: %v, want: %v", err, wantErr) + } +} + +func testStreamSetSendCompressorFailure(t *testing.T, resCompressor string, wantErr error) { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + + if err := grpc.SetSendCompressor(stream.Context(), resCompressor); err != nil { + return err + } + + return stream.Send(&testpb.StreamingOutputCallResponse{}) + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v, want: nil", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + s, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("Unexpected full duplex call error, got: %v, want: nil", err) + } + + if err := s.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("Unexpected full duplex call send error, got: %v, want: nil", err) + } + + if _, err := s.Recv(); !equalError(err, wantErr) { + t.Fatalf("Unexpected full duplex recv error, got: %v, want: nil", err) + } +} + +func (s) TestUnarySetSendCompressorAfterHeaderSendFailure(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + // Send headers early and then set send compressor. + grpc.SendHeader(ctx, metadata.MD{}) + err := grpc.SetSendCompressor(ctx, "gzip") + if err == nil { + t.Error("Wanted set send compressor error") + return &testpb.Empty{}, nil + } + return nil, err + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + wantErr := status.Error(codes.Unknown, "transport: set send compressor called after headers sent or stream done") + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); !equalError(err, wantErr) { + t.Fatalf("Unexpected unary call error, got: %v, want: %v", err, wantErr) + } +} + +func (s) TestStreamSetSendCompressorAfterHeaderSendFailure(t *testing.T) { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + // Send headers early and then set send compressor. + grpc.SendHeader(stream.Context(), metadata.MD{}) + err := grpc.SetSendCompressor(stream.Context(), "gzip") + if err == nil { + t.Error("Wanted set send compressor error") + } + return err + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + wantErr := status.Error(codes.Unknown, "transport: set send compressor called after headers sent or stream done") + s, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("Unexpected full duplex call error, got: %v, want: nil", err) + } + + if _, err := s.Recv(); !equalError(err, wantErr) { + t.Fatalf("Unexpected full duplex recv error, got: %v, want: %v", err, wantErr) + } +} + +func (s) TestClientSupportedCompressors(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + for _, tt := range []struct { + desc string + ctx context.Context + want []string + }{ + { + desc: "No additional grpc-accept-encoding header", + ctx: ctx, + want: []string{"gzip"}, + }, + { + desc: "With additional grpc-accept-encoding header", + ctx: metadata.AppendToOutgoingContext(ctx, + "grpc-accept-encoding", "test-compressor-1", + "grpc-accept-encoding", "test-compressor-2", + ), + want: []string{"gzip", "test-compressor-1", "test-compressor-2"}, + }, + { + desc: "With additional empty grpc-accept-encoding header", + ctx: metadata.AppendToOutgoingContext(ctx, + "grpc-accept-encoding", "", + ), + want: []string{"gzip"}, + }, + { + desc: "With additional grpc-accept-encoding header with spaces between values", + ctx: metadata.AppendToOutgoingContext(ctx, + "grpc-accept-encoding", "identity, deflate", + ), + want: []string{"gzip", "identity", "deflate"}, + }, + } { + t.Run(tt.desc, func(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + got, err := grpc.ClientSupportedCompressors(ctx) + if err != nil { + return nil, err + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("unexpected client compressors got: %v, want: %v", got, tt.want) + } + + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v, want: nil", err) + } + defer ss.Stop() + + _, err := ss.Client.EmptyCall(tt.ctx, &testpb.Empty{}) + if err != nil { + t.Fatalf("Unexpected unary call error, got: %v, want: nil", err) + } + }) + } +} + +func (s) TestAcceptCompressorsCallOption(t *testing.T) { + tests := []struct { + name string + callOption grpc.CallOption + wantHeader string + }{ + { + name: "with AcceptCompressors", + callOption: experimental.AcceptCompressors("gzip"), + wantHeader: "gzip", + }, + { + name: "without AcceptCompressors uses default", + callOption: nil, + wantHeader: grpcutil.RegisteredCompressors(), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, _ := metadata.FromIncomingContext(ctx) + header := md.Get("grpc-accept-encoding") + + if len(header) != 1 || header[0] != tt.wantHeader { + t.Errorf("unexpected grpc-accept-encoding header: got %v, want %v", header, tt.wantHeader) + } + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("failed to start server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + opts := []grpc.CallOption{} + if tt.callOption != nil { + opts = append(opts, tt.callOption) + } + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, opts...); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } + }) + } +} + +func (s) TestCompressorRegister(t *testing.T) { + for _, e := range listTestEnv() { + testCompressorRegister(t, e) + } +} + +func testCompressorRegister(t *testing.T, e env) { + te := newTest(t, e) + te.clientCompression = false + te.serverCompression = false + te.clientUseCompression = true + + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + tc := testgrpc.NewTestServiceClient(te.clientConn()) + + // Unary call + const argSize = 271828 + const respSize = 314159 + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, argSize) + if err != nil { + t.Fatal(err) + } + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: respSize, + Payload: payload, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("something", "something")) + if _, err := tc.UnaryCall(ctx, req); err != nil { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, ", err) + } + // Streaming RPC + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + respParam := []*testpb.ResponseParameters{ + { + Size: 31415, + }, + } + payload, err = newPayload(testpb.PayloadType_COMPRESSABLE, int32(31415)) + if err != nil { + t.Fatal(err) + } + sreq := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: payload, + } + if err := stream.Send(sreq); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = %v, want ", stream, err) + } +} + +type badGzipCompressor struct{} + +func (badGzipCompressor) Do(w io.Writer, p []byte) error { + buf := &bytes.Buffer{} + gzw := gzip.NewWriter(buf) + if _, err := gzw.Write(p); err != nil { + return err + } + err := gzw.Close() + bs := buf.Bytes() + if len(bs) >= 6 { + bs[len(bs)-6] ^= 1 // modify checksum at end by 1 byte + } + w.Write(bs) + return err +} + +func (badGzipCompressor) Type() string { + return "gzip" +} + +func (s) TestGzipBadChecksum(t *testing.T) { + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + } + if err := ss.Start(nil, grpc.WithCompressor(badGzipCompressor{})); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + p, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(1024)) + if err != nil { + t.Fatalf("Unexpected error from newPayload: %v", err) + } + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: p}); err == nil || + status.Code(err) != codes.Internal || + !strings.Contains(status.Convert(err).Message(), gzip.ErrChecksum.Error()) { + t.Errorf("ss.Client.UnaryCall(_) = _, %v\n\twant: _, status(codes.Internal, contains %q)", err, gzip.ErrChecksum) + } +} diff --git a/test/config_selector_test.go b/test/config_selector_test.go new file mode 100644 index 0000000..d4a7b43 --- /dev/null +++ b/test/config_selector_test.go @@ -0,0 +1,216 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/codes" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" +) + +type funcConfigSelector struct { + f func(iresolver.RPCInfo) (*iresolver.RPCConfig, error) +} + +func (f funcConfigSelector) SelectConfig(i iresolver.RPCInfo) (*iresolver.RPCConfig, error) { + return f.f(i) +} + +func (s) TestConfigSelector(t *testing.T) { + gotContextChan := testutils.NewChannelWithSize(1) + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + gotContextChan.SendContext(ctx, ctx) + return &testpb.Empty{}, nil + }, + } + ss.R = manual.NewBuilderWithScheme("confSel") + + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + const normalTimeout = 10 * time.Second + ctxDeadline := time.Now().Add(normalTimeout) + ctx, cancel := context.WithTimeout(context.Background(), normalTimeout) + defer cancel() + + const longTimeout = 30 * time.Second + longCtxDeadline := time.Now().Add(longTimeout) + longdeadlineCtx, cancel := context.WithTimeout(context.Background(), longTimeout) + defer cancel() + shorterTimeout := 3 * time.Second + + testMD := metadata.MD{"footest": []string{"bazbar"}} + mdOut := metadata.MD{"handler": []string{"value"}} + + var onCommittedCalled bool + + testCases := []struct { + name string + md metadata.MD // MD sent with RPC + config *iresolver.RPCConfig // config returned by config selector + csErr error // error returned by config selector + + wantMD metadata.MD + wantDeadline time.Time + wantTimeout time.Duration + wantErr error + }{{ + name: "basic", + md: testMD, + config: &iresolver.RPCConfig{}, + wantMD: testMD, + wantDeadline: ctxDeadline, + }, { + name: "alter MD", + md: testMD, + config: &iresolver.RPCConfig{ + Context: metadata.NewOutgoingContext(ctx, mdOut), + }, + wantMD: mdOut, + wantDeadline: ctxDeadline, + }, { + name: "erroring SelectConfig", + csErr: status.Errorf(codes.Unavailable, "cannot send RPC"), + wantErr: status.Errorf(codes.Unavailable, "cannot send RPC"), + }, { + name: "alter timeout; remove MD", + md: testMD, + config: &iresolver.RPCConfig{ + Context: longdeadlineCtx, // no metadata + }, + wantMD: nil, + wantDeadline: longCtxDeadline, + }, { + name: "nil config", + md: metadata.MD{}, + config: nil, + wantMD: nil, + wantDeadline: ctxDeadline, + }, { + name: "alter timeout via method config; remove MD", + md: testMD, + config: &iresolver.RPCConfig{ + MethodConfig: serviceconfig.MethodConfig{ + Timeout: &shorterTimeout, + }, + }, + wantMD: nil, + wantTimeout: shorterTimeout, + }, { + name: "onCommitted callback", + md: testMD, + config: &iresolver.RPCConfig{ + OnCommitted: func() { + onCommittedCalled = true + }, + }, + wantMD: testMD, + wantDeadline: ctxDeadline, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var gotInfo *iresolver.RPCInfo + state := iresolver.SetConfigSelector(resolver.State{ + Addresses: []resolver.Address{{Addr: ss.Address}}, + ServiceConfig: parseServiceConfig(t, ss.R, "{}"), + }, funcConfigSelector{ + f: func(i iresolver.RPCInfo) (*iresolver.RPCConfig, error) { + gotInfo = &i + cfg := tc.config + if cfg != nil && cfg.Context == nil { + cfg.Context = i.Context + } + return cfg, tc.csErr + }, + }) + ss.R.UpdateState(state) // Blocks until config selector is applied + + onCommittedCalled = false + ctx := metadata.NewOutgoingContext(ctx, tc.md) + startTime := time.Now() + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); fmt.Sprint(err) != fmt.Sprint(tc.wantErr) { + t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", err, tc.wantErr) + } else if err != nil { + return // remaining checks are invalid + } + + if gotInfo == nil { + t.Fatalf("no config selector data") + } + + if want := "/grpc.testing.TestService/EmptyCall"; gotInfo.Method != want { + t.Errorf("gotInfo.Method = %q; want %q", gotInfo.Method, want) + } + + gotContextI, ok := gotContextChan.ReceiveOrFail() + if !ok { + t.Fatalf("no context received") + } + gotContext := gotContextI.(context.Context) + + gotMD, _ := metadata.FromOutgoingContext(gotInfo.Context) + if diff := cmp.Diff(tc.md, gotMD); diff != "" { + t.Errorf("gotInfo.Context contains MD %v; want %v\nDiffs: %v", gotMD, tc.md, diff) + } + + gotMD, _ = metadata.FromIncomingContext(gotContext) + // Remove entries from gotMD not in tc.wantMD (e.g. authority header). + for k := range gotMD { + if _, ok := tc.wantMD[k]; !ok { + delete(gotMD, k) + } + } + if diff := cmp.Diff(tc.wantMD, gotMD, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("received md = %v; want %v\nDiffs: %v", gotMD, tc.wantMD, diff) + } + + wantDeadline := tc.wantDeadline + if wantDeadline.Equal(time.Time{}) { + wantDeadline = startTime.Add(tc.wantTimeout) + } + deadlineGot, _ := gotContext.Deadline() + if diff := deadlineGot.Sub(wantDeadline); diff > time.Second || diff < -time.Second { + t.Errorf("received deadline = %v; want ~%v", deadlineGot, wantDeadline) + } + + if tc.config != nil && tc.config.OnCommitted != nil && !onCommittedCalled { + t.Errorf("OnCommitted callback not called") + } + }) + } +} diff --git a/test/context_canceled_test.go b/test/context_canceled_test.go new file mode 100644 index 0000000..472f5b4 --- /dev/null +++ b/test/context_canceled_test.go @@ -0,0 +1,164 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/encoding/gzip" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestContextCanceled(t *testing.T) { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + stream.SetTrailer(metadata.New(map[string]string{"a": "b"})) + return status.Error(codes.PermissionDenied, "perm denied") + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + // Runs 10 rounds of tests with the given delay and returns counts of status codes. + // Fails in case of trailer/status code inconsistency. + const cntRetry uint = 10 + runTest := func(delay time.Duration) (cntCanceled, cntPermDenied uint) { + for i := uint(0); i < cntRetry; i++ { + ctx, cancel := context.WithTimeout(context.Background(), delay) + defer cancel() + + str, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + continue + } + + _, err = str.Recv() + if err == nil { + t.Fatalf("non-nil error expected from Recv()") + } + + _, trlOk := str.Trailer()["a"] + switch status.Code(err) { + case codes.PermissionDenied: + if !trlOk { + t.Fatalf(`status err: %v; wanted key "a" in trailer but didn't get it`, err) + } + cntPermDenied++ + case codes.DeadlineExceeded: + if trlOk { + t.Fatalf(`status err: %v; didn't want key "a" in trailer but got it`, err) + } + cntCanceled++ + default: + t.Fatalf(`unexpected status err: %v`, err) + } + } + return cntCanceled, cntPermDenied + } + + // Tries to find the delay that causes canceled/perm denied race. + canceledOk, permDeniedOk := false, false + for lower, upper := time.Duration(0), 2*time.Millisecond; lower <= upper; { + delay := lower + (upper-lower)/2 + cntCanceled, cntPermDenied := runTest(delay) + if cntPermDenied > 0 && cntCanceled > 0 { + // Delay that causes the race is found. + return + } + + // Set OK flags. + if cntCanceled > 0 { + canceledOk = true + } + if cntPermDenied > 0 { + permDeniedOk = true + } + + if cntPermDenied == 0 { + // No perm denied, increase the delay. + lower += (upper-lower)/10 + 1 + } else { + // All perm denied, decrease the delay. + upper -= (upper-lower)/10 + 1 + } + } + + if !canceledOk || !permDeniedOk { + t.Fatalf(`couldn't find the delay that causes canceled/perm denied race.`) + } +} + +// To make sure that canceling a stream with compression enabled won't result in +// internal error, compressed flag set with identity or empty encoding. +// +// The root cause is a select race on stream headerChan and ctx. Stream gets +// whether compression is enabled and the compression type from two separate +// functions, both include select with context. If the `case non-ctx:` wins the +// first one, but `case ctx.Done()` wins the second one, the compression info +// will be inconsistent, and it causes internal error. +func (s) TestCancelWhileRecvingWithCompression(t *testing.T) { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + if err := stream.Send(&testpb.StreamingOutputCallResponse{ + Payload: nil, + }); err != nil { + return err + } + } + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + for i := 0; i < 10; i++ { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + s, err := ss.Client.FullDuplexCall(ctx, grpc.UseCompressor(gzip.Name)) + if err != nil { + t.Fatalf("failed to start bidi streaming RPC: %v", err) + } + // Cancel the stream while receiving to trigger the internal error. + time.AfterFunc(time.Millisecond, cancel) + for { + _, err := s.Recv() + if err != nil { + if status.Code(err) != codes.Canceled { + t.Fatalf("recv failed with %v, want Canceled", err) + } + break + } + } + } + if err := ss.CC.Close(); err != nil { + t.Fatalf("Close failed with %v, want nil", err) + } +} diff --git a/test/control_plane_status_test.go b/test/control_plane_status_test.go new file mode 100644 index 0000000..d58035b --- /dev/null +++ b/test/control_plane_status_test.go @@ -0,0 +1,234 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/stub" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/stubserver" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" +) + +func (s) TestConfigSelectorStatusCodes(t *testing.T) { + testCases := []struct { + name string + csErr error + want error + }{{ + name: "legal status code", + csErr: status.Errorf(codes.Unavailable, "this error is fine"), + want: status.Errorf(codes.Unavailable, "this error is fine"), + }, { + name: "illegal status code", + csErr: status.Errorf(codes.NotFound, "this error is bad"), + want: status.Errorf(codes.Internal, "this error is bad"), + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + ss.R = manual.NewBuilderWithScheme("confSel") + + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + state := iresolver.SetConfigSelector(resolver.State{ + Addresses: []resolver.Address{{Addr: ss.Address}}, + ServiceConfig: parseServiceConfig(t, ss.R, "{}"), + }, funcConfigSelector{ + f: func(iresolver.RPCInfo) (*iresolver.RPCConfig, error) { + return nil, tc.csErr + }, + }) + ss.R.UpdateState(state) // Blocks until config selector is applied + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != status.Code(tc.want) || !strings.Contains(err.Error(), status.Convert(tc.want).Message()) { + t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", err, tc.want) + } + }) + } +} + +func (s) TestPickerStatusCodes(t *testing.T) { + testCases := []struct { + name string + pickerErr error + want error + }{{ + name: "legal status code", + pickerErr: status.Errorf(codes.Unavailable, "this error is fine"), + want: status.Errorf(codes.Unavailable, "this error is fine"), + }, { + name: "illegal status code", + pickerErr: status.Errorf(codes.NotFound, "this error is bad"), + want: status.Errorf(codes.Internal, "this error is bad"), + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + // Create a stub balancer that creates a picker that always returns + // an error. + sbf := stub.BalancerFuncs{ + UpdateClientConnState: func(d *stub.BalancerData, _ balancer.ClientConnState) error { + d.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(tc.pickerErr), + }) + return nil + }, + } + stub.Register("testPickerStatusCodesBalancer", sbf) + + ss.NewServiceConfig(`{"loadBalancingConfig": [{"testPickerStatusCodesBalancer":{}}] }`) + + // Make calls until pickerErr is received. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + var lastErr error + for ctx.Err() == nil { + if _, lastErr = ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(lastErr) == status.Code(tc.want) && strings.Contains(lastErr.Error(), status.Convert(tc.want).Message()) { + // Success! + return + } + time.Sleep(time.Millisecond) + } + + t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", lastErr, tc.want) + }) + } +} + +func (s) TestCallCredsFromDialOptionsStatusCodes(t *testing.T) { + testCases := []struct { + name string + credsErr error + want error + }{{ + name: "legal status code", + credsErr: status.Errorf(codes.Unavailable, "this error is fine"), + want: status.Errorf(codes.Unavailable, "this error is fine"), + }, { + name: "illegal status code", + credsErr: status.Errorf(codes.NotFound, "this error is bad"), + want: status.Errorf(codes.Internal, "this error is bad"), + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + errChan := make(chan error, 1) + creds := &testPerRPCCredentials{errChan: errChan} + + if err := ss.Start(nil, grpc.WithPerRPCCredentials(creds)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + errChan <- tc.credsErr + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != status.Code(tc.want) || !strings.Contains(err.Error(), status.Convert(tc.want).Message()) { + t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", err, tc.want) + } + }) + } +} + +func (s) TestCallCredsFromCallOptionsStatusCodes(t *testing.T) { + testCases := []struct { + name string + credsErr error + want error + }{{ + name: "legal status code", + credsErr: status.Errorf(codes.Unavailable, "this error is fine"), + want: status.Errorf(codes.Unavailable, "this error is fine"), + }, { + name: "illegal status code", + credsErr: status.Errorf(codes.NotFound, "this error is bad"), + want: status.Errorf(codes.Internal, "this error is bad"), + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + errChan := make(chan error, 1) + creds := &testPerRPCCredentials{errChan: errChan} + + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + errChan <- tc.credsErr + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(creds)); status.Code(err) != status.Code(tc.want) || !strings.Contains(err.Error(), status.Convert(tc.want).Message()) { + t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", err, tc.want) + } + }) + } +} diff --git a/test/creds_test.go b/test/creds_test.go new file mode 100644 index 0000000..3034064 --- /dev/null +++ b/test/creds_test.go @@ -0,0 +1,626 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + "google.golang.org/grpc/tap" + "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const ( + bundlePerRPCOnly = "perRPCOnly" + bundleTLSOnly = "tlsOnly" +) + +type testCredsBundle struct { + t *testing.T + mode string +} + +func (c *testCredsBundle) TransportCredentials() credentials.TransportCredentials { + if c.mode == bundlePerRPCOnly { + return insecure.NewCredentials() + } + + creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") + if err != nil { + c.t.Logf("Failed to load credentials: %v", err) + return nil + } + return creds +} + +func (c *testCredsBundle) PerRPCCredentials() credentials.PerRPCCredentials { + if c.mode == bundleTLSOnly { + return nil + } + return testPerRPCCredentials{authdata: authdata} +} + +func (c *testCredsBundle) NewWithMode(mode string) (credentials.Bundle, error) { + return &testCredsBundle{mode: mode}, nil +} + +func (s) TestCredsBundleBoth(t *testing.T) { + te := newTest(t, env{name: "creds-bundle", network: "tcp", security: "empty"}) + te.tapHandle = authHandle + te.customDialOptions = []grpc.DialOption{ + grpc.WithCredentialsBundle(&testCredsBundle{t: t}), + grpc.WithAuthority("x.test.example.com"), + } + creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("Failed to generate credentials %v", err) + } + te.customServerOptions = []grpc.ServerOption{ + grpc.Creds(creds), + } + te.startServer(&testServer{}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("Test failed. Reason: %v", err) + } +} + +func (s) TestCredsBundleTransportCredentials(t *testing.T) { + te := newTest(t, env{name: "creds-bundle", network: "tcp", security: "empty"}) + te.customDialOptions = []grpc.DialOption{ + grpc.WithCredentialsBundle(&testCredsBundle{t: t, mode: bundleTLSOnly}), + grpc.WithAuthority("x.test.example.com"), + } + creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("Failed to generate credentials %v", err) + } + te.customServerOptions = []grpc.ServerOption{ + grpc.Creds(creds), + } + te.startServer(&testServer{}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("Test failed. Reason: %v", err) + } +} + +func (s) TestCredsBundlePerRPCCredentials(t *testing.T) { + te := newTest(t, env{name: "creds-bundle", network: "tcp", security: "empty"}) + te.tapHandle = authHandle + te.customDialOptions = []grpc.DialOption{ + grpc.WithCredentialsBundle(&testCredsBundle{t: t, mode: bundlePerRPCOnly}), + } + te.startServer(&testServer{}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("Test failed. Reason: %v", err) + } +} + +type clientTimeoutCreds struct { + credentials.TransportCredentials + timeoutReturned bool +} + +func (c *clientTimeoutCreds) ClientHandshake(_ context.Context, _ string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + if !c.timeoutReturned { + c.timeoutReturned = true + return nil, nil, context.DeadlineExceeded + } + return rawConn, nil, nil +} + +func (c *clientTimeoutCreds) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{} +} + +func (c *clientTimeoutCreds) Clone() credentials.TransportCredentials { + return nil +} + +func (s) TestNonFailFastRPCSucceedOnTimeoutCreds(t *testing.T) { + te := newTest(t, env{name: "timeout-cred", network: "tcp", security: "empty"}) + te.userAgent = testAppUA + te.startServer(&testServer{security: te.e.security}) + defer te.tearDown() + + cc := te.clientConn(grpc.WithTransportCredentials(&clientTimeoutCreds{})) + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // This unary call should succeed, because ClientHandshake will succeed for the second time. + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + te.t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want ", err) + } +} + +type methodTestCreds struct{} + +func (m *methodTestCreds) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) { + ri, _ := credentials.RequestInfoFromContext(ctx) + return nil, status.Error(codes.Unknown, ri.Method) +} + +func (m *methodTestCreds) RequireTransportSecurity() bool { return false } + +func (s) TestGRPCMethodAccessibleToCredsViaContextRequestInfo(t *testing.T) { + const wantMethod = "/grpc.testing.TestService/EmptyCall" + te := newTest(t, env{name: "context-request-info", network: "tcp"}) + te.userAgent = testAppUA + te.startServer(&testServer{security: te.e.security}) + defer te.tearDown() + + cc := te.clientConn(grpc.WithPerRPCCredentials(&methodTestCreds{})) + tc := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Convert(err).Message() != wantMethod { + t.Fatalf("ss.client.EmptyCall(_, _) = _, %v; want _, _.Message()=%q", err, wantMethod) + } + + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Convert(err).Message() != wantMethod { + t.Fatalf("ss.client.EmptyCall(_, _) = _, %v; want _, _.Message()=%q", err, wantMethod) + } +} + +const clientAlwaysFailCredErrorMsg = "clientAlwaysFailCred always fails" + +type clientAlwaysFailCred struct { + credentials.TransportCredentials +} + +func (c clientAlwaysFailCred) ClientHandshake(context.Context, string, net.Conn) (net.Conn, credentials.AuthInfo, error) { + return nil, nil, errors.New(clientAlwaysFailCredErrorMsg) +} +func (c clientAlwaysFailCred) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{} +} +func (c clientAlwaysFailCred) Clone() credentials.TransportCredentials { + return nil +} + +func (s) TestFailFastRPCErrorOnBadCertificates(t *testing.T) { + te := newTest(t, env{name: "bad-cred", network: "tcp", security: "empty", balancer: "round_robin"}) + te.startServer(&testServer{security: te.e.security}) + defer te.tearDown() + + opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc, err := grpc.NewClient(te.srvAddr, opts...) + if err != nil { + t.Fatalf("NewClient(_) = %v, want %v", err, nil) + } + defer cc.Close() + + tc := testgrpc.NewTestServiceClient(cc) + if _, err = tc.EmptyCall(ctx, &testpb.Empty{}); strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) { + return + } + te.t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want err.Error() contains %q", err, clientAlwaysFailCredErrorMsg) +} + +func (s) TestWaitForReadyRPCErrorOnBadCertificates(t *testing.T) { + te := newTest(t, env{name: "bad-cred", network: "tcp", security: "empty", balancer: "round_robin"}) + te.startServer(&testServer{security: te.e.security}) + defer te.tearDown() + + opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})} + cc, err := grpc.NewClient(te.srvAddr, opts...) + if err != nil { + t.Fatalf("NewClient(_) = %v, want %v", err, nil) + } + defer cc.Close() + + // The DNS resolver may take more than defaultTestShortTimeout, we let the + // channel enter TransientFailure signalling that the first resolver state + // has been produced. + cc.Connect() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + tc := testgrpc.NewTestServiceClient(cc) + // Use a short context as WaitForReady waits for context expiration before + // failing the RPC. + ctx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err = tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); !strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want err.Error() contains %q", err, clientAlwaysFailCredErrorMsg) + } +} + +var ( + // test authdata + authdata = map[string]string{ + "test-key": "test-value", + "test-key2-bin": string([]byte{1, 2, 3}), + } +) + +type testPerRPCCredentials struct { + authdata map[string]string + errChan chan error +} + +func (cr testPerRPCCredentials) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { + var err error + if cr.errChan != nil { + err = <-cr.errChan + } + return cr.authdata, err +} + +func (cr testPerRPCCredentials) RequireTransportSecurity() bool { + return false +} + +func authHandle(ctx context.Context, _ *tap.Info) (context.Context, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return ctx, fmt.Errorf("didn't find metadata in context") + } + for k, vwant := range authdata { + vgot, ok := md[k] + if !ok { + return ctx, fmt.Errorf("didn't find authdata key %v in context", k) + } + if vgot[0] != vwant { + return ctx, fmt.Errorf("for key %v, got value %v, want %v", k, vgot, vwant) + } + } + return ctx, nil +} + +func (s) TestPerRPCCredentialsViaDialOptions(t *testing.T) { + for _, e := range listTestEnv() { + testPerRPCCredentialsViaDialOptions(t, e) + } +} + +func testPerRPCCredentialsViaDialOptions(t *testing.T, e env) { + te := newTest(t, e) + te.tapHandle = authHandle + te.perRPCCreds = testPerRPCCredentials{authdata: authdata} + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("Test failed. Reason: %v", err) + } +} + +func (s) TestPerRPCCredentialsViaCallOptions(t *testing.T) { + for _, e := range listTestEnv() { + testPerRPCCredentialsViaCallOptions(t, e) + } +} + +func testPerRPCCredentialsViaCallOptions(t *testing.T, e env) { + te := newTest(t, e) + te.tapHandle = authHandle + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{authdata: authdata})); err != nil { + t.Fatalf("Test failed. Reason: %v", err) + } +} + +func (s) TestPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T) { + for _, e := range listTestEnv() { + testPerRPCCredentialsViaDialOptionsAndCallOptions(t, e) + } +} + +func testPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T, e env) { + te := newTest(t, e) + te.perRPCCreds = testPerRPCCredentials{authdata: authdata} + // When credentials are provided via both dial options and call options, + // we apply both sets. + te.tapHandle = func(ctx context.Context, _ *tap.Info) (context.Context, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return ctx, fmt.Errorf("couldn't find metadata in context") + } + for k, vwant := range authdata { + vgot, ok := md[k] + if !ok { + return ctx, fmt.Errorf("couldn't find metadata for key %v", k) + } + if len(vgot) != 2 { + return ctx, fmt.Errorf("len of value for key %v was %v, want 2", k, len(vgot)) + } + if vgot[0] != vwant || vgot[1] != vwant { + return ctx, fmt.Errorf("value for %v was %v, want [%v, %v]", k, vgot, vwant, vwant) + } + } + return ctx, nil + } + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{authdata: authdata})); err != nil { + t.Fatalf("Test failed. Reason: %v", err) + } +} + +func (s) TestPerRPCCredentialMetadataValidationRejectsInvalidDialOptions(t *testing.T) { + for _, e := range listTestEnv() { + for _, test := range []struct { + name string + authdata map[string]string + wantErr string + }{ + { + name: "invalid key", + authdata: map[string]string{"bad key": "value"}, + wantErr: fmt.Sprintf("header key %q contains illegal characters not in [0-9a-z-_.]", "bad key"), + }, + { + name: "invalid value", + authdata: map[string]string{"valid-key": "bad\x01value"}, + wantErr: fmt.Sprintf("header key %q contains value with non-printable ASCII characters", "valid-key"), + }, + } { + t.Run(e.name+"/"+test.name, func(t *testing.T) { + testPerRPCCredentialMetadataValidationRejectsInvalidDialOptions(t, e, test.authdata, test.wantErr) + }) + } + } +} + +func testPerRPCCredentialMetadataValidationRejectsInvalidDialOptions(t *testing.T, e env, authdata map[string]string, wantErr string) { + te := newTest(t, e) + te.perRPCCreds = testPerRPCCredentials{authdata: authdata} + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Internal || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall() error = %v, want code %v and message containing %q", err, codes.Internal, wantErr) + } +} + +func (s) TestPerRPCCredentialMetadataValidationRejectsInvalidCallOptions(t *testing.T) { + for _, e := range listTestEnv() { + for _, test := range []struct { + name string + authdata map[string]string + wantErr string + }{ + { + name: "invalid key", + authdata: map[string]string{"bad key": "value"}, + wantErr: fmt.Sprintf("header key %q contains illegal characters not in [0-9a-z-_.]", "bad key"), + }, + { + name: "invalid value", + authdata: map[string]string{"valid-key": "bad\x01value"}, + wantErr: fmt.Sprintf("header key %q contains value with non-printable ASCII characters", "valid-key"), + }, + } { + t.Run(e.name+"/"+test.name, func(t *testing.T) { + testPerRPCCredentialMetadataValidationRejectsInvalidCallOptions(t, e, test.authdata, test.wantErr) + }) + } + } +} + +func testPerRPCCredentialMetadataValidationRejectsInvalidCallOptions(t *testing.T, e env, authdata map[string]string, wantErr string) { + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{authdata: authdata})); status.Code(err) != codes.Internal || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall() error = %v, want code %v and message containing %q", err, codes.Internal, wantErr) + } +} + +const testAuthority = "test.auth.ori.ty" + +type authorityCheckCreds struct { + credentials.TransportCredentials + got string +} + +func (c *authorityCheckCreds) ClientHandshake(_ context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + c.got = authority + return rawConn, nil, nil +} +func (c *authorityCheckCreds) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{} +} +func (c *authorityCheckCreds) Clone() credentials.TransportCredentials { + return c +} + +// This test makes sure that the authority client handshake gets is the endpoint +// in dial target, not the resolved ip address. +func (s) TestCredsHandshakeAuthority(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + cred := &authorityCheckCreds{} + s := grpc.NewServer() + go s.Serve(lis) + defer s.Stop() + + r := manual.NewBuilderWithScheme("whatever") + + cc, err := grpc.NewClient(r.Scheme()+":///"+testAuthority, grpc.WithTransportCredentials(cred), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err) + } + defer cc.Close() + cc.Connect() + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if cred.got != testAuthority { + t.Fatalf("client creds got authority: %q, want: %q", cred.got, testAuthority) + } +} + +// This test makes sure that the authority client handshake gets is the endpoint +// of the ServerName of the address when it is set. +func (s) TestCredsHandshakeServerNameAuthority(t *testing.T) { + const testServerName = "test.server.name" + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + cred := &authorityCheckCreds{} + s := grpc.NewServer() + go s.Serve(lis) + defer s.Stop() + + r := manual.NewBuilderWithScheme("whatever") + + cc, err := grpc.NewClient(r.Scheme()+":///"+testAuthority, grpc.WithTransportCredentials(cred), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err) + } + defer cc.Close() + cc.Connect() + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String(), ServerName: testServerName}}}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if cred.got != testServerName { + t.Fatalf("client creds got authority: %q, want: %q", cred.got, testAuthority) + } +} + +type serverDispatchCred struct { + rawConnCh chan net.Conn +} + +func (c *serverDispatchCred) ClientHandshake(_ context.Context, _ string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + return rawConn, nil, nil +} +func (c *serverDispatchCred) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + select { + case c.rawConnCh <- rawConn: + default: + } + return nil, nil, credentials.ErrConnDispatched +} +func (c *serverDispatchCred) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{} +} +func (c *serverDispatchCred) Clone() credentials.TransportCredentials { + return nil +} +func (c *serverDispatchCred) OverrideServerName(string) error { + return nil +} +func (c *serverDispatchCred) getRawConn() net.Conn { + return <-c.rawConnCh +} + +func (s) TestServerCredsDispatch(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + cred := &serverDispatchCred{ + rawConnCh: make(chan net.Conn, 1), + } + s := grpc.NewServer(grpc.Creds(cred)) + go s.Serve(lis) + defer s.Stop() + + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(cred)) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err) + } + defer cc.Close() + cc.Connect() + + rawConn := cred.getRawConn() + // Give grpc a chance to see the error and potentially close the connection. + // And check that connection is not closed after that. + time.Sleep(100 * time.Millisecond) + // Check rawConn is not closed. + if n, err := rawConn.Write([]byte{0}); n <= 0 || err != nil { + t.Errorf("Read() = %v, %v; want n>0, ", n, err) + } +} diff --git a/test/goaway_test.go b/test/goaway_test.go new file mode 100644 index 0000000..6f90ecc --- /dev/null +++ b/test/goaway_test.go @@ -0,0 +1,823 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "io" + "net" + "strings" + "testing" + "time" + + "golang.org/x/net/http2" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestGracefulClientOnGoAway attempts to ensure that when the server sends a +// GOAWAY (in this test, by configuring max connection age on the server), a +// client will never see an error. This requires that the client is appraised +// of the GOAWAY and updates its state accordingly before the transport stops +// accepting new streams. If a subconn is chosen by a picker and receives the +// goaway before creating the stream, an error will occur, but upon transparent +// retry, the clientconn will ensure a ready subconn is chosen. +func (s) TestGracefulClientOnGoAway(t *testing.T) { + const maxConnAge = 100 * time.Millisecond + const testTime = maxConnAge * 10 + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + ss := &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + S: grpc.NewServer(grpc.KeepaliveParams(keepalive.ServerParameters{MaxConnectionAge: maxConnAge})), + } + stubserver.StartTestService(t, ss) + defer ss.S.Stop() + + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial server: %v", err) + } + defer cc.Close() + c := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + endTime := time.Now().Add(testTime) + for time.Now().Before(endTime) { + if _, err := c.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall(_, _) = _, %v; want _, ", err) + } + } +} + +func (s) TestDetailedGoAwayErrorOnGracefulClosePropagatesToRPCError(t *testing.T) { + rpcDoneOnClient := make(chan struct{}) + ss := &stubserver.StubServer{ + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + <-rpcDoneOnClient + return status.Error(codes.Internal, "arbitrary status") + }, + } + sopts := []grpc.ServerOption{ + grpc.KeepaliveParams(keepalive.ServerParameters{ + MaxConnectionAge: time.Millisecond * 100, + MaxConnectionAgeGrace: time.Nanosecond, // ~instantaneously, but non-zero to avoid default + }), + } + if err := ss.Start(sopts); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall = _, %v, want _, ", ss.Client, err) + } + const expectedErrorMessageSubstring = "received prior goaway: code: NO_ERROR" + _, err = stream.Recv() + close(rpcDoneOnClient) + if err == nil || !strings.Contains(err.Error(), expectedErrorMessageSubstring) { + t.Fatalf("%v.Recv() = _, %v, want _, rpc error containing substring: %q", stream, err, expectedErrorMessageSubstring) + } +} + +func (s) TestDetailedGoAwayErrorOnAbruptClosePropagatesToRPCError(t *testing.T) { + grpctest.ExpectError("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\"") + // set the min keepalive time very low so that this test can take + // a reasonable amount of time + prev := internal.KeepaliveMinPingTime + internal.KeepaliveMinPingTime = time.Millisecond + defer func() { internal.KeepaliveMinPingTime = prev }() + + rpcDoneOnClient := make(chan struct{}) + ss := &stubserver.StubServer{ + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + <-rpcDoneOnClient + return status.Error(codes.Internal, "arbitrary status") + }, + } + sopts := []grpc.ServerOption{ + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: time.Second * 1000, /* arbitrary, large value */ + }), + } + dopts := []grpc.DialOption{ + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: time.Millisecond, /* should trigger "too many pings" error quickly */ + Timeout: time.Second * 1000, /* arbitrary, large value */ + PermitWithoutStream: false, + }), + } + if err := ss.Start(sopts, dopts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall = _, %v, want _, ", ss.Client, err) + } + const expectedErrorMessageSubstring = `received prior goaway: code: ENHANCE_YOUR_CALM, debug data: "too_many_pings"` + _, err = stream.Recv() + close(rpcDoneOnClient) + if err == nil || !strings.Contains(err.Error(), expectedErrorMessageSubstring) { + t.Fatalf("%v.Recv() = _, %v, want _, rpc error containing substring: |%v|", stream, err, expectedErrorMessageSubstring) + } +} + +func (s) TestClientConnCloseAfterGoAwayWithActiveStream(t *testing.T) { + for _, e := range listTestEnv() { + if e.name == "handler-tls" { + continue + } + testClientConnCloseAfterGoAwayWithActiveStream(t, e) + } +} + +func testClientConnCloseAfterGoAwayWithActiveStream(t *testing.T, e env) { + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.FullDuplexCall(ctx); err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want _, ", tc, err) + } + done := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(done) + }() + time.Sleep(50 * time.Millisecond) + cc.Close() + timeout := time.NewTimer(time.Second) + select { + case <-done: + case <-timeout.C: + t.Fatalf("Test timed-out.") + } +} + +func (s) TestServerGoAway(t *testing.T) { + for _, e := range listTestEnv() { + if e.name == "handler-tls" { + continue + } + testServerGoAway(t, e) + } +} + +func testServerGoAway(t *testing.T, e env) { + te := newTest(t, e) + te.userAgent = testAppUA + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + // Finish an RPC to make sure the connection is good. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + ch := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(ch) + }() + // Loop until the server side GoAway signal is propagated to the client. + for { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil && status.Code(err) != codes.DeadlineExceeded { + cancel() + break + } + cancel() + } + // A new RPC should fail. + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable && status.Code(err) != codes.Internal { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s or %s", err, codes.Unavailable, codes.Internal) + } + <-ch + awaitNewConnLogOutput() +} + +func (s) TestServerGoAwayPendingRPC(t *testing.T) { + for _, e := range listTestEnv() { + if e.name == "handler-tls" { + continue + } + testServerGoAwayPendingRPC(t, e) + } +} + +func testServerGoAwayPendingRPC(t *testing.T, e env) { + te := newTest(t, e) + te.userAgent = testAppUA + te.declareLogNoise( + "transport: http2Client.notifyError got notified that the client transport was broken EOF", + "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", + "grpc: addrConn.resetTransport failed to create client transport: connection error", + ) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + // Finish an RPC to make sure the connection is good. + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, ", tc, err) + } + ch := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(ch) + }() + // Loop until the server side GoAway signal is propagated to the client. + start := time.Now() + errored := false + for time.Since(start) < time.Second { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)) + cancel() + if err != nil { + errored = true + break + } + } + if !errored { + t.Fatalf("GoAway never received by client") + } + respParam := []*testpb.ResponseParameters{{Size: 1}} + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(100)) + if err != nil { + t.Fatal(err) + } + req := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: payload, + } + // The existing RPC should be still good to proceed. + if err := stream.Send(req); err != nil { + t.Fatalf("%v.Send(_) = %v, want ", stream, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = _, %v, want _, ", stream, err) + } + // The RPC will run until canceled. + cancel() + <-ch + awaitNewConnLogOutput() +} + +func (s) TestServerMultipleGoAwayPendingRPC(t *testing.T) { + for _, e := range listTestEnv() { + if e.name == "handler-tls" { + continue + } + testServerMultipleGoAwayPendingRPC(t, e) + } +} + +func testServerMultipleGoAwayPendingRPC(t *testing.T, e env) { + te := newTest(t, e) + te.userAgent = testAppUA + te.declareLogNoise( + "transport: http2Client.notifyError got notified that the client transport was broken EOF", + "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", + "grpc: addrConn.resetTransport failed to create client transport: connection error", + ) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + // Finish an RPC to make sure the connection is good. + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, ", tc, err) + } + ch1 := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(ch1) + }() + ch2 := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(ch2) + }() + // Loop until the server side GoAway signal is propagated to the client. + + for { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + cancel() + break + } + cancel() + } + select { + case <-ch1: + t.Fatal("GracefulStop() terminated early") + case <-ch2: + t.Fatal("GracefulStop() terminated early") + default: + } + respParam := []*testpb.ResponseParameters{ + { + Size: 1, + }, + } + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(100)) + if err != nil { + t.Fatal(err) + } + req := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: payload, + } + // The existing RPC should be still good to proceed. + if err := stream.Send(req); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, req, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = _, %v, want _, ", stream, err) + } + if err := stream.CloseSend(); err != nil { + t.Fatalf("%v.CloseSend() = %v, want ", stream, err) + } + + <-ch1 + <-ch2 + cancel() + awaitNewConnLogOutput() +} + +func (s) TestConcurrentClientConnCloseAndServerGoAway(t *testing.T) { + for _, e := range listTestEnv() { + if e.name == "handler-tls" { + continue + } + testConcurrentClientConnCloseAndServerGoAway(t, e) + } +} + +func testConcurrentClientConnCloseAndServerGoAway(t *testing.T, e env) { + te := newTest(t, e) + te.userAgent = testAppUA + te.declareLogNoise( + "transport: http2Client.notifyError got notified that the client transport was broken EOF", + "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", + "grpc: addrConn.resetTransport failed to create client transport: connection error", + ) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, ", tc, err) + } + ch := make(chan struct{}) + // Close ClientConn and Server concurrently. + go func() { + te.srv.GracefulStop() + close(ch) + }() + go func() { + cc.Close() + }() + <-ch +} + +func (s) TestConcurrentServerStopAndGoAway(t *testing.T) { + for _, e := range listTestEnv() { + if e.name == "handler-tls" { + continue + } + testConcurrentServerStopAndGoAway(t, e) + } +} + +func testConcurrentServerStopAndGoAway(t *testing.T, e env) { + te := newTest(t, e) + te.userAgent = testAppUA + te.declareLogNoise( + "transport: http2Client.notifyError got notified that the client transport was broken EOF", + "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", + "grpc: addrConn.resetTransport failed to create client transport: connection error", + ) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + + // Finish an RPC to make sure the connection is good. + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, ", tc, err) + } + + ch := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(ch) + }() + // Loop until the server side GoAway signal is propagated to the client. + for { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + cancel() + break + } + cancel() + } + // Stop the server and close all the connections. + te.srv.Stop() + respParam := []*testpb.ResponseParameters{ + { + Size: 1, + }, + } + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(100)) + if err != nil { + t.Fatal(err) + } + req := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: payload, + } + sendStart := time.Now() + for { + if err := stream.Send(req); err == io.EOF { + // stream.Send should eventually send io.EOF + break + } else if err != nil { + // Send should never return a transport-level error. + t.Fatalf("stream.Send(%v) = %v; want ", req, err) + } + if time.Since(sendStart) > 2*time.Second { + t.Fatalf("stream.Send(_) did not return io.EOF after 2s") + } + time.Sleep(time.Millisecond) + } + if _, err := stream.Recv(); err == nil || err == io.EOF { + t.Fatalf("%v.Recv() = _, %v, want _, ", stream, err) + } + <-ch + awaitNewConnLogOutput() +} + +// Proxies typically send GO_AWAY followed by connection closure a minute or so later. This +// test ensures that the connection is re-created after GO_AWAY and not affected by the +// subsequent (old) connection closure. +func (s) TestGoAwayThenClose(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + lis1, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + + unaryCallF := func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + } + fullDuplexCallF := func(stream testgrpc.TestService_FullDuplexCallServer) error { + if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil { + t.Errorf("Unexpected error from send: %v", err) + return err + } + // Wait until a message is received from client + _, err := stream.Recv() + if err == nil { + t.Error("Expected to never receive any message") + } + return err + } + ss1 := &stubserver.StubServer{ + Listener: lis1, + UnaryCallF: unaryCallF, + FullDuplexCallF: fullDuplexCallF, + S: grpc.NewServer(), + } + stubserver.StartTestService(t, ss1) + defer ss1.S.Stop() + + conn2Established := grpcsync.NewEvent() + lis2, err := listenWithNotifyingListener("tcp", "localhost:0", conn2Established) + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + ss2 := &stubserver.StubServer{ + Listener: lis2, + UnaryCallF: unaryCallF, + FullDuplexCallF: fullDuplexCallF, + S: grpc.NewServer(), + } + stubserver.StartTestService(t, ss2) + defer ss2.S.Stop() + + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{ + {Addr: lis1.Addr().String()}, + {Addr: lis2.Addr().String()}, + }}) + cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Error creating client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + // We make a streaming RPC and do an one-message-round-trip to make sure + // it's created on connection 1. + // + // We use a long-lived RPC because it will cause GracefulStop to send + // GO_AWAY, but the connection won't get closed until the server stops and + // the client receives the error. + t.Log("Creating first streaming RPC to server 1.") + stream, err := client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall(_) = _, %v; want _, nil", err) + } + if _, err = stream.Recv(); err != nil { + t.Fatalf("unexpected error from first recv: %v", err) + } + + t.Log("Gracefully stopping server 1.") + go ss1.S.GracefulStop() + + t.Log("Waiting for the ClientConn to enter IDLE state.") + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + t.Log("Performing another RPC to create a connection to server 2.") + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err) + } + + t.Log("Waiting for a connection to server 2.") + select { + case <-conn2Established.Done(): + case <-ctx.Done(): + t.Fatalf("timed out waiting for connection 2 to be established") + } + + // Close the listener for server2 to prevent it from allowing new connections. + lis2.Close() + + t.Log("Hard closing connection 1.") + ss1.S.Stop() + + t.Log("Waiting for the first stream to error.") + if _, err = stream.Recv(); err == nil { + t.Fatal("expected the stream to die, but got a successful Recv") + } + + t.Log("Ensuring connection 2 is stable.") + for i := 0; i < 10; i++ { + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err) + } + } +} + +// TestGoAwayStreamIDSmallerThanCreatedStreams tests the scenario where a server +// sends a goaway with a stream id that is smaller than some created streams on +// the client, while the client is simultaneously creating new streams. This +// should not induce a deadlock. +func (s) TestGoAwayStreamIDSmallerThanCreatedStreams(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("error listening: %v", err) + } + + ctCh := testutils.NewChannel() + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("error in lis.Accept(): %v", err) + } + ct := newClientTester(t, conn) + ctCh.Send(ct) + }() + + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err) + } + defer cc.Close() + cc.Connect() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + val, err := ctCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout waiting for client transport (should be given after http2 creation)") + } + ct := val.(*clientTester) + + tc := testgrpc.NewTestServiceClient(cc) + someStreamsCreated := grpcsync.NewEvent() + goAwayWritten := grpcsync.NewEvent() + go func() { + for i := 0; i < 20; i++ { + if i == 10 { + <-goAwayWritten.Done() + } + tc.FullDuplexCall(ctx) + if i == 4 { + someStreamsCreated.Fire() + } + } + }() + + <-someStreamsCreated.Done() + ct.writeGoAway(1, http2.ErrCodeNo, []byte{}) + goAwayWritten.Fire() +} + +// TestTwoGoAwayPingFrames tests the scenario where you get two go away ping +// frames from the client during graceful shutdown. This should not crash the +// server. +func (s) TestTwoGoAwayPingFrames(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen: %v", err) + } + defer lis.Close() + s := grpc.NewServer() + defer s.Stop() + go s.Serve(lis) + + conn, err := net.DialTimeout("tcp", lis.Addr().String(), defaultTestTimeout) + if err != nil { + t.Fatalf("Failed to dial: %v", err) + } + + st := newServerTesterFromConn(t, conn) + st.greet() + pingReceivedClientSide := testutils.NewChannel() + go func() { + for { + f, err := st.readFrame() + if err != nil { + return + } + switch f.(type) { + case *http2.GoAwayFrame: + case *http2.PingFrame: + pingReceivedClientSide.Send(nil) + default: + t.Errorf("server tester received unexpected frame type %T", f) + } + } + }() + gsDone := testutils.NewChannel() + go func() { + s.GracefulStop() + gsDone.Send(nil) + }() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := pingReceivedClientSide.Receive(ctx); err != nil { + t.Fatalf("Error waiting for ping frame client side from graceful shutdown: %v", err) + } + // Write two goaway pings here. + st.writePing(true, [8]byte{1, 6, 1, 8, 0, 3, 3, 9}) + st.writePing(true, [8]byte{1, 6, 1, 8, 0, 3, 3, 9}) + // Close the conn to finish up the Graceful Shutdown process. + conn.Close() + if _, err := gsDone.Receive(ctx); err != nil { + t.Fatalf("Error waiting for graceful shutdown of the server: %v", err) + } +} + +// TestClientSendsAGoAway tests the scenario where you get a go away ping +// frames from the client during graceful shutdown. +func (s) TestClientSendsAGoAway(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("error listening: %v", err) + } + defer lis.Close() + goAwayReceived := make(chan struct{}) + errCh := make(chan error) + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("error in lis.Accept(): %v", err) + } + ct := newClientTester(t, conn) + defer ct.conn.Close() + for { + f, err := ct.fr.ReadFrame() + if err != nil { + errCh <- fmt.Errorf("error reading frame: %v", err) + return + } + switch fr := f.(type) { + case *http2.GoAwayFrame: + fr = f.(*http2.GoAwayFrame) + if fr.ErrCode == http2.ErrCodeNo { + t.Logf("GoAway received from client") + close(goAwayReceived) + return + } + default: + t.Errorf("server tester received unexpected frame type %T", f) + errCh <- fmt.Errorf("server tester received unexpected frame type %T", f) + close(errCh) + } + } + }() + + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("error dialing: %v", err) + } + cc.Connect() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + cc.Close() + select { + case <-goAwayReceived: + case err := <-errCh: + t.Errorf("Error receiving the goAway: %v", err) + case <-ctx.Done(): + t.Errorf("Context timed out") + } +} diff --git a/test/gracefulstop_test.go b/test/gracefulstop_test.go new file mode 100644 index 0000000..1038c7b --- /dev/null +++ b/test/gracefulstop_test.go @@ -0,0 +1,364 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "golang.org/x/net/http2" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type delayListener struct { + net.Listener + closeCalled chan struct{} + acceptCalled chan struct{} + allowCloseCh chan struct{} + dialed bool +} + +func (d *delayListener) Accept() (net.Conn, error) { + select { + case <-d.acceptCalled: + // On the second call, block until closed, then return an error. + <-d.closeCalled + <-d.allowCloseCh + return nil, fmt.Errorf("listener is closed") + default: + close(d.acceptCalled) + conn, err := d.Listener.Accept() + if err != nil { + return nil, err + } + // Allow closing of listener only after accept. + // Note: Dial can return successfully, yet Accept + // might now have finished. + d.allowClose() + return conn, nil + } +} + +func (d *delayListener) allowClose() { + close(d.allowCloseCh) +} +func (d *delayListener) Close() error { + close(d.closeCalled) + go func() { + <-d.allowCloseCh + d.Listener.Close() + }() + return nil +} + +func (d *delayListener) Dial(ctx context.Context) (net.Conn, error) { + if d.dialed { + // Only hand out one connection (net.Dial can return more even after the + // listener is closed). This is not thread-safe, but Dial should never be + // called concurrently in this environment. + return nil, fmt.Errorf("no more conns") + } + d.dialed = true + return (&net.Dialer{}).DialContext(ctx, "tcp", d.Listener.Addr().String()) +} + +// TestGracefulStop ensures GracefulStop causes new connections to fail. +// +// Steps of this test: +// 1. Start Server +// 2. GracefulStop() Server after listener's Accept is called, but don't +// allow Accept() to exit when Close() is called on it. +// 3. Create a new connection to the server after listener.Close() is called. +// Server should close this connection immediately, before handshaking. +// 4. Send an RPC on the new connection. Should see Unavailable error +// because the ClientConn is in transient failure. +func (s) TestGracefulStop(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error listenening: %v", err) + } + dlis := &delayListener{ + Listener: lis, + acceptCalled: make(chan struct{}), + closeCalled: make(chan struct{}), + allowCloseCh: make(chan struct{}), + } + + ss := &stubserver.StubServer{ + Listener: dlis, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + return stream.Send(&testpb.StreamingOutputCallResponse{}) + }, + S: grpc.NewServer(), + } + // 1. Start Server and start serving by calling Serve(). + stubserver.StartTestService(t, ss) + + // 2. Call GracefulStop from a goroutine. It will trigger Close on the + // listener, but the listener will not actually close until a connection + // is accepted. + gracefulStopDone := make(chan struct{}) + <-dlis.acceptCalled + go func() { + ss.S.GracefulStop() + close(gracefulStopDone) + }() + + // 3. Create a new connection to the server after listener.Close() is called. + // Server should close this connection immediately, before handshaking. + + <-dlis.closeCalled // Block until GracefulStop calls dlis.Close() + + dialer := func(ctx context.Context, _ string) (net.Conn, error) { return dlis.Dial(ctx) } + cc, err := grpc.NewClient("passthrough:///", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialer)) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err) + } + client := testgrpc.NewTestServiceClient(cc) + defer cc.Close() + + // 4. Make an RPC. + // The server would send a GOAWAY first, but we are delaying the server's + // writes for now until the client writes more than the preface. + // This will cause a connection to be accepted. This will + // also unblock the Close method. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + if _, err = client.FullDuplexCall(ctx); err == nil || status.Code(err) != codes.Unavailable { + t.Fatalf("FullDuplexCall= _, %v; want _, ", err) + } + cancel() + <-gracefulStopDone +} + +// TestGracefulStopClosesConnAfterLastStream ensures that a server closes the +// connections to its clients when the final stream has completed after +// a GOAWAY. +func (s) TestGracefulStopClosesConnAfterLastStream(t *testing.T) { + + handlerCalled := make(chan struct{}) + gracefulStopCalled := make(chan struct{}) + + ts := &funcServer{streamingInputCall: func(testgrpc.TestService_StreamingInputCallServer) error { + close(handlerCalled) // Initiate call to GracefulStop. + <-gracefulStopCalled // Wait for GOAWAYs to be received by the client. + return nil + }} + + te := newTest(t, tcpClearEnv) + te.startServer(ts) + defer te.tearDown() + + te.withServerTester(func(st *serverTester) { + st.writeHeadersGRPC(1, "/grpc.testing.TestService/StreamingInputCall", false) + + <-handlerCalled // Wait for the server to invoke its handler. + + // Gracefully stop the server. + gracefulStopDone := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(gracefulStopDone) + }() + st.wantGoAway(http2.ErrCodeNo) // Server sends a GOAWAY due to GracefulStop. + pf := st.wantPing() // Server sends a ping to verify client receipt. + st.writePing(true, pf.Data) // Send ping ack to confirm. + st.wantGoAway(http2.ErrCodeNo) // Wait for subsequent GOAWAY to indicate no new stream processing. + + close(gracefulStopCalled) // Unblock server handler. + + fr := st.wantAnyFrame() // Wait for trailer. + hdr, ok := fr.(*http2.MetaHeadersFrame) + if !ok { + t.Fatalf("Received unexpected frame of type (%T) from server: %v; want HEADERS", fr, fr) + } + if !hdr.StreamEnded() { + t.Fatalf("Received unexpected HEADERS frame from server: %v; want END_STREAM set", fr) + } + + st.wantRSTStream(http2.ErrCodeNo) // Server should send RST_STREAM because client did not half-close. + + <-gracefulStopDone // Wait for GracefulStop to return. + }) +} + +// TestGracefulStopBlocksUntilGRPCConnectionsTerminate ensures that +// GracefulStop() blocks until all ongoing RPCs finished. +func (s) TestGracefulStopBlocksUntilGRPCConnectionsTerminate(t *testing.T) { + unblockGRPCCall := make(chan struct{}) + grpcCallExecuting := make(chan struct{}) + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + close(grpcCallExecuting) + <-unblockGRPCCall + return &testpb.SimpleResponse{}, nil + }, + } + + err := ss.Start(nil) + if err != nil { + t.Fatalf("StubServer.start failed: %s", err) + } + t.Cleanup(ss.Stop) + + grpcClientCallReturned := make(chan struct{}) + go func() { + clt := ss.Client + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := clt.UnaryCall(ctx, &testpb.SimpleRequest{}) + if err != nil { + t.Errorf("rpc failed with error: %s", err) + } + close(grpcClientCallReturned) + }() + + gracefulStopReturned := make(chan struct{}) + <-grpcCallExecuting + go func() { + ss.S.GracefulStop() + close(gracefulStopReturned) + }() + + select { + case <-gracefulStopReturned: + t.Error("GracefulStop returned before rpc method call ended") + case <-time.After(defaultTestShortTimeout): + } + + unblockGRPCCall <- struct{}{} + <-grpcClientCallReturned + <-gracefulStopReturned +} + +// TestStopAbortsBlockingGRPCCall ensures that when Stop() is called while an ongoing RPC +// is blocking that: +// - Stop() returns +// - and the RPC fails with an connection closed error on the client-side +func (s) TestStopAbortsBlockingGRPCCall(t *testing.T) { + unblockGRPCCall := make(chan struct{}) + grpcCallExecuting := make(chan struct{}) + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + close(grpcCallExecuting) + <-unblockGRPCCall + return &testpb.SimpleResponse{}, nil + }, + } + + err := ss.Start(nil) + if err != nil { + t.Fatalf("StubServer.start failed: %s", err) + } + t.Cleanup(ss.Stop) + + grpcClientCallReturned := make(chan struct{}) + go func() { + clt := ss.Client + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := clt.UnaryCall(ctx, &testpb.SimpleRequest{}) + if err == nil || !isConnClosedErr(err) { + t.Errorf("expected rpc to fail with connection closed error, got: %v", err) + } + close(grpcClientCallReturned) + }() + + <-grpcCallExecuting + ss.S.Stop() + + unblockGRPCCall <- struct{}{} + <-grpcClientCallReturned +} + +// TestServeHTTP_GracefulStop verifies that calling GracefulStop on a server +// using ServeHTTP returns properly. +func (s) TestServeHTTP_GracefulStop(t *testing.T) { + gs := grpc.NewServer() + defer gs.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Register a service that blocks until context is done to make sure the + // server is active when GracefulStop is called. + callStarted := make(chan struct{}) + testgrpc.RegisterTestServiceServer(gs, &stubserver.StubServer{ + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + close(callStarted) + <-ctx.Done() + return nil + }, + }) + + // Create a handler that delegates to ServeHTTP and signals when it returns. + serveHTTPDone := make(chan struct{}) + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gs.ServeHTTP(w, r) + close(serveHTTPDone) + }) + + // Start an HTTP/2 server with the wrapped handler. + ts := httptest.NewUnstartedServer(handler) + ts.EnableHTTP2 = true + ts.StartTLS() + defer ts.Close() + + // Make a new request to the gRPC server via HTTP/2 to ensure HandleStreams is called. + req, err := http.NewRequest("POST", ts.URL+"/grpc.testing.TestService/FullDuplexCall", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/grpc") + + client := ts.Client() + // Send the request in a goroutine. + go func() { + resp, err := client.Do(req) + if err != nil { + return + } + resp.Body.Close() + }() + + <-callStarted + // Call GracefulStop in a goroutine to avoid blocking the main thread until the RPC is done. + go gs.GracefulStop() + + // Verify that ServeHTTP returned because GracefulStop is called and ensure it doesn't panic. + select { + case <-serveHTTPDone: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for ServeHTTP to return after GracefulStop") + } +} diff --git a/test/healthcheck_test.go b/test/healthcheck_test.go new file mode 100644 index 0000000..c406d22 --- /dev/null +++ b/test/healthcheck_test.go @@ -0,0 +1,1281 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "errors" + "fmt" + "net" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func newTestHealthServer() *testHealthServer { + return newTestHealthServerWithWatchFunc(defaultWatchFunc) +} + +func newTestHealthServerWithWatchFunc(f healthWatchFunc) *testHealthServer { + return &testHealthServer{ + watchFunc: f, + update: make(chan struct{}, 1), + status: make(map[string]healthpb.HealthCheckResponse_ServingStatus), + } +} + +// defaultWatchFunc will send a HealthCheckResponse to the client whenever SetServingStatus is called. +func defaultWatchFunc(s *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { + if in.Service != "foo" { + return status.Error(codes.FailedPrecondition, + "the defaultWatchFunc only handles request with service name to be \"foo\"") + } + var done bool + for { + select { + case <-stream.Context().Done(): + done = true + case <-s.update: + } + if done { + break + } + s.mu.Lock() + resp := &healthpb.HealthCheckResponse{ + Status: s.status[in.Service], + } + s.mu.Unlock() + stream.SendMsg(resp) + } + return nil +} + +type healthWatchFunc func(*testHealthServer, *healthpb.HealthCheckRequest, healthgrpc.Health_WatchServer) error + +type testHealthServer struct { + healthgrpc.UnimplementedHealthServer + watchFunc healthWatchFunc + mu sync.Mutex + status map[string]healthpb.HealthCheckResponse_ServingStatus + update chan struct{} +} + +func (s *testHealthServer) Check(context.Context, *healthpb.HealthCheckRequest) (*healthpb.HealthCheckResponse, error) { + return &healthpb.HealthCheckResponse{ + Status: healthpb.HealthCheckResponse_SERVING, + }, nil +} + +func (s *testHealthServer) Watch(in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { + return s.watchFunc(s, in, stream) +} + +// SetServingStatus is called when need to reset the serving status of a service +// or insert a new service entry into the statusMap. +func (s *testHealthServer) SetServingStatus(service string, status healthpb.HealthCheckResponse_ServingStatus) { + s.mu.Lock() + s.status[service] = status + select { + case <-s.update: + default: + } + s.update <- struct{}{} + s.mu.Unlock() +} + +func setupHealthCheckWrapper(t *testing.T) (hcEnterChan chan struct{}, hcExitChan chan struct{}) { + t.Helper() + + hcEnterChan = make(chan struct{}) + hcExitChan = make(chan struct{}) + origHealthCheckFn := internal.HealthCheckFunc + internal.HealthCheckFunc = func(ctx context.Context, newStream func(string) (any, error), update func(connectivity.State, error), service string) error { + close(hcEnterChan) + defer close(hcExitChan) + return origHealthCheckFn(ctx, newStream, update, service) + } + + t.Cleanup(func() { + internal.HealthCheckFunc = origHealthCheckFn + }) + + return +} + +func setupServer(t *testing.T, watchFunc healthWatchFunc) (*grpc.Server, net.Listener, *testHealthServer) { + t.Helper() + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + + var ts *testHealthServer + if watchFunc != nil { + ts = newTestHealthServerWithWatchFunc(watchFunc) + } else { + ts = newTestHealthServer() + } + s := grpc.NewServer() + healthgrpc.RegisterHealthServer(s, ts) + testgrpc.RegisterTestServiceServer(s, &testServer{}) + go s.Serve(lis) + t.Cleanup(func() { s.Stop() }) + return s, lis, ts +} + +type clientConfig struct { + balancerName string + extraDialOption []grpc.DialOption +} + +func setupClient(t *testing.T, c *clientConfig) (*grpc.ClientConn, *manual.Resolver) { + t.Helper() + + r := manual.NewBuilderWithScheme("whatever") + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + } + if c != nil { + if c.balancerName != "" { + opts = append(opts, grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, c.balancerName))) + } + opts = append(opts, c.extraDialOption...) + } + + cc, err := grpc.NewClient(r.Scheme()+":///test.server", opts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + cc.Connect() + t.Cleanup(func() { cc.Close() }) + return cc, r +} + +func (s) TestHealthCheckWatchStateChange(t *testing.T) { + _, lis, ts := setupServer(t, nil) + + // The table below shows the expected series of addrConn connectivity transitions when server + // updates its health status. As there's only one addrConn corresponds with the ClientConn in this + // test, we use ClientConn's connectivity state as the addrConn connectivity state. + //+------------------------------+-------------------------------------------+ + //| Health Check Returned Status | Expected addrConn Connectivity Transition | + //+------------------------------+-------------------------------------------+ + //| NOT_SERVING | ->TRANSIENT FAILURE | + //| SERVING | ->READY | + //| SERVICE_UNKNOWN | ->TRANSIENT FAILURE | + //| SERVING | ->READY | + //| UNKNOWN | ->TRANSIENT FAILURE | + //+------------------------------+-------------------------------------------+ + ts.SetServingStatus("foo", healthpb.HealthCheckResponse_NOT_SERVING) + + cc, r := setupClient(t, nil) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, + ServiceConfig: parseServiceConfig(t, r, `{ + "healthCheckConfig": { + "serviceName": "foo" + }, + "loadBalancingConfig": [{"round_robin":{}}] +}`)}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitNotState(ctx, t, cc, connectivity.Idle) + testutils.AwaitNotState(ctx, t, cc, connectivity.Connecting) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + if s := cc.GetState(); s != connectivity.TransientFailure { + t.Fatalf("ClientConn is in %v state, want TRANSIENT FAILURE", s) + } + + ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) + testutils.AwaitNotState(ctx, t, cc, connectivity.TransientFailure) + if s := cc.GetState(); s != connectivity.Ready { + t.Fatalf("ClientConn is in %v state, want READY", s) + } + + ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVICE_UNKNOWN) + testutils.AwaitNotState(ctx, t, cc, connectivity.Ready) + if s := cc.GetState(); s != connectivity.TransientFailure { + t.Fatalf("ClientConn is in %v state, want TRANSIENT FAILURE", s) + } + + ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) + testutils.AwaitNotState(ctx, t, cc, connectivity.TransientFailure) + if s := cc.GetState(); s != connectivity.Ready { + t.Fatalf("ClientConn is in %v state, want READY", s) + } + + ts.SetServingStatus("foo", healthpb.HealthCheckResponse_UNKNOWN) + testutils.AwaitNotState(ctx, t, cc, connectivity.Ready) + if s := cc.GetState(); s != connectivity.TransientFailure { + t.Fatalf("ClientConn is in %v state, want TRANSIENT FAILURE", s) + } +} + +// If Watch returns Unimplemented, then the ClientConn should go into READY state. +func (s) TestHealthCheckHealthServerNotRegistered(t *testing.T) { + grpctest.ExpectError("Subchannel health check is unimplemented at server side, thus health check is disabled") + s := grpc.NewServer() + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to listen due to err: %v", err) + } + go s.Serve(lis) + defer s.Stop() + + cc, r := setupClient(t, nil) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, + ServiceConfig: parseServiceConfig(t, r, fmt.Sprintf(`{ + "healthCheckConfig": { + "serviceName": "foo" + }, + "loadBalancingConfig": [{"%s":{}}] + }`, roundrobin.Name))}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitNotState(ctx, t, cc, connectivity.Idle) + testutils.AwaitNotState(ctx, t, cc, connectivity.Connecting) + if s := cc.GetState(); s != connectivity.Ready { + t.Fatalf("ClientConn is in %v state, want READY", s) + } +} + +// In the case of a goaway received, the health check stream should be terminated and health check +// function should exit. +func (s) TestHealthCheckWithGoAway(t *testing.T) { + s, lis, ts := setupServer(t, nil) + ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) + + hcEnterChan, hcExitChan := setupHealthCheckWrapper(t) + cc, r := setupClient(t, &clientConfig{}) + tc := testgrpc.NewTestServiceClient(cc) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, + ServiceConfig: parseServiceConfig(t, r, fmt.Sprintf(`{ + "healthCheckConfig": { + "serviceName": "foo" + }, + "loadBalancingConfig": [{"%s":{}}] + }`, roundrobin.Name))}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // make some rpcs to make sure connection is working. + if err := verifyResultWithDelay(func() (bool, error) { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + // the stream rpc will persist through goaway event. + stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + respParam := []*testpb.ResponseParameters{{Size: 1}} + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(1)) + if err != nil { + t.Fatal(err) + } + req := &testpb.StreamingOutputCallRequest{ + ResponseParameters: respParam, + Payload: payload, + } + if err := stream.Send(req); err != nil { + t.Fatalf("%v.Send(_) = %v, want ", stream, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = _, %v, want _, ", stream, err) + } + + select { + case <-hcExitChan: + t.Fatal("Health check function has exited, which is not expected.") + default: + } + + // server sends GoAway + go s.GracefulStop() + + select { + case <-hcExitChan: + case <-time.After(5 * time.Second): + select { + case <-hcEnterChan: + default: + t.Fatal("Health check function has not entered after 5s.") + } + t.Fatal("Health check function has not exited after 5s.") + } + + // The existing RPC should be still good to proceed. + if err := stream.Send(req); err != nil { + t.Fatalf("%v.Send(_) = %v, want ", stream, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = _, %v, want _, ", stream, err) + } +} + +func (s) TestHealthCheckWithConnClose(t *testing.T) { + s, lis, ts := setupServer(t, nil) + ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) + + hcEnterChan, hcExitChan := setupHealthCheckWrapper(t) + cc, r := setupClient(t, &clientConfig{}) + tc := testgrpc.NewTestServiceClient(cc) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, + ServiceConfig: parseServiceConfig(t, r, fmt.Sprintf(`{ + "healthCheckConfig": { + "serviceName": "foo" + }, + "loadBalancingConfig": [{"%s":{}}] + }`, roundrobin.Name))}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // make some rpcs to make sure connection is working. + if err := verifyResultWithDelay(func() (bool, error) { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + select { + case <-hcExitChan: + t.Fatal("Health check function has exited, which is not expected.") + default: + } + // server closes the connection + s.Stop() + + select { + case <-hcExitChan: + case <-time.After(5 * time.Second): + select { + case <-hcEnterChan: + default: + t.Fatal("Health check function has not entered after 5s.") + } + t.Fatal("Health check function has not exited after 5s.") + } +} + +// addrConn drain happens when addrConn gets torn down due to its address being no longer in the +// address list returned by the resolver. +func (s) TestHealthCheckWithAddrConnDrain(t *testing.T) { + _, lis, ts := setupServer(t, nil) + ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) + + hcEnterChan, hcExitChan := setupHealthCheckWrapper(t) + cc, r := setupClient(t, &clientConfig{}) + tc := testgrpc.NewTestServiceClient(cc) + sc := parseServiceConfig(t, r, fmt.Sprintf(`{ + "healthCheckConfig": { + "serviceName": "foo" + }, + "loadBalancingConfig": [{"%s":{}}] + }`, roundrobin.Name)) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, + ServiceConfig: sc, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // make some rpcs to make sure connection is working. + if err := verifyResultWithDelay(func() (bool, error) { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + // the stream rpc will persist through goaway event. + stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + respParam := []*testpb.ResponseParameters{{Size: 1}} + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(1)) + if err != nil { + t.Fatal(err) + } + req := &testpb.StreamingOutputCallRequest{ + ResponseParameters: respParam, + Payload: payload, + } + if err := stream.Send(req); err != nil { + t.Fatalf("%v.Send(_) = %v, want ", stream, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = _, %v, want _, ", stream, err) + } + + select { + case <-hcExitChan: + t.Fatal("Health check function has exited, which is not expected.") + default: + } + // trigger teardown of the ac + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "fake address"}}, ServiceConfig: sc}) + + select { + case <-hcExitChan: + case <-time.After(5 * time.Second): + select { + case <-hcEnterChan: + default: + t.Fatal("Health check function has not entered after 5s.") + } + t.Fatal("Health check function has not exited after 5s.") + } + + // The existing RPC should be still good to proceed. + if err := stream.Send(req); err != nil { + t.Fatalf("%v.Send(_) = %v, want ", stream, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = _, %v, want _, ", stream, err) + } +} + +// ClientConn close will lead to its addrConns being torn down. +func (s) TestHealthCheckWithClientConnClose(t *testing.T) { + _, lis, ts := setupServer(t, nil) + ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) + + hcEnterChan, hcExitChan := setupHealthCheckWrapper(t) + cc, r := setupClient(t, &clientConfig{}) + tc := testgrpc.NewTestServiceClient(cc) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, + ServiceConfig: parseServiceConfig(t, r, (fmt.Sprintf(`{ + "healthCheckConfig": { + "serviceName": "foo" + }, + "loadBalancingConfig": [{"%s":{}}] + }`, roundrobin.Name)))}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // make some rpcs to make sure connection is working. + if err := verifyResultWithDelay(func() (bool, error) { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + select { + case <-hcExitChan: + t.Fatal("Health check function has exited, which is not expected.") + default: + } + + // trigger addrConn teardown + cc.Close() + + select { + case <-hcExitChan: + case <-time.After(5 * time.Second): + select { + case <-hcEnterChan: + default: + t.Fatal("Health check function has not entered after 5s.") + } + t.Fatal("Health check function has not exited after 5s.") + } +} + +// This test is to test the logic in the createTransport after the health check function returns which +// closes the skipReset channel(since it has not been closed inside health check func) to unblock +// onGoAway/onClose goroutine. +func (s) TestHealthCheckWithoutSetConnectivityStateCalledAddrConnShutDown(t *testing.T) { + watchFunc := func(_ *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { + if in.Service != "delay" { + return status.Error(codes.FailedPrecondition, + "this special Watch function only handles request with service name to be \"delay\"") + } + // Do nothing to mock a delay of health check response from server side. + // This case is to help with the test that covers the condition that setConnectivityState is not + // called inside HealthCheckFunc before the func returns. + select { + case <-stream.Context().Done(): + case <-time.After(5 * time.Second): + } + return nil + } + _, lis, ts := setupServer(t, watchFunc) + ts.SetServingStatus("delay", healthpb.HealthCheckResponse_SERVING) + + hcEnterChan, hcExitChan := setupHealthCheckWrapper(t) + _, r := setupClient(t, &clientConfig{}) + + // The serviceName "delay" is specially handled at server side, where response will not be sent + // back to client immediately upon receiving the request (client should receive no response until + // test ends). + sc := parseServiceConfig(t, r, fmt.Sprintf(`{ + "healthCheckConfig": { + "serviceName": "delay" + }, + "loadBalancingConfig": [{"%s":{}}] + }`, roundrobin.Name)) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, + ServiceConfig: sc, + }) + + select { + case <-hcExitChan: + t.Fatal("Health check function has exited, which is not expected.") + default: + } + + select { + case <-hcEnterChan: + case <-time.After(5 * time.Second): + t.Fatal("Health check function has not been invoked after 5s.") + } + // trigger teardown of the ac, ac in SHUTDOWN state + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "fake address"}}, ServiceConfig: sc}) + + // The health check func should exit without calling the setConnectivityState func, as server hasn't sent + // any response. + select { + case <-hcExitChan: + case <-time.After(5 * time.Second): + t.Fatal("Health check function has not exited after 5s.") + } + // The deferred leakcheck will check whether there's leaked goroutine, which is an indication + // whether we closes the skipReset channel to unblock onGoAway/onClose goroutine. +} + +// This test is to test the logic in the createTransport after the health check function returns which +// closes the allowedToReset channel(since it has not been closed inside health check func) to unblock +// onGoAway/onClose goroutine. +func (s) TestHealthCheckWithoutSetConnectivityStateCalled(t *testing.T) { + watchFunc := func(_ *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { + if in.Service != "delay" { + return status.Error(codes.FailedPrecondition, + "this special Watch function only handles request with service name to be \"delay\"") + } + // Do nothing to mock a delay of health check response from server side. + // This case is to help with the test that covers the condition that setConnectivityState is not + // called inside HealthCheckFunc before the func returns. + select { + case <-stream.Context().Done(): + case <-time.After(5 * time.Second): + } + return nil + } + s, lis, ts := setupServer(t, watchFunc) + ts.SetServingStatus("delay", healthpb.HealthCheckResponse_SERVING) + + hcEnterChan, hcExitChan := setupHealthCheckWrapper(t) + _, r := setupClient(t, &clientConfig{}) + + // The serviceName "delay" is specially handled at server side, where response will not be sent + // back to client immediately upon receiving the request (client should receive no response until + // test ends). + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, + ServiceConfig: parseServiceConfig(t, r, fmt.Sprintf(`{ + "healthCheckConfig": { + "serviceName": "delay" + }, + "loadBalancingConfig": [{"%s":{}}] + }`, roundrobin.Name))}) + + select { + case <-hcExitChan: + t.Fatal("Health check function has exited, which is not expected.") + default: + } + + select { + case <-hcEnterChan: + case <-time.After(5 * time.Second): + t.Fatal("Health check function has not been invoked after 5s.") + } + // trigger transport being closed + s.Stop() + + // The health check func should exit without calling the setConnectivityState func, as server hasn't sent + // any response. + select { + case <-hcExitChan: + case <-time.After(5 * time.Second): + t.Fatal("Health check function has not exited after 5s.") + } + // The deferred leakcheck will check whether there's leaked goroutine, which is an indication + // whether we closes the allowedToReset channel to unblock onGoAway/onClose goroutine. +} + +func testHealthCheckDisableWithDialOption(t *testing.T, addr string) { + hcEnterChan, _ := setupHealthCheckWrapper(t) + cc, r := setupClient(t, &clientConfig{extraDialOption: []grpc.DialOption{grpc.WithDisableHealthCheck()}}) + tc := testgrpc.NewTestServiceClient(cc) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: addr}}, + ServiceConfig: parseServiceConfig(t, r, fmt.Sprintf(`{ + "healthCheckConfig": { + "serviceName": "foo" + }, + "loadBalancingConfig": [{"%s":{}}] + }`, roundrobin.Name))}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // send some rpcs to make sure transport has been created and is ready for use. + if err := verifyResultWithDelay(func() (bool, error) { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + select { + case <-hcEnterChan: + t.Fatal("Health check function has exited, which is not expected.") + default: + } +} + +func testHealthCheckDisableWithBalancer(t *testing.T, addr string) { + hcEnterChan, _ := setupHealthCheckWrapper(t) + cc, r := setupClient(t, &clientConfig{}) + tc := testgrpc.NewTestServiceClient(cc) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: addr}}, + ServiceConfig: parseServiceConfig(t, r, `{ + "healthCheckConfig": { + "serviceName": "foo" + }, + "loadBalancingConfig": [{"pick_first":{}}] +}`)}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // send some rpcs to make sure transport has been created and is ready for use. + if err := verifyResultWithDelay(func() (bool, error) { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + select { + case <-hcEnterChan: + t.Fatal("Health check function has started, which is not expected.") + default: + } +} + +func testHealthCheckDisableWithServiceConfig(t *testing.T, addr string) { + hcEnterChan, _ := setupHealthCheckWrapper(t) + cc, r := setupClient(t, &clientConfig{}) + tc := testgrpc.NewTestServiceClient(cc) + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: addr}}}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // send some rpcs to make sure transport has been created and is ready for use. + if err := verifyResultWithDelay(func() (bool, error) { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + select { + case <-hcEnterChan: + t.Fatal("Health check function has started, which is not expected.") + default: + } +} + +func (s) TestHealthCheckDisable(t *testing.T) { + _, lis, ts := setupServer(t, nil) + ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) + + // test client side disabling configuration. + testHealthCheckDisableWithDialOption(t, lis.Addr().String()) + testHealthCheckDisableWithBalancer(t, lis.Addr().String()) + testHealthCheckDisableWithServiceConfig(t, lis.Addr().String()) +} + +func (s) TestHealthCheckChannelzCountingCallSuccess(t *testing.T) { + watchFunc := func(_ *testHealthServer, in *healthpb.HealthCheckRequest, _ healthgrpc.Health_WatchServer) error { + if in.Service != "channelzSuccess" { + return status.Error(codes.FailedPrecondition, + "this special Watch function only handles request with service name to be \"channelzSuccess\"") + } + return status.Error(codes.OK, "fake success") + } + _, lis, _ := setupServer(t, watchFunc) + + _, r := setupClient(t, nil) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, + ServiceConfig: parseServiceConfig(t, r, fmt.Sprintf(`{ + "healthCheckConfig": { + "serviceName": "channelzSuccess" + }, + "loadBalancingConfig": [{"%s":{}}] + }`, roundrobin.Name))}) + + if err := verifyResultWithDelay(func() (bool, error) { + cm, _ := channelz.GetTopChannels(0, 0) + if len(cm) == 0 { + return false, errors.New("channelz.GetTopChannels return 0 top channel") + } + subChans := cm[0].SubChans() + if len(subChans) == 0 { + return false, errors.New("there is 0 subchannel") + } + var id int64 + for k := range subChans { + id = k + break + } + scm := channelz.GetSubChannel(id) + if scm == nil { + return false, errors.New("nil subchannel returned") + } + // exponential backoff retry may result in more than one health check call. + cstart, csucc, cfail := scm.ChannelMetrics.CallsStarted.Load(), scm.ChannelMetrics.CallsSucceeded.Load(), scm.ChannelMetrics.CallsFailed.Load() + if cstart > 0 && csucc > 0 && cfail == 0 { + return true, nil + } + return false, fmt.Errorf("got %d CallsStarted, %d CallsSucceeded %d CallsFailed, want >0 >0 =0", cstart, csucc, cfail) + }); err != nil { + t.Fatal(err) + } +} + +func (s) TestHealthCheckChannelzCountingCallFailure(t *testing.T) { + watchFunc := func(_ *testHealthServer, in *healthpb.HealthCheckRequest, _ healthgrpc.Health_WatchServer) error { + if in.Service != "channelzFailure" { + return status.Error(codes.FailedPrecondition, + "this special Watch function only handles request with service name to be \"channelzFailure\"") + } + return status.Error(codes.Internal, "fake failure") + } + _, lis, _ := setupServer(t, watchFunc) + + _, r := setupClient(t, nil) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, + ServiceConfig: parseServiceConfig(t, r, fmt.Sprintf(`{ + "healthCheckConfig": { + "serviceName": "channelzFailure" + }, + "loadBalancingConfig": [{"%s":{}}] + }`, roundrobin.Name))}) + + if err := verifyResultWithDelay(func() (bool, error) { + cm, _ := channelz.GetTopChannels(0, 0) + if len(cm) == 0 { + return false, errors.New("channelz.GetTopChannels return 0 top channel") + } + subChans := cm[0].SubChans() + if len(subChans) == 0 { + return false, errors.New("there is 0 subchannel") + } + var id int64 + for k := range subChans { + id = k + break + } + scm := channelz.GetSubChannel(id) + if scm == nil { + return false, errors.New("nil subchannel returned") + } + // exponential backoff retry may result in more than one health check call. + cstart, cfail, csucc := scm.ChannelMetrics.CallsStarted.Load(), scm.ChannelMetrics.CallsFailed.Load(), scm.ChannelMetrics.CallsSucceeded.Load() + if cstart > 0 && cfail > 0 && csucc == 0 { + return true, nil + } + return false, fmt.Errorf("got %d CallsStarted, %d CallsFailed, %d CallsSucceeded, want >0, >0", cstart, cfail, csucc) + }); err != nil { + t.Fatal(err) + } +} + +// healthCheck is a helper function to make a unary health check RPC and return +// the response. +func healthCheck(d time.Duration, cc *grpc.ClientConn, service string) (*healthpb.HealthCheckResponse, error) { + ctx, cancel := context.WithTimeout(context.Background(), d) + defer cancel() + hc := healthgrpc.NewHealthClient(cc) + return hc.Check(ctx, &healthpb.HealthCheckRequest{Service: service}) +} + +// verifyHealthCheckStatus is a helper function to verify that the current +// health status of the service matches the one passed in 'wantStatus'. +func verifyHealthCheckStatus(t *testing.T, d time.Duration, cc *grpc.ClientConn, service string, wantStatus healthpb.HealthCheckResponse_ServingStatus) { + t.Helper() + resp, err := healthCheck(d, cc, service) + if err != nil { + t.Fatalf("Health/Check(_, _) = _, %v, want _, ", err) + } + if resp.Status != wantStatus { + t.Fatalf("Got the serving status %v, want %v", resp.Status, wantStatus) + } +} + +// verifyHealthCheckErrCode is a helper function to verify that a unary health +// check RPC returns an error with a code set to 'wantCode'. +func verifyHealthCheckErrCode(t *testing.T, d time.Duration, cc *grpc.ClientConn, service string, wantCode codes.Code) { + t.Helper() + if _, err := healthCheck(d, cc, service); status.Code(err) != wantCode { + t.Fatalf("Health/Check() got errCode %v, want %v", status.Code(err), wantCode) + } +} + +// newHealthCheckStream is a helper function to start a health check streaming +// RPC, and returns the stream. +func newHealthCheckStream(t *testing.T, cc *grpc.ClientConn, service string) (healthgrpc.Health_WatchClient, context.CancelFunc) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + hc := healthgrpc.NewHealthClient(cc) + stream, err := hc.Watch(ctx, &healthpb.HealthCheckRequest{Service: service}) + if err != nil { + t.Fatalf("hc.Watch(_, %v) failed: %v", service, err) + } + return stream, cancel +} + +// healthWatchChecker is a helper function to verify that the next health +// status returned on the given stream matches the one passed in 'wantStatus'. +func healthWatchChecker(t *testing.T, stream healthgrpc.Health_WatchClient, wantStatus healthpb.HealthCheckResponse_ServingStatus) { + t.Helper() + response, err := stream.Recv() + if err != nil { + t.Fatalf("stream.Recv() failed: %v", err) + } + if response.Status != wantStatus { + t.Fatalf("got servingStatus %v, want %v", response.Status, wantStatus) + } +} + +// TestHealthCheckSuccess invokes the unary Check() RPC on the health server in +// a successful case. +func (s) TestHealthCheckSuccess(t *testing.T) { + for _, e := range listTestEnv() { + testHealthCheckSuccess(t, e) + } +} + +func testHealthCheckSuccess(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + defer te.tearDown() + + verifyHealthCheckErrCode(t, 1*time.Second, te.clientConn(), defaultHealthService, codes.OK) +} + +// TestHealthCheckFailure invokes the unary Check() RPC on the health server +// with an expired context and expects the RPC to fail. +func (s) TestHealthCheckFailure(t *testing.T) { + e := env{ + name: "tcp-tls", + network: "tcp", + security: "tls", + balancer: roundrobin.Name, + } + te := newTest(t, e) + te.declareLogNoise( + "Failed to dial ", + "grpc: the client connection is closing; please retry", + ) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + defer te.tearDown() + + verifyHealthCheckErrCode(t, 0*time.Second, te.clientConn(), defaultHealthService, codes.DeadlineExceeded) + awaitNewConnLogOutput() +} + +// TestHealthCheckOff makes a unary Check() RPC on the health server where the +// health status of the defaultHealthService is not set, and therefore expects +// an error code 'codes.NotFound'. +func (s) TestHealthCheckOff(t *testing.T) { + for _, e := range listTestEnv() { + // TODO(bradfitz): Temporarily skip this env due to #619. + if e.name == "handler-tls" { + continue + } + testHealthCheckOff(t, e) + } +} + +func testHealthCheckOff(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + verifyHealthCheckErrCode(t, 1*time.Second, te.clientConn(), defaultHealthService, codes.NotFound) +} + +// TestHealthWatchMultipleClients makes a streaming Watch() RPC on the health +// server with multiple clients and expects the same status on both streams. +func (s) TestHealthWatchMultipleClients(t *testing.T) { + for _, e := range listTestEnv() { + testHealthWatchMultipleClients(t, e) + } +} + +func testHealthWatchMultipleClients(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + stream1, cf1 := newHealthCheckStream(t, cc, defaultHealthService) + defer cf1() + healthWatchChecker(t, stream1, healthpb.HealthCheckResponse_SERVICE_UNKNOWN) + + stream2, cf2 := newHealthCheckStream(t, cc, defaultHealthService) + defer cf2() + healthWatchChecker(t, stream2, healthpb.HealthCheckResponse_SERVICE_UNKNOWN) + + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_NOT_SERVING) + healthWatchChecker(t, stream1, healthpb.HealthCheckResponse_NOT_SERVING) + healthWatchChecker(t, stream2, healthpb.HealthCheckResponse_NOT_SERVING) +} + +// TestHealthWatchSameStatus makes a streaming Watch() RPC on the health server +// and makes sure that the health status of the server is as expected after +// multiple calls to SetServingStatus with the same status. +func (s) TestHealthWatchSameStatus(t *testing.T) { + for _, e := range listTestEnv() { + testHealthWatchSameStatus(t, e) + } +} + +func testHealthWatchSameStatus(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService) + defer cf() + + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVICE_UNKNOWN) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_NOT_SERVING) + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_NOT_SERVING) +} + +// TestHealthWatchServiceStatusSetBeforeStartingServer starts a health server +// on which the health status for the defaultService is set before the gRPC +// server is started, and expects the correct health status to be returned. +func (s) TestHealthWatchServiceStatusSetBeforeStartingServer(t *testing.T) { + for _, e := range listTestEnv() { + testHealthWatchSetServiceStatusBeforeStartingServer(t, e) + } +} + +func testHealthWatchSetServiceStatusBeforeStartingServer(t *testing.T, e env) { + hs := health.NewServer() + te := newTest(t, e) + te.healthServer = hs + hs.SetServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService) + defer cf() + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) +} + +// TestHealthWatchDefaultStatusChange verifies the simple case where the +// service starts off with a SERVICE_UNKNOWN status (because SetServingStatus +// hasn't been called yet) and then moves to SERVING after SetServingStatus is +// called. +func (s) TestHealthWatchDefaultStatusChange(t *testing.T) { + for _, e := range listTestEnv() { + testHealthWatchDefaultStatusChange(t, e) + } +} + +func testHealthWatchDefaultStatusChange(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService) + defer cf() + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVICE_UNKNOWN) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) +} + +// TestHealthWatchSetServiceStatusBeforeClientCallsWatch verifies the case +// where the health status is set to SERVING before the client calls Watch(). +func (s) TestHealthWatchSetServiceStatusBeforeClientCallsWatch(t *testing.T) { + for _, e := range listTestEnv() { + testHealthWatchSetServiceStatusBeforeClientCallsWatch(t, e) + } +} + +func testHealthWatchSetServiceStatusBeforeClientCallsWatch(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + defer te.tearDown() + + stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService) + defer cf() + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) +} + +// TestHealthWatchOverallServerHealthChange verifies setting the overall status +// of the server by using the empty service name. +func (s) TestHealthWatchOverallServerHealthChange(t *testing.T) { + for _, e := range listTestEnv() { + testHealthWatchOverallServerHealthChange(t, e) + } +} + +func testHealthWatchOverallServerHealthChange(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + stream, cf := newHealthCheckStream(t, te.clientConn(), "") + defer cf() + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) + te.setHealthServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING) + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_NOT_SERVING) +} + +// TestUnknownHandler verifies that an expected error is returned (by setting +// the unknownHandler on the server) for a service which is not exposed to the +// client. +func (s) TestUnknownHandler(t *testing.T) { + // An example unknownHandler that returns a different code and a different + // method, making sure that we do not expose what methods are implemented to + // a client that is not authenticated. + unknownHandler := func(any, grpc.ServerStream) error { + return status.Error(codes.Unauthenticated, "user unauthenticated") + } + for _, e := range listTestEnv() { + // TODO(bradfitz): Temporarily skip this env due to #619. + if e.name == "handler-tls" { + continue + } + testUnknownHandler(t, e, unknownHandler) + } +} + +func testUnknownHandler(t *testing.T, e env, unknownHandler grpc.StreamHandler) { + te := newTest(t, e) + te.unknownHandler = unknownHandler + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + verifyHealthCheckErrCode(t, 1*time.Second, te.clientConn(), "", codes.Unauthenticated) +} + +// TestHealthCheckServingStatus makes a streaming Watch() RPC on the health +// server and verifies a bunch of health status transitions. +func (s) TestHealthCheckServingStatus(t *testing.T) { + for _, e := range listTestEnv() { + testHealthCheckServingStatus(t, e) + } +} + +func testHealthCheckServingStatus(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + verifyHealthCheckStatus(t, 1*time.Second, cc, "", healthpb.HealthCheckResponse_SERVING) + verifyHealthCheckErrCode(t, 1*time.Second, cc, defaultHealthService, codes.NotFound) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + verifyHealthCheckStatus(t, 1*time.Second, cc, defaultHealthService, healthpb.HealthCheckResponse_SERVING) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_NOT_SERVING) + verifyHealthCheckStatus(t, 1*time.Second, cc, defaultHealthService, healthpb.HealthCheckResponse_NOT_SERVING) +} + +// Test verifies that registering a nil health listener closes the health +// client. +func (s) TestHealthCheckUnregisterHealthListener(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + hcEnterChan, hcExitChan := setupHealthCheckWrapper(t) + scChan := make(chan balancer.SubConn, 1) + readyUpdateReceivedCh := make(chan struct{}) + bf := stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + cc := bd.ClientConn + ccw := &subConnStoringCCWrapper{ + ClientConn: cc, + scChan: scChan, + stateListener: func(scs balancer.SubConnState) { + if scs.ConnectivityState != connectivity.Ready { + return + } + close(readyUpdateReceivedCh) + }, + } + bd.ChildBalancer = balancer.Get(pickfirst.Name).Build(ccw, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + } + + stub.Register(t.Name(), bf) + _, lis, ts := setupServer(t, nil) + ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) + + _, r := setupClient(t, nil) + svcCfg := fmt.Sprintf(`{ + "healthCheckConfig": { + "serviceName": "foo" + }, + "loadBalancingConfig": [{"%s":{}}] + }`, t.Name()) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, + ServiceConfig: parseServiceConfig(t, r, svcCfg)}) + + var sc balancer.SubConn + select { + case sc = <-scChan: + case <-ctx.Done(): + t.Fatal("Context timed out waiting for SubConn creation") + } + + // Wait for the SubConn to enter READY. + select { + case <-readyUpdateReceivedCh: + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for SubConn to enter READY") + } + + // Health check should start only after a health listener is registered. + select { + case <-hcEnterChan: + t.Fatalf("Health service client created prematurely.") + case <-time.After(defaultTestShortTimeout): + } + + // Register a health listener and verify it receives updates. + healthChan := make(chan balancer.SubConnState, 1) + sc.RegisterHealthListener(func(scs balancer.SubConnState) { + healthChan <- scs + }) + + select { + case <-hcEnterChan: + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for health check to begin.") + } + + for readyReceived := false; !readyReceived; { + select { + case scs := <-healthChan: + t.Logf("Received health update: %v", scs) + readyReceived = scs.ConnectivityState == connectivity.Ready + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for healthy backend.") + } + } + + // Registering a nil listener should invalidate the previously registered + // listener and close the health service client. + sc.RegisterHealthListener(nil) + select { + case <-hcExitChan: + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for the health client to close.") + } + + ts.SetServingStatus("foo", healthpb.HealthCheckResponse_NOT_SERVING) + + // No updates should be received on the listener. + select { + case scs := <-healthChan: + t.Fatalf("Received unexpected health update on the listener: %v", scs) + case <-time.After(defaultTestShortTimeout): + } +} diff --git a/test/http_header_end2end_test.go b/test/http_header_end2end_test.go new file mode 100644 index 0000000..0f03f80 --- /dev/null +++ b/test/http_header_end2end_test.go @@ -0,0 +1,259 @@ +/* +* +* Copyright 2022 gRPC authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ +package test + +import ( + "context" + "fmt" + "net" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestHTTPHeaderFrameErrorHandlingHTTPMode(t *testing.T) { + type test struct { + name string + header []string + errCode codes.Code + } + + var tests []test + + // Non-gRPC content-type fallback path. + for httpCode := range transport.HTTPStatusConvTab { + tests = append(tests, test{ + name: fmt.Sprintf("Non-gRPC content-type fallback path with httpCode: %v", httpCode), + header: []string{ + ":status", fmt.Sprintf("%d", httpCode), + "content-type", "text/html", // non-gRPC content type to switch to HTTP mode. + "grpc-status", "1", // Make up a gRPC status error + "grpc-status-details-bin", "???", // Make up a gRPC field parsing error + }, + errCode: transport.HTTPStatusConvTab[int(httpCode)], + }) + } + + // Missing content-type fallback path. + for httpCode := range transport.HTTPStatusConvTab { + tests = append(tests, test{ + name: fmt.Sprintf("Missing content-type fallback path with httpCode: %v", httpCode), + header: []string{ + ":status", fmt.Sprintf("%d", httpCode), + // Omitting content type to switch to HTTP mode. + "grpc-status", "1", // Make up a gRPC status error + "grpc-status-details-bin", "???", // Make up a gRPC field parsing error + }, + errCode: transport.HTTPStatusConvTab[int(httpCode)], + }) + } + + // Malformed HTTP status when fallback. + tests = append(tests, test{ + name: "Malformed HTTP status when fallback", + header: []string{ + ":status", "abc", + // Omitting content type to switch to HTTP mode. + "grpc-status", "1", // Make up a gRPC status error + "grpc-status-details-bin", "???", // Make up a gRPC field parsing error + }, + errCode: codes.Internal, + }) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + serverAddr, cleanup, err := startServer(t, test.header) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if err := doHTTPHeaderTest(serverAddr, test.errCode); err != nil { + t.Error(err) + } + }) + } +} + +// Testing erroneous ResponseHeader or Trailers-only (delivered in the first HEADERS frame). +func (s) TestHTTPHeaderFrameErrorHandlingInitialHeader(t *testing.T) { + for _, test := range []struct { + name string + header []string + errCode codes.Code + }{ + { + name: "missing gRPC content type", + header: []string{ + ":status", "403", + }, + errCode: codes.PermissionDenied, + }, + { + name: "malformed grpc-status", + header: []string{ + ":status", "502", + "content-type", "application/grpc", + "grpc-status", "abc", + }, + errCode: codes.Unknown, + }, + { + name: "Malformed grpc-tags-bin field ignores http status", + header: []string{ + ":status", "502", + "content-type", "application/grpc", + "grpc-status", "0", + "grpc-tags-bin", "???", + }, + errCode: codes.Internal, + }, + { + name: "gRPC status error ignoring http status", + header: []string{ + ":status", "502", + "content-type", "application/grpc", + "grpc-status", "3", + }, + errCode: codes.InvalidArgument, + }, + } { + t.Run(test.name, func(t *testing.T) { + serverAddr, cleanup, err := startServer(t, test.header) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if err := doHTTPHeaderTest(serverAddr, test.errCode); err != nil { + t.Error(err) + } + }) + } +} + +// Testing non-Trailers-only Trailers (delivered in second HEADERS frame) +func (s) TestHTTPHeaderFrameErrorHandlingNormalTrailer(t *testing.T) { + tests := []struct { + name string + responseHeader []string + trailer []string + errCode codes.Code + }{ + { + name: "trailer missing grpc-status to ignore http status", + responseHeader: []string{ + ":status", "200", + "content-type", "application/grpc", + }, + trailer: []string{ + // trailer missing grpc-status + ":status", "502", + }, + errCode: codes.Unknown, + }, + { + name: "malformed grpc-status-details-bin field with status 404 to be ignored due to content type", + responseHeader: []string{ + ":status", "404", + "content-type", "application/grpc", + }, + trailer: []string{ + // malformed grpc-status-details-bin field + "grpc-status", "0", + "grpc-status-details-bin", "????", + }, + errCode: codes.Internal, + }, + { + name: "malformed grpc-status-details-bin field with status 404 and no content type", + responseHeader: []string{ + ":status", "404", + }, + trailer: []string{ + // malformed grpc-status-details-bin field + "grpc-status", "0", + "grpc-status-details-bin", "????", + }, + errCode: codes.Unimplemented, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + serverAddr, cleanup, err := startServer(t, test.responseHeader, test.trailer) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if err := doHTTPHeaderTest(serverAddr, test.errCode); err != nil { + t.Error(err) + } + }) + + } +} + +func (s) TestHTTPHeaderFrameErrorHandlingMoreThanTwoHeaders(t *testing.T) { + header := []string{ + ":status", "200", + "content-type", "application/grpc", + } + serverAddr, cleanup, err := startServer(t, header, header, header) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if err := doHTTPHeaderTest(serverAddr, codes.Internal); err != nil { + t.Fatal(err) + } +} + +func startServer(t *testing.T, headerFields ...[]string) (serverAddr string, cleanup func(), err error) { + t.Helper() + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + return "", nil, fmt.Errorf("listening on %q: %v", "localhost:0", err) + } + server := &httpServer{responses: []httpServerResponse{{trailers: headerFields}}} + server.start(t, lis) + return lis.Addr().String(), func() { lis.Close() }, nil +} + +func doHTTPHeaderTest(lisAddr string, errCode codes.Code) error { + cc, err := grpc.NewClient(lisAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return fmt.Errorf("NewClient(%q): %v", lisAddr, err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + stream, err := client.FullDuplexCall(ctx) + if err != nil { + return fmt.Errorf("creating FullDuplex stream: %v", err) + } + if _, err := stream.Recv(); err == nil || status.Code(err) != errCode { + return fmt.Errorf("stream.Recv() = %v, want error code: %v", err, errCode) + } + return nil +} diff --git a/test/insecure_creds_test.go b/test/insecure_creds_test.go new file mode 100644 index 0000000..5296433 --- /dev/null +++ b/test/insecure_creds_test.go @@ -0,0 +1,198 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "strings" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// testLegacyPerRPCCredentials is a PerRPCCredentials that has yet incorporated security level. +type testLegacyPerRPCCredentials struct{} + +func (cr testLegacyPerRPCCredentials) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { + return nil, nil +} + +func (cr testLegacyPerRPCCredentials) RequireTransportSecurity() bool { + return true +} + +func getSecurityLevel(ai credentials.AuthInfo) credentials.SecurityLevel { + if c, ok := ai.(interface { + GetCommonAuthInfo() credentials.CommonAuthInfo + }); ok { + return c.GetCommonAuthInfo().SecurityLevel + } + return credentials.InvalidSecurityLevel +} + +// TestInsecureCreds tests the use of insecure creds on the server and client +// side, and verifies that expect security level and auth info are returned. +// Also verifies that this credential can interop with existing `WithInsecure` +// DialOption. +func (s) TestInsecureCreds(t *testing.T) { + tests := []struct { + desc string + clientInsecureCreds bool + serverInsecureCreds bool + }{ + { + desc: "client and server insecure creds", + clientInsecureCreds: true, + serverInsecureCreds: true, + }, + { + desc: "client only insecure creds", + clientInsecureCreds: true, + }, + { + desc: "server only insecure creds", + serverInsecureCreds: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err) + } + + ss := &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + if !test.serverInsecureCreds { + return &testpb.Empty{}, nil + } + + pr, ok := peer.FromContext(ctx) + if !ok { + return nil, status.Error(codes.DataLoss, "Failed to get peer from ctx") + } + // Check security level. + secLevel := getSecurityLevel(pr.AuthInfo) + if secLevel == credentials.InvalidSecurityLevel { + return nil, status.Errorf(codes.Unauthenticated, "peer.AuthInfo does not implement GetCommonAuthInfo()") + } + if secLevel != credentials.NoSecurity { + return nil, status.Errorf(codes.Unauthenticated, "Wrong security level: got %q, want %q", secLevel, credentials.NoSecurity) + } + return &testpb.Empty{}, nil + }, + } + sOpts := []grpc.ServerOption{} + if test.serverInsecureCreds { + ss.S = grpc.NewServer(grpc.Creds(insecure.NewCredentials())) + } else { + ss.S = grpc.NewServer(sOpts...) + } + stubserver.StartTestService(t, ss) + defer ss.S.Stop() + addr := lis.Addr().String() + opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} + cc, err := grpc.NewClient(addr, opts...) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", addr, err) + } + defer cc.Close() + + c := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = c.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall(_, _) = _, %v; want _, ", err) + } + }) + } +} + +func (s) TestInsecureCreds_WithPerRPCCredentials_AsCallOption(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err) + } + + ss := &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + S: grpc.NewServer(grpc.Creds(insecure.NewCredentials())), + } + stubserver.StartTestService(t, ss) + defer ss.S.Stop() + + addr := lis.Addr().String() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + dopts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} + copts := []grpc.CallOption{grpc.PerRPCCredentials(testLegacyPerRPCCredentials{})} + cc, err := grpc.NewClient(addr, dopts...) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", addr, err) + } + defer cc.Close() + + const wantErr = "transport: cannot send secure credentials on an insecure connection" + c := testgrpc.NewTestServiceClient(cc) + if _, err = c.EmptyCall(ctx, &testpb.Empty{}, copts...); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("insecure credentials with per-RPC credentials requiring transport security returned error: %v; want %s", err, wantErr) + } +} + +func (s) TestInsecureCreds_WithPerRPCCredentials_AsDialOption(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err) + } + ss := &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + S: grpc.NewServer(grpc.Creds(insecure.NewCredentials())), + } + stubserver.StartTestService(t, ss) + defer ss.S.Stop() + + addr := lis.Addr().String() + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithPerRPCCredentials(testLegacyPerRPCCredentials{}), + } + const wantErr = "the credentials require transport level security" + if _, err := grpc.NewClient(addr, dopts...); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("grpc.NewClient(%q) returned err %v, want: %v", addr, err, wantErr) + } +} diff --git a/test/interceptor_test.go b/test/interceptor_test.go new file mode 100644 index 0000000..f6db1d2 --- /dev/null +++ b/test/interceptor_test.go @@ -0,0 +1,281 @@ +/* + * + * Copyright 2022 gRPC authors. + + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type parentCtxkey struct{} +type firstInterceptorCtxkey struct{} +type secondInterceptorCtxkey struct{} +type baseInterceptorCtxKey struct{} + +const ( + parentCtxVal = "parent" + firstInterceptorCtxVal = "firstInterceptor" + secondInterceptorCtxVal = "secondInterceptor" + baseInterceptorCtxVal = "baseInterceptor" +) + +// TestUnaryClientInterceptor_ContextValuePropagation verifies that a unary +// interceptor receives context values specified in the context passed to the +// RPC call. +func (s) TestUnaryClientInterceptor_ContextValuePropagation(t *testing.T) { + errCh := testutils.NewChannel() + unaryInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.Send(fmt.Errorf("unaryInt got %q in context.Val, want %q", got, parentCtxVal)) + } + errCh.Send(nil) + return invoker(ctx, method, req, reply, cc, opts...) + } + + // Start a stub server and use the above unary interceptor while creating a + // ClientConn to it. + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := ss.Start(nil, grpc.WithUnaryInterceptor(unaryInt)); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(context.WithValue(ctx, parentCtxkey{}, parentCtxVal), &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall() failed: %v", err) + } + val, err := errCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for unary interceptor to be invoked: %v", err) + } + if val != nil { + t.Fatalf("unary interceptor failed: %v", val) + } +} + +// TestChainUnaryClientInterceptor_ContextValuePropagation verifies that a chain +// of unary interceptors receive context values specified in the original call +// as well as the ones specified by prior interceptors in the chain. +func (s) TestChainUnaryClientInterceptor_ContextValuePropagation(t *testing.T) { + errCh := testutils.NewChannel() + firstInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("first interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if ctx.Value(firstInterceptorCtxkey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("first interceptor should not have %T in context", firstInterceptorCtxkey{})) + } + if ctx.Value(secondInterceptorCtxkey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("first interceptor should not have %T in context", secondInterceptorCtxkey{})) + } + firstCtx := context.WithValue(ctx, firstInterceptorCtxkey{}, firstInterceptorCtxVal) + return invoker(firstCtx, method, req, reply, cc, opts...) + } + + secondInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("second interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if got, ok := ctx.Value(firstInterceptorCtxkey{}).(string); !ok || got != firstInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("second interceptor got %q in context.Val, want %q", got, firstInterceptorCtxVal)) + } + if ctx.Value(secondInterceptorCtxkey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("second interceptor should not have %T in context", secondInterceptorCtxkey{})) + } + secondCtx := context.WithValue(ctx, secondInterceptorCtxkey{}, secondInterceptorCtxVal) + return invoker(secondCtx, method, req, reply, cc, opts...) + } + + lastInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if got, ok := ctx.Value(firstInterceptorCtxkey{}).(string); !ok || got != firstInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, firstInterceptorCtxVal)) + } + if got, ok := ctx.Value(secondInterceptorCtxkey{}).(string); !ok || got != secondInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, secondInterceptorCtxVal)) + } + errCh.SendContext(ctx, nil) + return invoker(ctx, method, req, reply, cc, opts...) + } + + // Start a stub server and use the above chain of interceptors while creating + // a ClientConn to it. + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := ss.Start(nil, grpc.WithChainUnaryInterceptor(firstInt, secondInt, lastInt)); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(context.WithValue(ctx, parentCtxkey{}, parentCtxVal), &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall() failed: %v", err) + } + val, err := errCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for unary interceptor to be invoked: %v", err) + } + if val != nil { + t.Fatalf("unary interceptor failed: %v", val) + } +} + +// TestChainOnBaseUnaryClientInterceptor_ContextValuePropagation verifies that +// unary interceptors specified as a base interceptor or as a chain interceptor +// receive context values specified in the original call as well as the ones +// specified by interceptors in the chain. +func (s) TestChainOnBaseUnaryClientInterceptor_ContextValuePropagation(t *testing.T) { + errCh := testutils.NewChannel() + baseInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("base interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if ctx.Value(baseInterceptorCtxKey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("baseinterceptor should not have %T in context", baseInterceptorCtxKey{})) + } + baseCtx := context.WithValue(ctx, baseInterceptorCtxKey{}, baseInterceptorCtxVal) + return invoker(baseCtx, method, req, reply, cc, opts...) + } + + chainInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("chain interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if got, ok := ctx.Value(baseInterceptorCtxKey{}).(string); !ok || got != baseInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("chain interceptor got %q in context.Val, want %q", got, baseInterceptorCtxVal)) + } + errCh.SendContext(ctx, nil) + return invoker(ctx, method, req, reply, cc, opts...) + } + + // Start a stub server and use the above chain of interceptors while creating + // a ClientConn to it. + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := ss.Start(nil, grpc.WithUnaryInterceptor(baseInt), grpc.WithChainUnaryInterceptor(chainInt)); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(context.WithValue(ctx, parentCtxkey{}, parentCtxVal), &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall() failed: %v", err) + } + val, err := errCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for unary interceptor to be invoked: %v", err) + } + if val != nil { + t.Fatalf("unary interceptor failed: %v", val) + } +} + +// TestChainStreamClientInterceptor_ContextValuePropagation verifies that a +// chain of stream interceptors receive context values specified in the original +// call as well as the ones specified by the prior interceptors in the chain. +func (s) TestChainStreamClientInterceptor_ContextValuePropagation(t *testing.T) { + errCh := testutils.NewChannel() + firstInt := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("first interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if ctx.Value(firstInterceptorCtxkey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("first interceptor should not have %T in context", firstInterceptorCtxkey{})) + } + if ctx.Value(secondInterceptorCtxkey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("first interceptor should not have %T in context", secondInterceptorCtxkey{})) + } + firstCtx := context.WithValue(ctx, firstInterceptorCtxkey{}, firstInterceptorCtxVal) + return streamer(firstCtx, desc, cc, method, opts...) + } + + secondInt := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("second interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if got, ok := ctx.Value(firstInterceptorCtxkey{}).(string); !ok || got != firstInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("second interceptor got %q in context.Val, want %q", got, firstInterceptorCtxVal)) + } + if ctx.Value(secondInterceptorCtxkey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("second interceptor should not have %T in context", secondInterceptorCtxkey{})) + } + secondCtx := context.WithValue(ctx, secondInterceptorCtxkey{}, secondInterceptorCtxVal) + return streamer(secondCtx, desc, cc, method, opts...) + } + + lastInt := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if got, ok := ctx.Value(firstInterceptorCtxkey{}).(string); !ok || got != firstInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, firstInterceptorCtxVal)) + } + if got, ok := ctx.Value(secondInterceptorCtxkey{}).(string); !ok || got != secondInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, secondInterceptorCtxVal)) + } + errCh.SendContext(ctx, nil) + return streamer(ctx, desc, cc, method, opts...) + } + + // Start a stub server and use the above chain of interceptors while creating + // a ClientConn to it. + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + return stream.Send(&testpb.StreamingOutputCallResponse{}) + }, + } + if err := ss.Start(nil, grpc.WithChainStreamInterceptor(firstInt, secondInt, lastInt)); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.FullDuplexCall(context.WithValue(ctx, parentCtxkey{}, parentCtxVal)); err != nil { + t.Fatalf("ss.Client.FullDuplexCall() failed: %v", err) + } + val, err := errCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for stream interceptor to be invoked: %v", err) + } + if val != nil { + t.Fatalf("stream interceptor failed: %v", val) + } +} diff --git a/test/invoke_test.go b/test/invoke_test.go new file mode 100644 index 0000000..4ee6525 --- /dev/null +++ b/test/invoke_test.go @@ -0,0 +1,152 @@ +/* + * + * Copyright 2022 gRPC authors. + + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "strings" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/stubserver" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/status" +) + +// TestInvoke verifies a straightforward invocation of ClientConn.Invoke(). +func (s) TestInvoke(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{}); err != nil { + t.Fatalf("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") failed: %v", err) + } +} + +// TestInvokeLargeErr verifies an invocation of ClientConn.Invoke() where the +// server returns a really large error message. +func (s) TestInvokeLargeErr(t *testing.T) { + largeErrorStr := strings.Repeat("A", 1024*1024) + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, status.Error(codes.Internal, largeErrorStr) + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + err := ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{}) + if err == nil { + t.Fatal("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") succeeded when expected to fail") + } + st, ok := status.FromError(err) + if !ok { + t.Fatal("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") received non-status error") + } + if status.Code(err) != codes.Internal || st.Message() != largeErrorStr { + t.Fatalf("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") failed with error: %v, want an error of code %d and desc size %d", err, codes.Internal, len(largeErrorStr)) + } +} + +// TestInvokeErrorSpecialChars tests an invocation of ClientConn.Invoke() and +// verifies that error messages don't get mangled. +func (s) TestInvokeErrorSpecialChars(t *testing.T) { + const weirdError = "format verbs: %v%s" + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, status.Error(codes.Internal, weirdError) + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + err := ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{}) + if err == nil { + t.Fatal("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") succeeded when expected to fail") + } + st, ok := status.FromError(err) + if !ok { + t.Fatal("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") received non-status error") + } + if status.Code(err) != codes.Internal || st.Message() != weirdError { + t.Fatalf("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") failed with error: %v, want %v", err, weirdError) + } +} + +// TestInvokeCancel tests an invocation of ClientConn.Invoke() with a cancelled +// context and verifies that the request is not actually sent to the server. +func (s) TestInvokeCancel(t *testing.T) { + cancelled := 0 + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + cancelled++ + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + for i := 0; i < 100; i++ { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + cancel() + ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{}) + } + if cancelled != 0 { + t.Fatalf("server received %d of 100 cancelled requests", cancelled) + } +} + +// TestInvokeCancelClosedNonFail tests an invocation of ClientConn.Invoke() with +// a cancelled non-failfast RPC on a closed ClientConn and verifies that the +// call terminates with an error. +func (s) TestInvokeCancelClosedNonFailFast(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ss.CC.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + cancel() + if err := ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{}, grpc.WaitForReady(true)); err == nil { + t.Fatal("ClientConn.Invoke() on closed connection succeeded when expected to fail") + } +} diff --git a/test/kokoro/README.md b/test/kokoro/README.md new file mode 100644 index 0000000..1a7bd0d --- /dev/null +++ b/test/kokoro/README.md @@ -0,0 +1 @@ +The scripts in this directory are intended to be run by Kokoro CI jobs. diff --git a/test/kokoro/psm-csm.cfg b/test/kokoro/psm-csm.cfg new file mode 100644 index 0000000..25174cd --- /dev/null +++ b/test/kokoro/psm-csm.cfg @@ -0,0 +1,17 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "artifacts/**/*sponge_log.xml" + regex: "artifacts/**/*.log" + strip_prefix: "artifacts" + } +} +env_vars { + key: "PSM_TEST_SUITE" + value: "csm" +} diff --git a/test/kokoro/psm-dualstack.cfg b/test/kokoro/psm-dualstack.cfg new file mode 100644 index 0000000..8c2ba9d --- /dev/null +++ b/test/kokoro/psm-dualstack.cfg @@ -0,0 +1,17 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh" +timeout_mins: 360 + +action { + define_artifacts { + regex: "artifacts/**/*sponge_log.xml" + regex: "artifacts/**/*.log" + strip_prefix: "artifacts" + } +} +env_vars { + key: "PSM_TEST_SUITE" + value: "dualstack" +} diff --git a/test/kokoro/psm-interop-build-go.sh b/test/kokoro/psm-interop-build-go.sh new file mode 100755 index 0000000..a2bba3d --- /dev/null +++ b/test/kokoro/psm-interop-build-go.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -eo pipefail + +####################################### +# Builds test app Docker images and pushes them to GCR. +# Called from psm_interop_kokoro_lib.sh. +# +# Globals: +# SRC_DIR: Absolute path to the source repo on Kokoro VM +# SERVER_IMAGE_NAME: Test server Docker image name +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# DOCKER_REGISTRY: Docker registry to push to +# Outputs: +# Writes the output of docker image build stdout, stderr +####################################### +psm::lang::build_docker_images() { + local client_dockerfile="interop/xds/client/Dockerfile" + local server_dockerfile="interop/xds/server/Dockerfile" + psm::build::docker_images_generic "${client_dockerfile}" "${server_dockerfile}" +} diff --git a/test/kokoro/psm-interop-test-go.sh b/test/kokoro/psm-interop-test-go.sh new file mode 100755 index 0000000..054dbfe --- /dev/null +++ b/test/kokoro/psm-interop-test-go.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -eo pipefail + +# Input parameters to psm:: methods of the install script. +readonly GRPC_LANGUAGE="go" +readonly BUILD_SCRIPT_DIR="$(dirname "$0")" + +# Used locally. +readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/psm-interop/${TEST_DRIVER_BRANCH:-main}/.kokoro/psm_interop_kokoro_lib.sh" + +psm::lang::source_install_lib() { + echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}" + local install_lib + # Download to a tmp file. + install_lib="$(mktemp -d)/psm_interop_kokoro_lib.sh" + curl -s --retry-connrefused --retry 5 -o "${install_lib}" "${TEST_DRIVER_INSTALL_SCRIPT_URL}" + # Checksum. + if command -v sha256sum &> /dev/null; then + echo "Install script checksum:" + sha256sum "${install_lib}" + fi + source "${install_lib}" +} + +psm::lang::source_install_lib +source "${BUILD_SCRIPT_DIR}/psm-interop-build-${GRPC_LANGUAGE}.sh" +psm::run "${PSM_TEST_SUITE}" diff --git a/test/kokoro/psm-light.cfg b/test/kokoro/psm-light.cfg new file mode 100644 index 0000000..ec8682e --- /dev/null +++ b/test/kokoro/psm-light.cfg @@ -0,0 +1,17 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh" +timeout_mins: 30 + +action { + define_artifacts { + regex: "artifacts/**/*sponge_log.xml" + regex: "artifacts/**/*.log" + strip_prefix: "artifacts" + } +} +env_vars { + key: "PSM_TEST_SUITE" + value: "light" +} diff --git a/test/kokoro/psm-regional_td.cfg b/test/kokoro/psm-regional_td.cfg new file mode 100644 index 0000000..7d171ed --- /dev/null +++ b/test/kokoro/psm-regional_td.cfg @@ -0,0 +1,17 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh" +timeout_mins: 30 + +action { + define_artifacts { + regex: "artifacts/**/*sponge_log.xml" + regex: "artifacts/**/*.log" + strip_prefix: "artifacts" + } +} +env_vars { + key: "PSM_TEST_SUITE" + value: "regional_td" +} diff --git a/test/kokoro/psm-security.cfg b/test/kokoro/psm-security.cfg new file mode 100644 index 0000000..ff1c219 --- /dev/null +++ b/test/kokoro/psm-security.cfg @@ -0,0 +1,17 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh" +timeout_mins: 240 + +action { + define_artifacts { + regex: "artifacts/**/*sponge_log.xml" + regex: "artifacts/**/*.log" + strip_prefix: "artifacts" + } +} +env_vars { + key: "PSM_TEST_SUITE" + value: "security" +} diff --git a/test/kokoro/psm-spiffe.cfg b/test/kokoro/psm-spiffe.cfg new file mode 100644 index 0000000..94fbaaa --- /dev/null +++ b/test/kokoro/psm-spiffe.cfg @@ -0,0 +1,17 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh" +timeout_mins: 240 + +action { + define_artifacts { + regex: "artifacts/**/*sponge_log.xml" + regex: "artifacts/**/*.log" + strip_prefix: "artifacts" + } +} +env_vars { + key: "PSM_TEST_SUITE" + value: "spiffe" +} diff --git a/test/kokoro/xds.cfg b/test/kokoro/xds.cfg new file mode 100644 index 0000000..a1e4ed0 --- /dev/null +++ b/test/kokoro/xds.cfg @@ -0,0 +1,11 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/xds.sh" +timeout_mins: 360 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} diff --git a/test/kokoro/xds.sh b/test/kokoro/xds.sh new file mode 100755 index 0000000..886146c --- /dev/null +++ b/test/kokoro/xds.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -exu -o pipefail +[[ -f /VERSION ]] && cat /VERSION + +cd github + +export GOPATH="${HOME}/gopath" +pushd grpc-go/interop/xds/client +# Install a version of Go supported by gRPC for the new features, e.g. +# errors.Is() +gofilename=go1.25.0.linux-amd64.tar.gz +curl --retry 3 -O -L "https://go.dev/dl/${gofilename}" +sudo tar -C /usr/local -xf "${gofilename}" +sudo ln -s /usr/local/go/bin/go /usr/bin/go +# Retry go build on errors (e.g. go get connection errors), for at most 3 times +for i in 1 2 3; do go build && break || sleep 5; done +popd + +git clone -b master --single-branch --depth=1 https://github.com/grpc/grpc.git + +grpc/tools/run_tests/helper_scripts/prep_xds.sh + +# Test cases "path_matching" and "header_matching" are not included in "all", +# because not all interop clients in all languages support these new tests. +# +# TODO: remove "path_matching" and "header_matching" from --test_case after +# they are added into "all". +GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \ + python3 grpc/tools/run_tests/run_xds_tests.py \ + --test_case="ping_pong,circuit_breaking" \ + --project_id=grpc-testing \ + --project_num=830293263384 \ + --source_image=projects/grpc-testing/global/images/xds-test-server-5 \ + --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \ + --gcp_suffix=$(date '+%s') \ + --verbose \ + ${XDS_V3_OPT-} \ + --client_cmd="grpc-go/interop/xds/client/client \ + --server=xds:///{server_uri} \ + --stats_port={stats_port} \ + --qps={qps} \ + {fail_on_failed_rpc} \ + {rpcs_to_send} \ + {metadata_to_send}" diff --git a/test/kokoro/xds_k8s_lb.cfg b/test/kokoro/xds_k8s_lb.cfg new file mode 100644 index 0000000..4928a7b --- /dev/null +++ b/test/kokoro/xds_k8s_lb.cfg @@ -0,0 +1,17 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh" +timeout_mins: 300 + +action { + define_artifacts { + regex: "artifacts/**/*sponge_log.xml" + regex: "artifacts/**/*.log" + strip_prefix: "artifacts" + } +} +env_vars { + key: "PSM_TEST_SUITE" + value: "lb" +} diff --git a/test/kokoro/xds_url_map.cfg b/test/kokoro/xds_url_map.cfg new file mode 100644 index 0000000..08c9463 --- /dev/null +++ b/test/kokoro/xds_url_map.cfg @@ -0,0 +1,17 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh" +timeout_mins: 60 + +action { + define_artifacts { + regex: "artifacts/**/*sponge_log.xml" + regex: "artifacts/**/*.log" + strip_prefix: "artifacts" + } +} +env_vars { + key: "PSM_TEST_SUITE" + value: "url_map" +} diff --git a/test/kokoro/xds_v3.cfg b/test/kokoro/xds_v3.cfg new file mode 100644 index 0000000..1991efd --- /dev/null +++ b/test/kokoro/xds_v3.cfg @@ -0,0 +1,11 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/xds_v3.sh" +timeout_mins: 360 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} diff --git a/test/kokoro/xds_v3.sh b/test/kokoro/xds_v3.sh new file mode 100755 index 0000000..73eb50d --- /dev/null +++ b/test/kokoro/xds_v3.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +XDS_V3_OPT="--xds_v3_support" `dirname $0`/xds.sh diff --git a/test/local_creds_test.go b/test/local_creds_test.go new file mode 100644 index 0000000..13dd4ae --- /dev/null +++ b/test/local_creds_test.go @@ -0,0 +1,218 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "net" + "net/netip" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/local" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func testLocalCredsE2ESucceed(t *testing.T, network, address string) error { + lis, err := net.Listen(network, address) + if err != nil { + return fmt.Errorf("Failed to create listener: %v", err) + } + ss := &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + pr, ok := peer.FromContext(ctx) + if !ok { + return nil, status.Error(codes.DataLoss, "Failed to get peer from ctx") + } + type internalInfo interface { + GetCommonAuthInfo() credentials.CommonAuthInfo + } + var secLevel credentials.SecurityLevel + if info, ok := (pr.AuthInfo).(internalInfo); ok { + secLevel = info.GetCommonAuthInfo().SecurityLevel + } else { + return nil, status.Errorf(codes.Unauthenticated, "peer.AuthInfo does not implement GetCommonAuthInfo()") + } + // Check security level + switch network { + case "unix": + if secLevel != credentials.PrivacyAndIntegrity { + return nil, status.Errorf(codes.Unauthenticated, "Wrong security level: got %q, want %q", secLevel, credentials.PrivacyAndIntegrity) + } + case "tcp": + if secLevel != credentials.NoSecurity { + return nil, status.Errorf(codes.Unauthenticated, "Wrong security level: got %q, want %q", secLevel, credentials.NoSecurity) + } + } + return &testpb.Empty{}, nil + }, + S: grpc.NewServer(grpc.Creds(local.NewCredentials())), + } + stubserver.StartTestService(t, ss) + defer ss.S.Stop() + + var cc *grpc.ClientConn + lisAddr := lis.Addr().String() + + switch network { + case "unix": + cc, err = grpc.NewClient("passthrough:///"+lisAddr, grpc.WithTransportCredentials(local.NewCredentials()), grpc.WithContextDialer( + func(_ context.Context, addr string) (net.Conn, error) { + return net.Dial("unix", addr) + })) + case "tcp": + cc, err = grpc.NewClient(lisAddr, grpc.WithTransportCredentials(local.NewCredentials())) + default: + return fmt.Errorf("unsupported network %q", network) + } + if err != nil { + return fmt.Errorf("Failed to create a client for server: %v, %v", err, lisAddr) + } + defer cc.Close() + + c := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if _, err = c.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return fmt.Errorf("EmptyCall(_, _) = _, %v; want _, ", err) + } + return nil +} + +func (s) TestLocalCredsLocalhost(t *testing.T) { + if err := testLocalCredsE2ESucceed(t, "tcp", "localhost:0"); err != nil { + t.Fatalf("Failed e2e test for localhost: %v", err) + } +} + +func (s) TestLocalCredsUDS(t *testing.T) { + addr := fmt.Sprintf("/tmp/grpc_fullstck_test%d", time.Now().UnixNano()) + if err := testLocalCredsE2ESucceed(t, "unix", addr); err != nil { + t.Fatalf("Failed e2e test for UDS: %v", err) + } +} + +type connWrapper struct { + net.Conn + remote net.Addr +} + +func (c connWrapper) RemoteAddr() net.Addr { + return c.remote +} + +type lisWrapper struct { + net.Listener + remote net.Addr +} + +func spoofListener(l net.Listener, remote net.Addr) net.Listener { + return &lisWrapper{l, remote} +} + +func (l *lisWrapper) Accept() (net.Conn, error) { + c, err := l.Listener.Accept() + if err != nil { + return nil, err + } + return connWrapper{c, l.remote}, nil +} + +func spoofDialer(addr net.Addr) func(target string, t time.Duration) (net.Conn, error) { + return func(t string, d time.Duration) (net.Conn, error) { + c, err := net.DialTimeout("tcp", t, d) + if err != nil { + return nil, err + } + return connWrapper{c, addr}, nil + } +} + +func testLocalCredsE2EFail(t *testing.T, dopts []grpc.DialOption) error { + lis, err := testutils.LocalTCPListener() + if err != nil { + return fmt.Errorf("Failed to create listener: %v", err) + } + var fakeClientAddr, fakeServerAddr net.Addr + fakeClientAddr = &net.IPAddr{ + IP: netip.MustParseAddr("10.8.9.10").AsSlice(), + Zone: "", + } + fakeServerAddr = &net.IPAddr{ + IP: netip.MustParseAddr("10.8.9.11").AsSlice(), + Zone: "", + } + ss := &stubserver.StubServer{ + Listener: spoofListener(lis, fakeClientAddr), + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + S: grpc.NewServer(grpc.Creds(local.NewCredentials())), + } + stubserver.StartTestService(t, ss) + defer ss.S.Stop() + + cc, err := grpc.NewClient(lis.Addr().String(), append(dopts, grpc.WithDialer(spoofDialer(fakeServerAddr)))...) + if err != nil { + return fmt.Errorf("Failed to dial server: %v, %v", err, lis.Addr().String()) + } + defer cc.Close() + + c := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + _, err = c.EmptyCall(ctx, &testpb.Empty{}) + return err +} + +func isExpected(got, want error) bool { + return status.Code(got) == status.Code(want) && strings.Contains(status.Convert(got).Message(), status.Convert(want).Message()) +} + +func (s) TestLocalCredsClientFail(t *testing.T) { + // Use local creds at client-side which should lead to client-side failure. + opts := []grpc.DialOption{grpc.WithTransportCredentials(local.NewCredentials())} + want := status.Error(codes.Unavailable, "transport: authentication handshake failed: local credentials rejected connection to non-local address") + if err := testLocalCredsE2EFail(t, opts); !isExpected(err, want) { + t.Fatalf("testLocalCredsE2EFail() = %v; want %v", err, want) + } +} + +func (s) TestLocalCredsServerFail(t *testing.T) { + // Use insecure at client-side which should lead to server-side failure. + opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} + if err := testLocalCredsE2EFail(t, opts); status.Code(err) != codes.Unavailable { + t.Fatalf("testLocalCredsE2EFail() = %v; want %v", err, codes.Unavailable) + } +} diff --git a/test/logging.go b/test/logging.go new file mode 100644 index 0000000..15a923a --- /dev/null +++ b/test/logging.go @@ -0,0 +1,23 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import "google.golang.org/grpc/grpclog" + +var logger = grpclog.Component("testing") diff --git a/test/malformed_method_test.go b/test/malformed_method_test.go new file mode 100644 index 0000000..c5bc179 --- /dev/null +++ b/test/malformed_method_test.go @@ -0,0 +1,159 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "bytes" + "context" + "net" + "testing" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" + "google.golang.org/grpc/internal/stubserver" + + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestMalformedMethodPath tests that the server responds with Unimplemented +// when the method path is malformed. This verifies that the server does not +// route requests with a malformed method path to the application handler. +func (s) TestMalformedMethodPath(t *testing.T) { + tests := []struct { + name string + path string + wantStatus string // string representation of codes.Code + }{ + { + name: "missing_leading_slash", + path: "grpc.testing.TestService/UnaryCall", + wantStatus: "12", // Unimplemented + }, + { + name: "empty_path", + path: "", + wantStatus: "12", // Unimplemented + }, + { + name: "just_slash", + path: "/", + wantStatus: "12", // Unimplemented + }, + { + name: "double_slash", + path: "//grpc.testing.TestService/UnaryCall", + wantStatus: "12", // Unimplemented + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{Payload: &testpb.Payload{Body: []byte("pwned")}}, nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + // Open a raw TCP connection to the server and speak HTTP/2 directly. + tcpConn, err := net.Dial("tcp", ss.Address) + if err != nil { + t.Fatalf("Failed to dial tcp: %v", err) + } + defer tcpConn.Close() + + // Write the HTTP/2 connection preface and the initial settings frame. + if _, err := tcpConn.Write([]byte("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n")); err != nil { + t.Fatalf("Failed to write preface: %v", err) + } + framer := http2.NewFramer(tcpConn, tcpConn) + if err := framer.WriteSettings(); err != nil { + t.Fatalf("Failed to write settings: %v", err) + } + + // Encode and write the HEADERS frame. + var headerBuf bytes.Buffer + enc := hpack.NewEncoder(&headerBuf) + writeHeader := func(name, value string) { + enc.WriteField(hpack.HeaderField{Name: name, Value: value}) + } + writeHeader(":method", "POST") + writeHeader(":scheme", "http") + writeHeader(":authority", ss.Address) + writeHeader(":path", tc.path) + writeHeader("content-type", "application/grpc") + writeHeader("te", "trailers") + if err := framer.WriteHeaders(http2.HeadersFrameParam{ + StreamID: 1, + BlockFragment: headerBuf.Bytes(), + EndStream: false, + EndHeaders: true, + }); err != nil { + t.Fatalf("Failed to write headers: %v", err) + } + + // Send a small gRPC-encoded data frame (0 length). + if err := framer.WriteData(1, true, []byte{0, 0, 0, 0, 0}); err != nil { + t.Fatalf("Failed to write data: %v", err) + } + + // Read responses and look for grpc-status. + gotStatus := "" + dec := hpack.NewDecoder(4096, func(f hpack.HeaderField) { + if f.Name == "grpc-status" { + gotStatus = f.Value + } + }) + done := make(chan struct{}) + go func() { + defer close(done) + for { + frame, err := framer.ReadFrame() + if err != nil { + return + } + if headers, ok := frame.(*http2.HeadersFrame); ok { + if _, err := dec.Write(headers.HeaderBlockFragment()); err != nil { + return + } + if headers.StreamEnded() { + return + } + } + } + }() + + select { + case <-done: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for response") + } + + if gotStatus != tc.wantStatus { + t.Errorf("Got grpc-status %v, want %v", gotStatus, tc.wantStatus) + } + }) + } +} diff --git a/test/metadata_test.go b/test/metadata_test.go new file mode 100644 index 0000000..6c469d2 --- /dev/null +++ b/test/metadata_test.go @@ -0,0 +1,155 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "io" + "reflect" + "strings" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestInvalidMetadata(t *testing.T) { + grpctest.ExpectErrorN("stream: failed to validate md when setting trailer", 5) + + tests := []struct { + name string + md metadata.MD + appendMD []string + want error + recv error + }{ + { + name: "invalid key", + md: map[string][]string{string(rune(0x19)): {"testVal"}}, + want: status.Error(codes.Internal, "header key \"\\x19\" contains illegal characters not in [0-9a-z-_.]"), + recv: status.Error(codes.Internal, "invalid header field"), + }, + { + name: "invalid value", + md: map[string][]string{"test": {string(rune(0x19))}}, + want: status.Error(codes.Internal, "header key \"test\" contains value with non-printable ASCII characters"), + recv: status.Error(codes.Internal, "invalid header field"), + }, + { + name: "invalid appended value", + md: map[string][]string{"test": {"test"}}, + appendMD: []string{"/", "value"}, + want: status.Error(codes.Internal, "header key \"/\" contains illegal characters not in [0-9a-z-_.]"), + recv: status.Error(codes.Internal, "invalid header field"), + }, + { + name: "empty appended key", + md: map[string][]string{"test": {"test"}}, + appendMD: []string{"", "value"}, + want: status.Error(codes.Internal, "there is an empty key in the header"), + recv: status.Error(codes.Internal, "invalid header field"), + }, + { + name: "empty key", + md: map[string][]string{"": {"test"}}, + want: status.Error(codes.Internal, "there is an empty key in the header"), + recv: status.Error(codes.Internal, "invalid header field"), + }, + { + name: "-bin key with arbitrary value", + md: map[string][]string{"test-bin": {string(rune(0x19))}}, + want: nil, + recv: io.EOF, + }, + { + name: "valid key and value", + md: map[string][]string{"test": {"value"}}, + want: nil, + recv: io.EOF, + }, + } + + testNum := 0 + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != nil { + return err + } + test := tests[testNum] + testNum++ + // merge original md and added md. + md := metadata.Join(test.md, metadata.Pairs(test.appendMD...)) + + if err := stream.SetHeader(md); !reflect.DeepEqual(test.want, err) { + return fmt.Errorf("call stream.SendHeader(md) validate metadata which is %v got err :%v, want err :%v", md, err, test.want) + } + if err := stream.SendHeader(md); !reflect.DeepEqual(test.want, err) { + return fmt.Errorf("call stream.SendHeader(md) validate metadata which is %v got err :%v, want err :%v", md, err, test.want) + } + stream.SetTrailer(md) + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting ss endpoint server: %v", err) + } + defer ss.Stop() + + for _, test := range tests { + t.Run("unary "+test.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, test.md) + ctx = metadata.AppendToOutgoingContext(ctx, test.appendMD...) + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); !reflect.DeepEqual(test.want, err) { + t.Errorf("call ss.Client.EmptyCall() validate metadata which is %v got err :%v, want err :%v", test.md, err, test.want) + } + }) + } + + // call the stream server's api to drive the server-side unit testing + for _, test := range tests { + t.Run("streaming "+test.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Errorf("call ss.Client.FullDuplexCall got err :%v", err) + return + } + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Errorf("call ss.Client stream Send(nil) will success but got err :%v", err) + } + if _, err := stream.Recv(); status.Code(err) != status.Code(test.recv) || !strings.Contains(err.Error(), test.recv.Error()) { + t.Errorf("stream.Recv() = _, get err :%v, want err :%v", err, test.recv) + } + }) + } +} diff --git a/test/parse_config.go b/test/parse_config.go new file mode 100644 index 0000000..b8355c7 --- /dev/null +++ b/test/parse_config.go @@ -0,0 +1,38 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "testing" + + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" +) + +// parseServiceConfig is a test helper which uses the manual resolver to parse +// the given service config. It calls t.Fatal() if service config parsing fails. +func parseServiceConfig(t *testing.T, r *manual.Resolver, sc string) *serviceconfig.ParseResult { + t.Helper() + + scpr := r.CC().ParseServiceConfig(sc) + if scpr.Err != nil { + t.Fatalf("Failed to parse service config %q: %v", sc, scpr.Err) + } + return scpr +} diff --git a/test/race_test.go b/test/race_test.go new file mode 100644 index 0000000..d99f0a4 --- /dev/null +++ b/test/race_test.go @@ -0,0 +1,25 @@ +//go:build race +// +build race + +/* + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +func init() { + raceMode = true +} diff --git a/test/rawConnWrapper.go b/test/rawConnWrapper.go new file mode 100644 index 0000000..4928056 --- /dev/null +++ b/test/rawConnWrapper.go @@ -0,0 +1,247 @@ +/* + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package test + +import ( + "bytes" + "fmt" + "io" + "net" + "strings" + "sync" + "time" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" +) + +type listenerWrapper struct { + net.Listener + mu sync.Mutex + rcw *rawConnWrapper +} + +func listenWithConnControl(network, address string) (net.Listener, error) { + l, err := net.Listen(network, address) + if err != nil { + return nil, err + } + return &listenerWrapper{Listener: l}, nil +} + +// Accept blocks until Dial is called, then returns a net.Conn for the server +// half of the connection. +func (l *listenerWrapper) Accept() (net.Conn, error) { + c, err := l.Listener.Accept() + if err != nil { + return nil, err + } + l.mu.Lock() + l.rcw = newRawConnWrapperFromConn(c) + l.mu.Unlock() + return c, nil +} + +func (l *listenerWrapper) getLastConn() *rawConnWrapper { + l.mu.Lock() + defer l.mu.Unlock() + return l.rcw +} + +type dialerWrapper struct { + c net.Conn + rcw *rawConnWrapper +} + +func (d *dialerWrapper) dialer(target string, t time.Duration) (net.Conn, error) { + c, err := net.DialTimeout("tcp", target, t) + d.c = c + d.rcw = newRawConnWrapperFromConn(c) + return c, err +} + +func (d *dialerWrapper) getRawConnWrapper() *rawConnWrapper { + return d.rcw +} + +type rawConnWrapper struct { + cc io.ReadWriteCloser + fr *http2.Framer + + // writing headers: + headerBuf bytes.Buffer + hpackEnc *hpack.Encoder + + // reading frames: + frc chan http2.Frame + frErrc chan error +} + +func newRawConnWrapperFromConn(cc io.ReadWriteCloser) *rawConnWrapper { + rcw := &rawConnWrapper{ + cc: cc, + frc: make(chan http2.Frame, 1), + frErrc: make(chan error, 1), + } + rcw.hpackEnc = hpack.NewEncoder(&rcw.headerBuf) + rcw.fr = http2.NewFramer(cc, cc) + rcw.fr.ReadMetaHeaders = hpack.NewDecoder(4096 /*initialHeaderTableSize*/, nil) + + return rcw +} + +func (rcw *rawConnWrapper) Close() error { + return rcw.cc.Close() +} + +func (rcw *rawConnWrapper) encodeHeaderField(k, v string) error { + err := rcw.hpackEnc.WriteField(hpack.HeaderField{Name: k, Value: v}) + if err != nil { + return fmt.Errorf("HPACK encoding error for %q/%q: %v", k, v, err) + } + return nil +} + +// encodeRawHeader is for usage on both client and server side to construct header based on the input +// key, value pairs. +func (rcw *rawConnWrapper) encodeRawHeader(headers ...string) []byte { + if len(headers)%2 == 1 { + panic("odd number of kv args") + } + + rcw.headerBuf.Reset() + + pseudoCount := map[string]int{} + var keys []string + vals := map[string][]string{} + + for len(headers) > 0 { + k, v := headers[0], headers[1] + headers = headers[2:] + if _, ok := vals[k]; !ok { + keys = append(keys, k) + } + if strings.HasPrefix(k, ":") { + pseudoCount[k]++ + if pseudoCount[k] == 1 { + vals[k] = []string{v} + } else { + // Allows testing of invalid headers w/ dup pseudo fields. + vals[k] = append(vals[k], v) + } + } else { + vals[k] = append(vals[k], v) + } + } + for _, k := range keys { + for _, v := range vals[k] { + rcw.encodeHeaderField(k, v) + } + } + return rcw.headerBuf.Bytes() +} + +// encodeHeader is for usage on client side to write request header. +// +// encodeHeader encodes headers and returns their HPACK bytes. headers +// must contain an even number of key/value pairs. There may be +// multiple pairs for keys (e.g. "cookie"). The :method, :path, and +// :scheme headers default to GET, / and https. +func (rcw *rawConnWrapper) encodeHeader(headers ...string) []byte { + if len(headers)%2 == 1 { + panic("odd number of kv args") + } + + rcw.headerBuf.Reset() + + if len(headers) == 0 { + // Fast path, mostly for benchmarks, so test code doesn't pollute + // profiles when we're looking to improve server allocations. + rcw.encodeHeaderField(":method", "GET") + rcw.encodeHeaderField(":path", "/") + rcw.encodeHeaderField(":scheme", "https") + return rcw.headerBuf.Bytes() + } + + if len(headers) == 2 && headers[0] == ":method" { + // Another fast path for benchmarks. + rcw.encodeHeaderField(":method", headers[1]) + rcw.encodeHeaderField(":path", "/") + rcw.encodeHeaderField(":scheme", "https") + return rcw.headerBuf.Bytes() + } + + pseudoCount := map[string]int{} + keys := []string{":method", ":path", ":scheme"} + vals := map[string][]string{ + ":method": {"GET"}, + ":path": {"/"}, + ":scheme": {"https"}, + } + for len(headers) > 0 { + k, v := headers[0], headers[1] + headers = headers[2:] + if _, ok := vals[k]; !ok { + keys = append(keys, k) + } + if strings.HasPrefix(k, ":") { + pseudoCount[k]++ + if pseudoCount[k] == 1 { + vals[k] = []string{v} + } else { + // Allows testing of invalid headers w/ dup pseudo fields. + vals[k] = append(vals[k], v) + } + } else { + vals[k] = append(vals[k], v) + } + } + for _, k := range keys { + for _, v := range vals[k] { + rcw.encodeHeaderField(k, v) + } + } + return rcw.headerBuf.Bytes() +} + +func (rcw *rawConnWrapper) writeHeaders(p http2.HeadersFrameParam) error { + if err := rcw.fr.WriteHeaders(p); err != nil { + return fmt.Errorf("error writing HEADERS: %v", err) + } + return nil +} + +func (rcw *rawConnWrapper) writeRSTStream(streamID uint32, code http2.ErrCode) error { + if err := rcw.fr.WriteRSTStream(streamID, code); err != nil { + return fmt.Errorf("error writing RST_STREAM: %v", err) + } + return nil +} + +func (rcw *rawConnWrapper) writeGoAway(maxStreamID uint32, code http2.ErrCode, debugData []byte) error { + if err := rcw.fr.WriteGoAway(maxStreamID, code, debugData); err != nil { + return fmt.Errorf("error writing GoAway: %v", err) + } + return nil +} + +func (rcw *rawConnWrapper) writeDataFrame(streamID uint32, payload []byte) error { + if err := rcw.fr.WriteData(streamID, false, payload); err != nil { + return fmt.Errorf("error writing Raw Frame: %v", err) + } + return nil +} diff --git a/test/resolver_update_test.go b/test/resolver_update_test.go new file mode 100644 index 0000000..e62d915 --- /dev/null +++ b/test/resolver_update_test.go @@ -0,0 +1,263 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestResolverUpdateDuringBuild_ServiceConfigParseError makes the +// resolver.Builder call into the ClientConn, during the Build call, with a +// service config parsing error. +// +// We use two separate mutexes in the code which make sure there is no data race +// in this code path, and also that there is no deadlock. +func (s) TestResolverUpdateDuringBuild_ServiceConfigParseError(t *testing.T) { + // Setting InitialState on the manual resolver makes it call into the + // ClientConn during the Build call. + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{ServiceConfig: &serviceconfig.ParseResult{Err: errors.New("resolver build err")}}) + + cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("NewClient(_, _) = _, %v; want _, nil", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + const wantMsg = "error parsing service config" + const wantCode = codes.Unavailable + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != wantCode || !strings.Contains(status.Convert(err).Message(), wantMsg) { + t.Fatalf("EmptyCall RPC failed: %v; want code: %v, want message: %q", err, wantCode, wantMsg) + } +} + +type fakeConfig struct { + serviceconfig.Config +} + +// TestResolverUpdateDuringBuild_ServiceConfigInvalidTypeError makes the +// resolver.Builder call into the ClientConn, during the Build call, with an +// invalid service config type. +// +// We use two separate mutexes in the code which make sure there is no data race +// in this code path, and also that there is no deadlock. +func (s) TestResolverUpdateDuringBuild_ServiceConfigInvalidTypeError(t *testing.T) { + // Setting InitialState on the manual resolver makes it call into the + // ClientConn during the Build call. + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{ServiceConfig: &serviceconfig.ParseResult{Config: fakeConfig{}}}) + + cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("NewClient(_, _) = _, %v; want _, nil", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + const wantMsg = "illegal service config type" + const wantCode = codes.Unavailable + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != wantCode || !strings.Contains(status.Convert(err).Message(), wantMsg) { + t.Fatalf("EmptyCall RPC failed: %v; want code: %v, want message: %q", err, wantCode, wantMsg) + } +} + +// TestResolverUpdate_InvalidServiceConfigAsFirstUpdate makes the resolver send +// an update with an invalid service config as its first update. This should +// make the ClientConn apply the failing LB policy, and should result in RPC +// errors indicating the failing service config. +func (s) TestResolverUpdate_InvalidServiceConfigAsFirstUpdate(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + + cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("NewClient(_, _) = _, %v; want _, nil", err) + } + cc.Connect() + defer cc.Close() + + scpr := r.CC().ParseServiceConfig("bad json service config") + r.UpdateState(resolver.State{ServiceConfig: scpr}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + const wantMsg = "error parsing service config" + const wantCode = codes.Unavailable + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != wantCode || !strings.Contains(status.Convert(err).Message(), wantMsg) { + t.Fatalf("EmptyCall RPC failed: %v; want code: %v, want message: %q", err, wantCode, wantMsg) + } +} + +func verifyClientConnStateUpdate(got, want balancer.ClientConnState) error { + if got, want := got.ResolverState.Addresses, want.ResolverState.Addresses; !cmp.Equal(got, want) { + return fmt.Errorf("update got unexpected addresses: %v, want %v", got, want) + } + if got, want := got.ResolverState.ServiceConfig.Config, want.ResolverState.ServiceConfig.Config; !internal.EqualServiceConfigForTesting(got, want) { + return fmt.Errorf("received unexpected service config: \ngot: %v \nwant: %v", got, want) + } + if got, want := got.BalancerConfig, want.BalancerConfig; !cmp.Equal(got, want) { + return fmt.Errorf("received unexpected balancer config: \ngot: %v \nwant: %v", cmp.Diff(nil, got), cmp.Diff(nil, want)) + } + return nil +} + +// TestResolverUpdate_InvalidServiceConfigAfterGoodUpdate tests the scenario +// where the resolver sends an update with an invalid service config after +// having sent a good update. This should result in the ClientConn discarding +// the new invalid service config, and continuing to use the old good config. +func (s) TestResolverUpdate_InvalidServiceConfigAfterGoodUpdate(t *testing.T) { + type wrappingBalancerConfig struct { + serviceconfig.LoadBalancingConfig + Config string `json:"config,omitempty"` + } + + // Register a stub balancer which uses a "pick_first" balancer underneath and + // signals on a channel when it receives ClientConn updates. + ccUpdateCh := testutils.NewChannel() + stub.Register(t.Name(), stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + pf := balancer.Get(pickfirst.Name) + bd.ChildBalancer = pf.Build(bd.ClientConn, bd.BuildOptions) + }, + Close: func(bd *stub.BalancerData) { + bd.ChildBalancer.Close() + }, + ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + cfg := &wrappingBalancerConfig{} + if err := json.Unmarshal(lbCfg, cfg); err != nil { + return nil, err + } + return cfg, nil + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + if _, ok := ccs.BalancerConfig.(*wrappingBalancerConfig); !ok { + return fmt.Errorf("received balancer config of unsupported type %T", ccs.BalancerConfig) + } + ccUpdateCh.Send(ccs) + ccs.BalancerConfig = nil + return bd.ChildBalancer.UpdateClientConnState(ccs) + }, + }) + + // Start a backend exposing the test service. + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + defer backend.Stop() + + r := manual.NewBuilderWithScheme("whatever") + + cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("NewClient(_, _) = _, %v; want _, nil", err) + } + defer cc.Close() + cc.Connect() + // Push a resolver update and verify that our balancer receives the update. + addrs := []resolver.Address{{Addr: backend.Address}} + const lbCfg = "wrapping balancer LB policy config" + goodSC := r.CC().ParseServiceConfig(fmt.Sprintf(` +{ + "loadBalancingConfig": [ + { + "%v": { + "config": "%s" + } + } + ] +}`, t.Name(), lbCfg)) + r.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: goodSC}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantCCS := balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: addrs, + ServiceConfig: goodSC, + }, + BalancerConfig: &wrappingBalancerConfig{Config: lbCfg}, + } + ccs, err := ccUpdateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for ClientConnState update from grpc") + } + gotCCS := ccs.(balancer.ClientConnState) + if err := verifyClientConnStateUpdate(gotCCS, wantCCS); err != nil { + t.Fatal(err) + } + + // Ensure RPCs are successful. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall RPC failed: %v", err) + } + + // Push a bad resolver update and ensure that the update is propagated to our + // stub balancer. But since the pushed update contains an invalid service + // config, our balancer should continue to see the old loadBalancingConfig. + badSC := r.CC().ParseServiceConfig("bad json service config") + wantCCS.ResolverState.ServiceConfig = badSC + r.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: badSC}) + ccs, err = ccUpdateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for ClientConnState update from grpc") + } + gotCCS = ccs.(balancer.ClientConnState) + if err := verifyClientConnStateUpdate(gotCCS, wantCCS); err != nil { + t.Fatal(err) + } + + // RPCs should continue to be successful since the ClientConn is using the old + // good service config. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall RPC failed: %v", err) + } +} diff --git a/test/retry_test.go b/test/retry_test.go new file mode 100644 index 0000000..5eb3dd1 --- /dev/null +++ b/test/retry_test.go @@ -0,0 +1,1069 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "io" + "net" + "reflect" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestRetryUnary(t *testing.T) { + serverMu := sync.Mutex{} + i := -1 + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (r *testpb.Empty, err error) { + serverMu.Lock() + defer serverMu.Unlock() + defer func() { t.Logf("server call %v returning err %v", i, err) }() + i++ + switch i { + case 0, 2, 5: + return &testpb.Empty{}, nil + case 6, 8, 11: + return nil, status.New(codes.Internal, "non-retryable error").Err() + } + return nil, status.New(codes.AlreadyExists, "retryable error").Err() + }, + } + if err := ss.Start([]grpc.ServerOption{}, + grpc.WithDefaultServiceConfig(`{ + "methodConfig": [{ + "name": [{"service": "grpc.testing.TestService"}], + "waitForReady": true, + "retryPolicy": { + "MaxAttempts": 4, + "InitialBackoff": ".01s", + "MaxBackoff": ".01s", + "BackoffMultiplier": 1.0, + "RetryableStatusCodes": [ "ALREADY_EXISTS" ] + } + }]}`)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + testCases := []struct { + wantCode codes.Code + wantCount int + }{ + {codes.OK, 0}, + {codes.OK, 2}, + {codes.OK, 5}, + {codes.Internal, 6}, + {codes.Internal, 8}, + {codes.Internal, 11}, + {codes.AlreadyExists, 15}, + } + for num, tc := range testCases { + t.Log("Case", num) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}) + cancel() + if status.Code(err) != tc.wantCode { + t.Fatalf("EmptyCall(_, _) = _, %v; want _, ", err, tc.wantCode) + } + serverMu.Lock() + if i != tc.wantCount { + serverMu.Unlock() + t.Fatalf("i = %v; want %v", i, tc.wantCount) + } + serverMu.Unlock() + } +} + +func (s) TestRetryThrottling(t *testing.T) { + serverMu := sync.Mutex{} + i := -1 + + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + serverMu.Lock() + defer serverMu.Unlock() + i++ + switch i { + case 0, 3, 6, 10, 11, 12, 13, 14, 16, 18: + return &testpb.Empty{}, nil + } + return nil, status.New(codes.Unavailable, "retryable error").Err() + }, + } + if err := ss.Start([]grpc.ServerOption{}, + grpc.WithDefaultServiceConfig(`{ + "methodConfig": [{ + "name": [{"service": "grpc.testing.TestService"}], + "waitForReady": true, + "retryPolicy": { + "MaxAttempts": 4, + "InitialBackoff": ".01s", + "MaxBackoff": ".01s", + "BackoffMultiplier": 1.0, + "RetryableStatusCodes": [ "UNAVAILABLE" ] + } + }], + "retryThrottling": { + "maxTokens": 10, + "tokenRatio": 0.5 + } + }`)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + testCases := []struct { + wantCode codes.Code + wantCount int + }{ + {codes.OK, 0}, // tokens = 10 + {codes.OK, 3}, // tokens = 8.5 (10 - 2 failures + 0.5 success) + {codes.OK, 6}, // tokens = 6 + {codes.Unavailable, 8}, // tokens = 5 -- first attempt is retried; second aborted. + {codes.Unavailable, 9}, // tokens = 4 + {codes.OK, 10}, // tokens = 4.5 + {codes.OK, 11}, // tokens = 5 + {codes.OK, 12}, // tokens = 5.5 + {codes.OK, 13}, // tokens = 6 + {codes.OK, 14}, // tokens = 6.5 + {codes.OK, 16}, // tokens = 5.5 + {codes.Unavailable, 17}, // tokens = 4.5 + } + for _, tc := range testCases { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}) + cancel() + if status.Code(err) != tc.wantCode { + t.Errorf("EmptyCall(_, _) = _, %v; want _, ", err, tc.wantCode) + } + serverMu.Lock() + if i != tc.wantCount { + t.Errorf("i = %v; want %v", i, tc.wantCount) + } + serverMu.Unlock() + } +} + +func (s) TestRetryStreaming(t *testing.T) { + req := func(b byte) *testpb.StreamingOutputCallRequest { + return &testpb.StreamingOutputCallRequest{Payload: &testpb.Payload{Body: []byte{b}}} + } + res := func(b byte) *testpb.StreamingOutputCallResponse { + return &testpb.StreamingOutputCallResponse{Payload: &testpb.Payload{Body: []byte{b}}} + } + + largePayload, _ := newPayload(testpb.PayloadType_COMPRESSABLE, 500) + + type serverOp func(stream testgrpc.TestService_FullDuplexCallServer) error + type clientOp func(stream testgrpc.TestService_FullDuplexCallClient) error + + // Server Operations + sAttempts := func(n int) serverOp { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { + const key = "grpc-previous-rpc-attempts" + md, ok := metadata.FromIncomingContext(stream.Context()) + if !ok { + return status.Errorf(codes.Internal, "server: no header metadata received") + } + if got := md[key]; len(got) != 1 || got[0] != strconv.Itoa(n) { + return status.Errorf(codes.Internal, "server: metadata = %v; want ", md, key, n) + } + return nil + } + } + sReq := func(b byte) serverOp { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { + want := req(b) + if got, err := stream.Recv(); err != nil || !proto.Equal(got, want) { + return status.Errorf(codes.Internal, "server: Recv() = %v, %v; want %v, ", got, err, want) + } + return nil + } + } + sReqPayload := func(p *testpb.Payload) serverOp { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { + want := &testpb.StreamingOutputCallRequest{Payload: p} + if got, err := stream.Recv(); err != nil || !proto.Equal(got, want) { + return status.Errorf(codes.Internal, "server: Recv() = %v, %v; want %v, ", got, err, want) + } + return nil + } + } + sHdr := func() serverOp { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { + return stream.SendHeader(metadata.Pairs("test_header", "test_value")) + } + } + sRes := func(b byte) serverOp { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { + msg := res(b) + if err := stream.Send(msg); err != nil { + return status.Errorf(codes.Internal, "server: Send(%v) = %v; want ", msg, err) + } + return nil + } + } + sErr := func(c codes.Code) serverOp { + return func(testgrpc.TestService_FullDuplexCallServer) error { + return status.New(c, "this is a test error").Err() + } + } + sCloseSend := func() serverOp { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { + if msg, err := stream.Recv(); msg != nil || err != io.EOF { + return status.Errorf(codes.Internal, "server: Recv() = %v, %v; want , io.EOF", msg, err) + } + return nil + } + } + sPushback := func(s string) serverOp { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { + stream.SetTrailer(metadata.MD{"grpc-retry-pushback-ms": []string{s}}) + return nil + } + } + + // Client Operations + cReq := func(b byte) clientOp { + return func(stream testgrpc.TestService_FullDuplexCallClient) error { + msg := req(b) + if err := stream.Send(msg); err != nil { + return fmt.Errorf("client: Send(%v) = %v; want ", msg, err) + } + return nil + } + } + cReqPayload := func(p *testpb.Payload) clientOp { + return func(stream testgrpc.TestService_FullDuplexCallClient) error { + msg := &testpb.StreamingOutputCallRequest{Payload: p} + if err := stream.Send(msg); err != nil { + return fmt.Errorf("client: Send(%v) = %v; want ", msg, err) + } + return nil + } + } + cRes := func(b byte) clientOp { + return func(stream testgrpc.TestService_FullDuplexCallClient) error { + want := res(b) + if got, err := stream.Recv(); err != nil || !proto.Equal(got, want) { + return fmt.Errorf("client: Recv() = %v, %v; want %v, ", got, err, want) + } + return nil + } + } + cErr := func(c codes.Code) clientOp { + return func(stream testgrpc.TestService_FullDuplexCallClient) error { + res, err := stream.Recv() + var gotCode codes.Code + if err == io.EOF { + gotCode = codes.OK + } else { + gotCode = status.Code(err) + } + if res != nil || gotCode != c { + return fmt.Errorf("client: Recv() = %v, %v; want , %v", res, err, c) + } + return nil + } + } + cCloseSend := func() clientOp { + return func(stream testgrpc.TestService_FullDuplexCallClient) error { + if err := stream.CloseSend(); err != nil { + return fmt.Errorf("client: CloseSend() = %v; want ", err) + } + return nil + } + } + var curTime time.Time + cGetTime := func() clientOp { + return func(_ testgrpc.TestService_FullDuplexCallClient) error { + curTime = time.Now() + return nil + } + } + cCheckElapsed := func(d time.Duration) clientOp { + return func(_ testgrpc.TestService_FullDuplexCallClient) error { + if elapsed := time.Since(curTime); elapsed < d { + return fmt.Errorf("elapsed time: %v; want >= %v", elapsed, d) + } + return nil + } + } + cHdr := func() clientOp { + return func(stream testgrpc.TestService_FullDuplexCallClient) error { + _, err := stream.Header() + if err == io.EOF { + // The stream ended successfully; convert to nil to avoid + // erroring the test case. + err = nil + } + return err + } + } + cCtx := func() clientOp { + return func(stream testgrpc.TestService_FullDuplexCallClient) error { + stream.Context() + return nil + } + } + + testCases := []struct { + desc string + serverOps []serverOp + clientOps []clientOp + }{{ + desc: "Non-retryable error code", + serverOps: []serverOp{sReq(1), sErr(codes.Internal)}, + clientOps: []clientOp{cReq(1), cErr(codes.Internal)}, + }, { + desc: "One retry necessary", + serverOps: []serverOp{sReq(1), sErr(codes.Unavailable), sReq(1), sAttempts(1), sRes(1)}, + clientOps: []clientOp{cReq(1), cRes(1), cErr(codes.OK)}, + }, { + desc: "Exceed max attempts (4); check attempts header on server", + serverOps: []serverOp{ + sReq(1), sErr(codes.Unavailable), + sReq(1), sAttempts(1), sErr(codes.Unavailable), + sAttempts(2), sReq(1), sErr(codes.Unavailable), + sAttempts(3), sReq(1), sErr(codes.Unavailable), + }, + clientOps: []clientOp{cReq(1), cErr(codes.Unavailable)}, + }, { + desc: "Multiple requests", + serverOps: []serverOp{ + sReq(1), sReq(2), sErr(codes.Unavailable), + sReq(1), sReq(2), sRes(5), + }, + clientOps: []clientOp{cReq(1), cReq(2), cRes(5), cErr(codes.OK)}, + }, { + desc: "Multiple successive requests", + serverOps: []serverOp{ + sReq(1), sErr(codes.Unavailable), + sReq(1), sReq(2), sErr(codes.Unavailable), + sReq(1), sReq(2), sReq(3), sRes(5), + }, + clientOps: []clientOp{cReq(1), cReq(2), cReq(3), cRes(5), cErr(codes.OK)}, + }, { + desc: "No retry after receiving", + serverOps: []serverOp{ + sReq(1), sErr(codes.Unavailable), + sReq(1), sRes(3), sErr(codes.Unavailable), + }, + clientOps: []clientOp{cReq(1), cRes(3), cErr(codes.Unavailable)}, + }, { + desc: "Retry via ClientStream.Header()", + serverOps: []serverOp{sReq(1), sErr(codes.Unavailable), sReq(1), sAttempts(1)}, + clientOps: []clientOp{cReq(1), cHdr() /* this should cause a retry */, cErr(codes.OK)}, + }, { + desc: "No retry after header", + serverOps: []serverOp{sReq(1), sHdr(), sErr(codes.Unavailable)}, + clientOps: []clientOp{cReq(1), cHdr(), cErr(codes.Unavailable)}, + }, { + desc: "No retry after context", + serverOps: []serverOp{sReq(1), sErr(codes.Unavailable)}, + clientOps: []clientOp{cReq(1), cCtx(), cErr(codes.Unavailable)}, + }, { + desc: "Replaying close send", + serverOps: []serverOp{ + sReq(1), sReq(2), sCloseSend(), sErr(codes.Unavailable), + sReq(1), sReq(2), sCloseSend(), sRes(1), sRes(3), sRes(5), + }, + clientOps: []clientOp{cReq(1), cReq(2), cCloseSend(), cRes(1), cRes(3), cRes(5), cErr(codes.OK)}, + }, { + desc: "Negative server pushback - no retry", + serverOps: []serverOp{sReq(1), sPushback("-1"), sErr(codes.Unavailable)}, + clientOps: []clientOp{cReq(1), cErr(codes.Unavailable)}, + }, { + desc: "Non-numeric server pushback - no retry", + serverOps: []serverOp{sReq(1), sPushback("xxx"), sErr(codes.Unavailable)}, + clientOps: []clientOp{cReq(1), cErr(codes.Unavailable)}, + }, { + desc: "Multiple server pushback values - no retry", + serverOps: []serverOp{sReq(1), sPushback("100"), sPushback("10"), sErr(codes.Unavailable)}, + clientOps: []clientOp{cReq(1), cErr(codes.Unavailable)}, + }, { + desc: "1s server pushback - delayed retry", + serverOps: []serverOp{sReq(1), sPushback("1000"), sErr(codes.Unavailable), sReq(1), sRes(2)}, + clientOps: []clientOp{cGetTime(), cReq(1), cRes(2), cCheckElapsed(time.Second), cErr(codes.OK)}, + }, { + desc: "Overflowing buffer - no retry", + serverOps: []serverOp{sReqPayload(largePayload), sErr(codes.Unavailable)}, + clientOps: []clientOp{cReqPayload(largePayload), cErr(codes.Unavailable)}, + }} + + serverMu := sync.Mutex{} + var serverOpIter int + var serverOps []serverOp + + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + serverMu.Lock() + defer serverMu.Unlock() + for serverOpIter < len(serverOps) { + op := serverOps[serverOpIter] + serverOpIter++ + if err := op(stream); err != nil { + return err + } + } + return nil + }, + } + if err := ss.Start([]grpc.ServerOption{}, grpc.WithDefaultCallOptions(grpc.MaxRetryRPCBufferSize(200)), + grpc.WithDefaultServiceConfig(`{ + "methodConfig": [{ + "name": [{"service": "grpc.testing.TestService"}], + "waitForReady": true, + "retryPolicy": { + "MaxAttempts": 4, + "InitialBackoff": ".01s", + "MaxBackoff": ".01s", + "BackoffMultiplier": 1.0, + "RetryableStatusCodes": [ "UNAVAILABLE" ] + } + }]}`)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for { + if ctx.Err() != nil { + t.Fatalf("Timed out waiting for service config update") + } + if ss.CC.GetMethodConfig("/grpc.testing.TestService/FullDuplexCall").WaitForReady != nil { + break + } + time.Sleep(time.Millisecond) + } + + for i, tc := range testCases { + func() { + serverMu.Lock() + serverOpIter = 0 + serverOps = tc.serverOps + serverMu.Unlock() + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("%v: Error while creating stream: %v", tc.desc, err) + } + for j, op := range tc.clientOps { + if err := op(stream); err != nil { + t.Errorf("%d %d %v: %v", i, j, tc.desc, err) + break + } + } + serverMu.Lock() + defer serverMu.Unlock() + if serverOpIter != len(serverOps) { + t.Errorf("%v: serverOpIter = %v; want %v", tc.desc, serverOpIter, len(serverOps)) + } + }() + } +} + +func (s) TestMaxCallAttempts(t *testing.T) { + testCases := []struct { + serviceMaxAttempts int + clientMaxAttempts int + expectedAttempts int + }{ + {serviceMaxAttempts: 9, clientMaxAttempts: 4, expectedAttempts: 4}, + {serviceMaxAttempts: 9, clientMaxAttempts: 7, expectedAttempts: 7}, + {serviceMaxAttempts: 3, clientMaxAttempts: 10, expectedAttempts: 3}, + {serviceMaxAttempts: 8, clientMaxAttempts: -1, expectedAttempts: 5}, // 5 is default max + {serviceMaxAttempts: 3, clientMaxAttempts: 0, expectedAttempts: 3}, + } + + for _, tc := range testCases { + clientOpts := []grpc.DialOption{ + grpc.WithMaxCallAttempts(tc.clientMaxAttempts), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{ + "methodConfig": [{ + "name": [{"service": "grpc.testing.TestService"}], + "waitForReady": true, + "retryPolicy": { + "MaxAttempts": %d, + "InitialBackoff": ".01s", + "MaxBackoff": ".01s", + "BackoffMultiplier": 1.0, + "RetryableStatusCodes": [ "UNAVAILABLE" ] + } + }]}`, tc.serviceMaxAttempts), + ), + } + + serverMu := sync.Mutex{} + streamCallCount := 0 + unaryCallCount := 0 + + ss := &stubserver.StubServer{ + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + serverMu.Lock() + defer serverMu.Unlock() + streamCallCount++ + return status.New(codes.Unavailable, "this is a test error").Err() + }, + EmptyCallF: func(context.Context, *testpb.Empty) (r *testpb.Empty, err error) { + serverMu.Lock() + defer serverMu.Unlock() + unaryCallCount++ + return nil, status.New(codes.Unavailable, "this is a test error").Err() + }, + } + + func() { + + if err := ss.Start([]grpc.ServerOption{}, clientOpts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + for { + if ctx.Err() != nil { + t.Fatalf("Timed out waiting for service config update") + } + if ss.CC.GetMethodConfig("/grpc.testing.TestService/FullDuplexCall").WaitForReady != nil { + break + } + time.Sleep(time.Millisecond) + } + + // Test streaming RPC + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("Error while creating stream: %v", err) + } + if got, err := stream.Recv(); err == nil { + t.Fatalf("client: Recv() = %s, %v; want , error", got, err) + } else if status.Code(err) != codes.Unavailable { + t.Fatalf("client: Recv() = _, %v; want _, Unavailable", err) + } else if !strings.Contains(err.Error(), "retries exhausted") { + t.Fatalf("want: retries exhausted, got: %v", err) + } + + serverMu.Lock() + if streamCallCount != tc.expectedAttempts { + serverMu.Unlock() + t.Fatalf("stream expectedAttempts = %v; want %v", streamCallCount, tc.expectedAttempts) + } + serverMu.Unlock() + + // Test unary RPC + if ugot, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Fatalf("client: EmptyCall() = %s, %v; want , error", ugot, err) + } else if status.Code(err) != codes.Unavailable { + t.Fatalf("client: EmptyCall() = _, %v; want _, Unavailable", err) + } + serverMu.Lock() + defer serverMu.Unlock() + if unaryCallCount != tc.expectedAttempts { + t.Fatalf("unary expectedAttempts = %v; want %v", unaryCallCount, tc.expectedAttempts) + } + }() + } +} + +type retryStatsHandler struct { + mu sync.Mutex + s []stats.RPCStats +} + +func (*retryStatsHandler) TagRPC(ctx context.Context, _ *stats.RPCTagInfo) context.Context { + return ctx +} +func (h *retryStatsHandler) HandleRPC(_ context.Context, s stats.RPCStats) { + // these calls come in nondeterministically - so can just ignore + if _, ok := s.(*stats.DelayedPickComplete); ok { + return + } + h.mu.Lock() + h.s = append(h.s, s) + h.mu.Unlock() +} +func (*retryStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} +func (*retryStatsHandler) HandleConn(context.Context, stats.ConnStats) {} + +func (s) TestRetryStats(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen. Err: %v", err) + } + defer lis.Close() + server := &httpServer{ + waitForEndStream: true, + responses: []httpServerResponse{{ + trailers: [][]string{{ + ":status", "200", + "content-type", "application/grpc", + "grpc-status", "14", // UNAVAILABLE + "grpc-message", "unavailable retry", + "grpc-retry-pushback-ms", "10", + }}, + }, { + headers: [][]string{{ + ":status", "200", + "content-type", "application/grpc", + }}, + payloads: [][]byte{{0, 0, 0, 0, 0}}, // header for 0-byte response message. + trailers: [][]string{{ + "grpc-status", "0", // OK + }}, + }}, + refuseStream: func(i uint32) bool { + return i == 1 + }, + } + server.start(t, lis) + handler := &retryStatsHandler{} + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithStatsHandler(handler), + grpc.WithDefaultServiceConfig((`{ + "methodConfig": [{ + "name": [{"service": "grpc.testing.TestService"}], + "retryPolicy": { + "MaxAttempts": 4, + "InitialBackoff": ".01s", + "MaxBackoff": ".01s", + "BackoffMultiplier": 1.0, + "RetryableStatusCodes": [ "UNAVAILABLE" ] + } + }]}`))) + if err != nil { + t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + client := testgrpc.NewTestServiceClient(cc) + + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("unexpected EmptyCall error: %v", err) + } + handler.mu.Lock() + want := []stats.RPCStats{ + &stats.Begin{}, + &stats.OutHeader{FullMethod: "/grpc.testing.TestService/EmptyCall"}, + &stats.OutPayload{WireLength: 5}, + &stats.End{}, + + &stats.Begin{IsTransparentRetryAttempt: true}, + &stats.OutHeader{FullMethod: "/grpc.testing.TestService/EmptyCall"}, + &stats.OutPayload{WireLength: 5}, + &stats.InTrailer{Trailer: metadata.Pairs("content-type", "application/grpc", "grpc-retry-pushback-ms", "10")}, + &stats.End{}, + + &stats.Begin{}, + &stats.OutHeader{FullMethod: "/grpc.testing.TestService/EmptyCall"}, + &stats.OutPayload{WireLength: 5}, + &stats.InHeader{}, + &stats.InPayload{WireLength: 5}, + &stats.InTrailer{}, + &stats.End{}, + } + + toString := func(ss []stats.RPCStats) (ret []string) { + for _, s := range ss { + ret = append(ret, fmt.Sprintf("%T - %v", s, s)) + } + return ret + } + t.Logf("Handler received frames:\n%v\n---\nwant:\n%v\n", + strings.Join(toString(handler.s), "\n"), + strings.Join(toString(want), "\n")) + + if len(handler.s) != len(want) { + t.Fatalf("received unexpected number of RPCStats: got %v; want %v", len(handler.s), len(want)) + } + + // There is a race between receiving the payload (triggered by the + // application / gRPC library) and receiving the trailer (triggered at the + // transport layer). Adjust the received stats accordingly if necessary. + const tIdx, pIdx = 13, 14 + _, okT := handler.s[tIdx].(*stats.InTrailer) + _, okP := handler.s[pIdx].(*stats.InPayload) + if okT && okP { + handler.s[pIdx], handler.s[tIdx] = handler.s[tIdx], handler.s[pIdx] + } + + // The transparent retry attempt has a similar race, but between sending + // the payload and receiving the trailer. OutPayload is reported once the + // data frame is queued on the transport, not once it is written, so the + // server can respond and the trailer can be reported first. Adjust the + // received stats accordingly if necessary. + const tIdx2, pIdx2 = 6, 7 + _, okT2 := handler.s[tIdx2].(*stats.InTrailer) + _, okP2 := handler.s[pIdx2].(*stats.OutPayload) + if okT2 && okP2 { + handler.s[pIdx2], handler.s[tIdx2] = handler.s[tIdx2], handler.s[pIdx2] + } + + for i := range handler.s { + w, s := want[i], handler.s[i] + + // Validate the event type + if reflect.TypeOf(w) != reflect.TypeOf(s) { + t.Fatalf("at position %v: got %T; want %T", i, s, w) + } + wv, sv := reflect.ValueOf(w).Elem(), reflect.ValueOf(s).Elem() + + // Validate that Client is always true + if sv.FieldByName("Client").Interface().(bool) != true { + t.Fatalf("at position %v: got Client=false; want true", i) + } + + // Validate any set fields in want + for i := 0; i < wv.NumField(); i++ { + if !wv.Field(i).IsZero() { + if got, want := sv.Field(i).Interface(), wv.Field(i).Interface(); !reflect.DeepEqual(got, want) { + name := reflect.TypeOf(w).Elem().Field(i).Name + t.Fatalf("at position %v, field %v: got %v; want %v", i, name, got, want) + } + } + } + + // Since the above only tests non-zero-value fields, test + // IsTransparentRetryAttempt=false explicitly when needed. + if wb, ok := w.(*stats.Begin); ok && !wb.IsTransparentRetryAttempt { + if s.(*stats.Begin).IsTransparentRetryAttempt { + t.Fatalf("at position %v: got IsTransparentRetryAttempt=true; want false", i) + } + } + } + + // Validate timings between last Begin and preceding End. + end := handler.s[8].(*stats.End) + begin := handler.s[9].(*stats.Begin) + diff := begin.BeginTime.Sub(end.EndTime) + if diff < 10*time.Millisecond || diff > 50*time.Millisecond { + t.Fatalf("pushback time before final attempt = %v; want ~10ms", diff) + } +} + +func (s) TestRetryTransparentWhenCommitted(t *testing.T) { + // With MaxConcurrentStreams=1: + // + // 1. Create stream 1 that is retriable. + // 2. Stream 1 is created and fails with a retriable code. + // 3. Create dummy stream 2, blocking indefinitely. + // 4. Stream 1 retries (and blocks until stream 2 finishes) + // 5. Stream 1 is canceled manually. + // + // If there is no bug, the stream is done and errors with CANCELED. With a bug: + // + // 6. Stream 1 has a nil stream (attempt.s). Operations like CloseSend will panic. + + first := grpcsync.NewEvent() + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + // signal? + if !first.HasFired() { + first.Fire() + t.Log("returned first error") + return status.Error(codes.AlreadyExists, "first attempt fails and is retriable") + } + t.Log("blocking") + <-stream.Context().Done() + return stream.Context().Err() + }, + } + + if err := ss.Start([]grpc.ServerOption{grpc.MaxConcurrentStreams(1)}, + grpc.WithDefaultServiceConfig(`{ + "methodConfig": [{ + "name": [{"service": "grpc.testing.TestService"}], + "waitForReady": true, + "retryPolicy": { + "MaxAttempts": 2, + "InitialBackoff": ".1s", + "MaxBackoff": ".1s", + "BackoffMultiplier": 1.0, + "RetryableStatusCodes": [ "ALREADY_EXISTS" ] + } + }]}`)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx1, cancel1 := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel1() + ctx2, cancel2 := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel2() + + stream1, err := ss.Client.FullDuplexCall(ctx1) + if err != nil { + t.Fatalf("Error creating stream 1: %v", err) + } + + // Create dummy stream to block indefinitely. + _, err = ss.Client.FullDuplexCall(ctx2) + if err != nil { + t.Errorf("Error creating stream 2: %v", err) + } + + stream1Closed := grpcsync.NewEvent() + go func() { + _, err := stream1.Recv() + // Will trigger a retry when it sees the ALREADY_EXISTS error + if status.Code(err) != codes.Canceled { + t.Errorf("Expected stream1 to be canceled; got error: %v", err) + } + stream1Closed.Fire() + }() + + // Wait longer than the retry backoff timer. + time.Sleep(200 * time.Millisecond) + cancel1() + + // Operations on the stream should not panic. + <-stream1Closed.Done() + stream1.CloseSend() + stream1.Recv() + stream1.Send(&testpb.StreamingOutputCallRequest{}) +} + +func (s) TestNoRetry(t *testing.T) { + scJSON := `{ + "methodConfig": [{ + "name": [{"service": "grpc.testing.TestService"}], + "retryPolicy": { + "MaxAttempts": 4, + "InitialBackoff": ".01s", + "MaxBackoff": ".01s", + "BackoffMultiplier": 1.0, + "RetryableStatusCodes": [ "UNAVAILABLE" ] + } + }]}` + tests := []struct { + name string + dialOpts []grpc.DialOption + }{ + { + name: "disabled", + dialOpts: []grpc.DialOption{ + grpc.WithDefaultServiceConfig(scJSON), + grpc.WithDisableRetry(), + }, + }, + { + name: "not_configured", + dialOpts: []grpc.DialOption{}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + return status.New(codes.Unavailable, "retryable error").Err() + }, + EmptyCallF: func(context.Context, *testpb.Empty) (r *testpb.Empty, err error) { + return nil, status.New(codes.Unavailable, "retryable error").Err() + }, + } + if err := ss.Start([]grpc.ServerOption{}, tc.dialOpts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Test streaming RPC + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("Error while creating stream: %v", err) + } + _, err = stream.Recv() + if err == nil { + t.Fatal("stream.Recv() succeeded when expected to fail") + } + if status.Code(err) != codes.Unavailable { + t.Fatalf("client: Recv() = _, %v; want _, Unavailable", err) + } + if strings.Contains(err.Error(), "retries exhausted") { + t.Fatalf("client: EmptyCall() failed with an unexpected 'retries exhausted' error: %v", err) + } + + // Test unary RPC + _, err = ss.Client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Fatal("EmptyCall() succeeded when expected to fail") + } + if status.Code(err) != codes.Unavailable { + t.Fatalf("client: EmptyCall() = _, %v; want _, Unavailable", err) + } + + if strings.Contains(err.Error(), "retries exhausted") { + t.Fatalf("client: EmptyCall() failed with an unexpected 'retries exhausted' error: %v", err) + } + }) + } +} + +// authorityOverridePicker returns an authority override in the pick result +// metadata for every pick made before the server has seen the first attempt +// of the RPC, and no metadata for later picks. This mirrors an xDS cluster +// (gRFC A81) where only some of the endpoints carry a hostname, so only some +// picks rewrite the authority. The decision is keyed off the server-side +// attempt count, rather than a per-picker pick count, so that it stays stable +// if the channel re-picks for the first attempt or installs a new picker +// instance on a connectivity state change. +type authorityOverridePicker struct { + sc balancer.SubConn + authority string + serverAttempts *atomic.Int32 +} + +func (p *authorityOverridePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + res := balancer.PickResult{SubConn: p.sc} + if p.serverAttempts.Load() == 0 { + res.Metadata = metadata.Pairs(":authority", p.authority) + } + return res, nil +} + +// TestAuthorityOverrideNotReusedAcrossAttempts verifies that an authority +// override supplied by the LB picker applies only to the attempt it was picked +// for. A retry attempt whose pick carries no override must fall back to the +// channel's authority instead of reusing the previous attempt's override. +func (s) TestAuthorityOverrideNotReusedAcrossAttempts(t *testing.T) { + const ( + balancerName = "authority-override-retry-balancer" + overrideAuthority = "picked-endpoint.example.com" + wantAuthority = "test.server" + ) + + // Incremented by the server handler as attempts arrive, and read by the + // picker to decide whether to return the authority override. + serverAttempts := &atomic.Int32{} + + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + addrs := ccs.ResolverState.Addresses + if len(addrs) == 0 { + return nil + } + var sc balancer.SubConn + sc, err := bd.ClientConn.NewSubConn(addrs[:1], balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &authorityOverridePicker{sc: sc, authority: overrideAuthority, serverAttempts: serverAttempts}, + }) + }, + }) + if err != nil { + return err + } + sc.Connect() + return nil + }, + } + stub.Register(balancerName, bf) + + // Attempts run sequentially (the retry is only scheduled after the first + // attempt's handler has returned), so authorities needs no extra + // synchronization. + var authorities []string + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, _ := metadata.FromIncomingContext(ctx) + authorities = append(authorities, md.Get(":authority")...) + // Fail the first attempt with a retryable code so that the RPC is + // retried, and let the second attempt succeed. + if serverAttempts.Add(1) == 1 { + return nil, status.Error(codes.Unavailable, "forcing a retry") + } + return &testpb.Empty{}, nil + }, + } + if err := ss.StartServer(); err != nil { + t.Fatalf("Failed to start server: %v", err) + } + defer ss.Stop() + + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}}) + + sc := fmt.Sprintf(`{ + "loadBalancingConfig": [{%q: {}}], + "methodConfig": [{ + "name": [{"service": "grpc.testing.TestService"}], + "retryPolicy": { + "maxAttempts": 2, + "initialBackoff": "0.01s", + "maxBackoff": "0.01s", + "backoffMultiplier": 1.0, + "retryableStatusCodes": ["UNAVAILABLE"] + } + }] + }`, balancerName) + + cc, err := grpc.NewClient(r.Scheme()+":///"+wantAuthority, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r), grpc.WithDefaultServiceConfig(sc)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + if len(authorities) != 2 { + t.Fatalf("Server saw %d attempts (%q), want 2", len(authorities), authorities) + } + if authorities[0] != overrideAuthority { + t.Errorf("First attempt used authority %q, want %q", authorities[0], overrideAuthority) + } + if authorities[1] != wantAuthority { + t.Errorf("Retry attempt used authority %q, want %q", authorities[1], wantAuthority) + } +} diff --git a/test/roundrobin_test.go b/test/roundrobin_test.go new file mode 100644 index 0000000..7fa92aa --- /dev/null +++ b/test/roundrobin_test.go @@ -0,0 +1,318 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/channelz" + imetadata "google.golang.org/grpc/internal/metadata" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + rrutil "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const rrServiceConfig = `{"loadBalancingConfig": [{"round_robin":{}}]}` + +func testRoundRobinBasic(ctx context.Context, t *testing.T, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, []*stubserver.StubServer) { + t.Helper() + + r := manual.NewBuilderWithScheme("whatever") + + const backendCount = 5 + backends := make([]*stubserver.StubServer, backendCount) + endpoints := make([]resolver.Endpoint, backendCount) + addrs := make([]resolver.Address, backendCount) + for i := 0; i < backendCount; i++ { + backend := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + + backends[i] = backend + addrs[i] = resolver.Address{Addr: backend.Address} + endpoints[i] = resolver.Endpoint{Addresses: []resolver.Address{addrs[i]}} + } + + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(rrServiceConfig), + } + dopts = append(dopts, opts...) + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + client := testgrpc.NewTestServiceClient(cc) + + // At this point, the resolver has not returned any addresses to the channel. + // This RPC must block until the context expires. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + + r.UpdateState(resolver.State{Addresses: addrs}) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil { + t.Fatal(err) + } + return cc, r, backends +} + +// TestRoundRobin_Basic tests the most basic scenario for round_robin. It brings +// up a bunch of backends and verifies that RPCs are getting round robin-ed +// across these backends. +func (s) TestRoundRobin_Basic(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testRoundRobinBasic(ctx, t) +} + +// TestRoundRobin_AddressesRemoved tests the scenario where a bunch of backends +// are brought up, and round_robin is configured as the LB policy and RPCs are +// being correctly round robin-ed across these backends. We then send a resolver +// update with no addresses and verify that the channel enters TransientFailure +// and RPCs fail with an expected error message. +func (s) TestRoundRobin_AddressesRemoved(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc, r, _ := testRoundRobinBasic(ctx, t) + + // Send a resolver update with no addresses. This should push the channel into + // TransientFailure. + r.UpdateState(resolver.State{Endpoints: []resolver.Endpoint{}}) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + const msgWant = "no children to pick from" + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); !strings.Contains(status.Convert(err).Message(), msgWant) { + t.Fatalf("EmptyCall() = %v, want Contains(Message(), %q)", err, msgWant) + } +} + +// TestRoundRobin_NewAddressWhileBlocking tests the case where round_robin is +// configured on a channel, things are working as expected and then a resolver +// updates removes all addresses. An RPC attempted at this point in time will be +// blocked because there are no valid ¡ds. This test verifies that when new +// backends are added, the RPC is able to complete. +func (s) TestRoundRobin_NewAddressWhileBlocking(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc, r, backends := testRoundRobinBasic(ctx, t) + + // Send a resolver update with no addresses. This should push the channel into + // TransientFailure. + r.UpdateState(resolver.State{Addresses: []resolver.Address{}}) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + client := testgrpc.NewTestServiceClient(cc) + doneCh := make(chan struct{}) + go func() { + // The channel is currently in TransientFailure and this RPC will block + // until the channel becomes Ready, which will only happen when we push a + // resolver update with a valid backend address. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Errorf("EmptyCall() = %v, want ", err) + } + close(doneCh) + }() + + // Make sure that there is one pending RPC on the ClientConn before attempting + // to push new addresses through the name resolver. If we don't do this, the + // resolver update can happen before the above goroutine gets to make the RPC. + for { + if err := ctx.Err(); err != nil { + t.Fatal(err) + } + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + t.Fatalf("there should only be one top channel, not %d", len(tcs)) + } + started := tcs[0].ChannelMetrics.CallsStarted.Load() + completed := tcs[0].ChannelMetrics.CallsSucceeded.Load() + tcs[0].ChannelMetrics.CallsFailed.Load() + if (started - completed) == 1 { + break + } + time.Sleep(defaultTestShortTimeout) + } + + // Send a resolver update with a valid backend to push the channel to Ready + // and unblock the above RPC. + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backends[0].Address}}}) + + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for blocked RPC to complete") + case <-doneCh: + } +} + +// TestRoundRobin_OneServerDown tests the scenario where a channel is configured +// to round robin across a set of backends, and things are working correctly. +// One backend goes down. The test verifies that going forward, RPCs are round +// robin-ed across the remaining set of backends. +func (s) TestRoundRobin_OneServerDown(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc, _, backends := testRoundRobinBasic(ctx, t) + + // Stop one backend. RPCs should round robin across the remaining backends. + backends[len(backends)-1].Stop() + + addrs := make([]resolver.Address, len(backends)-1) + for i := 0; i < len(backends)-1; i++ { + addrs[i] = resolver.Address{Addr: backends[i].Address} + } + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil { + t.Fatalf("RPCs are not being round robined across remaining servers: %v", err) + } +} + +// TestRoundRobin_AllServersDown tests the scenario where a channel is +// configured to round robin across a set of backends, and things are working +// correctly. Then, all backends go down. The test verifies that the channel +// moves to TransientFailure and failfast RPCs fail with Unavailable. +func (s) TestRoundRobin_AllServersDown(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc, _, backends := testRoundRobinBasic(ctx, t) + + // Stop all backends. + for _, b := range backends { + b.Stop() + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Failfast RPCs should fail with Unavailable. + client := testgrpc.NewTestServiceClient(cc) + + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable { + t.Fatalf("EmptyCall got err: %v; want Unavailable", err) + } +} + +// TestRoundRobin_UpdateAddressAttributes tests the scenario where the addresses +// returned by the resolver contain attributes. The test verifies that the +// attributes contained in the addresses show up as RPC metadata in the backend. +func (s) TestRoundRobin_UpdateAddressAttributes(t *testing.T) { + const ( + testMDKey = "test-md" + testMDValue = "test-md-value" + ) + r := manual.NewBuilderWithScheme("whatever") + + // Spin up a StubServer to serve as a backend. The implementation verifies + // that the expected metadata is received. + testMDChan := make(chan []string, 1) + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if ok { + select { + case testMDChan <- md[testMDKey]: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + + // Dial the backend with round_robin as the LB policy. + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(rrServiceConfig), + } + // Set an initial resolver update with no address attributes. + addr := resolver.Address{Addr: backend.Address} + r.InitialState(resolver.State{Addresses: []resolver.Address{addr}}) + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // Make an RPC and ensure it does not contain the metadata we are looking for. + client := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() = %v, want ", err) + } + select { + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for metadata received in RPC") + case md := <-testMDChan: + if len(md) != 0 { + t.Fatalf("received metadata %v, want nil", md) + } + } + + // Send a resolver update with address attributes. + addrWithAttributes := imetadata.Set(addr, metadata.Pairs(testMDKey, testMDValue)) + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrWithAttributes}}) + + // Make an RPC and ensure it contains the metadata we are looking for. The + // resolver update isn't processed synchronously, so we wait some time before + // failing if some RPCs do not contain it. +Done: + for { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() = %v, want ", err) + } + select { + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for metadata received in RPC") + case md := <-testMDChan: + if len(md) == 1 && md[0] == testMDValue { + break Done + } + } + time.Sleep(defaultTestShortTimeout) + } +} diff --git a/test/server_test.go b/test/server_test.go new file mode 100644 index 0000000..d475754 --- /dev/null +++ b/test/server_test.go @@ -0,0 +1,587 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "io" + "runtime/pprof" + "sync/atomic" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type ctxKey string + +// TestServerReturningContextError verifies that if a context error is returned +// by the service handler, the status will have the correct status code, not +// Unknown. +func (s) TestServerReturningContextError(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return nil, context.DeadlineExceeded + }, + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + return context.DeadlineExceeded + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}) + if s, ok := status.FromError(err); !ok || s.Code() != codes.DeadlineExceeded { + t.Fatalf("ss.Client.EmptyCall() got error %v; want ", err) + } + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("unexpected error starting the stream: %v", err) + } + _, err = stream.Recv() + if s, ok := status.FromError(err); !ok || s.Code() != codes.DeadlineExceeded { + t.Fatalf("ss.Client.FullDuplexCall().Recv() got error %v; want ", err) + } + +} + +func pprofCtxCollectLabels(ctx context.Context) map[string]string { + seenLabels := map[string]string{} + pprof.ForLabels(ctx, func(k, val string) bool { + seenLabels[k] = val + return true + }) + return seenLabels +} + +// TestServerSetGoroutineLabelsInContext verifies that when enabled, the +// grpc.method runtime/pprof goroutine label gets set in the context that's +// passed to the handlers. +func (s) TestServerSetGoroutineLabelsInContext(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.LabelServerGoroutines, envconfig.GoroutineLabelServerMethod) + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + ctxLabels := pprofCtxCollectLabels(ctx) + if val, ok := ctxLabels["grpc.method"]; !ok { + t.Errorf("missing \"grpc.method\" label; found labels: %v", ctxLabels) + } else if wantVal := "/grpc.testing.TestService/EmptyCall"; val != wantVal { + t.Errorf("unexpected value for \"grpc.method\" label %q; want %q", ctxLabels["grpc.method"], wantVal) + } + return &testpb.Empty{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + ctxLabels := pprofCtxCollectLabels(stream.Context()) + if val, ok := ctxLabels["grpc.method"]; !ok { + t.Errorf("missing \"grpc.method\" label; found labels: %v", ctxLabels) + } else if wantVal := "/grpc.testing.TestService/FullDuplexCall"; val != wantVal { + t.Errorf("unexpected value for \"grpc.method\" label %q; want %q", ctxLabels["grpc.method"], wantVal) + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall() got error %v; want OK", err) + } + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("unexpected error starting the stream: %v", err) + } + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("ss.Client.FullDuplexCall().Recv() got error %v; want io.EOF", err) + } +} + +// TestServerSetGoroutineLabelsInContextEnvVarDisabled verifies that when +// disable, the grpc.method runtime/pprof goroutine label does _not_ get set in +// the context that's passed to the handlers. +func (s) TestServerSetGoroutineLabelsInContextEnvVarDisabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.LabelServerGoroutines, 0) + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + ctxLabels := pprofCtxCollectLabels(ctx) + if val, ok := ctxLabels["grpc.method"]; ok { + t.Errorf("\"grpc.method\" label set with value %q; found labels: %v", val, ctxLabels) + } + return &testpb.Empty{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + ctxLabels := pprofCtxCollectLabels(stream.Context()) + if val, ok := ctxLabels["grpc.method"]; ok { + t.Errorf("\"grpc.method\" label set with value %q; found labels: %v", val, ctxLabels) + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall() got error %v; want OK", err) + } + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("unexpected error starting the stream: %v", err) + } + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("ss.Client.FullDuplexCall().Recv() got error %v; want io.EOF", err) + } +} + +func (s) TestChainUnaryServerInterceptor(t *testing.T) { + var ( + firstIntKey = ctxKey("firstIntKey") + secondIntKey = ctxKey("secondIntKey") + ) + + firstInt := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + if ctx.Value(firstIntKey) != nil { + return nil, status.Errorf(codes.Internal, "first interceptor should not have %v in context", firstIntKey) + } + if ctx.Value(secondIntKey) != nil { + return nil, status.Errorf(codes.Internal, "first interceptor should not have %v in context", secondIntKey) + } + + firstCtx := context.WithValue(ctx, firstIntKey, 0) + resp, err := handler(firstCtx, req) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to handle request at firstInt") + } + + simpleResp, ok := resp.(*testpb.SimpleResponse) + if !ok { + return nil, status.Errorf(codes.Internal, "failed to get *testpb.SimpleResponse at firstInt") + } + return &testpb.SimpleResponse{ + Payload: &testpb.Payload{ + Type: simpleResp.GetPayload().GetType(), + Body: append(simpleResp.GetPayload().GetBody(), '1'), + }, + }, nil + } + + secondInt := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + if ctx.Value(firstIntKey) == nil { + return nil, status.Errorf(codes.Internal, "second interceptor should have %v in context", firstIntKey) + } + if ctx.Value(secondIntKey) != nil { + return nil, status.Errorf(codes.Internal, "second interceptor should not have %v in context", secondIntKey) + } + + secondCtx := context.WithValue(ctx, secondIntKey, 1) + resp, err := handler(secondCtx, req) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to handle request at secondInt") + } + + simpleResp, ok := resp.(*testpb.SimpleResponse) + if !ok { + return nil, status.Errorf(codes.Internal, "failed to get *testpb.SimpleResponse at secondInt") + } + return &testpb.SimpleResponse{ + Payload: &testpb.Payload{ + Type: simpleResp.GetPayload().GetType(), + Body: append(simpleResp.GetPayload().GetBody(), '2'), + }, + }, nil + } + + lastInt := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + if ctx.Value(firstIntKey) == nil { + return nil, status.Errorf(codes.Internal, "last interceptor should have %v in context", firstIntKey) + } + if ctx.Value(secondIntKey) == nil { + return nil, status.Errorf(codes.Internal, "last interceptor should not have %v in context", secondIntKey) + } + + resp, err := handler(ctx, req) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to handle request at lastInt at lastInt") + } + + simpleResp, ok := resp.(*testpb.SimpleResponse) + if !ok { + return nil, status.Errorf(codes.Internal, "failed to get *testpb.SimpleResponse at lastInt") + } + return &testpb.SimpleResponse{ + Payload: &testpb.Payload{ + Type: simpleResp.GetPayload().GetType(), + Body: append(simpleResp.GetPayload().GetBody(), '3'), + }, + }, nil + } + + sopts := []grpc.ServerOption{ + grpc.ChainUnaryInterceptor(firstInt, secondInt, lastInt), + } + + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 0) + if err != nil { + return nil, status.Errorf(codes.Aborted, "failed to make payload: %v", err) + } + + return &testpb.SimpleResponse{ + Payload: payload, + }, nil + }, + } + if err := ss.Start(sopts); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if s, ok := status.FromError(err); !ok || s.Code() != codes.OK { + t.Fatalf("ss.Client.UnaryCall(ctx, _) = %v, %v; want nil, ", resp, err) + } + + respBytes := resp.Payload.GetBody() + if string(respBytes) != "321" { + t.Fatalf("invalid response: want=%s, but got=%s", "321", resp) + } +} + +func (s) TestChainOnBaseUnaryServerInterceptor(t *testing.T) { + baseIntKey := ctxKey("baseIntKey") + + baseInt := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + if ctx.Value(baseIntKey) != nil { + return nil, status.Errorf(codes.Internal, "base interceptor should not have %v in context", baseIntKey) + } + + baseCtx := context.WithValue(ctx, baseIntKey, 1) + return handler(baseCtx, req) + } + + chainInt := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + if ctx.Value(baseIntKey) == nil { + return nil, status.Errorf(codes.Internal, "chain interceptor should have %v in context", baseIntKey) + } + + return handler(ctx, req) + } + + sopts := []grpc.ServerOption{ + grpc.UnaryInterceptor(baseInt), + grpc.ChainUnaryInterceptor(chainInt), + } + + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(sopts); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}) + if s, ok := status.FromError(err); !ok || s.Code() != codes.OK { + t.Fatalf("ss.Client.EmptyCall(ctx, _) = %v, %v; want nil, ", resp, err) + } +} + +func (s) TestChainStreamServerInterceptor(t *testing.T) { + callCounts := make([]atomic.Int32, 4) + + firstInt := func(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + if callCounts[0].Load() != 0 { + return status.Errorf(codes.Internal, "callCounts[0] should be 0, but got=%d", callCounts[0].Load()) + } + if callCounts[1].Load() != 0 { + return status.Errorf(codes.Internal, "callCounts[1] should be 0, but got=%d", callCounts[1].Load()) + } + if callCounts[2].Load() != 0 { + return status.Errorf(codes.Internal, "callCounts[2] should be 0, but got=%d", callCounts[2].Load()) + } + if callCounts[3].Load() != 0 { + return status.Errorf(codes.Internal, "callCounts[3] should be 0, but got=%d", callCounts[3].Load()) + } + callCounts[0].Add(1) + return handler(srv, stream) + } + + secondInt := func(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + if callCounts[0].Load() != 1 { + return status.Errorf(codes.Internal, "callCounts[0] should be 1, but got=%d", callCounts[0].Load()) + } + if callCounts[1].Load() != 0 { + return status.Errorf(codes.Internal, "callCounts[1] should be 0, but got=%d", callCounts[1].Load()) + } + if callCounts[2].Load() != 0 { + return status.Errorf(codes.Internal, "callCounts[2] should be 0, but got=%d", callCounts[2].Load()) + } + if callCounts[3].Load() != 0 { + return status.Errorf(codes.Internal, "callCounts[3] should be 0, but got=%d", callCounts[3].Load()) + } + callCounts[1].Add(1) + return handler(srv, stream) + } + + lastInt := func(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + if callCounts[0].Load() != 1 { + return status.Errorf(codes.Internal, "callCounts[0] should be 1, but got=%d", callCounts[0].Load()) + } + if callCounts[1].Load() != 1 { + return status.Errorf(codes.Internal, "callCounts[1] should be 1, but got=%d", callCounts[1].Load()) + } + if callCounts[2].Load() != 0 { + return status.Errorf(codes.Internal, "callCounts[2] should be 0, but got=%d", callCounts[2].Load()) + } + if callCounts[3].Load() != 0 { + return status.Errorf(codes.Internal, "callCounts[3] should be 0, but got=%d", callCounts[3].Load()) + } + callCounts[2].Add(1) + return handler(srv, stream) + } + + sopts := []grpc.ServerOption{ + grpc.ChainStreamInterceptor(firstInt, secondInt, lastInt), + } + + ss := &stubserver.StubServer{ + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + if callCounts[0].Load() != 1 { + return status.Errorf(codes.Internal, "callCounts[0] should be 1, but got=%d", callCounts[0].Load()) + } + if callCounts[1].Load() != 1 { + return status.Errorf(codes.Internal, "callCounts[1] should be 1, but got=%d", callCounts[1].Load()) + } + if callCounts[2].Load() != 1 { + return status.Errorf(codes.Internal, "callCounts[2] should be 0, but got=%d", callCounts[2].Load()) + } + if callCounts[3].Load() != 0 { + return status.Errorf(codes.Internal, "callCounts[3] should be 0, but got=%d", callCounts[3].Load()) + } + callCounts[3].Add(1) + return nil + }, + } + if err := ss.Start(sopts); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("failed to FullDuplexCall: %v", err) + } + + _, err = stream.Recv() + if err != io.EOF { + t.Fatalf("failed to recv from stream: %v", err) + } + + if callCounts[3].Load() != 1 { + t.Fatalf("callCounts[3] should be 1, but got=%d", callCounts[3].Load()) + } +} + +// Test verifies that only unary interceptors are invoked for unary RPCs and +// only streaming interceptors are invoked for streaming RPCs. +func (s) TestInterceptorSegregation(t *testing.T) { + var unaryCalled, streamCalled atomic.Bool + + unaryInt := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + unaryCalled.Store(true) + return handler(ctx, req) + } + streamInt := func(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + streamCalled.Store(true) + return handler(srv, ss) + } + sopts := []grpc.ServerOption{grpc.UnaryInterceptor(unaryInt), grpc.StreamInterceptor(streamInt)} + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + return nil + }, + } + if err := ss.Start(sopts); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + // Make a unary RPC and ensure that only the unary interceptor was invoked. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("ss.Client.UnaryCall failed: %v", err) + } + if !unaryCalled.Load() { + t.Error("Unary interceptor was not called for Unary RPC") + } + if streamCalled.Load() { + t.Error("Stream interceptor was called for Unary RPC") + } + + // Make a streaming RPC and ensure that only the streaming interceptor was + // invoked. + unaryCalled.Store(false) + streamCalled.Store(false) + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %v", err) + } + if _, err := stream.Recv(); err != io.EOF { + t.Fatalf("stream.Recv() returned %v, want io.EOF", err) + } + if unaryCalled.Load() { + t.Error("Unary interceptor was called for Streaming RPC") + } + if !streamCalled.Load() { + t.Error("Stream interceptor was not called for Streaming RPC") + } +} + +type wrappedTestStreamKey struct{} + +type wrappedTestStream struct { + grpc.ServerStream +} + +func (w *wrappedTestStream) Context() context.Context { + return context.WithValue(w.ServerStream.Context(), wrappedTestStreamKey{}, w) +} + +// Test verifies that an internal xDS filter wrapper option configured on the +// server gets invoked and can wrap the ServerStream for both Unary and +// Streaming RPCs. +func (s) TestXDSFilterWrapperOption(t *testing.T) { + var wrapperCallCount atomic.Int32 + wrapper := func(ss grpc.ServerStream) (grpc.ServerStream, error) { + wrapperCallCount.Add(1) + return &wrappedTestStream{ServerStream: ss}, nil + } + + opt := internal.XDSFilterWrapperOption.(func(func(grpc.ServerStream) (grpc.ServerStream, error)) grpc.ServerOption)(wrapper) + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + if _, ok := ctx.Value(wrappedTestStreamKey{}).(*wrappedTestStream); !ok { + return nil, status.Errorf(codes.Internal, "context value is %T, want *wrappedTestStream", ctx.Value(wrappedTestStreamKey{})) + } + return &testpb.Empty{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, ok := stream.Context().Value(wrappedTestStreamKey{}).(*wrappedTestStream); !ok { + return status.Errorf(codes.Internal, "context value is %T, want *wrappedTestStream", stream.Context().Value(wrappedTestStreamKey{})) + } + return nil + }, + } + if err := ss.Start([]grpc.ServerOption{opt}); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } + if got := wrapperCallCount.Load(); got != 1 { + t.Fatalf("XDSFilterWrapperOption callback call count for Unary RPC got %d, want 1", got) + } + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall failed: %v", err) + } + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("Recv failed: %v", err) + } + if got := wrapperCallCount.Load(); got != 2 { + t.Fatalf("XDSFilterWrapperOption callback call count for Streaming RPC got %d, want 2", got) + } +} + +// Test verifies that if an internal xDS filter wrapper returns an error, +// the RPC is rejected early with that status error before executing +// handlers. +func (s) TestXDSFilterWrapperOption_EarlyRejection(t *testing.T) { + wrapper := func(grpc.ServerStream) (grpc.ServerStream, error) { + return nil, status.Error(codes.PermissionDenied, "early rejection by internal wrapper") + } + + opt := internal.XDSFilterWrapperOption.(func(func(grpc.ServerStream) (grpc.ServerStream, error)) grpc.ServerOption)(wrapper) + + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error { + return nil + }, + } + if err := ss.Start([]grpc.ServerOption{opt}); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.PermissionDenied { + t.Fatalf("EmptyCall failed with error %v; want PermissionDenied", err) + } + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall failed: %v", err) + } + if _, err = stream.Recv(); status.Code(err) != codes.PermissionDenied { + t.Fatalf("stream.Recv() got error %v; want PermissionDenied", err) + } +} diff --git a/test/servertester.go b/test/servertester.go new file mode 100644 index 0000000..c70a4fa --- /dev/null +++ b/test/servertester.go @@ -0,0 +1,320 @@ +/* + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package test contains tests. +package test + +import ( + "bytes" + "errors" + "io" + "strings" + "testing" + "time" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" +) + +// This is a subset of http2's serverTester type. +// +// serverTester wraps a io.ReadWriter (acting like the underlying +// network connection) and provides utility methods to read and write +// http2 frames. +// +// NOTE(bradfitz): this could eventually be exported somewhere. Others +// have asked for it too. For now I'm still experimenting with the +// API and don't feel like maintaining a stable testing API. + +type serverTester struct { + cc io.ReadWriteCloser // client conn + t testing.TB + fr *http2.Framer + + // writing headers: + headerBuf bytes.Buffer + hpackEnc *hpack.Encoder + + // reading frames: + frc chan http2.Frame + frErrc chan error +} + +func newServerTesterFromConn(t testing.TB, cc io.ReadWriteCloser) *serverTester { + st := &serverTester{ + t: t, + cc: cc, + frc: make(chan http2.Frame, 1), + frErrc: make(chan error, 1), + } + st.hpackEnc = hpack.NewEncoder(&st.headerBuf) + st.fr = http2.NewFramer(cc, cc) + st.fr.ReadMetaHeaders = hpack.NewDecoder(4096 /*initialHeaderTableSize*/, nil) + + return st +} + +func (st *serverTester) readFrame() (http2.Frame, error) { + go func() { + fr, err := st.fr.ReadFrame() + if err != nil { + st.frErrc <- err + } else { + st.frc <- fr + } + }() + t := time.NewTimer(2 * time.Second) + defer t.Stop() + select { + case f := <-st.frc: + return f, nil + case err := <-st.frErrc: + return nil, err + case <-t.C: + return nil, errors.New("timeout waiting for frame") + } +} + +// greet initiates the client's HTTP/2 connection into a state where +// frames may be sent. +func (st *serverTester) greet() { + st.greetWithSettings() +} + +// greetWithSettings initiates the client's HTTP/2 connection with custom settings. +func (st *serverTester) greetWithSettings(settings ...http2.Setting) { + st.writePreface() + if len(settings) > 0 { + if err := st.fr.WriteSettings(settings...); err != nil { + st.t.Fatalf("Error writing initial SETTINGS frame from client to server: %v", err) + } + } else { + st.writeInitialSettings() + } + st.wantSettings() + st.writeSettingsAck() + for { + f, err := st.readFrame() + if err != nil { + st.t.Fatal(err) + } + switch f := f.(type) { + case *http2.WindowUpdateFrame: + // grpc's transport/http2_server sends this + // before the settings ack. The Go http2 + // server uses a setting instead. + case *http2.SettingsFrame: + if f.IsAck() { + return + } + st.t.Fatalf("during greet, got non-ACK settings frame") + default: + st.t.Fatalf("during greet, unexpected frame type %T", f) + } + } +} + +func (st *serverTester) writePreface() { + n, err := st.cc.Write([]byte(http2.ClientPreface)) + if err != nil { + st.t.Fatalf("Error writing client preface: %v", err) + } + if n != len(http2.ClientPreface) { + st.t.Fatalf("Writing client preface, wrote %d bytes; want %d", n, len(http2.ClientPreface)) + } +} + +func (st *serverTester) writeInitialSettings() { + if err := st.fr.WriteSettings(); err != nil { + st.t.Fatalf("Error writing initial SETTINGS frame from client to server: %v", err) + } +} + +func (st *serverTester) writeSettingsAck() { + if err := st.fr.WriteSettingsAck(); err != nil { + st.t.Fatalf("Error writing ACK of server's SETTINGS: %v", err) + } +} + +func (st *serverTester) wantGoAway(errCode http2.ErrCode) *http2.GoAwayFrame { + f, err := st.readFrame() + if err != nil { + st.t.Fatalf("Error while expecting an RST frame: %v", err) + } + gaf, ok := f.(*http2.GoAwayFrame) + if !ok { + st.t.Fatalf("got a %T; want *http2.GoAwayFrame", f) + } + if gaf.ErrCode != errCode { + st.t.Fatalf("expected GOAWAY error code '%v', got '%v'", errCode.String(), gaf.ErrCode.String()) + } + return gaf +} + +func (st *serverTester) wantPing() *http2.PingFrame { + f, err := st.readFrame() + if err != nil { + st.t.Fatalf("Error while expecting an RST frame: %v", err) + } + pf, ok := f.(*http2.PingFrame) + if !ok { + st.t.Fatalf("got a %T; want *http2.GoAwayFrame", f) + } + return pf +} + +func (st *serverTester) wantRSTStream(errCode http2.ErrCode) *http2.RSTStreamFrame { + f, err := st.readFrame() + if err != nil { + st.t.Fatalf("Error while expecting an RST frame: %v", err) + } + rf, ok := f.(*http2.RSTStreamFrame) + if !ok { + st.t.Fatalf("got a %T; want *http2.RSTStreamFrame", f) + } + if rf.ErrCode != errCode { + st.t.Fatalf("expected RST error code '%v', got '%v'", errCode.String(), rf.ErrCode.String()) + } + return rf +} + +func (st *serverTester) wantSettings() *http2.SettingsFrame { + f, err := st.readFrame() + if err != nil { + st.t.Fatalf("Error while expecting a SETTINGS frame: %v", err) + } + sf, ok := f.(*http2.SettingsFrame) + if !ok { + st.t.Fatalf("got a %T; want *SettingsFrame", f) + } + return sf +} + +// wait for any activity from the server +func (st *serverTester) wantAnyFrame() http2.Frame { + f, err := st.fr.ReadFrame() + if err != nil { + st.t.Fatal(err) + } + return f +} + +func (st *serverTester) encodeHeaderField(k, v string) { + err := st.hpackEnc.WriteField(hpack.HeaderField{Name: k, Value: v}) + if err != nil { + st.t.Fatalf("HPACK encoding error for %q/%q: %v", k, v, err) + } +} + +// encodeHeader encodes headers and returns their HPACK bytes. headers +// must contain an even number of key/value pairs. There may be +// multiple pairs for keys (e.g. "cookie"). The :method, :path, and +// :scheme headers default to GET, / and https. +func (st *serverTester) encodeHeader(headers ...string) []byte { + if len(headers)%2 == 1 { + panic("odd number of kv args") + } + + st.headerBuf.Reset() + + if len(headers) == 0 { + // Fast path, mostly for benchmarks, so test code doesn't pollute + // profiles when we're looking to improve server allocations. + st.encodeHeaderField(":method", "GET") + st.encodeHeaderField(":path", "/") + st.encodeHeaderField(":scheme", "https") + return st.headerBuf.Bytes() + } + + if len(headers) == 2 && headers[0] == ":method" { + // Another fast path for benchmarks. + st.encodeHeaderField(":method", headers[1]) + st.encodeHeaderField(":path", "/") + st.encodeHeaderField(":scheme", "https") + return st.headerBuf.Bytes() + } + + pseudoCount := map[string]int{} + keys := []string{":method", ":path", ":scheme"} + vals := map[string][]string{ + ":method": {"GET"}, + ":path": {"/"}, + ":scheme": {"https"}, + } + for len(headers) > 0 { + k, v := headers[0], headers[1] + headers = headers[2:] + if _, ok := vals[k]; !ok { + keys = append(keys, k) + } + if strings.HasPrefix(k, ":") { + pseudoCount[k]++ + if pseudoCount[k] == 1 { + vals[k] = []string{v} + } else { + // Allows testing of invalid headers w/ dup pseudo fields. + vals[k] = append(vals[k], v) + } + } else { + vals[k] = append(vals[k], v) + } + } + for _, k := range keys { + for _, v := range vals[k] { + st.encodeHeaderField(k, v) + } + } + return st.headerBuf.Bytes() +} + +func (st *serverTester) writeHeadersGRPC(streamID uint32, path string, endStream bool) { + st.writeHeaders(http2.HeadersFrameParam{ + StreamID: streamID, + BlockFragment: st.encodeHeader( + ":method", "POST", + ":path", path, + ":authority", "localhost", + "content-type", "application/grpc", + "te", "trailers", + ), + EndStream: endStream, + EndHeaders: true, + }) +} + +func (st *serverTester) writeHeaders(p http2.HeadersFrameParam) { + if err := st.fr.WriteHeaders(p); err != nil { + st.t.Fatalf("Error writing HEADERS: %v", err) + } +} + +func (st *serverTester) writeData(streamID uint32, endStream bool, data []byte) { + if err := st.fr.WriteData(streamID, endStream, data); err != nil { + st.t.Fatalf("Error writing DATA: %v", err) + } +} + +func (st *serverTester) writeRSTStream(streamID uint32, code http2.ErrCode) { + if err := st.fr.WriteRSTStream(streamID, code); err != nil { + st.t.Fatalf("Error writing RST_STREAM: %v", err) + } +} + +func (st *serverTester) writePing(ack bool, data [8]byte) { + if err := st.fr.WritePing(ack, data); err != nil { + st.t.Fatalf("Error writing PING: %v", err) + } +} diff --git a/test/stats_test.go b/test/stats_test.go new file mode 100644 index 0000000..28156ed --- /dev/null +++ b/test/stats_test.go @@ -0,0 +1,142 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "net" + "sync" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/interop" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/stats" +) + +// TestPeerForClientStatsHandler configures a stats handler that +// verifies that peer is sent all stats handler callouts instead +// of Begin and PickerUpdated. +func (s) TestPeerForClientStatsHandler(t *testing.T) { + psh := &peerStatsHandler{} + + // Stats callouts & peer object population. + // Note: + // * Begin stats lack peer info (RPC starts pre-resolution). + // * PickerUpdated: no peer info (picker lacks transport details). + expectedCallouts := map[stats.RPCStats]bool{ + &stats.OutPayload{}: true, + &stats.InHeader{}: true, + &stats.OutHeader{}: true, + &stats.InTrailer{}: true, + &stats.OutTrailer{}: true, + &stats.End{}: true, + &stats.Begin{}: false, + &stats.DelayedPickComplete{}: false, + } + + // Start server. + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + ss := &stubserver.StubServer{ + Listener: l, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + S: grpc.NewServer(), + } + stubserver.StartTestService(t, ss) + defer ss.S.Stop() + + // Create client with stats handler and do some calls. + cc, err := grpc.NewClient( + l.Addr().String(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithStatsHandler(psh)) + if err != nil { + t.Fatal(err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + interop.DoEmptyUnaryCall(ctx, client) + + psh.mu.Lock() + pshArgs := psh.args + psh.mu.Unlock() + + // Fetch the total unique stats handlers with peer != nil + uniqueStatsTypes := make(map[string]struct{}) + for _, callbackArgs := range pshArgs { + key := fmt.Sprintf("%T", callbackArgs.rpcStats) + if _, exists := uniqueStatsTypes[key]; exists { + continue + } + uniqueStatsTypes[fmt.Sprintf("%T", callbackArgs.rpcStats)] = struct{}{} + } + if len(uniqueStatsTypes) != len(expectedCallouts) { + t.Errorf("Unexpected number of stats handler callouts. Got %v, want %v", len(uniqueStatsTypes), len(expectedCallouts)) + } + + for _, callbackArgs := range pshArgs { + expectedPeer, found := expectedCallouts[callbackArgs.rpcStats] + // In case expectation is set to false and still we got the peer, + // then it's good to have it. So no need to assert those conditions. + if found && expectedPeer && callbackArgs.peer != nil { + continue + } else if expectedPeer && callbackArgs.peer == nil { + t.Errorf("peer not populated for: %T", callbackArgs.rpcStats) + } + } +} + +type peerStats struct { + rpcStats stats.RPCStats + peer *peer.Peer +} + +type peerStatsHandler struct { + args []peerStats + mu sync.Mutex +} + +func (h *peerStatsHandler) TagRPC(ctx context.Context, _ *stats.RPCTagInfo) context.Context { + return ctx +} + +func (h *peerStatsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { + p, _ := peer.FromContext(ctx) + h.mu.Lock() + defer h.mu.Unlock() + h.args = append(h.args, peerStats{rs, p}) +} + +func (h *peerStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +func (h *peerStatsHandler) HandleConn(context.Context, stats.ConnStats) {} diff --git a/test/stream_cleanup_test.go b/test/stream_cleanup_test.go new file mode 100644 index 0000000..600fb1f --- /dev/null +++ b/test/stream_cleanup_test.go @@ -0,0 +1,139 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "io" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestStreamCleanup(t *testing.T) { + const initialWindowSize uint = 70 * 1024 // Must be higher than default 64K, ignored otherwise + const bodySize = 2 * initialWindowSize // Something that is not going to fit in a single window + const callRecvMsgSize uint = 1 // The maximum message size the client can receive + + ss := &stubserver.StubServer{ + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{Payload: &testpb.Payload{ + Body: make([]byte, bodySize), + }}, nil + }, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(callRecvMsgSize))), grpc.WithInitialWindowSize(int32(initialWindowSize))); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.ResourceExhausted { + t.Fatalf("should fail with ResourceExhausted, message's body size: %v, maximum message size the client can receive: %v", bodySize, callRecvMsgSize) + } + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("should succeed, err: %v", err) + } +} + +func (s) TestStreamCleanupAfterSendStatus(t *testing.T) { + const initialWindowSize uint = 70 * 1024 // Must be higher than default 64K, ignored otherwise + const bodySize = 2 * initialWindowSize // Something that is not going to fit in a single window + + serverReturnedStatus := make(chan struct{}) + + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + defer func() { + close(serverReturnedStatus) + }() + return stream.Send(&testpb.StreamingOutputCallResponse{ + Payload: &testpb.Payload{ + Body: make([]byte, bodySize), + }, + }) + }, + } + if err := ss.Start(nil, grpc.WithInitialWindowSize(int32(initialWindowSize))); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + // This test makes sure we don't delete stream from server transport's + // activeStreams list too aggressively. + + // 1. Make a long living stream RPC. So server's activeStream list is not + // empty. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall= _, %v; want _, ", err) + } + + // 2. Wait for service handler to return status. + // + // This will trigger a stream cleanup code, which will eventually remove + // this stream from activeStream. + // + // But the stream removal won't happen because it's supposed to be done + // after the status is sent by loopyWriter, and the status send is blocked + // by flow control. + <-serverReturnedStatus + + // 3. GracefulStop (besides sending goaway) checks the number of + // activeStreams. + // + // It will close the connection if there's no active streams. This won't + // happen because of the pending stream. But if there's a bug in stream + // cleanup that causes stream to be removed too aggressively, the connection + // will be closed and the stream will be broken. + gracefulStopDone := make(chan struct{}) + go func() { + defer close(gracefulStopDone) + ss.S.GracefulStop() + }() + + // 4. Make sure the stream is not broken. + if _, err := stream.Recv(); err != nil { + t.Fatalf("stream.Recv() = _, %v, want _, ", err) + } + if _, err := stream.Recv(); err != io.EOF { + t.Fatalf("stream.Recv() = _, %v, want _, io.EOF", err) + } + + timer := time.NewTimer(time.Second) + select { + case <-gracefulStopDone: + timer.Stop() + case <-timer.C: + t.Fatalf("s.GracefulStop() didn't finish within 1 second after the last RPC") + } +} diff --git a/test/subconn_test.go b/test/subconn_test.go new file mode 100644 index 0000000..8e3d0e7 --- /dev/null +++ b/test/subconn_test.go @@ -0,0 +1,126 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "errors" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/resolver" +) + +type tsccPicker struct { + sc balancer.SubConn +} + +func (p *tsccPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + return balancer.PickResult{SubConn: p.sc}, nil +} + +// TestSubConnEmpty tests that removing all addresses from a SubConn and then +// re-adding them does not cause a panic and properly reconnects. +func (s) TestSubConnEmpty(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // sc is the one SubConn used throughout the test. Created on demand and + // re-used on every update. + var sc balancer.SubConn + + // Simple custom balancer that sets the address list to empty if the + // resolver produces no addresses. Pickfirst, by default, will remove the + // SubConn in this case instead. + bal := stub.BalancerFuncs{ + UpdateClientConnState: func(d *stub.BalancerData, ccs balancer.ClientConnState) error { + if sc == nil { + var err error + sc, err = d.ClientConn.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + switch state.ConnectivityState { + case connectivity.Ready: + d.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &tsccPicker{sc: sc}, + }) + case connectivity.TransientFailure: + d.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(fmt.Errorf("error connecting: %v", state.ConnectionError)), + }) + } + }, + }) + if err != nil { + t.Errorf("error creating initial subconn: %v", err) + } + } else { + d.ClientConn.UpdateAddresses(sc, ccs.ResolverState.Addresses) + } + sc.Connect() + + if len(ccs.ResolverState.Addresses) == 0 { + d.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(errors.New("no addresses")), + }) + } else { + d.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &tsccPicker{sc: sc}, + }) + } + return nil + }, + } + stub.Register("tscc", bal) + + // Start the stub server with our stub balancer. + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil, grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"tscc":{}}]}`)); err != nil { + t.Fatalf("Error starting server: %v", err) + } + defer ss.Stop() + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } + + t.Log("Removing addresses from resolver and SubConn") + ss.R.UpdateState(resolver.State{Addresses: []resolver.Address{}}) + testutils.AwaitState(ctx, t, ss.CC, connectivity.TransientFailure) + + t.Log("Re-adding addresses to resolver and SubConn") + ss.R.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}}) + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } +} diff --git a/test/timeouts.go b/test/timeouts.go new file mode 100644 index 0000000..1c0c212 --- /dev/null +++ b/test/timeouts.go @@ -0,0 +1,29 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import "time" + +const ( + // Default timeout for tests in this package. + defaultTestTimeout = 10 * time.Second + // Default short timeout, to be used when waiting for events which are not + // expected to happen. + defaultTestShortTimeout = 100 * time.Millisecond +) diff --git a/test/tools/go.mod b/test/tools/go.mod new file mode 100644 index 0000000..90a1cae --- /dev/null +++ b/test/tools/go.mod @@ -0,0 +1,29 @@ +module google.golang.org/grpc/test/tools + +go 1.25.0 + +require ( + github.com/client9/misspell v0.3.4 + github.com/mgechev/revive v1.15.0 + golang.org/x/tools v0.49.0 + google.golang.org/protobuf v1.36.12 + honnef.co/go/tools v0.7.0 +) + +require ( + codeberg.org/chavacava/garif v0.2.1 // indirect + github.com/BurntSushi/toml v1.6.0 // indirect + github.com/fatih/color v1.19.0 // indirect + github.com/fatih/structtag v1.2.0 // indirect + github.com/hashicorp/go-version v1.9.0 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mgechev/dots v1.0.0 // indirect + github.com/spf13/afero v1.15.0 // indirect + golang.org/x/exp/typeparams v0.0.0-20260813180055-c1d0aacb2297 // indirect + golang.org/x/mod v0.40.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/telemetry v0.0.0-20260814151720-d8c169486af1 // indirect + golang.org/x/text v0.41.0 // indirect +) diff --git a/test/tools/go.sum b/test/tools/go.sum new file mode 100644 index 0000000..c25d481 --- /dev/null +++ b/test/tools/go.sum @@ -0,0 +1,44 @@ +codeberg.org/chavacava/garif v0.2.1 h1:K9oYxSlvlXrHXyW26Z4q61bTpJDo1wbvXcYKar/F/LM= +codeberg.org/chavacava/garif v0.2.1/go.mod h1:oHnDSmc0f9K1MeE+MQD/yjkiIB5Xsn5y3S9Dg96Xk84= +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= +github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= +github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mgechev/dots v1.0.0 h1:o+4OJ3OjWzgQHGJXKfJ8rbH4dqDugu5BiEy84nxg0k4= +github.com/mgechev/dots v1.0.0/go.mod h1:rykuMydC9t3wfkM+ccYH3U3ss03vZGg6h3hmOznXLH0= +github.com/mgechev/revive v1.15.0 h1:vJ0HzSBzfNyPbHKolgiFjHxLek9KUijhqh42yGoqZ8Q= +github.com/mgechev/revive v1.15.0/go.mod h1:LlAKO3QQe9OJ0pVZzI2GPa8CbXGZ/9lNpCGvK4T/a8A= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +golang.org/x/exp/typeparams v0.0.0-20260813180055-c1d0aacb2297 h1:5MlQaTIH1l83SH0C4Xd5VtW0muW6qnokTvh0uybX1Lw= +golang.org/x/exp/typeparams v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:PqrXSW65cXDZH0k4IeUbhmg/bcAZDbzNz3byBpKCsXo= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260814151720-d8c169486af1 h1:tw/dhR3C+0y5Y8sTyXtV5GzjFE9hWblSlUUjF6Xs+Ko= +golang.org/x/telemetry v0.0.0-20260814151720-d8c169486af1/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= +honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= diff --git a/test/tools/tools.go b/test/tools/tools.go new file mode 100644 index 0000000..2edd384 --- /dev/null +++ b/test/tools/tools.go @@ -0,0 +1,34 @@ +//go:build tools +// +build tools + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// This file is not intended to be compiled. Because some of these imports are +// not actual go packages, we use a build constraint at the top of this file to +// prevent tools from inspecting the imports. + +package tools + +import ( + _ "github.com/client9/misspell/cmd/misspell" + _ "github.com/mgechev/revive" + _ "golang.org/x/tools/cmd/goimports" + _ "google.golang.org/protobuf/cmd/protoc-gen-go" + _ "honnef.co/go/tools/cmd/staticcheck" +) diff --git a/test/tools/tools_vet.go b/test/tools/tools_vet.go new file mode 100644 index 0000000..06ab2fd --- /dev/null +++ b/test/tools/tools_vet.go @@ -0,0 +1,21 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package tools is used to pin specific versions of external tools in this +// module's go.mod that gRPC uses for internal testing. +package tools diff --git a/test/transport_test.go b/test/transport_test.go new file mode 100644 index 0000000..d082cd0 --- /dev/null +++ b/test/transport_test.go @@ -0,0 +1,355 @@ +/* +* +* Copyright 2023 gRPC authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ +package test + +import ( + "context" + "encoding/binary" + "io" + "net" + "sync" + "testing" + "time" + + "golang.org/x/net/http2" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// connWrapperWithCloseCh wraps a net.Conn and fires an event when closed. +type connWrapperWithCloseCh struct { + net.Conn + close *grpcsync.Event +} + +// Close closes the connection and sends a value on the close channel. +func (cw *connWrapperWithCloseCh) Close() error { + cw.close.Fire() + return cw.Conn.Close() +} + +// These custom creds are used for storing the connections made by the client. +// The closeCh in conn can be used to detect when conn is closed. +type transportRestartCheckCreds struct { + mu sync.Mutex + connections []*connWrapperWithCloseCh +} + +func (c *transportRestartCheckCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + return rawConn, nil, nil +} +func (c *transportRestartCheckCreds) ClientHandshake(_ context.Context, _ string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + c.mu.Lock() + defer c.mu.Unlock() + conn := &connWrapperWithCloseCh{Conn: rawConn, close: grpcsync.NewEvent()} + c.connections = append(c.connections, conn) + return conn, nil, nil +} +func (c *transportRestartCheckCreds) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{} +} +func (c *transportRestartCheckCreds) Clone() credentials.TransportCredentials { + return c +} +func (c *transportRestartCheckCreds) OverrideServerName(string) error { + return nil +} + +// Tests that the client transport drains and restarts when next stream ID exceeds +// MaxStreamID. This test also verifies that subsequent RPCs use a new client +// transport and the old transport is closed. +func (s) TestClientTransportRestartsAfterStreamIDExhausted(t *testing.T) { + // Set the transport's MaxStreamID to 4 to cause connection to drain after 2 RPCs. + originalMaxStreamID := transport.MaxStreamID + transport.MaxStreamID = 4 + defer func() { + transport.MaxStreamID = originalMaxStreamID + }() + + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil { + return status.Errorf(codes.Internal, "unexpected error receiving: %v", err) + } + if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil { + return status.Errorf(codes.Internal, "unexpected error sending: %v", err) + } + if recv, err := stream.Recv(); err != io.EOF { + return status.Errorf(codes.Internal, "Recv = %v, %v; want _, io.EOF", recv, err) + } + return nil + }, + } + + creds := &transportRestartCheckCreds{} + if err := ss.Start(nil, grpc.WithTransportCredentials(creds)); err != nil { + t.Fatalf("Starting stubServer: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + var streams []testgrpc.TestService_FullDuplexCallClient + + const numStreams = 3 + // expected number of conns when each stream is created i.e., 3rd stream is created + // on a new connection. + expectedNumConns := [numStreams]int{1, 1, 2} + + // Set up 3 streams. + for i := 0; i < numStreams; i++ { + s, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("Creating FullDuplex stream: %v", err) + } + streams = append(streams, s) + // Verify expected num of conns after each stream is created. + if len(creds.connections) != expectedNumConns[i] { + t.Fatalf("Got number of connections created: %v, want: %v", len(creds.connections), expectedNumConns[i]) + } + } + + // Verify all streams still work. + for i, stream := range streams { + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("Sending on stream %d: %v", i, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("Receiving on stream %d: %v", i, err) + } + } + + for i, stream := range streams { + if err := stream.CloseSend(); err != nil { + t.Fatalf("CloseSend() on stream %d: %v", i, err) + } + } + + // Verifying first connection was closed. + select { + case <-creds.connections[0].close.Done(): + case <-ctx.Done(): + t.Fatal("Timeout expired when waiting for first client transport to close") + } +} + +// Tests that an RST_STREAM frame that causes an io.ErrUnexpectedEOF while +// reading a gRPC message is correctly converted to a gRPC status with code +// CANCELLED. The test sends a data frame with a partial gRPC message, followed +// by an RST_STREAM frame with HTTP/2 code CANCELLED. The test asserts the +// client receives the correct status. +func (s) TestRSTDuringMessageRead(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatal(err) + } + defer lis.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%s) = %v", lis.Addr().String(), err) + } + defer cc.Close() + + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("lis.Accept() = %v", err) + return + } + defer conn.Close() + framer := http2.NewFramer(conn, conn) + + if _, err := io.ReadFull(conn, make([]byte, len(clientPreface))); err != nil { + t.Errorf("Error while reading client preface: %v", err) + return + } + if err := framer.WriteSettings(); err != nil { + t.Errorf("Error while writing settings: %v", err) + return + } + if err := framer.WriteSettingsAck(); err != nil { + t.Errorf("Error while writing settings: %v", err) + return + } + for ctx.Err() == nil { + frame, err := framer.ReadFrame() + if err != nil { + return + } + switch frame := frame.(type) { + case *http2.HeadersFrame: + // When the client creates a stream, write a partial gRPC + // message followed by an RST_STREAM. + const messageLen = 2048 + buf := make([]byte, messageLen/2) + // Write the gRPC message length header. + binary.BigEndian.PutUint32(buf[1:5], uint32(messageLen)) + if err := framer.WriteData(1, false, buf); err != nil { + return + } + framer.WriteRSTStream(1, http2.ErrCodeCancel) + default: + t.Logf("Server received frame: %v", frame) + } + } + }() + + // The server will send a partial gRPC message before cancelling the stream. + // The client should get a gRPC status with code CANCELLED. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Canceled { + t.Fatalf("client.EmptyCall() returned %v; want status with code %v", err, codes.Canceled) + } +} + +// Test verifies that a client-side cancellation correctly frees up resources on +// the server. The test setup is designed to simulate a scenario where a server +// is blocked from sending a large message due to a full client-side flow +// control window. The client-side cancellation of this blocked RPC then frees +// up the max concurrent streams quota on the server, allowing a new RPC to be +// created successfully. +func (s) TestCancelWhileServerWaitingForFlowControl(t *testing.T) { + serverDoneCh := make(chan struct{}, 2) + const flowControlWindowSize = 65535 + ss := &stubserver.StubServer{ + StreamingOutputCallF: func(_ *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error { + // Send a large message to exhaust the client's flow control window. + stream.Send(&testpb.StreamingOutputCallResponse{ + Payload: &testpb.Payload{ + Body: make([]byte, flowControlWindowSize+1), + }, + }) + serverDoneCh <- struct{}{} + return nil + }, + } + + // Create a server that allows only 1 stream at a time. + ss = stubserver.StartTestService(t, ss, grpc.MaxConcurrentStreams(1)) + defer ss.Stop() + // Use a static flow control window. + if err := ss.StartClient(grpc.WithStaticStreamWindowSize(flowControlWindowSize)); err != nil { + t.Fatalf("Error while start test service client: %v", err) + } + client := ss.Client + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + streamCtx, streamCancel := context.WithCancel(ctx) + defer streamCancel() + + if _, err := client.StreamingOutputCall(streamCtx, &testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("Failed to create server streaming RPC: %v", err) + } + + // Wait for the server handler to return. This should cause the trailers to + // be buffered on the server, waiting for flow control quota to first send + // the data frame. + select { + case <-ctx.Done(): + t.Fatal("Context timed out waiting for server handler to return.") + case <-serverDoneCh: + } + + // Attempt to create a stream. It should fail since the previous stream is + // still blocked. + shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer shortCancel() + _, err := client.StreamingOutputCall(shortCtx, &testpb.StreamingOutputCallRequest{}) + if status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("Server stream creation returned error with unexpected status code: %v, want code: %v", err, codes.DeadlineExceeded) + } + + // Cancel the RPC, this should free up concurrent stream quota on the + // server. + streamCancel() + + // Attempt to create another stream. + stream, err := client.StreamingOutputCall(ctx, &testpb.StreamingOutputCallRequest{}) + if err != nil { + t.Fatalf("Failed to create server streaming RPC: %v", err) + } + _, err = stream.Recv() + if err != nil { + t.Fatalf("Failed to read from the stream: %v", err) + } +} + +// Tests that when a client sends a HEADERS frame with EndStream=true, the +// server-side stream receives an io.EOF on Recv() and does not hang waiting +// for data frames. +func (s) TestHeadersEndStreamNoHang(t *testing.T) { + receivedErr := make(chan error, 1) + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + receivedErr <- err + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + conn, err := net.DialTimeout("tcp", ss.Address, defaultTestTimeout) + if err != nil { + t.Fatalf("Failed to dial: %v", err) + } + defer conn.Close() + + st := newServerTesterFromConn(t, conn) + st.greet() + + // Send HEADERS with EndStream = true and no grpc-timeout header. + st.writeHeaders(http2.HeadersFrameParam{ + StreamID: 1, + BlockFragment: st.encodeHeader( + ":method", "POST", + ":path", "/grpc.testing.TestService/FullDuplexCall", + ":authority", "localhost", + "content-type", "application/grpc", + "te", "trailers", + ), + EndStream: true, + EndHeaders: true, + }) + + select { + case err := <-receivedErr: + if err != io.EOF { + t.Errorf("Streaming handler returned error = %v, expected io.EOF", err) + } + case <-time.After(defaultTestTimeout): + t.Fatalf("Timed out waiting for Recv() on the server to complete") + } +} diff --git a/test/xds/xds_client_ack_nack_test.go b/test/xds/xds_client_ack_nack_test.go new file mode 100644 index 0000000..05f5a1b --- /dev/null +++ b/test/xds/xds_client_ack_nack_test.go @@ -0,0 +1,202 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/resolver" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// We are interested in LDS, RDS, CDS and EDS resources as part of the regular +// xDS flow on the client. +const wantResources = 4 + +// seenAllACKs returns true if the provided ackVersions map contains valid acks +// for all the resources that we are interested in. If `wantNonEmpty` is true, +// only non-empty ack versions are considered valid. +func seenAllACKs(acksVersions map[string]string, wantNonEmpty bool) bool { + if len(acksVersions) != wantResources { + return false + } + for _, ack := range acksVersions { + if wantNonEmpty && ack == "" { + return false + } + } + return true +} + +// TestClientResourceVersionAfterStreamRestart tests the scenario where the +// xdsClient's ADS stream to the management server gets broken. This test +// verifies that the version number on the initial request on the new stream +// indicates the most recent version seen by the client on the previous stream. +func (s) TestClientResourceVersionAfterStreamRestart(t *testing.T) { + // Create a restartable listener which can close existing connections. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + + // We depend on the fact that the management server assigns monotonically + // increasing stream IDs starting at 1. + const ( + idBeforeRestart = 1 + idAfterRestart = 2 + ) + + // Events of importance in the test, in the order in which they are expected + // to happen. + acksReceivedBeforeRestart := grpcsync.NewEvent() + streamRestarted := grpcsync.NewEvent() + acksReceivedAfterRestart := grpcsync.NewEvent() + + // Map from stream id to a map of resource type to resource version. + ackVersionsMap := make(map[int64]map[string]string) + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { + // Return early under the following circumstances: + // - Received all the requests we wanted to see. This is to avoid + // any stray requests leading to test flakes. + // - Request contains no resource names. Such requests are usually + // seen when the xdsclient is shutting down and is no longer + // interested in the resources that it had subscribed to earlier. + if acksReceivedAfterRestart.HasFired() || len(req.GetResourceNames()) == 0 { + return nil + } + // Create a stream specific map to store ack versions if this is the + // first time we are seeing this stream id. + if ackVersionsMap[id] == nil { + ackVersionsMap[id] = make(map[string]string) + } + ackVersionsMap[id][req.GetTypeUrl()] = req.GetVersionInfo() + // Prior to stream restart, we are interested only in non-empty + // resource versions. The xdsclient first sends out requests with an + // empty version string. After receipt of requested resource, it + // sends out another request for the same resource, but this time + // with a non-empty version string, to serve as an ACK. + if seenAllACKs(ackVersionsMap[idBeforeRestart], true) { + acksReceivedBeforeRestart.Fire() + } + // After stream restart, we expect the xdsclient to send out + // requests with version string set to the previously ACKed + // versions. If it sends out requests with empty version string, it + // is a bug and we want this test to catch it. + if seenAllACKs(ackVersionsMap[idAfterRestart], false) { + acksReceivedAfterRestart.Fire() + } + return nil + }, + OnStreamClosed: func(int64, *v3corepb.Node) { + streamRestarted.Fire() + }, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + xdsResolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // A successful RPC means that the xdsclient received all requested + // resources. The ACKs from the xdsclient may get a little delayed. So, we + // need to wait for all ACKs to be received on the management server before + // restarting the stream. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for all resources to be ACKed prior to stream restart") + case <-acksReceivedBeforeRestart.Done(): + } + + // Stop the listener on the management server. This will cause the client to + // backoff and recreate the stream. + lis.Stop() + + // Wait for the stream to be closed on the server. + <-streamRestarted.Done() + + // Restart the listener on the management server to be able to accept + // reconnect attempts from the client. + lis.Restart() + + // Wait for all the previously sent resources to be re-requested. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for all resources to be ACKed post stream restart") + case <-acksReceivedAfterRestart.Done(): + } + + if diff := cmp.Diff(ackVersionsMap[idBeforeRestart], ackVersionsMap[idAfterRestart]); diff != "" { + t.Fatalf("unexpected diff in ack versions before and after stream restart (-want, +got):\n%s", diff) + } + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} diff --git a/test/xds/xds_client_affinity_test.go b/test/xds/xds_client_affinity_test.go new file mode 100644 index 0000000..3a4b33e --- /dev/null +++ b/test/xds/xds_client_affinity_test.go @@ -0,0 +1,128 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// hashRouteConfig returns a RouteConfig resource with hash policy set to +// header "session_id". +func hashRouteConfig(routeName, ldsTarget, clusterName string) *v3routepb.RouteConfiguration { + return &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + HashPolicy: []*v3routepb.RouteAction_HashPolicy{{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: "session_id", + }, + }, + Terminal: true, + }}, + }}, + }}, + }}, + } +} + +// ringhashCluster returns a Cluster resource that picks ringhash as the lb +// policy. +func ringhashCluster(clusterName, edsServiceName string) *v3clusterpb.Cluster { + return &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: edsServiceName, + }, + LbPolicy: v3clusterpb.Cluster_RING_HASH, + } +} + +// TestClientSideAffinitySanityCheck tests that the affinity config can be +// propagated to pick the ring_hash policy. It doesn't test the affinity +// behavior in ring_hash policy. +func (s) TestClientSideAffinitySanityCheck(t *testing.T) { + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + // Replace RDS and CDS resources with ringhash config, but keep the resource + // names. + resources.Routes = []*v3routepb.RouteConfiguration{hashRouteConfig( + resources.Routes[0].Name, + resources.Listeners[0].Name, + resources.Clusters[0].Name, + )} + resources.Clusters = []*v3clusterpb.Cluster{ringhashCluster( + resources.Clusters[0].Name, + resources.Clusters[0].EdsClusterConfig.ServiceName, + )} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to create a client for server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} diff --git a/test/xds/xds_client_certificate_providers_test.go b/test/xds/xds_client_certificate_providers_test.go new file mode 100644 index 0000000..b7b2fc4 --- /dev/null +++ b/test/xds/xds_client_certificate_providers_test.go @@ -0,0 +1,508 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "crypto/tls" + "fmt" + "strings" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// Tests the case where the bootstrap configuration contains no certificate +// providers, and xDS credentials with an insecure fallback is specified at dial +// time. The management server is configured to return client side xDS resources +// with no security configuration. The test verifies that the gRPC client is +// able to make RPCs to the backend which is configured to accept plaintext +// connections. This ensures that the insecure fallback credentials are getting +// used on the client. +func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Success(t *testing.T) { + // Spin up an xDS management server. + mgmtServer, nodeID, _, resolverBuilder := setup.ManagementServerAndResolver(t) + + // Spin up a test backend. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure client side xDS resources on the management server, with no + // security configuration in the Cluster resource. + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} + +// Tests the case where the bootstrap configuration contains no certificate +// providers, and xDS credentials with an insecure fallback is specified at dial +// time. The management server is configured to return client side xDS resources +// with an mTLS security configuration. The test verifies that the gRPC client +// moves to TRANSIENT_FAILURE and rpcs fail with the expected error code and +// string. This ensures that when the certificate provider instance name +// specified in the security configuration is not present in the bootstrap, +// channel creation does not fail, but it moves to TRANSIENT_FAILURE and +// subsequent rpcs fail. +func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Failure(t *testing.T) { + // Start an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server, + // with no certificate providers. + nodeID := uuid.New().String() + bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create an xDS resolver with the above bootstrap configuration. + resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Spin up a test backend. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure client side xDS resources on the management server, with mTLS + // security configuration in the Cluster resource. + const serviceName = "my-service-client-side-xds" + const clusterName = "cluster-" + serviceName + const endpointsName = "endpoints-" + serviceName + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + resources.Clusters = []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelMTLS)} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn and ensure that it moves to TRANSIENT_FAILURE. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + cc.Connect() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Make an RPC and ensure that expected error is returned. + wantErr := fmt.Sprintf("identity certificate provider instance name %q missing in bootstrap configuration", e2e.ClientSideCertProviderInstance) + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr) + } +} + +// Tests the case where the bootstrap configuration contains one certificate +// provider, and xDS credentials with an insecure fallback is specified at dial +// time. The management server responds with three clusters: +// 1. contains valid security configuration pointing to the certificate provider +// instance specified in the bootstrap +// 2. contains no security configuration, hence should use insecure fallback +// 3. contains invalid security configuration pointing to a non-existent +// certificate provider instance +// +// The test verifies that RPCs to the first two clusters succeed, while RPCs to +// the third cluster fails with an appropriate code and error message. +func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T) { + // Spin up an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + var xdsResolver resolver.Builder + if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { + var err error + xdsResolver, err = newResolver.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } + + // Create test backends for all three clusters + // backend1 configured with TLS creds, represents cluster1 + // backend2 configured with insecure creds, represents cluster2 + // backend3 configured with insecure creds, represents cluster3 + serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert) + server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server1.Stop() + server2 := stubserver.StartTestService(t, nil) + defer server2.Stop() + server3 := stubserver.StartTestService(t, nil) + defer server3.Stop() + + // Configure client side xDS resources on the management server. + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName1 = "cluster1-" + serviceName + const clusterName2 = "cluster2-" + serviceName + const clusterName3 = "cluster3-" + serviceName + const endpointsName1 = "endpoints1-" + serviceName + const endpointsName2 = "endpoints2-" + serviceName + const endpointsName3 = "endpoints3-" + serviceName + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)} + // Route configuration: + // - "/grpc.testing.TestService/EmptyCall" --> cluster1 + // - "/grpc.testing.TestService/UnaryCall" --> cluster2 + // - "/grpc.testing.TestService/FullDuplexCall" --> cluster3 + routes := []*v3routepb.RouteConfiguration{{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}, + }}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}, + }}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/FullDuplexCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName3}, + }}, + }, + }, + }}, + }} + // Clusters: + // - cluster1 with cert provider name e2e.ClientSideCertProviderInstance. + // - cluster2 with no security configuration. + // - cluster3 with non-existent cert provider name. + clusters := []*v3clusterpb.Cluster{ + e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelMTLS), + e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelNone), + func() *v3clusterpb.Cluster { + cluster3 := e2e.DefaultCluster(clusterName3, endpointsName3, e2e.SecurityLevelMTLS) + cluster3.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "non-existent-certificate-provider-instance-name", + }, + }, + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "non-existent-certificate-provider-instance-name", + }, + }, + }), + }, + } + return cluster3 + }(), + } + // Endpoints for each of the above clusters with backends created earlier. + endpoints := []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}), + e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}), + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Make an RPC to be routed to cluster1 and verify that it succeeds. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if got, want := peer.Addr.String(), server1.Address; got != want { + t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want) + + } + + // Make an RPC to be routed to cluster2 and verify that it succeeds. + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil { + t.Fatalf("UnaryCall() failed: %v", err) + } + if got, want := peer.Addr.String(), server2.Address; got != want { + t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want) + } + + // Make an RPC to be routed to cluster3 and verify that it fails. + const wantErr = `identity certificate provider instance name "non-existent-certificate-provider-instance-name" missing in bootstrap configuration` + if _, err := client.FullDuplexCall(ctx); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("FullDuplexCall failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr) + } +} + +// Tests the case where the bootstrap configuration contains one certificate +// provider configured with SPIFFE Bundle Map roots on the client side, and xDS +// credentials with an insecure fallback is specified at dial time. The +// management server responds with three clusters: +// 1. contains valid security configuration pointing to the certificate provider +// instance specified in the bootstrap, and the server uses a SPIFFE cert. +// 2. contains valid security configuration pointing to the certificate provider +// instance specified in the bootstrap, and the server uses a SPIFFE cert chain. +// 3. contains invalid security configuration pointing to a non-existent +// certificate provider instance +// +// The test verifies that RPCs to the first two clusters succeed, while RPCs to +// the third cluster fails with an appropriate code and error message. +func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfigurationSPIFFE(t *testing.T) { + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolverWithSPIFFE(t) + + // Create test backends for all three clusters + // backend1 configured with a SPIFFE cert, represents cluster1 + // backend2 configured with a SPIFFE cert chain, represents cluster2 + // backend3 configured with insecure creds, represents cluster3 + serverCreds := testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert) + server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server1.Stop() + serverCreds2 := testutils.CreateServerTLSCredentialsCompatibleWithSPIFFEChain(t, tls.RequireAndVerifyClientCert) + server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds2)) + defer server2.Stop() + server3 := stubserver.StartTestService(t, nil) + defer server3.Stop() + + // Configure client side xDS resources on the management server. + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName1 = "cluster1-" + serviceName + const clusterName2 = "cluster2-" + serviceName + const clusterName3 = "cluster3-" + serviceName + const endpointsName1 = "endpoints1-" + serviceName + const endpointsName2 = "endpoints2-" + serviceName + const endpointsName3 = "endpoints3-" + serviceName + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)} + // Route configuration: + // - "/grpc.testing.TestService/EmptyCall" --> cluster1 + // - "/grpc.testing.TestService/UnaryCall" --> cluster2 + // - "/grpc.testing.TestService/FullDuplexCall" --> cluster3 + routes := []*v3routepb.RouteConfiguration{{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}, + }}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}, + }}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/FullDuplexCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName3}, + }}, + }, + }, + }}, + }} + // Clusters: + // - cluster1 with cert provider name e2e.ClientSideCertProviderInstance and mTLS. + // - cluster2 with cert provider name e2e.ClientSideCertProviderInstance and mTLS. + // - cluster3 with non-existent cert provider name. + clusters := []*v3clusterpb.Cluster{ + e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelMTLS), + e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelMTLS), + func() *v3clusterpb.Cluster { + cluster3 := e2e.DefaultCluster(clusterName3, endpointsName3, e2e.SecurityLevelMTLS) + cluster3.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "non-existent-certificate-provider-instance-name", + }, + }, + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "non-existent-certificate-provider-instance-name", + }, + }, + }), + }, + } + return cluster3 + }(), + } + // Endpoints for each of the above clusters with backends created earlier. + endpoints := []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}), + e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}), + e2e.DefaultEndpoint(endpointsName3, "localhost", []uint32{testutils.ParsePort(t, server3.Address)}), + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Make an RPC to be routed to cluster1 and verify that it succeeds. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if got, want := peer.Addr.String(), server1.Address; got != want { + t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want) + } + verifySecurityInformationFromPeerSPIFFE(t, peer, e2e.SecurityLevelMTLS, 1) + + // Make an RPC to be routed to cluster2 and verify that it succeeds. + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil { + t.Fatalf("UnaryCall() failed: %v", err) + } + if got, want := peer.Addr.String(), server2.Address; got != want { + t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want) + } + // In this call the server contains a peer chain of length 2 + verifySecurityInformationFromPeerSPIFFE(t, peer, e2e.SecurityLevelMTLS, 2) + + // Make an RPC to be routed to cluster3 and verify that it fails. + const wantErr = `identity certificate provider instance name "non-existent-certificate-provider-instance-name" missing in bootstrap configuration` + if _, err := client.FullDuplexCall(ctx); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("FullDuplexCall failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr) + } +} diff --git a/test/xds/xds_client_custom_lb_test.go b/test/xds/xds_client_custom_lb_test.go new file mode 100644 index 0000000..84b8a20 --- /dev/null +++ b/test/xds/xds_client_custom_lb_test.go @@ -0,0 +1,269 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "testing" + "time" + + "google.golang.org/grpc" + _ "google.golang.org/grpc/balancer/leastrequest" // To register least_request + _ "google.golang.org/grpc/balancer/weightedroundrobin" // To register weighted_round_robin + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/resolver" + + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3clientsideweightedroundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3" + v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3" + v3roundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/round_robin/v3" + v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// wrrLocality is a helper that takes a proto message and returns a +// WrrLocalityProto with the proto message marshaled into a proto.Any as a +// child. +func wrrLocality(t *testing.T, m proto.Message) *v3wrrlocalitypb.WrrLocality { + return &v3wrrlocalitypb.WrrLocality{ + EndpointPickingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, m), + }, + }, + }, + }, + } +} + +// clusterWithLBConfiguration returns a cluster resource with the proto message +// passed Marshaled to an any and specified through the load_balancing_policy +// field. +func clusterWithLBConfiguration(t *testing.T, clusterName, edsServiceName string, secLevel e2e.SecurityLevel, m proto.Message) *v3clusterpb.Cluster { + cluster := e2e.DefaultCluster(clusterName, edsServiceName, secLevel) + cluster.LoadBalancingPolicy = &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(t, m), + }, + }, + }, + } + return cluster +} + +// TestWRRLocality tests RPC distribution across a scenario with 5 backends, +// with 2 backends in a locality with weight 1, and 3 backends in a second +// locality with weight 2. Through xDS, the test configures a +// wrr_locality_balancer with either a round robin or custom (specifying pick +// first) child load balancing policy, and asserts the correct distribution +// based on the locality weights and the endpoint picking policy specified. +func (s) TestWrrLocality(t *testing.T) { + backend1 := stubserver.StartTestService(t, nil) + port1 := testutils.ParsePort(t, backend1.Address) + defer backend1.Stop() + backend2 := stubserver.StartTestService(t, nil) + port2 := testutils.ParsePort(t, backend2.Address) + defer backend2.Stop() + backend3 := stubserver.StartTestService(t, nil) + port3 := testutils.ParsePort(t, backend3.Address) + defer backend3.Stop() + backend4 := stubserver.StartTestService(t, nil) + port4 := testutils.ParsePort(t, backend4.Address) + defer backend4.Stop() + backend5 := stubserver.StartTestService(t, nil) + port5 := testutils.ParsePort(t, backend5.Address) + defer backend5.Stop() + const serviceName = "my-service-client-side-xds" + + tests := []struct { + name string + // Configuration will be specified through load_balancing_policy field. + wrrLocalityConfiguration *v3wrrlocalitypb.WrrLocality + addressDistributionWant []struct { + addr string + count int + } + }{ + { + name: "rr_child", + wrrLocalityConfiguration: wrrLocality(t, &v3roundrobinpb.RoundRobin{}), + // Each addresses expected probability is locality weight of + // locality / total locality weights multiplied by 1 / number of + // endpoints in each locality (due to round robin across endpoints + // in a locality). Thus, address 1 and address 2 have 1/3 * 1/2 + // probability, and addresses 3 4 5 have 2/3 * 1/3 probability of + // being routed to. + addressDistributionWant: []struct { + addr string + count int + }{ + {addr: backend1.Address, count: 6}, + {addr: backend2.Address, count: 6}, + {addr: backend3.Address, count: 8}, + {addr: backend4.Address, count: 8}, + {addr: backend5.Address, count: 8}, + }, + }, + // This configures custom lb as the child of wrr_locality, which points + // to our pick_first implementation. Thus, the expected distribution of + // addresses is locality weight of locality / total locality weights as + // the probability of picking the first backend within the locality + // (e.g. Address 1 for locality 1, and Address 3 for locality 2). + { + name: "custom_lb_child_pick_first", + wrrLocalityConfiguration: wrrLocality(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/pick_first", + Value: &structpb.Struct{}, + }), + addressDistributionWant: []struct { + addr string + count int + }{ + {addr: backend1.Address, count: 1}, + {addr: backend3.Address, count: 2}, + }, + }, + // Sanity check for weighted round robin. Don't need to test super + // specific behaviors, as that is covered in unit tests. Set up weighted + // round robin as the endpoint picking policy with per RPC load reports + // enabled. Due the server not sending trailers with load reports, the + // weighted round robin policy should essentially function as round + // robin, and thus should have the same distribution as round robin + // above. + { + name: "custom_lb_child_wrr/", + wrrLocalityConfiguration: wrrLocality(t, &v3clientsideweightedroundrobinpb.ClientSideWeightedRoundRobin{ + EnableOobLoadReport: &wrapperspb.BoolValue{ + Value: false, + }, + // BlackoutPeriod long enough to cause load report weights to + // trigger in the scope of test case, but no load reports + // configured anyway. + BlackoutPeriod: durationpb.New(10 * time.Second), + WeightExpirationPeriod: durationpb.New(10 * time.Second), + WeightUpdatePeriod: durationpb.New(time.Second), + ErrorUtilizationPenalty: &wrapperspb.FloatValue{Value: 1}, + }), + addressDistributionWant: []struct { + addr string + count int + }{ + {addr: backend1.Address, count: 6}, + {addr: backend2.Address, count: 6}, + {addr: backend3.Address, count: 8}, + {addr: backend4.Address, count: 8}, + {addr: backend5.Address, count: 8}, + }, + }, + { + name: "custom_lb_least_request", + wrrLocalityConfiguration: wrrLocality(t, &v3leastrequestpb.LeastRequest{ + ChoiceCount: wrapperspb.UInt32(2), + }), + // The test performs a Unary RPC, and blocks until the RPC returns, + // and then makes the next Unary RPC. Thus, over iterations, no RPC + // counts are present. This causes least request's randomness of + // indexes to sample to converge onto a round robin distribution per + // locality. Thus, expect the same distribution as round robin + // above. + addressDistributionWant: []struct { + addr string + count int + }{ + {addr: backend1.Address, count: 6}, + {addr: backend2.Address, count: 6}, + {addr: backend3.Address, count: 8}, + {addr: backend4.Address, count: 8}, + {addr: backend5.Address, count: 8}, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Start an xDS management server. + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + routeConfigName := "route-" + serviceName + clusterName := "cluster-" + serviceName + endpointsName := "endpoints-" + serviceName + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{clusterWithLBConfiguration(t, clusterName, endpointsName, e2e.SecurityLevelNone, test.wrrLocalityConfiguration)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: endpointsName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Backends: []e2e.BackendOptions{{Ports: []uint32{port1}}, {Ports: []uint32{port2}}}, + Weight: 1, + }, + { + Backends: []e2e.BackendOptions{{Ports: []uint32{port3}}, {Ports: []uint32{port4}}, {Ports: []uint32{port5}}}, + Weight: 2, + }, + }, + })}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + var addrDistWant []resolver.Address + for _, addrAndCount := range test.addressDistributionWant { + for i := 0; i < addrAndCount.count; i++ { + addrDistWant = append(addrDistWant, resolver.Address{Addr: addrAndCount.addr}) + } + } + if err := roundrobin.CheckWeightedRoundRobinRPCs(ctx, t, client, addrDistWant); err != nil { + t.Fatalf("Error in expected round robin: %v", err) + } + }) + } +} diff --git a/test/xds/xds_client_federation_test.go b/test/xds/xds_client_federation_test.go new file mode 100644 index 0000000..815b520 --- /dev/null +++ b/test/xds/xds_client_federation_test.go @@ -0,0 +1,343 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestClientSideFederation tests that federation is supported. +// +// In this test, some xDS responses contain resource names in another authority +// (in the new resource name style): +// - LDS: old style, no authority (default authority) +// - RDS: new style, in a different authority +// - CDS: old style, no authority (default authority) +// - EDS: new style, in a different authority +func (s) TestClientSideFederation(t *testing.T) { + // Start a management server as the default authority. + serverDefaultAuth := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Start another management server as the other authority. + const nonDefaultAuth = "non-default-auth" + serverAnotherAuth := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create a bootstrap file in a temporary directory. + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, serverDefaultAuth.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + // Specify the address of the non-default authority. + Authorities: map[string]json.RawMessage{ + nonDefaultAuth: []byte(fmt.Sprintf(`{ + "xds_servers": [ + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + } + ] + }`, serverAnotherAuth.Address)), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap file: %v", err) + } + + resolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + // LDS is old style name. + ldsName := serviceName + // RDS is new style, with the non default authority. + rdsName := fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/%s", nonDefaultAuth, "route-"+serviceName) + // CDS is old style name. + cdsName := "cluster-" + serviceName + // EDS is new style, with the non default authority. + edsName := fmt.Sprintf("xdstp://%s/envoy.config.route.v3.ClusterLoadAssignment/%s", nonDefaultAuth, "endpoints-"+serviceName) + + // Split resources, put LDS/CDS in the default authority, and put RDS/EDS in + // the other authority. + resourcesDefault := e2e.UpdateOptions{ + NodeID: nodeID, + // This has only LDS and CDS. + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + resourcesAnother := e2e.UpdateOptions{ + NodeID: nodeID, + // This has only RDS and EDS. + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, cdsName)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // This has only LDS and CDS. + if err := serverDefaultAuth.Update(ctx, resourcesDefault); err != nil { + t.Fatal(err) + } + // This has only RDS and EDS. + if err := serverAnotherAuth.Update(ctx, resourcesAnother); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} + +// TestClientSideFederationWithOnlyXDSTPStyleLDS tests that federation is +// supported with new xdstp style names for LDS only while using the old style +// for other resources. This test in addition also checks that when service name +// contains escapable characters, we "fully" encode it for looking up +// VirtualHosts in xDS RouteConfiguration. +func (s) TestClientSideFederationWithOnlyXDSTPStyleLDS(t *testing.T) { + // Start a management server as a sophisticated authority. + const authority = "traffic-manager.xds.notgoogleapis.com" + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create a bootstrap file in a temporary directory. + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + ClientDefaultListenerResourceNameTemplate: fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/%%s", authority), + // Specify the address of the non-default authority. + Authorities: map[string]json.RawMessage{ + authority: []byte(fmt.Sprintf(`{ + "xds_servers": [ + { + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + } + ] + }`, mgmtServer.Address)), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap file: %v", err) + } + + resolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // serviceName with escapable characters - ' ', and '/'. + const serviceName = "my-service-client-side-xds/2nd component" + + // All other resources are with old style name. + const rdsName = "route-" + serviceName + const cdsName = "cluster-" + serviceName + const edsName = "endpoints-" + serviceName + + // Resource update sent to go-control-plane mgmt server. + resourceUpdate := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: func() []*v3listenerpb.Listener { + // LDS is new style xdstp name. Since the LDS resource name is prefixed + // with xdstp, the string will be %-encoded excluding '/'s. See + // bootstrap.PopulateResourceTemplate(). + const specialEscapedServiceName = "my-service-client-side-xds/2nd%20component" // same as bootstrap.percentEncode(serviceName) + ldsName := fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/%s", authority, specialEscapedServiceName) + return []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)} + }(), + Routes: func() []*v3routepb.RouteConfiguration { + // RouteConfiguration will has one entry in []VirtualHosts that contains the + // "fully" escaped service name in []Domains. This is to assert that gRPC + // uses the escaped service name to lookup VirtualHosts. RDS is also with + // old style name. + const fullyEscapedServiceName = "my-service-client-side-xds%2F2nd%20component" // same as url.PathEscape(serviceName) + return []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, fullyEscapedServiceName, cdsName)} + }(), + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resourceUpdate); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} + +// TestFederation_UnknownAuthorityInDialTarget tests the case where a ClientConn +// is created with a dial target containing an authority which is not specified +// in the bootstrap configuration. The test verifies that RPCs on the ClientConn +// fail with an appropriate error. +func (s) TestFederation_UnknownAuthorityInDialTarget(t *testing.T) { + // Setting up the management server is not *really* required for this test + // case. All we need is a bootstrap configuration which does not contain the + // authority mentioned in the dial target. But setting up the management + // server and actually making an RPC ensures that the xDS client is + // configured properly, and when we dial with an unknown authority in the + // next step, we can be sure that the error we receive is legitimate. + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + target := fmt.Sprintf("xds:///%s", serviceName) + cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed %q: %v", target, err) + } + defer cc.Close() + t.Log("Created ClientConn to test service") + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() RPC: %v", err) + } + t.Log("Successfully performed an EmptyCall RPC") + + target = fmt.Sprintf("xds://unknown-authority/%s", serviceName) + t.Logf("Creating a channel with unknown authority %q, expecting failure", target) + wantErr := fmt.Sprintf("authority \"unknown-authority\" specified in dial target %q is not found in the bootstrap file", target) + cc, err = grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("Unexpected error while creating ClientConn: %v", err) + } + defer cc.Close() + client = testgrpc.NewTestServiceClient(cc) + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall(_, _) = _, %v; want _, %q", err, wantErr) + } +} + +// TestFederation_UnknownAuthorityInReceivedResponse tests the case where the +// LDS resource associated with the dial target contains an RDS resource name +// with an authority which is not specified in the bootstrap configuration. The +// test verifies that RPCs fail with an appropriate error. +func (s) TestFederation_UnknownAuthorityInReceivedResponse(t *testing.T) { + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + // LDS is old style name. + // RDS is new style, with an unknown authority. + const serviceName = "my-service-client-side-xds" + const unknownAuthority = "unknown-authority" + ldsName := serviceName + rdsName := fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/%s", unknownAuthority, "route-"+serviceName) + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, "cluster-"+serviceName)}, + SkipValidation: true, // This update has only LDS and RDS resources. + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + target := fmt.Sprintf("xds:///%s", serviceName) + cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed %q: %v", target, err) + } + defer cc.Close() + t.Log("Created ClientConn to test service") + + client := testgrpc.NewTestServiceClient(cc) + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Fatal("EmptyCall RPC succeeded for target with unknown authority when expected to fail") + } + if got, want := status.Code(err), codes.Unavailable; got != want { + t.Fatalf("EmptyCall RPC returned status code: %v, want %v", got, want) + } +} diff --git a/test/xds/xds_client_filter_e2e_test.go b/test/xds/xds_client_filter_e2e_test.go new file mode 100644 index 0000000..95f7354 --- /dev/null +++ b/test/xds/xds_client_filter_e2e_test.go @@ -0,0 +1,239 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "io" + "testing" + + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/internal/xds/httpfilter" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// Tests the interaction between the defaultStreamInterceptor and xDS filters. +// It verifies that the SendMsg, RecvMsg, and CloseSend methods are called the +// expected number of times for different RPC types (unary, client streaming, +// server streaming, and bidi streaming) when an xDS filter is registered and +// used in the call chain. +func (s) TestDefaultStreamInterceptor_InteractionWithXDSFilters(t *testing.T) { + // Register a custom xDS filter builder for the test. + testFilterTypeURL := t.Name() + filterBuilder := newTrackingHTTPFilterBuilder(testFilterTypeURL) + httpfilter.Register(filterBuilder) + defer httpfilter.UnregisterForTesting(testFilterTypeURL) + + // Setup a test backend implementing both all four RPC types. + testServer := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + StreamingOutputCallF: func(_ *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error { + return stream.Send(&testpb.StreamingOutputCallResponse{}) + }, + StreamingInputCallF: func(stream testgrpc.TestService_StreamingInputCallServer) error { + for { + _, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + return err + } + } + return stream.SendAndClose(&testpb.StreamingInputCallResponse{}) + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + _, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil { + return err + } + } + }, + } + if err := testServer.Start(nil); err != nil { + t.Fatal("Error starting server:", err) + } + defer testServer.Stop() + + // Start an xDS management server. + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + const serviceName = "my-service-xds" + clusterSpec := &v3routepb.RouteAction_Cluster{Cluster: "cluster-A"} + hcm := &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "route-" + serviceName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: clusterSpec, + }}, + }}, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + { + Name: "tracking-filter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: testFilterTypeURL, + }), + }, + }, + e2e.RouterHTTPFilter, + }, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster("cluster-A", "cluster-A", e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint("cluster-A", "localhost", []uint32{testutils.ParsePort(t, testServer.Address)})}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + + tests := []struct { + name string + run func(ctx context.Context, client testgrpc.TestServiceClient) + wantSendMsg int32 + wantRecvMsg int32 + }{ + { + name: "unary", + run: func(ctx context.Context, client testgrpc.TestServiceClient) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + }, + wantSendMsg: 1, // 1 request + wantRecvMsg: 2, // 1 response + 1 trailers + }, + { + name: "client_streaming", + run: func(ctx context.Context, client testgrpc.TestServiceClient) { + stream, err := client.StreamingInputCall(ctx) + if err != nil { + t.Fatalf("StreamingInputCall() failed: %v", err) + } + for i := 0; i < 2; i++ { + if err := stream.Send(&testpb.StreamingInputCallRequest{}); err != nil { + t.Fatalf("Send() failed: %v", err) + } + } + if _, err := stream.CloseAndRecv(); err != nil { + t.Fatalf("CloseAndRecv() failed: %v", err) + } + }, + wantSendMsg: 2, // 2 requests + wantRecvMsg: 2, // 1 response + 1 trailers + }, + { + name: "server_streaming", + run: func(ctx context.Context, client testgrpc.TestServiceClient) { + stream, err := client.StreamingOutputCall(ctx, &testpb.StreamingOutputCallRequest{}) + if err != nil { + t.Fatalf("StreamingOutputCall() failed: %v", err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("Recv() failed: %v", err) + } + }, + wantSendMsg: 1, // 1 request + wantRecvMsg: 1, // 1 response (trailers not yet consumed since stream not read to EOF) + }, + { + name: "bidi_streaming", + run: func(ctx context.Context, client testgrpc.TestServiceClient) { + stream, err := client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall() failed: %v", err) + } + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("Send() failed: %v", err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("Recv() failed: %v", err) + } + if err := stream.CloseSend(); err != nil { + t.Fatalf("CloseSend() failed: %v", err) + } + }, + wantSendMsg: 1, // 1 request + wantRecvMsg: 1, // 1 response + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + filterBuilder.sendMsgCount.Store(0) + filterBuilder.recvMsgCount.Store(0) + filterBuilder.closeSendCount.Store(0) + + tc.run(ctx, client) + + if got := filterBuilder.sendMsgCount.Load(); got != tc.wantSendMsg { + t.Fatalf("SendMsg() count = %d, want %d", got, tc.wantSendMsg) + } + if got := filterBuilder.recvMsgCount.Load(); got != tc.wantRecvMsg { + t.Fatalf("RecvMsg() count = %d, want %d", got, tc.wantRecvMsg) + } + if got := filterBuilder.closeSendCount.Load(); got != 1 { + t.Fatalf("CloseSend() count = %d, want 1", got) + } + }) + } +} diff --git a/test/xds/xds_client_http_connect_test.go b/test/xds/xds_client_http_connect_test.go new file mode 100644 index 0000000..f82886e --- /dev/null +++ b/test/xds/xds_client_http_connect_test.go @@ -0,0 +1,642 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package xds_test + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "strconv" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/proxyserver" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3http11proxypb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/http_11_proxy/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" +) + +func enableXDSHTTPConnect(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSHTTPConnectEnabled, true) + cleanup, err := xdsresource.RegisterMetadataConverterForTesting(version.V3AddressURL) + if err != nil { + t.Fatalf("RegisterMetadataConverterForTesting(%q) failed: %v", version.V3AddressURL, err) + } + t.Cleanup(cleanup) +} + +func splitHostPort(t *testing.T, addr string) (string, int) { + t.Helper() + host, portStr, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("Failed to split host and port for address %q: %v", addr, err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + t.Fatalf("Failed to parse port %q: %v", portStr, err) + } + return host, port +} + +// Test veifies client-side HTTP CONNECT proxying support where the proxy +// address is configured via endpoint metadata. It verifies that gRPC requests +// are successfully routed through the HTTP CONNECT proxy to the correct +// backend server. +func (s) TestXDSHTTPConnect(t *testing.T) { + enableXDSHTTPConnect(t) + + // Spin up a mock HTTP CONNECT proxy server to capture client connections. + connectCh := make(chan string, 1) + pServer := proxyserver.New(t, func(req *http.Request) { + connectCh <- req.URL.Host + }, false) + + proxyHost, proxyPort := splitHostPort(t, pServer.Addr) + + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + backendHost, backendPort := splitHostPort(t, server.Address) + + const ( + serviceName = "my-service-xds-http-connect" + routeConfigName = "route-my-service-xds-http-connect" + clusterName = "cluster-my-service-xds-http-connect" + endpointsName = "endpoints-my-service-xds-http-connect" + ) + + cluster := &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}}, + ServiceName: endpointsName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.http_11_proxy", + ConfigType: &v3corepb.TransportSocket_TypedConfig{TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{})}, + }, + } + + loadAssignment := &v3endpointpb.ClusterLoadAssignment{ + ClusterName: endpointsName, + Endpoints: []*v3endpointpb.LocalityLbEndpoints{ + { + Locality: &v3corepb.Locality{SubZone: "locality-1"}, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + LbEndpoints: []*v3endpointpb.LbEndpoint{ + { + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: backendHost, + PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(backendPort)}, + }, + }, + }, + }, + }, + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: proxyHost, + PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(proxyPort)}, + }, + }, + }), + }, + }, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + }, + }, + }, + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{loadAssignment}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Verify that the request was routed through the proxy to the expected + // backend. + select { + case gotTarget := <-connectCh: + if gotTarget != server.Address { + t.Errorf("Unexpected server address from proxy server, got %s want %s", gotTarget, server.Address) + } + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout while waiting for server address from proxy server") + } +} + +// Test verifies client-side HTTP CONNECT proxying support where the proxy +// address is configured via metadata at the locality level. It verifies that +// the client successfully fallback to using the locality-level proxy address +// if no endpoint-level metadata is present. +func (s) TestXDSHTTPConnect_LocalityFallback(t *testing.T) { + enableXDSHTTPConnect(t) + + // Spin up a mock HTTP CONNECT proxy server to capture client connections. + connectCh := make(chan string, 1) + pServer := proxyserver.New(t, func(req *http.Request) { + connectCh <- req.URL.Host + }, false) + + proxyHost, proxyPort := splitHostPort(t, pServer.Addr) + + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + backendHost, backendPort := splitHostPort(t, server.Address) + + const ( + serviceName = "my-service-xds-http-connect" + routeConfigName = "route-my-service-xds-http-connect" + clusterName = "cluster-my-service-xds-http-connect" + endpointsName = "endpoints-my-service-xds-http-connect" + ) + + cluster := &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}}, + ServiceName: endpointsName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.http_11_proxy", + ConfigType: &v3corepb.TransportSocket_TypedConfig{TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{})}, + }, + } + + loadAssignment := &v3endpointpb.ClusterLoadAssignment{ + ClusterName: endpointsName, + Endpoints: []*v3endpointpb.LocalityLbEndpoints{ + { + Locality: &v3corepb.Locality{SubZone: "locality-1"}, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + // Locality-level metadata contains the proxy address. + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: proxyHost, + PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(proxyPort)}, + }, + }, + }), + }, + }, + LbEndpoints: []*v3endpointpb.LbEndpoint{ + { + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: backendHost, + PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(backendPort)}, + }, + }, + }, + }, + }, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + }, + }, + }, + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{loadAssignment}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Verify that the request was routed through the proxy to the expected + // backend. + select { + case gotTarget := <-connectCh: + if gotTarget != server.Address { + t.Errorf("Unexpected server address from proxy server, got %s want %s", gotTarget, server.Address) + } + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout while waiting for server address from proxy server") + } +} + +// Test verifies client-side HTTP CONNECT proxying support when the client +// uses XDS credentials. It verifies that when a plaintext Http11Proxy wrapper +// is present in CDS, the XDS client successfully falls back to insecure +// credentials and establishes a proxy CONNECT tunnel to the correct backend +// server without security handshaking. +func (s) TestXDSHTTPConnect_WithInsecureXDSCredentials(t *testing.T) { + enableXDSHTTPConnect(t) + + // Spin up a mock HTTP CONNECT proxy server to capture client connections. + connectCh := make(chan string, 1) + pServer := proxyserver.New(t, func(req *http.Request) { + connectCh <- req.URL.Host + }, false) + + proxyHost, proxyPort := splitHostPort(t, pServer.Addr) + + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + backendHost, backendPort := splitHostPort(t, server.Address) + + const ( + serviceName = "my-service-xds-http-connect" + routeConfigName = "route-my-service-xds-http-connect" + clusterName = "cluster-my-service-xds-http-connect" + endpointsName = "endpoints-my-service-xds-http-connect" + ) + + cluster := &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}}, + ServiceName: endpointsName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.http_11_proxy", + ConfigType: &v3corepb.TransportSocket_TypedConfig{TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{})}, + }, + } + + loadAssignment := &v3endpointpb.ClusterLoadAssignment{ + ClusterName: endpointsName, + Endpoints: []*v3endpointpb.LocalityLbEndpoints{ + { + Locality: &v3corepb.Locality{SubZone: "locality-1"}, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + LbEndpoints: []*v3endpointpb.LbEndpoint{ + { + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: backendHost, + PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(backendPort)}, + }, + }, + }, + }, + }, + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: proxyHost, + PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(proxyPort)}, + }, + }, + }), + }, + }, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + }, + }, + }, + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{loadAssignment}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("Failed to create client XDS credentials: %v", err) + } + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Verify that the request was routed through the proxy to the expected + // backend. + select { + case gotTarget := <-connectCh: + if gotTarget != server.Address { + t.Errorf("Unexpected server address from proxy server, got %s want %s", gotTarget, server.Address) + } + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout while waiting for server address from proxy server") + } +} + +// Test verifies client-side HTTP CONNECT proxying support when the cluster +// uses an Http11ProxyUpstreamTransport wrapping an inner UpstreamTlsContext. +// It verifies that a secure TLS session is established with the backend over +// the HTTP CONNECT proxy connection. +func (s) TestXDSHTTPConnect_WithUpstreamTLSContext(t *testing.T) { + enableXDSHTTPConnect(t) + + // Spin up a mock HTTP CONNECT proxy server to capture client connections. + connectCh := make(chan string, 1) + pServer := proxyserver.New(t, func(req *http.Request) { + connectCh <- req.URL.Host + }, false) + + proxyHost, proxyPort := splitHostPort(t, pServer.Addr) + + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert) + server := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server.Stop() + + backendHost, backendPort := splitHostPort(t, server.Address) + + const ( + serviceName = "my-service-xds-http-connect" + routeConfigName = "route-my-service-xds-http-connect" + clusterName = "cluster-my-service-xds-http-connect" + endpointsName = "endpoints-my-service-xds-http-connect" + ) + + cluster := &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}}, + ServiceName: endpointsName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.http_11_proxy", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{ + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + }, + }, + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + }, + }, + }), + }, + }, + }), + }, + }, + } + + loadAssignment := &v3endpointpb.ClusterLoadAssignment{ + ClusterName: endpointsName, + Endpoints: []*v3endpointpb.LocalityLbEndpoints{ + { + Locality: &v3corepb.Locality{SubZone: "locality-1"}, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + LbEndpoints: []*v3endpointpb.LbEndpoint{ + { + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: backendHost, + PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(backendPort)}, + }, + }, + }, + }, + }, + Metadata: &v3corepb.Metadata{ + TypedFilterMetadata: map[string]*anypb.Any{ + "envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: proxyHost, + PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(proxyPort)}, + }, + }, + }), + }, + }, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + }, + }, + }, + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{loadAssignment}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("Failed to create client XDS credentials: %v", err) + } + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Verify that the request was routed through the proxy to the expected + // backend. + select { + case gotTarget := <-connectCh: + if gotTarget != server.Address { + t.Errorf("Unexpected server address from proxy server, got %s want %s", gotTarget, server.Address) + } + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout while waiting for server address from proxy server") + } +} + +// Test verifies client-side HTTP CONNECT proxying support when cluster +// specifies an HTTP CONNECT proxy transport socket, but no proxy address +// metadata is supplied. It verifies that the client gracefully fallback +// to dialing the backend directly. +func (s) TestXDSHTTPConnect_NoMetadata(t *testing.T) { + enableXDSHTTPConnect(t) + + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const ( + serviceName = "my-service-xds-http-connect" + routeConfigName = "route-my-service-xds-http-connect" + clusterName = "cluster-my-service-xds-http-connect" + endpointsName = "endpoints-my-service-xds-http-connect" + ) + + cluster := &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}}, + ServiceName: endpointsName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.http_11_proxy", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{})}, + }, + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} diff --git a/test/xds/xds_client_ignore_resource_deletion_test.go b/test/xds/xds_client_ignore_resource_deletion_test.go new file mode 100644 index 0000000..dd55adb --- /dev/null +++ b/test/xds/xds_client_ignore_resource_deletion_test.go @@ -0,0 +1,502 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "encoding/json" + "fmt" + "net" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds" + + clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const ( + serviceName = "my-service-xds" + rdsName = "route-" + serviceName + cdsName1 = "cluster1-" + serviceName + cdsName2 = "cluster2-" + serviceName + edsName1 = "eds1-" + serviceName + edsName2 = "eds2-" + serviceName +) + +var ( + // This route configuration resource contains two routes: + // - a route for the EmptyCall rpc, to be sent to cluster1 + // - a route for the UnaryCall rpc, to be sent to cluster2 + defaultRouteConfigWithTwoRoutes = &routepb.RouteConfiguration{ + Name: rdsName, + VirtualHosts: []*routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*routepb.Route{ + { + Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}}, + Action: &routepb.Route_Route{Route: &routepb.RouteAction{ + ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: cdsName1}, + }}, + }, + { + Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}}, + Action: &routepb.Route_Route{Route: &routepb.RouteAction{ + ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: cdsName2}, + }}, + }, + }, + }}, + } +) + +// This test runs subtest each for a Listener resource and a Cluster resource deletion +// in the response from the server for the following cases: +// - testResourceDeletionIgnored: When ignore_resource_deletion is set, the +// xDSClient should not delete the resource. +// - testResourceDeletionNotIgnored: When ignore_resource_deletion is unset, +// the xDSClient should delete the resource. +// +// Resource deletion is only applicable to Listener and Cluster resources. +func (s) TestIgnoreResourceDeletionOnClient(t *testing.T) { + server1 := stubserver.StartTestService(t, nil) + t.Cleanup(server1.Stop) + + server2 := stubserver.StartTestService(t, nil) + t.Cleanup(server2.Stop) + + initialResourceOnServer := func(nodeID string) e2e.UpdateOptions { + return e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)}, + Routes: []*routepb.RouteConfiguration{defaultRouteConfigWithTwoRoutes}, + Clusters: []*clusterpb.Cluster{ + e2e.DefaultCluster(cdsName1, edsName1, e2e.SecurityLevelNone), + e2e.DefaultCluster(cdsName2, edsName2, e2e.SecurityLevelNone), + }, + Endpoints: []*endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(edsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}), + e2e.DefaultEndpoint(edsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}), + }, + SkipValidation: true, + } + } + + tests := []struct { + name string + updateResource func(r *e2e.UpdateOptions) + }{ + { + name: "listener", + updateResource: func(r *e2e.UpdateOptions) { + r.Listeners = nil + }, + }, + { + name: "cluster", + updateResource: func(r *e2e.UpdateOptions) { + r.Clusters = nil + }, + }, + } + for _, test := range tests { + t.Run(fmt.Sprintf("%s resource deletion ignored", test.name), func(t *testing.T) { + testResourceDeletionIgnored(t, initialResourceOnServer, test.updateResource) + }) + t.Run(fmt.Sprintf("%s resource deletion not ignored", test.name), func(t *testing.T) { + testResourceDeletionNotIgnored(t, initialResourceOnServer, test.updateResource) + }) + } +} + +// This subtest tests the scenario where the bootstrap config has "ignore_resource_deletion" +// set in "server_features" field. This subtest verifies that the resource was +// not deleted by the xDSClient when a resource is missing the xDS response and +// RPCs continue to succeed. +func testResourceDeletionIgnored(t *testing.T, initialResource func(string) e2e.UpdateOptions, updateResource func(r *e2e.UpdateOptions)) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + t.Cleanup(cancel) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + nodeID := uuid.New().String() + bs := generateBootstrapContents(t, mgmtServer.Address, true, nodeID) + xdsR := xdsResolverBuilder(t, bs) + resources := initialResource(nodeID) + + // Update the management server with initial resources setup. + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR)) + if err != nil { + t.Fatalf("Failed to dial local test server: %v.", err) + } + t.Cleanup(func() { cc.Close() }) + + if err := verifyRPCtoAllEndpoints(cc); err != nil { + t.Fatal(err) + } + + // Mutate resource and update on the server. + updateResource(&resources) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Make an RPC every 50ms for the next 500ms. This is to ensure that the + // updated resource is received from the management server and is processed by + // gRPC. Since resource deletions are ignored by the xDS client, we expect RPCs + // to all endpoints to keep succeeding. + timer := time.NewTimer(500 * time.Millisecond) + ticker := time.NewTicker(50 * time.Millisecond) + t.Cleanup(ticker.Stop) + for { + if err := verifyRPCtoAllEndpoints(cc); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + return + case <-timer.C: + return + case <-ticker.C: + } + } +} + +// This subtest tests the scenario where the bootstrap config has "ignore_resource_deletion" +// not set in "server_features" field. This subtest verifies that the resource was +// deleted by the xDSClient when a resource is missing the xDS response and subsequent +// RPCs fail. +func testResourceDeletionNotIgnored(t *testing.T, initialResource func(string) e2e.UpdateOptions, updateResource func(r *e2e.UpdateOptions)) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + t.Cleanup(cancel) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + nodeID := uuid.New().String() + bs := generateBootstrapContents(t, mgmtServer.Address, false, nodeID) + xdsR := xdsResolverBuilder(t, bs) + resources := initialResource(nodeID) + + // Update the management server with initial resources setup. + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + if err := verifyRPCtoAllEndpoints(cc); err != nil { + t.Fatal(err) + } + + // Mutate resource and update on the server. + updateResource(&resources) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Spin up go routines to verify RPCs fail after the update. The xDS node ID + // needs to be part of the error seen by the RPC caller. + client := testgrpc.NewTestServiceClient(cc) + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + continue + } + if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), nodeID) { + return + } + } + }() + go func() { + defer wg.Done() + for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) { + _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if err == nil { + continue + } + if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), nodeID) { + return + } + } + }() + + wg.Wait() + if ctx.Err() != nil { + t.Fatal("Context expired before RPCs failed.") + } +} + +// This helper generates a custom bootstrap config for the test. +func generateBootstrapContents(t *testing.T, serverURI string, ignoreResourceDeletion bool, nodeID string) []byte { + t.Helper() + var serverCfgs json.RawMessage + if ignoreResourceDeletion { + serverCfgs = []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}], + "server_features": ["ignore_resource_deletion"] + }]`, serverURI)) + } else { + serverCfgs = []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, serverURI)) + + } + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: serverCfgs, + Node: fmt.Appendf(nil, `{"id": "%s"}`, nodeID), + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatal(err) + } + return bootstrapContents +} + +// This helper creates an XDS resolver Builder from the bootstrap config passed +// as parameter. +func xdsResolverBuilder(t *testing.T, bs []byte) resolver.Builder { + t.Helper() + xdsR, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bs) + if err != nil { + t.Fatalf("Creating xDS resolver for testing failed for config %q: %v", string(bs), err) + } + return xdsR +} + +// This helper creates an xDS-enabled gRPC server using the listener and the +// bootstrap config passed. It then registers the test service on the newly +// created gRPC server and starts serving. +func setupGRPCServerWithModeChangeChannelAndServe(t *testing.T, bootstrapContents []byte, lis net.Listener) chan connectivity.ServingMode { + t.Helper() + updateCh := make(chan connectivity.ServingMode, 1) + + // Create a server option to get notified about serving mode changes. + modeChangeOpt := xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) { + t.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err) + updateCh <- args.Mode + }) + stub := &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + } + server, err := xds.NewGRPCServer(grpc.Creds(insecure.NewCredentials()), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents)) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + stub.S = server + t.Cleanup(stub.S.Stop) + + stubserver.StartTestService(t, stub) + + return updateCh +} + +// This helper creates a new TCP listener. This helper also uses this listener to +// create a resource update with a listener resource. This helper returns the +// resource update and the TCP listener. +func resourceWithListenerForGRPCServer(t *testing.T, nodeID string) (e2e.UpdateOptions, net.Listener) { + t.Helper() + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + t.Cleanup(func() { lis.Close() }) + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of listener at %q: %v", lis.Addr(), err) + } + listener := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName") + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*listenerpb.Listener{listener}, + } + return resources, lis +} + +// This test creates a gRPC server which provides server-side xDS functionality +// by talking to a custom management server. This tests the scenario where bootstrap +// config with "server_features" includes "ignore_resource_deletion". In which +// case, when the listener resource is deleted on the management server, the gRPC +// server should continue to serve RPCs. +func (s) TestListenerResourceDeletionOnServerIgnored(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + nodeID := uuid.New().String() + bs := generateBootstrapContents(t, mgmtServer.Address, true, nodeID) + xdsR := xdsResolverBuilder(t, bs) + resources, lis := resourceWithListenerForGRPCServer(t, nodeID) + modeChangeCh := setupGRPCServerWithModeChangeChannelAndServe(t, bs, lis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the server to update to ServingModeServing mode. + select { + case <-ctx.Done(): + t.Fatal("Test timed out waiting for a server to change to ServingModeServing.") + case mode := <-modeChangeCh: + if mode != connectivity.ServingModeServing { + t.Fatalf("Server switched to mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + + // Create a ClientConn and make a successful RPCs. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + if err := verifyRPCtoAllEndpoints(cc); err != nil { + t.Fatal(err) + } + + // Update without a listener resource. + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*listenerpb.Listener{}, + }); err != nil { + t.Fatal(err) + } + + // Perform RPCs every 100 ms for 1s and verify that the serving mode does not + // change on gRPC server. + timer := time.NewTimer(500 * time.Millisecond) + ticker := time.NewTicker(50 * time.Millisecond) + t.Cleanup(ticker.Stop) + for { + if err := verifyRPCtoAllEndpoints(cc); err != nil { + t.Fatal(err) + } + select { + case <-timer.C: + return + case mode := <-modeChangeCh: + t.Fatalf("Server switched to mode: %v when no switch was expected", mode) + case <-ticker.C: + } + } +} + +// This test creates a gRPC server which provides server-side xDS functionality +// by talking to a custom management server. This tests the scenario where bootstrap +// config with "server_features" does not include "ignore_resource_deletion". In +// which case, when the listener resource is deleted on the management server, the +// gRPC server should stop serving RPCs and switch mode to ServingModeNotServing. +func (s) TestListenerResourceDeletionOnServerNotIgnored(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + nodeID := uuid.New().String() + bs := generateBootstrapContents(t, mgmtServer.Address, false, nodeID) + xdsR := xdsResolverBuilder(t, bs) + resources, lis := resourceWithListenerForGRPCServer(t, nodeID) + updateCh := setupGRPCServerWithModeChangeChannelAndServe(t, bs, lis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the listener to move to "serving" mode. + select { + case <-ctx.Done(): + t.Fatal("Test timed out waiting for a mode change update.") + case mode := <-updateCh: + if mode != connectivity.ServingModeServing { + t.Fatalf("Listener received new mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + + // Create a ClientConn and make a successful RPCs. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + if err := verifyRPCtoAllEndpoints(cc); err != nil { + t.Fatal(err) + } + + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*listenerpb.Listener{}, // empty listener resource + }); err != nil { + t.Fatal(err) + } + + select { + case <-ctx.Done(): + t.Fatalf("timed out waiting for a mode change update: %v", err) + case mode := <-updateCh: + if mode != connectivity.ServingModeNotServing { + t.Fatalf("listener received new mode %v, want %v", mode, connectivity.ServingModeNotServing) + } + } +} + +// This helper makes both UnaryCall and EmptyCall RPCs using the ClientConn that +// is passed to this function. This helper panics for any failed RPCs. +func verifyRPCtoAllEndpoints(cc grpc.ClientConnInterface) error { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return fmt.Errorf("rpc EmptyCall() failed: %v", err) + } + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + return fmt.Errorf("rpc UnaryCall() failed: %v", err) + } + return nil +} diff --git a/test/xds/xds_client_integration_test.go b/test/xds/xds_client_integration_test.go new file mode 100644 index 0000000..3898a3a --- /dev/null +++ b/test/xds/xds_client_integration_test.go @@ -0,0 +1,244 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) + +func (s) TestClientSideXDS(t *testing.T) { + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} + +// TestClient_ConcurrentRPC ensures thread safety for xDS clients executing +// concurrent RPCs, particularly verifying that the regex matchers do not cause +// data races. +func (s) TestClient_ConcurrentRPC(t *testing.T) { + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName = "cluster-" + serviceName + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_SafeRegex{ + SafeRegex: &v3matcherpb.RegexMatcher{Regex: "/.*"}, + }, + }, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }}, + }}, + }}, + }}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, clusterName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + + const numRPCs = 20 + wg := sync.WaitGroup{} + for i := 0; i < numRPCs; i++ { + wg.Go(func() { + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("EmptyCall() failed: %v", err) + } + }) + } + wg.Wait() +} + +// Test verifies that client-side xDS routing strictly evaluates +// OutgoingContext metadata and ignores IncomingContext metadata. This +// prevents incoming headers on a server handler context from leaking +// into and contaminating outgoing client routing decisions. +func (s) TestClientSideXDS_RouteMatching_IgnoreIncomingMetadata(t *testing.T) { + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + serverA := stubserver.StartTestService(t, nil) + defer serverA.Stop() + serverB := stubserver.StartTestService(t, nil) + defer serverB.Stop() + + const ( + serviceName = "my-service-client" + routeConfigName = "route-" + serviceName + clusterA = "cluster-A-" + serviceName + clusterB = "cluster-B-" + serviceName + headerName = "x-route-target" + headerValue = "cluster-a" + ) + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + Headers: []*v3routepb.HeaderMatcher{{ + Name: headerName, + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ + ExactMatch: headerValue, + }, + }}, + }, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterA}, + }}, + }, + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + }, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterB}, + }}, + }, + }, + }}, + }}, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone), + e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, serverA.Address)}), + e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, serverB.Address)}), + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + // Simulate an intermediary server that receives an incoming RPC containing + // the header "x-route-target: cluster-a" in its IncomingContext, and then + // initiates an outgoing client RPC using that same context without setting + // any OutgoingContext metadata. Client-side routing must strictly evaluate + // OutgoingContext (which is empty) and select Route 2. + incomingCtx := metadata.NewIncomingContext(ctx, metadata.Pairs(headerName, headerValue)) + var p peer.Peer + if _, err := client.EmptyCall(incomingCtx, &testpb.Empty{}, grpc.Peer(&p)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if p.Addr.String() != serverB.Address { + t.Fatalf("Unexpected client routing to peer %s, want fallback Route 2 (clusterB at %s)", p.Addr.String(), serverB.Address) + } +} diff --git a/test/xds/xds_client_outlier_detection_test.go b/test/xds/xds_client_outlier_detection_test.go new file mode 100644 index 0000000..4df142f --- /dev/null +++ b/test/xds/xds_client_outlier_detection_test.go @@ -0,0 +1,323 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// TestOutlierDetection_NoopConfig tests the scenario where the Outlier +// Detection feature is enabled on the gRPC client, but it receives no Outlier +// Detection configuration from the management server. This should result in a +// no-op Outlier Detection configuration being used to configure the Outlier +// Detection balancer. This test verifies that an RPC is able to proceed +// normally with this configuration. +func (s) TestOutlierDetection_NoopConfig(t *testing.T) { + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + server := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + server.StartServer() + t.Logf("Started test service backend at %q", server.Address) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} + +// clientResourcesMultipleBackendsAndOD returns xDS resources which correspond +// to multiple upstreams, corresponding different backends listening on +// different localhost:port combinations. The resources also configure an +// Outlier Detection Balancer configured through the passed in Outlier Detection +// proto. +func clientResourcesMultipleBackendsAndOD(params e2e.ResourceParams, ports []uint32, od *v3clusterpb.OutlierDetection) e2e.UpdateOptions { + routeConfigName := "route-" + params.DialTarget + clusterName := "cluster-" + params.DialTarget + endpointsName := "endpoints-" + params.DialTarget + return e2e.UpdateOptions{ + NodeID: params.NodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(params.DialTarget, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, params.DialTarget, clusterName)}, + Clusters: []*v3clusterpb.Cluster{clusterWithOutlierDetection(clusterName, endpointsName, params.SecLevel, od)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, params.Host, ports)}, + } +} + +func clusterWithOutlierDetection(clusterName, edsServiceName string, secLevel e2e.SecurityLevel, od *v3clusterpb.OutlierDetection) *v3clusterpb.Cluster { + cluster := e2e.DefaultCluster(clusterName, edsServiceName, secLevel) + cluster.OutlierDetection = od + return cluster +} + +// checkRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn, +// connected to a server exposing the test.grpc_testing.TestService, are +// roundrobined across the given backend addresses. +// +// Returns a non-nil error if context deadline expires before RPCs start to get +// roundrobined across the given backends. +func checkRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error { + wantAddrCount := make(map[string]int) + for _, addr := range addrs { + wantAddrCount[addr.Addr]++ + } + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + // Perform 3 iterations. + var iterations [][]string + for i := 0; i < 3; i++ { + iteration := make([]string, len(addrs)) + for c := 0; c < len(addrs); c++ { + var peer peer.Peer + client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)) + if peer.Addr != nil { + iteration[c] = peer.Addr.String() + } + } + iterations = append(iterations, iteration) + } + // Ensure the first iteration contains all addresses in addrs. + gotAddrCount := make(map[string]int) + for _, addr := range iterations[0] { + gotAddrCount[addr]++ + } + if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" { + continue + } + // Ensure all three iterations contain the same addresses. + if !cmp.Equal(iterations[0], iterations[1]) || !cmp.Equal(iterations[0], iterations[2]) { + continue + } + return nil + } + return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v", addrs) +} + +// TestOutlierDetectionWithOutlier tests the Outlier Detection Balancer e2e. It +// spins up three backends, one which consistently errors, and configures the +// ClientConn using xDS to connect to all three of those backends. The Outlier +// Detection Balancer should eject the connection to the backend which +// constantly errors, causing RPC's to not be routed to that upstream, and only +// be Round Robined across the two healthy upstreams. Other than the intervals +// the unhealthy upstream is ejected, RPC's should regularly round robin across +// all three upstreams. +func (s) TestOutlierDetectionWithOutlier(t *testing.T) { + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + // Working backend 1. + backend1 := stubserver.StartTestService(t, nil) + port1 := testutils.ParsePort(t, backend1.Address) + defer backend1.Stop() + + // Working backend 2. + backend2 := stubserver.StartTestService(t, nil) + port2 := testutils.ParsePort(t, backend2.Address) + defer backend2.Stop() + + // Backend 3 that will always return an error and eventually ejected. + backend3 := stubserver.StartTestService(t, &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return nil, errors.New("some error") }, + }) + port3 := testutils.ParsePort(t, backend3.Address) + defer backend3.Stop() + + const serviceName = "my-service-client-side-xds" + resources := clientResourcesMultipleBackendsAndOD(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + SecLevel: e2e.SecurityLevelNone, + }, []uint32{port1, port2, port3}, &v3clusterpb.OutlierDetection{ + Interval: &durationpb.Duration{Nanos: 50000000}, // .5 seconds + BaseEjectionTime: &durationpb.Duration{Seconds: 30}, + MaxEjectionTime: &durationpb.Duration{Seconds: 300}, + MaxEjectionPercent: &wrapperspb.UInt32Value{Value: 1}, + FailurePercentageThreshold: &wrapperspb.UInt32Value{Value: 50}, + EnforcingFailurePercentage: &wrapperspb.UInt32Value{Value: 100}, + FailurePercentageRequestVolume: &wrapperspb.UInt32Value{Value: 8}, + FailurePercentageMinimumHosts: &wrapperspb.UInt32Value{Value: 3}, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + fullAddresses := []resolver.Address{ + {Addr: backend1.Address}, + {Addr: backend2.Address}, + {Addr: backend3.Address}, + } + // At first, due to no statistics on each of the backends, the 3 + // upstreams should all be round robined across. + if err = checkRoundRobinRPCs(ctx, client, fullAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // The addresses which don't return errors. + okAddresses := []resolver.Address{ + {Addr: backend1.Address}, + {Addr: backend2.Address}, + } + // After calling the three upstreams, one of them constantly error + // and should eventually be ejected for a period of time. This + // period of time should cause the RPC's to be round robined only + // across the two that are healthy. + if err = checkRoundRobinRPCs(ctx, client, okAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } +} + +// TestOutlierDetectionXDSDefaultOn tests that Outlier Detection is by default +// configured on in the xDS Flow. If the Outlier Detection proto message is +// present with SuccessRateEjection unset, then Outlier Detection should be +// turned on. The test setups and xDS system with xDS resources with Outlier +// Detection present in the CDS update, but with SuccessRateEjection unset, and +// asserts that Outlier Detection is turned on and ejects upstreams. +func (s) TestOutlierDetectionXDSDefaultOn(t *testing.T) { + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + // Working backend 1. + backend1 := stubserver.StartTestService(t, nil) + port1 := testutils.ParsePort(t, backend1.Address) + defer backend1.Stop() + + // Working backend 2. + backend2 := stubserver.StartTestService(t, nil) + port2 := testutils.ParsePort(t, backend2.Address) + defer backend2.Stop() + + // Backend 3 that will always return an error and eventually ejected. + backend3 := stubserver.StartTestService(t, &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return nil, errors.New("some error") }, + }) + port3 := testutils.ParsePort(t, backend3.Address) + defer backend3.Stop() + + // Configure CDS resources with Outlier Detection set but + // EnforcingSuccessRate unset. This should cause Outlier Detection to be + // configured with SuccessRateEjection present in configuration, which will + // eventually be populated with its default values along with the knobs set + // as SuccessRate fields in the proto, and thus Outlier Detection should be + // on and actively eject upstreams. + const serviceName = "my-service-client-side-xds" + resources := clientResourcesMultipleBackendsAndOD(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + SecLevel: e2e.SecurityLevelNone, + }, []uint32{port1, port2, port3}, &v3clusterpb.OutlierDetection{ + // Need to set knobs to trigger ejection within the test time frame. + Interval: &durationpb.Duration{Nanos: 50000000}, + // EnforcingSuccessRateSet to nil, causes success rate algorithm to be + // turned on. + SuccessRateMinimumHosts: &wrapperspb.UInt32Value{Value: 1}, + SuccessRateRequestVolume: &wrapperspb.UInt32Value{Value: 8}, + SuccessRateStdevFactor: &wrapperspb.UInt32Value{Value: 1}, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + fullAddresses := []resolver.Address{ + {Addr: backend1.Address}, + {Addr: backend2.Address}, + {Addr: backend3.Address}, + } + // At first, due to no statistics on each of the backends, the 3 + // upstreams should all be round robined across. + if err = checkRoundRobinRPCs(ctx, client, fullAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // The addresses which don't return errors. + okAddresses := []resolver.Address{ + {Addr: backend1.Address}, + {Addr: backend2.Address}, + } + // After calling the three upstreams, one of them constantly error + // and should eventually be ejected for a period of time. This + // period of time should cause the RPC's to be round robined only + // across the two that are healthy. + if err = checkRoundRobinRPCs(ctx, client, okAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } +} diff --git a/test/xds/xds_client_priority_locality_test.go b/test/xds/xds_client_priority_locality_test.go new file mode 100644 index 0000000..607e6ea --- /dev/null +++ b/test/xds/xds_client_priority_locality_test.go @@ -0,0 +1,211 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + rrutil "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/resolver" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +// backendAddressesAndPorts extracts the address and port of each of the +// StubServers passed in and returns them. Fails the test if any of the +// StubServers passed have an invalid address. +func backendAddressesAndPorts(t *testing.T, servers []*stubserver.StubServer) ([]resolver.Address, []uint32) { + addrs := make([]resolver.Address, len(servers)) + ports := make([]uint32, len(servers)) + for i := 0; i < len(servers); i++ { + addrs[i] = resolver.Address{Addr: servers[i].Address} + ports[i] = testutils.ParsePort(t, servers[i].Address) + } + return addrs, ports +} + +// Tests scenarios involving localities moving between priorities. +// - The test starts off with a cluster that contains two priorities, one +// locality in each, and one endpoint in each. Verifies that traffic reaches +// the endpoint in the higher priority. +// - The test then moves the locality in the lower priority over to the higher +// priority. At that point, we would have a cluster with a single priority, +// but two localities, and one endpoint in each. Verifies that traffic is +// split between the endpoints. +// - The test then deletes the locality that was originally in the higher +// priority.Verifies that all traffic is now reaching the only remaining +// endpoint. +func (s) TestClientSideXDS_LocalityChangesPriority(t *testing.T) { + // Spin up a management server and two test service backends. + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + backend0 := stubserver.StartTestService(t, nil) + defer backend0.Stop() + backend1 := stubserver.StartTestService(t, nil) + defer backend1.Stop() + addrs, ports := backendAddressesAndPorts(t, []*stubserver.StubServer{backend0, backend1}) + + // Configure resources on the management server. We use default client side + // resources for listener, route configuration and cluster. For the + // endpoints resource though, we create one with two priorities, and one + // locality each, and one endpoint each. + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName = "cluster-" + serviceName + const endpointsName = "endpoints-" + serviceName + locality1 := e2e.LocalityID{Region: "my-region-1", Zone: "my-zone-1", SubZone: "my-subzone-1"} + locality2 := e2e.LocalityID{Region: "my-region-2", Zone: "my-zone-2", SubZone: "my-subzone-2"} + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: endpointsName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Name: "my-locality-1", + Weight: 1000000, + Priority: 0, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[0]}}}, + Locality: locality1, + }, + { + Name: "my-locality-2", + Weight: 1000000, + Priority: 1, + Backends: []e2e.BackendOptions{{Ports: []uint32{ports[1]}}}, + Locality: locality2, + }, + }, + })}, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // // Ensure that RPCs get routed to the backend in the higher priority. + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil { + t.Fatal(err) + } + + // Update the endpoints resource to contain a single priority with two + // localities, and one endpoint each. The locality weights are equal at this + // point, and we expect RPCs to be round-robined across the two localities. + resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: endpointsName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Name: "my-locality-1", + Weight: 500000, + Priority: 0, + Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend0.Address)}}}, + Locality: locality1, + }, + { + Name: "my-locality-2", + Weight: 500000, + Priority: 0, + Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend1.Address)}}}, + Locality: locality2, + }, + }, + })} + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil { + t.Fatal(err) + } + + // Update the locality weights ever so slightly. We still expect RPCs to be + // round-robined across the two localities. + resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: endpointsName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Name: "my-locality-1", + Weight: 499884, + Priority: 0, + Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend0.Address)}}}, + Locality: locality1, + }, + { + Name: "my-locality-2", + Weight: 500115, + Priority: 0, + Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend1.Address)}}}, + Locality: locality2, + }, + }, + })} + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil { + t.Fatal(err) + } + + // Update the endpoints resource to contain a single priority with one + // locality. The locality which was originally in the higher priority is now + // dropped. + resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: endpointsName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Name: "my-locality-2", + Weight: 1000000, + Priority: 0, + Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend1.Address)}}}, + Locality: locality2, + }, + }, + })} + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil { + t.Fatal(err) + } +} diff --git a/test/xds/xds_client_retry_test.go b/test/xds/xds_client_retry_test.go new file mode 100644 index 0000000..aebcef6 --- /dev/null +++ b/test/xds/xds_client_retry_test.go @@ -0,0 +1,192 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "sync" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestClientSideRetry(t *testing.T) { + // serverMu guards access to ctr and errs. + serverMu := sync.Mutex{} + ctr := 0 + errs := []codes.Code{codes.ResourceExhausted} + + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + server := stubserver.StartTestService(t, &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + serverMu.Lock() + defer func() { + ctr++ + serverMu.Unlock() + }() + if ctr < len(errs) { + return nil, status.Errorf(errs[ctr], "this should be retried") + } + return &testpb.Empty{}, nil + }, + }) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + defer cancel() + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.ResourceExhausted { + t.Fatalf("rpc EmptyCall() = _, %v; want _, ResourceExhausted", err) + } + + testCases := []struct { + name string + vhPolicy *v3routepb.RetryPolicy + routePolicy *v3routepb.RetryPolicy + errs []codes.Code // the errors returned by the server for each RPC + tryAgainErr codes.Code // the error that would be returned if we are still using the old retry policies. + errWant codes.Code + }{{ + name: "virtualHost only, fail", + vhPolicy: &v3routepb.RetryPolicy{ + RetryOn: "resource-exhausted,unavailable", + NumRetries: &wrapperspb.UInt32Value{Value: 1}, + }, + errs: []codes.Code{codes.ResourceExhausted, codes.Unavailable}, + routePolicy: nil, + tryAgainErr: codes.ResourceExhausted, + errWant: codes.Unavailable, + }, { + name: "virtualHost only", + vhPolicy: &v3routepb.RetryPolicy{ + RetryOn: "resource-exhausted, unavailable", + NumRetries: &wrapperspb.UInt32Value{Value: 2}, + }, + errs: []codes.Code{codes.ResourceExhausted, codes.Unavailable}, + routePolicy: nil, + tryAgainErr: codes.Unavailable, + errWant: codes.OK, + }, { + name: "virtualHost+route, fail", + vhPolicy: &v3routepb.RetryPolicy{ + RetryOn: "resource-exhausted,unavailable", + NumRetries: &wrapperspb.UInt32Value{Value: 2}, + }, + routePolicy: &v3routepb.RetryPolicy{ + RetryOn: "resource-exhausted", + NumRetries: &wrapperspb.UInt32Value{Value: 2}, + }, + errs: []codes.Code{codes.ResourceExhausted, codes.Unavailable}, + tryAgainErr: codes.OK, + errWant: codes.Unavailable, + }, { + name: "virtualHost+route", + vhPolicy: &v3routepb.RetryPolicy{ + RetryOn: "resource-exhausted", + NumRetries: &wrapperspb.UInt32Value{Value: 2}, + }, + routePolicy: &v3routepb.RetryPolicy{ + RetryOn: "unavailable", + NumRetries: &wrapperspb.UInt32Value{Value: 2}, + }, + errs: []codes.Code{codes.Unavailable}, + tryAgainErr: codes.Unavailable, + errWant: codes.OK, + }, { + name: "virtualHost+route, not enough attempts", + vhPolicy: &v3routepb.RetryPolicy{ + RetryOn: "unavailable", + NumRetries: &wrapperspb.UInt32Value{Value: 2}, + }, + routePolicy: &v3routepb.RetryPolicy{ + RetryOn: "unavailable", + NumRetries: &wrapperspb.UInt32Value{Value: 1}, + }, + errs: []codes.Code{codes.Unavailable, codes.Unavailable}, + tryAgainErr: codes.OK, + errWant: codes.Unavailable, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + serverMu.Lock() + errs = tc.errs + + // Confirm tryAgainErr is correct before updating resources. + ctr = 0 + serverMu.Unlock() + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if code := status.Code(err); code != tc.tryAgainErr { + t.Fatalf("with old retry policy: EmptyCall() = _, %v; want _, %v", err, tc.tryAgainErr) + } + + resources.Routes[0].VirtualHosts[0].RetryPolicy = tc.vhPolicy + resources.Routes[0].VirtualHosts[0].Routes[0].GetRoute().RetryPolicy = tc.routePolicy + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + for { + serverMu.Lock() + ctr = 0 + serverMu.Unlock() + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if code := status.Code(err); code == tc.tryAgainErr { + continue + } else if code != tc.errWant { + t.Fatalf("rpc EmptyCall() = _, %v; want _, %v", err, tc.errWant) + } + break + } + }) + } +} diff --git a/test/xds/xds_client_sni_test.go b/test/xds/xds_client_sni_test.go new file mode 100644 index 0000000..335641e --- /dev/null +++ b/test/xds/xds_client_sni_test.go @@ -0,0 +1,851 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "crypto/tls" + "fmt" + "strings" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// validSNI matches the wildcard SAN (*.test.example.com) in the test server +// certificate. +const validSNI = "x.test.example.com" + +// Tests the SNI and SAN validation logic by verifying that RPCs succeed when +// AutoSNISANValidation is enabled and the SNI matches a server certificate DNS +// SAN. Also verifies that RPCs fail with an 'Unavailable' status if the SNI is +// present but does not match any DNS SAN in the certificate. +func (s) TestClientSideXDS_SNISANValidation(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true) + + // Spin up an xDS management server. + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + // Create test backends for two clusters: + // - backend1 configured with TLS creds, represents cluster1 (valid SNI) + // - backend2 configured with TLS creds, represents cluster2 (invalid SNI) + serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert) + server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server1.Stop() + server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server2.Stop() + + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName1 = "cluster1-" + serviceName + const clusterName2 = "cluster2-" + serviceName + const endpointName1 = "endpoint1-" + serviceName + const endpointName2 = "endpoint2-" + serviceName + + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)} + + // Route configuration: + // - "/grpc.testing.TestService/EmptyCall" --> cluster1 (valid SNI) + // - "/grpc.testing.TestService/UnaryCall" --> cluster2 (invalid SNI) + routes := []*v3routepb.RouteConfiguration{{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}}}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}}}, + }, + }, + }}, + }} + + // Configure cluster1 with valid SNI and AutoSniSanValidation set to true. + cluster1 := e2e.DefaultCluster(clusterName1, endpointName1, e2e.SecurityLevelMTLS) + cluster1.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + Sni: validSNI, + AutoSniSanValidation: true, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: e2e.ClientSideCertProviderInstance}, + }, + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: e2e.ClientSideCertProviderInstance}, + }, + }), + }, + } + + // Configure cluster2 with invalid SNI and AutoSniSanValidation set to true. + cluster2 := e2e.DefaultCluster(clusterName2, endpointName2, e2e.SecurityLevelMTLS) + cluster2.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + Sni: "wrong.sni.domain", + AutoSniSanValidation: true, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "root", + }, + }, + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "identity", + }, + }, + }), + }, + } + + clusters := []*v3clusterpb.Cluster{cluster1, cluster2} + endpoints := []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(endpointName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}), + e2e.DefaultEndpoint(endpointName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}), + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + // RPC to cluster1 should succeed because auto_sni_san_validation is true + // and sni matches server cert SAN. + peerInfo := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if got, want := peerInfo.Addr.String(), server1.Address; got != want { + t.Fatalf("EmptyCall() routed to %q, want to be routed to: %q", got, want) + } + + // RPC to cluster2 should fail because even though auto_sni_san_validation + // is true, sni doesn't match server cert SAN. + const wantErr = "do not match the SNI" + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("UnaryCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr) + } +} + +// Tests that when AutoHostSNI is enabled, the endpoint's hostname is used for +// the ServerName even though SNI is specified. It verifies that the TLS +// handshake and subsequent RPC succeed because the resolved SNI i.e. the +// hostname matches the server's certificate SAN. +func (s) TestClientSideXDS_AutoHostSNI(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true) + + // Spin up an xDS management server. + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + // Create test backend + serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert) + server := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server.Stop() + + // Configure client side xDS resources on the management server. + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName = "cluster-" + serviceName + const endpointName = "endpoint-" + serviceName + + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)} + routes := []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)} + + // Configure the cluster with AutoHostSni and AutoSniSanValidation set to + // true with a invalid SNI. + cluster := e2e.DefaultCluster(clusterName, endpointName, e2e.SecurityLevelMTLS) + cluster.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + AutoHostSni: true, + AutoSniSanValidation: true, + Sni: "wrong.sni.domain", + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "root", + }, + }, + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "identity", + }, + }, + }), + }, + } + + // Endpoints configuring Hostname to the defaultTestCertSAN to verify + // AutoHostSni usage. + endpoints := []*v3endpointpb.ClusterLoadAssignment{ + e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: endpointName, + Host: "localhost", + Localities: []e2e.LocalityOptions{{ + Weight: 1, + Backends: []e2e.BackendOptions{{ + Ports: []uint32{testutils.ParsePort(t, server.Address)}, + Hostname: validSNI, + }}, + }}, + }), + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: endpoints, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + // RPC should succeed because auto_host_sni sets SNI from the endpoint + // hostname and auto_sni_san_validation validates that the SNI matches + // server cert SAN. + peerInfo := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if got, want := peerInfo.Addr.String(), server.Address; got != want { + t.Fatalf("EmptyCall() routed to %q, want to be routed to: %q", got, want) + } +} + +// Tests the scenario where multiple endpoints are configured with distinct +// hostnames. It verifies that the client uses the specific hostname associated +// with each endpoint during the TLS handshake by simulating an EDS failover and +// confirming that the SNI is dynamically updated to match the active endpoint. +func (s) TestClientSideXDS_AutoHostSNI_MultipleEndpoints(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true) + + // Spin up an xDS management server. + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + // Create test backends. + serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert) + server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server1.Stop() + server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server2.Stop() + + // Configure client side xDS resources on the management server. + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName = "cluster-" + serviceName + const endpointName = "endpoint-" + serviceName + + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)} + routes := []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)} + + clusters := []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointName, e2e.SecurityLevelMTLS)} + clusters[0].TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + AutoHostSni: true, + AutoSniSanValidation: true, + Sni: "wrong.sni.domain", + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "root", + }, + }, + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "identity", + }, + }, + }), + }, + } + + // Configure only Priority 0 with an invalid hostname. RPC should fail due + // to SAN mismatch. + endpoints := []*v3endpointpb.ClusterLoadAssignment{ + e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: endpointName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Name: "locality-0", + Weight: 1, + Priority: 0, + Backends: []e2e.BackendOptions{{ + Ports: []uint32{testutils.ParsePort(t, server1.Address)}, + Hostname: "wrong.sni.domain", + }}, + }, + }, + }), + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + // RPC should fail because hostname for endpoints in Priority 0 is invalid. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Fatalf("EmptyCall() unexpectedly succeeded with invalid SNI and no failover") + } else if !strings.Contains(err.Error(), "authentication handshake failed") { + t.Fatalf("EmptyCall() failed with %v, want error containing 'authentication handshake failed'", err) + } + + // Update the same EDS resource to add Priority 1 with an endpoint with + // valid hostname. + endpoints = []*v3endpointpb.ClusterLoadAssignment{ + e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: endpointName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Name: "locality-0", + Weight: 1, + Priority: 0, + Backends: []e2e.BackendOptions{{ + Ports: []uint32{testutils.ParsePort(t, server1.Address)}, + Hostname: "wrong.sni.domain", + }}, + }, + { + Name: "locality-1", + Weight: 1, + Priority: 1, + Backends: []e2e.BackendOptions{{ + Ports: []uint32{testutils.ParsePort(t, server2.Address)}, + Hostname: validSNI, + }}, + }, + }, + }), + } + resources.Endpoints = endpoints + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // RPC should succeed because it automatically fails over to Priority 1. Id + // only one hostname would have been used per leaf cluster , this RPC should + // have failed due to invalid SNI in Priority 0. + peerInfo := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed after adding Priority 1: %v", err) + } + if got, want := peerInfo.Addr.String(), server2.Address; got != want { + t.Fatalf("EmptyCall() successes routed to %q, want %q", got, want) + } +} + +// TestClientSideXDS_FallbackSANMatchers tests that when AutoSniSanValidation is +// true, and no SNI is provided by the control plane and AutoHostSNI is not set, +// the validation falls back to using the explicit SAN matchers specified in the +// configuration. It verifies that RPCs succeed when the fallback matchers match +// the server certificate SAN and fail when they do not. +func (s) TestClientSideXDS_FallbackSANMatchers(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true) + + // Spin up an xDS management server. + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + // Create test backends. + serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert) + server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server1.Stop() + server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server2.Stop() + + // Configure client side xDS resources on the management server. + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName1 = "cluster1-" + serviceName + const clusterName2 = "cluster2-" + serviceName + const endpointName1 = "endpoint1-" + serviceName + const endpointName2 = "endpoint2-" + serviceName + + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)} + + routes := []*v3routepb.RouteConfiguration{{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}}}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}}}, + }, + }, + }}, + }} + + // Configure cluster1 with AutoSniSanValidation set to true and no SNI + // provided for the handshake. The validation falls back to using the explicit + // SAN matchers specified in the configuration which matches the server1's + // certificate SAN. + cluster1 := e2e.DefaultCluster(clusterName1, endpointName1, e2e.SecurityLevelMTLS) + cluster1.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + AutoSniSanValidation: true, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "*.test.example.com"}}, + }, + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "root", + }, + }, + }, + }, + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "identity", + }, + }, + }), + }, + } + + // Configure cluster2 with AutoSniSanValidation set to true and no SNI + // provided for the handshake. The validation falls back to using the explicit + // SAN matchers specified in the configuration which does not match the + // server2's certificate SAN. + cluster2 := e2e.DefaultCluster(clusterName2, endpointName2, e2e.SecurityLevelMTLS) + cluster2.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + AutoSniSanValidation: true, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "wrong.san.domain"}}}, + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "root", + }, + }, + }, + }, + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "identity", + }, + }, + }), + }, + } + + clusters := []*v3clusterpb.Cluster{cluster1, cluster2} + endpoints := []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(endpointName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}), + e2e.DefaultEndpoint(endpointName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}), + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + } + + ctx, cancel := context.WithTimeout(context.Background(), 50*defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + // RPC to cluster1 should succeed because fallback SAN mathchers are used + // and they match server cert SAN. + peerInfo := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if got, want := peerInfo.Addr.String(), server1.Address; got != want { + t.Fatalf("EmptyCall() routed to %q, want to be routed to: %q", got, want) + } + + // RPC to cluster2 should fail because fallback SAN matchers are used + // but they don't match server cert SAN. + const wantErr = "do not match any of the accepted SANs" + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("UnaryCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr) + } +} + +// Tests that when the XDSSNIEnabled environment variable is set to false, SNI +// is not used for validation even if AutoSniSanValidation is true and SNI is +// set. It verifies that the system falls back to using explicit SAN matchers if +// provided, and the TLS handshake succeeds when they match the server +// certificate SAN. +func (s) TestClientSideXDS_SNIEnvVarDisabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, false) + + // Spin up an xDS management server. + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + // Create test backends. + serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert) + server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server1.Stop() + server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server2.Stop() + + // Configure client side xDS resources on the management server. + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName1 = "cluster1-" + serviceName + const clusterName2 = "cluster2-" + serviceName + const endpointName1 = "endpoint1-" + serviceName + const endpointName2 = "endpoint2-" + serviceName + + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)} + + routes := []*v3routepb.RouteConfiguration{{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}}}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}}}, + }, + }, + }}, + }} + + // cluster1 configuration with AutoSniSanValidation set to true and wrong SNI + // provided for the handshake. The validation falls back to using the explicit + // SAN matchers specified in the configuration which matches the server1's + // certificate SAN. + cluster1 := e2e.DefaultCluster(clusterName1, endpointName1, e2e.SecurityLevelMTLS) + cluster1.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + Sni: "incorrect.sni", + AutoSniSanValidation: true, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "*.test.example.com"}}, + }, + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "root", + }, + }, + }, + }, + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "identity", + }, + }, + }), + }, + } + + // Configure cluster2 with AutoSniSanValidation set to true and correct SNI + // provided for the handshake. The validation falls back to using the + // explicit SAN matchers specified in the configuration which does not match + // the server2's certificate SAN. + cluster2 := e2e.DefaultCluster(clusterName2, endpointName2, e2e.SecurityLevelMTLS) + cluster2.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + Sni: validSNI, + AutoSniSanValidation: true, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "wrong.san.domain"}}}, + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "root", + }, + }, + }, + }, + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + CertificateName: "identity", + }, + }, + }), + }, + } + + clusters := []*v3clusterpb.Cluster{cluster1, cluster2} + endpoints := []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(endpointName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}), + e2e.DefaultEndpoint(endpointName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}), + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + } + + ctx, cancel := context.WithTimeout(context.Background(), 50*defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + // RPC to cluster1 should succeed because fallback SAN mathchers are used + // and they match server cert SAN. + peerInfo := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if got, want := peerInfo.Addr.String(), server1.Address; got != want { + t.Fatalf("EmptyCall() routed to %q, want to be routed to: %q", got, want) + } + + // RPC to cluster2 should fail because fallback SAN matchers are used but they + // don't match server cert SAN. + const wantErr = "do not match any of the accepted SANs" + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("UnaryCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr) + } +} + +// Tests that when AutoHostSNI is enabled for a Logical DNS cluster, the SNI is +// resolved from the DNSHostName in the cluster configuration. It verifies that +// the TLS handshake succeeds when the DNSHostName matches the server's +// certificate SAN. +func (s) TestClientSideXDS_AutoHostSNI_LogicalDNS(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true) + + // Spin up an xDS management server. + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + // Create test backend + serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert) + server := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server.Stop() + + // Replace DNS resolver with a manual resolver. + dnsR := manual.NewBuilderWithScheme("dns") + originalDNS := resolver.Get("dns") + resolver.Register(dnsR) + t.Cleanup(func() { resolver.Register(originalDNS) }) + + dnsR.UpdateState(resolver.State{Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: server.Address}}}}}) + + // Configure client side xDS resources on the management server. + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName = "cluster-" + serviceName + + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)} + routes := []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)} + + // Cluster of Type LogicalDNS. with DNSHostName set to match the server's cert + // SAN. + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + Type: e2e.ClusterTypeLogicalDNS, + ClusterName: clusterName, + DNSHostName: validSNI, + DNSPort: uint32(testutils.ParsePort(t, server.Address)), + SecurityLevel: e2e.SecurityLevelMTLS, + }) + cluster.TransportSocket.ConfigType.(*v3corepb.TransportSocket_TypedConfig).TypedConfig = testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + AutoHostSni: true, + AutoSniSanValidation: true, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + }, + }, + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + }, + }, + }) + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: []*v3clusterpb.Cluster{cluster}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + // RPC should succeed because DNSHostName matches the server's certificate + // SAN. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} diff --git a/test/xds/xds_resolver_e2e_test.go b/test/xds/xds_resolver_e2e_test.go new file mode 100644 index 0000000..9d8625b --- /dev/null +++ b/test/xds/xds_resolver_e2e_test.go @@ -0,0 +1,754 @@ +/* +* +* Copyright 2026 gRPC authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package xds_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + iresolver "google.golang.org/grpc/internal/resolver" + + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// wrappingBuilder wraps a resolver.Builder and intercepts the ClientConn +// passed to Build by wrapping it with an interceptingClientConn. +type wrappingBuilder struct { + resolver.Builder + jsonCh chan string +} + +// Build wraps the provided resolver.ClientConn with an interceptingClientConn +// before delegating to the underlying resolver.Builder. +func (ib *wrappingBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { + icc := &interceptingClientConn{ + ClientConn: cc, + jsonCh: ib.jsonCh, + } + return ib.Builder.Build(target, icc, opts) +} + +// interceptingClientConn wraps a resolver.ClientConn and intercepts calls to +// ParseServiceConfig to send the raw JSON service config. +type interceptingClientConn struct { + resolver.ClientConn + jsonCh chan string +} + +func (icc *interceptingClientConn) ParseServiceConfig(js string) *serviceconfig.ParseResult { + select { + case icc.jsonCh <- js: + default: + } + return icc.ClientConn.ParseServiceConfig(js) +} + +type dummyFilterCfg struct { + httpfilter.FilterConfig +} + +type testFilterBuilder struct { + httpfilter.Builder + typeURL string + blockChan chan struct{} + enteredChan chan struct{} + closeChan chan struct{} + newStreamErr error +} + +func (tb *testFilterBuilder) TypeURLs() []string { return []string{tb.typeURL} } + +func (*testFilterBuilder) ParseFilterConfig(proto.Message, httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return dummyFilterCfg{}, nil +} + +func (*testFilterBuilder) ParseFilterConfigOverride(proto.Message, httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return dummyFilterCfg{}, nil +} + +func (*testFilterBuilder) IsTerminal() bool { return false } + +func (tb *testFilterBuilder) BuildClientFilter(httpfilter.ClientFilterOptions) httpfilter.ClientFilter { + return tb +} + +func (*testFilterBuilder) Close() {} + +// ClientFilterBuilder is an optional interface for filters to implement. This +// compile time check ensures the test filter implements it. +var _ httpfilter.ClientFilterBuilder = &testFilterBuilder{} + +func (tb *testFilterBuilder) BuildClientInterceptor(httpfilter.FilterConfig, httpfilter.FilterConfig) (httpfilter.ClientInterceptor, error) { + return &testInterceptor{ + blockChan: tb.blockChan, + enteredChan: tb.enteredChan, + closeChan: tb.closeChan, + newStreamErr: tb.newStreamErr, + }, nil +} + +type testInterceptor struct { + blockChan chan struct{} + enteredChan chan struct{} + closeChan chan struct{} + newStreamErr error +} + +func (i *testInterceptor) NewStream(ctx context.Context, _ iresolver.RPCInfo, newStream func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) { + // Signal that we have entered the filter + i.enteredChan <- struct{}{} + + select { + case <-i.blockChan: + case <-ctx.Done(): + return nil, ctx.Err() + } + + if i.newStreamErr != nil { + return nil, i.newStreamErr + } + return newStream(ctx, opts...) +} + +func (i *testInterceptor) Close() { + i.closeChan <- struct{}{} +} + +func newTestFilterBuilder(t testing.TB, typeURL string) *testFilterBuilder { + tb := &testFilterBuilder{ + typeURL: typeURL, + blockChan: make(chan struct{}), + enteredChan: make(chan struct{}, 1), + closeChan: make(chan struct{}, 3), + } + httpfilter.Register(tb) + t.Cleanup(func() { + httpfilter.UnregisterForTesting(typeURL) + }) + return tb +} + +// wantServiceConfig returns a JSON representation of a service config with +// xds_cluster_manager_experimental LB policy with child policies of +// cds_experimental for the provided cluster names. +func wantServiceConfig(clusters ...string) string { + var children []string + for _, cluster := range clusters { + children = append(children, fmt.Sprintf(`"cluster:%s": { + "childPolicy": [{ + "cds_experimental": { + "cluster": "%s" + } + }] + }`, cluster, cluster)) + } + return fmt.Sprintf(`{ + "loadBalancingConfig": [{ + "xds_cluster_manager_experimental": { + "children": { + %s + } + } + }] + }`, strings.Join(children, ",")) +} + +// compareJSONConfigs parses gotJSON and wantJSON using the internal gRPC +// Service Config parser and asserts that they represent the same configuration. +func compareJSONConfigs(t *testing.T, gotJSON, wantJSON string) { + gotParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(gotJSON) + if gotParsed.Err != nil { + t.Fatalf("Failed to parse got service config %q: %v", gotJSON, gotParsed.Err) + } + wantParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(wantJSON) + if wantParsed.Err != nil { + t.Fatalf("Failed to parse want service config %q: %v", wantJSON, wantParsed.Err) + } + if !internal.EqualServiceConfigForTesting(gotParsed.Config, wantParsed.Config) { + t.Fatalf("Service config mismatch.\nGot:\n%s\nWant:\n%s", gotJSON, wantJSON) + } +} + +func verifyServiceConfig(ctx context.Context, t *testing.T, jsonCh chan string, wantClusters ...string) { + t.Helper() + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for resolver state update") + case js := <-jsonCh: + compareJSONConfigs(t, js, wantServiceConfig(wantClusters...)) + } +} + +// Test verifies that when RPCs are in flight holding references to an old +// cluster, that cluster remains in the service config until all in-flight +// RPCs finish. +func (s) TestResolverDelayedClusterRemoval_MultipleInFlightRPCs(t *testing.T) { + testFilterTypeURL := t.Name() + tb := newTestFilterBuilder(t, testFilterTypeURL) + blockChan, enteredChan := tb.blockChan, tb.enteredChan + + // Start an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create the xDS resolver builder. + r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Create an intercepting resolver builder. + jsonCh := make(chan string, 1) + ib := &wrappingBuilder{ + Builder: r, + jsonCh: jsonCh, + } + + // Start test backends. + serverA := stubserver.StartTestService(t, nil) + defer serverA.Stop() + serverB := stubserver.StartTestService(t, nil) + defer serverB.Stop() + + const ( + serviceName = "my-service-xds" + clusterA = "cluster-A" + clusterB = "cluster-B" + ) + clusterSpec := &v3routepb.RouteAction_Cluster{Cluster: clusterA} + hcm := &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "route-" + serviceName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: clusterSpec, + }}, + }}, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + { + Name: "delaying-filter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: testFilterTypeURL, + }), + }, + }, + e2e.RouterHTTPFilter, + }, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, serverA.Address)})}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a gRPC client using the intercepting resolver. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(ib)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + // Trigger first RPC to cluster A. This RPC should reach our custom HTTP + // filter, and get blocked. + errCh := make(chan error, 2) + go func() { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + errCh <- err + }() + + // Verify that first RPC has entered the HTTP filter's NewStream and is + // currently blocked. + select { + case <-enteredChan: + case <-ctx.Done(): + t.Fatal("Timeout waiting for first RPC to reach HTTP filter") + } + + // Read the first state update from the intercepting resolver. + // This should contain cluster-A. + verifyServiceConfig(ctx, t, jsonCh, clusterA) + + // Trigger second RPC to cluster A. This RPC should reach our custom HTTP + // filter, and get blocked. + go func() { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + errCh <- err + }() + + // Verify that second RPC has entered the HTTP filter's NewStream and is + // currently blocked. + select { + case <-enteredChan: + case <-ctx.Done(): + t.Fatal("Timeout waiting for second RPC to reach HTTP filter") + } + + // Now update the route configuration on the management server to point to cluster B. + clusterSpec.Cluster = clusterB + resources.Listeners = []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}} + resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone)) + resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, serverB.Address)})) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + verifyServiceConfig(ctx, t, jsonCh, clusterA, clusterB) + + // Unblock one of the RPCs and verify it completes successfully. + blockChan <- struct{}{} + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for an RPC to succeed") + case err := <-errCh: + if err != nil { + t.Fatalf("RPC failed with unexpected error: %v", err) + } + } + + // Verify that because the other RPC to cluster-A is still in flight, the + // cluster reference count does not drop to 0 and no service config update + // is produced. + select { + case js := <-jsonCh: + t.Fatalf("Unexpected service config update received: %s", js) + case <-time.After(defaultTestShortTimeout): + } + + // Unblock the remaining RPC and verify it completes successfully. + blockChan <- struct{}{} + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for remaining RPC to succeed") + case err := <-errCh: + if err != nil { + t.Fatalf("Remaining RPC failed with unexpected error: %v", err) + } + } + + // Once the second RPC finishes (refCount drops 1 -> 0), cluster-A should be + // removed from the service config. + verifyServiceConfig(ctx, t, jsonCh, clusterB) +} + +// Test verifies that if stream creation fails early inside an HTTP filter or +// interceptor (before the stream is created), the OnFinish CallOption is still +// executed. This decrements the cluster reference count to 0, allowing the +// resolver to unsubscribe and prune the old cluster from the service config. +func (s) TestResolverPrunesCluster_StreamCreationFailure(t *testing.T) { + testFilterTypeURL := t.Name() + const wantErr = "blocking filter error" + tb := newTestFilterBuilder(t, testFilterTypeURL) + tb.newStreamErr = status.Error(codes.Unavailable, wantErr) + close(tb.blockChan) // Close blockChan immediately so that the RPC fails without blocking + + // Start an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Create the underlying xDS resolver builder. + r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Intercept resolver builder. + jsonCh := make(chan string, 1) + ib := &wrappingBuilder{ + Builder: r, + jsonCh: jsonCh, + } + + // Start test backends. + serverA := stubserver.StartTestService(t, nil) + defer serverA.Stop() + serverB := stubserver.StartTestService(t, nil) + defer serverB.Stop() + + const ( + serviceName = "my-service-xds" + clusterA = "cluster-A" + clusterB = "cluster-B" + ) + clusterSpec := &v3routepb.RouteAction_Cluster{Cluster: clusterA} + hcm := &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "route-" + serviceName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: clusterSpec, + }}, + }}, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + { + Name: "blocking-filter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: testFilterTypeURL, + }), + }, + }, + e2e.RouterHTTPFilter, + }, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, serverA.Address)})}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a gRPC client using the xDS resolver. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(ib)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + cc.Connect() + + // Read the first state update from the intercepting resolver. + // This should contain cluster-A. + verifyServiceConfig(ctx, t, jsonCh, clusterA) + + // Make an RPC. Since the filter immediately fails, this should fail with + // the expected error. + client := testgrpc.NewTestServiceClient(cc) + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("RPC failed with error %v, want code %v with desc containing %q", err, codes.Unavailable, wantErr) + } + + // Update the route configuration on the management server to point + // to cluster B. + clusterSpec.Cluster = clusterB + resources.Listeners = []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}} + resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone)) + resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, serverB.Address)})) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Read the second state update from the intercepting resolver. This should + // contain both cluster-A and cluster-B. + verifyServiceConfig(ctx, t, jsonCh, clusterA, clusterB) + + // Read the third state update from the intercepting resolver. Cluster-A's + // reference count drops to 0, causing the resolver to unsubscribe and + // prune it. + verifyServiceConfig(ctx, t, jsonCh, clusterB) +} + +// TestResolverDelayedInterceptorClose_ActiveRPCs verifies that when a route +// configuration is updated (causing the old config selector to stop), an HTTP +// filter interceptor for a route/cluster with an active in-flight RPC is not +// closed immediately. It remains active until all active RPCs to that cluster +// in that route complete and its refcount drops to 0. +func (s) TestResolverDelayedInterceptorClose_ActiveRPCs(t *testing.T) { + testFilterTypeURL := t.Name() + tb := newTestFilterBuilder(t, testFilterTypeURL) + blockChan, enteredChan, closeChan := tb.blockChan, tb.enteredChan, tb.closeChan + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Intercept resolver builder. + jsonCh := make(chan string, 1) + ib := &wrappingBuilder{ + Builder: r, + jsonCh: jsonCh, + } + + // Start test backend. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const ( + serviceName = "my-service-xds" + clusterA = "cluster-A" + clusterB = "cluster-B" + ) + hcm := &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "route-" + serviceName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterA}}}, + }}, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + { + Name: "delaying-filter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: testFilterTypeURL, + }), + }, + }, + e2e.RouterHTTPFilter, + }, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(ib)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + errCh := make(chan error, 1) + go func() { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + errCh <- err + }() + + select { + case <-enteredChan: + case <-ctx.Done(): + t.Fatal("Timeout waiting for RPC to reach HTTP filter") + } + + verifyServiceConfig(ctx, t, jsonCh, clusterA) + + // Update route configuration on the management server to point to cluster B. + hcm.GetRouteConfig().VirtualHosts[0].Routes[0].Action = &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterB}}} + resources.Listeners = []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}} + resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone)) + resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, server.Address)})) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + verifyServiceConfig(ctx, t, jsonCh, clusterA, clusterB) + + // Verify that because an RPC is still in flight on cluster-A, the interceptor + // is NOT closed prematurely when the old config selector is stopped. + select { + case <-closeChan: + t.Fatal("Unexpected interceptor Close() called while RPC is still active") + case <-time.After(defaultTestShortTimeout): + } + + // Unblock the active RPC and verify it completes successfully. + blockChan <- struct{}{} + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for RPC to succeed") + case err := <-errCh: + if err != nil { + t.Fatalf("RPC failed with unexpected error: %v", err) + } + } + + // Once the RPC finishes, its OnFinish CallOption decrements the refcount + // of cluster-A to 0. Verifies that interceptor.Close() is now called. + select { + case <-closeChan: + case <-ctx.Done(): + t.Fatal("Timeout waiting for interceptor Close() after RPC finished") + } + + verifyServiceConfig(ctx, t, jsonCh, clusterB) +} + +// Test verifies that when a route configuration is updated and there are no +// active in-flight RPCs on a route/cluster, the HTTP filter interceptor is +// closed immediately when the old config selector is stopped. +func (s) TestResolverImmediateInterceptorClose_NoActiveRPCs(t *testing.T) { + testFilterTypeURL := t.Name() + tb := newTestFilterBuilder(t, testFilterTypeURL) + closeChan := tb.closeChan + close(tb.blockChan) // Do not block RPCs in the filter + + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Intercept resolver builder. + jsonCh := make(chan string, 1) + ib := &wrappingBuilder{ + Builder: r, + jsonCh: jsonCh, + } + + // Start test backends. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const ( + serviceName = "my-service-xds" + clusterA = "cluster-A" + clusterB = "cluster-B" + ) + hcm := &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "route-" + serviceName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterA}}}, + }}, + }}, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{ + { + Name: "closing-filter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: testFilterTypeURL, + }), + }, + }, + e2e.RouterHTTPFilter, + }, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(ib)) + if err != nil { + t.Fatalf("Failed to create a gRPC client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + verifyServiceConfig(ctx, t, jsonCh, clusterA) + + // Update route configuration on the management server to point to cluster B. + hcm.GetRouteConfig().VirtualHosts[0].Routes[0].Action = &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterB}}} + resources.Listeners = []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}} + resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone)) + resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, server.Address)})) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + verifyServiceConfig(ctx, t, jsonCh, clusterA, clusterB) + + // Because there are no active RPCs on cluster-A, interceptor.Close() should + // be invoked immediately when the old config selector is stopped. + select { + case <-closeChan: + case <-ctx.Done(): + t.Fatal("Timeout waiting for interceptor Close() when no active RPCs exist") + } + + verifyServiceConfig(ctx, t, jsonCh, clusterB) +} diff --git a/test/xds/xds_rls_clusterspecifier_plugin_test.go b/test/xds/xds_rls_clusterspecifier_plugin_test.go new file mode 100644 index 0000000..d2c55dc --- /dev/null +++ b/test/xds/xds_rls_clusterspecifier_plugin_test.go @@ -0,0 +1,168 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/rls" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/protobuf/types/known/durationpb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/balancer/rls" // Register the RLS Load Balancing policy. +) + +// defaultClientResourcesWithRLSCSP returns a set of resources (LDS, RDS, CDS, EDS) for a +// client to connect to a server with a RLS Load Balancer as a child of Cluster Manager. +func defaultClientResourcesWithRLSCSP(t *testing.T, lb e2e.LoadBalancingPolicy, params e2e.ResourceParams, rlsProto *rlspb.RouteLookupConfig) e2e.UpdateOptions { + routeConfigName := "route-" + params.DialTarget + clusterName := "cluster-" + params.DialTarget + endpointsName := "endpoints-" + params.DialTarget + return e2e.UpdateOptions{ + NodeID: params.NodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(params.DialTarget, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: routeConfigName, + ListenerName: params.DialTarget, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeClusterSpecifierPlugin, + ClusterSpecifierPluginName: "rls-csp", + ClusterSpecifierPluginConfig: testutils.MarshalAny(t, &rlspb.RouteLookupClusterSpecifier{ + RouteLookupConfig: rlsProto, + }), + })}, + Clusters: []*v3clusterpb.Cluster{e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: endpointsName, + Policy: lb, + SecurityLevel: params.SecLevel, + })}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, params.Host, []uint32{params.Port})}, + } +} + +// TestRLSinxDS tests an xDS configured system with an RLS Balancer present. +// +// This test sets up the RLS Balancer using the RLS Cluster Specifier Plugin, +// spins up a test service and has a fake RLS Server correctly respond with a +// target corresponding to this test service. This test asserts an RPC proceeds +// as normal with the RLS Balancer as part of system. +func (s) TestRLSinxDS(t *testing.T) { + tests := []struct { + name string + lbPolicy e2e.LoadBalancingPolicy + }{ + { + name: "roundrobin", + lbPolicy: e2e.LoadBalancingPolicyRoundRobin, + }, + { + name: "ringhash", + lbPolicy: e2e.LoadBalancingPolicyRingHash, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testRLSinxDS(t, test.lbPolicy) + }) + } +} + +func testRLSinxDS(t *testing.T, lbPolicy e2e.LoadBalancingPolicy) { + // Set up all components and configuration necessary - management server, + // xDS resolver, fake RLS Server, and xDS configuration which specifies an + // RLS Balancer that communicates to this set up fake RLS Server. + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + lis := testutils.NewListenerWrapper(t, nil) + rlsServer, rlsRequestCh := rls.SetupFakeRLSServer(t, lis) + rlsProto := &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{{Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "grpc.testing.TestService"}}}}, + LookupService: rlsServer.Address, + LookupServiceTimeout: durationpb.New(defaultTestTimeout), + CacheSizeBytes: 1024, + } + + const serviceName = "my-service-client-side-xds" + resources := defaultClientResourcesWithRLSCSP(t, lbPolicy, e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }, rlsProto) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Configure the fake RLS Server to set the RLS Balancers child CDS + // Cluster's name as the target for the RPC to use. + rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rls.RouteLookupResponse { + return &rls.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{"cluster-" + serviceName}}} + }) + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + // Successfully sending the RPC will require the RLS Load Balancer to + // communicate with the fake RLS Server for information about the target. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // These RLS Verifications makes sure the RLS Load Balancer is actually part + // of the xDS Configured system that correctly sends out RPC. + + // Verify connection is established to RLS Server. + if _, err = lis.NewConnCh.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for RLS LB policy to create control channel") + } + + // Verify an rls request is sent out to fake RLS Server. + select { + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for an RLS request to be sent out") + case <-rlsRequestCh: + } +} diff --git a/test/xds/xds_security_config_nack_test.go b/test/xds/xds_security_config_nack_test.go new file mode 100644 index 0000000..0efc080 --- /dev/null +++ b/test/xds/xds_security_config_nack_test.go @@ -0,0 +1,377 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/resolver" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/google/uuid" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestUnmarshalListener_WithUpdateValidatorFunc(t *testing.T) { + const ( + serviceName = "my-service-client-side-xds" + missingIdentityProviderInstance = "missing-identity-provider-instance" + missingRootProviderInstance = "missing-root-provider-instance" + ) + + tests := []struct { + name string + securityConfig *v3corepb.TransportSocket + wantErr bool + }{ + { + name: "both identity and root providers are not present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingIdentityProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingRootProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: true, + }, + { + name: "only identity provider is not present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingIdentityProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ServerSideCertProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: true, + }, + { + name: "only root provider is not present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ServerSideCertProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingRootProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: true, + }, + { + name: "both identity and root providers are present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ServerSideCertProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ServerSideCertProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t) + + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + // Grab the host and port of the server and create client side xDS + // resources corresponding to it. + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + + // Create xDS resources to be consumed on the client side. This + // includes the listener, route configuration, cluster (with + // security configuration) and endpoint resources. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelMTLS, + }) + + // Create an inbound xDS listener resource for the server side. + inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS, "routeName") + for _, fc := range inboundLis.GetFilterChains() { + fc.TransportSocket = test.securityConfig + } + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn with the xds scheme and make an RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + // Make a context with a shorter timeout from the top level test + // context for cases where we expect failures. + timeout := defaultTestTimeout + if test.wantErr { + timeout = defaultTestShortTimeout + } + ctx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); (err != nil) != test.wantErr { + t.Fatalf("EmptyCall() returned err: %v, wantErr %v", err, test.wantErr) + } + }) + } +} + +func (s) TestUnmarshalCluster_WithUpdateValidatorFunc(t *testing.T) { + const ( + serviceName = "my-service-client-side-xds" + missingIdentityProviderInstance = "missing-identity-provider-instance" + missingRootProviderInstance = "missing-root-provider-instance" + ) + + tests := []struct { + name string + securityConfig *v3corepb.TransportSocket + wantErr bool + }{ + { + name: "both identity and root providers are not present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingIdentityProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingRootProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: true, + }, + { + name: "only identity provider is not present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingIdentityProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: true, + }, + { + name: "only root provider is not present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingRootProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: true, + }, + { + name: "both identity and root providers are present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management + // server with certificate provider configuration. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + xdsResolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // This creates a `Cluster` resource with a security config which + // refers to `e2e.ClientSideCertProviderInstance` for both root and + // identity certs. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelMTLS, + }) + resources.Clusters[0].TransportSocket = test.securityConfig + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + // Make a context with a shorter timeout from the top level test + // context for cases where we expect failures. + timeout := defaultTestTimeout + if test.wantErr { + timeout = defaultTestShortTimeout + } + ctx2, cancel2 := context.WithTimeout(ctx, timeout) + defer cancel2() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx2, &testpb.Empty{}, grpc.WaitForReady(true)); (err != nil) != test.wantErr { + t.Fatalf("EmptyCall() returned err: %v, wantErr %v", err, test.wantErr) + } + }) + } +} diff --git a/test/xds/xds_server_filter_e2e_test.go b/test/xds/xds_server_filter_e2e_test.go new file mode 100644 index 0000000..42931a5 --- /dev/null +++ b/test/xds/xds_server_filter_e2e_test.go @@ -0,0 +1,402 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "strconv" + "strings" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func makeUnaryRPC(ctx context.Context, client testgrpc.TestServiceClient) error { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return fmt.Errorf("EmptyCall() failed: %w", err) + } + return nil +} + +func makeStreamingRPC(ctx context.Context, client testgrpc.TestServiceClient) error { + stream, err := client.FullDuplexCall(ctx) + if err != nil { + return fmt.Errorf("FullDuplexCall() failed: %w", err) + } + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + return fmt.Errorf("stream.Send() failed: %w", err) + } + if err := stream.CloseSend(); err != nil { + return fmt.Errorf("stream.CloseSend() failed: %w", err) + } + if _, err := stream.Recv(); err != io.EOF { + return err + } + return nil +} + +// setupXDSListenerAndClient starts an xDS management server and a gRPC server, +// configures the management server with an inbound listener containing the +// specified httpFilters, dials the server using xDS, waits for the server to +// enter SERVING mode, and returns a TestServiceClient. +func setupXDSListenerAndClient(t *testing.T, httpFilters ...*v3httppb.HttpFilter) testgrpc.TestServiceClient { + t.Helper() + const serviceName = "my-service" + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t) + + // Wait for the server to enter SERVING mode before making RPCs to avoid + // flakes due to the server closing connections. + servingCh := make(chan struct{}) + opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) { + if args.Mode == connectivity.ServingModeServing { + close(servingCh) + } + }) + lis, stopServer := setupGRPCServer(t, bootstrapContents, opt) + t.Cleanup(stopServer) + + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + vhs := []*v3routepb.VirtualHost{{ + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}, + }} + + filters := append([]*v3httppb.HttpFilter{}, httpFilters...) + filters = append(filters, e2e.HTTPFilter("router", &v3routerpb.Router{})) + + networkFilters := []*v3listenerpb.Filter{{ + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: filters, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: vhs, + }, + }, + }), + }, + }} + + inboundLis := &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: networkFilters}, + } + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatalf("managementServer.Update() failed: %v", err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + select { + case <-servingCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for server to enter SERVING mode") + } + + return testgrpc.NewTestServiceClient(cc) +} + +// Test verifies that the xDS server-side HTTP filter chain correctly executes +// InterceptRPC and wraps grpc.ServerStream for both Unary and Streaming RPCs. +func (s) TestServerSideXDSHTTPFilter_InterceptRPC(t *testing.T) { + dummyTypeURL := t.Name() + fb := newTrackingHTTPFilterBuilder(dummyTypeURL) + httpfilter.Register(fb) + defer httpfilter.UnregisterForTesting(fb.typeURL) + + dummyFilter := newHTTPFilter(t, "dummy-filter", dummyTypeURL, "") + client := setupXDSListenerAndClient(t, dummyFilter) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Verify Unary RPC execution. + if err := makeUnaryRPC(ctx, client); err != nil { + t.Fatalf("makeUnaryRPC() failed: %v", err) + } + if got := fb.interceptRPCCount.Load(); got != 1 { + t.Fatalf("Unexpected interceptRPCCount for unary RPC, got %d, want 1", got) + } + if got := fb.recvMsgCount.Load(); got != 1 { + t.Fatalf("Unexpected recvMsgCount for unary RPC, got %d, want 1", got) + } + if got := fb.sendMsgCount.Load(); got != 1 { + t.Fatalf("Unexpected sendMsgCount for unary RPC, got %d, want 1", got) + } + + // Verify Streaming RPC execution. + if err := makeStreamingRPC(ctx, client); err != nil { + t.Fatalf("makeStreamingRPC() failed: %v", err) + } + if got := fb.interceptRPCCount.Load(); got != 2 { + t.Fatalf("Unexpected interceptRPCCount for streaming RPC, got %d, want 2", got) + } + if got := fb.recvMsgCount.Load(); got != 2 { + t.Fatalf("Unexpected recvMsgCount for streaming RPC, got %d, want 2", got) + } + // Since the server handler did not call SendMsg for the streaming RPC, the + // sendMsg count for this RPC is zero, so the total count remains 1. + if got := fb.sendMsgCount.Load(); got != 1 { + t.Fatalf("Unexpected sendMsgCount for streaming RPC, got %d, want 1", got) + } +} + +// Test verifies that when a server-side xDS HTTP filter returns an error from +// InterceptRPC, the RPC is rejected early with the expected error code and +// message for both Unary and Streaming RPCs. +func (s) TestServerSideXDSHTTPFilter_InterceptRPCEarlyRejection(t *testing.T) { + tests := []struct { + name string + errToRet error + wantCode codes.Code + wantErrMsg string + }{ + { + name: "InterceptRPC_FailsWithStatusError", + errToRet: status.Error(codes.PermissionDenied, "access denied by xDS server filter"), + wantCode: codes.PermissionDenied, + wantErrMsg: "access denied by xDS server filter", + }, + { + name: "InterceptRPC_FailsWithNonStatusError", + errToRet: errors.New("non-status error from xDS filter"), + wantCode: codes.Unknown, + wantErrMsg: "non-status error from xDS filter", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + typeURL := t.Name() + fb := newTrackingHTTPFilterBuilder(typeURL) + fb.interceptRPCFunc = func(grpc.ServerStream) (grpc.ServerStream, error) { + return nil, test.errToRet + } + httpfilter.Register(fb) + defer httpfilter.UnregisterForTesting(fb.typeURL) + + rejectFilter := newHTTPFilter(t, "reject-filter", typeURL, "") + client := setupXDSListenerAndClient(t, rejectFilter) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Verify Unary RPC rejection. + if err := makeUnaryRPC(ctx, client); status.Code(err) != test.wantCode || !strings.Contains(err.Error(), test.wantErrMsg) { + t.Fatalf("makeUnaryRPC() returned err = %v (code: %v), want code %v and msg %q", err, status.Code(err), test.wantCode, test.wantErrMsg) + } + if got := fb.interceptRPCCount.Load(); got != 1 { + t.Fatalf("Unexpected interceptRPCCount for unary RPC, got %d, want 1", got) + } + + // Verify Streaming RPC rejection. + if err := makeStreamingRPC(ctx, client); status.Code(err) != test.wantCode || !strings.Contains(err.Error(), test.wantErrMsg) { + t.Fatalf("makeStreamingRPC() returned err = %v (code: %v), want code %v and msg %q", err, status.Code(err), test.wantCode, test.wantErrMsg) + } + if got := fb.interceptRPCCount.Load(); got != 2 { + t.Fatalf("Unexpected interceptRPCCount for streaming RPC, got %d, want 2", got) + } + }) + } +} + +// Test verifies that multiple server-side xDS HTTP filters in a chain execute +// their InterceptRPC methods sequentially in the configured order for both +// Unary and Streaming RPCs. +func (s) TestServerSideXDS_InterceptRPCMultiFilterChaining(t *testing.T) { + typeURL1, typeURL2 := fmt.Sprintf("%s-1", t.Name()), fmt.Sprintf("%s-2", t.Name()) + eventsCh := make(chan string, 2) + const filter1, filter2 = "intercepted-filter-1", "intercepted-filter-2" + + fb1 := newTrackingHTTPFilterBuilder(typeURL1) + fb1.interceptRPCFunc = func(ss grpc.ServerStream) (grpc.ServerStream, error) { + eventsCh <- filter1 + return ss, nil + } + fb2 := newTrackingHTTPFilterBuilder(typeURL2) + fb2.interceptRPCFunc = func(ss grpc.ServerStream) (grpc.ServerStream, error) { + eventsCh <- filter2 + return ss, nil + } + + httpfilter.Register(fb1) + defer httpfilter.UnregisterForTesting(fb1.typeURL) + httpfilter.Register(fb2) + defer httpfilter.UnregisterForTesting(fb2.typeURL) + + f1 := newHTTPFilter(t, "filter-1", typeURL1, "") + f2 := newHTTPFilter(t, "filter-2", typeURL2, "") + + client := setupXDSListenerAndClient(t, f1, f2) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Verify Unary RPC filter chaining. + if err := makeUnaryRPC(ctx, client); err != nil { + t.Fatalf("makeUnaryRPC() failed: %v", err) + } + select { + case e := <-eventsCh: + if e != filter1 { + t.Fatalf("First intercept event = %q, want %q", e, filter1) + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for %q", filter1) + } + + select { + case e := <-eventsCh: + if e != filter2 { + t.Fatalf("Second intercept event = %q, want %q", e, filter2) + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for %q", filter2) + } + + // Verify Streaming RPC filter chaining. + if err := makeStreamingRPC(ctx, client); err != nil { + t.Fatalf("makeStreamingRPC() failed: %v", err) + } + select { + case e := <-eventsCh: + if e != filter1 { + t.Fatalf("First intercept event = %q, want %q", e, filter1) + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for %q", filter1) + } + + select { + case e := <-eventsCh: + if e != filter2 { + t.Fatalf("Second intercept event = %q, want %q", e, filter2) + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for %q", filter2) + } +} + +type errorWrappedStream struct { + grpc.ServerStream + recvErr string +} + +func (w *errorWrappedStream) RecvMsg(m any) error { + if w.recvErr != "" { + return status.Error(codes.Internal, w.recvErr) + } + return w.ServerStream.RecvMsg(m) +} + +// Test verifies that when multiple filters wrap the server stream and a +// filter's overridden RecvMsg returns an error, that error is properly +// propagated back to the client for both Unary and Streaming RPCs. +func (s) TestServerSideXDS_InterceptRPCMultiFilterErrorPropagation(t *testing.T) { + typeURL1, typeURL2 := fmt.Sprintf("%s-1", t.Name()), fmt.Sprintf("%s-2", t.Name()) + + fb1 := newTrackingHTTPFilterBuilder(typeURL1) + + const wantErr = "recvMsg failed in filter 2" + fb2 := newTrackingHTTPFilterBuilder(typeURL2) + fb2.interceptRPCFunc = func(ss grpc.ServerStream) (grpc.ServerStream, error) { + return &errorWrappedStream{ServerStream: ss, recvErr: wantErr}, nil + } + + httpfilter.Register(fb1) + defer httpfilter.UnregisterForTesting(fb1.typeURL) + httpfilter.Register(fb2) + defer httpfilter.UnregisterForTesting(fb2.typeURL) + + filter1 := newHTTPFilter(t, "filter-1", typeURL1, "") + filter2 := newHTTPFilter(t, "filter-2", typeURL2, "") + + client := setupXDSListenerAndClient(t, filter1, filter2) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Verify Unary RPC error propagation. + if err := makeUnaryRPC(ctx, client); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("makeUnaryRPC() returned error = %v, want error containing %q", err, wantErr) + } + + // Verify Streaming RPC error propagation. + if err := makeStreamingRPC(ctx, client); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("makeStreamingRPC() returned error = %v, want error containing %q", err, wantErr) + } +} diff --git a/test/xds/xds_server_filter_override_test.go b/test/xds/xds_server_filter_override_test.go new file mode 100644 index 0000000..60c4971 --- /dev/null +++ b/test/xds/xds_server_filter_override_test.go @@ -0,0 +1,393 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "net" + "strconv" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/internal/xds/httpfilter" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/xds" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" + + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" +) + +type overrideInfo struct { + disabled bool + path string +} + +func makeOverrideConfig(t *testing.T, typeURL string, info *overrideInfo) *anypb.Any { + if info == nil { + return nil + } + cfg := &v3routepb.FilterConfig{Disabled: info.disabled} + if info.path != "" { + cfg.Config = testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: typeURL, + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "path": {Kind: &structpb.Value_StringValue{StringValue: info.path}}, + }, + }, + }) + } + return testutils.MarshalAny(t, cfg) +} + +// TestServerSideXDS_FilterOverride_Disabled verifies that filters are NOT +// created when disabled by either base config or overrides at either +// VirtualHost or Route level. +func (s) TestServerSideXDS_FilterOverride_Disabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true) + + tests := []struct { + name string + baseDisabled bool + vhost *overrideInfo + route *overrideInfo + }{ + { + name: "BaseDisabled_NoOverrides", + baseDisabled: true, + }, + { + name: "BaseEnabled_VHostDisables", + baseDisabled: false, + vhost: &overrideInfo{disabled: true}, + }, + { + name: "BaseEnabled_VHostEnabled_RouteDisables", + baseDisabled: false, + vhost: &overrideInfo{disabled: false, path: "vhost-value"}, + route: &overrideInfo{disabled: true}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + testFilterTypeURL := t.Name() + fb := newTrackingHTTPFilterBuilder(testFilterTypeURL) + httpfilter.Register(fb) + defer httpfilter.UnregisterForTesting(fb.typeURL) + + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t) + + // Wait for the server to enter SERVING mode before making RPCs to avoid + // flakes due to the server closing connections. + servingCh := make(chan struct{}) + opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) { + if args.Mode == connectivity.ServingModeServing { + close(servingCh) + } + }) + lis, stopServer := setupGRPCServer(t, bootstrapContents, opt) + defer stopServer() + + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + baseFilter := newHTTPFilter(t, "test-filter", testFilterTypeURL, "base-path") + baseFilter.Disabled = tc.baseDisabled + + var vhOverrideConfig map[string]*anypb.Any + if tc.vhost != nil { + vhOverrideConfig = map[string]*anypb.Any{"test-filter": makeOverrideConfig(t, testFilterTypeURL, tc.vhost)} + } + var routeOverrideConfig map[string]*anypb.Any + if tc.route != nil { + routeOverrideConfig = map[string]*anypb.Any{"test-filter": makeOverrideConfig(t, testFilterTypeURL, tc.route)} + } + + vhs := []*v3routepb.VirtualHost{{ + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + TypedPerFilterConfig: routeOverrideConfig, + }}, + TypedPerFilterConfig: vhOverrideConfig, + }} + + networkFilters := []*v3listenerpb.Filter{{ + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + baseFilter, + e2e.HTTPFilter("router", &v3routerpb.Router{}), + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: vhs, + }, + }, + }), + }, + }} + + inboundLis := &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: networkFilters}, + } + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + select { + case <-servingCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for server to enter SERVING mode") + } + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Verify no filters or interceptors were created. + if got := fb.filtersCreated.Load(); got != 0 { + t.Fatalf("filtersCreated = %d; want 0", got) + } + if got := fb.interceptorsCreated.Load(); got != 0 { + t.Fatalf("interceptorsCreated = %d; want 0", got) + } + }) + } +} + +// TestServerSideXDS_FilterOverride_Enabled verifies that filters ARE created +// and receive the correct configuration when enabled by either base config or +// overrides at either VirtualHost or Route level. +func (s) TestServerSideXDS_FilterOverride_Enabled(t *testing.T) { + testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true) + + const basePath = "base-path" + + tests := []struct { + name string + baseDisabled bool + vhost *overrideInfo + route *overrideInfo + wantPath string + }{ + { + name: "BaseEnabled_NoOverrides", + baseDisabled: false, + wantPath: basePath, + }, + { + name: "VHostEnables", + baseDisabled: true, + vhost: &overrideInfo{disabled: false, path: "vhost-value"}, + wantPath: "vhost-value", + }, + { + name: "RouteEnables", + baseDisabled: true, + vhost: &overrideInfo{disabled: true}, + route: &overrideInfo{disabled: false, path: "route-value"}, + wantPath: "route-value", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pathCh := make(chan string, 1) + testFilterTypeURL := t.Name() + fb := newTrackingHTTPFilterBuilder(testFilterTypeURL) + fb.pathCh = pathCh + httpfilter.Register(fb) + defer httpfilter.UnregisterForTesting(fb.typeURL) + + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t) + + servingCh := make(chan struct{}) + opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) { + if args.Mode == connectivity.ServingModeServing { + close(servingCh) + } + }) + lis, stopServer := setupGRPCServer(t, bootstrapContents, opt) + defer stopServer() + + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + baseFilter := newHTTPFilter(t, "test-filter", testFilterTypeURL, basePath) + baseFilter.Disabled = tc.baseDisabled + + var vhOverrideConfig map[string]*anypb.Any + if tc.vhost != nil { + vhOverrideConfig = map[string]*anypb.Any{"test-filter": makeOverrideConfig(t, testFilterTypeURL, tc.vhost)} + } + var routeOverrideConfig map[string]*anypb.Any + if tc.route != nil { + routeOverrideConfig = map[string]*anypb.Any{"test-filter": makeOverrideConfig(t, testFilterTypeURL, tc.route)} + } + + vhs := []*v3routepb.VirtualHost{{ + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + TypedPerFilterConfig: routeOverrideConfig, + }}, + TypedPerFilterConfig: vhOverrideConfig, + }} + + networkFilters := []*v3listenerpb.Filter{{ + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + baseFilter, + e2e.HTTPFilter("router", &v3routerpb.Router{}), + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: vhs, + }, + }, + }), + }, + }} + + inboundLis := &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: networkFilters}, + } + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + select { + case <-servingCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for server to enter SERVING mode") + } + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Verify invoked with expected path. + select { + case p := <-pathCh: + if p != tc.wantPath { + t.Fatalf("Unexpected path received by filter: got %q, want %q", p, tc.wantPath) + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for filter to be invoked") + } + if got, want := fb.filtersCreated.Load(), int32(1); got != want { + t.Fatalf("Created %d filter instances, want: %d", got, want) + } + if got, want := fb.interceptorsCreated.Load(), int32(1); got != want { + t.Fatalf("Created %d interceptor instances, want: %d", got, want) + } + }) + } +} diff --git a/test/xds/xds_server_filter_state_retention_test.go b/test/xds/xds_server_filter_state_retention_test.go new file mode 100644 index 0000000..97d83ca --- /dev/null +++ b/test/xds/xds_server_filter_state_retention_test.go @@ -0,0 +1,699 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "net" + "strconv" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/xds" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const filterCfgPathFieldName = "path" + +// testFilterCfg is the internal representation of the filter config proto. It +// is returned by filter's config parsing methods. +type testFilterCfg struct { + httpfilter.FilterConfig + path string +} + +// filterConfigFromProto parses filter config specified as a v3.TypedStruct into +// a testFilterCfg. +func filterConfigFromProto(cfg proto.Message) (httpfilter.FilterConfig, error) { + ts, ok := cfg.(*v3xdsxdstypepb.TypedStruct) + if !ok { + return nil, fmt.Errorf("unsupported filter config type: %T, want %T", cfg, &v3xdsxdstypepb.TypedStruct{}) + } + + if ts.GetValue() == nil { + return testFilterCfg{}, nil + } + ret := testFilterCfg{} + if v := ts.GetValue().GetFields()[filterCfgPathFieldName]; v != nil { + ret.path = v.GetStringValue() + } + return ret, nil +} + +// trackingHTTPFilterBuilder is a test filter that allows counting and tracking +// the lifecycle and execution of filter and interceptor instances. +type trackingHTTPFilterBuilder struct { + httpfilter.Builder + filtersCreated atomic.Int32 + filtersDestroyed atomic.Int32 + interceptorsCreated atomic.Int32 + interceptorsDestroyed atomic.Int32 + interceptRPCCount atomic.Int32 + recvMsgCount atomic.Int32 + sendMsgCount atomic.Int32 + closeSendCount atomic.Int32 + typeURL string + pathCh chan string + interceptRPCFunc func(ss grpc.ServerStream) (grpc.ServerStream, error) +} + +func newTrackingHTTPFilterBuilder(typeURL string) *trackingHTTPFilterBuilder { + return &trackingHTTPFilterBuilder{typeURL: typeURL} +} + +func (t *trackingHTTPFilterBuilder) IsTerminal() bool { return false } + +func (t *trackingHTTPFilterBuilder) TypeURLs() []string { return []string{t.typeURL} } + +func (*trackingHTTPFilterBuilder) ParseFilterConfig(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return filterConfigFromProto(cfg) +} + +func (*trackingHTTPFilterBuilder) ParseFilterConfigOverride(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) { + return filterConfigFromProto(cfg) +} + +func (t *trackingHTTPFilterBuilder) BuildServerFilter() httpfilter.ServerFilter { + t.filtersCreated.Add(1) + return t +} + +func (t *trackingHTTPFilterBuilder) Close() { + t.filtersDestroyed.Add(1) +} + +var _ httpfilter.ClientFilterBuilder = &trackingHTTPFilterBuilder{} +var _ httpfilter.ServerFilterBuilder = &trackingHTTPFilterBuilder{} + +func (t *trackingHTTPFilterBuilder) BuildServerInterceptor(config, override httpfilter.FilterConfig) (httpfilter.ServerInterceptor, error) { + t.interceptorsCreated.Add(1) + + var effectiveCfg testFilterCfg + if override != nil { + effectiveCfg = override.(testFilterCfg) + } else if config != nil { + effectiveCfg = config.(testFilterCfg) + } else { + return nil, fmt.Errorf("unexpected missing config") + } + + interceptor := &trackingServerInterceptor{ + parent: t, + pathCh: t.pathCh, + basePath: effectiveCfg.path, + } + return interceptor, nil +} + +type trackingServerInterceptor struct { + parent *trackingHTTPFilterBuilder + pathCh chan string + basePath string +} + +func (i *trackingServerInterceptor) InterceptRPC(ss grpc.ServerStream) (grpc.ServerStream, error) { + i.parent.interceptRPCCount.Add(1) + if i.pathCh != nil { + i.pathCh <- i.basePath + } + if i.parent.interceptRPCFunc != nil { + return i.parent.interceptRPCFunc(ss) + } + return &wrappedServerStream{ + ServerStream: ss, + parent: i.parent, + }, nil +} + +func (i *trackingServerInterceptor) Close() { + i.parent.interceptorsDestroyed.Add(1) +} + +// wrappedServerStream wraps grpc.ServerStream to intercept and count +// RecvMsg and SendMsg invocations for testing. +type wrappedServerStream struct { + grpc.ServerStream + parent *trackingHTTPFilterBuilder +} + +func (w *wrappedServerStream) RecvMsg(m any) error { + if err := w.ServerStream.RecvMsg(m); err != nil { + return err + } + w.parent.recvMsgCount.Add(1) + return nil +} + +func (w *wrappedServerStream) SendMsg(m any) error { + if err := w.ServerStream.SendMsg(m); err != nil { + return err + } + w.parent.sendMsgCount.Add(1) + return nil +} + +func newHTTPFilter(t *testing.T, name, typeURL, path string) *v3httppb.HttpFilter { + return &v3httppb.HttpFilter{ + Name: name, + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{ + TypeUrl: typeURL, + Value: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: path}}, + }, + }, + }), + }, + } +} + +func (t *trackingHTTPFilterBuilder) BuildClientFilter(httpfilter.ClientFilterOptions) httpfilter.ClientFilter { + t.filtersCreated.Add(1) + return t +} + +func (t *trackingHTTPFilterBuilder) BuildClientInterceptor(_, _ httpfilter.FilterConfig) (httpfilter.ClientInterceptor, error) { + t.interceptorsCreated.Add(1) + return &trackingClientInterceptor{parent: t}, nil +} + +type trackingClientInterceptor struct { + parent *trackingHTTPFilterBuilder +} + +func (i *trackingClientInterceptor) NewStream(ctx context.Context, _ iresolver.RPCInfo, newStream func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) { + s, err := newStream(ctx, opts...) + if err != nil { + return nil, err + } + + return &wrappedClientStream{ + ClientStream: s, + parent: i.parent, + }, nil +} + +func (i *trackingClientInterceptor) Close() { + i.parent.interceptorsDestroyed.Add(1) +} + +type wrappedClientStream struct { + grpc.ClientStream + parent *trackingHTTPFilterBuilder +} + +func (w *wrappedClientStream) RecvMsg(m any) error { + w.parent.recvMsgCount.Add(1) + return w.ClientStream.RecvMsg(m) +} + +func (w *wrappedClientStream) SendMsg(m any) error { + w.parent.sendMsgCount.Add(1) + return w.ClientStream.SendMsg(m) +} + +func (w *wrappedClientStream) CloseSend() error { + w.parent.closeSendCount.Add(1) + return w.ClientStream.CloseSend() +} + +// Tests the filter state retention behavior when filter configs in the existing +// filter chain are updated, but there are no changes to the filter names. In +// this case, the existing filter instance should be retained, while new +// interceptor instances should be created with the updated config. +func (s) TestServerSideXDS_FilterStateRetention_AcrossUpdates_FilterConfigChange(t *testing.T) { + // Register a custom httpFilter builder for the test. + pathCh := make(chan string, 1) + testFilterTypeURL := t.Name() + fb := newTrackingHTTPFilterBuilder(testFilterTypeURL) + fb.pathCh = pathCh + httpfilter.Register(fb) + defer httpfilter.UnregisterForTesting(fb.typeURL) + + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t) + + // Wait for the server to enter SERVING mode before making RPCs to avoid + // flakes due to the server closing connections. + servingCh := make(chan struct{}) + opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) { + if args.Mode == connectivity.ServingModeServing { + close(servingCh) + } + }) + lis, stopServer := setupGRPCServer(t, bootstrapContents, opt) + defer stopServer() + + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + vhs := []*v3routepb.VirtualHost{{ + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}, + }} + networkFilters := []*v3listenerpb.Filter{{ + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + newHTTPFilter(t, "tracker", testFilterTypeURL, "initial-path"), + e2e.HTTPFilter("router", &v3routerpb.Router{}), + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: vhs, + }, + }, + }), + }, + }} + inboundLis := &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "v4-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{{ + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{Value: uint32(0)}, + }}, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{{ + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{Value: uint32(0)}, + }}, + }, + Filters: networkFilters, + }, + { + Name: "v6-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{{ + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{Value: uint32(0)}, + }}, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{{ + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{Value: uint32(0)}, + }}, + }, + Filters: networkFilters, + }, + }, + } + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Setup the management server with client and server-side resources. + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + select { + case <-servingCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for server to enter SERVING mode") + } + + // Make an RPC and verify that one filter and two interceptors are created + // (one per filter chain). + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + select { + case cfg := <-pathCh: + if got, want := cfg, "initial-path"; got != want { + t.Fatalf("Unexpected config sent to filter, got: %q, want: %q", got, want) + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for interceptor to be invoked") + } + if got, want := fb.filtersCreated.Load(), int32(1); got != want { + t.Fatalf("Created %d filter instances, want: %d", got, want) + } + if got, want := fb.interceptorsCreated.Load(), int32(2); got != want { + t.Fatalf("Created %d interceptor instances, want: %d", got, want) + } + + // Update the filter config in the listener resource. + networkFilters = []*v3listenerpb.Filter{{ + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + newHTTPFilter(t, "tracker", testFilterTypeURL, "final-path"), + e2e.HTTPFilter("router", &v3routerpb.Router{}), + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: vhs, + }, + }, + }), + }, + }} + inboundLis.FilterChains[0].Filters = networkFilters + inboundLis.FilterChains[1].Filters = networkFilters + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the updated config to be applied on the gRPC server. +WaitForUpdatedConfig: + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + select { + case cfg := <-pathCh: + if got, want := cfg, "final-path"; got == want { + break WaitForUpdatedConfig + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for interceptor get updated config") + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for updated config to be applied: %v", ctx.Err()) + } + + // Verify the filter instance is retained, while the interceptor instances + // are replaced with the updated config. + if got, want := fb.filtersCreated.Load(), int32(1); got != want { + t.Fatalf("Created %d filter instances, want: %d", got, want) + } + if got, want := fb.interceptorsCreated.Load(), int32(4); got != want { + t.Fatalf("Created %d interceptor instances, want: %d", got, want) + } + if got, want := fb.interceptorsDestroyed.Load(), int32(2); got != want { + t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want) + } + + // Stop the server (which causes the listener to be closed) to trigger + // cleanup of filters and interceptors, and verify that all instances are + // cleaned up. + stopServer() + if got, want := fb.filtersDestroyed.Load(), int32(1); got != want { + t.Fatalf("Destroyed %d filter instances, want: %d", got, want) + } + if got, want := fb.interceptorsDestroyed.Load(), int32(4); got != want { + t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want) + } +} + +// Tests the filter state retention behavior when a new filter chain with a +// filter of the existing type but different filter name is added to the +// listener resource. In this case, a filter instance should be created because +// the filter name is part of the key used to identify filter instances. +func (s) TestServerSideXDS_FilterStateRetention_AcrossUpdates_FilterChainsChange(t *testing.T) { + // Register a custom httpFilter builder for the test. + pathCh := make(chan string, 1) + testFilterTypeURL := t.Name() + fb := newTrackingHTTPFilterBuilder(testFilterTypeURL) + fb.pathCh = pathCh + httpfilter.Register(fb) + defer httpfilter.UnregisterForTesting(fb.typeURL) + + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t) + + // Wait for the server to enter SERVING mode before making RPCs to avoid + // flakes due to the server closing connections. + servingCh := make(chan struct{}) + opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) { + if args.Mode == connectivity.ServingModeServing { + close(servingCh) + } + }) + lis, stopServer := setupGRPCServer(t, bootstrapContents, opt) + defer stopServer() + + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + vhs := []*v3routepb.VirtualHost{{ + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}, + }} + networkFilters := []*v3listenerpb.Filter{{ + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + newHTTPFilter(t, "tracker", testFilterTypeURL, "initial-path"), + e2e.HTTPFilter("router", &v3routerpb.Router{}), + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: vhs, + }, + }, + }), + }, + }} + inboundLis := &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: networkFilters}, + } + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Setup the management server with client and server-side resources. + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + select { + case <-servingCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for server to enter SERVING mode") + } + + // Make an RPC and verify that one filter and one interceptor (for the + // default filter chain) is created. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + select { + case cfg := <-pathCh: + if got, want := cfg, "initial-path"; got != want { + t.Fatalf("Unexpected config sent to filter, got: %q, want: %q", got, want) + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for interceptor to be invoked") + } + if got, want := fb.filtersCreated.Load(), int32(1); got != want { + t.Fatalf("Created %d filter instances, want: %d", got, want) + } + if got, want := fb.interceptorsCreated.Load(), int32(1); got != want { + t.Fatalf("Created %d interceptor instances, want: %d", got, want) + } + + // Add a new filter chain to the listener resource that contains HTTP + // filters with a different filter name. This should result in a new filter + // instance being created because the filter name is part of the key used to + // identify filter instances. + networkFilters = []*v3listenerpb.Filter{{ + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + newHTTPFilter(t, "tracker-new", testFilterTypeURL, "final-path"), + e2e.HTTPFilter("router", &v3routerpb.Router{}), + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: vhs, + }, + }, + }), + }, + }} + inboundLis.FilterChains = []*v3listenerpb.FilterChain{ + { + Name: "v4-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{{ + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{Value: uint32(0)}, + }}, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{{ + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{Value: uint32(0)}, + }}, + }, + Filters: networkFilters, + }, + { + Name: "v6-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{{ + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{Value: uint32(0)}, + }}, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{{ + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{Value: uint32(0)}, + }}, + }, + Filters: networkFilters, + }, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the updated config to be applied on the gRPC server. +WaitForUpdatedConfig: + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + select { + case cfg := <-pathCh: + if got, want := cfg, "final-path"; got == want { + break WaitForUpdatedConfig + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for interceptor get updated config") + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for updated config to be applied: %v", ctx.Err()) + } + + // Verify the a new filter instance is created because of the new filter + // name. Three new interceptor instances should also be created (one for the + // default filter chain, and two for the newly added filter chains). + if got, want := fb.filtersCreated.Load(), int32(2); got != want { + t.Fatalf("Created %d filter instances, want: %d", got, want) + } + if got, want := fb.interceptorsCreated.Load(), int32(4); got != want { + t.Fatalf("Created %d interceptor instances, want: %d", got, want) + } + if got, want := fb.interceptorsDestroyed.Load(), int32(1); got != want { + t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want) + } + + // Stop the server (which causes the listener to be closed) to trigger + // cleanup of filters and interceptors, and verify that all instances are + // cleaned up. + stopServer() + if got, want := fb.filtersDestroyed.Load(), int32(2); got != want { + t.Fatalf("Destroyed %d filter instances, want: %d", got, want) + } + if got, want := fb.interceptorsDestroyed.Load(), int32(4); got != want { + t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want) + } +} diff --git a/test/xds/xds_server_integration_test.go b/test/xds/xds_server_integration_test.go new file mode 100644 index 0000000..40645f1 --- /dev/null +++ b/test/xds/xds_server_integration_test.go @@ -0,0 +1,548 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "io" + "net" + "strconv" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func testModeChangeServerOption(t *testing.T) grpc.ServerOption { + // Create a server option to get notified about serving mode changes. We don't + // do anything other than throwing a log entry here. But this is required, + // since the server code emits a log entry at the default level (which is + // ERROR) if no callback is registered for serving mode changes. Our + // testLogger fails the test if there is any log entry at ERROR level. It does + // provide an ExpectError() method, but that takes a string and it would be + // painful to construct the exact error message expected here. Instead this + // works just fine. + return xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) { + t.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err) + }) +} + +// acceptNotifyingListener wraps a listener and notifies users when a server +// calls the Listener.Accept() method. This can be used to ensure that the +// server is ready before requests are sent to it. +type acceptNotifyingListener struct { + net.Listener + serverReady grpcsync.Event +} + +func (l *acceptNotifyingListener) Accept() (net.Conn, error) { + l.serverReady.Fire() + return l.Listener.Accept() +} + +// setupGRPCServer performs the following: +// - spin up an xDS-enabled gRPC server, configure it with xdsCredentials and +// register the test service on it +// - create a local TCP listener and start serving on it +// +// Returns the following: +// - local listener on which the xDS-enabled gRPC server is serving on +// - cleanup function to be invoked by the tests when done +func setupGRPCServer(t *testing.T, bootstrapContents []byte, opts ...grpc.ServerOption) (net.Listener, func()) { + t.Helper() + + // Configure xDS credentials to be used on the server-side. + creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{ + FallbackCreds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatal(err) + } + + // Initialize a test gRPC server, assign it to the stub server, and start + // the test service. + stub := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + _, err := stream.Recv() // hangs here forever if stream doesn't shut down...doesn't receive EOF without any errors + if err == io.EOF { + return nil + } + if err != nil { + return err + } + } + }, + } + + opts = append([]grpc.ServerOption{ + grpc.Creds(creds), + testModeChangeServerOption(t), + xds.BootstrapContentsForTesting(bootstrapContents), + }, opts...) + if stub.S, err = xds.NewGRPCServer(opts...); err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + + // Create a local listener and pass it to Serve(). + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + + readyLis := &acceptNotifyingListener{ + Listener: lis, + serverReady: *grpcsync.NewEvent(), + } + + stub.Listener = readyLis + stubserver.StartTestService(t, stub) + + // Wait for the server to start running. + select { + case <-readyLis.serverReady.Done(): + case <-time.After(defaultTestTimeout): + t.Fatalf("Timed out while waiting for the backend server to start serving") + } + + return lis, func() { + stub.S.Stop() + } +} + +func hostPortFromListener(lis net.Listener) (string, uint32, error) { + host, p, err := net.SplitHostPort(lis.Addr().String()) + if err != nil { + return "", 0, fmt.Errorf("net.SplitHostPort(%s) failed: %v", lis.Addr().String(), err) + } + port, err := strconv.ParseInt(p, 10, 32) + if err != nil { + return "", 0, fmt.Errorf("strconv.ParseInt(%s, 10, 32) failed: %v", p, err) + } + return host, uint32(port), nil +} + +// TestServerSideXDS_Fallback is an e2e test which verifies xDS credentials +// fallback functionality. +// +// The following sequence of events happen as part of this test: +// - An xDS-enabled gRPC server is created and xDS credentials are configured. +// - xDS is enabled on the client by the use of the xds:/// scheme, and xDS +// credentials are configured. +// - Control plane is configured to not send any security configuration to both +// the client and the server. This results in both of them using the +// configured fallback credentials (which is insecure creds in this case). +func (s) TestServerSideXDS_Fallback(t *testing.T) { + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t) + + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + // Grab the host and port of the server and create client side xDS resources + // corresponding to it. This contains default resources with no security + // configuration in the Cluster resources. + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service-fallback" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + // Create an inbound xDS listener resource for the server side that does not + // contain any security configuration. This should force the server-side + // xdsCredentials to use fallback. + inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName") + resources.Listeners = append(resources.Listeners, inboundLis) + + // Setup the management server with client and server-side resources. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{ + FallbackCreds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn with the xds scheme and make a successful RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to create a client for server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Errorf("rpc EmptyCall() failed: %v", err) + } +} + +// TestServerSideXDS_FileWatcherCerts is an e2e test which verifies xDS +// credentials with file watcher certificate provider. +// +// The following sequence of events happen as part of this test: +// - An xDS-enabled gRPC server is created and xDS credentials are configured. +// - xDS is enabled on the client by the use of the xds:/// scheme, and xDS +// credentials are configured. +// - Control plane is configured to send security configuration to both the +// client and the server, pointing to the file watcher certificate provider. +// We verify both TLS and mTLS scenarios. +func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) { + tests := []struct { + name string + secLevel e2e.SecurityLevel + }{ + { + name: "tls", + secLevel: e2e.SecurityLevelTLS, + }, + { + name: "mtls", + secLevel: e2e.SecurityLevelMTLS, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t) + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + // Grab the host and port of the server and create client side xDS + // resources corresponding to it. + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + + // Create xDS resources to be consumed on the client side. This + // includes the listener, route configuration, cluster (with + // security configuration) and endpoint resources. + serviceName := "my-service-file-watcher-certs-" + test.name + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: test.secLevel, + }) + + // Create an inbound xDS listener resource for the server side that + // contains security configuration pointing to the file watcher + // plugin. + inboundLis := e2e.DefaultServerListener(host, port, test.secLevel, "routeName") + resources.Listeners = append(resources.Listeners, inboundLis) + + // Setup the management server with client and server resources. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{ + FallbackCreds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn with the xds scheme and make an RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to create a client for server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + }) + } +} + +// TestServerSideXDS_SecurityConfigChange is an e2e test where xDS is enabled on +// the server-side and xdsCredentials are configured for security. The control +// plane initially does not any security configuration. This forces the +// xdsCredentials to use fallback creds, which is this case is insecure creds. +// We verify that a client connecting with TLS creds is not able to successfully +// make an RPC. The control plane then sends a listener resource with security +// configuration pointing to the use of the file_watcher plugin and we verify +// that the same client is now able to successfully make an RPC. +func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) { + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + xdsResolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + // Grab the host and port of the server and create client side xDS resources + // corresponding to it. This contains default resources with no security + // configuration in the Cluster resource. This should force the xDS + // credentials on the client to use its fallback. + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service-security-config-change" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + // Create an inbound xDS listener resource for the server side that does not + // contain any security configuration. This should force the xDS credentials + // on server to use its fallback. + inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName") + resources.Listeners = append(resources.Listeners, inboundLis) + + // Setup the management server with client and server-side resources. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + xdsCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{ + FallbackCreds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn with the xds scheme and make a successful RPC. + xdsCC, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(xdsCreds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to create a client for server: %v", err) + } + defer xdsCC.Close() + + client := testgrpc.NewTestServiceClient(xdsCC) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // Create a ClientConn with TLS creds. This should fail since the server is + // using fallback credentials which in this case in insecure creds. + tlsCreds := testutils.CreateClientTLSCredentials(t) + tlsCC, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(tlsCreds)) + if err != nil { + t.Fatalf("failed to create a client for server: %v", err) + } + defer tlsCC.Close() + + // We don't set 'waitForReady` here since we want this call to failfast. + client = testgrpc.NewTestServiceClient(tlsCC) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable { + t.Fatal("rpc EmptyCall() succeeded when expected to fail") + } + + // Switch server and client side resources with ones that contain required + // security configuration for mTLS with a file watcher certificate provider. + resources = e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelMTLS, + }) + inboundLis = e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS, "routeName") + resources.Listeners = append(resources.Listeners, inboundLis) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Make another RPC with `waitForReady` set and expect this to succeed. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} + +// TestServerSideXDS_FileWatcherCertsSPIFFE is an e2e test which verifies xDS +// credentials with file watcher certificate provider that is configured with a +// SPIFFE Bundle Map for it's roots. +// +// The following sequence of events happen as part of this test: +// - An xDS-enabled gRPC server is created and xDS credentials are configured. +// - xDS is enabled on the client by the use of the xds:/// scheme, and xDS +// credentials are configured. +// - Control plane is configured to send security configuration to both the +// client and the server, pointing to the file watcher certificate provider. +// We verify both TLS and mTLS scenarios. +func (s) TestServerSideXDS_FileWatcherCertsSPIFFE(t *testing.T) { + tests := []struct { + name string + secLevel e2e.SecurityLevel + }{ + { + name: "tls", + secLevel: e2e.SecurityLevelTLS, + }, + { + name: "mtls", + secLevel: e2e.SecurityLevelMTLS, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolverWithSPIFFE(t) + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + // Grab the host and port of the server and create client side xDS + // resources corresponding to it. + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + + // Create xDS resources to be consumed on the client side. This + // includes the listener, route configuration, cluster (with + // security configuration) and endpoint resources. + serviceName := "my-service-file-watcher-certs-" + test.name + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: test.secLevel, + }) + + // Create an inbound xDS listener resource for the server side that + // contains security configuration pointing to the file watcher + // plugin. + inboundLis := e2e.DefaultServerListener(host, port, test.secLevel, "routeName") + resources.Listeners = append(resources.Listeners, inboundLis) + + // Setup the management server with client and server resources. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{ + FallbackCreds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn with the xds scheme and make an RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to create a client for server: %v", err) + } + defer cc.Close() + + peer := &peer.Peer{} + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeerSPIFFE(t, peer, test.secLevel, 1) + }) + } +} + +// Checks the AuthInfo available in the peer if it matches the expected security +// level of the connection. +func verifySecurityInformationFromPeerSPIFFE(t *testing.T, pr *peer.Peer, wantSecLevel e2e.SecurityLevel, wantPeerChainLen int) { + // This is not a true helper in the Go sense, because it does not perform + // setup or cleanup tasks. Marking it a helper is to ensure that when the + // test fails, the line information of the caller is outputted instead of + // from here. + // + // And this function directly calls t.Fatalf() instead of returning an error + // and letting the caller decide what to do with it. This is also OK since + // all callers will simply end up calling t.Fatalf() with the returned + // error, and can't add any contextual information of value to the error + // message. + t.Helper() + + authType := pr.AuthInfo.AuthType() + switch wantSecLevel { + case e2e.SecurityLevelNone: + if authType != "insecure" { + t.Fatalf("AuthType() is %s, want insecure", authType) + } + case e2e.SecurityLevelMTLS: + if authType != "tls" { + t.Fatalf("AuthType() is %s, want tls", authType) + } + ai, ok := pr.AuthInfo.(credentials.TLSInfo) + if !ok { + t.Fatalf("AuthInfo type is %T, want %T", pr.AuthInfo, credentials.TLSInfo{}) + } + if len(ai.State.PeerCertificates) != wantPeerChainLen { + t.Fatalf("Number of peer certificates is %d, want %d", len(ai.State.PeerCertificates), wantPeerChainLen) + } + } +} diff --git a/test/xds/xds_server_interceptor_leak_test.go b/test/xds/xds_server_interceptor_leak_test.go new file mode 100644 index 0000000..1a2d27b --- /dev/null +++ b/test/xds/xds_server_interceptor_leak_test.go @@ -0,0 +1,274 @@ +/* + * + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "net" + "strconv" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/internal/xds/httpfilter" + "google.golang.org/grpc/xds" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// Tests that interceptor instances are properly closed when a new RDS +// configuration is received for the same route configuration name. +func (s) TestServerSideXDS_InterceptorLeak_RDSUpdate(t *testing.T) { + // Register a custom httpFilter builder for the test. + pathCh := make(chan string, 1) + testFilterTypeURL := t.Name() + fb := newTrackingHTTPFilterBuilder(testFilterTypeURL) + fb.pathCh = pathCh + httpfilter.Register(fb) + defer httpfilter.UnregisterForTesting(fb.typeURL) + + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t) + + // Wait for the server to enter SERVING mode before making RPCs to avoid + // flakes due to the server closing connections. + servingCh := make(chan struct{}) + opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) { + if args.Mode == connectivity.ServingModeServing { + close(servingCh) + } + }) + lis, stopServer := setupGRPCServer(t, bootstrapContents, opt) + defer stopServer() + + // Add xDS resources for the client. + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + // Add xDS resources for the server. + vhs := []*v3routepb.VirtualHost{{ + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}, + }} + const routeConfigName = "routeName" + rds1 := &v3routepb.RouteConfiguration{ + Name: routeConfigName, + VirtualHosts: vhs, + } + networkFilters := []*v3listenerpb.Filter{{ + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + newHTTPFilter(t, "tracker", testFilterTypeURL, "path1"), + e2e.HTTPFilter("router", &v3routerpb.Router{}), + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: routeConfigName, + }, + }, + }), + }, + }} + inboundLis := &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "v4-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{{ + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{Value: uint32(0)}, + }}, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{{ + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{Value: uint32(0)}, + }}, + }, + Filters: networkFilters, + }, + { + Name: "v6-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{{ + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{Value: uint32(0)}, + }}, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{{ + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{Value: uint32(0)}, + }}, + }, + Filters: networkFilters, + }, + }, + } + resources.Listeners = append(resources.Listeners, inboundLis) + resources.Routes = append(resources.Routes, rds1) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + select { + case <-servingCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for server to enter SERVING mode") + } + + // Make an RPC and verify that one filter and two interceptors are created + // (one per filter chain). + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + select { + case cfg := <-pathCh: + if got, want := cfg, "path1"; got != want { + t.Fatalf("Unexpected config sent to filter, got: %q, want: %q", got, want) + } + case <-ctx.Done(): + t.Fatalf("Timeout waiting for interceptor to be invoked") + } + if got, want := fb.filtersCreated.Load(), int32(1); got != want { + t.Fatalf("Created %d filter instances, want: %d", got, want) + } + if got, want := fb.interceptorsCreated.Load(), int32(2); got != want { + t.Fatalf("Created %d interceptor instances, want: %d", got, want) + } + + // Update the RDS configuration with a new route for the same route configuration + // name. This should result in a new interceptor instance being created for + // the new route. + rds2 := &v3routepb.RouteConfiguration{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"*"}, + Routes: []*v3routepb.Route{ + { + Name: "EmptyCall", + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }, + { + Name: "Wildcard", + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }, + }, + }}, + } + resources.Routes = []*v3routepb.RouteConfiguration{resources.Routes[0], rds2} + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the updated config to be applied. Since the new configuration has + // two routes, we expect two interceptor instances to be created for each + // filter chain, for a total of 6 interceptor instances. We also wait for + // the old interceptors (one per filter chain, two total) to be destroyed, + // since destruction happens after the new interceptors are created within + // the same RDS update callback. Breaking only on interceptorsCreated==6 + // would race with the subsequent interceptorsDestroyed increment. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + select { + case <-pathCh: + default: + } + if fb.interceptorsCreated.Load() == 6 && fb.interceptorsDestroyed.Load() == 2 { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for updated config to be applied: %v", ctx.Err()) + } + + // Verify the filter instance is retained, while the interceptor instances + // are replaced with the updated config. + if got, want := fb.filtersCreated.Load(), int32(1); got != want { + t.Fatalf("Created %d filter instances, want: %d", got, want) + } + + stopServer() + if got, want := fb.filtersDestroyed.Load(), int32(1); got != want { + t.Fatalf("Destroyed %d filter instances, want: %d", got, want) + } + if got, want := fb.interceptorsDestroyed.Load(), int32(6); got != want { + t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want) + } +} diff --git a/test/xds/xds_server_rbac_test.go b/test/xds/xds_server_rbac_test.go new file mode 100644 index 0000000..6d1660e --- /dev/null +++ b/test/xds/xds_server_rbac_test.go @@ -0,0 +1,1201 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "encoding/json" + "fmt" + "net" + "strconv" + "strings" + "sync" + "testing" + + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + rpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestServerSideXDS_RouteConfiguration is an e2e test which verifies routing +// functionality. The xDS enabled server will be set up with route configuration +// where the route configuration has routes with the correct routing actions +// (NonForwardingAction), and the RPC's matching those routes should proceed as +// normal. +func (s) TestServerSideXDS_RouteConfiguration(t *testing.T) { + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t) + + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service-fallback" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + // Create an inbound xDS listener resource with route configuration which + // selectively will allow RPC's through or not. This will test routing in + // xds(Unary|Stream)Interceptors. + vhs := []*v3routepb.VirtualHost{ + // Virtual host that will never be matched to test Virtual Host selection. + { + Domains: []string{"this will not match*"}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }, + }, + }, + // This Virtual Host will actually get matched to. + { + Domains: []string{"*"}, + Routes: []*v3routepb.Route{ + // A routing rule that can be selectively triggered based on properties about incoming RPC. + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + // "Fully-qualified RPC method name with leading slash. Same as :path header". + }, + // Correct Action, so RPC's that match this route should proceed to interceptor processing. + Action: &v3routepb.Route_NonForwardingAction{}, + }, + // This routing rule is matched the same way as the one above, + // except has an incorrect action for the server side. However, + // since routing chooses the first route which matches an + // incoming RPC, this should never get invoked (iteration + // through this route slice is deterministic). + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + // "Fully-qualified RPC method name with leading slash. Same as :path header". + }, + // Incorrect Action, so RPC's that match this route should get denied. + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: ""}}, + }, + }, + // Another routing rule that can be selectively triggered based on incoming RPC. + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}, + }, + // Wrong action (!Non_Forwarding_Action) so RPC's that match this route should get denied. + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: ""}}, + }, + }, + // Another routing rule that can be selectively triggered based on incoming RPC. + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/StreamingInputCall"}, + }, + // Wrong action (!Non_Forwarding_Action) so RPC's that match this route should get denied. + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: ""}}, + }, + }, + // Not matching route, this is be able to get invoked logically (i.e. doesn't have to match the Route configurations above). + }}, + } + inboundLis := &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "v4-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: vhs, + }, + }, + }), + }, + }, + }, + }, + { + Name: "v6-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: vhs, + }, + }, + }), + }, + }, + }, + }, + }, + } + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Setup the management server with client and server-side resources. + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + // This Empty Call should match to a route with a correct action + // (NonForwardingAction). Thus, this RPC should proceed as normal. There is + // a routing rule that this RPC would match to that has an incorrect action, + // but the server should only use the first route matched to with the + // correct action. + if _, err = client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // This Unary Call should match to a route with an incorrect action. Thus, + // this RPC should not go through as per A36, and this call should receive + // an error with codes.Unavailable. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if status.Code(err) != codes.Unavailable { + t.Fatalf("client.UnaryCall() = _, %v, want _, error code %s", err, codes.Unavailable) + } + if !strings.Contains(err.Error(), nodeID) { + t.Fatalf("client.UnaryCall() = %v, want xDS node id %q", err, nodeID) + } + + // This Streaming Call should match to a route with an incorrect action. + // Thus, this RPC should not go through as per A36, and this call should + // receive an error with codes.Unavailable. + stream, err := client.StreamingInputCall(ctx) + if err != nil { + t.Fatalf("StreamingInputCall(_) = _, %v, want ", err) + } + _, err = stream.CloseAndRecv() + const wantStreamingErr = "the incoming RPC matched to a route that was not of action type non forwarding" + if status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantStreamingErr) { + t.Fatalf("client.StreamingInputCall() = %v, want error with code %s and message %q", err, codes.Unavailable, wantStreamingErr) + } + if !strings.Contains(err.Error(), nodeID) { + t.Fatalf("client.StreamingInputCall() = %v, want xDS node id %q", err, nodeID) + } + + // This Full Duplex should not match to a route, and thus should return an + // error and not proceed. + dStream, err := client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall(_) = _, %v, want ", err) + } + _, err = dStream.Recv() + const wantFullDuplexErr = "the incoming RPC did not match a configured Route" + if status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantFullDuplexErr) { + t.Fatalf("client.FullDuplexCall() = %v, want error with code %s and message %q", err, codes.Unavailable, wantFullDuplexErr) + } + if !strings.Contains(err.Error(), nodeID) { + t.Fatalf("client.FullDuplexCall() = %v, want xDS node id %q", err, nodeID) + } +} + +// serverListenerWithRBACHTTPFilters returns an xds Listener resource with HTTP Filters defined in the HCM, and a route +// configuration that always matches to a route and a VH. +func serverListenerWithRBACHTTPFilters(t *testing.T, host string, port uint32, rbacCfg *rpb.RBAC) *v3listenerpb.Listener { + // Rather than declare typed config inline, take a HCM proto and append the + // RBAC Filters to it. + hcm := &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}, + // This tests override parsing + building when RBAC Filter + // passed both normal and override config. + TypedPerFilterConfig: map[string]*anypb.Any{ + "rbac": testutils.MarshalAny(t, &rpb.RBACPerRoute{Rbac: rbacCfg}), + }, + }}}, + }, + } + hcm.HttpFilters = nil + hcm.HttpFilters = append(hcm.HttpFilters, e2e.HTTPFilter("rbac", rbacCfg)) + hcm.HttpFilters = append(hcm.HttpFilters, e2e.RouterHTTPFilter) + + return &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "v4-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, hcm), + }, + }, + }, + }, + { + Name: "v6-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, hcm), + }, + }, + }, + }, + }, + } +} + +// TestRBACHTTPFilter tests the xds configured RBAC HTTP Filter. It sets up the +// full end to end flow, and makes sure certain RPC's are successful and proceed +// as normal and certain RPC's are denied by the RBAC HTTP Filter which gets +// called by hooked xds interceptors. +func (s) TestRBACHTTPFilter(t *testing.T) { + tests := []struct { + name string + rbacCfg *rpb.RBAC + wantStatusEmptyCall codes.Code + wantStatusUnaryCall codes.Code + wantAuthzOutcomes map[bool]int + eventContent *audit.Event + }{ + // This test tests an RBAC HTTP Filter which is configured to allow any RPC. + // Any RPC passing through this RBAC HTTP Filter should proceed as normal. + { + name: "allow-anything", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + { + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "stat_logger", + TypedConfig: createXDSTypedStruct(t, map[string]any{}, "stat_logger"), + }, + IsOptional: false, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.OK, + wantStatusUnaryCall: codes.OK, + wantAuthzOutcomes: map[bool]int{true: 2, false: 0}, + // TODO(gtcooke94) add policy name (RBAC filter name) once + // https://github.com/grpc/grpc-go/pull/6327 is merged. + eventContent: &audit.Event{ + FullMethodName: "/grpc.testing.TestService/UnaryCall", + MatchedRule: "anyone", + Authorized: true, + }, + }, + // This test tests an RBAC HTTP Filter which is configured to allow only + // RPC's with certain paths ("UnaryCall"). Only unary calls passing + // through this RBAC HTTP Filter should proceed as normal, and any + // others should be denied. + { + name: "allow-certain-path", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "certain-path": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "/grpc.testing.TestService/UnaryCall"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.PermissionDenied, + wantStatusUnaryCall: codes.OK, + }, + // This test tests an RBAC HTTP Filter which is configured to allow only + // RPC's with certain paths ("UnaryCall") via the ":path" header. Only + // unary calls passing through this RBAC HTTP Filter should proceed as + // normal, and any others should be denied. + { + name: "allow-certain-path-by-header", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "certain-path": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: ":path", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "/grpc.testing.TestService/UnaryCall"}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.PermissionDenied, + wantStatusUnaryCall: codes.OK, + }, + // This test that a RBAC Config with nil rules means that every RPC is + // allowed. This maps to the line "If absent, no enforcing RBAC policy + // will be applied" from the RBAC Proto documentation for the Rules + // field. + { + name: "absent-rules", + rbacCfg: &rpb.RBAC{ + Rules: nil, + }, + wantStatusEmptyCall: codes.OK, + wantStatusUnaryCall: codes.OK, + }, + // The two tests below test that configuring the xDS RBAC HTTP Filter + // with :authority and host header matchers end up being logically + // equivalent. This represents functionality from this line in A41 - + // "As documented for HeaderMatcher, Envoy aliases :authority and Host + // in its header map implementation, so they should be treated + // equivalent for the RBAC matchers; there must be no behavior change + // depending on which of the two header names is used in the RBAC + // policy." + + // This test tests an xDS RBAC Filter with an :authority header matcher. + { + name: "match-on-authority", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "match-on-authority": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: ":authority", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: "my-service-fallback"}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.OK, + wantStatusUnaryCall: codes.OK, + }, + // This test tests that configuring an xDS RBAC Filter with a host + // header matcher has the same behavior as if it was configured with + // :authority. Since host and authority are aliased, this should still + // continue to match on incoming RPC's :authority, just as the test + // above. + { + name: "match-on-host", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "match-on-authority": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: "host", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: "my-service-fallback"}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.OK, + wantStatusUnaryCall: codes.OK, + }, + // This test tests that a "Host" header matcher behaves the same as a + // "host" one. gRPC lowercases every metadata key, so the alias to + // :authority must not depend on how the control plane spells the name. + { + name: "match-on-host-canonical-case", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "match-on-authority": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: "Host", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: "my-service-fallback"}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.OK, + wantStatusUnaryCall: codes.OK, + }, + // This test tests that a header matcher whose name is not lowercase + // still matches the header. Every RPC carries a user agent, so the + // RBAC Configuration below denies every RPC tried. + { + name: "deny-header-name-in-canonical-case", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "user-agent": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: "User-Agent", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PresentMatch{PresentMatch: true}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.PermissionDenied, + wantStatusUnaryCall: codes.PermissionDenied, + }, + // This test tests that the RBAC HTTP Filter hard codes the :method + // header to POST. Since the RBAC Configuration says to deny every RPC + // with a method :POST, every RPC tried should be denied. + { + name: "deny-post", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "post-method": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "POST"}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.PermissionDenied, + wantStatusUnaryCall: codes.PermissionDenied, + }, + { + name: "match-on-principal-remote-ip", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "match-on-principal-remote-ip": { + Permissions: []*v3rbacpb.Permission{ + { + Rule: &v3rbacpb.Permission_Header{ + Header: &v3routepb.HeaderMatcher{ + Name: "host", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: "my-service-fallback"}, + }, + }, + }, + }, + Principals: []*v3rbacpb.Principal{ + { + Identifier: &v3rbacpb.Principal_RemoteIp{ + RemoteIp: &v3corepb.CidrRange{ + AddressPrefix: "127.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{Value: 8}, + }, + }, + }, + { + Identifier: &v3rbacpb.Principal_RemoteIp{ + RemoteIp: &v3corepb.CidrRange{ + AddressPrefix: "::1", + PrefixLen: &wrapperspb.UInt32Value{Value: 128}, + }, + }, + }, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.OK, + wantStatusUnaryCall: codes.OK, + }, + // This test tests that RBAC ignores the TE: trailers header (which is + // hardcoded in http2_client.go for every RPC). Since the RBAC + // Configuration says to only ALLOW RPC's with a TE: Trailers, every RPC + // tried should be denied. + { + name: "allow-only-te", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "post-method": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: "TE", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "trailers"}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.PermissionDenied, + wantStatusUnaryCall: codes.PermissionDenied, + }, + // This test tests that an RBAC Config with Action.LOG configured allows + // every RPC through. This maps to the line "At this time, if the + // RBAC.action is Action.LOG then the policy will be completely ignored, + // as if RBAC was not configured." from A41 + { + name: "action-log", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_LOG, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.OK, + wantStatusUnaryCall: codes.OK, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + func() { + lb := &loggerBuilder{ + authzDecisionStat: map[bool]int{true: 0, false: 0}, + lastEvent: &audit.Event{}, + } + audit.RegisterLoggerBuilder(lb) + + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t) + + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service-fallback" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + inboundLis := serverListenerWithRBACHTTPFilters(t, host, port, test.rbacCfg) + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Setup the management server with client and server-side resources. + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != test.wantStatusEmptyCall { + t.Fatalf("EmptyCall() returned err with status: %v, wantStatusEmptyCall: %v", status.Code(err), test.wantStatusEmptyCall) + } + + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != test.wantStatusUnaryCall { + t.Fatalf("UnaryCall() returned err with status: %v, wantStatusUnaryCall: %v", err, test.wantStatusUnaryCall) + } + + lb.mu.Lock() + defer lb.mu.Unlock() + if test.wantAuthzOutcomes != nil { + if diff := cmp.Diff(lb.authzDecisionStat, test.wantAuthzOutcomes); diff != "" { + t.Fatalf("authorization decision do not match\ndiff (-got +want):\n%s", diff) + } + } + if test.eventContent != nil { + if diff := cmp.Diff(lb.lastEvent, test.eventContent); diff != "" { + t.Fatalf("unexpected event\ndiff (-got +want):\n%s", diff) + } + } + }() + }) + } +} + +// serverListenerWithBadRouteConfiguration returns an xds Listener resource with +// a Route Configuration that will never successfully match in order to test +// RBAC Environment variable being toggled on and off. +func serverListenerWithBadRouteConfiguration(t *testing.T, host string, port uint32) *v3listenerpb.Listener { + return &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "v4-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + // Incoming RPC's will try and match to Virtual Hosts based on their :authority header. + // Thus, incoming RPC's will never match to a Virtual Host (server side requires matching + // to a VH/Route of type Non Forwarding Action to proceed normally), and all incoming RPC's + // with this route configuration will be denied. + Domains: []string{"will-never-match"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}}, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + }), + }, + }, + }, + }, + { + Name: "v6-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + // Incoming RPC's will try and match to Virtual Hosts based on their :authority header. + // Thus, incoming RPC's will never match to a Virtual Host (server side requires matching + // to a VH/Route of type Non Forwarding Action to proceed normally), and all incoming RPC's + // with this route configuration will be denied. + Domains: []string{"will-never-match"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}}, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + }), + }, + }, + }, + }, + }, + } +} + +func (s) TestRBAC_WithBadRouteConfiguration(t *testing.T) { + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t) + // We need to wait for the server to enter SERVING mode before making RPCs + // to avoid flakes due to the server closing connections. + servingCh := make(chan struct{}) + + // Initialize a test gRPC server, assign it to the stub server, and start + // the test service. + opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) { + if args.Mode == connectivity.ServingModeServing { + close(servingCh) + } + }) + lis, cleanup2 := setupGRPCServer(t, bootstrapContents, opt) + defer cleanup2() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service-fallback" + + // The inbound listener needs a route table that will never match on a VH, + // and thus shouldn't allow incoming RPC's to proceed. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + // Since RBAC support is turned ON, all the RPC's should get denied with + // status code Unavailable due to not matching to a route of type Non + // Forwarding Action (Route Table not configured properly). + inboundLis := serverListenerWithBadRouteConfiguration(t, host, port) + resources.Listeners = append(resources.Listeners, inboundLis) + + // Setup the management server with client and server-side resources. + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the xDS-enabled gRPC server to go SERVING") + case <-servingCh: + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if status.Code(err) != codes.Unavailable { + t.Fatalf("EmptyCall() returned %v, want Unavailable", err) + } + if !strings.Contains(err.Error(), nodeID) { + t.Fatalf("EmptyCall() = %v, want xDS node id %q", err, nodeID) + } + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if status.Code(err) != codes.Unavailable { + t.Fatalf("UnaryCall() returned %v, want Unavailable", err) + } + if !strings.Contains(err.Error(), nodeID) { + t.Fatalf("UnaryCall() = %v, want xDS node id %q", err, nodeID) + } +} + +type statAuditLogger struct { + builder *loggerBuilder +} + +func (s *statAuditLogger) Log(event *audit.Event) { + b := s.builder + b.mu.Lock() + defer b.mu.Unlock() + b.authzDecisionStat[event.Authorized]++ + *b.lastEvent = *event +} + +type loggerBuilder struct { + mu sync.Mutex + authzDecisionStat map[bool]int // Map to hold counts of authorization decisions + lastEvent *audit.Event // Field to store last received event +} + +func (*loggerBuilder) Name() string { + return "stat_logger" +} + +func (lb *loggerBuilder) Build(audit.LoggerConfig) audit.Logger { + return &statAuditLogger{ + builder: lb, + } +} + +func (*loggerBuilder) ParseLoggerConfig(json.RawMessage) (audit.LoggerConfig, error) { + return nil, nil +} + +// This is used when converting a custom config from raw JSON to a TypedStruct. +// The TypeURL of the TypeStruct will be "grpc.authz.audit_logging/". +const typeURLPrefix = "grpc.authz.audit_logging/" + +// Builds custom configs for audit logger RBAC protos. +func createXDSTypedStruct(t *testing.T, in map[string]any, name string) *anypb.Any { + t.Helper() + pb, err := structpb.NewStruct(in) + if err != nil { + t.Fatalf("createXDSTypedStruct failed during structpb.NewStruct: %v", err) + } + typedStruct := &v3xdsxdstypepb.TypedStruct{ + TypeUrl: typeURLPrefix + name, + Value: pb, + } + customConfig, err := anypb.New(typedStruct) + if err != nil { + t.Fatalf("createXDSTypedStruct failed during anypb.New: %v", err) + } + return customConfig +} + +// Test verifies server-side route matching against incoming metadata. +// It focuses on a single Exact matcher to confirm restrictive per-route +// RBAC policy selection, while explicitly testing resistance to context +// conflation (Conflicting Contexts regression). +func (s) TestServerSideXDS_RBACRouteHeaderMatchers(t *testing.T) { + const serviceName = "my-service-regression" + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t) + + // Inject conflicting OutgoingContext on the server to verify it is ignored + // by routing. + opt := grpc.UnaryInterceptor(func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + // This simulates leakage or presence of OutgoingContext on the server. + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("x-route", "client-target")) + return handler(ctx, req) + }) + + lis, stopServer := setupGRPCServer(t, bootstrapContents, opt) + defer stopServer() + + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + // Setup a route with an Exact Matcher. + matcher := &v3routepb.HeaderMatcher{ + Name: "x-route", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ + ExactMatch: "server-target", + }, + } + + vhs := []*v3routepb.VirtualHost{{ + Domains: []string{"*"}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + Headers: []*v3routepb.HeaderMatcher{matcher}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + TypedPerFilterConfig: map[string]*anypb.Any{ + "rbac": testutils.MarshalAny(t, &rpb.RBACPerRoute{Rbac: &rpb.RBAC{Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "deny-all": { + Permissions: []*v3rbacpb.Permission{{Rule: &v3rbacpb.Permission_Any{Any: true}}}, + Principals: []*v3rbacpb.Principal{{Identifier: &v3rbacpb.Principal_Any{Any: true}}}, + }, + }, + }}}), + }, + }, + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }, + }, + }} + + inboundLis := &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: []*v3listenerpb.Filter{{ + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + e2e.HTTPFilter("rbac", &rpb.RBAC{Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "allow-all": { + Permissions: []*v3rbacpb.Permission{{Rule: &v3rbacpb.Permission_Any{Any: true}}}, + Principals: []*v3rbacpb.Principal{{Identifier: &v3rbacpb.Principal_Any{Any: true}}}, + }, + }, + }}), + e2e.RouterHTTPFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: vhs, + }, + }, + }), + }, + }}}, + } + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + // Make an RPC with metadata matching the header matcher. We pass + // "server-target" in OutgoingContext (client side). This becomes + // IncomingContext on server-side. If the server routing matches + // based on IncomingContext ("server-target") and ignores + // OutgoingContext ("client-target" injected by interceptor), + // it goes to Route 1 (Deny). + matchCtx := metadata.NewOutgoingContext(ctx, metadata.Pairs("x-route", "server-target")) + if _, err := client.EmptyCall(matchCtx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.PermissionDenied { + t.Fatalf("EmptyCall() failed with status code %v, want PermissionDenied", status.Code(err)) + } + + // Make an RPC with no metadata. This verifies that requests missing + // the matching header cleanly fall through. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.OK { + t.Fatalf("EmptyCall() failed with status code %v, want OK", status.Code(err)) + } +} diff --git a/test/xds/xds_telemetry_labels_test.go b/test/xds/xds_telemetry_labels_test.go new file mode 100644 index 0000000..7f1e79f --- /dev/null +++ b/test/xds/xds_telemetry_labels_test.go @@ -0,0 +1,281 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package xds_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/experimental/stats/telemetry" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/stats" + "google.golang.org/protobuf/types/known/structpb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const serviceNameKey = "service_name" +const serviceNameKeyCSM = "csm.service_name" +const serviceNamespaceKey = "service_namespace" +const serviceNamespaceKeyCSM = "csm.service_namespace_name" +const serviceNameValue = "grpc-service" +const serviceNamespaceValue = "grpc-service-namespace" +const backendServiceKey = "grpc.lb.backend_service" +const backendServiceValue = "cluster-my-service-client-side-xds" +const localityKey = "grpc.lb.locality" +const localityValue = `{region="region-1", zone="zone-1", sub_zone="subzone-1"}` + +// TestTelemetryLabels tests that telemetry labels from CDS make their way to +// the stats handler. The stats handler sets the mutable context value that the +// cluster impl picker will write telemetry labels to, and then the stats +// handler asserts that subsequent HandleRPC calls from the RPC lifecycle +// contain telemetry labels that it can see. +func (s) TestTelemetryLabels(t *testing.T) { + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const xdsServiceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: xdsServiceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + + resources.Clusters[0].Metadata = &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "com.google.csm.telemetry_labels": { + Fields: map[string]*structpb.Value{ + serviceNameKey: structpb.NewStringValue(serviceNameValue), + serviceNamespaceKey: structpb.NewStringValue(serviceNamespaceValue), + }, + }, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + fsh := &fakeStatsHandler{ + t: t, + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver), grpc.WithStatsHandler(fsh)) + if err != nil { + t.Fatalf("failed to create a new client to local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} + +// Tests that telemetry labels for an aggregate cluster hierarchy reflect the +// active leaf cluster receiving traffic, and correctly switch labels when +// failing over to a secondary leaf cluster. +func (s) TestTelemetryLabels_AggregateCluster(t *testing.T) { + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + const ( + xdsServiceName = "my-service-client-side-xds" + cluster1Name = "cluster-1" + cluster2Name = "cluster-2" + + csmName1 = "service-1" + csmNs1 = "namespace-1" + csmName2 = "service-2" + csmNs2 = "namespace-2" + ) + + servers := make([]*stubserver.StubServer, 2) + for i := range 2 { + servers[i] = stubserver.StartTestService(t, nil) + defer servers[i].Stop() + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(xdsServiceName, "route-"+xdsServiceName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig("route-"+xdsServiceName, xdsServiceName, xdsServiceName)}, + Clusters: []*v3clusterpb.Cluster{ + e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: xdsServiceName, + Type: e2e.ClusterTypeAggregate, + ChildNames: []string{cluster1Name, cluster2Name}, + }), + makeClusterResourceWithMetadata(cluster1Name, csmName1, csmNs1), + makeClusterResourceWithMetadata(cluster2Name, csmName2, csmNs2), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(cluster1Name, "localhost", []uint32{uint32(testutils.ParsePort(t, servers[0].Address))}), + e2e.DefaultEndpoint(cluster2Name, "localhost", []uint32{uint32(testutils.ParsePort(t, servers[1].Address))}), + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to create a new client to local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + // Make RPC to primary cluster and verify primary telemetry labels. + peer := &peer.Peer{} + var gotLabels map[string]string + callCtx := telemetry.NewContextWithLabelCallback(ctx, func(l map[string]string) { + gotLabels = l + }) + if _, err := client.EmptyCall(callCtx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if got, want := peer.Addr.String(), servers[0].Address; got != want { + t.Fatalf("EmptyCall() routed to %q, want %q", got, want) + } + + wantLabels := map[string]string{ + localityKey: localityValue, + backendServiceKey: cluster1Name, + } + if diff := cmp.Diff(gotLabels, wantLabels); diff != "" { + t.Fatalf("Telemetry labels for primary cluster (-got +want): %v", diff) + } + + // Trigger failover to secondary cluster by clearing primary endpoints. + resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(cluster1Name, "localhost", nil), + e2e.DefaultEndpoint(cluster2Name, "localhost", []uint32{uint32(testutils.ParsePort(t, servers[1].Address))}), + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Make RPCs until traffic switches to secondary cluster and capture + // secondary labels. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + callCtx := telemetry.NewContextWithLabelCallback(ctx, func(l map[string]string) { + gotLabels = l + }) + if _, err := client.EmptyCall(callCtx, &testpb.Empty{}, grpc.Peer(peer)); err == nil && peer.Addr.String() == servers[1].Address { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout waiting for RPCs to switch to secondary cluster %q", servers[1].Address) + } + + wantLabels = map[string]string{ + localityKey: localityValue, + backendServiceKey: cluster2Name, + } + if diff := cmp.Diff(gotLabels, wantLabels); diff != "" { + t.Fatalf("Telemetry labels after failover (-got +want): %v", diff) + } +} + +type fakeStatsHandler struct { + labels map[string]string + + t *testing.T +} + +func (fsh *fakeStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +func (fsh *fakeStatsHandler) HandleConn(context.Context, stats.ConnStats) {} + +func (fsh *fakeStatsHandler) TagRPC(ctx context.Context, _ *stats.RPCTagInfo) context.Context { + fsh.labels = make(map[string]string) + ctx = telemetry.NewContextWithLabelCallback(ctx, func(l map[string]string) { + for k, v := range l { + fsh.labels[k] = v + } + }) + return ctx +} + +func (fsh *fakeStatsHandler) HandleRPC(_ context.Context, rs stats.RPCStats) { + switch rs.(type) { + // stats.Begin is called before the picker runs, so it won't have telemetry + // labels. + // The following three stats callouts trigger OpenTelemetry metrics and are + // guaranteed to run after the picker has selected a subchannel. Therefore, + // they should have access to the desired telemetry labels. + case *stats.OutPayload, *stats.InPayload, *stats.End: + want := map[string]string{ + serviceNameKeyCSM: serviceNameValue, + serviceNamespaceKeyCSM: serviceNamespaceValue, + localityKey: localityValue, + backendServiceKey: backendServiceValue, + } + if diff := cmp.Diff(fsh.labels, want); diff != "" { + fsh.t.Fatalf("fsh.labels (-got +want): %v", diff) + } + default: + // Nothing to assert for the other stats.Handler callouts. + + } +} + +func makeClusterResourceWithMetadata(clusterName, serviceName, serviceNamespace string) *v3clusterpb.Cluster { + cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + Type: e2e.ClusterTypeEDS, + }) + cluster.Metadata = &v3corepb.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "com.google.csm.telemetry_labels": { + Fields: map[string]*structpb.Value{ + serviceNameKey: structpb.NewStringValue(serviceName), + serviceNamespaceKey: structpb.NewStringValue(serviceNamespace), + }, + }, + }, + } + return cluster +} diff --git a/testdata/README.md b/testdata/README.md new file mode 100644 index 0000000..c621e10 --- /dev/null +++ b/testdata/README.md @@ -0,0 +1,3 @@ +This directory contains x509 certificates used in cloud-to-prod interop tests. +For tests within gRPC-Go repo, please use the files in testdata/x509 +directory. diff --git a/testdata/ca.pem b/testdata/ca.pem new file mode 100644 index 0000000..49d39cd --- /dev/null +++ b/testdata/ca.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDWjCCAkKgAwIBAgIUWrP0VvHcy+LP6UuYNtiL9gBhD5owDQYJKoZIhvcNAQEL +BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTIw +MDMxNzE4NTk1MVoXDTMwMDMxNTE4NTk1MVowVjELMAkGA1UEBhMCQVUxEzARBgNV +BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 +ZDEPMA0GA1UEAwwGdGVzdGNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEAsGL0oXflF0LzoM+Bh+qUU9yhqzw2w8OOX5mu/iNCyUOBrqaHi7mGHx73GD01 +diNzCzvlcQqdNIH6NQSL7DTpBjca66jYT9u73vZe2MDrr1nVbuLvfu9850cdxiUO +Inv5xf8+sTHG0C+a+VAvMhsLiRjsq+lXKRJyk5zkbbsETybqpxoJ+K7CoSy3yc/k +QIY3TipwEtwkKP4hzyo6KiGd/DPexie4nBUInN3bS1BUeNZ5zeaIC2eg3bkeeW7c +qT55b+Yen6CxY0TEkzBK6AKt/WUialKMgT0wbTxRZO7kUCH3Sq6e/wXeFdJ+HvdV +LPlAg5TnMaNpRdQih/8nRFpsdwIDAQABoyAwHjAMBgNVHRMEBTADAQH/MA4GA1Ud +DwEB/wQEAwICBDANBgkqhkiG9w0BAQsFAAOCAQEAkTrKZjBrJXHps/HrjNCFPb5a +THuGPCSsepe1wkKdSp1h4HGRpLoCgcLysCJ5hZhRpHkRihhef+rFHEe60UePQO3S +CVTtdJB4CYWpcNyXOdqefrbJW5QNljxgi6Fhvs7JJkBqdXIkWXtFk2eRgOIP2Eo9 +/OHQHlYnwZFrk6sp4wPyR+A95S0toZBcyDVz7u+hOW0pGK3wviOe9lvRgj/H3Pwt +bewb0l+MhRig0/DVHamyVxrDRbqInU1/GTNCwcZkXKYFWSf92U+kIcTth24Q1gcw +eZiLl5FfrWokUNytFElXob0V0a5/kbhiLc3yWmvWqHTpqCALbVyF+rKJo2f5Kw== +-----END CERTIFICATE----- diff --git a/testdata/grpc_testing_not_regenerated/README.md b/testdata/grpc_testing_not_regenerated/README.md new file mode 100644 index 0000000..8ff72f0 --- /dev/null +++ b/testdata/grpc_testing_not_regenerated/README.md @@ -0,0 +1,12 @@ +`testv3.go` was generated with an older version of codegen, to test reflection +behavior with `grpc.SupportPackageIsVersion3`. DO NOT REGENERATE! + +`testv3.go` was then manually edited to replace `"golang.org/x/net/context"` +with `"context"`. + +`dynamic.go` was generated with a newer protoc and manually edited to remove +everything except the descriptor bytes var, which is renamed and exported. + +`simple_message_v1.go` was generated using protoc-gen-go v1.3.5 which doesn't +support the MesssageV2 API. As a result the generated code implements only the +old MessageV1 API. diff --git a/testdata/grpc_testing_not_regenerated/dynamic.go b/testdata/grpc_testing_not_regenerated/dynamic.go new file mode 100644 index 0000000..e75dbea --- /dev/null +++ b/testdata/grpc_testing_not_regenerated/dynamic.go @@ -0,0 +1,35 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc_testing_not_regenerated + +// FileDynamicProtoRawDesc is the descriptor for dynamic.proto, see README.md. +var FileDynamicProtoRawDesc = []byte{ + 0x0a, 0x0d, 0x64, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x0c, 0x0a, + 0x0a, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x73, 0x22, 0x0c, 0x0a, 0x0a, 0x44, + 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x71, 0x32, 0x57, 0x0a, 0x0e, 0x44, 0x79, 0x6e, + 0x61, 0x6d, 0x69, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0f, 0x44, + 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x31, 0x12, 0x18, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x79, + 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x71, 0x1a, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x52, + 0x65, 0x73, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, + 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} diff --git a/testdata/grpc_testing_not_regenerated/dynamic.proto b/testdata/grpc_testing_not_regenerated/dynamic.proto new file mode 100644 index 0000000..72e4915 --- /dev/null +++ b/testdata/grpc_testing_not_regenerated/dynamic.proto @@ -0,0 +1,33 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +syntax = "proto3"; + +package grpc.testing; + +option go_package = "google.golang.org/grpc/testdata/grpc_testing_not_regenerated"; + +message DynamicRes {} + +message DynamicReq {} + +// DynamicService is used to test reflection on dynamically constructed protocol +// buffer messages. +service DynamicService { + // DynamicMessage1 is a test RPC for dynamically constructed protobufs. + rpc DynamicMessage1(DynamicReq) returns (DynamicRes); +} diff --git a/testdata/grpc_testing_not_regenerated/simple.proto b/testdata/grpc_testing_not_regenerated/simple.proto new file mode 100644 index 0000000..c8b13c7 --- /dev/null +++ b/testdata/grpc_testing_not_regenerated/simple.proto @@ -0,0 +1,27 @@ +/* + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +syntax = "proto3"; + +package grpc.testdata.grpc_testing_not_regenerated; + +option go_package = "google.golang.org/grpc/testdata/grpc_testing_not_regenerated"; + +// SimpleMessage is used to hold string data. +message SimpleMessage { + string data = 1; +} diff --git a/testdata/grpc_testing_not_regenerated/simple_message_v1.go b/testdata/grpc_testing_not_regenerated/simple_message_v1.go new file mode 100644 index 0000000..7bf9ee2 --- /dev/null +++ b/testdata/grpc_testing_not_regenerated/simple_message_v1.go @@ -0,0 +1,83 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: simple.proto + +package grpc_testing_not_regenerated + +import ( + fmt "fmt" + math "math" + + proto "github.com/golang/protobuf/proto" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +// SimpleMessage is used to hold string data. +type SimpleMessage struct { + Data string `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SimpleMessage) Reset() { *m = SimpleMessage{} } +func (m *SimpleMessage) String() string { return proto.CompactTextString(m) } +func (*SimpleMessage) ProtoMessage() {} +func (*SimpleMessage) Descriptor() ([]byte, []int) { + return fileDescriptor_5ffd045dd4d042c1, []int{0} +} + +func (m *SimpleMessage) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SimpleMessage.Unmarshal(m, b) +} +func (m *SimpleMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SimpleMessage.Marshal(b, m, deterministic) +} +func (m *SimpleMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_SimpleMessage.Merge(m, src) +} +func (m *SimpleMessage) XXX_Size() int { + return xxx_messageInfo_SimpleMessage.Size(m) +} +func (m *SimpleMessage) XXX_DiscardUnknown() { + xxx_messageInfo_SimpleMessage.DiscardUnknown(m) +} + +var xxx_messageInfo_SimpleMessage proto.InternalMessageInfo + +func (m *SimpleMessage) GetData() string { + if m != nil { + return m.Data + } + return "" +} + +func init() { + proto.RegisterType((*SimpleMessage)(nil), "grpc.testdata.grpc_testing_not_regenerated.SimpleMessage") +} + +func init() { + proto.RegisterFile("simple.proto", fileDescriptor_5ffd045dd4d042c1) +} + +var fileDescriptor_5ffd045dd4d042c1 = []byte{ + // 142 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0xce, 0xcc, 0x2d, + 0xc8, 0x49, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xd2, 0x4a, 0x2f, 0x2a, 0x48, 0xd6, 0x2b, + 0x49, 0x2d, 0x2e, 0x49, 0x49, 0x2c, 0x49, 0xd4, 0x03, 0xf1, 0xe2, 0x41, 0xbc, 0xcc, 0xbc, 0xf4, + 0xf8, 0xbc, 0xfc, 0x92, 0xf8, 0xa2, 0xd4, 0xf4, 0xd4, 0xbc, 0xd4, 0xa2, 0xc4, 0x92, 0xd4, 0x14, + 0x25, 0x65, 0x2e, 0xde, 0x60, 0xb0, 0x5e, 0xdf, 0xd4, 0xe2, 0xe2, 0xc4, 0xf4, 0x54, 0x21, 0x21, + 0x2e, 0x16, 0x90, 0x2e, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xce, 0x20, 0x30, 0xdb, 0xc9, 0x2e, 0xca, + 0x26, 0x3d, 0x3f, 0x3f, 0x3d, 0x27, 0x55, 0x2f, 0x3d, 0x3f, 0x27, 0x31, 0x2f, 0x5d, 0x2f, 0xbf, + 0x28, 0x5d, 0x1f, 0x64, 0xac, 0x3e, 0xcc, 0x12, 0x7d, 0x7c, 0x96, 0x24, 0xb1, 0x81, 0xdd, 0x65, + 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0x2c, 0x80, 0xd6, 0x07, 0xa7, 0x00, 0x00, 0x00, +} diff --git a/testdata/grpc_testing_not_regenerated/testv3.go b/testdata/grpc_testing_not_regenerated/testv3.go new file mode 100644 index 0000000..7c55c9a --- /dev/null +++ b/testdata/grpc_testing_not_regenerated/testv3.go @@ -0,0 +1,476 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Code generated by protoc-gen-go. +// source: testv3.proto +// DO NOT EDIT! + +/* +Package grpc_testing_not_regenerated is a generated protocol buffer package. + +It is generated from these files: + + testv3.proto + +It has these top-level messages: + + SearchResponseV3 + SearchRequestV3 +*/ +package grpc_testing_not_regenerated + +import ( + context "context" + fmt "fmt" + math "math" + + proto "github.com/golang/protobuf/proto" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type SearchResponseV3_State int32 + +const ( + SearchResponseV3_UNKNOWN SearchResponseV3_State = 0 + SearchResponseV3_FRESH SearchResponseV3_State = 1 + SearchResponseV3_STALE SearchResponseV3_State = 2 +) + +var SearchResponseV3_State_name = map[int32]string{ + 0: "UNKNOWN", + 1: "FRESH", + 2: "STALE", +} +var SearchResponseV3_State_value = map[string]int32{ + "UNKNOWN": 0, + "FRESH": 1, + "STALE": 2, +} + +func (x SearchResponseV3_State) String() string { + return proto.EnumName(SearchResponseV3_State_name, int32(x)) +} +func (SearchResponseV3_State) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{0, 0} } + +type SearchResponseV3 struct { + Results []*SearchResponseV3_Result `protobuf:"bytes,1,rep,name=results" json:"results,omitempty"` + State SearchResponseV3_State `protobuf:"varint,2,opt,name=state,enum=grpc.testingv3.SearchResponseV3_State" json:"state,omitempty"` +} + +func (m *SearchResponseV3) Reset() { *m = SearchResponseV3{} } +func (m *SearchResponseV3) String() string { return proto.CompactTextString(m) } +func (*SearchResponseV3) ProtoMessage() {} +func (*SearchResponseV3) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *SearchResponseV3) GetResults() []*SearchResponseV3_Result { + if m != nil { + return m.Results + } + return nil +} + +func (m *SearchResponseV3) GetState() SearchResponseV3_State { + if m != nil { + return m.State + } + return SearchResponseV3_UNKNOWN +} + +type SearchResponseV3_Result struct { + Url string `protobuf:"bytes,1,opt,name=url" json:"url,omitempty"` + Title string `protobuf:"bytes,2,opt,name=title" json:"title,omitempty"` + Snippets []string `protobuf:"bytes,3,rep,name=snippets" json:"snippets,omitempty"` + Metadata map[string]*SearchResponseV3_Result_Value `protobuf:"bytes,4,rep,name=metadata" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` +} + +func (m *SearchResponseV3_Result) Reset() { *m = SearchResponseV3_Result{} } +func (m *SearchResponseV3_Result) String() string { return proto.CompactTextString(m) } +func (*SearchResponseV3_Result) ProtoMessage() {} +func (*SearchResponseV3_Result) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0, 0} } + +func (m *SearchResponseV3_Result) GetUrl() string { + if m != nil { + return m.Url + } + return "" +} + +func (m *SearchResponseV3_Result) GetTitle() string { + if m != nil { + return m.Title + } + return "" +} + +func (m *SearchResponseV3_Result) GetSnippets() []string { + if m != nil { + return m.Snippets + } + return nil +} + +func (m *SearchResponseV3_Result) GetMetadata() map[string]*SearchResponseV3_Result_Value { + if m != nil { + return m.Metadata + } + return nil +} + +type SearchResponseV3_Result_Value struct { + // Types that are valid to be assigned to Val: + // *SearchResponseV3_Result_Value_Str + // *SearchResponseV3_Result_Value_Int + // *SearchResponseV3_Result_Value_Real + Val isSearchResponseV3_Result_Value_Val `protobuf_oneof:"val"` +} + +func (m *SearchResponseV3_Result_Value) Reset() { *m = SearchResponseV3_Result_Value{} } +func (m *SearchResponseV3_Result_Value) String() string { return proto.CompactTextString(m) } +func (*SearchResponseV3_Result_Value) ProtoMessage() {} +func (*SearchResponseV3_Result_Value) Descriptor() ([]byte, []int) { + return fileDescriptor0, []int{0, 0, 0} +} + +type isSearchResponseV3_Result_Value_Val interface { + isSearchResponseV3_Result_Value_Val() +} + +type SearchResponseV3_Result_Value_Str struct { + Str string `protobuf:"bytes,1,opt,name=str,oneof"` +} +type SearchResponseV3_Result_Value_Int struct { + Int int64 `protobuf:"varint,2,opt,name=int,oneof"` +} +type SearchResponseV3_Result_Value_Real struct { + Real float64 `protobuf:"fixed64,3,opt,name=real,oneof"` +} + +func (*SearchResponseV3_Result_Value_Str) isSearchResponseV3_Result_Value_Val() {} +func (*SearchResponseV3_Result_Value_Int) isSearchResponseV3_Result_Value_Val() {} +func (*SearchResponseV3_Result_Value_Real) isSearchResponseV3_Result_Value_Val() {} + +func (m *SearchResponseV3_Result_Value) GetVal() isSearchResponseV3_Result_Value_Val { + if m != nil { + return m.Val + } + return nil +} + +func (m *SearchResponseV3_Result_Value) GetStr() string { + if x, ok := m.GetVal().(*SearchResponseV3_Result_Value_Str); ok { + return x.Str + } + return "" +} + +func (m *SearchResponseV3_Result_Value) GetInt() int64 { + if x, ok := m.GetVal().(*SearchResponseV3_Result_Value_Int); ok { + return x.Int + } + return 0 +} + +func (m *SearchResponseV3_Result_Value) GetReal() float64 { + if x, ok := m.GetVal().(*SearchResponseV3_Result_Value_Real); ok { + return x.Real + } + return 0 +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*SearchResponseV3_Result_Value) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _SearchResponseV3_Result_Value_OneofMarshaler, _SearchResponseV3_Result_Value_OneofUnmarshaler, _SearchResponseV3_Result_Value_OneofSizer, []interface{}{ + (*SearchResponseV3_Result_Value_Str)(nil), + (*SearchResponseV3_Result_Value_Int)(nil), + (*SearchResponseV3_Result_Value_Real)(nil), + } +} + +func _SearchResponseV3_Result_Value_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*SearchResponseV3_Result_Value) + // val + switch x := m.Val.(type) { + case *SearchResponseV3_Result_Value_Str: + b.EncodeVarint(1<<3 | proto.WireBytes) + b.EncodeStringBytes(x.Str) + case *SearchResponseV3_Result_Value_Int: + b.EncodeVarint(2<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.Int)) + case *SearchResponseV3_Result_Value_Real: + b.EncodeVarint(3<<3 | proto.WireFixed64) + b.EncodeFixed64(math.Float64bits(x.Real)) + case nil: + default: + return fmt.Errorf("SearchResponseV3_Result_Value.Val has unexpected type %T", x) + } + return nil +} + +func _SearchResponseV3_Result_Value_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*SearchResponseV3_Result_Value) + switch tag { + case 1: // val.str + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeStringBytes() + m.Val = &SearchResponseV3_Result_Value_Str{x} + return true, err + case 2: // val.int + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Val = &SearchResponseV3_Result_Value_Int{int64(x)} + return true, err + case 3: // val.real + if wire != proto.WireFixed64 { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeFixed64() + m.Val = &SearchResponseV3_Result_Value_Real{math.Float64frombits(x)} + return true, err + default: + return false, nil + } +} + +func _SearchResponseV3_Result_Value_OneofSizer(msg proto.Message) (n int) { + m := msg.(*SearchResponseV3_Result_Value) + // val + switch x := m.Val.(type) { + case *SearchResponseV3_Result_Value_Str: + n += proto.SizeVarint(1<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(len(x.Str))) + n += len(x.Str) + case *SearchResponseV3_Result_Value_Int: + n += proto.SizeVarint(2<<3 | proto.WireVarint) + n += proto.SizeVarint(uint64(x.Int)) + case *SearchResponseV3_Result_Value_Real: + n += proto.SizeVarint(3<<3 | proto.WireFixed64) + n += 8 + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +type SearchRequestV3 struct { + Query string `protobuf:"bytes,1,opt,name=query" json:"query,omitempty"` +} + +func (m *SearchRequestV3) Reset() { *m = SearchRequestV3{} } +func (m *SearchRequestV3) String() string { return proto.CompactTextString(m) } +func (*SearchRequestV3) ProtoMessage() {} +func (*SearchRequestV3) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *SearchRequestV3) GetQuery() string { + if m != nil { + return m.Query + } + return "" +} + +func init() { + proto.RegisterType((*SearchResponseV3)(nil), "grpc.testingv3.SearchResponseV3") + proto.RegisterType((*SearchResponseV3_Result)(nil), "grpc.testingv3.SearchResponseV3.Result") + proto.RegisterType((*SearchResponseV3_Result_Value)(nil), "grpc.testingv3.SearchResponseV3.Result.Value") + proto.RegisterType((*SearchRequestV3)(nil), "grpc.testingv3.SearchRequestV3") + proto.RegisterEnum("grpc.testingv3.SearchResponseV3_State", SearchResponseV3_State_name, SearchResponseV3_State_value) +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion3 + +// Client API for SearchServiceV3 service + +type SearchServiceV3Client interface { + Search(ctx context.Context, in *SearchRequestV3, opts ...grpc.CallOption) (*SearchResponseV3, error) + StreamingSearch(ctx context.Context, opts ...grpc.CallOption) (SearchServiceV3_StreamingSearchClient, error) +} + +type searchServiceV3Client struct { + cc *grpc.ClientConn +} + +func NewSearchServiceV3Client(cc *grpc.ClientConn) SearchServiceV3Client { + return &searchServiceV3Client{cc} +} + +func (c *searchServiceV3Client) Search(ctx context.Context, in *SearchRequestV3, opts ...grpc.CallOption) (*SearchResponseV3, error) { + out := new(SearchResponseV3) + err := grpc.Invoke(ctx, "/grpc.testingv3.SearchServiceV3/Search", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *searchServiceV3Client) StreamingSearch(ctx context.Context, opts ...grpc.CallOption) (SearchServiceV3_StreamingSearchClient, error) { + stream, err := grpc.NewClientStream(ctx, &_SearchServiceV3_serviceDesc.Streams[0], c.cc, "/grpc.testingv3.SearchServiceV3/StreamingSearch", opts...) + if err != nil { + return nil, err + } + x := &searchServiceV3StreamingSearchClient{stream} + return x, nil +} + +type SearchServiceV3_StreamingSearchClient interface { + Send(*SearchRequestV3) error + Recv() (*SearchResponseV3, error) + grpc.ClientStream +} + +type searchServiceV3StreamingSearchClient struct { + grpc.ClientStream +} + +func (x *searchServiceV3StreamingSearchClient) Send(m *SearchRequestV3) error { + return x.ClientStream.SendMsg(m) +} + +func (x *searchServiceV3StreamingSearchClient) Recv() (*SearchResponseV3, error) { + m := new(SearchResponseV3) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// Server API for SearchServiceV3 service + +type SearchServiceV3Server interface { + Search(context.Context, *SearchRequestV3) (*SearchResponseV3, error) + StreamingSearch(SearchServiceV3_StreamingSearchServer) error +} + +func RegisterSearchServiceV3Server(s *grpc.Server, srv SearchServiceV3Server) { + s.RegisterService(&_SearchServiceV3_serviceDesc, srv) +} + +func _SearchServiceV3_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SearchRequestV3) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SearchServiceV3Server).Search(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.testingv3.SearchServiceV3/Search", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SearchServiceV3Server).Search(ctx, req.(*SearchRequestV3)) + } + return interceptor(ctx, in, info, handler) +} + +func _SearchServiceV3_StreamingSearch_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SearchServiceV3Server).StreamingSearch(&searchServiceV3StreamingSearchServer{stream}) +} + +type SearchServiceV3_StreamingSearchServer interface { + Send(*SearchResponseV3) error + Recv() (*SearchRequestV3, error) + grpc.ServerStream +} + +type searchServiceV3StreamingSearchServer struct { + grpc.ServerStream +} + +func (x *searchServiceV3StreamingSearchServer) Send(m *SearchResponseV3) error { + return x.ServerStream.SendMsg(m) +} + +func (x *searchServiceV3StreamingSearchServer) Recv() (*SearchRequestV3, error) { + m := new(SearchRequestV3) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +var _SearchServiceV3_serviceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testingv3.SearchServiceV3", + HandlerType: (*SearchServiceV3Server)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Search", + Handler: _SearchServiceV3_Search_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "StreamingSearch", + Handler: _SearchServiceV3_StreamingSearch_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: fileDescriptor0, +} + +func init() { proto.RegisterFile("testv3.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 416 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x93, 0xd1, 0x6a, 0xd4, 0x40, + 0x14, 0x86, 0x77, 0x36, 0x9b, 0x6d, 0xf7, 0xac, 0xb6, 0x61, 0xe8, 0x45, 0xc8, 0x8d, 0x61, 0x2f, + 0x6c, 0x10, 0x0c, 0x92, 0x20, 0x88, 0x78, 0x53, 0x65, 0x65, 0xa1, 0x75, 0xc5, 0x89, 0xae, 0xde, + 0x8e, 0xeb, 0x61, 0x8d, 0x4d, 0xb3, 0xe9, 0xcc, 0x49, 0x60, 0x9f, 0xc5, 0x17, 0xf1, 0x55, 0x7c, + 0x1b, 0x99, 0x99, 0xa6, 0x50, 0x41, 0xba, 0x17, 0xde, 0xcd, 0x7f, 0x38, 0xff, 0x37, 0xff, 0x3f, + 0x24, 0xf0, 0x80, 0x50, 0x53, 0x97, 0xa7, 0x8d, 0xda, 0xd2, 0x96, 0x1f, 0x6d, 0x54, 0xb3, 0x4e, + 0xcd, 0xa8, 0xac, 0x37, 0x5d, 0x3e, 0xfb, 0x39, 0x82, 0xa0, 0x40, 0xa9, 0xd6, 0xdf, 0x05, 0xea, + 0x66, 0x5b, 0x6b, 0x5c, 0xe5, 0xfc, 0x0c, 0x0e, 0x14, 0xea, 0xb6, 0x22, 0x1d, 0xb2, 0xd8, 0x4b, + 0xa6, 0xd9, 0x69, 0x7a, 0xd7, 0x96, 0xfe, 0x6d, 0x49, 0x85, 0xdd, 0x17, 0xbd, 0x8f, 0xbf, 0x02, + 0x5f, 0x93, 0x24, 0x0c, 0x87, 0x31, 0x4b, 0x8e, 0xb2, 0xc7, 0xf7, 0x02, 0x0a, 0xb3, 0x2d, 0x9c, + 0x29, 0xfa, 0x3d, 0x84, 0xb1, 0x23, 0xf2, 0x00, 0xbc, 0x56, 0x55, 0x21, 0x8b, 0x59, 0x32, 0x11, + 0xe6, 0xc8, 0x4f, 0xc0, 0xa7, 0x92, 0x2a, 0x87, 0x9e, 0x08, 0x27, 0x78, 0x04, 0x87, 0xba, 0x2e, + 0x9b, 0x06, 0x49, 0x87, 0x5e, 0xec, 0x25, 0x13, 0x71, 0xab, 0xf9, 0x07, 0x38, 0xbc, 0x42, 0x92, + 0xdf, 0x24, 0xc9, 0x70, 0x64, 0x0b, 0x3d, 0xdf, 0xb3, 0x50, 0xfa, 0xee, 0xc6, 0x37, 0xaf, 0x49, + 0xed, 0xc4, 0x2d, 0x26, 0xba, 0x00, 0x7f, 0x25, 0xab, 0x16, 0x39, 0x07, 0x4f, 0x93, 0x72, 0xf9, + 0x16, 0x03, 0x61, 0x84, 0x99, 0x95, 0x35, 0xd9, 0x7c, 0x9e, 0x99, 0x95, 0x35, 0xf1, 0x13, 0x18, + 0x29, 0x94, 0x55, 0xe8, 0xc5, 0x2c, 0x61, 0x8b, 0x81, 0xb0, 0xea, 0xb5, 0x0f, 0x5e, 0x27, 0xab, + 0xe8, 0x07, 0x3c, 0xbc, 0x73, 0x91, 0x69, 0x7d, 0x89, 0xbb, 0xbe, 0xf5, 0x25, 0xee, 0xf8, 0x1b, + 0xf0, 0x3b, 0x73, 0xa1, 0xa5, 0x4e, 0xb3, 0xa7, 0xfb, 0x16, 0xb0, 0x29, 0x85, 0xf3, 0xbe, 0x1c, + 0xbe, 0x60, 0xb3, 0x27, 0xe0, 0xdb, 0xb7, 0xe6, 0x53, 0x38, 0xf8, 0xb4, 0x3c, 0x5f, 0xbe, 0xff, + 0xbc, 0x0c, 0x06, 0x7c, 0x02, 0xfe, 0x5b, 0x31, 0x2f, 0x16, 0x01, 0x33, 0xc7, 0xe2, 0xe3, 0xd9, + 0xc5, 0x3c, 0x18, 0xce, 0x4e, 0xe1, 0xb8, 0xe7, 0x5e, 0xb7, 0xa8, 0x69, 0x95, 0x9b, 0xd7, 0xbf, + 0x6e, 0x51, 0xf5, 0xd9, 0x9c, 0xc8, 0x7e, 0xb1, 0x7e, 0xb3, 0x40, 0xd5, 0x95, 0x6b, 0xf3, 0x15, + 0x9d, 0xc3, 0xd8, 0x8d, 0xf8, 0xa3, 0x7f, 0x85, 0xbd, 0x81, 0x46, 0xf1, 0x7d, 0x6d, 0xf8, 0x17, + 0x38, 0x2e, 0x48, 0xa1, 0xbc, 0x2a, 0xeb, 0xcd, 0x7f, 0xa3, 0x26, 0xec, 0x19, 0xfb, 0x3a, 0xb6, + 0x3f, 0x46, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xed, 0xa2, 0x8d, 0x75, 0x28, 0x03, 0x00, 0x00, +} diff --git a/testdata/grpc_testing_not_regenerated/testv3.proto b/testdata/grpc_testing_not_regenerated/testv3.proto new file mode 100644 index 0000000..c8eee85 --- /dev/null +++ b/testdata/grpc_testing_not_regenerated/testv3.proto @@ -0,0 +1,58 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +syntax = "proto3"; + +package grpc.testingv3; + +option go_package = "google.golang.org/grpc/testdata/grpc_testing_not_regenerated"; + +message SearchResponseV3 { + message Result { + string url = 1; + string title = 2; + repeated string snippets = 3; + message Value { + oneof val { + string str = 1; + int64 int = 2; + double real = 3; + } + } + map metadata = 4; + } + enum State { + UNKNOWN = 0; + FRESH = 1; + STALE = 2; + } + repeated Result results = 1; + State state = 2; +} + +message SearchRequestV3 { + string query = 1; +} + +// SearchServiceV3 is used to test grpc server reflection. +service SearchServiceV3 { + // Search is a unary RPC. + rpc Search(SearchRequestV3) returns (SearchResponseV3); + + // StreamingSearch is a streaming RPC. + rpc StreamingSearch(stream SearchRequestV3) returns (stream SearchResponseV3); +} diff --git a/testdata/server1.key b/testdata/server1.key new file mode 100644 index 0000000..0864629 --- /dev/null +++ b/testdata/server1.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDnE443EknxvxBq +6+hvn/t09hl8hx366EBYvZmVM/NC+7igXRAjiJiA/mIaCvL3MS0Iz5hBLxSGICU+ +WproA3GCIFITIwcf/ETyWj/5xpgZ4AKrLrjQmmX8mhwUajfF3UvwMJrCOVqPp67t +PtP+2kBXaqrXdvnvXR41FsIB8V7zIAuIZB6bHQhiGVlc1sgZYsE2EGG9WMmHtS86 +qkAOTjG2XyjmPTGAwhGDpYkYrpzp99IiDh4/Veai81hn0ssQkbry0XRD/Ig3jcHh +23WiriPNJ0JsbgXUSLKRPZObA9VgOLy2aXoN84IMaeK3yy+cwSYG/99w93fUZJte +MXwz4oYZAgMBAAECggEBAIVn2Ncai+4xbH0OLWckabwgyJ4IM9rDc0LIU368O1kU +koais8qP9dujAWgfoh3sGh/YGgKn96VnsZjKHlyMgF+r4TaDJn3k2rlAOWcurGlj +1qaVlsV4HiEzp7pxiDmHhWvp4672Bb6iBG+bsjCUOEk/n9o9KhZzIBluRhtxCmw5 +nw4Do7z00PTvN81260uPWSc04IrytvZUiAIx/5qxD72bij2xJ8t/I9GI8g4FtoVB +8pB6S/hJX1PZhh9VlU6Yk+TOfOVnbebG4W5138LkB835eqk3Zz0qsbc2euoi8Hxi +y1VGwQEmMQ63jXz4c6g+X55ifvUK9Jpn5E8pq+pMd7ECgYEA93lYq+Cr54K4ey5t +sWMa+ye5RqxjzgXj2Kqr55jb54VWG7wp2iGbg8FMlkQwzTJwebzDyCSatguEZLuB +gRGroRnsUOy9vBvhKPOch9bfKIl6qOgzMJB267fBVWx5ybnRbWN/I7RvMQf3k+9y +biCIVnxDLEEYyx7z85/5qxsXg/MCgYEA7wmWKtCTn032Hy9P8OL49T0X6Z8FlkDC +Rk42ygrc/MUbugq9RGUxcCxoImOG9JXUpEtUe31YDm2j+/nbvrjl6/bP2qWs0V7l +dTJl6dABP51pCw8+l4cWgBBX08Lkeen812AAFNrjmDCjX6rHjWHLJcpS18fnRRkP +V1d/AHWX7MMCgYEA6Gsw2guhp0Zf2GCcaNK5DlQab8OL4Hwrpttzo4kuTlwtqNKp +Q9H4al9qfF4Cr1TFya98+EVYf8yFRM3NLNjZpe3gwYf2EerlJj7VLcahw0KKzoN1 +QBENfwgPLRk5sDkx9VhSmcfl/diLroZdpAwtv3vo4nEoxeuGFbKTGx3Qkf0CgYEA +xyR+dcb05Ygm3w4klHQTowQ10s1H80iaUcZBgQuR1ghEtDbUPZHsoR5t1xCB02ys +DgAwLv1bChIvxvH/L6KM8ovZ2LekBX4AviWxoBxJnfz/EVau98B0b1auRN6eSC83 +FRuGldlSOW1z/nSh8ViizSYE5H5HX1qkXEippvFRE88CgYB3Bfu3YQY60ITWIShv +nNkdcbTT9eoP9suaRJjw92Ln+7ZpALYlQMKUZmJ/5uBmLs4RFwUTQruLOPL4yLTH +awADWUzs3IRr1fwn9E+zM8JVyKCnUEM3w4N5UZskGO2klashAd30hWO+knRv/y0r +uGIYs9Ek7YXlXIRVrzMwcsrt1w== +-----END PRIVATE KEY----- diff --git a/testdata/server1.pem b/testdata/server1.pem new file mode 100644 index 0000000..88244f8 --- /dev/null +++ b/testdata/server1.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDtDCCApygAwIBAgIUbJfTREJ6k6/+oInWhV1O1j3ZT0IwDQYJKoZIhvcNAQEL +BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTIw +MDMxODAzMTA0MloXDTMwMDMxNjAzMTA0MlowZTELMAkGA1UEBhMCVVMxETAPBgNV +BAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRUwEwYDVQQKDAxFeGFtcGxl +LCBDby4xGjAYBgNVBAMMESoudGVzdC5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEA5xOONxJJ8b8Qauvob5/7dPYZfIcd+uhAWL2ZlTPz +Qvu4oF0QI4iYgP5iGgry9zEtCM+YQS8UhiAlPlqa6ANxgiBSEyMHH/xE8lo/+caY +GeACqy640Jpl/JocFGo3xd1L8DCawjlaj6eu7T7T/tpAV2qq13b5710eNRbCAfFe +8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c +6ffSIg4eP1XmovNYZ9LLEJG68tF0Q/yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPV +YDi8tml6DfOCDGnit8svnMEmBv/fcPd31GSbXjF8M+KGGQIDAQABo2swaTAJBgNV +HRMEAjAAMAsGA1UdDwQEAwIF4DBPBgNVHREESDBGghAqLnRlc3QuZ29vZ2xlLmZy +ghh3YXRlcnpvb2kudGVzdC5nb29nbGUuYmWCEioudGVzdC55b3V0dWJlLmNvbYcE +wKgBAzANBgkqhkiG9w0BAQsFAAOCAQEAS8hDQA8PSgipgAml7Q3/djwQ644ghWQv +C2Kb+r30RCY1EyKNhnQnIIh/OUbBZvh0M0iYsy6xqXgfDhCB93AA6j0i5cS8fkhH +Jl4RK0tSkGQ3YNY4NzXwQP/vmUgfkw8VBAZ4Y4GKxppdATjffIW+srbAmdDruIRM +wPeikgOoRrXf0LA1fi4TqxARzeRwenQpayNfGHTvVF9aJkl8HoaMunTAdG5pIVcr +9GKi/gEMpXUJbbVv3U5frX1Wo4CFo+rZWJ/LyCMeb0jciNLxSdMwj/E/ZuExlyeZ +gc9ctPjSMvgSyXEKv6Vwobleeg88V2ZgzenziORoWj4KszG/lbQZvg== +-----END CERTIFICATE----- diff --git a/testdata/spiffe/README.md b/testdata/spiffe/README.md new file mode 100644 index 0000000..99dfa90 --- /dev/null +++ b/testdata/spiffe/README.md @@ -0,0 +1,76 @@ +## File Purposes + +* spiffe_cert.pem - the certificate that is placed in spiffe bundles (copied + into the `x5c` field) +* server1_spiffe.pem - another certificate placed in spiffe bundles +* spiffe_multi_uri_san_cert.pem - another certificate placed in spiffe bundles +* spiffe-openssl.cnf - the configuration file passed to the openssl CLI when + creating these certificate files +* spiffebundle.json - the valid spiffe bundle for happy path testing +* spiffebundle2.json - Another valid spiffe bundle that is used in testing + file reloading (a different file is needed to ensure changes are picked up). + It is just the `example.com` trust domain from spiffebundle.json. +* spiffebundle_corrupted_cert.json - manually modifies the `x5c` field and + removes a character to create an invalid certificate +* spiffebundle_empty_keys.json - the `keys` field is an empty array +* spiffebundle_empty_string_keys.json - the `keys` field contains an entry +* with an empty string key +* spiffebundle_invalid_trustdomain - uses a `#` in the trust domain which is a + disallowed character per the spec +* spiffebundle_malformed.json - a fully wrong json +* spiffebundle_match_client_spiffe.json - a valid spiffe bundle with a trust + domain matching the SPIFFE ID in spiffe_cert.pem +* spiffebundle_wrong_kid.json - has the `kid` field instead of the `kty` field +* spiffebundle_wrong_kty.json - Uses `EC` instead of `RSA` in the `kty` field +* spiffebundle_wrong_multi_certs.json - place 2 certificates in the `x5c` + field +* spiffebundle_wrong_root.json - The top level json string is `trustDomains` + instead of `trust_domains` +* spiffebundle_wrong_seq_type.json - the `spiffe_sequence` number must be an + integer +* spiffebundle_wrong_use.json - The `use` field must be `x509-svid` or + `jwt-svid` (we are expecting and support `x509-svid` per the gRFC) + +## Test File Creation: + +The SPIFFE related extensions are listed in spiffe-openssl.cnf config. Both +client_spiffe.pem and server1_spiffe.pem are generated in the same way as the +client and server certificates described in the testdata/x509 with the same CAs. +Specifically they were made with the following commands: + +``` +$ openssl req -new -key client.key -out spiffe-cert.csr \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=testclient/ \ + -config spiffe-openssl.cnf -reqexts spiffe_client_e2e +$ openssl x509 -req -CA ca.pem -CAkey ca.key -CAcreateserial \ + -in spiffe-cert.csr -out client_spiffe.pem -extensions spiffe_client_e2e \ + -extfile spiffe-openssl.cnf -days 3650 -sha256 +$ openssl req -new -key server1.key -out spiffe-cert.csr \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=*.test.google.com/ \ + -config spiffe-openssl.cnf -reqexts spiffe_server_e2e +$ openssl x509 -req -CA ca.pem -CAkey ca.key -CAcreateserial \ + -in spiffe-cert.csr -out server1_spiffe.pem -extensions spiffe_server_e2e \ + -extfile spiffe-openssl.cnf -days 3650 -sha256 +``` + +Additionally, the SPIFFE trust bundle map files (spiffebundle*.json) are +manually created for end to end testing. The spiffebundle.json contains the +"example.com" trust domain (only this entry is used in e2e tests) matching URI +SAN of server1_spiffe.pem, and the CA certificate there is ca.pem. The +spiffebundle.json file contains "foo.bar.com" trust domain (only this entry is +used in e2e tests) matching URI SAN of client_spiffe.pem, and the CA certificate +there is also ca.pem. + +If updating these files, the `x5c` field in the json is the raw PEM certificates +and can be copy pasted from the certificate file. `n` and `e` are values from +the public key. `e` should *probably* be `AQAB` as it is the exponent. `n` can +be fetched from the certificate by getting the RSA key from the cert and +extracting the value. This can be done in golang with the following codeblock: +``` func GetBase64ModulusFromPublicKey(key *rsa.PublicKey) string { return +base64.RawURLEncoding.EncodeToString(key.N.Bytes()) } + +block, _ := pem.Decode(rawPemCert) cert, _ := x509.ParseCertificate(block.Bytes) +publicKey := cert.PublicKey.(*rsa.PublicKey) +fmt.Println(GetBase64ModulusFromPublicKey(publicKey)) ``` + +The rest of the files are manually modified as described above. diff --git a/testdata/spiffe/client_spiffe.pem b/testdata/spiffe/client_spiffe.pem new file mode 100644 index 0000000..c70981a --- /dev/null +++ b/testdata/spiffe/client_spiffe.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEMjCCAxqgAwIBAgIUVXGlXjNENtOZbI12epjgIhMaShUwDQYJKoZIhvcNAQEL +BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTI0 +MTAyNDE2NDAzN1oXDTM0MTAyMjE2NDAzN1owaDELMAkGA1UEBhMCQVUxEzARBgNV +BAgMClNvbWUtU3RhdGUxDDAKBgNVBAcMA1NWTDEhMB8GA1UECgwYSW50ZXJuZXQg +V2lkZ2l0cyBQdHkgTHRkMRMwEQYDVQQDDAp0ZXN0Y2xpZW50MIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsqmEafg11ae9jRW0B/IXYU2S8nGVzpSYZjLK +yZq459qe6SP/Jk2f9BQvkhlgRmVfhC4h65gl+c32iC6/SLsOxoa91c6Hn4vK+tqy +7qVTzYv6naso1pNnRAhwvWd/gINysyk8nq11oynL8ilZjNGcRNEV4Q1v0aEG6mbF +NhioNQdq4VFPCjdIFZip9KyRzsc0VUmHmC2KeWJ+yq7TyXCsqPWlbhK+3RgDc6ch +epYP52AVnPvUhsJKC3RbyrwAWCTMq2zYR1EH79H82mdD/OnX0xDaw8cwC68xp6nM +dyk68CY5Gf2kq9bcg9P7V77pERYj8VgSYYx0O9BqkxUGNfUW4QIDAQABo4HlMIHi +MEQGA1UdEQQ9MDuGOXNwaWZmZTovL2Zvby5iYXIuY29tLzllZWJjY2QyLTEyYmYt +NDBhNi1iMjYyLTY1ZmUwNDg3ZDQ1MzAdBgNVHQ4EFgQU28U8sUTGNEDyeCrvJDJd +AALabSMwewYDVR0jBHQwcqFapFgwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNv +bWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0G +A1UEAwwGdGVzdGNhghRas/RW8dzL4s/pS5g22Iv2AGEPmjANBgkqhkiG9w0BAQsF +AAOCAQEAE3LLE8GR283q/aE646SgAfltqpESP38NmYdJMdZgWRxbOqdWabYDfibt +9r8j+IRvVuuTWuH2eNS5wXJtS1BZ+z24wTLa+a2KjOV12gChP+3N7jhqId4eolSL +1fjscPY6luZP4Pm3D73lBvIoBvXpDGyrxleiUCEEkKXmTOA8doFvbrcbwm+yUJOP +VKUKvAzTNztb0BGDzKKU4E2yK5PSyv2n5m2NpzxYYfHoGeVcxvj7nCnSfoX/EWHb +d8ztJYDg9X0iNcfQXt7PZ+j6VcxfDpGCDxe2rFQoYvlWjhr3xOi/1e5A1zx1Ly07 +m9MB4hntu4e2656ZDWbgOHLpO0q1iQ== +-----END CERTIFICATE----- diff --git a/testdata/spiffe/server1_spiffe.pem b/testdata/spiffe/server1_spiffe.pem new file mode 100644 index 0000000..76cb41d --- /dev/null +++ b/testdata/spiffe/server1_spiffe.pem @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEZDCCA0ygAwIBAgIUVXGlXjNENtOZbI12epjgIhMaShMwDQYJKoZIhvcNAQEL +BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTI0 +MTAyMTAyMTQxNVoXDTM0MTAxOTAyMTQxNVowZTELMAkGA1UEBhMCVVMxETAPBgNV +BAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRUwEwYDVQQKDAxFeGFtcGxl +LCBDby4xGjAYBgNVBAMMESoudGVzdC5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEA5xOONxJJ8b8Qauvob5/7dPYZfIcd+uhAWL2ZlTPz +Qvu4oF0QI4iYgP5iGgry9zEtCM+YQS8UhiAlPlqa6ANxgiBSEyMHH/xE8lo/+caY +GeACqy640Jpl/JocFGo3xd1L8DCawjlaj6eu7T7T/tpAV2qq13b5710eNRbCAfFe +8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c +6ffSIg4eP1XmovNYZ9LLEJG68tF0Q/yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPV +YDi8tml6DfOCDGnit8svnMEmBv/fcPd31GSbXjF8M+KGGQIDAQABo4IBGTCCARUw +dwYDVR0RBHAwboIQKi50ZXN0Lmdvb2dsZS5mcoIYd2F0ZXJ6b29pLnRlc3QuZ29v +Z2xlLmJlghIqLnRlc3QueW91dHViZS5jb22HBMCoAQOGJnNwaWZmZTovL2V4YW1w +bGUuY29tL3dvcmtsb2FkLzllZWJjY2QyMB0GA1UdDgQWBBRvRpAYHQYP6dFPf5V7 +/MyCftnNjTB7BgNVHSMEdDByoVqkWDBWMQswCQYDVQQGEwJBVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8w +DQYDVQQDDAZ0ZXN0Y2GCFFqz9Fbx3Mviz+lLmDbYi/YAYQ+aMA0GCSqGSIb3DQEB +CwUAA4IBAQBJ1bnbBHa1n15vvhpGIzokuiJ+9q/zim63UuVDnkhrQM2N+RQbStGT +Tis2tNse1bh460dJFm6ArgHWogzx6fQZzgaDeCOAXvrAe4jM9IHr9K7lkq/33CZS +BDV+jCmm2sRsqSMkKUcX6JhyqWGFHuTDAKJzsEV2MlcswleKlGHDkeelAaxlLzpz +RHOSQd0N9xAs18lzx95SQEx90PtrBOmvIDDiI5o5z9Oz12Iy1toiksFl4jmknkDD +5VF3AyCRgN8NPW0uNC8D2vo4L+tgj9U6NPlmMOrjRsEH257LJ1wopAGr+yezkIId +QQodGSVm5cOuw/K7Ma4nBDjVJkjcdY3t +-----END CERTIFICATE----- diff --git a/testdata/spiffe/spiffe-openssl.cnf b/testdata/spiffe/spiffe-openssl.cnf new file mode 100644 index 0000000..0092743 --- /dev/null +++ b/testdata/spiffe/spiffe-openssl.cnf @@ -0,0 +1,28 @@ +[spiffe_client] +subjectAltName = @alt_names + +[spiffe_client_multi] +subjectAltName = @alt_names_multi + +[spiffe_server_e2e] +subjectAltName = @alt_names_server_e2e + +[spiffe_client_e2e] +subjectAltName = @alt_names_client_e2e + +[alt_names] +URI = spiffe://foo.bar.com/client/workload/1 + +[alt_names_multi] +URI.1 = spiffe://foo.bar.com/client/workload/1 +URI.2 = spiffe://foo.bar.com/client/workload/2 + +[alt_names_server_e2e] +DNS.1 = *.test.google.fr +DNS.2 = waterzooi.test.google.be +DNS.3 = *.test.youtube.com +IP.1 = "192.168.1.3" +URI = spiffe://example.com/workload/9eebccd2 + +[alt_names_client_e2e] +URI = spiffe://foo.bar.com/9eebccd2-12bf-40a6-b262-65fe0487d453 diff --git a/testdata/spiffe/spiffe_cert.pem b/testdata/spiffe/spiffe_cert.pem new file mode 100644 index 0000000..bbe7d66 --- /dev/null +++ b/testdata/spiffe/spiffe_cert.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQEL +BQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQy +NTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEM +MAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVu +dDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJ +LVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/Z +G5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqO +a6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3Z +JPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XV +m0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW75 +7PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfc +msHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740Yc +DmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPN +zHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRs +vvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkI +sK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRH +HvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAP +BgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29t +L2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/ +aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK3 +4Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0 +IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+ +PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV ++j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2D +vUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gq +yjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvV +z6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hx +x0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U +0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EX +GA91fn0891b5eEW8BJHXX0jri0aN8g== +-----END CERTIFICATE----- diff --git a/testdata/spiffe/spiffe_multi_uri_san_cert.pem b/testdata/spiffe/spiffe_multi_uri_san_cert.pem new file mode 100644 index 0000000..14b0872 --- /dev/null +++ b/testdata/spiffe/spiffe_multi_uri_san_cert.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIELTCCAxWgAwIBAgIUVXGlXjNENtOZbI12epjgIhMaShEwDQYJKoZIhvcNAQEL +BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTI0 +MDkxNzE2MTk0NFoXDTM0MDkxNTE2MTk0NFowTjELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRl +c3QtY2xpZW50MTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOcTjjcS +SfG/EGrr6G+f+3T2GXyHHfroQFi9mZUz80L7uKBdECOImID+YhoK8vcxLQjPmEEv +FIYgJT5amugDcYIgUhMjBx/8RPJaP/nGmBngAqsuuNCaZfyaHBRqN8XdS/AwmsI5 +Wo+nru0+0/7aQFdqqtd2+e9dHjUWwgHxXvMgC4hkHpsdCGIZWVzWyBliwTYQYb1Y +yYe1LzqqQA5OMbZfKOY9MYDCEYOliRiunOn30iIOHj9V5qLzWGfSyxCRuvLRdEP8 +iDeNweHbdaKuI80nQmxuBdRIspE9k5sD1WA4vLZpeg3zggxp4rfLL5zBJgb/33D3 +d9Rkm14xfDPihhkCAwEAAaOB+jCB9zBZBgNVHREEUjBQhiZzcGlmZmU6Ly9mb28u +YmFyLmNvbS9jbGllbnQvd29ya2xvYWQvMYYmc3BpZmZlOi8vZm9vLmJhci5jb20v +Y2xpZW50L3dvcmtsb2FkLzIwHQYDVR0OBBYEFG9GkBgdBg/p0U9/lXv8zIJ+2c2N +MHsGA1UdIwR0MHKhWqRYMFYxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0 +YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxDzANBgNVBAMM +BnRlc3RjYYIUWrP0VvHcy+LP6UuYNtiL9gBhD5owDQYJKoZIhvcNAQELBQADggEB +AJ4Cbxv+02SpUgkEu4hP/1+8DtSBXUxNxI0VG4e3Ap2+Rhjm3YiFeS/UeaZhNrrw +UEjkSTPFODyXR7wI7UO9OO1StyD6CMkp3SEvevU5JsZtGL6mTiTLTi3Qkywa91Bt +GlyZdVMghA1bBJLBMwiD5VT5noqoJBD7hDy6v9yNmt1Sw2iYBJPqI3Gnf5bMjR3s +UICaxmFyqaMCZsPkfJh0DmZpInGJys3m4QqGz6ZE2DWgcSr1r/ML7/5bSPjjr8j4 +WFFSqFR3dMu8CbGnfZTCTXa4GTX/rARXbAO67Z/oJbJBK7VKayskL+PzKuohb9ox +jGL772hQMbwtFCOFXu5VP0s= +-----END CERTIFICATE----- diff --git a/testdata/spiffe/spiffe_test.json b/testdata/spiffe/spiffe_test.json new file mode 100644 index 0000000..5af9a08 --- /dev/null +++ b/testdata/spiffe/spiffe_test.json @@ -0,0 +1,13 @@ +{ + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8g==" + ], + "n": "yeAKqRnrclUqhlLgSS1RZWvu2xH3Gu1GM31UkmS2f1_iQ5liFTfFCD7hYm-Y4TBHN12lJxV1JjO8xLc_2RuYWqcC8Dm9gt2cTj-j0I2AicW5C4nvm-N1EKxbtccXd7zIVSYHI_wCMRKZrM7qjmupFK5GIn4yQA0PuuJGYJeum7E3VzAs6cchZfcVH33-fS_Dl_h1HwHKGc93po4N2ST2FEUBJo3WUzmrtRIxYpcB4HffPSbezSoMd9DV6B5sO57rFQSGPeHy9lSjd39V1ZtHR7uTSNu_UOs69GQVO5OU9yHh587FHrEEuDRpZJQ1qFUypO91L_T52vMkLElu-ezxGxzhqfSp7uG56BjzpbBvNciD4qTuoU4BhStuKRfezPZKRm1qmF-Ptry5awgH3JrB6mlD5kJWHkVU0-r8O3LWKNpMlGBrZzg15O-i6nsEQVb5M8e-UFIX4wMm--NGHA5iHp3Oek6uoHmyR44RJlUnoUesSKVz-0lXrq08oT7BEJ6SR7YLOHXMFjmaewbzzcx15ian6-77LIYnt70uZENLtXtcqgKl4Mg46ilzpMfnaYFE3oUiR4TzR703OS10bL71ymM-ha8Efw2fLewTRLz3i4xYMaR-0FSRSwrwm17OSusmG7Jy38RSAT0Z62yZCLCvrEMJ0XZ7IlHT7E5BuQdpUH0", + "e": "AQAB" + } + ] +} diff --git a/testdata/spiffe/spiffebundle.json b/testdata/spiffe/spiffebundle.json new file mode 100644 index 0000000..c0cc1cf --- /dev/null +++ b/testdata/spiffe/spiffebundle.json @@ -0,0 +1,31 @@ +{ + "trust_domains": { + "example.com": { + "spiffe_sequence": 12035488, + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8g==" + ], + "n": "yeAKqRnrclUqhlLgSS1RZWvu2xH3Gu1GM31UkmS2f1_iQ5liFTfFCD7hYm-Y4TBHN12lJxV1JjO8xLc_2RuYWqcC8Dm9gt2cTj-j0I2AicW5C4nvm-N1EKxbtccXd7zIVSYHI_wCMRKZrM7qjmupFK5GIn4yQA0PuuJGYJeum7E3VzAs6cchZfcVH33-fS_Dl_h1HwHKGc93po4N2ST2FEUBJo3WUzmrtRIxYpcB4HffPSbezSoMd9DV6B5sO57rFQSGPeHy9lSjd39V1ZtHR7uTSNu_UOs69GQVO5OU9yHh587FHrEEuDRpZJQ1qFUypO91L_T52vMkLElu-ezxGxzhqfSp7uG56BjzpbBvNciD4qTuoU4BhStuKRfezPZKRm1qmF-Ptry5awgH3JrB6mlD5kJWHkVU0-r8O3LWKNpMlGBrZzg15O-i6nsEQVb5M8e-UFIX4wMm--NGHA5iHp3Oek6uoHmyR44RJlUnoUesSKVz-0lXrq08oT7BEJ6SR7YLOHXMFjmaewbzzcx15ian6-77LIYnt70uZENLtXtcqgKl4Mg46ilzpMfnaYFE3oUiR4TzR703OS10bL71ymM-ha8Efw2fLewTRLz3i4xYMaR-0FSRSwrwm17OSusmG7Jy38RSAT0Z62yZCLCvrEMJ0XZ7IlHT7E5BuQdpUH0", + "e": "AQAB" + } + ] + }, + "test.example.com": { + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIEZDCCA0ygAwIBAgIUVXGlXjNENtOZbI12epjgIhMaShMwDQYJKoZIhvcNAQELBQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTI0MTAyMTAyMTQxNVoXDTM0MTAxOTAyMTQxNVowZTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRUwEwYDVQQKDAxFeGFtcGxlLCBDby4xGjAYBgNVBAMMESoudGVzdC5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5xOONxJJ8b8Qauvob5/7dPYZfIcd+uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM+YQS8UhiAlPlqa6ANxgiBSEyMHH/xE8lo/+caYGeACqy640Jpl/JocFGo3xd1L8DCawjlaj6eu7T7T/tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q/yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv/fcPd31GSbXjF8M+KGGQIDAQABo4IBGTCCARUwdwYDVR0RBHAwboIQKi50ZXN0Lmdvb2dsZS5mcoIYd2F0ZXJ6b29pLnRlc3QuZ29vZ2xlLmJlghIqLnRlc3QueW91dHViZS5jb22HBMCoAQOGJnNwaWZmZTovL2V4YW1wbGUuY29tL3dvcmtsb2FkLzllZWJjY2QyMB0GA1UdDgQWBBRvRpAYHQYP6dFPf5V7/MyCftnNjTB7BgNVHSMEdDByoVqkWDBWMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8wDQYDVQQDDAZ0ZXN0Y2GCFFqz9Fbx3Mviz+lLmDbYi/YAYQ+aMA0GCSqGSIb3DQEBCwUAA4IBAQBJ1bnbBHa1n15vvhpGIzokuiJ+9q/zim63UuVDnkhrQM2N+RQbStGTTis2tNse1bh460dJFm6ArgHWogzx6fQZzgaDeCOAXvrAe4jM9IHr9K7lkq/33CZSBDV+jCmm2sRsqSMkKUcX6JhyqWGFHuTDAKJzsEV2MlcswleKlGHDkeelAaxlLzpzRHOSQd0N9xAs18lzx95SQEx90PtrBOmvIDDiI5o5z9Oz12Iy1toiksFl4jmknkDD5VF3AyCRgN8NPW0uNC8D2vo4L+tgj9U6NPlmMOrjRsEH257LJ1wopAGr+yezkIIdQQodGSVm5cOuw/K7Ma4nBDjVJkjcdY3t" + ], + "n": "5xOONxJJ8b8Qauvob5_7dPYZfIcd-uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM-YQS8UhiAlPlqa6ANxgiBSEyMHH_xE8lo_-caYGeACqy640Jpl_JocFGo3xd1L8DCawjlaj6eu7T7T_tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q_yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv_fcPd31GSbXjF8M-KGGQ", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe/spiffebundle2.json b/testdata/spiffe/spiffebundle2.json new file mode 100644 index 0000000..69a7ba0 --- /dev/null +++ b/testdata/spiffe/spiffebundle2.json @@ -0,0 +1,18 @@ +{ + "trust_domains": { + "example.com": { + "spiffe_sequence": 12035488, + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8g==" + ], + "n": "yeAKqRnrclUqhlLgSS1RZWvu2xH3Gu1GM31UkmS2f1_iQ5liFTfFCD7hYm-Y4TBHN12lJxV1JjO8xLc_2RuYWqcC8Dm9gt2cTj-j0I2AicW5C4nvm-N1EKxbtccXd7zIVSYHI_wCMRKZrM7qjmupFK5GIn4yQA0PuuJGYJeum7E3VzAs6cchZfcVH33-fS_Dl_h1HwHKGc93po4N2ST2FEUBJo3WUzmrtRIxYpcB4HffPSbezSoMd9DV6B5sO57rFQSGPeHy9lSjd39V1ZtHR7uTSNu_UOs69GQVO5OU9yHh587FHrEEuDRpZJQ1qFUypO91L_T52vMkLElu-ezxGxzhqfSp7uG56BjzpbBvNciD4qTuoU4BhStuKRfezPZKRm1qmF-Ptry5awgH3JrB6mlD5kJWHkVU0-r8O3LWKNpMlGBrZzg15O-i6nsEQVb5M8e-UFIX4wMm--NGHA5iHp3Oek6uoHmyR44RJlUnoUesSKVz-0lXrq08oT7BEJ6SR7YLOHXMFjmaewbzzcx15ian6-77LIYnt70uZENLtXtcqgKl4Mg46ilzpMfnaYFE3oUiR4TzR703OS10bL71ymM-ha8Efw2fLewTRLz3i4xYMaR-0FSRSwrwm17OSusmG7Jy38RSAT0Z62yZCLCvrEMJ0XZ7IlHT7E5BuQdpUH0", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe/spiffebundle_corrupted_cert.json b/testdata/spiffe/spiffebundle_corrupted_cert.json new file mode 100644 index 0000000..44a17e7 --- /dev/null +++ b/testdata/spiffe/spiffebundle_corrupted_cert.json @@ -0,0 +1,31 @@ +{ + "trust_domains": { + "example.com": { + "spiffe_sequence": 12035488, + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8" + ], + "n": "yeAKqRnrclUqhlLgSS1RZWvu2xH3Gu1GM31UkmS2f1_iQ5liFTfFCD7hYm-Y4TBHN12lJxV1JjO8xLc_2RuYWqcC8Dm9gt2cTj-j0I2AicW5C4nvm-N1EKxbtccXd7zIVSYHI_wCMRKZrM7qjmupFK5GIn4yQA0PuuJGYJeum7E3VzAs6cchZfcVH33-fS_Dl_h1HwHKGc93po4N2ST2FEUBJo3WUzmrtRIxYpcB4HffPSbezSoMd9DV6B5sO57rFQSGPeHy9lSjd39V1ZtHR7uTSNu_UOs69GQVO5OU9yHh587FHrEEuDRpZJQ1qFUypO91L_T52vMkLElu-ezxGxzhqfSp7uG56BjzpbBvNciD4qTuoU4BhStuKRfezPZKRm1qmF-Ptry5awgH3JrB6mlD5kJWHkVU0-r8O3LWKNpMlGBrZzg15O-i6nsEQVb5M8e-UFIX4wMm--NGHA5iHp3Oek6uoHmyR44RJlUnoUesSKVz-0lXrq08oT7BEJ6SR7YLOHXMFjmaewbzzcx15ian6-77LIYnt70uZENLtXtcqgKl4Mg46ilzpMfnaYFE3oUiR4TzR703OS10bL71ymM-ha8Efw2fLewTRLz3i4xYMaR-0FSRSwrwm17OSusmG7Jy38RSAT0Z62yZCLCvrEMJ0XZ7IlHT7E5BuQdpUH0", + "e": "AQAB" + } + ] + }, + "test.example.com": { + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIEZDCCA0ygAwIBAgIUVXGlXjNENtOZbI12epjgIhMaShMwDQYJKoZIhvcNAQELBQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTI0MTAyMTAyMTQxNVoXDTM0MTAxOTAyMTQxNVowZTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRUwEwYDVQQKDAxFeGFtcGxlLCBDby4xGjAYBgNVBAMMESoudGVzdC5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5xOONxJJ8b8Qauvob5/7dPYZfIcd+uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM+YQS8UhiAlPlqa6ANxgiBSEyMHH/xE8lo/+caYGeACqy640Jpl/JocFGo3xd1L8DCawjlaj6eu7T7T/tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q/yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv/fcPd31GSbXjF8M+KGGQIDAQABo4IBGTCCARUwdwYDVR0RBHAwboIQKi50ZXN0Lmdvb2dsZS5mcoIYd2F0ZXJ6b29pLnRlc3QuZ29vZ2xlLmJlghIqLnRlc3QueW91dHViZS5jb22HBMCoAQOGJnNwaWZmZTovL2V4YW1wbGUuY29tL3dvcmtsb2FkLzllZWJjY2QyMB0GA1UdDgQWBBRvRpAYHQYP6dFPf5V7/MyCftnNjTB7BgNVHSMEdDByoVqkWDBWMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8wDQYDVQQDDAZ0ZXN0Y2GCFFqz9Fbx3Mviz+lLmDbYi/YAYQ+aMA0GCSqGSIb3DQEBCwUAA4IBAQBJ1bnbBHa1n15vvhpGIzokuiJ+9q/zim63UuVDnkhrQM2N+RQbStGTTis2tNse1bh460dJFm6ArgHWogzx6fQZzgaDeCOAXvrAe4jM9IHr9K7lkq/33CZSBDV+jCmm2sRsqSMkKUcX6JhyqWGFHuTDAKJzsEV2MlcswleKlGHDkeelAaxlLzpzRHOSQd0N9xAs18lzx95SQEx90PtrBOmvIDDiI5o5z9Oz12Iy1toiksFl4jmknkDD5VF3AyCRgN8NPW0uNC8D2vo4L+tgj9U6NPlmMOrjRsEH257LJ1wopAGr+yezkIIdQQodGSVm5cOuw/K7Ma4nBDjVJkjcdY3t" + ], + "n": "5xOONxJJ8b8Qauvob5_7dPYZfIcd-uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM-YQS8UhiAlPlqa6ANxgiBSEyMHH_xE8lo_-caYGeACqy640Jpl_JocFGo3xd1L8DCawjlaj6eu7T7T_tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q_yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv_fcPd31GSbXjF8M-KGGQ", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe/spiffebundle_empty_keys.json b/testdata/spiffe/spiffebundle_empty_keys.json new file mode 100644 index 0000000..f12b203 --- /dev/null +++ b/testdata/spiffe/spiffebundle_empty_keys.json @@ -0,0 +1,9 @@ +{ + "trust_domains": { + "": { + "spiffe_sequence": 12035488, + "keys": [ + ] + } + } +} diff --git a/testdata/spiffe/spiffebundle_empty_string_key.json b/testdata/spiffe/spiffebundle_empty_string_key.json new file mode 100644 index 0000000..f213f61 --- /dev/null +++ b/testdata/spiffe/spiffebundle_empty_string_key.json @@ -0,0 +1,18 @@ +{ + "trust_domains": { + "": { + "spiffe_sequence": 12035488, + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8g==" + ], + "n": "yeAKqRnrclUqhlLgSS1RZWvu2xH3Gu1GM31UkmS2f1_iQ5liFTfFCD7hYm-Y4TBHN12lJxV1JjO8xLc_2RuYWqcC8Dm9gt2cTj-j0I2AicW5C4nvm-N1EKxbtccXd7zIVSYHI_wCMRKZrM7qjmupFK5GIn4yQA0PuuJGYJeum7E3VzAs6cchZfcVH33-fS_Dl_h1HwHKGc93po4N2ST2FEUBJo3WUzmrtRIxYpcB4HffPSbezSoMd9DV6B5sO57rFQSGPeHy9lSjd39V1ZtHR7uTSNu_UOs69GQVO5OU9yHh587FHrEEuDRpZJQ1qFUypO91L_T52vMkLElu-ezxGxzhqfSp7uG56BjzpbBvNciD4qTuoU4BhStuKRfezPZKRm1qmF-Ptry5awgH3JrB6mlD5kJWHkVU0-r8O3LWKNpMlGBrZzg15O-i6nsEQVb5M8e-UFIX4wMm--NGHA5iHp3Oek6uoHmyR44RJlUnoUesSKVz-0lXrq08oT7BEJ6SR7YLOHXMFjmaewbzzcx15ian6-77LIYnt70uZENLtXtcqgKl4Mg46ilzpMfnaYFE3oUiR4TzR703OS10bL71ymM-ha8Efw2fLewTRLz3i4xYMaR-0FSRSwrwm17OSusmG7Jy38RSAT0Z62yZCLCvrEMJ0XZ7IlHT7E5BuQdpUH0", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe/spiffebundle_invalid_trustdomain.json b/testdata/spiffe/spiffebundle_invalid_trustdomain.json new file mode 100644 index 0000000..b16ace6 --- /dev/null +++ b/testdata/spiffe/spiffebundle_invalid_trustdomain.json @@ -0,0 +1,18 @@ +{ + "trust_domains": { + "invalid#character": { + "spiffe_sequence": 12035488, + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8g==" + ], + "n": "yeAKqRnrclUqhlLgSS1RZWvu2xH3Gu1GM31UkmS2f1_iQ5liFTfFCD7hYm-Y4TBHN12lJxV1JjO8xLc_2RuYWqcC8Dm9gt2cTj-j0I2AicW5C4nvm-N1EKxbtccXd7zIVSYHI_wCMRKZrM7qjmupFK5GIn4yQA0PuuJGYJeum7E3VzAs6cchZfcVH33-fS_Dl_h1HwHKGc93po4N2ST2FEUBJo3WUzmrtRIxYpcB4HffPSbezSoMd9DV6B5sO57rFQSGPeHy9lSjd39V1ZtHR7uTSNu_UOs69GQVO5OU9yHh587FHrEEuDRpZJQ1qFUypO91L_T52vMkLElu-ezxGxzhqfSp7uG56BjzpbBvNciD4qTuoU4BhStuKRfezPZKRm1qmF-Ptry5awgH3JrB6mlD5kJWHkVU0-r8O3LWKNpMlGBrZzg15O-i6nsEQVb5M8e-UFIX4wMm--NGHA5iHp3Oek6uoHmyR44RJlUnoUesSKVz-0lXrq08oT7BEJ6SR7YLOHXMFjmaewbzzcx15ian6-77LIYnt70uZENLtXtcqgKl4Mg46ilzpMfnaYFE3oUiR4TzR703OS10bL71ymM-ha8Efw2fLewTRLz3i4xYMaR-0FSRSwrwm17OSusmG7Jy38RSAT0Z62yZCLCvrEMJ0XZ7IlHT7E5BuQdpUH0", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe/spiffebundle_malformed.json b/testdata/spiffe/spiffebundle_malformed.json new file mode 100644 index 0000000..9284cfc --- /dev/null +++ b/testdata/spiffe/spiffebundle_malformed.json @@ -0,0 +1,4 @@ +[ + "test.google.com", + "test.google.com.au" +] diff --git a/testdata/spiffe/spiffebundle_match_client_spiffe.json b/testdata/spiffe/spiffebundle_match_client_spiffe.json new file mode 100644 index 0000000..1149436 --- /dev/null +++ b/testdata/spiffe/spiffebundle_match_client_spiffe.json @@ -0,0 +1,31 @@ +{ + "trust_domains": { + "foo.bar.com": { + "spiffe_sequence": 12035488, + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8g==" + ], + "n": "yeAKqRnrclUqhlLgSS1RZWvu2xH3Gu1GM31UkmS2f1_iQ5liFTfFCD7hYm-Y4TBHN12lJxV1JjO8xLc_2RuYWqcC8Dm9gt2cTj-j0I2AicW5C4nvm-N1EKxbtccXd7zIVSYHI_wCMRKZrM7qjmupFK5GIn4yQA0PuuJGYJeum7E3VzAs6cchZfcVH33-fS_Dl_h1HwHKGc93po4N2ST2FEUBJo3WUzmrtRIxYpcB4HffPSbezSoMd9DV6B5sO57rFQSGPeHy9lSjd39V1ZtHR7uTSNu_UOs69GQVO5OU9yHh587FHrEEuDRpZJQ1qFUypO91L_T52vMkLElu-ezxGxzhqfSp7uG56BjzpbBvNciD4qTuoU4BhStuKRfezPZKRm1qmF-Ptry5awgH3JrB6mlD5kJWHkVU0-r8O3LWKNpMlGBrZzg15O-i6nsEQVb5M8e-UFIX4wMm--NGHA5iHp3Oek6uoHmyR44RJlUnoUesSKVz-0lXrq08oT7BEJ6SR7YLOHXMFjmaewbzzcx15ian6-77LIYnt70uZENLtXtcqgKl4Mg46ilzpMfnaYFE3oUiR4TzR703OS10bL71ymM-ha8Efw2fLewTRLz3i4xYMaR-0FSRSwrwm17OSusmG7Jy38RSAT0Z62yZCLCvrEMJ0XZ7IlHT7E5BuQdpUH0", + "e": "AQAB" + } + ] + }, + "test.example.com": { + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIEZDCCA0ygAwIBAgIUVXGlXjNENtOZbI12epjgIhMaShMwDQYJKoZIhvcNAQELBQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTI0MTAyMTAyMTQxNVoXDTM0MTAxOTAyMTQxNVowZTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRUwEwYDVQQKDAxFeGFtcGxlLCBDby4xGjAYBgNVBAMMESoudGVzdC5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5xOONxJJ8b8Qauvob5/7dPYZfIcd+uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM+YQS8UhiAlPlqa6ANxgiBSEyMHH/xE8lo/+caYGeACqy640Jpl/JocFGo3xd1L8DCawjlaj6eu7T7T/tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q/yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv/fcPd31GSbXjF8M+KGGQIDAQABo4IBGTCCARUwdwYDVR0RBHAwboIQKi50ZXN0Lmdvb2dsZS5mcoIYd2F0ZXJ6b29pLnRlc3QuZ29vZ2xlLmJlghIqLnRlc3QueW91dHViZS5jb22HBMCoAQOGJnNwaWZmZTovL2V4YW1wbGUuY29tL3dvcmtsb2FkLzllZWJjY2QyMB0GA1UdDgQWBBRvRpAYHQYP6dFPf5V7/MyCftnNjTB7BgNVHSMEdDByoVqkWDBWMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8wDQYDVQQDDAZ0ZXN0Y2GCFFqz9Fbx3Mviz+lLmDbYi/YAYQ+aMA0GCSqGSIb3DQEBCwUAA4IBAQBJ1bnbBHa1n15vvhpGIzokuiJ+9q/zim63UuVDnkhrQM2N+RQbStGTTis2tNse1bh460dJFm6ArgHWogzx6fQZzgaDeCOAXvrAe4jM9IHr9K7lkq/33CZSBDV+jCmm2sRsqSMkKUcX6JhyqWGFHuTDAKJzsEV2MlcswleKlGHDkeelAaxlLzpzRHOSQd0N9xAs18lzx95SQEx90PtrBOmvIDDiI5o5z9Oz12Iy1toiksFl4jmknkDD5VF3AyCRgN8NPW0uNC8D2vo4L+tgj9U6NPlmMOrjRsEH257LJ1wopAGr+yezkIIdQQodGSVm5cOuw/K7Ma4nBDjVJkjcdY3t" + ], + "n": "5xOONxJJ8b8Qauvob5_7dPYZfIcd-uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM-YQS8UhiAlPlqa6ANxgiBSEyMHH_xE8lo_-caYGeACqy640Jpl_JocFGo3xd1L8DCawjlaj6eu7T7T_tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q_yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv_fcPd31GSbXjF8M-KGGQ", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe/spiffebundle_wrong_kid.json b/testdata/spiffe/spiffebundle_wrong_kid.json new file mode 100644 index 0000000..7f1c96b --- /dev/null +++ b/testdata/spiffe/spiffebundle_wrong_kid.json @@ -0,0 +1,31 @@ +{ + "trust_domains": { + "example.com": { + "spiffe_sequence": 12035488, + "keys": [ + { + "kid": "Some value", + "use": "x509-svid", + "x5c": [ + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8g==" + ], + "n": "yeAKqRnrclUqhlLgSS1RZWvu2xH3Gu1GM31UkmS2f1_iQ5liFTfFCD7hYm-Y4TBHN12lJxV1JjO8xLc_2RuYWqcC8Dm9gt2cTj-j0I2AicW5C4nvm-N1EKxbtccXd7zIVSYHI_wCMRKZrM7qjmupFK5GIn4yQA0PuuJGYJeum7E3VzAs6cchZfcVH33-fS_Dl_h1HwHKGc93po4N2ST2FEUBJo3WUzmrtRIxYpcB4HffPSbezSoMd9DV6B5sO57rFQSGPeHy9lSjd39V1ZtHR7uTSNu_UOs69GQVO5OU9yHh587FHrEEuDRpZJQ1qFUypO91L_T52vMkLElu-ezxGxzhqfSp7uG56BjzpbBvNciD4qTuoU4BhStuKRfezPZKRm1qmF-Ptry5awgH3JrB6mlD5kJWHkVU0-r8O3LWKNpMlGBrZzg15O-i6nsEQVb5M8e-UFIX4wMm--NGHA5iHp3Oek6uoHmyR44RJlUnoUesSKVz-0lXrq08oT7BEJ6SR7YLOHXMFjmaewbzzcx15ian6-77LIYnt70uZENLtXtcqgKl4Mg46ilzpMfnaYFE3oUiR4TzR703OS10bL71ymM-ha8Efw2fLewTRLz3i4xYMaR-0FSRSwrwm17OSusmG7Jy38RSAT0Z62yZCLCvrEMJ0XZ7IlHT7E5BuQdpUH0", + "e": "AQAB" + } + ] + }, + "test.example.com": { + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIEZDCCA0ygAwIBAgIUVXGlXjNENtOZbI12epjgIhMaShMwDQYJKoZIhvcNAQELBQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTI0MTAyMTAyMTQxNVoXDTM0MTAxOTAyMTQxNVowZTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRUwEwYDVQQKDAxFeGFtcGxlLCBDby4xGjAYBgNVBAMMESoudGVzdC5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5xOONxJJ8b8Qauvob5/7dPYZfIcd+uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM+YQS8UhiAlPlqa6ANxgiBSEyMHH/xE8lo/+caYGeACqy640Jpl/JocFGo3xd1L8DCawjlaj6eu7T7T/tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q/yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv/fcPd31GSbXjF8M+KGGQIDAQABo4IBGTCCARUwdwYDVR0RBHAwboIQKi50ZXN0Lmdvb2dsZS5mcoIYd2F0ZXJ6b29pLnRlc3QuZ29vZ2xlLmJlghIqLnRlc3QueW91dHViZS5jb22HBMCoAQOGJnNwaWZmZTovL2V4YW1wbGUuY29tL3dvcmtsb2FkLzllZWJjY2QyMB0GA1UdDgQWBBRvRpAYHQYP6dFPf5V7/MyCftnNjTB7BgNVHSMEdDByoVqkWDBWMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8wDQYDVQQDDAZ0ZXN0Y2GCFFqz9Fbx3Mviz+lLmDbYi/YAYQ+aMA0GCSqGSIb3DQEBCwUAA4IBAQBJ1bnbBHa1n15vvhpGIzokuiJ+9q/zim63UuVDnkhrQM2N+RQbStGTTis2tNse1bh460dJFm6ArgHWogzx6fQZzgaDeCOAXvrAe4jM9IHr9K7lkq/33CZSBDV+jCmm2sRsqSMkKUcX6JhyqWGFHuTDAKJzsEV2MlcswleKlGHDkeelAaxlLzpzRHOSQd0N9xAs18lzx95SQEx90PtrBOmvIDDiI5o5z9Oz12Iy1toiksFl4jmknkDD5VF3AyCRgN8NPW0uNC8D2vo4L+tgj9U6NPlmMOrjRsEH257LJ1wopAGr+yezkIIdQQodGSVm5cOuw/K7Ma4nBDjVJkjcdY3t" + ], + "n": "5xOONxJJ8b8Qauvob5_7dPYZfIcd-uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM-YQS8UhiAlPlqa6ANxgiBSEyMHH_xE8lo_-caYGeACqy640Jpl_JocFGo3xd1L8DCawjlaj6eu7T7T_tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q_yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv_fcPd31GSbXjF8M-KGGQ", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe/spiffebundle_wrong_kty.json b/testdata/spiffe/spiffebundle_wrong_kty.json new file mode 100644 index 0000000..a786355 --- /dev/null +++ b/testdata/spiffe/spiffebundle_wrong_kty.json @@ -0,0 +1,31 @@ +{ + "trust_domains": { + "example.com": { + "spiffe_sequence": 12035488, + "keys": [ + { + "kty": "DSA", + "use": "x509-svid", + "x5c": [ + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8g==" + ], + "n": "yeAKqRnrclUqhlLgSS1RZWvu2xH3Gu1GM31UkmS2f1_iQ5liFTfFCD7hYm-Y4TBHN12lJxV1JjO8xLc_2RuYWqcC8Dm9gt2cTj-j0I2AicW5C4nvm-N1EKxbtccXd7zIVSYHI_wCMRKZrM7qjmupFK5GIn4yQA0PuuJGYJeum7E3VzAs6cchZfcVH33-fS_Dl_h1HwHKGc93po4N2ST2FEUBJo3WUzmrtRIxYpcB4HffPSbezSoMd9DV6B5sO57rFQSGPeHy9lSjd39V1ZtHR7uTSNu_UOs69GQVO5OU9yHh587FHrEEuDRpZJQ1qFUypO91L_T52vMkLElu-ezxGxzhqfSp7uG56BjzpbBvNciD4qTuoU4BhStuKRfezPZKRm1qmF-Ptry5awgH3JrB6mlD5kJWHkVU0-r8O3LWKNpMlGBrZzg15O-i6nsEQVb5M8e-UFIX4wMm--NGHA5iHp3Oek6uoHmyR44RJlUnoUesSKVz-0lXrq08oT7BEJ6SR7YLOHXMFjmaewbzzcx15ian6-77LIYnt70uZENLtXtcqgKl4Mg46ilzpMfnaYFE3oUiR4TzR703OS10bL71ymM-ha8Efw2fLewTRLz3i4xYMaR-0FSRSwrwm17OSusmG7Jy38RSAT0Z62yZCLCvrEMJ0XZ7IlHT7E5BuQdpUH0", + "e": "AQAB" + } + ] + }, + "test.example.com": { + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIEZDCCA0ygAwIBAgIUVXGlXjNENtOZbI12epjgIhMaShMwDQYJKoZIhvcNAQELBQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTI0MTAyMTAyMTQxNVoXDTM0MTAxOTAyMTQxNVowZTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRUwEwYDVQQKDAxFeGFtcGxlLCBDby4xGjAYBgNVBAMMESoudGVzdC5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5xOONxJJ8b8Qauvob5/7dPYZfIcd+uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM+YQS8UhiAlPlqa6ANxgiBSEyMHH/xE8lo/+caYGeACqy640Jpl/JocFGo3xd1L8DCawjlaj6eu7T7T/tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q/yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv/fcPd31GSbXjF8M+KGGQIDAQABo4IBGTCCARUwdwYDVR0RBHAwboIQKi50ZXN0Lmdvb2dsZS5mcoIYd2F0ZXJ6b29pLnRlc3QuZ29vZ2xlLmJlghIqLnRlc3QueW91dHViZS5jb22HBMCoAQOGJnNwaWZmZTovL2V4YW1wbGUuY29tL3dvcmtsb2FkLzllZWJjY2QyMB0GA1UdDgQWBBRvRpAYHQYP6dFPf5V7/MyCftnNjTB7BgNVHSMEdDByoVqkWDBWMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8wDQYDVQQDDAZ0ZXN0Y2GCFFqz9Fbx3Mviz+lLmDbYi/YAYQ+aMA0GCSqGSIb3DQEBCwUAA4IBAQBJ1bnbBHa1n15vvhpGIzokuiJ+9q/zim63UuVDnkhrQM2N+RQbStGTTis2tNse1bh460dJFm6ArgHWogzx6fQZzgaDeCOAXvrAe4jM9IHr9K7lkq/33CZSBDV+jCmm2sRsqSMkKUcX6JhyqWGFHuTDAKJzsEV2MlcswleKlGHDkeelAaxlLzpzRHOSQd0N9xAs18lzx95SQEx90PtrBOmvIDDiI5o5z9Oz12Iy1toiksFl4jmknkDD5VF3AyCRgN8NPW0uNC8D2vo4L+tgj9U6NPlmMOrjRsEH257LJ1wopAGr+yezkIIdQQodGSVm5cOuw/K7Ma4nBDjVJkjcdY3t" + ], + "n": "5xOONxJJ8b8Qauvob5_7dPYZfIcd-uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM-YQS8UhiAlPlqa6ANxgiBSEyMHH_xE8lo_-caYGeACqy640Jpl_JocFGo3xd1L8DCawjlaj6eu7T7T_tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q_yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv_fcPd31GSbXjF8M-KGGQ", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe/spiffebundle_wrong_multi_certs.json b/testdata/spiffe/spiffebundle_wrong_multi_certs.json new file mode 100644 index 0000000..5efbb99 --- /dev/null +++ b/testdata/spiffe/spiffebundle_wrong_multi_certs.json @@ -0,0 +1,19 @@ +{ + "trust_domains": { + "google.com": { + "spiffe_sequence": 123, + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8g==", + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8g==" + ], + "n": "yeAKqRnrclUqhlLgSS1RZWvu2xH3Gu1GM31UkmS2f1_iQ5liFTfFCD7hYm-Y4TBHN12lJxV1JjO8xLc_2RuYWqcC8Dm9gt2cTj-j0I2AicW5C4nvm-N1EKxbtccXd7zIVSYHI_wCMRKZrM7qjmupFK5GIn4yQA0PuuJGYJeum7E3VzAs6cchZfcVH33-fS_Dl_h1HwHKGc93po4N2ST2FEUBJo3WUzmrtRIxYpcB4HffPSbezSoMd9DV6B5sO57rFQSGPeHy9lSjd39V1ZtHR7uTSNu_UOs69GQVO5OU9yHh587FHrEEuDRpZJQ1qFUypO91L_T52vMkLElu-ezxGxzhqfSp7uG56BjzpbBvNciD4qTuoU4BhStuKRfezPZKRm1qmF-Ptry5awgH3JrB6mlD5kJWHkVU0-r8O3LWKNpMlGBrZzg15O-i6nsEQVb5M8e-UFIX4wMm--NGHA5iHp3Oek6uoHmyR44RJlUnoUesSKVz-0lXrq08oT7BEJ6SR7YLOHXMFjmaewbzzcx15ian6-77LIYnt70uZENLtXtcqgKl4Mg46ilzpMfnaYFE3oUiR4TzR703OS10bL71ymM-ha8Efw2fLewTRLz3i4xYMaR-0FSRSwrwm17OSusmG7Jy38RSAT0Z62yZCLCvrEMJ0XZ7IlHT7E5BuQdpUH0", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe/spiffebundle_wrong_root.json b/testdata/spiffe/spiffebundle_wrong_root.json new file mode 100644 index 0000000..d92465e --- /dev/null +++ b/testdata/spiffe/spiffebundle_wrong_root.json @@ -0,0 +1,31 @@ +{ + "trustDomains": { + "example.com": { + "spiffe_sequence": 12035488, + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8g==" + ], + "n": "yeAKqRnrclUqhlLgSS1RZWvu2xH3Gu1GM31UkmS2f1_iQ5liFTfFCD7hYm-Y4TBHN12lJxV1JjO8xLc_2RuYWqcC8Dm9gt2cTj-j0I2AicW5C4nvm-N1EKxbtccXd7zIVSYHI_wCMRKZrM7qjmupFK5GIn4yQA0PuuJGYJeum7E3VzAs6cchZfcVH33-fS_Dl_h1HwHKGc93po4N2ST2FEUBJo3WUzmrtRIxYpcB4HffPSbezSoMd9DV6B5sO57rFQSGPeHy9lSjd39V1ZtHR7uTSNu_UOs69GQVO5OU9yHh587FHrEEuDRpZJQ1qFUypO91L_T52vMkLElu-ezxGxzhqfSp7uG56BjzpbBvNciD4qTuoU4BhStuKRfezPZKRm1qmF-Ptry5awgH3JrB6mlD5kJWHkVU0-r8O3LWKNpMlGBrZzg15O-i6nsEQVb5M8e-UFIX4wMm--NGHA5iHp3Oek6uoHmyR44RJlUnoUesSKVz-0lXrq08oT7BEJ6SR7YLOHXMFjmaewbzzcx15ian6-77LIYnt70uZENLtXtcqgKl4Mg46ilzpMfnaYFE3oUiR4TzR703OS10bL71ymM-ha8Efw2fLewTRLz3i4xYMaR-0FSRSwrwm17OSusmG7Jy38RSAT0Z62yZCLCvrEMJ0XZ7IlHT7E5BuQdpUH0", + "e": "AQAB" + } + ] + }, + "test.example.com": { + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIEZDCCA0ygAwIBAgIUVXGlXjNENtOZbI12epjgIhMaShMwDQYJKoZIhvcNAQELBQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTI0MTAyMTAyMTQxNVoXDTM0MTAxOTAyMTQxNVowZTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRUwEwYDVQQKDAxFeGFtcGxlLCBDby4xGjAYBgNVBAMMESoudGVzdC5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5xOONxJJ8b8Qauvob5/7dPYZfIcd+uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM+YQS8UhiAlPlqa6ANxgiBSEyMHH/xE8lo/+caYGeACqy640Jpl/JocFGo3xd1L8DCawjlaj6eu7T7T/tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q/yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv/fcPd31GSbXjF8M+KGGQIDAQABo4IBGTCCARUwdwYDVR0RBHAwboIQKi50ZXN0Lmdvb2dsZS5mcoIYd2F0ZXJ6b29pLnRlc3QuZ29vZ2xlLmJlghIqLnRlc3QueW91dHViZS5jb22HBMCoAQOGJnNwaWZmZTovL2V4YW1wbGUuY29tL3dvcmtsb2FkLzllZWJjY2QyMB0GA1UdDgQWBBRvRpAYHQYP6dFPf5V7/MyCftnNjTB7BgNVHSMEdDByoVqkWDBWMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8wDQYDVQQDDAZ0ZXN0Y2GCFFqz9Fbx3Mviz+lLmDbYi/YAYQ+aMA0GCSqGSIb3DQEBCwUAA4IBAQBJ1bnbBHa1n15vvhpGIzokuiJ+9q/zim63UuVDnkhrQM2N+RQbStGTTis2tNse1bh460dJFm6ArgHWogzx6fQZzgaDeCOAXvrAe4jM9IHr9K7lkq/33CZSBDV+jCmm2sRsqSMkKUcX6JhyqWGFHuTDAKJzsEV2MlcswleKlGHDkeelAaxlLzpzRHOSQd0N9xAs18lzx95SQEx90PtrBOmvIDDiI5o5z9Oz12Iy1toiksFl4jmknkDD5VF3AyCRgN8NPW0uNC8D2vo4L+tgj9U6NPlmMOrjRsEH257LJ1wopAGr+yezkIIdQQodGSVm5cOuw/K7Ma4nBDjVJkjcdY3t" + ], + "n": "5xOONxJJ8b8Qauvob5_7dPYZfIcd-uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM-YQS8UhiAlPlqa6ANxgiBSEyMHH_xE8lo_-caYGeACqy640Jpl_JocFGo3xd1L8DCawjlaj6eu7T7T_tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q_yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv_fcPd31GSbXjF8M-KGGQ", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe/spiffebundle_wrong_seq_type.json b/testdata/spiffe/spiffebundle_wrong_seq_type.json new file mode 100644 index 0000000..3bbc073 --- /dev/null +++ b/testdata/spiffe/spiffebundle_wrong_seq_type.json @@ -0,0 +1,31 @@ +{ + "trust_domains": { + "example.com": { + "spiffe_sequence": 12035488.5, + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8g==" + ], + "n": "yeAKqRnrclUqhlLgSS1RZWvu2xH3Gu1GM31UkmS2f1_iQ5liFTfFCD7hYm-Y4TBHN12lJxV1JjO8xLc_2RuYWqcC8Dm9gt2cTj-j0I2AicW5C4nvm-N1EKxbtccXd7zIVSYHI_wCMRKZrM7qjmupFK5GIn4yQA0PuuJGYJeum7E3VzAs6cchZfcVH33-fS_Dl_h1HwHKGc93po4N2ST2FEUBJo3WUzmrtRIxYpcB4HffPSbezSoMd9DV6B5sO57rFQSGPeHy9lSjd39V1ZtHR7uTSNu_UOs69GQVO5OU9yHh587FHrEEuDRpZJQ1qFUypO91L_T52vMkLElu-ezxGxzhqfSp7uG56BjzpbBvNciD4qTuoU4BhStuKRfezPZKRm1qmF-Ptry5awgH3JrB6mlD5kJWHkVU0-r8O3LWKNpMlGBrZzg15O-i6nsEQVb5M8e-UFIX4wMm--NGHA5iHp3Oek6uoHmyR44RJlUnoUesSKVz-0lXrq08oT7BEJ6SR7YLOHXMFjmaewbzzcx15ian6-77LIYnt70uZENLtXtcqgKl4Mg46ilzpMfnaYFE3oUiR4TzR703OS10bL71ymM-ha8Efw2fLewTRLz3i4xYMaR-0FSRSwrwm17OSusmG7Jy38RSAT0Z62yZCLCvrEMJ0XZ7IlHT7E5BuQdpUH0", + "e": "AQAB" + } + ] + }, + "test.example.com": { + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": [ + "MIIEZDCCA0ygAwIBAgIUVXGlXjNENtOZbI12epjgIhMaShMwDQYJKoZIhvcNAQELBQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTI0MTAyMTAyMTQxNVoXDTM0MTAxOTAyMTQxNVowZTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRUwEwYDVQQKDAxFeGFtcGxlLCBDby4xGjAYBgNVBAMMESoudGVzdC5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5xOONxJJ8b8Qauvob5/7dPYZfIcd+uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM+YQS8UhiAlPlqa6ANxgiBSEyMHH/xE8lo/+caYGeACqy640Jpl/JocFGo3xd1L8DCawjlaj6eu7T7T/tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q/yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv/fcPd31GSbXjF8M+KGGQIDAQABo4IBGTCCARUwdwYDVR0RBHAwboIQKi50ZXN0Lmdvb2dsZS5mcoIYd2F0ZXJ6b29pLnRlc3QuZ29vZ2xlLmJlghIqLnRlc3QueW91dHViZS5jb22HBMCoAQOGJnNwaWZmZTovL2V4YW1wbGUuY29tL3dvcmtsb2FkLzllZWJjY2QyMB0GA1UdDgQWBBRvRpAYHQYP6dFPf5V7/MyCftnNjTB7BgNVHSMEdDByoVqkWDBWMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8wDQYDVQQDDAZ0ZXN0Y2GCFFqz9Fbx3Mviz+lLmDbYi/YAYQ+aMA0GCSqGSIb3DQEBCwUAA4IBAQBJ1bnbBHa1n15vvhpGIzokuiJ+9q/zim63UuVDnkhrQM2N+RQbStGTTis2tNse1bh460dJFm6ArgHWogzx6fQZzgaDeCOAXvrAe4jM9IHr9K7lkq/33CZSBDV+jCmm2sRsqSMkKUcX6JhyqWGFHuTDAKJzsEV2MlcswleKlGHDkeelAaxlLzpzRHOSQd0N9xAs18lzx95SQEx90PtrBOmvIDDiI5o5z9Oz12Iy1toiksFl4jmknkDD5VF3AyCRgN8NPW0uNC8D2vo4L+tgj9U6NPlmMOrjRsEH257LJ1wopAGr+yezkIIdQQodGSVm5cOuw/K7Ma4nBDjVJkjcdY3t" + ], + "n": "5xOONxJJ8b8Qauvob5_7dPYZfIcd-uhAWL2ZlTPzQvu4oF0QI4iYgP5iGgry9zEtCM-YQS8UhiAlPlqa6ANxgiBSEyMHH_xE8lo_-caYGeACqy640Jpl_JocFGo3xd1L8DCawjlaj6eu7T7T_tpAV2qq13b5710eNRbCAfFe8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c6ffSIg4eP1XmovNYZ9LLEJG68tF0Q_yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPVYDi8tml6DfOCDGnit8svnMEmBv_fcPd31GSbXjF8M-KGGQ", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe/spiffebundle_wrong_use.json b/testdata/spiffe/spiffebundle_wrong_use.json new file mode 100644 index 0000000..8aaf913 --- /dev/null +++ b/testdata/spiffe/spiffebundle_wrong_use.json @@ -0,0 +1,18 @@ +{ + "trust_domains": { + "example.com": { + "spiffe_sequence": 12035488, + "keys": [ + { + "kty": "RSA", + "use": "NOT-x509-svid", + "x5c": [ + "MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQELBQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQyNTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVudDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAPBgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK34Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV+j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2DvUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gqyjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvVz6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hxx0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EXGA91fn0891b5eEW8BJHXX0jri0aN8g==" + ], + "n": "yeAKqRnrclUqhlLgSS1RZWvu2xH3Gu1GM31UkmS2f1_iQ5liFTfFCD7hYm-Y4TBHN12lJxV1JjO8xLc_2RuYWqcC8Dm9gt2cTj-j0I2AicW5C4nvm-N1EKxbtccXd7zIVSYHI_wCMRKZrM7qjmupFK5GIn4yQA0PuuJGYJeum7E3VzAs6cchZfcVH33-fS_Dl_h1HwHKGc93po4N2ST2FEUBJo3WUzmrtRIxYpcB4HffPSbezSoMd9DV6B5sO57rFQSGPeHy9lSjd39V1ZtHR7uTSNu_UOs69GQVO5OU9yHh587FHrEEuDRpZJQ1qFUypO91L_T52vMkLElu-ezxGxzhqfSp7uG56BjzpbBvNciD4qTuoU4BhStuKRfezPZKRm1qmF-Ptry5awgH3JrB6mlD5kJWHkVU0-r8O3LWKNpMlGBrZzg15O-i6nsEQVb5M8e-UFIX4wMm--NGHA5iHp3Oek6uoHmyR44RJlUnoUesSKVz-0lXrq08oT7BEJ6SR7YLOHXMFjmaewbzzcx15ian6-77LIYnt70uZENLtXtcqgKl4Mg46ilzpMfnaYFE3oUiR4TzR703OS10bL71ymM-ha8Efw2fLewTRLz3i4xYMaR-0FSRSwrwm17OSusmG7Jy38RSAT0Z62yZCLCvrEMJ0XZ7IlHT7E5BuQdpUH0", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe_end2end/README.md b/testdata/spiffe_end2end/README.md new file mode 100644 index 0000000..171cc8b --- /dev/null +++ b/testdata/spiffe_end2end/README.md @@ -0,0 +1,38 @@ +All of the following files in this directory except `server_spiffebundle.json` +and `client_spiffebundle.json` are generated with the `generate.sh` and +`intermediate_gen.sh` script in this directory. + +These comprise a root trust certificate authority (CA) that signs two +certificates - `client_spiffe.pem` and `server_spiffe.pem`. These are valid +SPIFFE certificates (via the configuration in `spiffe-openssl.cnf`), and the +`*_spiffebundle.json` files are SPIFFE Bundle Maps for the client and server +respectively. + +The SPIFFE trust bundle map files (`*_spiffebundle.json`) are manually created +for end to end testing. The `server_spiffebundle.json` contains the +`foo.bar.com` trust domain (only this entry is used in e2e tests) matching URI +SAN of `client_spiffe.pem`, and the CA certificate is `ca.pem`. The client +`spiffebundle.json` file contains `example.com` trust domain matching the URI +SAN of `server_spiffe.pem`, and the CA certificate there is also `ca.pem`. + +`leaf_and_intermediate_chain.pem` is a certificate chain whose leaf is a valid +SPIFFE cert that is signed by an intermediate CA (`intermediate_ca.pem`). The +intermediate CA is signed by the root CA (`ca.pem`). Thus, this setup yields a +valid chain to the root of trust `ca.pem`. + +If updating these files, the `x5c` field in the json is the raw PEM CA +certificate and can be copy pasted from the certificate file `ca.pem`. `n` and +`e` are values from the public key attached to this certificate. `e` should +*probably* be `AQAB` as it is the exponent. `n` can be fetched from the +certificate by getting the RSA key from the cert and extracting the value. This +can be done in golang with the following codeblock: + +``` +func GetBase64ModulusFromPublicKey(key *rsa.PublicKey) string { + return base64.RawURLEncoding.EncodeToString(key.N.Bytes()) +} + +block, _ := pem.Decode(rawPemCert) cert, _ := x509.ParseCertificate(block.Bytes) +publicKey := cert.PublicKey.(*rsa.PublicKey) +fmt.Println(GetBase64ModulusFromPublicKey(publicKey)) +``` diff --git a/testdata/spiffe_end2end/ca.key b/testdata/spiffe_end2end/ca.key new file mode 100644 index 0000000..282b76b --- /dev/null +++ b/testdata/spiffe_end2end/ca.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDZRezL1GWM/0um +gcLNGI9fsp0sabLUN1rUTwXjCGHvGAt9cPY1xK6P6+/QXeBkYogmwC3Udhwl+8x3 +AOu2FKP9QeveW+hB4N3SCtY4D+nGxPWnL1Yjh0170tkVRPkh89wPX4XdtAcIgZjj +NgPH9wR3eWH8E3cDRvxjosy4e9aw0/VTHb0lZ2Ko6mY8x2bJNFUExmuh157b//yH +dcL8O1K1h13o6YSqSlnLmnaSHpHA5/21erSjszE3ZjiPZFR6cl/RDjpJRKrh/eQA +Aeg7XO00VokrvZnK07mZ2nNyp4z1yCV4fwWCdBMfFfUSm0EgN8D65JxGj1He5bm4 +LVv6WuWj+cqgAzRhHjp8iZsmLFKvAoXSg1eJ+mG8vw5P9ZVd38M8N2DtifvmCkE1 +fln84oQj42wDgNQuwSafbGQIhNYrnsgbf+2lNj5IIYm/ScUcexbmCmKyOZfNcVQN +AF26vQOvibBDOFyDW2qdi2tIGJgTPhkJPbDqKJx4UqNxIuoam28PifmK51Cn4i9r +WOpQ2nM7JGX1Q3IE6S5iaVnogp2+SSh3PhktxQOK9IUX7ysX9eVQAdN8lnKgaIZu +yYOx9H9jWValZ/dCLa2Y/BuFslx4NRVLtoJkUIakdTepo83ZrrSs3c4DSEIJJkgV +PBmBkua3RdYB1Cw+A8T0C8nvExyJbwIDAQABAoICACiqtFwgWJOP8y5qqjXuK2O7 +kDyQWhbrd3Y2hkEMwBva0ce8XXEJ66xnej7zWz0eziIGpW1XYARiAU6i3vt5GIqe +DJJVfH7cIliei6L5o4V/NwWAUCydNgz/lX8sJj9LH/zGoc0x1YErlJyubv2+B9l+ +4C6ozAVSg4n4nj3HfBMj0Q6qCc2W8LnyCYnFvsDZKTKYGuwzafn+i1slWQYR6nSs +rhCaqKig+vanZhsKWriTE4MkwOvE7ejTGrIIEuHKR5HfClzNW/ipp0OzF68IwSMj +MVB9D+yNPXeFEke7AaC9QfowOC1HIY8XcoP1iDaKCPZM1M1GJQpF0EGfR9Bvs/xp +Jp1Yp8idegZKz75S1fKq5EvOp0sIfyFST6aeSlYHz1/MRKf4H+8klRQeOpAnVqhO +8wT/NT0NHYU6d/fa74H7BkE146P8I3N2fsmuUkmwpn9h3nruTIh92PnDs+krI+TF +/TZzfQxS4BZ1ky/IgqmmhWK3RwFVr65v7Reyo4F56vHkxukcJh2L1WM0RyFeZM17 +9aSbdQpLj7x6jOLM4JlYa8+KUarmzfHs8A4ulApY5RnRs898mCE6XeuI3Rl9EsY0 +JcvS0vQcunGY89dk8tw2Lay9BIFgGcPKI1zKJlbbf6nmcZEruQdRl5+Z2q8ZaTtb +DmOGV6aTa9h50t20pxqNAoIBAQD5kblDIzGlSh5sn8WMOMgsoiH9L0vw8amw6PFy +JohYiMtgrUovuV84a6G3xe2dGya/9t38Z7ywTocKkxaYM6dqQ41COisJdUwD4tvR +RrTbZqnyiFjC3iLYqpXgxUy0IVZd1uzkoiUB95fgMr3TsZGHipvo2HTj/G2nIgSO +4mNN39L4tQNSqig9FOUvYj4T9qac1D0kbucd3gZ7NhifuuGRVF3yH600+6niNyCr +7wPwEcsZCdz5N6O9upKmQ/zhMmwyTT5XskamJw7ltMOZ6S+ihSy/y0ZuIXYCDir8 +VrNFXutt1dntyK0IIVb2Zk4n7/ETbkoZfPqMaIU/xsNfKQmtAoIBAQDe3yk8aC+q +TbvVKwU+v4OqUwlWNOVgLg50ZfvU4OQ9CbfFtvDmKbTxEmNHN2ENO3/6092QUIQs +ZrkZ3QBDkjABTFvRA5a6wOqgxZ2goIwwZWJkYrJFA5d+ExB+AciT1ZaVUKbBWM5R +V90NfnJWdqlssnw3+lkvBkfOUNt/JUaY0V3cVIdZkCnu+/hGbeZFFwpdnkALXDGu +WIy3zDFv196bO8qdX4UU3t15UYrkpdvM/QEEF9RNVpSkOB/SuEExI5h6YWaiGiiw +HcFqZ2CF3ZoBf8ATL+c5WLbIvwSuGW8QEv69hb198tpQswa+R7DX+IA22xL3pvgv +7NqzcQIT33sLAoIBAHEK3aSYa2NYGEHReNST36+/3K9m3foMLHWyfbLb20Rm6eAn +fgPx7jyLHBw2rfNMmhe3hUNP5briRu62QzS7qOhMIs7NtDK7i6vy9OhtI3yBmxb9 +RV826QfE9NBz7dNlik5FDNZez259rLBjq3IY6zc+xHIKoZ+m9jAPC4uA5cQfTttS +emfWJRXNwiXdVQsL3NoKlItNJKh1qe/jR/IJ3yRJ16fVS0pFd+S8XbMjN2BlXt4/ +hnToC+XjfYuMHh4PDc0XCdcFLFUUOf44C6VKZ0YxFlzlgUhfJam2qyfTSa5xWShO +BkFbdWzKVS6UlnAmkcbgXPYAkyhIK5sAt/wBhZ0CggEAMPI/wyV6emNyAgHduAcl +am2sGkOpsHLM9+FB6mGtnn4Y3xIrW9EDDQKlzQkrhlVv6O1Itp4IORwiQLzZhv1/ +D3nunDu2ibM+lCpyUMmRoDtT3YoTbra4OZcEQzgvDdCVrps01DelsBWk1YbUo4qR +8O5N+5k+puYxNO1rF0RfecZZX78sro3Lt9GcmBMgxEGoJCFSHWyU+J434VG19cMp +/1ulRuSofInph/BRmZ+XYzCZXYXCOW3vXRV6X7PZlWok3ZOwj59BGlSemrizaRLe +9L9StqQJmv2Rvwq8g2PQkW4qhgLuN8/zBFAdBgMGopfPyLxaMQt5bEUPTuNdunGV +OwKCAQEA9GEIEXos7dTzp68cqytYN/SeiuRgHgtvi2zDlD1HUL/7V3c192NBgAR1 +tPfR/Gl9TrKZox45sHvUCyt395aqB7fpSkmJBBtPmqXkX//E7wtjWiry/TSAjYVP +6hqo70XlgBNPpqNV7/PoXagfGzFxp2l3LIB52xePuHbwOa3Z7UB/2IItsu4liNfG +9mYRloSNALglbWZaFyzG51OUywCM+s7DylAOv2dRJEZ/wPybDpQsTG/h9AJsb6Wa +ZrO+m0pyvFjAGnHuqVpwjkKT0Y4sTqZUOaQMQsG1fB6wfYIagypKrdxMtF0w7PvJ +vmWw2Uw4DH8zFu1I9deCqW6ilOly7w== +-----END PRIVATE KEY----- diff --git a/testdata/spiffe_end2end/ca.pem b/testdata/spiffe_end2end/ca.pem new file mode 100644 index 0000000..3d2f55b --- /dev/null +++ b/testdata/spiffe_end2end/ca.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFlTCCA32gAwIBAgIUZ4e1KRtWw0c8V5NnA9VEoUMvzq4wDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMSEwHwYDVQQKDBhJbnRlcm5l +dCBXaWRnaXRzIFB0eSBMdGQxGzAZBgNVBAMMEmZvby5iYXIuaG9vLmNhLmNvbTAe +Fw0yNjAzMDkwNTI4MjBaFw0zNjAzMDYwNTI4MjBaMFoxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJWQTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRsw +GQYDVQQDDBJmb28uYmFyLmhvby5jYS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4IC +DwAwggIKAoICAQDZRezL1GWM/0umgcLNGI9fsp0sabLUN1rUTwXjCGHvGAt9cPY1 +xK6P6+/QXeBkYogmwC3Udhwl+8x3AOu2FKP9QeveW+hB4N3SCtY4D+nGxPWnL1Yj +h0170tkVRPkh89wPX4XdtAcIgZjjNgPH9wR3eWH8E3cDRvxjosy4e9aw0/VTHb0l +Z2Ko6mY8x2bJNFUExmuh157b//yHdcL8O1K1h13o6YSqSlnLmnaSHpHA5/21erSj +szE3ZjiPZFR6cl/RDjpJRKrh/eQAAeg7XO00VokrvZnK07mZ2nNyp4z1yCV4fwWC +dBMfFfUSm0EgN8D65JxGj1He5bm4LVv6WuWj+cqgAzRhHjp8iZsmLFKvAoXSg1eJ ++mG8vw5P9ZVd38M8N2DtifvmCkE1fln84oQj42wDgNQuwSafbGQIhNYrnsgbf+2l +Nj5IIYm/ScUcexbmCmKyOZfNcVQNAF26vQOvibBDOFyDW2qdi2tIGJgTPhkJPbDq +KJx4UqNxIuoam28PifmK51Cn4i9rWOpQ2nM7JGX1Q3IE6S5iaVnogp2+SSh3Phkt +xQOK9IUX7ysX9eVQAdN8lnKgaIZuyYOx9H9jWValZ/dCLa2Y/BuFslx4NRVLtoJk +UIakdTepo83ZrrSs3c4DSEIJJkgVPBmBkua3RdYB1Cw+A8T0C8nvExyJbwIDAQAB +o1MwUTAdBgNVHQ4EFgQU/LjtA3oVWCTz3qfAQXuc/dC7b84wHwYDVR0jBBgwFoAU +/LjtA3oVWCTz3qfAQXuc/dC7b84wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B +AQsFAAOCAgEAZFJDbhKL1M16Ya7eAvrgUWI9vbC4xm/68HzIGuQLeP9yr7QMfk5w +zh9umYcGWs/hkz2jg/88PbPUEP35psTw+dZGtnQwi/0icM9w7m0HZrDx8vMYvX5A +oEaPb5PqbjwhBxzd2Hpv/Y+PLLRO9uh1Xeh95J4yp5t/xOPyimljKLTam+rpj7Be +vYftlSXuIWrR6sldIt9I1zT5CfcBSefmSVo2WtgLdqfOBIAD73OMbcxEPDiTF4/q +8S1qHeA8eNUXc5IrWEI2NirhpvR0ZXMyjY1gJoZ4XqPFSXg+1oi9qpbjMhKU5J+R +m8brv/HmACaP90hSPXjuRl2Tp/NJ6idV1heNd4MdTeAlH+deQ+UQ/rFo6HY95S3l +tdcaDX8V7WdpRUQddv2DS5VPPi60LJuOdxKB+XFtjdLUtxwOw3Hoj91YWZLuMu2x +zVeASeA6FsB0vZwQdqn8FX1BQNSIEHGs7tieV7N0AlLavQBpyRYfhgKOL9Utbl4C +p6C6k42uFo/tFREUdzgtt0JqT+/NMJRA6rOJYHVUWNeG2ggGpuCtHXkwktC4JBnU +wnvdVZX3sXPslmiFzMO6S/+PesHA0TclifTZNOETHphF8efGsRFL/lkhcBRkZKBV +OkWKdKMUnXPgDKLyu/e1WQXCXmyTjv8FEiG2Ni8U4P0c2PldooYVjw4= +-----END CERTIFICATE----- diff --git a/testdata/spiffe_end2end/client.key b/testdata/spiffe_end2end/client.key new file mode 100644 index 0000000..15c27d5 --- /dev/null +++ b/testdata/spiffe_end2end/client.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCeb12R71ES72b6 +ag0gDK+8Hf6DPc2szn1Iwa6L7qoqBfFXxZImPhqVIuBTISxIlweib+lVOKtJDFym +a/ILGiWwKy19H+9dwf/R5keufOjs+dkSt2kwNNAgednGiMiuBi6186QSCxhp1+lF +q4UnBTZDsVjm232NPF3w7xUMLzB7XesY7A7cSLFBOdB/ObDB+UYMR6BpyN7C8gH3 +Y2t9wYHpw8NpMNsGshGSG5Ti8R48j7ZS4lZslW6qPzquYMrdNQx8XhsHOr3q9LDw +SIfR7j4M0A5FRQQlWpslmAUlxDqmrXQ6FdAMSyU8Qa9VVHzO0PrrEXOFijQdguL0 +QT0WWd9JAgMBAAECggEANHlaGl2TEpxsFQBO/JB3G+0rQLiViGiToidT8lDH10dE +mfd74mKrVz3H4oCYNCqhgrFiLLGqOXV4sj4KWpb8aI7EcC7Fjt4UJqcIksgYNXTC +6qoIksjLLhZthI4FOW4exnC9pKQ8H8I9JrAaV8QoJt90PHHx8XdO/d+RrwlqG1GI +a1iIPLEEx54hMkpUDojmzTSaWihZTeBnl55fy0bXcCK1WA6lp9UODhP1HXgf5HyT +hopVGwIzP2xr4hTsKujwGp0rIF+42ty9llGKP8mZQ4kDaf2gHDflORjvXE0GS7Sd +6dLv/uOBvRVqiG/FnUKCFsPQab1I8iIJSCUw9wxuKQKBgQDRipuRFCkPa/pBWa9j +xLg1r7zbb2fLX4HMC5+DeQYnrURDDGUglAgUUcgLeALTKpDbZdpPoDm62nCWNPbM +CMreMuz6yxPpV6Gc+8aIVg25ExxyuBBc8x8raVVRUaQQJblur6LnPHTP6fme5D9Q +fCPQAmop/8RIozNuYNULvlt7wwKBgQDBkABkPsGR1TbGXG9b9RriAomYY0pFxQZs +yUey+SqPrNJ3Y9lFbCU2yuG8BUaXKGYHWji9Zn10Jv393d2Y1mIUA+O+KWOmMdce +NBBkSHsOJkeQSc16bZpcZ7gwSaLiJp/LJ8tFoerY72kpoqk14zLHBb0RjbnPpp7b ++7kTjKAkAwKBgQCAYmZye2G0+zl3tNWLUUp0SlpyME3uA3Rpam2vhgVJZhW+5udH +EKvqlzj0HfHNI/VhF4Ss6MS2QYRd49GarYBup9Ee0DJA89onbvPzMJZz4Mu7Vh9g +c+2TEZSeoGDfK93zfVVYTGhdw0OYIMzSKV1f4zrcxMKkpqmqZVXjPhybNQKBgEdo +ytww0tTsZiLUIzk3uc07xmtz5gjLYU1tDIiYp/0NczAcpCGafjgyrQjioWJOwyVf +QaOOViYt5HJuEby7Cr/7l1+mgV7k8EnyR0HYA536vVgcAjRyjwocMbWO1Qq92SHn +8nQkAjI6UP/NRMPep/MIyPHa6XwUKnNZ8LOno8TlAoGAYGlDV2mbgTLKbIyXfKhe +EX+NP1OvhKrk+GJhkeTXwnu0Vz11KwLdRSSt2olLiMgl7phKinWlspWiNU0RGf3i +BZkmGzRHiSn/ykKY792ujN7iodjjuR2644e1FdDGCOJVH9erh+H7lwI6v2om32Rf +W4GdaqQ/t7TaabJqPPtLBFs= +-----END PRIVATE KEY----- diff --git a/testdata/spiffe_end2end/client_spiffe.pem b/testdata/spiffe_end2end/client_spiffe.pem new file mode 100644 index 0000000..2458a44 --- /dev/null +++ b/testdata/spiffe_end2end/client_spiffe.pem @@ -0,0 +1,28 @@ +-----BEGIN CERTIFICATE----- +MIIEvjCCAqagAwIBAgIUCzY7jU+NrvxZf0WMdg/5AM5i8XUwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMSEwHwYDVQQKDBhJbnRlcm5l +dCBXaWRnaXRzIFB0eSBMdGQxGzAZBgNVBAMMEmZvby5iYXIuaG9vLmNhLmNvbTAe +Fw0yNjAzMDkwNTI4MjBaFw0zNjAzMDYwNTI4MjBaMEwxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRMwEQYDVQQD +DAp0ZXN0Y2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnm9d +ke9REu9m+moNIAyvvB3+gz3NrM59SMGui+6qKgXxV8WSJj4alSLgUyEsSJcHom/p +VTirSQxcpmvyCxolsCstfR/vXcH/0eZHrnzo7PnZErdpMDTQIHnZxojIrgYutfOk +EgsYadfpRauFJwU2Q7FY5tt9jTxd8O8VDC8we13rGOwO3EixQTnQfzmwwflGDEeg +acjewvIB92NrfcGB6cPDaTDbBrIRkhuU4vEePI+2UuJWbJVuqj86rmDK3TUMfF4b +Bzq96vSw8EiH0e4+DNAORUUEJVqbJZgFJcQ6pq10OhXQDEslPEGvVVR8ztD66xFz +hYo0HYLi9EE9FlnfSQIDAQABo4GJMIGGMEQGA1UdEQQ9MDuGOXNwaWZmZTovL2Zv +by5iYXIuY29tLzllZWJjY2QyLTEyYmYtNDBhNi1iMjYyLTY1ZmUwNDg3ZDQ1MzAd +BgNVHQ4EFgQUWhXxI5pA878mNWlbVFscM7rPD30wHwYDVR0jBBgwFoAU/LjtA3oV +WCTz3qfAQXuc/dC7b84wDQYJKoZIhvcNAQELBQADggIBAMqDRhSCS+cSozh7oeqQ +RJehfP/3pX8grzezk/PmohG8NiPx0Zwy9Huqf9BV5VG1iRwF1DJmbj2VV59jTuLa +B6LNWVDwmToI3nVCedM4pTLGEKIvtKEMZQserR/FrN9WEHG3nU2SnS1jqTRVu+lv +69sOtkenXs51X6YzyRvY/MD+b78JuyuFu9/33Tzn+mJ82CBWuGspKveWanwKsYZp +N5XXPNLF6YWO8MfcX9ephjQXguPxmavoWno/QDmPuZzAwX6T9MqvNKk6aK8hdQfU +Gw5E9nQ+oyZyQXZw7FwsEsj+MOhPp3wR4Ubwth2PXfXWs6icvDr0Cq2O/x5eCQ9w +F1fy8ORBFLSqwmqCuTjt0rF2dqMUDiV7EVRzg8ACWB4gCoSAZhT1/uIkO452Tasa +zF5c29bWo1TCe8lJ2QkFBxZyVTPwJU1uIMhWUHrykqaO6oueZZFp+qLK9HevTqjd +5Cp4OzdemWyhQhk8u30sNUM0u+Elk125BnGkuBr6DH3AbKP5JK1gtyWGt/6CfDR/ +iBG18B7aQEEb46vmLd4EsgLnwssOir1+t/IBFQ0cXsuqQekuez1GqA9+IQfcg028 +lrjLvMejJjfofvIUmQ5R5IgZX1xThb2kM05CifJANceu2sORxieefVC6izDc3gOB +2cRdNb9kp9E89P7FnBkku+9Y +-----END CERTIFICATE----- diff --git a/testdata/spiffe_end2end/client_spiffebundle.json b/testdata/spiffe_end2end/client_spiffebundle.json new file mode 100644 index 0000000..892ce0b --- /dev/null +++ b/testdata/spiffe_end2end/client_spiffebundle.json @@ -0,0 +1,16 @@ +{ + "trust_domains": { + "example.com": { + "spiffe_sequence": 12035488, + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": ["MIIFlTCCA32gAwIBAgIUZ4e1KRtWw0c8V5NnA9VEoUMvzq4wDQYJKoZIhvcNAQELBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxGzAZBgNVBAMMEmZvby5iYXIuaG9vLmNhLmNvbTAeFw0yNjAzMDkwNTI4MjBaFw0zNjAzMDYwNTI4MjBaMFoxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJWQTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRswGQYDVQQDDBJmb28uYmFyLmhvby5jYS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDZRezL1GWM/0umgcLNGI9fsp0sabLUN1rUTwXjCGHvGAt9cPY1xK6P6+/QXeBkYogmwC3Udhwl+8x3AOu2FKP9QeveW+hB4N3SCtY4D+nGxPWnL1Yjh0170tkVRPkh89wPX4XdtAcIgZjjNgPH9wR3eWH8E3cDRvxjosy4e9aw0/VTHb0lZ2Ko6mY8x2bJNFUExmuh157b//yHdcL8O1K1h13o6YSqSlnLmnaSHpHA5/21erSjszE3ZjiPZFR6cl/RDjpJRKrh/eQAAeg7XO00VokrvZnK07mZ2nNyp4z1yCV4fwWCdBMfFfUSm0EgN8D65JxGj1He5bm4LVv6WuWj+cqgAzRhHjp8iZsmLFKvAoXSg1eJ+mG8vw5P9ZVd38M8N2DtifvmCkE1fln84oQj42wDgNQuwSafbGQIhNYrnsgbf+2lNj5IIYm/ScUcexbmCmKyOZfNcVQNAF26vQOvibBDOFyDW2qdi2tIGJgTPhkJPbDqKJx4UqNxIuoam28PifmK51Cn4i9rWOpQ2nM7JGX1Q3IE6S5iaVnogp2+SSh3PhktxQOK9IUX7ysX9eVQAdN8lnKgaIZuyYOx9H9jWValZ/dCLa2Y/BuFslx4NRVLtoJkUIakdTepo83ZrrSs3c4DSEIJJkgVPBmBkua3RdYB1Cw+A8T0C8nvExyJbwIDAQABo1MwUTAdBgNVHQ4EFgQU/LjtA3oVWCTz3qfAQXuc/dC7b84wHwYDVR0jBBgwFoAU/LjtA3oVWCTz3qfAQXuc/dC7b84wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAZFJDbhKL1M16Ya7eAvrgUWI9vbC4xm/68HzIGuQLeP9yr7QMfk5wzh9umYcGWs/hkz2jg/88PbPUEP35psTw+dZGtnQwi/0icM9w7m0HZrDx8vMYvX5AoEaPb5PqbjwhBxzd2Hpv/Y+PLLRO9uh1Xeh95J4yp5t/xOPyimljKLTam+rpj7BevYftlSXuIWrR6sldIt9I1zT5CfcBSefmSVo2WtgLdqfOBIAD73OMbcxEPDiTF4/q8S1qHeA8eNUXc5IrWEI2NirhpvR0ZXMyjY1gJoZ4XqPFSXg+1oi9qpbjMhKU5J+Rm8brv/HmACaP90hSPXjuRl2Tp/NJ6idV1heNd4MdTeAlH+deQ+UQ/rFo6HY95S3ltdcaDX8V7WdpRUQddv2DS5VPPi60LJuOdxKB+XFtjdLUtxwOw3Hoj91YWZLuMu2xzVeASeA6FsB0vZwQdqn8FX1BQNSIEHGs7tieV7N0AlLavQBpyRYfhgKOL9Utbl4Cp6C6k42uFo/tFREUdzgtt0JqT+/NMJRA6rOJYHVUWNeG2ggGpuCtHXkwktC4JBnUwnvdVZX3sXPslmiFzMO6S/+PesHA0TclifTZNOETHphF8efGsRFL/lkhcBRkZKBVOkWKdKMUnXPgDKLyu/e1WQXCXmyTjv8FEiG2Ni8U4P0c2PldooYVjw4="], + "n": "2UXsy9RljP9LpoHCzRiPX7KdLGmy1Dda1E8F4whh7xgLfXD2NcSuj-vv0F3gZGKIJsAt1HYcJfvMdwDrthSj_UHr3lvoQeDd0grWOA_pxsT1py9WI4dNe9LZFUT5IfPcD1-F3bQHCIGY4zYDx_cEd3lh_BN3A0b8Y6LMuHvWsNP1Ux29JWdiqOpmPMdmyTRVBMZrodee2__8h3XC_DtStYdd6OmEqkpZy5p2kh6RwOf9tXq0o7MxN2Y4j2RUenJf0Q46SUSq4f3kAAHoO1ztNFaJK72ZytO5mdpzcqeM9cgleH8FgnQTHxX1EptBIDfA-uScRo9R3uW5uC1b-lrlo_nKoAM0YR46fImbJixSrwKF0oNXifphvL8OT_WVXd_DPDdg7Yn75gpBNX5Z_OKEI-NsA4DULsEmn2xkCITWK57IG3_tpTY-SCGJv0nFHHsW5gpisjmXzXFUDQBdur0Dr4mwQzhcg1tqnYtrSBiYEz4ZCT2w6iiceFKjcSLqGptvD4n5iudQp-Iva1jqUNpzOyRl9UNyBOkuYmlZ6IKdvkkodz4ZLcUDivSFF-8rF_XlUAHTfJZyoGiGbsmDsfR_Y1lWpWf3Qi2tmPwbhbJceDUVS7aCZFCGpHU3qaPN2a60rN3OA0hCCSZIFTwZgZLmt0XWAdQsPgPE9AvJ7xMciW8", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe_end2end/generate.sh b/testdata/spiffe_end2end/generate.sh new file mode 100755 index 0000000..d52c81f --- /dev/null +++ b/testdata/spiffe_end2end/generate.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Generate client/server self signed CAs and certs. +openssl req -x509 -newkey rsa:4096 -keyout ca.key -out ca.pem -days 3650 -nodes -subj "/C=US/ST=VA/O=Internet Widgits Pty Ltd/CN=foo.bar.hoo.ca.com" + +# The SPIFFE related extensions are listed in spiffe-openssl.cnf config. Both +# client_spiffe.pem and server_spiffe.pem are generated in the same way with +# original client.pem and server.pem but with using that config. Here are the +# exact commands (we pass "-subj" as argument in this case): +openssl genrsa -out client.key.rsa 2048 +openssl pkcs8 -topk8 -in client.key.rsa -out client.key -nocrypt +openssl req -new -key client.key -out spiffe-cert.csr \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=testclient/ \ + -config spiffe-openssl.cnf -reqexts spiffe_client_e2e +openssl x509 -req -CA ca.pem -CAkey ca.key -CAcreateserial \ + -in spiffe-cert.csr -out client_spiffe.pem -extensions spiffe_client_e2e \ + -extfile spiffe-openssl.cnf -days 3650 -sha256 + +openssl genrsa -out server.key.rsa 2048 +openssl pkcs8 -topk8 -in server.key.rsa -out server.key -nocrypt +openssl req -new -key server.key -out spiffe-cert.csr \ + -subj "/C=US/ST=CA/L=SVL/O=gRPC/CN=*.test.google.com/" \ + -config spiffe-openssl.cnf -reqexts spiffe_server_e2e +openssl x509 -req -CA ca.pem -CAkey ca.key -CAcreateserial \ + -in spiffe-cert.csr -out server_spiffe.pem -extensions spiffe_server_e2e \ + -extfile spiffe-openssl.cnf -days 3650 -sha256 + +rm *.rsa +rm *.csr +rm *.srl diff --git a/testdata/spiffe_end2end/intermediate.cnf b/testdata/spiffe_end2end/intermediate.cnf new file mode 100644 index 0000000..eeeb3af --- /dev/null +++ b/testdata/spiffe_end2end/intermediate.cnf @@ -0,0 +1,38 @@ +[ca] +default_ca = CA_intermediate + +[CA_intermediate] +dir = . +certs = $dir/certs +crl_dir = $dir/crl +new_certs_dir = $dir/newcerts +database = $dir/index.txt +serial = $dir/serial +RANDFILE = $dir/private/.rand +private_key = $dir/intermediate_ca.key +certificate = $dir/intermediate_ca.pem +crl = $dir/intermediate.crl + +# For certificate revocation lists. +crlnumber = $dir/crlnumber +crl = $dir/crl/intermediate.crl +crl_extensions = crl_ext +default_crl_days = 3650 + +default_md = sha256 + +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req +prompt = no + +[req_distinguished_name] +CN = intermediatecert.example.com + +[crl_ext] +authorityKeyIdentifier=keyid:always + +[v3_req] +keyUsage = critical, digitalSignature, keyEncipherment, keyCertSign, cRLSign +extendedKeyUsage = clientAuth, serverAuth +basicConstraints = critical, CA:true diff --git a/testdata/spiffe_end2end/intermediate_ca.key b/testdata/spiffe_end2end/intermediate_ca.key new file mode 100644 index 0000000..4400555 --- /dev/null +++ b/testdata/spiffe_end2end/intermediate_ca.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDnXje+M6pAPFnr +Ve2XjBMSslqLQc/PBLMGplOloIwgoqcw3UnC8LfZjoYXT7dfjA0IOxIg4mj+XGiQ +voNMaudWEjTwf0goRE6kBUCVy+mxVq1XHR9Oooc49fGQZgx+YkIMGAZPgg6T4THd +tQezPg4cEjx6b4C4x+pmWs0ZvoYPi0MvZCDSt4sN20sw9/mYH4wfZyEVBWpqJKUN +jjwc5lzcweA0QkuO8LAZVQy9pwXeVhLEyCCtaDrnT4wRsiL2hRZ0s+RFHKIG6I88 +uv8ZAWzthladh19a5/C74vACtpU/UyNipJrTpyOPePmhESAP3ut2arMryAvaDi4Q +Lb2oQtG7AgMBAAECggEAV5qKk7d/QrQ3Pc661/NfM2iZtPm1xAZc/OsUZ/WaqS69 +DFj1eVzo1/TJm+EApyphstie/BmONKVrqEaic1hVAHeDWP5wpWJ7vkoI0s8zTITr +vDzYIk+S0MP687hYCbCNnoOoPAmcGG0fkvldEYaxE/rpsFgwzoZPn/LIjvNfUlmp +yRIj5qTPl0XyDJE7cDgc7sjohAeKH3IQYZ9llgRlXIX8I8cPo6WOus7RmBx8xAyR +Z3bsdxeTV6jjJUiAQi8vPb5GM23ghKmZnoRmYgwzetTwktIlWiFRKPQ+EZ9EVFlV +SWitTH3af1q2xn736o/xTmCta5mnZw4+wgGfIQWXfQKBgQDz7KtGn+YOru2qcA89 +REIE/4VMzT32DobSCRLBBdnKnArYO0gVXsvdne7SPeIC64oQxUw/rpF4wwSpdfnU +FEgmKlTTQ40tHFstpjqNiPd1eHTu7aG0uBn7NY86hBKfF4R26aar7KxI+2cNv5kK +nBFpUMQnMZ0Yup+7/R9lamUzLwKBgQDy0mq5tB+tfK5H8DEu/cs5hkaJSO3Iug1h +AJMdJ1GIlPTSB5/z+r5MMA/ufMwteQ2TOpbU44p32ZFAT83alAYSInt1STfj7/ZK +3U8loiYVh5o9Uxzs5CHgRRNpLJwxlTEnctQCgIEJP5H0HxpP+6RomutAfkqCC8wp +vhXEHCoXNQKBgAPBZA2tToxxUwVpvkJSN7X9/R5mloqgRKEdNKW2IllFN8GGgCCc +GgVqdg/UlhM3byO89eSRGnpCfmLhhxwlx8qWokGya40DP8AfLA2byzuKxDodfHzc +zMGaXH8pI2RBp29xP3isJybkf/ytM3z/VCFL9gkocWO9E9KAHiigj7hPAoGAbrsh +zDml0HlxCIEyDJnT2RGwjN5jAQxHGZsnez344m37DuRHPv1zVk78lOb4PSxc0mz/ +Z2m0NV9T653449powlBTOHMBN8Kv8AfoFeNRtrO11I1YPXbzM9CMP4QGXl4IolKs +988eCNeieU7NsvewS7uJ2Ek/NPqoScjTKDEnyJkCgYAXX/BzB0FtxW9Rbm3YAQrY +kFh/R5aHpkuz18U3u961A62HSfSR19ZfQUqmguea1jfZvsidd3+OeepW6qeqCUFt +PchZKTjdYpNRyyZ3HeMof4ciitns119tsnTcpGwknGgiiQUjeGd9dTPqwvOhkc0K +Hwb1z80PO5jU95PTWdkPYQ== +-----END PRIVATE KEY----- diff --git a/testdata/spiffe_end2end/intermediate_ca.pem b/testdata/spiffe_end2end/intermediate_ca.pem new file mode 100644 index 0000000..b6a0f25 --- /dev/null +++ b/testdata/spiffe_end2end/intermediate_ca.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEkzCCAnugAwIBAgIUScL7tRZAHT2l5ZGw+YblRNRRuO4wDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMSEwHwYDVQQKDBhJbnRlcm5l +dCBXaWRnaXRzIFB0eSBMdGQxGzAZBgNVBAMMEmZvby5iYXIuaG9vLmNhLmNvbTAe +Fw0yNjAzMDkwNTI4MzdaFw0zNjAzMDYwNTI4MzdaMCcxJTAjBgNVBAMMHGludGVy +bWVkaWF0ZWNlcnQuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDnXje+M6pAPFnrVe2XjBMSslqLQc/PBLMGplOloIwgoqcw3UnC8LfZ +joYXT7dfjA0IOxIg4mj+XGiQvoNMaudWEjTwf0goRE6kBUCVy+mxVq1XHR9Oooc4 +9fGQZgx+YkIMGAZPgg6T4THdtQezPg4cEjx6b4C4x+pmWs0ZvoYPi0MvZCDSt4sN +20sw9/mYH4wfZyEVBWpqJKUNjjwc5lzcweA0QkuO8LAZVQy9pwXeVhLEyCCtaDrn +T4wRsiL2hRZ0s+RFHKIG6I88uv8ZAWzthladh19a5/C74vACtpU/UyNipJrTpyOP +ePmhESAP3ut2arMryAvaDi4QLb2oQtG7AgMBAAGjgYMwgYAwDgYDVR0PAQH/BAQD +AgGmMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBROacRArm/eG2joV3rxYZUe/WpdpDAfBgNVHSMEGDAWgBT8 +uO0DehVYJPPep8BBe5z90LtvzjANBgkqhkiG9w0BAQsFAAOCAgEAcGPtuw1yoTGp +nOe1GxBmokrKQ+K2WTeSoAjAx1kkCTk+VCCzgZWFDZRCdIY94U2I2jviJbKttZmE +TIRd3uHXhVQKH93paQxdo1xfhL4akSg9CCdeiqkUBZ0JCZKovm0UdSnz0VBEPRZ5 +jZ5WGxD/VnSTt6Hw4u6OjwV6GrZ8ld8ftqbfy34zWOnK2zdpE1gKUrpsWGOHUhd/ +e1wZklsJ/Vyu2L4y2hCWvmAG1NBJgE6AhQMpM9McbcKs4012VZczfV0xUIo1wD7c +ccvhlz7KSIppAnXrvCrahCpv/vmpWnSExTuYmohW+LCvFSO89eSryfIQ+xHEb7Ck +PPiwja7lWGr7KCFX7E98LMl6HX1P76snNXOyxM6++r2/KzGnb0xV84g8VGbdgzhN +RUPRdoV89i0OwfNZlLMklNr0ilNTEYiWEjnD2bz1sHG6Ck5RK6tZfFew8JX+KA30 +WNs7s3YgKNS3s0/JJ3b7YrvByBQQnFGtVMNEk27RfGLd7dzNvlFtofX/d0GYyg9Y +b5pQrVd78mUCDg7MWZCW6unk8BrJXF+d6nw0V3FgY03eGr3wOAGkSzJUdJdqtycx +04JeL1n3Sl8r6mJJMJLzn1YAqDgM4Gt9W9corJFPCm2XnfUGjC/rDM7ACQYic8II +J9QTW19kU+UwDdBp1lX+hb8AbA4otBM= +-----END CERTIFICATE----- diff --git a/testdata/spiffe_end2end/intermediate_gen.sh b/testdata/spiffe_end2end/intermediate_gen.sh new file mode 100755 index 0000000..6fa8698 --- /dev/null +++ b/testdata/spiffe_end2end/intermediate_gen.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2025 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Meant to be run from testdata/spiffe_end2end/ +# Sets up an intermediate ca, generates certificates, then copies then up and deletes unnecessary files + +rm -rf intermediate_ca +mkdir intermediate_ca +cp intermediate.cnf intermediate_ca/ +cp spiffe-openssl.cnf intermediate_ca/ +pushd intermediate_ca + +# Generating the intermediate CA +openssl genrsa -out temp.rsa 2048 +openssl pkcs8 -topk8 -in temp.rsa -out intermediate_ca.key -nocrypt +rm temp.rsa +openssl req -key intermediate_ca.key -new -out temp.csr -config intermediate.cnf +openssl x509 -req -days 3650 -in temp.csr -CA "../ca.pem" -CAkey "../ca.key" -CAcreateserial -out intermediate_ca.pem -extfile intermediate.cnf -extensions 'v3_req' + +# Generating the leaf and chain +openssl genrsa -out temp.rsa 2048 +openssl pkcs8 -topk8 -in temp.rsa -out leaf_signed_by_intermediate.key -nocrypt +openssl req -new -key leaf_signed_by_intermediate.key -out spiffe-cert.csr \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=testserver/ \ + -config spiffe-openssl.cnf -reqexts spiffe_server_e2e +openssl x509 -req -CA intermediate_ca.pem -CAkey intermediate_ca.key -CAcreateserial \ + -in spiffe-cert.csr -out leaf_signed_by_intermediate.pem -extensions spiffe_server_e2e \ + -extfile spiffe-openssl.cnf -days 3650 -sha256 +cat leaf_signed_by_intermediate.pem intermediate_ca.pem > leaf_and_intermediate_chain.pem + +popd + +# Copy files up to the higher directory +cp "./intermediate_ca/leaf_signed_by_intermediate.key" ./ +cp "./intermediate_ca/leaf_signed_by_intermediate.pem" ./ +cp "./intermediate_ca/leaf_and_intermediate_chain.pem" ./ +cp "./intermediate_ca/intermediate_ca.key" ./ +cp "./intermediate_ca/intermediate_ca.pem" ./ + +rm ca.srl +rm -rf intermediate_ca diff --git a/testdata/spiffe_end2end/leaf_and_intermediate_chain.pem b/testdata/spiffe_end2end/leaf_and_intermediate_chain.pem new file mode 100644 index 0000000..0092a48 --- /dev/null +++ b/testdata/spiffe_end2end/leaf_and_intermediate_chain.pem @@ -0,0 +1,50 @@ +-----BEGIN CERTIFICATE----- +MIIDvjCCAqagAwIBAgIUSWu6PDuiG6sb3Xq6vlh6VJJaM8EwDQYJKoZIhvcNAQEL +BQAwJzElMCMGA1UEAwwcaW50ZXJtZWRpYXRlY2VydC5leGFtcGxlLmNvbTAeFw0y +NjAzMDkwNTI4MzdaFw0zNjAzMDYwNTI4MzdaMEwxCzAJBgNVBAYTAlVTMQswCQYD +VQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRMwEQYDVQQDDAp0 +ZXN0c2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn0RCm1Zy +1skITm0HxKRPj0c4eI98kUay30ZsdgWR0BWv76+LoleJ8CYn5t7m1E2fvhTH7yye +hiZLSMC3I9/653/wjhOSK3dnow43ErGgStGr22TdQPIiUAj9UNuQ9LhWJtTUwQXm +ial5SZzMS+OfQhjiK6lxUFh6sDk8KTRqDVVF65RLYMhGyhSOn+aI8SXGoWu+3y/G +UrOOLo8MlBN1YrK9Sgu4JZzYgdUAxs6PR8WSssoBykdq7Ak9Kr3/P1k0aF+eLEN+ +pjUgwC8z+Lx0xJcXZMKAHsGIFoFWP3hCEmtHWW7OzZ/FOeZARJ4EDg3u+cQr+Dqn +11/Vi9ruZH12mwIDAQABo4G8MIG5MHcGA1UdEQRwMG6CECoudGVzdC5nb29nbGUu +ZnKCGHdhdGVyem9vaS50ZXN0Lmdvb2dsZS5iZYISKi50ZXN0LnlvdXR1YmUuY29t +hwTAqAEDhiZzcGlmZmU6Ly9leGFtcGxlLmNvbS93b3JrbG9hZC85ZWViY2NkMjAd +BgNVHQ4EFgQUXaDeLUuKpyHPOFRSzjDft8OqIRIwHwYDVR0jBBgwFoAUTmnEQK5v +3hto6Fd68WGVHv1qXaQwDQYJKoZIhvcNAQELBQADggEBABjU2ZTLcaMSM6uwbw48 +WrTMwxhCKhOWtirGxJkKzr/g6E5j7VMayWYWuDLDn+snWNOPuCEEFLOY15FYxvX8 +iVQ0uBYBNvbKmcLggm3eu3JleMGvHh4RnwJ7MprYB2UEHqGkD2p9CNm67eO4uqCh +FpNj0kwrlL2nzO8jgJVnV8gn60UJ5OUA5q09HaDg0Eh6QNTVHL5V5WJPp2aGgTK8 +Ku/gIjjAanWlpk9A5jsHz5UD0I97+E3jn+ZJGHBEM9u6pQUyhwL/K9VirLqCLd/x +UjCphAIfz1JN/+ZypSTyk+MDKU37hjbnk/uxh6ltv+Q3SnyRKM23FlDwG74jHLYJ +nVI= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEkzCCAnugAwIBAgIUScL7tRZAHT2l5ZGw+YblRNRRuO4wDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMSEwHwYDVQQKDBhJbnRlcm5l +dCBXaWRnaXRzIFB0eSBMdGQxGzAZBgNVBAMMEmZvby5iYXIuaG9vLmNhLmNvbTAe +Fw0yNjAzMDkwNTI4MzdaFw0zNjAzMDYwNTI4MzdaMCcxJTAjBgNVBAMMHGludGVy +bWVkaWF0ZWNlcnQuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDnXje+M6pAPFnrVe2XjBMSslqLQc/PBLMGplOloIwgoqcw3UnC8LfZ +joYXT7dfjA0IOxIg4mj+XGiQvoNMaudWEjTwf0goRE6kBUCVy+mxVq1XHR9Oooc4 +9fGQZgx+YkIMGAZPgg6T4THdtQezPg4cEjx6b4C4x+pmWs0ZvoYPi0MvZCDSt4sN +20sw9/mYH4wfZyEVBWpqJKUNjjwc5lzcweA0QkuO8LAZVQy9pwXeVhLEyCCtaDrn +T4wRsiL2hRZ0s+RFHKIG6I88uv8ZAWzthladh19a5/C74vACtpU/UyNipJrTpyOP +ePmhESAP3ut2arMryAvaDi4QLb2oQtG7AgMBAAGjgYMwgYAwDgYDVR0PAQH/BAQD +AgGmMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBROacRArm/eG2joV3rxYZUe/WpdpDAfBgNVHSMEGDAWgBT8 +uO0DehVYJPPep8BBe5z90LtvzjANBgkqhkiG9w0BAQsFAAOCAgEAcGPtuw1yoTGp +nOe1GxBmokrKQ+K2WTeSoAjAx1kkCTk+VCCzgZWFDZRCdIY94U2I2jviJbKttZmE +TIRd3uHXhVQKH93paQxdo1xfhL4akSg9CCdeiqkUBZ0JCZKovm0UdSnz0VBEPRZ5 +jZ5WGxD/VnSTt6Hw4u6OjwV6GrZ8ld8ftqbfy34zWOnK2zdpE1gKUrpsWGOHUhd/ +e1wZklsJ/Vyu2L4y2hCWvmAG1NBJgE6AhQMpM9McbcKs4012VZczfV0xUIo1wD7c +ccvhlz7KSIppAnXrvCrahCpv/vmpWnSExTuYmohW+LCvFSO89eSryfIQ+xHEb7Ck +PPiwja7lWGr7KCFX7E98LMl6HX1P76snNXOyxM6++r2/KzGnb0xV84g8VGbdgzhN +RUPRdoV89i0OwfNZlLMklNr0ilNTEYiWEjnD2bz1sHG6Ck5RK6tZfFew8JX+KA30 +WNs7s3YgKNS3s0/JJ3b7YrvByBQQnFGtVMNEk27RfGLd7dzNvlFtofX/d0GYyg9Y +b5pQrVd78mUCDg7MWZCW6unk8BrJXF+d6nw0V3FgY03eGr3wOAGkSzJUdJdqtycx +04JeL1n3Sl8r6mJJMJLzn1YAqDgM4Gt9W9corJFPCm2XnfUGjC/rDM7ACQYic8II +J9QTW19kU+UwDdBp1lX+hb8AbA4otBM= +-----END CERTIFICATE----- diff --git a/testdata/spiffe_end2end/leaf_signed_by_intermediate.key b/testdata/spiffe_end2end/leaf_signed_by_intermediate.key new file mode 100644 index 0000000..2e5cca6 --- /dev/null +++ b/testdata/spiffe_end2end/leaf_signed_by_intermediate.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCfREKbVnLWyQhO +bQfEpE+PRzh4j3yRRrLfRmx2BZHQFa/vr4uiV4nwJifm3ubUTZ++FMfvLJ6GJktI +wLcj3/rnf/COE5Ird2ejDjcSsaBK0avbZN1A8iJQCP1Q25D0uFYm1NTBBeaJqXlJ +nMxL459CGOIrqXFQWHqwOTwpNGoNVUXrlEtgyEbKFI6f5ojxJcaha77fL8ZSs44u +jwyUE3Visr1KC7glnNiB1QDGzo9HxZKyygHKR2rsCT0qvf8/WTRoX54sQ36mNSDA +LzP4vHTElxdkwoAewYgWgVY/eEISa0dZbs7Nn8U55kBEngQODe75xCv4OqfXX9WL +2u5kfXabAgMBAAECgf9AcpewyOJaQLNeJWtBrB9DTRNdfOuMiKcIliruxqeGU58g +cYB3258uoXFxIPq5AUf4MUEoO60ZpUjkkA3XwmKfg+0ymln9iKlBboihvA+cSgyy +11bW1J+4k+qrW89DEf9+05dqEpDiGmZF8ahuFprplgu2qNDcU4ksJRlsBNysdO9q +w8HN25ERWniAzsuXuG+w9AGBFk2eNuumKmQUBJmeqYcJb45YsXhLM4k1rQOJlmYZ +OI+HhHexRn7/iLOsZns+Sm9DXadWy7Gdf+WaU4AR4M9JoBGi4qG9LMP2NDVFvPr0 +S+nfsri+4Z08SvcgdG+sHPzsxwojSAeSm8EIIpECgYEAysvpoISbu0j3noNAjA81 ++0YyN3XQ/dSkogTtbJFE8Zm0YjpvMYwpO5oJFXnbE2ofAHNUXicpnO2gw0lna8Bp +RmYfdjI8/w//+x9Mw3yBhBQ0YlDK4MC2A90G69/AsJsKqiqHOy6aQg0yeLLIVJn7 +bsAjOyUpuhpBzGQcG9bwZvMCgYEAyQzSU2tbxYiKiZcnrP87alPa/43XtUSsTILX +EbmsWL9gMyqz3UOf0QQomUis0njbuQHi8MU+8ecQPG43AjAnpyHuwtgDUUV4EDPb +IhGCE39Or6Jn9cNLkDpSdANCXE1Q+qjdvmaYChYpO/o4xR1036XUBz7qNYWXWj4O +pmk267kCgYEAi923dA4Bmlno7lp32iFjiboQSE/ppCdUpKnhVk+azUbFMjo7FmEk +zwad3UH95pX6a8UfGxDHkoQRrJ6jxZ0e/n7QlCRyDThrxDcCKpFkgkOtHWG7iude +oat/ao8XxrYn1NUgD6FEoumXNceYg0DwOKIrqk8nSENzvhQNjuXfSCECgYAymveV +58AByIyWdWWXNedOrCzDhoB1MAPufkCERagL7p/YQTdkylC/27wcWR6nG6SyvLbS +w9NEMFT14QgXlOdmOjRO9vBe1I2UBnlx6dZD8hdsPgTM54ttkkCO0wMxGIT5kue9 +tTUv1MQsRJ9lfjSc1rC34i4xqp6eKGCnonQggQKBgQCBYfAGIrZy3OLbr3EybT7w +lq5LXSnxRkVAXYAY9Ekrha2TjKZvnLWcPvej60wi9swf0756Oyz672XAoPe+cveJ +mVhE3HurAHtUyC/PQhh8foqvO0dEtRrOdpCZIGv0mGflrHDzU8qHWBGuCtwmYsZU +rJEMcQLW2d6IFQkizWc9+A== +-----END PRIVATE KEY----- diff --git a/testdata/spiffe_end2end/leaf_signed_by_intermediate.pem b/testdata/spiffe_end2end/leaf_signed_by_intermediate.pem new file mode 100644 index 0000000..75b2a0b --- /dev/null +++ b/testdata/spiffe_end2end/leaf_signed_by_intermediate.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDvjCCAqagAwIBAgIUSWu6PDuiG6sb3Xq6vlh6VJJaM8EwDQYJKoZIhvcNAQEL +BQAwJzElMCMGA1UEAwwcaW50ZXJtZWRpYXRlY2VydC5leGFtcGxlLmNvbTAeFw0y +NjAzMDkwNTI4MzdaFw0zNjAzMDYwNTI4MzdaMEwxCzAJBgNVBAYTAlVTMQswCQYD +VQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRMwEQYDVQQDDAp0 +ZXN0c2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn0RCm1Zy +1skITm0HxKRPj0c4eI98kUay30ZsdgWR0BWv76+LoleJ8CYn5t7m1E2fvhTH7yye +hiZLSMC3I9/653/wjhOSK3dnow43ErGgStGr22TdQPIiUAj9UNuQ9LhWJtTUwQXm +ial5SZzMS+OfQhjiK6lxUFh6sDk8KTRqDVVF65RLYMhGyhSOn+aI8SXGoWu+3y/G +UrOOLo8MlBN1YrK9Sgu4JZzYgdUAxs6PR8WSssoBykdq7Ak9Kr3/P1k0aF+eLEN+ +pjUgwC8z+Lx0xJcXZMKAHsGIFoFWP3hCEmtHWW7OzZ/FOeZARJ4EDg3u+cQr+Dqn +11/Vi9ruZH12mwIDAQABo4G8MIG5MHcGA1UdEQRwMG6CECoudGVzdC5nb29nbGUu +ZnKCGHdhdGVyem9vaS50ZXN0Lmdvb2dsZS5iZYISKi50ZXN0LnlvdXR1YmUuY29t +hwTAqAEDhiZzcGlmZmU6Ly9leGFtcGxlLmNvbS93b3JrbG9hZC85ZWViY2NkMjAd +BgNVHQ4EFgQUXaDeLUuKpyHPOFRSzjDft8OqIRIwHwYDVR0jBBgwFoAUTmnEQK5v +3hto6Fd68WGVHv1qXaQwDQYJKoZIhvcNAQELBQADggEBABjU2ZTLcaMSM6uwbw48 +WrTMwxhCKhOWtirGxJkKzr/g6E5j7VMayWYWuDLDn+snWNOPuCEEFLOY15FYxvX8 +iVQ0uBYBNvbKmcLggm3eu3JleMGvHh4RnwJ7MprYB2UEHqGkD2p9CNm67eO4uqCh +FpNj0kwrlL2nzO8jgJVnV8gn60UJ5OUA5q09HaDg0Eh6QNTVHL5V5WJPp2aGgTK8 +Ku/gIjjAanWlpk9A5jsHz5UD0I97+E3jn+ZJGHBEM9u6pQUyhwL/K9VirLqCLd/x +UjCphAIfz1JN/+ZypSTyk+MDKU37hjbnk/uxh6ltv+Q3SnyRKM23FlDwG74jHLYJ +nVI= +-----END CERTIFICATE----- diff --git a/testdata/spiffe_end2end/server.key b/testdata/spiffe_end2end/server.key new file mode 100644 index 0000000..a876ede --- /dev/null +++ b/testdata/spiffe_end2end/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7mwFKwCTD58lp +Pq+aQUpum9+CADQxiiFY2B9zWGCkotIaZhtCA/ZVdy5Wn17T0AosLAeyxWsqg9OV +cEBRgs7xvvZxXM8MUlfOjTNMgfFgI8VNYOK94Skwo/CTdNz1KkWLthkNwdL8vjWH +KQgdqwp3HvuqYL0N+b27MPEQdImY+sqGWqWtHqFbTIPJcyyWPS1Wyw9jh+YwTo/w +jPXW6mcODRNWv8ZZfbtOaQVNQ60t8U9sdJhESZ5XqQrWsTwulKxGw5mXl1K10iTW +mX28k8QS9NsgtZKYyBwIFGcjc+19EiP46xDlXmWcE3YL+huogP7KDXuUoKIo/oRB +xaYJ9mujAgMBAAECggEAEeoBO5QRbquJbgVQW1h0tQ8pTo6abUiVWph4mFkOEWqC +yYaKf8lFEnAo+piJQ4yQDBvAOG/lhc/EunZXjfEBtc/YVIbaNoD+ZXjSNzIJTHbd +9j+UJzGC72QYKtxz5O0+atLenZOug/fdwKRIZBzbCPjqayCFrPn2BhPsUPfv062l +AzP2om5rrT1ZdXLNlEy7BXjMs1qljj1b3VmfWvnrsA5khq0DHYJoJ7rP0f/tKhBR +t4+Gh7/6Wec4Qqoy10rco6Oa3OpIolKFJMYQ3enA2aaTIM7L7E+kVrRm62+OB7mn +pkpkPMYN9IC52FqHSaWc16vftynr+PjbNbkcElk4gQKBgQDsnleyATqTNtX2qDH9 +eYv4I05hulbVyI1OgsiWKLHQooGuaHdReUwW/RuofSVo/owJXSIvqnldltyuQ9Da +ZPYbOlpEl4twSRIx5To4w7zuYgpFr8643lPCpij+PLGPryvn2JuJAoNPU4rzEN7y +1vxU2U7Bqoh/X0S9ZYpBwzIyQwKBgQDK+OhJZdBVVeta3Nip9Bli7XXhcZV++7ip +JDxzIMrDqjYzY4sQDtkhpysuos2Mgp90C4HwhpDsfP2whVuypFLj6ChEspot/1DI +rFo2KhEOVOXOfdTnLrtWlitSp38i2E2C5g53T9Jigyq8ssQfZwoqlVP7fOK+79FV +CBJpIC27IQKBgEunTO6zCeFr1PlqSaF7rU8HKtaAV6c+2j9R/YRVOpU0gDYdXJkG +KVfoUWGLsdxiFrAfwQBwhyFvTNvC/xH02eNWwunPclvSYSjm27iujMfDPPmO/o+J +Nkq0CcNP8I26OlWEoiYqUYWZdoHE0SPfrQoL+Oxe9AmVkkrkHlJscK41AoGAB3Ut +08SR6xDFHQmQTG5ToHbpJedufsPw/QX/0psZ2Cag5zJ5IZXqFHp387a3proF8dWa +aKQJHydYiuvbeqze/tDA6gVF9Pq0lSsABY12IvirmPK2p+fnqj7KSLcuzLD16CFb +1rZwHH6FS3mmCyFWFkp2U387NZjKMD2jr4knJQECgYEAxexweu0XMPRujIL4/ONO +XjKf0N7/WJOaMt4cxP/po02taLSg+9Nw3/7rEM2sX6046mG3mPTKb2Aj6gpdmPIj +DdpkFBoxgZy3/QBscm4Xmi6AOgmAVy0cNgnXLDbFY8pzUiaBaxpqtHlBxzxNk2UG +3DB1bo79CURbSmdFaUklpkQ= +-----END PRIVATE KEY----- diff --git a/testdata/spiffe_end2end/server_spiffe.pem b/testdata/spiffe_end2end/server_spiffe.pem new file mode 100644 index 0000000..cc924d0 --- /dev/null +++ b/testdata/spiffe_end2end/server_spiffe.pem @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE+DCCAuCgAwIBAgIUCzY7jU+NrvxZf0WMdg/5AM5i8XYwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMSEwHwYDVQQKDBhJbnRlcm5l +dCBXaWRnaXRzIFB0eSBMdGQxGzAZBgNVBAMMEmZvby5iYXIuaG9vLmNhLmNvbTAe +Fw0yNjAzMDkwNTI4MjFaFw0zNjAzMDYwNTI4MjFaMFMxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRowGAYDVQQD +DBEqLnRlc3QuZ29vZ2xlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBALubAUrAJMPnyWk+r5pBSm6b34IANDGKIVjYH3NYYKSi0hpmG0ID9lV3Llaf +XtPQCiwsB7LFayqD05VwQFGCzvG+9nFczwxSV86NM0yB8WAjxU1g4r3hKTCj8JN0 +3PUqRYu2GQ3B0vy+NYcpCB2rCnce+6pgvQ35vbsw8RB0iZj6yoZapa0eoVtMg8lz +LJY9LVbLD2OH5jBOj/CM9dbqZw4NE1a/xll9u05pBU1DrS3xT2x0mERJnlepCtax +PC6UrEbDmZeXUrXSJNaZfbyTxBL02yC1kpjIHAgUZyNz7X0SI/jrEOVeZZwTdgv6 +G6iA/soNe5Sgoij+hEHFpgn2a6MCAwEAAaOBvDCBuTB3BgNVHREEcDBughAqLnRl +c3QuZ29vZ2xlLmZyghh3YXRlcnpvb2kudGVzdC5nb29nbGUuYmWCEioudGVzdC55 +b3V0dWJlLmNvbYcEwKgBA4Ymc3BpZmZlOi8vZXhhbXBsZS5jb20vd29ya2xvYWQv +OWVlYmNjZDIwHQYDVR0OBBYEFBX+f5l5bG5acfhtDIBG57WD+zAtMB8GA1UdIwQY +MBaAFPy47QN6FVgk896nwEF7nP3Qu2/OMA0GCSqGSIb3DQEBCwUAA4ICAQCWvSe3 +kD21ksNBbH5COlpLhKbbnnlrkBAzzZjeJfkdDROXXUWAepxubX81cd5aZwhucFWL +bEL08D9+f+cUrseAafMxNXqBoK9fzJJq2v8fD27Ian63oAfcR5pG7EuvoJ99zwqL +Bi3SYuakq+u59LfWOmq/ohSi9H9pFbSJHjmMi0CynfL6qYJKacJcEjDq8yJTC4t7 +/s8eJGz+w1sSIlpkgbnNva3MA2HrLVzdQwC7JzZ69FljX4JZH37PXTQfi6Ri1WJq +G8OlEnk4NVt/pimiK5OLmhcfk7fbFL274A7rLzrHT1IZE6NUmDq9i8BwqnTNumMI +mwNQWe7ZPO6SNyv8lVrcuZF/wRx4b5KP0OBqi9I5emjolU1n+OcfurClgkFgN46d +GSBeIToQekBOWFm1Hb4a/nBMqaiLHnslnBbGsZGZ+miD8Rue/yqtCD10NjYJkaUC +jHOSwZ3hNn1Po9S0HehKA9MZO/ES3MQHedqtP3K3HG52ZmKQ8NNKIQ34zSsgrNHP +vZgWpmdvcUrnzh1Ft1oq4nLws3Y2F4/c7YUfVWzo+ydePy5eaYQaMjYdhosBJWHk +3fMQJm+mkeLZmM2Akk5Zza68U6FuyBTyERbl60TV2pU7pZ5cqaOOO16/kfZTmYSq +9g6wsjV7y0tVXlPir4ezSeI0SyYikuS9/v6Ycw== +-----END CERTIFICATE----- diff --git a/testdata/spiffe_end2end/server_spiffebundle.json b/testdata/spiffe_end2end/server_spiffebundle.json new file mode 100644 index 0000000..ddc2382 --- /dev/null +++ b/testdata/spiffe_end2end/server_spiffebundle.json @@ -0,0 +1,16 @@ +{ + "trust_domains": { + "foo.bar.com": { + "spiffe_sequence": 12035488, + "keys": [ + { + "kty": "RSA", + "use": "x509-svid", + "x5c": ["MIIFlTCCA32gAwIBAgIUZ4e1KRtWw0c8V5NnA9VEoUMvzq4wDQYJKoZIhvcNAQELBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxGzAZBgNVBAMMEmZvby5iYXIuaG9vLmNhLmNvbTAeFw0yNjAzMDkwNTI4MjBaFw0zNjAzMDYwNTI4MjBaMFoxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJWQTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRswGQYDVQQDDBJmb28uYmFyLmhvby5jYS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDZRezL1GWM/0umgcLNGI9fsp0sabLUN1rUTwXjCGHvGAt9cPY1xK6P6+/QXeBkYogmwC3Udhwl+8x3AOu2FKP9QeveW+hB4N3SCtY4D+nGxPWnL1Yjh0170tkVRPkh89wPX4XdtAcIgZjjNgPH9wR3eWH8E3cDRvxjosy4e9aw0/VTHb0lZ2Ko6mY8x2bJNFUExmuh157b//yHdcL8O1K1h13o6YSqSlnLmnaSHpHA5/21erSjszE3ZjiPZFR6cl/RDjpJRKrh/eQAAeg7XO00VokrvZnK07mZ2nNyp4z1yCV4fwWCdBMfFfUSm0EgN8D65JxGj1He5bm4LVv6WuWj+cqgAzRhHjp8iZsmLFKvAoXSg1eJ+mG8vw5P9ZVd38M8N2DtifvmCkE1fln84oQj42wDgNQuwSafbGQIhNYrnsgbf+2lNj5IIYm/ScUcexbmCmKyOZfNcVQNAF26vQOvibBDOFyDW2qdi2tIGJgTPhkJPbDqKJx4UqNxIuoam28PifmK51Cn4i9rWOpQ2nM7JGX1Q3IE6S5iaVnogp2+SSh3PhktxQOK9IUX7ysX9eVQAdN8lnKgaIZuyYOx9H9jWValZ/dCLa2Y/BuFslx4NRVLtoJkUIakdTepo83ZrrSs3c4DSEIJJkgVPBmBkua3RdYB1Cw+A8T0C8nvExyJbwIDAQABo1MwUTAdBgNVHQ4EFgQU/LjtA3oVWCTz3qfAQXuc/dC7b84wHwYDVR0jBBgwFoAU/LjtA3oVWCTz3qfAQXuc/dC7b84wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAZFJDbhKL1M16Ya7eAvrgUWI9vbC4xm/68HzIGuQLeP9yr7QMfk5wzh9umYcGWs/hkz2jg/88PbPUEP35psTw+dZGtnQwi/0icM9w7m0HZrDx8vMYvX5AoEaPb5PqbjwhBxzd2Hpv/Y+PLLRO9uh1Xeh95J4yp5t/xOPyimljKLTam+rpj7BevYftlSXuIWrR6sldIt9I1zT5CfcBSefmSVo2WtgLdqfOBIAD73OMbcxEPDiTF4/q8S1qHeA8eNUXc5IrWEI2NirhpvR0ZXMyjY1gJoZ4XqPFSXg+1oi9qpbjMhKU5J+Rm8brv/HmACaP90hSPXjuRl2Tp/NJ6idV1heNd4MdTeAlH+deQ+UQ/rFo6HY95S3ltdcaDX8V7WdpRUQddv2DS5VPPi60LJuOdxKB+XFtjdLUtxwOw3Hoj91YWZLuMu2xzVeASeA6FsB0vZwQdqn8FX1BQNSIEHGs7tieV7N0AlLavQBpyRYfhgKOL9Utbl4Cp6C6k42uFo/tFREUdzgtt0JqT+/NMJRA6rOJYHVUWNeG2ggGpuCtHXkwktC4JBnUwnvdVZX3sXPslmiFzMO6S/+PesHA0TclifTZNOETHphF8efGsRFL/lkhcBRkZKBVOkWKdKMUnXPgDKLyu/e1WQXCXmyTjv8FEiG2Ni8U4P0c2PldooYVjw4="], + "n": "2UXsy9RljP9LpoHCzRiPX7KdLGmy1Dda1E8F4whh7xgLfXD2NcSuj-vv0F3gZGKIJsAt1HYcJfvMdwDrthSj_UHr3lvoQeDd0grWOA_pxsT1py9WI4dNe9LZFUT5IfPcD1-F3bQHCIGY4zYDx_cEd3lh_BN3A0b8Y6LMuHvWsNP1Ux29JWdiqOpmPMdmyTRVBMZrodee2__8h3XC_DtStYdd6OmEqkpZy5p2kh6RwOf9tXq0o7MxN2Y4j2RUenJf0Q46SUSq4f3kAAHoO1ztNFaJK72ZytO5mdpzcqeM9cgleH8FgnQTHxX1EptBIDfA-uScRo9R3uW5uC1b-lrlo_nKoAM0YR46fImbJixSrwKF0oNXifphvL8OT_WVXd_DPDdg7Yn75gpBNX5Z_OKEI-NsA4DULsEmn2xkCITWK57IG3_tpTY-SCGJv0nFHHsW5gpisjmXzXFUDQBdur0Dr4mwQzhcg1tqnYtrSBiYEz4ZCT2w6iiceFKjcSLqGptvD4n5iudQp-Iva1jqUNpzOyRl9UNyBOkuYmlZ6IKdvkkodz4ZLcUDivSFF-8rF_XlUAHTfJZyoGiGbsmDsfR_Y1lWpWf3Qi2tmPwbhbJceDUVS7aCZFCGpHU3qaPN2a60rN3OA0hCCSZIFTwZgZLmt0XWAdQsPgPE9AvJ7xMciW8", + "e": "AQAB" + } + ] + } + } +} diff --git a/testdata/spiffe_end2end/spiffe-openssl.cnf b/testdata/spiffe_end2end/spiffe-openssl.cnf new file mode 100644 index 0000000..0092743 --- /dev/null +++ b/testdata/spiffe_end2end/spiffe-openssl.cnf @@ -0,0 +1,28 @@ +[spiffe_client] +subjectAltName = @alt_names + +[spiffe_client_multi] +subjectAltName = @alt_names_multi + +[spiffe_server_e2e] +subjectAltName = @alt_names_server_e2e + +[spiffe_client_e2e] +subjectAltName = @alt_names_client_e2e + +[alt_names] +URI = spiffe://foo.bar.com/client/workload/1 + +[alt_names_multi] +URI.1 = spiffe://foo.bar.com/client/workload/1 +URI.2 = spiffe://foo.bar.com/client/workload/2 + +[alt_names_server_e2e] +DNS.1 = *.test.google.fr +DNS.2 = waterzooi.test.google.be +DNS.3 = *.test.youtube.com +IP.1 = "192.168.1.3" +URI = spiffe://example.com/workload/9eebccd2 + +[alt_names_client_e2e] +URI = spiffe://foo.bar.com/9eebccd2-12bf-40a6-b262-65fe0487d453 diff --git a/testdata/testdata.go b/testdata/testdata.go new file mode 100644 index 0000000..c7d2481 --- /dev/null +++ b/testdata/testdata.go @@ -0,0 +1,42 @@ +/* + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testdata + +import ( + "path/filepath" + "runtime" +) + +// basepath is the root directory of this package. +var basepath string + +func init() { + _, currentFile, _, _ := runtime.Caller(0) + basepath = filepath.Dir(currentFile) +} + +// Path returns the absolute path the given relative file or directory path, +// relative to the google.golang.org/grpc/testdata directory in the user's GOPATH. +// If rel is already absolute, it is returned unmodified. +func Path(rel string) string { + if filepath.IsAbs(rel) { + return rel + } + + return filepath.Join(basepath, rel) +} diff --git a/testdata/x509/README.md b/testdata/x509/README.md new file mode 100644 index 0000000..661caf4 --- /dev/null +++ b/testdata/x509/README.md @@ -0,0 +1,6 @@ +This directory contains x509 certificates and associated private keys used in +gRPC-Go tests. + +How were these test certs/keys generated ? +------------------------------------------ +Run `./create.sh` diff --git a/testdata/x509/client1_cert.pem b/testdata/x509/client1_cert.pem new file mode 100644 index 0000000..6f82cc3 --- /dev/null +++ b/testdata/x509/client1_cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFcTCCA1mgAwIBAgICA+gwDQYJKoZIhvcNAQELBQAwUDELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV +BAMMDnRlc3QtY2xpZW50X2NhMB4XDTIxMTIyMzE4NDI1MVoXDTMxMTIyMTE4NDI1 +MVowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBALUoje/J3uPOJ0dapY2s7mGLVPhYRaHyRnJE2/TY +zFOB0IisAF3R7BIDufQrHhk3fh0JazCw95TDD9rxsKEVs6Z50lmDkrg/bjlsniE/ +n+M1JacaLQW7xfh2L+Ei4jvMr101nAsimd6IxFU9m3+2SFbhPBG/GWWJ2ZKqQblz +DVMpNg9FYNmMe45vLevOhdPQBE4cVoAPhI9Je+P4Koslebhor0koUeQVeYdBbCq3 +3dQJPAHjBST6mD9mJI4yVrE3Xso3LO85WROUPhRYQyXhrgU15W6g9qTpMTfkriUe +FYLCtAPU9LBodyvjYLuwoEoyRVsA6Zh/vABteD8Afl552fV9KwN2fRVbTDAxQCp7 +P8gE3/rD1RKv7KBNJ/LrwMu7g4VO+tzYDxWee+eXPQ6M/zRWAb3E0v3UNHsF1ZBl +rlFhEiRShHrXDEKMQwCTSrRjwYajUpZ/Hq2USDgkLepKmTmCaoBfWHPyZwblqSTn +A4DNOh5N23eJyrLnJOPYjzZqEPfX5hDTjFRdVTQxtmYlJ1muwtlNyuwZDImhjO6G +54pPj/bV6gy1+YpIQBemPoXtqqmcRiEVWSV5zAizwRaWf85tqpxb1Tjuj2OpD9le +oO4JX0HLjhyQBoKspNohu2I4+s7ex/w92bf76cTpYTbMJqIp37YZmfPVztHVaMl4 +W0xRAgMBAAGjVzBVMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFMRdhhib+RS6IJpQ +zFsaKH1BNbyZMA4GA1UdDwEB/wQEAwIF4DAWBgNVHSUBAf8EDDAKBggrBgEFBQcD +AjANBgkqhkiG9w0BAQsFAAOCAgEAHyQwLSo/UdSoZCKcdeb0DCkeABRMUPIysroj +gQJCe3cAKOT+TxL6Qnp6jzM2/vt2T/lzlRp7RAB1jH3J4wy5re4cbCn1sbII3Rog +Nm4PKcw6hfUET44t1Gk9DsCjgvxEIirFBWVpxfn+YDI916iH1fkNURaMP+yxpQBL +3K4bmxanBiyBUHC8cyChLMD2NwXjOAA4pZFk0ohpmK0YUk4ra3Z3Q30DCH6NZ1ZP +aOMDHrCXU6MLlmPk8yiOnotgjqiYEgi3Bzxd/OHpR41Xo8k6g3UrN2GEQFs17ibQ +CQasxodOar5Vezu6ZKCYk5TaY4lugT34w+qxi8tVF54WY2jtWY5PUmU6ZT2Dw5cn +CQzlPUdEebOc1hltTvsD049/2lZmGlMXk0dykxy51jYAYznf2rb3cnC1vu1Wgi3w +J28xXBYD8AvME9jaJ6g3L+KR+AFCSLqpUsTxvu9zKf6pLrVtOCl+9G69uOK/wono +yMGNeel8rkzwzzr1LNrhmcKHqipkq83vqxIUT/mbpBUKO1ZXVG/TWKS6bpBTc4Pn +hBCIvGOSyoKuEiXnFr6fqLhLskUNcCNl7iOfA9h/MhS5ZufJXhhXu3Wbo/KC/mNh +y+fr1S9AyA+EJaYtJRKAOeewGvXYb881UNXWGCQU1aVNJnujRKFyhd07sEjxsad9 +Bn/aYes= +-----END CERTIFICATE----- diff --git a/testdata/x509/client1_key.pem b/testdata/x509/client1_key.pem new file mode 100644 index 0000000..6cd652c --- /dev/null +++ b/testdata/x509/client1_key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKgIBAAKCAgEAtSiN78ne484nR1qljazuYYtU+FhFofJGckTb9NjMU4HQiKwA +XdHsEgO59CseGTd+HQlrMLD3lMMP2vGwoRWzpnnSWYOSuD9uOWyeIT+f4zUlpxot +BbvF+HYv4SLiO8yvXTWcCyKZ3ojEVT2bf7ZIVuE8Eb8ZZYnZkqpBuXMNUyk2D0Vg +2Yx7jm8t686F09AEThxWgA+Ej0l74/gqiyV5uGivSShR5BV5h0FsKrfd1Ak8AeMF +JPqYP2YkjjJWsTdeyjcs7zlZE5Q+FFhDJeGuBTXlbqD2pOkxN+SuJR4VgsK0A9T0 +sGh3K+Ngu7CgSjJFWwDpmH+8AG14PwB+XnnZ9X0rA3Z9FVtMMDFAKns/yATf+sPV +Eq/soE0n8uvAy7uDhU763NgPFZ5755c9Doz/NFYBvcTS/dQ0ewXVkGWuUWESJFKE +etcMQoxDAJNKtGPBhqNSln8erZRIOCQt6kqZOYJqgF9Yc/JnBuWpJOcDgM06Hk3b +d4nKsuck49iPNmoQ99fmENOMVF1VNDG2ZiUnWa7C2U3K7BkMiaGM7obnik+P9tXq +DLX5ikhAF6Y+he2qqZxGIRVZJXnMCLPBFpZ/zm2qnFvVOO6PY6kP2V6g7glfQcuO +HJAGgqyk2iG7Yjj6zt7H/D3Zt/vpxOlhNswmoinfthmZ89XO0dVoyXhbTFECAwEA +AQKCAgEAjtzrijWVy+sQuMm4k1DUMSKzIKJkT4GDoqvBFoc+I4DVVmLmaxaYZ+B+ +bhruwo4rq3R5Ds4QgUWPJGfDllVJ9rhNdYA4XYrQPwL0dV36ljCcf/o5lTLuvbFe +stpStTwG86fKZlGkLIWI53wNPBshUzqOp6QfwB6E8Y/JAxnDYVi3pDVfWlDaQ4pU +GYklqtN6AauBX75dGK6nwDE+Q7uLES2lRjlA03FIBK1IQyv7CTM7GnXQ4cep9x1z +KJx0F4+F9kyq6AE+yRz4FA1C7wXZuYw2YhcYSxcHVH/IAceGyTcIxZjUWqYXjQnk +iD+TONAKN+kxTq01MtUhpfWasqC/i+6QU1eqf5YWpd6GsRKyrGgO02NND/SM6Z3V ++S9og4QAjdUyc8dkN+udd1K1CeYNFbmhrYpF2aS9k/PjDP3L137hDW6Cy+thIjZP +u9OB6ba2yUrbQDlmkCbh0vX+77HKAbT5bj8h9r7MqzNsPsgkaKS8gZ79T/Whr/ft +Xiu+eo/u1jtjwUjNMKGxQ9XiU2UU7QccthHHLcYaiv4eySHXA75h+Sho9cD1Vvs/ +ms1/nbCSuU9TSK0UK/V8YjeDA0eVGtDCX3weIW2ECQ80SoT7uf+fhjaLkvOadb7f +1O9DvYVYZvblxUm8ajOh+/n9VyB/I9R9Q8GdGiauXy16uXLZMdECggEBAPEx+4aR +XfuXmnEoPk3GpvyrRlrMy02aklrATKcxeAYo2uiIcuQv65E3AmE1GHpoxmAKjLty +fuUfGdT7f4uGeF6p+IEkW4jQm56UFbCdun9kduEaN9FRylTBqUKWIY2rtRS6nHZ8 +bAkL/6Uv3g9NWx95rV7HnAfC2n6AIvc8LRfQVVqSvjPbsEPvJAT2353D0Rb7vC2M +1hKeBrSNBiy57EKnrMDOhNpBvSBU0Zc+YsBRNAimKyBz7dt35H+THkFaEk9vGtG0 +QkDvngPzSX99Ojwk2mo9jGrh7LHErWih5C73IfvYUh3kyEwbZ5y25i9Z0F37boIG +jHSVvcPp+9x9PNUCggEBAMBHLyhBUAQVZFXtWysr0BjO34XffgkSt1XQa8cVxif7 +glWauUZtjfC7PT/qgY0mx2dI2bDcKiQQCBlVavP1RLRwj3rZv23eit7z13UgHSa6 +3dnsgpO2Zux6qoV48lO4xbuFqZtW+MP+9jthKwr95r8lmZ4cmGQwXXcqNsR7skFt +30Uhcyn+MTfyLwcqt8g9i98rrJmbPAuIME/Sz9DLIi6UxQLI6MeEn92AzECNDp18 +CypOL+sDrLw/7HNHNoSblgm628BHpBgT2qaOYnawRr0gni7MHXOAbDopKYDAtLuU +ZMFjlILdfiSDouhvKtMlZG9arTB0TasdAQJGPz53H40CggEBAJ4JDvJsOzVHb2Vn +ZfNWD0INA0spVqhheDXIPDFsg2UdzdmA1i7XizUZ4xBIVuKV1i1FnFKRwb1ktGtN +4pNMJ4B3RCFx7hvl+6FbDB8uKe2gqRfzMtGPEtCYF8xOTGvkLwEHCM/F1I/U8cuN +YqWKHQOxmTw58+1N6hXq5X4zSqSI1/RBpCiccJEClwo9q+VWUaEKjpEV74pBSslw +gbQ6mihOby3h40CSxFXz3WSI9vFmA38LScS40Qf1NZ21iqRtXQP5G4x93M9pcZLL +DMRhDBAuYYItE91QbONJqAmf0cBII1c9tQhrSCY96pTPbmFmKtX5kb3Whp85Ih7F +KEafNIUCggEBALMnoIDZmjyz0fFeX3wyLotu9kY+n6jEj56dvE6bsy694grxR4Cf +w4lybPeJAX0LjPBnqK5p9bn0VheEx0rYVVPrLUVCbmNo3+wtN6wiaAcWRnAvNtt7 +MRtWkFwc/W2U1GiNeiMLPm8guT1KpFhxiva/igsQic2QYwYNh0o8FzNvtIEtUajm +9+Uw+zCqVON2tUUT5JabVa9JDfrSamAZZZgRdh/KI1sD8BDrWWUsCVojoiOhBnTr +z5730ND4oYudjIc0XF0kY3krxqc6M/Ry+vZt1fW0qhxcpHrsr4cQB1ZgRiELL+1f +g5FyNfBs5HIofRRkYMqtE1FEjRQZcAQ76mECggEAaOUtM9BZuV9gEwmG4hmFfeXq +vJOMvlsDkRRbLuDQ1B8Vw3v7lt1+K+KfBt96MoQe08MyXM7sIMB+hn+zakNaM2W6 +UzTnAPQQAo+wELqj6U3DrV7zw7I1hZTA9G7qxMAQBEmk3u2q4/zWDAcyAx3D9JVj +L3G14pYf0drFLChnknVTPRaF0Q5upLYzCPLMa9w0FLKy6fkfdWdpzyjvW7+JEeFY +koA98hrottqJB2CcqehQDSCUHKKbd5U15y1NV1BQloaPJLwpPAVTkBszQSHanltN +l9POJBJlfQ1eWL88wHdKiLbtOg6PTfAmfghIRxakjHvxBgFO1/xG6Lxm7QwUDQ== +-----END RSA PRIVATE KEY----- diff --git a/testdata/x509/client2_cert.pem b/testdata/x509/client2_cert.pem new file mode 100644 index 0000000..2cffdea --- /dev/null +++ b/testdata/x509/client2_cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFcTCCA1mgAwIBAgICA+gwDQYJKoZIhvcNAQELBQAwUDELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV +BAMMDnRlc3QtY2xpZW50X2NhMB4XDTIxMTIyMzE4NDI1MloXDTMxMTIyMTE4NDI1 +MlowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MjCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBALb7KLFguOBiEHR8FBI/0AFs2X3s7fN5ZCOkTf4p +s9LwAcBWm5/zUqzvZCSui+4sr3qN1b1D+Xbc4xH9+WcxfbeoA2w4d2FKKJ0qaShD +Mu4XTQfj9B7g5GZ+FUeP9rScqgJ+WFeOM6QoAgRrFAS0AMP21TpDue4AVKmD1trX +z3f1DaRBtcUa4zlk2J0GBQDPyPB4worxhZ0IW+OLz2RIl8AWJBDFKMgscxEx239t +GTOY9H6hPI7Py2koknj7LBNc84lf2PVFw5OytQYglmtkFQqntyVxtETAOL4pFOjj +Zw6MAnQBGLS6nhiXG1LkZuvWJn1T5ewhci4qNVpv/8LtrPc2Fv7jb86I5XJvdOGv +hYC6IwS9Psg9oCYaIzyandoSj8SEwXaQuD98ROBUs1raasLedg3d4xNeZCRRmnzE +me+IpHm/wS4hTMxQJHYVewNB68fl6FoyRAqoXNy5yi8uMJKbjqb9E7niQCQRO9vQ +eX0TrB23uoUXbdTz95uMiw8yy/xz11/h59TxN9cOiqDf74tymgH0jTwO8eg/bzKU +zTXxTANcfGDvS1vR+yaZDxNbZ/3A4+NzNF0M/Z7AHgEzUcx4yu2shrXXr3dhNpzb +crk9yoCnEAiP1i54euqaOOrp6O34cRCCBE18j+oEEIIfYdMRXorCPxHRzdHKlJ5K +OfRZAgMBAAGjVzBVMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFCIxr8jA3srbonwd +2AoxV5teDkuzMA4GA1UdDwEB/wQEAwIF4DAWBgNVHSUBAf8EDDAKBggrBgEFBQcD +AjANBgkqhkiG9w0BAQsFAAOCAgEAT4hUMxXeg6cqFyyg8TeStBI3fWtmVKahlsba +Nh1KlZe5ZVTwWKh6ULn2zvSqy0t28wpER8Ky2a/yBxsssvPKGQBUgUUmUOSy0Zzk +ICU+pDJrVtZ02vOPlrx23SpnE3EFs3yXMGO5B0RGScHG62YjHyBDPJH3Gun4CB2W +PpthtYIX2FN5g15T3r6UZy62w7SjUEu5z5Ke7IOiAcnXNOXtozv4J9v3bt2Kg7WB +YS80r6b6cyOhO57jobdRBcdsWWogBXBn+ciaL8z3gLNWPs6YooA/6/95Vq0uV8t3 +xfq0XH1dbcdZOnalSwNEyOgLKxQ/yggOd9ridk9e5cGBBIfw1v1N1qDkOWjcdEoR +qjAjR4pgUa+d92/HNLYG8SqVGqACjUUQM6tigw6tHUbeqpk1iI7vT8Cl6Er6bEYE +tMTWEcWAI7GsqJXl2SJPMsObjBg34aZJnU+xxedMDF+OYZXzYeYk2De7uhXUi3iu +46alockzYqOdN6vE99Y7757C1X3N62PnEUhZN0Ri9D7i1yjQ9t0CCucam6hcqcEH +fcDIsXTQz0l97iztkPhcLd3kzAg2pXopwuHkhd3Ih5So5/m1V0rjHVVtrbSkN8/u +JlHy0/tNsJ1OaJKRqd665M9IhaRrc9KP0lzHoA1ZpUsRKo/Be8gNUaw9EP1CMDny +kKizg2s= +-----END CERTIFICATE----- diff --git a/testdata/x509/client2_key.pem b/testdata/x509/client2_key.pem new file mode 100644 index 0000000..a9563fb --- /dev/null +++ b/testdata/x509/client2_key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEAtvsosWC44GIQdHwUEj/QAWzZfezt83lkI6RN/imz0vABwFab +n/NSrO9kJK6L7iyveo3VvUP5dtzjEf35ZzF9t6gDbDh3YUoonSppKEMy7hdNB+P0 +HuDkZn4VR4/2tJyqAn5YV44zpCgCBGsUBLQAw/bVOkO57gBUqYPW2tfPd/UNpEG1 +xRrjOWTYnQYFAM/I8HjCivGFnQhb44vPZEiXwBYkEMUoyCxzETHbf20ZM5j0fqE8 +js/LaSiSePssE1zziV/Y9UXDk7K1BiCWa2QVCqe3JXG0RMA4vikU6ONnDowCdAEY +tLqeGJcbUuRm69YmfVPl7CFyLio1Wm//wu2s9zYW/uNvzojlcm904a+FgLojBL0+ +yD2gJhojPJqd2hKPxITBdpC4P3xE4FSzWtpqwt52Dd3jE15kJFGafMSZ74ikeb/B +LiFMzFAkdhV7A0Hrx+XoWjJECqhc3LnKLy4wkpuOpv0TueJAJBE729B5fROsHbe6 +hRdt1PP3m4yLDzLL/HPXX+Hn1PE31w6KoN/vi3KaAfSNPA7x6D9vMpTNNfFMA1x8 +YO9LW9H7JpkPE1tn/cDj43M0XQz9nsAeATNRzHjK7ayGtdevd2E2nNtyuT3KgKcQ +CI/WLnh66po46uno7fhxEIIETXyP6gQQgh9h0xFeisI/EdHN0cqUnko59FkCAwEA +AQKCAgEAtP73H42nEfyufiqFyA9q9x3ufMsyDFYVIdRSeYhSoeJaOSDyS2NqcjlR ++57UN0HoSfemZtKoHlUcHx3z54li65m72P55x7iNN/lNj0/5Pt25ioaHYUvfYSpy +bhkPVVRqLpE/XUwB9OzGIgyw/n33C+BKxplbfvrAw/TvQAWc6PFzDvkYjeGsxYbl +ZV0g8c6W2pb5CGsjWVN9YTVYbcAIqy67egMr9eVR5L5GemM2PH2dyuw+dJ1CfcBu +MlFxJa4aD9bJSsQ5Uw3AVlFBuPSEg8emN9mjESZ6ek80qbDWreL8QjcbcxntbDF8 +C6B11e48oFeu5MWopdWGdPC4Mt7a6Pjjy/ESGHcKqiDPP0VdcEgKpmowXI2CtXfz +k9bbIAoveMgFThX4eb/d5DzYXID9MkSd3GdZXMa2Z2xqX3/S4dujWKda0VlN61vd +3sX0Xd6Wno91vobjFx3tqhqumKpZ/1MjNDqzB6v0lRzxaiFPT5/h6hTIzGKslzvJ +H9bTUyoocXo6Xuskw5FHcM3VFriJljfFOi09eqVvldSvaBosYc6MIRuw8zuGuSon +q9ZBIYDgdnfuXhMh2cohEUOHoi692FgGsC651rl/bgHncx3q7IS995vz8NzmZF2V +dpN9q4v6nwBuePQs23s2MAEF5REyeOR/eA7gWbtGASnZMfmyk6kCggEBAOMGYgAZ +JMr3dY0bZsp2hdS2quHau2mUIEvV4FMvu/FLLiFq0JMHNe3vlmZrTLR/JW9TfK00 +ymPXEc/v01raJDeG2Y7I0086v65tDmdHyEE30MLEYNww2XsPqZIoacJNEvr/Jmki +O/nSaUszxJ4ygOGA6u7oJi1YJ+l2Oe8kQ6UMxoDHSDO0Y2Fjhg+TKCF0+a0Y9ddP +Q8k0B3MOXUcUuGk5ZtnXXJbDBq2Fvpf/pGsCp9twESyu/nbGSGKAClH3hfxXNU/C +jBUbX9Jyxgw5ZqWqRIt7O5NBliav3MClKKKVBYWQiju0SjV4GC2s7kF3lfohFq6R +ltGgn0pxaXsLqbMCggEBAM5Vubx62pQ9O36FPCp4yLCzMeGuuTA0Wq8LoC1OtBvq +OtHjKZdmx+Pe4W224iyceK1hEYNd18Byv1w/FSJPR5U9W+jk/GYTQ1WlMosAeGYG +fNvuLCJUDxO1cDimRIuCiQeAeYiAgCmyfmsdaUEiwMYsI22ITlSeKavIULiZDvc6 +JUQfDgfsmmD9ZtxVhwyGuLgqnHEOxXv4Cti511/iYbwM1NMB1tvmuDpjZAwpQMpl +/Fq4N/cNCe91/sAF/a3VXMZlxXey1kGmXLlCPFdzGGMGelI0v8cbB+dJ11NnZDC7 +EZPknj4jiEHVkN7/jl+WVk4zhfr8l85xh5Q+nP8/C8MCggEAPNUkA3S5WC2w8Qur +oorZ16LO7VAoMeVANjHsNz4uNTz48nllxFAFUmmFupH77s23ITqUyPDBXrlti3Nv +BgQ3+i0HNOx5Oty6KioM1v30Gg2zwczPS5FHZWNQA9sSY781W85s43UJ7ypDjqQj +hmRwBnz99uB8AmCB6VwFsB/ehGaE9lLv9PLcQmdhr+C1uylWEd0DWxthRZPMfzcV +JYvW0lNQTQUZSUifDHYvGRmmXApNIk7IO1n006zUDpjSqx4RaAmSPnoaATnhlkms +6e+joraaQWnXD+FeM6WiGHjpB4+4+A5ADDmGPQeeKvcQrLg3ltuw8TwP1sIcjN0Q +76izYwKCAQEAlYaQPCN3pTeelrhs+oZfQZYKjvb0oxc9pF6zbEH9ycD7cUDC0kIc +l2jcSore6t9VoKeYbm+iO4esX2gjo6J6SI+XvHW85ygMgtNdhlgH6D/JWgQGnbX2 +2xyAP71WLReiv/n9mMsulYkRjgRZU2eg9bvkzKqbwTyBDEj1HmFk9AqCGRS8MUfo +NGNOmFuuq4gx8tyGVHQU7xq4mYhLqOPAWeuei29oyiEv3rhKN3npxwMTVpbrj7A2 +Q/9pZrSwurnFKs1zxaOnGxo5VdPHMMRqptB58nrhg6N2HcloLrvdYmcefOOPPY64 +XqUrAD+IaILk9nTmIhXM2UFytB6P3XVNywKCAQBsOZZ7Bk3LEZHpqqlqOy7U9jjI +39tir93AEIf46i2Rn0YgynuTpsh458E4LEH88ZXJCDdfOtFPTEqa0wnm1DHhpai2 +qyiNeXWFpmhbsEgLR4RfiASVyl1W4febZT+JpcVkYtkMwro6u6Owy8L34SO+rPCb +X2IyPqQ1+lj/9ZvXU9AGaFgZNQ9bui3sK3ifvNYLGbPTBM939hNdOoI5mW44eEHQ +ZDBKjiMNnkWlNnFJk2DEyGVIQak7pVgSygX+RkMAP/OjDPNO3DyS81WXZKuxOlda +rWV0liu6hYAAz4Bq881oXzTviG24BgUjNJVq0qYtIzrsbW7fDYirkn5ap/7k +-----END RSA PRIVATE KEY----- diff --git a/testdata/x509/client_ca_cert.pem b/testdata/x509/client_ca_cert.pem new file mode 100644 index 0000000..026a4e4 --- /dev/null +++ b/testdata/x509/client_ca_cert.pem @@ -0,0 +1,35 @@ +-----BEGIN CERTIFICATE----- +MIIGAjCCA+qgAwIBAgIUZzkKhtgm6Y3RaksChHMIJFKV+U4wDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFzAVBgNVBAMMDnRlc3QtY2xpZW50X2NhMB4XDTIxMTIyMzE4 +NDI1MFoXDTMxMTIyMTE4NDI1MFowUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNB +MQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNVBAMMDnRlc3QtY2xp +ZW50X2NhMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1Rk7zsuwXn8r +KMHk+gmvaftFmlY+NHs1mKJPzyCGFnablnJtHU4hDpSvvNitoQ0OcurOo9V9ALlA +U2uw/1q6Yhg1Am4cXwSWHG0/GwCQAdPTVb7W1MiAd/IB5bx9xrwfjrpGLjVLS3/y +nOKP+kl1bf6WAcLEPClvH+kSG8xMwvg58ot7ipWQcWBTSuZLaz89d2yfxpvtwrvS +YDemY6f8Tkxil+kDjb2Jo/zdRDz8eIEOs1PcdztrdWWeQaYJVX6aEOHCfdVNOHw3 +jNQKyVREUgXjr/pkwo9fTnZjQdBUhZIo7NuPPG25t5qZK3dUDuLcVRQ5Vt0/45pZ +/HkZDCkxmSynZWz2gPClOHVPOG8Eqi0Mbd3XxQSsd1Go667oFotLvTuynbYhdh4s +xAJWXbFV26HgDXI5wXueXrs1n0stUlbD6KahfeoYBu+idX7gB4RftqhqlbIazu3y +hj22k8cMQEPkLhzmUwRt64juLA0+FRG0Hfr8vdZD+f91Qbv86Qw3c1/lckQIOlyI +MerljNbCbHJm9KOZGf1zizwvMVtVzuVtr6RY+Loov4gzhJ5kNSk/YDMQC42c2Yhz +Lr5y9EGe/cL8QXdKfjKNeJjCbzxTTFiVBq5XRKUgjz6ga+F7KGO7ayMBrexZ7+ap +z7ydlUYS+xp43hqdisAGmUMJdDVlHCMCAwEAAaOB0zCB0DAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBTq92tDG5TfVvTqbu1bA593K6aAwjCBjQYDVR0jBIGFMIGC +gBTq92tDG5TfVvTqbu1bA593K6aAwqFUpFIwUDELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNVBAMMDnRl +c3QtY2xpZW50X2NhghRnOQqG2CbpjdFqSwKEcwgkUpX5TjAOBgNVHQ8BAf8EBAMC +AgQwDQYJKoZIhvcNAQELBQADggIBAMHOXRUUq5vf9G2NvnAR1lb0fTKx4/6B9rhU +Nli9uIoWGQyMu8icEMistUp4AdHWdhutKX9NS0Fe3e5ef6qIYCng0gVBE3fTHJd4 +V8MhGtyaK0K/gpTrJdClwK/litRIEjCFwNYEK8vtuqNjR82d8IuFjnbinb+IGCH0 +sLRGvvZch+dwM5N9BVRq20M2FZhyI+fWZmt1ZiBwnfy3xM+enD2I+/LOUFoxAmGS +m2vnS+ULhq7fLaK6vgyUIGqRDQMxYEql9QGzRIspV9vVhRuOCmowlJbgCv++eOUG +FvjlAPlQRGJ+ShpXO5n2pEkdjIJOrLf4kyviLDHffIl5I80fRWzv7GJ1HP+Bb9qO +LZGaiO3SelPhvJGTSV5uSZpgkFsBbgdbbGI60W2QQIHEwG0HdjnNk17+TmVEUoCj +rWK/Kzw5py1Egtibju4CiJ8uIKeew+2pfdnnyHoCVwCfdACc4dwRpet6fQvkRcru +5PR5MzZqUI2+bjg/hJrHj7SVpxpjcr3OZdh05T+heCVuPp+9mHBmcxbeA8rkMZAq +vILLwgwEriSbKy9Y1GLs2oaPNaWEpN9Q6kZPUwtwlzjHG3OOtldeXPpMVpg6Sb0y +3NnRfvfV/g2gm68S21j6qhGM2aeQCdCu5insqnR8GS5/stmuyCNnlst24JBneE0i +louEQ0EV +-----END CERTIFICATE----- diff --git a/testdata/x509/client_ca_key.pem b/testdata/x509/client_ca_key.pem new file mode 100644 index 0000000..750e20b --- /dev/null +++ b/testdata/x509/client_ca_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQDVGTvOy7Befyso +weT6Ca9p+0WaVj40ezWYok/PIIYWdpuWcm0dTiEOlK+82K2hDQ5y6s6j1X0AuUBT +a7D/WrpiGDUCbhxfBJYcbT8bAJAB09NVvtbUyIB38gHlvH3GvB+OukYuNUtLf/Kc +4o/6SXVt/pYBwsQ8KW8f6RIbzEzC+Dnyi3uKlZBxYFNK5ktrPz13bJ/Gm+3Cu9Jg +N6Zjp/xOTGKX6QONvYmj/N1EPPx4gQ6zU9x3O2t1ZZ5BpglVfpoQ4cJ91U04fDeM +1ArJVERSBeOv+mTCj19OdmNB0FSFkijs2488bbm3mpkrd1QO4txVFDlW3T/jmln8 +eRkMKTGZLKdlbPaA8KU4dU84bwSqLQxt3dfFBKx3UajrrugWi0u9O7KdtiF2HizE +AlZdsVXboeANcjnBe55euzWfSy1SVsPopqF96hgG76J1fuAHhF+2qGqVshrO7fKG +PbaTxwxAQ+QuHOZTBG3riO4sDT4VEbQd+vy91kP5/3VBu/zpDDdzX+VyRAg6XIgx +6uWM1sJscmb0o5kZ/XOLPC8xW1XO5W2vpFj4uii/iDOEnmQ1KT9gMxALjZzZiHMu +vnL0QZ79wvxBd0p+Mo14mMJvPFNMWJUGrldEpSCPPqBr4XsoY7trIwGt7Fnv5qnP +vJ2VRhL7GnjeGp2KwAaZQwl0NWUcIwIDAQABAoICAQCgH7bmG/4p84qdtJx3GaH6 +k/noD9fsHYzXZVds/zZiWLtuoArHk3aZezZWQ8asFqB9z1x4lSm5ynnAdVJpfmZA +4Ymrisu8xjh5ocliY9jR1radXqoU95g5CNtOIoWsOJ3J5MRpYlhyofDO3Btt6ZbY +kQ1sw0orHsNGih62Tpx7gIQicZbiOqJv3v6XcFbJfpqUS0X/uhk9U16wOADKL2cR ++qm3Fjs6XWq4k4A8D0tyzR8btu8ZlMeZTkNNdxLacCgaeVlorke5IvWm14pHYA96 +Rryg9hiSbaMi1SieQonQWFRyLkUCFj0P7pYbqC28hdEkCO9RCy0/vDLT2LbugWGn +JBdPIQqRYggGnEdRocvflx6f2Xdid9I4zrI2XWnorbypqVIdmhVivCCWK8PNqKE2 +YcRg8TRQHvyOXoR55Sodrxp6KycUc65nduGe5jsyjA9hlQ0Jfxhr4gv1LuytnVCx +Z2q2PFF/cznrSLlU8uBT9Lb2gGQXRyI/rxp6g6zwnTKLvMXsQqBrt2hzlE2vkdiz +I8EcLp99IT4CwSJAyGFdR/2ZmXg2Hy20GiGc5RisMIsXvj2gVt26XHrbb+LnYHMq +0+5d5QoMnTMZC+JczoDiw64vQlzJGcM1VWFDOMn9g7UALgofCQv9/nZrWLjw5hIB +FCli4JhtwjNUP5Gz2sqx4QKCAQEA/zY1Op8i0j2xaxeaysZg6midIKyvS0a+E+CJ +qfNE0qmwCEmG/T67+IvKIwXqfBGBrBntg3De4rTDxhTVL4S4Ue89WYzB/sf9J52e +6HEpBCujRJcdb8ouxSfDkpkMYXsVsVTIjckbQl731cm4qk7L8DS1GuE0oZs9I3kx +iQSzJ1+GzRotnzO6a11NU5n5N7NM+97x9z/BHFmd3fUOlUkYdpr67PVBNKRaj46k +Ifs0Og7cZNh2JCzhVOOYrf/x9DybjCJnPHLVuNMqYOHTNI/LgpFytM26+Lnmu1X3 +mcohVacygr55oZaCC0dz6CijEOpX9SL9sUZJb/tJ01Sxv/pgMQKCAQEA1cG6Oa7h +1Hl1f6Qqg9qiWNTHur6dBOw4lt/ej8vexB6y9c85WoMfXUBFpiQta9Nt+XCrC/UU +wY0EqdQir+Ydwg92ddX+1eKb7NmNLi/moUF+s0V+9uPvgGcz6xVlSMQKTgsYxZnZ +CE8ZSBTSD3dYyIadGQHaFoP4PsABzGfzYjWnQpvk4SZf055Qs0Gt6vIBlbs7R/O/ +wPajzFYf/o0mAaPAAdPpuK6C3Q1J4Gp5LKkHtzY79XFl336uXQV3AwxU29sAkmVS +/COFl772Ev55P5nV8NsEQaChoyuNHO11YQtZEyh7zTwx+R4SfnTivffVQNfusnIa +gKuj2Eoq24XgkwKCAQAeHRJY0XA1aIwnu8hLBu9mmWN4+IdSlY1WIRd9UzQau2UH +BU4FUcKySCRYz5jkfNhVK1YIPWg/Td8P32NsUPfCyzzs9Rvq6UQoyYN3n+qcEF4a +eM5DY5LzNobwJFj+o5xiqUNk34b05OnPcxb0GYoc1MtN2abxLrUfG2zJ4yEUk0P/ +rYgWke78Pi0ioTdz6Bc8XQkmCILLypNDHmhTGyXk0NKs5R+Fi6MX71fUnqSB+UDu +MVB3YkhQUO6yEVJGZGRiO6j8y/wF6/zDI8JdIF5+EJV9Wg0mziC4mCM4JU6bobfn +D3ygoXbEx/CYQztCgrRQO4m9wjJmITuL0SGMKonxAoIBAQCCelJ2S23GCK3UUB0z +hw16M8gHEbs++gJA9j4ggE1mYWbT7L4RpeBLR6Q8GfEv1EtY65E9J0iYLMAf+kGC +JXEct9uTaiC35i9PkCxBeTPKUvRH8a/ifJgBRP3IDbNZi3DO2q8wTwzPqZjBCxR+ +JFepb6INVbgN7lhl1UZDw2ApHp8OZaJ8XLQ5tHWGNh03QKn+/97buMnfu62YWSoG +c5ozfgUCGJyeAsgWrrndpqB4xmTTTOOkmqeYmPdOCLvwvGJAIZpjwj25cuVlD0ed +qH/SdtDEyKv8c1S3CSqF8dyodAjXTOrlCE1oxxZ64lZVpyYhAq3NdyD+Uccdi4hF +n57JAoIBAQDFuv2cmOl34qQz1vd+R0axxNSQEwYC9wug9WG8PEARRmk6vCIMy/AZ +XnHXZ4aV9ds9q0J4hGrx0C1vjGHSpBFR+kulI+KcIITtHLPTAgE7e1UXGATVz0+B +ES3qvzJ1eXhl7hrrFfYUdmPok7pJUhf37qqhKajcRfVtHccaB6J7v/sbAMCIXP1B +ij7EESZgM+NLwOQ/iAM2Bpuphn+gxdV2oqgorx3kLymzffhmn0oq6qfn818DH5ps +sPgi2bndSxG9jNtpCIPPC9ltMNwWxuB+3f+wd2pKIjBulJ9tb+72s/Vb4v7EOmJ/ +c/xqN5lRsGXGduw76PipTrLpy3/LkZDL +-----END PRIVATE KEY----- diff --git a/testdata/x509/client_with_spiffe_cert.pem b/testdata/x509/client_with_spiffe_cert.pem new file mode 100644 index 0000000..b982fcb --- /dev/null +++ b/testdata/x509/client_with_spiffe_cert.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFxzCCA6+gAwIBAgICA+gwDQYJKoZIhvcNAQELBQAwUDELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV +BAMMDnRlc3QtY2xpZW50X2NhMB4XDTIzMDUyMjA1MDA1NVoXDTMzMDUxOTA1MDA1 +NVowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBANXyLXGYzQFwLGwjzkeuo/y41voDH1Y9J+ee4qJU +OFuMKKXx5ai7n7dik4//J12OqJbbr416cFkKmcojwwbAdncXMV58EF82Bt8QRov0 +Vtoio/wxlyRlxDlVYwr56W+0EVP9Q+kzA/dTnMgOQYIeSix96CUQRy8XDu1YX3rk +fiUkND9xxuQw8OXi3LXguv/lilLVC/lXiXwa0RWEgMZZU2S1/lAElAG3aZuuWULG +K+PpKPuqkcptbUPCvNN1eUs9/D82aoFuqRCmpTC+7bUO+SJSggpUHcgTbXT9i6OO +9eR0ijcaQjtb0Y6ro+Cv60YOnlGC8It3KoY2SxioyqdceRUohqs4T4hjBEckzz11 +AC0Pj0Gp4NJPcOY68EjhD5rvncn76RRr3z2XZpd+2Nz+Fldxk/aaejfdgqs9lo1g +C+aP+nk9oqSpFAc+rpHsblLZehUur/FHhenn1pYWqkSJsAG0sFW4sDHATRIfva3c +HNHB5kBzruGymywBGO0xOw7+s5XzPiNnbXT5FBY1rKG7RwlqdtDh6LWJRHmEblWV +tPHNiY+rrStv0rN7Hk/YKcSXd5JiTjk3GXjO1YJJVEraEWHlxzdGy+xu/m0iJLed +pxZwuxxdZ/Q2+Ht+X9pO2DsW8BQFbddCwbooxKygwSlmHCN1gRSWqWMZY5nzsxxY +tic9AgMBAAGjgawwgakwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUyiXne0d3g9zN +gjhCdl2d9ykxIfgwDgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUF +BwMCMDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3Jr +bG9hZC8xMB8GA1UdIwQYMBaAFOr3a0MblN9W9Opu7VsDn3crpoDCMA0GCSqGSIb3 +DQEBCwUAA4ICAQB3pt3mLXDDcReko9eTFahkNyU2zGP7CSi1RcgfP1aJDLBTjePb +JUhoY14tSpOGSliXWscXbNveW+Yk+tB411r8SJuXIkaYZM2BJQDWFzL7aLfAQSx5 +rf8tHVyKO89uBoQtgEnxZp9BFhBp/b2y5DLrZWjM6W9s21C9S9UIFjD8UwrKicNH +HGxIC6AZ6yc0x2Nsq/KW1IZ6HDueZRB4tud75wwkPVpS5fb+XqIJEBP7lgYrJjGp +aLLxV2vn1kX2/qbH31hhWVpNyPkpFsT+IbkPFLDyQoZKHbewD6M56+KBRTTENETQ +hFLgJB0HiICJ2I6cqw1UbDJMJFkcnThsuI8Wg9dxZ+OffYeZ5bnFCVIg0WUi9oMK +JDXZAqYDwBaQHyNszaYzZ5VE2Gd/K8PEDevW4RblI+vAOamIM5w1DjQHWf7n1byt +nGwnxt4IQ5vwlrdX3FDcEkhacHdcniX/FTpYrfOistPh+QpBAvA92DG1CbAf2nKY +yXLx+Ho7tUEBGioU4XvRHccwumfatf5z+JO/EvIi2yWd1tanl5J3o/sSs9ixJfx4 +aSuM+zAwf8EM+YGqYMCZ896+T6/r7NAg+YIDYu1K5b5QqYyPanqNqUf9VTR4oQ4v ++jdb5PkujXbjENvkAhNbUyUbQJ+IU0KHm3/sdhRPN5tuc9C+BTSQvlmKkw== +-----END CERTIFICATE----- diff --git a/testdata/x509/client_with_spiffe_key.pem b/testdata/x509/client_with_spiffe_key.pem new file mode 100644 index 0000000..6adcdc3 --- /dev/null +++ b/testdata/x509/client_with_spiffe_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDV8i1xmM0BcCxs +I85HrqP8uNb6Ax9WPSfnnuKiVDhbjCil8eWou5+3YpOP/yddjqiW26+NenBZCpnK +I8MGwHZ3FzFefBBfNgbfEEaL9FbaIqP8MZckZcQ5VWMK+elvtBFT/UPpMwP3U5zI +DkGCHkosfeglEEcvFw7tWF965H4lJDQ/ccbkMPDl4ty14Lr/5YpS1Qv5V4l8GtEV +hIDGWVNktf5QBJQBt2mbrllCxivj6Sj7qpHKbW1DwrzTdXlLPfw/NmqBbqkQpqUw +vu21DvkiUoIKVB3IE210/YujjvXkdIo3GkI7W9GOq6Pgr+tGDp5RgvCLdyqGNksY +qMqnXHkVKIarOE+IYwRHJM89dQAtD49BqeDST3DmOvBI4Q+a753J++kUa989l2aX +ftjc/hZXcZP2mno33YKrPZaNYAvmj/p5PaKkqRQHPq6R7G5S2XoVLq/xR4Xp59aW +FqpEibABtLBVuLAxwE0SH72t3BzRweZAc67hspssARjtMTsO/rOV8z4jZ210+RQW +Nayhu0cJanbQ4ei1iUR5hG5VlbTxzYmPq60rb9Kzex5P2CnEl3eSYk45Nxl4ztWC +SVRK2hFh5cc3Rsvsbv5tIiS3nacWcLscXWf0Nvh7fl/aTtg7FvAUBW3XQsG6KMSs +oMEpZhwjdYEUlqljGWOZ87McWLYnPQIDAQABAoICAAY5tM7QZm67R9+hrxfw4f6x +ljfSLXBB+U5JFkko8DbhvjEN9+PQCda5PJf9EbUsOIWjQNl6DZjZsR3rqnog0ZGn +kB0yuPs8RDjrbVIXOwu/5EurWb2KZIpSjL4+BWflsndiMD6x6FSjDzXXDFrv7LKc +u0uQzLF3F00avDSEP5NvGUIbWnE7Z1cZIdj9ABQAJuVAI8gOnwaIdTsODv02jjGp +BgxoBbKDFsSb7yb9QzuvhizEitd8FajaGsqAaZYh6JwiRjkb8jl0z+u6MoqJNACm +q/gG+JLg1deIpS6OM2OBbKAr2G+HvXJMVklsdQkl1b+DcuJsBkW/gLHn/3WdQDyq +t9sB8XrUx3S5dy6oroj9tcrwwlpUPbx3/37BX7OEn/NlIWZojI62hGexoFaggu3O +Jg0JJYH8THlQqs9G/oXmRTQKngse2FLEIPePie9vIIvokiQtG4T6miOK+6QmxYZq +H+KPT8AQG8j7AEexo4is1mEayapmTxftIYANOLFaT82BhsUOZksa698Sz7k1Cf/o +MSFn6CocGLflMEzdBqEq0wbBkdeuKUKlXG3ztXlqElN1xFdbzkaP/Tzl1ooq3kLR +0cLBCJNrHxTo1tuW4qTn+s4GLHpM4PE4YZgMehVWtyRFBb7mrSXsESw03POvulBx +65vA86DtCPm/jVuC5lQBAoIBAQD8IWDHYtQnvn/za6etc0fc7eTyq1jmoS/gh33y +eHaY6IccUN2LXCxgTJYwmfy57De58Im5AcOnkgGvw2Hw2i6+A5i4tKkyCa9awW4A +M20QOnyQpF+9uiIqGzI72skpfH20SvgTstTFtgGr3UBOqTfcApL+1X4guvGnY+Cx +uHUAPzbis9G3CNOWb4iiLhUcBnTDZyB3MPM4S1U8E5JLFo86+va6gbqdBP4ac+KH +08XDk/z6ohp9p796o6IiBQyZEsVaYLCrzjSOXeFfE5Fyj2z53oGlws+/PdhXKo02 +3++zRESiLVuGbCmAN17nKwDbZu9kFfGNP2WdwhJt9Yey91I9AoIBAQDZOsXWNkyP +zoDcSrvJznMPFcwQCbMNyU7A+axXpAsxDqn16AQj5/t1PWqufjRSdC7gVUWFcQ2K +ldUHkNyGtqHVCcNpqsMZJT51NlgTOl1A3GLnmm+tAiMCcr7aySeNnlj08fW278Ek +fnxpgUqGtXjTFpArULSFdZulXNPAP85ZDBburJtdhMfiT3GyQ1iRZcXkzsUVzNU1 +nGGk0jtCodlzQKiz3/aHO63G0GAjtdPuXpzGm7nBJSgLD0GabkCdkHDFULOaraYy +t1zsCsg7tQWa4KGRDNkcJKzoz3zf1sI4g87UJceGoXdB+mfluyKtnFhqjFalFW8Y +14Yb8YYdYHkBAoIBAC1pZaED7+poqWsSjNT02pC0WHRM4GpJxfHO9aRihhnsZ8l1 +1zFunJ+Lq9F9KsPiA/d9l5C2/KKF7b/WlSFoatrWkv9RqtfUXr0d8c4fdRljL2Rt +9sCZceXbmCSnt2u9fHaouh3yK9ige5SU+Swx1lnOLOOxWFJU2Ymot6PK8Wfl+uDC +OpeZA2MpG5b6bdrqXsWDIZnWOzh8eRGlBMh5e7rH0QCutQnrCEmDbd3BCvG7Cemq +oNLZD+fq6Rzvg+FePCWXHLsVHOo3how1XhEgPCSVKwzMFdcAMKMiiuTDWM0VEreT +K9T+TktFrdY9LJ5X3+5K9YLXVFohxmf/vT1CxpECggEBAIfegeVU+xgrYl/nAoPb +9A1oZcVWO78QvYhn4YrDmRhrApVDNGu86oPPEU3otBMqhjNcQmqPZpfa1W6xBa3g +x2H3hFkwLG0q5WDsx7PnGnK6JcaUyurcXkdmu8ceb/XdJ+i0+ioc1aJc1rYq3xFY +qiTlhPECvpaHE/4fDHa/sfHyZNmN7nNU3KzJYeTMyLXQgTF2vsC+6FBq6ovrzpMD +pn224I35NDorcqrapHdRgCgk10xGFK4g7mXUegT8lr+2m0JfEqdZm403MRCWQd1O +gR35CDUwYw9+RQQs2v8qVTqB/riklKK5lV0YISoInU0XcBncg0koGd/g1gneTDNN +pwECggEBAM4sDCCPplzbyd0yXLGo9P3RYIsNFnKnIm0YGRPrevBaiux3Qhr7Whpi +eV04BJ7Q58Z2WFzPFMhdXU45y4c6jIbmikdplEW1TASgXxOTvTfhg8P8ljdLPx+R +3CvQi4BPkJ3ZtYrHLKXKF/9aseyHLlSzuNUAJ6H0YxVi0tmzCFG82SWcFOzhR2Ec +cWDptGTRt9YY+Eo5rhPYbX/s8fCcW2u9QGnRnX35F8vJOp8Q7eCONIaN6faV4Yos +1wk6WXjZfDgEdjxmrnqXrgxdup82uD4Q1agmkxAjPl/9frLtHMW87Y0OixJb/Sve +eSCMKThlBQ57WubHTi2TbFBVKph/rP0= +-----END PRIVATE KEY----- diff --git a/testdata/x509/client_with_spiffe_openssl.cnf b/testdata/x509/client_with_spiffe_openssl.cnf new file mode 100644 index 0000000..b761983 --- /dev/null +++ b/testdata/x509/client_with_spiffe_openssl.cnf @@ -0,0 +1,17 @@ +[req] +distinguished_name = req_distinguished_name +attributes = req_attributes + +[req_distinguished_name] + +[req_attributes] + +[test_client] +basicConstraints = critical,CA:FALSE +subjectKeyIdentifier = hash +keyUsage = critical,nonRepudiation,digitalSignature,keyEncipherment +extendedKeyUsage = critical,clientAuth +subjectAltName = @alt_names + +[alt_names] +URI = spiffe://foo.bar.com/client/workload/1 diff --git a/testdata/x509/create.sh b/testdata/x509/create.sh new file mode 100755 index 0000000..378bd10 --- /dev/null +++ b/testdata/x509/create.sh @@ -0,0 +1,151 @@ +#!/bin/bash + +# Create the server CA certs. +openssl req -x509 \ + -newkey rsa:4096 \ + -nodes \ + -days 3650 \ + -keyout server_ca_key.pem \ + -out server_ca_cert.pem \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-server_ca/ \ + -config ./openssl.cnf \ + -extensions test_ca \ + -sha256 + +# Create the client CA certs. +openssl req -x509 \ + -newkey rsa:4096 \ + -nodes \ + -days 3650 \ + -keyout client_ca_key.pem \ + -out client_ca_cert.pem \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client_ca/ \ + -config ./openssl.cnf \ + -extensions test_ca \ + -sha256 + +# Generate two server certs. +openssl genrsa -out server1_key.pem 4096 +openssl req -new \ + -key server1_key.pem \ + -days 3650 \ + -out server1_csr.pem \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-server1/ \ + -config ./openssl.cnf \ + -reqexts test_server +openssl x509 -req \ + -in server1_csr.pem \ + -CAkey server_ca_key.pem \ + -CA server_ca_cert.pem \ + -days 3650 \ + -set_serial 1000 \ + -out server1_cert.pem \ + -extfile ./openssl.cnf \ + -extensions test_server \ + -sha256 +openssl verify -verbose -CAfile server_ca_cert.pem server1_cert.pem + +openssl genrsa -out server2_key.pem 4096 +openssl req -new \ + -key server2_key.pem \ + -days 3650 \ + -out server2_csr.pem \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-server2/ \ + -config ./openssl.cnf \ + -reqexts test_server +openssl x509 -req \ + -in server2_csr.pem \ + -CAkey server_ca_key.pem \ + -CA server_ca_cert.pem \ + -days 3650 \ + -set_serial 1000 \ + -out server2_cert.pem \ + -extfile ./openssl.cnf \ + -extensions test_server \ + -sha256 +openssl verify -verbose -CAfile server_ca_cert.pem server2_cert.pem + +# Generate two client certs. +openssl genrsa -out client1_key.pem 4096 +openssl req -new \ + -key client1_key.pem \ + -days 3650 \ + -out client1_csr.pem \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client1/ \ + -config ./openssl.cnf \ + -reqexts test_client +openssl x509 -req \ + -in client1_csr.pem \ + -CAkey client_ca_key.pem \ + -CA client_ca_cert.pem \ + -days 3650 \ + -set_serial 1000 \ + -out client1_cert.pem \ + -extfile ./openssl.cnf \ + -extensions test_client \ + -sha256 +openssl verify -verbose -CAfile client_ca_cert.pem client1_cert.pem + +openssl genrsa -out client2_key.pem 4096 +openssl req -new \ + -key client2_key.pem \ + -days 3650 \ + -out client2_csr.pem \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client2/ \ + -config ./openssl.cnf \ + -reqexts test_client +openssl x509 -req \ + -in client2_csr.pem \ + -CAkey client_ca_key.pem \ + -CA client_ca_cert.pem \ + -days 3650 \ + -set_serial 1000 \ + -out client2_cert.pem \ + -extfile ./openssl.cnf \ + -extensions test_client \ + -sha256 +openssl verify -verbose -CAfile client_ca_cert.pem client2_cert.pem + +# Generate a cert with SPIFFE ID. +openssl req -x509 \ + -newkey rsa:4096 \ + -keyout spiffe_key.pem \ + -out spiffe_cert.pem \ + -nodes \ + -days 3650 \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client1/ \ + -addext "subjectAltName = URI:spiffe://foo.bar.com/client/workload/1" \ + -sha256 + +# Generate a cert with SPIFFE ID and another SAN URI field(which doesn't meet SPIFFE specs). +openssl req -x509 \ + -newkey rsa:4096 \ + -keyout multiple_uri_key.pem \ + -out multiple_uri_cert.pem \ + -nodes \ + -days 3650 \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client1/ \ + -addext "subjectAltName = URI:spiffe://foo.bar.com/client/workload/1, URI:https://bar.baz.com/client" \ + -sha256 + +# Generate a cert with SPIFFE ID using client_with_spiffe_openssl.cnf +openssl req -new \ + -key client_with_spiffe_key.pem \ + -out client_with_spiffe_csr.pem \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client1/ \ + -config ./client_with_spiffe_openssl.cnf \ + -reqexts test_client +openssl x509 -req \ + -in client_with_spiffe_csr.pem \ + -CAkey client_ca_key.pem \ + -CA client_ca_cert.pem \ + -days 3650 \ + -set_serial 1000 \ + -out client_with_spiffe_cert.pem \ + -extfile ./client_with_spiffe_openssl.cnf \ + -extensions test_client \ + -sha256 +openssl verify -verbose -CAfile client_with_spiffe_cert.pem + +# Cleanup the CSRs. +rm *_csr.pem diff --git a/testdata/x509/multiple_uri_cert.pem b/testdata/x509/multiple_uri_cert.pem new file mode 100644 index 0000000..210b844 --- /dev/null +++ b/testdata/x509/multiple_uri_cert.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIFzjCCA7agAwIBAgIUA0Tqj0ezdOI2R+W4smJis+1NRucwDQYJKoZIhvcNAQEL +BQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQy +NTRaFw0zMTEyMjExODQyNTRaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEM +MAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVu +dDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCnfQMbGQzzWl17NuJQ +annAeiYFX9mcyKi0ywS1BOsNpDn8SRgeW7Ymj8EqMYIUv0VK5QDpsJKUQ4ZDBE/f +drplyXhbR0/aMpAATxP4AjnzH0pw49aIe/n3KTdBZj/KF52qO7WG2rcJ5GSVr8Z6 +H8FeP02GRt2iOAbLqV5/k52gBEEzjSJ1Be1DiRAMOQL/Wahyo1XTfvB0UeG7nIFN +OFiTdQmgS0s6OUYWns8J2jcSO8XoSKCxuNz+ZzkisX7xicRFjkweLKNLuYmS4U1x +wd19JMizaCRoK8E6NnOGVrxP/r9am5ft3QgC6AIkzZvNcylcHkkhZo5++X5qC18u +mKjPgxCOzAop/pGeiItfkjnshccLPgoPBI1W/gO6puxDaTV3HFGZdy5rtV/6MxjE +byUf69aKaBEY9d+mIw0OB8TfyqzNSoU578rTbuEqwiG2f1IDe0KMB2xkikDXGXPz +YYRVdm4bmgVmY3fjpoiM06+aSN1IbYEnmvuB98z1nf8VyRdR39jVk80udfHHdQWu +xTBMEHBtAjT6HC0tKIkeGKB5kmozPIVL+6EbI0JyuRVB6wyCrmnpv0Mw1NzSF5NM +JiYD/5ScSKhnFogYEstq8Lgj5atyrqi703bNGDVGmbTqChBG133r30WNT9JJ4rzS +KznDJhoHOgQqagkzGgHsKeAmiwIDAQABo4GjMIGgMB0GA1UdDgQWBBRHg8AR1psw +kslOTh5a2nxGAmo33zAfBgNVHSMEGDAWgBRHg8AR1pswkslOTh5a2nxGAmo33zAP +BgNVHRMBAf8EBTADAQH/ME0GA1UdEQRGMESGJnNwaWZmZTovL2Zvby5iYXIuY29t +L2NsaWVudC93b3JrbG9hZC8xhhpodHRwczovL2Jhci5iYXouY29tL2NsaWVudDAN +BgkqhkiG9w0BAQsFAAOCAgEAOkL6WsETiUWJT2lhMXEHGpLwu1Q4nETr4O51+V7t +AJJd7oGS/QRL0K6YNDgNQW6GOUZptvTEOSAO2irNohP+0+ITZAClF46ggB0pRAeD +COWjnG9h1aonMtVlnswh2xVYfg4jd+qfQZ07jN9tATn5ZBpFpcaxvcyAYc/eq6x/ +DKf7HBBWq9XWyRxZJuPD9qhyGPDzI/E2yr2ahLJFSGMRbTDivDUbw0yHbzmYnY2g +uPrVAAD4DuKsJxyZrA2/Hs7ZspBMTyUjWj7KSw64AcDvFDQgPBXDfG4CMSRH3Eh5 +J2F48ej7T6J1+PbJ81ISifGjUZH50haskBG5TKQqRX65p5LIVrDThsEM+YpfEyOB +mD2ylbxNs/X3b9fk07iS2HirfKZ0cKSINZPU+hEroasqxCcAY0E28Kzw0SdAGCGf +iZNRT0mNVgTPg7Bnrb7JhCBrm0aid0/nYFX+fqeKuS2lcdAcx6U5EgH0KnHg+9/N +NbSv+RtRiGWv5RqWF/Pk4bdHPvlzp/qiFfX9dQIOBtrFph9XUt/bEf6hZgaMKvT1 +QbQuM+rmf2ghjbqpCRP9iZUYBzOOvDZ8IeugguDvyBgrGaUSpreMzMC52B0fp2jB +Ib89u6yiKNNZzBGGE0d9y2qsju7q3IoV+eUwqbCUvGvcal+gdAfhO7Pvr3dD40z+ +g58= +-----END CERTIFICATE----- diff --git a/testdata/x509/multiple_uri_key.pem b/testdata/x509/multiple_uri_key.pem new file mode 100644 index 0000000..621c1b2 --- /dev/null +++ b/testdata/x509/multiple_uri_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQCnfQMbGQzzWl17 +NuJQannAeiYFX9mcyKi0ywS1BOsNpDn8SRgeW7Ymj8EqMYIUv0VK5QDpsJKUQ4ZD +BE/fdrplyXhbR0/aMpAATxP4AjnzH0pw49aIe/n3KTdBZj/KF52qO7WG2rcJ5GSV +r8Z6H8FeP02GRt2iOAbLqV5/k52gBEEzjSJ1Be1DiRAMOQL/Wahyo1XTfvB0UeG7 +nIFNOFiTdQmgS0s6OUYWns8J2jcSO8XoSKCxuNz+ZzkisX7xicRFjkweLKNLuYmS +4U1xwd19JMizaCRoK8E6NnOGVrxP/r9am5ft3QgC6AIkzZvNcylcHkkhZo5++X5q +C18umKjPgxCOzAop/pGeiItfkjnshccLPgoPBI1W/gO6puxDaTV3HFGZdy5rtV/6 +MxjEbyUf69aKaBEY9d+mIw0OB8TfyqzNSoU578rTbuEqwiG2f1IDe0KMB2xkikDX +GXPzYYRVdm4bmgVmY3fjpoiM06+aSN1IbYEnmvuB98z1nf8VyRdR39jVk80udfHH +dQWuxTBMEHBtAjT6HC0tKIkeGKB5kmozPIVL+6EbI0JyuRVB6wyCrmnpv0Mw1NzS +F5NMJiYD/5ScSKhnFogYEstq8Lgj5atyrqi703bNGDVGmbTqChBG133r30WNT9JJ +4rzSKznDJhoHOgQqagkzGgHsKeAmiwIDAQABAoICAHkAXOUP1QZe65hfz2LPecRv +utY5KCsX4KI05eKtee9yDR5R5GXSVidHxgLon5TDlpkEFwO9uDf7DJ2QGPBVg1aU +FirDu1HlI5nFh6SuXxVhLtOeFtil0LIaibvq1fz30MUyu/OAQaqY4X4u7lI+bOHd +E/IFcouGtIogg4/hoof/aueGeDVZIc+fzwM1kQ/Pw12G2TOhyrAOk+mJqPST15I4 +hMrUerXGuPcQpnz0tMKsgk9NYSLkbmwxQNrqps5zfGPP6PgHwbWshlKiCOQ9bfnC +QGk0vNCxg7i9q/qK4SNd5Prd3AZRoD8RRLM4A+6K23+ctbK2uA3Ny+Fq88njKlkN +jYxHPlkZ+b6nGzYxwZ4pbVYR/rpmrKdnrns1t4l/9GCOwMhDZe1jnRZaimQOoQQs +8hHMwxDisqsOjzd5ozQU3dVgxmG/n0jGjjtBIVp8usGe//AqRmZ7SVRNrglgG6FI +vqYxwCvum+DEJ4X5ONDyyKddmccGkCpj1lX6xPBtEkp7VupKx4KHW6ufteQYSkdh +U80RrCPaIKoFm1y5Jes9vOtICtvRk4PVNfLXBBycn0WR6aUQEkFHDvSHFlCnROPD +UdABH1r3bSMruz5vQrdIA/6XGHXzjHmq1WN0pKynwberFwazAxDlD/1G1ZbPD4Dg ++Z9cpyZ7Tlxj+T4hSfExAoIBAQDT8GU76CpiHL1VkIA9Khk3j6Nglq8ALQjRUH2S +ReqcAhvpUiSr6G9lt9PWGm5iWv4qpRoln+HyBDXpWOxaTR/PNi07J0NeTGxk2+in +IYNMDg9IILDGlbv5tKen4yKD9Yb7VHD73DxhDbSs2U+o49eTdfXmlQfErd15alZy +m19Xl2r2jPtYF8diPEp2m2dUamArnA8Vd3jdFAxTsfP8eNeAcCn6R3jEfIFyMm02 +X2H4iGO4Ec1ykVdiFxiemElLjm3Z7vVek88Md0KgjuAxhv5AuogRYtgMIMlhNhco +hfNgXzVtvxuDFJ3J2rlA20T0htSz6xsy9ZVUYRWkDIf0tPRJAoIBAQDKTufkXmcC +wXhkjnUMmYXqnmul6CPeYbXyfX1CZhtAlZSXtzAJ/2e7eRlqb7iFJuYfTdpkbN6i +pMrSb5wfcPtl8RRC+MERMCbe/LB4DND70QSLy/u8mWIsjbkdIT6SCi8rlNzfaL7U +cOb4uzmuXOhZHvdw8q9YTjXlT/EjKoMa2a+RggVLnICTm5sG6tsfkgiH5+DSsFCL +kKQt9Gtc66Q8HfyJW4ljK2JAOjYR7w3+bDXsodrxUQrl5maMALeBdkenc9uzwsmE +2an/R0NQToJJku+gRKVbJxN9jyMEVMygmygxfmW5qUyFZ8W2dd1Rkcr9jzNdVPlT +a6KgEGHPP5wzAoIBAQCqma3DpUTIqT227JRtp7+Yu+TVUTYZPCcG5nXOEr2hSwlQ +rTCbuIRDKtA4XhpQzdIeXbxIYQStnboP1eabYc2jLIcIQLi35WizX1lNf2qDBCZE +9xuVHt6rSEJUoD8eXbuEABraghOQREoVgO/gkVbsel2weHJCXXoTzAc+RddfWKFf +SWjhJnL2nnWKN9nbV62GLR7vNrZxrzuk+2/c4SEHYEJKFtIdx+MjG3hR9kGUn6U1 +fA8Wk+v1J4ZH02ncig/fB703nl9iN3XIbHoHJBTx4bS52gjy6klwGOxXUEvyXXFS +oCzzPNsuqwPIMzi0ZPw+v5erU4ga3fNflD60Oh0RAoIBAQCXV84MPj7rhdZNy3Bu +246eBKNdOrtSimA1poEFIiNy/jNqB+WNJR7x1VcZE7jDC2WNt40QIY2vuH3uTQZL +UxcOnPneW/76n74EhJ5zQIs6RpQTDKcm4Mvbrq3zx8HqOGovPS66hr5zaH6xRkaR +VPmQaiULvtFDy0ZwZIxtFUl81aqMvOq/NLXPNtITq6/+/x0YpnO+yZ2Hus3Hfxiu +K63yNzCLhQnTQUo/6Aw5AE/ErCju++oxKsJvWBwQ0hx1YgmakIakBK0CkF6nFSWb +NxAqgBx5FcFp3mKrRGAaxmFKKKg51me9K5SOHCKBK81ETz++zdjMElxudo/zFC5H +fzuXAoIBAQDTFfoLQ2XC0PI5x77zpse3zOvIEvsh3e3tLUhOHLoLTUqX0Nvp/7m7 +ohTHrPU4lf4ERInL4Kz7y2iI2yjRiKD+ARYHDYVx3QgttNeUFnsODnseJTnBcgFB +EWlThVuxbvEzJAZXNVtKTHmCLKFHqc9epyfdI59uD5lXQjvNp4uNEPiEE1AX0d8d +0OuFfQ6bSK1rZNv6IkRPTl+LCWuyHvZNWO1WZ4IUCg4XLr1IXtQaRIMRDxyxbHwc +6vwk0JtRxCt6wvIb/YWUK6qjjbfcfH70iDComZRTyXLGrr8w+vtmqvD43MUGIEmq +XHot6Ki4FhjC4ks5oT9m2q6TerUyIg8Z +-----END PRIVATE KEY----- diff --git a/testdata/x509/openssl.cnf b/testdata/x509/openssl.cnf new file mode 100644 index 0000000..d103421 --- /dev/null +++ b/testdata/x509/openssl.cnf @@ -0,0 +1,28 @@ +[req] +distinguished_name = req_distinguished_name +attributes = req_attributes + +[req_distinguished_name] + +[req_attributes] + +[test_ca] +basicConstraints = critical,CA:TRUE +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer:always +keyUsage = critical,keyCertSign + +[test_server] +basicConstraints = critical,CA:FALSE +subjectKeyIdentifier = hash +keyUsage = critical,digitalSignature,keyEncipherment,keyAgreement +subjectAltName = @server_alt_names + +[server_alt_names] +DNS.1 = *.test.example.com + +[test_client] +basicConstraints = critical,CA:FALSE +subjectKeyIdentifier = hash +keyUsage = critical,nonRepudiation,digitalSignature,keyEncipherment +extendedKeyUsage = critical,clientAuth diff --git a/testdata/x509/server1_cert.pem b/testdata/x509/server1_cert.pem new file mode 100644 index 0000000..ed6bc02 --- /dev/null +++ b/testdata/x509/server1_cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFeDCCA2CgAwIBAgICA+gwDQYJKoZIhvcNAQELBQAwUDELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV +BAMMDnRlc3Qtc2VydmVyX2NhMB4XDTIxMTIyMzE4NDI1MFoXDTMxMTIyMTE4NDI1 +MFowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3Qtc2VydmVyMTCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBAKtk472NGPcQhDL9U6wsYWGOachAw5XX/a7lUBh/ +yowV+qD/SRCbspeBfiNdMNoXh/LPgyePWhAhskT2XaSJZ5cYD6VpI9Q55lFnFzR7 +Q1bw7BLaD2q83BJkrUSGyDnxH+LdQc2+Gq7rj1PIpIDBaJDtdd8U9bcpP6rH+S9Q +yGQw5OniPCCsUrnx8ym/3lAhKdn2OWXLq+F1avim8AN5dQj0fvAI2kMQdswKgY2M +bs5E932WPtjwLbe80A7RtHPIrqvsdVIoaZav7g3liKekisBuJGLMtTX3hBct5an2 +eKu3Q901bEQXeMWrToekc+DnUsmQ5TwkXatWiE+/sMWg80KNyWt8rulI4ATF8go8 +7Jl8duyb/jvULXjTRdDae6w34gNZjq9jZH2qSVIiLV3Jy0GadyRVJDVyE8Lz8EiI +XkbhgbjL8fpNG8cjN+58sK3TNDuP480A/Pi/9I1BoPYTSPCD6H1KPQJwF2GZVmgj +epF328/RGjl0bfaY458RRYZafydblBpIhDsLBBRmDMkFh4SghAgOwoQBjsEZpCmF +efzzPmJybfloBdmBiqfrEXP8t3J4jBzP5+qhYZRxHik0ignOWwyDtQQSUa2JTJoE +/ET8bkO88XLL7hkAlF+eLVV8ao4oXRh5yjf1c4PvJ/Zfr80mYJYOvOlA8Me9/+A7 +jZr3AgMBAAGjXjBcMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFJmANOTd7MASs5/K +mnBYcpvzmgLcMA4GA1UdDwEB/wQEAwIDqDAdBgNVHREEFjAUghIqLnRlc3QuZXhh +bXBsZS5jb20wDQYJKoZIhvcNAQELBQADggIBAFH3XxzcRkP5053jgcyV1/L3hNY3 +gAKxdDjYSw9uk2saJXz4gjfvc8xyIsWv35QU2yzzvN3xaGfDP+qVh1fNmC/7DdUF +LJzYTJb/wm4atVM6oYzdBhu/b3NqtMPb06NFKqyEX4SSN2QjeUVUF1QgAkjmsCiE +79NOuoCO0aWcxgdKd18Wl9MLtG/PtlCMLRcPlx4FX6OYLcOeqFEtcOQXKYDWej0r +9m8JQ2DAGRSa3AOUYskN12GacEmchC86cnlMsL2AycnX1YzOBawgp4KKSur40iPg +S1+8LRjZA5Dz88+a+DXb619ckABO7v8b0AVlbkVVmaXNnhBEU4vqdfQa4BygaGGl +BG8hYYMoNBHpNDr8bBWwwl/WVpGUIMHpOnIJnG5gAGrNiAxH1/6DYFC+cXEOH5J2 +NpkJ5O3Jm9a1xtwBs3tSp8GEORqVrpIjiK+bUWQacss9nsyyO6Oo0S33javSR8AN +nJrGHBE01/QytEpJ53d3N0btZrByhiFZkh8BG4NdhXZsAaQjVy6EEHxfJBsfl8Z6 +UGX4T/TTkASNDWA4B+/nRD/BxrcSegDb8fE34GY9M0IWgQtmMIdR49bOxygzYMFK +lrh+dwGqZ9/xqJ3ro7sYphhJ+Gk5YL5lkZygF2/F9GJY3zOcKrFUalfJgaqKOaTO +6zW3ZjSyDhQoFNR1 +-----END CERTIFICATE----- diff --git a/testdata/x509/server1_key.pem b/testdata/x509/server1_key.pem new file mode 100644 index 0000000..5e5331a --- /dev/null +++ b/testdata/x509/server1_key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEAq2TjvY0Y9xCEMv1TrCxhYY5pyEDDldf9ruVQGH/KjBX6oP9J +EJuyl4F+I10w2heH8s+DJ49aECGyRPZdpIlnlxgPpWkj1DnmUWcXNHtDVvDsEtoP +arzcEmStRIbIOfEf4t1Bzb4aruuPU8ikgMFokO113xT1tyk/qsf5L1DIZDDk6eI8 +IKxSufHzKb/eUCEp2fY5Zcur4XVq+KbwA3l1CPR+8AjaQxB2zAqBjYxuzkT3fZY+ +2PAtt7zQDtG0c8iuq+x1Uihplq/uDeWIp6SKwG4kYsy1NfeEFy3lqfZ4q7dD3TVs +RBd4xatOh6Rz4OdSyZDlPCRdq1aIT7+wxaDzQo3Ja3yu6UjgBMXyCjzsmXx27Jv+ +O9QteNNF0Np7rDfiA1mOr2NkfapJUiItXcnLQZp3JFUkNXITwvPwSIheRuGBuMvx ++k0bxyM37nywrdM0O4/jzQD8+L/0jUGg9hNI8IPofUo9AnAXYZlWaCN6kXfbz9Ea +OXRt9pjjnxFFhlp/J1uUGkiEOwsEFGYMyQWHhKCECA7ChAGOwRmkKYV5/PM+YnJt ++WgF2YGKp+sRc/y3cniMHM/n6qFhlHEeKTSKCc5bDIO1BBJRrYlMmgT8RPxuQ7zx +csvuGQCUX54tVXxqjihdGHnKN/Vzg+8n9l+vzSZglg686UDwx73/4DuNmvcCAwEA +AQKCAgEAknXlUv42vjGD9pqZnMBT+uyaooANYoevBXx5ZGYXbHv/rwJXqnSSOXtz +kb651zRSfQAswGp0eOKClwG8ZbTxK6FpBV2CO4G6ugcRQkyu76Vy5m0mzXxTxvf3 +RF60zSaqq8+MwsbXwHAVC3CielBMDcSNfDNKAdmiyUqXOoKaq1tI0j/8R6NaEgGa +XCvUSr78J4CL7dwMpd4TqiXlZeKtSxi7PF0kPjjce2Hi8VV2/pbaspvoWrNrLd6Q +IIm83VA5SzsFyk40ZIs0LvXdP/yQgP3d4/uwQkyfuLsEzaeL2JkDyg0z1kAEeU35 +DlpOl3q1OP+zlCAzVw3b7+ILqeXu2KOLPBCoRTo2wWNKutxcExmA0oIfMSgyb/P9 +36bNsdmT6b/6C9ZeJpsXZWedx7bmlEYg1patfk1CgD6WMynsl+fQ34F658v/JWzY +b8JpqAG+2ov4j4EyRnwK267/6u/Yw4CvQw+8giKIDMv6W81b0GWpCBT7PkTKgUaW +1Hq46z6xZ/NttiTU6qMsgUskFheU7IuO1KHl1kpLBERio9S87ZxkjLGfvxl4+sS7 +7OdgIVsM1d50RYy4pipNplQXw998kitPvNFcSOK5vnxqW5sz4SRzVAUl5sAqJ5GC +0MPqq/I+H84BBQwwviQ9WsBuA1+YW6NxfY/mldAzAjg3/yUMwokCggEBANTyvgnw +3SjBOKzg56QXRs1L2eqQzqtDzyOn2BXlMvT3bJFdqOHROhqqD0oSSbYZuCoJAJPh +/W2bqengGSbsGwKKLHDN0yPWR12QVq3y9gbK/L6Qktjp2zDhHqNgr+4SSSl0gMMF +bz6Nzn+0SO6C3m+M6hAgfsuizIhSCxBSSLccFSIT0ZiYRzq7rN8FlirpSIhd9cQ5 +B3q41lebUHpfciPr7K+psmCXO9NqtSvXtcMA/n7GyIGPBVDp8kMUwzuH9OVOkZkK +Z1a42uuYKs/zgnbXV7kCZ6iBQkt7A2Scv/IIwdeaKxfTv87e8UqMkYPuf4wgqLq1 +USoMoHdz6JmQnw0CggEBAM4LfX4PIIGT+OQG5kn54ZcsqerPljPj19EBe8U3jyf3 +OzxDkSX0f2fmBVDjqj8QRO/PcXQbGUXWhJN1DvvuJITE93Y5Pj8DJX3CfLEnjE+b ++sfuSxNawH9NwvwNt42NWDleMAgfMot3J+MwlXb31BWixCMj41Vl4VrkHEzX/M6R +aAWcTqeY59KumtOvZO/4U98VvNyOipLHWcWZuwJLzeUjvyZ+hJQ6kBGJJYNezYm/ +qhHZ5k4bz8HAUSgih+Yb3DNmREk3YqIM4Skq843YdF8SEnFw/4b7PKMfmBXy3X72 +f51wdCiSkIm9o+/QLaduExQd7AhmFO8avZa5bRNNwRMCggEAKY/DBX+sOoMTw7IV +o9IjMHhobL6ch5Kxf/0HUKauPl94IhsMlh5W39NnLobJOjBk4FdndHV8GAN0sz/Y +yN72GpXLPKz/U5RD04ATWtn7qLG/iJYBAzMJY83cQ/jf/XA2NVAWvXl3D9dvgT83 +qM2ECnOPT1x4QthgYQ7aN/JHXO2vNjp2AvldlZoBkHmvqGpljLACAq06x3oB45Fd +sLSmO1qVlGdjeDSsKYQ/HfJ4+DlecnHrulWmrPcsIGmR/TF427Rs+FiueJ+VorvN +R074nKdE6MgOYTXxMXgt3lo1oFCTPLhLRtg+LGsY3vr2f7Bx1nCdXet7juBuBUJr +GGXAlQKCAQEAxNVHMfijdgX022kX8A2Ni4x4Wj+q3rFHR3viUDnOQUC2TtDBRX/3 +gjrEU0zaI1qYcHs8h80nbIcMqY1HHjaWnltHh6IRq8KGu0fjNJ1yNc7tWLd08u1c +PYD8xysXcVtYr50hx3B+KatP6IJOFpOUAIM4WdV74+XqzZhizKn88R0JQWrb3NF+ +jM6OS7EffPs+rDuo6w4kpSlZwiIk+4GNFNv8THrKjowPeyEIPCKBuZjmkB0YHQAG +jbH6FZw/NPzidBu7GjKVv/cL1fcZKiVgrj2mbsai5MD3YWHaOQWEwTgcGzwFS4kQ +GPWYOY0nP+4wvaQECtXyI6To/qbu42UBDwKCAQAd+/H4mepB8cgchZobFco6yTt3 +qgS1D9sOFgWYZLkxIuQTQH0C1cLHXIiqf0NrJAOBbaJ+vVuCrqv2MUASaxKoDK7h +1W413yKIELWbTk7yEttw0M0T2PXmI5dNdKuw5I5hw+MmjLOFmyRzvX870ihDnM4F +MISxV7K45t1EHjMsz66fMc8BIkophwK3/7FSok5XhYdLQdQS9Rshv0PXQmffkVUM +UTlrwgH/3WGRhkFbmcdBMlawHQGvjiyZ+Gz+wF1uhzEYweT6wUfaHZePxX0hXqom +WVS6ojlUji6NNJqFN8DB4q8V5/EShj4fpdjenDap5IxFgDSxgSCShR2FGTCW +-----END RSA PRIVATE KEY----- diff --git a/testdata/x509/server2_cert.pem b/testdata/x509/server2_cert.pem new file mode 100644 index 0000000..753065b --- /dev/null +++ b/testdata/x509/server2_cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFeDCCA2CgAwIBAgICA+gwDQYJKoZIhvcNAQELBQAwUDELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV +BAMMDnRlc3Qtc2VydmVyX2NhMB4XDTIxMTIyMzE4NDI1MVoXDTMxMTIyMTE4NDI1 +MVowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3Qtc2VydmVyMjCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBAMqkUFp6xBzIksawPZpDQCZS+ZE/Pjfab4q7CUd/ +pN0Ss0U9MRBBnYj767qvwrOGQYdpkK0NWh+BtUOQHaqsnD+ykr1x2i27uMvYBnkn +91t8EmfW8u5cj2lZGM8SRXaiCc2tv3ZFDKeizIp5BcxLsTubRZNMTOYxUCpFoABo +d6DGYXaU6LFNbIhXQ/vTlbDa85a5EjS5SWeBzEEFgBHHQDdjh+cqS4bzAgI+klYT +uYM3PMbUuOg49hVQjW1pXSGO8Yvha0fzZP4upUKi6h1+/xBUZdDiWuynEJZuYPAC +xqgnmgSkexNLJwrg/6JPr04TfbKagffKDHKhloI62CZJ9/VRwo7OF8dsl1Y+cr3T +XjDtsxmypDIi31szDYhE5V1P+FAGP3sewQBRh597x1RA8tGE3ijIV6iqSlSo/Lpx +HaVM5n3PWY5vUNFWdbawMDuCUvi9GYn3JVnt4YUxjsMxLC39FibzUmwD7JXqx86t +ArPRLU6knrcod72dKRnUGi446Z4dElqMngyu2Z08RB5iYzDwT+xugwki0gezQ2uB +59/968+MI3eVe/QWMwtIAoZuHKvPsAHOb9bAcBvLQi94MWLbBUkAga3XRF6CNAGq +LPNdhznCNYdXMnMdBQHAzu7V5+yIB/ZFQ519Fm4RinCG6clIi326A6GNYenAJ/Jr +Xa5tAgMBAAGjXjBcMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFBfefRYaa/BmcX6g +eSQSMrh0zktEMA4GA1UdDwEB/wQEAwIDqDAdBgNVHREEFjAUghIqLnRlc3QuZXhh +bXBsZS5jb20wDQYJKoZIhvcNAQELBQADggIBALpTW6UICT8SyCY7VNUja51t3+XF +QoD8xKu2qS69G/oDbxe5SF3ldymvkQPqVwnW9e34mjxD1Au4IF8zQWv2EMm96wDD +0Js0/yAMjw6/60f0hF9lpQEqe24W+wgbRV4Fzt3/rybLte+4L9chvq1plHqQ3sfa +s99D7bfPSLX8n82ppNmm0U81kXmtNAw3P+vStBqRgrZNkbkkgsoGZsTgzKuD+H81 +WUzIqmIAfTmkDN47SXYuneULlFNWwtHgTWv4jq2/ptYo75MQq+ExwTDGM168x17u +yaC634INTjfNd04exiktBJXWmAS8K4aYgvHPgcIlzyidR7taI0X1O4mR4qomh5W2 +fVmGkpQZCmkW80whgycY3ui2fdWYOs3XGdfz53fJdN2vWebpUjw7+1owlmqFJhEe +Ct0wqLLeE8rdOfKueh3/xi7CxoeYM2fVjN4gHPojcQ3Mcs7wiJDm0WFaITi6+KDS +LmGhSHKaiXiGKsbLykN0DygDQYa/c4t6NfzoRGWKMhdAcRXicZaxnwPD65psAijv +ZDgONwXeHKgfk7DnE3rs+D9xuh2ciw7lkcbImYmOCMoV88qG0t1uIwlM3xh8S7Oa +DH6q4vj3pF63QS57uRtwCBCOa3xcYKTDJbdRyUKgAejVoz8bqTI8lBjnRTtxlQFi +4ugkg86X1370IY84 +-----END CERTIFICATE----- diff --git a/testdata/x509/server2_key.pem b/testdata/x509/server2_key.pem new file mode 100644 index 0000000..82c947d --- /dev/null +++ b/testdata/x509/server2_key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEAyqRQWnrEHMiSxrA9mkNAJlL5kT8+N9pvirsJR3+k3RKzRT0x +EEGdiPvruq/Cs4ZBh2mQrQ1aH4G1Q5AdqqycP7KSvXHaLbu4y9gGeSf3W3wSZ9by +7lyPaVkYzxJFdqIJza2/dkUMp6LMinkFzEuxO5tFk0xM5jFQKkWgAGh3oMZhdpTo +sU1siFdD+9OVsNrzlrkSNLlJZ4HMQQWAEcdAN2OH5ypLhvMCAj6SVhO5gzc8xtS4 +6Dj2FVCNbWldIY7xi+FrR/Nk/i6lQqLqHX7/EFRl0OJa7KcQlm5g8ALGqCeaBKR7 +E0snCuD/ok+vThN9spqB98oMcqGWgjrYJkn39VHCjs4Xx2yXVj5yvdNeMO2zGbKk +MiLfWzMNiETlXU/4UAY/ex7BAFGHn3vHVEDy0YTeKMhXqKpKVKj8unEdpUzmfc9Z +jm9Q0VZ1trAwO4JS+L0ZifclWe3hhTGOwzEsLf0WJvNSbAPslerHzq0Cs9EtTqSe +tyh3vZ0pGdQaLjjpnh0SWoyeDK7ZnTxEHmJjMPBP7G6DCSLSB7NDa4Hn3/3rz4wj +d5V79BYzC0gChm4cq8+wAc5v1sBwG8tCL3gxYtsFSQCBrddEXoI0Aaos812HOcI1 +h1cycx0FAcDO7tXn7IgH9kVDnX0WbhGKcIbpyUiLfboDoY1h6cAn8mtdrm0CAwEA +AQKCAgBa3LaS+304Es+Ne7UDmKgJByeUczEoxi9Bm4AbqSZ5Yksz/q4jReinZZ5b +hTfeW5LCbxlKHzSL8BMhClvjDaa6AQ4/F+/mlcfUzzaH2N3XDZkLKpyfOK2tZR/0 +qZKwERQoP4IcO/XirOLeLEnnQwFjYsodtBa/GNmDOtj1leIeGxXUoAx+g+Lod4iq +QENcm7ChoraBIZvCZ7b4aMj2L8uhimWDx7k593itHPVs10dViM0dsoB+0Bu3jvj7 +WEVEKN4yBI+gIYjlWHENohMryqf/4HgO45A1kOulKDUbKYN+HtO2xTHSgt4syJqX +YveOILs5/IHOY7CVLdNY7Z3B/WTKtO4UCzGtFWsD0Ai6rtfnSj2aAmq5uHWDwPHl +3fHdTK6knHdlaPWbeQiBIk4bT6L/JjH38dqBU17/RathjoCDQngNmid7AgSnv5o0 +5ugTCTzzFTUz7FnA9uYcEWIq7xDB5gVFTcvWcARYd2BsLM/9gF1Hh7r4A8gsHj2i +0+7Saw6mvAsPXp0JH+8idBk8khV8v6Uy1arF5aYF8yNus/hVr5fUBnnK6MZ+F2Nr +VOa49n5BhbWm/IsVYdgnnk4uwUx4yNuwMZ9/nSsDEZ6IiX9KZjap6zKNwhoodLV2 +T9WYKC7JMOEBr1CzoQL9JzvyYulZUp0F5SBMbB9kZj6j1gQRGQKCAQEA5QbA1hqt +0iy5KjH26Rok2pwi3z49o/sFPyg4TBOs5Zx7T/iBQM3ZlyOUdGT4uYFX83Xljbo1 +As/UIM1wzCSUbyyHGs7RTuoAWJ35d94UwdfOmw5j5ETAtbA0EVxnLCqbp2ArqC60 +UT/M9Yk9HK8bj6wwb0kZ/wsfwn78Jts8GhtCozPg/c2KWq3ce2bcHMuGhlaWq5Jb +XlHrZBIoL1tYsT8LWe4wc8Agm/w0ZC4rM6B0A8rxdsrPAc3WmlvNvnA8DA7OAJZ5 +j6zThsQ6FdSic8CI6p5vw4pyFZXtdERIjX2jkWVLVdb8adQKhqcglBvYZt5e73tk +a1OfgG0tX3M8TwKCAQEA4oIOLgVJWc9lfWFaLiRIjKlVtnOqfR8Vy9e7i2dBylgO +ZAoVM3ROYLsbDRsMJqMN/IQIuSOMYP+lIeeLoJlcqor49+z6em45Kbt5ZiK1VRoG +78Zi/UZ6cb/vcB17zmeBuUHYEmHLbB8PWbL+qFEipVDAf0q1jD9VnRw6ntt5K0oM +AKPH7jiCxfo3GU62nOZgcT1rwnA0FHBl9vvcr+237a/+NoNkRqQxTX4y5kA5l0nP +9EEWYvOlCKbkYiPKHOZMGZ4MWb0FlFp46KPxiM/x8XxB6NFUacEJUKE27NYgj69+ +5C62A34YtLKptD2+CeAKqxYOYIt3Tj4qJBrJ4m6OgwKCAQBxoXwjvnDnipEEQm4D +EZmfbUBQCw2CQpVD1Ky58jkiYxU7hEx83qVKu7h4V3CgeXAttx0ByJVso7jX3ZZN +cwjCcBFIV7y5rpglX5vawTEDTBOSEv20z/fdLWNoCbSW0T0ROkHu291TQphqaoEL +rkW6bvBJBrgDNn23flGU5clYGpZhaugChOxUOVbfUxV6o/BGzsdKsP7sOTDVIb0W +YfgLWQBEykz34SdMvUExQ0bkAoQNLa/IBK/YcUw8obfe+MiSIvZKjF4bzt/USZ+Y +HTvMuoY0Ag/psNMRqqV5vjdRHDj/doZ+PIBX8YCXdmxPj9E6mLH5l/sm1QKaMZEF +fqM5AoIBAQCPe8lVt72Wab2lpgTFU/CtQhtsv2qRZh6diSRhk2BmuE8tagGyHYwE +1KG3NJoG46VZf54zAWTMkUTe7FlTu7KqyewayYCGC8qkOAEYBQaPSTR5sVdFj97C +rc4UXGjwADt5yk8AnfiJnkdQEAYnQ3ZJ+JRoTkAg/oHSS26K8QaZuIdP5HAi5KNa +nD1JB8bAL2OKeFkJy6ACDo1Y3oUW4ORxadoEWEkuQpaEu1us5aRVxMk5tf1jY2n4 +yBfGX1uJ4Qz18VtrgUTGjGUpIalAfFGMIqVxwSDS+RhYfjdX4fCwdIBSNZDRN5CY +7tB3v+DhSo4XgJpM6CwEYXa6dknK6TPXAoIBAQCo/+MThNIrKwIsbKxyz3Xzg4Ut +6JUQxd3YU1kNcNde8BSuNp+05RcUIc7wpRTrrJwAc9uNHVNjKz73WcpjMh3dHYJY +VF0nbUzM2m+KzLdTJtRTYMaFGJjiHbttetNGJoKomNrMea/vEjZh5WadkBagKBGp +u85H3Ff1vYdMyCTBiU6eyabxc08/ZEaFJaQALjVC4e/mQdluCHyfmqeY1awKLmCK +vf5ZvBC6ISOMMibqcRT5ocjAvO3j+3d9Ce3ExX5U1fu/xYb3YoWQtdX+qIAT2KYq +QG1vDy0VieGJlUiDPooWin/X830pxyYJ79w7XN67JIZdlrtIEVDJe4/xghXt +-----END RSA PRIVATE KEY----- diff --git a/testdata/x509/server_ca_cert.pem b/testdata/x509/server_ca_cert.pem new file mode 100644 index 0000000..2760c56 --- /dev/null +++ b/testdata/x509/server_ca_cert.pem @@ -0,0 +1,35 @@ +-----BEGIN CERTIFICATE----- +MIIGAjCCA+qgAwIBAgIUVaYPCm+rhznxJTRWV7wJKkmRuW0wDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFzAVBgNVBAMMDnRlc3Qtc2VydmVyX2NhMB4XDTIxMTIyMzE4 +NDI0OVoXDTMxMTIyMTE4NDI0OVowUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNB +MQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNVBAMMDnRlc3Qtc2Vy +dmVyX2NhMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxo6Cn80nk3i5 +PgmYnMBicmJEykEz5YbJEuyN+Mjv1wivqc23P75qvu7u0FPePptHZK+Q3PCnv7BZ +jc+MDQzZhUWN8jwenMGOxpVrX0zjK7Q0u92YbrHgxE9fkRA5fZcXGzZlrhsJQJUA +G+0QGCSzjWZvSab2JrVn/gYEzikcl81Q6zAJkTI9vACZC0vnTc6XsVC8QCpT71fb +qQwE4Bvr1tyuA6biB4H40RiLGWuG+8BoVn1pgSL/9GzRnsEnSN2KCfaqzk9VMDnP +TQRx0yJY+Zl5FB/ufeEJH8hh1OS3dAJhR7IYLktlm8S68dSI/oTs811BWIw1dOqa +KbpElXS5Tr9usGOehxy7q6dlazj2+nDzIhQ/20koX0dqyN1O8Pzi4OWcR5YQEBDO +8Bp9v6JNowwbMkZGSg/C1GMNwN4rEhLlAgpv8/4CoZwlQM0oROWiiZwczpVniDiq +6dYtTUhuJwC0cgJLSswDXpnAlp30hPB7EV5MIdr+9ybuRAx59Nl+ZB8g6utuWNaA +lNTrAsouwWBalHmY/f4/ltEnHkwgKCReYFHpDNuDVtnxhtEfGzd5IxQWNl2etWCR +Nnf7Z3DQHLTduIQD2R0qp73tqFK8T1DR/HZkbZnZPvBqmUIXvCnHJKKB3ntrkpqR +bQHMq6Tkv3NL+N3XpZwEz0AOeiRE+gsCAwEAAaOB0zCB0DAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBQl88evytJrL7t5BGRbJUeMOW4jaTCBjQYDVR0jBIGFMIGC +gBQl88evytJrL7t5BGRbJUeMOW4jaaFUpFIwUDELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNVBAMMDnRl +c3Qtc2VydmVyX2NhghRVpg8Kb6uHOfElNFZXvAkqSZG5bTAOBgNVHQ8BAf8EBAMC +AgQwDQYJKoZIhvcNAQELBQADggIBAMGqi2F7ccNQ0FSiALPUjO0VvQrUqdWLrc9Z +67rr7wBu4bEzchM+HQP9GwbnSnH9yT0pnYj2H6idAfqTww1kKuR4CYMkGsNJ9PYW +AgYdrC67HKT2xhy9YmrUItIe/pM6rRO6oNA8Np3IAEmC0gpVMqmPqHeLvwhxcf4f +izsi148gTGOxBIWVNupImFrOaztKV6SbVwA+wdHNJvXz4MEEYlMlgHFfkrAEXHfO +6QmHXru8C0BIQaMOiVZDN8YCwmsrcGFYjHFRS/OnYblrRxuVDdhpMmNiQRJLhZHi +jf6WOpJS7o50FmC8bG1CE0CqMNF/qz3Hap36Rm2w/xSems2dIqMr6FsH34KkyXzm +pCHN162g720orV1uExpgfRSfv+IaklN1sM98WkTqAkz9p6OPPEo4VHeVGUk/mFuv +aVnByrk7qmpTLBGDk7dFI0GjsNwOz619omgYZGliRU+7rDXP4fN6EPlF5sQO7MJX +REOSZvVcHPpIAIqTFRR4SBnwYGsEPQbTKTH7jJROg0TGmiKeN4N1syb4KNos2Wfp +ZZB+f2qmn6LXS6d2kI692UomRfGVNoBEsAhWBW7FzpU/WnT+aF97VpvWUEqzg/AS +61tKM/t/ap6kNTLaPGSWTk9Ade/KuMmg+nSrL6S1Xa0T1rl2Qjd5h7U6JLWHL94v +GPxPuBsN +-----END CERTIFICATE----- diff --git a/testdata/x509/server_ca_key.pem b/testdata/x509/server_ca_key.pem new file mode 100644 index 0000000..1f69f34 --- /dev/null +++ b/testdata/x509/server_ca_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDGjoKfzSeTeLk+ +CZicwGJyYkTKQTPlhskS7I34yO/XCK+pzbc/vmq+7u7QU94+m0dkr5Dc8Ke/sFmN +z4wNDNmFRY3yPB6cwY7GlWtfTOMrtDS73ZhuseDET1+REDl9lxcbNmWuGwlAlQAb +7RAYJLONZm9JpvYmtWf+BgTOKRyXzVDrMAmRMj28AJkLS+dNzpexULxAKlPvV9up +DATgG+vW3K4DpuIHgfjRGIsZa4b7wGhWfWmBIv/0bNGewSdI3YoJ9qrOT1UwOc9N +BHHTIlj5mXkUH+594QkfyGHU5Ld0AmFHshguS2WbxLrx1Ij+hOzzXUFYjDV06pop +ukSVdLlOv26wY56HHLurp2VrOPb6cPMiFD/bSShfR2rI3U7w/OLg5ZxHlhAQEM7w +Gn2/ok2jDBsyRkZKD8LUYw3A3isSEuUCCm/z/gKhnCVAzShE5aKJnBzOlWeIOKrp +1i1NSG4nALRyAktKzANemcCWnfSE8HsRXkwh2v73Ju5EDHn02X5kHyDq625Y1oCU +1OsCyi7BYFqUeZj9/j+W0SceTCAoJF5gUekM24NW2fGG0R8bN3kjFBY2XZ61YJE2 +d/tncNActN24hAPZHSqnve2oUrxPUNH8dmRtmdk+8GqZQhe8KcckooHee2uSmpFt +AcyrpOS/c0v43delnATPQA56JET6CwIDAQABAoICAQC80PSi5kMGWD1AI3v/RGvZ +/l0QQOULFhvMZSu1M8/wGxCBV2E1uuxj2W88qSSlQKCpvNLzZ979yMPAuWejWV7Y +/4W2nzk1NFODwL+0hrdY7itfo6C7U2g9BoYIuvcQ2Udd12LmKEuqIIdUByHQ88XT +Z1/ZGG7n7IaR6ENVkX7hVJvoq2vNqYtPZvoi5fF16koSkoYSNq5O4qu+m/Fe9O5X +CtBoJKC5Jv3oSYCtkbVxXk1aQjS8Wv4v//NvFps3DYWhZ/KR8ps+GxtpUBq1/unB +ohKj8qGnDwLQOIvgGgfiyAieV1vrWkOr1283XTdRYjK6Uyo6/Eoxfo9PsxRZVACK +lpkRGn2p1GTbx442INrOwhJbJtKAcSlM7V8m2dAhgMTXXJzPtl0eCNWGy0s1obpK +1p9qTUz23s3WA425TAXQwIFpq0CdrXPgaXmZYw5LlfuwQhUmw5SQBOg8dX30Y5rb +vrrBBj1+2kAEYyeC3aQ3HwP6qAevWtmlbyCLMS5pYcPTRIEJlVX7OMCZSmvYnOku +0Vghgr2g+FZjNgQMo0HYEq6bb1jmpnN6tY4y3H7Zq76xavkFBWzaz8aHEB1koBoW +e0z+AaojAXEMaAKzsA3hF74HZA4kZf9kFi4er4ZLlnZFrU7UCN99mQfsV0LU+75E +rpzoRVYE8CZN9egiphZFYQKCAQEA8AUt1mi3tDfbyuDnut9KVkraCDAQ5WX+31H9 +BTPdIoXaeYx29zCKSEaBAMvf43s2pDgVPVyuc7KVFp8l3LZZ+znNWxU3eVznE2Ua +1QUTR8ZxueUIyOZxuSHU8xqWvxZ7SLZYTqTVmYGpORjvuQIgS/+6GrAbCNcZLNIo +I7U4/Nx4zJ3bkF1xaKJcaVXQRwG0cHQEZZdgkaE+J/ak3WTRC/11tEr4agGo+UDF +6XBQryp3HxPMnZyAv44zOB9gqD/FvnxH6S5ybc93eISQXZoJQuuad3GuEn28Gz5x +Bsr6zzSkhsTp2cEr2AVSH+b/SCseZ/2JEE0mjPxgjXRVLQrnGQKCAQEA08akFJjH +nq0aEcL8HWbUE2PuuXcq/vUZHFNv1gG+e6vZ7gU7bq+5SS3+APuerPQrhE+3mKsn +WKUfkr2Rgn2Yxieo/u+SKMbDaNNT5h08KgA/joewRYDeInACw9QHsKPKArWmWn6d +6fHYl5d3rxUPGUaM0fsdOvayX+xeN4PDn+puqzInhh3TFbeyvkpu1b/xZZ17u7Xz +yvu1PRHNt5eZB6QZ/GF3d1r63CTC7m3GF/wikoV9Ygy0Me8gw8WxqnF42GFRyUIa +wHXfb3w4Y4rZ17b7x5H18FF4tHMgXN0pNbGk+PoIFB3Erzfqi95xVyZh/MROwFiL +qy2KeRoniUbCwwKCAQBDvxJ7DD+dzI5rKyP9KP1Qcfwsh3SdazaPThL+nu7xyZoq +6KzDhJ3jXJMY6HKfQK3hmDrWgQx0d5mBMxZ6v7WSJXSDGu/3f3Nxk/4I1k/k2GxN +LgpWukSrHpN+sqiN8wiFM4KlX/0yQNjE1vcC30jCasHauo5G5n+imQbfXU1igdBO +4NeSXe2evQUcbi5FfIOzoeuDyUBmmn5yxTkvjD89BSNt6iNHuIQ7Jj82bo83geLx +kKMWcZAdgUOPubuMgcOMyoN5m7SMrhxolfIxmUK38sw8noeljHvFrNA2PKCiT5eI +upfO8KkxZf8SJh8z/YetjnBbe4tADBQsmQNZnVQxAoIBABNKmxPNPxHzTtajXngH +L/Z8OfjnJCGJjjoIV7209vcpFncaPum8VDKYX/US9sdmjrhE0sKzhKgMkq25WxH6 +Avq6Dij7BeN1B8P6zD/AFgT1dNS1A5exP4r/jSDtpa2vne1VQswnkJcJEuPsRljK +oE97H8TZDTab1m/qhkKkXCOrJV2u+e67tMjbrQqsmSAblg/dorHcx1KMT1w6zPSW +eLg7eKqG7m0O+p8nMiKqGUuCClwykNNnuNp7oA51adPO9mUvqFWfEfTKSApN1I0s +zt9ZqeHqJ+82XLqDakVLWD+t6QtNK4M5mvsjKtiG8OgxdOejslDPQBnd0ilp+oQE +0CUCggEATL5jg3Kcbq9HcaFEdgqzVXXcqNI4hOVWjP3Vp6W390PUZkmls1Igms3l +zc9fhibHv5gRq/Y3xQGUuP0dKuPuTykjXUTRozeal5ErkiLU6PZeVlz7xUHbLrF9 +J/IARiZtnfhcDh8qo33QZ4m4HUB5b54/4Gdzr7go8o0UVI50gXY65kwz5hV79/7P +S38hxS+X+k2FkWuCNrP6ECkC/L8jNM1h2gR4Ez6FziiNxYbOhQgsaTQRKjca/v/E +tzX9oZjMqiWtWKcxgSCZs/1DscXjjCsi64f/oOBShF1x4ZeiMoZTMhQBLMlbVx/n +JxRWkMCYLEbAZXWJXEKHyqaiIAOpkw== +-----END PRIVATE KEY----- diff --git a/testdata/x509/spiffe_cert.pem b/testdata/x509/spiffe_cert.pem new file mode 100644 index 0000000..bbe7d66 --- /dev/null +++ b/testdata/x509/spiffe_cert.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQEL +BQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQy +NTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEM +MAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVu +dDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJ +LVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/Z +G5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqO +a6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3Z +JPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XV +m0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW75 +7PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfc +msHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740Yc +DmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPN +zHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRs +vvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkI +sK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRH +HvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAP +BgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29t +L2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/ +aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK3 +4Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0 +IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+ +PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV ++j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2D +vUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gq +yjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvV +z6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hx +x0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U +0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EX +GA91fn0891b5eEW8BJHXX0jri0aN8g== +-----END CERTIFICATE----- diff --git a/testdata/x509/spiffe_key.pem b/testdata/x509/spiffe_key.pem new file mode 100644 index 0000000..77a33e9 --- /dev/null +++ b/testdata/x509/spiffe_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDJ4AqpGetyVSqG +UuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zE +tz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpms +zuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3em +jg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3 +f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQs +SW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlr +CAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb7 +40YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7 +BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5 +LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnr +bJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABAoICAQCXbolwqfHVFQ/OLRLzsZvy +UZGeIY7UUxKrAyPSoNvJFpr7DG7n7arOcaTOG1p56aYyYPvHIrB7FKpQggnSCYIy +1j89BoMjTKu4gsKWad72+ivB/RmRPYGOHUy6QftXpXQ9c0Y99weJ2iMO3zRR2269 +BbG0pCd7ppCbJquWP5IKVlhl/cxjHS3vd/YPZorlS1QUhpsMxGHfFKLzfHHk5nX1 +ZIHlctZIHeWw8VG8W/xbbnA2RhcxnY42vqAG+/NCkgZUAM3WU8zWfU0WEZ3Imy9Q +HuPv7m9H+vyBYSYQR7eh3nfAVHnHeRBKQX6hpQ/PEwHneXzVmZVnaaZD1l35+oQs +9mCmIC5PkwGe+vJI+Rxt5UElgzRDMVF6YuHfmobQQn5mT0Jsbdn6tfaFXa+e8+Ja +4WSlv/rVNSvCxNSwV0fOGIIrV1CIElf9ei7Go/Jewz94Eh2PdoOL9gNPqBeFFLmt +mS7HgST/Dkn8yAvyYgY5IDDWDiauECMV2F37QRk9stX+dUFhDQU3P/CcMH0SRyjx +vXRRvWY/5rWuinJcIr1Kb0OHiDztrM/4wAYc1BQjsL6CPKT3aAmCs/fxf0JflK/o +pvzsK5+AyBrvEBh9SXPcQYYEj6ZYCGZ/rPVlmgOjT/d4+xZf0FBUi3g+np7V15Km +ao4LgfcSQF5xUVTiyGo1AQKCAQEA423AcschurSrHwnmL1E5UIaYoFyKok2W3GZ6 +nu8Gp49GlF1v+CQ4w0zMNgigTTHxPyShK4/lC1n8W7liuo2TD98Km4qsnkD04R1z +Jf2PNBoVfMFEwxTA0t3LYfqJc3xLxWTR7wbvIRwtrcM/tFbU0RQAZ0/eE5Vine6t +fU8HBn/kSnUeCXMjFqheRccluYQgwES5ayJjnvf4Yrwe+In89+C6JE01LetqtxiL +U9X6iO9VpF9JVI7CPTBd7cZI3jfO+N2dXGlnt/Dp95S3RymoRSyMA1ILCCFZOeTo +Kh26/6hfgc2ox6ttN+s/1J9xPaXFmvbEAT27n+bDFVIfvF9QwQKCAQEA4zx4xLsW +AdljDLSDcLziWE1ikvO25hWiH3Q7ZyFi7vqby4d7FdgazNERrFZaCNQeSLfMakLd +zO1URfWsG7+6XvY45VtgVlqw2+uAHlE/B3FtrSeaIDr68AvWfPo02vckRMgTCBch +MEvul58A658mlybOkTHoRyaeDDD+83sIHAnyFzublxfsfbdgqzFW1RxfiUwmAM7w +9rI+PFPQnBgBkfyjcOfaVx/I1uvm3Nnl4ZUAmEdz6YlJdN6EhM8SjCP2LsLvhDUz +kjZ5WJ50ybRs3DWhDH/d06DmFlGwgu894TKBHqrq2fgDDukpZkH7cHldcjugzlHJ +c2CAO9MxI4UyvQKCAQAogEgQaKP6EuiSe3nRnV5el8mgbTqHEtg14c4edaSyvFIu +Y8Fn6FNvfEK1sK2TcbxrqUNGdbatYdYOI6KQZFv3LJo//t8kw56YZF04O8J/3dFL +yUNMlmqMYtEwXqSRu2Xm/kBgl9SICfOciTPUEs6NeUllHJUI2caZJ4Mf2K4Am0/1 +bovt1OI/y7YWKRPvyLboZpS6noItMi26r5O4YSJ6pjuf8VvyFIWJm8ZcJLQcJLsU +rZ9qfo3axb1EddZONJQYP6chaOf+mtmfrI1DEAkWYIuCn961EPNJ2xj5PxgpJTv0 +6sIO5NlrZuqUG9zXxKi/Iwjey7aZEEhXiKt8KWFBAoIBABtcRaJScHTqit2VwpnJ +dGtzbeIJzETp5+pnoVtqjrH9pNKdznkz2w48QieBAjg76iWRU+Cbin9JODNwQDfb +HwKeHP2owfHD27WvJm8AE1m/E5icwxcMYviSRFIqAkE3LrvFZ107A7j/+4twDrlQ +IWJjvs2Gt9QRV0hagegpMTHHFMotWC+aJtSARvh16WGhl/M9IvpH8IWTsqCq6txQ +m6fLRpaqpASHhDQ0lUiUR/Sgb0DmoZNF/3096bDgCfirv9GjkRlXGo2JV5UPBzre +KZleL7UElF4N6oZXcaxiSA4ceaWKqNpz3VJnSp/QZAkH4/OEMHmHKX1l6irJ5AnF +2PUCggEAQzXltQN+24eNh9nD9BzJR7CYOJQ4CzmHbkxJJZXGJtSCv0mYcEsSRmpH +5mW9hr8w6Y5WL4XfMduUX9od8exJtXl5d7Fl8oUNh+CwDr9smEjEDchh+6d11ZCi +Ervr1XOmNyOtpnQ1+N2nbbBEMLVns9yX6oNnl9mBdwpvwko7Q9ahyTvWnE2oG5At +8VeX/34k6BWdqPCJfnISMVbt6D+J0kaqaVw6BplTNSRs93bmqKwpcrRHMTida+bO +l9t8Cy3TguZdRWTJZ0kFmze0fV6dXYookZoUIeisTZBl701tOsvysZjxtMQJfTLJ +Io+0lEzXxTbCbBP/iyizjo62XTgpdQ== +-----END PRIVATE KEY----- diff --git a/trace.go b/trace.go new file mode 100644 index 0000000..10f4f79 --- /dev/null +++ b/trace.go @@ -0,0 +1,143 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "bytes" + "fmt" + "io" + "net" + "strings" + "sync" + "time" +) + +// EnableTracing controls whether to trace RPCs using the golang.org/x/net/trace package. +// This should only be set before any RPCs are sent or received by this program. +var EnableTracing bool + +// methodFamily returns the trace family for the given method. +// It turns "/pkg.Service/GetFoo" into "pkg.Service". +func methodFamily(m string) string { + m = strings.TrimPrefix(m, "/") // remove leading slash + if i := strings.Index(m, "/"); i >= 0 { + m = m[:i] // remove everything from second slash + } + return m +} + +// traceEventLog mirrors golang.org/x/net/trace.EventLog. +// +// It exists in order to avoid importing x/net/trace on grpcnotrace builds. +type traceEventLog interface { + Printf(format string, a ...any) + Errorf(format string, a ...any) + Finish() +} + +// traceLog mirrors golang.org/x/net/trace.Trace. +// +// It exists in order to avoid importing x/net/trace on grpcnotrace builds. +type traceLog interface { + LazyLog(x fmt.Stringer, sensitive bool) + LazyPrintf(format string, a ...any) + SetError() + SetRecycler(f func(any)) + SetTraceInfo(traceID, spanID uint64) + SetMaxEvents(m int) + Finish() +} + +// traceInfo contains tracing information for an RPC. +type traceInfo struct { + tr traceLog + firstLine firstLine +} + +// firstLine is the first line of an RPC trace. +// It may be mutated after construction; remoteAddr specifically may change +// during client-side use. +type firstLine struct { + mu sync.Mutex + client bool // whether this is a client (outgoing) RPC + remoteAddr net.Addr + deadline time.Duration // may be zero +} + +func (f *firstLine) SetRemoteAddr(addr net.Addr) { + f.mu.Lock() + f.remoteAddr = addr + f.mu.Unlock() +} + +func (f *firstLine) String() string { + f.mu.Lock() + defer f.mu.Unlock() + + var line bytes.Buffer + io.WriteString(&line, "RPC: ") + if f.client { + io.WriteString(&line, "to") + } else { + io.WriteString(&line, "from") + } + fmt.Fprintf(&line, " %v deadline:", f.remoteAddr) + if f.deadline != 0 { + fmt.Fprint(&line, f.deadline) + } else { + io.WriteString(&line, "none") + } + return line.String() +} + +const truncateSize = 100 + +func truncate(x string, l int) string { + if l > len(x) { + return x + } + return x[:l] +} + +// payload represents an RPC request or response payload. +type payload struct { + sent bool // whether this is an outgoing payload + msg any // e.g. a proto.Message + // TODO(dsymonds): add stringifying info to codec, and limit how much we hold here? +} + +func (p payload) String() string { + if p.sent { + return truncate(fmt.Sprintf("sent: %v", p.msg), truncateSize) + } + return truncate(fmt.Sprintf("recv: %v", p.msg), truncateSize) +} + +type fmtStringer struct { + format string + a []any +} + +func (f *fmtStringer) String() string { + return fmt.Sprintf(f.format, f.a...) +} + +type stringer string + +func (s stringer) String() string { return string(s) } diff --git a/trace_notrace.go b/trace_notrace.go new file mode 100644 index 0000000..1da3a23 --- /dev/null +++ b/trace_notrace.go @@ -0,0 +1,52 @@ +//go:build grpcnotrace + +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +// grpcnotrace can be used to avoid importing golang.org/x/net/trace, which in +// turn enables binaries using gRPC-Go for dead code elimination, which can +// yield 10-15% improvements in binary size when tracing is not needed. + +import ( + "context" + "fmt" +) + +type notrace struct{} + +func (notrace) LazyLog(x fmt.Stringer, sensitive bool) {} +func (notrace) LazyPrintf(format string, a ...any) {} +func (notrace) SetError() {} +func (notrace) SetRecycler(f func(any)) {} +func (notrace) SetTraceInfo(traceID, spanID uint64) {} +func (notrace) SetMaxEvents(m int) {} +func (notrace) Finish() {} + +func newTrace(family, title string) traceLog { + return notrace{} +} + +func newTraceContext(ctx context.Context, tr traceLog) context.Context { + return ctx +} + +func newTraceEventLog(family, title string) traceEventLog { + return nil +} diff --git a/trace_test.go b/trace_test.go new file mode 100644 index 0000000..73bc4a5 --- /dev/null +++ b/trace_test.go @@ -0,0 +1,50 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "testing" +) + +func (s) TestMethodFamily(t *testing.T) { + cases := []struct { + desc string + method string + wantMethodFamily string + }{ + { + desc: "No leading slash", + method: "pkg.service/method", + wantMethodFamily: "pkg.service", + }, + { + desc: "Leading slash", + method: "/pkg.service/method", + wantMethodFamily: "pkg.service", + }, + } + + for _, ut := range cases { + t.Run(ut.desc, func(t *testing.T) { + if got := methodFamily(ut.method); got != ut.wantMethodFamily { + t.Fatalf("methodFamily(%s) = %s, want %s", ut.method, got, ut.wantMethodFamily) + } + }) + } +} diff --git a/trace_withtrace.go b/trace_withtrace.go new file mode 100644 index 0000000..88d6e85 --- /dev/null +++ b/trace_withtrace.go @@ -0,0 +1,39 @@ +//go:build !grpcnotrace + +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + + t "golang.org/x/net/trace" +) + +func newTrace(family, title string) traceLog { + return t.New(family, title) +} + +func newTraceContext(ctx context.Context, tr traceLog) context.Context { + return t.NewContext(ctx, tr) +} + +func newTraceEventLog(family, title string) traceEventLog { + return t.NewEventLog(family, title) +} diff --git a/version.go b/version.go new file mode 100644 index 0000000..75a21e9 --- /dev/null +++ b/version.go @@ -0,0 +1,22 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +// Version is the current grpc version. +const Version = "1.85.0-dev" diff --git a/xds/bootstrap/bootstrap.go b/xds/bootstrap/bootstrap.go new file mode 100644 index 0000000..3b7d1e9 --- /dev/null +++ b/xds/bootstrap/bootstrap.go @@ -0,0 +1,102 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package bootstrap provides the functionality to register possible options +// for aspects of the xDS client through the bootstrap file. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed +// in a later release. +package bootstrap + +import ( + "encoding/json" + + "google.golang.org/grpc/credentials" +) + +// channelCredsRegistry is a map from channel credential type name to +// ChannelCredential builder. +var channelCredsRegistry = make(map[string]ChannelCredentials) + +// callCredsRegistry is a map from call credential type name to +// ChannelCredential builder. +var callCredsRegistry = make(map[string]CallCredentials) + +// ChannelCredentials interface encapsulates a credentials.Bundle builder +// that can be used for communicating with the xDS Management server. +type ChannelCredentials interface { + // Build returns a credential bundle associated with this credential, and a + // function to clean up any additional resources associated with this bundle + // when it is no longer needed. + Build(config json.RawMessage) (credentials.Bundle, func(), error) + // Name returns the credential name associated with this credential. + Name() string +} + +// RegisterChannelCredentials registers ChannelCredentials used for connecting +// to the xDS management server. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple credentials are +// registered with the same name, the one registered last will take effect. +func RegisterChannelCredentials(c ChannelCredentials) { + channelCredsRegistry[c.Name()] = c +} + +// GetChannelCredentials returns the credentials associated with a given name. +// If no credentials are registered with the name, nil will be returned. +func GetChannelCredentials(name string) ChannelCredentials { + if c, ok := channelCredsRegistry[name]; ok { + return c + } + + return nil +} + +// CallCredentials interface encapsulates a credentials.PerRPCCredentials +// builder that can be used for communicating with the xDS Management server. +type CallCredentials interface { + // Build returns a PerRPCCredentials created from the provided + // configuration, and a function to clean up any additional resources + // associated with them when they are no longer needed. + Build(config json.RawMessage) (credentials.PerRPCCredentials, func(), error) + // Name returns the credential name associated with this credential. + Name() string +} + +// RegisterCallCredentials registers CallCredentials used for connecting +// to the xDS management server. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple credentials are +// registered with the same name, the one registered last will take effect. +func RegisterCallCredentials(c CallCredentials) { + callCredsRegistry[c.Name()] = c +} + +// GetCallCredentials returns the credentials associated with a given name. +// If no credentials are registered with the name, nil will be returned. +func GetCallCredentials(name string) CallCredentials { + if c, ok := callCredsRegistry[name]; ok { + return c + } + + return nil +} diff --git a/xds/bootstrap/bootstrap_test.go b/xds/bootstrap/bootstrap_test.go new file mode 100644 index 0000000..8cbc5bc --- /dev/null +++ b/xds/bootstrap/bootstrap_test.go @@ -0,0 +1,119 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package bootstrap + +import ( + "encoding/json" + "testing" + + "google.golang.org/grpc/credentials" +) + +const testCredsBuilderName = "test_creds" + +var builder = &testCredsBuilder{} + +func init() { + RegisterChannelCredentials(builder) +} + +type testCredsBuilder struct { + config json.RawMessage +} + +func (t *testCredsBuilder) Build(config json.RawMessage) (credentials.Bundle, func(), error) { + t.config = config + return nil, nil, nil +} + +func (t *testCredsBuilder) Name() string { + return testCredsBuilderName +} + +func TestRegisterNew(t *testing.T) { + c := GetChannelCredentials(testCredsBuilderName) + if c == nil { + t.Fatalf("GetCredentials(%q) credential = nil", testCredsBuilderName) + } + + const sampleConfig = "sample_config" + rawMessage := json.RawMessage(sampleConfig) + if _, _, err := c.Build(rawMessage); err != nil { + t.Errorf("Build(%v) error = %v, want nil", rawMessage, err) + } + + if got, want := string(builder.config), sampleConfig; got != want { + t.Errorf("Build config = %v, want %v", got, want) + } +} + +func TestChannelCredsBuilders(t *testing.T) { + tests := []struct { + typename string + builder ChannelCredentials + }{ + {"google_default", &googleDefaultCredsBuilder{}}, + {"insecure", &insecureCredsBuilder{}}, + {"tls", &tlsCredsBuilder{}}, + } + + for _, test := range tests { + t.Run(test.typename, func(t *testing.T) { + if got, want := test.builder.Name(), test.typename; got != want { + t.Errorf("%T.Name = %v, want %v", test.builder, got, want) + } + + bundle, stop, err := test.builder.Build(nil) + if err != nil { + t.Fatalf("%T.Build failed: %v", test.builder, err) + } + if bundle == nil { + t.Errorf("%T.Build returned nil bundle, expected non-nil", test.builder) + } + stop() + }) + } +} + +func TestJWTCallCredsBuilder(t *testing.T) { + builder := &jwtCallCredsBuilder{} + config := json.RawMessage(`{"jwt_token_file":"/path/to/token.jwt"}`) + + creds, stop, err := builder.Build(config) + if err != nil { + t.Fatalf("Build(%s) failed: %v", config, err) + } + defer stop() + if creds == nil { + t.Errorf("Build(%s) returned nil creds, expected non-nil", config) + } +} + +func TestTlsCredsBuilder(t *testing.T) { + tls := &tlsCredsBuilder{} + _, stop, err := tls.Build(json.RawMessage(`{}`)) + if err != nil { + t.Fatalf("tls.Build() failed with error %s when expected to succeed", err) + } + defer stop() + + if _, stop, err := tls.Build(json.RawMessage(`{"ca_certificate_file":"/ca_certificates.pem","refresh_interval": "asdf"}`)); err == nil { + defer stop() + t.Errorf("tls.Build() succeeded with an invalid refresh interval, when expected to fail") + } +} diff --git a/xds/bootstrap/credentials.go b/xds/bootstrap/credentials.go new file mode 100644 index 0000000..2547759 --- /dev/null +++ b/xds/bootstrap/credentials.go @@ -0,0 +1,85 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package bootstrap + +import ( + "encoding/json" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/xds/bootstrap/jwtcreds" + "google.golang.org/grpc/internal/xds/bootstrap/tlscreds" +) + +func init() { + RegisterChannelCredentials(&insecureCredsBuilder{}) + RegisterChannelCredentials(&googleDefaultCredsBuilder{}) + RegisterChannelCredentials(&tlsCredsBuilder{}) + + RegisterCallCredentials(&jwtCallCredsBuilder{}) +} + +// insecureCredsBuilder implements the `ChannelCredentials` interface defined in +// package `xds/bootstrap` and encapsulates an insecure credential. +type insecureCredsBuilder struct{} + +func (i *insecureCredsBuilder) Build(json.RawMessage) (credentials.Bundle, func(), error) { + return insecure.NewBundle(), func() {}, nil +} + +func (i *insecureCredsBuilder) Name() string { + return "insecure" +} + +// tlsCredsBuilder implements the `ChannelCredentials` interface defined in +// package `xds/bootstrap` and encapsulates a TLS credential. +type tlsCredsBuilder struct{} + +func (t *tlsCredsBuilder) Build(config json.RawMessage) (credentials.Bundle, func(), error) { + return tlscreds.NewBundle(config) +} + +func (t *tlsCredsBuilder) Name() string { + return "tls" +} + +// googleDefaultCredsBuilder implements the `ChannelCredentials` interface defined in +// package `xds/bootstrap` and encapsulates a Google Default credential. +type googleDefaultCredsBuilder struct{} + +func (d *googleDefaultCredsBuilder) Build(json.RawMessage) (credentials.Bundle, func(), error) { + return google.NewDefaultCredentials(), func() {}, nil +} + +func (d *googleDefaultCredsBuilder) Name() string { + return "google_default" +} + +// jwtCallCredsBuilder implements the `CallCredentials` interface defined in +// package `xds/bootstrap` and encapsulates JWT call credentials. +type jwtCallCredsBuilder struct{} + +func (j *jwtCallCredsBuilder) Build(configJSON json.RawMessage) (credentials.PerRPCCredentials, func(), error) { + return jwtcreds.NewCallCredentials(configJSON) +} + +func (j *jwtCallCredsBuilder) Name() string { + return "jwt_token_file" +} diff --git a/xds/csds/csds.go b/xds/csds/csds.go new file mode 100644 index 0000000..603819d --- /dev/null +++ b/xds/csds/csds.go @@ -0,0 +1,108 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package csds implements features to dump the status (xDS responses) the +// xds_client is using. +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a later +// release. +package csds + +import ( + "context" + "fmt" + "io" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/status" + + v3statusgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" +) + +var logger = grpclog.Component("xds") + +const prefix = "[csds-server %p] " + +func prefixLogger(s *ClientStatusDiscoveryServer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, s)) +} + +// ClientStatusDiscoveryServer provides an implementation of the Client Status +// Discovery Service (CSDS) for exposing the xDS config of a given client. See +// https://github.com/envoyproxy/envoy/blob/main/api/envoy/service/status/v3/csds.proto. +// +// For more details about the gRPC implementation of CSDS, refer to gRPC A40 at: +// https://github.com/grpc/proposal/blob/master/A40-csds-support.md. +type ClientStatusDiscoveryServer struct { + logger *internalgrpclog.PrefixLogger +} + +// NewClientStatusDiscoveryServer returns an implementation of the CSDS server +// that can be registered on a gRPC server. +func NewClientStatusDiscoveryServer() (*ClientStatusDiscoveryServer, error) { + s := &ClientStatusDiscoveryServer{} + s.logger = prefixLogger(s) + s.logger.Infof("Created CSDS server") + return s, nil +} + +// StreamClientStatus implements interface ClientStatusDiscoveryServiceServer. +func (s *ClientStatusDiscoveryServer) StreamClientStatus(stream v3statusgrpc.ClientStatusDiscoveryService_StreamClientStatusServer) error { + for { + req, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + resp, err := s.buildClientStatusRespForReq(req) + if err != nil { + return err + } + if err := stream.Send(resp); err != nil { + return err + } + } +} + +// FetchClientStatus implements interface ClientStatusDiscoveryServiceServer. +func (s *ClientStatusDiscoveryServer) FetchClientStatus(_ context.Context, req *v3statuspb.ClientStatusRequest) (*v3statuspb.ClientStatusResponse, error) { + return s.buildClientStatusRespForReq(req) +} + +// buildClientStatusRespForReq fetches the status of xDS resources from the +// xdsclient, and returns the response to be sent back to the csds client. +// +// If it returns an error, the error is a status error. +func (s *ClientStatusDiscoveryServer) buildClientStatusRespForReq(req *v3statuspb.ClientStatusRequest) (*v3statuspb.ClientStatusResponse, error) { + // Field NodeMatchers is unsupported, by design + // https://github.com/grpc/proposal/blob/master/A40-csds-support.md#detail-node-matching. + if len(req.NodeMatchers) != 0 { + return nil, status.Errorf(codes.InvalidArgument, "node_matchers are not supported, request contains node_matchers: %v", req.NodeMatchers) + } + + return xdsclient.DumpResources(), nil +} + +// Close cleans up the resources. +func (s *ClientStatusDiscoveryServer) Close() {} diff --git a/xds/csds/csds_e2e_test.go b/xds/csds/csds_e2e_test.go new file mode 100644 index 0000000..252e693 --- /dev/null +++ b/xds/csds/csds_e2e_test.go @@ -0,0 +1,699 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package csds_test + +import ( + "context" + "fmt" + "io" + "slices" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" + "google.golang.org/grpc/xds/csds" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + + v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + v3statuspbgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter +) + +const defaultTestTimeout = 5 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// The following watcher implementations are no-ops since we don't really care +// about the callback received by these watchers in the test. We only care +// whether CSDS reports the expected state. + +type nopListenerWatcher struct{} + +func (nopListenerWatcher) ResourceChanged(_ *xdsresource.ListenerUpdate, onDone func()) { + onDone() +} +func (nopListenerWatcher) ResourceError(_ error, onDone func()) { + onDone() +} +func (nopListenerWatcher) AmbientError(_ error, onDone func()) { + onDone() +} + +type nopRouteConfigWatcher struct{} + +func (nopRouteConfigWatcher) ResourceChanged(_ *xdsresource.RouteConfigUpdate, onDone func()) { + onDone() +} +func (nopRouteConfigWatcher) ResourceError(_ error, onDone func()) { + onDone() +} +func (nopRouteConfigWatcher) AmbientError(_ error, onDone func()) { + onDone() +} + +type nopClusterWatcher struct{} + +func (nopClusterWatcher) ResourceChanged(_ *xdsresource.ClusterUpdate, onDone func()) { + onDone() +} +func (nopClusterWatcher) ResourceError(_ error, onDone func()) { + onDone() +} +func (nopClusterWatcher) AmbientError(_ error, onDone func()) { + onDone() +} + +type nopEndpointsWatcher struct{} + +func (nopEndpointsWatcher) ResourceChanged(_ *xdsresource.EndpointsUpdate, onDone func()) { + onDone() +} +func (nopEndpointsWatcher) ResourceError(_ error, onDone func()) { + onDone() +} +func (nopEndpointsWatcher) AmbientError(_ error, onDone func()) { + onDone() +} + +// This watcher writes the onDone callback on to a channel for the test to +// invoke it when it wants to unblock the next read on the ADS stream in the xDS +// client. This is particularly useful when a resource is NACKed, because the +// go-control-plane management server continuously resends the same resource in +// this case, and applying flow control from these watchers ensures that xDS +// client does not spend all of its time receiving and NACKing updates from the +// management server. This was indeed the case on arm64 (before we had support +// for ADS stream level flow control), and was causing CSDS to not receive any +// updates from the xDS client. +type blockingListenerWatcher struct { + testCtxDone <-chan struct{} // Closed when the test is done. + onDoneCh chan func() // Channel to write the onDone callback to. +} + +func newBlockingListenerWatcher(testCtxDone <-chan struct{}) *blockingListenerWatcher { + return &blockingListenerWatcher{ + testCtxDone: testCtxDone, + onDoneCh: make(chan func(), 1), + } +} + +func (w *blockingListenerWatcher) ResourceChanged(_ *xdsresource.ListenerUpdate, onDone func()) { + writeOnDone(w.testCtxDone, w.onDoneCh, onDone) +} +func (w *blockingListenerWatcher) ResourceError(_ error, onDone func()) { + writeOnDone(w.testCtxDone, w.onDoneCh, onDone) +} +func (w *blockingListenerWatcher) AmbientError(_ error, onDone func()) { + writeOnDone(w.testCtxDone, w.onDoneCh, onDone) +} + +// writeOnDone attempts to write the onDone callback on the onDone channel. It +// returns when it can successfully write to the channel or when the test is +// done, which is signalled by testCtxDone being closed. +func writeOnDone(testCtxDone <-chan struct{}, onDoneCh chan func(), onDone func()) { + select { + case <-testCtxDone: + case onDoneCh <- onDone: + } +} + +// Creates a gRPC server and starts serving a CSDS service implementation on it. +// Returns the address of the newly created gRPC server. +// +// Registers cleanup functions on t to stop the gRPC server and the CSDS +// implementation. +func startCSDSServer(t *testing.T) string { + t.Helper() + + server := grpc.NewServer() + t.Cleanup(server.Stop) + + csdss, err := csds.NewClientStatusDiscoveryServer() + if err != nil { + t.Fatalf("Failed to create CSDS service implementation: %v", err) + } + v3statuspbgrpc.RegisterClientStatusDiscoveryServiceServer(server, csdss) + t.Cleanup(csdss.Close) + + // Create a local listener and pass it to Serve(). + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + go func() { + if err := server.Serve(lis); err != nil { + t.Errorf("Serve() failed: %v", err) + } + }() + return lis.Addr().String() +} + +func startCSDSClientStream(ctx context.Context, t *testing.T, serverAddr string) v3statuspbgrpc.ClientStatusDiscoveryService_StreamClientStatusClient { + conn, err := grpc.NewClient(serverAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial CSDS server %q: %v", serverAddr, err) + } + + client := v3statuspbgrpc.NewClientStatusDiscoveryServiceClient(conn) + stream, err := client.StreamClientStatus(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("Failed to create a stream for CSDS: %v", err) + } + t.Cleanup(func() { conn.Close() }) + return stream +} + +// Tests CSDS functionality. The test performs the following: +// - Spins up a management server and creates two xDS clients talking to it. +// - Registers a set of watches on the xDS clients, and verifies that the CSDS +// response reports resources in REQUESTED state. +// - Configures resources on the management server corresponding to the ones +// being watched by the clients, and verifies that the CSDS response reports +// resources in ACKED state. +// +// For the above operations, the test also verifies that the client_scope field +// in the CSDS response is populated appropriately. +func (s) TestCSDS(t *testing.T) { + // Spin up a xDS management server on a local port. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create a bootstrap contents pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + // We use the default xDS client pool here because the CSDS service reports + // on the state of the default xDS client which is implicitly managed + // within the xdsclient.DefaultPool. + xdsclient.DefaultPool.SetFallbackBootstrapConfig(config) + defer func() { xdsclient.DefaultPool.UnsetBootstrapConfigForTesting() }() + // Create two xDS clients, with different names. These should end up + // creating two different xDS clients. + const xdsClient1Name = "xds-csds-client-1" + xdsClient1, xdsClose1, err := xdsclient.DefaultPool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: xdsClient1Name, + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer xdsClose1() + const xdsClient2Name = "xds-csds-client-2" + xdsClient2, xdsClose2, err := xdsclient.DefaultPool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: xdsClient2Name, + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer xdsClose2() + + // Start a CSDS server and create a client stream to it. + addr := startCSDSServer(t) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream := startCSDSClientStream(ctx, t, addr) + + // Verify that the xDS client reports an empty config. + wantNode := &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + } + wantResp := &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + ClientScope: xdsClient1Name, + }, + { + Node: wantNode, + ClientScope: xdsClient2Name, + }, + }, + } + if err := checkClientStatusResponse(ctx, stream, wantResp); err != nil { + t.Fatal(err) + } + + // Initialize the xDS resources to be used in this test. + ldsTargets := []string{"lds.target.good:0000", "lds.target.good:1111"} + rdsTargets := []string{"route-config-0", "route-config-1"} + cdsTargets := []string{"cluster-0", "cluster-1"} + edsTargets := []string{"endpoints-0", "endpoints-1"} + listeners := make([]*v3listenerpb.Listener, len(ldsTargets)) + listenerAnys := make([]*anypb.Any, len(ldsTargets)) + for i := range ldsTargets { + listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i]) + listenerAnys[i] = testutils.MarshalAny(t, listeners[i]) + } + routes := make([]*v3routepb.RouteConfiguration, len(rdsTargets)) + routeAnys := make([]*anypb.Any, len(rdsTargets)) + for i := range rdsTargets { + routes[i] = e2e.DefaultRouteConfig(rdsTargets[i], ldsTargets[i], cdsTargets[i]) + routeAnys[i] = testutils.MarshalAny(t, routes[i]) + } + clusters := make([]*v3clusterpb.Cluster, len(cdsTargets)) + clusterAnys := make([]*anypb.Any, len(cdsTargets)) + for i := range cdsTargets { + clusters[i] = e2e.DefaultCluster(cdsTargets[i], edsTargets[i], e2e.SecurityLevelNone) + clusterAnys[i] = testutils.MarshalAny(t, clusters[i]) + } + endpoints := make([]*v3endpointpb.ClusterLoadAssignment, len(edsTargets)) + endpointAnys := make([]*anypb.Any, len(edsTargets)) + ips := []string{"0.0.0.0", "1.1.1.1"} + ports := []uint32{123, 456} + for i := range edsTargets { + endpoints[i] = e2e.DefaultEndpoint(edsTargets[i], ips[i], ports[i:i+1]) + endpointAnys[i] = testutils.MarshalAny(t, endpoints[i]) + } + + // Register watches on the xDS clients for two resources of each type. + for _, xdsC := range []xdsclient.XDSClient{xdsClient1, xdsClient2} { + for _, target := range ldsTargets { + xdsresource.WatchListener(xdsC, target, nopListenerWatcher{}) + } + for _, target := range rdsTargets { + xdsresource.WatchRouteConfig(xdsC, target, nopRouteConfigWatcher{}) + } + for _, target := range cdsTargets { + xdsresource.WatchCluster(xdsC, target, nopClusterWatcher{}) + } + for _, target := range edsTargets { + xdsresource.WatchEndpoints(xdsC, target, nopEndpointsWatcher{}) + } + } + + // Verify that the xDS client reports the resources as being in "Requested" + // state, and in version "0". + wantConfigs := []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + ClientScope: xdsClient1Name, + }, + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + ClientScope: xdsClient2Name, + }, + }, + } + if err := checkClientStatusResponse(ctx, stream, wantResp); err != nil { + t.Fatal(err) + } + + // Configure the management server with two resources of each type, + // corresponding to the watches registered above. + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + }); err != nil { + t.Fatal(err) + } + + // Verify that the xDS client reports the resources as being in "ACKed" + // state, and in version "1". + wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[0], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[1], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[0], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[1], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[0], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[0], nil), + makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[1], nil), + } + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + ClientScope: xdsClient1Name, + }, + { + Node: wantNode, + GenericXdsConfigs: wantConfigs, + ClientScope: xdsClient2Name, + }, + }, + } + if err := checkClientStatusResponse(ctx, stream, wantResp); err != nil { + t.Fatal(err) + } +} + +// Tests CSDS functionality. The test performs the following: +// - Spins up a management server and creates two xDS clients talking to it. +// - Registers one watch on each xDS client, and verifies that the CSDS +// response reports resources in REQUESTED state. +// - Configures two resources on the management server and verifies that the +// CSDS response reports the resources as being in ACKED state. +// - Updates one of two resources on the management server such that it is +// expected to be NACKed by the client. Verifies that the CSDS response +// contains one resource in ACKED state and one in NACKED state. +// +// For the above operations, the test also verifies that the client_scope field +// in the CSDS response is populated appropriately. +// +// This test does a bunch of similar things to the previous test, but has +// reduced complexity because of having to deal with a single resource type. +// This makes it possible to test the NACKing a resource (which results in +// continuous resending of the resource by the go-control-plane management +// server), in an easier and less flaky way. +func (s) TestCSDS_NACK(t *testing.T) { + // Spin up a xDS management server on a local port. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create a bootstrap contents pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + // We use the default xDS client pool here because the CSDS service reports + // on the state of the default xDS client which is implicitly managed + // within the xdsclient.DefaultPool. + xdsclient.DefaultPool.SetFallbackBootstrapConfig(config) + defer func() { xdsclient.DefaultPool.UnsetBootstrapConfigForTesting() }() + // Create two xDS clients, with different names. These should end up + // creating two different xDS clients. + const xdsClient1Name = "xds-csds-client-1" + xdsClient1, xdsClose1, err := xdsclient.DefaultPool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: xdsClient1Name, + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer xdsClose1() + const xdsClient2Name = "xds-csds-client-2" + xdsClient2, xdsClose2, err := xdsclient.DefaultPool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: xdsClient2Name, + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer xdsClose2() + + // Start a CSDS server and create a client stream to it. + addr := startCSDSServer(t) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream := startCSDSClientStream(ctx, t, addr) + + // Verify that the xDS client reports an empty config. + wantNode := &v3corepb.Node{ + Id: nodeID, + UserAgentName: "gRPC Go", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"}, + } + wantResp := &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + ClientScope: xdsClient1Name, + }, + { + Node: wantNode, + ClientScope: xdsClient2Name, + }, + }, + } + if err := checkClientStatusResponse(ctx, stream, wantResp); err != nil { + t.Fatal(err) + } + + // Initialize the xDS resources to be used in this test. + const ldsTarget0, ldsTarget1 = "lds.target.good:0000", "lds.target.good:1111" + listener0 := e2e.DefaultClientListener(ldsTarget0, "rds-name") + listener1 := e2e.DefaultClientListener(ldsTarget1, "rds-name") + listenerAny0 := testutils.MarshalAny(t, listener0) + listenerAny1 := testutils.MarshalAny(t, listener1) + + // Register the watchers, one for each xDS client. + watcher1 := nopListenerWatcher{} + watcher2 := newBlockingListenerWatcher(ctx.Done()) + xdsresource.WatchListener(xdsClient1, ldsTarget0, watcher1) + xdsresource.WatchListener(xdsClient2, ldsTarget1, watcher2) + + // Verify that the xDS client reports the resources as being in "Requested" + // state, and in version "0". + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTarget0, "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + }, + ClientScope: xdsClient1Name, + }, + { + Node: wantNode, + GenericXdsConfigs: []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTarget1, "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil), + }, + ClientScope: xdsClient2Name, + }, + }, + } + if err := checkClientStatusResponse(ctx, stream, wantResp); err != nil { + t.Fatal(err) + } + + // Configure the management server with two listener resources corresponding + // to the watches registered above. + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener0, listener1}, + SkipValidation: true, + }); err != nil { + t.Fatal(err) + } + + // Verify that the xDS client reports the resources as being in "ACKed" + // state, and in version "1". + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTarget0, "1", v3adminpb.ClientResourceStatus_ACKED, listenerAny0, nil), + }, + ClientScope: xdsClient1Name, + }, + { + Node: wantNode, + GenericXdsConfigs: []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTarget1, "1", v3adminpb.ClientResourceStatus_ACKED, listenerAny1, nil), + }, + ClientScope: xdsClient2Name, + }, + }, + } + if err := checkClientStatusResponse(ctx, stream, wantResp); err != nil { + t.Fatal(err) + } + + // Unblock reads on the ADS stream by calling the onDone callback sent to + // the watcher. + select { + case <-ctx.Done(): + t.Fatal("Timed out waiting for watch callback") + case onDone := <-watcher2.onDoneCh: + onDone() + } + + // Update the second resource with an empty ApiListener field which is + // expected to be NACK'ed by the xDS client. + listener1.ApiListener = nil + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener0, listener1}, + SkipValidation: true, + }); err != nil { + t.Fatal(err) + } + + // Wait for the update to reach the watchers. + select { + case <-ctx.Done(): + t.Fatal("Timed out waiting for watch callback") + case onDone := <-watcher2.onDoneCh: + onDone() + } + + // Verify that the xDS client reports the first listener resource as being + // ACKed and the second listener resource as being NACKed. The version for + // the ACKed resource would be "2", while that for the NACKed resource would + // be "1". In the NACKed resource, the version which is NACKed is stored in + // the ErrorState field. + wantResp = &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: wantNode, + GenericXdsConfigs: []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTarget0, "2", v3adminpb.ClientResourceStatus_ACKED, listenerAny0, nil), + }, + ClientScope: xdsClient1Name, + }, + { + Node: wantNode, + GenericXdsConfigs: []*v3statuspb.ClientConfig_GenericXdsConfig{ + makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTarget1, "1", v3adminpb.ClientResourceStatus_NACKED, listenerAny1, &v3adminpb.UpdateFailureState{VersionInfo: "2"}), + }, + ClientScope: xdsClient2Name, + }, + }, + } + if err := checkClientStatusResponse(ctx, stream, wantResp); err != nil { + t.Fatal(err) + } +} + +func makeGenericXdsConfig(typeURL, name, version string, status v3adminpb.ClientResourceStatus, config *anypb.Any, failure *v3adminpb.UpdateFailureState) *v3statuspb.ClientConfig_GenericXdsConfig { + return &v3statuspb.ClientConfig_GenericXdsConfig{ + TypeUrl: typeURL, + Name: name, + VersionInfo: version, + ClientStatus: status, + XdsConfig: config, + ErrorState: failure, + } +} + +// Repeatedly sends CSDS requests and receives CSDS responses on the provided +// stream and verifies that the response matches `want`. Returns an error if +// sending or receiving on the stream fails, or if the context expires before a +// response matching `want` is received. +// +// Expects client configs in `want` to be sorted on `client_scope` and the +// resource dump to be sorted on type_url and resource name. +func checkClientStatusResponse(ctx context.Context, stream v3statuspbgrpc.ClientStatusDiscoveryService_StreamClientStatusClient, want *v3statuspb.ClientStatusResponse) error { + var cmpOpts = cmp.Options{ + protocmp.Transform(), + protocmp.IgnoreFields((*v3statuspb.ClientConfig_GenericXdsConfig)(nil), "last_updated"), + protocmp.IgnoreFields((*v3adminpb.UpdateFailureState)(nil), "last_update_attempt", "details"), + } + + var lastErr error + for ; ctx.Err() == nil; <-time.After(100 * time.Millisecond) { + if err := stream.Send(&v3statuspb.ClientStatusRequest{Node: nil}); err != nil { + if err != io.EOF { + return fmt.Errorf("failed to send ClientStatusRequest: %v", err) + } + // If the stream has closed, we call Recv() until it returns a non-nil + // error to get the actual error on the stream. + for { + if _, err := stream.Recv(); err != nil { + return fmt.Errorf("failed to recv ClientStatusResponse: %v", err) + } + } + } + got, err := stream.Recv() + if err != nil { + return fmt.Errorf("failed to recv ClientStatusResponse: %v", err) + } + // Sort the client configs based on the `client_scope` field. + slices.SortFunc(got.GetConfig(), func(a, b *v3statuspb.ClientConfig) int { + return strings.Compare(a.ClientScope, b.ClientScope) + }) + // Sort the resource configs based on the type_url and name fields. + for _, cc := range got.GetConfig() { + slices.SortFunc(cc.GetGenericXdsConfigs(), func(a, b *v3statuspb.ClientConfig_GenericXdsConfig) int { + if strings.Compare(a.TypeUrl, b.TypeUrl) == 0 { + return strings.Compare(a.Name, b.Name) + } + return strings.Compare(a.TypeUrl, b.TypeUrl) + }) + } + diff := cmp.Diff(want, got, cmpOpts) + if diff == "" { + return nil + } + lastErr = fmt.Errorf("received unexpected resource dump, diff (-got, +want):\n%s, got: %s\n want:%s", diff, pretty.ToJSON(got), pretty.ToJSON(want)) + } + return fmt.Errorf("timeout when waiting for resource dump to reach expected state: ctxErr: %v, otherErr: %v", ctx.Err(), lastErr) +} + +func (s) TestCSDSNoXDSClient(t *testing.T) { + // Create a bootstrap file in a temporary directory. Since we pass an empty + // bootstrap configuration, it will fail xDS client creation because the + // `server_uri` field is unset. + testutils.CreateBootstrapFileForTesting(t, []byte(``)) + + // Start a CSDS server and create a client stream to it. + addr := startCSDSServer(t) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream := startCSDSClientStream(ctx, t, addr) + + if err := stream.Send(&v3statuspb.ClientStatusRequest{Node: nil}); err != nil { + t.Fatalf("Failed to send ClientStatusRequest: %v", err) + } + r, err := stream.Recv() + if err != nil { + // io.EOF is not ok. + t.Fatalf("Failed to recv ClientStatusResponse: %v", err) + } + if n := len(r.Config); n != 0 { + t.Fatalf("got %d configs, want 0: %v", n, prototext.Format(r)) + } +} diff --git a/xds/googledirectpath/googlec2p.go b/xds/googledirectpath/googlec2p.go new file mode 100644 index 0000000..ed0e8e2 --- /dev/null +++ b/xds/googledirectpath/googlec2p.go @@ -0,0 +1,250 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package googledirectpath implements a resolver that configures xds to make +// cloud to prod directpath connection. +// +// It's a combo of DNS and xDS resolvers. It delegates to DNS if +// - not on GCE, or +// - xDS bootstrap env var is set (so this client needs to do normal xDS, not +// direct path, and clients with this scheme is not part of the xDS mesh). +package googledirectpath + +import ( + "encoding/json" + "fmt" + rand "math/rand/v2" + "net/url" + "sync" + "time" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/googlecloud" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/resolver" + + _ "google.golang.org/grpc/xds" // To register xds resolvers and balancers. +) + +const ( + c2pScheme = "google-c2p" + c2pAuthority = "traffic-director-c2p.xds.googleapis.com" + + defaultUniverseDomain = "googleapis.com" + zoneURL = "http://metadata.google.internal/computeMetadata/v1/instance/zone" + ipv6URL = "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ipv6s" + ipv6CapableMetadataName = "TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE" + httpReqTimeout = 10 * time.Second + + logPrefix = "[google-c2p-resolver]" + dnsName, xdsName = "dns", "xds" +) + +var ( + logger = internalgrpclog.NewPrefixLogger(grpclog.Component("directpath"), logPrefix) + universeDomainMu sync.Mutex + universeDomain = "" + // For overriding in unittests. + onGCE = googlecloud.OnGCE + randInt = rand.Int + xdsClientPool = xdsclient.DefaultPool +) + +func init() { + resolver.Register(c2pResolverBuilder{}) +} + +// SetUniverseDomain informs the gRPC library of the universe domain +// in which the process is running (for example, "googleapis.com"). +// It is the caller's responsibility to ensure that the domain is correct. +// +// This setting is used by the "google-c2p" resolver (the resolver used +// for URIs with the "google-c2p" scheme) to configure its dependencies. +// +// If a gRPC channel is created with the "google-c2p" URI scheme and this +// function has NOT been called, then gRPC configures the universe domain as +// "googleapis.com". +// +// Returns nil if either: +// +// a) The universe domain has not yet been configured. +// b) The universe domain has been configured and matches the provided value. +// +// Otherwise, returns an error. +func SetUniverseDomain(domain string) error { + universeDomainMu.Lock() + defer universeDomainMu.Unlock() + if domain == "" { + return fmt.Errorf("universe domain cannot be empty") + } + if universeDomain == "" { + universeDomain = domain + return nil + } + if universeDomain != domain { + return fmt.Errorf("universe domain cannot be set to %s, already set to different value: %s", domain, universeDomain) + } + return nil +} + +func getXdsServerURI() string { + universeDomainMu.Lock() + defer universeDomainMu.Unlock() + if universeDomain == "" { + universeDomain = defaultUniverseDomain + } + // Put env var override logic after default value logic so + // that tests still run the default value logic. + if envconfig.C2PResolverTestOnlyTrafficDirectorURI != "" { + return envconfig.C2PResolverTestOnlyTrafficDirectorURI + } + return fmt.Sprintf("dns:///directpath-pa.%s", universeDomain) +} + +type c2pResolverWrapper struct { + resolver.Resolver + cancel func() // Release the reference to the xDS client that was created in Build(). +} + +func (r *c2pResolverWrapper) Close() { + r.Resolver.Close() + r.cancel() +} + +type c2pResolverBuilder struct{} + +func (c2pResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { + if t.URL.Host != "" { + return nil, fmt.Errorf("google-c2p URI scheme does not support authorities") + } + + isGCE := onGCE() + // We check only for the presence of the "force-xds" query parameter key, + // and do not evaluate its value. + _, forceXDS := t.URL.Query()["force-xds"] + if !forceXDS && !isGCE { + // If not xDS, fallback to DNS. + t.URL.Scheme = dnsName + return resolver.Get(dnsName).Build(t, cc, opts) + } + + var zone string + var ipv6Capable bool + if forceXDS { + // If the force-xds query parameter is present, GCE metadata server + // queries are bypassed. + zone = "" + ipv6Capable = true + } else { + // Note that the following calls to getZone() and getIPv6Capable() does I/O, + // and has 10 seconds timeout each. + // + // This should be fine in most of the cases. In certain error cases, this + // could block Dial() for up to 10 seconds (each blocking call has its own + // goroutine). + var wg sync.WaitGroup + wg.Go(func() { zone = getZone(httpReqTimeout) }) + wg.Go(func() { ipv6Capable = getIPv6Capable(httpReqTimeout) }) + wg.Wait() + } + + xdsServerURI := getXdsServerURI() + nodeCfg := newNodeConfig(zone, ipv6Capable, forceXDS) + xdsServerCfg := newXdsServerConfig(xdsServerURI) + authoritiesCfg := newAuthoritiesConfig(xdsServerCfg) + + cfg := map[string]any{ + "xds_servers": []any{xdsServerCfg}, + "client_default_listener_resource_name_template": "%s", + "authorities": authoritiesCfg, + "node": nodeCfg, + } + cfgJSON, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("failed to marshal bootstrap configuration: %v", err) + } + config, err := bootstrap.NewConfigFromContents(cfgJSON) + if err != nil { + return nil, fmt.Errorf("failed to parse bootstrap contents: %s, %v", string(cfgJSON), err) + } + + t = resolver.Target{ + URL: url.URL{ + Scheme: xdsName, + Host: c2pAuthority, + Path: t.URL.Path, + }, + } + + // Create a new xDS client for this target using the provided bootstrap + // configuration. This client is stored in the xdsclient pool’s internal + // cache, keeping it alive and associated with this resolver until Closed(). + // While the c2p resolver itself does not directly use the client, creating + // it ensures that when the xDS resolver later requests a client for the + // same target, the existing instance will be reused. + _, cancel, err := xdsClientPool.NewClientWithConfig(t.String(), opts.MetricsRecorder, config) + if err != nil { + return nil, fmt.Errorf("failed to create xds client: %v", err) + } + + r, err := resolver.Get(xdsName).Build(t, cc, opts) + if err != nil { + cancel() + return nil, err + } + return &c2pResolverWrapper{Resolver: r, cancel: cancel}, nil +} + +func (b c2pResolverBuilder) Scheme() string { + return c2pScheme +} + +func newNodeConfig(zone string, ipv6Capable bool, forceXDS bool) map[string]any { + prefix := "C2P" + if forceXDS { + prefix = "C2P-non-gcp" + } + node := map[string]any{ + "id": fmt.Sprintf("%s-%d", prefix, randInt()), + } + if zone != "" { + node["locality"] = map[string]any{"zone": zone} + } + // Enable dualstack endpoints in TD. + if ipv6Capable { + node["metadata"] = map[string]any{ipv6CapableMetadataName: true} + } + return node +} + +func newAuthoritiesConfig(serverCfg map[string]any) map[string]any { + return map[string]any{ + c2pAuthority: map[string]any{"xds_servers": []any{serverCfg}}, + } +} + +func newXdsServerConfig(uri string) map[string]any { + return map[string]any{ + "server_uri": uri, + "channel_creds": []map[string]any{{"type": "google_default"}}, + "server_features": []any{"ignore_resource_deletion"}, + } +} diff --git a/xds/googledirectpath/googlec2p_test.go b/xds/googledirectpath/googlec2p_test.go new file mode 100644 index 0000000..d0fafe7 --- /dev/null +++ b/xds/googledirectpath/googlec2p_test.go @@ -0,0 +1,768 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package googledirectpath + +import ( + "context" + "encoding/json" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/resolver" +) + +const defaultTestTimeout = 5 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type emptyResolver struct { + resolver.Resolver + scheme string +} + +func (er *emptyResolver) Build(_ resolver.Target, _ resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) { + return er, nil +} + +func (er *emptyResolver) Scheme() string { + return er.scheme +} + +func (er *emptyResolver) Close() {} + +var ( + testDNSResolver = &emptyResolver{scheme: "dns"} + testXDSResolver = &emptyResolver{scheme: "xds"} +) + +// replaceResolvers unregisters the real resolvers for schemes `dns` and `xds` +// and registers test resolvers instead. This allows the test to verify that +// expected resolvers are built. +func replaceResolvers(t *testing.T) { + oldDNS := resolver.Get("dns") + resolver.Register(testDNSResolver) + oldXDS := resolver.Get("xds") + resolver.Register(testXDSResolver) + t.Cleanup(func() { + resolver.Register(oldDNS) + resolver.Register(oldXDS) + }) +} + +func simulateRunningOnGCE(t *testing.T, gce bool) { + oldOnGCE := onGCE + onGCE = func() bool { return gce } + t.Cleanup(func() { onGCE = oldOnGCE }) +} + +// ensure universeDomain is set to the expected default, +// and clean it up again after the test. +func useCleanUniverseDomain(t *testing.T) { + universeDomainMu.Lock() + defer universeDomainMu.Unlock() + if universeDomain != "" { + t.Fatalf("universe domain unexpectedly initialized: %v", universeDomain) + } + t.Cleanup(func() { + universeDomainMu.Lock() + universeDomain = "" + universeDomainMu.Unlock() + }) +} + +// verifyXDSClientBootstrapConfig checks that an xDS client with the given name +// exists in the pool and that its bootstrap config matches the expected config. +func verifyXDSClientBootstrapConfig(t *testing.T, pool *xdsclient.Pool, name string, wantConfig *bootstrap.Config) { + t.Helper() + client, close, err := pool.GetClientForTesting(name) + if err != nil { + t.Fatalf("GetClientForTesting(%q) failed to get xds client: %v", name, err) + } + defer close() + + gotConfig := client.BootstrapConfig() + if gotConfig == nil { + t.Fatalf("Failed to get bootstrap config: %v", err) + } + if diff := cmp.Diff(wantConfig, gotConfig); diff != "" { + t.Fatalf("xdsClient.BootstrapConfig() for client %q returned unexpected diff (-want +got):\n%s", name, diff) + } +} + +// Tests the scenario where the bootstrap env vars are set and we're running on +// GCE. The test builds a google-c2p resolver and verifies that an xDS resolver +// is built and that we don't fallback to DNS (because federation is enabled by +// default). +func (s) TestBuildWithBootstrapEnvSet(t *testing.T) { + replaceResolvers(t) + simulateRunningOnGCE(t, true) + useCleanUniverseDomain(t) + + builder := resolver.Get(c2pScheme) + for i, envP := range []*string{&envconfig.XDSBootstrapFileName, &envconfig.XDSBootstrapFileContent} { + t.Run(strconv.Itoa(i), func(t *testing.T) { + // Set bootstrap config env var. + oldEnv := *envP + *envP = "does not matter" + defer func() { *envP = oldEnv }() + + // Override xDS client pool. + oldXdsClientPool := xdsClientPool + xdsClientPool = xdsclient.NewPool(nil) + defer func() { xdsClientPool = oldXdsClientPool }() + + // Build the google-c2p resolver. + r, err := builder.Build(resolver.Target{}, nil, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("failed to build resolver: %v", err) + } + defer r.Close() + + // Build should return wrapped xDS resolver, not DNS. + if r, ok := r.(*c2pResolverWrapper); !ok || r.Resolver != testXDSResolver { + t.Fatalf("Build() returned %#v, want c2pResolverWrapper", r) + } + }) + } +} + +// Tests the scenario where we are not running on GCE. The test builds a +// google-c2p resolver and verifies that we fallback to DNS. +func (s) TestBuildNotOnGCE(t *testing.T) { + replaceResolvers(t) + simulateRunningOnGCE(t, false) + useCleanUniverseDomain(t) + builder := resolver.Get(c2pScheme) + + // Build the google-c2p resolver. + r, err := builder.Build(resolver.Target{}, nil, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("failed to build resolver: %v", err) + } + defer r.Close() + + // Build should return DNS, not xDS. + if r != testDNSResolver { + t.Fatalf("Build() returned %#v, want dns resolver", r) + } +} + +func bootstrapConfig(t *testing.T, opts bootstrap.ConfigOptionsForTesting) *bootstrap.Config { + t.Helper() + + contents, err := bootstrap.NewContentsForTesting(opts) + if err != nil { + t.Fatalf("Failed to create bootstrap contents: %v", err) + } + cfg, err := bootstrap.NewConfigFromContents(contents) + if err != nil { + t.Fatalf("Failed to create bootstrap config: %v", err) + } + return cfg +} + +// Test that when a google-c2p resolver is built, the xDS client is built with +// the expected config. +func (s) TestBuildXDS(t *testing.T) { + replaceResolvers(t) + simulateRunningOnGCE(t, true) + useCleanUniverseDomain(t) + builder := resolver.Get(c2pScheme) + + // Override the zone returned by the metadata server. + oldGetZone := getZone + getZone = func(time.Duration) string { return "test-zone" } + defer func() { getZone = oldGetZone }() + + // Override the random func used in the node ID. + origRandInd := randInt + randInt = func() int { return 666 } + defer func() { randInt = origRandInd }() + + for _, tt := range []struct { + desc string + ipv6Capable bool + tdURIOverride string + wantBootstrapConfig *bootstrap.Config + }{ + { + desc: "ipv6 false", + wantBootstrapConfig: bootstrapConfig(t, bootstrap.ConfigOptionsForTesting{ + Servers: []byte(`[{ + "server_uri": "dns:///directpath-pa.googleapis.com", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + }]`), + Authorities: map[string]json.RawMessage{ + "traffic-director-c2p.xds.googleapis.com": []byte(`{ + "xds_servers": [ + { + "server_uri": "dns:///directpath-pa.googleapis.com", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + } + ] + }`), + }, + Node: []byte(`{ + "id": "C2P-666", + "locality": { + "zone": "test-zone" + } + }`), + }), + }, + { + desc: "ipv6 true", + ipv6Capable: true, + wantBootstrapConfig: bootstrapConfig(t, bootstrap.ConfigOptionsForTesting{ + Servers: []byte(`[{ + "server_uri": "dns:///directpath-pa.googleapis.com", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + }]`), + Authorities: map[string]json.RawMessage{ + "traffic-director-c2p.xds.googleapis.com": []byte(`{ + "xds_servers": [ + { + "server_uri": "dns:///directpath-pa.googleapis.com", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + } + ] + }`), + }, + Node: []byte(`{ + "id": "C2P-666", + "locality": { + "zone": "test-zone" + }, + "metadata": { + "TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE": true + } + }`), + }), + }, + { + desc: "override TD URI", + ipv6Capable: true, + tdURIOverride: "test-uri", + wantBootstrapConfig: bootstrapConfig(t, bootstrap.ConfigOptionsForTesting{ + Servers: []byte(`[{ + "server_uri": "test-uri", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + }]`), + Authorities: map[string]json.RawMessage{ + "traffic-director-c2p.xds.googleapis.com": []byte(`{ + "xds_servers": [ + { + "server_uri": "test-uri", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + } + ] + }`), + }, + Node: []byte(`{ + "id": "C2P-666", + "locality": { + "zone": "test-zone" + }, + "metadata": { + "TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE": true + } + }`), + }), + }, + } { + t.Run(tt.desc, func(t *testing.T) { + // Override IPv6 capability returned by the metadata server. + oldGetIPv6Capability := getIPv6Capable + getIPv6Capable = func(time.Duration) bool { return tt.ipv6Capable } + defer func() { getIPv6Capable = oldGetIPv6Capability }() + + // Override TD URI test only env var. + if tt.tdURIOverride != "" { + oldURI := envconfig.C2PResolverTestOnlyTrafficDirectorURI + envconfig.C2PResolverTestOnlyTrafficDirectorURI = tt.tdURIOverride + defer func() { envconfig.C2PResolverTestOnlyTrafficDirectorURI = oldURI }() + } + + // Override xDS client pool. + oldXdsClientPool := xdsClientPool + xdsClientPool = xdsclient.NewPool(nil) + defer func() { xdsClientPool = oldXdsClientPool }() + + getIPv6Capable = func(time.Duration) bool { return tt.ipv6Capable } + defer func() { getIPv6Capable = oldGetIPv6Capability }() + + // Build the google-c2p resolver. + target := resolver.Target{URL: url.URL{Scheme: c2pScheme, Path: "test-path"}} + r, err := builder.Build(target, nil, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("failed to build resolver: %v", err) + } + defer r.Close() + + // Build should return wrapped xDS resolver, not DNS. + if r, ok := r.(*c2pResolverWrapper); !ok || r.Resolver != testXDSResolver { + t.Fatalf("Build() returned %#v, want c2pResolverWrapper", r) + } + + xdsTarget := resolver.Target{URL: url.URL{Scheme: xdsName, Host: c2pAuthority, Path: target.URL.Path}} + verifyXDSClientBootstrapConfig(t, xdsClientPool, xdsTarget.String(), tt.wantBootstrapConfig) + }) + } +} + +// TestDialFailsWhenTargetContainsAuthority attempts to Dial a target URI of +// google-c2p scheme with a non-empty authority and verifies that it fails with +// an expected error. +func (s) TestBuildFailsWhenCalledWithAuthority(t *testing.T) { + useCleanUniverseDomain(t) + uri := "google-c2p://an-authority/resource" + cc, err := grpc.NewClient(uri, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("failed to create a client for server: %v", err) + } + defer func() { + if cc != nil { + cc.Close() + } + }() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + wantErr := "google-c2p URI scheme does not support authorities" + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("client.EmptyCall(%s) returned error: %v, want: %v", uri, err, wantErr) + } +} + +func (s) TestSetUniverseDomainNonDefault(t *testing.T) { + replaceResolvers(t) + simulateRunningOnGCE(t, true) + useCleanUniverseDomain(t) + builder := resolver.Get(c2pScheme) + + // Override the zone returned by the metadata server. + oldGetZone := getZone + getZone = func(time.Duration) string { return "test-zone" } + defer func() { getZone = oldGetZone }() + + // Override IPv6 capability returned by the metadata server. + oldGetIPv6Capability := getIPv6Capable + getIPv6Capable = func(time.Duration) bool { return false } + defer func() { getIPv6Capable = oldGetIPv6Capability }() + + // Override the random func used in the node ID. + origRandInd := randInt + randInt = func() int { return 666 } + defer func() { randInt = origRandInd }() + + // Set the universe domain + testUniverseDomain := "test-universe-domain.test" + if err := SetUniverseDomain(testUniverseDomain); err != nil { + t.Fatalf("SetUniverseDomain(%s) failed: %v", testUniverseDomain, err) + } + + // Now set universe domain to something different, it should fail + domain := "test-universe-domain-2.test" + err := SetUniverseDomain(domain) + wantErr := "already set" + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("googlec2p.SetUniverseDomain(%s) returned error: %v, want: %v", domain, err, wantErr) + } + + // Now explicitly set universe domain to the default, it should also fail + domain = "googleapis.com" + err = SetUniverseDomain(domain) + wantErr = "already set" + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("googlec2p.SetUniverseDomain(%s) returned error: %v, want: %v", domain, err, wantErr) + } + + // Now set universe domain to the original value, it should work + if err := SetUniverseDomain(testUniverseDomain); err != nil { + t.Fatalf("googlec2p.SetUniverseDomain(%s) failed: %v", testUniverseDomain, err) + } + + // Override xDS client pool. + oldXdsClientPool := xdsClientPool + xdsClientPool = xdsclient.NewPool(nil) + defer func() { xdsClientPool = oldXdsClientPool }() + + // Build the google-c2p resolver. + target := resolver.Target{URL: url.URL{Scheme: c2pScheme, Path: "test-path"}} + r, err := builder.Build(target, nil, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("failed to build resolver: %v", err) + } + defer r.Close() + + // Build should return wrapped xDS resolver, not DNS. + if r, ok := r.(*c2pResolverWrapper); !ok || r.Resolver != testXDSResolver { + t.Fatalf("Build() returned %#v, want c2pResolverWrapper", r) + } + + // Check that we use directpath-pa.test-universe-domain.test in the + // bootstrap config. + wantBootstrapConfig := bootstrapConfig(t, bootstrap.ConfigOptionsForTesting{ + Servers: []byte(`[{ + "server_uri": "dns:///directpath-pa.test-universe-domain.test", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + }]`), + Authorities: map[string]json.RawMessage{ + "traffic-director-c2p.xds.googleapis.com": []byte(`{ + "xds_servers": [ + { + "server_uri": "dns:///directpath-pa.test-universe-domain.test", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + } + ] + }`), + }, + Node: []byte(`{ + "id": "C2P-666", + "locality": { + "zone": "test-zone" + } + }`), + }) + + xdsTarget := resolver.Target{URL: url.URL{Scheme: xdsName, Host: c2pAuthority, Path: target.URL.Path}} + verifyXDSClientBootstrapConfig(t, xdsClientPool, xdsTarget.String(), wantBootstrapConfig) +} + +func (s) TestDefaultUniverseDomain(t *testing.T) { + replaceResolvers(t) + simulateRunningOnGCE(t, true) + useCleanUniverseDomain(t) + builder := resolver.Get(c2pScheme) + + // Override the zone returned by the metadata server. + oldGetZone := getZone + getZone = func(time.Duration) string { return "test-zone" } + defer func() { getZone = oldGetZone }() + + // Override IPv6 capability returned by the metadata server. + oldGetIPv6Capability := getIPv6Capable + getIPv6Capable = func(time.Duration) bool { return false } + defer func() { getIPv6Capable = oldGetIPv6Capability }() + + // Override the random func used in the node ID. + origRandInd := randInt + randInt = func() int { return 666 } + defer func() { randInt = origRandInd }() + + // Override xDS client pool. + oldXdsClientPool := xdsClientPool + xdsClientPool = xdsclient.NewPool(nil) + defer func() { xdsClientPool = oldXdsClientPool }() + + // Build the google-c2p resolver. + target := resolver.Target{URL: url.URL{Scheme: c2pScheme, Path: "test-path"}} + r, err := builder.Build(target, nil, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("failed to build resolver: %v", err) + } + defer r.Close() + + // Build should return xDS, not DNS. + if r, ok := r.(*c2pResolverWrapper); !ok || r.Resolver != testXDSResolver { + t.Fatalf("Build() returned %#v, want c2pResolverWrapper", r) + } + + // Check that we use directpath-pa.googleapis.com in the bootstrap config + wantBootstrapConfig := bootstrapConfig(t, bootstrap.ConfigOptionsForTesting{ + Servers: []byte(`[{ + "server_uri": "dns:///directpath-pa.googleapis.com", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + }]`), + Authorities: map[string]json.RawMessage{ + "traffic-director-c2p.xds.googleapis.com": []byte(`{ + "xds_servers": [ + { + "server_uri": "dns:///directpath-pa.googleapis.com", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + } + ] + }`), + }, + Node: []byte(`{ + "id": "C2P-666", + "locality": { + "zone": "test-zone" + } + }`), + }) + xdsTarget := resolver.Target{URL: url.URL{Scheme: xdsName, Host: c2pAuthority, Path: target.URL.Path}} + verifyXDSClientBootstrapConfig(t, xdsClientPool, xdsTarget.String(), wantBootstrapConfig) + + // Now set universe domain to something different than the default, it should fail + domain := "test-universe-domain.test" + err = SetUniverseDomain(domain) + wantErr := "already set" + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("googlec2p.SetUniverseDomain(%s) returned error: %v, want: %v", domain, err, wantErr) + } + + // Now explicitly set universe domain to the default, it should work + domain = "googleapis.com" + if err := SetUniverseDomain(domain); err != nil { + t.Fatalf("googlec2p.SetUniverseDomain(%s) failed: %v", domain, err) + } +} + +func (s) TestSetUniverseDomainEmptyString(t *testing.T) { + replaceResolvers(t) + simulateRunningOnGCE(t, true) + useCleanUniverseDomain(t) + wantErr := "cannot be empty" + err := SetUniverseDomain("") + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("googlec2p.SetUniverseDomain(\"\") returned error: %v, want: %v", err, wantErr) + } +} + +// TestCreateMultipleXDSClients validates that multiple xds clients with +// different bootstrap config coexist in the same pool. It confirms that +// a client created by the google-c2p resolver does not interfere with an +// explicitly created client using a different bootstrap configuration. +func (s) TestCreateMultipleXDSClients(t *testing.T) { + replaceResolvers(t) + simulateRunningOnGCE(t, true) + useCleanUniverseDomain(t) + + // Override the zone returned by the metadata server. + oldGetZone := getZone + getZone = func(time.Duration) string { return "test-zone" } + defer func() { getZone = oldGetZone }() + + // Override the random func used in the node ID. + origRandInd := randInt + randInt = func() int { return 666 } + defer func() { randInt = origRandInd }() + + // Override IPv6 capability returned by the metadata server. + oldGetIPv6Capability := getIPv6Capable + getIPv6Capable = func(time.Duration) bool { return false } + defer func() { getIPv6Capable = oldGetIPv6Capability }() + + // Define bootstrap config for generic xds resolver + genericXDSConfig := bootstrapConfig(t, bootstrap.ConfigOptionsForTesting{ + Servers: []byte(`[{ + "server_uri": "dns:///regular-xds.googleapis.com", + "channel_creds": [{"type": "google_default"}], + "server_features": [] + }]`), + Node: []byte(`{"id": "regular-xds-node", "locality": {"zone": "test-zone"}}`), + }) + + // Override xDS client pool. + oldXdsClientPool := xdsClientPool + xdsClientPool = xdsclient.NewPool(genericXDSConfig) + defer func() { xdsClientPool = oldXdsClientPool }() + + // Create generic xds client. + xdsTarget := resolver.Target{URL: *testutils.MustParseURL("xds:///target")} + _, closeGeneric, err := xdsClientPool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: xdsTarget.String(), + Config: genericXDSConfig, + }) + if err != nil { + t.Fatalf("xdsClientPool.NewClientForTesting(%q) failed: %v", xdsTarget.String(), err) + } + defer closeGeneric() + + verifyXDSClientBootstrapConfig(t, xdsClientPool, xdsTarget.String(), genericXDSConfig) + + // Build the google-c2p resolver. + c2pBuilder := resolver.Get(c2pScheme) + c2pTarget := resolver.Target{URL: url.URL{Scheme: c2pScheme, Path: "test-path"}} + tcc := &testutils.ResolverClientConn{Logger: t} + c2pRes, err := c2pBuilder.Build(c2pTarget, tcc, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("Failed to build resolver: %v", err) + } + defer c2pRes.Close() + + // Bootstrap config for c2p resolver. + c2pConfig := bootstrapConfig(t, bootstrap.ConfigOptionsForTesting{ + Servers: []byte(`[{ + "server_uri": "dns:///directpath-pa.googleapis.com", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + }]`), + Authorities: map[string]json.RawMessage{ + "traffic-director-c2p.xds.googleapis.com": []byte(`{ + "xds_servers": [{ + "server_uri": "dns:///directpath-pa.googleapis.com", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + }] + }`), + }, + Node: []byte(`{ + "id": "C2P-666", + "locality": { + "zone": "test-zone" + } + }`), + }) + + c2pXDSTarget := resolver.Target{URL: url.URL{Scheme: xdsName, Host: c2pAuthority, Path: c2pTarget.URL.Path}} + verifyXDSClientBootstrapConfig(t, xdsClientPool, c2pXDSTarget.String(), c2pConfig) +} + +// Test validates that the C2P resolver handles non-GCP (Google Cloud +// Interconnect) clients correctly when the force-xds query parameter is +// provided in various valid formats. +// It verifies that: +// - GCE metadata server queries for zone and ipv6 capability are bypassed. +// - The node ID is formatted using the prefix ("C2P-non-gcp"). +// - Zone information is completely omitted from the node config. +// - Dualstack IPv6 capability (TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE) +// is set to true. +func (s) TestBuild_NotOnGCE_WithForceXDS(t *testing.T) { + wantBootstrapConfig := bootstrapConfig(t, bootstrap.ConfigOptionsForTesting{ + Servers: []byte(`[{ + "server_uri": "dns:///directpath-pa.googleapis.com", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + }]`), + Authorities: map[string]json.RawMessage{ + "traffic-director-c2p.xds.googleapis.com": []byte(`{ + "xds_servers": [ + { + "server_uri": "dns:///directpath-pa.googleapis.com", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + } + ] + }`), + }, + Node: []byte(`{ + "id": "C2P-non-gcp-666", + "metadata": { + "TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE": true + } + }`), + }) + tests := []struct { + desc string + rawQuery string + onGCE bool + }{ + { + desc: "query_param_key_only", + rawQuery: "force-xds", + }, + { + desc: "query_param_trailing_equals_value_empty", + rawQuery: "force-xds=", + }, + { + desc: "query_param_value_true", + rawQuery: "force-xds=true", + }, + { + desc: "query_param_value_false", + rawQuery: "force-xds=false", + }, + { + desc: "query_param_multiple_query_params", + rawQuery: "foo=bar&force-xds", + }, + { + desc: "onGCE_with_force-xds", + rawQuery: "force-xds", + onGCE: true, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + replaceResolvers(t) + simulateRunningOnGCE(t, tt.onGCE) + useCleanUniverseDomain(t) + + // Override the random func used in the node ID. + origRandInd := randInt + randInt = func() int { return 666 } + defer func() { randInt = origRandInd }() + + // Override xDS client pool. + oldXdsClientPool := xdsClientPool + xdsClientPool = xdsclient.NewPool(nil) + defer func() { xdsClientPool = oldXdsClientPool }() + + // Target URI with force-xds query parameter. + builder := resolver.Get(c2pScheme) + target := resolver.Target{URL: url.URL{ + Scheme: c2pScheme, + Path: "test-path", + RawQuery: tt.rawQuery, + }} + + // Build the google-c2p resolver. + res, err := builder.Build(target, nil, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("Failed to build resolver: %v", err) + } + defer res.Close() + + // Query parameters are omitted since the downstream xDS target is + // configured using only active fields. + xdsTarget := resolver.Target{ + URL: url.URL{ + Scheme: xdsName, + Host: c2pAuthority, + Path: target.URL.Path, + }, + } + verifyXDSClientBootstrapConfig(t, xdsClientPool, xdsTarget.String(), wantBootstrapConfig) + }) + } +} diff --git a/xds/googledirectpath/utils.go b/xds/googledirectpath/utils.go new file mode 100644 index 0000000..32a28a1 --- /dev/null +++ b/xds/googledirectpath/utils.go @@ -0,0 +1,101 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package googledirectpath + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +func getFromMetadata(timeout time.Duration, urlStr string) ([]byte, error) { + parsedURL, err := url.Parse(urlStr) + if err != nil { + return nil, err + } + client := &http.Client{Timeout: timeout} + req := &http.Request{ + Method: http.MethodGet, + URL: parsedURL, + Header: http.Header{"Metadata-Flavor": {"Google"}}, + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed communicating with metadata server: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("metadata server returned resp with non-OK: %v", resp) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed reading from metadata server: %v", err) + } + return body, nil +} + +var ( + zone string + zoneOnce sync.Once +) + +// Defined as var to be overridden in tests. +var getZone = func(timeout time.Duration) string { + zoneOnce.Do(func() { + qualifiedZone, err := getFromMetadata(timeout, zoneURL) + if err != nil { + logger.Warningf("could not discover instance zone: %v", err) + return + } + i := bytes.LastIndexByte(qualifiedZone, '/') + if i == -1 { + logger.Warningf("could not parse zone from metadata server: %s", qualifiedZone) + return + } + zone = string(qualifiedZone[i+1:]) + }) + return zone +} + +var ( + ipv6Capable bool + ipv6CapableOnce sync.Once +) + +// Defined as var to be overridden in tests. +var getIPv6Capable = func(timeout time.Duration) bool { + ipv6CapableOnce.Do(func() { + addr, err := getFromMetadata(timeout, ipv6URL) + if err != nil { + logger.Warningf("could not discover ipv6 capability: %v", err) + return + } + if trimmedAddr := strings.TrimSpace(string(addr)); trimmedAddr == "" { + logger.Warningf("metadata server returned empty ipv6 address") + return + } + ipv6Capable = true + }) + return ipv6Capable +} diff --git a/xds/server.go b/xds/server.go new file mode 100644 index 0000000..f7dca98 --- /dev/null +++ b/xds/server.go @@ -0,0 +1,235 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "errors" + "fmt" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + estats "google.golang.org/grpc/experimental/stats" + "google.golang.org/grpc/internal" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + istats "google.golang.org/grpc/internal/stats" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/server" + "google.golang.org/grpc/internal/xds/xdsclient" +) + +const serverPrefix = "[xds-server %p] " + +var ( + // These will be overridden in unit tests. + xdsClientPool = xdsclient.DefaultPool + newGRPCServer = func(opts ...grpc.ServerOption) grpcServer { + return grpc.NewServer(opts...) + } +) + +// grpcServer contains methods from grpc.Server which are used by the +// GRPCServer type here. This is useful for overriding in unit tests. +type grpcServer interface { + RegisterService(*grpc.ServiceDesc, any) + Serve(net.Listener) error + Stop() + GracefulStop() + GetServiceInfo() map[string]grpc.ServiceInfo +} + +// GRPCServer wraps a gRPC server and provides server-side xDS functionality, by +// communication with a management server using xDS APIs. It implements the +// grpc.ServiceRegistrar interface and can be passed to service registration +// functions in IDL generated code. +type GRPCServer struct { + gs grpcServer + quit *grpcsync.Event + logger *internalgrpclog.PrefixLogger + opts *server.Options + xdsC xdsclient.XDSClient + xdsClientClose func() +} + +// NewGRPCServer creates an xDS-enabled gRPC server using the passed in opts. +// The underlying gRPC server has no service registered and has not started to +// accept requests yet. +func NewGRPCServer(opts ...grpc.ServerOption) (*GRPCServer, error) { + // xdsFilterWrapper is a callback passed to the underlying gRPC server. + // It adapts the generic stream parameter and delegates to RouteAndProcess + // to execute xDS HTTP filter interceptors for each incoming RPC. + xDSFilterWrapper := func(ss grpc.ServerStream) (grpc.ServerStream, error) { + return server.RouteAndProcess(ss) + } + // Construct a grpc.ServerOption that registers xDSFilterWrapper on + // the server. + xDSFilterWrapperOption := internal.XDSFilterWrapperOption.(func(func(grpc.ServerStream) (grpc.ServerStream, error)) grpc.ServerOption) + newOpts := append([]grpc.ServerOption{xDSFilterWrapperOption(xDSFilterWrapper)}, opts...) + s := &GRPCServer{ + gs: newGRPCServer(newOpts...), + quit: grpcsync.NewEvent(), + } + s.handleServerOptions(opts) + + var mrl estats.MetricsRecorder + mrl = istats.NewMetricsRecorderList(nil) + if srv, ok := s.gs.(*grpc.Server); ok { // Will hit in prod but not for testing. + mrl = internal.MetricsRecorderForServer.(func(*grpc.Server) estats.MetricsRecorder)(srv) + } + + // Initializing the xDS client upfront (instead of at serving time) + // simplifies the code by eliminating the need for a mutex to protect the + // xdsC and xdsClientClose fields. + pool := xdsClientPool + if s.opts.ClientPoolForTesting != nil { + pool = s.opts.ClientPoolForTesting + } + xdsClient, xdsClientClose, err := pool.NewClient(xdsclient.NameForServer, mrl) + if err != nil { + return nil, fmt.Errorf("xDS client creation failed: %v", err) + } + + // Validate the bootstrap configuration for server specific fields. + + // Listener resource name template is mandatory on the server side unless a + // listener resource name override is provided. + if s.opts.OverrideListenerResourceName == nil { + cfg := xdsClient.BootstrapConfig() + if cfg.ServerListenerResourceNameTemplate() == "" { + xdsClientClose() + return nil, errors.New("missing server_listener_resource_name_template in the bootstrap configuration") + } + } + + s.xdsC = xdsClient + s.xdsClientClose = xdsClientClose + + s.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(serverPrefix, s)) + s.logger.Infof("Created xds.GRPCServer") + + return s, nil +} + +// handleServerOptions iterates through the list of server options passed in by +// the user, and handles the xDS server specific options. +func (s *GRPCServer) handleServerOptions(opts []grpc.ServerOption) { + so := s.defaultServerOptions() + server.ApplyServerOptions(opts, so) + s.opts = so +} + +func (s *GRPCServer) defaultServerOptions() *server.Options { + return &server.Options{ + // A default serving mode change callback which simply logs at the + // default-visible log level. This will be used if the application does not + // register a mode change callback. + // + // Note that this means that `s.opts.ModeCallback` will never be nil and can + // safely be invoked directly from `handleServingModeChanges`. + ModeCallback: s.loggingServerModeChangeCallback, + } +} + +func (s *GRPCServer) loggingServerModeChangeCallback(addr net.Addr, mode connectivity.ServingMode, err error) { + switch mode { + case connectivity.ServingModeServing: + s.logger.Errorf("Listener %q entering mode: %q", addr.String(), mode) + case connectivity.ServingModeNotServing: + s.logger.Errorf("Listener %q entering mode: %q due to error: %v", addr.String(), mode, err) + } +} + +// RegisterService registers a service and its implementation to the underlying +// gRPC server. It is called from the IDL generated code. This must be called +// before invoking Serve. +func (s *GRPCServer) RegisterService(sd *grpc.ServiceDesc, ss any) { + s.gs.RegisterService(sd, ss) +} + +// GetServiceInfo returns a map from service names to ServiceInfo. +// Service names include the package names, in the form of .. +func (s *GRPCServer) GetServiceInfo() map[string]grpc.ServiceInfo { + return s.gs.GetServiceInfo() +} + +// Serve gets the underlying gRPC server to accept incoming connections on the +// listener lis, which is expected to be listening on a TCP port. +// +// A connection to the management server, to receive xDS configuration, is +// initiated here. +// +// Serve will return a non-nil error unless Stop or GracefulStop is called. +func (s *GRPCServer) Serve(lis net.Listener) error { + s.logger.Infof("Serve() passed a net.Listener on %s", lis.Addr().String()) + if _, ok := lis.Addr().(*net.TCPAddr); !ok { + return fmt.Errorf("xds: GRPCServer expects listener to return a net.TCPAddr. Got %T", lis.Addr()) + } + + if s.quit.HasFired() { + return grpc.ErrServerStopped + } + + // The server listener resource name template from the bootstrap + // configuration contains a template for the name of the Listener resource + // to subscribe to for a gRPC server. If the token `%s` is present in the + // string, it will be replaced with the server's listening "IP:port" (e.g., + // "0.0.0.0:8080", "[::]:8080"). + var name string + if s.opts.OverrideListenerResourceName != nil { + name = s.opts.OverrideListenerResourceName(lis.Addr()) + } else { + cfg := s.xdsC.BootstrapConfig() + name = cfg.ServerListenerResourceNameTemplate() + name = bootstrap.PopulateResourceTemplate(name, lis.Addr().String()) + } + + // Create a listenerWrapper which handles all functionality required by + // this particular instance of Serve(). + lw := server.NewListenerWrapper(server.ListenerWrapperParams{ + Listener: lis, + ListenerResourceName: name, + XDSClient: s.xdsC, + ModeCallback: s.opts.ModeCallback, + }) + return s.gs.Serve(lw) +} + +// Stop stops the underlying gRPC server. It immediately closes all open +// connections. It cancels all active RPCs on the server side and the +// corresponding pending RPCs on the client side will get notified by connection +// errors. +func (s *GRPCServer) Stop() { + s.quit.Fire() + s.gs.Stop() + if s.xdsC != nil { + s.xdsClientClose() + } +} + +// GracefulStop stops the underlying gRPC server gracefully. It stops the server +// from accepting new connections and RPCs and blocks until all the pending RPCs +// are finished. +func (s *GRPCServer) GracefulStop() { + s.quit.Fire() + s.gs.GracefulStop() + if s.xdsC != nil { + s.xdsClientClose() + } +} diff --git a/xds/server_ext_test.go b/xds/server_ext_test.go new file mode 100644 index 0000000..7e97a90 --- /dev/null +++ b/xds/server_ext_test.go @@ -0,0 +1,532 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "io" + "net" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) + +func hostPortFromListener(lis net.Listener) (string, uint32, error) { + host, p, err := net.SplitHostPort(lis.Addr().String()) + if err != nil { + return "", 0, fmt.Errorf("net.SplitHostPort(%s) failed: %v", lis.Addr().String(), err) + } + port, err := strconv.ParseInt(p, 10, 32) + if err != nil { + return "", 0, fmt.Errorf("strconv.ParseInt(%s, 10, 32) failed: %v", p, err) + } + return host, uint32(port), nil +} + +// servingModeChangeHandler handles changes to the serving mode of an +// xDS-enabled gRPC server. It logs the changes and sends the new mode and any +// errors on appropriate channels for the test to consume. +type servingModeChangeHandler struct { + logger interface { + Logf(format string, args ...any) + } + // Access to the below fields are guarded by this mutex. + mu sync.Mutex + modeCh chan connectivity.ServingMode + errCh chan error + currentMode connectivity.ServingMode + currentErr error +} + +func newServingModeChangeHandler(t *testing.T) *servingModeChangeHandler { + return &servingModeChangeHandler{ + logger: t, + modeCh: make(chan connectivity.ServingMode, 1), + errCh: make(chan error, 1), + } +} + +func (m *servingModeChangeHandler) modeChangeCallback(addr net.Addr, args xds.ServingModeChangeArgs) { + m.mu.Lock() + defer m.mu.Unlock() + // Suppress pushing duplicate mode change and error if the mode is staying + // in NOT_SERVING and the error is the same. + // + // TODO(purnesh42h): Should we move this check to listener wrapper? This + // shouldn't happen in practice a lot. But we never know what kind of + // management servers users run. + if m.currentMode == args.Mode && m.currentMode == connectivity.ServingModeNotServing && m.currentErr.Error() == args.Err.Error() { + return + } + m.logger.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err) + m.modeCh <- args.Mode + m.currentMode = args.Mode + if args.Err != nil { + m.errCh <- args.Err + } + m.currentErr = args.Err +} + +// createStubServer creates a new xDS-enabled gRPC server and returns a +// stubserver.StubServer that can be used for testing. The server is configured +// with the provided modeChangeOpt and xdsclient.Pool. +func createStubServer(t *testing.T, lis net.Listener, opts ...grpc.ServerOption) *stubserver.StubServer { + stub := &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + if _, err := stream.Recv(); err == io.EOF { + return nil + } else if err != nil { + return err + } + } + }, + } + server, err := xds.NewGRPCServer(opts...) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + stub.S = server + stubserver.StartTestService(t, stub) + t.Cleanup(stub.Stop) + return stub +} + +// waitForSuccessfulRPC waits for an RPC to succeed, repeatedly calling the +// EmptyCall RPC on the provided client connection until the call succeeds. +// If the context is canceled or the expected error is not before the context +// timeout expires, the test will fail. +func waitForSuccessfulRPC(ctx context.Context, t *testing.T, cc *grpc.ClientConn, opts ...grpc.CallOption) { + t.Helper() + + client := testgrpc.NewTestServiceClient(cc) + for { + select { + case <-ctx.Done(): + t.Fatalf("Timeout waiting for RPCs to succeed") + case <-time.After(defaultTestShortTimeout): + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, opts...); err == nil { + return + } + } + } +} + +// waitForFailedRPCWithStatus waits for an RPC to fail with the expected status +// code, error message, and node ID. It repeatedly calls the EmptyCall RPC on +// the provided client connection until the error matches the expected values. +// If the context is canceled or the expected error is not before the context +// timeout expires, the test will fail. +func waitForFailedRPCWithStatus(ctx context.Context, t *testing.T, cc *grpc.ClientConn, wantCode codes.Code, wantErr, wantNodeID string) { + t.Helper() + + client := testgrpc.NewTestServiceClient(cc) + var err error + for { + select { + case <-ctx.Done(): + t.Fatalf("RPCs failed with most recent error: %v. Want status code %v, error: %s, node id: %s", err, wantCode, wantErr, wantNodeID) + case <-time.After(defaultTestShortTimeout): + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if gotCode := status.Code(err); gotCode != wantCode { + continue + } + if gotErr := err.Error(); !strings.Contains(gotErr, wantErr) { + continue + } + if !strings.Contains(err.Error(), wantNodeID) { + continue + } + t.Logf("Most recent error happy case: %v", err.Error()) + return + } + } +} + +// Tests the basic scenario for an xDS enabled gRPC server. +// +// - Verifies that the xDS enabled gRPC server requests for the expected +// listener resource. +// - Once the listener resource is received from the management server, it +// verifies that the xDS enabled gRPC server requests for the appropriate +// route configuration name. Also verifies that at this point, the server has +// not yet started serving RPCs +// - Once the route configuration is received from the management server, it +// verifies that the server can serve RPCs successfully. +func (s) TestServer_Basic(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server. + listenerNamesCh := make(chan []string, 1) + routeNamesCh := make(chan []string, 1) + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + // The test needs to only verify the names of resources being + // subscribed to, we can skip ACKs. + if req.GetVersionInfo() != "" { + return nil + } + switch req.GetTypeUrl() { + case "type.googleapis.com/envoy.config.listener.v3.Listener": + select { + case listenerNamesCh <- req.GetResourceNames(): + case <-ctx.Done(): + } + case "type.googleapis.com/envoy.config.route.v3.RouteConfiguration": + select { + case routeNamesCh <- req.GetResourceNames(): + case <-ctx.Done(): + } + } + return nil + }, + AllowResourceSubset: true, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create a listener on a local port to act as the xDS enabled gRPC server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to listen to local port: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + + // Configure the managegement server with a listener resource for the above + // xDS enabled gRPC server. + const routeConfigName = "routeName" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListenerWithRouteConfigName(host, port, e2e.SecurityLevelNone, "routeName")}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Start an xDS-enabled gRPC server with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + modeChangeHandler := newServingModeChangeHandler(t) + modeChangeOpt := xds.ServingModeCallback(modeChangeHandler.modeChangeCallback) + createStubServer(t, lis, modeChangeOpt, xds.ClientPoolForTesting(pool)) + + // Wait for the expected listener resource to be requested. + wantLisResourceNames := []string{fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port))))} + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the expected listener resource to be requested") + case gotLisResourceName := <-listenerNamesCh: + if !cmp.Equal(gotLisResourceName, wantLisResourceNames) { + t.Fatalf("Got unexpected listener resource names: %v, want %v", gotLisResourceName, wantLisResourceNames) + } + } + + // Wait for the expected route config resource to be requested. + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the expected route config resource to be requested") + case gotRouteNames := <-routeNamesCh: + if !cmp.Equal(gotRouteNames, []string{routeConfigName}) { + t.Fatalf("Got unexpected route config resource names: %v, want %v", gotRouteNames, []string{routeConfigName}) + } + } + + // Ensure that the server is not serving RPCs yet. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-modeChangeHandler.modeCh: + t.Fatal("Server started serving RPCs before the route config was received") + } + + // Create a gRPC channel to the xDS enabled server. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", lis.Addr(), err) + } + defer cc.Close() + + // Ensure that the server isnt't serving RPCs successfully. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err == nil || status.Code(err) != codes.Unavailable { + t.Fatalf("EmptyCall() returned %v, want %v", err, codes.Unavailable) + } + + // Configure the management server with the expected route config resource, + // and expext RPCs to succeed. + resources.Routes = []*v3routepb.RouteConfiguration{e2e.RouteConfigNonForwardingAction(routeConfigName)} + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the server to start serving RPCs") + case gotMode := <-modeChangeHandler.modeCh: + if gotMode != connectivity.ServingModeServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeServing) + } + } + waitForSuccessfulRPC(ctx, t, cc) +} + +// Tests that the xDS-enabled gRPC server cleans up all its resources when all +// connections to it are closed. +func (s) TestServer_ConnectionCleanup(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create a listener on a local port to act as the xDS enabled gRPC server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to listen to local port: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + + // Configure the managegement server with a listener and route configuration + // resource for the above xDS enabled gRPC server. + const routeConfigName = "routeName" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListenerWithRouteConfigName(host, port, e2e.SecurityLevelNone, "routeName")}, + Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigNonForwardingAction(routeConfigName)}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Start an xDS-enabled gRPC server with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + modeChangeOpt := xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) { + t.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err) + }) + createStubServer(t, lis, modeChangeOpt, xds.ClientPoolForTesting(pool)) + + // Create a gRPC channel and verify that RPCs succeed. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", lis.Addr(), err) + } + defer cc.Close() + waitForSuccessfulRPC(ctx, t, cc) + + // Create multiple channels to the server, and make an RPC on each one. When + // everything is closed, the server should have cleaned up all its resources + // as well (and this will be verified by the leakchecker). + const numConns = 100 + var wg sync.WaitGroup + wg.Add(numConns) + for range numConns { + go func() { + defer wg.Done() + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Errorf("grpc.NewClient failed with err: %v", err) + } + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("EmptyCall() failed: %v", err) + } + cc.Close() + }() + } + wg.Wait() +} + +// Tests that multiple xDS-enabled gRPC servers can be created with different +// bootstrap configurations, and that they correctly request different LDS +// resources from the management server based on their respective listening +// ports. It also ensures that gRPC clients can connect to the intended server +// and that RPCs function correctly. The test uses the grpc.Peer() call option +// to validate that the client is connected to the correct server. +func (s) TestServer_MultipleServers_DifferentBootstrapConfigurations(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create two bootstrap configurations pointing to the above management server. + nodeID1 := uuid.New().String() + bootstrapContents1 := e2e.DefaultBootstrapContents(t, nodeID1, mgmtServer.Address) + nodeID2 := uuid.New().String() + bootstrapContents2 := e2e.DefaultBootstrapContents(t, nodeID2, mgmtServer.Address) + + // Create two xDS-enabled gRPC servers using the above bootstrap configs. + lis1, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + lis2, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + + modeChangeHandler1 := newServingModeChangeHandler(t) + modeChangeOpt1 := xds.ServingModeCallback(modeChangeHandler1.modeChangeCallback) + modeChangeHandler2 := newServingModeChangeHandler(t) + modeChangeOpt2 := xds.ServingModeCallback(modeChangeHandler2.modeChangeCallback) + config1, err := bootstrap.NewConfigFromContents(bootstrapContents1) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents1), err) + } + pool1 := xdsclient.NewPool(config1) + config2, err := bootstrap.NewConfigFromContents(bootstrapContents2) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents2), err) + } + pool2 := xdsclient.NewPool(config2) + createStubServer(t, lis1, modeChangeOpt1, xds.ClientPoolForTesting(pool1)) + createStubServer(t, lis2, modeChangeOpt2, xds.ClientPoolForTesting(pool2)) + + // Update the management server with the listener resources pointing to the + // corresponding gRPC servers. + host1, port1, err := hostPortFromListener(lis1) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + host2, port2, err := hostPortFromListener(lis2) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + + resources1 := e2e.UpdateOptions{ + NodeID: nodeID1, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListener(host1, port1, e2e.SecurityLevelNone, "routeName")}, + } + if err := mgmtServer.Update(ctx, resources1); err != nil { + t.Fatal(err) + } + + resources2 := e2e.UpdateOptions{ + NodeID: nodeID2, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListener(host2, port2, e2e.SecurityLevelNone, "routeName")}, + } + if err := mgmtServer.Update(ctx, resources2); err != nil { + t.Fatal(err) + } + + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the xDS-enabled gRPC server to go SERVING") + case gotMode := <-modeChangeHandler1.modeCh: + if gotMode != connectivity.ServingModeServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeServing) + } + } + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the xDS-enabled gRPC server to go SERVING") + case gotMode := <-modeChangeHandler2.modeCh: + if gotMode != connectivity.ServingModeServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeServing) + } + } + + // Create two gRPC clients, one for each server. + cc1, err := grpc.NewClient(lis1.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client for test server 1: %s, %v", lis1.Addr().String(), err) + } + defer cc1.Close() + + cc2, err := grpc.NewClient(lis2.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to create client for test server 2: %s, %v", lis2.Addr().String(), err) + } + defer cc2.Close() + + // Both unary RPCs should work once the servers transitions into serving. + var peer1 peer.Peer + waitForSuccessfulRPC(ctx, t, cc1, grpc.Peer(&peer1)) + if peer1.Addr.String() != lis1.Addr().String() { + t.Errorf("Connected to wrong peer: %s, want %s", peer1.Addr, lis1.Addr()) + } + + var peer2 peer.Peer + waitForSuccessfulRPC(ctx, t, cc2, grpc.Peer(&peer2)) + if peer2.Addr.String() != lis2.Addr().String() { + t.Errorf("Connected to wrong peer: %s, want %s", peer2.Addr, lis2.Addr()) + } +} diff --git a/xds/server_options.go b/xds/server_options.go new file mode 100644 index 0000000..2afda67 --- /dev/null +++ b/xds/server_options.go @@ -0,0 +1,102 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/xds/bootstrap" + internalserver "google.golang.org/grpc/internal/xds/server" + "google.golang.org/grpc/internal/xds/xdsclient" +) + +// ServingModeCallback returns a grpc.ServerOption which allows users to +// register a callback to get notified about serving mode changes. +func ServingModeCallback(cb ServingModeCallbackFunc) grpc.ServerOption { + return internalserver.NewServerOption(func(o *internalserver.Options) { + o.ModeCallback = func(addr net.Addr, mode connectivity.ServingMode, err error) { + cb(addr, ServingModeChangeArgs{ + Mode: mode, + Err: err, + }) + } + }) +} + +// ServingModeCallbackFunc is the callback that users can register to get +// notified about the server's serving mode changes. The callback is invoked +// with the address of the listener and its new mode. +// +// Users must not perform any blocking operations in this callback. +type ServingModeCallbackFunc func(addr net.Addr, args ServingModeChangeArgs) + +// ServingModeChangeArgs wraps the arguments passed to the serving mode callback +// function. +type ServingModeChangeArgs struct { + // Mode is the new serving mode of the server listener. + Mode connectivity.ServingMode + // Err is set to a non-nil error if the server has transitioned into + // not-serving mode. + Err error +} + +// BootstrapContentsForTesting returns a grpc.ServerOption which allows users +// to inject a bootstrap configuration used by only this server, instead of the +// global configuration from the environment variables. +// +// # Testing Only +// +// This function should ONLY be used for testing and may not work with some +// other features, including the CSDS service. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func BootstrapContentsForTesting(bootstrapContents []byte) grpc.ServerOption { + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + logger.Warningf("Failed to parse bootstrap contents %s for server options: %v", string(bootstrapContents), err) + return internalserver.NewServerOption(func(o *internalserver.Options) { + o.ClientPoolForTesting = nil + }) + } + return ClientPoolForTesting(xdsclient.NewPool(config)) +} + +// ClientPoolForTesting returns a grpc.ServerOption with the pool for xds +// clients. It allows users to set a pool for xDS clients sharing the bootstrap +// contents for this server. +// +// # Testing Only +// +// This function should ONLY be used for testing and may not work with some +// other features, including the CSDS service. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func ClientPoolForTesting(pool *xdsclient.Pool) grpc.ServerOption { + return internalserver.NewServerOption(func(o *internalserver.Options) { + o.ClientPoolForTesting = pool + }) +} diff --git a/xds/server_resource_ext_test.go b/xds/server_resource_ext_test.go new file mode 100644 index 0000000..8641316 --- /dev/null +++ b/xds/server_resource_ext_test.go @@ -0,0 +1,654 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "io" + "net" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + "google.golang.org/grpc/xds" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// Tests the case where an LDS points to an RDS which returns resource not +// found. Before getting the resource not found, the xDS Server has not received +// all configuration needed, so it should Accept and Close any new connections. +// After it has received the resource not found error (due to short watch +// expiry), the server should move to serving, successfully Accept Connections, +// and fail at the L7 level with resource not found specified. +func (s) TestServer_RouteConfiguration_ResourceNotFound(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + routeConfigNamesCh := make(chan []string, 1) + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.TypeUrl == version.V3RouteConfigURL { + select { + case routeConfigNamesCh <- req.GetResourceNames(): + case <-ctx.Done(): + } + } + return nil + }, + AllowResourceSubset: true, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Setup the management server to respond with a listener resource that + // specifies a route name to watch, and no RDS resource corresponding to + // this route name. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + const routeConfigResourceName = "routeName" + listener := e2e.DefaultServerListenerWithRouteConfigName(host, port, e2e.SecurityLevelNone, routeConfigResourceName) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + modeChangeHandler := newServingModeChangeHandler(t) + modeChangeOpt := xds.ServingModeCallback(modeChangeHandler.modeChangeCallback) + + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + // Create a specific xDS client instance within that pool for the server, + // configuring it with a short WatchExpiryTimeout. + pool := xdsclient.NewPool(config) + _, serverXDSClientClose, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: xdsclient.NameForServer, + WatchExpiryTimeout: 500 * time.Millisecond, + }) + if err != nil { + t.Fatalf("Failed to create xDS client for server: %v", err) + } + defer serverXDSClientClose() + // Start an xDS-enabled gRPC server using the above client from the pool. + createStubServer(t, lis, modeChangeOpt, xds.ClientPoolForTesting(pool)) + + // Wait for the route configuration resource to be requested from the + // management server. + select { + case gotNames := <-routeConfigNamesCh: + if !cmp.Equal(gotNames, []string{routeConfigResourceName}) { + t.Fatalf("Requested route config resource names: %v, want %v", gotNames, []string{routeConfigResourceName}) + } + case <-ctx.Done(): + t.Fatal("Timeout waiting for route config resource to be requested") + } + + // Do NOT send the RDS resource. The xDS client's watch expiry timer will + // fire. After the RDS resource is deemed "not found" (due to the short + // watch expiry), the server will transition to SERVING mode. + + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + // Before the watch expiry, the server is NOT_SERVING, RPCs should fail with UNAVAILABLE. + waitForFailedRPCWithStatus(ctx, t, cc, codes.Unavailable, "", "") + + // Wait for the xDS-enabled gRPC server to go SERVING. This should happen + // after the RDS watch expiry timer fires. + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the xDS-enabled gRPC server to go SERVING") + case gotMode := <-modeChangeHandler.modeCh: + if gotMode != connectivity.ServingModeServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeServing) + } + } + // After watch expiry, the server should be SERVING, but RPCs should fail + // at the L7 level with resource not found. + waitForFailedRPCWithStatus(ctx, t, cc, codes.Unavailable, "error from xDS configuration for matched route configuration", nodeID) +} + +// Tests the scenario where the control plane sends the same resource update. It +// verifies that the mode change callback is not invoked and client connections +// to the server are not recycled. +func (s) TestServer_RedundantUpdateSuppression(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Setup the management server to respond with the listener resources. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + listener := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName") + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Start an xDS-enabled gRPC server with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + modeChangeHandler := newServingModeChangeHandler(t) + modeChangeOpt := xds.ServingModeCallback(modeChangeHandler.modeChangeCallback) + createStubServer(t, lis, modeChangeOpt, xds.ClientPoolForTesting(pool)) + + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for a mode change update: %v", err) + case mode := <-modeChangeHandler.modeCh: + if mode != connectivity.ServingModeServing { + t.Fatalf("Listener received new mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + + // Create a ClientConn and make a successful RPCs. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", lis.Addr(), err) + } + defer cc.Close() + waitForSuccessfulRPC(ctx, t, cc) + + // Start a goroutine to make sure that we do not see any connectivity state + // changes on the client connection. If redundant updates are not + // suppressed, server will recycle client connections. + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + errCh := make(chan error, 1) + go func() { + prev := connectivity.Ready + for { + curr := cc.GetState() + if !(curr == connectivity.Ready || curr == connectivity.Idle) { + errCh <- fmt.Errorf("unexpected connectivity state change {%s --> %s} on the client connection", prev, curr) + return + } + if !cc.WaitForStateChange(ctx, curr) { + // Break out of the for loop when the context has been cancelled. + break + } + prev = curr + } + errCh <- nil + }() + + // Update the management server with the same listener resource. This will + // update the resource version though, and should result in a the management + // server sending the same resource to the xDS-enabled gRPC server. + if err := managementServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + }); err != nil { + t.Fatal(err) + } + + // Since redundant resource updates are suppressed, we should not see the + // mode change callback being invoked. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case mode := <-modeChangeHandler.modeCh: + t.Fatalf("Unexpected mode change callback with new mode %v", mode) + } + + // Make sure RPCs continue to succeed. + waitForSuccessfulRPC(ctx, t, cc) + + // Cancel the context to ensure that the WaitForStateChange call exits early + // and returns false. + cancel() + if err := <-errCh; err != nil { + t.Fatal(err) + } +} + +// Tests the case where the route configuration contains an unsupported route +// action. Verifies that RPCs fail with UNAVAILABLE. +func (s) TestServer_FailWithRouteActionRoute(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Configure the managegement server with a listener and route configuration + // resource for the above xDS enabled gRPC server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to listen to local port: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + const routeConfigName = "routeName" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListenerWithRouteConfigName(host, port, e2e.SecurityLevelNone, "routeName")}, + Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigNonForwardingAction(routeConfigName)}, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Start an xDS-enabled gRPC server with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + modeChangeOpt := xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) { + t.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err) + }) + createStubServer(t, lis, modeChangeOpt, xds.ClientPoolForTesting(pool)) + + // Create a gRPC channel and verify that RPCs succeed. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", lis.Addr(), err) + } + defer cc.Close() + waitForSuccessfulRPC(ctx, t, cc) + + // Update the route config resource to contain an unsupported action. + // + // "NonForwardingAction is expected for all Routes used on server-side; a + // route with an inappropriate action causes RPCs matching that route to + // fail with UNAVAILABLE." - A36 + resources.Routes = []*v3routepb.RouteConfiguration{e2e.RouteConfigFilterAction("routeName")} + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + waitForFailedRPCWithStatus(ctx, t, cc, codes.Unavailable, "the incoming RPC matched to a route that was not of action type non forwarding", nodeID) +} + +// Tests the case where the listener resource is removed from the management +// server. This should cause the xDS server to transition to NOT_SERVING mode, +// and the error message should contain the xDS node ID. +func (s) TestServer_ListenerResourceRemoved(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Configure the managegement server with a listener and route configuration + // resource for the above xDS enabled gRPC server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to listen to local port: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + const routeConfigName = "routeName" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListenerWithRouteConfigName(host, port, e2e.SecurityLevelNone, "routeName")}, + Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigNonForwardingAction(routeConfigName)}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Start an xDS-enabled gRPC server with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + modeChangeHandler := newServingModeChangeHandler(t) + modeChangeOpt := xds.ServingModeCallback(modeChangeHandler.modeChangeCallback) + createStubServer(t, lis, modeChangeOpt, xds.ClientPoolForTesting(pool)) + + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the xDS-enabled gRPC server to go SERVING") + case gotMode := <-modeChangeHandler.modeCh: + if gotMode != connectivity.ServingModeServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeServing) + } + } + + // Create a gRPC channel and verify that RPCs succeed. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", lis.Addr(), err) + } + defer cc.Close() + waitForSuccessfulRPC(ctx, t, cc) + + // Remove the listener resource from the management server. This should + // cause the server to go NOT_SERVING, and the error message should contain + // the xDS node ID. + resources.Listeners = nil + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for server to go NOT_SERVING") + case gotMode := <-modeChangeHandler.modeCh: + if gotMode != connectivity.ServingModeNotServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeNotServing) + } + gotErr := <-modeChangeHandler.errCh + if gotErr == nil || !strings.Contains(gotErr.Error(), nodeID) { + t.Fatalf("Unexpected error: %v, want xDS Node id: %s", gotErr, nodeID) + } + } +} + +// Tests the case where the listener resource points to a route configuration +// name that is NACKed. This should trigger the server to move to SERVING, +// successfully accept connections, and fail at RPCs with an expected error +// message. +func (s) TestServer_RouteConfiguration_ResourceNACK(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Configure the managegement server with a listener and route configuration + // resource (that will be NACKed) for the above xDS enabled gRPC server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to listen to local port: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + const routeConfigName = "routeName" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListenerWithRouteConfigName(host, port, e2e.SecurityLevelNone, "routeName")}, + Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigNoRouteMatch(routeConfigName)}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Start an xDS-enabled gRPC server with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + modeChangeHandler := newServingModeChangeHandler(t) + modeChangeOpt := xds.ServingModeCallback(modeChangeHandler.modeChangeCallback) + createStubServer(t, lis, modeChangeOpt, xds.ClientPoolForTesting(pool)) + + // Create a gRPC channel and verify that RPCs succeed. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", lis.Addr(), err) + } + defer cc.Close() + + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the server to start serving RPCs") + case gotMode := <-modeChangeHandler.modeCh: + if gotMode != connectivity.ServingModeServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeServing) + } + } + waitForFailedRPCWithStatus(ctx, t, cc, codes.Unavailable, "error from xDS configuration for matched route configuration", nodeID) +} + +// Tests the case where the listener resource points to multiple route +// configuration resources. +// +// - Initially the listener resource points to three route configuration +// resources (A, B and C). The filter chain in the listener matches incoming +// connections to route A, and RPCs are expected to succeed. +// - A streaming RPC is also kept open at this point. +// - The listener resource is then updated to point to two route configuration +// resources (A and B). The filter chain in the listener resource does not +// match to any of the configured routes. The default filter chain though +// matches to route B, which contains a route action of type "Route", and this +// is not supported on the server side. New RPCs are expected to fail, while +// any ongoing RPCs should be allowed to complete. +// - The listener resource is then updated to point to a single route +// configuration (A), and the filter chain in the listener matches to route A. +// New RPCs are expected to succeed at this point. +func (s) TestServer_MultipleRouteConfigurations(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create a listener on a local port to act as the xDS enabled gRPC server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to listen to local port: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + + // Setup the management server to respond with a listener resource that + // specifies three route names to watch, and the corresponding route + // configuration resources. + const routeConfigNameA = "routeName-A" + const routeConfigNameB = "routeName-B" + const routeConfigNameC = "routeName-C" + ldsResource := e2e.DefaultServerListenerWithRouteConfigName(host, port, e2e.SecurityLevelNone, routeConfigNameA) + ldsResource.FilterChains = append(ldsResource.FilterChains, + filterChainWontMatch(t, routeConfigNameB, "1.1.1.1", []uint32{1}), + filterChainWontMatch(t, routeConfigNameC, "2.2.2.2", []uint32{2}), + ) + routeConfigA := e2e.RouteConfigNonForwardingAction(routeConfigNameA) + routeConfigB := e2e.RouteConfigFilterAction(routeConfigNameB) // Unsupported route action on server. + routeConfigC := e2e.RouteConfigFilterAction(routeConfigNameC) // Unsupported route action on server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ldsResource}, + Routes: []*v3routepb.RouteConfiguration{routeConfigA, routeConfigB, routeConfigC}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Start an xDS-enabled gRPC server with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + modeChangeOpt := xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) { + t.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err) + }) + createStubServer(t, lis, modeChangeOpt, xds.ClientPoolForTesting(pool)) + + // Create a gRPC channel and verify that RPCs succeed. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient(%q) failed: %v", lis.Addr(), err) + } + defer cc.Close() + waitForSuccessfulRPC(ctx, t, cc) + + // Start a streaming RPC and keep the stream open. + client := testgrpc.NewTestServiceClient(cc) + stream, err := client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall failed: %v", err) + } + if err = stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("stream.Send() failed: %v, should continue to work due to graceful stop", err) + } + + // Update the listener resource such that the filter chain does not match + // incoming connections to route A. Instead a default filter chain matches + // to route B, which contains an unsupported route action. + ldsResource = e2e.DefaultServerListenerWithRouteConfigName(host, port, e2e.SecurityLevelNone, routeConfigNameA) + ldsResource.FilterChains = []*v3listenerpb.FilterChain{filterChainWontMatch(t, routeConfigNameA, "1.1.1.1", []uint32{1})} + ldsResource.DefaultFilterChain = filterChainWontMatch(t, routeConfigNameB, "2.2.2.2", []uint32{2}) + resources.Listeners = []*v3listenerpb.Listener{ldsResource} + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // xDS is eventually consistent. So simply poll for the new change to be + // reflected. + // "NonForwardingAction is expected for all Routes used on server-side; a + // route with an inappropriate action causes RPCs matching that route to + // fail with UNAVAILABLE." - A36 + waitForFailedRPCWithStatus(ctx, t, cc, codes.Unavailable, "the incoming RPC matched to a route that was not of action type non forwarding", nodeID) + + // Stream should be allowed to continue on the old working configuration - + // as it on a connection that is gracefully closed (old FCM/LDS + // Configuration which is allowed to continue). + if err = stream.CloseSend(); err != nil { + t.Fatalf("stream.CloseSend() failed: %v, should continue to work due to graceful stop", err) + } + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + + // Update the listener resource to point to a single route configuration + // that is expected to match and verify that RPCs succeed. + resources.Listeners = []*v3listenerpb.Listener{e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, routeConfigNameA)} + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + waitForSuccessfulRPC(ctx, t, cc) +} + +// filterChainWontMatch returns a filter chain that won't match if running the +// test locally. +func filterChainWontMatch(t *testing.T, routeName string, addressPrefix string, srcPorts []uint32) *v3listenerpb.FilterChain { + hcm := &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: routeName, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, + } + return &v3listenerpb.FilterChain{ + Name: routeName + "-wont-match", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: addressPrefix, + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePorts: srcPorts, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: addressPrefix, + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: testutils.MarshalAny(t, hcm)}, + }, + }, + } +} diff --git a/xds/server_security_ext_test.go b/xds/server_security_ext_test.go new file mode 100644 index 0000000..fbc21e5 --- /dev/null +++ b/xds/server_security_ext_test.go @@ -0,0 +1,475 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "net" + "strconv" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/xds" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// Tests the case where the bootstrap configuration contains no certificate +// providers, and xDS credentials with an insecure fallback is specified at +// server creation time. The management server is configured to return a +// server-side xDS Listener resource with no security configuration. The test +// verifies that a gRPC client configured with insecure credentials is able to +// make RPCs to the backend. This ensures that the insecure fallback +// credentials are getting used on the server. +func (s) TestServer_Security_NoCertProvidersInBootstrap_Success(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server. + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create a listener on a local port to act as the xDS enabled gRPC server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to listen to local port: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + + // Configure the managegement server with a listener and route configuration + // resource for the above xDS enabled gRPC server. + const routeConfigName = "routeName" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListenerWithRouteConfigName(host, port, e2e.SecurityLevelNone, "routeName")}, + Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigNonForwardingAction(routeConfigName)}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Start an xDS-enabled gRPC server with the above bootstrap configuration + // and configure xDS credentials to be used on the server-side. + creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{ + FallbackCreds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatal(err) + } + + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + modeChangeOpt := xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) { + t.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err) + }) + createStubServer(t, lis, grpc.Creds(creds), modeChangeOpt, xds.ClientPoolForTesting(pool)) + + // Create a client that uses insecure creds and verify RPCs. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } +} + +// Tests the case where the bootstrap configuration contains no certificate +// providers, and xDS credentials with an insecure fallback is specified at +// server creation time. The management server is configured to return a +// server-side xDS Listener resource with mTLS security configuration. The xDS +// client is expected to NACK this resource because the certificate provider +// instance name specified in the Listener resource will not be present in the +// bootstrap file. The test verifies that server creation does not fail and that +// if the xDS-enabled gRPC server receives resource error causing mode change, +// it does not enter "serving" mode. +func (s) TestServer_Security_NoCertificateProvidersInBootstrap_Failure(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Spin up an xDS management server that pushes on a channel when it + // receives a NACK for an LDS response. + nackCh := make(chan struct{}, 1) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() != "type.googleapis.com/envoy.config.listener.v3.Listener" { + return nil + } + if req.GetErrorDetail() == nil { + return nil + } + select { + case nackCh <- struct{}{}: + default: + } + return nil + }, + AllowResourceSubset: true, + }) + + // Create bootstrap configuration with no certificate providers. + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create a listener on a local port to act as the xDS enabled gRPC server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to listen to local port: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + // Start an xDS-enabled gRPC server with the above bootstrap configuration + // and configure xDS credentials to be used on the server-side. + creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{ + FallbackCreds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatal(err) + } + + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + modeChangeHandler := newServingModeChangeHandler(t) + modeChangeOpt := xds.ServingModeCallback(modeChangeHandler.modeChangeCallback) + createStubServer(t, lis, grpc.Creds(creds), modeChangeOpt, xds.ClientPoolForTesting(pool)) + + // Create an inbound xDS listener resource for the server side that contains + // mTLS security configuration. Since the received certificate provider + // instance name would be missing in the bootstrap configuration, this + // resource is expected to NACKed by the xDS client. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS, "routeName")}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the NACK from the xDS client. + select { + case <-nackCh: + case <-ctx.Done(): + t.Fatal("Timeout when waiting for an NACK from the xDS client for the LDS response") + } + + // Wait a short duration and ensure that if the server receive mode change + // it does not enter "serving" mode. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case stateCh := <-modeChangeHandler.modeCh: + if stateCh == connectivity.ServingModeServing { + t.Fatal("Server entered serving mode before the route config was received") + } + } + + // Create a client that uses insecure creds and verify that RPCs don't + // succeed. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + + waitForFailedRPCWithStatus(ctx, t, cc, codes.Unavailable, "", "") +} + +// Tests the case where the bootstrap configuration contains one certificate +// provider, and xDS credentials with an insecure fallback is specified at +// server creation time. Two listeners are configured on the xDS-enabled gRPC +// server. The management server responds with two listener resources: +// 1. contains valid security configuration pointing to the certificate provider +// instance specified in the bootstrap +// 2. contains invalid security configuration pointing to a non-existent +// certificate provider instance +// +// The test verifies that an RPC to the first listener succeeds, while the +// second listener receive a resource error which cause the server mode change +// but never moves to "serving" mode. +func (s) TestServer_Security_WithValidAndInvalidSecurityConfiguration(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Spin up an xDS management server that pushes on a channel when it + // receives a NACK for an LDS response. + nackCh := make(chan struct{}, 1) + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() != "type.googleapis.com/envoy.config.listener.v3.Listener" { + return nil + } + if req.GetErrorDetail() == nil { + return nil + } + select { + case nackCh <- struct{}{}: + default: + } + return nil + }, + AllowResourceSubset: true, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create two xDS-enabled gRPC servers using the above bootstrap configs. + lis1, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + lis2, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + + modeChangeHandler1 := newServingModeChangeHandler(t) + modeChangeOpt1 := xds.ServingModeCallback(modeChangeHandler1.modeChangeCallback) + modeChangeHandler2 := newServingModeChangeHandler(t) + modeChangeOpt2 := xds.ServingModeCallback(modeChangeHandler2.modeChangeCallback) + creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + createStubServer(t, lis1, grpc.Creds(creds), modeChangeOpt1, xds.ClientPoolForTesting(pool)) + createStubServer(t, lis2, grpc.Creds(creds), modeChangeOpt2, xds.ClientPoolForTesting(pool)) + + // Create inbound xDS listener resources for the server side that contains + // mTLS security configuration. + // lis1 --> security configuration pointing to a valid cert provider + // lis2 --> security configuration pointing to a non-existent cert provider + host1, port1, err := hostPortFromListener(lis1) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + resource1 := e2e.DefaultServerListener(host1, port1, e2e.SecurityLevelMTLS, "routeName") + host2, port2, err := hostPortFromListener(lis2) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + hcm := &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}}, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + } + ts := &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "non-existent-certificate-provider", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "non-existent-certificate-provider", + }, + }, + }, + }), + }, + } + resource2 := &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host2, strconv.Itoa(int(port2)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host2, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port2, + }, + }, + }, + }, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "v4-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: testutils.MarshalAny(t, hcm)}, + }, + }, + TransportSocket: ts, + }, + { + Name: "v6-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: testutils.MarshalAny(t, hcm)}, + }, + }, + TransportSocket: ts, + }, + }, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{resource1, resource2}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a client that uses TLS creds and verify RPCs to listener1. + clientCreds := testutils.CreateClientTLSCredentials(t) + cc1, err := grpc.NewClient(lis1.Addr().String(), grpc.WithTransportCredentials(clientCreds)) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc1.Close() + + client1 := testgrpc.NewTestServiceClient(cc1) + if _, err := client1.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Wait for the NACK from the xDS client. + select { + case <-nackCh: + case <-ctx.Done(): + t.Fatal("Timeout when waiting for an NACK from the xDS client for the LDS response") + } + + // Wait a short duration and ensure that if the server receives mode change + // it does not enter "serving" mode. + select { + case <-time.After(2 * defaultTestShortTimeout): + case mode := <-modeChangeHandler2.modeCh: + if mode == connectivity.ServingModeServing { + t.Fatal("Server changed to serving mode when not expected to") + } + } + + // Create a client that uses insecure creds and verify that RPCs don't + // succeed to listener2. + cc2, err := grpc.NewClient(lis2.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc2.Close() + + waitForFailedRPCWithStatus(ctx, t, cc2, codes.Unavailable, "", "") +} diff --git a/xds/server_serving_mode_ext_test.go b/xds/server_serving_mode_ext_test.go new file mode 100644 index 0000000..43fcc7b --- /dev/null +++ b/xds/server_serving_mode_ext_test.go @@ -0,0 +1,606 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "io" + "net" + "strconv" + "strings" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// Tests the Server's logic as it transitions from NOT_SERVING to SERVING, then +// to NOT_SERVING again. Before it goes to SERVING, connections should be +// accepted and closed. After it goes SERVING, RPC's should proceed as normal +// according to matched route configuration. After it transitions back into +// NOT_SERVING, (through an explicit LDS Resource Not Found), previously running +// RPC's should be gracefully closed and still work, and new RPC's should fail. +func (s) TestServer_ServingModeChanges_SingleServer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Setup the management server to respond with a listener resource that + // specifies a route name to watch. Due to not having received the full + // configuration, this should cause the server to be in mode NOT_SERVING. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + listener := e2e.DefaultServerListenerWithRouteConfigName(host, port, e2e.SecurityLevelNone, "routeName") + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Start an xDS-enabled gRPC server with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + modeChangeHandler := newServingModeChangeHandler(t) + modeChangeOpt := xds.ServingModeCallback(modeChangeHandler.modeChangeCallback) + createStubServer(t, lis, modeChangeOpt, xds.ClientPoolForTesting(pool)) + + // Start a gRPC channel to the above server. The server is yet to receive + // route configuration, and therefore RPCs must fail at this time. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + waitForFailedRPCWithStatus(ctx, t, cc, codes.Unavailable, "", "") + + // Setup the route configuration resource on the management server. This + // should cause the xDS-enabled gRPC server to move to SERVING mode. + routeConfig := e2e.RouteConfigNonForwardingAction("routeName") + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + Routes: []*v3routepb.RouteConfiguration{routeConfig}, + SkipValidation: true, + } + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the xDS-enabled gRPC server to go SERVING") + case gotMode := <-modeChangeHandler.modeCh: + if gotMode != connectivity.ServingModeServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeServing) + } + } + waitForSuccessfulRPC(ctx, t, cc) + + // Start a stream before switching the server to not serving. Due to the + // stream being created before the graceful stop of the underlying + // connection, it should be able to continue even after the server switches + // to not serving. + c := testgrpc.NewTestServiceClient(cc) + stream, err := c.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("cc.FullDuplexCall failed: %f", err) + } + + // Remove the listener resource from the management server. + resources.Listeners = nil + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure the server is in NOT_SERVING mode. + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the xDS-enabled gRPC server to go NOT_SERVING") + case gotMode := <-modeChangeHandler.modeCh: + if gotMode != connectivity.ServingModeNotServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeNotServing) + } + gotErr := <-modeChangeHandler.errCh + if gotErr == nil || !strings.Contains(gotErr.Error(), nodeID) { + t.Fatalf("Unexpected error: %v, want xDS Node id: %s", gotErr, nodeID) + } + } + + // Due to graceful stop, any started streams continue to work. + if err = stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("stream.Send() failed: %v, should continue to work due to graceful stop", err) + } + if err = stream.CloseSend(); err != nil { + t.Fatalf("stream.CloseSend() failed: %v, should continue to work due to graceful stop", err) + } + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("stream.Recv() failed with %v, want io.EOF", err) + } + + // New RPCs on that connection should eventually start failing. + waitForFailedRPCWithStatus(ctx, t, cc, codes.Unavailable, "", "") +} + +// Tests the serving mode functionality with multiple xDS enabled gRPC servers. +func (s) TestServer_ServingModeChanges_MultipleServers(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + managementServer, nodeID, bootstrapContents, _ := setup.ManagementServerAndResolver(t) + + // Create two local listeners and pass it to Serve(). + lis1, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + lis2, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + + // Create a server option to get notified about serving mode changes. + modeChangeHandler1 := newServingModeChangeHandler(t) + modeChangeOpt1 := xds.ServingModeCallback(modeChangeHandler1.modeChangeCallback) + modeChangeHandler2 := newServingModeChangeHandler(t) + modeChangeOpt2 := xds.ServingModeCallback(modeChangeHandler2.modeChangeCallback) + + // Start two xDS-enabled gRPC servers with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + createStubServer(t, lis1, modeChangeOpt1, xds.ClientPoolForTesting(pool)) + createStubServer(t, lis2, modeChangeOpt2, xds.ClientPoolForTesting(pool)) + + // Setup the management server to respond with server-side Listener + // resources for both listeners. + host1, port1, err := hostPortFromListener(lis1) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + listener1 := e2e.DefaultServerListener(host1, port1, e2e.SecurityLevelNone, "routeName") + host2, port2, err := hostPortFromListener(lis2) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + listener2 := e2e.DefaultServerListener(host2, port2, e2e.SecurityLevelNone, "routeName") + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener1, listener2}, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for both listeners to move to "serving" mode. + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for a mode change update: %v", err) + case mode := <-modeChangeHandler1.modeCh: + if mode != connectivity.ServingModeServing { + t.Fatalf("Listener 1 received new mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for a mode change update: %v", err) + case mode := <-modeChangeHandler2.modeCh: + if mode != connectivity.ServingModeServing { + t.Fatalf("Listener 2 received new mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + + // Create a ClientConn to the first listener and make a successful RPCs. + cc1, err := grpc.NewClient(lis1.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc1.Close() + waitForSuccessfulRPC(ctx, t, cc1) + + // Create a ClientConn to the second listener and make a successful RPCs. + cc2, err := grpc.NewClient(lis2.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + defer cc2.Close() + waitForSuccessfulRPC(ctx, t, cc2) + + // Update the management server to remove the second listener resource. This + // should push only the second listener into "not-serving" mode. + if err := managementServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener1}, + }); err != nil { + t.Fatal(err) + } + + // Wait for lis2 to move to "not-serving" mode. + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for a mode change update: %v", err) + case mode := <-modeChangeHandler2.modeCh: + if mode != connectivity.ServingModeNotServing { + t.Fatalf("Listener received new mode %v, want %v", mode, connectivity.ServingModeNotServing) + } + gotErr := <-modeChangeHandler2.errCh + if gotErr == nil || !strings.Contains(gotErr.Error(), nodeID) { + t.Fatalf("Unexpected error: %v, want xDS Node id: %s", gotErr, nodeID) + } + } + + // Make sure RPCs succeed on cc1 and fail on cc2. + waitForSuccessfulRPC(ctx, t, cc1) + waitForFailedRPCWithStatus(ctx, t, cc2, codes.Unavailable, "", "") + + // Update the management server to remove the first listener resource as + // well. This should push the first listener into "not-serving" mode. Second + // listener is already in "not-serving" mode. + if err := managementServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{}, + }); err != nil { + t.Fatal(err) + } + + // Wait for lis1 to move to "not-serving" mode. lis2 was already removed + // from the xdsclient's resource cache. So, lis2's callback will not be + // invoked this time around. + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for a mode change update: %v", err) + case mode := <-modeChangeHandler1.modeCh: + if mode != connectivity.ServingModeNotServing { + t.Fatalf("Listener received new mode %v, want %v", mode, connectivity.ServingModeNotServing) + } + gotErr := <-modeChangeHandler1.errCh + if gotErr == nil || !strings.Contains(gotErr.Error(), nodeID) { + t.Fatalf("Unexpected error: %v, want xDS Node id: %s", gotErr, nodeID) + } + } + + // Make sure RPCs fail on both. + waitForFailedRPCWithStatus(ctx, t, cc1, codes.Unavailable, "", "") + waitForFailedRPCWithStatus(ctx, t, cc2, codes.Unavailable, "", "") + + // Make sure new connection attempts to "not-serving" servers fail. + if cc1, err = grpc.NewClient(lis1.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())); err != nil { + t.Fatal("Failed to create clientConn to a server in \"not-serving\" state") + } + defer cc1.Close() + if _, err := testgrpc.NewTestServiceClient(cc1).FullDuplexCall(ctx); status.Code(err) != codes.Unavailable { + t.Fatalf("FullDuplexCall failed with status code: %v, want: Unavailable", status.Code(err)) + } + + // Update the management server with both listener resources. + if err := managementServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener1, listener2}, + }); err != nil { + t.Fatal(err) + } + + // Wait for both listeners to move to "serving" mode. + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for a mode change update: %v", err) + case mode := <-modeChangeHandler1.modeCh: + if mode != connectivity.ServingModeServing { + t.Fatalf("Listener received new mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for a mode change update: %v", err) + case mode := <-modeChangeHandler2.modeCh: + if mode != connectivity.ServingModeServing { + t.Fatalf("Listener received new mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + + // The clientConns created earlier should be able to make RPCs now. + waitForSuccessfulRPC(ctx, t, cc1) + waitForSuccessfulRPC(ctx, t, cc2) +} + +// Tests the case where the server receives a client-side listener resource. +// The server should transition to NotServing mode and reject incoming connections. +func (s) TestServer_ClientSideListenerReceivedOnServer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create a listener on a local port to act as the xDS enabled gRPC server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + + // Configure a client-side listener resource on the management server. + // We use the server-side listening address as the resource name, but return a + // client-side listener resource (meaning it has ApiListener and lacks inbound TCP config). + listenerName := fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))) + listener := e2e.DefaultClientListener(listenerName, "routeName") + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Start an xDS-enabled gRPC server with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + modeChangeHandler := newServingModeChangeHandler(t) + modeChangeOpt := xds.ServingModeCallback(modeChangeHandler.modeChangeCallback) + createStubServer(t, lis, modeChangeOpt, xds.ClientPoolForTesting(pool)) + + // Ensure the server transitions to NOT_SERVING mode due to client-side listener being sent. + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the xDS-enabled gRPC server to go NOT_SERVING") + case gotMode := <-modeChangeHandler.modeCh: + if gotMode != connectivity.ServingModeNotServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeNotServing) + } + const wantErr = "received client-side listener resource" + gotErr := <-modeChangeHandler.errCh + if gotErr == nil || !strings.Contains(gotErr.Error(), wantErr) { + t.Fatalf("Unexpected error: %v, want error containing '%s'", gotErr, wantErr) + } + } + + // Update the management server with the expected server-side listener and route config. + // This should cause the server to recover and transition back to SERVING mode. + listenerServer := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName") + routeConfig := e2e.RouteConfigNonForwardingAction("routeName") + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerServer}, + Routes: []*v3routepb.RouteConfiguration{routeConfig}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the xDS-enabled gRPC server to go SERVING") + case gotMode := <-modeChangeHandler.modeCh: + if gotMode != connectivity.ServingModeServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeServing) + } + } + + // Create a gRPC client to the server and verify that we can make a successful RPC. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + waitForSuccessfulRPC(ctx, t, cc) +} + +// Tests the case where the server receives a listener resource with a mismatched port/address. +// The server should transition to NotServing mode and reject incoming connections. +func (s) TestServer_MismatchedAddressListenerReceivedOnServer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create a listener on a local port to act as the xDS enabled gRPC server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + + // Configure a server-side listener with a mismatched port on the management server. + listener := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName") + listener.Address.GetSocketAddress().PortSpecifier = &v3corepb.SocketAddress_PortValue{ + PortValue: port + 1, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Start an xDS-enabled gRPC server with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + modeChangeHandler := newServingModeChangeHandler(t) + modeChangeOpt := xds.ServingModeCallback(modeChangeHandler.modeChangeCallback) + createStubServer(t, lis, modeChangeOpt, xds.ClientPoolForTesting(pool)) + + // Ensure the server transitions to NOT_SERVING mode due to the address/port mismatch. + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the xDS-enabled gRPC server to go NOT_SERVING") + case gotMode := <-modeChangeHandler.modeCh: + if gotMode != connectivity.ServingModeNotServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeNotServing) + } + const wantErr = "does not match listening address" + gotErr := <-modeChangeHandler.errCh + if gotErr == nil || !strings.Contains(gotErr.Error(), wantErr) { + t.Fatalf("Unexpected error: %v, want error containing '%s'", gotErr, wantErr) + } + } + + // Update the management server with the expected matching server-side listener and route config. + // This should cause the server to recover and transition back to SERVING mode. + listenerServer := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName") + routeConfig := e2e.RouteConfigNonForwardingAction("routeName") + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerServer}, + Routes: []*v3routepb.RouteConfiguration{routeConfig}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the xDS-enabled gRPC server to go SERVING") + case gotMode := <-modeChangeHandler.modeCh: + if gotMode != connectivity.ServingModeServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeServing) + } + } + + // Create a gRPC client to the server and verify that we can make a successful RPC. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + waitForSuccessfulRPC(ctx, t, cc) +} + +// Tests the case where the server receives a listener resource with a wildcard +// port. The server should match the listener address and transition to Serving +// mode regardless of the port on which it is listening. +func (s) TestServer_ListenerMatchWithWildcardPort(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address) + + // Create a listener on a local port to act as the xDS enabled gRPC server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of server: %v", err) + } + + // Configure a server-side listener with a wildcard port on the management + // server. Construct the listener with the actual port first so its resource + // name matches the server's default watch name, then replace only the port + // value in the socket address with zero. + listener := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName") + listener.Address.GetSocketAddress().PortSpecifier = &v3corepb.SocketAddress_PortValue{ + PortValue: 0, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Start an xDS-enabled gRPC server with the above bootstrap configuration. + config, err := bootstrap.NewConfigFromContents(bootstrapContents) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err) + } + pool := xdsclient.NewPool(config) + modeChangeHandler := newServingModeChangeHandler(t) + modeChangeOpt := xds.ServingModeCallback(modeChangeHandler.modeChangeCallback) + createStubServer(t, lis, modeChangeOpt, xds.ClientPoolForTesting(pool)) + + // Ensure the server transitions to SERVING mode despite the wildcard port. + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for the xDS-enabled gRPC server to go SERVING") + case gotMode := <-modeChangeHandler.modeCh: + if gotMode != connectivity.ServingModeServing { + t.Fatalf("Mode changed to %v, want %v", gotMode, connectivity.ServingModeServing) + } + } + + // Create a gRPC client to the server and verify that we can make a + // successful RPC. + cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + waitForSuccessfulRPC(ctx, t, cc) +} diff --git a/xds/server_test.go b/xds/server_test.go new file mode 100644 index 0000000..e21a6a1 --- /dev/null +++ b/xds/server_test.go @@ -0,0 +1,814 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/xds/bootstrap" + internalserver "google.golang.org/grpc/internal/xds/server" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + nonExistentManagementServer = "non-existent-management-server" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type fakeGRPCServer struct { + done chan struct{} + registerServiceCh *testutils.Channel + serveCh *testutils.Channel +} + +func (f *fakeGRPCServer) RegisterService(*grpc.ServiceDesc, any) { + f.registerServiceCh.Send(nil) +} + +func (f *fakeGRPCServer) Serve(lis net.Listener) error { + f.serveCh.Send(nil) + <-f.done + lis.Close() + return nil +} + +func (f *fakeGRPCServer) Stop() { + close(f.done) +} +func (f *fakeGRPCServer) GracefulStop() { + close(f.done) +} + +func (f *fakeGRPCServer) GetServiceInfo() map[string]grpc.ServiceInfo { + panic("implement me") +} + +func newFakeGRPCServer() *fakeGRPCServer { + return &fakeGRPCServer{ + done: make(chan struct{}), + registerServiceCh: testutils.NewChannel(), + serveCh: testutils.NewChannel(), + } +} + +func generateBootstrapContents(t *testing.T, nodeID, serverURI string) []byte { + bs := e2e.DefaultBootstrapContents(t, nodeID, serverURI) + return bs +} + +func (s) TestNewServer_Success(t *testing.T) { + xdsCreds, err := xds.NewServerCredentials(xds.ServerOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("failed to create xds server credentials: %v", err) + } + + tests := []struct { + desc string + serverOpts []grpc.ServerOption + wantXDSCredsInUse bool + }{ + { + desc: "without_xds_creds", + serverOpts: []grpc.ServerOption{ + grpc.Creds(insecure.NewCredentials()), + BootstrapContentsForTesting(generateBootstrapContents(t, uuid.NewString(), nonExistentManagementServer)), + }, + }, + { + desc: "with_xds_creds", + serverOpts: []grpc.ServerOption{ + grpc.Creds(xdsCreds), + BootstrapContentsForTesting(generateBootstrapContents(t, uuid.NewString(), nonExistentManagementServer)), + }, + wantXDSCredsInUse: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // The xds package adds 1 server option (stream wrapper) to the server + // options passed in by the user. + wantServerOpts := len(test.serverOpts) + 1 + + origNewGRPCServer := newGRPCServer + newGRPCServer = func(opts ...grpc.ServerOption) grpcServer { + if got := len(opts); got != wantServerOpts { + t.Fatalf("%d ServerOptions passed to grpc.Server, want %d", got, wantServerOpts) + } + // Verify that the user passed ServerOptions are forwarded as is. + if diff := cmp.Diff(test.serverOpts, opts[1:], cmp.Comparer(func(a, b grpc.ServerOption) bool { return a == b })); diff != "" { + t.Fatalf("ServerOptions mismatch (-want +got):\n%s", diff) + } + return grpc.NewServer(opts...) + } + defer func() { + newGRPCServer = origNewGRPCServer + }() + + s, err := NewGRPCServer(test.serverOpts...) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + defer s.Stop() + }) + } +} + +func (s) TestNewServer_Failure(t *testing.T) { + xdsCreds, err := xds.NewServerCredentials(xds.ServerOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("failed to create xds server credentials: %v", err) + } + + tests := []struct { + desc string + serverOpts []grpc.ServerOption + wantErr string + }{ + { + desc: "bootstrap env var not set", + serverOpts: []grpc.ServerOption{grpc.Creds(xdsCreds), BootstrapContentsForTesting(nil)}, + wantErr: "failed to read xDS bootstrap config from env vars", + }, + { + desc: "empty bootstrap config", + serverOpts: []grpc.ServerOption{ + grpc.Creds(xdsCreds), + BootstrapContentsForTesting(nil), + }, + wantErr: "xDS client creation failed", + }, + { + desc: "server_listener_resource_name_template is missing", + serverOpts: []grpc.ServerOption{ + grpc.Creds(xdsCreds), + func() grpc.ServerOption { + bs, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, nonExistentManagementServer)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, uuid.New().String())), + CertificateProviders: map[string]json.RawMessage{ + "cert-provider-instance": json.RawMessage("{}"), + }, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + return BootstrapContentsForTesting(bs) + }(), + }, + wantErr: "missing server_listener_resource_name_template in the bootstrap configuration", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + s, err := NewGRPCServer(test.serverOpts...) + if err == nil { + s.Stop() + t.Fatal("NewGRPCServer() succeeded when expected to fail") + } + if !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("NewGRPCServer() failed with error: %v, want: %s", err, test.wantErr) + } + }) + } +} + +// TestServer_OverrideListenerResourceNameOverridesMissingTemplate verifies +// that an internal listener resource name override takes precedence over +// server_listener_resource_name_template from bootstrap, including when the +// template is absent, and that its returned name is used for the LDS watch. +func (s) TestServer_OverrideListenerResourceNameOverridesMissingTemplate(t *testing.T) { + const wantResourceName = "xdstp://foo/bar" + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + ldsResourceNamesCh := make(chan []string, 1) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3ListenerURL { + select { + case ldsResourceNamesCh <- req.GetResourceNames(): + case <-ctx.Done(): + } + } + return nil + }, + }) + + bs, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, uuid.New().String())), + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + fs := newFakeGRPCServer() + origNewGRPCServer := newGRPCServer + newGRPCServer = func(...grpc.ServerOption) grpcServer { return fs } + defer func() { newGRPCServer = origNewGRPCServer }() + + lisAddrCh := make(chan net.Addr, 1) + resourceNameOpt := internalserver.OverrideListenerResourceName(func(addr net.Addr) string { + lisAddrCh <- addr + return wantResourceName + }) + srv, err := NewGRPCServer(resourceNameOpt, BootstrapContentsForTesting(bs)) + if err != nil { + t.Fatalf("NewGRPCServer() failed: %v", err) + } + defer srv.Stop() + + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + go func() { _ = srv.Serve(lis) }() + + // Verify that OverrideListenerResourceName receives the listener address. + select { + case gotAddr := <-lisAddrCh: + if gotAddr.String() != lis.Addr().String() { + t.Fatalf("OverrideListenerResourceName() called with address %q, want %q", gotAddr, lis.Addr()) + } + case <-ctx.Done(): + t.Fatal("Timeout waiting for OverrideListenerResourceName to be called") + } + + // Verify that the LDS watch uses the resource name returned by the override. + var gotResourceNames []string + select { + case gotResourceNames = <-ldsResourceNamesCh: + case <-ctx.Done(): + t.Fatal("Timeout waiting for an LDS request") + } + wantResourceNames := []string{wantResourceName} + if !cmp.Equal(gotResourceNames, wantResourceNames) { + t.Fatalf("LDS watch registered for names %v, want %v", gotResourceNames, wantResourceNames) + } +} + +func (s) TestRegisterService(t *testing.T) { + fs := newFakeGRPCServer() + + origNewGRPCServer := newGRPCServer + newGRPCServer = func(...grpc.ServerOption) grpcServer { return fs } + defer func() { newGRPCServer = origNewGRPCServer }() + + s, err := NewGRPCServer(BootstrapContentsForTesting(generateBootstrapContents(t, uuid.NewString(), "non-existent-management-server"))) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + defer s.Stop() + + s.RegisterService(&grpc.ServiceDesc{}, nil) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := fs.registerServiceCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when expecting RegisterService() to called on grpc.Server: %v", err) + } +} + +const ( + fakeProvider1Name = "fake-certificate-provider-1" + fakeProvider2Name = "fake-certificate-provider-2" +) + +var ( + fpb1, fpb2 *fakeProviderBuilder + fakeProvider1Config json.RawMessage + fakeProvider2Config json.RawMessage +) + +func init() { + fpb1 = &fakeProviderBuilder{ + name: fakeProvider1Name, + buildCh: testutils.NewChannel(), + } + fpb2 = &fakeProviderBuilder{ + name: fakeProvider2Name, + buildCh: testutils.NewChannel(), + } + certprovider.Register(fpb1) + certprovider.Register(fpb2) + + fakeProvider1Config = json.RawMessage(fmt.Sprintf(`{ + "plugin_name": "%s", + "config": "my fake config 1" + }`, fakeProvider1Name)) + fakeProvider2Config = json.RawMessage(fmt.Sprintf(`{ + "plugin_name": "%s", + "config": "my fake config 2" + }`, fakeProvider2Name)) +} + +// fakeProviderBuilder builds new instances of fakeProvider and interprets the +// config provided to it as a string. +type fakeProviderBuilder struct { + name string + buildCh *testutils.Channel +} + +func (b *fakeProviderBuilder) ParseConfig(cfg any) (*certprovider.BuildableConfig, error) { + var config string + if err := json.Unmarshal(cfg.(json.RawMessage), &config); err != nil { + return nil, fmt.Errorf("providerBuilder %s failed to unmarshal config: %v", b.name, cfg) + } + return certprovider.NewBuildableConfig(b.name, []byte(config), func(certprovider.BuildOptions) certprovider.Provider { + b.buildCh.Send(nil) + return &fakeProvider{ + Distributor: certprovider.NewDistributor(), + config: config, + } + }), nil +} + +func (b *fakeProviderBuilder) Name() string { + return b.name +} + +// fakeProvider is an implementation of the Provider interface which provides a +// method for tests to invoke to push new key materials. +type fakeProvider struct { + *certprovider.Distributor + config string +} + +// Close helps implement the Provider interface. +func (p *fakeProvider) Close() { + p.Distributor.Stop() +} + +func verifyCertProviderNotCreated() error { + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := fpb1.buildCh.Receive(sCtx); err != context.DeadlineExceeded { + return errors.New("certificate provider created when no xDS creds were specified") + } + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := fpb2.buildCh.Receive(sCtx); err != context.DeadlineExceeded { + return errors.New("certificate provider created when no xDS creds were specified") + } + return nil +} + +func hostPortFromListener(t *testing.T, lis net.Listener) (string, uint32) { + t.Helper() + + host, p, err := net.SplitHostPort(lis.Addr().String()) + if err != nil { + t.Fatalf("net.SplitHostPort(%s) failed: %v", lis.Addr().String(), err) + } + port, err := strconv.ParseInt(p, 10, 32) + if err != nil { + t.Fatalf("strconv.ParseInt(%s, 10, 32) failed: %v", p, err) + } + return host, uint32(port) +} + +// TestServeSuccess tests the successful case of creating an xDS enabled gRPC +// server and calling Serve() on it. The test verifies that an LDS request is +// sent out for the expected name, and also verifies that the serving mode +// changes appropriately. +func (s) TestServeSuccess(t *testing.T) { + // Setup an xDS management server that pushes on a channel when an LDS + // request is received by it. + ldsRequestCh := make(chan []string, 1) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3ListenerURL { + select { + case ldsRequestCh <- req.GetResourceNames(): + default: + } + } + return nil + }, + }) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + + // Override the function to create the underlying grpc.Server to allow the + // test to verify that Serve() is called on the underlying server. + fs := newFakeGRPCServer() + origNewGRPCServer := newGRPCServer + newGRPCServer = func(...grpc.ServerOption) grpcServer { return fs } + defer func() { newGRPCServer = origNewGRPCServer }() + + // Create a new xDS enabled gRPC server and pass it a server option to get + // notified about serving mode changes. + modeChangeCh := testutils.NewChannel() + modeChangeOption := ServingModeCallback(func(addr net.Addr, args ServingModeChangeArgs) { + t.Logf("Server mode change callback invoked for listener %q with mode %q and error %v", addr.String(), args.Mode, args.Err) + modeChangeCh.Send(args.Mode) + }) + server, err := NewGRPCServer(modeChangeOption, BootstrapContentsForTesting(bootstrapContents)) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + defer server.Stop() + + // Call Serve() in a goroutine. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + go func() { + if err := server.Serve(lis); err != nil { + t.Error(err) + } + }() + + // Ensure that the LDS request is sent out for the expected name. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + var gotNames []string + select { + case gotNames = <-ldsRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for an LDS request to be sent out") + } + wantNames := []string{strings.ReplaceAll(e2e.ServerListenerResourceNameTemplate, "%s", lis.Addr().String())} + if !cmp.Equal(gotNames, wantNames) { + t.Fatalf("LDS watch registered for names %v, want %v", gotNames, wantNames) + } + + // Update the management server with a good listener resource. + host, port := hostPortFromListener(t, lis) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName")}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify the serving mode reports SERVING. + v, err := modeChangeCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for serving mode to change: %v", err) + } + if mode := v.(connectivity.ServingMode); mode != connectivity.ServingModeServing { + t.Fatalf("Serving mode is %q, want %q", mode, connectivity.ServingModeServing) + } + + // Verify that Serve() is called on the underlying gRPC server. + if _, err := fs.serveCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for Serve() to be invoked on the grpc.Server") + } + + // Update the listener resource on the management server in such a way that + // it will be NACKed by our xDS client. The listener_filters field is + // unsupported and will be NACKed. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName")}, + } + resources.Listeners[0].ListenerFilters = []*v3listenerpb.ListenerFilter{{Name: "foo"}} + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that there is no change in the serving mode. The server should + // continue using the previously received good configuration. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if v, err := modeChangeCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected change in serving mode. New mode is %v", v.(connectivity.ServingMode)) + } + + // Remove the listener resource from the management server. This should + // result in a resource-not-found error from the xDS client and should + // result in the server moving to NOT_SERVING mode. + resources.Listeners = nil + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + v, err = modeChangeCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for serving mode to change: %v", err) + } + if mode := v.(connectivity.ServingMode); mode != connectivity.ServingModeNotServing { + t.Fatalf("Serving mode is %q, want %q", mode, connectivity.ServingModeNotServing) + } +} + +// TestNewServer_ClientCreationFailure tests the case where the xDS client +// creation fails and verifies that the call to NewGRPCServer() fails. +func (s) TestNewServer_ClientCreationFailure(t *testing.T) { + origXDSClientPool := xdsClientPool + xdsClientPool = xdsclient.NewPool(nil) + defer func() { xdsClientPool = origXDSClientPool }() + + if _, err := NewGRPCServer(); err == nil { + t.Fatal("NewGRPCServer() succeeded when expected to fail") + } +} + +// TestHandleListenerUpdate_NoXDSCreds tests the case where an xds-enabled gRPC +// server is not configured with xDS credentials. Verifies that the security +// config received as part of a Listener update is not acted upon. +func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) { + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{}) + + // Generate bootstrap configuration pointing to the above management server + // with certificate provider configuration pointing to fake certificate + // providers. + nodeID := uuid.NewString() + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + CertificateProviders: map[string]json.RawMessage{ + e2e.ServerSideCertProviderInstance: fakeProvider1Config, + e2e.ClientSideCertProviderInstance: fakeProvider2Config, + }, + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create a new xDS enabled gRPC server and pass it a server option to get + // notified about serving mode changes. Also pass the above bootstrap + // configuration to be used during xDS client creation. + modeChangeCh := testutils.NewChannel() + modeChangeOption := ServingModeCallback(func(addr net.Addr, args ServingModeChangeArgs) { + t.Logf("Server mode change callback invoked for listener %q with mode %q and error %v", addr.String(), args.Mode, args.Err) + modeChangeCh.Send(args.Mode) + }) + server, err := NewGRPCServer(modeChangeOption, BootstrapContentsForTesting(bootstrapContents)) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + defer server.Stop() + + // Call Serve() in a goroutine. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + go func() { + if err := server.Serve(lis); err != nil { + t.Error(err) + } + }() + + // Update the management server with a good listener resource that contains + // security configuration. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + host, port := hostPortFromListener(t, lis) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS, "routeName")}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify the serving mode reports SERVING. + v, err := modeChangeCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for serving mode to change: %v", err) + } + if mode := v.(connectivity.ServingMode); mode != connectivity.ServingModeServing { + t.Fatalf("Serving mode is %q, want %q", mode, connectivity.ServingModeServing) + } + + // Make sure the security configuration is not acted upon. + if err := verifyCertProviderNotCreated(); err != nil { + t.Fatal(err) + } +} + +// TestHandleListenerUpdate_ErrorUpdate tests the case where an xds-enabled gRPC +// server is configured with xDS credentials, but receives a Listener update +// with an error. Verifies that no certificate providers are created. +func (s) TestHandleListenerUpdate_ErrorUpdate(t *testing.T) { + // Setup an xDS management server that pushes on a channel when an LDS + // request is received by it. + ldsRequestCh := make(chan []string, 1) + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3ListenerURL { + select { + case ldsRequestCh <- req.GetResourceNames(): + default: + } + } + return nil + }, + }) + + // Generate bootstrap configuration pointing to the above management server + // with certificate provider configuration pointing to fake certificate + // providers. + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": %q, + "channel_creds": [{"type": "insecure"}] + }]`, mgmtServer.Address)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + CertificateProviders: map[string]json.RawMessage{ + e2e.ServerSideCertProviderInstance: fakeProvider1Config, + e2e.ClientSideCertProviderInstance: fakeProvider2Config, + }, + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create a new xDS enabled gRPC server and pass it a server option to get + // notified about serving mode changes. Also pass the above bootstrap + // configuration to be used during xDS client creation. + modeChangeCh := testutils.NewChannel() + modeChangeOption := ServingModeCallback(func(addr net.Addr, args ServingModeChangeArgs) { + // The serving mode callback may be invoked multiple times as the xDS + // client NACKs the updates and the management server responds back with + // the same version of the LDS resource. + t.Logf("Server mode change callback invoked for listener %q with mode %q and error %v", addr.String(), args.Mode, args.Err) + modeChangeCh.Replace(args.Mode) + }) + server, err := NewGRPCServer(modeChangeOption, BootstrapContentsForTesting(bootstrapContents)) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + defer server.Stop() + + // Call Serve() in a goroutine. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + go server.Serve(lis) + + // Update the listener resource on the management server in such a way that + // it will be NACKed by our xDS client. The listener_filters field is + // unsupported and will be NACKed. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + host, port := hostPortFromListener(t, lis) + listener := e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS, "routeName") + listener.ListenerFilters = []*v3listenerpb.ListenerFilter{{Name: "foo"}} + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that the LDS request is sent out for the expected name. + var gotNames []string + select { + case gotNames = <-ldsRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for an LDS request to be sent out") + } + wantNames := []string{strings.ReplaceAll(e2e.ServerListenerResourceNameTemplate, "%s", lis.Addr().String())} + if !cmp.Equal(gotNames, wantNames) { + t.Fatalf("LDS watch registered for names %v, want %v", gotNames, wantNames) + } + + // Make sure that no certificate providers are created. + if err := verifyCertProviderNotCreated(); err != nil { + t.Fatal(err) + } + + // Also make sure that serving mode updates are received. The serving + // mode changes to NOT_SERVING. This happens because watcher received a + // resource error for the invalid resource from the server. + mode, err := modeChangeCh.Receive(ctx) + if err == context.DeadlineExceeded { + t.Fatal("Serving mode did not change when expected to change") + } + if mode != connectivity.ServingModeNotServing { + t.Fatalf("Serving mode = %q, want %q", mode, connectivity.ServingModeNotServing) + } +} + +// TestServeReturnsErrorAfterClose tests that the xds Server returns +// grpc.ErrServerStopped if Serve is called after Close on the server. +func (s) TestServeReturnsErrorAfterClose(t *testing.T) { + server, err := NewGRPCServer(BootstrapContentsForTesting(generateBootstrapContents(t, uuid.NewString(), nonExistentManagementServer))) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + server.Stop() + err = server.Serve(lis) + if err == nil || !strings.Contains(err.Error(), grpc.ErrServerStopped.Error()) { + t.Fatalf("server erorred with wrong error, want: %v, got :%v", grpc.ErrServerStopped, err) + } +} + +// TestServeAndCloseDoNotRace tests that Serve and Close on the xDS Server do +// not race and leak the xDS Client. A leak would be found by the leak checker. +func (s) TestServeAndCloseDoNotRace(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + + // Generate bootstrap contents up front for all servers. + bootstrapContents := generateBootstrapContents(t, uuid.NewString(), nonExistentManagementServer) + + // Override the default ServingModeCallback with a noop function because the + // serverURI is invalid which will result in xDS channel creation failure + // while registering the watch for listener resource. This will trigger + // resource error notifications for the invalid listener resource leading + // to service mode change to "not serving" each time. + // + // Even if the server is currently NOT_SERVING and the new mode is also + // NOT_SERVING, the update is not suppressed as: + // 1. the error may have change + // 2. it provides a timestamp of the last backoff attempt + noopModeChangeCallback := func(_ net.Addr, _ ServingModeChangeArgs) {} + + wg := sync.WaitGroup{} + wg.Add(200) + for i := 0; i < 100; i++ { + server, err := NewGRPCServer(BootstrapContentsForTesting(bootstrapContents), ServingModeCallback(noopModeChangeCallback)) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + go func() { + server.Serve(lis) + wg.Done() + }() + go func() { + server.Stop() + wg.Done() + }() + } + wg.Wait() +} diff --git a/xds/test/eds_resource_missing_test.go b/xds/test/eds_resource_missing_test.go new file mode 100644 index 0000000..2472705 --- /dev/null +++ b/xds/test/eds_resource_missing_test.go @@ -0,0 +1,188 @@ +/* + * + * Copyright 2025 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/internal/xds/bootstrap" + "google.golang.org/grpc/internal/xds/xdsclient" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/xds" // To register the xDS resolver and LB policies. +) + +const ( + defaultTestWatchExpiryTimeout = 500 * time.Millisecond + defaultTestTimeout = 5 * time.Second +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// Test verifies the xDS-enabled gRPC channel's behavior when the management +// server fails to send an EDS resource referenced by a Cluster resource. The +// expected outcome is an RPC failure with a status code Unavailable and a +// message indicating the absence of available targets. +func (s) TestEDS_MissingResource(t *testing.T) { + // Start an xDS management server. + mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address) + config, err := bootstrap.NewConfigFromContents(bc) + if err != nil { + t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err) + } + + // Create an xDS client with a short resource expiry timer. + pool := xdsclient.NewPool(config) + xdsC, xdsClose, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{ + Name: t.Name(), + WatchExpiryTimeout: defaultTestWatchExpiryTimeout, + }) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer xdsClose() + + // Create an xDS resolver for the test that uses the above xDS client. + resolverBuilder := internal.NewXDSResolverWithClientForTesting.(func(xdsclient.XDSClient) (resolver.Builder, error)) + xdsResolver, err := resolverBuilder(xdsC) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + // Create resources on the management server. No EDS resource is configured. + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName = "cluster-" + serviceName + const endpointsName = "endpoints-" + serviceName + resources := e2e.UpdateOptions{ + NodeID: nodeID, + SkipValidation: true, // Cluster resource refers to an EDS resource that is not configured. + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelNone)}, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn with the xds:/// scheme. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("Failed to create a grpc channel: %v", err) + } + defer cc.Close() + + // Make an RPC and verify that it fails with the expected error. + client := testgrpc.NewTestServiceClient(cc) + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Fatal("EmptyCall() succeeded, want failure") + } + if gotCode, wantCode := status.Code(err), codes.Unavailable; gotCode != wantCode { + t.Errorf("EmptyCall() failed with code = %v, want %s", gotCode, wantCode) + } + if gotMsg, wantMsg := err.Error(), "no targets to pick from"; !strings.Contains(gotMsg, wantMsg) { + t.Errorf("EmptyCall() failed with message = %q, want to contain %q", gotMsg, wantMsg) + } +} + +// Test verifies the xDS-enabled gRPC channel's behavior when the management +// server sends an EDS resource with no endpoints. The expected outcome is an +// RPC failure with a status code Unavailable and a message indicating the +// absence of available targets. +func (s) TestEDS_NoEndpointsInResource(t *testing.T) { + managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t) + + // Create resources on the management server, with the EDS resource + // containing no endpoints. + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName = "cluster-" + serviceName + const endpointsName = "endpoints-" + serviceName + resources := e2e.UpdateOptions{ + NodeID: nodeID, + SkipValidation: true, // Cluster resource refers to an EDS resource that is not configured. + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: "endpoints-" + serviceName, + Host: "localhost", + }), + }, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn with the xds:/// scheme. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("Failed to create a grpc channel: %v", err) + } + defer cc.Close() + + // Make an RPC and verify that it fails with the expected error. + client := testgrpc.NewTestServiceClient(cc) + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Fatal("EmptyCall() succeeded, want failure") + } + if gotCode, wantCode := status.Code(err), codes.Unavailable; gotCode != wantCode { + t.Errorf("EmptyCall() failed with code = %v, want %s", gotCode, wantCode) + } + if gotMsg, wantMsg := err.Error(), "no targets to pick from"; !strings.Contains(gotMsg, wantMsg) { + t.Errorf("EmptyCall() failed with message = %q, want to contain %q", gotMsg, wantMsg) + } +} diff --git a/xds/xds.go b/xds/xds.go new file mode 100644 index 0000000..182c32d --- /dev/null +++ b/xds/xds.go @@ -0,0 +1,100 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xds contains an implementation of the xDS suite of protocols, to be +// used by gRPC client and server applications. +// +// On the client-side, users simply need to import this package to get all xDS +// functionality. On the server-side, users need to use the GRPCServer type +// exported by this package instead of the regular grpc.Server. +// +// See https://github.com/grpc/grpc-go/tree/master/examples/features/xds for +// example. +package xds + +import ( + "fmt" + + "google.golang.org/grpc" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + internaladmin "google.golang.org/grpc/internal/admin" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/xds/csds" + + _ "google.golang.org/grpc/credentials/tls/certprovider/pemfile" // Register the file watcher certificate provider plugin. + _ "google.golang.org/grpc/internal/xds/balancer" // Register the balancers. + _ "google.golang.org/grpc/internal/xds/clusterspecifier/rls" // Register the RLS cluster specifier plugin. Note that this does not register the RLS LB policy. + _ "google.golang.org/grpc/internal/xds/httpfilter/extproc" // Register the extproc filter. + _ "google.golang.org/grpc/internal/xds/httpfilter/fault" // Register the fault injection filter. + _ "google.golang.org/grpc/internal/xds/httpfilter/rbac" // Register the RBAC filter. + _ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter. + _ "google.golang.org/grpc/internal/xds/resolver" // Register the xds_resolver. + _ "google.golang.org/grpc/internal/xds/xdsclient/xdslbregistry/converter" // Register the xDS LB Registry Converters. + + v3statusgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" +) + +var logger = grpclog.Component("xds") + +func init() { + internaladmin.AddService(func(registrar grpc.ServiceRegistrar) (func(), error) { + var grpcServer *grpc.Server + switch ss := registrar.(type) { + case *grpc.Server: + grpcServer = ss + case *GRPCServer: + sss, ok := ss.gs.(*grpc.Server) + if !ok { + logger.Warning("grpc server within xds.GRPCServer is not *grpc.Server, CSDS will not be registered") + return nil, nil + } + grpcServer = sss + default: + // Returning an error would cause the top level admin.Register() to + // fail. Log a warning instead. + logger.Error("Server to register service on is neither a *grpc.Server or a *xds.GRPCServer, CSDS will not be registered") + return nil, nil + } + + csdss, err := csds.NewClientStatusDiscoveryServer() + if err != nil { + return nil, fmt.Errorf("failed to create csds server: %v", err) + } + v3statusgrpc.RegisterClientStatusDiscoveryServiceServer(grpcServer, csdss) + return csdss.Close, nil + }) +} + +// NewXDSResolverWithConfigForTesting creates a new xDS resolver builder using +// the provided xDS bootstrap config instead of the global configuration from +// the supported environment variables. The resolver.Builder is meant to be +// used in conjunction with the grpc.WithResolvers DialOption. +// +// # Testing Only +// +// This function should ONLY be used for testing and may not work with some +// other features, including the CSDS service. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func NewXDSResolverWithConfigForTesting(bootstrapConfig []byte) (resolver.Builder, error) { + return internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapConfig) +}